{
"coverage": {
"total": 173,
"tables": 142,
"withMissingFields": 0,
"systemDataAudited": 142,
"sourcedLibraryAdditions": 31
},
"readingCount": 9,
"taxonomy": {
"methods": {
"label": "Methods · 方法",
"description": "按持久改进的对象进一步区分四类。"
},
"evaluation": {
"label": "Evaluation · 评估",
"description": "评测框架、诊断与实证分析。"
},
"dataset": {
"label": "Dataset · 数据集",
"description": "以可复用数据集为主要贡献；不把所有 benchmark 归入数据集。"
},
"theory": {
"label": "Theory · 理论",
"description": "形式定义、自指与程序搜索基础。"
},
"overview": {
"label": "综述与观点",
"description": "领域路线、概念梳理及产业观察。"
},
"resources": {
"label": "工具与基础设施",
"description": "可复用仓库、服务和实验基础设施。"
}
},
"methodTypes": {
"harness": "Harness 进化",
"artifact": "产物进化",
"weights": "模型参数进化",
"joint": "Harness + 模型参数进化"
},
"updated": "2026-09-21",
"originalCount": 128,
"papers": [
{
"id": "cs-0309048",
"title": "Gödel Machines: Self-Referential Universal Problem Solvers Making Provably Optimal Self-Improvements",
"url": "https://arxiv.org/abs/cs/0309048",
"date": "2003-09 / 2006",
"priority": "C",
"categories": [
"theory"
],
"tags": [],
"fields": {
"本质定位": "把整个机器代码（包括 proof searcher 自身）纳入可重写对象；只有证明 rewrite 提高期望 utility 才执行。",
"什么在变": "机器任意代码，包括负责证明/寻找 rewrite 的代码。",
"谁来改 / 谁执行": "**改**：机器内部 proof searcher；rewrite 后 modifier 本身可改变。
**执行**：同一 Gödel Machine。",
"基础 harness": "初始 program + axioms + utility + proof-searcher。",
"Feedback": "形式证明：rewrite 的预期 utility 优于继续搜索。",
"Evolution → Eval": "理论构造，无现代 benchmark。",
"Meta-depth": "M3。",
"相对之前真正新增什么": "这是最干净的理论 M3：不是“optimizer 固定地改 policy”，而是 optimizer/proof searcher 也在同一可编辑代码里。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "把整个机器代码（包括 proof searcher 自身）纳入可重写对象；只有证明 rewrite 提高期望 utility 才执行。 **相对前序：** 这是最干净的理论 M3：不是“optimizer 固定地改 policy”，而是 optimizer/proof searcher 也在同一可编辑代码里。"
},
{
"label": "什么在变",
"text": "机器任意代码，包括负责证明/寻找 rewrite 的代码。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 机器内部 proof searcher；rewrite 后 modifier 本身可改变。 **执行：** 同一 Gödel Machine。"
},
{
"label": "基础 harness",
"text": "初始 program + axioms + utility + proof-searcher。"
},
{
"label": "Feedback",
"text": "形式证明：rewrite 的预期 utility 优于继续搜索。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "理论构造，无现代 benchmark。 **Meta-depth：** M3。"
},
{
"label": "主要结果",
"text": "条件式全局最优 self-rewrite 理论。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "需要可形式化 utility/axioms/proof，现实 agent 很难满足。 **对我们：** 我们讨论“严格 recursive”的上界定义；现代 M2 工作大都仍固定 evaluator/selection。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 40,
"fields": {
"时间": "2003/2006",
"论文": "**[Gödel Machines](https://arxiv.org/abs/cs/0309048)**",
"为什么是主干": "严格 RSI 的理论原型：连 improver/proof-searcher 自身也可被重写。",
"证据边界要记住": "理论构造，不是现代 agent 实证。"
}
},
{
"section": "0A. 理论源头",
"line": 252,
"fields": {
"优先级": "**C**",
"时间": "2003-09 / 2006",
"论文": "[Gödel Machines: Self-Referential Universal Problem Solvers Making Provably Optimal Self-Improvements](https://arxiv.org/abs/cs/0309048)",
"本质定位": "把整个机器代码（包括 proof searcher 自身）纳入可重写对象；只有证明 rewrite 提高期望 utility 才执行。",
"什么在变": "机器任意代码，包括负责证明/寻找 rewrite 的代码。",
"谁来改 / 谁执行": "**改**：机器内部 proof searcher；rewrite 后 modifier 本身可改变。
**执行**：同一 Gödel Machine。",
"基础 harness": "初始 program + axioms + utility + proof-searcher。",
"Feedback": "形式证明：rewrite 的预期 utility 优于继续搜索。",
"Evolution → Eval": "理论构造，无现代 benchmark。",
"Meta-depth": "M3。",
"相对之前真正新增什么": "这是最干净的理论 M3：不是“optimizer 固定地改 policy”，而是 optimizer/proof searcher 也在同一可编辑代码里。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "理论 / 不适用",
"protocolBasis": "理论或形式化前驱，不能套用现代 agent 数据隔离标签。",
"year": "2003",
"depth": [
"M3"
],
"legacyCategories": [
"F"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "把整个机器代码（包括 proof searcher 自身）纳入可重写对象；只有证明 rewrite 提高期望 utility 才执行。",
"novelty": "把寻找改进方案、证明改进有效的程序也纳入自修改范围；每次切换都要求形式证明，保证依赖预设公理与可证明性。",
"object": "机器自身的代码，包括负责寻找改进方案与证明改进有效的程序。",
"executor": "理论上的 Gödel Machine：运行任务程序，并在内部搜索证明；不是某款已训练的语言模型。",
"modifier": "机器内部的证明搜索器寻找“改写程序会提高效用”的证明，满足条件后执行改写；改写也可以涉及证明搜索器自身。",
"roleContext": "**改**：机器内部 proof searcher；rewrite 后 modifier 本身可改变。
**执行**：同一 Gödel Machine。",
"seed": "初始程序含任务求解器、公理系统、效用定义和证明搜索器。只有证明某个自修改比继续原搜索更有利后才切换；可改的包括证明搜索机制本身，前提是初始形式系统能够证明所需命题。",
"fixed": "",
"verdict": "通过形式证明判断：改写程序的预期效用是否高于继续运行原有搜索。这里的效用是系统预先定义、希望最大化的目标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "理论 / 不适用",
"note": "理论构造，无现代 benchmark。"
}
],
"takeaway": "需要可形式化 utility/axioms/proof，现实 agent 很难满足。 **对我们：** 我们讨论“严格 recursive”的上界定义；现代 M2 工作大都仍固定 evaluator/selection。",
"result": "条件式全局最优 self-rewrite 理论。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：不适用：理论机器，不以训练数据集定义学习。\n\n调试 / 选版本数据：形式证明检查修改的预期效用；不是从评测基准错题诊断。\n\n最终测试数据：理论最优性结论，没有本文 task agent 评测基准测试。\n\n数据隔离与证据边界：关键边界是公理的正确性和证明可达性，而非 训练／测试 划分。",
"cycle": "证明搜索器同时寻找自修改程序及其收益证明。只有证明现在切换优于继续原搜索，才执行代码替换；可被改写的代码也包括证明搜索器自身。",
"train": "不适用：理论机器，不以训练数据集定义学习。",
"debug": "形式证明检查修改的预期效用；不是从评测基准错题诊断。",
"test": "理论最优性结论，没有本文 task agent 评测基准测试。",
"isolation": "关键边界是公理的正确性和证明可达性，而非 训练／测试 划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "初始程序含任务求解器、公理系统、效用定义和证明搜索器。只有证明某个自修改比继续原搜索更有利后才切换；可改的包括证明搜索机制本身，前提是初始形式系统能够证明所需命题。",
"protocol": "**无 benchmark 训练/测试：**这是形式化自修改构造；环境与奖励通过公理和效用描述，而非某份训练题库。论文的机器人等例子说明适用问题，不是独立实验数据证明。应比较证明成立的条件，不能与 SWE-bench 涨分放在同一证据口径。",
"sections": "§2.3、§5–6",
"source": "https://arxiv.org/abs/cs/0309048",
"version": "cs/0309048v5",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e19760737f414482f5b00a908343b984521fb74098d6b67e92fdf56c548612c5",
"seedStatus": "not-applicable",
"protocolStatus": "not-applicable"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "理论上的 Gödel Machine：运行任务程序，并在内部搜索证明；不是某款已训练的语言模型。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "机器内部的证明搜索器寻找“改写程序会提高效用”的证明，满足条件后执行改写；改写也可以涉及证明搜索器自身。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "机器自身的代码，包括负责寻找改进方案与证明改进有效的程序。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "通过形式证明判断：改写程序的预期效用是否高于继续运行原有搜索。这里的效用是系统预先定义、希望最大化的目标。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "初始程序含任务求解器、公理系统、效用定义和证明搜索器。只有证明某个自修改比继续原搜索更有利后才切换；可改的包括证明搜索机制本身，前提是初始形式系统能够证明所需命题。",
"sources": [
{
"label": "§2.3、§5–6",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "证明搜索器同时寻找自修改程序及其收益证明。只有证明现在切换优于继续原搜索，才执行代码替换；可被改写的代码也包括证明搜索器自身。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不适用：理论机器，不以训练数据集定义学习。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "形式证明检查修改的预期效用；不是从评测基准错题诊断。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "理论最优性结论，没有本文 task agent 评测基准测试。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "关键边界是公理的正确性和证明可达性，而非 训练／测试 划分。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把寻找改进方案、证明改进有效的程序也纳入自修改范围；每次切换都要求形式证明，保证依赖预设公理与可证明性。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/abs/cs/0309048",
"version": "",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "这是最干净的理论 M3：不是“optimizer 固定地改 policy”，而是 optimizer/proof searcher 也在同一可编辑代码里。",
"feedbackCases": [
{
"label": "理论判据：没有实验数据集",
"data": "理论中的环境、公理和预定义效用目标；不使用现代训练集/测试集。",
"scoring": "证明检查器验证自修改的形式证明：预期效用是否高于继续运行原程序。",
"visible": "可被系统检查的证明，而不是任务数据上的经验分数或模型自评。",
"use": "只有证明满足触发条件才执行改写；论文的最优性结论不能当作某个 benchmark 上实测涨分。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"judgment": "形式证明检查；没有 benchmark 答案或模型裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "不适用：理论机器，不以训练数据集定义学习。",
"selection": "形式证明检查修改的预期效用；不是从评测基准错题诊断。",
"evaluation": "理论最优性结论，没有本文 task agent 评测基准测试。",
"isolation": "关键边界是公理的正确性和证明可达性，而非 训练／测试 划分。",
"roles": {
"executor": {
"value": "理论上的 Gödel Machine：运行任务程序，并在内部搜索证明；不是某款已训练的语言模型。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
},
"modifier": {
"value": "机器内部的证明搜索器寻找“改写程序会提高效用”的证明，满足条件后执行改写；改写也可以涉及证明搜索器自身。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
},
"seed": {
"value": "初始程序含任务求解器、公理系统、效用定义和证明搜索器。只有证明某个自修改比继续原搜索更有利后才切换；可改的包括证明搜索机制本身，前提是初始形式系统能够证明所需命题。",
"sources": [
{
"label": "§2.3、§5–6",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"selection": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"evaluation": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"isolation": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "传统学习算法即使能改进某类策略，负责改进的算法本身仍是写死的，需要人提出更好的算法并证明它有用。作者希望消除这一限制：让系统连自己的改进过程也能重写，同时用形式化证明判断修改是否值得，从而为通用自改进提供理论依据。",
"sources": [
{
"label": "§1 Introduction and Outline（第 2 页）",
"url": "https://arxiv.org/pdf/cs/0309048v5#page=2"
}
]
},
{
"key": "position",
"value": "讨论自修改系统在什么条件下能够证明：改变自身程序的预期收益高于继续使用原程序，并由此获得最优性保证。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
},
{
"key": "conclusion",
"value": "给出依赖公理和可证明性的最优性保证；证明搜索器本身也能被改写，尚不是现代 task agent 的实证系统。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
}
],
"fields": {
"object": "机器自身的代码，包括负责寻找改进方案与证明改进有效的程序。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2303.11366",
"title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
"url": "https://arxiv.org/abs/2303.11366",
"date": "2023-03-20",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"ExecutableVerifier",
"M0",
"MemoryContent",
"org:princeton",
"org:mit",
"person:shunyu-yao"
],
"fields": {
"本质定位": "actor 执行任务，evaluator 给 outcome，reflector 把失败/成功转成自然语言 reflection 存入 episodic memory，供同任务后续 trial 使用。",
"什么在变": "episodic reflection memory。",
"谁来改 / 谁执行": "**改**：固定 reflector prompt；actor/evaluator/reflector 可由同/不同 LLM 角色承担。
**执行**：LLM actor。",
"基础 harness": "actor + evaluator + reflector + memory buffer。",
"Feedback": "binary/scalar task feedback；可来自环境、unit tests、answer metric。",
"Evolution → Eval": "ALFWorld/HotPotQA/HumanEval 等，多为同 task 多 trial。",
"Meta-depth": "M0：memory content 变，memory update mechanism fixed。",
"相对之前真正新增什么": "相对 Self-Refine 的关键差别是 reflection 成为 persistent memory；但主要用于同一任务重试，不是跨 task lifelong skill learning。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "actor 执行任务，evaluator 给 outcome，reflector 把失败/成功转成自然语言 reflection 存入 episodic memory，供同任务后续 trial 使用。 **相对前序：** 相对 Self-Refine 的关键差别是 reflection 成为 persistent memory；但主要用于同一任务重试，不是跨 task lifelong skill learning。"
},
{
"label": "什么在变",
"text": "episodic reflection memory。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 固定 reflector prompt；actor/evaluator/reflector 可由同/不同 LLM 角色承担。 **执行：** LLM actor。"
},
{
"label": "基础 harness",
"text": "actor + evaluator + reflector + memory buffer。"
},
{
"label": "Feedback",
"text": "binary/scalar task feedback；可来自环境、unit tests、answer metric。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "ALFWorld/HotPotQA/HumanEval 等，多为同 task 多 trial。 **Meta-depth：** M0：memory content 变，memory update mechanism fixed。"
},
{
"label": "主要结果",
"text": "多任务显著提升；HumanEval Rust ablation 中无 generated tests 的 self-reflection 可退化，而 tests+reflection 提升。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "容易把同题重试收益误解为跨任务 self-evolution。 **对我们：** persistent textual experience 的经典起点；也直接支持“grounded feedback 才可靠”。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 41,
"fields": {
"时间": "2023-03",
"论文": "**[Reflexion](https://arxiv.org/abs/2303.11366)**",
"为什么是主干": "`feedback→reflection→persistent experience→future behavior` 的经典 LLM 前史。",
"证据边界要记住": "updater 固定，更像 persistent memory 而非 harness mechanism evolution。"
}
},
{
"section": "A2. Context / Memory Evolution",
"line": 287,
"fields": {
"优先级": "**C**",
"时间": "2023-03-20",
"论文": "[Reflexion: Language Agents with Verbal Reinforcement Learning](https://arxiv.org/abs/2303.11366)",
"本质定位": "actor 执行任务，evaluator 给 outcome，reflector 把失败/成功转成自然语言 reflection 存入 episodic memory，供同任务后续 trial 使用。",
"什么在变": "episodic reflection memory。",
"谁来改 / 谁执行": "**改**：固定 reflector prompt；actor/evaluator/reflector 可由同/不同 LLM 角色承担。
**执行**：LLM actor。",
"基础 harness": "actor + evaluator + reflector + memory buffer。",
"Feedback": "binary/scalar task feedback；可来自环境、unit tests、answer metric。",
"Evolution → Eval": "ALFWorld/HotPotQA/HumanEval 等，多为同 task 多 trial。",
"Meta-depth": "M0：memory content 变，memory update mechanism fixed。",
"相对之前真正新增什么": "相对 Self-Refine 的关键差别是 reflection 成为 persistent memory；但主要用于同一任务重试，不是跨 task lifelong skill learning。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "同任务多次重试",
"protocolBasis": "Reflexion 原记录指出，多为同一 task 的多次 trial，不等同于跨任务迁移。",
"year": "2023",
"depth": [
"M0"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "actor 执行任务，evaluator 给 outcome，reflector 把失败/成功转成自然语言 reflection 存入 episodic memory，供同任务后续 trial 使用。",
"novelty": "把失败原因写成下一次尝试会读取的文字记忆，使同一任务的重试能够利用先前教训；参数和反思流程保持固定。",
"object": "跨尝试保留的反思记忆：记录失败原因和下一次的改进建议。模型参数及执行、评估、反思流程固定。",
"executor": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 使用 GPT-3 执行动作；编程实验以 GPT-4 生成代码，并运行自建单元测试。模型配置随任务变化，不能统一写成一款 语言模型 执行任务的模型。",
"modifier": "固定反思提示让语言模型把失败反馈写成文字经验，下一次执行时读取。算法将执行任务的模型、评估器、reflector 分为角色；§4 没有逐项列出所有角色对应的 API 快照，不能据此断言都用同一个版本。",
"roleContext": "**改**：固定 reflector prompt；actor/evaluator/reflector 可由同/不同 LLM 角色承担。
**执行**：LLM actor。",
"seed": "任务执行者、结果评估器、反思器和短期经验缓冲区组成固定流程。失败后把结果转成文字反思再重试；问答/家务设置可保留三条经验，编程设置保留一条，不更新模型参数。",
"fixed": "M0：memory content 变，memory update mechanism fixed",
"verdict": "编程任务运行测试检查代码，问答任务对照答案，交互任务读取环境给出的成功或失败。模型把这些结果与执行过程一起用于反思；不同任务的具体判分方式见展开表格。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "同任务多次重试",
"note": "ALFWorld/HotPotQA/HumanEval 等，多为同 task 多 trial。"
}
],
"takeaway": "容易把同题重试收益误解为跨任务 self-evolution。 **对我们：** persistent textual experience 的经典起点；也直接支持“grounded feedback 才可靠”。",
"result": "多任务显著提升；HumanEval Rust ablation 中无 generated tests 的 self-reflection 可退化，而 tests+reflection 提升。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不做参数训练。ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 134 个环境、HotpotQA（需要结合多份资料作答的多跳问答基准） 的 100 道题在当前题上反思重试。\n\n调试 / 选版本数据：动作/问答读取任务结果；编程使用自行生成并通过语法检查的至多 6 个单元测试。\n\n最终测试数据：编程在 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP（Python/Rust）及 LeetcodeHardGym 40 题上，用正式测试判断最终代码。\n\n数据隔离与证据边界：动作/问答的反馈与报告来自同题重试；编程把自建测试与正式测试分开。",
"cycle": "反思器读取本次轨迹、成功/失败信号和此前反思，指出错误动作及下次应采取的替代做法。把文字反思追加到记忆，再从重置后的同一任务重试；不改权重或反思算法。任务成功或达到重试上限后停止；不是先筛选一套新运行框架再部署。",
"train": "不做参数训练。ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 134 个环境、HotpotQA（需要结合多份资料作答的多跳问答基准） 的 100 道题在当前题上反思重试。",
"debug": "动作/问答读取任务结果；编程使用自行生成并通过语法检查的至多 6 个单元测试。",
"test": "编程在 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP（Python/Rust）及 LeetcodeHardGym 40 题上，用正式测试判断最终代码。",
"isolation": "动作/问答的反馈与报告来自同题重试；编程把自建测试与正式测试分开。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "任务执行者、结果评估器、反思器和短期经验缓冲区组成固定流程。失败后把结果转成文字反思再重试；问答/家务设置可保留三条经验，编程设置保留一条，不更新模型参数。",
"protocol": "**问答/动作：**HotpotQA 100 题，ALFWorld 134 个任务，在同一题多次尝试并接收结果；这不是先在前一批题学习后冻结测试。\n\n**编程：**HumanEval、MBPP 的 Python/Rust 任务及自建 LeetcodeHardGym 40 道较新难题。迭代时使用模型生成、经语法过滤的至多 6 个单元测试；最终再由 benchmark 检查。编程的自测和正式测试必须分开，不能一概写成直接给官方答案。",
"sections": "§4.1–4.3",
"source": "https://arxiv.org/abs/2303.11366",
"version": "2303.11366v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "cd0d76073ad1f5e370e977860e9272b5219abedb3b9bf5c6b88299743ab75ae3",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 使用 GPT-3 执行动作；编程实验以 GPT-4 生成代码，并运行自建单元测试。模型配置随任务变化，不能统一写成一款 语言模型 执行任务的模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定反思提示让语言模型把失败反馈写成文字经验，下一次执行时读取。算法将执行任务的模型、评估器、reflector 分为角色；§4 没有逐项列出所有角色对应的 API 快照，不能据此断言都用同一个版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨尝试保留的反思记忆：记录失败原因和下一次的改进建议。模型参数及执行、评估、反思流程固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "编程任务运行测试检查代码，问答任务对照答案，交互任务读取环境给出的成功或失败。模型把这些结果与执行过程一起用于反思；不同任务的具体判分方式见展开表格。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "任务执行者、结果评估器、反思器和短期经验缓冲区组成固定流程。失败后把结果转成文字反思再重试；问答/家务设置可保留三条经验，编程设置保留一条，不更新模型参数。",
"sources": [
{
"label": "§4.1–4.3",
"url": "https://arxiv.org/abs/2303.11366"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "反思器读取本次轨迹、成功/失败信号和此前反思，指出错误动作及下次应采取的替代做法。把文字反思追加到记忆，再从重置后的同一任务重试；不改权重或反思算法。任务成功或达到重试上限后停止；不是先筛选一套新运行框架再部署。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不做参数训练。ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 134 个环境、HotpotQA（需要结合多份资料作答的多跳问答基准） 的 100 道题在当前题上反思重试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "动作/问答读取任务结果；编程使用自行生成并通过语法检查的至多 6 个单元测试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "编程在 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP（Python/Rust）及 LeetcodeHardGym 40 题上，用正式测试判断最终代码。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "动作/问答的反馈与报告来自同题重试；编程把自建测试与正式测试分开。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把失败原因写成下一次尝试会读取的文字记忆，使同一任务的重试能够利用先前教训；参数和反思流程保持固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2303.11366v4",
"version": "2303.11366v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Self-Refine 的关键差别是 reflection 成为 persistent memory；但主要用于同一任务重试，不是跨 task lifelong skill learning。",
"feedbackCases": [
{
"label": "同题反思：ALFWorld",
"data": "134 个环境任务，在失败任务上反复尝试。",
"scoring": "依据环境任务完成信号及失败检测判断本次尝试；模型把失败记录转成文字反思。",
"visible": "环境观察、动作记录和成功/失败反馈。",
"use": "反思写入记忆后重试当前任务，没有参数训练，也不是在另一套测试题上验证后才采用反思。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"judgment": "ALFWorld 环境成功信号；另用失败检测触发反思"
},
{
"label": "同题反思：HotpotQA",
"data": "100 道问答题。",
"scoring": "将最终答案与参考答案作精确匹配，给正确/错误信号；反思阶段不提供参考答案本身。",
"visible": "模型读取自己的推理/检索记录及错误信号。",
"use": "修改解题思路后再试当前题；连续失败达到上限时停止。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"judgment": "规则将最终回答与 HotpotQA 标准答案精确匹配"
},
{
"label": "代码反思与正式判分",
"data": "HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP（Python/Rust）、LeetcodeHardGym 40 题。",
"scoring": "调试使用模型自己生成并通过语法检查的至多六个单元测试；正式代码成绩由基准的保密测试计算。",
"visible": "调试可见自建测试的执行结果与错误，自建测试可能漏检或误判。",
"use": "先据自建测试修改代码，再交正式测试；不能把内部自测通过当作正式正确率。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"judgment": "调试：模型生成的单元测试；正式判分：基准隐藏测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "不做参数训练。ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 134 个环境、HotpotQA（需要结合多份资料作答的多跳问答基准） 的 100 道题在当前题上反思重试。",
"selection": "动作/问答读取任务结果；编程使用自行生成并通过语法检查的至多 6 个单元测试。",
"evaluation": "编程在 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP（Python/Rust）及 LeetcodeHardGym 40 题上，用正式测试判断最终代码。",
"isolation": "动作/问答的反馈与报告来自同题重试；编程把自建测试与正式测试分开。",
"roles": {
"executor": {
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 使用 GPT-3 执行动作；编程实验以 GPT-4 生成代码，并运行自建单元测试。模型配置随任务变化，不能统一写成一款 语言模型 执行任务的模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3.SSS0.Px3"
}
]
},
"modifier": {
"value": "固定反思提示让语言模型把失败反馈写成文字经验，下一次执行时读取。算法将执行任务的模型、评估器、reflector 分为角色；§4 没有逐项列出所有角色对应的 API 快照，不能据此断言都用同一个版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3.SSS0.Px3"
}
]
},
"seed": {
"value": "任务执行者、结果评估器、反思器和短期经验缓冲区组成固定流程。失败后把结果转成文字反思再重试；问答/家务设置可保留三条经验，编程设置保留一条，不更新模型参数。",
"sources": [
{
"label": "§4.1–4.3",
"url": "https://arxiv.org/abs/2303.11366"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "大型语言 agent 若靠强化学习更新参数来吸收失败经验，需要大量计算与时间；一个成败分数又难说明究竟哪步出了错。因此，作者关注如何把失败转成下一次可执行的改进建议，使 agent 无需重新训练模型也能从试错中学习。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2303.11366#S1"
}
]
},
{
"key": "position",
"value": "研究冻结参数的语言 agent 能否从试错中学习，在决策、问答和编程任务中把失败经验转化为后续尝试的改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.11366"
}
]
},
{
"key": "conclusion",
"value": "在决策、问答和编程任务上提升表现，说明不更新权重也能利用失败经验改进后续尝试。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2303.11366"
}
]
}
],
"fields": {
"modifier": "语言模型按固定反思提示，将失败反馈写成下一次可读取的经验。执行、评估和反思是不同角色；原文没有逐一披露全部角色的 API 版本。",
"object": "跨尝试保留的反思记忆：记录失败原因和下一次的改进建议。模型参数及执行、评估、反思流程固定。",
"verdict": "编程任务运行测试检查代码，问答任务对照答案，交互任务读取环境给出的成功或失败。模型把这些结果与执行过程一起用于反思；不同任务的具体判分方式见展开表格。"
}
},
"attributions": [
{
"tag": "org:princeton",
"label": "Princeton University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.11366"
}
]
},
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.11366"
}
]
},
{
"tag": "person:shunyu-yao",
"label": "Shunyu Yao",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2303.11366"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience",
"feedback"
]
},
{
"id": "2305.16291",
"title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
"url": "https://arxiv.org/abs/2305.16291",
"date": "2023-05-25",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"M0",
"Online",
"SelfFeedback",
"Skill",
"Tool",
"org:stanford"
],
"fields": {
"本质定位": "GPT-4 在 Minecraft 中自动提出 curriculum、写/调试 executable code skills，把通过环境验证的程序存入 skill library 并检索复用。",
"什么在变": "skill library contents（code skills）。",
"谁来改 / 谁执行": "**改**：GPT-4 skill synthesizer/debugger；curriculum/retrieval procedure fixed。
**执行**：GPT-4-based Minecraft agent + Mineflayer。",
"基础 harness": "automatic curriculum + skill library + iterative prompting。",
"Feedback": "environment state、execution errors、self-verification。",
"Evolution → Eval": "开放式 Minecraft lifelong run；另测新世界/新任务 skill transfer。",
"Meta-depth": "M0/M1：skill content evolves，skill system fixed。",
"相对之前真正新增什么": "相对文字 reflection/memory，关键跃迁是经验变成 **可执行、可复用 code skill**。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "GPT-4 在 Minecraft 中自动提出 curriculum、写/调试 executable code skills，把通过环境验证的程序存入 skill library 并检索复用。 **相对前序：** 相对文字 reflection/memory，关键跃迁是经验变成 **可执行、可复用 code skill**。"
},
{
"label": "什么在变",
"text": "skill library contents（code skills）。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** GPT-4 skill synthesizer/debugger；curriculum/retrieval procedure fixed。 **执行：** GPT-4-based Minecraft agent + Mineflayer。"
},
{
"label": "基础 harness",
"text": "automatic curriculum + skill library + iterative prompting。"
},
{
"label": "Feedback",
"text": "environment state、execution errors、self-verification。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "开放式 Minecraft lifelong run；另测新世界/新任务 skill transfer。 **Meta-depth：** M0/M1：skill content evolves，skill system fixed。"
},
{
"label": "主要结果",
"text": "3.3× unique items、15.3× faster tech milestones、2.3× farther travel（论文口径）。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "没有独立 held-out evolution protocol；domain 特殊。 **对我们：** executable skill evolution 的经典起点，后续 AgentFactory/ASPIRE/Alita-G 都应与它比较。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 42,
"fields": {
"时间": "2023-05",
"论文": "**[Voyager](https://arxiv.org/abs/2305.16291)**",
"为什么是主干": "executable skill library 的经典起点。",
"证据边界要记住": "Minecraft/open-ended embodied setting；不是 full harness。"
}
},
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 302,
"fields": {
"优先级": "**C**",
"时间": "2023-05-25",
"论文": "[Voyager: An Open-Ended Embodied Agent with Large Language Models](https://arxiv.org/abs/2305.16291)",
"本质定位": "GPT-4 在 Minecraft 中自动提出 curriculum、写/调试 executable code skills，把通过环境验证的程序存入 skill library 并检索复用。",
"什么在变": "skill library contents（code skills）。",
"谁来改 / 谁执行": "**改**：GPT-4 skill synthesizer/debugger；curriculum/retrieval procedure fixed。
**执行**：GPT-4-based Minecraft agent + Mineflayer。",
"基础 harness": "automatic curriculum + skill library + iterative prompting。",
"Feedback": "environment state、execution errors、self-verification。",
"Evolution → Eval": "开放式 Minecraft lifelong run；另测新世界/新任务 skill transfer。",
"Meta-depth": "M0/M1：skill content evolves，skill system fixed。",
"相对之前真正新增什么": "相对文字 reflection/memory，关键跃迁是经验变成 **可执行、可复用 code skill**。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Online / continual",
"protocolBasis": "开放式或不重置的连续运行；不同于标准 freeze-held-out。",
"year": "2023",
"depth": [
"M0",
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "GPT-4 在 Minecraft 中自动提出 curriculum、写/调试 executable code skills，把通过环境验证的程序存入 skill library 并检索复用。",
"novelty": "把成功行为保存成可检索、可再次执行的代码技能，并让自动课程决定接下来探索什么；经验因此能用于新的 Minecraft 任务。",
"object": "Minecraft 技能库中的可执行代码，用于保存和复用已学会的动作。模型参数、课程选择、技能检索和验证流程固定。",
"executor": "主实验由 gpt-4-0314 为 Minecraft 游戏编写行动代码，再通过 Mineflayer（让程序控制游戏角色的工具库）执行；gpt-3.5-turbo-0301 辅助问答，text-embedding-ada-002 将技能描述转成可按相似度检索的向量。",
"modifier": "gpt-4-0314 生成并调试技能代码，也负责课程提议和任务成功检查；补充实验将主模型换为 gpt-4-0613。模型参数不更新。",
"roleContext": "**改**：GPT-4 skill synthesizer/debugger；curriculum/retrieval procedure fixed。
**执行**：GPT-4-based Minecraft agent + Mineflayer。",
"seed": "自建 Minecraft agent 包含自动课程、代码技能库和失败后修改代码的流程。自动课程选择探索目标，技能库复用已完成的行为，Mineflayer 工具库把代码变成游戏动作；模型参数不更新。",
"fixed": "M0/M1：skill content evolves，skill system fixed",
"verdict": "读取 Minecraft 中的物品与环境状态、代码报错，以及模型对目标是否完成的检查。环境观察说明实际发生了什么，模型据此判断下一次应怎样修代码。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Online / continual",
"note": "开放式 Minecraft lifelong run；另测新世界/新任务 skill transfer。"
}
],
"takeaway": "没有独立 held-out evolution protocol；domain 特殊。 **对我们：** executable skill evolution 的经典起点，后续 AgentFactory/ASPIRE/Alita-G 都应与它比较。",
"result": "3.3× unique items、15.3× faster tech milestones、2.3× farther travel（论文口径）。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Minecraft 在线探索；主实验每次最多 160 轮提示，共 3 次运行。经验是采集、制作和探索时生成的技能代码。\n\n调试 / 选版本数据：当前世界的任务状态和代码异常；完成情况由提供批评意见的模型检查。\n\n最终测试数据：另清空背包、重置新世界，测试 4 个未见目标；每目标 3 次尝试，上限 50 轮提示。\n\n数据隔离与证据边界：探索曲线来自持续学习；新世界目标实验另测技能迁移，不能把两者混成一套测试。",
"cycle": "同时读环境状态、代码运行错误和提供批评意见的模型的完成判断，据此修复本轮技能。GPT-4 编写并修改 Mineflayer 可执行技能；课程模块根据已完成和失败任务选择下一目标。每个目标最多修订 4 轮；成功的代码加入技能库，失败目标记录后继续课程。",
"train": "Minecraft 在线探索；主实验每次最多 160 轮提示，共 3 次运行。经验是采集、制作和探索时生成的技能代码。",
"debug": "当前世界的任务状态和代码异常；完成情况由提供批评意见的模型检查。",
"test": "另清空背包、重置新世界，测试 4 个未见目标；每目标 3 次尝试，上限 50 轮提示。",
"isolation": "探索曲线来自持续学习；新世界目标实验另测技能迁移，不能把两者混成一套测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "Minecraft agent 有自动课程、可执行技能库、迭代代码提示和环境反馈；通过 Mineflayer 类程序接口执行技能。课程决定探索目标，技能检索复用已有代码，失败后修改代码，权重不更新。",
"protocol": "**经验构建：**在开放式 Minecraft 世界内探索、采集物品并生成技能，记录探索进展；不是从静态训练文本学习。\n\n**测试：**另在新世界/任务检查技能迁移，同时主探索曲线仍来自持续运行。需分开看同一世界内积累和新环境迁移；每组世界种子、任务数本轮尚未核实，不把所有探索记录称为留出测试。",
"sections": "自动课程、技能库与迁移实验",
"source": "https://arxiv.org/abs/2305.16291",
"version": "2305.16291v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6eab7f00a75734cb4bba9fc631e2f4b30d1ce3982459592642512a80a4038fc8",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验由 gpt-4-0314 为 Minecraft 游戏编写行动代码，再通过 Mineflayer（让程序控制游戏角色的工具库）执行；gpt-3.5-turbo-0301 辅助问答，text-embedding-ada-002 将技能描述转成可按相似度检索的向量。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.16291#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2305.16291#A1.SS3.SSS2"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2305.16291#A1.SS4.SSS1"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2305.16291#A1.SS5.SSS1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2305.16291#A2.SS4.SSS5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "gpt-4-0314 生成并调试技能代码，也负责课程提议和任务成功检查；补充实验将主模型换为 gpt-4-0613。模型参数不更新。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.16291#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2305.16291#A1.SS3.SSS2"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2305.16291#A1.SS4.SSS1"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2305.16291#A1.SS5.SSS1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2305.16291#A2.SS4.SSS5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "Minecraft 技能库中的可执行代码，用于保存和复用已学会的动作。模型参数、课程选择、技能检索和验证流程固定。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "读取 Minecraft 中的物品与环境状态、代码报错，以及模型对目标是否完成的检查。环境观察说明实际发生了什么，模型据此判断下一次应怎样修代码。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "自建 Minecraft agent 包含自动课程、代码技能库和失败后修改代码的流程。自动课程选择探索目标，技能库复用已完成的行为，Mineflayer 工具库把代码变成游戏动作；模型参数不更新。",
"sources": [
{
"label": "自动课程、技能库与迁移实验",
"url": "https://arxiv.org/abs/2305.16291"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "同时读环境状态、代码运行错误和提供批评意见的模型的完成判断，据此修复本轮技能。GPT-4 编写并修改 Mineflayer 可执行技能；课程模块根据已完成和失败任务选择下一目标。每个目标最多修订 4 轮；成功的代码加入技能库，失败目标记录后继续课程。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Minecraft 在线探索；主实验每次最多 160 轮提示，共 3 次运行。经验是采集、制作和探索时生成的技能代码。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前世界的任务状态和代码异常；完成情况由提供批评意见的模型检查。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "另清空背包、重置新世界，测试 4 个未见目标；每目标 3 次尝试，上限 50 轮提示。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "探索曲线来自持续学习；新世界目标实验另测技能迁移，不能把两者混成一套测试。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把成功行为保存成可检索、可再次执行的代码技能，并让自动课程决定接下来探索什么；经验因此能用于新的 Minecraft 任务。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2305.16291v2",
"version": "2305.16291v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对文字 reflection/memory，关键跃迁是经验变成 **可执行、可复用 code skill**。",
"feedbackCases": [
{
"label": "Minecraft 探索与技能学习",
"data": "当前 Minecraft 世界；主实验每次最多 160 轮提示，共三次运行。",
"scoring": "环境提供状态变化和代码执行错误；GPT-4 的批评角色根据目标及当前状态检查是否完成，提供成功判定与改进意见。",
"visible": "可见环境状态、异常以及批评角色的文字反馈。",
"use": "修改技能代码，成功后存入库，再由课程选择后续目标；这里没有人工标注问答集。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"judgment": "GPT-4 依据任务目标与环境状态判断成功并给批评；代码错误来自执行器"
},
{
"label": "新世界目标测试",
"data": "重置世界并清空背包后的四个未见目标；每目标三次尝试、最多 50 轮提示。",
"scoring": "按新世界中的目标完成情况评估技能复用。",
"visible": "执行时仍会接收环境观察与错误。",
"use": "检验已积累技能是否帮助解决新目标，不是训练集准确率。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"judgment": "按 Minecraft 新目标完成情况评测，区别于探索期技能累计数"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Minecraft 在线探索；主实验每次最多 160 轮提示，共 3 次运行。经验是采集、制作和探索时生成的技能代码。",
"selection": "当前世界的任务状态和代码异常；完成情况由提供批评意见的模型检查。",
"evaluation": "另清空背包、重置新世界，测试 4 个未见目标；每目标 3 次尝试，上限 50 轮提示。",
"isolation": "探索曲线来自持续学习；新世界目标实验另测技能迁移，不能把两者混成一套测试。",
"roles": {
"executor": {
"value": "主实验由 gpt-4-0314 为 Minecraft 游戏编写行动代码，再通过 Mineflayer（让程序控制游戏角色的工具库）执行；gpt-3.5-turbo-0301 辅助问答，text-embedding-ada-002 将技能描述转成可按相似度检索的向量。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.16291#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2305.16291#A1.SS3.SSS2"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2305.16291#A1.SS4.SSS1"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2305.16291#A1.SS5.SSS1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2305.16291#A2.SS4.SSS5"
}
]
},
"modifier": {
"value": "gpt-4-0314 生成并调试技能代码，也负责课程提议和任务成功检查；补充实验将主模型换为 gpt-4-0613。模型参数不更新。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.16291#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2305.16291#A1.SS3.SSS2"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2305.16291#A1.SS4.SSS1"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2305.16291#A1.SS5.SSS1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2305.16291#A2.SS4.SSS5"
}
]
},
"seed": {
"value": "自建 Minecraft agent 包含自动课程、代码技能库和失败后修改代码的流程。自动课程选择探索目标，技能库复用已完成的行为，Mineflayer 工具库把代码变成游戏动作；模型参数不更新。",
"sources": [
{
"label": "自动课程、技能库与迁移实验",
"url": "https://arxiv.org/abs/2305.16291"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"selection": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"evaluation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
],
"isolation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2305.16291#S3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有强化学习和模仿学习 agent 直接探索底层动作，难以高效探索并迁移行为；已有语言模型 agent 也没有持续获取、修订和积累技能的终身学习能力。这使它们难在没有固定终点的开放世界中越探索越能干，因此需要把新经历变成后续可复用的技能。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2305.16291#S1"
}
]
},
{
"key": "position",
"value": "面向 Minecraft 开放世界，研究 agent 能否在无人持续指定任务的情况下自主探索、积累技能，并将能力迁移到新任务和新世界。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.16291"
}
]
},
{
"key": "conclusion",
"value": "获得更多物品、加快技术树探索；积累的技能可在新的 Minecraft 世界中复用。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.16291"
}
]
}
],
"fields": {
"object": "Minecraft 技能库中的可执行代码，用于保存和复用已学会的动作。模型参数、课程选择、技能检索和验证流程固定。",
"verdict": "读取 Minecraft 中的物品与环境状态、代码报错，以及模型对目标是否完成的检查。环境观察说明实际发生了什么，模型据此判断下一次应怎样修代码。",
"executor": "主实验由 gpt-4-0314 为 Minecraft 游戏编写行动代码，再通过 Mineflayer（让程序控制游戏角色的工具库）执行；gpt-3.5-turbo-0301 辅助问答，text-embedding-ada-002 将技能描述转成可按相似度检索的向量。",
"seed": "自建 Minecraft agent 包含自动课程、代码技能库和失败后修改代码的流程。自动课程选择探索目标，技能库复用已完成的行为，Mineflayer 工具库把代码变成游戏动作；模型参数不更新。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2305.16291"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience",
"curriculum"
]
},
{
"id": "2505.22954",
"title": "Darwin Gödel Machine (DGM): Open-Ended Evolution of Self-Improving Agents",
"url": "https://arxiv.org/abs/2505.22954",
"date": "2025-05-29",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Archive",
"ExecutableVerifier",
"HarnessCode",
"Improver",
"M2",
"Population",
"org:ubc",
"org:sakana",
"person:jeff-clune"
],
"fields": {
"本质定位": "让 coding agents 修改自己的 code，并保留多个已验证 stepping stones 形成 open-ended archive，而不是只沿单一 best chain。",
"谁来改 → 谁执行；基础 harness": "self-mod coding agent → coding-agent descendants；seed 为 coding-agent source/harness。",
"Feedback / evidence": "executable coding benchmarks；archive selection。",
"什么在变": "coding-agent code/harness。",
"谁来改 / 谁执行": "**改**：archive 中 candidate agents 自己生成 descendants。
**执行**：descendant agents。",
"基础 harness": "self-mod coding agent seed。",
"Feedback": "SWE/Polyglot executable tests + performance。",
"Evolution → Eval": "SWE/Polyglot search；部分 final Polyglot 覆盖更多未用于 search tasks。",
"Meta-depth": "M2。",
"相对之前真正新增什么": "相对 SICA 的新增主要是 **open-ended archive / diversity**，降低 greedy self-improvement 卡死在局部最优。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "让 coding agents 修改自己的 code，并保留多个已验证 stepping stones 形成 open-ended archive，而不是只沿单一 best chain。 **相对前序：** 相对 SICA 的新增主要是 **open-ended archive / diversity**，降低 greedy self-improvement 卡死在局部最优。"
},
{
"label": "什么在变",
"text": "coding-agent code/harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** archive 中 candidate agents 自己生成 descendants。 **执行：** descendant agents。"
},
{
"label": "基础 harness",
"text": "self-mod coding agent seed。"
},
{
"label": "Feedback",
"text": "SWE/Polyglot executable tests + performance。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "SWE/Polyglot search；部分 final Polyglot 覆盖更多未用于 search tasks。 **Meta-depth：** M2。"
},
{
"label": "主要结果",
"text": "SWE 约20→50、Polyglot 14.2→30.7；open-ended archive ablation 显著。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "self-mod prompt/selection/evaluator仍 fixed；SWE 泛化隔离有限。 **对我们：** 核心启发不是“能改代码”，而是 evolution 需要 population/stepping-stone preservation。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 43,
"fields": {
"时间": "2025-05",
"论文": "**[Darwin Gödel Machine](https://arxiv.org/abs/2505.22954)**",
"为什么是主干": "self-modifying agent source + open-ended archive 的代表，是现代 RSI/harness program evolution 的直接前驱。",
"证据边界要记住": "outer archive / selection mechanism 仍固定。"
}
},
{
"section": "A. Foundations / agent-program / improver-level",
"line": 134,
"fields": {
"时间": "2025-05-29",
"论文": "[Darwin Gödel Machine](https://arxiv.org/abs/2505.22954)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**F / H-Full / Meta**。不沿单一路径贪心改，而维护 coding-agent archive，让不同版本成为 stepping stones；modern open-ended self-modifying agent 的主干。",
"谁来改 → 谁执行；基础 harness": "self-mod coding agent → coding-agent descendants；seed 为 coding-agent source/harness。",
"Feedback / evidence": "executable coding benchmarks；archive selection。",
"标签": "`#HarnessCode #Improver #Archive #Population #ExecutableVerifier #M2`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 323,
"fields": {
"优先级": "**C**",
"时间": "2025-05-29",
"论文": "[Darwin Gödel Machine (DGM): Open-Ended Evolution of Self-Improving Agents](https://arxiv.org/abs/2505.22954)",
"本质定位": "让 coding agents 修改自己的 code，并保留多个已验证 stepping stones 形成 open-ended archive，而不是只沿单一 best chain。",
"什么在变": "coding-agent code/harness。",
"谁来改 / 谁执行": "**改**：archive 中 candidate agents 自己生成 descendants。
**执行**：descendant agents。",
"基础 harness": "self-mod coding agent seed。",
"Feedback": "SWE/Polyglot executable tests + performance。",
"Evolution → Eval": "SWE/Polyglot search；部分 final Polyglot 覆盖更多未用于 search tasks。",
"Meta-depth": "M2。",
"相对之前真正新增什么": "相对 SICA 的新增主要是 **open-ended archive / diversity**，降低 greedy self-improvement 卡死在局部最优。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "混合协议",
"protocolBasis": "不同实验的划分不同，不能用一个 held-out 标签概括；见逐任务协议。",
"year": "2025",
"depth": [
"M2"
],
"legacyCategories": [
"F",
"H-Full",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "archive 中的 coding agent 修改自己的代码形成后代，后代经过 coding benchmark 评估后留在 archive，后续可再次作为修改者。",
"novelty": "让已有 coding agent 修改自身代码，并保留不同历史分支；暂时分数不高的版本也可能成为后来改进的起点。",
"object": "coding agent 的代码与 harness，包括任务执行方式和工具使用逻辑。基础模型参数、外层父代选择和评测器固定。",
"executor": "SWE-bench用Claude 3.5 Sonnet（New）；Polyglot用o3-mini。两者均由档案中的候选 task agent 代码组织执行，模型权重冻结。",
"modifier": "o1 读取评估日志提出改进问题，父版本 编程 task agent 用 Claude 3.5 Sonnet 实施源码修改。",
"roleContext": "**改**：archive 中 candidate agents 自己生成 descendants。
**执行**：descendant agents。",
"seed": "轻量编程 task agent，只有 Bash 和查看/编辑文件两个工具；输入包括仓库位置、问题和测试框架。",
"fixed": "本实验不训练 foundation model；outer parent selection / evaluator 仍固定。",
"verdict": "在 SWE-bench 和 Polyglot 中运行代码测试，判断修复是否成功；诊断者结合测试结果和执行日志提出改进建议。",
"diagnosis": "从自己的 benchmark failures 和 codebase 找可改进能力。",
"update": "修改 agent 实现，再让后代继续执行并参与后续 self-modification。",
"acceptance": "评估后加入 archive，以分数和探索策略影响 parent selection。",
"experiments": [
{
"name": "SWE / Polyglot",
"evolve": "benchmark 子集上演化",
"selection": "archive benchmark scores",
"test": "包含更大/full-set 与跨语言设置",
"isolation": "混合协议",
"note": "full-set 包含原训练题时不能称完全 disjoint test。"
}
],
"takeaway": "self-mod prompt/selection/evaluator仍 fixed；SWE 泛化隔离有限。 **对我们：** 核心启发不是“能改代码”，而是 evolution 需要 population/stepping-stone preservation。",
"result": "SWE 约20→50、Polyglot 14.2→30.7；open-ended archive ablation 显著。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2505.22954v3"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SWE-bench：先 10 题，再加 50 题，强候选扩到共 200 题；Polyglot：10 题门控，再评另外 50 题。\n\n调试 / 选版本数据：上述搜索子集的运行日志和成绩用于诊断、采样父代和筛选候选。\n\n最终测试数据：报告同域成绩及 full Polyglot；另将 SWE-bench 进化版本迁移到 Polyglot，反向亦然，并测 Python→其他语言。\n\n数据隔离与证据边界：同域扩展集合含搜索题；跨评测基准实验才是该次进化从未接触的任务来源。",
"cycle": "独立的 o1 模型诊断评估日志并提出建议，修改者实现新 agent 版本，再运行编程任务。候选必须能编译并保留编辑仓库的能力才能入档；不要求每个后代立即超过父代，较弱版本也可成为后续改进的起点。",
"train": "SWE-bench：先 10 题，再加 50 题，强候选扩到共 200 题；Polyglot：10 题门控，再评另外 50 题。",
"debug": "上述搜索子集的运行日志和成绩用于诊断、采样父代和筛选候选。",
"test": "报告同域成绩及 full Polyglot；另将 SWE-bench 进化版本迁移到 Polyglot，反向亦然，并测 Python→其他语言。",
"isolation": "同域扩展集合含搜索题；跨评测基准实验才是该次进化从未接触的任务来源。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "一个可编辑自己 Python 源码的编程 agent，配有仓库/命令执行接口。每代从档案选父版本，由父 agent 生成自身修改，再执行 benchmark 决定是否加入档案；不是人固定所有改进规则后只微调提示。",
"protocol": "**进化：**在 SWE-bench 或 Polyglot 的搜索子集上改 agent，按执行成绩维护档案；更大集合的同域报告不能自动算全新任务测试。\n\n**明确留出迁移：**SWE-bench 上进化的版本转到从未用于该次搜索的 Polyglot，反向亦然；另有仅用 Polyglot Python 任务进化后测其他语言。各搜索子集数量本轮待核实，跨 benchmark 结果与含训练题的 full-set 结果分别解释。",
"sections": "算法与跨 benchmark/语言迁移",
"source": "https://arxiv.org/abs/2505.22954",
"version": "2505.22954v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "06f675e0edc49dcdb27a04d1dfe82f112dd719b6755ad28387a32551fdf91d7b",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "SWE-bench用Claude 3.5 Sonnet（New）；Polyglot用o3-mini。两者均由档案中的候选 task agent 代码组织执行，模型权重冻结。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2505.22954#A4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "o1 读取评估日志提出改进问题，父版本 编程 task agent 用 Claude 3.5 Sonnet 实施源码修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "coding agent 的代码与 harness，包括任务执行方式和工具使用逻辑。基础模型参数、外层父代选择和评测器固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "在 SWE-bench 和 Polyglot 中运行代码测试，判断修复是否成功；诊断者结合测试结果和执行日志提出改进建议。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "轻量编程 task agent，只有 Bash 和查看/编辑文件两个工具；输入包括仓库位置、问题和测试框架。",
"sources": [
{
"label": "算法与跨 benchmark/语言迁移",
"url": "https://arxiv.org/abs/2505.22954"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "独立的 o1 模型诊断评估日志并提出建议，修改者实现新 agent 版本，再运行编程任务。候选必须能编译并保留编辑仓库的能力才能入档；不要求每个后代立即超过父代，较弱版本也可成为后续改进的起点。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SWE-bench：先 10 题，再加 50 题，强候选扩到共 200 题；Polyglot：10 题门控，再评另外 50 题。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "上述搜索子集的运行日志和成绩用于诊断、采样父代和筛选候选。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "报告同域成绩及 full Polyglot；另将 SWE-bench 进化版本迁移到 Polyglot，反向亦然，并测 Python→其他语言。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同域扩展集合含搜索题；跨评测基准实验才是该次进化从未接触的任务来源。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让已有 coding agent 修改自身代码，并保留不同历史分支；暂时分数不高的版本也可能成为后来改进的起点。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2505.22954v3",
"version": "2505.22954v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "保留不立即最优的 stepping stones 与多样分支；不是普通固定外部 proposer 的单链 greedy search。",
"feedbackCases": [
{
"label": "进化与同域评测：SWE-bench",
"data": "先检查 10 题，再增加 50 题，强候选扩至共 200 题。",
"scoring": "用仓库测试判断候选 agent 生成的补丁是否修复问题，计算解题比例。",
"visible": "父代 agent 读取自己的评测日志与得分，据失败提出功能改进；代码能运行且保留编辑能力才入库。",
"use": "分数用于选父代及评价子代；这些搜索子集成绩不是纯独立测试。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"judgment": "SWE-bench 仓库测试检查补丁功能"
},
{
"label": "进化 / 迁移：Polyglot",
"data": "搜索先 10 题门控，再另外 50 题；报告完整多语言基准及与 SWE-bench 之间的迁移。",
"scoring": "按题目的代码测试判对错。作者采用 pass@1（单次尝试完成任务的比例）：任务执行 agent 不能看到正式测试结果后再修答案。",
"visible": "外层改进过程使用 agent 评测日志；与单题执行者能否访问正式测试须分开。",
"use": "比较进化 agent 及跨语言/跨基准迁移；不能按 Aider 榜单的“看一次正式测试再重答”协议解释。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"judgment": "Polyglot 编程题自带测试检查代码输出"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "SWE-bench：先 10 题，再加 50 题，强候选扩到共 200 题；Polyglot：10 题门控，再评另外 50 题。",
"selection": "上述搜索子集的运行日志和成绩用于诊断、采样父代和筛选候选。",
"evaluation": "报告同域成绩及 full Polyglot；另将 SWE-bench 进化版本迁移到 Polyglot，反向亦然，并测 Python→其他语言。",
"isolation": "同域扩展集合含搜索题；跨评测基准实验才是该次进化从未接触的任务来源。",
"roles": {
"executor": {
"value": "SWE-bench用Claude 3.5 Sonnet（New）；Polyglot用o3-mini。两者均由档案中的候选 task agent 代码组织执行，模型权重冻结。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2505.22954#A4.SS1"
}
]
},
"modifier": {
"value": "o1 读取评估日志提出改进问题，父版本 编程 task agent 用 Claude 3.5 Sonnet 实施源码修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
]
},
"seed": {
"value": "轻量编程 task agent，只有 Bash 和查看/编辑文件两个工具；输入包括仓库位置、问题和测试框架。",
"sources": [
{
"label": "算法与跨 benchmark/语言迁移",
"url": "https://arxiv.org/abs/2505.22954"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"selection": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"evaluation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
],
"isolation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.22954#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2505.22954#S4.SS4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2505.22954#A5.SS2"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2505.22954#A5.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "多数AI仍在人工预设的架构边界内学习，不能自主改写自身系统。Gödel Machine要求先证明改动有益，但作者认为这在实际复杂AI系统中难以做到，因而需要用可执行的实证评估支持自修改，并保留能成为未来改进起点的多样版本。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2505.22954#S1"
}
]
},
{
"key": "position",
"value": "探索 coding agent 能否减少对人工系统设计的依赖，通过持续改进自身实现，逐步提升解题能力及进一步自改进的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.22954"
}
]
},
{
"key": "conclusion",
"value": "SWE-bench 从 20% 提升到 50%；开放档案允许暂时不占优的方案成为后续改进的起点。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2505.22954"
}
]
}
],
"fields": {
"object": "coding agent 的代码与 harness，包括任务执行方式和工具使用逻辑。基础模型参数、外层父代选择和评测器固定。",
"verdict": "在 SWE-bench 和 Polyglot 中运行代码测试，判断修复是否成功；诊断者结合测试结果和执行日志提出改进建议。"
}
},
"attributions": [
{
"tag": "org:ubc",
"label": "University of British Columbia",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2505.22954"
}
]
},
{
"tag": "org:sakana",
"label": "Sakana AI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2505.22954"
}
]
},
{
"tag": "person:jeff-clune",
"label": "Jeff Clune",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2505.22954"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness",
"improver"
]
},
{
"id": "2511.10395",
"title": "AgentEvolver: Towards Efficient Self-Evolving Agent System",
"url": "https://arxiv.org/abs/2511.10395",
"date": "2025-11-13",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Data",
"EnvironmentReward",
"LLMJudge",
"M1",
"Weights"
],
"fields": {
"本质定位": "把 self-evolving agent 训练拆成 Self-Questioning（自生成 task）、Self-Navigating（利用历史经验探索）、Self-Attributing（细粒度 credit），形成 2025–26 很有代表性的 training-loop self-evolution。",
"什么在变": "policy weights + 自生成 task / experience / credit strategy；deployment harness不是主要对象。",
"谁来改 / 谁执行": "**改**：固定 RL/training pipeline + 多个 LLM 辅助模块。
**执行**：Qwen2.5-7B/14B policy。",
"基础 harness": "标准 agent policy / AppWorld、BFCL interaction scaffold。",
"Feedback": "environment outcome + LLM judge / attribution。",
"Evolution → Eval": "AppWorld、BFCL v3；训练与最终评估按论文 split。",
"Meta-depth": "model-level M1；outer learning algorithm fixed。",
"相对之前真正新增什么": "代表性在于把“数据从哪来、怎么探索、怎么分 credit”统一成 self-evolving training system；它是后续 RSI/model-level self-evolution 的重要基线，而不是 harness-only 方法。"
},
"details": [
{
"label": "本质定位",
"text": "不是 deployment harness evolution，而是 **agent training-loop self-evolution**：同时解决 task data、trajectory exploration、credit assignment 三个瓶颈。"
},
{
"label": "什么在变",
"text": "Qwen2.5 policy weights；同时训练过程持续自生成 tasks、复用 experience、产生更细粒度 credit。"
},
{
"label": "谁来改 / 谁执行",
"text": "执行者是 Qwen2.5-7B/14B policy；外层 Self-Questioning / Navigating / Attributing 与 RL pipeline 固定，另使用外部 LLM 做 synthesis/judge/attribution。"
},
{
"label": "基础 harness",
"text": "AppWorld / BFCL agent interaction scaffold；不是把 runtime harness code 当 edit target。"
},
{
"label": "Feedback",
"text": "environment outcome + judge / attribution signal。"
},
{
"label": "核心结果",
"text": "旧调研记录中：avg@8 7B **15.8→45.2**，14B **29.8→57.6**；Self-Questioning 是最大的第一步 gain，三模块联合最好。"
},
{
"label": "为什么经典",
"text": "它是“self-evolving agents”这波里非常早、传播很广的完整训练框架之一；之后许多工作都会把“自生成经验→利用经验→写回 policy”作为 model-level 对照。"
},
{
"label": "边界",
"text": "improver/training algorithm 本身并没有递归改变；严格说更像 model-level adaptive RL，而不是 harness RSI。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 44,
"fields": {
"时间": "2025-11",
"论文": "**[AgentEvolver](https://arxiv.org/abs/2511.10395)**",
"为什么是主干": "2025–26 “self-evolving agent training loop” 的高影响代表：task generation、experience navigation、credit assignment 一体化。",
"证据边界要记住": "主要更新 policy/data/credit loop，不是 deployment harness evolution。"
}
},
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 346,
"fields": {
"优先级": "**C**",
"时间": "2025-11-13",
"论文": "[AgentEvolver: Towards Efficient Self-Evolving Agent System](https://arxiv.org/abs/2511.10395)",
"本质定位": "把 self-evolving agent 训练拆成 Self-Questioning（自生成 task）、Self-Navigating（利用历史经验探索）、Self-Attributing（细粒度 credit），形成 2025–26 很有代表性的 training-loop self-evolution。",
"什么在变": "policy weights + 自生成 task / experience / credit strategy；deployment harness不是主要对象。",
"谁来改 / 谁执行": "**改**：固定 RL/training pipeline + 多个 LLM 辅助模块。
**执行**：Qwen2.5-7B/14B policy。",
"基础 harness": "标准 agent policy / AppWorld、BFCL interaction scaffold。",
"Feedback": "environment outcome + LLM judge / attribution。",
"Evolution → Eval": "AppWorld、BFCL v3；训练与最终评估按论文 split。",
"Meta-depth": "model-level M1；outer learning algorithm fixed。",
"相对之前真正新增什么": "代表性在于把“数据从哪来、怎么探索、怎么分 credit”统一成 self-evolving training system；它是后续 RSI/model-level self-evolution 的重要基线，而不是 harness-only 方法。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"M-Weight",
"M-Data"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "把 self-evolving agent 训练拆成 Self-Questioning（自生成 task）、Self-Navigating（利用历史经验探索）、Self-Attributing（细粒度 credit），形成 2025–26 很有代表性的 training-loop self-evolution。",
"novelty": "将自主生成任务、利用已有经验探索、判断哪些步骤贡献了奖励接入参数训练，分别处理学习材料、探索效率和训练信号的问题。",
"object": "策略模型参数，以及训练中生成的任务和经验；环境接口与训练规则固定。",
"executor": "Qwen2.5-7B-Instruct和Qwen2.5-14B-Instruct，经过本文训练后执行AppWorld/BFCL任务。",
"modifier": "固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）训练程序更新Qwen2.5-7B/14B；Self-Questioning的探索和任务合成用Qwen-Plus，任务评分者用Qwen3-235B-A22B。步骤归因另见Self-Attributing配置。",
"roleContext": "**改**：固定 RL/training pipeline + 多个 LLM 辅助模块。
**执行**：Qwen2.5-7B/14B policy。",
"seed": "以可训练 task agent 的行为策略接入 AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 和 BFCL 的工具交互，系统自行构造探索任务、利用已有成功经验导航，并给轨迹分配训练信用。基础工具环境固定，主要训练行为策略，而非随意改底层 API。",
"fixed": "outer learning algorithm fixed",
"verdict": "最终任务结果用于训练；Qwen-Max 另对整条轨迹的各步标记好坏，把最终收益分配到具体步骤。合成任务的评审由 Qwen3-235B-A22B 承担，两种评审角色不同。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "AppWorld、BFCL v3；训练与最终评估按论文 split。"
}
],
"takeaway": "improver/training algorithm 本身并没有递归改变；严格说更像 model-level adaptive RL，而不是 harness RSI。",
"result": "旧调研记录中：avg@8 7B **15.8→45.2**，14B **29.8→57.6**；Self-Questioning 是最大的第一步 gain，三模块联合最好。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：在 AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、BFCL 环境探索后合成任务。消融分别比较 100/200/500 条合成数据；它们不是三套最终测试集。\n\n调试 / 选版本数据：自建任务由 Qwen3-235B-A22B 评判并去重；经验质量由 语言模型 校验。\n\n最终测试数据：AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 任务完成率 TGC；BFCL v3 的 multi-turn split，检查后端状态及执行路径。主实验每题 8 次运行，报告 avg@8/best@8。\n\n数据隔离与证据边界：§7.1 指定环境与 BFCL 子集，但没有给出可复现的合成题—最终评估题逐题去重清单；不能据环境相同断言题目重叠。",
"cycle": "Qwen-Max 阅读整条轨迹，给各步标好坏，将步骤贡献与最终任务奖励结合。固定训练程序据此更新 Qwen2.5-7B/14B；合成任务和经验检索帮助收集有效训练经历，模型不自行改写训练算法。",
"train": "在 AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、BFCL 环境探索后合成任务。消融分别比较 100/200/500 条合成数据；它们不是三套最终测试集。",
"debug": "自建任务由 Qwen3-235B-A22B 评判并去重；经验质量由 语言模型 校验。",
"test": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 任务完成率 TGC；BFCL v3 的 multi-turn split，检查后端状态及执行路径。主实验每题 8 次运行，报告 avg@8/best@8。",
"isolation": "§7.1 指定环境与 BFCL 子集，但没有给出可复现的合成题—最终评估题逐题去重清单；不能据环境相同断言题目重叠。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "以可训练 agent policy 接入 AppWorld 和 BFCL 的工具交互，系统自行构造探索任务、利用已有成功经验导航，并给轨迹分配训练信用。基础工具环境固定，主要训练行为策略，而非随意改底层 API。",
"protocol": "**训练数据：**Self-Questioning 在 AppWorld、BFCL 环境中探索，再合成含两个实体、三个属性、三项操作的高难度任务；Qwen-Plus 负责探索和合成，Qwen3-235B-A22B 评判，词面相似度阈值 0.8 去重。主实验冷启动经验池也来自这些合成题，训练 Qwen2.5-7B/14B 策略。\n\n**正式评估：**AppWorld 以全部程序测试通过率 TGC 判分；BFCL v3 仅使用 multi-turn split，每回合同时检查后端状态和执行路径，强制终止算错。主结果是每题 8 次运行的 avg@8/best@8，最长 30 步；消融用 avg@4/best@4。\n\n**尚待核实：**合成训练池数量、AppWorld 的具体评估 split，以及训练任务与评估题的去重隔离细节；环境相同本身不能证明题目重叠或独立。",
"sections": "§6.3、§7.1.1–7.1.3",
"source": "https://arxiv.org/abs/2511.10395",
"version": "2511.10395v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "0f98f8349931da8cc79df3940579e573954aead91a9d4d2fc96947648b8333a6",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen2.5-7B-Instruct和Qwen2.5-14B-Instruct，经过本文训练后执行AppWorld/BFCL任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS2"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）训练程序更新Qwen2.5-7B/14B；Self-Questioning的探索和任务合成用Qwen-Plus，任务评分者用Qwen3-235B-A22B。步骤归因另见Self-Attributing配置。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS2"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "策略模型参数，以及训练中生成的任务和经验；环境接口与训练规则固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "最终任务结果用于训练；Qwen-Max 另对整条轨迹的各步标记好坏，把最终收益分配到具体步骤。合成任务的评审由 Qwen3-235B-A22B 承担，两种评审角色不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px3"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "以可训练 task agent 的行为策略接入 AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 和 BFCL 的工具交互，系统自行构造探索任务、利用已有成功经验导航，并给轨迹分配训练信用。基础工具环境固定，主要训练行为策略，而非随意改底层 API。",
"sources": [
{
"label": "§6.3、§7.1.1–7.1.3",
"url": "https://arxiv.org/abs/2511.10395"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "Qwen-Max 阅读整条轨迹，给各步标好坏，将步骤贡献与最终任务奖励结合。固定训练程序据此更新 Qwen2.5-7B/14B；合成任务和经验检索帮助收集有效训练经历，模型不自行改写训练算法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "在 AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、BFCL 环境探索后合成任务。消融分别比较 100/200/500 条合成数据；它们不是三套最终测试集。",
"sources": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "自建任务由 Qwen3-235B-A22B 评判并去重；经验质量由 语言模型 校验。",
"sources": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 任务完成率 TGC；BFCL v3 的 multi-turn split，检查后端状态及执行路径。主实验每题 8 次运行，报告 avg@8/best@8。",
"sources": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§7.1 指定环境与 BFCL 子集，但没有给出可复现的合成题—最终评估题逐题去重清单；不能据环境相同断言题目重叠。",
"sources": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将自主生成任务、利用已有经验探索、判断哪些步骤贡献了奖励接入参数训练，分别处理学习材料、探索效率和训练信号的问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2511.10395v1",
"version": "2511.10395v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "代表性在于把“数据从哪来、怎么探索、怎么分 credit”统一成 self-evolving training system；它是后续 RSI/model-level self-evolution 的重要基线，而不是 harness-only 方法。",
"feedbackCases": [
{
"label": "自建任务与训练奖励",
"data": "在 AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、BFCL 的交互环境探索，生成新的训练任务；100/200/500 是合成数据量消融。",
"scoring": "Qwen3-235B-A22B 参与合成任务评判与去重；另由 Qwen-Max 一次读取完整执行记录，标出各步 GOOD（好）/BAD（差），再结合任务最终结果形成训练奖励。",
"visible": "任务/经验质量反馈、最终任务结果和分步贡献评价。",
"use": "用于学习任务模型；任务质量评审、分步归因模型与最终基准判分必须分开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px3"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"judgment": "Qwen3-235B-A22B 审合成任务；Qwen-Max 评轨迹步骤，两者与终局任务奖励分开"
},
{
"label": "最终评估：AppWorld / BFCL v3",
"data": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 任务与 BFCL v3 多轮划分；每题八次运行。",
"scoring": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 使用任务目标完成率 TGC；BFCL 检查函数执行后的后端状态和执行路径。",
"visible": "报告 avg@8 与 best@8；这些是真实任务完成指标，而非单独平均 Qwen-Max 的 G分布外任务/BAD 标签。",
"use": "判断训练后策略是否更能完成官方任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px3"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"judgment": "AppWorld 程序验收目标；BFCL 程序检查状态与执行路径"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "在 AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、BFCL 环境探索后合成任务。消融分别比较 100/200/500 条合成数据；它们不是三套最终测试集。",
"selection": "自建任务由 Qwen3-235B-A22B 评判并去重；经验质量由 语言模型 校验。",
"evaluation": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 任务完成率 TGC；BFCL v3 的 multi-turn split，检查后端状态及执行路径。主实验每题 8 次运行，报告 avg@8/best@8。",
"isolation": "§7.1 指定环境与 BFCL 子集，但没有给出可复现的合成题—最终评估题逐题去重清单；不能据环境相同断言题目重叠。",
"roles": {
"executor": {
"value": "Qwen2.5-7B-Instruct和Qwen2.5-14B-Instruct，经过本文训练后执行AppWorld/BFCL任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS2"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
}
]
},
"modifier": {
"value": "固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）训练程序更新Qwen2.5-7B/14B；Self-Questioning的探索和任务合成用Qwen-Plus，任务评分者用Qwen3-235B-A22B。步骤归因另见Self-Attributing配置。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS2"
},
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS3.Px1"
}
]
},
"seed": {
"value": "以可训练 task agent 的行为策略接入 AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 和 BFCL 的工具交互，系统自行构造探索任务、利用已有成功经验导航，并给轨迹分配训练信用。基础工具环境固定，主要训练行为策略，而非随意改底层 API。",
"sources": [
{
"label": "§6.3、§7.1.1–7.1.3",
"url": "https://arxiv.org/abs/2511.10395"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"selection": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"evaluation": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
],
"isolation": [
{
"label": "§7.1",
"url": "https://arxiv.org/html/2511.10395#S7.SS1.SSS1"
},
{
"label": "§7.3",
"url": "https://arxiv.org/html/2511.10395#S7.SS3.SSS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 进入新环境后，人工构造强化学习任务昂贵；大量盲目采样又产生重复、低学习价值的轨迹，长程工具调用进一步放大计算与金钱成本。作者因此关注让模型主动组织学习任务、利用经验探索并提高轨迹利用率，减少适应新环境所需的人工和样本。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2511.10395#S1"
}
]
},
{
"key": "position",
"value": "面向工具交互任务，研究如何减少人工构造训练任务与低效探索的成本，使 agent 更高效地自主学习新环境中的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2511.10395"
}
]
},
{
"key": "conclusion",
"value": "三种机制共同提高探索及训练效率；改进发生在数据生成、经验利用和参数学习的整条流程中。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2511.10395"
}
]
}
],
"fields": {
"object": "策略模型参数，以及训练中生成的任务和经验；环境接口与训练规则固定。",
"verdict": "最终任务结果用于训练；Qwen-Max 另对整条轨迹的各步标记好坏，把最终收益分配到具体步骤。合成任务的评审由 Qwen3-235B-A22B 承担，两种评审角色不同。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"curriculum"
]
},
{
"id": "2603.19461",
"title": "Hyperagents",
"url": "https://arxiv.org/abs/2603.19461",
"date": "2026-03-19",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Improver",
"M2",
"org:meta-superintelligence",
"org:meta-fair",
"org:ubc",
"person:jeff-clune"
],
"fields": {
"本质定位": "把 task agent 与 editable meta-agent 放入同一 program；meta-agent 不只改 task agent，也能修改自己的 self-modification procedure，并把 meta-improvements 跨 domain transfer。",
"什么在变": "task-agent code + meta-agent code/self-mod procedure。",
"谁来改 / 谁执行": "**改**：editable meta-agent。
**执行**：task agent + meta-agent 同一 program。",
"基础 harness": "DGM-H / task+meta agent seed。",
"Feedback": "domain task reward / evaluator；不同 domain 用不同 execution/judge。",
"Evolution → Eval": "Polyglot、paper review、robotics reward design、IMO grading；有 largely unseen/full-set 与 cross-domain transfer。",
"Meta-depth": "M2 强；outer parent-selection/evaluator 在主实验仍 fixed。",
"相对之前真正新增什么": "相对 DGM 的核心新点：coding domain 中“会写代码→会改自己”天然对齐，Hyperagents 显式拆出/可编辑 meta-agent，验证 **改进能力本身**能跨非 coding domain 迁移。"
},
"details": [
{
"label": "本质定位",
"text": "把 task agent 与 meta-agent 放进同一个 editable program；**meta-agent 可以修改“如何修改 agent”的 procedure**，直接触及 improver-level RSI。"
},
{
"label": "什么在变",
"text": "task-agent code + meta-agent code / self-modification procedure。"
},
{
"label": "谁来改 / 谁执行",
"text": "editable meta-agent 产生修改；task agent 执行 domain task。官方实现允许 meta-agent 修改 repo 任意部分，包括自身 prompt/code。"
},
{
"label": "Feedback",
"text": "domain-specific empirical utility / evaluator + archive selection。"
},
{
"label": "核心新点",
"text": "相比 DGM，最重要不是“又会改 agent code”，而是把 DGM 中相对隐含/固定的 modification procedure 显式做成 editable meta-agent，并测跨 coding、paper review、robotics、grading 的迁移。"
},
{
"label": "为什么经典",
"text": "它几乎就是“improver improvement”这一分支的定义性工作之一；和我们的 meta-depth / modifier 是否也在变问题直接对应。"
},
{
"label": "边界",
"text": "outer parent selection、archive、evaluator 仍固定，因此还不是 full M3 self-reference。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 45,
"fields": {
"时间": "2026-03",
"论文": "**[Hyperagents](https://arxiv.org/abs/2603.19461)**",
"为什么是主干": "明确把 **modification procedure 本身**放入 editable program，是 improver-level RSI 的代表。",
"证据边界要记住": "evaluator/archive/outer selection 仍大体固定。"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 325,
"fields": {
"优先级": "**C**",
"时间": "2026-03-19",
"论文": "[Hyperagents](https://arxiv.org/abs/2603.19461)",
"本质定位": "把 task agent 与 editable meta-agent 放入同一 program；meta-agent 不只改 task agent，也能修改自己的 self-modification procedure，并把 meta-improvements 跨 domain transfer。",
"什么在变": "task-agent code + meta-agent code/self-mod procedure。",
"谁来改 / 谁执行": "**改**：editable meta-agent。
**执行**：task agent + meta-agent 同一 program。",
"基础 harness": "DGM-H / task+meta agent seed。",
"Feedback": "domain task reward / evaluator；不同 domain 用不同 execution/judge。",
"Evolution → Eval": "Polyglot、paper review、robotics reward design、IMO grading；有 largely unseen/full-set 与 cross-domain transfer。",
"Meta-depth": "M2 强；outer parent-selection/evaluator 在主实验仍 fixed。",
"相对之前真正新增什么": "相对 DGM 的核心新点：coding domain 中“会写代码→会改自己”天然对齐，Hyperagents 显式拆出/可编辑 meta-agent，验证 **改进能力本身**能跨非 coding domain 迁移。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 task agent 与可编辑 meta-agent 放入一个 program。meta-agent 不只改任务求解器，还能改产生后续 modifications 的程序。",
"novelty": "把 task agent 和 meta-agent 的代码都设为可修改对象；再把改进程序迁移到新领域，检验它是否学会了通用的 agent 修改方法。",
"object": "两层代码：执行任务的 agent，以及负责修改它的 meta-agent。后者的改进程序也能被修改；外层父代选择和评测协议固定。",
"executor": "按任务分开：Polyglot 编程由 o3-mini 执行；论文评审由 GPT-4o 执行；机器人奖励函数由 Claude 4.5 Sonnet 编写并交给模拟器运行；奥数解答评分由 o4-mini 执行。",
"modifier": "Polyglot 的自修改模型是 Claude 3.5 Sonnet (New)；其余三个领域均用 Claude 4.5 Sonnet 修改 task agent 和 meta-agent 代码。",
"roleContext": "**改**：editable meta-agent。
**执行**：task agent + meta-agent 同一 program。",
"seed": "DGM-H 同时保存可编辑任务程序和改进程序。论文评审/机器人奖励设计的共同起点仅直接返回一次模型调用结果，没有任务专用解析或后处理，因此初始格式错误可能直接导致零分；不是从成熟领域 task agent 开始。",
"fixed": "外层 parent selection / evaluation protocols 仍固定。",
"verdict": "反馈随领域变化：编程看执行测试，机器人看环境任务收益，论文评审和数学评分看评价结果。修改者读取相应成绩及执行日志，详细任务协议分行列在表格中。",
"diagnosis": "根据 task performance 与 meta-level improvement 证据改 task/meta 程序。",
"update": "同时开放 task-agent 与 modification procedure 的代码。",
"acceptance": "以 train/validation 选择 best agent，再在相应测试协议评估。",
"experiments": [
{
"name": "Polyglot",
"evolve": "60 tasks：先 10 题门控，再测剩余 50",
"selection": "training score",
"test": "full Polyglot，包含大量未见题但非完全 disjoint",
"isolation": "混合协议",
"note": ""
},
{
"name": "Review / Robotics / IMO grading",
"evolve": "domain train tasks",
"selection": "validation；没有 validation 时用 train",
"test": "domain test；另做 meta-agent transfer / improvement@k",
"isolation": "独立测试",
"note": "固定 meta-agent 的 transfer 实验与继续 self-improve 是不同对照。"
}
],
"takeaway": "outer parent selection、archive、evaluator 仍固定，因此还不是 full M3 self-reference。",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2603.19461v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：论文评审：100 篇训练文章；Polyglot：60 道搜索题；机器人：Go2 前进任务；迁移后的 IMO 评分：100 道训练题。\n\n调试 / 选版本数据：评审及 IMO 各另有 100 道 验证集；Polyglot 与机器人无单独 验证集，以训练成绩选版本。\n\n最终测试数据：评审及 IMO 各 100 道 测试集；机器人改测抬高躯干；Polyglot 报 full 评测基准，原文同时称含 165 个未见任务。\n\n数据隔离与证据边界：评审/IMO 三段划分；机器人跨目标。Polyglot 的 full 评测基准与未见部分口径需区分；IMO 全集附加结果也不等于独立测试子集。",
"cycle": "上层修改者读取解题程序、改进程序以及历次成绩和日志，决定改哪一部分。两套程序都可编辑；用训练或验证成绩选版本，再按各领域的测试协议评价。",
"train": "论文评审：100 篇训练文章；Polyglot：60 道搜索题；机器人：Go2 前进任务；迁移后的 IMO 评分：100 道训练题。",
"debug": "评审及 IMO 各另有 100 道 验证集；Polyglot 与机器人无单独 验证集，以训练成绩选版本。",
"test": "评审及 IMO 各 100 道 测试集；机器人改测抬高躯干；Polyglot 报 full 评测基准，原文同时称含 165 个未见任务。",
"isolation": "评审/IMO 三段划分；机器人跨目标。Polyglot 的 full 评测基准与未见部分口径需区分；IMO 全集附加结果也不等于独立测试子集。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "DGM-H 同时保存可编辑任务程序和改进程序。论文评审/机器人奖励设计的共同起点仅直接返回一次模型调用结果，没有任务专用解析或后处理，因此初始格式错误可能直接导致零分；不是从成熟领域 agent 开始。",
"protocol": "**论文评审：**来自实际会议接收/拒绝数据，train/validation/test 各 100 题，先以 10 题小批门控再评完整训练集。\n\n**Polyglot：**60 个搜索任务，先测 10 题、超过阈值再测剩余 50；该域没有独立 validation。原文同时称最终评估 full benchmark，并提到 165 个未见任务，应区分完整集合与未见部分，不把 50 误写成全部训练量。另有机器人奖励设计和 IMO 评分任务，逐域数量本轮待核实。",
"sections": "初始 agent；各领域数据设置",
"source": "https://arxiv.org/abs/2603.19461",
"version": "2603.19461v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "865f3a94a88bb0357b376a59c8c93150695f24fef8761b6f9e55062bdef85575",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "按任务分开：Polyglot 编程由 o3-mini 执行；论文评审由 GPT-4o 执行；机器人奖励函数由 Claude 4.5 Sonnet 编写并交给模拟器运行；奥数解答评分由 o4-mini 执行。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2603.19461#A4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Polyglot 的自修改模型是 Claude 3.5 Sonnet (New)；其余三个领域均用 Claude 4.5 Sonnet 修改 task agent 和 meta-agent 代码。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2603.19461#A4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "两层代码：执行任务的 agent，以及负责修改它的 meta-agent。后者的改进程序也能被修改；外层父代选择和评测协议固定。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "反馈随领域变化：编程看执行测试，机器人看环境任务收益，论文评审和数学评分看评价结果。修改者读取相应成绩及执行日志，详细任务协议分行列在表格中。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "DGM-H 同时保存可编辑任务程序和改进程序。论文评审/机器人奖励设计的共同起点仅直接返回一次模型调用结果，没有任务专用解析或后处理，因此初始格式错误可能直接导致零分；不是从成熟领域 task agent 开始。",
"sources": [
{
"label": "初始 agent；各领域数据设置",
"url": "https://arxiv.org/abs/2603.19461"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "上层修改者读取解题程序、改进程序以及历次成绩和日志，决定改哪一部分。两套程序都可编辑；用训练或验证成绩选版本，再按各领域的测试协议评价。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "论文评审：100 篇训练文章；Polyglot：60 道搜索题；机器人：Go2 前进任务；迁移后的 IMO 评分：100 道训练题。",
"sources": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "评审及 IMO 各另有 100 道 验证集；Polyglot 与机器人无单独 验证集，以训练成绩选版本。",
"sources": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "评审及 IMO 各 100 道 测试集；机器人改测抬高躯干；Polyglot 报 full 评测基准，原文同时称含 165 个未见任务。",
"sources": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "评审/IMO 三段划分；机器人跨目标。Polyglot 的 full 评测基准与未见部分口径需区分；IMO 全集附加结果也不等于独立测试子集。",
"sources": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把 task agent 和 meta-agent 的代码都设为可修改对象；再把改进程序迁移到新领域，检验它是否学会了通用的 agent 修改方法。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.19461v1",
"version": "2603.19461v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "显式把“改进能力”拆出来，并用新 domain 上生成更好 task agent 的能力测试 meta-level transfer。",
"feedbackCases": [
{
"label": "论文接收预测",
"data": "来自公开会议投稿与接收结果的数据，训练/验证/测试各 100 篇",
"scoring": "把 agent 预测的接收/拒绝与真实会议接收结果比较；这是对人类决定的拟合，不是论文质量的客观真值。",
"visible": "任务执行结果与评测记录供上层 agent 分析并修改任务程序和改进程序。",
"use": "论文评审与 IMO 用单独验证成绩选版本；Polyglot、机器人用训练表现选版本。最终报告需按各自留出设置解释。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"judgment": "规则比较预测接收／拒绝标签与会议真实结果"
},
{
"label": "IMO-GradingBench",
"data": "问题、候选证明、参考解与评分指南；迁移后训练/验证/测试各 100 题",
"scoring": "比较 agent 给出的离散分数与专家人类评分，按匹配准确率评价。",
"visible": "任务执行结果与评测记录供上层 agent 分析并修改任务程序和改进程序。",
"use": "论文评审与 IMO 用单独验证成绩选版本；Polyglot、机器人用训练表现选版本。最终报告需按各自留出设置解释。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"judgment": "规则比较 agent 输出的分数与人类专家分数；agent 本身负责评证明"
},
{
"label": "Polyglot",
"data": "60 道搜索题；最终评完整基准，文中称有 165 道未见题",
"scoring": "用编程题测试检查答案是否正确，按任务通过率评分。",
"visible": "任务执行结果与评测记录供上层 agent 分析并修改任务程序和改进程序。",
"use": "论文评审与 IMO 用单独验证成绩选版本；Polyglot、机器人用训练表现选版本。最终报告需按各自留出设置解释。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"judgment": "Polyglot 的可执行代码测试"
},
{
"label": "Go2 机器人奖励设计",
"data": "进化目标为前进；迁移目标为抬高躯干",
"scoring": "执行 agent 写的奖励函数并训练机器人策略，再以环境任务表现评价这个奖励设计。",
"visible": "任务执行结果与评测记录供上层 agent 分析并修改任务程序和改进程序。",
"use": "论文评审与 IMO 用单独验证成绩选版本；Polyglot、机器人用训练表现选版本。最终报告需按各自留出设置解释。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"judgment": "实际训练机器人策略，再由环境指标评价奖励函数"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "论文评审：100 篇训练文章；Polyglot：60 道搜索题；机器人：Go2 前进任务；迁移后的 IMO 评分：100 道训练题。",
"selection": "评审及 IMO 各另有 100 道 验证集；Polyglot 与机器人无单独 验证集，以训练成绩选版本。",
"evaluation": "评审及 IMO 各 100 道 测试集；机器人改测抬高躯干；Polyglot 报 full 评测基准，原文同时称含 165 个未见任务。",
"isolation": "评审/IMO 三段划分；机器人跨目标。Polyglot 的 full 评测基准与未见部分口径需区分；IMO 全集附加结果也不等于独立测试子集。",
"roles": {
"executor": {
"value": "按任务分开：Polyglot 编程由 o3-mini 执行；论文评审由 GPT-4o 执行；机器人奖励函数由 Claude 4.5 Sonnet 编写并交给模拟器运行；奥数解答评分由 o4-mini 执行。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2603.19461#A4.SS1"
}
]
},
"modifier": {
"value": "Polyglot 的自修改模型是 Claude 3.5 Sonnet (New)；其余三个领域均用 Claude 4.5 Sonnet 修改 task agent 和 meta-agent 代码。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2603.19461#A4.SS1"
}
]
},
"seed": {
"value": "DGM-H 同时保存可编辑任务程序和改进程序。论文评审/机器人奖励设计的共同起点仅直接返回一次模型调用结果，没有任务专用解析或后处理，因此初始格式错误可能直接导致零分；不是从成熟领域 task agent 开始。",
"sources": [
{
"label": "初始 agent；各领域数据设置",
"url": "https://arxiv.org/abs/2603.19461"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"selection": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"evaluation": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
],
"isolation": [
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2603.19461#A3.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.19461#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2603.19461#A3.SS3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2603.19461#A3.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "如果修改者的程序固定，增加一层修改者仍不能让系统学习新的改进方法。DGM 还假设任务技能能帮助系统修改自身，但这一假设在编程之外未必成立，因此需要检验改进程序能否自我修改并跨任务迁移。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.19461#S1"
}
]
},
{
"key": "position",
"value": "研究能否同时改进 task agent 的解题代码与 meta-agent 的修改程序，并将学到的修改方法迁移到编程之外的任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.19461"
}
]
},
{
"key": "conclusion",
"value": "在编程、评审、奖励设计和数学评分中改进；修改机制本身积累的经验可以跨领域迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.19461"
}
]
}
],
"fields": {
"object": "两层代码：执行任务的 agent，以及负责修改它的 meta-agent。后者的改进程序也能被修改；外层父代选择和评测协议固定。",
"verdict": "反馈随领域变化：编程看执行测试，机器人看环境任务收益，论文评审和数学评分看评价结果。修改者读取相应成绩及执行日志，详细任务协议分行列在表格中。"
}
},
"attributions": [
{
"tag": "org:meta-superintelligence",
"label": "Meta Superintelligence Labs",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.19461"
}
]
},
{
"tag": "org:meta-fair",
"label": "Meta FAIR",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.19461"
}
]
},
{
"tag": "org:ubc",
"label": "University of British Columbia",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.19461"
}
]
},
{
"tag": "person:jeff-clune",
"label": "Jeff Clune",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2603.19461"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2603.28052",
"title": "Meta-Harness: End-to-End Optimization of Model Harnesses",
"url": "https://arxiv.org/abs/2603.28052",
"date": "2026-03-30",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HarnessCode",
"M1",
"MemoryContent",
"OfflineSearch",
"Prompt",
"StrongerBuilder",
"Workflow",
"org:stanford",
"org:mit",
"org:krafton",
"person:chelsea-finn",
"person:omar-khattab"
],
"fields": {
"本质定位": "Claude Opus 4.6 + Claude Code 直接读取所有历史 harness source/score/full traces，搜索完整 executable harness。",
"谁来改 → 谁执行；基础 harness": "stronger/separate coding proposer → frozen executor + current harness；seed 从简单 context/retrieval 到成熟 TB harness 不等。",
"Feedback / evidence": "search-set score + raw execution traces。",
"什么在变": "完整 executable harness。",
"谁来改 / 谁执行": "**改**：Claude Opus 4.6 + Claude Code proposer。
**执行**：domain-specific frozen base model + candidate harness。",
"基础 harness": "classification/math 从较简单 seed；TB2 从 Terminus/Terminus-KIRA strong harness。",
"Feedback": "search-set reward + full execution traces + historical harness code。",
"Evolution → Eval": "classification/math 有 held-out；TB2 89 tasks same-set search/eval。",
"Meta-depth": "M1 strong-to-weak/external modifier。",
"相对之前真正新增什么": "相对 ADAS/AFlow 更明确把 **model harness**（context/memory/retrieval/control flow/coding harness）当统一 edit object，并强调 raw-history observability。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "Claude Opus 4.6 + Claude Code 直接读取所有历史 harness source/score/full traces，搜索完整 executable harness。 **相对前序：** 相对 ADAS/AFlow 更明确把 **model harness**（context/memory/retrieval/control flow/coding harness）当统一 edit object，并强调 raw-history observability。"
},
{
"label": "什么在变",
"text": "完整 executable harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** Claude Opus 4.6 + Claude Code proposer。 **执行：** domain-specific frozen base model + candidate harness。"
},
{
"label": "基础 harness",
"text": "classification/math 从较简单 seed；TB2 从 Terminus/Terminus-KIRA strong harness。"
},
{
"label": "Feedback",
"text": "search-set reward + full execution traces + historical harness code。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "classification/math 有 held-out；TB2 89 tasks same-set search/eval。 **Meta-depth：** M1 strong-to-weak/external modifier。"
},
{
"label": "主要结果",
"text": "full trace/history 显著优于 score/summary；classification/math held-out 和 transfer 有收益；TB2 strong seed 仍可涨。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "前两类 seed/simple baselines 较弱；最有说服力的 TB2 又没有 held-out。 **对我们：** 我们当前最直接 baseline：应复制其 modifier 可见信息，但改进评估协议。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 46,
"fields": {
"时间": "2026-03",
"论文": "**[Meta-Harness](https://arxiv.org/abs/2603.28052)**",
"为什么是主干": "full executable harness optimization 的主干 baseline：完整 source/history/raw trace 都进入 proposer context。",
"证据边界要记住": "TerminalBench headline 同集 search/eval；其他任务更干净。"
}
},
{
"section": "D. Full harness / executable harness optimization",
"line": 170,
"fields": {
"时间": "2026-03-30",
"论文": "[Meta-Harness](https://arxiv.org/abs/2603.28052)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**H-Full**。把 executable harness code 作为 search object，并把**所有历史 candidate code/score/raw traces**暴露给 coding proposer；是 full-harness 主干 baseline。",
"谁来改 → 谁执行；基础 harness": "stronger/separate coding proposer → frozen executor + current harness；seed 从简单 context/retrieval 到成熟 TB harness 不等。",
"Feedback / evidence": "search-set score + raw execution traces。",
"标签": "`#HarnessCode #Prompt #MemoryContent #Workflow #StrongerBuilder #ExecutableVerifier #OfflineSearch #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 326,
"fields": {
"优先级": "**C**",
"时间": "2026-03-30",
"论文": "[Meta-Harness: End-to-End Optimization of Model Harnesses](https://arxiv.org/abs/2603.28052)",
"本质定位": "Claude Opus 4.6 + Claude Code 直接读取所有历史 harness source/score/full traces，搜索完整 executable harness。",
"什么在变": "完整 executable harness。",
"谁来改 / 谁执行": "**改**：Claude Opus 4.6 + Claude Code proposer。
**执行**：domain-specific frozen base model + candidate harness。",
"基础 harness": "classification/math 从较简单 seed；TB2 从 Terminus/Terminus-KIRA strong harness。",
"Feedback": "search-set reward + full execution traces + historical harness code。",
"Evolution → Eval": "classification/math 有 held-out；TB2 89 tasks same-set search/eval。",
"Meta-depth": "M1 strong-to-weak/external modifier。",
"相对之前真正新增什么": "相对 ADAS/AFlow 更明确把 **model harness**（context/memory/retrieval/control flow/coding harness）当统一 edit object，并强调 raw-history observability。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "混合协议",
"protocolBasis": "不同实验的划分不同，不能用一个 held-out 标签概括；见逐任务协议。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "让 coding proposer 在文件系统里检索历代 harness 源码、分数和完整执行日志，再直接改写目标模型的运行代码。历史不是一段固定摘要，而是 proposer 能主动查阅的搜索经验。",
"novelty": "修改者可以主动检索历史版本的代码、分数和执行记录，再改写组织模型与工具的程序；历史保留到可重新诊断的细节层面。",
"object": "可执行 harness 的代码：模型调用、工具使用、记忆与检索、执行步骤。目标模型参数和外层搜索规则固定。",
"executor": "分类：GPT-OSS-120B；数学：GPT-OSS-20B 等；TB2：Opus 4.6 / Haiku 4.5。各实验目标模型固定。",
"modifier": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 中的 Claude Opus 4.6 作为候选方案提出者，读取历史运行框架、分数和执行轨迹，改写单文件 Python 运行框架；执行任务的基础模型按领域另设并保持冻结。",
"roleContext": "**改**：Claude Opus 4.6 + Claude Code proposer。
**执行**：domain-specific frozen base model + candidate harness。",
"seed": "不同实验有不同起点。分类比较无示范提示、少量示范提示、ACE（通过反思整理经验来构造上下文）和 MCE（维护文字技能库来构造上下文）；数学任务提供检索语料及提示、路由逻辑；终端任务与 Terminus 等已有执行框架比较。不能据某个实验推断全篇只有一套初始工具。",
"fixed": "目标模型权重、外层 propose→evaluate→log 程序与评分接口固定。",
"verdict": "用于搜索和改进候选方案的数据集标签、任务分数或可执行结果检查器（按测试或判分规则检查任务结果）；不同任务使用不同评分器。",
"diagnosis": "主动检索历史 candidate code 与 raw traces，分析具体失败步骤及旧修改为何退化。",
"update": "直接编辑检索、状态管理、提示构造与执行控制代码；每轮留下完整候选和日志。",
"acceptance": "继续用 search-set 评分指导候选搜索；隔离边界必须按实验分别看。",
"experiments": [
{
"name": "分类",
"evolve": "在线分类 search tasks",
"selection": "search-set 结果",
"test": "额外 OOD 分类数据集",
"isolation": "迁移评估",
"note": "分类执行时也有在线状态积累。"
},
{
"name": "数学检索",
"evolve": "250 道 OlympiadBench + Omni-MATH hard 搜索题",
"selection": "40 轮产生的候选按 search score 选择",
"test": "200 道未见 IMO-level 题；另换目标模型",
"isolation": "独立测试",
"note": "检索语料另作去重/去污染。"
},
{
"name": "Terminal-Bench 2",
"evolve": "同一 TB2 benchmark 上搜索",
"selection": "benchmark score 参与迭代",
"test": "同一 full benchmark 报告结果",
"isolation": "同集适应",
"note": "不能把数学/分类的 held-out 结论移到 TB2。"
}
],
"takeaway": "前两类 seed/simple baselines 较弱；最有说服力的 TB2 又没有 held-out。 **对我们：** 我们当前最直接 baseline：应复制其 modifier 可见信息，但改进评估协议。",
"result": "full trace/history 显著优于 score/summary；classification/math held-out 和 transfer 有收益；TB2 strong seed 仍可涨。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2603.28052v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：分类：LawBench、Symptom2Disease、USPTO-50k 的搜索子集；数学：250 道搜索题；Terminal-Bench 2：全部 89 题。\n\n调试 / 选版本数据：候选方案提出者读取搜索题的标签/成绩和轨迹，自主提出后继版本。\n\n最终测试数据：分类用留出题；数学另用 IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 两个月份共 40 题；TB2 仍测同一 89 题。\n\n数据隔离与证据边界：分类/数学的测试不返回给候选方案提出者；TB2 明确同集搜索和报告，不能统一标成独立测试。",
"cycle": "修改者通过终端查阅历代代码、分数和完整执行记录，定位问题后直接改检索、状态管理、提示或执行控制代码。候选先检查接口有效性，再在搜索题上运行；保留效果或成本各有优势的版本，最后评价入选框架。",
"train": "分类：LawBench、Symptom2Disease、USPTO-50k 的搜索子集；数学：250 道搜索题；Terminal-Bench 2：全部 89 题。",
"debug": "候选方案提出者读取搜索题的标签/成绩和轨迹，自主提出后继版本。",
"test": "分类用留出题；数学另用 IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 两个月份共 40 题；TB2 仍测同一 89 题。",
"isolation": "分类/数学的测试不返回给候选方案提出者；TB2 明确同集搜索和报告，不能统一标成独立测试。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "把完整 executable harness code 作为搜索对象，用 coding-agent proposer 基于历史 harness、score 和完整 execution traces 自动做 harness engineering。",
"index": {
"date": "2026-03-30",
"paper": "[Meta-Harness: End-to-End Optimization of Model Harnesses](https://arxiv.org/abs/2603.28052)",
"executor": "**Text classification**：GPT-OSS-120B + context/memory harness（seed：zero-shot / few-shot / ACE / MCE）
**Math retrieval**：搜索时 GPT-OSS-20B + BM25-based retrieval harness；最终同一 harness 还测 GPT-5.4-nano / GPT-5.4-mini / Gemini-3.1-Flash-Lite / Gemini-3-Flash
**TerminalBench-2**：Claude Opus 4.6 或 Haiku 4.5 + 从 Terminus 2 / Terminus-KIRA 出发演化的 coding harness",
"modifier": "**Claude Opus 4.6 + Claude Code** coding-agent proposer",
"feedback": "search-set score / pass rate + **完整 execution traces** + 历史 harness source",
"evolve": "**Classification**：LawBench / Symptom2Disease / USPTO-50k 的 search split
**Math**：250 个 olympiad-difficulty search problems
**TerminalBench-2**：**同一 89 tasks** 直接用于 search",
"eval": "**Classification**：同三数据集 held-out test split
**Math**：200 个 unseen IMO-level problems；另做 cross-model transfer
**TerminalBench-2**：**仍是同一 89 tasks，无 held-out**",
"tldr": "直接搜索/重写 executable harness。前两个任务有 held-out test，但初始 harness 和 baseline 都较简单；TerminalBench 从成熟 harness 出发，却直接对最终 89 tasks 优化，因此还没有同时证明“成熟 harness 可持续改进 + held-out 泛化”。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | Harness engineering 仍主要靠人工；现有 text optimizer 多只看当前 candidate、scalar score 或压缩后的短 summary，难对长 horizon harness 中“早期设计选择→后续失败”做 credit assignment。 |\n| 本文定位 | 提出 **Meta-Harness**：把 harness 变成 executable code search space，让 agentic proposer 自己决定读哪些历史 code/trace、诊断什么 failure、做局部修改还是完整 rewrite。 |\n| 核心贡献 | ① 搜索完整 harness program，而非只改 prompt；② 所有历史 candidate 的 source code、scores、execution traces 通过 filesystem 暴露给 proposer；③ proposer 自主检索历史并修改代码；④ 用 population + Pareto frontier 保留候选，在 text classification、math retrieval、TerminalBench-2 三类任务上验证。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **模型权重固定**；进化的是 task-specific executable harness，包括 prompt construction、retrieval、memory/state update、orchestration/control flow 等。 |\n| 谁来改 | **Claude Code + Claude Opus 4.6** 作为 coding-agent proposer。 |\n| 谁来执行 | 每个 domain 的固定 base model 执行 candidate harness；base model 不训练。 |\n| 修改时能看到什么 | filesystem 中**所有历史 candidate 的 source code、evaluation scores、execution traces**（prompt、tool call、model output、state update 等）；proposer 用 grep/cat 等工具按需读取，而不是把历史压进一个 prompt。 |\n| Feedback | search-set task reward/accuracy/pass rate + 完整 rollout traces；没有独立人工 improvement feedback。 |\n| Search | 反复 `inspect history → propose harness → evaluate → log`；维护 population/Pareto frontier，但不规定 parent-selection rule，proposer 可读任意历史 candidate。 |\n| 方法本质 | **固定模型上的 outer-loop harness code search**；不是 model-weight RSI。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark | 执行者：模型 + harness | Evolution feedback | Search → Final Eval | 人工标注 / GT |\n|---|---|---|---|---|\n| Online text classification：LawBench / Symptom2Disease / USPTO-50k | GPT-OSS-120B + candidate context/memory harness | search-set accuracy + traces；20 iterations × 2 candidates | **Search set → held-out test set**；test 直到最终 evaluation 才使用 | search-set label 用于自动 accuracy feedback；test label 不进入 search |\n| Retrieval-augmented math reasoning | Search 时 GPT-OSS-20B + candidate retrieval harness | 250 个 olympiad-difficulty search problems 上的 accuracy + traces | **250 search problems → 200 previously unseen IMO-level problems**；同一 harness 还测 4 个 search 时未见过的模型 | correctness 用于 search reward；held-out problems 不进入 search |\n| TerminalBench-2 | Claude Opus 4.6 / Haiku 4.5 + candidate agentic harness；从 Terminus 2 / Terminus-KIRA 强 baseline 出发 | 89 tasks 的 verifier/pass rate +完整 traces | **Search 和 final evaluation 都直接使用同一 89-task benchmark** | task-native verifier；没有独立 held-out task |"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| 完整 trace/history interface 显著优于压缩 feedback | text-classification search-set：Meta-Harness median/best **50.0/56.7**；scores-only 约 **34.6/41.3**，scores+summary 约 **34.9/38.7** | 在该 setting 下，raw execution traces 对 harness optimization 很关键，scalar/summary 丢失诊断信息 |\n| Classification harness 优于现有 context-management baseline | held-out test avg **48.6%**；ACE 40.9%，MCE 40.0%；context 11.4K vs ACE 50.8K | 搜出的 context/memory policy 可在 held-out classification test 上提升 accuracy/context tradeoff |\n| Math harness 对任务和模型都有迁移 | 200 unseen IMO-level problems 上，5 个模型相对 no-retrieval 平均 **+4.7 pt**；其中 4 个模型 search 时未见 | 说明至少该 retrieval policy 不只是拟合 search problems 或单一 target model |\n| TerminalBench-2 能从成熟 harness 继续搜到提升 | Opus-4.6：**76.4%**；Haiku-4.5：**37.6%**，均超过文中主要 hand-engineered comparison | 自动 harness code search 能在成熟 agentic coding setting 中找到有效改动 |\n| Proposer 会利用历史失败形成修改假设 | TerminalBench trajectory 中先因 prompt+structural edits regression，后隔离变量并转向 safer additive modification | 提供 qualitative evidence：proposer 会基于 trace 做 diagnosis，而不是纯随机 mutation |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| Story 与实验匹配 | **整体匹配。** 真正搜索的是 executable harness code，proposer 也确实能访问完整历史 code+trace，区别于 prompt-only optimization。 |\n| 前两个 benchmark 的关键限制 | **初始 harness 很简单**，主要从基础 prompt/context/memory/retrieval 方案开始搜索；对比 baseline 也主要是较简单的 prompt/context optimization 方法。因此虽然这两个 setting 有较干净的 held-out test，**但从简单 harness 搜到明显提升相对容易**，不能据此说明 Meta-Harness 已经能稳定改进成熟、复杂的 agent harness。 |\n| TerminalBench 的关键问题 | 虽然从 **Terminus / Terminus-KIRA 等成熟 harness** 出发，更能说明在强 baseline 上还能搜到有效改动；但作者明确 **search 和 final eval 都在同一 89 tasks 上**，因此主要证明“针对这批公开任务能自动搜出更好的 harness”，不能证明得到的是可泛化的 harness improvement。 |\n| 数据隔离不统一 | 前两个 setting 有较干净 held-out test；TerminalBench 没有。横向汇总时不能把三个 benchmark 的泛化证据等量看待。 |\n| Self-RSI 口径 | proposer 是更强的 Claude Opus 4.6 + Claude Code，base executor 固定；本质上是 **cross-system automated harness engineering**，不是执行者自身递归改进自己。 |\n| 归因 | full-trace interface 有 ablation，是较干净的机制证据；但最终 harness 内部可能同时改变 prompt、retrieval、state/control flow，最终性能提升未逐组件因果拆开。 |"
},
{
"title": "总评",
"body": "**Meta-Harness 的核心价值是把完整 executable harness 变成可搜索对象，并让强 coding agent 直接读取所有历史 code、score 和 execution trace 后修改 harness。前两个任务有较干净的 held-out 泛化证据，但初始 harness 和主要 baseline 都较简单，因此提升相对容易获得；TerminalBench-2 则从成熟 harness 出发，更能说明在强 baseline 上还能搜到有效改动，但它直接对最终 89 个任务优化。综合来看，论文证明了 automated harness search 有效，但还没有同时满足“从成熟 harness 出发 + 独立 held-out task 验证”的最强证据。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "按实验给不同起点：分类从 zero-shot、few-shot、ACE、MCE 等候选出发；数学有检索语料与路由/提示；终端任务另有 Terminus 系列对照。不存在一套能概括全篇的单一初始工具表。",
"protocol": "**分类：**LawBench、Symptom2Disease、USPTO-50k 的搜索集→各自留出题，具体数量见详细实验记录。\n\n**数学：**250 道奥赛难度题搜索，另测 200 道 IMO 级未见题：IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 2025-12 的 17、2026-01 的 23。检索库共 535,356 题，来自 OpenMathReasoning、DeepMath、NuminaMath、PolyMath、Omni-MATH、FineProofs、历史 AIME、Putnam-AXIOM；检索材料数量不等于搜索训练题数。\n\n**Terminal-Bench 2：**同一 89 题用于搜索和报告，没有独立留出任务。",
"sections": "分类实验；数学附录 C、Tables 10–11；终端实验",
"source": "https://arxiv.org/abs/2603.28052",
"version": "2603.28052v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "83278baf4b11d265b2e5ebf9b7a3d6d45294f9e4a3c3363cd86290d3046742e0",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "分类：GPT-OSS-120B；数学：GPT-OSS-20B 等；TB2：Opus 4.6 / Haiku 4.5。各实验目标模型固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 中的 Claude Opus 4.6 作为候选方案提出者，读取历史运行框架、分数和执行轨迹，改写单文件 Python 运行框架；执行任务的基础模型按领域另设并保持冻结。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可执行 harness 的代码：模型调用、工具使用、记忆与检索、执行步骤。目标模型参数和外层搜索规则固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用于搜索和改进候选方案的数据集标签、任务分数或可执行结果检查器（按测试或判分规则检查任务结果）；不同任务使用不同评分器。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "不同实验有不同起点。分类比较无示范提示、少量示范提示、ACE（通过反思整理经验来构造上下文）和 MCE（维护文字技能库来构造上下文）；数学任务提供检索语料及提示、路由逻辑；终端任务与 Terminus 等已有执行框架比较。不能据某个实验推断全篇只有一套初始工具。",
"sources": [
{
"label": "分类实验；数学附录 C、Tables 10–11；终端实验",
"url": "https://arxiv.org/abs/2603.28052"
},
{
"label": "§4.1：ACE / MCE 起点说明",
"url": "https://arxiv.org/html/2603.28052#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "修改者通过终端查阅历代代码、分数和完整执行记录，定位问题后直接改检索、状态管理、提示或执行控制代码。候选先检查接口有效性，再在搜索题上运行；保留效果或成本各有优势的版本，最后评价入选框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "分类：LawBench、Symptom2Disease、USPTO-50k 的搜索子集；数学：250 道搜索题；Terminal-Bench 2：全部 89 题。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "候选方案提出者读取搜索题的标签/成绩和轨迹，自主提出后继版本。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "分类用留出题；数学另用 IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 两个月份共 40 题；TB2 仍测同一 89 题。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "分类/数学的测试不返回给候选方案提出者；TB2 明确同集搜索和报告，不能统一标成独立测试。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "修改者可以主动检索历史版本的代码、分数和执行记录，再改写组织模型与工具的程序；历史保留到可重新诊断的细节层面。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.28052v1",
"version": "2603.28052v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 prompt optimizer，核心变化是完整 executable harness 的搜索空间，以及对原始历史证据的主动访问；不是只把最终分数写进下一轮 prompt。",
"feedbackCases": [
{
"label": "分类框架搜索",
"data": "LawBench、Symptom2Disease、USPTO-50k 的搜索子集；最终分类结果在各自留出集。",
"scoring": "把预测类别与数据集类别标签比较，计算分类准确率；同时记录额外上下文用量。",
"visible": "修改者可检索历次候选代码、成绩和执行记录，包括提示、模型输出及状态更新。",
"use": "根据搜索集表现选框架；分类测试集保留到最后。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"judgment": "规则比较预测类别与标注类别"
},
{
"label": "数学检索框架搜索与测试",
"data": "250 道搜索题；最终为 IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 2025-12 的 17 题和 2026-01 的 23 题。",
"scoring": "按对应数学任务得分评价回答；四套最终题混合最终答案、证明与研究型问题，不能全部解释为字符串精确匹配。该文未提供足以逐题还原的统一判分实现。",
"visible": "搜索端可读取候选分数和原始执行记录。",
"use": "据搜索题修改检索、路由与提示；四套最终题用于冻结框架的泛化测试。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"judgment": "数学任务各用自身评分协议；混合答案题、证明题与研究题，未公开完整判分实现"
},
{
"label": "终端框架搜索与报告",
"data": "Terminal-Bench 2 全部 89 题。",
"scoring": "按终端基准的任务验收程序判断最终产物是否通过。",
"visible": "完整候选代码、分数和执行日志对修改者开放。",
"use": "在同一 89 题上搜索并报告结果；这部分不具有分类实验那样的留出测试边界。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"judgment": "终端基准原生测试程序"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "分类：LawBench、Symptom2Disease、USPTO-50k 的搜索子集；数学：250 道搜索题；Terminal-Bench 2：全部 89 题。",
"selection": "候选方案提出者读取搜索题的标签/成绩和轨迹，自主提出后继版本。",
"evaluation": "分类用留出题；数学另用 IMO-AnswerBench 100、IMO-ProofBench 60、ArXivMath 两个月份共 40 题；TB2 仍测同一 89 题。",
"isolation": "分类/数学的测试不返回给候选方案提出者；TB2 明确同集搜索和报告，不能统一标成独立测试。",
"roles": {
"executor": {
"value": "分类：GPT-OSS-120B；数学：GPT-OSS-20B 等；TB2：Opus 4.6 / Haiku 4.5。各实验目标模型固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
]
},
"modifier": {
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 中的 Claude Opus 4.6 作为候选方案提出者，读取历史运行框架、分数和执行轨迹，改写单文件 Python 运行框架；执行任务的基础模型按领域另设并保持冻结。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "不同实验有不同起点。分类比较无示范提示、少量示范提示、ACE（通过反思整理经验来构造上下文）和 MCE（维护文字技能库来构造上下文）；数学任务提供检索语料及提示、路由逻辑；终端任务与 Terminus 等已有执行框架比较。不能据某个实验推断全篇只有一套初始工具。",
"sources": [
{
"label": "分类实验；数学附录 C、Tables 10–11；终端实验",
"url": "https://arxiv.org/abs/2603.28052"
},
{
"label": "§4.1：ACE / MCE 起点说明",
"url": "https://arxiv.org/html/2603.28052#S4.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"evaluation": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
],
"isolation": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.28052#S4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "框架工程仍主要靠人工读失败记录、试少数设计；自动文本优化器又常只保留当前候选、总分或压缩摘要，删掉了把后续失败追溯到早期框架决定的证据。作者因此要求修改者能按需检查长期保留的代码与执行历史，避免每轮基于丢失细节的反馈猜测修改。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.28052#S1"
}
]
},
{
"key": "position",
"value": "研究自动运行框架设计能否充分利用历次试验的经验，找到优于人工设计、并能迁移到其他任务或模型的运行方式。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.28052"
}
]
},
{
"key": "conclusion",
"value": "在分类、数学检索和终端任务上获得收益，支持“更完整的历史证据有助于自动设计运行框架”。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.28052"
}
]
}
],
"fields": {
"object": "可执行 harness 的代码：模型调用、工具使用、记忆与检索、执行步骤。目标模型参数和外层搜索规则固定。",
"seed": "不同实验有不同起点。分类比较无示范提示、少量示范提示、ACE（通过反思整理经验来构造上下文）和 MCE（维护文字技能库来构造上下文）；数学任务提供检索语料及提示、路由逻辑；终端任务与 Terminus 等已有执行框架比较。不能据某个实验推断全篇只有一套初始工具。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.28052"
}
]
},
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.28052"
}
]
},
{
"tag": "org:krafton",
"label": "KRAFTON",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.28052"
}
]
},
{
"tag": "person:chelsea-finn",
"label": "Chelsea Finn",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2603.28052"
}
]
},
{
"tag": "person:omar-khattab",
"label": "Omar Khattab",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2603.28052"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2604.25850",
"title": "Agentic Harness Engineering (AHE)",
"url": "https://arxiv.org/abs/2604.25850",
"date": "2026-04-28",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HarnessCode",
"M1",
"MemoryContent",
"RegressionGate",
"SameSet",
"SeparateEvolver",
"Tool",
"Workflow",
"org:peking"
],
"fields": {
"本质定位": "冻结 GPT-5.4，Evolve Agent 编辑 system prompt、tools、middleware、skills、subagents、memory；Debugger 把超长 rollout 压成 evidence，edit manifest 记录 expected fixes/regressions 并验证/rollback。",
"谁来改 → 谁执行；基础 harness": "Evolve Agent + Debugger → frozen GPT executor + NexAU/AHE harness。",
"Feedback / evidence": "verifier + layered traces + cross-iteration task delta。",
"什么在变": "full executable coding harness。",
"谁来改 / 谁执行": "**改**：GPT-5.4 xhigh Evolve Agent；GPT-5.4 Debugger。
**执行**：GPT-5.4 high + current NexAU/AHE harness。",
"基础 harness": "NexAU strong coding harness。",
"Feedback": "TB2 verifier + full traces + cross-iteration task delta。",
"Evolution → Eval": "TB2 same 89 tasks 10 rounds；SWE-bench-V frozen transfer。",
"Meta-depth": "M1 external modifier。",
"相对之前真正新增什么": "相对 Meta-Harness 最清楚的新点是 **Component / Experience / Decision Observability** 与 edit-level attribution/rollback；还做 component transplant。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "冻结 GPT-5.4，Evolve Agent 编辑 system prompt、tools、middleware、skills、subagents、memory；Debugger 把超长 rollout 压成 evidence，edit manifest 记录 expected fixes/regressions 并验证/rollback。 **相对前序：** 相对 Meta-Harness 最清楚的新点是 **Component / Experience / Decision Observability** 与 edit-level attribution/rollback；还做 component transplant。"
},
{
"label": "什么在变",
"text": "full executable coding harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** GPT-5.4 xhigh Evolve Agent；GPT-5.4 Debugger。 **执行：** GPT-5.4 high + current NexAU/AHE harness。"
},
{
"label": "基础 harness",
"text": "NexAU strong coding harness。"
},
{
"label": "Feedback",
"text": "TB2 verifier + full traces + cross-iteration task delta。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "TB2 same 89 tasks 10 rounds；SWE-bench-V frozen transfer。 **Meta-depth：** M1 external modifier。"
},
{
"label": "主要结果",
"text": "TB2 69.7→77.0；收益主要 memory/tool/middleware；SWE-V accuracy only +0.4pp但token -12%；发现 regression blindness。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "headline same-set；held-out accuracy gain 很小；Explore Agent 还注入外部 prior。 **对我们：** 特别值得学 component attribution、regression prediction 与 rollback。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 47,
"fields": {
"时间": "2026-04",
"论文": "**[Agentic Harness Engineering (AHE)](https://arxiv.org/abs/2604.25850)**",
"为什么是主干": "full-harness evolution 工程化代表：observability、attribution、rollback、regression。",
"证据边界要记住": "TB2 主结果同集；cross-benchmark accuracy transfer 较小。"
}
},
{
"section": "D. Full harness / executable harness optimization",
"line": 171,
"fields": {
"时间": "2026-04-28",
"论文": "[Agentic Harness Engineering (AHE)](https://arxiv.org/abs/2604.25850)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**H-Full**。新增不是“full harness 可改”本身，而是 component/experience/decision observability + edit hypothesis + rollback，专门处理 attribution/interference。",
"谁来改 → 谁执行；基础 harness": "Evolve Agent + Debugger → frozen GPT executor + NexAU/AHE harness。",
"Feedback / evidence": "verifier + layered traces + cross-iteration task delta。",
"标签": "`#HarnessCode #MemoryContent #Tool #Workflow #SeparateEvolver #ExecutableVerifier #RegressionGate #SameSet #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 327,
"fields": {
"优先级": "**C**",
"时间": "2026-04-28",
"论文": "[Agentic Harness Engineering (AHE)](https://arxiv.org/abs/2604.25850)",
"本质定位": "冻结 GPT-5.4，Evolve Agent 编辑 system prompt、tools、middleware、skills、subagents、memory；Debugger 把超长 rollout 压成 evidence，edit manifest 记录 expected fixes/regressions 并验证/rollback。",
"什么在变": "full executable coding harness。",
"谁来改 / 谁执行": "**改**：GPT-5.4 xhigh Evolve Agent；GPT-5.4 Debugger。
**执行**：GPT-5.4 high + current NexAU/AHE harness。",
"基础 harness": "NexAU strong coding harness。",
"Feedback": "TB2 verifier + full traces + cross-iteration task delta。",
"Evolution → Eval": "TB2 same 89 tasks 10 rounds；SWE-bench-V frozen transfer。",
"Meta-depth": "M1 external modifier。",
"相对之前真正新增什么": "相对 Meta-Harness 最清楚的新点是 **Component / Experience / Decision Observability** 与 edit-level attribution/rollback；还做 component transplant。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "混合协议",
"protocolBasis": "不同实验的划分不同，不能用一个 held-out 标签概括；见逐任务协议。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 coding harness 拆成可编辑文件，把轨迹组织成可诊断的证据，再要求每次改动写下可检验的“修复谁、可能伤害谁”的预测。下一轮对照真实结果继续修改或回退。",
"novelty": "把一次框架修改关联到具体组件和执行证据，并通过移植组件检查改动能否复用、是否与其他组件相互干扰。",
"object": "coding agent 的 harness：执行代码、工具、中间处理逻辑和长期记忆。模型参数及外层诊断、修改流程固定。",
"executor": "GPT-5.4（high）在 NexAU（组织模型、工具和任务执行的模块化 agent 框架） 的 Code Agent 中运行任务；跨模型测试再替换这个执行端，复用已进化的运行框架。",
"modifier": "GPT-5.4 驱动 Evolve Agent 改代码，另一个 GPT-5.4 Agent Debugger 提供诊断。推理强度存在原文差异：正文 §4.1 称三个角色均为 high，附录 A 表 4 的 Evolve Agent 配置是 xhigh。",
"roleContext": "**改**：GPT-5.4 xhigh Evolve Agent；GPT-5.4 Debugger。
**执行**：GPT-5.4 high + current NexAU/AHE harness。",
"seed": "从 NexAU 的初始版本出发，仅提供 Bash 命令行工具。NexAU 是组织模型、工具和执行过程的 agent 框架；这个实验的起点未预置完整专用工具、中间处理代码和长期记忆，进化逐步补充这些机制，例如改命令超时处理。",
"fixed": "模型权重与评估器固定；外层 Debugger / Evolve 的角色与流程固定。",
"verdict": "Terminal-Bench 2 的任务验收程序判断是否完成；同时比较同一题在修改前后是否由失败变成功、是否反而退化，再检查改动是否符合原先预测。",
"diagnosis": "Debugger 从原始轨迹生成逐任务根因/成功分析，再汇总成 benchmark 层证据。",
"update": "增删改 system prompt、tools、middleware、memory 等文件；manifest 绑定改动与预期影响。",
"acceptance": "下一轮验证 manifest 的修复/退化预测，保留或撤回修改；还做单组件移植分析。",
"experiments": [
{
"name": "主实验",
"evolve": "TB2 同一 89 tasks，10 轮",
"selection": "按 TB2 分数选择版本",
"test": "同一 TB2 上报告 69.7→77.0",
"isolation": "同集适应",
"note": ""
},
{
"name": "迁移",
"evolve": "沿用 TB2 已进化的 workspace",
"selection": "不做目标域重新进化",
"test": "SWE-bench Verified 与替换 base model",
"isolation": "冻结后迁移",
"note": "不能等同于 TB2 内部独立测试。"
}
],
"takeaway": "原记录的 seed 强度及 xhigh modifier 描述已按当前原文更正。69.7→77.0 是同集迭代结果；转移收益与组件干扰应分开阅读。",
"result": "TB2 69.7→77.0；收益主要 memory/tool/middleware；SWE-V accuracy only +0.4pp但token -12%；发现 regression blindness。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2604.25850v4"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Terminal-Bench 2 全部 89 题，连续 10 轮修改运行框架；不训练模型参数。\n\n调试 / 选版本数据：每题每轮 2 次执行尝试（从开始做任务到得到结果的过程），Agent Debugger 对照成功/失败轨迹及跨轮变化，检验改动 改动清单。\n\n最终测试数据：同一 TB2 89 题报告最终成绩；冻结运行框架后，另测 SWE-bench Verified 500 题。\n\n数据隔离与证据边界：TB2 是同集迭代；SWE-bench Verified 是跨评测基准迁移。换模型但仍测 TB2 不等于换了任务。",
"cycle": "诊断角色先从逐题记录分析成功或失败原因，再汇总共同问题。修改者编辑提示、工具、中间处理和记忆代码，同时写出预计修好或破坏哪些任务。下一轮实际执行核对预测，再保留或撤回改动。",
"train": "Terminal-Bench 2 全部 89 题，连续 10 轮修改运行框架；不训练模型参数。",
"debug": "每题每轮 2 次执行尝试（从开始做任务到得到结果的过程），Agent Debugger 对照成功/失败轨迹及跨轮变化，检验改动 改动清单。",
"test": "同一 TB2 89 题报告最终成绩；冻结运行框架后，另测 SWE-bench Verified 500 题。",
"isolation": "TB2 是同集迭代；SWE-bench Verified 是跨评测基准迁移。换模型但仍测 TB2 不等于换了任务。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "冻结 base model，把完整 executable coding-agent harness 作为进化对象；通过 **Component / Experience / Decision Observability**，让 harness 修改可定位、可验证、可回滚。",
"index": {
"date": "2026-04-28",
"paper": "[Agentic Harness Engineering: Observability-Driven Automatic Evolution of Coding-Agent Harnesses](https://arxiv.org/abs/2604.25850)",
"executor": "**Terminal-Bench 2**：GPT-5.4 high + 当前 NexAU/AHE coding-agent harness；另做 SWE-bench-Verified frozen transfer 与跨模型 transfer",
"modifier": "**GPT-5.4 xhigh Evolve Agent**；**GPT-5.4 Agent Debugger** 负责把 rollout 压缩成结构化 evidence；iteration 1 另有 Explore Agent 提炼外部 coding-agent knowledge",
"feedback": "**task verifier pass/fail + 完整/分层 execution traces + 跨 iteration task-level outcome delta**；每次 edit 还声明 expected fixes / regressions",
"evolve": "**Terminal-Bench 2 全部 89 tasks**，10 iterations，每 task 每轮 2 rollouts；直接在这 89 tasks 上反复 evolve",
"eval": "headline final score **仍是同一 TB2 89 tasks，无 held-out**；另有 **SWE-bench-Verified 500 tasks frozen cross-benchmark transfer**；cross-model transfer 仍在同一 TB2 tasks",
"tldr": "冻结 base model，直接进化完整 executable harness（prompt/tools/middleware/memory 等）。TB2 **69.7→77.0**，但主结果是同任务集 adaptive optimization；真正 held-out 的 SWE-bench-V 只 **+0.4pp**、token 约 **−12%**。收益主要来自 memory/tool/middleware，prompt-only 反而退化；还发现 component interference 与 regression blindness。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | Coding-agent performance 很大程度取决于 system prompt、tools、middleware、memory 等 harness design，但这些仍主要靠人工工程；已有自动优化多只改 prompt、skill、workflow 等单个 surface。完整 harness evolution 还面临 **trajectory 过长、组件耦合、修改效果难归因** 等问题。 |\n| 本文定位 | 将核心瓶颈定位为 **observability**：modifier 需要知道 **可以改什么、发生了什么、某次修改是否真的产生了预期效果**，而不是仅给一个 scalar reward 后让模型盲目搜索。 |\n| 核心贡献 | 提出 **AHE**：① Component Observability，把 harness 拆成可独立编辑的组件；② Experience Observability，把长 rollout 压缩成 per-task diagnosis / benchmark overview，同时保留 raw trace；③ Decision Observability，每次 edit 显式记录 evidence、root cause、expected fixes / regressions，并在下一轮验证、必要时 rollback。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **模型权重完全冻结**；可编辑对象覆盖 system prompt、tool description、tool implementation、middleware、skills、sub-agent configuration、long-term memory 等 **executable / persistent harness components**。 |\n| 谁来执行 | **GPT-5.4 high + 当前 NexAU/AHE coding-agent harness**。 |\n| 谁来改 | **GPT-5.4 xhigh Evolve Agent + evolution harness/skills**；另有 **GPT-5.4 Agent Debugger** 负责把任务 rollout 组织成可用 evidence。三者属于同一模型 family，但 role / harness 不同。 |\n| 修改者能看到什么 | benchmark overview、per-task failure/success diagnosis；需要时可 drill down 到 cleaned/raw trajectories；还可看到 task verifier reward、历史 edit manifest 与跨轮 task-level performance delta。 |\n| Feedback | **自动 feedback**：task verifier pass/fail + execution trajectory + previous-edit outcome delta；没有运行时人工 improvement feedback。 |\n| 修改约束 | verifier、tracer、LLM config 等不属于可编辑 harness；不能通过换更强模型、增加 reasoning budget、关闭 verifier 等方式涨分。 |\n| 外部先验 | iteration 1 有一次性 **Explore Agent**，会从 NexAU source 与公开 coding-agent references 中提炼 reusable evolution skills。因此不是完全 closed-world、只靠自身 rollout 的 self-evolution。 |\n| 方法本质 | **frozen-model full-harness search / evolution**；模型能力不通过 weight update 改变，而通过外部 adaptation surface 持续修改。 |\n\n##### 三层 Observability\n\n| 层 | 做法 | 作用 |\n|---|---|---|\n| Component Observability | prompt / tools / middleware / skills / memory 等拆成独立组件；logical edit 单独 git commit | 明确“改了什么”，便于组件级 attribution 与 rollback |\n| Experience Observability | Agent Debugger 将百万 token 级 rollout 分层压缩成 per-task diagnosis → benchmark overview，同时保留 raw trace | 从长 trajectory 中提炼 actionable failure / success evidence |\n| Decision Observability | 每次 edit 保存 `evidence → root cause → proposed fix → expected fixes / regressions` | 把修改变成可证伪 hypothesis，下一轮直接用 task outcome 验证 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark / 实验 | 执行者：模型 + harness | Feedback / verifier | Train / Evolve → Eval | 人工标注 / GT |\n|---|---|---|---|---|\n| **Terminal-Bench 2 主 evolution** | **GPT-5.4 high + 当前 NexAU/AHE harness** | 官方 task verifier + 完整 trajectory；Debugger 生成 diagnosis；Evolver 读取跨轮 task delta | **同一完整 89 tasks 上做 10 轮 evolution，每轮每 task 2 rollouts；headline 77.0 仍在这 89 tasks 上报告** | 无运行时人工 feedback；自动 verifier 提供结果 |\n| **SWE-bench-Verified frozen transfer** | **GPT-5.4 + 在 TB2 上 evolve 后冻结的 AHE harness** | SWE-bench official tests / evaluator | **Evolve：TB2 89 tasks → Eval：SWE-bench-V 500 tasks**；真正 cross-benchmark held-out | 无 |\n| **Cross-model transfer** | evolved harness 分别换给 GPT-5.4 medium/xhigh、Gemini-3.1-flash-lite-preview、Qwen-3.6-plus、DeepSeek-v4-flash | TB2 verifier | harness 在 GPT-5.4 high + TB2 上 evolve；换模型后仍测试 **同一 TB2 89 tasks** | 无 |"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| 完整 harness evolution 能显著提高目标任务集表现 | TB2：NexAU seed **69.7 → AHE 77.0**；Codex 71.9，ACE 68.9，TF-GRPO 72.3 | 在固定 TB2 任务集上，冻结 GPT-5.4 权重、仅改 harness 可以搜索到更高性能配置；**但这不是 held-out task generalization 证据**。 |\n| 收益主要来自 executable / persistent components，而非 prompt | 单组件 transplant：memory-only **75.3**，tool-only **73.0**，middleware-only **71.9**，system-prompt-only **67.4**，full AHE **77.0** | Full-harness evolution 的收益不等同于 prompt optimization；memory、tools、middleware 是主要正向来源。 |\n| 组件存在 interaction / interference | 正向单组件 gain 不能线性叠加；部分 hard tasks 上 memory-only 甚至优于 full AHE，重复 verification/check 会消耗 step budget | Harness evolution 不只是“各组件分别变好”，**component coupling / interference** 本身就是关键变量。 |\n| Cross-benchmark transfer 有限但存在 | SWE-bench-V：seed **75.2 → AHE 75.6**；tokens **526k → 461k** | 真正 held-out 的 performance gain 只有 **+0.4pp**，更明显的 transfer 是约 **12% token reduction**；不能表述成“大幅跨 benchmark 提升”。 |\n| Harness 可跨模型迁移 | TB2 上换到多个 model family 后均有正 gain，部分弱模型约 +5～10pp | evolved harness 包含一定 model-agnostic coordination pattern；但 task 与 evolve set 相同，因此不是 task generalization。 |\n| Modifier 存在明显 regression blindness | Fix prediction precision/recall **33.7% / 51.4%**；Regression prediction **11.8% / 11.1%**，虽高于 random 但明显更弱 | Evolve Agent 有一定 self-attribution 能力，但更会预测“会修好什么”，不擅长预测 collateral damage。 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| **主结果 train/eval 同集** | 最核心的 **69.7→77.0** 是直接在 Terminal-Bench 2 完整 89 tasks 上反复执行、修改、再选择 best harness；**没有 task-level held-out split**。因此主要证明针对这批 tasks 的 adaptive harness optimization。 |\n| Held-out 泛化证据强度有限 | SWE-bench-V 是真正 frozen cross-benchmark transfer，但 accuracy 仅 **+0.4pp**；更明显的是 token efficiency。因此它补充了泛化证据，但远弱于 headline TB2 gain。 |\n| Self-evolution 不是纯 closed-world | 初始 Explore Agent 会读取公开 coding-agent references 与 NexAU source，将外部 human-engineered prior 注入 evolution skills；不能把全部提升解释为只从自身失败经验中产生新能力。 |\n| “Observability 是核心瓶颈”仍缺完全正交验证 | Debugger、结构化 evidence、manifest、rollback 等机制一起加入；虽然 edit attribution 很有价值，但缺三层 observability 的完整独立因果拆解。 |\n| 对副作用感知弱 | regression prediction 显著弱于 fix prediction，说明当前 evolution loop 更擅长寻找 improvement，较难提前发现 collateral damage。 |\n| 能力上限 | 证明固定模型仅改 harness 可提高目标任务集成绩，并有一定 transfer；但**没有严格证明超过该 base model 在充分强人工 / oracle harness 下的能力上限**。 |\n| 与我们的关系 | 这是非常直接的 Harness Self-Evolution 前驱：它已经研究 edit-level task regression，但“副作用”仍只看 benchmark pass/fail，没有系统测 **confidence/calibration、长度、bias、拒答、judge gaming 等行为偏移**；也没有系统测 harness evolution 的 capability ceiling。 |"
},
{
"title": "总评",
"body": "**AHE 是很典型的 Harness Self-Evolution：base model 冻结，完整 executable harness（prompt、tools、middleware、memory 等）直接成为修改对象；GPT-5.4 Code Agent 产生带 verifier reward 的 rollout，Debugger 将长轨迹压缩为结构化 evidence，Evolve Agent 再修改 harness，并通过下一轮 task delta 验证 / rollback。最重要的实证是收益主要来自 memory、tools、middleware，而不是 prompt，同时出现 component interference 与 regression blindness。最大问题是 headline TB2 结果直接在同一 89 tasks 上 evolve 和 eval；真正 held-out 的 SWE-bench transfer 只有轻微 accuracy gain。因此它较强地证明“针对固定任务集可以自动搜出更好 harness”，但对通用 harness capability improvement 的证明仍有限。**"
}
],
"caveats": [
{
"text": "模型设置的原文口径不一致：正文 §4.1 称三个角色均用 GPT-5.4 high；附录 A 表 4 的 reference-run 配置却列出 Code Agent 为 high、Evolve Agent 为 xhigh。下文沿用新笔记及附录的 xhigh，并保留这一差异；此前网站统一写成 high 过于确定。",
"url": "https://arxiv.org/html/2604.25850v4#A1"
}
]
},
"systemDataAudit": {
"seed": "从 NexAU₀ 的 bash-only 起点出发，没有预置完整专用工具、middleware 和长期记忆；通过改提示、工具和运行逻辑建立这些能力。具体示例包括改 shell timeout 接口，而非仅换一句提示词。",
"protocol": "**进化/同集报告：**Terminal-Bench 2 全部 89 题（4 easy、55 medium、30 hard），十轮演化，每题每轮两次 rollout，单题 timeout 延长到一小时。最终 TB2 仍是这 89 题。\n\n**留出迁移：**冻结 harness 后测 SWE-bench Verified 500 题；跨模型但仍用 TB2 的成绩要与跨 benchmark 留出分开。正文和附录对修改者推理档位存在差异，保留原有补注。",
"sections": "实验设置与转移实验；原文口径补注",
"source": "https://arxiv.org/abs/2604.25850",
"version": "2604.25850v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "445e241b48fbde2335f2f051969b2be50a2a84c1766ae577800370c33c7c84e8",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5.4（high）在 NexAU（组织模型、工具和任务执行的模块化 agent 框架） 的 Code Agent 中运行任务；跨模型测试再替换这个执行端，复用已进化的运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-5.4 驱动 Evolve Agent 改代码，另一个 GPT-5.4 Agent Debugger 提供诊断。推理强度存在原文差异：正文 §4.1 称三个角色均为 high，附录 A 表 4 的 Evolve Agent 配置是 xhigh。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "coding agent 的 harness：执行代码、工具、中间处理逻辑和长期记忆。模型参数及外层诊断、修改流程固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "Terminal-Bench 2 的任务验收程序判断是否完成；同时比较同一题在修改前后是否由失败变成功、是否反而退化，再检查改动是否符合原先预测。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从 NexAU 的初始版本出发，仅提供 Bash 命令行工具。NexAU 是组织模型、工具和执行过程的 agent 框架；这个实验的起点未预置完整专用工具、中间处理代码和长期记忆，进化逐步补充这些机制，例如改命令超时处理。",
"sources": [
{
"label": "实验设置与转移实验；原文口径补注",
"url": "https://arxiv.org/abs/2604.25850"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "诊断角色先从逐题记录分析成功或失败原因，再汇总共同问题。修改者编辑提示、工具、中间处理和记忆代码，同时写出预计修好或破坏哪些任务。下一轮实际执行核对预测，再保留或撤回改动。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Terminal-Bench 2 全部 89 题，连续 10 轮修改运行框架；不训练模型参数。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "每题每轮 2 次执行尝试（从开始做任务到得到结果的过程），Agent Debugger 对照成功/失败轨迹及跨轮变化，检验改动 改动清单。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "同一 TB2 89 题报告最终成绩；冻结运行框架后，另测 SWE-bench Verified 500 题。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "TB2 是同集迭代；SWE-bench Verified 是跨评测基准迁移。换模型但仍测 TB2 不等于换了任务。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把一次框架修改关联到具体组件和执行证据，并通过移植组件检查改动能否复用、是否与其他组件相互干扰。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.25850v4",
"version": "2604.25850v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "不只是 full-harness search，而是把组件、经验与编辑决策的可观测性连接起来，并用 component transplant 检查改动间的干扰。",
"feedbackCases": [
{
"label": "进化：Terminal-Bench 2",
"data": "全部 89 题，连续 10 轮，每题每轮执行两次。",
"scoring": "官方任务验收程序运行测试，给出通过/失败；不是用修改者的预测代替验收。",
"visible": "Agent Debugger 读取成功/失败轨迹与跨轮变化；修改者另写预计修复和可能退化的任务清单，再与下一轮真实测试结果比较。",
"use": "同一批题上定位问题、改运行框架和检验改动；最终 TB2 分数仍来自这批题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"judgment": "TB2 原生测试程序判通过／失败"
},
{
"label": "冻结迁移：SWE-bench Verified",
"data": "500 道代码修复题，使用 TB2 进化后冻结的框架。",
"scoring": "基准的仓库测试验收补丁，计算任务通过率。",
"visible": "用于报告跨任务迁移的成绩。",
"use": "这一组不用于上述 TB2 的十轮框架修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"judgment": "SWE-bench 仓库测试验收补丁"
}
],
"experiments": [
{
"label": "Terminal-Bench 2：十轮框架进化",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "全部 89 题，连续十轮，每题每轮运行两次；根据验收结果与轨迹修改框架。",
"selection": "在同一 89 题上检查改动并继续进化。",
"evaluation": "仍在同一 89 题上报告成绩；更换执行模型的比较也沿用 TB2。",
"isolation": "这是同题集上的持续改进，不是新任务上的独立测试。",
"roles": {
"executor": {
"value": "GPT-5.4（high）在 NexAU（组织模型、工具和任务执行的模块化 agent 框架） 的 Code Agent 中运行任务；跨模型测试再替换这个执行端，复用已进化的运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
]
},
"modifier": {
"value": "GPT-5.4 驱动 Evolve Agent 改代码，另一个 GPT-5.4 Agent Debugger 提供诊断。推理强度存在原文差异：正文 §4.1 称三个角色均为 high，附录 A 表 4 的 Evolve Agent 配置是 xhigh。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
}
]
},
"seed": {
"value": "从 NexAU 的初始版本出发，仅提供 Bash 命令行工具。NexAU 是组织模型、工具和执行过程的 agent 框架；这个实验的起点未预置完整专用工具、中间处理代码和长期记忆，进化逐步补充这些机制，例如改命令超时处理。",
"sources": [
{
"label": "实验设置与转移实验；原文口径补注",
"url": "https://arxiv.org/abs/2604.25850"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
]
}
},
{
"label": "冻结框架迁移到 SWE-bench Verified",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "使用上述 TB2 的 89 题进化得到的框架。",
"selection": "迁移前冻结框架。",
"evaluation": "SWE-bench Verified 的 500 道代码修复题。",
"isolation": "这 500 题不参与 TB2 的十轮修改；与同题集 TB2 成绩分开解释。",
"roles": {
"executor": {
"value": "GPT-5.4（high）在 NexAU（组织模型、工具和任务执行的模块化 agent 框架） 的 Code Agent 中运行任务；跨模型测试再替换这个执行端，复用已进化的运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
]
},
"modifier": {
"value": "迁移测试不再调用修改者更新框架；使用 TB2 阶段由 GPT-5.4 Evolve Agent 与 Agent Debugger 进化出的冻结版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.25850#S4.SS1.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1.SS0.SSS0.Px3"
}
]
},
"seed": {
"value": "测试起点是 TB2 十轮进化后冻结的 NexAU 框架，已可能包含新工具和处理机制；不是重新从仅 Bash 的原始框架开始。",
"sources": [
{
"label": "实验设置与转移实验；原文口径补注",
"url": "https://arxiv.org/abs/2604.25850"
},
{
"label": "NexAU 官方说明",
"url": "https://github.com/nex-agi/NexAU"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2604.25850#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.25850#S4.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.25850#A1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "人工框架调优跟不上模型更新，已有自动方法又多只改提示或技能等单一部分。若联合修改，冗长原始轨迹难提供可行动的线索，紧密耦合的组件又容易改坏；因此作者关注怎样让修改者看清组件、失败证据和改动后果，稳定优化整个运行框架。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.25850#S1"
}
]
},
{
"key": "position",
"value": "面向工具、记忆和控制流程共同作用的复杂 agent 系统，研究如何使自动框架改进可诊断、可验证，并产生可迁移的工程收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.25850"
}
]
},
{
"key": "conclusion",
"value": "Terminal-Bench 2 成功率从 69.7% 到 77.0%；主要收益来自工具、中间处理和长期记忆，而非只改系统提示。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.25850"
}
]
}
],
"fields": {
"modifier": "GPT-5.4 驱动代码修改者和独立诊断者，分别编辑运行框架与分析失败。原文对修改者推理强度的记载有差异，具体配置见展开表格。",
"object": "coding agent 的 harness：执行代码、工具、中间处理逻辑和长期记忆。模型参数及外层诊断、修改流程固定。",
"verdict": "Terminal-Bench 2 的任务验收程序判断是否完成；同时比较同一题在修改前后是否由失败变成功、是否反而退化，再检查改动是否符合原先预测。",
"seed": "从 NexAU 的初始版本出发，仅提供 Bash 命令行工具。NexAU 是组织模型、工具和执行过程的 agent 框架；这个实验的起点未预置完整专用工具、中间处理代码和长期记忆，进化逐步补充这些机制，例如改命令超时处理。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.25850"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2606.09498",
"title": "Self-Harness: Harnesses That Improve Themselves",
"url": "https://arxiv.org/abs/2606.09498",
"date": "2026-06-08",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HarnessCode",
"M1",
"OfflineSearch",
"RegressionGate",
"SameModel",
"org:shanghai-ai-lab"
],
"fields": {
"本质定位": "不依赖更强 external builder：当前固定模型从自身 execution traces 中聚类 model-specific failures，提出 diverse/minimal harness edits，并经 regression validation 后 merge。",
"谁来改 → 谁执行；基础 harness": "same frozen model → 自己的 harness；seed 为已有 agent harness。",
"Feedback / evidence": "failure patterns + held-in/held-out regression score；注意 held-out 参与 acceptance。",
"什么在变": "non-parametric agent harness：instructions/tools/memory/state/runtime mechanisms 等 bounded executable edits；weights fixed。",
"谁来改 / 谁执行": "**改/执行**：同一 fixed base model，在 task-agent 与 proposer role 间切换；MiniMax M2.5、Qwen3.5-35B-A3B、GLM-5。",
"基础 harness": "minimal initial harness。",
"Feedback": "held-in verifier-grounded execution failures + passing behaviors + previous edit summaries；candidate 由 held-in/held-out regression score gate。",
"Evolution → Eval": "Terminal-Bench-2.0、SWE-bench Verified、AppWorld；held-out traces 不给 proposer，但 **held-out performance 每轮进入 acceptance rule**，因此不是 untouched final test。",
"Meta-depth": "M1 same-model；proposal/evaluator/acceptance machinery fixed。",
"相对之前真正新增什么": "相对 Meta-Harness/AHE 的代表性新点是 **把 harness improver 内化到 target model 自身**，并围绕 model-specific weakness mining + bounded edit + regression gate 组织 loop；但 held-out score 参与 search，泛化证据需谨慎。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "same-model harness self-improvement 的代表。它把 Meta-Harness/AHE 中“外部更强 proposer 改 target harness”内化为：**target model 自己根据自己在当前 harness 下暴露的 failure pattern 改自己的 harness**。核心 loop 是 Weakness Mining → Harness Proposal → Proposal Validation。"
},
{
"label": "什么在变",
"text": "模型权重固定；变的是 non-parametric agent harness，包括 instructions、tools、memory/state/runtime mechanisms 等 bounded edits。"
},
{
"label": "谁来改 / 谁执行",
"text": "**同一个 fixed base model**既执行任务又在 proposer role 下产生 harness edits；实验覆盖 MiniMax M2.5、Qwen3.5-35B-A3B、GLM-5。"
},
{
"label": "基础 harness",
"text": "minimal initial harness；因此能较清楚观察不同模型最后长出不同 model-specific harness。"
},
{
"label": "Feedback",
"text": "proposer 只看 held-in 失败的 verifier-grounded cluster、passing behavior 与 previous edit summaries；不是看全部 raw failed tasks 逐题 patch。"
},
{
"label": "Evolution → Eval",
"text": "Terminal-Bench-2.0、SWE-bench Verified、AppWorld。需要特别注意：论文把 split 称为 held-in / held-out，**held-out trace 不给 proposer，但 held-out score 在每轮 regression validation 中参与 candidate acceptance**。所以它不是严格 `evolve→freeze→untouched test`。"
},
{
"label": "主要结果",
"text": "九个 model×benchmark setting 的最终 harness 都同时提升 held-in 与 held-out pass rate；TB2 held-out 例如 MiniMax 40.5→61.9、Qwen 23.8→38.1、GLM 42.9→57.1。"
},
{
"label": "为什么是经典 / 关键限制",
"text": "经典性在于它非常直接地回答“**模型能不能自己改自己的 operating harness？**”；但 evidence 不能 overclaim 为严格 unseen generalization，因为 held-out score 进入 search/acceptance loop。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 48,
"fields": {
"时间": "2026-06",
"论文": "**[Self-Harness](https://arxiv.org/abs/2606.09498)**",
"为什么是主干": "same-model self-harness 的最直接代表：target 根据自己的 failure 修改自己的 harness。",
"证据边界要记住": "held-out regression score 参与 acceptance，不是 untouched test。"
}
},
{
"section": "D. Full harness / executable harness optimization",
"line": 175,
"fields": {
"时间": "2026-06-08",
"论文": "[Self-Harness](https://arxiv.org/abs/2606.09498)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**H-Full**。same fixed model 自己从 recurring failures 提 bounded edit，再 merge；相对 strong-builder 路线真正关键是 **modifier=executor backbone**。",
"谁来改 → 谁执行；基础 harness": "same frozen model → 自己的 harness；seed 为已有 agent harness。",
"Feedback / evidence": "failure patterns + held-in/held-out regression score；注意 held-out 参与 acceptance。",
"标签": "`#HarnessCode #SameModel #ExecutableVerifier #RegressionGate #OfflineSearch #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 330,
"fields": {
"优先级": "**C**",
"时间": "2026-06-08",
"论文": "[Self-Harness: Harnesses That Improve Themselves](https://arxiv.org/abs/2606.09498)",
"本质定位": "不依赖更强 external builder：当前固定模型从自身 execution traces 中聚类 model-specific failures，提出 diverse/minimal harness edits，并经 regression validation 后 merge。",
"什么在变": "non-parametric agent harness：instructions/tools/memory/state/runtime mechanisms 等 bounded executable edits；weights fixed。",
"谁来改 / 谁执行": "**改/执行**：同一 fixed base model，在 task-agent 与 proposer role 间切换；MiniMax M2.5、Qwen3.5-35B-A3B、GLM-5。",
"基础 harness": "minimal initial harness。",
"Feedback": "held-in verifier-grounded execution failures + passing behaviors + previous edit summaries；candidate 由 held-in/held-out regression score gate。",
"Evolution → Eval": "Terminal-Bench-2.0、SWE-bench Verified、AppWorld；held-out traces 不给 proposer，但 **held-out performance 每轮进入 acceptance rule**，因此不是 untouched final test。",
"Meta-depth": "M1 same-model；proposal/evaluator/acceptance machinery fixed。",
"相对之前真正新增什么": "相对 Meta-Harness/AHE 的代表性新点是 **把 harness improver 内化到 target model 自身**，并围绕 model-specific weakness mining + bounded edit + regression gate 组织 loop；但 held-out score 参与 search，泛化证据需谨慎。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "验证集参与接受",
"protocolBasis": "held-out score 每轮参与 regression acceptance；不是 untouched final test。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "同一个固定模型一边执行任务，一边根据自身失败模式提出小范围 harness 修改。循环由 weakness mining、proposal 和 regression validation 三步组成。",
"novelty": "让执行任务的同一种模型归纳自己的共性失败，再修改自己的运行框架；接受修改仍由预设检查控制，包含留出题的分数。",
"object": "harness 配置允许修改的提示词、工具、记忆、状态和执行机制；不修改模型参数、任务环境或候选接受规则，也不能任意重写整个 SDK。",
"executor": "MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5。",
"modifier": "每个配置使用对应的 MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5 担任提案者，读取评测器返回的失败证据并提出编辑；基础模型参数冻结。",
"roleContext": "**改/执行**：同一 fixed base model，在 task-agent 与 proposer role 间切换；MiniMax M2.5、Qwen3.5-35B-A3B、GLM-5。",
"seed": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和命令行工具。没有多模态输入处理，故对应任务被排除；只能改运行框架定义的配置面，不是任意改整个 SDK。",
"fixed": "权重、解码、预算、任务环境、evaluator 和 promotion rule 固定。",
"verdict": "评测任务自带的判分程序给出任务通过或失败。",
"diagnosis": "只向 proposer 暴露 held-in traces、失败簇、passing behavior 和既往改动摘要。",
"update": "提出少量 diverse / minimal 配置改动，分别对应可解释的失败机制。",
"acceptance": "自动 promotion gate 同时检查 held-in 与 held-out 分数；held-out trace 不暴露，但分数参与接受。",
"experiments": [
{
"name": "TB2",
"evolve": "固定 64-case 子集的 held-in 部分",
"selection": "held-in + held-out regression gate",
"test": "同一 held-out 部分报告结果",
"isolation": "验证集参与接受",
"note": ""
},
{
"name": "SWE-bench Verified",
"evolve": "100-case 子集中的 67 held-in",
"selection": "33 held-out 分数参与 gate",
"test": "该 33-case held-out",
"isolation": "验证集参与接受",
"note": ""
},
{
"name": "AppWorld",
"evolve": "90 official training tasks",
"selection": "90 normal/challenge test 抽样任务参与 gate",
"test": "同一 held-out 集",
"isolation": "验证集参与接受",
"note": ""
}
],
"takeaway": "不是 evolve→freeze→untouched final test。隔离了 proposer 对 held-out 轨迹的访问，没有隔离 search 对 held-out 分数的适应。",
"result": "九个 model×benchmark setting 的最终 harness 都同时提升 held-in 与 held-out pass rate；TB2 held-out 例如 MiniMax 40.5→61.9、Qwen 23.8→38.1、GLM 42.9→57.1。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2606.09498v3"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不训权重。TB2 固定 64 题子集；SWE-bench Verified 67 道 用于诊断和改进的题；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道官方训练题，提供失败轨迹。\n\n调试 / 选版本数据：SWE 33 道单独留出的、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道从 test_normal/test_challenge 抽取的题，加上各自 用于诊断和改进的题 分数共同决定是否接受候选。\n\n最终测试数据：最终报告同一 用于诊断和改进的题/预先留出的题 的通过率，没有再加第三套完全不参与门控的测试集。\n\n数据隔离与证据边界：单独留出的轨迹不向候选方案提出者展示，但成绩参与自动选择；“看不到题”不等于“未用于选版本”。",
"cycle": "修改者只读取用于诊断的题的轨迹、失败类型、成功行为和历史改动，提出针对性的少量修改。接受时同时检查这批题和留出检查题的分数；留出题的轨迹不公开，但分数参与选择，因此它不是最后才使用的独立测试。",
"train": "不训权重。TB2 固定 64 题子集；SWE-bench Verified 67 道 用于诊断和改进的题；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道官方训练题，提供失败轨迹。",
"debug": "SWE 33 道单独留出的、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道从 test_normal/test_challenge 抽取的题，加上各自 用于诊断和改进的题 分数共同决定是否接受候选。",
"test": "最终报告同一 用于诊断和改进的题/预先留出的题 的通过率，没有再加第三套完全不参与门控的测试集。",
"isolation": "单独留出的轨迹不向候选方案提出者展示，但成绩参与自动选择；“看不到题”不等于“未用于选版本”。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和 shell 工具。没有多模态输入处理，故对应任务被排除；只能改 harness 定义的配置面，不是任意改整个 SDK。",
"protocol": "**数据：**TB2 固定 64 题子集，排除不稳定外网/多模态任务；SWE-bench Verified 固定 100 题，67 held-in、33 held-out；AppWorld 90 官方训练题＋90 从 normal/challenge 抽样题。\n\n**重要隔离区别：**held-in 的轨迹给提案者看，held-out 不给提案者，但它的分数参与自动接受门控，最终也报告同一 held-out。所以它是选版本的一部分，不是完全未参与选择的盲测。TB2 两部分的具体数量本轮待核实。",
"sections": "实验任务子集与 promotion gate",
"source": "https://arxiv.org/abs/2606.09498",
"version": "2606.09498v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "7c061603ab1f45da3482e7391aa798f8c0c09b5e2ff2ed1b03313e0af680c336",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "每个配置使用对应的 MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5 担任提案者，读取评测器返回的失败证据并提出编辑；基础模型参数冻结。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "harness 配置允许修改的提示词、工具、记忆、状态和执行机制；不修改模型参数、任务环境或候选接受规则，也不能任意重写整个 SDK。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "评测任务自带的判分程序给出任务通过或失败。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和命令行工具。没有多模态输入处理，故对应任务被排除；只能改运行框架定义的配置面，不是任意改整个 SDK。",
"sources": [
{
"label": "实验任务子集与 promotion gate",
"url": "https://arxiv.org/abs/2606.09498"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "修改者只读取用于诊断的题的轨迹、失败类型、成功行为和历史改动，提出针对性的少量修改。接受时同时检查这批题和留出检查题的分数；留出题的轨迹不公开，但分数参与选择，因此它不是最后才使用的独立测试。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训权重。TB2 固定 64 题子集；SWE-bench Verified 67 道 用于诊断和改进的题；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道官方训练题，提供失败轨迹。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "SWE 33 道单独留出的、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90 道从 test_normal/test_challenge 抽取的题，加上各自 用于诊断和改进的题 分数共同决定是否接受候选。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "最终报告同一 用于诊断和改进的题/预先留出的题 的通过率，没有再加第三套完全不参与门控的测试集。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "单独留出的轨迹不向候选方案提出者展示，但成绩参与自动选择；“看不到题”不等于“未用于选版本”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让执行任务的同一种模型归纳自己的共性失败，再修改自己的运行框架；接受修改仍由预设检查控制，包含留出题的分数。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.09498v3",
"version": "2606.09498v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 proposer 内化到目标模型自身，并聚类共性失败而不是逐题硬编码修补；“同模型”不意味着没有固定外层 gate。",
"feedbackCases": [
{
"label": "TB2：提出修改与回归检查",
"data": "固定 64 题子集",
"scoring": "运行终端任务原生判分程序检查最终产物是否通过。",
"visible": "修改者读取用于分析失败的题的消息、工具调用和判分结果；候选还要在用于回归检查的题上重新评估。",
"use": "两部分成绩共同约束是否保留修改，留出题参与了版本选择；最终报告没有再增加第三套完全不参与选择的测试集。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"judgment": "TB2 原生测试程序检查终端产物"
},
{
"label": "SWE-bench Verified：提出修改与回归检查",
"data": "67 道用于分析失败的题，33 道用于回归检查的留出题",
"scoring": "运行代码修复任务的测试，按通过/失败评分。",
"visible": "修改者读取用于分析失败的题的消息、工具调用和判分结果；候选还要在用于回归检查的题上重新评估。",
"use": "两部分成绩共同约束是否保留修改，留出题参与了版本选择；最终报告没有再增加第三套完全不参与选择的测试集。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"judgment": "SWE-bench 仓库测试验收补丁"
},
{
"label": "AppWorld：提出修改与回归检查",
"data": "90 道官方训练题用于失败分析；从 test_normal/test_challenge 另抽 90 题作回归检查",
"scoring": "由 AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 的任务验收检查任务执行后的应用状态和目标完成情况。",
"visible": "修改者读取用于分析失败的题的消息、工具调用和判分结果；候选还要在用于回归检查的题上重新评估。",
"use": "两部分成绩共同约束是否保留修改，留出题参与了版本选择；最终报告没有再增加第三套完全不参与选择的测试集。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"judgment": "AppWorld 任务自带验收程序检查应用状态与目标"
}
],
"experiments": [
{
"label": "Terminal-Bench 2",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "固定 64 道终端任务；运行失败与验收结果用于提出修改。",
"selection": "同一任务子集上的成绩约束是否保留修改。",
"evaluation": "仍报告这 64 题的成绩。",
"isolation": "没有另设第三套不参与修改或选版本的最终测试集。",
"roles": {
"executor": {
"value": "MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"modifier": {
"value": "每个配置使用对应的 MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5 担任提案者，读取评测器返回的失败证据并提出编辑；基础模型参数冻结。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"seed": {
"value": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和命令行工具。没有多模态输入处理，故对应任务被排除；只能改运行框架定义的配置面，不是任意改整个 SDK。",
"sources": [
{
"label": "实验任务子集与 promotion gate",
"url": "https://arxiv.org/abs/2606.09498"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
]
}
},
{
"label": "SWE-bench Verified",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "67 道代码修复题用于执行、分析失败和提出修改。",
"selection": "另 33 道留出题做回归检查；轨迹不展示给提议者，但成绩参与选版本。",
"evaluation": "报告上述进化题与留出题上的结果。",
"isolation": "33 题是选版本用的留出集，不是选定后才首次使用的最终测试集。",
"roles": {
"executor": {
"value": "MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"modifier": {
"value": "每个配置使用对应的 MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5 担任提案者，读取评测器返回的失败证据并提出编辑；基础模型参数冻结。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"seed": {
"value": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和命令行工具。没有多模态输入处理，故对应任务被排除；只能改运行框架定义的配置面，不是任意改整个 SDK。",
"sources": [
{
"label": "实验任务子集与 promotion gate",
"url": "https://arxiv.org/abs/2606.09498"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
]
}
},
{
"label": "AppWorld",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "90 道官方训练题用于执行、失败分析和修改框架。",
"selection": "从 test_normal / test_challenge 另抽 90 题检查回归，成绩参与选版本。",
"evaluation": "报告上述训练题与回归检查题上的结果。",
"isolation": "回归题虽来自官方 test split，但已用于选择框架；没有第三套最终盲测。",
"roles": {
"executor": {
"value": "MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"modifier": {
"value": "每个配置使用对应的 MiniMax M2.5、Qwen3.5-35B-A3B 或 GLM-5 担任提案者，读取评测器返回的失败证据并提出编辑；基础模型参数冻结。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
}
]
},
"seed": {
"value": "DeepAgent SDK 上的简化配置：简短任务提示、默认文件系统和命令行工具。没有多模态输入处理，故对应任务被排除；只能改运行框架定义的配置面，不是任意改整个 SDK。",
"sources": [
{
"label": "实验任务子集与 promotion gate",
"url": "https://arxiv.org/abs/2606.09498"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.09498#A1.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "各模型的能力和失败模式不同，同一套框架未必都适合；模型更新越快，逐个靠人工重调框架就越难持续。借助更强外部模型还可能昂贵、不可用或不了解目标模型的弱点，因此作者研究能否由目标模型根据自身执行证据适配自己的框架。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.09498#S1"
}
]
},
{
"key": "position",
"value": "研究冻结模型能否在没有人工工程师或更强外部模型指导时，识别自身特有的执行弱点，并改进适合自己的运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.09498"
}
]
},
{
"key": "conclusion",
"value": "九种模型与任务组合的开发和留出表现都改善，展示了不依赖更强外部修改者的适配路线。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.09498"
}
]
}
],
"fields": {
"object": "harness 配置允许修改的提示词、工具、记忆、状态和执行机制；不修改模型参数、任务环境或候选接受规则，也不能任意重写整个 SDK。",
"verdict": "TB2 用原生测试检查终端产物；SWE-bench 用仓库测试验收补丁；AppWorld 用任务验收程序检查应用状态。结果用于提出修改和检查已有能力是否退步。"
}
},
"attributions": [
{
"tag": "org:shanghai-ai-lab",
"label": "Shanghai AI Laboratory",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.09498"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2606.17546",
"title": "SEAGym: An Evaluation Environment for Self-Evolving LLM Agents",
"url": "https://arxiv.org/abs/2606.17546",
"date": "2026-06-16",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"CrossBenchmark",
"HeldOut",
"RegressionGate",
"org:tsinghua"
],
"fields": {
"本质定位": "专门评估 self-evolving harness 的可靠性：train/update-validation、held-out ID/OOD、replay、cost、snapshot 分开记录，关注“持续更新是否真的持续变强”。",
"什么在变": "benchmark本身不进化；被测的是不同 self-evolving methods 的 persistent harness/state。",
"谁来改 / 谁执行": "**改**：各被测方法自己的 updater。
**执行**：ACE、TF-GRPO、AHE 等方法/模型。",
"基础 harness": "统一 evaluation environment / benchmark-specific harness。",
"Feedback": "task verifier + held-out/replay/cost/snapshot metrics。",
"Evolution → Eval": "Terminal-Bench 2.0 + HLE；多视角 evolution evaluation。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "重要性在负面证据：frequent update 不保证 held-out gain，best intermediate snapshot 继续 evolve 可能 collapse；把 retention/generalization/cost 从 final score 中拆出来。"
},
"details": [
{
"label": "本质定位",
"text": "不提出新的 evolver，而是专门问：**一个 self-evolving agent 到底是真的变强，还是只在 update data 上涨分？**"
},
{
"label": "评什么",
"text": "train / update-validation / held-out ID / OOD / replay / cost / snapshots；把 discovery、retention、generalization 和 cost 拆开。"
},
{
"label": "基础 setting",
"text": "Terminal-Bench 2.0 + HLE；比较 ACE、TF-GRPO、AHE 等不同 evolution styles。"
},
{
"label": "核心结论",
"text": "frequent update 不保证 held-out improvement；最好 intermediate snapshot 继续 evolve 可能 collapse；experience source、backend、update schedule 都会影响 reliability。"
},
{
"label": "为什么经典",
"text": "它代表 2026 后半段非常重要的 **evaluation correction wave**：以后不能只报一个 final score，就声称“self-evolution 成功”。"
},
{
"label": "对我们最重要",
"text": "我们做 capability ceiling / 系统性行为偏移时，本质上也应该采用这种“evolution trajectory + snapshot + held-out counterfactual”的思路。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 49,
"fields": {
"时间": "2026-06",
"论文": "**[SEAGym](https://arxiv.org/abs/2606.17546)**",
"为什么是主干": "self-evolving harness 的 reliability/evaluation 代表：held-out ID/OOD、replay、snapshot、cost 分开评。",
"证据边界要记住": "自己不提出新的 evolution 方法；价值在评测框架和负面证据。"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 362,
"fields": {
"优先级": "**C**",
"时间": "2026-06-16",
"论文": "[SEAGym: An Evaluation Environment for Self-Evolving LLM Agents](https://arxiv.org/abs/2606.17546)",
"本质定位": "专门评估 self-evolving harness 的可靠性：train/update-validation、held-out ID/OOD、replay、cost、snapshot 分开记录，关注“持续更新是否真的持续变强”。",
"什么在变": "benchmark本身不进化；被测的是不同 self-evolving methods 的 persistent harness/state。",
"谁来改 / 谁执行": "**改**：各被测方法自己的 updater。
**执行**：ACE、TF-GRPO、AHE 等方法/模型。",
"基础 harness": "统一 evaluation environment / benchmark-specific harness。",
"Feedback": "task verifier + held-out/replay/cost/snapshot metrics。",
"Evolution → Eval": "Terminal-Bench 2.0 + HLE；多视角 evolution evaluation。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "重要性在负面证据：frequent update 不保证 held-out gain，best intermediate snapshot 继续 evolve 可能 collapse；把 retention/generalization/cost 从 final score 中拆出来。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录报告独立任务评估；查看详细记录中的数据与选模限制。",
"year": "2026",
"depth": [
"未明确"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"B-Harness",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把静态 benchmark 转成可连续更新的环境，同时保存 agent snapshots，分别测 update-validation、ID/OOD transfer、replay 和成本。",
"novelty": "在连续更新过程中反复测新任务、旧能力和成本，观察中间好版本是否随后退化；重点是测量改进过程。",
"object": "基准本身固定；比较的是不同方法如何更新可持续保留的运行框架与状态。",
"executor": "主对照用DeepSeek-V4-Flash执行ACE、TF-GRPO、AHE；跨模型实验另用GLM-5.1、GPT-5.4，并交换运行框架快照与执行执行端。",
"modifier": "ACE、TF-GRPO、AHE各自实现更新；主实验执行端为DeepSeek-V4-Flash，跨模型AHE分别在GLM-5.1和GPT-5.4下进化。SEAGym负责调用和记录。",
"roleContext": "**改**：各被测方法自己的 updater。
**执行**：ACE、TF-GRPO、AHE 等方法/模型。",
"seed": "作者自建评估层，分别提供“执行任务”和“更新方法状态”两类接口。前者返回可公开的执行记录和奖励，后者按各方法原有规则更新跨任务保留的运行框架与状态。任务环境和判分程序仍由原评测基准运行，私有答案及数据划分信息不提供给 task agent。",
"fixed": "",
"verdict": "各任务原生评分器给出成绩；平台另外比较不同版本在验证、新任务、旧任务重做和成本上的表现，区分能力提升、遗忘与额外开销。",
"diagnosis": "对照更新前后、不同源任务/批量/backend，以及遗忘、恢复和迁移曲线。",
"update": "由各 baseline 自己完成；SEAGym 负责采样、记录与评估。",
"acceptance": "update-validation 与 final held-out / replay views 分开记录，不能只看最终一个分数。",
"experiments": [
{
"name": "统一 evolution evaluation",
"evolve": "80 source train tasks：TB2 + HLE text-only Math/Physics",
"selection": "35 source validation tasks",
"test": "55 source test；80 HLE CS/AI 与 Engineering OOD tasks",
"isolation": "学习、选模、ID/OOD 分开",
"note": "snapshot、replay 与成本单独测量，不只报最终最高分。"
}
],
"takeaway": "我们做 capability ceiling / 系统性行为偏移时，本质上也应该采用这种“evolution trajectory + snapshot + held-out counterfactual”的思路。",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2606.17546v1"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：Terminal-Bench 2 与 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 数学/物理文本题组成 80 道 训练集，按批运行并更新。\n\n调试 / 选版本数据：35 道 验证集 用于观察进化轮次快照；更新规则由 ACE、TF-GRPO、AHE 等被测方法决定。\n\n最终测试数据：55 道 同分布测试；另有 80 道 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 计算机科学／人工智能、Engineering 题测 分布外任务；额外重放训练题以观察遗忘。\n\n数据隔离与证据边界：明确区分训练、阶段验证、最终 ID/分布外任务 和训练重放；重放成绩不当作新题泛化。",
"cycle": "被测方法各自负责更新系统，SEAGym 负责提供任务、保存版本并记录表现。更新后的验证、新任务测试、旧任务重做及成本分别统计，从而观察收益是否保留、是否遗忘。",
"train": "Terminal-Bench 2 与 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 数学/物理文本题组成 80 道 训练集，按批运行并更新。",
"debug": "35 道 验证集 用于观察进化轮次快照；更新规则由 ACE、TF-GRPO、AHE 等被测方法决定。",
"test": "55 道 同分布测试；另有 80 道 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 计算机科学／人工智能、Engineering 题测 分布外任务；额外重放训练题以观察遗忘。",
"isolation": "明确区分训练、阶段验证、最终 ID/分布外任务 和训练重放；重放成绩不当作新题泛化。"
},
"reviewed": true,
"focusedReview": true,
"systemDataAudit": {
"seed": "作者自建评估层把 rollout 和 update 接口分开：前者执行任务并返回公开轨迹/奖励，后者按各方法原生规则更新持久 harness。实际环境和 verifier 留在原 benchmark 后端，私有答案与 split 元数据不暴露给 agent。",
"protocol": "**数据：**TB2 与 HLE 文本数学/物理组成源任务，80 train、35 validation、55 test；另有 80 道 HLE CS/AI 与 Engineering 题测 OOD。\n\n**流程：**按批运行 train 并更新，保存 epoch 快照；验证用于阶段评估，最后区分 ID、OOD 和训练重放。重放看修复与遗忘，不算新增留出任务；不同方法的原生初始化仍需逐方法看。",
"sections": "算法 1；数据划分；附录 E",
"source": "https://arxiv.org/abs/2606.17546",
"version": "2606.17546v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "02e45db3c87d953de7e94d0dc507b8b520a517c9ee64f4e6f5b131eeff87ebd6",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主对照用DeepSeek-V4-Flash执行ACE、TF-GRPO、AHE；跨模型实验另用GLM-5.1、GPT-5.4，并交换运行框架快照与执行执行端。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.17546#S4.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2606.17546#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2606.17546#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "ACE、TF-GRPO、AHE各自实现更新；主实验执行端为DeepSeek-V4-Flash，跨模型AHE分别在GLM-5.1和GPT-5.4下进化。SEAGym负责调用和记录。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.17546#S4.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2606.17546#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2606.17546#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "基准本身固定；比较的是不同方法如何更新可持续保留的运行框架与状态。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "各任务原生评分器给出成绩；平台另外比较不同版本在验证、新任务、旧任务重做和成本上的表现，区分能力提升、遗忘与额外开销。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建评估层，分别提供“执行任务”和“更新方法状态”两类接口。前者返回可公开的执行记录和奖励，后者按各方法原有规则更新跨任务保留的运行框架与状态。任务环境和判分程序仍由原评测基准运行，私有答案及数据划分信息不提供给 task agent。",
"sources": [
{
"label": "算法 1；数据划分；附录 E",
"url": "https://arxiv.org/abs/2606.17546"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "被测方法各自负责更新系统，SEAGym 负责提供任务、保存版本并记录表现。更新后的验证、新任务测试、旧任务重做及成本分别统计，从而观察收益是否保留、是否遗忘。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Terminal-Bench 2 与 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 数学/物理文本题组成 80 道 训练集，按批运行并更新。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "35 道 验证集 用于观察进化轮次快照；更新规则由 ACE、TF-GRPO、AHE 等被测方法决定。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "55 道 同分布测试；另有 80 道 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 计算机科学／人工智能、Engineering 题测 分布外任务；额外重放训练题以观察遗忘。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "明确区分训练、阶段验证、最终 ID/分布外任务 和训练重放；重放成绩不当作新题泛化。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在连续更新过程中反复测新任务、旧能力和成本，观察中间好版本是否随后退化；重点是测量改进过程。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.17546v1",
"version": "2606.17546v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它提供测量和诊断 protocol，不发明一个统一的 harness updater；需要问“哪个方法在更新、哪一版 snapshot 被评”。",
"feedbackCases": [
{
"label": "进化反馈：Terminal-Bench 2 / HLE",
"data": "两者的终端任务和数学/物理文本题合为 80 道训练题，按批执行。",
"scoring": "任务后端提供判分奖励：终端任务按可执行验收检查产物；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 按其题目评分元数据检查回答。论文的统一层不重新定义所有后端判分细节。",
"visible": "训练批次的执行记录、公开错误、奖励和方法允许访问的产物；私有答案、评分资产与数据划分信息不送入修改过程。",
"use": "ACE、TF-GRPO、AHE 分别按原方法更新状态，不是统一用验证集搜索最优版本。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"judgment": "TB2 用程序验收；HLE 用题目评分元数据，本文未展开具体比对器"
},
{
"label": "冻结快照评估与重放",
"data": "35 道验证、55 道同分布测试；另 80 道 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 计算机科学／人工智能 与工程题测分布外迁移；还重做 80 道训练题。",
"scoring": "继续用任务后端的固定判分，比较成功率、退步/恢复情况及耗时费用。",
"visible": "这些评估记录用于研究者观察快照，主协议不将验证或测试结果用作更新证据。",
"use": "区分对新题泛化与对旧题遗忘；不能将“更新后的验证”误写为用于改版本的开发集。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"judgment": "固定任务后端判分；不是用快照评审模型代替任务裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Terminal-Bench 2 与 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 数学/物理文本题组成 80 道 训练集，按批运行并更新。",
"selection": "35 道 验证集 用于观察进化轮次快照；更新规则由 ACE、TF-GRPO、AHE 等被测方法决定。",
"evaluation": "55 道 同分布测试；另有 80 道 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 计算机科学／人工智能、Engineering 题测 分布外任务；额外重放训练题以观察遗忘。",
"isolation": "明确区分训练、阶段验证、最终 ID/分布外任务 和训练重放；重放成绩不当作新题泛化。",
"roles": {
"executor": {
"value": "主对照用DeepSeek-V4-Flash执行ACE、TF-GRPO、AHE；跨模型实验另用GLM-5.1、GPT-5.4，并交换运行框架快照与执行执行端。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.17546#S4.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2606.17546#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2606.17546#A2.SS4"
}
]
},
"modifier": {
"value": "ACE、TF-GRPO、AHE各自实现更新；主实验执行端为DeepSeek-V4-Flash，跨模型AHE分别在GLM-5.1和GPT-5.4下进化。SEAGym负责调用和记录。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.17546#S4.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2606.17546#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2606.17546#A2.SS4"
}
]
},
"seed": {
"value": "作者自建评估层，分别提供“执行任务”和“更新方法状态”两类接口。前者返回可公开的执行记录和奖励，后者按各方法原有规则更新跨任务保留的运行框架与状态。任务环境和判分程序仍由原评测基准运行，私有答案及数据划分信息不提供给 task agent。",
"sources": [
{
"label": "算法 1；数据划分；附录 E",
"url": "https://arxiv.org/abs/2606.17546"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"selection": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"evaluation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
],
"isolation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.17546#S3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "静态 agent 评测逐题重置，只能说明某个固定版本有多强；自进化却会改变后续任务使用的系统。只报最后涨分，无法区分泛化、对开发题适配和原有能力退化；已有连续评测也未覆盖不同更新节奏，因此需要保留版本并分开测学习、迁移与遗忘。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.17546#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 框架的连续更新是否产生可复用的能力提升，同时区分新任务泛化、旧能力遗忘、执行成本与短期过拟合。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17546"
}
]
},
{
"key": "conclusion",
"value": "发现频繁更新不一定改善留出表现，中间的好版本也可能在后续进化中退化。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.17546"
}
]
}
],
"fields": {
"object": "基准本身固定；比较的是不同方法如何更新可持续保留的运行框架与状态。",
"verdict": "各任务原生评分器给出成绩；平台另外比较不同版本在验证、新任务、旧任务重做和成本上的表现，区分能力提升、遗忘与额外开销。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.17546"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2607.14777",
"title": "SEED: Self-Evolving On-Policy Distillation for Agentic RL",
"url": "https://arxiv.org/abs/2607.14777",
"date": "2026-07-16",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"HeldOut",
"M1",
"SameModel",
"Weights",
"org:tsinghua",
"org:ntu"
],
"fields": {
"本质定位": "当前 policy rollout 后自己充当 trajectory analyzer 生成 hindsight skill，再通过 OPD + GRPO 把 skill-conditioned action preference 蒸馏进 weights。",
"什么在变": "model weights / policy；临时 hindsight skill 不持久部署。",
"谁来改 / 谁执行": "**改**：固定 GRPO+OPD；Stage1 外部 GLM-5.2 bootstrap，Stage2 current policy analyzer。
**执行**：当前 policy checkpoint。",
"基础 harness": "普通 agent policy，无 persistent skill harness。",
"Feedback": "environment terminal reward + trajectory。",
"Evolution → Eval": "ALFWorld/WebShop/SearchQA train/eval 分开。",
"Meta-depth": "model-level M1。",
"相对之前真正新增什么": "它不是 harness evolution；价值是作为对照：经验最终写回权重，skill 只是 training-time privileged signal。"
},
"details": [
{
"label": "本质定位",
"text": "当前 policy 自己 rollout，再用当前 checkpoint 充当 trajectory analyzer 生成 hindsight skill，把 skill 对 sampled action 的影响通过 OPD + GRPO 写回 weights。"
},
{
"label": "什么在变",
"text": "policy weights；hindsight skill 是训练时 privileged signal，不作为 persistent deployment harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "Stage-2 actor/analyzer 是当前 policy checkpoint；外层 GRPO+OPD algorithm 固定；Stage-1 有 GLM-5.2 bootstrap。"
},
{
"label": "Feedback",
"text": "environment terminal reward +完整 trajectory。"
},
{
"label": "核心结果",
"text": "旧记录：ALFWorld 75.0→91.8、WebShop 63.3→78.9；dynamic/on-policy skill 优于 static skill，并有 unseen generalization。"
},
{
"label": "为什么经典",
"text": "它是 model-level endogenous self-evolution 的一个非常清晰、效果强、开源的代表；用来和 harness-only 路线对照很重要。"
},
{
"label": "边界",
"text": "“self-evolving”主要是 actor/analyzer checkpoint 同步刷新；training algorithm 并未被模型自己重写。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 50,
"fields": {
"时间": "2026-07",
"论文": "**[SEED](https://arxiv.org/abs/2607.14777)**",
"为什么是主干": "endogenous model self-improvement 代表：当前 policy 自己产生/分析 on-policy experience，再蒸馏回 weights。",
"证据边界要记住": "Stage-1 有外部 bootstrap；外层训练算法固定。"
}
},
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 348,
"fields": {
"优先级": "**C**",
"时间": "2026-07-16",
"论文": "[SEED: Self-Evolving On-Policy Distillation for Agentic RL](https://arxiv.org/abs/2607.14777)",
"本质定位": "当前 policy rollout 后自己充当 trajectory analyzer 生成 hindsight skill，再通过 OPD + GRPO 把 skill-conditioned action preference 蒸馏进 weights。",
"什么在变": "model weights / policy；临时 hindsight skill 不持久部署。",
"谁来改 / 谁执行": "**改**：固定 GRPO+OPD；Stage1 外部 GLM-5.2 bootstrap，Stage2 current policy analyzer。
**执行**：当前 policy checkpoint。",
"基础 harness": "普通 agent policy，无 persistent skill harness。",
"Feedback": "environment terminal reward + trajectory。",
"Evolution → Eval": "ALFWorld/WebShop/SearchQA train/eval 分开。",
"Meta-depth": "model-level M1。",
"相对之前真正新增什么": "它不是 harness evolution；价值是作为对照：经验最终写回权重，skill 只是 training-time privileged signal。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录报告独立任务评估；查看详细记录中的数据与选模限制。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"M-Weight"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "当前 policy rollout 后自己充当 trajectory analyzer 生成 hindsight skill，再通过 OPD + GRPO 把 skill-conditioned action preference 蒸馏进 weights。",
"novelty": "从模型自己的执行经历提炼文字指导，在训练时帮助产生更好的行为，再把指导转入模型参数；最终执行不依赖额外的这份指导。",
"object": "策略模型的参数；事后技能用于训练，不作为永久部署记忆。",
"executor": "Qwen2.5-3B-Instruct、Qwen2.5-7B-Instruct、Qwen3-1.7B-Instruct的当前checkpoint（某个时刻保存的模型或系统版本）；多模态扩展用Qwen2.5-VL-3B-Instruct。",
"modifier": "Stage1由GLM-5.2从轨迹抽取事后技能以构造监督微调（用示范数据训练模型）；Stage2由当前策略checkpoint（某个时刻保存的模型或系统版本）兼任分析器，固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）与从当前模型实际执行的记录中进行蒸馏学习程序更新参数。",
"roleContext": "**改**：固定 GRPO+OPD；Stage1 外部 GLM-5.2 bootstrap，Stage2 current policy analyzer。
**执行**：当前 policy checkpoint。",
"seed": "固定任务交互接口上的可训练 task agent，先用轨迹—技能数据 监督微调（用示范数据训练模型），再进行 on-policy distillation/强化学习（根据奖励调整模型行为）。技能是训练信号，不等同于部署时另外维护一个永久技能库；ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、搜索各有原生动作接口。",
"fixed": "",
"verdict": "环境在任务结束时返回的奖励，以及完成任务的逐步执行记录。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "ALFWorld/WebShop/SearchQA train/eval 分开。"
}
],
"takeaway": "“self-evolving”主要是 actor/analyzer checkpoint 同步刷新；training algorithm 并未被模型自己重写。",
"result": "旧记录：ALFWorld 75.0→91.8、WebShop 63.3→78.9；dynamic/on-policy skill 优于 static skill，并有 unseen generalization。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：监督微调（用示范数据训练模型） 每类 180 题×8 条轨迹；强化学习（根据奖励调整模型行为）：ALFWorld（通过文字动作完成家居物体操作的交互环境） 2,400、WebShop（根据用户要求挑选和购买商品的交互基准） 2,400、搜索 QA 19,200 条训练实例。\n\n调试 / 选版本数据：训练轨迹的终局奖励和事后技能用于学习；数据比例实验比较样本效率。\n\n最终测试数据：ALFWorld（通过文字动作完成家居物体操作的交互环境） 140 已见＋134 未见；WebShop（根据用户要求挑选和购买商品的交互基准） 128 题；搜索 QA 共 51,713 题，涵盖 NQ、TriviaQA、PopQA、HotpotQA（需要结合多份资料作答的多跳问答基准）、2Wiki、MuSiQue、Bamboogle。\n\n数据隔离与证据边界：训练和评估分别配置；ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 seen/unseen 结果分开报告。七个 QA 评估集不能全部当成训练集。",
"cycle": "分析器从当前策略生成的轨迹中提取事后技能；技能条件下的 词元 概率提供更密集的训练信号。先做轨迹—技能 监督微调（用示范数据训练模型），再联合 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 与从当前模型实际执行的记录中进行蒸馏学习；分析器随策略同步更新。按 强化学习（根据奖励调整模型行为）/蒸馏目标更新参数，不是用测试分数接受某段运行框架补丁。",
"train": "监督微调（用示范数据训练模型） 每类 180 题×8 条轨迹；强化学习（根据奖励调整模型行为）：ALFWorld（通过文字动作完成家居物体操作的交互环境） 2,400、WebShop（根据用户要求挑选和购买商品的交互基准） 2,400、搜索 QA 19,200 条训练实例。",
"debug": "训练轨迹的终局奖励和事后技能用于学习；数据比例实验比较样本效率。",
"test": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 140 已见＋134 未见；WebShop（根据用户要求挑选和购买商品的交互基准） 128 题；搜索 QA 共 51,713 题，涵盖 NQ、TriviaQA、PopQA、HotpotQA（需要结合多份资料作答的多跳问答基准）、2Wiki、MuSiQue、Bamboogle。",
"isolation": "训练和评估分别配置；ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 seen/unseen 结果分开报告。七个 QA 评估集不能全部当成训练集。"
},
"reviewed": false,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "针对 agentic RL 只有 sparse trajectory-level reward、难做细粒度 credit assignment 的问题，让最新 policy 同时作为 actor 和 trajectory analyzer，把自己的 on-policy rollout 总结成 hindsight skill，再将 skill 对 sampled actions 的行为影响蒸馏回模型权重。",
"index": {
"date": "2026-07-16",
"paper": "[SEED: Self-Evolving On-Policy Distillation for Agentic Reinforcement Learning](https://arxiv.org/abs/2607.14777)",
"executor": "当前 policy checkpoint 作为 agent 执行 ALFWorld / WebShop / Search QA 等任务",
"modifier": "固定的 **GRPO + OPD training algorithm** 更新 policy weights；Stage 2 同一 checkpoint 还充当 trajectory analyzer",
"feedback": "environment terminal reward / task outcome + 完整 trajectory；Stage 1 额外有 **GLM-5.2 hindsight-skill supervision**",
"evolve": "**ALFWorld**：180-task SFT seed + 2,400 RL tasks
**WebShop**：180 + 2,400
**Search QA**：180 + 19,200；另有 Sokoban / EZPoints multimodal extension",
"eval": "**ALFWorld**：140 seen + 134 unseen test
**WebShop**：128 test
**Search QA**：51,713 eval questions；multimodal 也用 held-out eval",
"tldr": "当前 policy 自己 rollout、再分析 trajectory/outcome 生成 hindsight skill，并蒸馏回权重；held-out generalization 较规范，但 Stage 1 依赖外部 bootstrap，且没有直接证明 analyzer quality 随 checkpoint 持续变强，因此更像 model-level adaptive self-distillation，而非 harness RSI。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | Outcome-based agentic RL 通常只给 episode-level sparse/delayed reward，不能指出中间哪些 observation、tool call、action 应强化或纠正；已有 reflection / memory / skill 方法又常把 hindsight 当作 static experience、固定 teacher 或 inference-time context，随着 policy 变化可能与当前 failure mode 失配。 |\n| 本文定位 | 提出 **SEED（Self-Evolving On-Policy Distillation）**：把当前 policy 自己完成的 trajectory 转成 natural-language hindsight skill，并把 skill-conditioned 与 ordinary context 下对同一 sampled action 的 probability shift 转成 dense token-level OPD signal，与 outcome RL 联合训练。 |\n| 为什么叫 self-evolving | Stage 2 中**最新 policy checkpoint 同时充当 actor 和 analyzer**；policy 更新后，下一轮 rollout distribution 和负责分析这些 rollout 的 model checkpoint 一起更新，因此作者称 decision making 与 hindsight supervision co-evolve。 |\n| 核心贡献 | `on-policy rollout → self-analysis → hindsight skill → token-level OPD + GRPO → next policy`；skill 只作为 training-time privileged supervision，deployment 时不需要 skill memory、retrieval 或额外 analyzer。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **模型权重**及随之变化的 on-policy trajectory distribution；不是 harness、memory 或外部 persistent skill bank。 |\n| Stage 1：外部 bootstrap | 每个 backbone 从 180 个 training tasks 各采 8 条 rollout，共 **1,440 trajectories**；外部 **GLM-5.2** 读取完整 task/trajectory/reward/final outcome，生成 hindsight skill；再用 `(trajectory → skill)` 做 3-epoch SFT，让 backbone 先学会 trajectory analysis。 |\n| Stage 2：self-evolving loop | 当前 frozen snapshot $\\pi_t$ 自己 rollout，并用**同一 checkpoint**作为 analyzer 对完成的 trajectory 生成 hindsight skill；对同一 sampled action 比较普通 context 与 skill-augmented context 的 log-probability，形成 OPD signal，与 GRPO 联合更新得到 $\\pi_{t+1}$。 |\n| 谁来改 | 固定人工设计的 **GRPO + OPD optimization algorithm** 更新 weights；不是 coding agent 主动决定如何重写 optimizer/workflow。 |\n| Analyzer 能看到什么 | 完整 task、observations/actions、rewards 和 terminal outcome；因此 hindsight skill 是 **reward-grounded self-analysis**，不是无 verifier 的纯自监督 reflection。 |\n| Skill 生命周期 | Stage 2 每轮根据当前 policy 的新 trajectory 重新生成；不作为跨任务 persistent memory 保存，也不在 inference 时提供给 actor。 |\n| 方法本质 | **model-weight agentic RL / adaptive self-distillation**，不是 harness RSI。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark | Backbone / training | Feedback | Train → Test | GT / 外部 supervision |\n|---|---|---|---|---|\n| ALFWorld | Qwen2.5-3B/7B、Qwen3-1.7B；SFT 180 tasks，RL 2,400 | environment success / terminal reward + trajectory | RL train 与 eval 分开；140 seen test + **134 unseen test** | Stage 1 有 GLM-5.2 skill supervision；Stage 2 依赖 environment outcome |\n| WebShop | 同类 backbone；SFT 180，RL 2,400 | normalized task score + exact success | **128 test samples**，不进入 RL training | 同上 |\n| Search-Augmented QA | 同类 backbone；SFT 180，RL 19,200 | answer correctness / reward + search trajectory | **51,713 eval questions** | outcome-grounded |\n| Multimodal extension：Sokoban / EZPoints | Qwen2.5-VL-3B | environment/task success | held-out evaluation | 同样的 SEED training mechanism |\n\n> 数据隔离相对规范：**training/evolution tasks 与最终 evaluation tasks 分开**，没有像 RSIBench-Data 那样反复利用同一最终 test subset 的 reward 来改模型。"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| SEED 明显优于 outcome-only GRPO | 三个 backbone 上，相对 GRPO：ALFWorld **+14.9～45.9 pt**，Search QA **+1.4～9.3**，WebShop score **+8.7～19.8**，success **+5.5～39.0** | trajectory hindsight 转成 dense on-policy supervision 对 agentic RL 有显著价值 |\n| 3B backbone 上提升很大 | Qwen2.5-3B：ALFWorld **75.0 GRPO → 91.8 SEED**；WebShop success **63.3→78.9**；Search QA **36.4→45.7** | 方法效果不只来自更大 backbone |\n| Dynamic/on-policy skill 优于 static skill | ALFWorld：SEED **91.8**；w/o Hindsight Skill SFT 86.0；w/o Self-Evolving OPD 87.0；w/o On-Policy Skill、改 static offline skills 84.4 | 当前 policy 自己的新 trajectory 所产生的 guidance 比固定旧 skill 更匹配当前行为分布 |\n| Sample efficiency 更高 | ALFWorld 仅用 60% data 的 SEED **80.7**，已经超过 full-data GRPO 75.0 | dense hindsight supervision 提高训练样本利用率 |\n| 有 unseen generalization | ALFWorld unseen：GRPO **70.9 → SEED 86.2**，6 类中 5 类提升 | 改进不是只体现在 RL training trajectories 上，存在较干净 held-out transfer evidence |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| 是否属于 Harness Evolution | **不是。** 最终改变的是 policy weights；inference-time harness 没有被搜索或改写。 |\n| “Self-evolving”口径 | 比较弱：核心 loop 是人工预先规定的 `rollout → analyze → OPD+GRPO`；模型不会自己决定“应该怎样修改自己的训练机制”。 |\n| External bootstrap | **Stage 1 明确依赖 GLM-5.2** 给 1,440 rollout 标 hindsight skills，先把 trajectory-analysis 能力教给 target model；因此整个系统不是从零开始的纯 self-generated supervision。 |\n| Analyzer 是否真的越进化越强 | **没有直接验证。** actor/analyzer 共享 checkpoint，weights 更新后 analyzer 随之变化，但论文没有单独测不同 checkpoint 的 hindsight-skill quality 是否持续提高。 |\n| 最关键的 co-evolution 归因是否干净 | 不完全。论文有 static-skill / w/o OPD 等 ablation，能说明 dynamic hindsight 有作用；但缺一个严格的 **current synchronized analyzer vs frozen old analyzer** matched comparison，因此不能完全隔离“analyzer 自身同步进化”这一因素。 |\n| Skill correctness | self-generated skill 不保证正确；它是根据 trajectory + terminal outcome 生成的 behavioral hypothesis，dense/informative 不等于 ground-truth process supervision。 |\n| 对 RSI 调研的意义 | 更适合作为 **model-level self-evolution / adaptive self-distillation**：说明当前 policy 产生的经验可以动态转成改进信号并蒸馏进 weights，但与 harness self-evolution 的核心研究对象不同。 |"
},
{
"title": "总评",
"body": "**SEED 是一个效果较强、数据隔离也比较规范的 agentic RL/self-distillation 方法，但它的“self-evolving”主要来自 actor 与 analyzer 共享并同步刷新同一个 policy checkpoint，而不是 agent 主动修改自己的 harness 或 training algorithm。Stage 1 还依赖 GLM-5.2 外部 bootstrap。论文较好证明了 dynamic on-policy hindsight 优于 static skill，但没有直接证明 analyzer 本身随着 evolution 持续变强，因此应归入 model-level self-evolution，而不是 Harness Evolution。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "固定任务交互接口上的可训练 agent，先用轨迹—技能数据 SFT，再进行 on-policy distillation/RL。技能是训练信号，不等同于部署时另外维护一个永久技能库；ALFWorld、WebShop、搜索各有原生动作接口。",
"protocol": "**SFT：**ALFWorld、WebShop、搜索 QA 每类选 180 个训练任务，每题 8 条 rollout，共每类 1,440 条轨迹。\n\n**RL→测试：**ALFWorld 2,400 训练→140 seen＋134 unseen；WebShop 2,400→128；搜索 QA 19,200→51,713 个评估问题。搜索来源为 NQ、TriviaQA、PopQA、HotpotQA、2WikiMultiHopQA、MuSiQue、Bamboogle。不同数据来源是否都参与训练需按 Search-R1 配置看，不把七个名字都默认成训练集。",
"sections": "Table 3 与训练数据配置",
"source": "https://arxiv.org/abs/2607.14777",
"version": "2607.14777v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "69fdf736c447ae970e321ad0c67b012bdaebb3745e625ad31bf9e45369dad30a",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen2.5-3B-Instruct、Qwen2.5-7B-Instruct、Qwen3-1.7B-Instruct的当前checkpoint（某个时刻保存的模型或系统版本）；多模态扩展用Qwen2.5-VL-3B-Instruct。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px4"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2607.14777#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Stage1由GLM-5.2从轨迹抽取事后技能以构造监督微调（用示范数据训练模型）；Stage2由当前策略checkpoint（某个时刻保存的模型或系统版本）兼任分析器，固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）与从当前模型实际执行的记录中进行蒸馏学习程序更新参数。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px4"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2607.14777#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "策略模型的参数；事后技能用于训练，不作为永久部署记忆。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "环境在任务结束时返回的奖励，以及完成任务的逐步执行记录。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定任务交互接口上的可训练 task agent，先用轨迹—技能数据 监督微调（用示范数据训练模型），再进行 on-policy distillation/强化学习（根据奖励调整模型行为）。技能是训练信号，不等同于部署时另外维护一个永久技能库；ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、搜索各有原生动作接口。",
"sources": [
{
"label": "Table 3 与训练数据配置",
"url": "https://arxiv.org/abs/2607.14777"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "分析器从当前策略生成的轨迹中提取事后技能；技能条件下的 词元 概率提供更密集的训练信号。先做轨迹—技能 监督微调（用示范数据训练模型），再联合 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 与从当前模型实际执行的记录中进行蒸馏学习；分析器随策略同步更新。按 强化学习（根据奖励调整模型行为）/蒸馏目标更新参数，不是用测试分数接受某段运行框架补丁。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "监督微调（用示范数据训练模型） 每类 180 题×8 条轨迹；强化学习（根据奖励调整模型行为）：ALFWorld（通过文字动作完成家居物体操作的交互环境） 2,400、WebShop（根据用户要求挑选和购买商品的交互基准） 2,400、搜索 QA 19,200 条训练实例。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练轨迹的终局奖励和事后技能用于学习；数据比例实验比较样本效率。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 140 已见＋134 未见；WebShop（根据用户要求挑选和购买商品的交互基准） 128 题；搜索 QA 共 51,713 题，涵盖 NQ、TriviaQA、PopQA、HotpotQA（需要结合多份资料作答的多跳问答基准）、2Wiki、MuSiQue、Bamboogle。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "训练和评估分别配置；ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 seen/unseen 结果分开报告。七个 QA 评估集不能全部当成训练集。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从模型自己的执行经历提炼文字指导，在训练时帮助产生更好的行为，再把指导转入模型参数；最终执行不依赖额外的这份指导。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.14777v1",
"version": "2607.14777v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它不是 harness evolution；价值是作为对照：经验最终写回权重，skill 只是 training-time privileged signal。",
"feedbackCases": [
{
"label": "ALFWorld",
"data": "强化学习 2,400 条实例；测试 140 已见 与 134 未见",
"scoring": "环境检查操作目标是否完成并返回终局成功/失败。",
"visible": "完整执行记录和终局结果；分析模型事后提炼技能，再比较有/无技能指导时的模型输出概率，形成更细粒度的训练信号。",
"use": "终局奖励和技能条件下的蒸馏共同训练参数；测试不需要注入这些事后技能。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"judgment": "ALFWorld 环境程序检查目标完成"
},
{
"label": "WebShop",
"data": "强化学习 2,400 条实例；测试 128 题",
"scoring": "环境给购物约束满足程度的连续分数；另以完全满足目标的二元指标报告成功率。",
"visible": "完整执行记录和终局结果；分析模型事后提炼技能，再比较有/无技能指导时的模型输出概率，形成更细粒度的训练信号。",
"use": "终局奖励和技能条件下的蒸馏共同训练参数；测试不需要注入这些事后技能。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"judgment": "WebShop 环境计算目标约束满足分"
},
{
"label": "搜索问答",
"data": "强化学习 19,200 条实例；测试涵盖 NQ、TriviaQA、PopQA、HotpotQA（需要结合多份资料作答的多跳问答基准）、2Wiki、MuSiQue、Bamboogle 共 51,713 题",
"scoring": "根据问答基准参考答案评价最终回答；搜索工具返回的网页只是解题证据，不等于正确性标签。",
"visible": "完整执行记录和终局结果；分析模型事后提炼技能，再比较有/无技能指导时的模型输出概率，形成更细粒度的训练信号。",
"use": "终局奖励和技能条件下的蒸馏共同训练参数；测试不需要注入这些事后技能。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"judgment": "参考答案比对；所引方法段未说明规则匹配或模型裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "监督微调（用示范数据训练模型） 每类 180 题×8 条轨迹；强化学习（根据奖励调整模型行为）：ALFWorld（通过文字动作完成家居物体操作的交互环境） 2,400、WebShop（根据用户要求挑选和购买商品的交互基准） 2,400、搜索 QA 19,200 条训练实例。",
"selection": "训练轨迹的终局奖励和事后技能用于学习；数据比例实验比较样本效率。",
"evaluation": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 140 已见＋134 未见；WebShop（根据用户要求挑选和购买商品的交互基准） 128 题；搜索 QA 共 51,713 题，涵盖 NQ、TriviaQA、PopQA、HotpotQA（需要结合多份资料作答的多跳问答基准）、2Wiki、MuSiQue、Bamboogle。",
"isolation": "训练和评估分别配置；ALFWorld（通过文字动作完成家居物体操作的交互环境） 的 seen/unseen 结果分开报告。七个 QA 评估集不能全部当成训练集。",
"roles": {
"executor": {
"value": "Qwen2.5-3B-Instruct、Qwen2.5-7B-Instruct、Qwen3-1.7B-Instruct的当前checkpoint（某个时刻保存的模型或系统版本）；多模态扩展用Qwen2.5-VL-3B-Instruct。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px4"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2607.14777#A3.SS3"
}
]
},
"modifier": {
"value": "Stage1由GLM-5.2从轨迹抽取事后技能以构造监督微调（用示范数据训练模型）；Stage2由当前策略checkpoint（某个时刻保存的模型或系统版本）兼任分析器，固定GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）与从当前模型实际执行的记录中进行蒸馏学习程序更新参数。",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px4"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2607.14777#A3.SS3"
}
]
},
"seed": {
"value": "固定任务交互接口上的可训练 task agent，先用轨迹—技能数据 监督微调（用示范数据训练模型），再进行 on-policy distillation/强化学习（根据奖励调整模型行为）。技能是训练信号，不等同于部署时另外维护一个永久技能库；ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、搜索各有原生动作接口。",
"sources": [
{
"label": "Table 3 与训练数据配置",
"url": "https://arxiv.org/abs/2607.14777"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"selection": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"evaluation": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
],
"isolation": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.14777#A2.SS1.SSS0.Px4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.14777#S4.SS1.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务的最终成败奖励太粗：失败轨迹可能只有局部错误，成功轨迹里的关键决策也不会被单个分数点明，模型因而难学到中间哪一步该保留或纠正。静态经验和固定教师又跟不上模型行为变化，因此需要从当前轨迹持续提炼能用于逐步学习的监督。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.14777#S1"
}
]
},
{
"key": "position",
"value": "研究交互 agent 能否从自己刚完成的经历中学到更具体的行动指导，弥补只看任务最终成败时缺少中间监督的问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.14777"
}
]
},
{
"key": "conclusion",
"value": "在交互任务上提高学习效果；自然语言经验在训练期间使用，最终转化为参数中的行为能力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.14777"
}
]
}
],
"fields": {
"object": "策略模型的参数；事后技能用于训练，不作为永久部署记忆。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.14777"
}
]
},
{
"tag": "org:ntu",
"label": "Nanyang Technological University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.14777"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights",
"feedback"
]
},
{
"id": "2607.15524",
"title": "Recursive Harness Self-Improvement",
"url": "https://arxiv.org/abs/2607.15524",
"date": "2026-07-17",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Prompt",
"Workflow",
"SeparateEvolver",
"PairwiseFeedback",
"Sequential",
"M1",
"org:berkeley",
"org:sakana"
],
"fields": {
"本质定位": "把 user-constructed harness 表示成 **prompt-level agent-loop specification**，每轮只与上一 revision 的产物做一次 pairwise LLM preference，并把累计 self-comparison history 当作后续 update 的 momentum-like signal。",
"谁来改 → 谁执行；基础 harness": "optimizer/reviser → prompt-level agent loop executor。",
"Feedback / evidence": "pairwise comparison across versions。",
"什么在变": "agent roles/instructions、communication contracts、workflow hops、context-management prompt specification；**不搜索 executable code**。",
"谁来改 / 谁执行": "**改**：LLM harness optimizer 根据 current spec + revision-history preferences 写下一版。
**执行**：低 reasoning-effort research/coding agent under current harness。",
"基础 harness": "user-constructed prompt-level multi-agent loop。",
"Feedback": "successive outputs/repositories 的 pairwise LLM judgment + accumulated revision history；不是 task-native formal verifier。",
"Evolution → Eval": "30 个 synthetic ML-research tasks（quant finance/robotics/pharmacy）上 task-specific iterative refinement；主证据不是独立 held-out task generalization。",
"Meta-depth": "M1；optimizer/judge protocol fixed。",
"相对之前真正新增什么": "相对 Meta-Harness/Self-Harness 的核心差异是 **低成本 trajectory-local preference optimization**：不用 population search，也不要求 verifier pass-count；提升主要来自更好的 inter-agent information flow/context sparsification，而非更长 reasoning。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "prompt-level harness self-improvement 的代表。与 Meta-Harness 的 population/code search、Self-Harness 的 verifier regression 不同，RHI 每轮只比较当前 revision 与上一 revision 的产物，用累计 pairwise preference history 引导下一次修改。"
},
{
"label": "什么在变",
"text": "prompt-level agent loop：roles/instructions、subagent–orchestrator communication contracts、workflow hops、context-management rules；**不是 full executable harness code**。"
},
{
"label": "谁来改 / 谁执行",
"text": "LLM harness optimizer 写下一版 prompt specification；target research/coding agent 在当前 harness 下执行 30 个 synthetic ML-research tasks。"
},
{
"label": "基础 harness",
"text": "user-constructed low-reasoning-effort multi-agent loop。"
},
{
"label": "Feedback",
"text": "LLM evaluator 对 successive repositories/outputs 做 pairwise preference；累计 self-comparison history作为 update signal。"
},
{
"label": "Evolution → Eval",
"text": "30 个 quant-finance / robotics / pharmacy synthetic ML-research tasks 上做 task-specific refinement；主证据不是独立 held-out task transfer。"
},
{
"label": "主要结果",
"text": "几轮修改即可让 low-reasoning-effort agent 超过对应 maximum-reasoning-effort setting，并报告最高约 60% inference-cost reduction；分析认为收益主要来自更有效的 inter-agent information flow，而非生成更长 reasoning。"
},
{
"label": "为什么是经典 / 关键限制",
"text": "它代表了 **廉价、局部、preference-based harness refinement** 这条路线；但 editable space 偏 prompt/workflow，pairwise judge 也是外部固定边界，且不等于 full-harness recursive self-improvement。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 51,
"fields": {
"时间": "2026-07",
"论文": "**[Recursive Harness Self-Improvement (RHI)](https://arxiv.org/abs/2607.15524)**",
"为什么是主干": "harness revision-history / recursive revision 这条路线的代表。",
"证据边界要记住": "evaluator/optimizer role 分离，不是纯 same-model self-loop。"
}
},
{
"section": "D. Full harness / executable harness optimization",
"line": 178,
"fields": {
"时间": "2026-07-17",
"论文": "[Recursive Harness Self-Improvement (RHI)](https://arxiv.org/abs/2607.15524)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**H-Prompt / H-Full-lite**。把 agent loop 表成 prompt-level harness，用 consecutive versions 的 **pairwise feedback history** 递归 revision；不是 full source-code foundry。",
"谁来改 → 谁执行；基础 harness": "optimizer/reviser → prompt-level agent loop executor。",
"Feedback / evidence": "pairwise comparison across versions。",
"标签": "`#Prompt #Workflow #PairwiseFeedback #Sequential #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 333,
"fields": {
"优先级": "**C**",
"时间": "2026-07-17",
"论文": "[Recursive Harness Self-Improvement](https://arxiv.org/abs/2607.15524)",
"本质定位": "把 user-constructed harness 表示成 **prompt-level agent-loop specification**，每轮只与上一 revision 的产物做一次 pairwise LLM preference，并把累计 self-comparison history 当作后续 update 的 momentum-like signal。",
"什么在变": "agent roles/instructions、communication contracts、workflow hops、context-management prompt specification；**不搜索 executable code**。",
"谁来改 / 谁执行": "**改**：LLM harness optimizer 根据 current spec + revision-history preferences 写下一版。
**执行**：低 reasoning-effort research/coding agent under current harness。",
"基础 harness": "user-constructed prompt-level multi-agent loop。",
"Feedback": "successive outputs/repositories 的 pairwise LLM judgment + accumulated revision history；不是 task-native formal verifier。",
"Evolution → Eval": "30 个 synthetic ML-research tasks（quant finance/robotics/pharmacy）上 task-specific iterative refinement；主证据不是独立 held-out task generalization。",
"Meta-depth": "M1；optimizer/judge protocol fixed。",
"相对之前真正新增什么": "相对 Meta-Harness/Self-Harness 的核心差异是 **低成本 trajectory-local preference optimization**：不用 population search，也不要求 verifier pass-count；提升主要来自更好的 inter-agent information flow/context sparsification，而非更长 reasoning。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Same-set adaptive",
"protocolBasis": "依据原记录：主实验在同一任务/子集上反复优化或评估；不能直接当作冻结后的泛化。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full",
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 roles、instructions、communication contracts 和 workflow hops 写成 prompt-level agent-loop specification。每轮比较当前与上一版产物，用累计偏好历史指导下一次改写。",
"novelty": "比较相邻两个版本的任务产物来调整提示、角色间信息传递和上下文配置；修改范围是这些用户可配置的说明与设置。",
"object": "描述 agent 角色、通信方式、工作流和上下文管理的提示文本；不修改可执行代码或模型参数。",
"executor": "sonnet-4.6、opus-4.7、opus-4.8 的 research/coding task agent，在当前 loop specification 下生成仓库。",
"modifier": "语言模型 运行框架优化器读取历次仓库比较反馈并改写工作流提示。§5.1 列明了执行模型，附录 C 给出优化提示，但这两处没有明确指定优化器的模型型号；不能把评价用 GPT-5.5 当成优化器型号。",
"roleContext": "**改**：LLM harness optimizer 根据 current spec + revision-history preferences 写下一版。
**执行**：低 reasoning-effort research/coding agent under current harness。",
"seed": "研究任务专用的多 task agent 提示设计，初始定义角色、交接、产物与检查规则，再迭代这些说明。展示的工具/数据清单属于方案规定，不能自动当作已经执行并验证的系统功能。",
"fixed": "模型权重、optimizer/judge 流程固定。",
"verdict": "模型按功能、任务要求和可复现性等标准比较前后两个产物，判断哪版更好；这是成对质量评审，并非所有任务都由代码测试给出通过率。",
"diagnosis": "从相邻输出/代码仓库的比较及累积 revision history 判断信息流瓶颈。",
"update": "改 roles、instructions、contracts 与 hops；目标是减少上下文冗余、改善任务相关信息传递。",
"acceptance": "用后续 pairwise 比较衡量 revision；不能读成 verifier-gated 全代码自修改。",
"experiments": [
{
"name": "ML research tasks",
"evolve": "30 个 synthetic tasks：量化金融、机器人、制药",
"selection": "同任务上相邻 revision 比较",
"test": "相同任务上的 pairwise 评估；多个模型/seed",
"isolation": "同任务迭代",
"note": "不是跨任务冻结后泛化的主要证据。"
}
],
"takeaway": "它代表了 **廉价、局部、preference-based harness refinement** 这条路线；但 editable space 偏 prompt/workflow，pairwise judge 也是外部固定边界，且不等于 full-harness recursive self-improvement。",
"result": "几轮修改即可让 low-reasoning-effort agent 超过对应 maximum-reasoning-effort setting，并报告最高约 60% inference-cost reduction；分析认为收益主要来自更有效的 inter-agent information flow，而非生成更长 reasoning。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2607.15524v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：30 个合成 ML 研究任务：量化金融、机器人、制药各 10 个，由行业招聘描述转成研究任务。没有模型参数训练。\n\n调试 / 选版本数据：同一任务上比较相邻版本生成的仓库；评审反馈写入该任务的运行框架历史。\n\n最终测试数据：继续在这 30 个任务上比较版本，另比较不同模型和随机运行。\n\n数据隔离与证据边界：同题定制与重跑，不是从训练题学习通用运行框架后在另一批题盲测。",
"cycle": "比较相邻版本的产物及历史修改，找出多 agent 之间的信息传递问题。修改的是文字规定的角色、指令、交付要求和通信步骤；后续仍按产物成对评审比较，不能据此声称完整运行代码都经过执行测试。",
"train": "30 个合成 ML 研究任务：量化金融、机器人、制药各 10 个，由行业招聘描述转成研究任务。没有模型参数训练。",
"debug": "同一任务上比较相邻版本生成的仓库；评审反馈写入该任务的运行框架历史。",
"test": "继续在这 30 个任务上比较版本，另比较不同模型和随机运行。",
"isolation": "同题定制与重跑，不是从训练题学习通用运行框架后在另一批题盲测。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "研究任务专用的多 agent 提示设计，初始定义角色、交接、产物与检查规则，再迭代这些说明。展示的工具/数据清单属于方案规定，不能自动当作已经执行并验证的系统功能。",
"protocol": "**数据：**30 个合成 ML 研究任务，涉及量化金融、机器人、制药等；例如蛋白侧链任务要求 RCSB 数据来源和 train/val/test manifest。\n\n**评估：**同任务比较相邻 harness revision，并做多模型/seed 的成对评审；不是先在 30 题进化后到另一批任务盲测。任务方案写有 split 要求不等于每个实际产物都正确完成隔离。",
"sections": "任务设定、初始/迭代 harness 示例与评估",
"source": "https://arxiv.org/abs/2607.15524",
"version": "2607.15524v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "df7af1e9e183f0ce0b6d462101ef7e20880b3f4a50d8dd01ed7c3634874dd5cd",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "sonnet-4.6、opus-4.7、opus-4.8 的 research/coding task agent，在当前 loop specification 下生成仓库。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "语言模型 运行框架优化器读取历次仓库比较反馈并改写工作流提示。§5.1 列明了执行模型，附录 C 给出优化提示，但这两处没有明确指定优化器的模型型号；不能把评价用 GPT-5.5 当成优化器型号。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.15524#A3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "描述 agent 角色、通信方式、工作流和上下文管理的提示文本；不修改可执行代码或模型参数。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "模型按功能、任务要求和可复现性等标准比较前后两个产物，判断哪版更好；这是成对质量评审，并非所有任务都由代码测试给出通过率。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "研究任务专用的多 task agent 提示设计，初始定义角色、交接、产物与检查规则，再迭代这些说明。展示的工具/数据清单属于方案规定，不能自动当作已经执行并验证的系统功能。",
"sources": [
{
"label": "任务设定、初始/迭代 harness 示例与评估",
"url": "https://arxiv.org/abs/2607.15524"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "比较相邻版本的产物及历史修改，找出多 agent 之间的信息传递问题。修改的是文字规定的角色、指令、交付要求和通信步骤；后续仍按产物成对评审比较，不能据此声称完整运行代码都经过执行测试。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "30 个合成 ML 研究任务：量化金融、机器人、制药各 10 个，由行业招聘描述转成研究任务。没有模型参数训练。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同一任务上比较相邻版本生成的仓库；评审反馈写入该任务的运行框架历史。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "继续在这 30 个任务上比较版本，另比较不同模型和随机运行。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同题定制与重跑，不是从训练题学习通用运行框架后在另一批题盲测。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "比较相邻两个版本的任务产物来调整提示、角色间信息传递和上下文配置；修改范围是这些用户可配置的说明与设置。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.15524v1",
"version": "2607.15524v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用相邻 revision 的低成本 pairwise preference 优化信息流；不搜索 executable harness code，不等于修改 improver 自身。",
"feedbackCases": [
{
"label": "相邻仓库版本的模型比较",
"data": "30 个由行业职位描述构造的 ML 研究任务：量化金融、机器人、制药各十个。",
"scoring": "评审模型按功能、任务对齐、复现性等评估提示比较相邻版本产出的代码仓库，给优劣偏好；它不是运行统一验收测试后的正确率。",
"visible": "修改者读取累计的版本比较反馈，但不直接看到完整评估提示；该提示的偏好会通过反馈间接影响修改。",
"use": "同任务反复改文字角色和交接说明；最后仍在这 30 个任务上比较，不能称为跨题独立测试。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"judgment": "模型按功能、任务对齐和可复现性等评审要求比较两版代码仓库"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "30 个合成 ML 研究任务：量化金融、机器人、制药各 10 个，由行业招聘描述转成研究任务。没有模型参数训练。",
"selection": "同一任务上比较相邻版本生成的仓库；评审反馈写入该任务的运行框架历史。",
"evaluation": "继续在这 30 个任务上比较版本，另比较不同模型和随机运行。",
"isolation": "同题定制与重跑，不是从训练题学习通用运行框架后在另一批题盲测。",
"roles": {
"executor": {
"value": "sonnet-4.6、opus-4.7、opus-4.8 的 research/coding task agent，在当前 loop specification 下生成仓库。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
]
},
"modifier": {
"value": "语言模型 运行框架优化器读取历次仓库比较反馈并改写工作流提示。§5.1 列明了执行模型，附录 C 给出优化提示，但这两处没有明确指定优化器的模型型号；不能把评价用 GPT-5.5 当成优化器型号。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.15524#A3"
}
]
},
"seed": {
"value": "研究任务专用的多 task agent 提示设计，初始定义角色、交接、产物与检查规则，再迭代这些说明。展示的工具/数据清单属于方案规定，不能自动当作已经执行并验证的系统功能。",
"sources": [
{
"label": "任务设定、初始/迭代 harness 示例与评估",
"url": "https://arxiv.org/abs/2607.15524"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.15524#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.15524#S5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "框架组织出的执行轨迹质量，会限制当前任务表现，也会限制未来可用的模型训练材料。但若每次适配都要昂贵的大规模框架搜索，就难在有限单任务预算下推广。作者因此研究固定基础模型时，少量轻量迭代能否改善框架并提高增加推理计算后的性能上限。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.15524#S1"
}
]
},
{
"key": "position",
"value": "研究用户可配置的轻量运行框架能否以少量迭代提升科研任务的执行质量，并降低对高推理强度和人工持续维护的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.15524"
}
]
},
{
"key": "conclusion",
"value": "在合成研究任务中，改进后的较低推理强度配置可超过更高强度对照；收益主要来自信息传递与上下文管理。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.15524"
}
]
}
],
"fields": {
"modifier": "语言模型读取历次仓库比较反馈并改写工作流提示。原文未明确指定这个修改者的模型型号；评价用的 GPT-5.5 不能直接当作修改者型号。",
"object": "描述 agent 角色、通信方式、工作流和上下文管理的提示文本；不修改可执行代码或模型参数。",
"verdict": "模型按功能、任务要求和可复现性等标准比较前后两个产物，判断哪版更好；这是成对质量评审，并非所有任务都由代码测试给出通过率。"
}
},
"attributions": [
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.15524"
}
]
},
{
"tag": "org:sakana",
"label": "Sakana AI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.15524"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2607.25886",
"title": "RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement",
"url": "https://arxiv.org/abs/2607.25886",
"date": "2026-07-28",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"BenchmarkScore",
"Data",
"Improver",
"SameSet",
"org:nus"
],
"fields": {
"本质定位": "固定 post-training/serving/eval stack，只让 frontier researcher 根据 checkpoint feedback 改 training-data strategy。",
"被测系统 / feedback": "4 frontier researcher systems；fixed Qwen target candidates。",
"证据边界与关键结论": "**58.33%** setting 能发现 later-better candidate，但 peak 后继续搜的 runs 中 **78.26%** 以更差 final 结束；selection/eval 同 task subset。定义了 discovery–reliability gap。",
"什么在变": "data strategy + candidate LoRA weights。",
"谁来改 / 谁执行": "**改**：Claude Code/Codex researcher agents。
**执行**：固定 Qwen3.5-35B-A3B target candidates。",
"基础 harness": "fixed Tinker/post-train/eval stack。",
"Feedback": "同一 eval tasks 的 score/trajectory/verifier outcome。",
"Evolution → Eval": "selection 与 final official eval 使用同一 task subset。",
"Meta-depth": "non-harness M1。",
"相对之前真正新增什么": "最重要发现不是“成功 RSI”，而是 discovery–reliability gap：能偶尔找到更好 strategy，却常在 peak 后 regression。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "RSI evaluation 的代表作之一：固定 post-training / serving / evaluation stack，只开放 **training-data strategy**，看 frontier researcher 能不能利用 checkpoint feedback 持续找到更好的 candidate。它最重要的不是某个方法，而是把 **discovery 与 reliability** 拆开。"
},
{
"label": "什么在变",
"text": "researcher 的 data strategy；每轮从同一 base 训练新的 target LoRA candidate。deployment harness 不进化。"
},
{
"label": "谁来改 / 谁执行",
"text": "Claude Code / Codex researcher systems 改 data strategy；固定 Qwen3.5-35B-A3B-Base candidate checkpoint执行评测。"
},
{
"label": "基础 harness",
"text": "fixed Tinker post-training backend + fixed serving/eval stack，尽量隔离 data-centric research 变量。"
},
{
"label": "Feedback",
"text": "evaluation score、trajectory、verifier outcome、execution diagnostics；protected eval labels/trajectories不能直接作为 supervision，但同一任务产生的 feedback会进入下一轮决策。"
},
{
"label": "Evolution → Eval",
"text": "最大问题：**selection 和 final official eval 使用同一 task subset**（含 SWE/TB2/GPQA/AIME 等）；因此更像 adaptive optimization set，不是 held-out generalization。"
},
{
"label": "主要结果",
"text": "14/24 settings 后续 candidate 超过 first valid；但达到 historical best 后继续搜的 23 settings 中 **18/23 最终低于 peak**。这就是 discovery–reliability gap。"
},
{
"label": "为什么是经典 / 对我们意义",
"text": "它把我们最关心的“**能不能发现提升 ≠ 能不能稳定持续提升**”直接变成 benchmark observation；rollback、best preservation、stopping、真正 held-out eval 都因此变成 RSI 的一等问题。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 52,
"fields": {
"时间": "2026-07",
"论文": "**[RSIBench-Data](https://arxiv.org/abs/2607.25886)**",
"为什么是主干": "RSI benchmark/研究过程的经典代表；提出最重要的 **discovery–reliability gap** 证据之一。",
"证据边界要记住": "selection 与 final official eval 使用同一 task subset。"
}
},
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 228,
"fields": {
"时间": "2026-07-28",
"论文": "[RSIBench-Data](https://arxiv.org/abs/2607.25886)",
"级别": "**C**",
"它真正测什么": "**B-RSI / M-Data**。固定 post-training stack，只测 researcher 能否把 checkpoint feedback 转成更好的 training-data strategy。",
"被测系统 / feedback": "4 frontier researcher systems；fixed Qwen target candidates。",
"证据边界与关键结论": "**58.33%** setting 能发现 later-better candidate，但 peak 后继续搜的 runs 中 **78.26%** 以更差 final 结束；selection/eval 同 task subset。定义了 discovery–reliability gap。",
"标签": "`#Data #Improver #BenchmarkScore #SameSet #B-Reliability`"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 363,
"fields": {
"优先级": "**C**",
"时间": "2026-07-28",
"论文": "[RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement](https://arxiv.org/abs/2607.25886)",
"本质定位": "固定 post-training/serving/eval stack，只让 frontier researcher 根据 checkpoint feedback 改 training-data strategy。",
"什么在变": "data strategy + candidate LoRA weights。",
"谁来改 / 谁执行": "**改**：Claude Code/Codex researcher agents。
**执行**：固定 Qwen3.5-35B-A3B target candidates。",
"基础 harness": "fixed Tinker/post-train/eval stack。",
"Feedback": "同一 eval tasks 的 score/trajectory/verifier outcome。",
"Evolution → Eval": "selection 与 final official eval 使用同一 task subset。",
"Meta-depth": "non-harness M1。",
"相对之前真正新增什么": "最重要发现不是“成功 RSI”，而是 discovery–reliability gap：能偶尔找到更好 strategy，却常在 peak 后 regression。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Same-set adaptive",
"protocolBasis": "依据原记录：主实验在同一任务/子集上反复优化或评估；不能直接当作冻结后的泛化。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"B-RSI",
"M-Data"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "固定训练、serving 与评估栈，只让 researcher 根据模型失败反馈提出或修订训练数据策略，再生成候选 LoRA checkpoint。",
"novelty": "平台接手训练和数据处理的工程执行，让被测模型主要决定研究什么数据；同时记录找到的最好方案与最终保留方案的差距。",
"object": "训练数据的选择／构造策略，以及用这些数据训练出的 LoRA 参数（少量附加可训练参数）。训练、推理和评分基础设施固定。",
"executor": "Qwen3.5-35B-A3B-Base 派生的目标 checkpoints；通过固定 task agent 运行框架执行。",
"modifier": "主矩阵：Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus-4.8/Sonnet-5（high），Codex + GPT-5.6-Sol/Terra（max）。生成训练轨迹的外部模型固定Claude Opus4.8，Tinker执行训练。另有Claude Code + Kimi-K2.6指导Kimi-K2.6的案例。",
"roleContext": "**改**：Claude Code/Codex researcher agents。
**执行**：固定 Qwen3.5-35B-A3B target candidates。",
"seed": "平台通过 Tinker 训练服务执行参数更新并提供训练后的模型调用。软件修复任务使用 mini-swe-agent（主要用命令行读写文件和运行测试），其他任务使用 Terminus-2（组织模型操作终端环境）。研究 agent 主要决定数据策略和合成数据内容，最终评分所用框架固定。",
"fixed": "",
"verdict": "训练服务实际生成候选模型，评估端返回其成绩及诊断记录；研究 agent 用这些反馈调整数据策略和选择模型版本。",
"diagnosis": "researcher 从错误提出能力缺口假设，再判断该合成什么训练经验。",
"update": "修改 data strategy / 数据集，训练 candidate LoRA；不是重写 deployment harness。",
"acceptance": "按 selection score 选择 checkpoint；另运行 official evaluation。",
"experiments": [
{
"name": "六个 benchmark",
"evolve": "允许的公共 seed 与合成数据用于训练",
"selection": "同一指定 task subset 上的 selection feedback",
"test": "fresh environment 下的 official eval，但 task subset 相同",
"isolation": "同集适应",
"note": "环境重置不等于统计上未见测试题。"
}
],
"takeaway": "**58.33%** setting 能发现 later-better candidate，但 peak 后继续搜的 runs 中 **78.26%** 以更差 final 结束；selection/eval 同 task subset。定义了 discovery–reliability gap。",
"result": "14/24 settings 后续 candidate 超过 first valid；但达到 historical best 后继续搜的 23 settings 中 **18/23 最终低于 peak**。这就是 discovery–reliability gap。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2607.25886v1"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：研究者围绕六个目标基准，以公开 seed 仓库/示例生成训练记录，交给固定服务训练 Qwen3.5-35B-A3B-Base 的 LoRA（只训练少量适配参数）；不是统一预制训练集。\n\n调试 / 选版本数据：SWE-bench Verified/Multilingual/Pro 各 100 题、GPQA（研究生级科学问答基准） Diamond 100 题、TB2 89 题、AIME 2026 30 题提供候选反馈。\n\n最终测试数据：选定 checkpoint（某个时刻保存的模型或系统版本） 后，在新环境用同一目标任务子集重新评估；AIME 每题 4 次解码。\n\n数据隔离与证据边界：新建沙箱隔离执行状态，但没有额外不参与候选选择的测试题。",
"cycle": "研究 agent 从错误提出能力缺口假设，决定合成或筛选什么数据。训练服务用新数据训练候选适配参数，再返回成绩；研究 agent 选择保存的模型版本。最终部署框架固定，不允许通过更换它来提高分数。",
"train": "研究者围绕六个目标基准，以公开 seed 仓库/示例生成训练记录，交给固定服务训练 Qwen3.5-35B-A3B-Base 的 LoRA（只训练少量适配参数）；不是统一预制训练集。",
"debug": "SWE-bench Verified/Multilingual/Pro 各 100 题、GPQA（研究生级科学问答基准） Diamond 100 题、TB2 89 题、AIME 2026 30 题提供候选反馈。",
"test": "选定 checkpoint（某个时刻保存的模型或系统版本） 后，在新环境用同一目标任务子集重新评估；AIME 每题 4 次解码。",
"isolation": "新建沙箱隔离执行状态，但没有额外不参与候选选择的测试题。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "固定 post-training / serving / evaluation stack，单独评估 frontier agent 能否通过 checkpoint feedback 反复研究和改进训练数据。",
"index": {
"date": "2026-07-28",
"paper": "[RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement](https://arxiv.org/abs/2607.25886)",
"executor": "固定 target **Qwen3.5-35B-A3B-Base** 的 candidate checkpoint",
"modifier": "Claude Code / Codex + frontier LLM researcher 反复修改 **training-data strategy**，共享 Tinker backend 从同一 base 训练新 LoRA candidate",
"feedback": "evaluation/selection **score、trajectory、verifier outcome、execution diagnostics**；由 evaluation GT/verifier 产生，但 protected label/trajectory 不能直接作为 supervision",
"evolve": "**Evolution/selection tasks**：SWE-bench Verified 100、SWE-bench Multilingual 100、SWE-bench Pro 100、Terminal-Bench 2.0 **89/89**、GPQA Diamond 100、AIME 2026 **30/30**；researcher 另可利用对应 public seed data / synthetic data 生成训练集",
"eval": "**Final official eval 仍使用完全相同的 task subset**；只是在 fresh environment 重新执行，**没有独立 held-out tasks**",
"tldr": "固定 post-training stack，只研究如何根据 checkpoint feedback 改训练数据。能偶尔发现更优策略但 peak 后常 regression，而且 selection 与 final eval 使用同一 task subset，因此更能说明面向固定 evaluation tasks 的 adaptive data optimization，而非已证明可泛化 RSI。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 现有 automated post-training benchmark 往往把数据设计、优化、serving、evaluation、system implementation 混在一起，最终模型变好时难判断提升是否来自 agent 的 **data-centric research** 能力。 |\n| 本文定位 | 提出 **RSIBench-Data**：固定 target model 和 surrounding post-training stack，只让 researcher agent 决定训练 experience/data strategy 与白名单内的少量 config，研究其能否“诊断 capability gap → 构造数据 → 训练 checkpoint → 根据反馈继续改”。 |\n| 核心贡献 | ① 受控 data-centric research benchmark；② 统一 Tinker SFT、Harbor/E2B evaluator 和 budget；③ 记录每轮 hypothesis/data/checkpoint/feedback；④ 用 4 researcher × 6 benchmark 分析 feedback-driven improvement 是否可靠。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | Researcher 的 **training-data strategy** 在迭代；每轮还训练新的 target LoRA weights。Harness 本身不进化。 |\n| 谁来改 | 主实验 4 个 researcher system：**Claude Code + Opus-4.8 / Sonnet-5；Codex + gpt-5.6-sol / gpt-5.6-terra**。 |\n| 被改对象 | 固定 **Qwen3.5-35B-A3B-Base**；每个 attempt 经共享 Tinker backend 做 LoRA SFT。 |\n| 外部 rollout model | 所有实验固定 **Claude Opus 4.8** 生成 reasoning trace、tool-use sequence 或 full trajectory。 |\n| 迭代关系 | 每个 candidate 都是从同一固定 base model 训练一个 LoRA checkpoint；不是持续的 $M_t\\rightarrow M_{t+1}$ 权重自修改。真正跨轮持续变化的是 data-research policy / strategy。 |\n| Researcher 能看到什么 | permitted selection score、task trajectories、verifier outcomes、execution/infrastructure errors、token usage、training/sampling cost、elapsed time 等。 |\n| Feedback | **evaluation GT / verifier-grounded feedback**：correctness/score/verifier outcome 会进入下一轮研究决策。 |\n| 数据使用限制 | evaluation-only task、label、trajectory 等 protected materials **不能直接作为 training supervision**；但由这些 evaluation tasks 的 gold/verifier 产生的 feedback 可以用于改下一轮 data strategy。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark | Eval subset / runner | Feedback | Evolution / Train → Test | GT / label 使用 |\n|---|---|---|---|---|\n| SWE-bench Verified | 固定 100 tasks；Mini-SWE-Agent | official-style verifier + trajectory | selection 与 final official eval 使用**同一 task subset** | protected task/label/trajectory 不能直接训练；verifier feedback 进入 loop |\n| SWE-bench Multilingual | 固定 100 tasks；Mini-SWE-Agent | 同上 | 同一 subset | 同上 |\n| SWE-bench Pro | 固定 100 tasks；Mini-SWE-Agent | 同上 | 同一 subset | 同上 |\n| Terminal-Bench 2.0 | **89/89 全量**；Terminus-2 | task verifier/pass + trajectory | 直接对完整 benchmark 反复 selection；final 仍同一 tasks | verifier-grounded feedback |\n| GPQA Diamond | 固定 100 tasks；Terminus-2 | correctness/score + trajectory | selection 与 final official eval 同一 subset | gold-grounded correctness feedback，但不能把 eval label 直接做 supervision |\n| AIME 2026 | **30/30 全量**，每题 4 rollout；Terminus-2 | correctness/score | 同一 full set | gold-grounded correctness feedback |\n\n> **关键口径**：这篇不是“把 test label 直接教给模型”，但确实是**面向最终 evaluation/test tasks 做 iterative optimization**：researcher 反复获得同一批任务上由 gold/verifier 产生的 score、correctness、trajectory 等 feedback，再修改训练数据；最终 official evaluation 仍是同一 task subset，只是在 fresh environment 中重新执行。"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| Iteration 经常能找到比 first attempt 更好的 candidate | **14/24 = 58.33%** setting 中 later candidate > first valid candidate | 当前 researcher 已具备一定 feedback→data-strategy discovery 能力 |\n| Improvement 不可靠、非单调 | 达到 historical best 后继续搜索的 23 settings 中，**18/23 = 78.26%** 最终 attempt 低于 peak，其余 5 个只回到 peak | 核心 **discovery–reliability gap**：能偶尔发现更好的策略，但不能稳定把更多 feedback 转化为持续 improvement |\n| Historical-best preservation 很重要 | 多个 run 在 peak 后继续花预算但 regression | rollback / checkpoint preservation / stopping 是 self-improvement system 的关键能力 |\n| 各 researcher 没有稳定排名 | 三个 SWE-style task 分别由不同 researcher 赢；非 SWE 多由 gpt-5.6-sol 领先 | improvement 强烈依赖 researcher × benchmark interaction |\n| Main target 上部分任务大幅提升 | 例如 SWE Verified base 12% → 46%；AIME 30% → 53.33%；TerminalBench 1.12% → 20.22% | data-centric research 可以产生很大 task-specific gain，但并非所有任务都改善 |\n| Same-family RSI 未突破 base | Kimi-K2.6 researcher → Kimi-K2.6 target：有效 data strategy 从 8% 改到约 21%；near-no-op adapter 22%，但 unadapted reference **33%** | 更接近 self-RSI 的 setting 中，当前 loop 仍没有找到真正超过原始模型的 training distribution |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| Story 与实验匹配 | 如果定位为“评估 data-centric researcher ability”，**匹配**；若把主实验解释为严格 RSI，则偏强，因为主 setting 是强 researcher 改固定弱 target。 |\n| 最重要的数据隔离问题 | 作者明确承认：**checkpoint selection 和 official evaluation 使用同一 task subset**。因此 official score 只证明 fresh execution performance，不证明对 statistically held-out tasks 的 adaptive generalization。 |\n| Test-label leakage 怎么看 | 没有 direct label-as-supervision leakage；但 **test/eval gold 或 verifier 产生的 feedback 明确进入 optimization loop**。所以从评估角度，这批 task 已经承担 dev/optimization set 的角色。 |\n| Self-evolution 口径 | 主实验不是严格 self-RSI；每轮 target checkpoint 也不是 $M_t\\rightarrow M_{t+1}$ 连续更新，而是围绕同一 base 重做 candidate training。 |\n| 归因 | 固定 training/serving/eval stack 是优点；但 researcher identity 同时捆绑 base LLM、Claude Code/Codex scaffold、reasoning effort，agent 间差异不能归因到单一因素。 |\n| 统计稳定性 | 每个 setting 主要只有 one representative run，成本高、缺多 seed；58.33%/78.26% 是当前 24 个主 setting 的经验统计，不是稳定性充分估计。 |\n| 对我们的启发 | **不能只看是否搜到过更高分 candidate**；要评估 feedback→improvement 是否稳定、peak 后是否 regression、何时 stop/rollback，以及 freeze 后能否在真正 held-out tasks 上保留提升。 |"
},
{
"title": "总评",
"body": "**RSIBench-Data 的价值主要是把“能否发现 improvement”和“能否可靠持续 improvement”区分开，并直接观察到 discovery–reliability gap。其受控 post-training stack 有利于归因，但 selection 与 final evaluation 没有 task-level held-out，因此结果仍更接近针对固定 evaluation tasks 的 adaptive data optimization，而不是已经证明可泛化的 recursive self-improvement。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "固定 Tinker 后训练与模型服务栈；SWE 任务用 mini-swe-agent，其他任务用 Terminus-2。研究 agent 的主要自由度是合成数据、修改数据策略，不能同时改最终评估 harness 来涨分。",
"protocol": "**合成/选择反馈：**SWE-bench Verified、Multilingual、Pro 各 100 题，GPQA Diamond 100 题，TB2 全 89 题、AIME 2026 全 30 题；研究者用对应公开 seed data 和自合成数据训练候选。AIME 每题四次解码。\n\n**正式结果：**选好 checkpoint 后在新环境重跑的仍是同一任务子集，没有额外独立 held-out tasks。环境重新创建不等于换了测试数据；具体合成训练集随 run 改变，见详细笔记。",
"sections": "§3 固定设施；§4.3 与附录 A",
"source": "https://arxiv.org/abs/2607.25886",
"version": "2607.25886v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "695b148cd2207dbb644144b5a4cd2e09b0438850eab21021f1dc90e4e1c10648",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3.5-35B-A3B-Base 派生的目标 checkpoints；通过固定 task agent 运行框架执行。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "主矩阵：Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus-4.8/Sonnet-5（high），Codex + GPT-5.6-Sol/Terra（max）。生成训练轨迹的外部模型固定Claude Opus4.8，Tinker执行训练。另有Claude Code + Kimi-K2.6指导Kimi-K2.6的案例。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.25886#S4.SS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2607.25886#S5.SS5.SSS0.Px1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "训练数据的选择／构造策略，以及用这些数据训练出的 LoRA 参数（少量附加可训练参数）。训练、推理和评分基础设施固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "训练服务实际生成候选模型，评估端返回其成绩及诊断记录；研究 agent 用这些反馈调整数据策略和选择模型版本。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "平台通过 Tinker 训练服务执行参数更新并提供训练后的模型调用。软件修复任务使用 mini-swe-agent（主要用命令行读写文件和运行测试），其他任务使用 Terminus-2（组织模型操作终端环境）。研究 agent 主要决定数据策略和合成数据内容，最终评分所用框架固定。",
"sources": [
{
"label": "§3 固定设施；§4.3 与附录 A",
"url": "https://arxiv.org/abs/2607.25886"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "研究 agent 从错误提出能力缺口假设，决定合成或筛选什么数据。训练服务用新数据训练候选适配参数，再返回成绩；研究 agent 选择保存的模型版本。最终部署框架固定，不允许通过更换它来提高分数。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "研究者围绕六个目标基准，以公开 seed 仓库/示例生成训练记录，交给固定服务训练 Qwen3.5-35B-A3B-Base 的 LoRA（只训练少量适配参数）；不是统一预制训练集。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "SWE-bench Verified/Multilingual/Pro 各 100 题、GPQA（研究生级科学问答基准） Diamond 100 题、TB2 89 题、AIME 2026 30 题提供候选反馈。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "选定 checkpoint（某个时刻保存的模型或系统版本） 后，在新环境用同一目标任务子集重新评估；AIME 每题 4 次解码。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "新建沙箱隔离执行状态，但没有额外不参与候选选择的测试题。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "平台接手训练和数据处理的工程执行，让被测模型主要决定研究什么数据；同时记录找到的最好方案与最终保留方案的差距。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.25886v1",
"version": "2607.25886v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "隔离 data-centric research 决策能力，发现“能找到更好候选”和“能可靠地持续改进”之间的差距。名字有 Data，主要贡献仍是 benchmark。",
"feedbackCases": [
{
"label": "SWE-bench Verified / Multilingual / Pro",
"data": "每基准 100 道目标题",
"scoring": "固定评测服务在仓库环境执行补丁验收测试，计算任务解决率。",
"visible": "研究者提交训练数据和白名单配置后，固定 Tinker 服务训练检查点；评测返回允许的分数、轨迹、判分结果和执行诊断。",
"use": "反馈用于改合成数据/训练数据策略，不能修改评分器；最终在新环境重评同一目标子集，不是另外一套从未用于反馈的测试题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"judgment": "受保护的仓库测试程序检查补丁"
},
{
"label": "Terminal-Bench 2",
"data": "89 道目标题",
"scoring": "固定服务运行任务验收程序，判断终端产物是否通过。",
"visible": "研究者提交训练数据和白名单配置后，固定 Tinker 服务训练检查点；评测返回允许的分数、轨迹、判分结果和执行诊断。",
"use": "反馈用于改合成数据/训练数据策略，不能修改评分器；最终在新环境重评同一目标子集，不是另外一套从未用于反馈的测试题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"judgment": "受保护的终端任务测试程序检查产物"
},
{
"label": "GPQA Diamond",
"data": "100 道目标题",
"scoring": "按科学选择题参考答案判定回答正确性。",
"visible": "研究者提交训练数据和白名单配置后，固定 Tinker 服务训练检查点；评测返回允许的分数、轨迹、判分结果和执行诊断。",
"use": "反馈用于改合成数据/训练数据策略，不能修改评分器；最终在新环境重评同一目标子集，不是另外一套从未用于反馈的测试题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"judgment": "对照 GPQA 参考选项；本文未展开答案抽取实现"
},
{
"label": "AIME 2026",
"data": "30 道目标题，最终每题四次解码",
"scoring": "按数学题参考答案判定最终答案，汇总正确率。",
"visible": "研究者提交训练数据和白名单配置后，固定 Tinker 服务训练检查点；评测返回允许的分数、轨迹、判分结果和执行诊断。",
"use": "反馈用于改合成数据/训练数据策略，不能修改评分器；最终在新环境重评同一目标子集，不是另外一套从未用于反馈的测试题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"judgment": "对照 AIME 标准答案；本文未展开答案抽取和比对实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "研究者围绕六个目标基准，以公开 seed 仓库/示例生成训练记录，交给固定服务训练 Qwen3.5-35B-A3B-Base 的 LoRA（只训练少量适配参数）；不是统一预制训练集。",
"selection": "SWE-bench Verified/Multilingual/Pro 各 100 题、GPQA（研究生级科学问答基准） Diamond 100 题、TB2 89 题、AIME 2026 30 题提供候选反馈。",
"evaluation": "选定 checkpoint（某个时刻保存的模型或系统版本） 后，在新环境用同一目标任务子集重新评估；AIME 每题 4 次解码。",
"isolation": "新建沙箱隔离执行状态，但没有额外不参与候选选择的测试题。",
"roles": {
"executor": {
"value": "Qwen3.5-35B-A3B-Base 派生的目标 checkpoints；通过固定 task agent 运行框架执行。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
}
]
},
"modifier": {
"value": "主矩阵：Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus-4.8/Sonnet-5（high），Codex + GPT-5.6-Sol/Terra（max）。生成训练轨迹的外部模型固定Claude Opus4.8，Tinker执行训练。另有Claude Code + Kimi-K2.6指导Kimi-K2.6的案例。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.25886#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.25886#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.25886#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.25886#S3.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.25886#S4.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.25886#S4.SS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2607.25886#S5.SS5.SSS0.Px1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "平台通过 Tinker 训练服务执行参数更新并提供训练后的模型调用。软件修复任务使用 mini-swe-agent（主要用命令行读写文件和运行测试），其他任务使用 Terminus-2（组织模型操作终端环境）。研究 agent 主要决定数据策略和合成数据内容，最终评分所用框架固定。",
"sources": [
{
"label": "§3 固定设施；§4.3 与附录 A",
"url": "https://arxiv.org/abs/2607.25886"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"selection": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"evaluation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
],
"isolation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.25886#S3.SS5"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.25886#S4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "失败轨迹只说明模型哪里不会，并不会自动变成有效训练数据；已有自动后训练评测又把数据研究、训练优化、模型服务和系统实现混在一起。这样难判断 agent 是否真正会提出并验证数据改进假设，因此需要固定周围工程条件，单独评估数据研究过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.25886#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 在训练基础设施已提供的条件下，能否诊断模型能力缺口、提出有效的数据研究方案，并稳定保留后训练收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.25886"
}
]
},
{
"key": "conclusion",
"value": "agent 能发现更好数据方案，却常在达到峰值后丢掉好版本；发现改进与稳定保留改进仍有差距。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.25886"
}
]
}
],
"fields": {
"object": "训练数据的选择／构造策略，以及用这些数据训练出的 LoRA 参数（少量附加可训练参数）。训练、推理和评分基础设施固定。",
"verdict": "训练服务实际生成候选模型，评估端返回其成绩及诊断记录；研究 agent 用这些反馈调整数据策略和选择模型版本。",
"seed": "平台通过 Tinker 训练服务执行参数更新并提供训练后的模型调用。软件修复任务使用 mini-swe-agent（主要用命令行读写文件和运行测试），其他任务使用 Terminus-2（组织模型操作终端环境）。研究 agent 主要决定数据策略和合成数据内容，最终评分所用框架固定。"
}
},
"attributions": [
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.25886"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"curriculum",
"evaluation"
]
},
{
"id": "2607.28568",
"title": "Frontis-MA1: Training an AI4AI Model towards Recursive Self-Improvement in Machine Learning Engineering",
"url": "https://arxiv.org/abs/2607.28568",
"date": "2026-07-30",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"CrossBenchmark",
"EditorWeights",
"Improver",
"LearnedUpdater",
"Weights"
],
"fields": {
"本质定位": "OpenMLE 全栈 + execution-grounded SFT/RL 训练 35B meta-evolution model，学习 Draft/Improve/Debug/Crossover operators，再用于长程 MLE evolutionary search。",
"什么在变": "meta-evolution model weights + MLE program artifacts；outer evolutionary framework仍固定。",
"谁来改 / 谁执行": "**改**：Frontis-MA1 learned operators + OpenMLE-Evo。
**执行**：Frontis-MA1-35B 在 MLE research tasks 上搜索/改程序。",
"基础 harness": "OpenMLE-Gym/RL/Evo full-stack research harness。",
"Feedback": "executable MLE score / benchmark outcome。",
"Evolution → Eval": "MLE-Bench Lite + NatureBench Lite transfer；model-swap/framework-swap analysis。",
"Meta-depth": "improver-level M2-ish。",
"相对之前真正新增什么": "代表性在于把“improver”本身训练成专门模型能力，并且模型权重、训练框架、搜索环境都开放；是 AI4AI→RSI 线的重要节点。"
},
"details": [
{
"label": "本质定位",
"text": "**learned improver / AI4AI**：不是 prompt 一个通用 frontier model，而是专门训练 35B meta-evolution model 学 Draft / Improve / Debug / Crossover。"
},
{
"label": "什么在变",
"text": "Frontis-MA1 的 meta-evolution能力写进 weights；运行时再持续修改 MLE programs。"
},
{
"label": "基础 harness",
"text": "OpenMLE-Gym / RL / Evo 全栈；execution-grounded program-evolution environment。"
},
{
"label": "Feedback",
"text": "executable MLE score / benchmark outcome。"
},
{
"label": "核心结果",
"text": "MLE-Bench Lite：base **39.39→60.61**，Evo-Max **71.21**；NatureBench 做 model-swap / framework-swap transfer。"
},
{
"label": "为什么经典/热度高",
"text": "这篇不是“小众 harness trick”，而是 2026 AI4AI/RSI 主线里非常显眼的系统性工作；OpenMLE 与模型权重均开放，社区关注也高。"
},
{
"label": "边界",
"text": "learned improver 已变强，但 outer evolutionary search framework 没有再被下一代 improver递归重写。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 53,
"fields": {
"时间": "2026-07",
"论文": "**[Frontis-MA1](https://arxiv.org/abs/2607.28568)**",
"为什么是主干": "“训练一个专门的 meta-evolution / AI4AI improver” 的代表，且 OpenMLE 全栈开源。",
"证据边界要记住": "learned improver 变强，但 outer evolution system 仍固定。"
}
},
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 354,
"fields": {
"优先级": "**C**",
"时间": "2026-07-30",
"论文": "[Frontis-MA1: Training an AI4AI Model towards Recursive Self-Improvement in Machine Learning Engineering](https://arxiv.org/abs/2607.28568)",
"本质定位": "OpenMLE 全栈 + execution-grounded SFT/RL 训练 35B meta-evolution model，学习 Draft/Improve/Debug/Crossover operators，再用于长程 MLE evolutionary search。",
"什么在变": "meta-evolution model weights + MLE program artifacts；outer evolutionary framework仍固定。",
"谁来改 / 谁执行": "**改**：Frontis-MA1 learned operators + OpenMLE-Evo。
**执行**：Frontis-MA1-35B 在 MLE research tasks 上搜索/改程序。",
"基础 harness": "OpenMLE-Gym/RL/Evo full-stack research harness。",
"Feedback": "executable MLE score / benchmark outcome。",
"Evolution → Eval": "MLE-Bench Lite + NatureBench Lite transfer；model-swap/framework-swap analysis。",
"Meta-depth": "improver-level M2-ish。",
"相对之前真正新增什么": "代表性在于把“improver”本身训练成专门模型能力，并且模型权重、训练框架、搜索环境都开放；是 AI4AI→RSI 线的重要节点。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"M-Weight",
"Meta"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "OpenMLE 全栈 + execution-grounded SFT/RL 训练 35B meta-evolution model，学习 Draft/Improve/Debug/Crossover operators，再用于长程 MLE evolutionary search。",
"novelty": "专门训练执行机器学习工程研究的模型，并提供与训练操作相配套的搜索环境，分别检查模型能力和搜索所得经验的迁移。",
"object": "负责程序进化的模型参数，以及它生成的机器学习程序；外层搜索框架固定。",
"executor": "Frontis-MA1-35B在OpenMLE-Evo中执行ML研究操作；35B训练基座为Qwen3.6-35B-A3B，另有从Qwen3-30B-A3B-Thinking-2507开始的配置。",
"modifier": "部署时由训练后的Frontis-MA1执行草拟／调试／改进／交叉组合。监督微调（用示范数据训练模型）教师包括GLM-4.7与Qwen3-30B-A3B-Thinking-2507，随后固定强化学习（根据奖励调整模型行为）流程训练这些操作。",
"roleContext": "**改**：Frontis-MA1 learned operators + OpenMLE-Evo。
**执行**：Frontis-MA1-35B 在 MLE research tasks 上搜索/改程序。",
"seed": "作者的 OpenMLE-Gym 是标准任务环境，OpenMLE-RL 负责训练研究模型，OpenMLE-Evo 负责提出、组合和筛选候选改进。每题已有公开数据、示例提交和 metric.py 评分程序；隐藏答案留在私有区域，研究者有可运行的起点。",
"fixed": "",
"verdict": "运行机器学习实验后得到的任务评分或评测结果。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "MLE-Bench Lite + NatureBench Lite transfer；model-swap/framework-swap analysis。"
}
],
"takeaway": "learned improver 已变强，但 outer evolutionary search framework 没有再被下一代 improver递归重写。",
"result": "MLE-Bench Lite：base **39.39→60.61**，Evo-Max **71.21**；NatureBench 做 model-swap / framework-swap transfer。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：OpenMLE-Gym 5,758 个可执行任务：156 个精选任务、3,362 个 Kaggle Dataset 任务、2,240 个 Kaggle Competition 任务；其中执行轨迹用于 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为）。\n\n调试 / 选版本数据：任务内部可见训练数据及验证成绩供代码搜索；程序执行异常也返回操作模型。\n\n最终测试数据：MLE-Bench Lite 官方 22 题；另用固定 10 题 NatureBench Lite 检查跨科研领域迁移。\n\n数据隔离与证据边界：框架训练任务与评估评测基准去重；竞赛内部的验证/私有测试是另一层划分。NatureBench 子集仅 10 题，结论范围有限。",
"cycle": "模型学习四类研究操作：草拟程序、修复错误、改进方案、组合不同方案。先用示范训练，再根据执行效果强化这些操作；部署时由 OpenMLE-Evo 搜索系统调度，保存程序、成绩及分支经验，继续选择父代。",
"train": "OpenMLE-Gym 5,758 个可执行任务：156 个精选任务、3,362 个 Kaggle Dataset 任务、2,240 个 Kaggle Competition 任务；其中执行轨迹用于 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为）。",
"debug": "任务内部可见训练数据及验证成绩供代码搜索；程序执行异常也返回操作模型。",
"test": "MLE-Bench Lite 官方 22 题；另用固定 10 题 NatureBench Lite 检查跨科研领域迁移。",
"isolation": "框架训练任务与评估评测基准去重；竞赛内部的验证/私有测试是另一层划分。NatureBench 子集仅 10 题，结论范围有限。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "OpenMLE-Gym 提供标准任务包，OpenMLE-RL 训练研究操作，OpenMLE-Evo 组合改进/交叉等搜索。每题已含 public 数据、示例提交和 metric.py；隐藏答案在 private 区，起点不是无评估支持的空工作区。",
"protocol": "**训练来源：**将竞赛和研究任务转为可执行包，经构建检查、语义过滤并对全部评估 benchmark 去重；执行轨迹用于 SFT/RL。训练包准确总量本轮未核实。\n\n**测试：**MLE-Bench Lite 22 题，每题单 RTX 4090、12GB VRAM 上限、12 小时；另在完全留出的 NatureBench Lite 10 题做模型替换/框架替换对照。任务内 validation 供搜索，最终竞赛成绩与框架训练去重是不同层级的隔离。",
"sections": "任务包构造、MLE-Bench Lite 设置和 NatureBench 迁移",
"source": "https://arxiv.org/abs/2607.28568",
"version": "2607.28568v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "68885074c4ebd4a321640e22a125cb5c7eeb87ca828afd5e4dd499d1a74ee3e1",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Frontis-MA1-35B在OpenMLE-Evo中执行ML研究操作；35B训练基座为Qwen3.6-35B-A3B，另有从Qwen3-30B-A3B-Thinking-2507开始的配置。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.28568#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.28568#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "部署时由训练后的Frontis-MA1执行草拟／调试／改进／交叉组合。监督微调（用示范数据训练模型）教师包括GLM-4.7与Qwen3-30B-A3B-Thinking-2507，随后固定强化学习（根据奖励调整模型行为）流程训练这些操作。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.28568#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.28568#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "负责程序进化的模型参数，以及它生成的机器学习程序；外层搜索框架固定。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行机器学习实验后得到的任务评分或评测结果。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者的 OpenMLE-Gym 是标准任务环境，OpenMLE-RL 负责训练研究模型，OpenMLE-Evo 负责提出、组合和筛选候选改进。每题已有公开数据、示例提交和 metric.py 评分程序；隐藏答案留在私有区域，研究者有可运行的起点。",
"sources": [
{
"label": "任务包构造、MLE-Bench Lite 设置和 NatureBench 迁移",
"url": "https://arxiv.org/abs/2607.28568"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "模型学习四类研究操作：草拟程序、修复错误、改进方案、组合不同方案。先用示范训练，再根据执行效果强化这些操作；部署时由 OpenMLE-Evo 搜索系统调度，保存程序、成绩及分支经验，继续选择父代。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "OpenMLE-Gym 5,758 个可执行任务：156 个精选任务、3,362 个 Kaggle Dataset 任务、2,240 个 Kaggle Competition 任务；其中执行轨迹用于 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为）。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "任务内部可见训练数据及验证成绩供代码搜索；程序执行异常也返回操作模型。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "MLE-Bench Lite 官方 22 题；另用固定 10 题 NatureBench Lite 检查跨科研领域迁移。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "框架训练任务与评估评测基准去重；竞赛内部的验证/私有测试是另一层划分。NatureBench 子集仅 10 题，结论范围有限。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "专门训练执行机器学习工程研究的模型，并提供与训练操作相配套的搜索环境，分别检查模型能力和搜索所得经验的迁移。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.28568v1",
"version": "2607.28568v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "代表性在于把“improver”本身训练成专门模型能力，并且模型权重、训练框架、搜索环境都开放；是 AI4AI→RSI 线的重要节点。",
"feedbackCases": [
{
"label": "训练修改模型：OpenMLE-Gym",
"data": "5,758 个任务：156 个精选任务、3,362 个 Kaggle Dataset 任务、2,240 个 Kaggle Competition 任务。",
"scoring": "构造程序按有标签数据生成数据划分与 metric.py；实际运行 ML 程序，用预测结果按任务指标计分。代码异常也作为反馈；训练奖励按方向及历史得分范围归一化，不能把不同任务原始指标直接平均。",
"visible": "父代程序、验证分数、相对父代改进、执行耗时及错误记录等，供 Draft/Improve/Debug/Crossover 操作使用。",
"use": "筛选监督示范并强化训练改进模型；私有参考答案与公开输入在任务构造时分开。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"judgment": "实际运行训练程序；任务生成的 metric.py 按预测与标签计算指标"
},
{
"label": "最终 ML / 科研迁移",
"data": "MLE-Bench Lite 官方 22 题、NatureBench Lite 固定十题。",
"scoring": "在各任务中实际执行候选程序，按该任务的数据及规定指标评价；不同竞赛/科研任务没有一种统一“答对”规则。",
"visible": "开发中可用任务允许的验证成绩和错误；最终成绩按各基准的评测协议报告。",
"use": "检验训练出的程序改进模型跨任务表现，不是用这 32 题代替 5,758 个训练任务。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"judgment": "运行任务后按原任务指标计分；不同竞赛的指标分别定义"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "OpenMLE-Gym 5,758 个可执行任务：156 个精选任务、3,362 个 Kaggle Dataset 任务、2,240 个 Kaggle Competition 任务；其中执行轨迹用于 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为）。",
"selection": "任务内部可见训练数据及验证成绩供代码搜索；程序执行异常也返回操作模型。",
"evaluation": "MLE-Bench Lite 官方 22 题；另用固定 10 题 NatureBench Lite 检查跨科研领域迁移。",
"isolation": "框架训练任务与评估评测基准去重；竞赛内部的验证/私有测试是另一层划分。NatureBench 子集仅 10 题，结论范围有限。",
"roles": {
"executor": {
"value": "Frontis-MA1-35B在OpenMLE-Evo中执行ML研究操作；35B训练基座为Qwen3.6-35B-A3B，另有从Qwen3-30B-A3B-Thinking-2507开始的配置。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.28568#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.28568#A2.SS3"
}
]
},
"modifier": {
"value": "部署时由训练后的Frontis-MA1执行草拟／调试／改进／交叉组合。监督微调（用示范数据训练模型）教师包括GLM-4.7与Qwen3-30B-A3B-Thinking-2507，随后固定强化学习（根据奖励调整模型行为）流程训练这些操作。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2607.28568#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.28568#A2.SS3"
}
]
},
"seed": {
"value": "作者的 OpenMLE-Gym 是标准任务环境，OpenMLE-RL 负责训练研究模型，OpenMLE-Evo 负责提出、组合和筛选候选改进。每题已有公开数据、示例提交和 metric.py 评分程序；隐藏答案留在私有区域，研究者有可运行的起点。",
"sources": [
{
"label": "任务包构造、MLE-Bench Lite 设置和 NatureBench 迁移",
"url": "https://arxiv.org/abs/2607.28568"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"selection": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"evaluation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
],
"isolation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2607.28568#S6.SS1.SSS0.Px1"
},
{
"label": "§6.6",
"url": "https://arxiv.org/html/2607.28568#S6.SS6"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2607.28568#A4.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "机器学习工程需要在耗时、带噪的实验反馈下反复改进，单次写出方案并不足够。已有公开系统分别推进任务环境、执行反馈训练或搜索框架，却未同时打通可复现的完整循环；训练与实际搜索操作脱节，也使改进经验难直接培养下一轮提出方案的能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.28568#S1"
}
]
},
{
"key": "position",
"value": "面向机器学习工程，研究能否训练出擅长改进 AI 系统的专门模型，并使其工程改进能力与搜索经验迁移到新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.28568"
}
]
},
{
"key": "conclusion",
"value": "在机器学习工程基准上同时验证模型训练和进化搜索的收益，并分别测试二者的迁移能力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.28568"
}
]
}
],
"fields": {
"object": "负责程序进化的模型参数，以及它生成的机器学习程序；外层搜索框架固定。",
"seed": "作者的 OpenMLE-Gym 是标准任务环境，OpenMLE-RL 负责训练研究模型，OpenMLE-Evo 负责提出、组合和筛选候选改进。每题已有公开数据、示例提交和 metric.py 评分程序；隐藏答案留在私有区域，研究者有可运行的起点。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2608.02276",
"title": "Harness-R1: Learning to Edit Executable Runtime Harnesses from Agent Failure Trajectories",
"url": "https://arxiv.org/abs/2608.02276",
"date": "2026-08-03",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"EditorWeights",
"LearnedUpdater",
"SeparateEvolver",
"ExecutableVerifier",
"M1",
"org:sjtu"
],
"fields": {
"本质定位": "专门 post-train 一个 9B Harness Engineer，把“读 failure trajectory→写 executable lifecycle hooks→target fresh rerun”训练成独立能力；target frozen。",
"什么在变": "harness-editor model weights + executable runtime hooks；target weights不变。",
"谁来改 / 谁执行": "**改**：独立 Qwen3.5-9B Harness Engineer；cold-start teacher GPT-5.5，之后 online GRPO。
**执行**：frozen Qwen3.5-9B target。",
"基础 harness": "自建 runtime-hook harness API（episode-init / pre-decision / pre-action / post-feedback）。",
"Feedback": "target fresh rerun 的真实 task reward change。",
"Evolution → Eval": "WebShop / ALFWorld / DBBench；held-out/eval tasks；另测 target fine-tune 后 editor 是否仍增益。",
"Meta-depth": "learned-improver M1/M2-ish；editor 与 target 分离。",
"相对之前真正新增什么": "相对 Meta-Harness/Self-Harness 的真正新点是 **harness editing 本身被训练成模型能力**，而不是每次 prompting 一个 proposer。"
},
"details": [
{
"label": "本质定位",
"text": "把 **harness editing 当成可以 post-train 的独立模型能力**。这是和 Meta-Harness / Self-Harness 本质不同的一条路线。"
},
{
"label": "什么在变",
"text": "Harness Engineer weights + 它生成的 executable lifecycle hooks；target frozen。"
},
{
"label": "谁来改 / 谁执行",
"text": "target=Qwen3.5-9B；editor=另一个 Qwen3.5-9B Harness Engineer；cold-start teacher=GPT-5.5。"
},
{
"label": "基础 harness",
"text": "runtime hook API：episode-init / pre-decision / pre-action / post-feedback。"
},
{
"label": "Feedback",
"text": "candidate patch 后让 frozen target fresh rerun；真实 task reward delta 用作 online GRPO reward。"
},
{
"label": "核心结果",
"text": "target 平均 success **44.3→53.6 (+9.3pp)**；target 自己先 fine-tune 后仍能 **59.2→64.2**。"
},
{
"label": "为什么经典",
"text": "它第一次把“会不会改 harness”从一次次 prompting frontier model，变成专门可学习、可比较的 editor capability；对我们以后拆 self-diagnosis / self-edit capability 非常重要。"
},
{
"label": "边界",
"text": "不是同一个 target 自己改自己；editor 冷启动也依赖 GPT-5.5 teacher。"
}
],
"occurrences": [
{
"section": "Core-15：主干论文（优先完整读）",
"line": 54,
"fields": {
"时间": "2026-08",
"论文": "**[Harness-R1](https://arxiv.org/abs/2608.02276)**",
"为什么是主干": "把 harness editing 从 prompt frontier LLM 变成 **可 post-train 的独立能力**；trained harness engineer 是清晰的新范式。",
"证据边界要记住": "target 与 editor 是分开的；cold start 依赖更强 teacher。"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 340,
"fields": {
"优先级": "**C**",
"时间": "2026-08-03",
"论文": "[Harness-R1: Learning to Edit Executable Runtime Harnesses from Agent Failure Trajectories](https://arxiv.org/abs/2608.02276)",
"本质定位": "专门 post-train 一个 9B Harness Engineer，把“读 failure trajectory→写 executable lifecycle hooks→target fresh rerun”训练成独立能力；target frozen。",
"什么在变": "harness-editor model weights + executable runtime hooks；target weights不变。",
"谁来改 / 谁执行": "**改**：独立 Qwen3.5-9B Harness Engineer；cold-start teacher GPT-5.5，之后 online GRPO。
**执行**：frozen Qwen3.5-9B target。",
"基础 harness": "自建 runtime-hook harness API（episode-init / pre-decision / pre-action / post-feedback）。",
"Feedback": "target fresh rerun 的真实 task reward change。",
"Evolution → Eval": "WebShop / ALFWorld / DBBench；held-out/eval tasks；另测 target fine-tune 后 editor 是否仍增益。",
"Meta-depth": "learned-improver M1/M2-ish；editor 与 target 分离。",
"相对之前真正新增什么": "相对 Meta-Harness/Self-Harness 的真正新点是 **harness editing 本身被训练成模型能力**，而不是每次 prompting 一个 proposer。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录报告独立任务评估；查看详细记录中的数据与选模限制。",
"year": "2026",
"depth": [
"M1",
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "训练一个专门的 Harness Engineer，把目标 agent 的失败批次变成可执行 runtime hooks。冻结 target 重新跑相同任务，真实 reward delta 用于更新 engineer 权重。",
"novelty": "用示范及补丁安装后的任务奖励训练一个独立编辑模型，让它从失败记录生成辅助执行代码；答题模型的参数保持冻结。",
"object": "两种变化：harness 编辑模型的参数，以及它生成的执行补丁。被修复的任务模型参数固定。",
"executor": "冻结的 Qwen3.5-9B 运行生成的运行框架完成任务；跨模型实验再把运行框架迁移给其他目标模型。",
"modifier": "独立的 Qwen3.5-9B 框架工程师 生成和修改运行框架；先用 GPT-5.5 提供冷启动示例，再用在线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练工程师模型。",
"roleContext": "**改**：独立 Qwen3.5-9B Harness Engineer；cold-start teacher GPT-5.5，之后 online GRPO。
**执行**：frozen Qwen3.5-9B target。",
"seed": "初始框架已经能组织模型与环境交互，并在任务开始、模型决策前、动作执行前、收到反馈后预留四处可插入辅助代码的位置。编辑模型可以在这里调整输入、检查动作或处理停滞；答题模型参数及任务环境接口固定。",
"fixed": "每阶段 target weights 固定；engineer 权重受训练，hook API / 外层优化规则固定。",
"verdict": "修改框架后，让固定任务模型在同一批题上重新执行，比较真实任务奖励的变化；这份变化训练修改者，不能当作独立留出题的泛化成绩。",
"diagnosis": "engineer 读取 target 的 failure batch，决定应在哪个 lifecycle 阶段干预。",
"update": "生成并验证 executable runtime patch；reward 反传给 engineer。",
"acceptance": "先验证 patch 可执行，再用 same-batch outcome 构造训练 reward；这不等于 held-out regression gate。",
"experiments": [
{
"name": "WebShop / ALFWorld / DBBench",
"evolve": "target failure batches 与 same-batch reruns 训练 engineer",
"selection": "SFT + GRPO 的训练选择",
"test": "Table D.1 列出 1,300 test tasks（WebShop 500、ALFWorld 500、DBBench 300）；另测未见目标模型",
"isolation": "独立测试",
"note": "“目标已 fine-tune 后再训练 engineer”是分阶段设置，不是已证实多代闭环。"
}
],
"takeaway": "不是同一个 target 自己改自己；editor 冷启动也依赖 GPT-5.5 teacher。",
"result": "target 平均 success **44.3→53.6 (+9.3pp)**；target 自己先 fine-tune 后仍能 **59.2→64.2**。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.02276v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：编辑器的 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为） 任务：WebShop（根据用户要求挑选和购买商品的交互基准） 5,290/5,190；ALFWorld（通过文字动作完成家居物体操作的交互环境） 1,380/1,280；DBBench 2,401/2,302。先分任务再采轨迹，两个训练分区互斥。\n\n调试 / 选版本数据：299 道 验证集 用于选择编辑器 checkpoint（某个时刻保存的模型或系统版本）；另一个迁移实验每个评测基准给编辑器 10 条失败示例，据此生成补丁。\n\n最终测试数据：完整测试 1,300 题：WebShop（根据用户要求挑选和购买商品的交互基准） 500、ALFWorld（通过文字动作完成家居物体操作的交互环境） 500、DBBench 300。附录 F 的未见题结果为扣除 30 道示例后的 1,270 题。\n\n数据隔离与证据边界：区分编辑器训练、checkpoint（某个时刻保存的模型或系统版本） 验证和最终测试；测试内又区分补丁生成时可见的 30 道示例与未见的 1,270 道题。",
"cycle": "工程师模型读取固定 task agent 的一批失败记录，决定改哪个执行环节。补丁先检查能否运行，再让 task agent 在同一批题上重做；前后奖励变化用于训练工程师。这个训练奖励不等于独立留出题上的收益。",
"train": "编辑器的 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为） 任务：WebShop（根据用户要求挑选和购买商品的交互基准） 5,290/5,190；ALFWorld（通过文字动作完成家居物体操作的交互环境） 1,380/1,280；DBBench 2,401/2,302。先分任务再采轨迹，两个训练分区互斥。",
"debug": "299 道 验证集 用于选择编辑器 checkpoint（某个时刻保存的模型或系统版本）；另一个迁移实验每个评测基准给编辑器 10 条失败示例，据此生成补丁。",
"test": "完整测试 1,300 题：WebShop（根据用户要求挑选和购买商品的交互基准） 500、ALFWorld（通过文字动作完成家居物体操作的交互环境） 500、DBBench 300。附录 F 的未见题结果为扣除 30 道示例后的 1,270 题。",
"isolation": "区分编辑器训练、checkpoint（某个时刻保存的模型或系统版本） 验证和最终测试；测试内又区分补丁生成时可见的 30 道示例与未见的 1,270 道题。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "固定生命周期钩子接口：episode-init、pre-decision、pre-action、post-feedback。编辑模型从目标失败轨迹写可执行钩子；目标 agent 的任务接口不随意变，编辑器则单独经过 SFT 和结果训练。",
"protocol": "**任务级 SFT/RL/验证/测试：**WebShop 5,290/5,190/100/500；ALFWorld 1,380/1,280/99/500；DBBench 2,401/2,302/100/300。SFT 和 RL 任务先互斥拆分，总训练 17,843，验证 299，最终测试 1,300。\n\n**特点：**ALFWorld 测试是 109 个 new_std＋391 个 train_valid 抽样，不能套用其他论文的 134 unseen。先分任务再采轨迹/补丁，样本记录数不是新增任务数；修正旧笔记中的 1,270 口径。",
"sections": "附录 D、Table D.1",
"source": "https://arxiv.org/abs/2608.02276",
"version": "2608.02276v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "8a5063f2c30dbe84d574d98bfec80488693778d138e07a249d630c3557afd9fd",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "冻结的 Qwen3.5-9B 运行生成的运行框架完成任务；跨模型实验再把运行框架迁移给其他目标模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "独立的 Qwen3.5-9B 框架工程师 生成和修改运行框架；先用 GPT-5.5 提供冷启动示例，再用在线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练工程师模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "两种变化：harness 编辑模型的参数，以及它生成的执行补丁。被修复的任务模型参数固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "修改框架后，让固定任务模型在同一批题上重新执行，比较真实任务奖励的变化；这份变化训练修改者，不能当作独立留出题的泛化成绩。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "初始框架已经能组织模型与环境交互，并在任务开始、模型决策前、动作执行前、收到反馈后预留四处可插入辅助代码的位置。编辑模型可以在这里调整输入、检查动作或处理停滞；答题模型参数及任务环境接口固定。",
"sources": [
{
"label": "附录 D、Table D.1",
"url": "https://arxiv.org/abs/2608.02276"
},
{
"label": "§3.1：四处代码介入位置",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "工程师模型读取固定 task agent 的一批失败记录，决定改哪个执行环节。补丁先检查能否运行，再让 task agent 在同一批题上重做；前后奖励变化用于训练工程师。这个训练奖励不等于独立留出题上的收益。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "编辑器的 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为） 任务：WebShop（根据用户要求挑选和购买商品的交互基准） 5,290/5,190；ALFWorld（通过文字动作完成家居物体操作的交互环境） 1,380/1,280；DBBench 2,401/2,302。先分任务再采轨迹，两个训练分区互斥。",
"sources": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "299 道 验证集 用于选择编辑器 checkpoint（某个时刻保存的模型或系统版本）；另一个迁移实验每个评测基准给编辑器 10 条失败示例，据此生成补丁。",
"sources": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "完整测试 1,300 题：WebShop（根据用户要求挑选和购买商品的交互基准） 500、ALFWorld（通过文字动作完成家居物体操作的交互环境） 500、DBBench 300。附录 F 的未见题结果为扣除 30 道示例后的 1,270 题。",
"sources": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "区分编辑器训练、checkpoint（某个时刻保存的模型或系统版本） 验证和最终测试；测试内又区分补丁生成时可见的 30 道示例与未见的 1,270 道题。",
"sources": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用示范及补丁安装后的任务奖励训练一个独立编辑模型，让它从失败记录生成辅助执行代码；答题模型的参数保持冻结。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.02276v1",
"version": "2608.02276v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "改进器的 editing policy 可以通过 SFT + GRPO 学出来；被训练的是独立 editor，不是同一个 target 的权重与 harness 同时自演化。",
"feedbackCases": [
{
"label": "编辑器训练 / WebShop",
"data": "监督训练 5,290 题、强化学习 5,190 题；完整测试 500 题",
"scoring": "环境按购物目标的约束满足程度给连续奖励，可对部分完成给分。",
"visible": "从目标模型的失败记录提取任务约束、动作—观察片段和结果。编辑器提交补丁后，让固定目标模型在同一批任务中重新执行，取得真实新奖励。",
"use": "按修补前后任务表现训练编辑模型；不是只奖励补丁语法正确。另有 299 道验证题选检查点；迁移实验每基准给 10 条失败示例后冻结补丁，扣除示例的独立结果共 1,270 题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"judgment": "WebShop 环境按商品与购物要求的匹配程度计算奖励"
},
{
"label": "编辑器训练 / ALFWorld",
"data": "监督训练 1,380 题、强化学习 1,280 题；完整测试 500 题",
"scoring": "环境检查物体操作目标是否完成，给二元成功/失败。",
"visible": "从目标模型的失败记录提取任务约束、动作—观察片段和结果。编辑器提交补丁后，让固定目标模型在同一批任务中重新执行，取得真实新奖励。",
"use": "按修补前后任务表现训练编辑模型；不是只奖励补丁语法正确。另有 299 道验证题选检查点；迁移实验每基准给 10 条失败示例后冻结补丁，扣除示例的独立结果共 1,270 题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"judgment": "ALFWorld 环境程序检查物体操作目标"
},
{
"label": "编辑器训练 / DBBench",
"data": "监督训练 2,401 题、强化学习 2,302 题；完整测试 300 题",
"scoring": "数据库任务环境给二元任务成功/失败，不是让编辑器自评补丁格式。",
"visible": "从目标模型的失败记录提取任务约束、动作—观察片段和结果。编辑器提交补丁后，让固定目标模型在同一批任务中重新执行，取得真实新奖励。",
"use": "按修补前后任务表现训练编辑模型；不是只奖励补丁语法正确。另有 299 道验证题选检查点；迁移实验每基准给 10 条失败示例后冻结补丁，扣除示例的独立结果共 1,270 题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"judgment": "DBBench 环境给二元成功信号；本文未展开每题内部检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "编辑器的 监督微调（用示范数据训练模型）/强化学习（根据奖励调整模型行为） 任务：WebShop（根据用户要求挑选和购买商品的交互基准） 5,290/5,190；ALFWorld（通过文字动作完成家居物体操作的交互环境） 1,380/1,280；DBBench 2,401/2,302。先分任务再采轨迹，两个训练分区互斥。",
"selection": "299 道 验证集 用于选择编辑器 checkpoint（某个时刻保存的模型或系统版本）；另一个迁移实验每个评测基准给编辑器 10 条失败示例，据此生成补丁。",
"evaluation": "完整测试 1,300 题：WebShop（根据用户要求挑选和购买商品的交互基准） 500、ALFWorld（通过文字动作完成家居物体操作的交互环境） 500、DBBench 300。附录 F 的未见题结果为扣除 30 道示例后的 1,270 题。",
"isolation": "区分编辑器训练、checkpoint（某个时刻保存的模型或系统版本） 验证和最终测试；测试内又区分补丁生成时可见的 30 道示例与未见的 1,270 道题。",
"roles": {
"executor": {
"value": "冻结的 Qwen3.5-9B 运行生成的运行框架完成任务；跨模型实验再把运行框架迁移给其他目标模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
]
},
"modifier": {
"value": "独立的 Qwen3.5-9B 框架工程师 生成和修改运行框架；先用 GPT-5.5 提供冷启动示例，再用在线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练工程师模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
}
]
},
"seed": {
"value": "初始框架已经能组织模型与环境交互，并在任务开始、模型决策前、动作执行前、收到反馈后预留四处可插入辅助代码的位置。编辑模型可以在这里调整输入、检查动作或处理停滞；答题模型参数及任务环境接口固定。",
"sources": [
{
"label": "附录 D、Table D.1",
"url": "https://arxiv.org/abs/2608.02276"
},
{
"label": "§3.1：四处代码介入位置",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"selection": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"evaluation": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
],
"isolation": [
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "直接提示固定模型修改框架并不可靠：作者的初步对照中，强模型编辑有时收益有限甚至掉分。已有方法通常用结果挑补丁，却不训练提出补丁的模型；而补丁好坏只有装进运行系统后才能判断，因此需要把失败、可执行修改与真实任务增益连成修改者的学习信号。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.02276#S1"
}
]
},
{
"key": "position",
"value": "研究“根据失败记录修改运行框架”能否成为可专门学习的工程能力，使修改者可靠地提升另一个冻结 agent 的任务表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.02276"
}
]
},
{
"key": "conclusion",
"value": "冻结 Qwen3.5-9B 的平均成功率从 44.3% 到 53.6%；更新的是工程师权重及其生成的补丁。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.02276"
}
]
}
],
"fields": {
"object": "两种变化：harness 编辑模型的参数，以及它生成的执行补丁。被修复的任务模型参数固定。",
"verdict": "修改框架后，让固定任务模型在同一批题上重新执行，比较真实任务奖励的变化；这份变化训练修改者，不能当作独立留出题的泛化成绩。",
"seed": "初始框架已经能组织模型与环境交互，并在任务开始、模型决策前、动作执行前、收到反馈后预留四处可插入辅助代码的位置。编辑模型可以在这里调整输入、检查动作或处理停滞；答题模型参数及任务环境接口固定。"
}
},
"attributions": [
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.02276"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver",
"harness",
"weights"
]
},
{
"id": "2608.05144",
"title": "Argus: A General-Purpose Agentic Reasoning Runtime for Long-Horizon Tasks",
"url": "https://arxiv.org/abs/2608.05144",
"date": "2026-08-05",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Evaluator",
"M1",
"MemoryContent",
"RegressionGate",
"Skill",
"Workflow"
],
"fields": {
"本质定位": "Manager/Planner/Engineer/Reviewer 分权；persistent state 中保存 memory/skills/verifiers/routing/rejected routes，并以 evidence/authority gate 才 commit 更新或 objective refinement。",
"什么在变": "persistent runtime state、skills/memory/verifier guidance/routing，部分 operational objective。",
"谁来改 / 谁执行": "**改**：Argus 内部多角色共同更新。
**执行**：GPT-5.5 + Argus/Codex/Copilot backend。",
"基础 harness": "固定 Manager–Planner–Engineer–Reviewer runtime。",
"Feedback": "task-native evidence + Reviewer + authority confirmation。",
"Evolution → Eval": "7 arenas 多为 sequential/current benchmark；Math synthesis 有 dev/test。",
"Meta-depth": "M1；framework roles fixed。",
"相对之前真正新增什么": "独特性不在“又能记忆”，而在 **verification-gated durable state + role separation + authorized pivot**。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "persistent runtime / harness-state evolution 的代表。不是直接重写整个 harness source，而是让 memory、skills、verifier guidance、routing、rejected routes、operational contract 等 durable state 在长期执行中被验证后 commit。"
},
{
"label": "什么在变",
"text": "persistent memory/skills/knowledge/verifier/routing/failure state，部分 operational objective/constraint；模型权重与 Manager–Planner–Engineer–Reviewer 大框架基本固定。"
},
{
"label": "谁来改 / 谁执行",
"text": "Argus 内部 Manager / Planner / Engineer / Reviewer 分权更新；GPT-5.5 + Codex/Copilot 等 backend执行。"
},
{
"label": "基础 harness",
"text": "固定四角色 runtime + durable project state + review/revision/rollback gate。"
},
{
"label": "Feedback",
"text": "task-native executable/quantitative evidence + Reviewer judgment；material objective change 还需要 authority/user confirmation。"
},
{
"label": "Evolution → Eval",
"text": "SWE-Bench Pro 等主 arenas 多是 sequential/current benchmark accumulation，没有统一 `evolve→freeze→held-out`；Math synthesis 有 dev/test，因而证据强度不统一。"
},
{
"label": "主要结果",
"text": "完整 runtime 在 SWE-Bench Pro 等任务上显著优于 direct backend，并观察到 reviewer rescue、state reuse 与 objective pivot；但多个机制和额外 test-time compute 同时变化。"
},
{
"label": "为什么是经典 / 关键限制",
"text": "经典性在于它把“持续改进”做成了 **有治理的 durable runtime state**：不是所有 reflection 都写回，只有 evidence-backed candidate 才 commit；同时它也是 story–evidence gap 的好例子——主 benchmark 并不能干净证明 objective pivot/self-evolution 本身带来 gain。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 60,
"fields": {
"论文": "**[Argus](https://arxiv.org/abs/2608.05144)**",
"为什么仍应视为 Core / 强代表": "persistent runtime / durable-state / verified objective pivot 的代表；强调 role separation 与 commit authority。"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 335,
"fields": {
"优先级": "**C**",
"时间": "2026-08-05",
"论文": "[Argus: A General-Purpose Agentic Reasoning Runtime for Long-Horizon Tasks](https://arxiv.org/abs/2608.05144)",
"本质定位": "Manager/Planner/Engineer/Reviewer 分权；persistent state 中保存 memory/skills/verifiers/routing/rejected routes，并以 evidence/authority gate 才 commit 更新或 objective refinement。",
"什么在变": "persistent runtime state、skills/memory/verifier guidance/routing，部分 operational objective。",
"谁来改 / 谁执行": "**改**：Argus 内部多角色共同更新。
**执行**：GPT-5.5 + Argus/Codex/Copilot backend。",
"基础 harness": "固定 Manager–Planner–Engineer–Reviewer runtime。",
"Feedback": "task-native evidence + Reviewer + authority confirmation。",
"Evolution → Eval": "7 arenas 多为 sequential/current benchmark；Math synthesis 有 dev/test。",
"Meta-depth": "M1；framework roles fixed。",
"相对之前真正新增什么": "独特性不在“又能记忆”，而在 **verification-gated durable state + role separation + authorized pivot**。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "混合协议",
"protocolBasis": "不同实验的划分不同，不能用一个 held-out 标签概括；见逐任务协议。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Manager/Planner/Engineer/Reviewer 分权；persistent state 中保存 memory/skills/verifiers/routing/rejected routes，并以 evidence/authority gate 才 commit 更新或 objective refinement。",
"novelty": "将规划、执行、审核和目标管理分给不同角色；经验需经相应权限与证据检查才长期复用，改变研究方向也要保留理由和必要的人类决策。",
"object": "跨任务保留的记忆、技能、验证指导、路由规则及部分操作目标；角色分工固定。",
"executor": "七个主基准均用 GPT-5.5：SWE-Bench Pro 经 Copilot 运行，推理强度 xhigh；其余六项经 Codex 运行。GLM-5.2 经 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 的结果属于尚在进行、没有匹配 Direct 基线的补充实验。",
"modifier": "GPT-5.5 驱动 Argus 的 管理角色、规划角色、工程角色、审阅角色 分工，写入或审核记忆、技能、流程和验证器；运行时规定持久化及审核规则，涉及目标歧义时由人作决定。",
"roleContext": "**改**：Argus 内部多角色共同更新。
**执行**：GPT-5.5 + Argus/Codex/Copilot backend。",
"seed": "固定 管理角色–规划角色–工程角色–审阅角色 四角色运行系统，预算由外层控制；规划角色 定任务边界，工程角色 执行，审阅角色 验证。Skill/Wiki 路径向 task agent 开放供自行检索，不是框架自动把所有经验注入。",
"fixed": "framework roles fixed",
"verdict": "审阅角色检查任务产物、运行记录及其证据是否支持目标完成；需要确认的事项依照系统权限处理。不同任务有不同验收条件，开放科研没有统一自动正确答案。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "混合协议",
"note": "7 arenas 多为 sequential/current benchmark；Math synthesis 有 dev/test。"
}
],
"takeaway": "经典性在于它把“持续改进”做成了 **有治理的 durable runtime state**：不是所有 reflection 都写回，只有 evidence-backed candidate 才 commit；同时它也是 story–evidence gap 的好例子——主 benchmark 并不能干净证明 objective pivot/self-evolution 本身带来 gain。",
"result": "完整 runtime 在 SWE-Bench Pro 等任务上显著优于 direct backend，并观察到 reviewer rescue、state reuse 与 objective pivot；但多个机制和额外 test-time compute 同时变化。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：固定模型下的在线任务经验；SWE-bench Pro 731 题顺序执行并积累运行状态，没有统一离线训练集。\n\n调试 / 选版本数据：各任务原生信号：仓库测试、kernel 正确性/速度、nanochat 验证 BPB、nanoGPT 目标 loss，以及 审阅角色 的产物检查。\n\n最终测试数据：七个 arena：SWE-Pro、SOL-ExecBench、nanochat B200、nanochat H100、nanoGPT speedrun、AARRI-Bench 82 题、Arbor 数学推理数据合成。\n\n数据隔离与证据边界：这些 arena 的评分和适应协议各不相同；在线 SWE 成绩不能视作冻结状态后在全新任务上的独立因果验证。",
"cycle": "工程或科研角色提出产物、技能和记忆修改，由相应权限角色依据证据决定是否接收。要求独立审核的任务交给审阅者；部分低风险任务允许记录工程角色自审。规划角色维护后续工作，管理角色维护目标与阶段；改研究方向需记录依据，必要时交由人类决定。",
"train": "固定模型下的在线任务经验；SWE-bench Pro 731 题顺序执行并积累运行状态，没有统一离线训练集。",
"debug": "各任务原生信号：仓库测试、kernel 正确性/速度、nanochat 验证 BPB、nanoGPT 目标 loss，以及 审阅角色 的产物检查。",
"test": "七个 arena：SWE-Pro、SOL-ExecBench、nanochat B200、nanochat H100、nanoGPT speedrun、AARRI-Bench 82 题、Arbor 数学推理数据合成。",
"isolation": "这些 arena 的评分和适应协议各不相同；在线 SWE 成绩不能视作冻结状态后在全新任务上的独立因果验证。"
},
"reviewed": false,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "固定模型下的 verification-gated persistent runtime / harness-state evolution。",
"index": {
"date": "2026-08-05",
"paper": "[Argus: A General-Purpose Agentic Reasoning Runtime for Long-Horizon Tasks](https://arxiv.org/abs/2608.05144)",
"executor": "**GPT-5.5 + Argus runtime**（不同任务接 Codex/Copilot 等 backend）",
"modifier": "Argus 内部 **Planner / Engineer / Reviewer / Manager** 分权更新 persistent runtime state",
"feedback": "execution evidence、task-native verifier、Reviewer judgment；material objective change 还需要 authority/user confirmation",
"evolve": "**没有统一独立训练集**：SWE-Bench Pro 731 tasks 顺序执行并积累 state；SOL-ExecBench、nanochat B200/H100、nanoGPT speedrun、AARRI-Bench 也在任务执行过程中直接适应；Math data synthesis 有独立 dev",
"eval": "**同一批主 benchmark arenas**：SWE-Bench Pro、SOL-ExecBench、nanochat B200/H100、nanoGPT speedrun、AARRI-Bench；**Math synthesis 另有 held-out test**",
"tldr": "固定模型，通过 persistent memory/skill/verifier/routing/state 与 review gate 持续积累和修正 runtime state。完整 runtime 明显提升任务表现，但主 benchmark 多是 fixed objective，且大多没有独立 evolve→freeze→held-out protocol，缺 frozen-state / fixed-objective counterfactual，因此还不能干净归因到 self-evolution 或 objective pivot 本身。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 现有 self-improvement 多依赖固定、dense、快速的 objective/evaluator；真实 long-horizon research 中 objective、constraint、verifier 本身可能随 evidence 逐渐明确。长期运行还面临跨 session 状态丢失、执行者 self-certification、失败经验无法复用等问题。 |\n| 本文定位 | 提出 **Argus**：固定模型的 long-horizon agent runtime，用 durable state + role separation + verification gate 支持持续执行、回滚和 **verified pivoting**。稳定的是 user intent，operational objective / constraint / verifier 可基于证据 refinement。 |\n| 核心贡献 | ① Manager–Planner–Engineer–Reviewer 分权；② 将 memory、skills、verifiers、routing、rejected routes、operational contract 等作为可持久化 runtime state；③ candidate update 必须经过 evidence/authority gate 后才能 commit；④ 在 7 个 benchmark 和 long-horizon research cases 中验证通用执行、review/revision、persistent state 与 objective pivot。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 核心流程 | **Manager → Planner → Engineer ↔ Reviewer → Manager**。Planner 拆 bounded mission；Engineer 实际调用工具/改代码/跑实验；Reviewer 检查真实 artifact + execution evidence，输出 `done / continue / blocked`；必要时返工、rollback 或 pivot。 |\n| 什么在进化 | **模型权重不变**；变化的是 memory、skills、knowledge、verifier guidance、routing、失败路线、task/evaluation definition，以及可 refinement 的 operational objective/constraint。大框架 Manager–Planner–Engineer–Reviewer 本身基本固定，不是直接重写完整 harness source。 |\n| 谁来改 | 同一 Argus runtime 中的 Planner / Engineer / Reviewer / Manager 分权更新不同 state；没有独立更强 proposer。 |\n| 修改时能看到什么 | 当前 durable project state、execution trajectory、artifact、review/evaluation evidence、历史 accepted/rejected state。 |\n| Feedback | 同模型 Reviewer + task-native executable/quantitative evidence；material objective change 需要对应 authority/user confirmation。 |\n| Evolution 口径 | 只有 `candidate → verify → authorized commit → later reuse` 才算 self-evolution；单纯 reflection 不算。 |\n| 方法本质 | **test-time agentic scaling + 跨 mission persistent harness-state adaptation**，而非模型训练式 self-improvement。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark | 任务 | 执行者：模型 + harness | Feedback / verifier | Evolution / Train → Test | 人工标注 / feedback |\n|---|---|---|---|---|---|\n| SWE-Bench Pro | 731 个真实 repo issue，修改 production code | **GPT-5.5/xhigh + Argus；Copilot backend** | Argus Reviewer / execution evidence；最终 official executable verifier 判分 | **无独立 evolve-train → freeze → held-out test**；731 个任务顺序执行并积累 state；无 frozen-state matched replay | **无 gold patch / 正确答案作为迭代 feedback**；benchmark test 只用于最终验证 |\n| SOL-ExecBench | GPU kernel correctness + speed optimization | **GPT-5.5 + Argus；Codex** | kernel correctness + 实际运行性能 / SOL score | 直接在 optimization tasks 上执行；无独立 harness train/test split | 无人工 improvement feedback |\n| nanochat B200 | 1×B200、5 min 内优化 LM training program，使 validation BPB 越低越好 | **GPT-5.5 + Argus；Codex** | 实际训练得到的 validation BPB | 直接反复修改→训练→测 BPB | 无 |\n| nanochat H100 | 同上，换 1×H100 | **GPT-5.5 + Argus；Codex** | validation BPB | 同上 | 无 |\n| nanoGPT speedrun | 8×H100 上尽快达到固定 validation loss=3.28 | **GPT-5.5 + Argus；Codex** | validation loss + 达到目标所需时间 | 直接针对固定目标优化 | 无 |\n| AARRI-Bench | 82 个 research-intern 任务，测试科研 workflow/judgment | **GPT-5.5 + Argus；Codex** | 每题人工预先编写的 deterministic tests / pattern matching，最终 0/1 pass | Argus 直接跑 evaluation tasks；无独立 evolution split | **有人类预先定义正确行为并写 tests**，但无运行时人工 improvement feedback |\n| Math-Reasoning Data Synthesis | 生成 AIME-style 数学题，使题目 first attempt 难、更多 attempts 可解 | **GPT-5.5 + Argus；Codex** | validity/novelty/consistency filters + 固定 GPT-5.5 solver；优化 `pass@4-pass@1` | **有 dev/test split**：dev 上迭代，held-out test 做最终验证 | 无人工逐轮 feedback；数学正确性主要依赖自动 consistency/solver proxy，不是 formal proof |"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 真正能支持的结论 |\n|---|---|---|\n| 完整 Argus runtime 能显著提高 task performance | SWE-Bench Pro **≈78% vs Direct Copilot ≈59%**；Argus 用 **1.41× aggregate tokens** | planning/review/revision/persistent runtime 整体有效，但无法区分各组件贡献，也不能排除更多 test-time compute 的作用 |\n| Reviewer 能救回部分失败 trajectory | 43 个任务被要求 revise，34 个后来通过 official verifier；22 个满足严格 `continue→revision→done` rescue | review/revision loop 确实改变了 trajectory 并产生 recoveries |\n| 同一 runtime 跨任务类型可工作 | SWE repair、kernel、training optimization、research、data synthesis 等 7 个 arena 均取得较强结果 | 支持 Argus 作为 general-purpose runtime，而非单 benchmark workflow |\n| persistent state 可能降低后续执行成本 | SWE mature waves 相比 startup：solve-input token/task **−21%**，active time/task **−15%** | 与经验复用一致，但只是 observational evidence，不能因果归因于 state evolution |\n| objective / success contract 可以随 evidence pivot | 数学 campaign、6 个 paper campaigns；典型项目连续 no-go 后从 positive method search 转为 negative-results audit | 展示 **verified objective refinement 可以发生并持续推进**；但没有证明允许 pivot 比 fixed-objective 更好 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| **Story 与主实验错位** | 论文最强调 **underdefined objective / verified pivoting**，但 7 个主 benchmark 大多是 **fixed objective + 明确 verifier**。这些 benchmark 能证明 runtime 强，**不能说明“可修改 objective”带来性能提升**；真正对应该 claim 的主要是 long-horizon case studies。 |\n| Objective revision 缺直接 ablation | 没有 `pivot-enabled vs fixed-objective` controlled comparison，也没有统计 benchmark 中 objective revision 的频率及其 performance gain；因此只证明“能 pivot”，没证明“pivot 有益”。 |\n| Self-evolution 缺因果验证 | SWE 的 startup→mature token/time 改善没有 `evolved state vs frozen initial state` matched-task replay；task composition/difficulty 同时变化，因此不能证明 persistent evolution 本身提升 capability。 |\n| 提升归因不干净 | SWE 78% vs 59% 同时引入 planning、Reviewer、多轮 revision、persistent state 和 **1.41× token**，缺组件级 ablation，无法判断主要收益来自 evolution 还是普通 test-time scaling。 |\n| 数据隔离有限 | SWE 的 state accumulation 发生在 731-task evaluation sequence 上，不是先独立 evolve 后 freeze 再在 held-out tasks 测泛化；Math synthesis 的 dev/test 设计相对更干净。 |\n| 能力上限 | 论文证明完整 runtime 能提高平均任务表现和长期执行能力，但**没有证明 evolved harness state 突破 initial model/harness 的能力上限**。 |"
},
{
"title": "总评",
"body": "**Argus 的亮点是把 persistent harness evolution 的对象与治理机制做得很完整：不仅积累成功 memory/skill，也保留失败路线，并允许 evidence-backed objective refinement。其主 benchmark 较强地证明了完整 runtime 和 review/revision loop 有用；但论文最有特色的 objective pivot 主要由 case study 支撑，同时缺少 frozen-state、fixed-objective 等关键 counterfactual，因此还不能严格证明“self-evolution / objective revision 本身导致 capability improvement”。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "固定 Manager–Planner–Engineer–Reviewer 四角色 runtime，预算由外层控制；Planner 定任务边界，Engineer 执行，Reviewer 验证。Skill/Wiki 路径向 agent 开放供自行检索，不是框架自动把所有经验注入。",
"protocol": "**七个 arena：**SWE-bench Pro 731 题顺序做并积累状态；SOL-ExecBench GPU kernel；nanochat B200 与 H100 各五分钟训练优化；nanoGPT speedrun 测到目标 loss 的耗时；AARRI-Bench 82 项研究助理任务；数学推理数据合成。\n\n**隔离：**没有统一离线进化集。前述多数在同一任务过程适应并报告，数学合成另有 dev/held-out 设计。不能把系统其他 benchmark 分数当作 SWE 在线状态增长的因果留出证据。",
"sections": "§5 与各 arena 评估定义",
"source": "https://arxiv.org/abs/2608.05144",
"version": "2608.05144v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "d1c37d8e43aaa0f4a0c7bfb819640e9c5a2cde0134658313153858946d529a55",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "七个主基准均用 GPT-5.5：SWE-Bench Pro 经 Copilot 运行，推理强度 xhigh；其余六项经 Codex 运行。GLM-5.2 经 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 的结果属于尚在进行、没有匹配 Direct 基线的补充实验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.05144#S6.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-5.5 驱动 Argus 的 管理角色、规划角色、工程角色、审阅角色 分工，写入或审核记忆、技能、流程和验证器；运行时规定持久化及审核规则，涉及目标歧义时由人作决定。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.05144#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨任务保留的记忆、技能、验证指导、路由规则及部分操作目标；角色分工固定。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "审阅角色检查任务产物、运行记录及其证据是否支持目标完成；需要确认的事项依照系统权限处理。不同任务有不同验收条件，开放科研没有统一自动正确答案。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定 管理角色–规划角色–工程角色–审阅角色 四角色运行系统，预算由外层控制；规划角色 定任务边界，工程角色 执行，审阅角色 验证。Skill/Wiki 路径向 task agent 开放供自行检索，不是框架自动把所有经验注入。",
"sources": [
{
"label": "§5 与各 arena 评估定义",
"url": "https://arxiv.org/abs/2608.05144"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "工程或科研角色提出产物、技能和记忆修改，由相应权限角色依据证据决定是否接收。要求独立审核的任务交给审阅者；部分低风险任务允许记录工程角色自审。规划角色维护后续工作，管理角色维护目标与阶段；改研究方向需记录依据，必要时交由人类决定。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "固定模型下的在线任务经验；SWE-bench Pro 731 题顺序执行并积累运行状态，没有统一离线训练集。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "各任务原生信号：仓库测试、kernel 正确性/速度、nanochat 验证 BPB、nanoGPT 目标 loss，以及 审阅角色 的产物检查。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "七个 arena：SWE-Pro、SOL-ExecBench、nanochat B200、nanochat H100、nanoGPT speedrun、AARRI-Bench 82 题、Arbor 数学推理数据合成。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这些 arena 的评分和适应协议各不相同；在线 SWE 成绩不能视作冻结状态后在全新任务上的独立因果验证。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将规划、执行、审核和目标管理分给不同角色；经验需经相应权限与证据检查才长期复用，改变研究方向也要保留理由和必要的人类决策。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.05144v2",
"version": "2608.05144v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "独特性不在“又能记忆”，而在 **verification-gated durable state + role separation + authorized pivot**。",
"feedbackCases": [
{
"label": "SWE-bench Pro",
"data": "731 题顺序运行",
"scoring": "仓库可执行验收测试检查补丁。",
"visible": "执行证据交给 审阅角色 检查产物与目标是否一致；候选记忆/技能须经角色审核及可用任务验收才复用。",
"use": "同一持久项目中积累经验；这几类任务的成功率、BPB、耗时和差距指标不应合成一个“任务奖励”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"judgment": "SWE-bench Pro 仓库测试"
},
{
"label": "SOL-ExecBench",
"data": "B200 GPU kernel 优化任务",
"scoring": "先检查 kernel 计算正确性，再测运行速度。",
"visible": "执行证据交给 审阅角色 检查产物与目标是否一致；候选记忆/技能须经角色审核及可用任务验收才复用。",
"use": "同一持久项目中积累经验；这几类任务的成功率、BPB、耗时和差距指标不应合成一个“任务奖励”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"judgment": "可执行数值正确性检查 + 内核运行计时"
},
{
"label": "nanochat B200 / H100",
"data": "两套硬件上的训练任务",
"scoring": "验证数据上的 BPB（每字节所需编码位数）衡量模型质量，结合资源限制评价。",
"visible": "执行证据交给 审阅角色 检查产物与目标是否一致；候选记忆/技能须经角色审核及可用任务验收才复用。",
"use": "同一持久项目中积累经验；这几类任务的成功率、BPB、耗时和差距指标不应合成一个“任务奖励”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"judgment": "验证集损失指标 BPB，由程序计算"
},
{
"label": "nanoGPT speedrun",
"data": "固定训练任务及预算",
"scoring": "测达到规定验证损失的时间。",
"visible": "执行证据交给 审阅角色 检查产物与目标是否一致；候选记忆/技能须经角色审核及可用任务验收才复用。",
"use": "同一持久项目中积累经验；这几类任务的成功率、BPB、耗时和差距指标不应合成一个“任务奖励”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"judgment": "达到指定验证损失的训练耗时，由程序计时"
},
{
"label": "AARRI-Bench / Arbor 数学数据合成",
"data": "AARRI 82 个研究复现题及 Arbor 合成研究任务",
"scoring": "按对应科研任务的完成/复现或训练效果评价；各自目标和评分单位不同。",
"visible": "执行证据交给 审阅角色 检查产物与目标是否一致；候选记忆/技能须经角色审核及可用任务验收才复用。",
"use": "同一持久项目中积累经验；这几类任务的成功率、BPB、耗时和差距指标不应合成一个“任务奖励”。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"judgment": "按各科研任务的复现或训练指标验收；未统一成答案裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4
],
"evolution": "固定模型下的在线任务经验；SWE-bench Pro 731 题顺序执行并积累运行状态，没有统一离线训练集。",
"selection": "各任务原生信号：仓库测试、kernel 正确性/速度、nanochat 验证 BPB、nanoGPT 目标 loss，以及 审阅角色 的产物检查。",
"evaluation": "七个 arena：SWE-Pro、SOL-ExecBench、nanochat B200、nanochat H100、nanoGPT speedrun、AARRI-Bench 82 题、Arbor 数学推理数据合成。",
"isolation": "这些 arena 的评分和适应协议各不相同；在线 SWE 成绩不能视作冻结状态后在全新任务上的独立因果验证。",
"roles": {
"executor": {
"value": "七个主基准均用 GPT-5.5：SWE-Bench Pro 经 Copilot 运行，推理强度 xhigh；其余六项经 Codex 运行。GLM-5.2 经 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 的结果属于尚在进行、没有匹配 Direct 基线的补充实验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.05144#S6.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "GPT-5.5 驱动 Argus 的 管理角色、规划角色、工程角色、审阅角色 分工，写入或审核记忆、技能、流程和验证器；运行时规定持久化及审核规则，涉及目标歧义时由人作决定。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.05144#S6.SS1"
}
]
},
"seed": {
"value": "固定 管理角色–规划角色–工程角色–审阅角色 四角色运行系统，预算由外层控制；规划角色 定任务边界，工程角色 执行，审阅角色 验证。Skill/Wiki 路径向 task agent 开放供自行检索，不是框架自动把所有经验注入。",
"sources": [
{
"label": "§5 与各 arena 评估定义",
"url": "https://arxiv.org/abs/2608.05144"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"selection": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"evaluation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
],
"isolation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.05144#S5.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "开放科研开始时，目标往往尚未定义到可直接优化的程度，也没有可信分数持续指路；沿用目标固定的 agent 流程，可能把修正错误目标与失败后的自我辩解混淆。因此需要保留促成目标修订的证据，并在 agent 无法判断的关键处引入专家，而不只按现有分数继续搜索。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.05144#S1"
}
]
},
{
"key": "position",
"value": "面向目标可能变化、正确性信号不充分的开放科研，解决 agent 如何长期维持研究方向、积累可信证据并持续推进工作的问题。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05144"
}
]
},
{
"key": "conclusion",
"value": "展示跨任务执行与长期研究案例；关键特点是持续维护目标、证据和决策，而非只围绕一个分数搜索。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05144"
}
]
}
],
"fields": {
"object": "跨任务保留的记忆、技能、验证指导、路由规则及部分操作目标；角色分工固定。",
"verdict": "审阅角色检查任务产物、运行记录及其证据是否支持目标完成；需要确认的事项依照系统权限处理。不同任务有不同验收条件，开放科研没有统一自动正确答案。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"curriculum"
]
},
{
"id": "2608.09819",
"title": "Macaron-V1: Towards Open Continual Learning with Self-Improvement and Mixture-of-LoRA",
"url": "https://arxiv.org/abs/2608.09819",
"date": "2026-08-10",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"M1",
"SameSet",
"Weights"
],
"fields": {
"本质定位": "系统把 deployable successor 定义为 versioned model–harness pair：MoL specialist adapters + HCP-carried prompts/skills/tools/hooks；MindForge Discovery→Expansion→Update。",
"什么在变": "设计上 model adapters + harness config；直接实验只 harness/config。",
"谁来改 / 谁执行": "**改**：MindForge/Expansion adaptive search。
**执行**：frozen GLM-5.2 base（RSI coverage study）或 Venti/Tall systems。",
"基础 harness": "HCP versioned model–harness resources。",
"Feedback": "TerminalBench official reward + trajectory/config outcome。",
"Evolution → Eval": "122 selected frozen-base failure tasks 上 adaptive search，same-set cumulative coverage；无 frozen held-out successor。",
"Meta-depth": "M1 hybrid substrate。",
"相对之前真正新增什么": "架构层面连接 model/harness continual learning，但论文最直接 RSI 实验只隔离 frozen-model configuration Expansion。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "model–harness continual learning / co-evolution 的代表系统。它最重要的 framing 是把 successor 定义成 **versioned model–harness pair**，用 HCP 把 prompt/skill/tool/hook/session/workspace 等 harness resources 与 adapter/data/eval lineage 绑定。"
},
{
"label": "什么在变",
"text": "设计上同时允许 specialist LoRA/model revision + harness/config revision；但论文最直接 RSI coverage experiment **只改 harness/config，model frozen**。"
},
{
"label": "谁来改 / 谁执行",
"text": "MindForge/Expansion adaptive search 改 HCP-carried resources；直接 coverage study 用 frozen GLM-5.2-FP8 base，整体产品线另有 Venti/Tall model–harness systems。"
},
{
"label": "基础 harness",
"text": "HCP-versioned prompts/skills/tools/hooks + Mixture-of-LoRA architecture / MindForge lifecycle。"
},
{
"label": "Feedback",
"text": "TerminalBench official reward + trajectory/configuration outcomes + version/audit lineage。"
},
{
"label": "Evolution → Eval",
"text": "122 个预先选出的 frozen-base failure tasks 上 adaptive search；最终 **仍在同一 122 tasks 上统计 cumulative coverage**，没有一个 frozen single-successor held-out test。"
},
{
"label": "主要结果",
"text": "adaptive search 最终可覆盖 122/122 failure tasks，但 best single full-set configuration 只有 11/122；说明不同 failures 能被不同 harness config 激活，不等于存在一个 100% successor。"
},
{
"label": "为什么是经典 / 关键限制",
"text": "它是目前 **model–harness pair / continual-learning infrastructure** 最有代表性的系统之一；但必须明确：论文没有证明完整 Expansion→Update→下一代 pair 的多代 compounding，直接实验主要还是 frozen-model harness search。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 61,
"fields": {
"论文": "**[Macaron-V1](https://arxiv.org/abs/2608.09819)**",
"为什么仍应视为 Core / 强代表": "versioned **model–harness pair** / continual learning framing 的高热代表；系统设计很完整。"
}
},
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 352,
"fields": {
"优先级": "**C**",
"时间": "2026-08-10",
"论文": "[Macaron-V1: Towards Open Continual Learning with Self-Improvement and Mixture-of-LoRA](https://arxiv.org/abs/2608.09819)",
"本质定位": "系统把 deployable successor 定义为 versioned model–harness pair：MoL specialist adapters + HCP-carried prompts/skills/tools/hooks；MindForge Discovery→Expansion→Update。",
"什么在变": "设计上 model adapters + harness config；直接实验只 harness/config。",
"谁来改 / 谁执行": "**改**：MindForge/Expansion adaptive search。
**执行**：frozen GLM-5.2 base（RSI coverage study）或 Venti/Tall systems。",
"基础 harness": "HCP versioned model–harness resources。",
"Feedback": "TerminalBench official reward + trajectory/config outcome。",
"Evolution → Eval": "122 selected frozen-base failure tasks 上 adaptive search，same-set cumulative coverage；无 frozen held-out successor。",
"Meta-depth": "M1 hybrid substrate。",
"相对之前真正新增什么": "架构层面连接 model/harness continual learning，但论文最直接 RSI 实验只隔离 frozen-model configuration Expansion。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Same-set adaptive",
"protocolBasis": "依据原记录：主实验在同一任务/子集上反复优化或评估；不能直接当作冻结后的泛化。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "把部署单元定义为可版本化的 model–harness pair：LoRA specialists、路由，以及 HCP 携带的 prompts/skills/tools/hooks。MindForge 组织发现、扩张和更新。",
"novelty": "把可更新的专长适配参数与可改写的工具、技能和运行配置组织到同一系统；直接框架搜索实验冻结模型，衡量多个配置合起来覆盖多少失败题。",
"object": "122 题实证搜索只改提示、技能、工具配置和执行钩子，不训练参数。系统设计还包括专长 LoRA 的训练，但不能把该训练当作 122 题实验的一部分。",
"executor": "Venti：冻结GLM-5.2（744B）配四个LoRA（只训练少量适配参数） 专长模块；Tall：Qwen3.6-35B-A3B配四个LoRA（只训练少量适配参数）。覆盖率搜索实验需与最终适配参数配置分开读。",
"modifier": "MindForge 是本文的学习任务生成与执行记录筛选模块；Expansion search 是运行配置搜索流程，负责尝试不同提示、工具和技能配置。需要更新模型适配参数时，由对应训练程序执行。",
"roleContext": "**改**：MindForge/Expansion adaptive search。
**执行**：frozen GLM-5.2 base（RSI coverage study）或 Venti/Tall systems。",
"seed": "HCP（本文保存和管理运行配置的组件）保存版本化运行配置，MoL（多个专长适配器及其路由机制）在冻结基础模型上路由 聊天、代理任务、编程、界面生成四类 LoRA（只训练少量适配参数） 专家。基础设施已有模型服务与适配器管理；Expansion（本文的运行配置搜索实验）研究的配置搜索与整套参数训练是两层不同过程。",
"fixed": "HCP 是配置协议，不接收梯度；不同实验实际可变对象不同。",
"verdict": "任务执行结果、轨迹/config 任务结果；部分系统评估使用 语言模型 评分者。",
"diagnosis": "从失败分布识别缺失或未被引出的能力，再搜索配置或学习 specialist。",
"update": "系统设计允许 harness config 与 adapters 更新；需区分实际执行的阶段。",
"acceptance": "候选配置经重新执行验证；不能用累计 coverage 代替冻结 successor 性能。",
"experiments": [
{
"name": "RSI coverage study",
"evolve": "选定 frozen-base failure tasks 上的 adaptive search",
"selection": "搜索内选择可解决配置",
"test": "same-set 累计 coverage",
"isolation": "同集适应",
"note": "此实验不是 untouched held-out 的 joint model–harness successor。"
}
],
"takeaway": "归为联合系统，同时明确“架构层联合”与“当前实验的实际更新”两列，避免把设计宣言当成多代实证。",
"result": "adaptive search 最终可覆盖 122/122 failure tasks，但 best single full-set configuration 只有 11/122；说明不同 failures 能被不同 harness config 激活，不等于存在一个 100% successor。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.09819v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：MindForge 生成并评估任务轨迹，用于更新 专长模块 LoRA（只训练少量适配参数）；Expansion 单独固定基础模型，在 TB2.1 的 29 个来源族、122 个失败模拟任务上搜索配置。\n\n调试 / 选版本数据：Expansion 的 69 个 jobs、450 次尝试针对尚未通过的同批题；helper 必须通过私有参考验证才进入可复用工具池。\n\n最终测试数据：Expansion 报同一 122 题的累计覆盖；整系统另在 ChatBench、LivingBench、VitaBench/2、τ³、PinchBench、ClawGym、SWE-Verified、TB2.1、DeepSWE、SWE Atlas QnA、UI4A-Bench 评估。\n\n数据隔离与证据边界：122/122 指不同配置累计至少通过一次，不是一个冻结配置通过全部未见题。6,448 条路由诊断样本来自训练数据，也不是独立泛化测试。",
"cycle": "从失败任务识别缺失能力，尝试修改运行配置；系统设计还支持训练专长适配器，但需要分清具体实验有没有执行该步骤。每个候选配置重新运行任务验证；多个配置累计解决的题数不能当成单一最终版本的成功率。",
"train": "MindForge 生成并评估任务轨迹，用于更新 专长模块 LoRA（只训练少量适配参数）；Expansion 单独固定基础模型，在 TB2.1 的 29 个来源族、122 个失败模拟任务上搜索配置。",
"debug": "Expansion 的 69 个 jobs、450 次尝试针对尚未通过的同批题；helper 必须通过私有参考验证才进入可复用工具池。",
"test": "Expansion 报同一 122 题的累计覆盖；整系统另在 ChatBench、LivingBench、VitaBench/2、τ³、PinchBench、ClawGym、SWE-Verified、TB2.1、DeepSWE、SWE Atlas QnA、UI4A-Bench 评估。",
"isolation": "122/122 指不同配置累计至少通过一次，不是一个冻结配置通过全部未见题。6,448 条路由诊断样本来自训练数据，也不是独立泛化测试。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "把 post-deployment adaptation 组织成 **versioned model–harness pair** 的 successor construction，并用 Mixture-of-LoRA 承载可分离 specialist updates；但论文直接隔离验证的 RSI 实验只覆盖 **frozen-model harness/configuration Expansion**。",
"index": {
"date": "2026-08-10",
"paper": "[Macaron-V1: Towards Open Continual Learning with Self-Improvement and Mixture-of-LoRA](https://arxiv.org/abs/2608.09819)",
"executor": "**直接 RSI/Expansion 实验**：frozen GLM-5.2-FP8 base + 当前 HCP-carried harness/config；整体产品线另有 Venti（GLM-5.2 744B + 4 LoRA specialists）与 Tall（Qwen3.6-based 50B + adapters）",
"modifier": "**MindForge / Expansion adaptive search loop** 修改 HCP-carried resources、task skills、tool exposure、hooks；论文未把该隔离实验归因给一个固定更强 builder model",
"feedback": "official task reward + trajectory / configuration outcomes + HCP version/audit lineage",
"evolve": "**TerminalBench 2.1 的 29 个 source families 中挑出的 122 个 frozen-base failure simulation tasks**；69 jobs / 450 attempts，adaptive search 直接针对仍未覆盖 tasks",
"eval": "**仍是同一 122-task failure slice**，报告 cumulative coverage；没有 frozen single-successor held-out task test。整体 Personal Intelligence / agent / coding / GenUI benchmark 不能当作该 Expansion 实验的因果 held-out eval",
"tldr": "系统目标是 versioned model–harness continual learning，但论文直接 RSI 证据只隔离 **frozen-model harness/config search**：adaptive search 覆盖 122/122 failure tasks，而最佳 single full-set config 仅 11/122；作者明确说这不是 full RSI training-cycle gain，也未验证跨代 compounding。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 部署后的 agent/model 需要从真实 experience 中持续学习，但完整 weight update 成本高、容易产生 interference；同时 model update、harness/config update、trajectory/data/eval 往往缺少统一 version lineage，难判断 successor 到底改了什么。 |\n| 本文定位 | 将 adaptation 定义为 **versioned model–harness pairs 的 recursive improvement**；model 侧通过 Mixture-of-LoRA（MoL）维护可扩展 specialist adapters，harness 侧通过 Harness Context Protocol（HCP）把 prompts、skills、tools、hooks、sessions、workspace 等变成可版本化、可审计配置。 |\n| 核心贡献 | ① MoL：冻结大 base、每个 user turn 路由一个 specialist LoRA；② Model–Harness Co-design + HCP；③ MindForge 三阶段 `Discovery → Expansion → Update` RSI lifecycle；④ 用 122 个 base-failure tasks 隔离测试 Expansion/configuration-search 能力。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 设计上什么在进化 | **Model–harness pair**。Model 侧可产生新的 specialist LoRA revision；harness 侧可修改 HCP-carried prompts、skills、tool exposure、hooks、workspace/session resources。HCP 协议本身不是 gradient-updated object，而是 versioned boundary。 |\n| MoL / 执行结构 | Base model 冻结；多个 specialist LoRA 分别承载 chat、agent、coding、GenUI 等能力，每个 user turn 选择一个 adapter。Venti 为 GLM-5.2 744B base + 4 LoRAs；Tall 为 Qwen3.6-based 50B + adapters。 |\n| RSI 三阶段 | **Discovery** 找 failure / opportunity；**Expansion** 在 language/configuration space 搜有效 behavior；**Update** 把选中的 trajectories 转成新的 adapter revision，并与对应 HCP configuration / evaluation artifact 绑定。 |\n| 谁来改 | 论文把 **MindForge / Expansion search** 作为 lifecycle/orchestration mechanism；在直接隔离实验中，不把修改归因于一个固定更强 external builder，而是持续修改 HCP-carried resources、skills、tool exposure、hooks。 |\n| Feedback | official task reward + trajectory / configuration outcomes；trajectory selection 要求通过 evaluator、能归因到目标 task/config，并保留 HCP / adapter / data / eval lineage。 |\n| **直接 RSI 实验实际改什么** | **只改 harness/configuration。** 论文明确写明 model frozen、全程 **no optimizer step**；因此这组实验证明的是 configuration search，不是 model–harness pair 的完整联合更新。 |\n| 方法本质 | 系统设计属于 **model–harness continual-learning architecture**；当前最直接可归因的 RSI evidence 则属于 **frozen-model harness evolution**。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| 实验 | 执行者：模型 + harness | Feedback | Train / Evolve → Eval | 人工标注 / GT |\n|---|---|---|---|---|\n| **RSI Expansion coverage study** | **Frozen GLM-5.2-FP8 base + 当前 HCP-carried harness/config** | TerminalBench official reward + execution/config outcomes | 从 TerminalBench 2.1 的 **29 个 source families** 中选出 frozen base 全部 not-pass 的 **122 simulation tasks**；69 chronological jobs、450 attempts；adaptive search 持续针对未覆盖 tasks。**Coverage 仍在同一 122 tasks 上统计，无 held-out task split。** | 依赖 official reward；论文没有把该 experiment 描述为人工逐轮 improvement feedback |\n| Overall Macaron model-family evaluation | Venti / Tall model–harness systems | 各 benchmark official / reproduced evaluation protocol | Personal Intelligence、agent、coding、GenUI、general capability 等 broad evaluation | 这些结果验证当前 model family / harness design，但**不能用来隔离 Expansion 自进化的因果贡献** |\n\n> **关键数据隔离口径**：122 tasks 是故意选出的 **base-failure slice**；adaptive search 会根据“哪些任务还没被覆盖”决定下一轮 target，因此 `122/122` 是 search coverage ceiling，不是一个 frozen successor 在 unseen tasks 上的 generalization accuracy。"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 真正能支持的结论 |\n|---|---|---|\n| Frozen model 的能力可被不同 harness/config 大量“激活” | 69 jobs / 450 attempts 后 cumulative unique coverage 达 **122/122**；每个 task 至少在某个 configuration 下 pass 一次 | baseline failure 不等于 frozen model 完全没有该行为；harness/configuration 对 capability elicitation 有很大影响。 |\n| 单一 global configuration 远远不够 | 两次 full-set single-configuration sweep 分别 **4/122** 与 **11/122**；adaptive search 最终 122/122 | 异质 failure 更适合 targeted configuration search，而不是寻找一个万能 config。 |\n| 后期 search yield 明显提高，但不能直接归因给某个 hook | Skill/HCP search pooled pass **64.5%**；最后 `+ stop-gate hooks` 阶段 **81.2%** | Search trajectory 越来越有效；但论文明确指出 task mix、targeting policy、harness-error rate 同时变化，因此不能把阶段差简单解释成某个 component 的 causal effect。 |\n| **没有验证 Expansion→adapter Update 的 transfer** | 论文明确说明本实验没有执行后续 $\\phi$/adapter update | 122/122 **不是** full RSI training-cycle capability gain，也不证明搜出的 behavior 被稳定写入下一代 model。 |\n| 多代 continual learning 仍未证明 | Abstract / discussion 明确将 **compounding gains from continual learning** 留作 open question | Macaron 更像搭好了 recursive continual-learning substrate，而不是已经展示稳定 generation-to-generation compounding。 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| **Story 与直接实验的距离** | 论文整体 story 是 `versioned model–harness pair → next model–harness pair`；但最直接 RSI experiment **只测 Expansion**，model frozen、没有 optimizer step。不能把 122/122 写成完整 model–harness recursive improvement 的实证。 |\n| **Train/Eval 同集** | Adaptive search 和 coverage 都发生在同一 122 个 selected failure tasks；而且 search 明确知道哪些 tasks 仍未覆盖。它证明“针对这些 failure 可以搜到有效 config”，不是 unseen-task generalization。 |\n| **Coverage ≠ 一个更强 successor** | 122/122 可以来自 122 个 task 各自不同的 configuration；不存在“一个最终统一 harness 在 122 tasks 上 100%”这一结论。 |\n| Weight / harness attribution 尚未完成 | 论文明确说 generation-by-generation 的 quantitative attribution 尚未报告；因此 model update、harness update、interaction 各自贡献仍不清楚。 |\n| 能力上限 | 结果很有价值地说明 **harness 能暴露 frozen base 的大量 latent capability**；但不证明 model 本身能力上限被突破，更不证明 successor improver 出现 compounding gain。 |"
},
{
"title": "总评",
"body": "**Macaron-V1 的系统设计比一篇普通 harness-search 论文更宽：它把 deployable successor 定义成有完整 lineage 的 model–harness pair，并用 specialist LoRA + HCP + MindForge 为 continual learning 准备更新通道。但论文最干净的 RSI 实验恰恰主动把 model update 拿掉，只验证 frozen base 上的 adaptive harness/configuration search：122 个预先选出的 base-failure tasks 最终都能在某个 configuration 下被覆盖，而单一 full-set config 最多只过 11 个。这非常能说明 harness 对 latent capability elicitation 的作用，却不能说明完整 RSI cycle 或多代 model–harness compounding 已经成立。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "HCP 保存版本化运行配置，MoL 在冻结基础模型上路由 Chat/Agent/Coding/GenUI 四类 LoRA 专家。基础设施已有模型服务与适配器管理；Expansion 研究的配置搜索与整套参数训练是两层不同过程。",
"protocol": "**Expansion：**TerminalBench 2.1 的 29 个来源族中取 122 个冻结基础系统失败的模拟任务，69 jobs/450 attempts 针对尚未覆盖任务搜索；最后报告同一 122 题的累计覆盖，不是一个冻结候选在 122 个独立新题上的成绩。\n\n**整体测试另列：**ChatBench、LivingBench、VitaBench/2、τ³、PinchBench、ClawGym、SWE-Verified、TB2.1、DeepSWE、SWE Atlas QnA、UI4A-Bench。它们测整体模型系统，不自动证明 Expansion 的迁移收益；逐套专家训练数据本轮未完整核实。",
"sections": "§2–3 Expansion；Table 8",
"source": "https://arxiv.org/abs/2608.09819",
"version": "2608.09819v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "3cbb659d859349cd05b83cbf91e277f5d434a2f9be461560e07e271db2085001",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Venti：冻结GLM-5.2（744B）配四个LoRA（只训练少量适配参数） 专长模块；Tall：Qwen3.6-35B-A3B配四个LoRA（只训练少量适配参数）。覆盖率搜索实验需与最终适配参数配置分开读。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.09819#S6.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "MindForge 是本文的学习任务生成与执行记录筛选模块；Expansion search 是运行配置搜索流程，负责尝试不同提示、工具和技能配置。需要更新模型适配参数时，由对应训练程序执行。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "122 题实证搜索只改提示、技能、工具配置和执行钩子，不训练参数。系统设计还包括专长 LoRA 的训练，但不能把该训练当作 122 题实验的一部分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务执行结果、轨迹/config 任务结果；部分系统评估使用 语言模型 评分者。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "HCP（本文保存和管理运行配置的组件）保存版本化运行配置，MoL（多个专长适配器及其路由机制）在冻结基础模型上路由 聊天、代理任务、编程、界面生成四类 LoRA（只训练少量适配参数） 专家。基础设施已有模型服务与适配器管理；Expansion（本文的运行配置搜索实验）研究的配置搜索与整套参数训练是两层不同过程。",
"sources": [
{
"label": "§2–3 Expansion；Table 8",
"url": "https://arxiv.org/abs/2608.09819"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "从失败任务识别缺失能力，尝试修改运行配置；系统设计还支持训练专长适配器，但需要分清具体实验有没有执行该步骤。每个候选配置重新运行任务验证；多个配置累计解决的题数不能当成单一最终版本的成功率。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "MindForge 生成并评估任务轨迹，用于更新 专长模块 LoRA（只训练少量适配参数）；Expansion 单独固定基础模型，在 TB2.1 的 29 个来源族、122 个失败模拟任务上搜索配置。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Expansion 的 69 个 jobs、450 次尝试针对尚未通过的同批题；helper 必须通过私有参考验证才进入可复用工具池。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Expansion 报同一 122 题的累计覆盖；整系统另在 ChatBench、LivingBench、VitaBench/2、τ³、PinchBench、ClawGym、SWE-Verified、TB2.1、DeepSWE、SWE Atlas QnA、UI4A-Bench 评估。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "122/122 指不同配置累计至少通过一次，不是一个冻结配置通过全部未见题。6,448 条路由诊断样本来自训练数据，也不是独立泛化测试。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把可更新的专长适配参数与可改写的工具、技能和运行配置组织到同一系统；直接框架搜索实验冻结模型，衡量多个配置合起来覆盖多少失败题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.09819v2",
"version": "2608.09819v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "协同学习的系统架构很完整；但架构允许共同更新，不等于所有 RSI 实验都同时更新模型和 harness。",
"feedbackCases": [
{
"label": "实际框架搜索：TB2.1 失败任务",
"data": "TerminalBench 2.1 的 29 个来源族所构造的 122 个模拟任务，入选条件是固定 GLM-5.2-FP8 在官方奖励下全部未通过。",
"scoring": "任务结束后，由任务自带的验收测试检查容器中的代码或其他产物，产生官方奖励，再据此记通过/失败；不是让另一模型凭印象判断。Harbor（运行隔离任务环境、agent 和验收程序的评测工具） 的标准接口执行 tests/test.sh 并读取奖励文件；具体测试断言随任务不同。",
"visible": "搜索利用任务是否已通过和执行记录，后续优先处理仍未通过的题；共 69 个任务批次、450 次尝试。",
"use": "修改提示、技能、开放工具与执行钩子；同一批题的累计覆盖达到 122/122，表示每题至少被某个配置通过一次，不代表一个冻结配置全过。",
"sources": [
{
"label": "§3.2.5；图 7、表 4",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "Harbor 官方协议：测试脚本与奖励文件（接口说明）",
"url": "https://www.harborframework.com/docs/tasks"
}
],
"judgment": "Terminal-Bench 任务自带测试程序检查容器产物"
},
{
"label": "系统设计：合成任务与训练轨迹",
"data": "MindForge（本文的任务生成与筛选模块）从版本化种子任务库生成更难变体，如增加约束、隐藏偏好或串联子目标；论文未给出一个可直接列举全部样本的固定训练集。",
"scoring": "每个生成任务须带可验证答案或评分细则；具体任务后端负责检查题目质量、难度与执行结果，并可附过程评价。有效且有学习价值的执行记录用于训练适配参数。",
"visible": "任务结果、过程判断、运行配置和来源记录关联保存；论文未给出所有训练任务逐一对应的判分器/评审模型清单。",
"use": "框架描述了“构题→执行/筛选→GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新适配参数”的流程；上述 122 题搜索实验本身没有执行参数更新，不可把架构描述当成多代训练实证。",
"sources": [
{
"label": "§3.2.1–3.2.4",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2.3：任务生成与筛选",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS3.Px1"
}
],
"judgment": "合成题配答案或评分细则；论文未逐任务披露程序／模型裁判配置"
},
{
"label": "工具复用检查",
"data": "模型产生并暂存的辅助工具；使用私有、预先留出的参考进行验证。",
"scoring": "save_tool 只进入候选池；通过私有参考验证后，promote_tool 才使其可供后续任务调用。",
"visible": "验证决定工具是否获准复用；后续发现错误会记录并撤回。论文未列出每个辅助工具的私有测试内容。",
"use": "这是工具入库条件，不是 ChatBench 等成品榜单分数。",
"sources": [
{
"label": "§3.1.2：Validated Reuse",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
}
],
"judgment": "私有参考验证工具；未逐工具公开验证实现"
},
{
"label": "成品评测：ChatBench",
"data": "46 个脱敏真实多轮对话案例，每个模型—案例组合生成三次。",
"scoring": "私有部署的 GLM-5.2 按针对该案例的 1–5 分细则评审，细则来自六条交互原则；三次取平均。",
"visible": "报告案例评分的汇总；这是模型评审，不是标准答案字符串匹配。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.1：ChatBench",
"url": "https://arxiv.org/html/2608.09819#A2.SS1.SSS0.Px1"
}
],
"judgment": "GLM-5.2 按案例专属 1–5 分细则评审"
},
{
"label": "成品评测：LivingBench",
"data": "40 个日常场景，中英文各 20；每例最多 10 轮，重复三次。",
"scoring": "Claude Opus 4.6 评审；总分 = 70% 需求满足度（按场景要求逐项判定）+ 30% 过程质量（依据交互记录）。Kimi K2.6 模拟用户，Gemini 3.1 Pro 承担其余环境角色，它们不是同一个评分角色。",
"visible": "按固定权重聚合三次结果；部分用户私有状态与检查要求对被测 agent 隐藏。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.1：LivingBench",
"url": "https://arxiv.org/html/2608.09819#A2.SS1.SSS0.Px2"
}
],
"judgment": "Claude Opus 4.6 按需求满足与过程质量评分"
},
{
"label": "成品评测：VitaBench",
"data": "Delivery、In-Store、OTA、Cross 四类任务。",
"scoring": "GLM-5.1 同时承担用户模拟和评审，四类任务的成功率作宏平均。作者替换了已不可用的原官方评审器。",
"visible": "报告这一重建协议下的任务成功率；不能与原官方判分配置视为完全相同。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.3：VitaBench",
"url": "https://arxiv.org/html/2608.09819#A2.SS3.SSS0.Px1"
}
],
"judgment": "GLM-5.1 作模型裁判；本文替换原基准裁判"
},
{
"label": "成品评测：VitaBench2",
"data": "中文个性化任务，使用 Rewrite 记忆设置；每条用户序列运行一次。",
"scoring": "先平均同一用户序列内的子任务奖励，再跨用户平均，称为 Avg@1；论文这一段没有逐项交代奖励判定函数或评审模型。",
"visible": "给出序列与用户层面的平均奖励；官方 Avg@4 使用四次执行，与此不同。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.3：VitaBench2",
"url": "https://arxiv.org/html/2608.09819#A2.SS3.SSS0.Px2"
}
],
"judgment": "子任务奖励汇总；此处未披露奖励内部是规则还是模型判定"
},
{
"label": "成品评测：τ³-Bench",
"data": "同一批工具—用户交互任务，单次执行。",
"scoring": "沿用任务成功判定，报告 pass@1（单次尝试完成任务的比例）；GPT-5.2（low）是用户模拟器。该段没有把它指定为评分模型，也没有展开成功判定的内部规则。",
"visible": "报告单次任务成功率。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.3：τ³-Bench",
"url": "https://arxiv.org/html/2608.09819#A2.SS3.SSS0.Px3"
}
],
"judgment": "任务成功验收；未展开内部规则，GPT-5.2 是模拟用户"
},
{
"label": "成品评测：PinchBench / ClawGym",
"data": "各自固定评测任务；PinchBench 使用 Perplexity 搜索。",
"scoring": "PinchBench 由 Claude Haiku 4.5 评审，报告观察到的最好分数；ClawGym 由 GPT-5.4 评审，报告单次成功率。",
"visible": "两者均有模型评审，但评审型号和汇总方式不同。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.3：PinchBench / ClawGym",
"url": "https://arxiv.org/html/2608.09819#A2.SS3.SSS0.Px4"
},
{
"label": "ClawGym",
"url": "https://arxiv.org/html/2608.09819#A2.SS3.SSS0.Px5"
}
],
"judgment": "PinchBench：Claude Haiku 4.5；ClawGym：GPT-5.4 模型评审"
},
{
"label": "成品评测：SWE-Verified / TB2.1 / DeepSWE",
"data": "三套编程/终端任务，均通过 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 运行；TB2.1 另用 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） 沙箱，限时四小时。",
"scoring": "按各基准任务的验收规则评分。TB2.1 报单次通过率；SWE-Verified 仅在评测系统报错时最多重试三次，不从有效答案中挑最好；DeepSWE 最多尝试三次并取最好。",
"visible": "报告任务是否通过及对应汇总；附录没有列出逐题验收测试的内容。不能把这三种“重试”混为一种。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.4",
"url": "https://arxiv.org/html/2608.09819#A2.SS4.SSS0.Px1"
},
{
"label": "TB2.1",
"url": "https://arxiv.org/html/2608.09819#A2.SS4.SSS0.Px2"
},
{
"label": "DeepSWE",
"url": "https://arxiv.org/html/2608.09819#A2.SS4.SSS0.Px3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
},
{
"label": "Harbor：任务与验收接口",
"url": "https://www.harborframework.com/docs/tasks"
}
],
"judgment": "软件／终端基准的可执行测试验收产物"
},
{
"label": "成品评测：SWE Atlas QnA",
"data": "软件仓库问答任务，使用 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 执行。",
"scoring": "Claude Opus 4.8 评审，报告 pass@3（每题最多尝试三次、至少成功一次的比例）。",
"visible": "这是问答的模型评审，不能与补丁执行测试等同。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.4：SWE Atlas QnA",
"url": "https://arxiv.org/html/2608.09819#A2.SS4.SSS0.Px4"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"judgment": "Claude Opus 4.8 作问答裁判；该段未展开具体比对提示"
},
{
"label": "成品评测：UI4A-Bench",
"data": "161 个交互界面生成任务，覆盖八个领域；只应用与当前任务相关的评分条目。",
"scoring": "手机视口的交互运行器配 Gemini 3.5 Flash 评审；检查工程可运行性、任务质量、视觉、交互和约束五维度，按固定权重汇总后再作版本化线性归一化。",
"visible": "五个维度分及最终汇总分；不是只让模型看截图随意给一个总分。",
"use": "论文将此列为已发布系统的成品评测；未证明该分数逐轮返回给上述框架搜索或直接作为适配器训练奖励。",
"sources": [
{
"label": "附录 B.5",
"url": "https://arxiv.org/html/2608.09819#A2.SS5.SSS0.Px1"
}
],
"judgment": "Gemini 3.5 Flash 按五维评分要求评审，并结合可运行性检查"
}
],
"experiments": [
{
"label": "实际框架搜索：Terminal-Bench 2.1",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "29 个来源族构造的 122 个模拟任务，固定 GLM-5.2-FP8 初始全部未通过；运行失败题获得验收反馈并修改框架。",
"selection": "根据已通过／未通过状态与轨迹，后续优先继续处理失败任务。",
"evaluation": "报告同一 122 题上多个配置的累计覆盖，69 批、450 次尝试。",
"isolation": "122/122 表示每题至少被某个配置通过，不是一个冻结框架在独立测试集上全过。",
"roles": {
"executor": {
"value": "固定 GLM-5.2-FP8；122 题覆盖率搜索全程没有参数更新，不能把成品 Venti／Tall 的四适配器配置套到这项搜索实验。",
"sources": [
{
"label": "§3.2.5：122 题覆盖率实验",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
}
]
},
"modifier": {
"value": "Expansion 配置搜索流程根据未通过任务继续尝试运行配置；该实验描述没有单独报告一个可确认型号的修改模型。没有执行适配参数训练。",
"sources": [
{
"label": "§3.2.5：搜索轨迹与边界",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
}
]
},
"seed": {
"value": "冻结模型配 HCP 保存的运行资源配置，逐步尝试重试控制、技能／工具暴露和停止检查钩子。HCP 是保存运行配置的组件；起始任务集特意选择原始配置全部失败的 122 题。",
"sources": [
{
"label": "§3.2.5、表 4",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
]
}
},
{
"label": "系统设计：合成任务、训练轨迹与工具入库",
"learningCases": [
1,
2
],
"testCases": [],
"evolution": "MindForge 从种子任务库生成变体，配答案或评分细则；验证后的轨迹用于所描述的适配参数训练。候选工具先进入暂存池。",
"selection": "工具须通过私有参考验证才能入库；任务质量、难度与轨迹另经筛选。",
"evaluation": "该架构描述不能单独对应为某项已完成的多代联合训练测试；成品基准另列。",
"isolation": "122 题框架搜索没有执行参数更新，不能把它当作整个训练架构的实证。",
"roles": {
"executor": {
"value": "这是系统训练架构说明：基础模型配专长适配器完成任务并收集轨迹；不代表 122 题搜索期间执行了四适配器训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.09819#S6.SS1.SSS0.Px1"
}
]
},
"modifier": {
"value": "MindForge 是本文的学习任务生成与执行记录筛选模块；Expansion search 是运行配置搜索流程，负责尝试不同提示、工具和技能配置。需要更新模型适配参数时，由对应训练程序执行。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
]
},
"seed": {
"value": "HCP（本文保存和管理运行配置的组件）保存版本化运行配置，MoL（多个专长适配器及其路由机制）在冻结基础模型上路由 聊天、代理任务、编程、界面生成四类 LoRA（只训练少量适配参数） 专家。基础设施已有模型服务与适配器管理；Expansion（本文的运行配置搜索实验）研究的配置搜索与整套参数训练是两层不同过程。",
"sources": [
{
"label": "§2–3 Expansion；Table 8",
"url": "https://arxiv.org/abs/2608.09819"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
]
}
},
{
"label": "成品系统：问答、交互、编程与界面评测",
"learningCases": [],
"testCases": [
3,
4,
5,
6,
7,
8,
9,
10,
11
],
"evolution": "这些基准在论文中作为已发布系统的成品评测，没有证据表明其分数逐轮用于上述框架搜索。",
"selection": "各基准按其规定次数重复、平均或取最好；具体规则见本行测试说明。",
"evaluation": "ChatBench、LivingBench、VitaBench、VitaBench2、τ³-Bench、PinchBench、ClawGym、SWE-Verified、TB2.1、DeepSWE、SWE Atlas QnA、UI4A-Bench；每项数据规模与判分分别保留在下方。",
"isolation": "成品成绩与 122 题框架搜索、合成训练架构分开解释。",
"roles": {
"executor": {
"value": "Venti：冻结GLM-5.2（744B）配四个LoRA（只训练少量适配参数） 专长模块；Tall：Qwen3.6-35B-A3B配四个LoRA（只训练少量适配参数）。覆盖率搜索实验需与最终适配参数配置分开读。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.09819#S6.SS1.SSS0.Px1"
}
]
},
"modifier": {
"value": "成品评测阶段不逐题进化；MindForge 的任务生成、轨迹筛选和适配器训练属于系统构建阶段，不能把成品基准分数全部当成更新反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
]
},
"seed": {
"value": "HCP（本文保存和管理运行配置的组件）保存版本化运行配置，MoL（多个专长适配器及其路由机制）在冻结基础模型上路由 聊天、代理任务、编程、界面生成四类 LoRA（只训练少量适配参数） 专家。基础设施已有模型服务与适配器管理；Expansion（本文的运行配置搜索实验）研究的配置搜索与整套参数训练是两层不同过程。",
"sources": [
{
"label": "§2–3 Expansion；Table 8",
"url": "https://arxiv.org/abs/2608.09819"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2608.09819#S2.SS3.SSS0.Px5"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.09819#A2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2608.09819#A2.SS6"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "一次性后训练使模型绑定于发布前的任务与环境，部署后仍会出现新知识、工具和用户需求，却缺少把这些经历转成系统改进的机制。把异质任务全部挤进共享参数还会产生相互干扰，因此作者关注部署后的持续适应以及不同专长如何共同服务用户。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.09819#S1"
}
]
},
{
"key": "position",
"value": "面向部署后的开放持续学习，探索 agent 如何把新经验转化为长期能力，并协调模型专长与运行框架的共同适应；直接搜索实证主要覆盖框架配置。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09819"
}
]
},
{
"key": "conclusion",
"value": "提供开放模型与持续改进系统；不同任务可由不同配置解决，不等于存在一个能统一解决全部失败的版本。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.09819"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.09819#S3.SS1.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.09819#S3.SS2.SSS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09819#S4"
}
]
}
],
"fields": {
"object": "122 题实证搜索只改提示、技能、工具配置和执行钩子，不训练参数。系统设计还包括专长 LoRA 的训练，但不能把该训练当作 122 题实验的一部分。",
"verdict": "框架搜索用 Terminal-Bench 自带测试程序验收产物，并读取失败轨迹。合成训练数据另配答案或评分细则；成品基准另用代码测试、参考答案或模型评分，不是全部用于框架进化。",
"executor": "122 题框架搜索固定 GLM-5.2-FP8；成品 Venti 用 GLM-5.2，Tall 用 Qwen3.6-35B-A3B，均配四个专长适配器。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "endlessfrontier.tech-assets-paper.pdf",
"title": "BigBang: Pursuing Open-Ended Intelligence through Self-Evolving Synthesis of Verifiable Frontier Tasks",
"url": "https://endlessfrontier.tech/assets/paper.pdf",
"date": "2026-08",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Data"
],
"fields": {
"本质定位": "Generator Agent 改 data-synthesis code，Critic 用可验证 proxy，Meta-Critic 用 downstream training effect 校准生成/评价策略，然后 post-train model。",
"什么在变": "data-synthesis program、critic criteria、data distribution、model weights。",
"谁来改 / 谁执行": "**改**：Generator/Critic/Meta-Critic loop。
**执行**：Qwen3.6 base→BigBang-V1 等。",
"基础 harness": "data-synthesis/research harness。",
"Feedback": "formal/computation/simulation/tool verifier + downstream training effect。",
"Evolution → Eval": "outer-loop real tasks用于 calibration；final 9 suites/11 rows。",
"Meta-depth": "data-level M1/M2-ish。",
"相对之前真正新增什么": "这是 data/model co-evolution，不是 deployment harness evolution；但 feedback 设计（verifier + downstream outcome）很值得借鉴。"
},
"details": [],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 62,
"fields": {
"论文": "**[BigBang](https://endlessfrontier.tech/assets/paper.pdf)**",
"为什么仍应视为 Core / 强代表": "data-level RSI / self-evolving frontier-task synthesis 的系统级代表；不是 deployment harness，但 RSI 叙事里绕不开。"
}
},
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 350,
"fields": {
"优先级": "**C**",
"时间": "2026-08",
"论文": "[BigBang: Pursuing Open-Ended Intelligence through Self-Evolving Synthesis of Verifiable Frontier Tasks](https://endlessfrontier.tech/assets/paper.pdf)",
"本质定位": "Generator Agent 改 data-synthesis code，Critic 用可验证 proxy，Meta-Critic 用 downstream training effect 校准生成/评价策略，然后 post-train model。",
"什么在变": "data-synthesis program、critic criteria、data distribution、model weights。",
"谁来改 / 谁执行": "**改**：Generator/Critic/Meta-Critic loop。
**执行**：Qwen3.6 base→BigBang-V1 等。",
"基础 harness": "data-synthesis/research harness。",
"Feedback": "formal/computation/simulation/tool verifier + downstream training effect。",
"Evolution → Eval": "outer-loop real tasks用于 calibration；final 9 suites/11 rows。",
"Meta-depth": "data-level M1/M2-ish。",
"相对之前真正新增什么": "这是 data/model co-evolution，不是 deployment harness evolution；但 feedback 设计（verifier + downstream outcome）很值得借鉴。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1",
"M2"
],
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "Generator Agent 改 data-synthesis code，Critic 用可验证 proxy，Meta-Critic 用 downstream training effect 校准生成/评价策略，然后 post-train model。",
"novelty": "把任务生成、任务验证和模型学习连起来，并根据学习后的效果调整任务供应；变化涉及训练材料及模型能力。",
"object": "任务合成代码、评价标准、训练数据分布，以及接受后训练的模型参数。",
"executor": "最终由 BigBang-V1 答题，它从 Qwen3.6-35B-A3B 进行后训练得到。生成训练题的 agent 和最终受训模型是两种角色。",
"modifier": "生成角色 生成题目及解答，Critic 检查可验证性与质量，Meta-Critic 校准评价。技术报告没有给出这三个数据生产角色各自的模型型号；已明确的 Qwen3.6-35B-A3B 是受训基础模型，不能据此填成三个角色的型号。",
"roleContext": "**改**：Generator/Critic/Meta-Critic loop。
**执行**：Qwen3.6 base→BigBang-V1 等。",
"seed": "数据合成/研究框架能生成可验证问题、运行候选模型做校准，并修改合成程序。最终模型多数任务接 search、visit、code_exec，专属评测基准用各自运行框架；不能把合成器和最终执行器混写成一个 task agent。",
"fixed": "data-level M1/M2-ish。",
"verdict": "先通过计算、模拟、工具或形式检查验证合成任务，再观察用这些任务训练出的模型是否在下游评测中提升；题目能验证与题目有训练价值分开判断。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "outer-loop real tasks用于 calibration；final 9 suites/11 rows。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：生成角色合成并求解可验证科学任务，Critic筛查后对Qwen3.6-35B-A3B做后训练；不是直接拿某个固定评测基准做统一训练集。\n\n调试 / 选版本数据：Critic评正确性、难度、可验证性及训练价值；真实研究任务表现用于校准Critic并调整下一批合成数据。\n\n最终测试数据：BrowseComp、xbench-DeepSearch、SWE-bench Pro、SciCode-Verified（Sub/Main）、FrontierScience-Research、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、BioMysteryBench（HS/HD）、MLE-Bench Lite、PaperBench Code-Dev。\n\n数据隔离与证据边界：真实任务虽称单独留出的，却会反馈数据合成方向；不能把整个多轮流程解释为最终成绩从未反馈。SciCode经作者审查形成Verified版本。",
"cycle": "生成者编写并运行合成任务代码，记录失败和结论；批评者判断这些任务是否具有训练价值。再训练候选模型，用真实下游任务表现校准任务生产过程。",
"train": "生成角色合成并求解可验证科学任务，Critic筛查后对Qwen3.6-35B-A3B做后训练；不是直接拿某个固定评测基准做统一训练集。",
"debug": "Critic评正确性、难度、可验证性及训练价值；真实研究任务表现用于校准Critic并调整下一批合成数据。",
"test": "BrowseComp、xbench-DeepSearch、SWE-bench Pro、SciCode-Verified（Sub/Main）、FrontierScience-Research、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、BioMysteryBench（HS/HD）、MLE-Bench Lite、PaperBench Code-Dev。",
"isolation": "真实任务虽称单独留出的，却会反馈数据合成方向；不能把整个多轮流程解释为最终成绩从未反馈。SciCode经作者审查形成Verified版本。"
},
"reviewed": false,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "不是修改 deployment harness，而是让 **synthetic-task / data-synthesis pipeline 自己进化**：Generator 直接改 synthesis code，Critic 提供可验证的 dense proxy，Meta-Critic 再用真实 downstream training effect 校准“什么数据真的能让模型变强”，最终反复 post-train model。",
"index": {
"date": "2026-08",
"paper": "[BigBang: Pursuing Open-Ended Intelligence through Self-Evolving Synthesis of Verifiable Frontier Tasks](https://endlessfrontier.tech/assets/paper.pdf)",
"executor": "**BigBang-V1（由 Qwen3.6-35B-A3B post-train）+ task-specific agent/eval harness**；35B total / 3B active，256K context，trajectory 最多 500 tool calls",
"modifier": "**Generator Agent** 直接修改/执行/debug data-synthesis code；**Critic Agent** 评估 generated tasks；**Meta-Critic** 用真实 training outcome 校准 critic 与 generation strategy",
"feedback": "formal rules / computation / simulation / domain tools 等 verifiable feedback + Critic 的 correctness/verifiability/difficulty/scalability/diversity judgments + **held-out real research tasks 上的 downstream training effect**",
"evolve": "evolving synthetic frontier tasks / synthesis programs；代表性 pipeline 生成的数据被用于训练 model variants，**held-out real research tasks 也进入 outer loop 用于 outcome calibration**",
"eval": "最终 BigBang-V1 在 **9 个 benchmark suites / 11 个 rows** 上评测，覆盖 long-horizon search、coding、scientific research、AI research；其中部分是直接 targeted capability，部分作为 transfer",
"tldr": "这是 **data-level RSI**，不是 deployment harness evolution：generator–critic–meta-critic 持续改进 synthetic-task generation/evaluation，再 post-train model。BigBang 相对 Qwen3.6 base 在 11/11 rows 都提升、35B 组 9 项第一+1 项并列第一；但主表测的是完整 pipeline，generator/critic/meta-critic 贡献未分开，而且所谓 held-out real tasks 本身参与 outer-loop calibration。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 随模型接近/超过 human-expert performance，继续依赖人类已知问题和人工设计 task 会限制 capability growth；但 frontier problem 又存在张力：**足够新/难的任务未必容易验证，而容易自动验证的任务可能过窄、很快饱和。** |\n| 本文定位 | 提出 **verifiable frontier tasks**：位于当前知识边界、甚至没有已知最优解，但 candidate solution 可以通过 formal methods、computation、simulation 或 domain-specific tools 客观评估。 |\n| 核心贡献 | 构造 `real frontier gaps → Generator–Critic synthesis → synthetic data → model post-training → real-task outcome calibration → next synthesis round` 的闭环；论文将其称为 **early data-level form of recursive self-improvement**。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **data-synthesis program / generation strategy + critic evaluation criteria/calibration + synthetic-data distribution + model weights**。Deployment agent harness 不是核心 evolution target。 |\n| Generator Agent | 一个 code agent 直接**修改、执行、debug data-synthesis code**，而不只是在固定 prompt 中抽样新题；其 long-horizon agent harness 记录每次修改的 motivation、results、failure causes，支持后续搜索。 |\n| Critic Agent | 两层审计：基础层检查 trajectory information、tool validity、format/integrity 等；高层进一步评价 correctness/evidence、contradiction、verifiability、difficulty/research depth、scalability/diversity 等。它是低成本 proxy，不需要每次都完整训练模型才能给 feedback。 |\n| Meta-Critic / Outcome calibration | 周期性采样不同 synthesis pipelines，用它们的数据训练 **不同 model variants**，再到 **held-out real-world / real research tasks** 上测实际 training effect；将真实 outcome 与 Critic prediction 比较，反过来修正 evaluation criteria 和 generation strategy。 |\n| Co-evolution | 固定 synthesis pipeline 会随着 model 变强而饱和，所以 pipeline 需要持续发现新 capability gaps、提高 difficulty/diversity/structure；模型训练后的能力变化又成为下一轮 task synthesis 的 target。 |\n| Feedback | formal rules、computation、simulators、domain-specific tools / experimental outcome + Critic proxy + **实际 downstream training outcome**。 |\n| 人工 GT 口径 | 核心 synthesis loop 不以“为每个 frontier task 提供人工 canonical answer”为前提；但论文也没有声称整个 training/evaluation stack 完全零人工数据/零人工 rubric，因此不应扩大为“完全无人工监督”。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| 项目 | 内容 |\n|---|---|\n| Base / successor | **Qwen3.6-35B-A3B → BigBang-V1**；35B total parameters、约 3B active parameters。 |\n| Agent runtime | 256K context window；每条 trajectory 最多 **500 tool calls**。 |\n| Train / Evolve | Generator–Critic 持续生成和筛选 **verifiable frontier synthetic tasks**；部分 representative synthesis pipelines 的数据用于训练 model variants；**held-out real research tasks 被用来做 Meta-Critic 的 outcome calibration，因此它们是 outer-loop validation signal，不是最终完全 untouched test。** |\n| Final Eval | **9 个 benchmark suites / 11 个 score rows**，覆盖 long-horizon search、coding、scientific research、AI research：BrowseComp、xbench、SWE-Bench Pro、SciCode-V-Main/Sub、FS-R、HLE、BioMystery-HS/HD、MLE-Bench Lite、PaperBench Code-Dev。 |\n| Eval 与 training 的关系 | 论文明确说 scientific research evaluations 与训练 pipeline 的 target 最接近；BrowseComp / SWE-Bench Pro / MLE-Bench 等用于展示向 web search、software engineering、ML engineering 的 transfer。不能把全部 11 rows 都简单叫“训练时完全未见的 held-out benchmark”。 |\n| Verifier / GT | 不同 frontier task 使用 formal/computational/simulation/tool-based verification；最终 benchmarks 使用各自 official harness/evaluator/reference。 |"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 真正能支持的结论 |\n|---|---|---|\n| BigBang 对 base model 是广泛而非单点提升 | 相比 Qwen3.6-35B-A3B，**11/11 rows 全部提高**；最大增益包括 FS-R **11.9→46.2 (+34.3)**、xbench **32.6→58.4 (+25.8)**、SciCode-V-Main **26.6→50.0 (+23.4)**、HLE **36.2→50.3 (+14.1)**、BioMystery-HD **2.0→15.7 (+13.7)** | 完整 self-evolving synthesis + post-training pipeline 确实能大幅扩展 35B base 的 research/reasoning/coding/tool-use performance。 |\n| 35B 规模组很强 | 35B comparison 中 **9/11 rows 单独最高**，另在 SciCode-V-Main 与 Agents-A1 并列最高，即 10/11 项 first/joint-first | 说明这种 data-centric post-training 能把较小 MoE model 推到同规模 frontier。 |\n| 能逼近甚至局部超过更大 frontier models | BigBang-V1 在 11 rows 中 **6 项超过 DeepSeek V4 Flash**；FS-R、HLE、BioMystery-HD、PaperBench Code-Dev 等还超过 V4 Pro | 强化了“数据/任务生成 pipeline 本身可以带来大规模 capability gain”的 evidence，但不等于 BigBang 在所有能力上优于更大模型。 |\n| Generator 在固定 Critic 下也能逐轮改进数据 | 论文报告 critic held fixed 时，Generator 产生的数据在 independent benchmark 上随 rounds 持续改善；并出现合并多 source、重写 generation prompts、增加 quality checks / candidate elimination 等行为 | 支持 Generator 不只是重复采样，而是在修改 synthesis process 本身。 |\n| Search-and-Verify 进一步提高 inference performance | 6 个 search/science benchmarks 全部提升，例如 xbench **58.4→64.6**、HLE **50.3→52.5**、BioMystery-HS **57.5→63.0** | 这是 **inference-time search/verification** 的额外收益，应与训练阶段 self-evolution 分开看。 |\n| **主结果无法逐组件归因** | 论文明确说明 Table 1 测量的是 **complete BigBang post-training pipeline**，Generator、Critic、outcome calibration 的 individual contribution 仍需要单独 ablation | 能证明完整闭环有效，但不能从当前主结果断言哪一个 self-evolution component 是主要增益来源。 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| **是否属于 Harness Self-Evolution** | **不属于。** Generator 虽然运行在 long-horizon agent harness 中，真正被自动改的是 data-synthesis code/strategy，随后还更新 model weights；应该归入 **Non-Harness Self-Evolution / data-level RSI**。 |\n| Held-out 的含义需谨慎 | “held-out real research tasks”是相对 synthetic training data held out，但它们的 training-effect 被 **Meta-Critic 反复用于校准 Critic / Generator**，因此从整个 evolution loop 看更像 outer-loop validation，而不是完全 untouched final test。 |\n| 归因仍不够干净 | 主表比较的是 base 与完整 BigBang pipeline，没有完整的 `generator only / +critic / +meta-critic outcome calibration` controlled ablation；因此 story 中最关键的 co-evolution机制还缺独立因果量化。 |\n| “Recursive” 的强度 | 论文的 recursive 主要体现在 **模型变强 → 固定 synthesis pipeline 饱和 → outcome signal 反过来改变 task-generation/evaluation strategy → 继续训练模型**。这是较强的 data/model co-evolution，但并没有证明一个 general-purpose improver 自身在多代中稳定产生 compounding improvement。 |\n| Feedback 独立性 | 相比只用 self-judge 的方法更可靠：核心 anchor 是 executable verification 与真实 downstream training effect；但 evaluation criterion / real-task set 仍是外部固定边界，并没有一起开放式进化。 |\n| 能力上限 | 相比 base 的提升巨大，而且能在部分 benchmark 超过大得多的模型，说明 synthetic frontier-task evolution 能显著改变能力边界；但目前不能据此断言“突破了该 base architecture 在任意充分训练/充分强 data pipeline 下的理论上限”。 |"
},
{
"title": "总评",
"body": "**BigBang 是目前很有代表性的 data-level self-evolution：不是让 deployed agent 改 prompt/memory/harness，而是让 Generator Agent 直接改生成训练任务的程序，Critic 用可验证 frontier feedback 做低成本筛选，Meta-Critic 再用真实 downstream training effect 校准“哪些看起来困难的任务真的值得训练”，最终持续改变 synthetic-data distribution 和 model weights。BigBang-V1 相对 Qwen3.6 base 在全部 11 个 benchmark rows 上都提升，说明完整 flywheel 很有效；但作者自己的结果也明确留下两条边界：主表没有拆开 Generator/Critic/Meta-Critic 的贡献，而且用于 outcome calibration 的 held-out real tasks 已经处于 outer loop 内。因此它是很强的 data/model co-evolution evidence，但还不是对通用多代 compounding RSI 的证明。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "数据合成/研究框架能生成可验证问题、运行候选模型做校准，并修改合成程序。最终模型多数任务接 search、visit、code_exec，专属 benchmark 用各自 harness；不能把合成器和最终执行器混写成一个 agent。",
"protocol": "**训练数据：**演化合成前沿任务及生成程序，选出的数据训练模型；部分真实研究评估参与外层能力校准，不是所有评测都与生成器选择无关。\n\n**最终评测：**包含 BrowseComp、xbench-DeepSearch、SWE-Bench Pro、SciCode-Verified（Sub/Main）、FrontierScience-Research（FS-R）、HLE、BioMysteryBench（HS/HD）、MLE-Bench Lite、PaperBench Code-Dev，共九套 benchmark/十一行结果；FS-R 用 avg@30、xbench 用 avg@5，次数不是题量。逐套合成数据量及重叠审计本轮未完整核实，保留详细笔记的协议差异。",
"sections": "数据合成循环；§3 评估协议与主表",
"source": "https://endlessfrontier.tech/assets/paper.pdf",
"version": "2606.30616,",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "9c3f4366de559df1f32392928a029c299c63486b21752cbb6ea7f7aaab310f79",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "最终由 BigBang-V1 答题，它从 Qwen3.6-35B-A3B 进行后训练得到。生成训练题的 agent 和最终受训模型是两种角色。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "生成角色 生成题目及解答，Critic 检查可验证性与质量，Meta-Critic 校准评价。技术报告没有给出这三个数据生产角色各自的模型型号；已明确的 Qwen3.6-35B-A3B 是受训基础模型，不能据此填成三个角色的型号。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务合成代码、评价标准、训练数据分布，以及接受后训练的模型参数。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "先通过计算、模拟、工具或形式检查验证合成任务，再观察用这些任务训练出的模型是否在下游评测中提升；题目能验证与题目有训练价值分开判断。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "数据合成/研究框架能生成可验证问题、运行候选模型做校准，并修改合成程序。最终模型多数任务接 search、visit、code_exec，专属评测基准用各自运行框架；不能把合成器和最终执行器混写成一个 task agent。",
"sources": [
{
"label": "数据合成循环；§3 评估协议与主表",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "生成者编写并运行合成任务代码，记录失败和结论；批评者判断这些任务是否具有训练价值。再训练候选模型，用真实下游任务表现校准任务生产过程。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "生成角色合成并求解可验证科学任务，Critic筛查后对Qwen3.6-35B-A3B做后训练；不是直接拿某个固定评测基准做统一训练集。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Critic评正确性、难度、可验证性及训练价值；真实研究任务表现用于校准Critic并调整下一批合成数据。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "BrowseComp、xbench-DeepSearch、SWE-bench Pro、SciCode-Verified（Sub/Main）、FrontierScience-Research、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、BioMysteryBench（HS/HD）、MLE-Bench Lite、PaperBench Code-Dev。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "真实任务虽称单独留出的，却会反馈数据合成方向；不能把整个多轮流程解释为最终成绩从未反馈。SciCode经作者审查形成Verified版本。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把任务生成、任务验证和模型学习连起来，并根据学习后的效果调整任务供应；变化涉及训练材料及模型能力。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://endlessfrontier.tech/assets/paper.pdf",
"version": "",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "这是 data/model co-evolution，不是 deployment harness evolution；但 feedback 设计（verifier + downstream outcome）很值得借鉴。",
"feedbackCases": [
{
"label": "合成任务的训练反馈",
"data": "生成角色 构造可验证的科学任务，供 Qwen3.6-35B-A3B 后训练；不是一套固定现成训练集。",
"scoring": "Critic 检查正确性、难度、可验证性和训练价值，任务可使用形式证明、计算、模拟或工具结果验证；验证方式随合成任务变化。",
"visible": "经检验的任务、解答与质量判断进入训练数据，后续真实研究表现用于校准下一轮构题。",
"use": "改数据生成及批评机制，再训练任务模型；原文的多种验证手段不是一份统一的逐题奖励函数。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"judgment": "Critic 评任务质量，配合形式证明／计算／模拟等任务验证；非统一判分器"
},
{
"label": "搜索 / 问答 / 科研评估",
"data": "BrowseComp、xbench-DeepSearch、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、FrontierScience-Research、BioMysteryBench（HS/HD）。",
"scoring": "按各基准问题的参考答案或科学任务评分细则评价；这些属于下游成绩，不能直接当作全部合成训练样本的奖励来源。",
"visible": "汇总下游问答/研究表现。",
"use": "用于观察合成数据训练的迁移价值；各基准细则不同，不能仅写“环境奖励”。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"judgment": "下游基准的答案或评分细则；本文未逐项披露规则／模型比对配置"
},
{
"label": "代码执行与机器学习评估",
"data": "SWE-bench Pro、SciCode-Verified（Sub/Main）、MLE-Bench Lite。",
"scoring": "代码任务以执行验证和测试为依据，ML 任务以真实运行后的任务指标为依据。",
"visible": "代码验收或实验运行成绩。",
"use": "评估训练后模型的研究执行能力。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"judgment": "代码测试／实际训练后的数值指标"
},
{
"label": "PaperBench Code-Dev",
"data": "根据研究论文构建复现代码仓库。",
"scoring": "gpt-5.5 按层级评分细则树评价提交代码；该 Code-Dev 设置不要求完整执行论文全部实验。",
"visible": "层级要求的满足分数。",
"use": "评估复现代码产物；不能与实际复现实验成功率等同。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"judgment": "gpt-5.5 按分层评分细则审查代码"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1,
2,
3
],
"evolution": "生成角色合成并求解可验证科学任务，Critic筛查后对Qwen3.6-35B-A3B做后训练；不是直接拿某个固定评测基准做统一训练集。",
"selection": "Critic评正确性、难度、可验证性及训练价值；真实研究任务表现用于校准Critic并调整下一批合成数据。",
"evaluation": "BrowseComp、xbench-DeepSearch、SWE-bench Pro、SciCode-Verified（Sub/Main）、FrontierScience-Research、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、BioMysteryBench（HS/HD）、MLE-Bench Lite、PaperBench Code-Dev。",
"isolation": "真实任务虽称单独留出的，却会反馈数据合成方向；不能把整个多轮流程解释为最终成绩从未反馈。SciCode经作者审查形成Verified版本。",
"roles": {
"executor": {
"value": "最终由 BigBang-V1 答题，它从 Qwen3.6-35B-A3B 进行后训练得到。生成训练题的 agent 和最终受训模型是两种角色。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
},
"modifier": {
"value": "生成角色 生成题目及解答，Critic 检查可验证性与质量，Meta-Critic 校准评价。技术报告没有给出这三个数据生产角色各自的模型型号；已明确的 Qwen3.6-35B-A3B 是受训基础模型，不能据此填成三个角色的型号。",
"sources": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
},
"seed": {
"value": "数据合成/研究框架能生成可验证问题、运行候选模型做校准，并修改合成程序。最终模型多数任务接 search、visit、code_exec，专属评测基准用各自运行框架；不能把合成器和最终执行器混写成一个 task agent。",
"sources": [
{
"label": "数据合成循环；§3 评估协议与主表",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"selection": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"evaluation": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"isolation": [
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "当模型接近人类专家水平，训练任务仍受人类知识和出题能力限制，难持续提供足够困难的新挑战。单纯生成更多合成题也不够：固定生成器会达到难度上限，缺乏可靠验证又无法信任学习信号；作者因此关注能不断产生、验证并提高难度的前沿任务来源。",
"sources": [
{
"label": "§1 Introduction（第 1–3 页）",
"url": "https://endlessfrontier.tech/assets/paper.pdf#page=1"
}
]
},
{
"key": "position",
"value": "研究前沿科学任务的供应能否摆脱对人工整理的持续依赖，并由系统自主扩展可验证的学习任务，推动模型能力继续增长。",
"sources": [
{
"label": "摘要",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
},
{
"key": "conclusion",
"value": "BigBang 相对基础模型在科研及通用任务上提升；特色是连任务生成与评价策略也随经验调整。",
"sources": [
{
"label": "摘要",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
}
],
"fields": {
"object": "任务合成代码、评价标准、训练数据分布，以及接受后训练的模型参数。",
"verdict": "先通过计算、模拟、工具或形式检查验证合成任务，再观察用这些任务训练出的模型是否在下游评测中提升；题目能验证与题目有训练价值分开判断。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"curriculum"
]
},
{
"id": "2607.05297",
"title": "MetaSkill-Evolve: Recursive Self-Improvement of LLM Agents via Two-Timescale Meta-Skill Evolution",
"url": "https://arxiv.org/abs/2607.05297",
"date": "2026-07-06",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"GoldLabel",
"HeldOut",
"Improver",
"M2",
"SameModel",
"Skill"
],
"fields": {
"本质定位": "每个 branch 同时带 task skill s 与 meta-skill m=(Analyzer ψ, Retriever σ, Allocator α, Proposer π, Evolver ε)；task skill 快速进化，meta-skill 慢速进化，而且同一 improvement pipeline 被用来改它自己。",
"什么在变": "task skill + 五部分 meta-skill；共享 frozen backbone weights 不变。",
"谁来改 / 谁执行": "**改**：同一五角色 improvement pipeline；slow loop 让该 pipeline 对自己的 meta-skill 做更新。
**执行**：Analyzer/Retriever/Allocator/Proposer/Evolver 全部共享一个 frozen backbone；task solver使用当前 skill。",
"基础 harness": "固定五角色 wiring + 当前 task skill/meta-skill files。",
"Feedback": "execution traces + task outcome/labels/verifier；meta-skill 的价值通过后续 task-skill evolution 的 downstream performance体现。",
"Evolution → Eval": "OfficeQA、SealQA、ALFWorld 上 evolution data 与 held-out test 分开；test 不参与 branch evolution。",
"Meta-depth": "M2：improver representation 自身进化；但 outer task objective、五角色 wiring、selection/eval protocol仍固定。",
"相对之前真正新增什么": "这是和我们 meta-depth 最直接的工作之一：以前 skill evolution 只改“做什么”，这里把 **“怎么分析、取经验、分配 credit、提案、合并 skill”** 五个 improver component 也变成 branch-local persistent editable state。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "每个 branch 同时带 task skill s 与 meta-skill m=(Analyzer ψ, Retriever σ, Allocator α, Proposer π, Evolver ε)；task skill 快速进化，meta-skill 慢速进化，而且同一 improvement pipeline 被用来改它自己。 **相对前序：** 这是和我们 meta-depth 最直接的工作之一：以前 skill evolution 只改“做什么”，这里把 **“怎么分析、取经验、分配 credit、提案、合并 skill”** 五个 improver component 也变成 branch-local persistent editable state。"
},
{
"label": "什么在变",
"text": "task skill + 五部分 meta-skill；共享 frozen backbone weights 不变。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 同一五角色 improvement pipeline；slow loop 让该 pipeline 对自己的 meta-skill 做更新。 **执行：** Analyzer/Retriever/Allocator/Proposer/Evolver 全部共享一个 frozen backbone；task solver使用当前 skill。"
},
{
"label": "基础 harness",
"text": "固定五角色 wiring + 当前 task skill/meta-skill files。"
},
{
"label": "Feedback",
"text": "execution traces + task outcome/labels/verifier；meta-skill 的价值通过后续 task-skill evolution 的 downstream performance体现。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "OfficeQA、SealQA、ALFWorld 上 evolution data 与 held-out test 分开；test 不参与 branch evolution。 **Meta-depth：** M2：improver representation 自身进化；但 outer task objective、五角色 wiring、selection/eval protocol仍固定。"
},
{
"label": "主要结果",
"text": "held-out test 相对 raw backbone +23.54 / +16.09 / +1.92 pt；也优于 static-skill 与 single-level evolution。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "不能说成完全 self-referential RSI：最外层 pipeline topology和 objective没变；meta-loop收益也会和更大 search/state capacity混在一起。 **对我们：** 这篇应作为我们研究“meta-depth 是否真的带来额外 improvement”的核心比较对象。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 63,
"fields": {
"论文": "**[MetaSkill-Evolve](https://arxiv.org/abs/2607.05297)**",
"为什么仍应视为 Core / 强代表": "明确 fast skill + slow meta-skill，两层 improvement procedure evolution；meta-depth 讨论的关键参照。"
}
},
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 312,
"fields": {
"优先级": "**C**",
"时间": "2026-07-06",
"论文": "[MetaSkill-Evolve: Recursive Self-Improvement of LLM Agents via Two-Timescale Meta-Skill Evolution](https://arxiv.org/abs/2607.05297)",
"本质定位": "每个 branch 同时带 task skill s 与 meta-skill m=(Analyzer ψ, Retriever σ, Allocator α, Proposer π, Evolver ε)；task skill 快速进化，meta-skill 慢速进化，而且同一 improvement pipeline 被用来改它自己。",
"什么在变": "task skill + 五部分 meta-skill；共享 frozen backbone weights 不变。",
"谁来改 / 谁执行": "**改**：同一五角色 improvement pipeline；slow loop 让该 pipeline 对自己的 meta-skill 做更新。
**执行**：Analyzer/Retriever/Allocator/Proposer/Evolver 全部共享一个 frozen backbone；task solver使用当前 skill。",
"基础 harness": "固定五角色 wiring + 当前 task skill/meta-skill files。",
"Feedback": "execution traces + task outcome/labels/verifier；meta-skill 的价值通过后续 task-skill evolution 的 downstream performance体现。",
"Evolution → Eval": "OfficeQA、SealQA、ALFWorld 上 evolution data 与 held-out test 分开；test 不参与 branch evolution。",
"Meta-depth": "M2：improver representation 自身进化；但 outer task objective、五角色 wiring、selection/eval protocol仍固定。",
"相对之前真正新增什么": "这是和我们 meta-depth 最直接的工作之一：以前 skill evolution 只改“做什么”，这里把 **“怎么分析、取经验、分配 credit、提案、合并 skill”** 五个 improver component 也变成 branch-local persistent editable state。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2026",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Skill",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "每个 branch 同时携带 task skill 和五个 meta-skill 文件。快循环改任务技能；慢循环用同一套流水线改 Analyzer、Retriever、Allocator、Proposer、Evolver 的行为说明。",
"novelty": "维护两类文字说明：一类指导做任务，另一类指导怎样改前一类说明；前者频繁更新，后者隔若干轮总结更新，外层角色流程固定。",
"object": "任务技能和指导五个改进角色的元技能；基础模型、角色连接及评测规则固定。",
"executor": "冻结的 Gemma-4 31B 读取当前任务技能并解题。",
"modifier": "五个角色共用 Gemma-4 31B；快速循环改任务技能，慢速循环改指导这些角色的指导如何改进其他技能的说明文件，模型权重保持不变。",
"roleContext": "**改**：同一五角色 improvement pipeline；slow loop 让该 pipeline 对自己的 meta-skill 做更新。
**执行**：Analyzer/Retriever/Allocator/Proposer/Evolver 全部共享一个 frozen backbone；task solver使用当前 skill。",
"seed": "固定五角色流水线维护两类 Markdown 说明：一类指导完成任务，另一类指导如何修改前一类技能。快循环修改任务技能，慢循环整理改进经验并跨分支共享；角色和运行程序保持固定。",
"fixed": "无权重训练；角色数量、连接方式、任务目标、selection/eval protocol 固定。",
"verdict": "OfficeQA、SealQA 和 ALFWorld 各按自己的答案或环境判定给出结果；训练题的失败轨迹用于提出改动，验证题的成绩用于选择技能版本。",
"diagnosis": "Analyzer 找失败，Retriever 取分支经验，Allocator 分配 credit。",
"update": "快循环改 task skill；每隔 H 次触发慢循环修改 meta-skills。",
"acceptance": "training failure mining；validation 为 child 打分与选择 best skill。",
"experiments": [
{
"name": "OfficeQA / SealQA / ALFWorld",
"evolve": "按 category 分层的 train：failure mining",
"selection": "disjoint validation：child scoring / best selection",
"test": "独立 test，由 separate benchmark-mode pass 运行",
"isolation": "独立测试",
"note": ""
}
],
"takeaway": "不能说成完全 self-referential RSI：最外层 pipeline topology和 objective没变；meta-loop收益也会和更大 search/state capacity混在一起。 **对我们：** 这篇应作为我们研究“meta-depth 是否真的带来额外 improvement”的核心比较对象。",
"result": "held-out test 相对 raw backbone +23.54 / +16.09 / +1.92 pt；也优于 static-skill 与 single-level evolution。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2607.05297v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：OfficeQA、SealQA、ALFWorld（通过文字动作完成家居物体操作的交互环境），按任务类别分层划分；训练集 用于采失败轨迹，不更新模型参数。\n\n调试 / 选版本数据：独立 验证集 用于评分子技能及选择最好版本；快循环改任务技能，慢循环根据近期改进收益修改五类指导如何改进其他技能的说明。\n\n最终测试数据：冻结选出的技能，以单独评测基准模式运行未被演化循环看到的 测试集。\n\n数据隔离与证据边界：任务级 训练／验证／测试 互斥；比例消融使用共同 测试集 交集重新计分，不能与主表绝对分数混比。",
"cycle": "分析角色定位失败，检索角色寻找分支经验，分配角色判断各部分贡献。较频繁地修改任务技能，每隔若干轮再修改指导这些角色的元技能；训练题用于找错误，验证题用于评价新技能并选版本。",
"train": "OfficeQA、SealQA、ALFWorld（通过文字动作完成家居物体操作的交互环境），按任务类别分层划分；训练集 用于采失败轨迹，不更新模型参数。",
"debug": "独立 验证集 用于评分子技能及选择最好版本；快循环改任务技能，慢循环根据近期改进收益修改五类指导如何改进其他技能的说明。",
"test": "冻结选出的技能，以单独评测基准模式运行未被演化循环看到的 测试集。",
"isolation": "任务级 训练／验证／测试 互斥；比例消融使用共同 测试集 交集重新计分，不能与主表绝对分数混比。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "固定五角色流水线操作任务 skill 和指导改技能的 meta-skill Markdown；快循环改任务技能，慢循环更新改进经验并跨分支共享。角色和运行程序不一起任意重写。",
"protocol": "**数据：**OfficeQA、SealQA、ALFWorld；按 category 分层分成三个互斥部分。training 供失败挖掘，validation 供子候选评分和最佳技能选择，test 只由独立 benchmark-mode 最终运行。\n\n**划分实验：**ALFWorld 分析改变 train 比例 0.05/0.10/0.15 与 validation 0.10→0.25 的影响，不能给全篇固定一个比例。各主设置绝对数量本轮待核实。",
"sections": "实验设置、划分敏感性 Table 5",
"source": "https://arxiv.org/abs/2607.05297",
"version": "2607.05297v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "fae5466f1181fd777e462a7cfdce67981a06159da9062d520d37a863f1fec341",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "冻结的 Gemma-4 31B 读取当前任务技能并解题。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "五个角色共用 Gemma-4 31B；快速循环改任务技能，慢速循环改指导这些角色的指导如何改进其他技能的说明文件，模型权重保持不变。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务技能和指导五个改进角色的元技能；基础模型、角色连接及评测规则固定。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "OfficeQA、SealQA 和 ALFWorld 各按自己的答案或环境判定给出结果；训练题的失败轨迹用于提出改动，验证题的成绩用于选择技能版本。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定五角色流水线维护两类 Markdown 说明：一类指导完成任务，另一类指导如何修改前一类技能。快循环修改任务技能，慢循环整理改进经验并跨分支共享；角色和运行程序保持固定。",
"sources": [
{
"label": "实验设置、划分敏感性 Table 5",
"url": "https://arxiv.org/abs/2607.05297"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "分析角色定位失败，检索角色寻找分支经验，分配角色判断各部分贡献。较频繁地修改任务技能，每隔若干轮再修改指导这些角色的元技能；训练题用于找错误，验证题用于评价新技能并选版本。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "OfficeQA、SealQA、ALFWorld（通过文字动作完成家居物体操作的交互环境），按任务类别分层划分；训练集 用于采失败轨迹，不更新模型参数。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "独立 验证集 用于评分子技能及选择最好版本；快循环改任务技能，慢循环根据近期改进收益修改五类指导如何改进其他技能的说明。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "冻结选出的技能，以单独评测基准模式运行未被演化循环看到的 测试集。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "任务级 训练／验证／测试 互斥；比例消融使用共同 测试集 交集重新计分，不能与主表绝对分数混比。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "维护两类文字说明：一类指导做任务，另一类指导怎样改前一类说明；前者频繁更新，后者隔若干轮总结更新，外层角色流程固定。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.05297v1",
"version": "2607.05297v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把“怎么改进”的持久表示也纳入可编辑对象；但五角色 wiring 与外层 selection 仍固定。",
"feedbackCases": [
{
"label": "OfficeQA / SealQA",
"data": "各基准按任务类别分层分成互斥训练、验证、测试集。",
"scoring": "使用问答任务的正确性分数衡量技能表现；这篇论文报告准确率，没有单独展开两套问答评分器的实现或统一指定评审模型。",
"visible": "训练执行中挑选得分最低的失败例，分析其记录；候选技能另在验证集评分。",
"use": "快循环只保留验证收益为正的技能修改；慢循环依据近期修改收益调整“如何改技能”的说明。冻结后才在测试集报告结果。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"judgment": "问答参考答案判分；本文未明确规则／模型比对实现"
},
{
"label": "ALFWorld",
"data": "各基准按任务类别分层分成互斥训练、验证、测试集。",
"scoring": "按环境任务目标是否完成给成功/失败。",
"visible": "训练执行中挑选得分最低的失败例，分析其记录；候选技能另在验证集评分。",
"use": "快循环只保留验证收益为正的技能修改；慢循环依据近期修改收益调整“如何改技能”的说明。冻结后才在测试集报告结果。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"judgment": "ALFWorld 环境程序检查目标完成"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "OfficeQA、SealQA、ALFWorld（通过文字动作完成家居物体操作的交互环境），按任务类别分层划分；训练集 用于采失败轨迹，不更新模型参数。",
"selection": "独立 验证集 用于评分子技能及选择最好版本；快循环改任务技能，慢循环根据近期改进收益修改五类指导如何改进其他技能的说明。",
"evaluation": "冻结选出的技能，以单独评测基准模式运行未被演化循环看到的 测试集。",
"isolation": "任务级 训练／验证／测试 互斥；比例消融使用共同 测试集 交集重新计分，不能与主表绝对分数混比。",
"roles": {
"executor": {
"value": "冻结的 Gemma-4 31B 读取当前任务技能并解题。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
]
},
"modifier": {
"value": "五个角色共用 Gemma-4 31B；快速循环改任务技能，慢速循环改指导这些角色的指导如何改进其他技能的说明文件，模型权重保持不变。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
]
},
"seed": {
"value": "固定五角色流水线维护两类 Markdown 说明：一类指导完成任务，另一类指导如何修改前一类技能。快循环修改任务技能，慢循环整理改进经验并跨分支共享；角色和运行程序保持固定。",
"sources": [
{
"label": "实验设置、划分敏感性 Table 5",
"url": "https://arxiv.org/abs/2607.05297"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05297#S4.SS1.SSS0.Px1"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7"
},
{
"label": "附录G",
"url": "https://arxiv.org/html/2607.05297#A7.SS0.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有技能进化会改任务技能，却固定分析失败和提出修补的流程；如果错误来自诊断方式，后续仍会重复提出同类无效修改。只按当前任务分数选分支，也可能丢掉更能产生好后代的改进策略，因此作者把怎样修改技能以及这种修改策略的未来收益一起纳入研究。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.05297#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否在改善任务技能的同时，学会更有效地分析经验和改进技能，从而突破固定改进流程的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05297"
}
]
},
{
"key": "conclusion",
"value": "共享同一个冻结模型时，在三种 agent 基准上优于静态技能和单层进化对照。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.05297"
}
]
}
],
"fields": {
"object": "任务技能和指导五个改进角色的元技能；基础模型、角色连接及评测规则固定。",
"verdict": "OfficeQA、SealQA 和 ALFWorld 各按自己的答案或环境判定给出结果；训练题的失败轨迹用于提出改动，验证题的成绩用于选择技能版本。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2608.15071",
"title": "Evo-Harness: Context-to-Harness Skill Compilation for Self-Evolving Agents",
"url": "https://arxiv.org/abs/2608.15071",
"date": "2026-08-15",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HeldOut",
"M1",
"Prequential",
"SeparateEvolver",
"Skill",
"org:uiuc"
],
"fields": {
"本质定位": "sequential one-shot tasks：失败后把完整 trajectory + grounded feedback 编译成 General/Topic natural-language skills，更新 external skill harness，只影响未来任务。",
"谁来改 → 谁执行；基础 harness": "separate evolver → frozen solver + skill harness。",
"Feedback / evidence": "environment/verifier/rubric feedback；有 self-feedback ablation。",
"什么在变": "natural-language skill harness。",
"谁来改 / 谁执行": "**改**：默认 Claude Opus 4.6 reflector/evolver；retrieval Sonnet 4.5。
**执行**：默认 Claude Opus 4.6 solver + current skill harness。",
"基础 harness": "无/已有 skill harness；模型冻结。",
"Feedback": "environment/verifier/unit-test/rubric feedback + full trajectory。",
"Evolution → Eval": "5 benchmark sequential prequential；另有 SWE train→freeze→unseen test。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 earlier skill learning 的关键是系统比较 feedback grounding、skill granularity、solver/evolver pairing，并给 SWE train→freeze→test transfer。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "sequential one-shot tasks：失败后把完整 trajectory + grounded feedback 编译成 General/Topic natural-language skills，更新 external skill harness，只影响未来任务。 **相对前序：** 相对 earlier skill learning 的关键是系统比较 feedback grounding、skill granularity、solver/evolver pairing，并给 SWE train→freeze→test transfer。"
},
{
"label": "什么在变",
"text": "natural-language skill harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 默认 Claude Opus 4.6 reflector/evolver；retrieval Sonnet 4.5。 **执行：** 默认 Claude Opus 4.6 solver + current skill harness。"
},
{
"label": "基础 harness",
"text": "无/已有 skill harness；模型冻结。"
},
{
"label": "Feedback",
"text": "environment/verifier/unit-test/rubric feedback + full trajectory。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "5 benchmark sequential prequential；另有 SWE train→freeze→unseen test。 **Meta-depth：** M1。"
},
{
"label": "主要结果",
"text": "5 benchmarks 全提升；纯 self-generated feedback 在 CL/SWE 反而退化；SWE frozen skill 68.8→73.4。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "General/Topic 都是 bench 内，不是跨 benchmark universal skill；主 protocol 仍用同 benchmark stream。 **对我们：** 和我们 feedback×model×editable-space 分析最直接，尤其验证了 self-feedback contamination。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 64,
"fields": {
"论文": "**[Evo-Harness](https://arxiv.org/abs/2608.15071)**",
"为什么仍应视为 Core / 强代表": "skill-level harness evolution + grounded feedback vs self-feedback 的直接证据，对我们当前问题非常相关。"
}
},
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 161,
"fields": {
"时间": "2026-08-15",
"论文": "[Evo-Harness](https://arxiv.org/abs/2608.15071)",
"级别": "**C**",
"我们的定位：什么在变、真正新点": "**H-Skill**。one-shot task 后把 trajectory + grounded feedback 编译为 reusable natural-language skills；它最有价值的证据是 **self-feedback 会退化，grounded feedback 才稳定**。",
"谁来改 → 谁执行；基础 harness": "separate evolver → frozen solver + skill harness。",
"Feedback / evidence": "environment/verifier/rubric feedback；有 self-feedback ablation。",
"标签": "`#Skill #SeparateEvolver #ExecutableVerifier #Prequential #HeldOut #M1`"
}
},
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 315,
"fields": {
"优先级": "**C**",
"时间": "2026-08-15",
"论文": "[Evo-Harness: Context-to-Harness Skill Compilation for Self-Evolving Agents](https://arxiv.org/abs/2608.15071)",
"本质定位": "sequential one-shot tasks：失败后把完整 trajectory + grounded feedback 编译成 General/Topic natural-language skills，更新 external skill harness，只影响未来任务。",
"什么在变": "natural-language skill harness。",
"谁来改 / 谁执行": "**改**：默认 Claude Opus 4.6 reflector/evolver；retrieval Sonnet 4.5。
**执行**：默认 Claude Opus 4.6 solver + current skill harness。",
"基础 harness": "无/已有 skill harness；模型冻结。",
"Feedback": "environment/verifier/unit-test/rubric feedback + full trajectory。",
"Evolution → Eval": "5 benchmark sequential prequential；另有 SWE train→freeze→unseen test。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 earlier skill learning 的关键是系统比较 feedback grounding、skill granularity、solver/evolver pairing，并给 SWE train→freeze→test transfer。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Prequential + held-out",
"protocolBasis": "五个 benchmark 采用 sequential prequential；另有 SWE train → freeze → unseen test。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "每个任务只尝试一次，执行结束后把轨迹和外部反馈编译成 General / Topic 自然语言技能，供后续任务检索使用。",
"novelty": "把已完成任务的轨迹提炼成后续可读取的文字操作说明，并比较反馈来源、说明粒度及修改模型的影响；执行程序本身不被重写。",
"object": "供后续任务读取的文字技能库：新增、合并或修改通用经验和任务操作说明。模型参数与底层执行代码固定。",
"executor": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"modifier": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"roleContext": "**改**：默认 Claude Opus 4.6 reflector/evolver；retrieval Sonnet 4.5。
**执行**：默认 Claude Opus 4.6 solver + current skill harness。",
"seed": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"fixed": "模型权重、技能编译/检索流程固定。",
"verdict": "反馈来自实际任务环境、代码测试结果或逐项评分细则。论文对比模型自行评价、仅告知通过或失败，以及提供更完整诊断信息的反馈。",
"diagnosis": "结合本次完整轨迹与 grounded outcome 提炼可复用经验。",
"update": "修改 General / Topic skills，只影响未来任务。",
"acceptance": "按顺序继承技能；self-feedback 是否可靠由消融检验，不等于每次更新都有 held-out gate。",
"experiments": [
{
"name": "在线五个 benchmark",
"evolve": "顺序单次任务形成经验",
"selection": "后续任务使用当前 skills",
"test": "任务流中的后续表现",
"isolation": "Prequential",
"note": ""
},
{
"name": "SWE transfer",
"evolve": "training split 上由 Sonnet 4.5 进化 skills",
"selection": "冻结已学技能",
"test": "Opus 4.7 在 test split 使用；另有 online-updated 对照",
"isolation": "独立测试",
"note": ""
}
],
"takeaway": "self-generated feedback 在两组消融中低于 No-Evolve；外部判断和“怎么把判断写进技能”都需要单独研究。",
"result": "5 benchmarks 全提升；纯 self-generated feedback 在 CL/SWE 反而退化；SWE frozen skill 68.8→73.4。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.15071v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：主实验在线积累任务经验并编译成技能；另有 Sonnet 4.5 在 训练 split 上构建技能的独立迁移实验。\n\n调试 / 选版本数据：当前题作答后收到评测基准反馈，生成供后续任务使用的技能；对比细诊断与简化通过/失败反馈。\n\n最终测试数据：在线主表：WebArena-Infinity 80、TB2 89、SWE-bench Lite 300、CL-Bench 1,899、τ-bench 165。迁移实验另用 Opus 4.7 在 测试集 执行冻结技能。\n\n数据隔离与证据边界：在线更新和冻结迁移是不同协议；前者允许从已做题学习，后者才检验预先学到技能的跨题/模型迁移。",
"cycle": "任务失败或收到负面反馈后，求解模型先从执行记录中提炼经验；每批任务结束，修改者对照已有说明，选择新增、合并、改写或跳过。修改后的说明供未来任务使用；这套流程并不要求每次修改都先通过独立验证集的检查。",
"train": "主实验在线积累任务经验并编译成技能；另有 Sonnet 4.5 在 训练 split 上构建技能的独立迁移实验。",
"debug": "当前题作答后收到评测基准反馈，生成供后续任务使用的技能；对比细诊断与简化通过/失败反馈。",
"test": "在线主表：WebArena-Infinity 80、TB2 89、SWE-bench Lite 300、CL-Bench 1,899、τ-bench 165。迁移实验另用 Opus 4.7 在 测试集 执行冻结技能。",
"isolation": "在线更新和冻结迁移是不同协议；前者允许从已做题学习，后者才检验预先学到技能的跨题/模型迁移。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "提出 **online harness learning**：solver 权重冻结，任务按 sequential stream 到来；每个 task 只执行一次，失败后把 trajectory + grounded feedback 编译成可复用 natural-language skills，更新外部 skill harness，只影响后续任务。",
"index": {
"date": "2026-08-15",
"paper": "[Evo-Harness: Context-to-Harness Skill Compilation for Self-Evolving Agents](https://arxiv.org/abs/2608.15071)",
"executor": "默认 **Claude Opus 4.6 + 当前 natural-language skill harness**；skill retrieval 用 Claude Sonnet 4.5",
"modifier": "默认 **Claude Opus 4.6 reflector/evolver**：失败后读完整 trajectory + grounded feedback，生成 candidate skill，再做 ADD / MERGE / REVISE / SKIP",
"feedback": "environment / verifier / test / rubric feedback + 完整 trajectory；比较 self-generated、minimal grounded、standard grounded feedback",
"evolve": "各 benchmark 以 **sequential online task stream** 运行：WebArena-Infinity 80、TerminalBench-2 89、SWE-bench Lite 300、CL-Bench 1,899、τ-bench 165；每 task one-shot，完成后才成为未来 experience",
"eval": "主结果为同一 benchmark stream 上的 online/prequential evaluation；另有 **SWE train split → freeze skills → unseen test split** held-out transfer",
"tldr": "冻结模型，只进化 **natural-language skill harness**，不是完整 executable harness。**General / Topic skill 都只在单个 benchmark 内形成和使用**：General 是该 bench 内跨 task-type 的通用 procedure，Topic 是该 bench 内 recurring task type/domain/interface 的局部 procedure；**论文没有做跨 benchmark skill sharing/transfer**。Grounded feedback 才可靠，纯 self-feedback 会退化；SWE frozen transfer 68.8→73.4。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 直接保存完整 trajectory / episodic memory 容易混入 task-specific noise；很多经验学习方法偏 offline；同时既有工作常只报最终 gain，没有系统拆解 feedback、skill 粒度、solver/evolver pairing 对 evolution 的影响。 |\n| 本文定位 | 将一次 execution context **compile 成 harness skill**，而不是直接 replay/retrieve 旧 trajectory；从失败中提炼可泛化 procedure，持续更新 external skill harness。 |\n| 核心贡献 | ① General / Topic 两层 skill harness；② sequential one-shot online update；③ grounded feedback vs self-feedback；④ solver/evolver 能力与 transfer 分析；⑤ SWE train→test frozen-skill transfer。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **模型权重完全冻结**；进化的是外部 **natural-language skill harness**。没有修改 tool implementation、middleware、control flow 或 executable code，因此属于 **skill-level harness evolution**，不是 full executable harness evolution。 |\n| 谁来执行 | 默认 **Claude Opus 4.6 + 当前 skill harness**。 |\n| 谁来改 | 默认同为 **Claude Opus 4.6**：Reflector 读取失败 trajectory + feedback 生成 candidate skills；Evolver 做 `ADD / MERGE / REVISE / SKIP`。 |\n| Skill retrieval | 每个 task 前由 **Claude Sonnet 4.5** 从当前 skill library 中选择 relevant skills 注入 solver。 |\n| 更新频率 | 默认每 **16 tasks** 汇总/更新一次；skill 数量有预算限制。 |\n| Feedback | 完整 execution trajectory + environment / executable verifier / unit test / rubric feedback；比较 self-generated、minimal grounded、standard grounded 三种 feedback。 |\n| 成功轨迹是否更新 | 默认主要从 **failure** 中提炼 skill；failure 更直接暴露 capability gap。 |\n| 方法本质 | **online/prequential skill-harness learning**：task $t$ 完成后的经验只能影响 $t+1$ 及之后的任务，不会回到 task $t$ 反复优化。 |\n\n##### General skill 与 Topic skill：必须明确的范围\n\n| 类型 | 论文里的范围 | 例子 / 直觉 |\n|---|---|---|\n| **General skill** | **单个 benchmark 内**跨多类 task 都可复用的通用 procedure / operational pattern | SWE-bench 中通用 debugging、读 failing assertion、targeted test、verification、recovery 流程 |\n| **Topic skill** | **同一个 benchmark 内**针对 recurring task type / domain / interface 的局部 procedure / convention | SWE-bench 中 Django / SymPy 等 repo/domain convention；CL-Bench 中某类 rule system / output schema 的 local manual |\n| **不是 case memory** | Topic skill 也不是某一道题的答案，而是从相似 case 中抽象出的可复用局部知识 | 目标仍是未来 unseen tasks 可用，而不是 replay exact solution |\n\n> **General skill 和 Topic skill 都是在各自 benchmark 内形成和使用。论文没有实验“在 SWE-bench 学到 General skill，再拿到 WebArena / TerminalBench 使用”这种 cross-benchmark skill sharing / transfer。** \n> 因而这里的层级是：`具体 case → Topic skill（同 bench 内局部类型）→ General skill（同 bench 内跨 task-type 共性 procedure）`，不是 `case → benchmark skill → cross-benchmark universal skill`。"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| Benchmark | Tasks | Feedback / verifier | Evolve → Eval protocol | 数据隔离判断 |\n|---|---:|---|---|---|\n| WebArena-Infinity | 80 | programmatic state verifier | sequential stream；每 task one-shot，做完后才用于未来 skill update | online/prequential，不重复优化已做 task |\n| TerminalBench-2 | 89 | Docker / executable verifier | 同上 | 同上 |\n| SWE-bench Lite | 300 | unit tests | 同上 | 同上 |\n| CL-Bench | 1,899 | rubric judge | 同上 | 同上 |\n| τ-bench | 165 | final DB state verifier | 同上 | 同上 |\n| **SWE Train-Split Transfer** | train split → test split | SWE evaluator | train split 上 evolve → **freeze skills** → unseen test split | **真正 held-out transfer** |\n\n> 主表不是传统 `train → freeze → held-out test`，但也不同于 Meta-Harness / AHE 对同一 tasks 反复 search：**每个 task 只做一次，反馈只能影响未来任务。**"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| 5 个 benchmark 全部提升 | No-Evolve→Evo-Harness：CL-Bench **29.54→34.02**；TB2 **62.92→73.03**；SWE Lite **63.67→67.00**；τ-bench **72.73→76.97**；WebArena **72.50→76.25** | 从 sequential failures 中编译 skill 能提高后续任务表现；TB2 gain 最大。 |\n| General / Topic 的有效性依 task 而异 | SWE：General Only **66.67** > Topic Only 64.33；CL：Topic Only **33.70** > General Only 30.28 | SWE 更依赖跨 task debugging procedure；CL 更依赖局部 task-type/rule manuals。 |\n| **纯 self-generated feedback 会退化** | CL：29.54→**27.96**；SWE：63.67→**61.67** | 错误 reflection 会污染 persistent skill harness；self-evolution 不能只靠模型自判。 |\n| Grounded feedback 才可靠，但不是越详细越好 | CL：Standard **34.02** > Minimal 29.86；SWE：Minimal **67.33** ≈/略高于 Standard 67.00 | feedback informativeness 与 over-specificity 存在 trade-off。 |\n| Frozen skill 能泛化到 unseen tasks / 新 solver | SWE：No-Evolve **68.8**；Train-Split Transfer **73.4**；Online Updated **75.0** | 至少在 SWE 上存在真正 held-out procedural transfer。 |\n| Skill benefit 受 solver 能力限制 | Opus solver 可从 same/cross-model evolver skill 获益；Sonnet solver 即使用更强 Opus evolver 的 skill 也会退化 | evolution 同时受 `experience→skill` 与 `skill→behavior realization` 两个瓶颈限制。 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| 是否属于 Harness Evolution | **属于。** solver frozen，external skill harness 随 experience 更新。 |\n| 是否 full harness evolution | **不是。** Editable space 只有 natural-language procedural skills，没有 tools、middleware、workflow/code、sub-agent architecture。 |\n| Main protocol 是否 test-set overfitting | 比重复 search 同一 test tasks 更干净：每 task one-shot、不会根据自己的 reward 回头重做；但整个 benchmark stream 同时承担 evaluation 与 experience source，仍不是标准 frozen held-out protocol。 |\n| **Cross-benchmark 泛化** | **没有验证。** General / Topic skill 都在单个 benchmark 内定义和使用；不能把 “General” 理解成跨 SWE / WebArena / TB2 共用的 universal skill。 |\n| Feedback 结论 | 最有价值的发现之一：grounded verifier/environment feedback 是 evolution work 的关键，self-feedback 甚至会形成负向 cumulative effect。 |\n| Skill contamination | Persistent skill 一旦由错误 reflection 写入，可能长期伤害后续任务；论文结果直接观察到这种 regression。 |\n| 对我们的启发 | 很适合研究 `Task × Model × Feedback × Editable Space`：失败可拆成 **feedback 不可靠、experience compilation 失败、solver 无法利用 skill**。 |"
},
{
"title": "总评",
"body": "**Evo-Harness 是一篇相对干净的 skill-level Harness Self-Evolution 工作：模型冻结、任务 sequential one-shot，失败 trajectory 被 grounded feedback 驱动地编译成 General / Topic natural-language skills，只用于后续任务。它没有进化完整 executable harness，也没有跨 benchmark skill sharing；General 和 Topic 都只是“单个 benchmark 内”不同泛化粒度。论文最重要的结论是：grounded feedback 是 evolution work 的必要条件之一，纯 self-generated feedback 甚至会导致持续退化；同时 SWE train→test 的 frozen-skill transfer（68.8→73.4）说明至少部分 procedural skill 能真正泛化到 unseen tasks。**"
}
],
"caveats": []
},
"systemDataAudit": {
"seed": "冻结 solver 配合外部结构化技能 harness，检索选中的技能后执行任务；每批任务结束才反思并编译技能修改。技能分任务类型和跨任务知识，底层求解模型不训练。",
"protocol": "**在线主实验：**WebArena-Infinity 80、TerminalBench-2 89、SWE-bench Lite 300、CL-Bench 1,899、τ-bench 165，顺序执行，每题先作答再成为未来经验。不能把这些主表数字理解成训练完后冻结测试。\n\n**独立迁移实验：**Sonnet 4.5 在 training split 演化技能，冻结后给 Opus 4.7 在 test 使用；另对比 test stream 继续更新。两个实验分别说明跨题/模型迁移与在线适应。其 CL-Bench 应按该文引用的版本理解，不套用同名 benchmark 的别套统计。",
"sections": "§4.1、§4.5；附录数据设置",
"source": "https://arxiv.org/abs/2608.15071",
"version": "2608.15071v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6aca496c32269ac332acba44c4a94c5fff5a14be8da0a1664524664ad85a294e",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "供后续任务读取的文字技能库：新增、合并或修改通用经验和任务操作说明。模型参数与底层执行代码固定。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "反馈来自实际任务环境、代码测试结果或逐项评分细则。论文对比模型自行评价、仅告知通过或失败，以及提供更完整诊断信息的反馈。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "任务失败或收到负面反馈后，求解模型先从执行记录中提炼经验；每批任务结束，修改者对照已有说明，选择新增、合并、改写或跳过。修改后的说明供未来任务使用；这套流程并不要求每次修改都先通过独立验证集的检查。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "主实验在线积累任务经验并编译成技能；另有 Sonnet 4.5 在 训练 split 上构建技能的独立迁移实验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前题作答后收到评测基准反馈，生成供后续任务使用的技能；对比细诊断与简化通过/失败反馈。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "在线主表：WebArena-Infinity 80、TB2 89、SWE-bench Lite 300、CL-Bench 1,899、τ-bench 165。迁移实验另用 Opus 4.7 在 测试集 执行冻结技能。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "在线更新和冻结迁移是不同协议；前者允许从已做题学习，后者才检验预先学到技能的跨题/模型迁移。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把已完成任务的轨迹提炼成后续可读取的文字操作说明，并比较反馈来源、说明粒度及修改模型的影响；执行程序本身不被重写。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.15071v2",
"version": "2608.15071v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重点是 feedback grounding、技能粒度和 solver/evolver 配对；单看名字会误以为它改完整 harness code，实际主要改外部技能。",
"feedbackCases": [
{
"label": "在线技能进化：WebArena-Infinity",
"data": "80 个邮件及相关网页应用任务",
"scoring": "程序读取最终网页应用状态，检查是否完成要求，例如标签、设置或邮件操作。",
"visible": "当前任务完成后，读取执行记录和该基准可提供的失败信息；模型从失败或负面反馈中提炼候选经验。",
"use": "每批任务后修改文字技能，供后续任务使用；当前任务的反馈不是预先学完技能后的独立测试。",
"sources": [
{
"label": "附录 A：逐基准判分",
"url": "https://arxiv.org/html/2608.15071#A1"
},
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"judgment": "基准程序检查最终网页应用状态"
},
{
"label": "在线技能进化：Terminal-Bench 2",
"data": "89 个终端任务",
"scoring": "任务专属 Docker 容器中的验收脚本检查产物或答案，返回是否通过及可用测试错误。",
"visible": "当前任务完成后，读取执行记录和该基准可提供的失败信息；模型从失败或负面反馈中提炼候选经验。",
"use": "每批任务后修改文字技能，供后续任务使用；当前任务的反馈不是预先学完技能后的独立测试。",
"sources": [
{
"label": "附录 A：逐基准判分",
"url": "https://arxiv.org/html/2608.15071#A1"
},
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"judgment": "基准容器内的可执行测试检查产物"
},
{
"label": "在线技能进化：SWE-bench Lite",
"data": "300 个真实 GitHub 问题",
"scoring": "将代码补丁应用到仓库，运行仓库测试检查问题是否解决。",
"visible": "当前任务完成后，读取执行记录和该基准可提供的失败信息；模型从失败或负面反馈中提炼候选经验。",
"use": "每批任务后修改文字技能，供后续任务使用；当前任务的反馈不是预先学完技能后的独立测试。",
"sources": [
{
"label": "附录 A：逐基准判分",
"url": "https://arxiv.org/html/2608.15071#A1"
},
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"judgment": "仓库测试检查补丁：应修好的测试通过、原本通过的测试保持通过"
},
{
"label": "在线技能进化：CL-Bench",
"data": "1,899 个知识推理、发现/模拟、程序执行和规则应用任务",
"scoring": "模型读取回答和当前题的评分细则，逐项判断是否满足要求；附录 F 明确使用 rubric-based judge pipeline，即按评分细则进行模型评审。本文没有给出统一裁判模型型号或完整逐题细则。",
"visible": "当前任务完成后，读取执行记录和该基准可提供的失败信息；模型从失败或负面反馈中提炼候选经验。",
"use": "每批任务后修改文字技能，供后续任务使用；当前任务的反馈不是预先学完技能后的独立测试。",
"sources": [
{
"label": "附录 A：逐基准判分",
"url": "https://arxiv.org/html/2608.15071#A1"
},
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
},
{
"label": "附录 F：CL-Bench 的 rubric-based judge pipeline",
"url": "https://arxiv.org/html/2608.15071#A6"
}
],
"judgment": "模型按逐题评分细则评审；本文未披露统一裁判型号"
},
{
"label": "在线技能进化：τ-bench",
"data": "165 个航空和零售客服工具任务",
"scoring": "官方工具环境检查任务结束时的数据库状态是否符合目标。",
"visible": "当前任务完成后，读取执行记录和该基准可提供的失败信息；模型从失败或负面反馈中提炼候选经验。",
"use": "每批任务后修改文字技能，供后续任务使用；当前任务的反馈不是预先学完技能后的独立测试。",
"sources": [
{
"label": "附录 A：逐基准判分",
"url": "https://arxiv.org/html/2608.15071#A1"
},
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
],
"judgment": "官方程序比较最终数据库状态与任务目标"
},
{
"label": "反馈消融与冻结迁移",
"data": "反馈消融只在 CL-Bench、SWE-bench Lite；迁移另由 Sonnet 4.5 在训练划分构造技能，再由 Opus 4.7 在测试划分使用。",
"scoring": "消融比较：模型自行判断成功；仅给环境通过/失败；提供可用的错误信息、测试失败与判分输出。",
"visible": "三种反馈的信息量不同；不能把模型自评解释为获得了测试答案。",
"use": "消融检验反馈来源与详细程度；迁移时技能冻结，不用测试反馈继续改技能。",
"sources": [
{
"label": "§4.5–4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6：反馈消融",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
}
],
"judgment": "分别比较模型自评、环境成败信号、环境成败加诊断信息"
}
],
"experiments": [
{
"label": "WebArena-Infinity：在线技能学习",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "80 个网页应用任务形成连续任务流；每批任务结束后获取反馈并修改文字技能，供后续任务使用。",
"selection": "依据当前已完成任务的反馈修订技能。",
"evaluation": "报告同一任务流中逐步执行的成功率；后续任务使用此前积累的技能。",
"isolation": "这是在线“先做题、再从反馈学习”，不是先在全部题上进化后进行一次独立盲测。",
"roles": {
"executor": {
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
},
{
"label": "Terminal-Bench 2：在线技能学习",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "89 个终端任务形成连续任务流；每批任务结束后获取反馈并修改文字技能，供后续任务使用。",
"selection": "依据当前已完成任务的反馈修订技能。",
"evaluation": "报告同一任务流中逐步执行的成功率；后续任务使用此前积累的技能。",
"isolation": "这是在线“先做题、再从反馈学习”，不是先在全部题上进化后进行一次独立盲测。",
"roles": {
"executor": {
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
},
{
"label": "SWE-bench Lite：在线技能学习",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "300 个 GitHub 问题形成连续任务流；每批任务结束后获取反馈并修改文字技能，供后续任务使用。",
"selection": "依据当前已完成任务的反馈修订技能。",
"evaluation": "报告同一任务流中逐步执行的成功率；后续任务使用此前积累的技能。",
"isolation": "这是在线“先做题、再从反馈学习”，不是先在全部题上进化后进行一次独立盲测。",
"roles": {
"executor": {
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
},
{
"label": "CL-Bench：在线技能学习",
"learningCases": [
3
],
"testCases": [
3
],
"evolution": "1,899 个知识推理、发现与规则任务形成连续任务流；每批任务结束后获取反馈并修改文字技能，供后续任务使用。",
"selection": "依据当前已完成任务的反馈修订技能。",
"evaluation": "报告同一任务流中逐步执行的成功率；后续任务使用此前积累的技能。",
"isolation": "这是在线“先做题、再从反馈学习”，不是先在全部题上进化后进行一次独立盲测。",
"roles": {
"executor": {
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
},
{
"label": "τ-bench：在线技能学习",
"learningCases": [
4
],
"testCases": [
4
],
"evolution": "165 个航空与零售客服任务形成连续任务流；每批任务结束后获取反馈并修改文字技能，供后续任务使用。",
"selection": "依据当前已完成任务的反馈修订技能。",
"evaluation": "报告同一任务流中逐步执行的成功率；后续任务使用此前积累的技能。",
"isolation": "这是在线“先做题、再从反馈学习”，不是先在全部题上进化后进行一次独立盲测。",
"roles": {
"executor": {
"value": "默认Claude Opus4.6作为任务求解模型；另评Claude Opus4.7、Opus4.5、Kimi-K2.5和GPT-OSS配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为修改者；任务求解模型和修改者是分开的角色，实验还替换这些角色的模型组合，不能把所有结果算作同一配置。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
},
{
"label": "反馈消融与冻结迁移",
"learningCases": [
5
],
"testCases": [
5
],
"evolution": "反馈消融在 CL-Bench、SWE-bench Lite 比较自评、通过／失败和详细诊断；独立迁移实验由 Sonnet 4.5 在训练划分构造技能。",
"selection": "消融保持相应实验设置并改变反馈；迁移前冻结技能。",
"evaluation": "迁移由 Opus 4.7 在测试划分使用冻结技能。",
"isolation": "迁移实验不根据测试反馈继续改技能，须与在线主表分开解释。",
"roles": {
"executor": {
"value": "反馈消融沿用相应主实验配置；独立冻结迁移由 Opus 4.7 执行测试任务。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"modifier": {
"value": "冻结迁移的技能由 Sonnet 4.5 在训练划分构建，测试时不再修改；反馈消融另比较反馈条件，不能与迁移角色混用。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
}
]
},
"seed": {
"value": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。",
"sources": [
{
"label": "§4.1、§4.5；附录数据设置",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.15071#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "如果失败没有转成可复用教训，agent 会在后续任务中重复犯相似错误。已有方法多离线汇总轨迹，但实际任务连续到来，每次经历又混有题目细节、局部失败和工具噪声，简单检索旧记录不足以指导新任务。作者因此关注怎样从单次经历提炼可用指导，并指出既有研究缺少在这些真实约束下的验证，也未分清哪些设计和反馈真正带来了提升。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.15071#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 面对连续到来的新任务时，能否从已完成的经历中学到可复用经验、改善后续任务表现，并分清反馈来源、经验表示和修改者能力分别如何影响学习效果。",
"sources": [
{
"label": "§1 Introduction · 研究目标与分析范围",
"url": "https://arxiv.org/html/2608.15071#S1"
}
]
},
{
"key": "conclusion",
"value": "在五种真实任务基准上验证在线学习；外部反馈质量重要，单靠自评并不总能带来收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.15071"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
}
]
}
],
"fields": {
"object": "供后续任务读取的文字技能库：新增、合并或修改通用经验和任务操作说明。模型参数与底层执行代码固定。",
"seed": "参数固定的任务求解模型配合外部文字说明库。每次执行前，系统挑选相关说明加入模型上下文；每批任务结束后，再把失败经验整理为新增、合并或修改的说明。这里“技能编译”指整理文字经验，不是把它编译成可执行代码。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.15071"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2608.09096",
"title": "Evo-Bench: Can Language Models Improve Agent Harness?",
"url": "https://arxiv.org/abs/2608.09096",
"date": "2026-08-10",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"BenchmarkScore",
"CrossModel",
"HarnessCode",
"HeldOut",
"M1"
],
"fields": {
"本质定位": "专门测 model intrinsic harness-evolving capability：固定 policy model与minimal CodeAct seed，evolver在validation上长程改 executable harness，freeze后在sealed evaluation测。",
"被测系统 / seed harness / feedback": "9 evolver models；fixed policy + minimal CodeAct seed；160 val 上 evolve。",
"证据边界与关键结论": "**160 val→448 sealed eval**；Search含 BrowseComp/HLE。top gain 可到 +16.6，但 Office难、存在 early saturation；非常接近我们要的 harness-evolution benchmark 骨架。",
"什么在变": "evolver产出的 executable harness；policy model固定。",
"谁来改 / 谁执行": "**改**：9个 frontier/open-weight evolver models；同预算20 iterations/1000 evolver steps/48h。
**执行**：固定 policy model 在 candidate harness 下执行 Search/Office/General tasks。",
"基础 harness": "统一 minimal CodeAct harness + fixed policy model。",
"Feedback": "160 validation tasks上的native scorer/trajectory/evaluation feedback；448 evaluation tasks sealed。",
"Evolution → Eval": "608 harness-sensitive tasks：Search(BrowseComp,HLE) 320，Office(GDPval,APEX-Agents)192，General(Claw-Eval)96；160 val→448 sealed eval。",
"Meta-depth": "Evaluation of M1 harness-evolving capability。",
"相对之前真正新增什么": "相对 HarnessOpt-Bench 更进一步做 **harness-sensitive task construction**：先用完全disjoint auxiliary tasks产生代表性 evolved harnesses，再筛选真正会对 harness变化敏感的任务，并按sensitivity/difficulty分层切分。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "专门测 model intrinsic harness-evolving capability：固定 policy model与minimal CodeAct seed，evolver在validation上长程改 executable harness，freeze后在sealed evaluation测。 **相对前序：** 相对 HarnessOpt-Bench 更进一步做 **harness-sensitive task construction**：先用完全disjoint auxiliary tasks产生代表性 evolved harnesses，再筛选真正会对 harness变化敏感的任务，并按sensitivity/difficulty分层切分。"
},
{
"label": "什么在变",
"text": "evolver产出的 executable harness；policy model固定。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 9个 frontier/open-weight evolver models；同预算20 iterations/1000 evolver steps/48h。 **执行：** 固定 policy model 在 candidate harness 下执行 Search/Office/General tasks。"
},
{
"label": "基础 harness",
"text": "统一 minimal CodeAct harness + fixed policy model。"
},
{
"label": "Feedback",
"text": "160 validation tasks上的native scorer/trajectory/evaluation feedback；448 evaluation tasks sealed。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "608 harness-sensitive tasks：Search(BrowseComp,HLE) 320，Office(GDPval,APEX-Agents)192，General(Claw-Eval)96；160 val→448 sealed eval。 **Meta-depth：** Evaluation of M1 harness-evolving capability。"
},
{
"label": "主要结果",
"text": "所有9个 evolver都提升 seed；best absolute +16.6；Search gain最大、Office最难；常 early saturation后regress；evolved harness跨policy model transfer。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "任务是先按 harness sensitivity筛过的，绝对 gain不能外推所有agent tasks；固定policy设定也不是same-model self-RSI。 **对我们：** 目前和我们最贴的 benchmark：尤其 Search 含 BrowseComp/HLE，可直接研究 Task×Model×Feedback×Editable与peak/regression。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 65,
"fields": {
"论文": "**[Evo-Bench](https://arxiv.org/abs/2608.09096)**",
"为什么仍应视为 Core / 强代表": "当前最接近“统一测试 harness-evolving capability”的 benchmark 骨架；Search 里含 BrowseComp/HLE。"
}
},
{
"section": "A. Harness optimization / autonomous agent development",
"line": 195,
"fields": {
"时间": "2026-08-10",
"论文": "[Evo-Bench](https://arxiv.org/abs/2608.09096)",
"级别": "**C**",
"它真正测什么": "**B-Harness**。测 model intrinsic harness-evolving ability，并先筛选“**真的对 harness 敏感**”的 tasks；Search/Office/General。",
"被测系统 / seed harness / feedback": "9 evolver models；fixed policy + minimal CodeAct seed；160 val 上 evolve。",
"证据边界与关键结论": "**160 val→448 sealed eval**；Search含 BrowseComp/HLE。top gain 可到 +16.6，但 Office难、存在 early saturation；非常接近我们要的 harness-evolution benchmark 骨架。",
"标签": "`#HarnessCode #HeldOut #CrossModel #BenchmarkScore #M1`"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 366,
"fields": {
"优先级": "**C**",
"时间": "2026-08-10",
"论文": "[Evo-Bench: Can Language Models Improve Agent Harness?](https://arxiv.org/abs/2608.09096)",
"本质定位": "专门测 model intrinsic harness-evolving capability：固定 policy model与minimal CodeAct seed，evolver在validation上长程改 executable harness，freeze后在sealed evaluation测。",
"什么在变": "evolver产出的 executable harness；policy model固定。",
"谁来改 / 谁执行": "**改**：9个 frontier/open-weight evolver models；同预算20 iterations/1000 evolver steps/48h。
**执行**：固定 policy model 在 candidate harness 下执行 Search/Office/General tasks。",
"基础 harness": "统一 minimal CodeAct harness + fixed policy model。",
"Feedback": "160 validation tasks上的native scorer/trajectory/evaluation feedback；448 evaluation tasks sealed。",
"Evolution → Eval": "608 harness-sensitive tasks：Search(BrowseComp,HLE) 320，Office(GDPval,APEX-Agents)192，General(Claw-Eval)96；160 val→448 sealed eval。",
"Meta-depth": "Evaluation of M1 harness-evolving capability。",
"相对之前真正新增什么": "相对 HarnessOpt-Bench 更进一步做 **harness-sensitive task construction**：先用完全disjoint auxiliary tasks产生代表性 evolved harnesses，再筛选真正会对 harness变化敏感的任务，并按sensitivity/difficulty分层切分。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"B-Harness",
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "固定 policy model 与 CodeAct seed，让不同 evolver 在同预算下改 harness，再冻结提交到 sealed evaluation。任务构造先检查对 harness 改动是否敏感。",
"novelty": "固定执行条件和预算，分别指定修改模型、执行模型和评分者，测生成的框架能否在未见任务上保持收益。",
"object": "不同修改者生成的可执行 harness；执行模型、初始接口、预算与测试服务固定。",
"executor": "主实验固定 DeepSeek-V4-Flash 执行任务的模型；部分任务由 Qwen3.7-Plus 评分者评分。",
"modifier": "九种修改模型分别独立进化运行框架：GPT-5.6 Sol、Claude Opus 4.8、GLM-5.2、Qwen3.7-Max、MiniMax-M3、DeepSeek-V4-Pro、Kimi K2.7 Code、Qwen3.6-27B、Gemma-4-31B。每次均改同一个 DeepSeek-V4-Flash 执行端，预算为 20 次迭代／1,000 步／48 小时。",
"roleContext": "**改**：9个 frontier/open-weight evolver models；同预算20 iterations/1000 evolver steps/48h。
**执行**：固定 policy model 在 candidate harness 下执行 Search/Office/General tasks。",
"seed": "参数固定的任务执行模型运行同一个最小 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） task agent，修改者可改任务运行代码；构造评测基准时另用多个修改者产生辅助运行框架，筛选对运行框架差异敏感的题。构造阶段与被测进化阶段不混为训练。",
"fixed": "policy model、seed 接口、预算与测试服务固定。",
"verdict": "修改阶段返回验证题分数、逐题结果和执行诊断。问答、办公产物及工具任务分别采用对应判分规则，需要模型评审时统一使用 Qwen3.7-Plus；最终测试不参与修改。",
"diagnosis": "evolver 读取验证轨迹、工具错误与分数，自行决定如何诊断。",
"update": "提交修改后的 executable harness；研究的对象是不同 evolver 能构建出什么。",
"acceptance": "validation 选择 candidate；最终冻结后才访问 sealed evaluation。",
"experiments": [
{
"name": "Search / Office / General",
"evolve": "160 validation tasks 可供 evolve",
"selection": "验证分数与轨迹供 candidate 选择",
"test": "448 sealed evaluation tasks",
"isolation": "独立测试",
"note": "608 tasks；构造敏感性 harness 的 auxiliary data 与五个来源集隔离。"
}
],
"takeaway": "任务是先按 harness sensitivity筛过的，绝对 gain不能外推所有agent tasks；固定policy设定也不是same-model self-RSI。 **对我们：** 目前和我们最贴的 benchmark：尤其 Search 含 BrowseComp/HLE，可直接研究 Task×Model×Feedback×Editable与peak/regression。",
"result": "所有9个 evolver都提升 seed；best absolute +16.6；Search gain最大、Office最难；常 early saturation后regress；evolved harness跨policy model transfer。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.09096v2"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：不训练行为策略权重；修改者用可见 验证集 改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 运行框架。构造评测基准时另用辅助任务产生候选运行框架。\n\n调试 / 选版本数据：160 道 验证集：BrowseComp、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、GDPval、APEX-Agents、Claw-Eval 各 32 题，返回成绩、轨迹和诊断。\n\n最终测试数据：冻结后测 448 道题：BrowseComp/HLE 各 128；GDPval/APEX-Agents/Claw-Eval 各 64。\n\n数据隔离与证据边界：160 与 448 互斥；构造阶段的 auxiliary tasks 不是被测模型的正式训练集。",
"cycle": "修改者读取验证轨迹、工具错误与分数，自行决定如何诊断。提交修改后的 harness（组织模型调用、工具使用和执行步骤的代码）；研究的对象是不同修改者能构建出什么。验证集 选择 候选；最终冻结后才访问进化期间不可访问的最终评测。",
"train": "不训练行为策略权重；修改者用可见 验证集 改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 运行框架。构造评测基准时另用辅助任务产生候选运行框架。",
"debug": "160 道 验证集：BrowseComp、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、GDPval、APEX-Agents、Claw-Eval 各 32 题，返回成绩、轨迹和诊断。",
"test": "冻结后测 448 道题：BrowseComp/HLE 各 128；GDPval/APEX-Agents/Claw-Eval 各 64。",
"isolation": "160 与 448 互斥；构造阶段的 auxiliary tasks 不是被测模型的正式训练集。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "固定政策模型运行同一个最小 CodeAct agent，修改者可改任务运行代码；构造 benchmark 时另用多个修改者产生辅助 harness，筛选对 harness 差异敏感的题。构造阶段与被测进化阶段不混为训练。",
"protocol": "**主评测：**Search/Office/General 三域，共 608 题，160 validation 供演化和选版本，448 sealed evaluation 最终测试，按 2:2:1 聚合。\n\n**构造数据：**另有 320 个 auxiliary tasks 用于生成候选辅助 harness（73 次评估、去重 65 个版本），与主来源隔离。辅助任务不是每个被测模型的进化训练集；主数据五个来源的完整名称本轮待核实。",
"sections": "主评估协议；附录 B.2",
"source": "https://arxiv.org/abs/2608.09096",
"version": "2608.09096v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "99b8eac1927b406f7be15e6f893b8638be5e376b4deece858d508f58fbe10328",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验固定 DeepSeek-V4-Flash 执行任务的模型；部分任务由 Qwen3.7-Plus 评分者评分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "九种修改模型分别独立进化运行框架：GPT-5.6 Sol、Claude Opus 4.8、GLM-5.2、Qwen3.7-Max、MiniMax-M3、DeepSeek-V4-Pro、Kimi K2.7 Code、Qwen3.6-27B、Gemma-4-31B。每次均改同一个 DeepSeek-V4-Flash 执行端，预算为 20 次迭代／1,000 步／48 小时。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.09096#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "不同修改者生成的可执行 harness；执行模型、初始接口、预算与测试服务固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "修改阶段返回验证题分数、逐题结果和执行诊断。问答、办公产物及工具任务分别采用对应判分规则，需要模型评审时统一使用 Qwen3.7-Plus；最终测试不参与修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
},
{
"label": "表 1：各基准评分器",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "§5.1.1：统一评审模型",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "参数固定的任务执行模型运行同一个最小 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） task agent，修改者可改任务运行代码；构造评测基准时另用多个修改者产生辅助运行框架，筛选对运行框架差异敏感的题。构造阶段与被测进化阶段不混为训练。",
"sources": [
{
"label": "主评估协议；附录 B.2",
"url": "https://arxiv.org/abs/2608.09096"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "修改者读取验证轨迹、工具错误与分数，自行决定如何诊断。提交修改后的 harness（组织模型调用、工具使用和执行步骤的代码）；研究的对象是不同修改者能构建出什么。验证集 选择 候选；最终冻结后才访问进化期间不可访问的最终评测。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训练行为策略权重；修改者用可见 验证集 改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 运行框架。构造评测基准时另用辅助任务产生候选运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "160 道 验证集：BrowseComp、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、GDPval、APEX-Agents、Claw-Eval 各 32 题，返回成绩、轨迹和诊断。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "冻结后测 448 道题：BrowseComp/HLE 各 128；GDPval/APEX-Agents/Claw-Eval 各 64。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "160 与 448 互斥；构造阶段的 auxiliary tasks 不是被测模型的正式训练集。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "固定执行条件和预算，分别指定修改模型、执行模型和评分者，测生成的框架能否在未见任务上保持收益。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.09096v2",
"version": "2608.09096v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "比较的是 harness-evolving capability；benchmark 本身不是 evolution method。它还把 evolver、executor 和 judge 明确分开。",
"feedbackCases": [
{
"label": "BrowseComp / HLE",
"data": "各 32 道可见验证题、128 道最终评测题",
"scoring": "Qwen3.7-Plus 按源任务问答评测方式判断回答，报告单次正确率。",
"visible": "进化阶段返回验证集总分、逐题结果、执行轨迹及诊断信息；最终评测题在提交冻结框架后才访问。",
"use": "修改者可反复利用验证反馈改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 框架；最终题用于衡量泛化，不作为下一轮修改依据。",
"sources": [
{
"label": "表 1：各基准评分器",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "§5.1.1：统一评审模型",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"judgment": "Qwen3.7-Plus 对照问答参考答案作模型判分"
},
{
"label": "GDPval",
"data": "32 道验证题、64 道最终评测题",
"scoring": "Qwen3.7-Plus 按任务评分细则评价产物，报告平均分。",
"visible": "进化阶段返回验证集总分、逐题结果、执行轨迹及诊断信息；最终评测题在提交冻结框架后才访问。",
"use": "修改者可反复利用验证反馈改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 框架；最终题用于衡量泛化，不作为下一轮修改依据。",
"sources": [
{
"label": "表 1：各基准评分器",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "§5.1.1：统一评审模型",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"judgment": "Qwen3.7-Plus 按 GDPval 任务评分细则评审"
},
{
"label": "APEX-Agents",
"data": "32 道验证题、64 道最终评测题",
"scoring": "Qwen3.7-Plus 按任务评分细则评价，报告单次通过率。",
"visible": "进化阶段返回验证集总分、逐题结果、执行轨迹及诊断信息；最终评测题在提交冻结框架后才访问。",
"use": "修改者可反复利用验证反馈改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 框架；最终题用于衡量泛化，不作为下一轮修改依据。",
"sources": [
{
"label": "表 1：各基准评分器",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "§5.1.1：统一评审模型",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"judgment": "Qwen3.7-Plus 按 APEX-Agents 任务评分细则评审"
},
{
"label": "Claw-Eval",
"data": "32 道验证题、64 道最终评测题",
"scoring": "结合模型评审和规则检查；模型评审统一 Qwen3.7-Plus，按原基准三次执行的 Pass^3 指标汇总。",
"visible": "进化阶段返回验证集总分、逐题结果、执行轨迹及诊断信息；最终评测题在提交冻结框架后才访问。",
"use": "修改者可反复利用验证反馈改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 框架；最终题用于衡量泛化，不作为下一轮修改依据。",
"sources": [
{
"label": "表 1：各基准评分器",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "§5.1.1：统一评审模型",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"judgment": "基准规则检查 + Qwen3.7-Plus 模型评审"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "不训练行为策略权重；修改者用可见 验证集 改 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） 运行框架。构造评测基准时另用辅助任务产生候选运行框架。",
"selection": "160 道 验证集：BrowseComp、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、GDPval、APEX-Agents、Claw-Eval 各 32 题，返回成绩、轨迹和诊断。",
"evaluation": "冻结后测 448 道题：BrowseComp/HLE 各 128；GDPval/APEX-Agents/Claw-Eval 各 64。",
"isolation": "160 与 448 互斥；构造阶段的 auxiliary tasks 不是被测模型的正式训练集。",
"roles": {
"executor": {
"value": "主实验固定 DeepSeek-V4-Flash 执行任务的模型；部分任务由 Qwen3.7-Plus 评分者评分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
}
]
},
"modifier": {
"value": "九种修改模型分别独立进化运行框架：GPT-5.6 Sol、Claude Opus 4.8、GLM-5.2、Qwen3.7-Max、MiniMax-M3、DeepSeek-V4-Pro、Kimi K2.7 Code、Qwen3.6-27B、Gemma-4-31B。每次均改同一个 DeepSeek-V4-Flash 执行端，预算为 20 次迭代／1,000 步／48 小时。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2608.09096#A5.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09096#S5.SS1.SSS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.09096#A3.SS1"
}
]
},
"seed": {
"value": "参数固定的任务执行模型运行同一个最小 CodeAct（让模型生成并执行代码来采取行动的 agent 方式） task agent，修改者可改任务运行代码；构造评测基准时另用多个修改者产生辅助运行框架，筛选对运行框架差异敏感的题。构造阶段与被测进化阶段不混为训练。",
"sources": [
{
"label": "主评估协议；附录 B.2",
"url": "https://arxiv.org/abs/2608.09096"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.09096#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.09096#S4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "框架优化后的涨分，可能来自执行模型更强，也可能只是记住了开发题；短程单次改写还看不出持续研究能力。作者因此要求控制执行模型、检查开发与测试的迁移关系，并提供足够长的搜索过程，单独衡量提出和验证框架改进的能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.09096#S1"
}
]
},
{
"key": "position",
"value": "评测语言模型作为运行框架修改者的能力：在执行模型与预算受控时，能否带来泛化到未见任务的提升，并在持续搜索中保住收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09096"
}
]
},
{
"key": "conclusion",
"value": "九种修改模型都改善初始系统，但搜索任务收益大、办公任务更难，继续进化还可能退化。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.09096"
}
]
}
],
"fields": {
"object": "不同修改者生成的可执行 harness；执行模型、初始接口、预算与测试服务固定。",
"verdict": "修改阶段返回验证题分数、逐题结果和执行诊断。问答、办公产物及工具任务分别采用对应判分规则，需要模型评审时统一使用 Qwen3.7-Plus；最终测试不参与修改。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.31111",
"title": "Aspire: Can Models Self-Evolve from Vague Goals?",
"url": "https://arxiv.org/abs/2608.31111",
"date": "2026-08-31",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"HarnessCode",
"HeldOut",
"Weights",
"org:bytedance-seed"
],
"fields": {
"本质定位": "只给自然语言 capability goal，不给 downstream eval；agent需自己解释目标、选数据/update method、构造train/validation signal、决定何时评估；最终在hidden expert-authored 520 items上测。",
"什么在变": "允许 model weights 与 agent harness evolution；具体由agent自行选择。",
"谁来改 / 谁执行": "**改**：被测 self-evolution agent/researcher 负责决定并实施 update。
**执行**：同一被测 system 既作为 learner/modifier，也执行自己的验证与训练/harness-edit workflow。",
"基础 harness": "统一 interactive environment，可执行训练与harness edit。",
"Feedback": "agent自建 training/validation signals；真正 downstream hidden eval完全不可见。",
"Evolution → Eval": "6 vague capability goals→hidden 520 expert items。",
"Meta-depth": "Evaluation of end-to-end self-evolution。",
"相对之前真正新增什么": "这是对现有 self-evolution benchmark 一个非常关键的反转：**不再把任务和metric都给清楚**，直接测试 agent能否建立正确的 improvement objective与self-evaluation。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "只给自然语言 capability goal，不给 downstream eval；agent需自己解释目标、选数据/update method、构造train/validation signal、决定何时评估；最终在hidden expert-authored 520 items上测。 **相对前序：** 这是对现有 self-evolution benchmark 一个非常关键的反转：**不再把任务和metric都给清楚**，直接测试 agent能否建立正确的 improvement objective与self-evaluation。"
},
{
"label": "什么在变",
"text": "允许 model weights 与 agent harness evolution；具体由agent自行选择。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 被测 self-evolution agent/researcher 负责决定并实施 update。 **执行：** 同一被测 system 既作为 learner/modifier，也执行自己的验证与训练/harness-edit workflow。"
},
{
"label": "基础 harness",
"text": "统一 interactive environment，可执行训练与harness edit。"
},
{
"label": "Feedback",
"text": "agent自建 training/validation signals；真正 downstream hidden eval完全不可见。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "6 vague capability goals→hidden 520 expert items。 **Meta-depth：** Evaluation of end-to-end self-evolution。"
},
{
"label": "主要结果",
"text": "当前 agents能完成training/harness-edit loops，但weight gain稀少不稳定；strongest evolved harness仍低于 engineered Qwen-Agent；常出现local self-eval gain不迁移hidden eval，继续search还擦除已有提升。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "goal interpretation与search混在一起，不能像Evo-Bench那样纯隔离 harness modifier能力。 **对我们：** 和我们的 reliability/ceiling问题高度一致：**会迭代不代表知道该优化什么；self-eval misalignment本身就是失败源。**"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 66,
"fields": {
"论文": "**[Aspire](https://arxiv.org/abs/2608.31111)**",
"为什么仍应视为 Core / 强代表": "hidden downstream eval + vague goal，直接测试 self-evaluation/objective construction 是否可靠。"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 368,
"fields": {
"优先级": "**C**",
"时间": "2026-08-31",
"论文": "[Aspire: Can Models Self-Evolve from Vague Goals?](https://arxiv.org/abs/2608.31111)",
"本质定位": "只给自然语言 capability goal，不给 downstream eval；agent需自己解释目标、选数据/update method、构造train/validation signal、决定何时评估；最终在hidden expert-authored 520 items上测。",
"什么在变": "允许 model weights 与 agent harness evolution；具体由agent自行选择。",
"谁来改 / 谁执行": "**改**：被测 self-evolution agent/researcher 负责决定并实施 update。
**执行**：同一被测 system 既作为 learner/modifier，也执行自己的验证与训练/harness-edit workflow。",
"基础 harness": "统一 interactive environment，可执行训练与harness edit。",
"Feedback": "agent自建 training/validation signals；真正 downstream hidden eval完全不可见。",
"Evolution → Eval": "6 vague capability goals→hidden 520 expert items。",
"Meta-depth": "Evaluation of end-to-end self-evolution。",
"相对之前真正新增什么": "这是对现有 self-evolution benchmark 一个非常关键的反转：**不再把任务和metric都给清楚**，直接测试 agent能否建立正确的 improvement objective与self-evaluation。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录报告独立任务评估；查看详细记录中的数据与选模限制。",
"year": "2026",
"depth": [
"未明确"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"priorityBasis": "用户指定阅读重点：ByteDance Seed / TokenWave 的 Self-Developing Agents 系列。",
"prioritySources": [
{
"label": "团队项目页",
"url": "https://self-developing-agents.github.io/"
}
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "给 agent 一个宽泛目标，例如提高科学推理能力，让它自己选学习材料、决定怎样训练或修改运行流程，并设计练习来检查效果。作者再用保密题目检验：它在自己练习上的进步，是否真的变成了目标能力的提升。",
"novelty": "只给宽泛能力目标，让系统自行选择学习内容和自测方式，再用作者保密题检查是否真正朝目标进步；由此暴露“练习进步、目标能力没进步”的落差。",
"object": "分别研究两种更新：训练模型参数；固定参数、修改提示、工具规则和执行流程。没有验证两者同步进化。",
"executor": "**参数更新实验：** 由 Qwen3.5-4B 或 Qwen3.5-9B 训练得到的候选模型答题。\n\n**运行框架实验：** 固定使用 Qwen3.5-4B，配合创建者生成的候选运行框架答题。",
"modifier": "**参数更新实验：** 自行改进时，由初始 Qwen3.5-4B 或 Qwen3.5-9B 决定学习数据与训练方案，平台执行训练；外部指导时，由论文命名为 Luna、Terra、Sol 的三个 GPT-5.6 配置指导同一个 Qwen3.5-4B。负责制定方案的初始模型始终固定，不由训练后的候选模型接替。\n\n**运行框架实验：** 创建者为 Qwen3.5-4B 或上述 GPT-5.6 配置，负责生成外围运行代码。",
"roleContext": "**改**：被测 self-evolution agent/researcher 负责决定并实施 update。
**执行**：同一被测 system 既作为 learner/modifier，也执行自己的验证与训练/harness-edit workflow。",
"seed": "作者自建平台提供数据搜索、导入、生成、训练、进度查询和评估工具，代办底层训练工程。框架进化实验让创建者设计模型外围的运行代码；Qwen-Agent 是 Qwen 团队提供的 agent 开发框架，负责组织工具调用和任务执行，在这里作为人工设计的对照。两者都配合同一 Qwen3.5-4B 模型比较。",
"fixed": "Evaluation of end-to-end self-evolution。",
"verdict": "**参数更新实验：** 创建者可用自建验证数据和检查方法调整训练；论文未公开每次本地检查的完整题目和判分脚本。允许中途反馈的设置可有限次查询专家题组的汇总分数，但看不到题目和逐题错误；仅最终提交的设置不返回中途分数。\n\n**运行框架实验：** 创建者依据本地写作检查调整框架。例如 Luna 使用自设八项检查表；正式 20 道写作题的成绩不返回创建者。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "只给自然语言 capability goal，不给 downstream eval；agent需自己解释目标、选数据/update method、构造train/validation signal、决定何时评估；最终在hidden expert-authored 520 items上测。",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "6 vague capability goals→hidden 520 expert items。"
}
],
"takeaway": "goal interpretation与search混在一起，不能像Evo-Bench那样纯隔离 harness modifier能力。 **对我们：** 和我们的 reliability/ceiling问题高度一致：**会迭代不代表知道该优化什么；self-eval misalignment本身就是失败源。**",
"result": "当前 agents能完成training/harness-edit loops，但weight gain稀少不稳定；strongest evolved harness仍低于 engineered Qwen-Agent；常出现local self-eval gain不迁移hidden eval，继续search还擦除已有提升。",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：参数训练：自行下载或合成数据，记录包括 GSM8K 和 Hendrycks 数学数据。框架修改：自建写作任务。目标描述对照：沿用 PostTrainBench。\n\n调试 / 选版本数据：参数训练：本地验证；部分配置还能调用评估工具获取专家题组总分。框架修改：Luna 用 1 条提示、8 项检查，再检查 2 条相似提示；Sol 检查 4 类任务。\n\n最终测试数据：参数训练：自建 520 道专家题中的对应目标题组。框架：其中 20 道写作题，每个冻结框架运行 3 次。PostTrainBench：AIME 2025、GPQA Main、HealthBench、HumanEval、GSM8K、ArenaHard、BFCL。\n\n数据隔离与证据边界：参数训练：可调用评估的配置用同一题组分数选模型；仅最终提交的配置在提交前不返回分数。框架：正式题目和成绩均不返回创建者。520 题是专家新编题，GPQA／MMLU-Pro／MedQA 仅供题型参考。",
"cycle": "**参数更新实验：** 初始决策模型选择数据与训练方案，平台训练出候选版本。仅最终提交的设置只评提交的最终版本；训练期间调用评估工具获取专家题组总分的设置可以用专家题组总分继续选数据、训练和比较版本。平台按预设资格规则选出候选，只有成绩超过基础模型时才保留，否则退回基础模型。\n\n**运行框架实验：** 创建者根据自己的本地检查修改、选择并提交框架；平台冻结框架后做正式评测，分数不用于继续修改。",
"train": "参数训练：自行下载或合成数据，记录包括 GSM8K 和 Hendrycks 数学数据。框架修改：自建写作任务。目标描述对照：沿用 PostTrainBench。",
"debug": "参数训练：本地验证；部分配置还能调用评估工具获取专家题组总分。框架修改：Luna 用 1 条提示、8 项检查，再检查 2 条相似提示；Sol 检查 4 类任务。",
"test": "参数训练：自建 520 道专家题中的对应目标题组。框架：其中 20 道写作题，每个冻结框架运行 3 次。PostTrainBench：AIME 2025、GPQA Main、HealthBench、HumanEval、GSM8K、ArenaHard、BFCL。",
"isolation": "参数训练：可调用评估的配置用同一题组分数选模型；仅最终提交的配置在提交前不返回分数。框架：正式题目和成绩均不返回创建者。520 题是专家新编题，GPQA／MMLU-Pro／MedQA 仅供题型参考。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建的受控交互环境，通过一个统一工具提供数据搜索/导入/生成、启动训练、查询进度、自测、分支和停止等操作。agent 不直接使用 shell，也不需要安装训练仓库或实现分布式运行；这些底层工作由控制器代办。这个“简化”指工程接口简单，学习策略仍要 agent 自己决定；它不是一套预先写好改进策略的成熟 harness。",
"protocol": "**参数进化（RQ2）—进化数据：**附录 C.4 的轨迹统计点名 GSM8K 和 Hendrycks mathematics 数据：Qwen3.5-4B Self 的 32 次数据导入中有 30 次使用这两类数学数据；9B 的 15 次导入中有 11 次使用数学数据。这是导入事件数，不是训练样本数，也不表示所有实验都采用同一训练集；这些导入所用的具体 split 和样本量本轮尚未核实。\n\n**参数进化—验证与测试：**作者自建 520 道专家题，按目标分为科学推理 75、人文社科 110、医学 100、数学 126、逻辑/可靠性/指令遵循 89、学术写作 20。每次针对对应目标题目评测。“仅最终评测”在提交最终模型前不返回分数；“允许中途反馈”可有限次查询同一目标题组的汇总分数并据此选版本，因此它不是选版本后从未接触过的独立测试集。\n\n**Harness 进化（RQ3）—进化与自测数据：**创建者自行构造验证任务。论文给出的具体案例是 Luna 使用 1 条验证提示和 8 项检查表，修改后又检查 2 条相似提示；Sol 检查 4 类验证任务。这些是本地自测，不是 GSM8K，也不是正式测试集。任务完整文本本轮尚未核实。\n\n**Harness 进化—最终测试：**使用上述题库中的 20 道学术与科学写作题。Qwen3.5-4B 配合原始 Qwen-Agent 或各个生成的 harness；每个候选冻结后在相同 20 题上运行 3 次，分数不返回给创建者。这里没有测试全部 520 题，也不是 3 次独立进化。\n\n**题库来源与其他实验：**520 题由专家从零编写，GPQA、MMLU-Pro、MedQA 只作为题型和难度参考，不是直接拿它们的题测试。RQ1 另沿用 PostTrainBench 的任务评测，与上述题库分开统计；逐任务数据划分尚待补查。依据：[原文 §3.2、§4.4、附录 C.4 与 D](https://arxiv.org/html/2608.31111)。",
"sections": "§3.2–3.3、§4.4、附录 C.4/D",
"source": "https://arxiv.org/abs/2608.31111",
"version": "2608.31111v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "47dc6f2f2d8738b4336093c02475916ca18f4c9281b542644caf58a2a067252c",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "**参数更新实验：** 由 Qwen3.5-4B 或 Qwen3.5-9B 训练得到的候选模型答题。\n\n**运行框架实验：** 固定使用 Qwen3.5-4B，配合创建者生成的候选运行框架答题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "**参数更新实验：** 自行改进时，由初始 Qwen3.5-4B 或 Qwen3.5-9B 决定学习数据与训练方案，平台执行训练；外部指导时，由论文命名为 Luna、Terra、Sol 的三个 GPT-5.6 配置指导同一个 Qwen3.5-4B。负责制定方案的初始模型始终固定，不由训练后的候选模型接替。\n\n**运行框架实验：** 创建者为 Qwen3.5-4B 或上述 GPT-5.6 配置，负责生成外围运行代码。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "分别研究两种更新：训练模型参数；固定参数、修改提示、工具规则和执行流程。没有验证两者同步进化。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "**参数更新实验：** 创建者可用自建验证数据和检查方法调整训练；论文未公开每次本地检查的完整题目和判分脚本。允许中途反馈的设置可有限次查询专家题组的汇总分数，但看不到题目和逐题错误；仅最终提交的设置不返回中途分数。\n\n**运行框架实验：** 创建者依据本地写作检查调整框架。例如 Luna 使用自设八项检查表；正式 20 道写作题的成绩不返回创建者。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建平台提供数据搜索、导入、生成、训练、进度查询和评估工具，代办底层训练工程。框架进化实验让创建者设计模型外围的运行代码；Qwen-Agent 是 Qwen 团队提供的 agent 开发框架，负责组织工具调用和任务执行，在这里作为人工设计的对照。两者都配合同一 Qwen3.5-4B 模型比较。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "**参数更新实验：** 初始决策模型选择数据与训练方案，平台训练出候选版本。仅最终提交的设置只评提交的最终版本；训练期间调用评估工具获取专家题组总分的设置可以用专家题组总分继续选数据、训练和比较版本。平台按预设资格规则选出候选，只有成绩超过基础模型时才保留，否则退回基础模型。\n\n**运行框架实验：** 创建者根据自己的本地检查修改、选择并提交框架；平台冻结框架后做正式评测，分数不用于继续修改。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录 C.1：选版本与回退规则",
"url": "https://arxiv.org/html/2608.31111#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "参数训练：自行下载或合成数据，记录包括 GSM8K 和 Hendrycks 数学数据。框架修改：自建写作任务。目标描述对照：沿用 PostTrainBench。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px9"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "参数训练：本地验证；部分配置还能调用评估工具获取专家题组总分。框架修改：Luna 用 1 条提示、8 项检查，再检查 2 条相似提示；Sol 检查 4 类任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "参数训练：自建 520 道专家题中的对应目标题组。框架：其中 20 道写作题，每个冻结框架运行 3 次。PostTrainBench：AIME 2025、GPQA Main、HealthBench、HumanEval、GSM8K、ArenaHard、BFCL。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "参数训练：可调用评估的配置用同一题组分数选模型；仅最终提交的配置在提交前不返回分数。框架：正式题目和成绩均不返回创建者。520 题是专家新编题，GPQA／MMLU-Pro／MedQA 仅供题型参考。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "只给宽泛能力目标，让系统自行选择学习内容和自测方式，再用作者保密题检查是否真正朝目标进步；由此暴露“练习进步、目标能力没进步”的落差。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.31111v1",
"version": "2608.31111v1",
"scope": "原文 §3.1–3.3、§4 与附录 C 定向核查：区分执行与决策角色、初始环境、题库构造，以及最终评测和有限汇总反馈两种协议；非全文逐项审读。"
},
"focus": "它考察 agent 能否把宽泛目标变成合理的学习任务和自测标准：即使能成功运行训练、也能让自测分数上涨，仍可能因为练错内容而无法通过作者的保密考试。",
"feedbackCases": [
{
"label": "参数进化：训练材料与本地检查",
"data": "创建者自行导入或生成数据；附录明确记录 GSM8K（小学数学应用题基准）、Hendrycks mathematics。4B Self 的 32 次导入有 30 次涉及这两类数学数据，9B Self 是 15 次中 11 次；这是导入次数，不是样本数。",
"scoring": "本地检查使用创建者自己的验证数据和验证方法。论文公开的记录不能还原每次检查的题目、参考答案及具体判分脚本，因此不能断言这些数学数据都按官方答案正确率反馈。",
"visible": "自有验证数据可见，可反复检查且不消耗保密考试的查询额度；公开轨迹统计不含原始模型消息和命令输出。",
"use": "决定后续数据、训练计划或停止时间。该本地检查不能等同于下列专家题库的成绩。",
"sources": [
{
"label": "§3.2；附录 C.2",
"url": "https://arxiv.org/html/2608.31111#A3.SS2"
},
{
"label": "附录 C.4：导入数据统计",
"url": "https://arxiv.org/html/2608.31111#A3.SS4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"judgment": "创建者自测；论文未逐次公开所用答案、规则脚本或模型裁判"
},
{
"label": "参数进化：训练期间调用评估工具获取专家题组总分",
"data": "专家从零编写的 520 道题，按当前目标选择其中一组：科学 75、人文社科 110、医学 100、数学 126、逻辑/可靠性/指令遵循 89、写作 20。",
"scoring": "专家预设参考答案或评分细则，平台按固定判分规则评分；能规则判定的使用确定性/结构化评分，开放回答使用固定模型评审。",
"visible": "仅返回目标题组的汇总分数及剩余查询额度；不返回试题、参考答案、候选回答、逐题分数、评分细则或评审理由。",
"use": "有限次反复查询同一目标题组，指导训练、选检查点和停止；论文最后报告的也是这些被反复查询的题，不是另一个独立确认集。",
"sources": [
{
"label": "§3.2：题库与反馈边界",
"url": "https://arxiv.org/html/2608.31111#S3.SS2"
},
{
"label": "附录 C.2、C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS2"
}
],
"judgment": "参考答案／评分细则：客观题用规则，开放题用模型评审"
},
{
"label": "参数进化：仅最终提交",
"data": "同一专家题库中对应目标的题组；4B/9B Self × 六目标 × 两次独立运行。",
"scoring": "提交的检查点由平台评测；附录 C.3 明确评审模型为 gemini-3.5-flash，按固定题目评分规则判分。",
"visible": "提交前不返回这套题库的任何分数；每次运行只评一个最终检查点。",
"use": "衡量最终模型能力；这一分数不能用于本次运行的继续训练或改选版本。",
"sources": [
{
"label": "附录 C.3",
"url": "https://arxiv.org/html/2608.31111#A3.SS3"
}
],
"judgment": "gemini-3.5-flash 按题目评分要求评审"
},
{
"label": "框架进化：Luna 的本地写作检查",
"data": "一条创建者设计的写作提示；修改后再检查这条提示和两条相似提示。",
"scoring": "Luna 用自己设计的八项检查表检查输出，第一次 7/8；加上“研究问题、设计、分析、风险、局限”五部分固定模板后达到 8/8。原文未公开完整提示及全部八项逐字内容。",
"visible": "创建者可看到本地输出和检查结果。这里的 8/8 是自设检查表得分，不是官方写作题库得分。",
"use": "Luna 约 40 分钟后停止并交付框架；案例显示它学会了满足窄检查表，却可能把技术问答错误套成研究设计。",
"sources": [
{
"label": "附录 D：Luna 案例",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
}
],
"judgment": "Luna 按自建八项检查表自评"
},
{
"label": "框架进化：Terra / Sol 的本地检查",
"data": "Terra 检查回答审阅、计算器调用和输出完整性；Sol 在四类自建验证任务上比较提示长度、工具调用恢复、禁用工具与审阅步骤。",
"scoring": "Terra 因审阅器编造显著性和效应量而删掉审阅步骤。Sol 综合本地判分及运行证据，拒绝延迟高、重复调用、编造数值或判分漏检的版本，并纠正过一个错误的本地参考答案。",
"visible": "本地回答、错误及工具记录对创建者可见；论文给出这些案例，但没有给出四类验证任务的完整数据集和统一自动判分脚本。",
"use": "用于决定保留哪些修改；Sol 最后恢复原 Qwen-Agent（Qwen 团队提供的 agent 开发框架，组织模型调用工具和执行任务） 策略，只增加最终回答为空的检查。",
"sources": [
{
"label": "附录 D：Terra / Sol 案例",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px3"
},
{
"label": "附录 D：Sol",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"judgment": "创建者检查本地输出与运行错误；未公开统一判分实现"
},
{
"label": "框架进化：最终写作评测",
"data": "专家题库中的 20 道学术/科学写作题；Qwen3.5-4B 配原 Qwen-Agent（Qwen 团队提供的 agent 开发框架，组织模型调用工具和执行任务） 或冻结后的候选框架，每个运行三次。",
"scoring": "平台按保密题目的固定评分规则评价输出，报告按题平均和按评分样例平均的两种汇总。原文没有在此实验单独明确评审模型型号，不能直接套用参数实验的型号。",
"visible": "正式题目与正式分数均不返回创建者。",
"use": "仅衡量已冻结框架；三次运行测执行波动，不是三代进化。",
"sources": [
{
"label": "§4.4：设计与表 1",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录 D：信息边界",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px1"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"judgment": "按专家题目的保密评分细则评审；该实验未单独披露裁判型号"
},
{
"label": "PostTrainBench 对照实验",
"data": "AIME 2025、GPQA（研究生级科学问答基准） Main、HealthBench、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、GSM8K（小学数学应用题基准）、ArenaHard、BFCL；这组实验独立于上述 520 道专家题。",
"scoring": "原 PostTrainBench 将 AIME25、GSM8K、GPQA、BFCL 列为答案精确匹配；HumanEval 执行单元测试；ArenaHard Writing 由 GPT-5-mini 与基线输出比较，HealthBench-Easy 由 GPT-5-mini 按医疗评分细则评审。Aspire 表示沿用其评测，未另列完整逐任务改动；这里的具体机制引用原 PostTrainBench，不能据此把自建练习也视为相同评分。",
"visible": "agent 自行构造训练和验证方法；正式分数、工具访问与训练过程按 PostTrainBench 对照协议记录。",
"use": "比较给不给具体任务定义的影响；附录 B 的分数不能混入新专家题库。逐基准评测规则应以原 PostTrainBench 协议为准。",
"sources": [
{
"label": "§4.2 与附录 B",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"judgment": "沿用 PostTrainBench：答案匹配、代码测试、模型评审分别使用"
}
],
"experiments": [
{
"label": "参数更新：训练期间调用评估工具获取专家题组总分",
"learningCases": [
0,
1
],
"testCases": [
1
],
"evolution": "创建者自行导入或合成训练材料；附录记录 GSM8K、Hendrycks 数学数据，但没有逐次公开样本及划分。本地检查之外，可有限次查询专家题组总分，并据此继续训练。",
"selection": "用本地检查和对应专家题组汇总分选择检查点或停止；题目、答案和逐题错误不可见。",
"evaluation": "自建 520 道专家题中对应目标的一组；最终仍报告这组题的分数。",
"isolation": "同一专家题组已提供适应与选模信号，不能当作从未参与选择的独立最终测试。",
"roles": {
"executor": {
"value": "由 Qwen3.5-4B 或 Qwen3.5-9B 训练得到的候选模型答题；负责制定训练方案的初始模型不随候选更新。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
]
},
"modifier": {
"value": "Self 配置：固定的初始 Qwen3.5-4B／9B 选择数据和训练方案。外部指导配置：GPT-5.6 的 Luna、Terra、Sol 三个配置指导 Qwen3.5-4B。平台控制器实际执行训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
]
},
"seed": {
"value": "作者自建训练平台提供数据搜索、导入、生成、训练、进度查询和评估工具；平台代办底层训练工程。本组修改模型参数。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px9"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
}
]
}
},
{
"label": "参数更新：只在最终提交后评测",
"learningCases": [
0
],
"testCases": [
2
],
"evolution": "创建者自行导入或合成训练材料，并用自行构建的本地验证判断训练；附录记录上述两类数学数据的导入。",
"selection": "用本地检查选一个最终检查点；提交前不能查询保密专家题组成绩。",
"evaluation": "同一 520 道专家题库中对应目标的一组；4B/9B × 六个目标，每个组合两次独立运行。",
"isolation": "正式题目和分数在提交前不可见。自造练习与专家题组是两种数据，不能混称验证集。",
"roles": {
"executor": {
"value": "由 Qwen3.5-4B 或 Qwen3.5-9B 训练得到的候选模型答题；负责制定训练方案的初始模型不随候选更新。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
]
},
"modifier": {
"value": "仅 Qwen3.5-4B Self 与 Qwen3.5-9B Self：固定的初始模型制定训练方案，平台执行训练。这里没有 Luna／Terra／Sol 外部指导配置。",
"sources": [
{
"label": "附录 C.3：最终提交协议",
"url": "https://arxiv.org/html/2608.31111#A3.SS3"
}
]
},
"seed": {
"value": "作者自建训练平台提供数据搜索、导入、生成、训练、进度查询和评估工具；平台代办底层训练工程。本组修改模型参数。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px9"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
}
]
}
},
{
"label": "运行框架：创建者自测后，冻结框架测写作",
"learningCases": [
3,
4
],
"testCases": [
5
],
"evolution": "Luna 用一条自建写作提示及八项检查，修改后再检查原提示和两条相似提示；Terra/Sol 也用本地任务与运行证据调试，Sol 覆盖四类验证任务。不更新模型参数。",
"selection": "依据创建者的本地输出、检查结果和运行问题保留或撤回修改。",
"evaluation": "专家题库中的 20 道学术／科学写作题。每个冻结框架配 Qwen3.5-4B 运行三次。",
"isolation": "20 题的正式题目及分数不返回创建者；三次测量执行波动，不是三代进化。",
"roles": {
"executor": {
"value": "固定 Qwen3.5-4B，分别配生成的候选框架或人工设计的 Qwen-Agent 对照框架答题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
]
},
"modifier": {
"value": "Qwen3.5-4B 或 GPT-5.6 的 Luna／Terra／Sol 配置生成外围运行代码；正式写作评测阶段不再改框架。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.31111#S3.SS1.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
}
]
},
"seed": {
"value": "创建者设计模型外围运行代码；Qwen-Agent 是组织工具调用和任务执行的 agent 开发框架，在本组用作人工设计的对照，不能把它说成所有候选共同继承的起点。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.31111#S3.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px9"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
}
]
}
},
{
"label": "目标描述对照：沿用 PostTrainBench",
"learningCases": [
6
],
"testCases": [
6
],
"evolution": "在 PostTrainBench 中自主选择数据和训练方式；分别给出明确任务名或宽泛能力目标。",
"selection": "沿用 PostTrainBench 的开发和评估接口。",
"evaluation": "AIME 2025、GPQA Main、HealthBench、HumanEval、GSM8K、ArenaHard、BFCL。",
"isolation": "这组实验独立于自建 520 道专家题库；数据权限和最终选模应按 PostTrainBench 协议解释。",
"roles": {
"executor": {
"value": "提交的训练后基础模型负责答题。附录 B 的对照按相同基础模型配对，但未逐项列出基础模型型号；不能沿用另一组实验的 Qwen3.5-4B／9B。",
"sources": [
{
"label": "§4.2、附录 B：目标描述对照",
"url": "https://arxiv.org/html/2608.31111#A2"
}
]
},
"modifier": {
"value": "宽泛目标运行由 Claude Opus 4.8 或 GPT-5.6 决策；官方参照为 Claude Opus 4.8 Max 与 GPT-5.6。轨迹配对分析仅使用 Claude Opus 4.8。",
"sources": [
{
"label": "§4.2、附录 B 表 3",
"url": "https://arxiv.org/html/2608.31111#A2"
}
]
},
"seed": {
"value": "沿用 PostTrainBench 的后训练接口和原任务评测器；本组每次运行使用一张 H20、最多十小时。不是参数更新组的自建专家题平台，也不是 Qwen-Agent 写作框架。",
"sources": [
{
"label": "附录 B：配对运行与预算",
"url": "https://arxiv.org/html/2608.31111#A2"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.31111#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px9"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2608.31111#A3.SS4.SSS0.Px2"
},
{
"label": "Qwen-Agent 官方说明",
"url": "https://github.com/QwenLM/Qwen-Agent"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px2"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.31111#A4.SS0.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有自进化实验通常由人预先指定任务、训练数据或评分标准，模型主要优化一个已定义的目标。因此，即使基准成绩提高，也不能证明模型能自行判断该学什么、如何验证学习效果；自建练习上的改善还可能与真正希望提高的能力脱节。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.31111#S1"
}
]
},
{
"key": "position",
"value": "评测模型只获得宽泛能力目标时，能否把目标转成有效的学习方案。论文分别考察：模糊目标相对明确任务会怎样影响训练；模型能否自行选择数据并更新参数；参数固定时能否改进外围运行框架。",
"sources": [
{
"label": "§1、§4.1：研究问题与实验安排",
"url": "https://arxiv.org/html/2608.31111#S4.SS1"
}
]
},
{
"key": "conclusion",
"value": "宽泛目标设置的总体成绩低于明确任务的参照系统，但不同任务并非都下降。自主参数训练收益不稳定，生成的运行框架也未超过人工设计的 Qwen-Agent 对照。结果说明，完成训练或提高自建验证分数，并不足以证明目标能力提高；可查询专家题组成绩的设置还需与只评测最终提交的设置分开解释。",
"sources": [
{
"label": "§4.2：目标描述对照结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
},
{
"label": "§4.3：参数更新结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS3"
},
{
"label": "§4.4：运行框架结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS4"
}
]
}
],
"fields": {
"executor": "参数训练：Qwen3.5-4B 或 Qwen3.5-9B 的训练后模型答题；框架实验：固定 Qwen3.5-4B。PostTrainBench 对照组另列配置。",
"modifier": "参数训练：初始 Qwen3.5-4B／9B 制定方案，平台执行训练；允许查询评测分数的配置另比较 GPT-5.6 Luna／Terra／Sol 指导 4B。框架代码由 4B 或这三个 GPT-5.6 配置生成。",
"verdict": "参数训练：创建者本地验证；一组可调用评估工具获取专家题组总分，按参考答案或评分细则判分，另一组只评测最终提交。框架修改：自建任务和检查表（Luna 用八项写作检查），正式 20 题成绩不返回创建者。",
"seed": "参数训练用作者平台，提供数据搜索、导入、生成和训练工具。框架实验由创建者编写运行代码；Qwen-Agent（组织工具调用与执行的 agent 框架）是对照，不是所有候选的共同起点。",
"object": "分别研究两种更新：训练模型参数；固定参数、修改提示、工具规则和执行流程。没有验证两者同步进化。"
}
},
"attributions": [
{
"tag": "org:bytedance-seed",
"label": "ByteDance Seed",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.31111"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"curriculum",
"evaluation"
]
},
{
"id": "2408.08435",
"title": "Automated Design of Agentic Systems (ADAS)",
"url": "https://arxiv.org/abs/2408.08435",
"date": "2024-08-15",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Archive",
"BenchmarkScore",
"HarnessCode",
"HeldOut",
"M1",
"StrongerBuilder",
"Workflow",
"org:ubc",
"person:jeff-clune"
],
"fields": {
"本质定位": "Meta Agent Search 让 LLM 直接发明 executable agent code，并把历史 agent archive 当作搜索经验。",
"什么在变": "完整 agent program（受接口约束）。",
"谁来改 / 谁执行": "**改**：通常更强 GPT-4-class meta-agent；search/meta prompt 固定。
**执行**：candidate agent 的 target LLM（实验常用 GPT-3.5 等）。",
"基础 harness": "一个极简可编程 agent skeleton / seed archive。",
"Feedback": "validation task metric + runtime/error feedback。",
"Evolution → Eval": "多个任务有 validation search → held-out test，并测跨 domain/model transfer。",
"Meta-depth": "M1：完整 task-agent code 变，但 meta-agent/search fixed。",
"相对之前真正新增什么": "相对 prompt optimizer 的核心跃迁：editable object 变成整个 code-represented agent architecture，可同时包含 prompt、tool use、control flow、multi-agent logic。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "Meta Agent Search 让 LLM 直接发明 executable agent code，并把历史 agent archive 当作搜索经验。 **相对前序：** 相对 prompt optimizer 的核心跃迁：editable object 变成整个 code-represented agent architecture，可同时包含 prompt、tool use、control flow、multi-agent logic。"
},
{
"label": "什么在变",
"text": "完整 agent program（受接口约束）。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** 通常更强 GPT-4-class meta-agent；search/meta prompt 固定。 **执行：** candidate agent 的 target LLM（实验常用 GPT-3.5 等）。"
},
{
"label": "基础 harness",
"text": "一个极简可编程 agent skeleton / seed archive。"
},
{
"label": "Feedback",
"text": "validation task metric + runtime/error feedback。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "多个任务有 validation search → held-out test，并测跨 domain/model transfer。 **Meta-depth：** M1：完整 task-agent code 变，但 meta-agent/search fixed。"
},
{
"label": "主要结果",
"text": "ARC/reasoning/coding 等自动发现的 agent design 可优于人工 baseline，并有迁移。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "主 setting 是 strong meta-agent 设计 target agent，不是严格 self-RSI。 **对我们：** Meta-Harness/AHE 的直接前驱之一，应该在 related work 中出现。"
}
],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 67,
"fields": {
"论文": "**[ADAS](https://arxiv.org/abs/2408.08435)**",
"为什么仍应视为 Core / 强代表": "executable agent program search 的重要前驱；related work 中解释 full-harness 从哪里来时必读。"
}
},
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 278,
"fields": {
"优先级": "**C**",
"时间": "2024-08-15",
"论文": "[Automated Design of Agentic Systems (ADAS)](https://arxiv.org/abs/2408.08435)",
"本质定位": "Meta Agent Search 让 LLM 直接发明 executable agent code，并把历史 agent archive 当作搜索经验。",
"什么在变": "完整 agent program（受接口约束）。",
"谁来改 / 谁执行": "**改**：通常更强 GPT-4-class meta-agent；search/meta prompt 固定。
**执行**：candidate agent 的 target LLM（实验常用 GPT-3.5 等）。",
"基础 harness": "一个极简可编程 agent skeleton / seed archive。",
"Feedback": "validation task metric + runtime/error feedback。",
"Evolution → Eval": "多个任务有 validation search → held-out test，并测跨 domain/model transfer。",
"Meta-depth": "M1：完整 task-agent code 变，但 meta-agent/search fixed。",
"相对之前真正新增什么": "相对 prompt optimizer 的核心跃迁：editable object 变成整个 code-represented agent architecture，可同时包含 prompt、tool use、control flow、multi-agent logic。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2024",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Meta Agent Search 在代码空间发明 task agent，并把历史 agent archive 作为搜索经验。代码可以组合多个 LLM 调用、prompt 和控制逻辑。",
"novelty": "让上层设计模型生成用代码表达的 agent 结构，利用历史设计与评分继续搜索；可以改变模块组合和控制流程。",
"object": "task agent 的完整代码，包括模型调用、提示和控制流程；外层 meta-agent 及搜索规则保持固定。",
"executor": "主实验候选 task agent 及对照方案用gpt-3.5-turbo-0125执行；迁移另用GPT-4o、Claude3 Haiku和Claude3.5 Sonnet。",
"modifier": "独立 meta-agent（负责设计或修改 task agent）用gpt-4o-2024-05-13读取档案、验证结果并生成候选 task agent 代码。",
"roleContext": "**改**：通常更强 GPT-4-class meta-agent；search/meta prompt 固定。
**执行**：candidate agent 的 target LLM（实验常用 GPT-3.5 等）。",
"seed": "Meta Agent Search 在受接口约束的代码空间生成 task agent，初始档案含 CoT、Self-Refine 等人写设计。每候选先自查，验证报错可修至多五轮，再把代码与分数写回档案。",
"fixed": "M1：完整 task-agent code 变，但 meta-agent/search fixed",
"verdict": "验证任务的得分、运行过程中的错误信息。",
"diagnosis": "阅读 prior agent designs、验证结果与错误反馈，提出新架构。",
"update": "编写候选 agent 程序，包含 reasoning/control-flow 结构。",
"acceptance": "验证后把结果记入 archive，按验证表现筛选。",
"experiments": [
{
"name": "ARC / reasoning",
"evolve": "各任务 validation 搜索",
"selection": "validation 选择优秀 designs",
"test": "held-out test；MGSM designs 转到其他任务/模型",
"isolation": "独立测试",
"note": ""
}
],
"takeaway": "主 setting 是 strong meta-agent 设计 target agent，不是严格 self-RSI。 **对我们：** Meta-Harness/AHE 的直接前驱之一，应该在 related work 中出现。",
"result": "ARC/reasoning/coding 等自动发现的 agent design 可优于人工 baseline，并有迁移。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2408.08435v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不做参数训练；meta-agent（负责设计或修改 task agent）生成 task agent 代码，以 验证集 表现搜索架构。\n\n调试 / 选版本数据：ARC 小网格题 20；GPQA（研究生级科学问答基准） Diamond 32；DROP、MGSM、MMLU 各 128。代码报错时最多再反思修复 5 次。\n\n最终测试数据：ARC 60；GPQA（研究生级科学问答基准） 166；其他推理领域各 800。另测 MGSM 设计向 GSM8K（小学数学应用题基准）、GSM-Hard 及不同模型迁移。\n\n数据隔离与证据边界：搜索用 验证集，测试集 分开。ARC 来自公开训练库内的抽样划分，不是 ARC 官方私有测试集。",
"cycle": "读取既往 agent 设计、验证结果和错误反馈，提出新架构并编写可运行程序。运行验证题后把设计和成绩存入档案，再据此选择和探索下一候选。",
"train": "不做参数训练；meta-agent（负责设计或修改 task agent）生成 task agent 代码，以 验证集 表现搜索架构。",
"debug": "ARC 小网格题 20；GPQA（研究生级科学问答基准） Diamond 32；DROP、MGSM、MMLU 各 128。代码报错时最多再反思修复 5 次。",
"test": "ARC 60；GPQA（研究生级科学问答基准） 166；其他推理领域各 800。另测 MGSM 设计向 GSM8K（小学数学应用题基准）、GSM-Hard 及不同模型迁移。",
"isolation": "搜索用 验证集，测试集 分开。ARC 来自公开训练库内的抽样划分，不是 ARC 官方私有测试集。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "Meta Agent Search 在受接口约束的代码空间生成 agent，初始档案含 CoT、Self-Refine 等人写设计。每候选先自查，验证报错可修至多五轮，再把代码与分数写回档案。",
"protocol": "**搜索→测试：**ARC 逻辑题及 DROP 阅读、MGSM 数学、GPQA 科学、MMLU 多任务。GPQA Diamond 32 validation/166 test；其他推理领域抽 128/800，ARC 需单独看设置。\n\n**角色：**GPT-4o 作为 meta-agent，GPT-3.5 运行候选；另把 MGSM 设计迁移到 GSM8K、GSM-Hard 等域/模型。验证用于设计搜索，不是独立训练模型权重。",
"sections": "§4；Algorithm 1 与数据采样",
"source": "https://arxiv.org/abs/2408.08435",
"version": "2408.08435v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "72c533f0dfaef62490e61a06b542de15ff22fcd829b9f0b2844cace24341b38d",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验候选 task agent 及对照方案用gpt-3.5-turbo-0125执行；迁移另用GPT-4o、Claude3 Haiku和Claude3.5 Sonnet。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2408.08435#A1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "独立 meta-agent（负责设计或修改 task agent）用gpt-4o-2024-05-13读取档案、验证结果并生成候选 task agent 代码。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2408.08435#A1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "task agent 的完整代码，包括模型调用、提示和控制流程；外层 meta-agent 及搜索规则保持固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "验证任务的得分、运行过程中的错误信息。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Meta Agent Search 在受接口约束的代码空间生成 task agent，初始档案含 CoT、Self-Refine 等人写设计。每候选先自查，验证报错可修至多五轮，再把代码与分数写回档案。",
"sources": [
{
"label": "§4；Algorithm 1 与数据采样",
"url": "https://arxiv.org/abs/2408.08435"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "读取既往 agent 设计、验证结果和错误反馈，提出新架构并编写可运行程序。运行验证题后把设计和成绩存入档案，再据此选择和探索下一候选。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不做参数训练；meta-agent（负责设计或修改 task agent）生成 task agent 代码，以 验证集 表现搜索架构。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "ARC 小网格题 20；GPQA（研究生级科学问答基准） Diamond 32；DROP、MGSM、MMLU 各 128。代码报错时最多再反思修复 5 次。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ARC 60；GPQA（研究生级科学问答基准） 166；其他推理领域各 800。另测 MGSM 设计向 GSM8K（小学数学应用题基准）、GSM-Hard 及不同模型迁移。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "搜索用 验证集，测试集 分开。ARC 来自公开训练库内的抽样划分，不是 ARC 官方私有测试集。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让上层设计模型生成用代码表达的 agent 结构，利用历史设计与评分继续搜索；可以改变模块组合和控制流程。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2408.08435v2",
"version": "2408.08435v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "从 prompt 搜索扩到 code-represented agent architecture；与后来的通用 coding runtime evolution 有谱系联系，但两者的 seed 和执行环境不同。",
"feedbackCases": [
{
"label": "ARC 抽象推理",
"data": "ARC 任务的示例网格与目标网格，搜索用验证题，最终用独立测试题。",
"scoring": "agent 生成变换代码；运行代码得到目标网格，与参考网格精确匹配计分。",
"visible": "meta-agent 收到候选的验证数值分数，代码报错可用于修复；候选 agent 可用示例网格自测。",
"use": "数值结果写入候选档案指导后续 agent 设计，不是主方法把全部成功/失败日志自动归因。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"judgment": "运行变换代码，规则比较输出网格与标准网格"
},
{
"label": "阅读、数学、综合与科学问答",
"data": "DROP、MGSM、MMLU、GPQA（研究生级科学问答基准），使用各任务的验证/测试划分；另做跨域迁移。",
"scoring": "DROP 按答案 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务） 计分；MGSM 按数学答案正确性；MMLU、GPQA（研究生级科学问答基准） 按参考选择项正确率。",
"visible": "验证分数与运行报错供搜索使用。",
"use": "搜索产生 agent 代码，最终测试及迁移结果另报；各任务评分尺度不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"judgment": "DROP 用词项 F1；MGSM 比最终答案；MMLU／GPQA 比参考选项"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "不做参数训练；meta-agent（负责设计或修改 task agent）生成 task agent 代码，以 验证集 表现搜索架构。",
"selection": "ARC 小网格题 20；GPQA（研究生级科学问答基准） Diamond 32；DROP、MGSM、MMLU 各 128。代码报错时最多再反思修复 5 次。",
"evaluation": "ARC 60；GPQA（研究生级科学问答基准） 166；其他推理领域各 800。另测 MGSM 设计向 GSM8K（小学数学应用题基准）、GSM-Hard 及不同模型迁移。",
"isolation": "搜索用 验证集，测试集 分开。ARC 来自公开训练库内的抽样划分，不是 ARC 官方私有测试集。",
"roles": {
"executor": {
"value": "主实验候选 task agent 及对照方案用gpt-3.5-turbo-0125执行；迁移另用GPT-4o、Claude3 Haiku和Claude3.5 Sonnet。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2408.08435#A1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
]
},
"modifier": {
"value": "独立 meta-agent（负责设计或修改 task agent）用gpt-4o-2024-05-13读取档案、验证结果并生成候选 task agent 代码。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2408.08435#A1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
]
},
"seed": {
"value": "Meta Agent Search 在受接口约束的代码空间生成 task agent，初始档案含 CoT、Self-Refine 等人写设计。每候选先自查，验证报错可修至多五轮，再把代码与分数写回档案。",
"sources": [
{
"label": "§4；Algorithm 1 与数据采样",
"url": "https://arxiv.org/abs/2408.08435"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2408.08435#S4.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2408.08435#A5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "可靠解决复杂任务通常需要组合推理模块、工具和控制流程，但这些模块及其组合主要靠领域专家手调，耗费大量工程工作，也把可发现的设计限制在人类已考虑的范围内。作者因此研究能否自动搜索整个 agent 系统，而不只调整某个提示词。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2408.08435#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 系统的设计能否由模型自主完成，从人工预设的提示和模块组合扩展到更开放的系统结构，并获得跨任务、跨模型的适用性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2408.08435"
}
]
},
{
"key": "conclusion",
"value": "在多个任务领域发现优于手工对照的设计，并展示跨任务和跨模型迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2408.08435"
}
]
}
],
"fields": {
"object": "task agent 的完整代码，包括模型调用、提示和控制流程；外层 meta-agent 及搜索规则保持固定。"
}
},
"attributions": [
{
"tag": "org:ubc",
"label": "University of British Columbia",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2408.08435"
}
]
},
{
"tag": "person:jeff-clune",
"label": "Jeff Clune",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2408.08435"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2609.01437",
"title": "HarnessDev",
"url": "https://arxiv.org/abs/2609.01437",
"date": "2026-09-01",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"BenchmarkScore",
"CrossModel",
"HarnessCode",
"HeldOut",
"M1",
"org:bytedance-seed"
],
"fields": {
"本质定位": "**B-Harness**。把 harness development 拆成 **Creation**（minimal seed→完整 runnable infrastructure）和 **Evolution**（从自己创建的 harness 再改），同时测 capability+token efficiency。",
"被测系统 / seed harness / feedback": "6 creator LLMs、4 domains、5 downstream benchmarks；hidden eval；Evolution 用 downstream execution feedback。",
"证据边界与关键结论": "2,207 hidden downstream instances。新发现：code/search/research 离成熟人工 harness 仍远；Evolution gain 不稳定、held-out transfer 只部分成立，而且 harness gain 强依赖 runtime model。"
},
"details": [],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 68,
"fields": {
"论文": "**[HarnessDev](https://arxiv.org/abs/2609.01437)**",
"为什么仍应视为 Core / 强代表": "2026-09-01 新出的直接 benchmark：把 **Creation 与 Evolution** 分开，hidden downstream eval 同时测 capability、efficiency 和 cross-runtime-model transfer；对我们当前实验设计极直接，但因过新暂不按“经典热度”评级。"
}
},
{
"section": "A. Harness optimization / autonomous agent development",
"line": 196,
"fields": {
"时间": "2026-09-01",
"论文": "[HarnessDev](https://arxiv.org/abs/2609.01437)",
"级别": "**K / 新近重点**",
"它真正测什么": "**B-Harness**。把 harness development 拆成 **Creation**（minimal seed→完整 runnable infrastructure）和 **Evolution**（从自己创建的 harness 再改），同时测 capability+token efficiency。",
"被测系统 / seed harness / feedback": "6 creator LLMs、4 domains、5 downstream benchmarks；hidden eval；Evolution 用 downstream execution feedback。",
"证据边界与关键结论": "2,207 hidden downstream instances。新发现：code/search/research 离成熟人工 harness 仍远；Evolution gain 不稳定、held-out transfer 只部分成立，而且 harness gain 强依赖 runtime model。",
"标签": "`#HarnessCode #HeldOut #CrossModel #BenchmarkScore #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录明确标注 #HeldOut；是否参与选模仍需结合协议原文。",
"year": "2026",
"depth": [
"M1"
],
"priorityBasis": "用户指定阅读重点：ByteDance Seed / TokenWave 的 Self-Developing Agents 系列。",
"prioritySources": [
{
"label": "团队项目页",
"url": "https://self-developing-agents.github.io/"
}
],
"legacyCategories": [
"B-Harness"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 agent infrastructure 的能力拆成 Creation 与 Evolution：先从不带 task-solving policy 的弱 seed 构建 runnable harness，再从所建 harness 继续迭代。",
"novelty": "分别测从简陋起点建设框架和继续改进已有框架，并在统一接口下用不可见的正式任务评价生成系统。",
"object": "可执行 harness 的控制循环、工具编排、上下文、状态、生命周期与验证机制；被测模型权重固定。",
"executor": "由创建者对应模型执行的评测由创建者对应模型执行；统一更换执行模型的评测统一换Gemini3.1 Pro。创建者共六个：Opus4.8、GPT-5.5、Gemini3.1 Pro、DeepSeekV4Pro、Qwen3.7Max、Seed2.0Pro。",
"modifier": "上述六种创建者各自修改运行框架；GPT-5.5用Codex0.144.3开发，其余用Claude Code2.1.177。",
"roleContext": "6 creator LLMs、4 domains、5 downstream benchmarks；hidden eval；Evolution 用 downstream execution feedback。",
"seed": "Creation 从 policy-free 底座开始：接口能启动但没有主动执行循环、上下文管理、恢复、验证或停止策略。Evolution 则从 Creation 产出的 H₀ 开始，不能把两个阶段都称空白 seed。",
"fixed": "输入输出合同、下游评估服务与预算受控制。",
"verdict": "创建阶段读取公开开发任务的成绩与日志；继续进化时，SWE-Pro 和 Terminal-Bench 的真实执行结果帮助定位框架缺陷。创建能力与继续修订能力分别评价。",
"diagnosis": "creator 从具体任务 artifacts、日志与 per-task feedback 定位缺失机制。",
"update": "创建或改写执行循环、tool policy、context/state、recovery 与验证机制。",
"acceptance": "按冻结 commit 提交成对评估；最终由 creator 声明版本。",
"experiments": [
{
"name": "Creation",
"evolve": "公开 dev cases",
"selection": "creator 自主修订，冻结提交",
"test": "隐藏的下游评估；Self / Unified runtime 分开",
"isolation": "独立测试",
"note": ""
},
{
"name": "Evolution",
"evolve": "SWE-Pro-100 + Terminal-Bench-89 反馈",
"selection": "成对 full evaluation 与受限 probe；声明最终 commit",
"test": "SWE-Pro-630 在之后评估",
"isolation": "混合协议",
"note": "TB2 feedback 集不是 held-out；held-out 最优与最终声明版本也有差异。"
}
],
"takeaway": "2,207 hidden downstream instances。新发现：code/search/research 离成熟人工 harness 仍远；Evolution gain 不稳定、held-out transfer 只部分成立，而且 harness gain 强依赖 runtime model。",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2609.01437v1"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：Creation 从仅有被动接口的弱 seed 构建运行框架；Evolution 从已创建版本继续改代码，不训练权重。\n\n调试 / 选版本数据：Evolution 使用 SWE-Pro 100 题＋Terminal-Bench 89 题，提供反馈和固定小子集 探测任务。\n\n最终测试数据：Creation 覆盖 SWE-Pro、TB2.1、MLE-bench、EQ-Bench3、BrowseComp 共 2,207 实例；Evolution 冻结后另测不重叠 SWE-Pro 630 题。\n\n数据隔离与证据边界：630 题分数只在所有演化结束后计算，不影响编辑、停止或选版本；TB89 始终属于反馈集。",
"cycle": "创建者从公开任务产物、日志和逐题反馈定位缺失能力，编写或修改执行循环、工具规则、上下文、状态与错误恢复机制。提交确定的代码版本后作成对评估，最终版本由创建者选定。",
"train": "Creation 从仅有被动接口的弱 seed 构建运行框架；Evolution 从已创建版本继续改代码，不训练权重。",
"debug": "Evolution 使用 SWE-Pro 100 题＋Terminal-Bench 89 题，提供反馈和固定小子集 探测任务。",
"test": "Creation 覆盖 SWE-Pro、TB2.1、MLE-bench、EQ-Bench3、BrowseComp 共 2,207 实例；Evolution 冻结后另测不重叠 SWE-Pro 630 题。",
"isolation": "630 题分数只在所有演化结束后计算，不影响编辑、停止或选版本；TB89 始终属于反馈集。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "Creation 从 policy-free 底座开始：接口能启动但没有主动执行循环、上下文管理、恢复、验证或停止策略。Evolution 则从 Creation 产出的 H₀ 开始，不能把两个阶段都称空白 seed。",
"protocol": "**Creation 覆盖：**SWE-Pro public 731、Terminal-Bench 2.1 89、MLE-bench 75、EQ-Bench3 46、BrowseComp 1,266，共 2,207 个实例。\n\n**Evolution：**SWE-Pro 100＋TB 89 题提供反馈，最后用与反馈不重叠的 SWE-Pro 630 题评估所有正式版本，分数不反馈创建者。731 的 Creation 集合与 100＋630 的 Evolution 子集不是同一统计口径；TB 全集仍为反馈，不能标成盲测。",
"sections": "§3.3、Table 2、§4.3",
"source": "https://arxiv.org/abs/2609.01437",
"version": "2609.01437v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c1427fd3d4c25f9b4452cb8534cbeabfa39fad9e25d77c489e6d2da7737bc2d0",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "由创建者对应模型执行的评测由创建者对应模型执行；统一更换执行模型的评测统一换Gemini3.1 Pro。创建者共六个：Opus4.8、GPT-5.5、Gemini3.1 Pro、DeepSeekV4Pro、Qwen3.7Max、Seed2.0Pro。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01437#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "上述六种创建者各自修改运行框架；GPT-5.5用Codex0.144.3开发，其余用Claude Code2.1.177。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01437#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可执行 harness 的控制循环、工具编排、上下文、状态、生命周期与验证机制；被测模型权重固定。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "创建阶段读取公开开发任务的成绩与日志；继续进化时，SWE-Pro 和 Terminal-Bench 的真实执行结果帮助定位框架缺陷。创建能力与继续修订能力分别评价。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Creation 从 policy-free 底座开始：接口能启动但没有主动执行循环、上下文管理、恢复、验证或停止策略。Evolution 则从 Creation 产出的 H₀ 开始，不能把两个阶段都称空白 seed。",
"sources": [
{
"label": "§3.3、Table 2、§4.3",
"url": "https://arxiv.org/abs/2609.01437"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "创建者从公开任务产物、日志和逐题反馈定位缺失能力，编写或修改执行循环、工具规则、上下文、状态与错误恢复机制。提交确定的代码版本后作成对评估，最终版本由创建者选定。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Creation 从仅有被动接口的弱 seed 构建运行框架；Evolution 从已创建版本继续改代码，不训练权重。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Evolution 使用 SWE-Pro 100 题＋Terminal-Bench 89 题，提供反馈和固定小子集 探测任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Creation 覆盖 SWE-Pro、TB2.1、MLE-bench、EQ-Bench3、BrowseComp 共 2,207 实例；Evolution 冻结后另测不重叠 SWE-Pro 630 题。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "630 题分数只在所有演化结束后计算，不影响编辑、停止或选版本；TB89 始终属于反馈集。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "分别测从简陋起点建设框架和继续改进已有框架，并在统一接口下用不可见的正式任务评价生成系统。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2609.01437v1",
"version": "2609.01437v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "将从弱 seed 创建 harness 与继续优化已有 harness 分开测；统一接口和 hidden downstream evaluation 使“会造 agent”与“会用 agent”可以分开讨论。",
"feedbackCases": [
{
"label": "Creation：公开开发任务",
"data": "从无主动执行能力的接口起步，允许运行公开开发样例；正式题库不可见。",
"scoring": "开发环境提供样例的日志和分数；创建者自行实现执行、验证与停止逻辑。",
"visible": "只可见公开开发材料、运行日志和开发分数，不提供正式题目、答案或官方测试分数。",
"use": "创建者据此写出完整框架。不能把正式五基准的分数说成创建阶段的训练反馈。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"judgment": "创建者自行设计本地测试；不存在统一公开的自测判分器"
},
{
"label": "Evolution：SWE-Pro / Terminal-Bench",
"data": "SWE-Pro 100 题和 Terminal-Bench 89 题，以及固定小子集探测。",
"scoring": "SWE-Pro 的仓库验收测试检查补丁；终端任务的可执行验收检查产物。",
"visible": "执行记录、测试结果及开发分数交给修改者；平台不代做归因、自动选最好版本或回滚。",
"use": "修改者自行选择何时再测、怎样修改和最终交哪个框架；冻结后另测不重叠的 SWE-Pro 630 题。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"judgment": "SWE-Pro 仓库测试；Terminal-Bench 容器验收测试"
},
{
"label": "Creation 的正式测试",
"data": "SWE-Pro、TB2.1、MLE-bench、EQ-Bench3、BrowseComp 共 2,207 个实例。",
"scoring": "软件与终端任务用可执行验收；ML 任务看实验指标；情感/交互与搜索问答按各基准评分协议。不能把五种测试统一解释为代码单元测试。",
"visible": "正式成绩由评测端记录，创建期间不可见。",
"use": "衡量已创建框架的跨任务能力；不是在正式五基准上自由调试后再当盲测。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"judgment": "软件／终端用程序验收，ML 用实验指标；其余按各基准评审协议"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0,
1
],
"testCases": [
2,
1
],
"evolution": "Creation 从仅有被动接口的弱 seed 构建运行框架；Evolution 从已创建版本继续改代码，不训练权重。",
"selection": "Evolution 使用 SWE-Pro 100 题＋Terminal-Bench 89 题，提供反馈和固定小子集 探测任务。",
"evaluation": "Creation 覆盖 SWE-Pro、TB2.1、MLE-bench、EQ-Bench3、BrowseComp 共 2,207 实例；Evolution 冻结后另测不重叠 SWE-Pro 630 题。",
"isolation": "630 题分数只在所有演化结束后计算，不影响编辑、停止或选版本；TB89 始终属于反馈集。",
"roles": {
"executor": {
"value": "由创建者对应模型执行的评测由创建者对应模型执行；统一更换执行模型的评测统一换Gemini3.1 Pro。创建者共六个：Opus4.8、GPT-5.5、Gemini3.1 Pro、DeepSeekV4Pro、Qwen3.7Max、Seed2.0Pro。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01437#S4.SS1"
}
]
},
"modifier": {
"value": "上述六种创建者各自修改运行框架；GPT-5.5用Codex0.144.3开发，其余用Claude Code2.1.177。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2609.01437#A5.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01437#S4.SS1"
}
]
},
"seed": {
"value": "Creation 从 policy-free 底座开始：接口能启动但没有主动执行循环、上下文管理、恢复、验证或停止策略。Evolution 则从 Creation 产出的 H₀ 开始，不能把两个阶段都称空白 seed。",
"sources": [
{
"label": "§3.3、Table 2、§4.3",
"url": "https://arxiv.org/abs/2609.01437"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"selection": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"evaluation": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
],
"isolation": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2609.01437#S4.SS3.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01437#S5.SS0.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有成绩同时受模型和人工运行框架影响，无法回答模型能否自己建设并持续改进这种基础设施。框架优化还可能只适配创建者、记住开发题或暗中损害其他能力，因此需要直接评测可运行、可持久复用的框架建设，而不以普通代码编辑或开发集涨分替代。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.01437#S1"
}
]
},
{
"key": "position",
"value": "评测模型能否自主建设并持续改进 agent 的运行基础设施，区分“会在现成系统中做任务”与“会设计有效系统”这两种能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01437"
}
]
},
{
"key": "conclusion",
"value": "生成系统的优势随领域变化；进化收益不稳定，而且依赖运行该运行框架的模型。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2609.01437"
}
]
}
],
"fields": {
"seed": "创建阶段只有能启动的最小接口，缺少主动执行循环、上下文管理、恢复和验证策略；进化阶段则从自己创建的完整运行框架继续改进。",
"verdict": "创建阶段读取公开开发任务的成绩与日志；继续进化时，SWE-Pro 和 Terminal-Bench 的真实执行结果帮助定位框架缺陷。创建能力与继续修订能力分别评价。",
"object": "可执行 harness 的控制循环、工具编排、上下文、状态、生命周期与验证机制；被测模型权重固定。"
}
},
"attributions": [
{
"tag": "org:bytedance-seed",
"label": "ByteDance Seed",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.01437"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.03764",
"title": "GDPevo",
"url": "https://arxiv.org/abs/2608.03764",
"date": "2026-08-04",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"CapabilityCeiling",
"GoldLabel",
"HeldOut",
"M0",
"MemoryContent",
"Skill"
],
"fields": {
"本质定位": "**B-Lifelong / B-Reliability**。用 rule hybridization 把 enterprise workflow 规则分散到 train tasks，再重组到 held-out test，让 transfer 可归因；还给 fully-informed oracle ceiling。",
"被测系统 / feedback": "4 agents，4 supervision types；CRM/ERP/finance/health/legal/data workflows。",
"证据边界与关键结论": "5 train + 5 held-out/test per group；evolution 最多 +16.44，但 best 仍远低于 **91.6% oracle**。非常适合我们“能力上限”问题。"
},
"details": [],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 69,
"fields": {
"论文": "**[GDPevo](https://arxiv.org/abs/2608.03764)**",
"为什么仍应视为 Core / 强代表": "evolution-native held-out business benchmark，并给 fully-informed **oracle ceiling**；对我们“是否接近/突破能力上限”问题非常直接。"
}
},
{
"section": "B. Continual / memory / skill self-evolution",
"line": 216,
"fields": {
"时间": "2026-08-04",
"论文": "[GDPevo](https://arxiv.org/abs/2608.03764)",
"级别": "**K / 我们重点**",
"它真正测什么": "**B-Lifelong / B-Reliability**。用 rule hybridization 把 enterprise workflow 规则分散到 train tasks，再重组到 held-out test，让 transfer 可归因；还给 fully-informed oracle ceiling。",
"被测系统 / feedback": "4 agents，4 supervision types；CRM/ERP/finance/health/legal/data workflows。",
"证据边界与关键结论": "5 train + 5 held-out/test per group；evolution 最多 +16.44，但 best 仍远低于 **91.6% oracle**。非常适合我们“能力上限”问题。",
"标签": "`#MemoryContent #Skill #HeldOut #GoldLabel #CapabilityCeiling #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录明确标注 #HeldOut；是否参与选模仍需结合协议原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "用 rule hybridization 把 enterprise workflow 规则分散到 train tasks，再重组到 held-out test，让 transfer 可归因；还给 fully-informed oracle ceiling。",
"novelty": "把企业业务规则分散在学习案例中，再在测试案例里重新组合；另设提前拿到全部规则的对照，帮助判断瓶颈是缺少经验还是不会运用规则。",
"object": "各任务组的 SKILL.md 文件：业务规则、操作步骤、输出格式和失败经验。修改的是技能内容。",
"executor": "Codex或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）加载所学技能，候选模型为GPT-5.5、Opus4.8、GLM5.2、DeepSeek-V4-Pro-Preview。",
"modifier": "对应模型–运行框架配置从5道训练任务构建技能；评估另启动新 task agent 加载该技能，避免沿用训练会话。",
"roleContext": "4 agents，4 supervision types；CRM/ERP/finance/health/legal/data workflows。",
"seed": "作者自建的企业任务环境，提供可查询的业务数据和按规则判分的程序。被测 task agent 从训练任务提炼可复用技能，测试时携带技能库；评测基准本身不限定只能改技能，也不训练一个统一的任务模型。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "三种设置给的信息不同：自主探索只见环境；示例监督额外给标准答案；反思监督给程序判分并允许三轮反思。测试考察学到的业务规则能否用于新案例。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "5 train + 5 held-out/test per group；evolution 最多 +16.44，但 best 仍远低于 **91.6% oracle**。非常适合我们“能力上限”问题。"
}
],
"takeaway": "5 train + 5 held-out/test per group；evolution 最多 +16.44，但 best 仍远低于 **91.6% oracle**。非常适合我们“能力上限”问题。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：V1 12 组、V2 24 组，每组 5 道训练题；环境包括 CRM、ERP、财务、医疗等业务流程。\n\n调试 / 选版本数据：训练组的题目与环境；额外反馈依 self/fewshot/reflect 设置提供。\n\n最终测试数据：每组另有 5 道题，合计 180 道，重新组合训练阶段涉及的业务规则。\n\n数据隔离与证据边界：同业务环境、不同任务；训练完才交给新 task agent 测试，比较有/无技能的增益。",
"cycle": "在同组训练环境尝试任务，把允许看到的答案或程序反馈总结为技能；随后由新 task agent 加载该文件执行测试，三种监督强度分别比较。",
"train": "V1 12 组、V2 24 组，每组 5 道训练题；环境包括 CRM、ERP、财务、医疗等业务流程。",
"debug": "训练组的题目与环境；额外反馈依 self/fewshot/reflect 设置提供。",
"test": "每组另有 5 道题，合计 180 道，重新组合训练阶段涉及的业务规则。",
"isolation": "同业务环境、不同任务；训练完才交给新 task agent 测试，比较有/无技能的增益。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建的企业任务环境，提供可查询的业务数据和按规则判分的程序。被测 agent 从训练任务提炼可复用 skill，测试时携带技能库；benchmark 本身不限定只能改技能，也不训练一个统一的任务模型。",
"protocol": "**数据构造：**GDPevo 把 CRM、ERP、财务、医疗、法律和数据工作流拆成业务规则，在训练题中分散展示规则，再在测试题中重新组合。V1 为 12 组共 120 题，V2 为 24 组共 240 题；每组均为 5 道训练题、5 道留出测试题，正文默认合用 V1 与 V2。\n\n**进化与测试：**在每组 5 道训练题上构建 skill，再测同组 5 道未见题。fewshot 给标准答案；reflect 给程序评分并迭代 3 轮；self 只允许探索题目与环境。三种反馈不可混写；额外独立验证集本轮未核实。",
"sections": "数据构造与实验设置",
"source": "https://arxiv.org/abs/2608.03764",
"version": "2608.03764v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e9ddd3252839d022563e6c547acf7a7caac58ea08da8c843a87f0e40fdcb6dca",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Codex或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）加载所学技能，候选模型为GPT-5.5、Opus4.8、GLM5.2、DeepSeek-V4-Pro-Preview。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应模型–运行框架配置从5道训练任务构建技能；评估另启动新 task agent 加载该技能，避免沿用训练会话。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "各任务组的 SKILL.md 文件：业务规则、操作步骤、输出格式和失败经验。修改的是技能内容。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "三种设置给的信息不同：自主探索只见环境；示例监督额外给标准答案；反思监督给程序判分并允许三轮反思。测试考察学到的业务规则能否用于新案例。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建的企业任务环境，提供可查询的业务数据和按规则判分的程序。被测 task agent 从训练任务提炼可复用技能，测试时携带技能库；评测基准本身不限定只能改技能，也不训练一个统一的任务模型。",
"sources": [
{
"label": "数据构造与实验设置",
"url": "https://arxiv.org/abs/2608.03764"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在同组训练环境尝试任务，把允许看到的答案或程序反馈总结为技能；随后由新 task agent 加载该文件执行测试，三种监督强度分别比较。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "V1 12 组、V2 24 组，每组 5 道训练题；环境包括 CRM、ERP、财务、医疗等业务流程。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练组的题目与环境；额外反馈依 self/fewshot/reflect 设置提供。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "每组另有 5 道题，合计 180 道，重新组合训练阶段涉及的业务规则。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同业务环境、不同任务；训练完才交给新 task agent 测试，比较有/无技能的增益。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把企业业务规则分散在学习案例中，再在测试案例里重新组合；另设提前拿到全部规则的对照，帮助判断瓶颈是缺少经验还是不会运用规则。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.03764v1",
"version": "2608.03764v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把企业规则拆散到学习任务、在测试中重组，并提供全部规则已知的 oracle；借此区分缺经验、不会迁移与模型能力上限。",
"feedbackCases": [
{
"label": "GDPevo：self / fewshot / reflect",
"data": "每组五道训练任务，覆盖 CRM、ERP、财务、医疗等业务环境。",
"scoring": "reflect 用作者写的确定性规则评分脚本检查加权评分点，返回分数，连续三轮；fewshot 直接给五道训练题的标准答案；self 只让 agent 探索题目与环境，不提供答案相关监督。",
"visible": "三种设置分别获得规则分数、标准答案、或只有环境观察；没有统一的 语言模型 裁判。",
"use": "从训练组形成文字技能，随后在每组另外五道规则重组题上测试；不更新模型参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"judgment": "reflect：人工编写的规则脚本按加权检查项评分；fewshot：直接给标注答案；self：无答案反馈"
},
{
"label": "GDPevo：最终测试",
"data": "每组与训练不重叠的五道题；该版本主实验合计 180 道测试任务。",
"scoring": "逐任务确定性脚本按业务规则检查输出，评分点和权重由基准预设。",
"visible": "测试容器不挂载答案或评分器源文件；测试反馈不用于重新生成训练技能。",
"use": "检验业务规则能否迁移；任务组版本与数量以实验对应版本为准。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"judgment": "人工编写的确定性业务检查脚本，按预设权重计分"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "V1 12 组、V2 24 组，每组 5 道训练题；环境包括 CRM、ERP、财务、医疗等业务流程。",
"selection": "训练组的题目与环境；额外反馈依 self/fewshot/reflect 设置提供。",
"evaluation": "每组另有 5 道题，合计 180 道，重新组合训练阶段涉及的业务规则。",
"isolation": "同业务环境、不同任务；训练完才交给新 task agent 测试，比较有/无技能的增益。",
"roles": {
"executor": {
"value": "Codex或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）加载所学技能，候选模型为GPT-5.5、Opus4.8、GLM5.2、DeepSeek-V4-Pro-Preview。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "对应模型–运行框架配置从5道训练任务构建技能；评估另启动新 task agent 加载该技能，避免沿用训练会话。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px3"
}
]
},
"seed": {
"value": "作者自建的企业任务环境，提供可查询的业务数据和按规则判分的程序。被测 task agent 从训练任务提炼可复用技能，测试时携带技能库；评测基准本身不限定只能改技能，也不训练一个统一的任务模型。",
"sources": [
{
"label": "数据构造与实验设置",
"url": "https://arxiv.org/abs/2608.03764"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.03764#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03764#S4.SS1.SSS0.Px5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "专为进化设计的基准能安排可迁移能力，却较少覆盖金融、法律、医疗等困难业务；从静态任务改造的基准虽更丰富，原有训练/测试关系又未保证测到所学能力。因此进化后涨分难归因，作者希望同时具备真实业务覆盖和明确的经验迁移关系。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.03764#S1"
}
]
},
{
"key": "position",
"value": "评测企业工作流 agent 能否从既往案例学到隐含业务规则，并在新案例中组合运用这些规则，而非只复用旧题答案。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03764"
}
]
},
{
"key": "conclusion",
"value": "形成可自动扩展的企业工作流基准；进化有收益，但仍明显低于预先知道全部规则的对照。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.03764"
}
]
}
],
"fields": {
"object": "各任务组的 SKILL.md 文件：业务规则、操作步骤、输出格式和失败经验。修改的是技能内容。",
"verdict": "三种设置给的信息不同：自主探索只见环境；示例监督额外给标准答案；反思监督给程序判分并允许三轮反思。测试考察学到的业务规则能否用于新案例。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2602.22480",
"title": "VeRO",
"url": "https://arxiv.org/abs/2602.22480",
"date": "2026-02-25",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"Archive",
"BenchmarkScore",
"HarnessCode",
"M1",
"OfflineSearch"
],
"fields": {
"本质定位": "**B-Harness**。把 harness optimization 变成 auditable outer-loop task：Versioning + Rewards + Observations，结构化记录 stochastic LLM harness 的中间 traces 和 outcome。",
"被测系统 / seed harness / feedback": "coding-agent optimizer 改 target harness；VeRO 提 version snapshots、budgeted eval、structured traces。",
"证据边界与关键结论": "主要贡献是 measurement substrate；后来的 HarnessOpt-Bench 可看成把这条线标准化得更严格。"
},
"details": [],
"occurrences": [
{
"section": "Core-扩展：同样是领域代表作，按问题补读",
"line": 70,
"fields": {
"论文": "**[VeRO](https://arxiv.org/abs/2602.22480)**",
"为什么仍应视为 Core / 强代表": "harness optimizer 的 outer instrumentation / audit substrate：versioning、budgeted rewards、structured observations，是 HarnessOpt-Bench 等统一评测路线的早期直接节点。"
}
},
{
"section": "A. Harness optimization / autonomous agent development",
"line": 192,
"fields": {
"时间": "2026-02-25",
"论文": "[VeRO / VeRO-Bench](https://arxiv.org/abs/2602.22480)",
"级别": "**K**",
"它真正测什么": "**B-Harness**。把 harness optimization 变成 auditable outer-loop task：Versioning + Rewards + Observations，结构化记录 stochastic LLM harness 的中间 traces 和 outcome。",
"被测系统 / seed harness / feedback": "coding-agent optimizer 改 target harness；VeRO 提 version snapshots、budgeted eval、structured traces。",
"证据边界与关键结论": "主要贡献是 measurement substrate；后来的 HarnessOpt-Bench 可看成把这条线标准化得更严格。",
"标签": "`#HarnessCode #BenchmarkScore #OfflineSearch #Archive #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"B-Harness"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 harness optimization 变成 auditable outer-loop task：Versioning + Rewards + Observations，结构化记录 stochastic LLM harness 的中间 traces 和 outcome。",
"novelty": "保存 agent 代码版本、执行观察与任务奖励，使代码 agent 的框架优化过程可以追踪和比较；数据隔离需要按具体任务分别读。",
"object": "目标 agent 的 Python 实现，包括提示、工具及调用编排。",
"executor": "主优化目标为GPT-4.1-mini的 task agent；冻结优化后的程序，再换GPT-4.1测跨模型效果。",
"modifier": "VeRO-Agent或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）负责改代码；默认Claude Sonnet4.5，调度角色另用Claude Opus4.5、GPT-5.2-Codex。",
"roleContext": "coding-agent optimizer 改 target harness；VeRO 提 version snapshots、budgeted eval、structured traces。",
"seed": "论文比较两个明确不同的起点：Pawn 有 4 个工具、基础搜索、文本文件读取、Python 和网页抓取，25 行系统提示、最多 20 轮；Knight 有 6 个工具，增加 Wikipedia、反思、复杂文件读取及更丰富 Python 库，140 行提示、最多 40 轮。不能把两者统称为一个基础 task agent。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "ExperimentRunner 返回任务成绩和结构化执行反馈，调用次数受预算限制。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "VeRO budget ablation",
"evolve": "5 benchmark-study tasks；预算按 ExperimentRunner 调用计",
"selection": "通常选 best-validation commit；GPQA 无 validation，按 train 选",
"test": "有 test 时用 test，否则使用 validation",
"isolation": "不同任务混合口径",
"note": "Claude Sonnet 4.5 optimizer；每组 3 runs。不能把所有 holdout 统一当成 untouched test。"
}
],
"takeaway": "主要贡献是 measurement substrate；后来的 HarnessOpt-Bench 可看成把这条线标准化得更严格。",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2602.22480v4"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 50、GPQA（研究生级科学问答基准） Diamond 98、MATH 59、τ-bench Retail 100、SimpleQA 46。\n\n调试 / 选版本数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 87、MATH 60、τ-bench Retail 20、SimpleQA 45；GPQA（研究生级科学问答基准） 直接使用训练成绩。\n\n最终测试数据：GPQA（研究生级科学问答基准） 100、MATH 486、τ-bench Retail 115、SimpleQA 80；GAIA（需要检索、推理和使用工具的通用助理任务基准） 最终仍报告 87 验证集。\n\n数据隔离与证据边界：四项有最终 测试集；GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 同时参与选择和报告，不能视为额外盲测。SimpleQA 先筛出三个模型均失败的 171 题再划分。",
"cycle": "编程 task agent 修改代码、提交版本化快照、读取运行反馈继续修复；按 验证集 选版本，GPQA（研究生级科学问答基准） 无 验证集 时按 训练集 选。",
"train": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 50、GPQA（研究生级科学问答基准） Diamond 98、MATH 59、τ-bench Retail 100、SimpleQA 46。",
"debug": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 87、MATH 60、τ-bench Retail 20、SimpleQA 45；GPQA（研究生级科学问答基准） 直接使用训练成绩。",
"test": "GPQA（研究生级科学问答基准） 100、MATH 486、τ-bench Retail 115、SimpleQA 80；GAIA（需要检索、推理和使用工具的通用助理任务基准） 最终仍报告 87 验证集。",
"isolation": "四项有最终 测试集；GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 同时参与选择和报告，不能视为额外盲测。SimpleQA 先筛出三个模型均失败的 171 题再划分。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "论文比较两个明确不同的起点：Pawn 有 4 个工具、基础搜索、文本文件读取、Python 和网页抓取，25 行系统提示、最多 20 轮；Knight 有 6 个工具，增加 Wikipedia、反思、复杂文件读取及更丰富 Python 库，140 行提示、最多 40 轮。不能把两者统称为一个基础 agent。",
"protocol": "**数据与选版本：**VeRO 按任务提供训练、验证接口，优化代码后按验证表现选版本；存在独立测试集时再测试，否则报告验证集。GPQA 的训练/验证调用有预算限制。\n\n**GAIA：**筛选不依赖图像和音频的 137 道文本题，从中抽 50 道供优化；87 道验证题用于报告，未再构造独立测试集。因验证参与选版本，不能把这 87 题称为选模后完全未接触的测试集。其他任务的逐项数量本轮仍待核实，不能把 GAIA 的划分套用到全部任务。",
"sections": "任务数据说明；Table 10",
"source": "https://arxiv.org/abs/2602.22480",
"version": "2602.22480v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f8c8078cf12fe1fbc273ad51cfd23da3ccdfd37f0e74dedf484f17a86e9efaff",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主优化目标为GPT-4.1-mini的 task agent；冻结优化后的程序，再换GPT-4.1测跨模型效果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.22480#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2602.22480#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "VeRO-Agent或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）负责改代码；默认Claude Sonnet4.5，调度角色另用Claude Opus4.5、GPT-5.2-Codex。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.22480#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2602.22480#S5.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "目标 agent 的 Python 实现，包括提示、工具及调用编排。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "ExperimentRunner 返回任务成绩和结构化执行反馈，调用次数受预算限制。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "论文比较两个明确不同的起点：Pawn 有 4 个工具、基础搜索、文本文件读取、Python 和网页抓取，25 行系统提示、最多 20 轮；Knight 有 6 个工具，增加 Wikipedia、反思、复杂文件读取及更丰富 Python 库，140 行提示、最多 40 轮。不能把两者统称为一个基础 task agent。",
"sources": [
{
"label": "任务数据说明；Table 10",
"url": "https://arxiv.org/abs/2602.22480"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "编程 task agent 修改代码、提交版本化快照、读取运行反馈继续修复；按 验证集 选版本，GPQA（研究生级科学问答基准） 无 验证集 时按 训练集 选。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 50、GPQA（研究生级科学问答基准） Diamond 98、MATH 59、τ-bench Retail 100、SimpleQA 46。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 87、MATH 60、τ-bench Retail 20、SimpleQA 45；GPQA（研究生级科学问答基准） 直接使用训练成绩。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GPQA（研究生级科学问答基准） 100、MATH 486、τ-bench Retail 115、SimpleQA 80；GAIA（需要检索、推理和使用工具的通用助理任务基准） 最终仍报告 87 验证集。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "四项有最终 测试集；GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 同时参与选择和报告，不能视为额外盲测。SimpleQA 先筛出三个模型均失败的 171 题再划分。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "保存 agent 代码版本、执行观察与任务奖励，使代码 agent 的框架优化过程可以追踪和比较；数据隔离需要按具体任务分别读。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2602.22480v4",
"version": "2602.22480v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用版本、奖励和观察记录把优化过程变成可审计实验；不同任务的 holdout 口径不完全相同，不能统一写成独立 test。",
"feedbackCases": [
{
"label": "GAIA / SimpleQA",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 训练 50、验证/最终 87；SimpleQA 训练 46、验证 45、测试 80",
"scoring": "按问答题参考答案评价输出；框架通过 ExperimentRunner 返回任务分数和结构化执行记录。",
"visible": "训练样例的输入、输出、错误和逐题分数供优化者诊断；论文不同权限实验允许访问的验证/测试信息不同，不能统称全部保密。",
"use": "在评估调用预算内搜索候选；GAIA（需要检索、推理和使用工具的通用助理任务基准） 的最终 87 题仍是验证集，GPQA（研究生级科学问答基准） 用训练分数选版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"judgment": "GAIA／SimpleQA 对照参考答案；本文未明确两项各自的规则／模型判分实现"
},
{
"label": "GPQA Diamond / MATH",
"data": "GPQA（研究生级科学问答基准） 训练 98、测试 100；MATH 训练 59、验证 60、测试 486",
"scoring": "GPQA（研究生级科学问答基准） 比较参考选项；MATH 根据参考数学答案计分。",
"visible": "训练样例的输入、输出、错误和逐题分数供优化者诊断；论文不同权限实验允许访问的验证/测试信息不同，不能统称全部保密。",
"use": "在评估调用预算内搜索候选；GAIA（需要检索、推理和使用工具的通用助理任务基准） 的最终 87 题仍是验证集，GPQA（研究生级科学问答基准） 用训练分数选版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"judgment": "GPQA 比参考选项；MATH 比参考数学答案，未逐项展开解析器"
},
{
"label": "τ-bench Retail",
"data": "训练 100、验证 20、测试 115",
"scoring": "按客服任务的数据库/目标状态验收工具操作。",
"visible": "训练样例的输入、输出、错误和逐题分数供优化者诊断；论文不同权限实验允许访问的验证/测试信息不同，不能统称全部保密。",
"use": "在评估调用预算内搜索候选；GAIA（需要检索、推理和使用工具的通用助理任务基准） 的最终 87 题仍是验证集，GPQA（研究生级科学问答基准） 用训练分数选版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"judgment": "τ-bench 官方程序比较最终数据库与目标状态"
},
{
"label": "TerminalBench-2 案例",
"data": "89 道终端任务，比较两种优化接口与三个运行",
"scoring": "判分器返回正奖励计为通过；崩溃或超时单独计错误，正常完成但奖励为零也不算通过。",
"visible": "训练样例的输入、输出、错误和逐题分数供优化者诊断；论文不同权限实验允许访问的验证/测试信息不同，不能统称全部保密。",
"use": "在评估调用预算内搜索候选；GAIA（需要检索、推理和使用工具的通用助理任务基准） 的最终 87 题仍是验证集，GPQA（研究生级科学问答基准） 用训练分数选版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"judgment": "Terminal-Bench 原生任务验收程序"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 50、GPQA（研究生级科学问答基准） Diamond 98、MATH 59、τ-bench Retail 100、SimpleQA 46。",
"selection": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 87、MATH 60、τ-bench Retail 20、SimpleQA 45；GPQA（研究生级科学问答基准） 直接使用训练成绩。",
"evaluation": "GPQA（研究生级科学问答基准） 100、MATH 486、τ-bench Retail 115、SimpleQA 80；GAIA（需要检索、推理和使用工具的通用助理任务基准） 最终仍报告 87 验证集。",
"isolation": "四项有最终 测试集；GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 同时参与选择和报告，不能视为额外盲测。SimpleQA 先筛出三个模型均失败的 171 题再划分。",
"roles": {
"executor": {
"value": "主优化目标为GPT-4.1-mini的 task agent；冻结优化后的程序，再换GPT-4.1测跨模型效果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.22480#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2602.22480#S5.SS2"
}
]
},
"modifier": {
"value": "VeRO-Agent或Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）负责改代码；默认Claude Sonnet4.5，调度角色另用Claude Opus4.5、GPT-5.2-Codex。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2602.22480#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.22480#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2602.22480#S5.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "论文比较两个明确不同的起点：Pawn 有 4 个工具、基础搜索、文本文件读取、Python 和网页抓取，25 行系统提示、最多 20 轮；Knight 有 6 个工具，增加 Wikipedia、反思、复杂文件读取及更丰富 Python 库，140 行提示、最多 40 轮。不能把两者统称为一个基础 task agent。",
"sources": [
{
"label": "任务数据说明；Table 10",
"url": "https://arxiv.org/abs/2602.22480"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"selection": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"evaluation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
],
"isolation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2602.22480#A1.SS2.SSS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "构建 agent 仍主要靠人工观察失败、改提示和工具，过程慢且难扩展；已有 agent 代码搜索也较少将优化作为开放的编程任务来评估。如果缺少受控运行基础设施，优化器还可能接触评测内容或无限耗用预算，因此需要能执行、观察并约束完整优化过程的环境。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2602.22480#S1"
}
]
},
{
"key": "position",
"value": "面向包含随机模型调用的 agent 软件，建立可比较的评测环境，考察代码 agent 能否诊断并有效改进另一个 agent 的运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2602.22480"
}
]
},
{
"key": "conclusion",
"value": "使不同代码 agent 的运行框架优化能力能被系统比较，并可追踪哪些编辑产生实际收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2602.22480"
}
]
}
],
"fields": {
"object": "目标 agent 的 Python 实现，包括提示、工具及调用编排。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2410.04444",
"title": "Gödel Agent: A Self-Referential Agent Framework for Recursive Self-Improvement",
"url": "https://arxiv.org/abs/2410.04444",
"date": "2024-10-06",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"HarnessCode",
"Improver",
"M2",
"OfflineSearch",
"SameModel",
"org:peking"
],
"fields": {
"本质定位": "agent 可 inspect/modify 自己整个 executable logic，包括负责分析和修改自己的 procedure；更新后新 agent 继续下一轮。",
"谁来改 → 谁执行；基础 harness": "当前 agent/self-mod routine → 后代 agent；seed 是可执行 agent program。",
"Feedback / evidence": "empirical benchmark score 选 descendant；outer selection 仍固定。",
"什么在变": "agent code/logic，包括 self-mod procedure。",
"谁来改 / 谁执行": "**改**：当前 agent 自己。
**执行**：修改后的同一 agent。",
"基础 harness": "CoT agent + think/error/Python/Bash/LLM API 等基础工具。",
"Feedback": "task utility/validation score + runtime evidence。",
"Evolution → Eval": "DROP/MGSM/MMLU/GPQA validation evolution → test；Game24 case。",
"Meta-depth": "M2。",
"相对之前真正新增什么": "相对 ADAS 的本质差别：不再明确分开 fixed meta-agent 与 target agent；self-modification logic 也位于可编辑 program 中。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 133,
"fields": {
"时间": "2024-10-06",
"论文": "[Gödel Agent](https://arxiv.org/abs/2410.04444)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**F / Meta**。直接让 agent 读取并重写自己的 executable logic，连后续 self-modification logic 也在代码里；比只改 task prompt 更接近 program-level self-reference。",
"谁来改 → 谁执行；基础 harness": "当前 agent/self-mod routine → 后代 agent；seed 是可执行 agent program。",
"Feedback / evidence": "empirical benchmark score 选 descendant；outer selection 仍固定。",
"标签": "`#HarnessCode #Improver #SameModel #BenchmarkScore #OfflineSearch #M2`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 321,
"fields": {
"优先级": "**K**",
"时间": "2024-10-06",
"论文": "[Gödel Agent: A Self-Referential Agent Framework for Recursive Self-Improvement](https://arxiv.org/abs/2410.04444)",
"本质定位": "agent 可 inspect/modify 自己整个 executable logic，包括负责分析和修改自己的 procedure；更新后新 agent 继续下一轮。",
"什么在变": "agent code/logic，包括 self-mod procedure。",
"谁来改 / 谁执行": "**改**：当前 agent 自己。
**执行**：修改后的同一 agent。",
"基础 harness": "CoT agent + think/error/Python/Bash/LLM API 等基础工具。",
"Feedback": "task utility/validation score + runtime evidence。",
"Evolution → Eval": "DROP/MGSM/MMLU/GPQA validation evolution → test；Game24 case。",
"Meta-depth": "M2。",
"相对之前真正新增什么": "相对 ADAS 的本质差别：不再明确分开 fixed meta-agent 与 target agent；self-modification logic 也位于可编辑 program 中。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M2"
],
"legacyCategories": [
"F",
"H-Full",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "agent 可 inspect/modify 自己整个 executable logic，包括负责分析和修改自己的 procedure；更新后新 agent 继续下一轮。",
"novelty": "把解题流程和调用自修改的逻辑放进同一可编辑程序，使用实际执行反馈决定修改方向。",
"object": "agent 执行逻辑及负责自修改的程序。",
"executor": "受限主实验用 gpt-3.5-turbo-0125 执行优化后的策略，与基线保持相同模型；开放实验允许 agent 求助 GPT-4o，需与受限结果区分。",
"modifier": "gpt-4o-2024-05-13 驱动 Gödel Agent 读取反馈并改写策略。修改代码的模型与受限测试时答题的 GPT-3.5 不同。",
"roleContext": "**改**：当前 agent 自己。
**执行**：修改后的同一 agent。",
"seed": "从逐步推理的任务求解代码开始，task agent 另有思考、错误记录、Python、Bash 与模型调用等工具，可以直接编辑自身代码；不是只有文字反思的记忆系统。不同任务有各自初始任务求解模型，例如 Game of 24 提供初始 CoT 求解函数。",
"fixed": "",
"verdict": "任务预设的效用或验证分数，以及实际运行记录。效用指该任务用来衡量方案好坏的目标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "DROP/MGSM/MMLU/GPQA validation evolution → test；Game24 case。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不训练模型权重；在目标任务的 验证集 上进行源码搜索，每任务 6 次独立循环、每次最多 30 轮。\n\n调试 / 选版本数据：GPQA（研究生级科学问答基准） 32 题；DROP、MGSM、MMLU 各 128 题，用于改进期间计分。\n\n最终测试数据：GPQA（研究生级科学问答基准） 剩余 166 题；DROP、MGSM、MMLU 各 800 题，均用 GPT-3.5 运行候选。\n\n数据隔离与证据边界：主实验 验证／测试 分开；Game of 24 等展示案例不套用主表划分。",
"cycle": "task agent 自读源码与任务效用，调用修改/执行动作改变自己的求解与改进逻辑，再通过任务运行结果继续迭代；不要求形式化证明。",
"train": "不训练模型权重；在目标任务的 验证集 上进行源码搜索，每任务 6 次独立循环、每次最多 30 轮。",
"debug": "GPQA（研究生级科学问答基准） 32 题；DROP、MGSM、MMLU 各 128 题，用于改进期间计分。",
"test": "GPQA（研究生级科学问答基准） 剩余 166 题；DROP、MGSM、MMLU 各 800 题，均用 GPT-3.5 运行候选。",
"isolation": "主实验 验证／测试 分开；Game of 24 等展示案例不套用主表划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "从逐步推理的任务求解代码开始，agent 另有思考、错误记录、Python、Bash 与模型调用等工具，可以直接编辑自身代码；不是只有文字反思的记忆系统。不同任务有各自初始 solver，例如 Game of 24 提供初始 CoT 求解函数。",
"protocol": "**搜索与测试：**GPQA 科学题用 32 题作验证、其余 166 题作测试；其他领域按论文设置抽 128 题验证、800 题测试。验证结果驱动代码搜索，最终测试与验证分开。论文还展示 MGSM 和 Game of 24 的具体求解程序；不能把“其他领域 128/800”无条件套到这些展示案例上，逐任务对应仍需查表。",
"sections": "实验数据采样；初始与演化代码示例",
"source": "https://arxiv.org/abs/2410.04444",
"version": "2410.04444v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "4620ea9cc430f8f678834edbe08934e673a21e9e27915160c8d3def6cd6f6b6f",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "受限主实验用 gpt-3.5-turbo-0125 执行优化后的策略，与基线保持相同模型；开放实验允许 agent 求助 GPT-4o，需与受限结果区分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2410.04444#S5.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "gpt-4o-2024-05-13 驱动 Gödel Agent 读取反馈并改写策略。修改代码的模型与受限测试时答题的 GPT-3.5 不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2410.04444#S5.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "agent 执行逻辑及负责自修改的程序。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务预设的效用或验证分数，以及实际运行记录。效用指该任务用来衡量方案好坏的目标。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从逐步推理的任务求解代码开始，task agent 另有思考、错误记录、Python、Bash 与模型调用等工具，可以直接编辑自身代码；不是只有文字反思的记忆系统。不同任务有各自初始任务求解模型，例如 Game of 24 提供初始 CoT 求解函数。",
"sources": [
{
"label": "实验数据采样；初始与演化代码示例",
"url": "https://arxiv.org/abs/2410.04444"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "task agent 自读源码与任务效用，调用修改/执行动作改变自己的求解与改进逻辑，再通过任务运行结果继续迭代；不要求形式化证明。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训练模型权重；在目标任务的 验证集 上进行源码搜索，每任务 6 次独立循环、每次最多 30 轮。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "GPQA（研究生级科学问答基准） 32 题；DROP、MGSM、MMLU 各 128 题，用于改进期间计分。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GPQA（研究生级科学问答基准） 剩余 166 题；DROP、MGSM、MMLU 各 800 题，均用 GPT-3.5 运行候选。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "主实验 验证／测试 分开；Game of 24 等展示案例不套用主表划分。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把解题流程和调用自修改的逻辑放进同一可编辑程序，使用实际执行反馈决定修改方向。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2410.04444v4",
"version": "2410.04444v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 ADAS 的本质差别：不再明确分开 fixed meta-agent 与 target agent；self-modification logic 也位于可编辑 program 中。",
"feedbackCases": [
{
"label": "GPQA / MMLU",
"data": "GPQA（研究生级科学问答基准） 验证 32、测试 166；MMLU 验证 128、测试 800",
"scoring": "与数据集参考选项匹配计算准确率。",
"visible": "可见验证分数和程序运行记录；每任务六次独立源码搜索，每次最多 30 轮。",
"use": "用验证成绩指导自改程序，最终用 GPT-3.5 运行选出的候选并在测试题报告结果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"judgment": "规则比较预测选项与数据集标签"
},
{
"label": "DROP / MGSM",
"data": "各 128 道验证题、800 道测试题",
"scoring": "DROP 使用答案 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）；MGSM 比较最终数学答案。",
"visible": "可见验证分数和程序运行记录；每任务六次独立源码搜索，每次最多 30 轮。",
"use": "用验证成绩指导自改程序，最终用 GPT-3.5 运行选出的候选并在测试题报告结果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"judgment": "DROP 用词项 F1；MGSM 比最终数学答案"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "不训练模型权重；在目标任务的 验证集 上进行源码搜索，每任务 6 次独立循环、每次最多 30 轮。",
"selection": "GPQA（研究生级科学问答基准） 32 题；DROP、MGSM、MMLU 各 128 题，用于改进期间计分。",
"evaluation": "GPQA（研究生级科学问答基准） 剩余 166 题；DROP、MGSM、MMLU 各 800 题，均用 GPT-3.5 运行候选。",
"isolation": "主实验 验证／测试 分开；Game of 24 等展示案例不套用主表划分。",
"roles": {
"executor": {
"value": "受限主实验用 gpt-3.5-turbo-0125 执行优化后的策略，与基线保持相同模型；开放实验允许 agent 求助 GPT-4o，需与受限结果区分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2410.04444#S5.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
]
},
"modifier": {
"value": "gpt-4o-2024-05-13 驱动 Gödel Agent 读取反馈并改写策略。修改代码的模型与受限测试时答题的 GPT-3.5 不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2410.04444#S3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2410.04444#S5.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
]
},
"seed": {
"value": "从逐步推理的任务求解代码开始，task agent 另有思考、错误记录、Python、Bash 与模型调用等工具，可以直接编辑自身代码；不是只有文字反思的记忆系统。不同任务有各自初始任务求解模型，例如 Game of 24 提供初始 CoT 求解函数。",
"sources": [
{
"label": "实验数据采样；初始与演化代码示例",
"url": "https://arxiv.org/abs/2410.04444"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"selection": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"evaluation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
],
"isolation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2410.04444#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2410.04444#A2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 虽然会根据反馈修正行为，但模块和迭代流程仍含无法在运行时改进的人类先验；固定的元学习算法也限定了搜索范围。作者认为这会限制进一步自主优化，因此研究让 agent 能够读取并修改自身运行逻辑，使改进不只发生在预留的模块内部。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2410.04444#S1"
}
]
},
{
"key": "position",
"value": "探索 agent 能否自主改变解题流程及负责自修改的机制，减少人工固定设计对递归自改进空间的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2410.04444"
}
]
},
{
"key": "conclusion",
"value": "在多个领域报告表现、效率与迁移收益；通过执行反馈开展自修改，而非要求形式证明。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2410.04444"
}
]
}
],
"fields": {
"object": "agent 执行逻辑及负责自修改的程序。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2410.04444"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2506.10943",
"title": "SEAL / Self-Adapting Language Models",
"url": "https://arxiv.org/abs/2506.10943",
"date": "2025-06-12",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"Data",
"M1",
"OfflineSearch",
"SameModel",
"Weights"
],
"fields": {
"本质定位": "**M-Weight**。模型自己写 self-edit，决定“拿什么数据、如何更新自己”，最终变化是 weights；不是 deployment harness evolution。",
"谁来改 → 谁执行；基础 harness": "model-generated self-edit → 同一 base model 的 fine-tuned successor。",
"Feedback / evidence": "update 后 downstream performance 奖励 self-edit。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 135,
"fields": {
"时间": "2025-06-12",
"论文": "[SEAL / Self-Adapting Language Models](https://arxiv.org/abs/2506.10943)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**M-Weight**。模型自己写 self-edit，决定“拿什么数据、如何更新自己”，最终变化是 weights；不是 deployment harness evolution。",
"谁来改 → 谁执行；基础 harness": "model-generated self-edit → 同一 base model 的 fine-tuned successor。",
"Feedback / evidence": "update 后 downstream performance 奖励 self-edit。",
"标签": "`#Data #Weights #SameModel #BenchmarkScore #OfflineSearch #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"legacyCategories": [
"M-Weight"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "SEAL 让模型先把新材料改写成适合自己学习的数据或更新指令，再实际做一次权重更新。外层训练奖励的不是这段 self-edit 看起来是否合理，而是更新后的模型能否更好完成任务，因此它学习的是“怎样准备自己的学习材料”。",
"novelty": "模型为新材料生成适合训练的内容与更新指令，实际训练后再用任务表现评价这次生成；因此也在学习怎样准备有效的参数更新。",
"object": "生成学习材料／训练配置的策略，以及实际适应后的模型参数。SQuAD 生成训练文本；ARC 选择数据增强、学习率和训练轮次。",
"executor": "知识学习：Qwen2.5-7B；ARC 少样本实验：Llama-3.2-1B-Instruct。",
"modifier": "知识学习配置由 Qwen2.5-7B 生成 self-edit；ARC 配置由 Llama-3.2-1B-Instruct 生成。固定内层梯度更新和外层 强化学习（根据奖励调整模型行为）／ReSTEM 程序据此训练对应模型。",
"roleContext": "model-generated self-edit → 同一 base model 的 fine-tuned successor。",
"seed": "SEAL 让模型生成自己的适应材料或训练配置，再调用真实梯度更新流程。知识任务从新文本段落生成训练文本；ARC 提供数据增强和测试时训练工具，模型选择增强开关、学习率和训练轮次等，不能概括为只改提示。",
"fixed": "工具接口、内外层训练算法和奖励定义预先设计；不是模型自行重写 optimizer。",
"verdict": "更新后在对应问题上的答题成绩；ARC 用正确/错误，知识学习的答案由 GPT-4.1 判分。",
"diagnosis": "没有独立的逐步错误定位器；通过多个 self-edit 实际训练后的效果，反向筛选有用的生成策略。",
"update": "生成数据/更新配置 → 内层微调 → 评估适应模型 → 外层强化有效 self-edit。",
"acceptance": "知识学习每个 context 比较 5 个生成、各跑 3 seeds，保留平均表现最好的一个用于外层训练。",
"experiments": [
{
"name": "ARC",
"evolve": "11 个经筛选、可由 TTT 解出的 training tasks",
"selection": "保留成功且符合计算限制的配置",
"test": "8 个 held-out tasks，每题生成 5 个配置",
"isolation": "任务划分独立",
"note": "小而经过可解性筛选的任务子集，不能代表完整 ARC 能力。"
},
{
"name": "知识学习",
"evolve": "SQuAD train；每轮抽 50 contexts 训练 self-edit policy",
"selection": "按适应后 QA 表现选 self-edit",
"test": "200 个 evaluation passages / 974 questions",
"isolation": "外层训练与测试材料分开",
"note": "测试时仍要读取新 passage 并进行内层适应；不是完全冻结模型直接答题。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2506.10943v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：知识任务用 SQuAD 训练集 学习生成 self-edit；ARC 用 11 个筛选后的训练任务学习数据增强/训练配置。\n\n调试 / 选版本数据：内层实际微调后，以问题作答或 ARC 输出的正确率给 self-edit 外层 强化学习（根据奖励调整模型行为） 奖励。\n\n最终测试数据：知识任务：SQuAD evaluation 的 200 个新文本段落、974 个问题；ARC：8 个未见任务。\n\n数据隔离与证据边界：外层训练材料与最终任务分开；测试时仍对当前 passage/ARC 示例进行内层适应，因此不是全程冻结模型。",
"cycle": "模型生成学习数据和更新配置，内层训练程序实际微调，再评价更新后模型。外层训练奖励能产生有效更新的生成方式。知识学习实验对每份上下文生成五种方案，每种用三个随机种子运行，按平均效果选择。",
"train": "知识任务用 SQuAD 训练集 学习生成 self-edit；ARC 用 11 个筛选后的训练任务学习数据增强/训练配置。",
"debug": "内层实际微调后，以问题作答或 ARC 输出的正确率给 self-edit 外层 强化学习（根据奖励调整模型行为） 奖励。",
"test": "知识任务：SQuAD evaluation 的 200 个新文本段落、974 个问题；ARC：8 个未见任务。",
"isolation": "外层训练材料与最终任务分开；测试时仍对当前 passage/ARC 示例进行内层适应，因此不是全程冻结模型。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "SEAL 让模型生成自己的适应材料或训练配置，再调用真实梯度更新流程。知识任务从新 passage 生成训练文本；ARC 提供数据增强和测试时训练工具，模型选择增强开关、学习率和 epoch 等，不能概括为只改提示。",
"protocol": "**知识学习：**Qwen2.5-7B 以 SQuAD train 构造 self-edit 学习；每轮抽 50 个上下文。最终使用 evaluation 中 200 个新 passage、974 个问题，测试时先读 passage 并做内层适应，再不带原文回答。外层训练与测试材料分开，但测试模型并非完全冻结。\n\n**ARC：**Llama3.2-1B 使用 11 个经可解性筛选的 training tasks，最终 8 个 held-out tasks，每题生成 5 套配置；不能当作完整 ARC 基准表现。内层训练数据是当前题的示例经增强而来，外层按适应后成绩训练配置生成能力。",
"sections": "知识学习与 ARC 实验；附录训练设置",
"source": "https://arxiv.org/abs/2506.10943",
"version": "2506.10943v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "52d3a249ac9ddd5ba7a9bb596c5ebb88491341147fef32d73267c043dcd3e599",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "知识学习：Qwen2.5-7B；ARC 少样本实验：Llama-3.2-1B-Instruct。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "知识学习配置由 Qwen2.5-7B 生成 self-edit；ARC 配置由 Llama-3.2-1B-Instruct 生成。固定内层梯度更新和外层 强化学习（根据奖励调整模型行为）／ReSTEM 程序据此训练对应模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "生成学习材料／训练配置的策略，以及实际适应后的模型参数。SQuAD 生成训练文本；ARC 选择数据增强、学习率和训练轮次。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "更新后在对应问题上的答题成绩；ARC 用正确/错误，知识学习的答案由 GPT-4.1 判分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "SEAL 让模型生成自己的适应材料或训练配置，再调用真实梯度更新流程。知识任务从新文本段落生成训练文本；ARC 提供数据增强和测试时训练工具，模型选择增强开关、学习率和训练轮次等，不能概括为只改提示。",
"sources": [
{
"label": "知识学习与 ARC 实验；附录训练设置",
"url": "https://arxiv.org/abs/2506.10943"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "模型生成学习数据和更新配置，内层训练程序实际微调，再评价更新后模型。外层训练奖励能产生有效更新的生成方式。知识学习实验对每份上下文生成五种方案，每种用三个随机种子运行，按平均效果选择。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "知识任务用 SQuAD 训练集 学习生成 self-edit；ARC 用 11 个筛选后的训练任务学习数据增强/训练配置。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "内层实际微调后，以问题作答或 ARC 输出的正确率给 self-edit 外层 强化学习（根据奖励调整模型行为） 奖励。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "知识任务：SQuAD evaluation 的 200 个新文本段落、974 个问题；ARC：8 个未见任务。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "外层训练材料与最终任务分开；测试时仍对当前 passage/ARC 示例进行内层适应，因此不是全程冻结模型。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "模型为新材料生成适合训练的内容与更新指令，实际训练后再用任务表现评价这次生成；因此也在学习怎样准备有效的参数更新。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2506.10943v2",
"version": "2506.10943v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "区别于把反思留在上下文中的方法，self-edit 会触发梯度更新；又区别于普通合成数据，生成策略用更新后的实际任务表现训练。内层改权重，外层学习怎样生成有效的改写。",
"feedbackCases": [
{
"label": "知识适应：SQuAD",
"data": "训练使用 SQuAD 训练集 的文本及配套问答；最终为 200 个新文本段落、974 个问题。",
"scoring": "模型根据文本生成训练材料，实际微调后再答配套问题；GPT-4.1 按参考答案判断回答。",
"visible": "外层学习获得微调后问答成绩，而不是仅评价生成训练文本看起来是否正确。",
"use": "奖励能真正提高知识问答表现的自生成材料；新文本和新问题检验学习到的适应方法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"judgment": "GPT-4.1 对照 SQuAD 参考答案作模型判分"
},
{
"label": "少样本适应：ARC",
"data": "11 个筛选后的训练任务；最终八个未见任务。",
"scoring": "执行选择的数据增强和测试时训练方案后，比较模型预测网格与正确网格，给成功/失败。",
"visible": "外层获得更新后任务表现；内层可用给定示例作适应。",
"use": "训练模型选择有效的适应方案，训练题答案可见，最终测试题答案不可见。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"judgment": "规则比较模型预测的 ARC 网格与标准网格"
}
],
"experiments": [
{
"label": "知识适应：SQuAD",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "SQuAD 训练集文本与配套问答；先根据自生成材料微调，再用答题正确性奖励材料生成策略。",
"selection": "内部微调后的问答表现用于外层策略训练。",
"evaluation": "200 个新段落及其 974 个问题；测试时仍根据新段落生成材料并适应。",
"isolation": "外层训练与最终新段落分开，但测试包含对当前段落的参数适应，不是完全冻结参数答题。",
"roles": {
"executor": {
"value": "Qwen2.5-7B，先用生成材料完成内部参数适应，再回答当前段落的问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
]
},
"modifier": {
"value": "Qwen2.5-7B 自己生成学习材料；固定的内部梯度更新程序训练答题模型，外层强化学习／ReSTEM 根据适应后的表现改进材料生成策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
]
},
"seed": {
"value": "提供从新段落生成训练文本、实际微调并评测问答的流程；变化不只是提示文本，还会触发参数更新。",
"sources": [
{
"label": "知识学习与 ARC 实验；附录训练设置",
"url": "https://arxiv.org/abs/2506.10943"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"selection": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"evaluation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"isolation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
]
}
},
{
"label": "少样本适应：ARC",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "11 个筛选后的 ARC 训练任务；实际执行适应方案后的正确性奖励方案生成策略。",
"selection": "训练任务上的适应效果用于策略更新。",
"evaluation": "八个未见 ARC 任务；根据任务示例进行内部适应后答题。",
"isolation": "外层训练任务与最终任务分开；最终任务内部仍允许从所给示例学习。",
"roles": {
"executor": {
"value": "Llama-3.2-1B-Instruct，按生成的训练配置适应当前 ARC 任务后输出答案。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
]
},
"modifier": {
"value": "Llama-3.2-1B-Instruct 自己生成适应配置；固定训练程序执行更新，外层训练根据任务表现改进配置生成策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
}
]
},
"seed": {
"value": "提供数据增强和测试时训练工具；模型可选择增强开关、学习率、训练轮次，再真正更新参数。",
"sources": [
{
"label": "知识学习与 ARC 实验；附录训练设置",
"url": "https://arxiv.org/abs/2506.10943"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"selection": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"evaluation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
],
"isolation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2506.10943#A1.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2506.10943#A2.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "适应新知识或推理任务时，可用数据往往少，而且原始材料的格式和数量未必适合模型学习。现有训练流程不能让模型学会怎样为自己变换、补充材料和选择学习策略，因此即使拿到同一份新信息，也可能无法把它有效吸收到参数中。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2506.10943#S1"
}
]
},
{
"key": "position",
"value": "研究模型能否面对新材料或少量示例，自主决定怎样把输入转成有效的参数更新，从而获得新的知识与任务能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2506.10943"
}
]
},
{
"key": "conclusion",
"value": "在知识吸收和少样本任务中验证自适应；学习的核心是怎样把新输入变成有效的参数更新。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2506.10943"
}
]
}
],
"fields": {
"object": "生成学习材料／训练配置的策略，以及实际适应后的模型参数。SQuAD 生成训练文本；ARC 选择数据增强、学习率和训练轮次。",
"modifier": "SQuAD：Qwen2.5-7B 生成训练文本；ARC：Llama-3.2-1B-Instruct 生成训练配置。训练程序执行参数更新，再用更新后的答题成绩改进材料或配置的生成策略。",
"verdict": "SQuAD：GPT-4.1 对照参考答案给更新后的回答判分；ARC：规则比较预测网格与标准网格。适应后的成绩用于训练改进策略。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2604.23472",
"title": "Escher-Loop",
"url": "https://arxiv.org/abs/2604.23472",
"date": "2026-04-25",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"HarnessCode",
"Improver",
"JointEvolution",
"M2",
"Population",
"org:tsinghua",
"org:nus"
],
"fields": {
"本质定位": "**Meta**。同时维护 task-agent 与 optimizer-agent population；optimizer 不只改 task program，也改 optimizer 自己，形成 mutual evolution。",
"谁来改 → 谁执行；基础 harness": "optimizer population → task programs + optimizer descendants；task agents 执行。",
"Feedback / evidence": "task-agent score 反向选择 optimizer。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 136,
"fields": {
"时间": "2026-04-25",
"论文": "[Escher-Loop](https://arxiv.org/abs/2604.23472)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**Meta**。同时维护 task-agent 与 optimizer-agent population；optimizer 不只改 task program，也改 optimizer 自己，形成 mutual evolution。",
"谁来改 → 谁执行；基础 harness": "optimizer population → task programs + optimizer descendants；task agents 执行。",
"Feedback / evidence": "task-agent score 反向选择 optimizer。",
"标签": "`#HarnessCode #Improver #JointEvolution #Population #BenchmarkScore #M2`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"legacyCategories": [
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "同时维护 task-agent 与 optimizer-agent population；optimizer 不只改 task program，也改 optimizer 自己，形成 mutual evolution。",
"novelty": "同时保留多种解题者和改进者；改进者的好坏由它能给解题者带来多大实测提升来判断。",
"object": "任务求解程序，以及负责优化它们的优化器提示。",
"executor": "候选任务/优化程序在Escher-Loop内执行；程序生成使用Gemini3 Flash ensemble。",
"modifier": "Gemini3 Flash生成程序变体；80%请求用gemini-3-flash-preview的low thinking，20%用默认动态thinking。外层种群选择程序固定。",
"roleContext": "optimizer population → task programs + optimizer descendants；task agents 执行。",
"seed": "作者自建双种群系统：任务程序负责产生几何解，优化器程序负责修改任务程序和优化器；配有程序执行评分、种群采样和优化器比较机制。OpenEvolve 是手工固定优化器对照，不应写成所有运行都用同一个冻结改进器。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "用 task agent 的得分判断提示词优化器好不好，再据此选择优化器。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Kissing Number、Circle Packing（26 圆）、Heilbronn Triangle（11 点）上的程序搜索，不做参数训练。\n\n调试 / 选版本数据：同一几何问题的合法性检查和目标值；优化器的 Elo 来自实际改进效果。\n\n最终测试数据：报告相同问题在 1,000 万等价 词元 预算内的历史最佳目标值。\n\n数据隔离与证据边界：优化已知问题的解，不是训练/测试题泛化；动态评分针对优化器相对能力。",
"cycle": "执行候选计算几何目标值，比较优化器带来的任务收益并更新 Elo；任务群体与优化器群体交替改进，保留演化分支。",
"train": "Kissing Number、Circle Packing（26 圆）、Heilbronn Triangle（11 点）上的程序搜索，不做参数训练。",
"debug": "同一几何问题的合法性检查和目标值；优化器的 Elo 来自实际改进效果。",
"test": "报告相同问题在 1,000 万等价 词元 预算内的历史最佳目标值。",
"isolation": "优化已知问题的解，不是训练/测试题泛化；动态评分针对优化器相对能力。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建双种群系统：任务程序负责产生几何解，优化器程序负责修改任务程序和优化器；配有程序执行评分、种群采样和优化器比较机制。OpenEvolve 是手工固定优化器对照，不应写成所有运行都用同一个冻结改进器。",
"protocol": "**进化及评估任务：**Kissing Number、Circle Packing、Heilbronn Triangle 三个几何优化问题；在搜索过程中直接执行候选程序、计算几何目标值。各方法限制为 1000 万等价 token，并画随预算变化的历史最佳成绩；优化器另按 Elo 比较。这里不是在一份训练问答集上训练、再到独立 benchmark 测试，不能标成常规 train/test 泛化。",
"sections": "§3 Experimental Protocol；附录 A",
"source": "https://arxiv.org/abs/2604.23472",
"version": "2604.23472v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6483f5c2727f0d3c118039fc8b5c1b48db09b429949ec82bab5e1edebbc1c013",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "候选任务/优化程序在Escher-Loop内执行；程序生成使用Gemini3 Flash ensemble。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.23472#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Gemini3 Flash生成程序变体；80%请求用gemini-3-flash-preview的low thinking，20%用默认动态thinking。外层种群选择程序固定。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.23472#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务求解程序，以及负责优化它们的优化器提示。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用 task agent 的得分判断提示词优化器好不好，再据此选择优化器。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建双种群系统：任务程序负责产生几何解，优化器程序负责修改任务程序和优化器；配有程序执行评分、种群采样和优化器比较机制。OpenEvolve 是手工固定优化器对照，不应写成所有运行都用同一个冻结改进器。",
"sources": [
{
"label": "§3 Experimental Protocol；附录 A",
"url": "https://arxiv.org/abs/2604.23472"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "执行候选计算几何目标值，比较优化器带来的任务收益并更新 Elo；任务群体与优化器群体交替改进，保留演化分支。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Kissing Number、Circle Packing（26 圆）、Heilbronn Triangle（11 点）上的程序搜索，不做参数训练。",
"sources": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同一几何问题的合法性检查和目标值；优化器的 Elo 来自实际改进效果。",
"sources": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "报告相同问题在 1,000 万等价 词元 预算内的历史最佳目标值。",
"sources": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "优化已知问题的解，不是训练/测试题泛化；动态评分针对优化器相对能力。",
"sources": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "同时保留多种解题者和改进者；改进者的好坏由它能给解题者带来多大实测提升来判断。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.23472v2",
"version": "2604.23472v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "不是一个固定 optimizer 改 task agent：两个人口分别保留解题者与改进者，用 task-agent 的实测改进反过来评价 optimizer。",
"feedbackCases": [
{
"label": "Kissing Number",
"data": "对同一个数学优化实例不断生成候选程序，没有普通问答 训练／测试 划分。",
"scoring": "程序检查几何构型合法性，并按问题目标评价接触数。",
"visible": "task agent 得到实测目标值；不同优化器产生的子代成绩再转成相对胜负，更新优化器 Elo 分数。",
"use": "任务分数选解法，子代优劣反过来选择优化器；报告固定计算预算内历史最佳目标值。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"judgment": "程序检查几何约束，再计算合法构型的接触数"
},
{
"label": "Circle Packing（26 圆）",
"data": "对同一个数学优化实例不断生成候选程序，没有普通问答 训练／测试 划分。",
"scoring": "检查圆是否满足边界及不重叠约束，评价合法构型的目标值。",
"visible": "task agent 得到实测目标值；不同优化器产生的子代成绩再转成相对胜负，更新优化器 Elo 分数。",
"use": "任务分数选解法，子代优劣反过来选择优化器；报告固定计算预算内历史最佳目标值。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"judgment": "程序检查不越界、不重叠，再计算半径和"
},
{
"label": "Heilbronn Triangle（11 点）",
"data": "对同一个数学优化实例不断生成候选程序，没有普通问答 训练／测试 划分。",
"scoring": "检查点构型并计算由三点构成三角形的目标面积指标。",
"visible": "task agent 得到实测目标值；不同优化器产生的子代成绩再转成相对胜负，更新优化器 Elo 分数。",
"use": "任务分数选解法，子代优劣反过来选择优化器；报告固定计算预算内历史最佳目标值。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"judgment": "程序检查点构型并计算最小三角形面积"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "Kissing Number、Circle Packing（26 圆）、Heilbronn Triangle（11 点）上的程序搜索，不做参数训练。",
"selection": "同一几何问题的合法性检查和目标值；优化器的 Elo 来自实际改进效果。",
"evaluation": "报告相同问题在 1,000 万等价 词元 预算内的历史最佳目标值。",
"isolation": "优化已知问题的解，不是训练/测试题泛化；动态评分针对优化器相对能力。",
"roles": {
"executor": {
"value": "候选任务/优化程序在Escher-Loop内执行；程序生成使用Gemini3 Flash ensemble。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.23472#A1.SS2"
}
]
},
"modifier": {
"value": "Gemini3 Flash生成程序变体；80%请求用gemini-3-flash-preview的low thinking，20%用默认动态thinking。外层种群选择程序固定。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.23472#S2.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.23472#S3.SS3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.23472#A1.SS2"
}
]
},
"seed": {
"value": "作者自建双种群系统：任务程序负责产生几何解，优化器程序负责修改任务程序和优化器；配有程序执行评分、种群采样和优化器比较机制。OpenEvolve 是手工固定优化器对照，不应写成所有运行都用同一个冻结改进器。",
"sources": [
{
"label": "§3 Experimental Protocol；附录 A",
"url": "https://arxiv.org/abs/2604.23472"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"selection": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"evaluation": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
],
"isolation": [
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2604.23472#A3.SS2"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2604.23472#A3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "手写工具和工作流虽能处理特定任务，却限制了 agent 的泛化与自主成长；只优化任务解，又把怎样寻找改进的机制固定下来。作者因此强调任务解和优化器都应能改进，但优化器的变化必须由实际任务效果提供依据，否则缺少判断改进方向的信号。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.23472#S1"
}
]
},
{
"key": "position",
"value": "研究任务求解能力和优化器的改进能力能否共同增长，使优化策略随着被优化系统的水平变化而适应。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.23472"
}
]
},
{
"key": "conclusion",
"value": "在数学优化任务中超过静态对照；优化器策略会随 task agent 水平变化而调整。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.23472"
}
]
}
],
"fields": {
"object": "任务求解程序，以及负责优化它们的优化器提示。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.23472"
}
]
},
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.23472"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2606.04465",
"title": "SePO",
"url": "https://arxiv.org/abs/2606.04465",
"date": "2026-06-03",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"Improver",
"M2",
"OfflineSearch",
"Prompt",
"SameModel",
"org:nus"
],
"fields": {
"本质定位": "**H-Prompt / Meta**。先让 prompt optimizer 的**自身 system prompt**在多任务上变好，再用它优化 target prompts；是 prompt-level improver evolution。",
"谁来改 → 谁执行；基础 harness": "prompt agent self-evolves → 再改 task-agent prompt；基础是固定 prompt-optimization workflow。",
"Feedback / evidence": "train/dev benchmark score / verifier。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 137,
"fields": {
"时间": "2026-06-03",
"论文": "[SePO](https://arxiv.org/abs/2606.04465)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Prompt / Meta**。先让 prompt optimizer 的**自身 system prompt**在多任务上变好，再用它优化 target prompts；是 prompt-level improver evolution。",
"谁来改 → 谁执行；基础 harness": "prompt agent self-evolves → 再改 task-agent prompt；基础是固定 prompt-optimization workflow。",
"Feedback / evidence": "train/dev benchmark score / verifier。",
"标签": "`#Prompt #Improver #SameModel #BenchmarkScore #OfflineSearch #M2`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"legacyCategories": [
"H-Prompt",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "先让 prompt optimizer 的**自身 system prompt**在多任务上变好，再用它优化 target prompts；是 prompt-level improver evolution。",
"novelty": "先跨任务优化“指导优化器怎样改提示”的系统提示，再用改好的优化器处理目标任务；两阶段主要改变文字提示。",
"object": "提示优化器自己的系统提示，以及它为各 task agent 修改的提示。",
"executor": "默认DeepSeek-V3.2作task task agent；跨模型设置换Gemini3.1 Flash-Lite Preview。",
"modifier": "默认Gemini3.1 Pro Preview作负责生成提示词的 agent；另一配置用Claude Opus4.6。负责生成提示词的 agent 优化任务提示，也进化自己的优化提示。",
"roleContext": "prompt agent self-evolves → 再改 task-agent prompt；基础是固定 prompt-optimization workflow。",
"seed": "SePO 包含提出系统提示词的负责生成提示词的 agent 和按提示做题的 task task agent。只演化负责生成提示词的 agent 的自然语言系统提示；任务代码、工具和模型权重不改。训练在本文指提示搜索，不是参数微调。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "任务题目的评分器给训练或开发成绩，用来比较新旧提示；保留更好的提示，并在后续任务中检验优化经验是否能迁移。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：任务提示搜索：s1K-1.1 数学 535、ARC 416、MBPP 474、Sudoku 440；GPQA（研究生级科学问答基准） 用 MMLU STEM 作 agent 训练池。通用优化器另用 LIMO/MMLU 任务混合。\n\n调试 / 选版本数据：训练池上的正确率决定候选是否入档；它是提示搜索，不是模型权重 监督微调（用示范数据训练模型）。\n\n最终测试数据：AIME’25 30、ARC-AGI-1 419、MBPP 500、Sudoku 100、GPQA（研究生级科学问答基准） 198。\n\n数据隔离与证据边界：agent 训练池与最终任务分别定义；GPQA（研究生级科学问答基准） 训练量在表中标 N/A，不能补造统一数量。",
"cycle": "读取任务执行结果后生成子提示；只有优于父提示的子代进入档案。预训练阶段让优化器改自己的提示，下游阶段固定这个优化器。",
"train": "任务提示搜索：s1K-1.1 数学 535、ARC 416、MBPP 474、Sudoku 440；GPQA（研究生级科学问答基准） 用 MMLU STEM 作 agent 训练池。通用优化器另用 LIMO/MMLU 任务混合。",
"debug": "训练池上的正确率决定候选是否入档；它是提示搜索，不是模型权重 监督微调（用示范数据训练模型）。",
"test": "AIME’25 30、ARC-AGI-1 419、MBPP 500、Sudoku 100、GPQA（研究生级科学问答基准） 198。",
"isolation": "agent 训练池与最终任务分别定义；GPQA（研究生级科学问答基准） 训练量在表中标 N/A，不能补造统一数量。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "SePO 包含提出系统提示词的 prompt agent 和按提示做题的 task agent。只演化 prompt agent 的自然语言系统提示；任务代码、工具和模型权重不改。训练在本文指提示搜索，不是参数微调。",
"protocol": "**进化→测试：**AIME’25：从 s1K-1.1 筛出的 535 道有验证解答的题→官方 30 题；ARC-AGI-1：论文使用的 416/419 划分；MBPP：474/500；Sudoku：抽样 440/100。GPQA 测试为 198 题，训练代理数据来自 MMLU STEM，但表中训练数量为 N/A，不能补造。\n\n**通用提示训练池：**另用 LIMO 800 题，以及 MMLU 人文 518、社科 337、其他 355 题。ARC 报告 pass@3，其他任务按多次运行平均 pass@1；不要把重复次数当样本数。独立选模集数量本轮未核实。",
"sections": "§4.1；附录 C、Table 4",
"source": "https://arxiv.org/abs/2606.04465",
"version": "2606.04465v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "df7bc20d58c88e49d792493b8cdb6c6dffae68bccdbdd3c84cc20589898dbdf9",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "默认DeepSeek-V3.2作task task agent；跨模型设置换Gemini3.1 Flash-Lite Preview。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.04465#S4.SS2.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "默认Gemini3.1 Pro Preview作负责生成提示词的 agent；另一配置用Claude Opus4.6。负责生成提示词的 agent 优化任务提示，也进化自己的优化提示。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.04465#S4.SS2.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "提示优化器自己的系统提示，以及它为各 task agent 修改的提示。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务题目的评分器给训练或开发成绩，用来比较新旧提示；保留更好的提示，并在后续任务中检验优化经验是否能迁移。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "SePO 包含提出系统提示词的负责生成提示词的 agent 和按提示做题的 task task agent。只演化负责生成提示词的 agent 的自然语言系统提示；任务代码、工具和模型权重不改。训练在本文指提示搜索，不是参数微调。",
"sources": [
{
"label": "§4.1；附录 C、Table 4",
"url": "https://arxiv.org/abs/2606.04465"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "读取任务执行结果后生成子提示；只有优于父提示的子代进入档案。预训练阶段让优化器改自己的提示，下游阶段固定这个优化器。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "任务提示搜索：s1K-1.1 数学 535、ARC 416、MBPP 474、Sudoku 440；GPQA（研究生级科学问答基准） 用 MMLU STEM 作 agent 训练池。通用优化器另用 LIMO/MMLU 任务混合。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练池上的正确率决定候选是否入档；它是提示搜索，不是模型权重 监督微调（用示范数据训练模型）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AIME’25 30、ARC-AGI-1 419、MBPP 500、Sudoku 100、GPQA（研究生级科学问答基准） 198。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "agent 训练池与最终任务分别定义；GPQA（研究生级科学问答基准） 训练量在表中标 N/A，不能补造统一数量。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先跨任务优化“指导优化器怎样改提示”的系统提示，再用改好的优化器处理目标任务；两阶段主要改变文字提示。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.04465v1",
"version": "2606.04465v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "先在任务混合上改 prompt optimizer 自己的 system prompt，再拿它优化目标任务；所谓 pretraining/fine-tuning 在这里主要是 prompt-space 阶段。",
"feedbackCases": [
{
"label": "数学",
"data": "s1K-1.1 数学 535 题搜索；AIME 2025 的 30 题测试",
"scoring": "从回答提取框出的整数，与题目标准整数比较。",
"visible": "训练池任务成绩用于评价提示词产生的 task agent，并决定候选提示是否入档。",
"use": "进化提示生成者的提示词，不训练模型权重；冻结后按各测试协议评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"judgment": "规则抽取方框内整数，与标准整数比较"
},
{
"label": "ARC",
"data": "416 题搜索；ARC-AGI-1 的 419 题测试",
"scoring": "预测网格与参考网格比较，报告 pass@3（每题最多尝试三次、至少成功一次的比例）。",
"visible": "训练池任务成绩用于评价提示词产生的 task agent，并决定候选提示是否入档。",
"use": "进化提示生成者的提示词，不训练模型权重；冻结后按各测试协议评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"judgment": "规则比较预测网格与标准网格"
},
{
"label": "GPQA",
"data": "MMLU STEM 用于 agent 训练；GPQA（研究生级科学问答基准） 198 题测试",
"scoring": "预测答案字母与参考选项比较。",
"visible": "训练池任务成绩用于评价提示词产生的 task agent，并决定候选提示是否入档。",
"use": "进化提示生成者的提示词，不训练模型权重；冻结后按各测试协议评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"judgment": "规则比较选项字母与标准选项"
},
{
"label": "MBPP",
"data": "474 题搜索、500 题测试",
"scoring": "实际运行生成 Python 程序，按保密单元测试判断功能正确性。",
"visible": "训练池任务成绩用于评价提示词产生的 task agent，并决定候选提示是否入档。",
"use": "进化提示生成者的提示词，不训练模型权重；冻结后按各测试协议评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"judgment": "MBPP 隐藏单元测试执行代码"
},
{
"label": "Sudoku",
"data": "440 题搜索、100 题测试",
"scoring": "规则验证器检查 4×4 网格的行、列、子块约束。",
"visible": "训练池任务成绩用于评价提示词产生的 task agent，并决定候选提示是否入档。",
"use": "进化提示生成者的提示词，不训练模型权重；冻结后按各测试协议评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"judgment": "程序检查数独行、列和子块约束"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4
],
"evolution": "任务提示搜索：s1K-1.1 数学 535、ARC 416、MBPP 474、Sudoku 440；GPQA（研究生级科学问答基准） 用 MMLU STEM 作 agent 训练池。通用优化器另用 LIMO/MMLU 任务混合。",
"selection": "训练池上的正确率决定候选是否入档；它是提示搜索，不是模型权重 监督微调（用示范数据训练模型）。",
"evaluation": "AIME’25 30、ARC-AGI-1 419、MBPP 500、Sudoku 100、GPQA（研究生级科学问答基准） 198。",
"isolation": "agent 训练池与最终任务分别定义；GPQA（研究生级科学问答基准） 训练量在表中标 N/A，不能补造统一数量。",
"roles": {
"executor": {
"value": "默认DeepSeek-V3.2作task task agent；跨模型设置换Gemini3.1 Flash-Lite Preview。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.04465#S4.SS2.SSS0.Px5"
}
]
},
"modifier": {
"value": "默认Gemini3.1 Pro Preview作负责生成提示词的 agent；另一配置用Claude Opus4.6。负责生成提示词的 agent 优化任务提示，也进化自己的优化提示。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.04465#S3.SS1.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04465#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2606.04465#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.04465#S4.SS2.SSS0.Px5"
}
]
},
"seed": {
"value": "SePO 包含提出系统提示词的负责生成提示词的 agent 和按提示做题的 task task agent。只演化负责生成提示词的 agent 的自然语言系统提示；任务代码、工具和模型权重不改。训练在本文指提示搜索，不是参数微调。",
"sources": [
{
"label": "§4.1；附录 C、Table 4",
"url": "https://arxiv.org/abs/2606.04465"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.04465#S4.SS1.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04465#A3.SS0.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "提示优化器本身通常是人工设计、固定不变的 agent，因此处理再多任务也不能积累“怎样优化提示”的能力，效果受最初设计限制。只增加一层变异提示仍留下固定的上层规则，所以作者研究如何把优化器自身的提示也纳入同一改进过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.04465#S1"
}
]
},
{
"key": "position",
"value": "研究提示优化器能否学习并迁移“怎样改提示”的经验，突破固定人工优化指令对任务提示搜索的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04465"
}
]
},
{
"key": "conclusion",
"value": "在五种推理与编程基准上优于对照，跨任务预先学到的提示优化经验能够迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.04465"
}
]
}
],
"fields": {
"object": "提示优化器自己的系统提示，以及它为各 task agent 修改的提示。",
"verdict": "任务题目的评分器给训练或开发成绩，用来比较新旧提示；保留更好的提示，并在后续任务中检验优化经验是否能迁移。"
}
},
"attributions": [
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.04465"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2606.26294",
"title": "Red Queen Gödel Machine",
"url": "https://arxiv.org/abs/2606.26294",
"date": "2026-06-24",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Archive",
"CoEvolution",
"Evaluator",
"HarnessCode",
"Improver",
"JointEvolution",
"M2"
],
"fields": {
"本质定位": "**Meta / Hybrid**。把“agent 变、evaluator/utility 也变”显式化；重要点是避免 evaluator 一直固定，但在每个 selection epoch 内仍冻结标准以维持可比较性。",
"谁来改 → 谁执行；基础 harness": "evolutionary controller 同时产生 agent/evaluator descendants。",
"Feedback / evidence": "evolving utility/evaluator + task outcome；epoch 内稳定 selection。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 138,
"fields": {
"时间": "2026-06-24",
"论文": "[Red Queen Gödel Machine](https://arxiv.org/abs/2606.26294)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**Meta / Hybrid**。把“agent 变、evaluator/utility 也变”显式化；重要点是避免 evaluator 一直固定，但在每个 selection epoch 内仍冻结标准以维持可比较性。",
"谁来改 → 谁执行；基础 harness": "evolutionary controller 同时产生 agent/evaluator descendants。",
"Feedback / evidence": "evolving utility/evaluator + task outcome；epoch 内稳定 selection。",
"标签": "`#HarnessCode #Evaluator #JointEvolution #CoEvolution #Archive #M2`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"legacyCategories": [
"Hybrid",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把“agent 变、evaluator/utility 也变”显式化；重要点是避免 evaluator 一直固定，但在每个 selection epoch 内仍冻结标准以维持可比较性。",
"novelty": "允许每轮之间修改评价标准，但同一轮内部固定标准以比较候选；跨轮的分数需要结合当轮目标解释。",
"object": "task agent 与学习式评审器；每个选择阶段内部暂时固定评分规则。",
"executor": "主实验用 GPT-5.5（low）执行编码、论文写作、证明及评审。成本消融仅将搜索期论文任务调用换为 Nemotron 3 Ultra，最终仍用 GPT-5.5（low）评审。",
"modifier": "meta-agent 使用 GPT-5.5（low）读取节点及祖先反馈、修改角色 agent；混合模型消融保持 meta-agent 不变。搜索程序选择节点，并在进化轮次边界替换通过检验的评审器。",
"roleContext": "evolutionary controller 同时产生 agent/evaluator descendants。",
"seed": "每个角色从相同的最小 task agent 模板加领域输出格式开始，代码可修改。系统同时维护任务执行者与可进化评审者；一轮内评审固定，轮间才替换。初始模板不是预先写好的领域专家流水线。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "根据任务结果评分，但评分目标和评审者也会进化；同一轮内保持评审规则固定，轮间才更新。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：论文写作/评审：APReS标题摘要及接受/拒绝；证明：IMO-GradingBench人工分；编码：Polyglot可执行测试，代码评审：CRAVE接受/拒绝PR。\n\n调试 / 选版本数据：独立验证集驱动节点选择；训练轨迹只指导修改、不进入搜索效用。生成物评分角色使用不同生成物作训练与搜索。\n\n最终测试数据：各领域另用与验证集分开的测试集；证明需进化轮次内冻结评分器给满分7/7。本文明确没有使用SWE-bench。\n\n数据隔离与证据边界：区分生成修改、搜索选节点、最终测试三套证据；评审器可变，但同进化轮次内固定，避免不同尺子的分数直接混用。",
"cycle": "训练反馈用于生成修改；独立验证成绩用于搜索节点，评审器替换发生在进化轮次边界，依赖它的效用历史随之重新处理。",
"train": "论文写作/评审：APReS标题摘要及接受/拒绝；证明：IMO-GradingBench人工分；编码：Polyglot可执行测试，代码评审：CRAVE接受/拒绝PR。",
"debug": "独立验证集驱动节点选择；训练轨迹只指导修改、不进入搜索效用。生成物评分角色使用不同生成物作训练与搜索。",
"test": "各领域另用与验证集分开的测试集；证明需进化轮次内冻结评分器给满分7/7。本文明确没有使用SWE-bench。",
"isolation": "区分生成修改、搜索选节点、最终测试三套证据；评审器可变，但同进化轮次内固定，避免不同尺子的分数直接混用。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "每个角色从相同的最小任务 agent 模板加领域输出格式开始，代码可修改。系统同时维护任务执行者与可进化评审者；一轮内评审固定，轮间才替换。初始模板不是预先写好的领域专家流水线。",
"protocol": "**任务与依据：**编程使用 Polyglot 可执行测试，代码评审用 CRAVE 已接受/拒绝的 PR；论文评审以 APReS 人类论文标签为锚，另有证明生成与评分任务。SWE-bench 因运行成本未纳入本版。\n\n**隔离：**训练题及祖先的训练反馈供修改者看；节点选择依据不可见的验证题，最后用另外的测试题。依赖评审的生成任务还分别生成训练和搜索用产物。各领域精确划分数量、证明数据来源本轮未核实，不能只写成一个“隐藏测试集”。",
"sections": "§3.2、实验设置；附录 C.5",
"source": "https://arxiv.org/abs/2606.26294",
"version": "2606.26294v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6ecbe8ef12d25d52469e95e076d1a28b1ca828a58e9c70850ff23139d3f0eaeb",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验用 GPT-5.5（low）执行编码、论文写作、证明及评审。成本消融仅将搜索期论文任务调用换为 Nemotron 3 Ultra，最终仍用 GPT-5.5（low）评审。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
},
{
"label": "§5.5、附录 C.2",
"url": "https://arxiv.org/pdf/2606.26294#page=20"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "meta-agent 使用 GPT-5.5（low）读取节点及祖先反馈、修改角色 agent；混合模型消融保持 meta-agent 不变。搜索程序选择节点，并在进化轮次边界替换通过检验的评审器。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
},
{
"label": "§5.5、附录 C.2",
"url": "https://arxiv.org/pdf/2606.26294#page=20"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "task agent 与学习式评审器；每个选择阶段内部暂时固定评分规则。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "根据任务结果评分，但评分目标和评审者也会进化；同一轮内保持评审规则固定，轮间才更新。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "每个角色从相同的最小 task agent 模板加领域输出格式开始，代码可修改。系统同时维护任务执行者与可进化评审者；一轮内评审固定，轮间才替换。初始模板不是预先写好的领域专家流水线。",
"sources": [
{
"label": "§3.2、实验设置；附录 C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "训练反馈用于生成修改；独立验证成绩用于搜索节点，评审器替换发生在进化轮次边界，依赖它的效用历史随之重新处理。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "论文写作/评审：APReS标题摘要及接受/拒绝；证明：IMO-GradingBench人工分；编码：Polyglot可执行测试，代码评审：CRAVE接受/拒绝PR。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "独立验证集驱动节点选择；训练轨迹只指导修改、不进入搜索效用。生成物评分角色使用不同生成物作训练与搜索。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各领域另用与验证集分开的测试集；证明需进化轮次内冻结评分器给满分7/7。本文明确没有使用SWE-bench。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "区分生成修改、搜索选节点、最终测试三套证据；评审器可变，但同进化轮次内固定，避免不同尺子的分数直接混用。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "允许每轮之间修改评价标准，但同一轮内部固定标准以比较候选；跨轮的分数需要结合当轮目标解释。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.26294v2",
"version": "2606.26294v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "让评价标准在 epoch 边界演化，但 epoch 内固定，使同轮候选可比；“证明变好”因此是相对当轮 utility，而非跨所有标准的绝对进步。",
"feedbackCases": [
{
"label": "论文写作 / 评审",
"data": "APReS 题名、摘要和论文接收/拒绝记录",
"scoring": "评审角色对齐真实接收结果；写作产物由当前轮的评审角色给分。",
"visible": "训练记录用于提出修改；独立验证结果用于节点选择。评审器可以变化，但同一进化轮内固定。",
"use": "各领域另有测试集；这是相对于当前评分标准的共同进化，不能将可进化评审器的高分当作跨轮不变的外部真值。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"judgment": "写作由当轮冻结评审代理评分；评审 agent 用真实接收标签验收"
},
{
"label": "数学证明 / 评分",
"data": "IMO-GradingBench 问题、证明及专家分数",
"scoring": "评分角色与人类分数比较；证明生成是否成功按该轮冻结评分器是否给 7/7 判定。",
"visible": "训练记录用于提出修改；独立验证结果用于节点选择。评审器可以变化，但同一进化轮内固定。",
"use": "各领域另有测试集；这是相对于当前评分标准的共同进化，不能将可进化评审器的高分当作跨轮不变的外部真值。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"judgment": "证明由当轮冻结评分 agent 按要求给分；评分 agent 再与人类分数比对"
},
{
"label": "编程",
"data": "Polyglot 题目与代码测试",
"scoring": "执行代码测试检验程序功能。",
"visible": "训练记录用于提出修改；独立验证结果用于节点选择。评审器可以变化，但同一进化轮内固定。",
"use": "各领域另有测试集；这是相对于当前评分标准的共同进化，不能将可进化评审器的高分当作跨轮不变的外部真值。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"judgment": "代码题可执行测试"
},
{
"label": "代码评审",
"data": "CRAVE 中被接收/拒绝的 PR",
"scoring": "与数据集的 PR 接收/拒绝标签对齐评价。",
"visible": "训练记录用于提出修改；独立验证结果用于节点选择。评审器可以变化，但同一进化轮内固定。",
"use": "各领域另有测试集；这是相对于当前评分标准的共同进化，不能将可进化评审器的高分当作跨轮不变的外部真值。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"judgment": "规则比较预测 PR 接收标签与数据集标签"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "论文写作/评审：APReS标题摘要及接受/拒绝；证明：IMO-GradingBench人工分；编码：Polyglot可执行测试，代码评审：CRAVE接受/拒绝PR。",
"selection": "独立验证集驱动节点选择；训练轨迹只指导修改、不进入搜索效用。生成物评分角色使用不同生成物作训练与搜索。",
"evaluation": "各领域另用与验证集分开的测试集；证明需进化轮次内冻结评分器给满分7/7。本文明确没有使用SWE-bench。",
"isolation": "区分生成修改、搜索选节点、最终测试三套证据；评审器可变，但同进化轮次内固定，避免不同尺子的分数直接混用。",
"roles": {
"executor": {
"value": "主实验用 GPT-5.5（low）执行编码、论文写作、证明及评审。成本消融仅将搜索期论文任务调用换为 Nemotron 3 Ultra，最终仍用 GPT-5.5（low）评审。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
},
{
"label": "§5.5、附录 C.2",
"url": "https://arxiv.org/pdf/2606.26294#page=20"
}
]
},
"modifier": {
"value": "meta-agent 使用 GPT-5.5（low）读取节点及祖先反馈、修改角色 agent；混合模型消融保持 meta-agent 不变。搜索程序选择节点，并在进化轮次边界替换通过检验的评审器。",
"sources": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
},
{
"label": "§5.5、附录 C.2",
"url": "https://arxiv.org/pdf/2606.26294#page=20"
}
]
},
"seed": {
"value": "每个角色从相同的最小 task agent 模板加领域输出格式开始，代码可修改。系统同时维护任务执行者与可进化评审者；一轮内评审固定，轮间才替换。初始模板不是预先写好的领域专家流水线。",
"sources": [
{
"label": "§3.2、实验设置；附录 C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
]
}
},
"sources": {
"evolution": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"selection": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"evaluation": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"isolation": [
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有自改进搜索依赖在循环外固定的评价标准，但论文写作等任务未必有直接基准，已有评测也可能昂贵、反馈不足，或随能力提升而饱和、被钻空子。这些限制阻碍自改进进入开放研究场景，因此作者希望让评价器也能随被评价的 agent 一起改进。",
"sources": [
{
"label": "§1 Introduction（第 1–2 页）",
"url": "https://arxiv.org/pdf/2606.26294v2#page=1"
}
]
},
{
"key": "position",
"value": "研究在缺少完美固定评价器的任务中，解题者与评审者能否共同改进，同时保持评价可信和版本比较有意义。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.26294"
}
]
},
{
"key": "conclusion",
"value": "探索写作、证明和编码中的共同进化；评价器的外部校准与分阶段比较是方法的关键。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.26294"
}
]
}
],
"fields": {
"object": "task agent 与学习式评审器；每个选择阶段内部暂时固定评分规则。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"feedback",
"improver",
"evaluation"
]
},
{
"id": "2608.07645",
"title": "Mendel Gödel Machine",
"url": "https://arxiv.org/abs/2608.07645",
"date": "2026-08-07",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Archive",
"ExecutableVerifier",
"HarnessCode",
"M2",
"Population"
],
"fields": {
"本质定位": "**F / H-Full**。在 DGM archive 上加入多任务 evidence、cross-lineage comparison、reaction-norm mutation 与 hybridization，重点从单个 lineage 改成 comparative evolution。",
"谁来改 → 谁执行；基础 harness": "coding-agent evolver → archive descendants；coding agent 执行。",
"Feedback / evidence": "multi-task/cross-lineage verifier evidence。"
},
"details": [],
"occurrences": [
{
"section": "A. Foundations / agent-program / improver-level",
"line": 139,
"fields": {
"时间": "2026-08-07",
"论文": "[Mendel Gödel Machine](https://arxiv.org/abs/2608.07645)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**F / H-Full**。在 DGM archive 上加入多任务 evidence、cross-lineage comparison、reaction-norm mutation 与 hybridization，重点从单个 lineage 改成 comparative evolution。",
"谁来改 → 谁执行；基础 harness": "coding-agent evolver → archive descendants；coding agent 执行。",
"Feedback / evidence": "multi-task/cross-lineage verifier evidence。",
"标签": "`#HarnessCode #Archive #Population #ExecutableVerifier #M2`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"legacyCategories": [
"F",
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "在 DGM archive 上加入多任务 evidence、cross-lineage comparison、reaction-norm mutation 与 hybridization，重点从单个 lineage 改成 comparative evolution。",
"novelty": "对照同一 agent 在不同任务上的失败，以及不同历史分支在同一任务上的表现，用这些比较证据定位可继承的框架改动。",
"object": "coding agent 的可执行代码与运行结构。",
"executor": "主进化用Qwen3.6-35B-A3B执行候选编码 task agent；另评Qwen3-Coder-Next-80B-A3B。跨模型迁移冻结运行框架后换DeepSeek-V4-Flash/Pro。",
"modifier": "编码 task agent 依据档案与执行反馈改进后代；主实验使用Qwen3.6-35B-A3B，模型权重固定，变的是 task agent 代码。",
"roleContext": "coding-agent evolver → archive descendants；coding agent 执行。",
"seed": "Polyglot 初始 forward 仅作一次代码生成，没有结构化仓库分析和测试反馈循环；与 HGM 使用相同祖先版本。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "读取不同任务、不同历史分支的代码测试结果和执行记录，比较哪些改动有效、哪些导致退化，再提出修改或组合已有方案。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SWE-bench Verified、Polyglot 各 60 题的搜索设置；统一 200 次评估预算。\n\n调试 / 选版本数据：失败任务池和档案中不同版本的行为用于诊断；私有测试用例不向 task agent 展示。\n\n最终测试数据：主表报告搜索所得版本；另测完整 Polyglot 225、SWE-Pro、SWE-Multilingual 及跨模型迁移。\n\n数据隔离与证据边界：60 题搜索成绩不等于独立测试；完整 Polyglot 包含这批题，跨评测基准结果单独看。",
"cycle": "比较历史档案中不同任务和分支的代码及失败经历，提出修改或组合不同谱系的有效部分。分配新候选生成与评估预算，再按方法估计的表现选择最终版本。",
"train": "SWE-bench Verified、Polyglot 各 60 题的搜索设置；统一 200 次评估预算。",
"debug": "失败任务池和档案中不同版本的行为用于诊断；私有测试用例不向 task agent 展示。",
"test": "主表报告搜索所得版本；另测完整 Polyglot 225、SWE-Pro、SWE-Multilingual 及跨模型迁移。",
"isolation": "60 题搜索成绩不等于独立测试；完整 Polyglot 包含这批题，跨评测基准结果单独看。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "沿用 HGM 比较设置中的共同祖先编程 agent，允许编辑源码，并保存种群中不同版本的任务执行记录。MGM 在单条失败记录之外比较同一版本在多题的表现、以及不同版本之间的差异；具体初始工具清单本轮未核实。",
"protocol": "**主实验：**SWE-bench Verified 和 Polyglot 各取与 HGM 对照相同的 60 题子集，在 200 次评估预算内搜索，报告最佳信念版本。\n\n**迁移：**另测 SWE-bench Pro、SWE-bench Multilingual 和跨模型设置。论文说私有测试用例及结果不向进化 agent 开放，但这不自动意味着主实验的 60 个任务从未参与搜索；进化任务与最终任务的精确重叠本轮待核实，不能标成明确互斥的 60 题测试集。",
"sections": "§5、§5.1 与迁移实验",
"source": "https://arxiv.org/abs/2608.07645",
"version": "2608.07645v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "bdef46ff20f9ffbf296f6335f3ba5c809d077f774e0327fc520a267e2d3884cd",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主进化用Qwen3.6-35B-A3B执行候选编码 task agent；另评Qwen3-Coder-Next-80B-A3B。跨模型迁移冻结运行框架后换DeepSeek-V4-Flash/Pro。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.07645#S5.SS2.SSS0.Px2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.07645#A3.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2608.07645#A6.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "编码 task agent 依据档案与执行反馈改进后代；主实验使用Qwen3.6-35B-A3B，模型权重固定，变的是 task agent 代码。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.07645#S5.SS2.SSS0.Px2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.07645#A3.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2608.07645#A6.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "coding agent 的可执行代码与运行结构。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "读取不同任务、不同历史分支的代码测试结果和执行记录，比较哪些改动有效、哪些导致退化，再提出修改或组合已有方案。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Polyglot 初始 forward 仅作一次代码生成，没有结构化仓库分析和测试反馈循环；与 HGM 使用相同祖先版本。",
"sources": [
{
"label": "§5、§5.1 与迁移实验",
"url": "https://arxiv.org/abs/2608.07645"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "比较历史档案中不同任务和分支的代码及失败经历，提出修改或组合不同谱系的有效部分。分配新候选生成与评估预算，再按方法估计的表现选择最终版本。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SWE-bench Verified、Polyglot 各 60 题的搜索设置；统一 200 次评估预算。",
"sources": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "失败任务池和档案中不同版本的行为用于诊断；私有测试用例不向 task agent 展示。",
"sources": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主表报告搜索所得版本；另测完整 Polyglot 225、SWE-Pro、SWE-Multilingual 及跨模型迁移。",
"sources": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "60 题搜索成绩不等于独立测试；完整 Polyglot 包含这批题，跨评测基准结果单独看。",
"sources": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "对照同一 agent 在不同任务上的失败，以及不同历史分支在同一任务上的表现，用这些比较证据定位可继承的框架改动。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.07645v1",
"version": "2608.07645v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用跨任务的 reaction-norm mutation 与跨 lineage 的同任务对照决定改动；不同祖先的执行证据成为重组依据。",
"feedbackCases": [
{
"label": "代码 agent 搜索：SWE-bench Verified / Polyglot",
"data": "各 60 道搜索题，统一 200 次评估预算。",
"scoring": "SWE 用仓库补丁测试，Polyglot 用多语言程序测试，均按功能是否通过计分。",
"visible": "外层看失败任务池、不同候选的行为和评估证据；task agent 不可查看私有验收测试。",
"use": "从跨版本证据提出/组合框架改动；完整 Polyglot 225、SWE-Pro、SWE-Multilingual 及跨模型迁移另报。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"judgment": "SWE-bench 仓库测试／Polyglot 编程题测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "SWE-bench Verified、Polyglot 各 60 题的搜索设置；统一 200 次评估预算。",
"selection": "失败任务池和档案中不同版本的行为用于诊断；私有测试用例不向 task agent 展示。",
"evaluation": "主表报告搜索所得版本；另测完整 Polyglot 225、SWE-Pro、SWE-Multilingual 及跨模型迁移。",
"isolation": "60 题搜索成绩不等于独立测试；完整 Polyglot 包含这批题，跨评测基准结果单独看。",
"roles": {
"executor": {
"value": "主进化用Qwen3.6-35B-A3B执行候选编码 task agent；另评Qwen3-Coder-Next-80B-A3B。跨模型迁移冻结运行框架后换DeepSeek-V4-Flash/Pro。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.07645#S5.SS2.SSS0.Px2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.07645#A3.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2608.07645#A6.SS2"
}
]
},
"modifier": {
"value": "编码 task agent 依据档案与执行反馈改进后代；主实验使用Qwen3.6-35B-A3B，模型权重固定，变的是 task agent 代码。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.07645#A1.SS1"
},
{
"label": "附录H.1",
"url": "https://arxiv.org/html/2608.07645#A8.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.07645#S5.SS2.SSS0.Px2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2608.07645#A3.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2608.07645#A6.SS2"
}
]
},
"seed": {
"value": "Polyglot 初始 forward 仅作一次代码生成，没有结构化仓库分析和测试反馈循环；与 HGM 使用相同祖先版本。",
"sources": [
{
"label": "§5、§5.1 与迁移实验",
"url": "https://arxiv.org/abs/2608.07645"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"selection": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"evaluation": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
],
"isolation": [
{
"label": "§5",
"url": "https://arxiv.org/html/2608.07645#S5"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.07645#S5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有代码自进化主要改进历史档案和分支选择，但真正写补丁时往往仍只看某个 agent 在一道题上的一次失败。这样难分辨反复出现的系统缺陷与偶发的题目问题，也浪费了档案中其他任务和分支的对照信息，因此作者研究更充分利用这些证据生成修改。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.07645#S1"
}
]
},
{
"key": "position",
"value": "研究自改进代码 agent 如何利用不同任务和不同进化分支之间的经验，提高后续系统改进的质量与搜索效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.07645"
}
]
},
{
"key": "conclusion",
"value": "在受控分析及编程基准中提高搜索效率与表现，展示进化档案除存档外的诊断价值。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.07645"
}
]
}
],
"fields": {
"object": "coding agent 的可执行代码与运行结构。",
"verdict": "读取不同任务、不同历史分支的代码测试结果和执行记录，比较哪些改动有效、哪些导致退化，再提出修改或组合已有方案。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2403.03186",
"title": "Cradle",
"url": "https://arxiv.org/abs/2403.03186",
"date": "2024-03-05",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"M0",
"MemoryContent",
"Online",
"Skill",
"Tool",
"org:peking",
"org:ntu"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。长程 computer-control 中持续记录 observation/reflection，并在出现重复 procedure 时固化成 executable skill；是“experience→executable skill”的重要早期扩展。",
"谁来改 → 谁执行；基础 harness": "固定 computer-control agent 的 memory/skill manager → 同 agent 后续执行。",
"Feedback / evidence": "environment observation/outcome + reflection。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 145,
"fields": {
"时间": "2024-03-05",
"论文": "[Cradle](https://arxiv.org/abs/2403.03186)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。长程 computer-control 中持续记录 observation/reflection，并在出现重复 procedure 时固化成 executable skill；是“experience→executable skill”的重要早期扩展。",
"谁来改 → 谁执行；基础 harness": "固定 computer-control agent 的 memory/skill manager → 同 agent 后续执行。",
"Feedback / evidence": "environment observation/outcome + reflection。",
"标签": "`#MemoryContent #Skill #Tool #Online #EnvironmentReward #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "长程 computer-control 中持续记录 observation/reflection，并在出现重复 procedure 时固化成 executable skill；是“experience→executable skill”的重要早期扩展。",
"novelty": "统一通过屏幕获取信息、用鼠标键盘操作，在这一接口上组织规划、反思、技能和记忆，面向完整的计算机使用过程。",
"object": "交互中积累的经验记忆和可执行技能代码。",
"executor": "默认 gpt-4o-2024-05-13 看屏幕、规划操作并生成控制动作；电脑实际执行键鼠操作。text-embedding-ada-002 只用于技能向量检索。",
"modifier": "同一 GPT-4o 驱动 Cradle 的反思、任务推断和技能整理模块，更新外部记忆与技能；固定框架组织这些调用。",
"roleContext": "固定 computer-control agent 的 memory/skill manager → 同 agent 后续执行。",
"seed": "作者自建通用电脑控制框架，通过截图理解界面、用键鼠执行操作，不调用游戏内部状态 API。模块包括信息获取、反思、任务推断、技能管理、动作规划和记忆；因此是完整的视觉交互底座，不是只有文本 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "读取操作前后的截图或视频帧，由视觉模型判断动作是否完成并解释失败，再调整计划。最终效果按对应游戏或软件任务评价。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：在电脑游戏/软件实际交互中学习；RDR2 从少量基本移动技能起步，部分环境另给预置原子操作。\n\n调试 / 选版本数据：屏幕视频、动作执行结果及模型反思，不依赖通用的离线标签训练集。\n\n最终测试数据：RDR2、Stardew Valley 等游戏及软件控制任务，各按任务完成或活动表现评估，通常重复 5 次。\n\n数据隔离与证据边界：实验强调统一键鼠/视频接口；各环境的预置技能不同，不能解释为同一训练集上的统一留出测试。",
"cycle": "多模态模型 看前后视频帧判断动作是否完成、解释失败，再规划和修订技能；新技能存入程序记忆，后续按相似度检索。",
"train": "在电脑游戏/软件实际交互中学习；RDR2 从少量基本移动技能起步，部分环境另给预置原子操作。",
"debug": "屏幕视频、动作执行结果及模型反思，不依赖通用的离线标签训练集。",
"test": "RDR2、Stardew Valley 等游戏及软件控制任务，各按任务完成或活动表现评估，通常重复 5 次。",
"isolation": "实验强调统一键鼠/视频接口；各环境的预置技能不同，不能解释为同一训练集上的统一留出测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建通用电脑控制框架，通过截图理解界面、用键鼠执行操作，不调用游戏内部状态 API。模块包括信息获取、反思、任务推断、技能管理、动作规划和记忆；因此是完整的视觉交互底座，不是只有文本 ReAct。",
"protocol": "**交互与评估环境：**包括 Red Dead Redemption 2、Stardew Valley 等电脑环境；Stardew Valley 实验使用 1.6.8。技能和记忆来自与环境实际交互，比较任务完成与长期活动表现。它不是把游戏轨迹划为一个统一训练集和测试集；各环境的完整任务数与独立迁移划分本轮未核实。",
"sections": "框架模块；实验与 Stardew Valley 环境说明",
"source": "https://arxiv.org/abs/2403.03186",
"version": "2403.03186v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "9f20961ffa1506aeab65cf06fc8265456c3861caccc210f9aadbf7505cc43d64",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "默认 gpt-4o-2024-05-13 看屏幕、规划操作并生成控制动作；电脑实际执行键鼠操作。text-embedding-ada-002 只用于技能向量检索。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2403.03186#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2403.03186#A1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一 GPT-4o 驱动 Cradle 的反思、任务推断和技能整理模块，更新外部记忆与技能；固定框架组织这些调用。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2403.03186#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2403.03186#A1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "交互中积累的经验记忆和可执行技能代码。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "读取操作前后的截图或视频帧，由视觉模型判断动作是否完成并解释失败，再调整计划。最终效果按对应游戏或软件任务评价。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建通用电脑控制框架，通过截图理解界面、用键鼠执行操作，不调用游戏内部状态 API。模块包括信息获取、反思、任务推断、技能管理、动作规划和记忆；因此是完整的视觉交互底座，不是只有文本 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"sources": [
{
"label": "框架模块；实验与 Stardew Valley 环境说明",
"url": "https://arxiv.org/abs/2403.03186"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "多模态模型 看前后视频帧判断动作是否完成、解释失败，再规划和修订技能；新技能存入程序记忆，后续按相似度检索。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "在电脑游戏/软件实际交互中学习；RDR2 从少量基本移动技能起步，部分环境另给预置原子操作。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "屏幕视频、动作执行结果及模型反思，不依赖通用的离线标签训练集。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "RDR2、Stardew Valley 等游戏及软件控制任务，各按任务完成或活动表现评估，通常重复 5 次。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "实验强调统一键鼠/视频接口；各环境的预置技能不同，不能解释为同一训练集上的统一留出测试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "统一通过屏幕获取信息、用鼠标键盘操作，在这一接口上组织规划、反思、技能和记忆，面向完整的计算机使用过程。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2403.03186v3",
"version": "2403.03186v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "统一要求屏幕输入和鼠标键盘输出，在这个接口上结合反思、任务推断、规划、技能维护和记忆；记忆进化只是完整 computer-control 系统的一部分。",
"feedbackCases": [
{
"label": "游戏在线学习",
"data": "RDR2、Stardew Valley 等游戏中的实际交互任务。",
"scoring": "模型读取屏幕/视频与动作执行后的变化，反思动作是否达到子目标；游戏任务完成和活动指标用于研究评估。",
"visible": "视觉观察、动作结果和模型自我反思；不是一套随时给精确正确答案的监督接口。",
"use": "修正动作和技能，再继续当前环境；缺少明确反馈本身就是论文讨论的失败来源。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"judgment": "模型从屏幕、视频与状态变化自评子目标；研究端另计游戏完成指标"
},
{
"label": "电脑软件 / OSWorld",
"data": "软件操作与 OSWorld 中的真实电脑任务。",
"scoring": "OSWorld 每题配人工编写的验收脚本；系统通过屏幕和鼠标/键盘执行任务，研究端以任务脚本检查是否完成。",
"visible": "执行时主要看到屏幕变化，不能据最终有脚本评分就假定操作 agent 每步都拿到脚本答案。",
"use": "以任务完成情况评估通用电脑控制能力，与游戏中的自我反思信号区分。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"judgment": "OSWorld 每题人工编写的可执行验收脚本"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
0,
1
],
"evolution": "在电脑游戏/软件实际交互中学习；RDR2 从少量基本移动技能起步，部分环境另给预置原子操作。",
"selection": "屏幕视频、动作执行结果及模型反思，不依赖通用的离线标签训练集。",
"evaluation": "RDR2、Stardew Valley 等游戏及软件控制任务，各按任务完成或活动表现评估，通常重复 5 次。",
"isolation": "实验强调统一键鼠/视频接口；各环境的预置技能不同，不能解释为同一训练集上的统一留出测试。",
"roles": {
"executor": {
"value": "默认 gpt-4o-2024-05-13 看屏幕、规划操作并生成控制动作；电脑实际执行键鼠操作。text-embedding-ada-002 只用于技能向量检索。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2403.03186#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2403.03186#A1"
}
]
},
"modifier": {
"value": "同一 GPT-4o 驱动 Cradle 的反思、任务推断和技能整理模块，更新外部记忆与技能；固定框架组织这些调用。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2403.03186#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2403.03186#S3.SS3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2403.03186#A2.SS3"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2403.03186#A4.SS3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2403.03186#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2403.03186#A1"
}
]
},
"seed": {
"value": "作者自建通用电脑控制框架，通过截图理解界面、用键鼠执行操作，不调用游戏内部状态 API。模块包括信息获取、反思、任务推断、技能管理、动作规划和记忆；因此是完整的视觉交互底座，不是只有文本 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"sources": [
{
"label": "框架模块；实验与 Stardew Valley 环境说明",
"url": "https://arxiv.org/abs/2403.03186"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2403.03186#S4.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 常依赖人为包装的专用观察和动作接口，环境间差异很大，导致换游戏或软件就难泛化。统一使用屏幕、音频和鼠标键盘又要求多模态理解、精细控制及长程记忆，因此作者把通用计算机交互与自主积累经验作为需要共同解决的问题。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2403.03186#S1"
}
]
},
{
"key": "position",
"value": "面向游戏和日常软件，探索 agent 能否像人一样依靠通用屏幕与操作接口完成长程任务，减少对逐环境专用接口的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2403.03186"
}
]
},
{
"key": "conclusion",
"value": "覆盖多个商业游戏、日常软件和 OSWorld，展示无需专用环境 API 的长程电脑操作。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2403.03186"
}
]
}
],
"fields": {
"object": "交互中积累的经验记忆和可执行技能代码。",
"verdict": "读取操作前后的截图或视频帧，由视觉模型判断动作是否完成并解释失败，再调整计划。最终效果按对应游戏或软件任务评价。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2403.03186"
}
]
},
{
"tag": "org:ntu",
"label": "Nanyang Technological University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2403.03186"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2409.07429",
"title": "Agent Workflow Memory (AWM)",
"url": "https://arxiv.org/abs/2409.07429",
"date": "2024-09-11",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HumanDemo",
"M0",
"MemoryContent",
"Online",
"Prequential",
"Workflow",
"org:mit",
"org:cmu",
"person:graham-neubig"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。从 demos 或 agent 自己完成的 web trajectories 抽象 reusable workflow，再检索指导 future tasks；关键是存 procedure 而不是 raw episode。",
"谁来改 → 谁执行；基础 harness": "workflow inducer/memory module → fixed web agent；基础为原 web-agent workflow。",
"Feedback / evidence": "demos / successful trajectories。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 146,
"fields": {
"时间": "2024-09-11",
"论文": "[Agent Workflow Memory (AWM)](https://arxiv.org/abs/2409.07429)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。从 demos 或 agent 自己完成的 web trajectories 抽象 reusable workflow，再检索指导 future tasks；关键是存 procedure 而不是 raw episode。",
"谁来改 → 谁执行；基础 harness": "workflow inducer/memory module → fixed web agent；基础为原 web-agent workflow。",
"Feedback / evidence": "demos / successful trajectories。",
"标签": "`#MemoryContent #Workflow #HumanDemo #Online #Prequential #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2024",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "AWM 从成功的网页操作轨迹中抽出可复用的多步 workflow，例如把某个具体页面上的操作抽象成未来任务可遵循的步骤。执行者遇到新任务时把这些 workflow 当作记忆来使用；因此积累的是操作程序，而不是原样堆叠整条案例。",
"novelty": "从具体网页操作案例归纳可复用步骤；既研究事先从训练示例建库，也研究边做测试任务边积累，两种数据使用方式不同。",
"object": "从轨迹归纳的多步工作流记忆；执行模型及工作流归纳、调用流程固定。",
"executor": "WebArena 网页交互实验使用 gpt-4-0613，并通过 BrowserGym 浏览器操作环境执行；Mind2Web 网页动作预测实验使用 gpt-3.5-turbo 或 gpt-4，采样温度设为 0，以减少输出随机性。",
"modifier": "工作流归纳与动作生成使用对应实验的同一模型：WebArena为gpt-4-0613；Mind2Web为gpt-3.5-turbo或gpt-4。固定归纳提示把轨迹转成工作流。",
"roleContext": "workflow inducer/memory module → fixed web agent；基础为原 web-agent workflow。",
"seed": "BrowserGym 网页 task agent 上加入从成功经历归纳的任务执行流程；Mind2Web 另使用元素过滤和文本化动作轨迹接口。任务执行流程给步骤方法，不替换浏览器底层执行器。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "在线成功轨迹用于归纳；评测基准最终用网页任务/动作指标评价，不能把最终 评估器 都当成归纳器可见监督。",
"diagnosis": "主要抽取成功轨迹里的公共子程序，并非以逐步定位失败代码为核心。",
"update": "成功轨迹 → 抽象 reusable workflow → 加入记忆 → 后续任务调用。",
"acceptance": "本轮未核实有独立 candidate regression gate；不能把成功轨迹筛选等同于独立验证。",
"experiments": [
{
"name": "Mind2Web offline",
"evolve": "提供的训练示例归纳 workflows",
"selection": "原文该段未单列独立选模集",
"test": "cross-task / cross-website / cross-domain",
"isolation": "offline 与 online 分开看",
"note": ""
},
{
"name": "WebArena / Mind2Web online",
"evolve": "在 test queries 流上持续归纳",
"selection": "任务流内更新记忆",
"test": "同一流中后续 queries",
"isolation": "在线经验积累",
"note": "跨网站/域设置存在，但 online 不是先 freeze 再测试。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2409.07429v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Mind2Web 离线 从训练轨迹归纳任务执行流程；WebArena/online 从此前任务交互累积。\n\n调试 / 选版本数据：轨迹及模型归纳结果决定写入内容；Mind2Web 以元素准确率、动作 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、步骤/任务成功率评估。\n\n最终测试数据：WebArena；Mind2Web 的 cross-task、cross-website、cross-domain 三种测试。\n\n数据隔离与证据边界：离线 使用固定流程库；在线 允许从已处理测试题更新记忆，跨网站/域不能当作全程冻结。",
"cycle": "从任务执行过程归纳可复用网页操作流程。离线设置先完成记忆构建再测试；在线设置在测试任务序列中继续积累。若把失败经历误总结为有效流程，错误也会传播到后续任务。",
"train": "Mind2Web 离线 从训练轨迹归纳任务执行流程；WebArena/online 从此前任务交互累积。",
"debug": "轨迹及模型归纳结果决定写入内容；Mind2Web 以元素准确率、动作 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、步骤/任务成功率评估。",
"test": "WebArena；Mind2Web 的 cross-task、cross-website、cross-domain 三种测试。",
"isolation": "离线 使用固定流程库；在线 允许从已处理测试题更新记忆，跨网站/域不能当作全程冻结。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "BrowserGym 网页 agent 上加入从成功经历归纳的 workflow；Mind2Web 另使用元素过滤和文本化动作轨迹接口。workflow 给步骤方法，不替换浏览器底层执行器。",
"protocol": "**数据：**WebArena 与 Mind2Web。离线设置先从 training 轨迹归纳流程，再测新任务；在线设置在执行序列中累积流程，两者不可混称冻结测试。\n\n**Mind2Web 泛化：**分别测 cross-task、cross-website、cross-domain，强调任务、网站和域的隔离层次不同。各设置实际抽样量与选 workflow 的独立验证集本轮待核实。",
"sections": "§3.1–3.2",
"source": "https://arxiv.org/abs/2409.07429",
"version": "2409.07429v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e1bcd8606c5375bab05094553e0b8408eedae38cb7d296b2fc7690d460265274",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "WebArena 网页交互实验使用 gpt-4-0613，并通过 BrowserGym 浏览器操作环境执行；Mind2Web 网页动作预测实验使用 gpt-3.5-turbo 或 gpt-4，采样温度设为 0，以减少输出随机性。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2409.07429#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "工作流归纳与动作生成使用对应实验的同一模型：WebArena为gpt-4-0613；Mind2Web为gpt-3.5-turbo或gpt-4。固定归纳提示把轨迹转成工作流。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2409.07429#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "从轨迹归纳的多步工作流记忆；执行模型及工作流归纳、调用流程固定。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "在线成功轨迹用于归纳；评测基准最终用网页任务/动作指标评价，不能把最终 评估器 都当成归纳器可见监督。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "BrowserGym 网页 task agent 上加入从成功经历归纳的任务执行流程；Mind2Web 另使用元素过滤和文本化动作轨迹接口。任务执行流程给步骤方法，不替换浏览器底层执行器。",
"sources": [
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/abs/2409.07429"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "从任务执行过程归纳可复用网页操作流程。离线设置先完成记忆构建再测试；在线设置在测试任务序列中继续积累。若把失败经历误总结为有效流程，错误也会传播到后续任务。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Mind2Web 离线 从训练轨迹归纳任务执行流程；WebArena/online 从此前任务交互累积。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "轨迹及模型归纳结果决定写入内容；Mind2Web 以元素准确率、动作 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、步骤/任务成功率评估。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "WebArena；Mind2Web 的 cross-task、cross-website、cross-domain 三种测试。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "离线 使用固定流程库；在线 允许从已处理测试题更新记忆，跨网站/域不能当作全程冻结。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从具体网页操作案例归纳可复用步骤；既研究事先从训练示例建库，也研究边做测试任务边积累，两种数据使用方式不同。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2409.07429v1",
"version": "2409.07429v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "核心对照是“复用具体案例”与“归纳案例里的 procedure”。论文同时有从训练示例预先归纳的 offline 版本，以及在测试任务流中边执行边积累的 online 版本，不能合写成一种 held-out 协议。",
"feedbackCases": [
{
"label": "离线归纳：Mind2Web",
"data": "从 Mind2Web 训练示范的操作轨迹归纳工作流；测试分跨任务、跨网站、跨领域。",
"scoring": "最终将预测网页元素和动作与标注操作比较，计算元素准确率、动作 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、步骤/任务成功率。",
"visible": "归纳阶段使用训练示范，不是每条新工作流都经最终测试分数筛选。",
"use": "用训练操作经验指导新任务；测试指标与记忆生成信号分开。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"judgment": "规则比较预测元素、动作与人工标注，计算准确率和 F1"
},
{
"label": "在线归纳：WebArena / Mind2Web",
"data": "按测试任务顺序执行，从此前任务积累工作流。",
"scoring": "在线阶段由语言模型评估器读取执行轨迹，给预测的成功/失败标签；被判成功才归纳工作流。最终 WebArena 任务成功由环境验收，Mind2Web 则按动作标注评价。",
"visible": "记忆更新使用模型预测标签，可能把错误轨迹误认为成功；不是拿最终基准答案直接归纳。",
"use": "工作流影响后续题，不回头修改已经报告的当前题答案。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
],
"judgment": "记忆写入用模型预测成功；正式 WebArena 用环境验收、Mind2Web 用标注动作比对"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0,
1
],
"testCases": [
0,
1
],
"evolution": "Mind2Web 离线 从训练轨迹归纳任务执行流程；WebArena/online 从此前任务交互累积。",
"selection": "轨迹及模型归纳结果决定写入内容；Mind2Web 以元素准确率、动作 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、步骤/任务成功率评估。",
"evaluation": "WebArena；Mind2Web 的 cross-task、cross-website、cross-domain 三种测试。",
"isolation": "离线 使用固定流程库；在线 允许从已处理测试题更新记忆，跨网站/域不能当作全程冻结。",
"roles": {
"executor": {
"value": "WebArena 网页交互实验使用 gpt-4-0613，并通过 BrowserGym 浏览器操作环境执行；Mind2Web 网页动作预测实验使用 gpt-3.5-turbo 或 gpt-4，采样温度设为 0，以减少输出随机性。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2409.07429#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS1"
}
]
},
"modifier": {
"value": "工作流归纳与动作生成使用对应实验的同一模型：WebArena为gpt-4-0613；Mind2Web为gpt-3.5-turbo或gpt-4。固定归纳提示把轨迹转成工作流。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2409.07429#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS1"
}
]
},
"seed": {
"value": "BrowserGym 网页 task agent 上加入从成功经历归纳的任务执行流程；Mind2Web 另使用元素过滤和文本化动作轨迹接口。任务执行流程给步骤方法，不替换浏览器底层执行器。",
"sources": [
{
"label": "§3.1–3.2",
"url": "https://arxiv.org/abs/2409.07429"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有网页 agent 主要学习固定示例，遇到页面或任务上下文变化就不够稳健；每题独立执行又丢掉了先前成功和失败的价值。因此，作者希望从旧任务抽出可组合的常用步骤，帮助 agent 适应新场景，并逐步解决比原示例更复杂的操作链。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2409.07429#S1"
}
]
},
{
"key": "position",
"value": "研究网页 agent 能否把既往操作经历转化为跨任务复用的工作流程，从而提高长操作链任务的可靠性与效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2409.07429"
}
]
},
{
"key": "conclusion",
"value": "在 Mind2Web 和 WebArena 上提高成功率、减少操作步数，并验证跨网站及领域的复用。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2409.07429"
}
]
}
],
"fields": {
"object": "从轨迹归纳的多步工作流记忆；执行模型及工作流归纳、调用流程固定。",
"executor": "WebArena 网页交互实验使用 gpt-4-0613，并通过 BrowserGym 浏览器操作环境执行；Mind2Web 网页动作预测实验使用 gpt-3.5-turbo 或 gpt-4，采样温度设为 0，以减少输出随机性。"
}
},
"attributions": [
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2409.07429"
}
]
},
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2409.07429"
}
]
},
{
"tag": "person:graham-neubig",
"label": "Graham Neubig",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2409.07429"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2502.12110",
"title": "A-MEM: Agentic Memory for LLM Agents",
"url": "https://arxiv.org/abs/2502.12110",
"date": "2025-02-17",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Continual",
"M0",
"MemoryContent",
"Online",
"SelfFeedback"
],
"fields": {
"本质定位": "每条新 memory 被写成带 context/keywords/tags/links 的 note；新 note 会触发历史 note 的链接与属性更新，形成动态 Zettelkasten network。",
"谁来改 → 谁执行；基础 harness": "fixed memory updater → agent with Zettelkasten-like graph。",
"Feedback / evidence": "interaction content + LLM-generated relations。",
"什么在变": "memory notes、links、attributes。",
"谁来改 / 谁执行": "**改**：固定 A-MEM agentic update procedure。
**执行**：多种 foundation-model agents。",
"基础 harness": "LLM agent + Zettelkasten-like memory network。",
"Feedback": "新 interaction/memory content + LLM relation judgment。",
"Evolution → Eval": "LoCoMo、DialSim 等 memory benchmarks。",
"Meta-depth": "M0/M1-：memory graph structure 变，memory algorithm fixed。",
"相对之前真正新增什么": "相对 fixed vector-store memory，新增的是 **memory structure/content 自组织**；但 encode/link/update algorithm 仍由作者固定。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 147,
"fields": {
"时间": "2025-02-17",
"论文": "[A-MEM](https://arxiv.org/abs/2502.12110)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem**。新 memory 不只是 append，而会回写 related old memories 的 summary/attributes/links，形成自组织 graph；仍主要是 memory state 进化，不是 memory algorithm 自修改。",
"谁来改 → 谁执行；基础 harness": "fixed memory updater → agent with Zettelkasten-like graph。",
"Feedback / evidence": "interaction content + LLM-generated relations。",
"标签": "`#MemoryContent #Online #Continual #SelfFeedback #M0`"
}
},
{
"section": "A2. Context / Memory Evolution",
"line": 291,
"fields": {
"优先级": "**K**",
"时间": "2025-02-17",
"论文": "[A-MEM: Agentic Memory for LLM Agents](https://arxiv.org/abs/2502.12110)",
"本质定位": "每条新 memory 被写成带 context/keywords/tags/links 的 note；新 note 会触发历史 note 的链接与属性更新，形成动态 Zettelkasten network。",
"什么在变": "memory notes、links、attributes。",
"谁来改 / 谁执行": "**改**：固定 A-MEM agentic update procedure。
**执行**：多种 foundation-model agents。",
"基础 harness": "LLM agent + Zettelkasten-like memory network。",
"Feedback": "新 interaction/memory content + LLM relation judgment。",
"Evolution → Eval": "LoCoMo、DialSim 等 memory benchmarks。",
"Meta-depth": "M0/M1-：memory graph structure 变，memory algorithm fixed。",
"相对之前真正新增什么": "相对 fixed vector-store memory，新增的是 **memory structure/content 自组织**；但 encode/link/update algorithm 仍由作者固定。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0",
"M1"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "每条新 memory 被写成带 context/keywords/tags/links 的 note；新 note 会触发历史 note 的链接与属性更新，形成动态 Zettelkasten network。",
"novelty": "新经历进入记忆时会生成关联并更新既有记录，让记忆内容和连接共同变化；生成、关联和更新规则仍由作者预设。",
"object": "记忆条目的内容、属性与相互链接，形成可持续更新的记忆网络；不修改模型参数或记忆管理算法。",
"executor": "主实验包括GPT-4o-mini/4o、Qwen2.5-1.5B/3B、Llama3.2-1B/3B；附录还测DeepSeek-R1-32B、Claude3 Haiku、Claude3.5 Haiku。",
"modifier": "对应实验的基础 语言模型 按固定提示生成记忆笔记、判断连接并修订旧记忆的上下文、关键词和标签；向量模型 all-MiniLM-L6-v2 只负责召回。这里的“进化”是记忆内容变化，不是另训练一个更新模型。",
"roleContext": "**改**：固定 A-MEM agentic update procedure。
**执行**：多种 foundation-model agents。",
"seed": "在对话模型外加入作者自建的关联记忆网络。新经历形成带上下文说明的原子笔记，与已有笔记建立链接；查询时取 top-k 相关记忆。已有记忆可被重组，变化的是外部笔记和关系，不是模型权重。",
"fixed": "M0/M1-：memory graph structure 变，memory algorithm fixed",
"verdict": "新对话提供记忆内容；模型判断新旧笔记之间的关联，决定连接或修订哪些信息。最终问答成绩用于评价记忆效果，不是每次写记忆时得到的奖励。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "LoCoMo、DialSim 等 memory benchmarks。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim 的长对话用于构建记忆，不进行模型参数训练。\n\n调试 / 选版本数据：新笔记与近邻内容驱动记忆维护；问答指标衡量效果，不是逐条记忆的外部纠错信号。\n\n最终测试数据：LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 多类长期问答（该版本报告 7,512 对）；DialSim 的多人长对话问答。\n\n数据隔离与证据边界：对话是可见记忆材料，问题检验能否从中找出/组合事实；不是普通监督训练集与测试集的划分。",
"cycle": "新对话写成原子笔记，语言模型 结合相邻笔记决定加强链接或修改已有上下文/标签；检索这些笔记回答问题，不按评测基准分数选择运行框架版本。",
"train": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim 的长对话用于构建记忆，不进行模型参数训练。",
"debug": "新笔记与近邻内容驱动记忆维护；问答指标衡量效果，不是逐条记忆的外部纠错信号。",
"test": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 多类长期问答（该版本报告 7,512 对）；DialSim 的多人长对话问答。",
"isolation": "对话是可见记忆材料，问题检验能否从中找出/组合事实；不是普通监督训练集与测试集的划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在对话模型外加入作者自建的关联记忆网络。新经历形成带上下文说明的原子笔记，与已有笔记建立链接；查询时取 top-k 相关记忆。已有记忆可被重组，变化的是外部笔记和关系，不是模型权重。",
"protocol": "**记忆来源→测试：**先处理 LoCoMo 的长期对话，再用基于这些对话的问题测试单跳、多跳、时间推理等能力；另用 DialSim 的长期多人对话问答作评估，内容来自 Friends、The Big Bang Theory、The Office。这里的对话是待记住的材料，问题是记忆能力测试，不能把它们写成普通监督训练/测试。论文此版给出 LoCoMo 7,512 个问答对；其他论文可能采用不同子集，不统一替换数量。独立调参/选模划分本轮未核实。",
"sections": "实验数据与记忆结构",
"source": "https://arxiv.org/abs/2502.12110",
"version": "2502.12110v11",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f4c560498d0f18a424cead61b365e78082f197b31c4aba68776e680200a633e5",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验包括GPT-4o-mini/4o、Qwen2.5-1.5B/3B、Llama3.2-1B/3B；附录还测DeepSeek-R1-32B、Claude3 Haiku、Claude3.5 Haiku。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2502.12110#S4.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2502.12110#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应实验的基础 语言模型 按固定提示生成记忆笔记、判断连接并修订旧记忆的上下文、关键词和标签；向量模型 all-MiniLM-L6-v2 只负责召回。这里的“进化”是记忆内容变化，不是另训练一个更新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2502.12110#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2502.12110#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2502.12110#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2502.12110#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "记忆条目的内容、属性与相互链接，形成可持续更新的记忆网络；不修改模型参数或记忆管理算法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "新对话提供记忆内容；模型判断新旧笔记之间的关联，决定连接或修订哪些信息。最终问答成绩用于评价记忆效果，不是每次写记忆时得到的奖励。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在对话模型外加入作者自建的关联记忆网络。新经历形成带上下文说明的原子笔记，与已有笔记建立链接；查询时取 top-k 相关记忆。已有记忆可被重组，变化的是外部笔记和关系，不是模型权重。",
"sources": [
{
"label": "实验数据与记忆结构",
"url": "https://arxiv.org/abs/2502.12110"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "新对话写成原子笔记，语言模型 结合相邻笔记决定加强链接或修改已有上下文/标签；检索这些笔记回答问题，不按评测基准分数选择运行框架版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim 的长对话用于构建记忆，不进行模型参数训练。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "新笔记与近邻内容驱动记忆维护；问答指标衡量效果，不是逐条记忆的外部纠错信号。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 多类长期问答（该版本报告 7,512 对）；DialSim 的多人长对话问答。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "对话是可见记忆材料，问题检验能否从中找出/组合事实；不是普通监督训练集与测试集的划分。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "新经历进入记忆时会生成关联并更新既有记录，让记忆内容和连接共同变化；生成、关联和更新规则仍由作者预设。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2502.12110v11",
"version": "2502.12110v11",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 fixed vector-store memory，新增的是 **memory structure/content 自组织**；但 encode/link/update algorithm 仍由作者固定。",
"feedbackCases": [
{
"label": "记忆写入与维护",
"data": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim 的长对话。",
"scoring": "模型根据新笔记和检索近邻判断语义关联、生成链接并更新已有记忆；不靠问答测试答案逐条奖励记忆。",
"visible": "可见对话内容、提炼的笔记及邻接记忆。",
"use": "构建和维护记忆网络；没有模型参数训练。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"judgment": "模型判断记忆语义关联；写入时无标准答案正确性奖励"
},
{
"label": "记忆问答评估",
"data": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 多类长期问答与 DialSim 多人对话问答。",
"scoring": "回答与参考答案比较；主指标 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、BLEU-1 衡量词项重合，另报告 ROUGE、METEOR、SBERT 等及文本用量。",
"visible": "这些是研究端的问答评估指标，不是每次写入记忆可获得的外部纠错标签。",
"use": "检验记忆有助于回答哪些问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"judgment": "规则计算回答与参考答案的词项重合指标；另计算语义相似度"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim 的长对话用于构建记忆，不进行模型参数训练。",
"selection": "新笔记与近邻内容驱动记忆维护；问答指标衡量效果，不是逐条记忆的外部纠错信号。",
"evaluation": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 多类长期问答（该版本报告 7,512 对）；DialSim 的多人长对话问答。",
"isolation": "对话是可见记忆材料，问题检验能否从中找出/组合事实；不是普通监督训练集与测试集的划分。",
"roles": {
"executor": {
"value": "主实验包括GPT-4o-mini/4o、Qwen2.5-1.5B/3B、Llama3.2-1B/3B；附录还测DeepSeek-R1-32B、Claude3 Haiku、Claude3.5 Haiku。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2502.12110#S4.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2502.12110#A1.SS3"
}
]
},
"modifier": {
"value": "对应实验的基础 语言模型 按固定提示生成记忆笔记、判断连接并修订旧记忆的上下文、关键词和标签；向量模型 all-MiniLM-L6-v2 只负责召回。这里的“进化”是记忆内容变化，不是另训练一个更新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2502.12110#S3"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2502.12110#A2.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2502.12110#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2502.12110#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2502.12110#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2502.12110#S4.SS2"
}
]
},
"seed": {
"value": "在对话模型外加入作者自建的关联记忆网络。新经历形成带上下文说明的原子笔记，与已有笔记建立链接；查询时取 top-k 相关记忆。已有记忆可被重组，变化的是外部笔记和关系，不是模型权重。",
"sources": [
{
"label": "实验数据与记忆结构",
"url": "https://arxiv.org/abs/2502.12110"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2502.12110#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2502.12110#S4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "记忆系统往往预先固定存储结构、关联方式和存取时机，新知识只能塞进旧框架，不能促成新的连接或组织方式。这种僵硬结构会限制新环境泛化与长期交互，因此需要让记忆随新经历主动重组，并允许新信息修订对旧记忆的理解。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2502.12110#S1"
}
]
},
{
"key": "position",
"value": "面向持续对话中的记忆管理，研究 agent 能否随新信息自主组织和修订历史知识，而不仅是存下记录后按相似度检索。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2502.12110"
}
]
},
{
"key": "conclusion",
"value": "新记忆会触发旧笔记及连接更新；多种基础模型上的实验支持这种结构化记忆的价值。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2502.12110"
}
]
}
],
"fields": {
"object": "记忆条目的内容、属性与相互链接，形成可持续更新的记忆网络；不修改模型参数或记忆管理算法。",
"executor": "GPT-4o-mini／4o、Qwen2.5-1.5B／3B、Llama3.2-1B／3B 配合记忆回答问题；扩展实验还比较 DeepSeek-R1-32B 和 Claude Haiku 系列。",
"verdict": "新对话提供记忆内容；模型判断新旧笔记之间的关联，决定连接或修订哪些信息。最终问答成绩用于评价记忆效果，不是每次写记忆时得到的奖励。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2508.06433",
"title": "Memp",
"url": "https://arxiv.org/abs/2508.06433",
"date": "2025-08-08",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"M0",
"MemoryContent",
"Online",
"Prequential",
"Skill",
"Workflow"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。把 trajectory 蒸馏为 step instructions 和 script-like procedural memory，并允许 add/correct/deprecate；更接近可维护 procedure library。",
"谁来改 → 谁执行；基础 harness": "procedural-memory updater → fixed task agent。",
"Feedback / evidence": "accumulated trajectories / outcome。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 148,
"fields": {
"时间": "2025-08-08",
"论文": "[Memp](https://arxiv.org/abs/2508.06433)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。把 trajectory 蒸馏为 step instructions 和 script-like procedural memory，并允许 add/correct/deprecate；更接近可维护 procedure library。",
"谁来改 → 谁执行；基础 harness": "procedural-memory updater → fixed task agent。",
"Feedback / evidence": "accumulated trajectories / outcome。",
"标签": "`#MemoryContent #Skill #Workflow #Online #Prequential #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 trajectory 蒸馏为 step instructions 和 script-like procedural memory，并允许 add/correct/deprecate；更接近可维护 procedure library。",
"novelty": "比较执行记录、逐步指令和抽象脚本等经验表示，并比较怎样建库、检索和更新，判断哪种操作知识更能减少重复探索。",
"object": "程序记忆中的具体轨迹与抽象操作脚本。",
"executor": "主实验用GPT-4o、Claude及Qwen2.5-72B-Instruct；迁移实验将GPT-4o积累的程序记忆交给Qwen2.5-14B-Instruct使用。",
"modifier": "程序记忆更新流程调用基础 语言模型 总结和修订操作经验；主配置为 GPT-4o、Claude、Qwen2.5-72B-Instruct。跨模型实验明确由 GPT-4o 生成记忆，再供 Qwen2.5-14B 使用；论文 Backbones 段没有给出 Claude 的具体版本。",
"roleContext": "procedural-memory updater → fixed task agent。",
"seed": "给 task agent 加上程序性记忆：把交互经历整理成可复用的操作步骤，并比较只存脚本、完整轨迹、抽象程序步骤三种粒度。底层仍通过工具或文本动作与环境交互；不是重新生成一套通用运行框架。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "用任务的执行过程及成败结果整理操作经验，后续做题时检索复用；经验可以增加、修改或删除，各任务的最终评价方法见表格。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "原记录明确标注 #Prequential；经验只应影响后续任务。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：TravelPlanner、ALFWorld（通过文字动作完成家居物体操作的交互环境） 交互轨迹用于构建程序记忆，比较完整轨迹、抽象脚本及两者结合。\n\n调试 / 选版本数据：ALFWorld（通过文字动作完成家居物体操作的交互环境） 环境返回 0/1；TravelPlanner 计划转 JSON 后检查常识与硬约束。\n\n最终测试数据：ALFWorld（通过文字动作完成家居物体操作的交互环境） 分别报告 开发／测试；TravelPlanner 使用 测试集 的两阶段计划评估。\n\n数据隔离与证据边界：论文同时研究顺序更新；开发／测试 列名本身不能证明记忆在测试期间冻结，需区分构建与更新实验。",
"cycle": "任务轨迹和成功/失败反馈交给记忆 构建者，保留具体过程与抽象规则；后续检索相似经验，并执行添加、删除、修改。",
"train": "TravelPlanner、ALFWorld（通过文字动作完成家居物体操作的交互环境） 交互轨迹用于构建程序记忆，比较完整轨迹、抽象脚本及两者结合。",
"debug": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 环境返回 0/1；TravelPlanner 计划转 JSON 后检查常识与硬约束。",
"test": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 分别报告 开发／测试；TravelPlanner 使用 测试集 的两阶段计划评估。",
"isolation": "论文同时研究顺序更新；开发／测试 列名本身不能证明记忆在测试期间冻结，需区分构建与更新实验。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "给任务 agent 加上程序性记忆：把交互经历整理成可复用的操作步骤，并比较只存脚本、完整轨迹、抽象程序步骤三种粒度。底层仍通过工具或文本动作与环境交互；不是重新生成一套通用运行框架。",
"protocol": "**任务数据：**TravelPlanner 用于工具调用和带约束的旅行计划；ALFWorld 用于多轮家务操作。ALFWorld 分别报告 dev 与 test 成功率，TravelPlanner 检查常识约束和硬约束。记忆来自任务交互经历；构建记忆所用的具体 split、数量及是否在测试序列持续更新，本轮尚未核实，不能据 dev/test 列名推断严格隔离。",
"sections": "§4.1、Table 1；附录 C",
"source": "https://arxiv.org/abs/2508.06433",
"version": "2508.06433v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "351e6409b54d5129bb35654f792a3dd6874faabdd127480ff784b274abe2ecb1",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验用GPT-4o、Claude及Qwen2.5-72B-Instruct；迁移实验将GPT-4o积累的程序记忆交给Qwen2.5-14B-Instruct使用。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "程序记忆更新流程调用基础 语言模型 总结和修订操作经验；主配置为 GPT-4o、Claude、Qwen2.5-72B-Instruct。跨模型实验明确由 GPT-4o 生成记忆，再供 Qwen2.5-14B 使用；论文 Backbones 段没有给出 Claude 的具体版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "程序记忆中的具体轨迹与抽象操作脚本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用任务的执行过程及成败结果整理操作经验，后续做题时检索复用；经验可以增加、修改或删除，各任务的最终评价方法见表格。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "给 task agent 加上程序性记忆：把交互经历整理成可复用的操作步骤，并比较只存脚本、完整轨迹、抽象程序步骤三种粒度。底层仍通过工具或文本动作与环境交互；不是重新生成一套通用运行框架。",
"sources": [
{
"label": "§4.1、Table 1；附录 C",
"url": "https://arxiv.org/abs/2508.06433"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "任务轨迹和成功/失败反馈交给记忆 构建者，保留具体过程与抽象规则；后续检索相似经验，并执行添加、删除、修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "TravelPlanner、ALFWorld（通过文字动作完成家居物体操作的交互环境） 交互轨迹用于构建程序记忆，比较完整轨迹、抽象脚本及两者结合。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 环境返回 0/1；TravelPlanner 计划转 JSON 后检查常识与硬约束。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 分别报告 开发／测试；TravelPlanner 使用 测试集 的两阶段计划评估。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "论文同时研究顺序更新；开发／测试 列名本身不能证明记忆在测试期间冻结，需区分构建与更新实验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "比较执行记录、逐步指令和抽象脚本等经验表示，并比较怎样建库、检索和更新，判断哪种操作知识更能减少重复探索。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2508.06433v4",
"version": "2508.06433v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "比较 trajectory、逐步指令与抽象脚本等程序记忆，以及 build/retrieve/update 策略；重点是可维护的操作知识怎样降低重复探索。",
"feedbackCases": [
{
"label": "ALFWorld",
"data": "交互轨迹构建程序性记忆，另报告 开发／测试。",
"scoring": "环境返回任务目标完成的 0/1 信号。",
"visible": "成功/失败和逐步操作记录用于构建、比较完整轨迹与抽象操作脚本。",
"use": "在后续任务检索复用程序记忆；不是让模型自行判定所有任务成功。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"judgment": "ALFWorld 环境程序判任务是否完成"
},
{
"label": "TravelPlanner",
"data": "行程规划任务，用经历构建记忆，在 测试集 上作两阶段计划评价。",
"scoring": "把生成计划转换成 JSON，按基准检查常识要求及预算、日期等硬约束。",
"visible": "任务判分与计划内容作为经验；可用的是显式基准奖励。",
"use": "比较不同记忆表示的泛化，不能解释为无监督从任意用户对话自学。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"judgment": "TravelPlanner 程序检查 JSON 行程中的常识与硬约束"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "TravelPlanner、ALFWorld（通过文字动作完成家居物体操作的交互环境） 交互轨迹用于构建程序记忆，比较完整轨迹、抽象脚本及两者结合。",
"selection": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 环境返回 0/1；TravelPlanner 计划转 JSON 后检查常识与硬约束。",
"evaluation": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 分别报告 开发／测试；TravelPlanner 使用 测试集 的两阶段计划评估。",
"isolation": "论文同时研究顺序更新；开发／测试 列名本身不能证明记忆在测试期间冻结，需区分构建与更新实验。",
"roles": {
"executor": {
"value": "主实验用GPT-4o、Claude及Qwen2.5-72B-Instruct；迁移实验将GPT-4o积累的程序记忆交给Qwen2.5-14B-Instruct使用。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px2"
}
]
},
"modifier": {
"value": "程序记忆更新流程调用基础 语言模型 总结和修订操作经验；主配置为 GPT-4o、Claude、Qwen2.5-72B-Instruct。跨模型实验明确由 GPT-4o 生成记忆，再供 Qwen2.5-14B 使用；论文 Backbones 段没有给出 Claude 的具体版本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.06433#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2508.06433#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px2"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2508.06433#S5.SS0.SSS0.Px2"
}
]
},
"seed": {
"value": "给 task agent 加上程序性记忆：把交互经历整理成可复用的操作步骤，并比较只存脚本、完整轨迹、抽象程序步骤三种粒度。底层仍通过工具或文本动作与环境交互；不是重新生成一套通用运行框架。",
"sources": [
{
"label": "§4.1、Table 1；附录 C",
"url": "https://arxiv.org/abs/2508.06433"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.06433#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2508.06433#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2508.06433#A3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务包含许多步骤，重复从头推理和试错代价高；但现有操作知识多是脆弱的手写提示，或混在昂贵才能更新的模型参数里。已有记忆框架也很少系统处理技能的建立、索引、修补和淘汰，因此作者关注可持续维护的操作经验，而不只是存下过去发生了什么。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2508.06433#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否在不更新模型参数的情况下积累、修正和复用操作经验，改善后续任务的规划与执行效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2508.06433"
}
]
},
{
"key": "conclusion",
"value": "在规划与交互任务中提高成功率和效率；强模型生成的操作记忆也可帮助弱模型。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2508.06433"
}
]
}
],
"fields": {
"object": "程序记忆中的具体轨迹与抽象操作脚本。",
"verdict": "用任务的执行过程及成败结果整理操作经验，后续做题时检索复用；经验可以增加、修改或删除，各任务的最终评价方法见表格。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2509.25140",
"title": "ReasoningBank",
"url": "https://arxiv.org/abs/2509.25140",
"date": "2025-09-29",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"M0",
"MemoryContent",
"Online",
"Prequential",
"SelfFeedback",
"org:google-cloud-ai",
"org:uiuc"
],
"fields": {
"本质定位": "**H-Mem**。从自判 success/failure 中抽 generalizable reasoning strategy；额外 test-time rollouts继续扩 memory bank。真正风险是 self-judge contamination。",
"谁来改 → 谁执行；基础 harness": "fixed distiller/retriever → same solver/agent 后续使用 reasoning memory。",
"Feedback / evidence": "**self-judged** success/failure。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 149,
"fields": {
"时间": "2025-09-29",
"论文": "[ReasoningBank](https://arxiv.org/abs/2509.25140)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem**。从自判 success/failure 中抽 generalizable reasoning strategy；额外 test-time rollouts继续扩 memory bank。真正风险是 self-judge contamination。",
"谁来改 → 谁执行；基础 harness": "fixed distiller/retriever → same solver/agent 后续使用 reasoning memory。",
"Feedback / evidence": "**self-judged** success/failure。",
"标签": "`#MemoryContent #SelfFeedback #Online #Prequential #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "ReasoningBank 同时从成功与失败轨迹中提炼推理策略：成功时解释为什么有效，失败时提炼以后如何避免。它先由与执行者同源的模型判断成败，再抽取少量记忆；MaTTS 另外增加多条尝试，让轨迹之间的比较产生更丰富的经验。",
"novelty": "把成功和失败都提炼成可复用策略，并研究与多次尝试结合的效果；需分别看记忆本身和增加尝试次数带来的收益。",
"object": "推理策略记忆条目；不是权重或记忆管理代码。\n\n固定部分：模型权重与抽取/检索流程固定。",
"executor": "Gemini2.5 Flash、Gemini2.5 Pro、Claude3.7 Sonnet；网页任务配BrowserGym，SWE配仅提供 Bash 命令行工具 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"modifier": "经验抽取与成败分类使用对应执行者的同一基础模型，如Gemini2.5 Flash；角色提示及外层追加/检索算法固定。",
"roleContext": "fixed distiller/retriever → same solver/agent 后续使用 reasoning memory。",
"seed": "Gemini-2.5/Claude-3.7 的 ReAct（交替进行推理、调用工具和读取结果的执行方式） task agent，网页用 BrowserGym，SWE 用仅提供 Bash 命令行工具；外部模块从成功和失败轨迹提取可检索原则。任务执行器与记忆提取不是不同训练出来的专用网络。",
"fixed": "模型权重与抽取/检索流程固定。",
"verdict": "轨迹和用户问题交给同一基础模型的二分类评分者，自判 Success/Failure。",
"diagnosis": "按自判成败选择不同反思提示，归纳成功策略或错误原因；每条轨迹最多抽取 3 条 memory items。",
"update": "新条目直接追加；MaTTS 从多条候选轨迹抽取更丰富经验。",
"acceptance": "原文采用最小 consolidation：直接加入、不额外 pruning；不是经独立 regression gate 才写入。",
"experiments": [
{
"name": "WebArena / Mind2Web / SWE-bench Verified",
"evolve": "运行中产生轨迹并积累记忆",
"selection": "自判与记忆抽取；MaTTS 另有候选选择",
"test": "同类任务序列及跨任务/网站/域设置",
"isolation": "不能自动等同于冻结后测试",
"note": "Memory-free、其他 memory 与额外 rollout 的对照需要分别比较。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2509.25140v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：无离线参数训练；在任务序列中从成功与失败轨迹提炼记忆。\n\n调试 / 选版本数据：语言模型 评分者判断轨迹结果并生成经验，不直接读取官方答案作为记忆更新标签。\n\n最终测试数据：WebArena 684；Mind2Web 1,341（cross-task 252、cross-website 177、cross-domain 912）；SWE-bench Verified 500。\n\n数据隔离与证据边界：先解当前题再更新供未来题用；MaTTS 另增加尝试预算，须与纯记忆效应区分。",
"cycle": "同一模型先自判本次任务成败，再提炼成功策略或避错经验，每条轨迹最多三条记忆。新记忆直接加入，未设置额外裁剪或独立回归检查；多次尝试版本利用同题不同轨迹获得更丰富的对比经验。",
"train": "无离线参数训练；在任务序列中从成功与失败轨迹提炼记忆。",
"debug": "语言模型 评分者判断轨迹结果并生成经验，不直接读取官方答案作为记忆更新标签。",
"test": "WebArena 684；Mind2Web 1,341（cross-task 252、cross-website 177、cross-domain 912）；SWE-bench Verified 500。",
"isolation": "先解当前题再更新供未来题用；MaTTS 另增加尝试预算，须与纯记忆效应区分。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "Gemini-2.5/Claude-3.7 的 ReAct agent，网页用 BrowserGym，SWE 用 bash-only；外部模块从成功和失败轨迹提取可检索原则。任务执行器与记忆提取不是不同训练出来的专用网络。",
"protocol": "**任务流：**WebArena 684 题；Mind2Web 1,341 题（cross-task 252、cross-website 177、cross-domain 912）；SWE-bench Verified 500 题。\n\n**更新：**经历在测试时序列中提炼入库、用于后续题；MaTTS 另增加尝试预算，不能把额外采样的收益都当成记忆。这些基准的 test 名称不意味着记忆在测试全过程冻结；顺序与正式分数反馈的具体控制见方法原文。",
"sections": "Datasets、agent 设置与记忆更新",
"source": "https://arxiv.org/abs/2509.25140",
"version": "2509.25140v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "47b41bc5cf03574aec8284ea1c42f963ed98220c1bcf38ee6a43366e0b88d991",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Gemini2.5 Flash、Gemini2.5 Pro、Claude3.7 Sonnet；网页任务配BrowserGym，SWE配仅提供 Bash 命令行工具 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2509.25140#A1.SS3.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "经验抽取与成败分类使用对应执行者的同一基础模型，如Gemini2.5 Flash；角色提示及外层追加/检索算法固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2509.25140#A1.SS3.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "推理策略记忆条目；不是权重或记忆管理代码。\n\n固定部分：模型权重与抽取/检索流程固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "轨迹和用户问题交给同一基础模型的二分类评分者，自判 Success/Failure。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Gemini-2.5/Claude-3.7 的 ReAct（交替进行推理、调用工具和读取结果的执行方式） task agent，网页用 BrowserGym，SWE 用仅提供 Bash 命令行工具；外部模块从成功和失败轨迹提取可检索原则。任务执行器与记忆提取不是不同训练出来的专用网络。",
"sources": [
{
"label": "Datasets、agent 设置与记忆更新",
"url": "https://arxiv.org/abs/2509.25140"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "同一模型先自判本次任务成败，再提炼成功策略或避错经验，每条轨迹最多三条记忆。新记忆直接加入，未设置额外裁剪或独立回归检查；多次尝试版本利用同题不同轨迹获得更丰富的对比经验。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无离线参数训练；在任务序列中从成功与失败轨迹提炼记忆。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "语言模型 评分者判断轨迹结果并生成经验，不直接读取官方答案作为记忆更新标签。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "WebArena 684；Mind2Web 1,341（cross-task 252、cross-website 177、cross-domain 912）；SWE-bench Verified 500。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "先解当前题再更新供未来题用；MaTTS 另增加尝试预算，须与纯记忆效应区分。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把成功和失败都提炼成可复用策略，并研究与多次尝试结合的效果；需分别看记忆本身和增加尝试次数带来的收益。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2509.25140v2",
"version": "2509.25140v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "关键不只是“有 memory”，而是成功/失败策略都可写入，并与额外 test-time rollouts 结合。要区分记忆本身的收益与更多尝试的收益，也要区分自判 feedback 与外部 verifier。",
"feedbackCases": [
{
"label": "在线记忆更新",
"data": "WebArena 684、Mind2Web 1,341、SWE-bench Verified 500 的任务序列。",
"scoring": "与执行者相同的基础模型读取用户问题与执行轨迹，自行二分类 Success/Failure；不读取官方标准答案生成记忆标签。",
"visible": "模型自判、成功/失败策略及执行记录；多次执行的对比还用于提炼经验。",
"use": "写入、检索和合并跨任务记忆；这属于可能有误差的自评反馈。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"judgment": "执行模型读取自己的轨迹自评，无标准答案"
},
{
"label": "最终任务评分",
"data": "WebArena 网页操作、Mind2Web 标注动作任务、SWE-bench Verified 修复任务。",
"scoring": "WebArena 以环境任务验收判成功；Mind2Web 用动作/元素标注评估；SWE 用仓库测试验收补丁。",
"visible": "官方任务成绩供报告效果，不能当成上一行记忆更新已直接使用的标签。",
"use": "比较记忆及额外执行预算的收益。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"judgment": "WebArena 用状态验收；Mind2Web 比标注动作；SWE 用仓库测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "无离线参数训练；在任务序列中从成功与失败轨迹提炼记忆。",
"selection": "语言模型 评分者判断轨迹结果并生成经验，不直接读取官方答案作为记忆更新标签。",
"evaluation": "WebArena 684；Mind2Web 1,341（cross-task 252、cross-website 177、cross-domain 912）；SWE-bench Verified 500。",
"isolation": "先解当前题再更新供未来题用；MaTTS 另增加尝试预算，须与纯记忆效应区分。",
"roles": {
"executor": {
"value": "Gemini2.5 Flash、Gemini2.5 Pro、Claude3.7 Sonnet；网页任务配BrowserGym，SWE配仅提供 Bash 命令行工具 ReAct（交替进行推理、调用工具和读取结果的执行方式）。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2509.25140#A1.SS3.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
}
]
},
"modifier": {
"value": "经验抽取与成败分类使用对应执行者的同一基础模型，如Gemini2.5 Flash；角色提示及外层追加/检索算法固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2509.25140#S3"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2509.25140#A5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2509.25140#A1.SS3.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
}
]
},
"seed": {
"value": "Gemini-2.5/Claude-3.7 的 ReAct（交替进行推理、调用工具和读取结果的执行方式） task agent，网页用 BrowserGym，SWE 用仅提供 Bash 命令行工具；外部模块从成功和失败轨迹提取可检索原则。任务执行器与记忆提取不是不同训练出来的专用网络。",
"sources": [
{
"label": "Datasets、agent 设置与记忆更新",
"url": "https://arxiv.org/abs/2509.25140"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2509.25140#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2509.25140#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2509.25140#A2.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 把每个新任务孤立处理，会重复旧错误、丢弃有用经验，系统也难随使用变强。已有记忆多存原始轨迹或成功流程，缺少可迁移的高层推理规律和失败教训，因而容易停留在被动记录；作者希望这些经验能转成指导下一次决策的原则。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2509.25140#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否从自行判断的成功与失败经历中提炼可迁移的推理经验，在缺少标准答案反馈时仍改善后续任务表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2509.25140"
}
]
},
{
"key": "conclusion",
"value": "在网页与软件工程任务中优于多种记忆对照，增加尝试次数与改进记忆可相互促进。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2509.25140"
}
]
}
],
"fields": {
"object": "推理策略记忆条目；不是权重或记忆管理代码。\n\n固定部分：模型权重与抽取/检索流程固定。"
}
},
"attributions": [
{
"tag": "org:google-cloud-ai",
"label": "Google Cloud AI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2509.25140"
}
]
},
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2509.25140"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2604.10923",
"title": "Mem²Evolve",
"url": "https://arxiv.org/abs/2604.10923",
"date": "2026-04-13",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"CoEvolution",
"M1",
"MemoryContent",
"Online",
"Skill",
"Subagent",
"Tool"
],
"fields": {
"本质定位": "**H-Mem / H-Skill / Hybrid**。不只积累经验，还让 experience memory 指导创建 tool/expert-agent asset，asset 使用后又产生新 experience，形成 experience↔capability asset loop。",
"谁来改 → 谁执行；基础 harness": "fixed co-evolution controller → agent + new tools/expert agents。",
"Feedback / evidence": "task/environment outcomes + distilled experience。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 150,
"fields": {
"时间": "2026-04-13",
"论文": "[Mem²Evolve](https://arxiv.org/abs/2604.10923)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill / Hybrid**。不只积累经验，还让 experience memory 指导创建 tool/expert-agent asset，asset 使用后又产生新 experience，形成 experience↔capability asset loop。",
"谁来改 → 谁执行；基础 harness": "fixed co-evolution controller → agent + new tools/expert agents。",
"Feedback / evidence": "task/environment outcomes + distilled experience。",
"标签": "`#MemoryContent #Skill #Tool #Subagent #CoEvolution #Online #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Mem",
"H-Skill",
"Hybrid"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "不只积累经验，还让 experience memory 指导创建 tool/expert-agent asset，asset 使用后又产生新 experience，形成 experience↔capability asset loop。",
"novelty": "让已有经验指导生成工具或专家 agent，执行这些新能力又产生新经验，把知识积累接到可执行能力的扩展上。",
"object": "工具与专家 agent 组成的能力库，以及从执行中提炼的策略经验库。",
"executor": "GPT-5-chat驱动 task agent 及本文比较的对照方案；可调用进化出的工具与专家 task agent。",
"modifier": "Mem2Evolve 用 GPT-5-chat 作为基础模型，通过固定流程总结经验、生成工具或专家 agent；新工具的代码交给 SandboxFusion 执行。工具内部也可能再调用别的模型，例如图像工具调用 GPT-4o，这不代表外层基础模型换成了 GPT-4o。",
"roleContext": "fixed co-evolution controller → agent + new tools/expert agents。",
"seed": "作者自建两类记忆：资产库存工具和专家 task agent，经验库存成功方法和失败教训。任务先规划、招募或创建资产，再执行；新工具须通过生成测试和修正流程才能入库。它既能积累文字经验，也能扩展可执行能力。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "模型评审者根据实际执行结果给出成功判断与批评，再提炼经验、保留有用工具或专家；更新阶段不提供标准答案。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：在在线执行任务中构建双记忆，没有统一预制的训练文本集。\n\n调试 / 选版本数据：当前任务轨迹、答案和 语言模型 评分者的成功/失败与批评用于后向更新。\n\n最终测试数据：GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）；HotpotQA/2Wiki 各 500；AIME 2024/2025 各 30；TravelPlanner 1,000；WebShop（根据用户要求挑选和购买商品的交互基准） 251。\n\n数据隔离与证据边界：主机制在完成题后更新再服务后续题；官方任务评分与更新用的 语言模型 评分者不是同一个信号。",
"cycle": "检索已有工具/专家，不足时参照经验创建；任务结束由 语言模型 评分者给成功标签和批评，再保留有效资产、提炼经验。更新阶段不使用真值答案。",
"train": "在在线执行任务中构建双记忆，没有统一预制的训练文本集。",
"debug": "当前任务轨迹、答案和 语言模型 评分者的成功/失败与批评用于后向更新。",
"test": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）；HotpotQA/2Wiki 各 500；AIME 2024/2025 各 30；TravelPlanner 1,000；WebShop（根据用户要求挑选和购买商品的交互基准） 251。",
"isolation": "主机制在完成题后更新再服务后续题；官方任务评分与更新用的 语言模型 评分者不是同一个信号。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建两类记忆：资产库存工具和专家 agent，经验库存成功方法和失败教训。任务先规划、招募或创建资产，再执行；新工具须通过生成测试和修正流程才能入库。它既能积累文字经验，也能扩展可执行能力。",
"protocol": "**评测任务：**GAIA、ALFWorld、HotpotQA、2Wiki、AIME 2024/2025、TravelPlanner、WebShop，共八个 benchmark；AIME 两年各 30 题。\n\n**进化来源：**从执行轨迹提炼经验并改进工具/专家库，另分析单任务和跨任务演化。主表列出 benchmark 名称不足以证明统一的训练/验证/测试隔离；各任务用于积累记忆和报告最终成绩的 split、数量本轮尚未核实。",
"sections": "§3.3、§4.1、Table 2；附录 B.2",
"source": "https://arxiv.org/abs/2604.10923",
"version": "2604.10923v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "228db72a51f19f6a3296ce5ad88114a0c19f09dd7e92f665d340e7a1bacc5b76",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5-chat驱动 task agent 及本文比较的对照方案；可调用进化出的工具与专家 task agent。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.10923#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Mem2Evolve 用 GPT-5-chat 作为基础模型，通过固定流程总结经验、生成工具或专家 agent；新工具的代码交给 SandboxFusion 执行。工具内部也可能再调用别的模型，例如图像工具调用 GPT-4o，这不代表外层基础模型换成了 GPT-4o。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.10923#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.10923#S4.SS2.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.10923#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "工具与专家 agent 组成的能力库，以及从执行中提炼的策略经验库。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "模型评审者根据实际执行结果给出成功判断与批评，再提炼经验、保留有用工具或专家；更新阶段不提供标准答案。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建两类记忆：资产库存工具和专家 task agent，经验库存成功方法和失败教训。任务先规划、招募或创建资产，再执行；新工具须通过生成测试和修正流程才能入库。它既能积累文字经验，也能扩展可执行能力。",
"sources": [
{
"label": "§3.3、§4.1、Table 2；附录 B.2",
"url": "https://arxiv.org/abs/2604.10923"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "检索已有工具/专家，不足时参照经验创建；任务结束由 语言模型 评分者给成功标签和批评，再保留有效资产、提炼经验。更新阶段不使用真值答案。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "在在线执行任务中构建双记忆，没有统一预制的训练文本集。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前任务轨迹、答案和 语言模型 评分者的成功/失败与批评用于后向更新。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）；HotpotQA/2Wiki 各 500；AIME 2024/2025 各 30；TravelPlanner 1,000；WebShop（根据用户要求挑选和购买商品的交互基准） 251。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "主机制在完成题后更新再服务后续题；官方任务评分与更新用的 语言模型 评分者不是同一个信号。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让已有经验指导生成工具或专家 agent，执行这些新能力又产生新经验，把知识积累接到可执行能力的扩展上。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.10923v1",
"version": "2604.10923v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "经验指导生成工具/专家 agent，新的能力资产又产生经验；这比只增加 memory 多了一条从知识到可执行能力的反馈路径。",
"feedbackCases": [
{
"label": "在线双记忆更新",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）、HotpotQA/2Wiki、AIME 2024/2025、TravelPlanner、WebShop（根据用户要求挑选和购买商品的交互基准） 的任务经历。",
"scoring": "模型评审读取当前题、完整执行轨迹和最终回答，输出成功/失败及批评；原文明确假设更新阶段无法访问标准答案。",
"visible": "自评结果与文字诊断供更新资产库和经验库；新工具还需生成测试、修正和入库检查。",
"use": "把成功方法与失败教训写入记忆，不能把最终问答标签算作在线更新监督。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"judgment": "LLM 读取任务、轨迹和回答自评，不访问标准答案"
},
{
"label": "最终问答 / 规划 / 操作评估",
"data": "HotpotQA/2Wiki 各 500、AIME 两年各 30、TravelPlanner 1,000、WebShop（根据用户要求挑选和购买商品的交互基准） 251，另有 GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）。",
"scoring": "问答/数学按基准参考答案评价；TravelPlanner 检查计划约束；ALFWorld（通过文字动作完成家居物体操作的交互环境） 检查操作目标，WebShop（根据用户要求挑选和购买商品的交互基准） 检查购物要求。",
"visible": "研究端按各基准报告任务表现，评分方式随领域不同。",
"use": "测试任务完成情况；这些官方分数与上一行无标准答案的模型评审不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"judgment": "问答／数学按参考答案，本文未逐项展开比对器；规划／操作按环境规则检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "在在线执行任务中构建双记忆，没有统一预制的训练文本集。",
"selection": "当前任务轨迹、答案和 语言模型 评分者的成功/失败与批评用于后向更新。",
"evaluation": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、ALFWorld（通过文字动作完成家居物体操作的交互环境）；HotpotQA/2Wiki 各 500；AIME 2024/2025 各 30；TravelPlanner 1,000；WebShop（根据用户要求挑选和购买商品的交互基准） 251。",
"isolation": "主机制在完成题后更新再服务后续题；官方任务评分与更新用的 语言模型 评分者不是同一个信号。",
"roles": {
"executor": {
"value": "GPT-5-chat驱动 task agent 及本文比较的对照方案；可调用进化出的工具与专家 task agent。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.10923#S4.SS1.SSS0.Px3"
}
]
},
"modifier": {
"value": "Mem2Evolve 用 GPT-5-chat 作为基础模型，通过固定流程总结经验、生成工具或专家 agent；新工具的代码交给 SandboxFusion 执行。工具内部也可能再调用别的模型，例如图像工具调用 GPT-4o，这不代表外层基础模型换成了 GPT-4o。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.10923#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10923#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.10923#S3.SS3.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.10923#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.10923#S4.SS2.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.10923#A4.SS3"
}
]
},
"seed": {
"value": "作者自建两类记忆：资产库存工具和专家 task agent，经验库存成功方法和失败教训。任务先规划、招募或创建资产，再执行；新工具须通过生成测试和修正流程才能入库。它既能积累文字经验，也能扩展可执行能力。",
"sources": [
{
"label": "§3.3、§4.1、Table 2；附录 B.2",
"url": "https://arxiv.org/abs/2604.10923"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"selection": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"evaluation": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
],
"isolation": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.10923#A2.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "只积累经验、却固定工具和专家库，能力无法超出预设资源；只从头创造新工具，又用不上已经验证的策略和避错教训，成功难复现且容易重犯错误。作者因此关注让经验提炼与能力资源扩展相互支持，而不是各自独立更新。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.10923#S1"
}
]
},
{
"key": "position",
"value": "研究经验积累与工具、专家能力扩展能否相互促进，使 agent 突破只增加记忆或只增加工具的单一路径。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.10923"
}
]
},
{
"key": "conclusion",
"value": "在多类任务上优于单独积累经验或单独扩充工具，突出经验与可执行能力的共同增长。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.10923"
}
]
}
],
"fields": {
"object": "工具与专家 agent 组成的能力库，以及从执行中提炼的策略经验库。",
"verdict": "模型评审者根据实际执行结果给出成功判断与批评，再提炼经验、保留有用工具或专家；更新阶段不提供标准答案。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2604.16839",
"title": "HeLa-Mem",
"url": "https://arxiv.org/abs/2604.16839",
"date": "2026-04-18",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Continual",
"M0",
"MemoryContent",
"Online"
],
"fields": {
"本质定位": "**H-Mem**。用 Hebbian co-activation 调整 episodic graph，并把 dense hubs 定期蒸馏为 semantic knowledge；重点是 association dynamics，不是 modifier self-improvement。",
"谁来改 → 谁执行；基础 harness": "fixed Hebbian/reflection memory module → fixed agent。",
"Feedback / evidence": "co-activation + interaction/reflection。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 151,
"fields": {
"时间": "2026-04-18",
"论文": "[HeLa-Mem](https://arxiv.org/abs/2604.16839)",
"级别": "**R**",
"我们的定位：什么在变、真正新点": "**H-Mem**。用 Hebbian co-activation 调整 episodic graph，并把 dense hubs 定期蒸馏为 semantic knowledge；重点是 association dynamics，不是 modifier self-improvement。",
"谁来改 → 谁执行；基础 harness": "fixed Hebbian/reflection memory module → fixed agent。",
"Feedback / evidence": "co-activation + interaction/reflection。",
"标签": "`#MemoryContent #Online #Continual #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "用 Hebbian co-activation 调整 episodic graph，并把 dense hubs 定期蒸馏为 semantic knowledge；重点是 association dynamics，不是 modifier self-improvement。",
"novelty": "经常一起被使用的记忆会加强连接，再将密集关联的具体经历归纳成更一般的知识；改变的是记忆图与内容。",
"object": "记忆图的关联权重、情节节点及压缩后的语义记忆。",
"executor": "GPT-4o-mini、GPT-4o、Qwen2.5-14B、Qwen2.5-3B分别配HeLa-Mem回答问题。",
"modifier": "反思步骤调用对应配置的 GPT-4o-mini、GPT-4o、Qwen2.5-14B 或 Qwen2.5-3B；记忆边的强化和衰减由固定 Hebbian 更新公式执行，不是模型参数训练。",
"roleContext": "fixed Hebbian/reflection memory module → fixed agent。",
"seed": "作者自建双层记忆：对话节点图＋语义记忆库；检索结合向量相似度和关联传播，没有改基础模型参数。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "记忆被共同检索和使用时增强关联，模型从高度关联的内容中总结知识；这与最后用问答成绩衡量效果是两个不同环节。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 10 段长对话写入记忆；无需参数训练。\n\n调试 / 选版本数据：关联/访问统计驱动维护；附录 D 报告 LongMemEval-S 使用的检索和阈值配置。\n\n最终测试数据：LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 1,986 个问题；另测 LongMemEval-S 500 题。\n\n数据隔离与证据边界：读入历史后回答对应记忆问题；附录给出超参数，但这些指标不能单独证明在另一组数据上选好全部阈值。",
"cycle": "共同激活加强边权；高连接节点触发 语言模型 蒸馏。低边权、长期不活跃且近期未访问三条件同时成立才删除记忆。",
"train": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 10 段长对话写入记忆；无需参数训练。",
"debug": "关联/访问统计驱动维护；附录 D 报告 LongMemEval-S 使用的检索和阈值配置。",
"test": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 1,986 个问题；另测 LongMemEval-S 500 题。",
"isolation": "读入历史后回答对应记忆问题；附录给出超参数，但这些指标不能单独证明在另一组数据上选好全部阈值。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在语言模型外维护会更新连接强度的关联记忆，用类似 Hebbian 学习的共激活关系组织和检索历史信息。更新对象是记忆关系及内容；不是给模型做梯度训练。底层调用工具与初始记忆配置的完整清单本轮未核实。",
"protocol": "**记忆材料与测试：**LoCoMo 长对话设置采用 10 段对话、1,986 个问答：单跳 841、多跳 282、时间 321、开放域 96、对抗题 446。另报告 LongMemEval-S。对话写入记忆后回答对应问题；这不是新建监督训练集。独立选择记忆超参数所用数据、LongMemEval-S 采用数量本轮待核实。",
"sections": "LoCoMo 数据说明、Tables 4–5；附录 D",
"source": "https://arxiv.org/abs/2604.16839",
"version": "2604.16839v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "7500c77567c286374edccc459ee51c6c6e0182eb9f4bd6224df09756ea4a857a",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-4o-mini、GPT-4o、Qwen2.5-14B、Qwen2.5-3B分别配HeLa-Mem回答问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.16839#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "反思步骤调用对应配置的 GPT-4o-mini、GPT-4o、Qwen2.5-14B 或 Qwen2.5-3B；记忆边的强化和衰减由固定 Hebbian 更新公式执行，不是模型参数训练。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.16839#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "记忆图的关联权重、情节节点及压缩后的语义记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "记忆被共同检索和使用时增强关联，模型从高度关联的内容中总结知识；这与最后用问答成绩衡量效果是两个不同环节。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建双层记忆：对话节点图＋语义记忆库；检索结合向量相似度和关联传播，没有改基础模型参数。",
"sources": [
{
"label": "LoCoMo 数据说明、Tables 4–5；附录 D",
"url": "https://arxiv.org/abs/2604.16839"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "共同激活加强边权；高连接节点触发 语言模型 蒸馏。低边权、长期不活跃且近期未访问三条件同时成立才删除记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 10 段长对话写入记忆；无需参数训练。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "关联/访问统计驱动维护；附录 D 报告 LongMemEval-S 使用的检索和阈值配置。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 1,986 个问题；另测 LongMemEval-S 500 题。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "读入历史后回答对应记忆问题；附录给出超参数，但这些指标不能单独证明在另一组数据上选好全部阈值。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "经常一起被使用的记忆会加强连接，再将密集关联的具体经历归纳成更一般的知识；改变的是记忆图与内容。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.16839v1",
"version": "2604.16839v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "记忆关联由共同激活增强，再把密集 episodic hubs 蒸馏成 semantic knowledge；变的是图关联和内容，Hebbian 更新机制仍预先定义。",
"feedbackCases": [
{
"label": "记忆维护",
"data": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 十段长对话。",
"scoring": "记忆共同激活、访问和关联统计驱动连接及维护，结合交互/反思内容；不是按测试题标签修改每条记忆。",
"visible": "对话与记忆访问记录。",
"use": "组织可检索记忆，无参数训练。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"judgment": "记忆访问、共同激活和关联统计驱动维护，无逐条标准答案奖励"
},
{
"label": "问答评估",
"data": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 1,986 个问题；LongMemEval-S 500 题。",
"scoring": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 报参考答案 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务） 与 BLEU-1；LongMemEval-S 按其问答评估协议评分，另给出检索及阈值设置。",
"visible": "研究端记录回答分数；没有证据表明问答答案逐条返回记忆更新器。",
"use": "比较长对话记忆的效果，不能将访问统计等同于答案正确性反馈。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"judgment": "LoCoMo 用词项重合指标；LongMemEval-S 沿用其问答协议，本文未展开裁判实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 10 段长对话写入记忆；无需参数训练。",
"selection": "关联/访问统计驱动维护；附录 D 报告 LongMemEval-S 使用的检索和阈值配置。",
"evaluation": "LoCoMo（检查模型能否利用长期多轮对话记忆的基准） 1,986 个问题；另测 LongMemEval-S 500 题。",
"isolation": "读入历史后回答对应记忆问题；附录给出超参数，但这些指标不能单独证明在另一组数据上选好全部阈值。",
"roles": {
"executor": {
"value": "GPT-4o-mini、GPT-4o、Qwen2.5-14B、Qwen2.5-3B分别配HeLa-Mem回答问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.16839#S4.SS1"
}
]
},
"modifier": {
"value": "反思步骤调用对应配置的 GPT-4o-mini、GPT-4o、Qwen2.5-14B 或 Qwen2.5-3B；记忆边的强化和衰减由固定 Hebbian 更新公式执行，不是模型参数训练。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.16839#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.16839#S3.SS1.SSS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.16839#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2604.16839#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.16839#S4.SS1"
}
]
},
"seed": {
"value": "作者自建双层记忆：对话节点图＋语义记忆库；检索结合向量相似度和关联传播，没有改基础模型参数。",
"sources": [
{
"label": "LoCoMo 数据说明、Tables 4–5；附录 D",
"url": "https://arxiv.org/abs/2604.16839"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"selection": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"evaluation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
],
"isolation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2604.16839#A3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2604.16839#A4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "有限上下文使跨会话记忆容易破碎，造成事实前后不一、个性化下降和行为不稳定。已有方法又常把存储和检索分开优化，忽略经验之间的联系会随互动演变，因此 agent 即使记住单个事实，也难维持连续的关系与任务背景。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.16839#S1"
}
]
},
{
"key": "position",
"value": "面向长对话记忆，研究经验在反复共同使用中形成的关联，能否改善知识组织、回答质量和上下文使用效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.16839"
}
]
},
{
"key": "conclusion",
"value": "在长对话记忆任务中提高回答表现、减少上下文使用；改变的是记忆关联和内容。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.16839"
}
]
}
],
"fields": {
"verdict": "记忆被共同检索和使用时增强关联，模型从高度关联的内容中总结知识；这与最后用问答成绩衡量效果是两个不同环节。",
"object": "记忆图的关联权重、情节节点及压缩后的语义记忆。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2608.16114",
"title": "HyperSkill",
"url": "https://arxiv.org/abs/2608.16114",
"date": "2026-08-17",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"M0",
"MemoryContent",
"Online",
"Prequential",
"Skill"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。把 subtask 和 skill 组织成 trajectory hypergraph，结构化 retrieval + utility-based prune/merge；新增主要是 representation，而不是新的 RSI 层级。",
"谁来改 → 谁执行；基础 harness": "fixed hypergraph updater → task agent。",
"Feedback / evidence": "observed skill utility / task outcome。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 152,
"fields": {
"时间": "2026-08-17",
"论文": "[HyperSkill](https://arxiv.org/abs/2608.16114)",
"级别": "**R**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。把 subtask 和 skill 组织成 trajectory hypergraph，结构化 retrieval + utility-based prune/merge；新增主要是 representation，而不是新的 RSI 层级。",
"谁来改 → 谁执行；基础 harness": "fixed hypergraph updater → task agent。",
"Feedback / evidence": "observed skill utility / task outcome。",
"标签": "`#MemoryContent #Skill #Online #Prequential #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 subtask 和 skill 组织成 trajectory hypergraph，结构化 retrieval + utility-based prune/merge；新增主要是 representation，而不是新的 RSI 层级。",
"novelty": "用能同时连接多个子任务和技能的图结构组织技能库，利用共享关系检索，并按使用价值去重、合并或删减技能。",
"object": "连接子任务、技能与整条轨迹的超图记忆。",
"executor": "GPT-4o或Qwen3-30B-A3B，分别加载HyperSkill超图经验；两者检索预算不同。",
"modifier": "GPT-4o 或 Qwen3-30B-A3B 配置下，系统从执行轨迹提取技能；固定维护程序根据效用与检索次数合并、裁剪超图。all-MiniLM-L6-v2 是编码器，不负责决定修改文本。",
"roleContext": "fixed hypergraph updater → task agent。",
"seed": "作者自建超图检索层：同时按子任务和轨迹取候选，再按共现与效用排序技能；每道题开始时组装记忆，本题内保持不变。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "观察技能使用后的任务结果，衡量技能是否有用。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "原记录明确标注 #Prequential；经验只应影响后续任务。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：在 xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA 的连续任务经历中积累技能及轨迹超边。\n\n调试 / 选版本数据：任务结果与执行步数更新效用；当前题的记忆上下文固定，结束后再维护。\n\n最终测试数据：xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA，报告成功率、步骤和工具调用数，并比较 GPT-4o/Qwen3 骨干。\n\n数据隔离与证据边界：这是跨任务在线记忆机制；不能把数据集名中的评估集解释成一套冻结记忆后的独立测试。",
"cycle": "任务成功次数和步数更新记忆效用；成功轨迹提炼正策略，失败轨迹提炼避错经验；周期性删除低质量节点并合并冗余节点。",
"train": "在 xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA 的连续任务经历中积累技能及轨迹超边。",
"debug": "任务结果与执行步数更新效用；当前题的记忆上下文固定，结束后再维护。",
"test": "xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA，报告成功率、步骤和工具调用数，并比较 GPT-4o/Qwen3 骨干。",
"isolation": "这是跨任务在线记忆机制；不能把数据集名中的评估集解释成一套冻结记忆后的独立测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在任务 agent 外增加超图技能记忆：节点保存技能，超边表示多个技能在轨迹中的共同使用关系；可合并相关技能并按任务组合检索。变化的是技能与组合结构，不能只概括成“有一个 memory”。具体底层工具列表本轮未核实。",
"protocol": "**评测：**xBench（规划与工具使用）、GAIA（多步现实问题）、WebWalkerQA（多轮网页导航）；比较 GPT-4o 与 Qwen3-30B-A3B 等骨干。技能由交互轨迹构建，随后按当前状态检索。各 benchmark 的技能构建 split、验证集和最终测试数量本轮未核实，不能把整套 benchmark 名称当作已确认的互斥测试集。",
"sections": "§4.1 与记忆结构",
"source": "https://arxiv.org/abs/2608.16114",
"version": "2608.16114v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "9603c369cb4b5c6824f83423258d7f586a35671e478162519ee2c881f3365711",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-4o或Qwen3-30B-A3B，分别加载HyperSkill超图经验；两者检索预算不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-4o 或 Qwen3-30B-A3B 配置下，系统从执行轨迹提取技能；固定维护程序根据效用与检索次数合并、裁剪超图。all-MiniLM-L6-v2 是编码器，不负责决定修改文本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "连接子任务、技能与整条轨迹的超图记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "观察技能使用后的任务结果，衡量技能是否有用。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建超图检索层：同时按子任务和轨迹取候选，再按共现与效用排序技能；每道题开始时组装记忆，本题内保持不变。",
"sources": [
{
"label": "§4.1 与记忆结构",
"url": "https://arxiv.org/abs/2608.16114"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "任务成功次数和步数更新记忆效用；成功轨迹提炼正策略，失败轨迹提炼避错经验；周期性删除低质量节点并合并冗余节点。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "在 xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA 的连续任务经历中积累技能及轨迹超边。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "任务结果与执行步数更新效用；当前题的记忆上下文固定，结束后再维护。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA，报告成功率、步骤和工具调用数，并比较 GPT-4o/Qwen3 骨干。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这是跨任务在线记忆机制；不能把数据集名中的评估集解释成一套冻结记忆后的独立测试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用能同时连接多个子任务和技能的图结构组织技能库，利用共享关系检索，并按使用价值去重、合并或删减技能。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.16114v1",
"version": "2608.16114v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用超图连接子任务与共享技能，检索利用结构关系而不只做向量相似度；维护也依据技能效用和结构进行去重、合并或剪枝。",
"feedbackCases": [
{
"label": "主实验：有参考答案的技能更新",
"data": "xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA 连续任务。",
"scoring": "主实验使用 GT-judge，即根据标准答案判断本次任务成功/失败，再据此分配技能效用；执行步数也参与效用更新。",
"visible": "任务结果与步骤信息，当前题结束后才更新技能。",
"use": "通过共现关系和效用维护技能超图；不能称主实验完全不需要答案监督。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"judgment": "GT-judge 对照标准答案判成败；本文未明确该 judge 是规则还是模型实现"
},
{
"label": "自评消融与最终指标",
"data": "同三套基准，另做不提供标准答案的 self-judge 设置。",
"scoring": "自评变体只让模型看自己的执行记录判断成功，和主实验的参考答案评审对比；最终报告任务成功率、步数和工具调用数。",
"visible": "是否可见参考答案是两种反馈设置的关键差别。",
"use": "检验技能机制在较弱监督下的效果，不能合并成一句“任务结果”。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"judgment": "自评变体由模型读取轨迹判断成败，不看标准答案"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0,
1
],
"testCases": [
0,
1
],
"evolution": "在 xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA 的连续任务经历中积累技能及轨迹超边。",
"selection": "任务结果与执行步数更新效用；当前题的记忆上下文固定，结束后再维护。",
"evaluation": "xBench、GAIA（需要检索、推理和使用工具的通用助理任务基准）、WebWalkerQA，报告成功率、步骤和工具调用数，并比较 GPT-4o/Qwen3 骨干。",
"isolation": "这是跨任务在线记忆机制；不能把数据集名中的评估集解释成一套冻结记忆后的独立测试。",
"roles": {
"executor": {
"value": "GPT-4o或Qwen3-30B-A3B，分别加载HyperSkill超图经验；两者检索预算不同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
]
},
"modifier": {
"value": "GPT-4o 或 Qwen3-30B-A3B 配置下，系统从执行轨迹提取技能；固定维护程序根据效用与检索次数合并、裁剪超图。all-MiniLM-L6-v2 是编码器，不负责决定修改文本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.16114#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.16114#S3.SS2"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.16114#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.16114#S3.SS5"
},
{
"label": "附录H",
"url": "https://arxiv.org/html/2608.16114#A8"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
]
},
"seed": {
"value": "作者自建超图检索层：同时按子任务和轨迹取候选，再按共现与效用排序技能；每道题开始时组装记忆，本题内保持不变。",
"sources": [
{
"label": "§4.1 与记忆结构",
"url": "https://arxiv.org/abs/2608.16114"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.16114#S4.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2608.16114#A1.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "把轨迹、技能和子任务保存为孤立条目，会丢掉它们如何组合成功的关系；只按文字相似度检索，也难发现表面不同任务共享的操作结构。因此作者希望记忆同时保留子任务、技能和结果的多方关联，让经验能够支持组合复用与持续更新。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.16114#S1"
}
]
},
{
"key": "position",
"value": "研究如何保留子任务与技能之间的组合关系，使 agent 能够更准确地检索、组合和维护可复用的执行经验。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.16114"
}
]
},
{
"key": "conclusion",
"value": "在三个工具与搜索基准、两种模型上优于多种记忆对照，验证关系结构对技能复用的帮助。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.16114"
}
]
}
],
"fields": {
"object": "连接子任务、技能与整条轨迹的超图记忆。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2608.24876",
"title": "Recuris",
"url": "https://arxiv.org/abs/2608.24876",
"date": "2026-08-25",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"M1",
"MemoryContent",
"RegressionGate",
"SeparateEvolver",
"Skill",
"org:princeton",
"org:stanford",
"org:nus",
"org:oxford",
"person:mengdi-wang"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。工作记忆不仅压缩 history，还把“当前 task progress→skill selection”对齐，并让 execution evidence 能定位到具体 memory component；固定 Meta-Agent 再做 localized validation-gated Skill Memory update。",
"谁来改 → 谁执行；基础 harness": "separate fixed Meta-Agent → solver with coupled Working + Experiential Memory。",
"Feedback / evidence": "execution evidence + validation gate；4 long-horizon benchmarks/10 models。"
},
"details": [],
"occurrences": [
{
"section": "B. Memory / experience evolution",
"line": 153,
"fields": {
"时间": "2026-08-25",
"论文": "[Recuris](https://arxiv.org/abs/2608.24876)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。工作记忆不仅压缩 history，还把“当前 task progress→skill selection”对齐，并让 execution evidence 能定位到具体 memory component；固定 Meta-Agent 再做 localized validation-gated Skill Memory update。",
"谁来改 → 谁执行；基础 harness": "separate fixed Meta-Agent → solver with coupled Working + Experiential Memory。",
"Feedback / evidence": "execution evidence + validation gate；4 long-horizon benchmarks/10 models。",
"标签": "`#MemoryContent #Skill #SeparateEvolver #RegressionGate #ExecutableVerifier #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"priorityBasis": "用户指定阅读重点：Mengdi Wang（王梦迪，Princeton）署名论文。",
"prioritySources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2608.24876"
}
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Recuris 把工作记忆与经验记忆接起来：当前任务进展决定调用什么经验，结构化执行证据又能定位是哪一块经验导致失败。Meta-Agent 因而只修局部 memory component；候选通过含已成功 anchor tasks 的 dev gate 后才进入下一版。",
"novelty": "围绕记忆控制层的具体问题修改该层，并用对照实验区分“多了控制层”和“层里学到有效内容”的收益；小验证集的拒绝也可能漏掉有用补丁。",
"object": "工作记忆与经验记忆中的局部技能及控制组件；meta-agent、任务框架和基础模型固定。",
"executor": "进化阶段由冻结的 doubao-seed-2-0-pro 执行任务并产生失败轨迹；τ²-Bench 中它也模拟用户。最终迁移评测另换表 1 的目标模型，复用同一份技能记忆。",
"modifier": "固定 Meta-Agent 读取失败轨迹并修改对应记忆组件。主实现使用 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具），替换实验使用 DeepSeek Harness；§3.1 和 §3.4.3 没有为这两个修改端分别给出底层模型型号，不能把工具名当成模型名。",
"roleContext": "separate fixed Meta-Agent → solver with coupled Working + Experiential Memory。",
"seed": "沿用每套评测基准的原 task agent：τ² 工具调用 task agent、SkillFlow 的 Qwen-Code（Qwen 团队的终端 coding agent 工具） CLI（通过终端命令使用的程序界面）、Terminal-Bench 2.1 的 Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）；外加中性 M0 记忆与控制层。固定 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） meta-agent（负责设计或修改 task agent）读失败并只改对应记忆组件，不修改自己。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "从任务成败和多次开发集执行中判断改动是否有效；失败轨迹负责定位记忆中的错误步骤，已成功任务用于检查新补丁是否破坏原有行为。",
"diagnosis": "结构化 trace 指向具体 memory workflow，例如换货时误用原商品 ID，定位后只修对应步骤。",
"update": "Meta-Agent 对被定位的经验组件提出局部补丁。",
"acceptance": "dev 中包含已成功 anchor tasks 检查副作用；在统计证据无法区分于零时拒绝提交。",
"experiments": [
{
"name": "τ² Retail",
"evolve": "16 evolve",
"selection": "12 dev",
"test": "86 test",
"isolation": "三路分离",
"note": ""
},
{
"name": "τ² Airline",
"evolve": "两条 lineage：10 / 11 evolve",
"selection": "15 / 10 dev",
"test": "各自未见的补集 25 / 29 tasks",
"isolation": "三路分离",
"note": "test 文件字段为空，但原文明示用 evolve/dev 的补集。"
},
{
"name": "SkillFlow",
"evolve": "family 内顺序任务",
"selection": "在 family 内选 winning template",
"test": "迁移给未参与构建的模型",
"isolation": "任务内选择 + 跨模型迁移",
"note": "没有 held-out task split。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2608.24876v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：τ² Retail 16 evolve；Airline 两条运行线分别 10/11 evolve；SkillFlow 按任务族构建技能。\n\n调试 / 选版本数据：Retail 12 开发集；Airline 15/10 开发集。Meta-Agent 从失败轨迹定位到记忆组件，再提交范围受限的补丁。\n\n最终测试数据：Retail 86；Airline 25/29；SkillFlow 166 题用于跨模型比较，另含 TB2.1 实验。\n\n数据隔离与证据边界：τ² 的 evolve/开发／测试 分开；SkillFlow 在同族选模板并使用，无任务级单独留出的，不能套用 τ² 的隔离结论。",
"cycle": "根据失败记录定位具体记忆步骤，例如换货时误用了原商品编号，再只修该部分。开发题中保留已成功任务检查副作用；如果重复运行不足以支持可靠提升，就拒绝修改。",
"train": "τ² Retail 16 evolve；Airline 两条运行线分别 10/11 evolve；SkillFlow 按任务族构建技能。",
"debug": "Retail 12 开发集；Airline 15/10 开发集。Meta-Agent 从失败轨迹定位到记忆组件，再提交范围受限的补丁。",
"test": "Retail 86；Airline 25/29；SkillFlow 166 题用于跨模型比较，另含 TB2.1 实验。",
"isolation": "τ² 的 evolve/开发／测试 分开；SkillFlow 在同族选模板并使用，无任务级单独留出的，不能套用 τ² 的隔离结论。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "沿用每套 benchmark 的原 agent：τ² 工具调用 agent、SkillFlow 的 Qwen-Code CLI、Terminal-Bench 2.1 的 Terminus-2；外加中性 M0 记忆与控制层。固定 Claude Code meta-agent 读失败并只改对应记忆组件，不修改自己。",
"protocol": "**τ² Retail：**16 evolve、12 dev、86 test。Airline 两条运行线分别 10/15/25 和 11/10/29；后者 test 是 evolve/dev 之外的补集。\n\n**SkillFlow：**没有 held-out 任务划分，在族内选择模板再用于该族，结果主要测跨模型迁移。Terminal-Bench 2.1 另有实验，具体划分本轮待核实，不能把 Retail 的三段隔离套给所有 benchmark。",
"sections": "§2.3；附录 C",
"source": "https://arxiv.org/abs/2608.24876",
"version": "2608.24876v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ce064241c6e95c5d38cf5eb0f4b749643bea5b3352e320dd33cb47e38942444d",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "进化阶段由冻结的 doubao-seed-2-0-pro 执行任务并产生失败轨迹；τ²-Bench 中它也模拟用户。最终迁移评测另换表 1 的目标模型，复用同一份技能记忆。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.24876#S3.SS4.SSS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定 Meta-Agent 读取失败轨迹并修改对应记忆组件。主实现使用 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具），替换实验使用 DeepSeek Harness；§3.1 和 §3.4.3 没有为这两个修改端分别给出底层模型型号，不能把工具名当成模型名。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.24876#S3.SS4.SSS3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "工作记忆与经验记忆中的局部技能及控制组件；meta-agent、任务框架和基础模型固定。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "从任务成败和多次开发集执行中判断改动是否有效；失败轨迹负责定位记忆中的错误步骤，已成功任务用于检查新补丁是否破坏原有行为。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "沿用每套评测基准的原 task agent：τ² 工具调用 task agent、SkillFlow 的 Qwen-Code（Qwen 团队的终端 coding agent 工具） CLI（通过终端命令使用的程序界面）、Terminal-Bench 2.1 的 Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）；外加中性 M0 记忆与控制层。固定 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） meta-agent（负责设计或修改 task agent）读失败并只改对应记忆组件，不修改自己。",
"sources": [
{
"label": "§2.3；附录 C",
"url": "https://arxiv.org/abs/2608.24876"
},
{
"label": "Qwen Code 官方说明",
"url": "https://github.com/QwenLM/qwen-code"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "根据失败记录定位具体记忆步骤，例如换货时误用了原商品编号，再只修该部分。开发题中保留已成功任务检查副作用；如果重复运行不足以支持可靠提升，就拒绝修改。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "τ² Retail 16 evolve；Airline 两条运行线分别 10/11 evolve；SkillFlow 按任务族构建技能。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Retail 12 开发集；Airline 15/10 开发集。Meta-Agent 从失败轨迹定位到记忆组件，再提交范围受限的补丁。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Retail 86；Airline 25/29；SkillFlow 166 题用于跨模型比较，另含 TB2.1 实验。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "τ² 的 evolve/开发／测试 分开；SkillFlow 在同族选模板并使用，无任务级单独留出的，不能套用 τ² 的隔离结论。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "围绕记忆控制层的具体问题修改该层，并用对照实验区分“多了控制层”和“层里学到有效内容”的收益；小验证集的拒绝也可能漏掉有用补丁。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.24876v1",
"version": "2608.24876v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重点是让诊断对象和被修改对象一致，并分离“加上记忆控制层”与“层里真正学到的内容”。小 dev 集的保守 gate 也会拒绝在更大 held-out 集上实际有效的补丁，不能把拒绝理解为补丁一定无效。",
"feedbackCases": [
{
"label": "τ² Retail / Airline",
"data": "Retail 16 道进化、12 道开发、86 道测试；Airline 两条运行线分别 10/15/25 和 11/10/29。",
"scoring": "环境验收给满奖励才算成功；仅与用户达成口头一致、没有实际改数据库仍算失败。另对参考动作清单计算读操作与必需写操作召回。",
"visible": "失败轨迹、开发重复运行证据及状态变化。",
"use": "将失败定位到记忆组件并改补丁；独立测试检验迁移。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"judgment": "τ² 环境验收最终状态；规则另计必需操作召回"
},
{
"label": "Terminal-Bench 2.1 同题重试",
"data": "每个终端任务自己的失败尝试。",
"scoring": "保密验收器判该次得零分；两次尝试之间只给任务说明、失败轨迹及一个失败位，不提供测试内容。",
"visible": "没有详细测试判分、标准答案或跨任务记忆池。",
"use": "改记忆后重做同题，与同预算、冻结初始记忆的重试对照，不能当跨题学习。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"judgment": "TB2.1 隐藏测试程序验收；重试只给失败标记"
},
{
"label": "SkillFlow",
"data": "166 题用于跨模型比较；按任务族积累技能。",
"scoring": "按任务族的环境验收与评分细则判断成功，读取完成任务的记录来建记忆。",
"visible": "可复用经验随任务族积累。",
"use": "检验技能迁移；与单题终端重试协议分开。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"judgment": "SkillFlow 按任务族配置的验收器／评分细则判分，本文未逐族列出实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "τ² Retail 16 evolve；Airline 两条运行线分别 10/11 evolve；SkillFlow 按任务族构建技能。",
"selection": "Retail 12 开发集；Airline 15/10 开发集。Meta-Agent 从失败轨迹定位到记忆组件，再提交范围受限的补丁。",
"evaluation": "Retail 86；Airline 25/29；SkillFlow 166 题用于跨模型比较，另含 TB2.1 实验。",
"isolation": "τ² 的 evolve/开发／测试 分开；SkillFlow 在同族选模板并使用，无任务级单独留出的，不能套用 τ² 的隔离结论。",
"roles": {
"executor": {
"value": "进化阶段由冻结的 doubao-seed-2-0-pro 执行任务并产生失败轨迹；τ²-Bench 中它也模拟用户。最终迁移评测另换表 1 的目标模型，复用同一份技能记忆。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.24876#S3.SS4.SSS3"
}
]
},
"modifier": {
"value": "固定 Meta-Agent 读取失败轨迹并修改对应记忆组件。主实现使用 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具），替换实验使用 DeepSeek Harness；§3.1 和 §3.4.3 没有为这两个修改端分别给出底层模型型号，不能把工具名当成模型名。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.24876#S3.SS4.SSS3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "沿用每套评测基准的原 task agent：τ² 工具调用 task agent、SkillFlow 的 Qwen-Code（Qwen 团队的终端 coding agent 工具） CLI（通过终端命令使用的程序界面）、Terminal-Bench 2.1 的 Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）；外加中性 M0 记忆与控制层。固定 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） meta-agent（负责设计或修改 task agent）读失败并只改对应记忆组件，不修改自己。",
"sources": [
{
"label": "§2.3；附录 C",
"url": "https://arxiv.org/abs/2608.24876"
},
{
"label": "Qwen Code 官方说明",
"url": "https://github.com/QwenLM/qwen-code"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.24876#A3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务中目标和问题状态不断变化，初始指令会过时，完整历史又混入已完成步骤与噪声；agent 因此会忘记未解决事项，调用不合时宜的技能。问题不只是经验不够，而是缺少能持续反映当前需求的紧凑状态，因此作者研究让经验组织、选择和修订随执行状态调整。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.24876#S1"
}
]
},
{
"key": "position",
"value": "面向长程任务，解决 agent 如何准确维护当前进度、及时调用相关经验，并把失败归因到需要修订的记忆部分。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.24876"
}
]
},
{
"key": "conclusion",
"value": "多数模型与任务组合获得提升，长程任务收益更突出；更新须经过验证门槛。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.24876"
}
]
}
],
"fields": {
"executor": "进化时由冻结的 doubao-seed-2-0-pro 执行任务、产生失败轨迹；在 τ²-Bench 中它也模拟用户。之后将同一份技能记忆迁移给其他目标模型测试。",
"modifier": "固定 meta-agent 读取失败轨迹并修订对应记忆组件。两种实现分别使用 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 和 DeepSeek Harness；原文没有明确给出它们各自的底层模型型号。",
"seed": "保留原基准的执行工具：τ²-Bench 的工具调用 agent、SkillFlow 的 Qwen-Code（Qwen 团队的终端 coding agent 工具） CLI（通过终端命令使用的程序界面）、Terminal-Bench 2.1 的 Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）。额外加入初始记忆和控制层，meta-agent 只改记忆组件。",
"object": "工作记忆与经验记忆中的局部技能及控制组件；meta-agent、任务框架和基础模型固定。",
"verdict": "从任务成败和多次开发集执行中判断改动是否有效；失败轨迹负责定位记忆中的错误步骤，已成功任务用于检查新补丁是否破坏原有行为。"
}
},
"attributions": [
{
"tag": "org:princeton",
"label": "Princeton University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.24876"
}
]
},
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.24876"
}
]
},
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.24876"
}
]
},
{
"tag": "org:oxford",
"label": "University of Oxford",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.24876"
}
]
},
{
"tag": "person:mengdi-wang",
"label": "Mengdi Wang",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2608.24876"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2606.01314",
"title": "SkillSmith",
"url": "https://arxiv.org/abs/2606.01314",
"date": "2026-05-31",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"M1",
"MemoryContent",
"RegressionGate",
"SeparateEvolver",
"Skill",
"Tool",
"org:sjtu"
],
"fields": {
"本质定位": "**H-Skill**。failure-driven **joint skill + executable tool** edit；还能 wrap/edit/compose/split/retire tools。更特别的是从 traces 估计 skill pair 的 complement/conflict interaction matrix，用“ecological utility”指导 retrieval/mutation/retirement，并用 anti-pattern veto 重复错误。",
"谁来改 → 谁执行；基础 harness": "separate reflector/evolver → current agent with skill/tool library。",
"Feedback / evidence": "failure trajectory + progressive executable tests；3 benchmarks×5 Qwen3.5 scales。"
},
"details": [],
"occurrences": [
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 159,
"fields": {
"时间": "2026-05-31",
"论文": "[SkillSmith](https://arxiv.org/abs/2606.01314)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Skill**。failure-driven **joint skill + executable tool** edit；还能 wrap/edit/compose/split/retire tools。更特别的是从 traces 估计 skill pair 的 complement/conflict interaction matrix，用“ecological utility”指导 retrieval/mutation/retirement，并用 anti-pattern veto 重复错误。",
"谁来改 → 谁执行；基础 harness": "separate reflector/evolver → current agent with skill/tool library。",
"Feedback / evidence": "failure trajectory + progressive executable tests；3 benchmarks×5 Qwen3.5 scales。",
"标签": "`#Skill #Tool #MemoryContent #SeparateEvolver #ExecutableVerifier #RegressionGate #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "SkillSmith 把一次修复看成 skill 与 tool 的联合变更：如果文字步骤要求的操作没有工具支持，就连工具实现一起改。它还统计技能共同使用时互相帮助还是冲突，用这个关系决定检索、修改、合并或淘汰，并记住以前被证实失败的配置，防止反复提出同一种错误。",
"novelty": "联合维护技能、工具及依赖关系；记录已经验证失败的修改，避免重复采用。",
"object": "技能说明、工具实现和失败模式记忆。工具可包装、修改、组合、拆分或停用；任务模型参数与评分接口固定。",
"executor": "OfficeQA 比较 Qwen3.5 的 9B、27B、35B、122B、397B 五档；SealQA 表 1 列出 9B、35B、122B、397B。执行模型配合技能库和工具库完成任务，122B 还用于长期恢复实验。",
"modifier": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"roleContext": "separate reflector/evolver → current agent with skill/tool library。",
"seed": "按任务给不同技能/工具库：OfficeQA 有文档检索、表格解析、数值计算技能及 pdf_parser/table_extractor/formula_calc/unit_converter；SealQA 有搜索计划、来源可信度判断及搜索/抓取/去重工具；WildClaw 还有消息、日历、文件、创作和 Git 工具。起点已具备专用能力，不是空白技能库。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "OfficeQA 和 WildClawBench 使用各自任务评测规则。SealQA 由独立、冻结的 Qwen3.5-122B 读取问题、标准答案和回答，判对或错；同一回答判三次，取多数票。",
"diagnosis": "失败轨迹定位涉及的 skill/tool、错误类型与症状；同时根据共同出现的结果估计 skill 间冲突。",
"update": "生成 skill–tool bundle，执行工具操作，按效用调整检索、合并和淘汰。",
"acceptance": "工具单测 → 集成验证 → regression 检查；失败指纹写入 anti-pattern memory，后续相似错误可被 veto。",
"experiments": [
{
"name": "OfficeQA",
"evolve": "24 train",
"selection": "17 validation：Pareto 选择与 regression",
"test": "205 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "SealQA seal-0",
"evolve": "11 train",
"selection": "8 validation",
"test": "92 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "WildClawBench",
"evolve": "6 个 day–night rounds，日间用户交互产经验",
"selection": "夜间反思、proposal、验证、部署",
"test": "任务每天重复评价",
"isolation": "连续运行协议",
"note": "不能把前两个 benchmark 的 held-out 结论直接移到这里。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2606.01314v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：OfficeQA 24 道训练题；SealQA 11 道训练题。WildClaw 另使用六个昼夜的在线经历。\n\n调试 / 选版本数据：OfficeQA：17 题；SealQA：8 题。均用于筛选候选、检查已有能力是否退步；工具另做单元和集成检查。\n\n最终测试数据：OfficeQA 205、SealQA 92 道未用于进化的题；WildClaw 报持续运行结果。\n\n数据隔离与证据边界：OfficeQA、SealQA 均分训练／验证／测试。WildClaw 持续在线更新；100 轮抗扰动另用受控任务池。",
"cycle": "先定位失败涉及的技能、工具和错误模式，再一起修改相关技能与工具。依次运行工具单元测试、组合使用检查和回归检查；保存历史失败模式，阻止再次引入相似错误，并据效用调整检索、合并或淘汰。",
"train": "OfficeQA 24 道训练题；SealQA 11 道训练题。WildClaw 另使用六个昼夜的在线经历。",
"debug": "OfficeQA：17 题；SealQA：8 题。均用于筛选候选、检查已有能力是否退步；工具另做单元和集成检查。",
"test": "OfficeQA 205、SealQA 92 道未用于进化的题；WildClaw 报持续运行结果。",
"isolation": "OfficeQA、SealQA 均分训练／验证／测试。WildClaw 持续在线更新；100 轮抗扰动另用受控任务池。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "按任务给不同技能/工具库：OfficeQA 有文档检索、表格解析、数值计算技能及 pdf_parser/table_extractor/formula_calc/unit_converter；SealQA 有搜索计划、来源可信度判断及搜索/抓取/去重工具；WildClaw 还有消息、日历、文件、创作和 Git 工具。起点已具备专用能力，不是空白技能库。",
"protocol": "**OfficeQA：**美国 Treasury Bulletin 约 89k 页、246 问，24 train 发现失败、17 validation 选 Pareto 候选和回归检查、205 test 全程不参与进化。\n\n**SealQA：**11 train、8 validation、92 test。\n\n**WildClaw：**六个昼夜轮次反复处理任务，不能把其在线协议描述成 OfficeQA 一样的三段隔离。工具需先通过单元/集成/回归检查，运行检查也不等于独立任务测试。",
"sections": "附录 D.2、Tables 3–4 与数据划分",
"source": "https://arxiv.org/abs/2606.01314",
"version": "2606.01314v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "1ee225af605e917b032bf6530ff43c3f0006cc7505f65f77f5ebcd1ca3ae4cd6",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "OfficeQA 比较 Qwen3.5 的 9B、27B、35B、122B、397B 五档；SealQA 表 1 列出 9B、35B、122B、397B。执行模型配合技能库和工具库完成任务，122B 还用于长期恢复实验。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2606.01314#S1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "技能说明、工具实现和失败模式记忆。工具可包装、修改、组合、拆分或停用；任务模型参数与评分接口固定。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "OfficeQA 和 WildClawBench 使用各自任务评测规则。SealQA 由独立、冻结的 Qwen3.5-122B 读取问题、标准答案和回答，判对或错；同一回答判三次，取多数票。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2606.01314#A4.SS4.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "按任务给不同技能/工具库：OfficeQA 有文档检索、表格解析、数值计算技能及 pdf_parser/table_extractor/formula_calc/unit_converter；SealQA 有搜索计划、来源可信度判断及搜索/抓取/去重工具；WildClaw 还有消息、日历、文件、创作和 Git 工具。起点已具备专用能力，不是空白技能库。",
"sources": [
{
"label": "附录 D.2、Tables 3–4 与数据划分",
"url": "https://arxiv.org/abs/2606.01314"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先定位失败涉及的技能、工具和错误模式，再一起修改相关技能与工具。依次运行工具单元测试、组合使用检查和回归检查；保存历史失败模式，阻止再次引入相似错误，并据效用调整检索、合并或淘汰。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "OfficeQA 24 道训练题；SealQA 11 道训练题。WildClaw 另使用六个昼夜的在线经历。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "OfficeQA：17 题；SealQA：8 题。均用于筛选候选、检查已有能力是否退步；工具另做单元和集成检查。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "OfficeQA 205、SealQA 92 道未用于进化的题；WildClaw 报持续运行结果。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "OfficeQA、SealQA 均分训练／验证／测试。WildClaw 持续在线更新；100 轮抗扰动另用受控任务池。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "联合维护技能、工具及依赖关系；记录已经验证失败的修改，避免重复采用。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.01314v1",
"version": "2606.01314v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "与只更新 skill.md 的方法相比，重点是 skill–tool 依赖和技能之间的交互。anti-pattern memory 阻止的是已验证失败的实现路径，而不必否决修复需求本身。",
"feedbackCases": [
{
"label": "OfficeQA",
"data": "24 道训练、17 道验证、205 道测试。",
"scoring": "官方五档容差模糊匹配：数值看相对误差，文字看归一化编辑距离，再映射为任务分数；低于阈值的样本进入失败池。 主文准确率除特别声明外使用 0% 容差的精确匹配；不能把较宽容差下的反馈分直接当成主表准确率。",
"visible": "除分数外还读取缺失文档、编译错误、约束违规等结构化诊断。",
"use": "据失败改技能或工具，以独立验证筛选/回归检查，最终在 205 题报告效果。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2606.01314#A4.SS4.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"judgment": "OfficeQA 规则比对数值误差／文字编辑距离；主表准确率采用 0% 容差"
},
{
"label": "SealQA",
"data": "11 道训练、8 道验证、92 道测试。",
"scoring": "独立冻结 Qwen3.5-122B 读取问题、标准答案和回答，二元判对错；同一回答判三次取多数票。",
"visible": "训练结果及失败诊断供修改；验证用于选候选。",
"use": "这是有标准答案的外部模型评审，不能写成执行模型自评。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2606.01314#A4.SS4.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"judgment": "Qwen3.5-122B 对照问题、标准答案和回答判对错，三次多数票"
},
{
"label": "WildClawBench",
"data": "六个昼夜的在线任务经历。",
"scoring": "每任务三到五个二元检查点，如收件人、附件、截止时间；得分为通过比例，但关键检查点失败则全题记零。",
"visible": "检查点结果及工具可靠性问题，工具另作单元/集成测试。",
"use": "持续调整技能—工具组合；不是普通问答准确率。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2606.01314#A4.SS4.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"judgment": "按每题二元检查点和关键项规则汇总；本文未逐项说明检查点由程序或模型判断"
}
],
"experiments": [
{
"label": "OfficeQA",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "24 道训练题产生失败案例，据此改技能或工具。",
"selection": "17 道独立验证题筛选候选并检查回归；工具另做单元／集成检查。",
"evaluation": "205 道未用于进化的测试题。",
"isolation": "训练、验证、测试三段隔离；反馈可用容差分，主表准确率采用 0% 容差。",
"roles": {
"executor": {
"value": "OfficeQA 主表分别使用 Qwen3.5 的 9B、27B、35B、122B、397B 五档模型。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2606.01314#S1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
]
},
"modifier": {
"value": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "OfficeQA 起点有文档检索、表格解析、数值计算技能；工具包括 pdf_parser、table_extractor、formula_calc、unit_converter。",
"sources": [
{
"label": "附录 D.2、Tables 3–4 与数据划分",
"url": "https://arxiv.org/abs/2606.01314"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"selection": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"evaluation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"isolation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
]
}
},
{
"label": "SealQA",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "11 道训练题提供有标准答案的评审反馈。",
"selection": "8 道验证题筛选与回归检查。",
"evaluation": "92 道未用于进化的测试题。",
"isolation": "三段隔离；外部模型对照标准答案评审，不是执行者自评。",
"roles": {
"executor": {
"value": "SealQA 主表分别使用 Qwen3.5 的 9B、35B、122B、397B 四档模型；判分用的冻结 Qwen3.5-122B 是评审角色，不意味着所有答题者都是 122B。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2606.01314#S1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
]
},
"modifier": {
"value": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "SealQA 起点有搜索计划和来源可信度判断技能，以及搜索、网页抓取、去重排序工具。",
"sources": [
{
"label": "附录 D.2、Tables 3–4 与数据划分",
"url": "https://arxiv.org/abs/2606.01314"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"selection": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"evaluation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"isolation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
]
}
},
{
"label": "WildClawBench：在线运行",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "六个昼夜的在线任务经历，按检查点反馈持续改技能和工具。",
"selection": "在持续运行中依据结果调整；不是独立固定的问答验证集。",
"evaluation": "报告持续运行的任务成绩；另有 100 轮任务池抗扰动实验。",
"isolation": "在线协议，不能套用 OfficeQA／SealQA 的三段数据划分。",
"roles": {
"executor": {
"value": "WildClawBench 图 4 比较 Qwen3.5-9B、35B、397B 三档模型的在线轨迹；图 5 的任务维度分析使用 397B。跨三基准的 100 轮恢复实验另用 Qwen3.5-122B，不能把该型号套到全部在线结果。",
"sources": [
{
"label": "§4 图 4–5；附录 D.5–D.6",
"url": "https://arxiv.org/html/2606.01314#S4"
},
{
"label": "图 4：三个执行模型",
"url": "https://arxiv.org/html/2606.01314v1/fig/wildclaw_evo.png"
}
]
},
"modifier": {
"value": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.01314#A1.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.01314#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.01314#S4.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "WildClawBench 起点包括消息、邮件、日历、文件、代码执行、图像／视频处理和 Git 工具及对应技能；不是空技能库。",
"sources": [
{
"label": "附录 D.2、Tables 3–4 与数据划分",
"url": "https://arxiv.org/abs/2606.01314"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"selection": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"evaluation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
],
"isolation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.01314#A4.SS1.SSS0.Px1"
},
{
"label": "附录D.6",
"url": "https://arxiv.org/html/2606.01314#A4.SS6"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "技能使用失败可能来自工具过时、技能冲突或未记录的旧错误，而非技能文字本身。只改说明会留下真正故障，单独看每个技能的得分又忽略互补与干扰，因此作者希望把工具、技能关系和失败诊断一起纳入进化。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.01314#S1"
}
]
},
{
"key": "position",
"value": "研究技能说明与可执行工具如何共同适应任务，尤其关注多技能相互依赖时，单独改一项无法修复的执行问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.01314"
}
]
},
{
"key": "conclusion",
"value": "在 OfficeQA 文档问答、SealQA 搜索问答和 WildClawBench 交互任务中，技能与工具联合修改优于只改技能的对照，复杂工具链和多技能配合时优势更明显。消融中，固定工具不让修改造成的下降最大；移除失败记忆则更易重复引入已知问题，说明收益不只是来自增加技能数量。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.01314"
}
]
}
],
"fields": {
"executor": "Qwen3.5：OfficeQA 用 9B／27B／35B／122B／397B；SealQA 不含 27B；WildClaw 六天实验用 9B／35B／397B，恢复实验用 122B。",
"object": "技能说明、工具实现和失败模式记忆。工具可包装、修改、组合、拆分或停用；任务模型参数与评分接口固定。",
"modifier": "反思模块提出技能和工具的联合修改；Tool-Smith（负责实现工具修改的角色）只能包装、编辑、组合、拆分或停用工具。实验使用 Qwen3.5 系列；角色说明没有为 Tool-Smith 另列一款独立模型。",
"verdict": "OfficeQA：规则比对数值／文字答案；SealQA：Qwen3.5-122B 对照标准答案判对错，三次多数票；WildClaw：按任务检查点汇总，未逐项披露检查器实现。",
"seed": "OfficeQA：文档检索、表格解析、数值计算技能和工具；SealQA：搜索、抓取、来源筛选；WildClaw：消息、日历、文件、创作和 Git 工具。各任务都有初始技能库。"
}
},
"attributions": [
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.01314"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2606.17220",
"title": "When Rules Learn",
"url": "https://arxiv.org/abs/2606.17220",
"date": "2026-06-15",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"M1",
"OfflineSearch",
"Prompt",
"Skill"
],
"fields": {
"本质定位": "**H-Prompt / H-Skill**。法律检索中自动产生 query-rewrite rules、规划 rule-combination experiments、删除无收益 rule；本质是 narrow rule-library optimization。",
"谁来改 → 谁执行；基础 harness": "rule-search agent → legal retrieval agent。",
"Feedback / evidence": "retrieval metric。"
},
"details": [],
"occurrences": [
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 160,
"fields": {
"时间": "2026-06-15",
"论文": "[When Rules Learn](https://arxiv.org/abs/2606.17220)",
"级别": "**R**",
"我们的定位：什么在变、真正新点": "**H-Prompt / H-Skill**。法律检索中自动产生 query-rewrite rules、规划 rule-combination experiments、删除无收益 rule；本质是 narrow rule-library optimization。",
"谁来改 → 谁执行；基础 harness": "rule-search agent → legal retrieval agent。",
"Feedback / evidence": "retrieval metric。",
"标签": "`#Prompt #Skill #BenchmarkScore #OfflineSearch #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Prompt",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "法律检索中自动产生 query-rewrite rules、规划 rule-combination experiments、删除无收益 rule；本质是 narrow rule-library optimization。",
"novelty": "搜索并筛掉法律检索中的查询改写规则，改变送给检索器的查询方式；最后仍由 BM25 词项匹配算法排列文档。",
"object": "法律检索的查询改写规则及规则组合。",
"executor": "所有实验都由Qwen3-4B-Thinking改写检索查询，再交BM25（根据查询词与文档词项匹配程度排序的检索算法）检索；不是由规则进化模型直接替代检索器。",
"modifier": "规则进化分别用Qwen3-4B-Thinking、Qwen3-30B-A3B-Thinking、gpt-oss-20b、gpt-oss-120b。四者只改变规则，查询改写模型固定。",
"roleContext": "rule-search agent → legal retrieval agent。",
"seed": "从一条人写规则开始，给 task agent 维护可增删的文字规则集合；执行器按规则改写检索查询。系统比较新增和删减规则的效果，并保存历史候选，而非改模型参数或任意工具代码。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "在法律案例检索中，对照标注的相关案例计算检索指标，再用历史实验分数比较查询规则及规则组合。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：LeCaRD-v2 共 800 个查询、55,192 篇候选文档；不训练模型权重。\n\n调试 / 选版本数据：100 个 开发集 查询用于最多 500 步规则探索，并按历史分数选组合。\n\n最终测试数据：其余 700 个查询，报告法律案例检索 Recall。\n\n数据隔离与证据边界：规则探索与最终查询分开；重复运行/改写次数不当作额外样本。",
"cycle": "task agent 在新增规则、试验规则组合和删无效规则之间选择；读基线 Recall、历史实验分数与最近行动，再决定下一步。",
"train": "LeCaRD-v2 共 800 个查询、55,192 篇候选文档；不训练模型权重。",
"debug": "100 个 开发集 查询用于最多 500 步规则探索，并按历史分数选组合。",
"test": "其余 700 个查询，报告法律案例检索 Recall。",
"isolation": "规则探索与最终查询分开；重复运行/改写次数不当作额外样本。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "从一条人写规则开始，给任务 agent 维护可增删的文字规则集合；执行器按规则改写检索查询。系统比较新增和删减规则的效果，并保存历史候选，而非改模型参数或任意工具代码。",
"protocol": "**进化/选版本→测试：**LeCaRD-v2 法律案例检索，在 development 集上完成最多 500 步规则探索，按历史表现选规则子集，再到 held-out test 集报告 Recall。报告含五次运行和重复查询改写，不应把次数当任务数；dev/test 的题目数量本轮尚未核实。",
"sections": "实验设置；Table 2",
"source": "https://arxiv.org/abs/2606.17220",
"version": "2606.17220v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a05178ba4925fa47bec86ec10363b4154ed5fab10345a86e6fc33aaa575c1b50",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "所有实验都由Qwen3-4B-Thinking改写检索查询，再交BM25（根据查询词与文档词项匹配程度排序的检索算法）检索；不是由规则进化模型直接替代检索器。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "规则进化分别用Qwen3-4B-Thinking、Qwen3-30B-A3B-Thinking、gpt-oss-20b、gpt-oss-120b。四者只改变规则，查询改写模型固定。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "法律检索的查询改写规则及规则组合。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "在法律案例检索中，对照标注的相关案例计算检索指标，再用历史实验分数比较查询规则及规则组合。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从一条人写规则开始，给 task agent 维护可增删的文字规则集合；执行器按规则改写检索查询。系统比较新增和删减规则的效果，并保存历史候选，而非改模型参数或任意工具代码。",
"sources": [
{
"label": "实验设置；Table 2",
"url": "https://arxiv.org/abs/2606.17220"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "task agent 在新增规则、试验规则组合和删无效规则之间选择；读基线 Recall、历史实验分数与最近行动，再决定下一步。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LeCaRD-v2 共 800 个查询、55,192 篇候选文档；不训练模型权重。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "100 个 开发集 查询用于最多 500 步规则探索，并按历史分数选组合。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "其余 700 个查询，报告法律案例检索 Recall。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "规则探索与最终查询分开；重复运行/改写次数不当作额外样本。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "搜索并筛掉法律检索中的查询改写规则，改变送给检索器的查询方式；最后仍由 BM25 词项匹配算法排列文档。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.17220v1",
"version": "2606.17220v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "在法律检索中搜索 query-rewrite rule 的组合，并移除无效规则；最终仍由 BM25 检索，改进没有写入模型权重。",
"feedbackCases": [
{
"label": "法律案例检索：LeCaRD-v2",
"data": "800 个查询、55,192 篇候选文档；100 个开发查询搜索规则，700 个测试查询。",
"scoring": "根据数据集相关案例标注计算 Recall，检验 BM25（根据查询词与文档词项匹配程度排序的检索算法） 检索是否找回应找到的案例。",
"visible": "自动评估环境返回规则组合的检索分数，保存历史结果。",
"use": "最多 500 步新增、删减和组合查询改写规则；最终在其余查询评估，无权重训练。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"judgment": "规则比较检索结果与人工相关性标注，计算召回率"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "LeCaRD-v2 共 800 个查询、55,192 篇候选文档；不训练模型权重。",
"selection": "100 个 开发集 查询用于最多 500 步规则探索，并按历史分数选组合。",
"evaluation": "其余 700 个查询，报告法律案例检索 Recall。",
"isolation": "规则探索与最终查询分开；重复运行/改写次数不当作额外样本。",
"roles": {
"executor": {
"value": "所有实验都由Qwen3-4B-Thinking改写检索查询，再交BM25（根据查询词与文档词项匹配程度排序的检索算法）检索；不是由规则进化模型直接替代检索器。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px3"
}
]
},
"modifier": {
"value": "规则进化分别用Qwen3-4B-Thinking、Qwen3-30B-A3B-Thinking、gpt-oss-20b、gpt-oss-120b。四者只改变规则，查询改写模型固定。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2606.17220#S4"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17220#A1.SS1.SSS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2606.17220#A1.SS3.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px3"
}
]
},
"seed": {
"value": "从一条人写规则开始，给 task agent 维护可增删的文字规则集合；执行器按规则改写检索查询。系统比较新增和删减规则的效果，并保存历史候选，而非改模型参数或任意工具代码。",
"sources": [
{
"label": "实验设置；Table 2",
"url": "https://arxiv.org/abs/2606.17220"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"selection": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
],
"isolation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2606.17220#S5.SS1.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "法律案例检索要求精确对齐法律事实、术语和裁判推理，通用语义检索未必胜过词项匹配。查询改写虽能缩小措辞差距，但好规则依赖法律专家，粗糙生成又可能降低检索效果，因此作者研究依据检索反馈持续改进规则，减少手工设计依赖。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.17220#S1"
}
]
},
{
"key": "position",
"value": "面向法律案例检索，研究 agent 能否自主积累有效的查询规则，在强词项匹配基线之上持续提高检索质量。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17220"
}
]
},
{
"key": "conclusion",
"value": "在 LeCaRD-v2 上优于手工规则和贪心选择，历史实验利用与规则淘汰是关键因素。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.17220"
}
]
}
],
"fields": {
"object": "法律检索的查询改写规则及规则组合。",
"verdict": "在法律案例检索中，对照标注的相关案例计算检索指标，再用历史实验分数比较查询规则及规则组合。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2608.22793",
"title": "TRACE",
"url": "https://arxiv.org/abs/2608.22793",
"date": "2026-08-24",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"BenchmarkScore",
"M1",
"SeparateEvolver",
"Sequential",
"Skill",
"org:tsinghua"
],
"fields": {
"本质定位": "**H-Skill**。按 invoked skill 聚合 trajectories，直接 contrast success vs failure 后重写 behavioral skill bank；比只看单次失败更强调 skill-conditioned evidence。",
"谁来改 → 谁执行；基础 harness": "fixed skill-bank evolver → task agent。",
"Feedback / evidence": "round-level success/failure contrast + score。"
},
"details": [],
"occurrences": [
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 162,
"fields": {
"时间": "2026-08-24",
"论文": "[TRACE](https://arxiv.org/abs/2608.22793)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Skill**。按 invoked skill 聚合 trajectories，直接 contrast success vs failure 后重写 behavioral skill bank；比只看单次失败更强调 skill-conditioned evidence。",
"谁来改 → 谁执行；基础 harness": "fixed skill-bank evolver → task agent。",
"Feedback / evidence": "round-level success/failure contrast + score。",
"标签": "`#Skill #SeparateEvolver #BenchmarkScore #Sequential #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "按 invoked skill 聚合 trajectories，直接 contrast success vs failure 后重写 behavioral skill bank；比只看单次失败更强调 skill-conditioned evidence。",
"novelty": "按真正调用过的技能汇总成功与失败轨迹，再对照修订该技能；评价更强调重复运行能否稳定成功。",
"object": "车载任务的行为技能文本与技能划分。",
"executor": "GPT-5.5（medium）与GLM-5.2（high）分别加载相同的TRACE Skill Bank执行。",
"modifier": "Skill Bank由GPT-5.5收集的轨迹驱动进化；测试时原样交给GPT-5.5和GLM-5.2，不针对GLM重新进化。",
"roleContext": "fixed skill-bank evolver → task agent。",
"seed": "以模型默认提示与原生工具调用为对照；TRACE 额外加入技能库和按执行状态选择技能的模块。技能库用 GPT-5.5 轨迹演化，测试时不改地迁移到不同骨干，不是让测试骨干重新训练技能。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "每轮的任务得分，以及成功和失败执行之间的对比。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：CAR-bench 公开 训练集 先建技能库，再用公开 测试集 扩充覆盖；环境有 58 个工具、19 条领域政策。\n\n调试 / 选版本数据：公开两部分的执行轨迹用于对比和改写，技能由 GPT-5.5 轨迹演化后跨模型复用。\n\n最终测试数据：公开主表为 训练集＋测试集 合集；另有官方 30 道隐藏题，每题 3 次运行。\n\n数据隔离与证据边界：公开 测试集 已参与进化；只有另行保留的官方 隐藏 30 题属于未见任务评估。",
"cycle": "经验整理角色 对同一技能的成功/失败轨迹作对照，修改或拆分技能；未调用技能的重复失败用于补新技能，入库前去除任务 ID、记忆答案和环境特定值。",
"train": "CAR-bench 公开 训练集 先建技能库，再用公开 测试集 扩充覆盖；环境有 58 个工具、19 条领域政策。",
"debug": "公开两部分的执行轨迹用于对比和改写，技能由 GPT-5.5 轨迹演化后跨模型复用。",
"test": "公开主表为 训练集＋测试集 合集；另有官方 30 道隐藏题，每题 3 次运行。",
"isolation": "公开 测试集 已参与进化；只有另行保留的官方 隐藏 30 题属于未见任务评估。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "以模型默认提示与原生工具调用为对照；TRACE 额外加入技能库和按执行状态选择技能的模块。技能库用 GPT-5.5 轨迹演化，测试时不改地迁移到不同骨干，不是让测试骨干重新训练技能。",
"protocol": "**进化：**CAR-bench 先用公开 training split 建技能库，再用公开 test split 扩充。因此论文的公开 train+test 成绩不是未见任务泛化。\n\n**真正留出评测：**官方另有此前未见的 30 道隐藏题，每题执行 3 次，共 90 次尝试；该隐藏集不同于已用于进化的公开 test split。需要把这两种“test”分别标出。",
"sections": "实验设置；§3.3 Official Hidden-Set Evaluation",
"source": "https://arxiv.org/abs/2608.22793",
"version": "2608.22793v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6a068686f05f4a30d2394d4f5e4a033d29086a981af593b5b097819f7a2653a2",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5.5（medium）与GLM-5.2（high）分别加载相同的TRACE Skill Bank执行。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Skill Bank由GPT-5.5收集的轨迹驱动进化；测试时原样交给GPT-5.5和GLM-5.2，不针对GLM重新进化。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "车载任务的行为技能文本与技能划分。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "每轮的任务得分，以及成功和失败执行之间的对比。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "以模型默认提示与原生工具调用为对照；TRACE 额外加入技能库和按执行状态选择技能的模块。技能库用 GPT-5.5 轨迹演化，测试时不改地迁移到不同骨干，不是让测试骨干重新训练技能。",
"sources": [
{
"label": "实验设置；§3.3 Official Hidden-Set Evaluation",
"url": "https://arxiv.org/abs/2608.22793"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "经验整理角色 对同一技能的成功/失败轨迹作对照，修改或拆分技能；未调用技能的重复失败用于补新技能，入库前去除任务 ID、记忆答案和环境特定值。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "CAR-bench 公开 训练集 先建技能库，再用公开 测试集 扩充覆盖；环境有 58 个工具、19 条领域政策。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "公开两部分的执行轨迹用于对比和改写，技能由 GPT-5.5 轨迹演化后跨模型复用。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "公开主表为 训练集＋测试集 合集；另有官方 30 道隐藏题，每题 3 次运行。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "公开 测试集 已参与进化；只有另行保留的官方 隐藏 30 题属于未见任务评估。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "按真正调用过的技能汇总成功与失败轨迹，再对照修订该技能；评价更强调重复运行能否稳定成功。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.22793v2",
"version": "2608.22793v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "按实际调用的 skill 聚合成功和失败轨迹，再对照重写该 skill；它关心重复运行是否稳定成功，而不仅是多次尝试里能成功一次。",
"feedbackCases": [
{
"label": "技能进化：CAR-bench 公开任务",
"data": "先用公开 训练集，再用公开 测试集 补充覆盖；环境有 58 个工具、19 条政策。",
"scoring": "用 CAR-bench 任务成功判定评价工具交互，再对比成功与失败执行，归纳处理歧义及约束的技能。",
"visible": "任务结果及具体交互/工具轨迹，不是单看模型口头声称完成。",
"use": "由 GPT-5.5 经历提炼技能再跨模型复用；公开主表包含训练与公开测试，不能视为全未见。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"judgment": "沿用 CAR-bench 任务成功判定；所引段落未展开内部规则／模型检查"
},
{
"label": "隐藏评测",
"data": "官方 30 道隐藏任务，每题三次执行。",
"scoring": "按任务成功统计至少一次成功与多次稳定成功等可靠性指标；二者不是同一标准。",
"visible": "隐藏任务的最终评测结果用于报告。",
"use": "检验在公开任务积累的技能能否迁移及稳定完成。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"judgment": "沿用 CAR-bench 成功判定，再统计多次运行可靠性"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "CAR-bench 公开 训练集 先建技能库，再用公开 测试集 扩充覆盖；环境有 58 个工具、19 条领域政策。",
"selection": "公开两部分的执行轨迹用于对比和改写，技能由 GPT-5.5 轨迹演化后跨模型复用。",
"evaluation": "公开主表为 训练集＋测试集 合集；另有官方 30 道隐藏题，每题 3 次运行。",
"isolation": "公开 测试集 已参与进化；只有另行保留的官方 隐藏 30 题属于未见任务评估。",
"roles": {
"executor": {
"value": "GPT-5.5（medium）与GLM-5.2（high）分别加载相同的TRACE Skill Bank执行。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
}
]
},
"modifier": {
"value": "Skill Bank由GPT-5.5收集的轨迹驱动进化；测试时原样交给GPT-5.5和GLM-5.2，不针对GLM重新进化。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.22793#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
}
]
},
"seed": {
"value": "以模型默认提示与原生工具调用为对照；TRACE 额外加入技能库和按执行状态选择技能的模块。技能库用 GPT-5.5 轨迹演化，测试时不改地迁移到不同骨干，不是让测试骨干重新训练技能。",
"sources": [
{
"label": "实验设置；§3.3 Official Hidden-Set Evaluation",
"url": "https://arxiv.org/abs/2608.22793"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.22793#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.22793#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.22793#S3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "基准上偶尔答对，不能保证多轮车载交互中稳定遵守政策；请求不完整或根本无法满足时，模型还可能声称成功。由于这些失败具有随机性，单次成绩会掩盖能力与可靠性的差距，因此作者关注从对照轨迹中学习一致行为，以及何时应澄清或承认做不到。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.22793#S1"
}
]
},
{
"key": "position",
"value": "面向车载助手的模糊请求和行为约束，研究如何把偶尔成功转化为重复执行时的稳定可靠表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.22793"
}
]
},
{
"key": "conclusion",
"value": "明显缩小偶尔成功与重复成功之间的差距，并在车载任务隐藏评测中取得较好结果。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.22793"
}
]
}
],
"fields": {
"object": "车载任务的行为技能文本与技能划分。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.22793"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.23397",
"title": "MediSkill-Evo",
"url": "https://arxiv.org/abs/2608.23397",
"date": "2026-08-24",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"M1",
"MemoryContent",
"RegressionGate",
"Skill"
],
"fields": {
"本质定位": "**H-Mem / H-Skill**。临床场景不追求开放式“越存越多”，而是用 provenance/scope/process constraints 决定 experience 能否写入四类知识库；重点是安全写入 gate。",
"谁来改 → 谁执行；基础 harness": "fixed constrained updater → clinical interaction agent。",
"Feedback / evidence": "provenance + process checks + environment/verifier。"
},
"details": [],
"occurrences": [
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 163,
"fields": {
"时间": "2026-08-24",
"论文": "[MediSkill-Evo](https://arxiv.org/abs/2608.23397)",
"级别": "**R**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill**。临床场景不追求开放式“越存越多”，而是用 provenance/scope/process constraints 决定 experience 能否写入四类知识库；重点是安全写入 gate。",
"谁来改 → 谁执行；基础 harness": "fixed constrained updater → clinical interaction agent。",
"Feedback / evidence": "provenance + process checks + environment/verifier。",
"标签": "`#MemoryContent #Skill #ExecutableVerifier #RegressionGate #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Mem",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "临床场景不追求开放式“越存越多”，而是用 provenance/scope/process constraints 决定 experience 能否写入四类知识库；重点是安全写入 gate。",
"novelty": "将临床、操作过程、符号和视觉知识分库存放，并限制各类证据能支持什么写入与使用；同时考察诊断正确性和过程安全。",
"object": "临床、流程、符号和视觉知识库中的经验及操作指导。",
"executor": "Qwen3.6-Flash与DeepSeek-V4-Flash作为Doctor回答临床问题；多模态设置只用Qwen3.6-Flash，每病例最多8次Doctor推理。",
"modifier": "回合结束后，reflector 从已完成的病例轨迹提出新增、合并、修补或淘汰记忆的建议，固定审核规则决定是否进入下一快照。实验基础模型为 Qwen3.6-Flash 或 DeepSeek-V4-Flash；§3.1 明确 Doctor 与 moderator 共用模型，但没有为 reflector 单列型号。",
"roleContext": "fixed constrained updater → clinical interaction agent。",
"seed": "自建医生、患者与检查结果交互框架：医生 agent 发出合法检查请求后才能获得结果，未请求或不可得的证据保持未知；控制程序限制技能知识怎样影响动作。隐藏诊断答案由评测端保存，医生 agent 不能提前读取。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "检查经验的来源与适用范围、临床流程约束和任务执行结果，再决定是否允许写入或使用。病例和压力测试分别评价诊断与流程表现。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：MIMIC-IV/AgentClinic 衍生 FullChain 700 例；NEJM 图像病例 200 例。压力测试另用 70 病例生成 420 种条件。\n\n调试 / 选版本数据：训练病例的交互轨迹与提供批评意见的模型检查；不可获得的检查结果保持未知，不能当阴性。\n\n最终测试数据：FullChain 300 例、NEJM 100 例；压力测试是另外 30 个病例的 180 种条件。\n\n数据隔离与证据边界：病例级分开；180 条压力测试不是 180 名独立病人。诊断/评测目标不向执行医生开放。",
"cycle": "在 Doctor–Patient–Measurement 交互中收集过程证据，提供批评意见的模型结合证据一致性、诊断、安全与检查效率反馈修订技能；最终 评估器 另看私有目标。",
"train": "MIMIC-IV/AgentClinic 衍生 FullChain 700 例；NEJM 图像病例 200 例。压力测试另用 70 病例生成 420 种条件。",
"debug": "训练病例的交互轨迹与提供批评意见的模型检查；不可获得的检查结果保持未知，不能当阴性。",
"test": "FullChain 300 例、NEJM 100 例；压力测试是另外 30 个病例的 180 种条件。",
"isolation": "病例级分开；180 条压力测试不是 180 名独立病人。诊断/评测目标不向执行医生开放。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "自建 Doctor–Patient–Measurement 交互框架：医生 agent 必须发出合法检查请求后才能获得结果，未请求或不可得的证据保持未知；技能知识通过受约束的控制器影响动作。隐藏诊断答案由评测端保存，不作为医生的已知信息。",
"protocol": "**进化→测试：**MIMIC-IV 衍生病历按 AgentClinic 协议转为问诊场景，700 个训练 encounter、300 个测试 encounter；NEJM 图像病例另构建 300 例，200 训练、100 测试。NEJM 图像是需主动请求的检查，不在开始时直接展示。\n\n**任务特点：**病例包含病史、检查、参考诊断和适用的管理/安全要求；标准诊断和评测目标不向执行医生开放。独立验证/选技能集本轮未核实，不能把两个测试集和训练病例混合报告。",
"sections": "数据构造与交互协议",
"source": "https://arxiv.org/abs/2608.23397",
"version": "2608.23397v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "eb107488a2884c343c6544af38672a2d0c880078961d19ece85f70c81f27ef0f",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3.6-Flash与DeepSeek-V4-Flash作为Doctor回答临床问题；多模态设置只用Qwen3.6-Flash，每病例最多8次Doctor推理。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.23397#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "回合结束后，reflector 从已完成的病例轨迹提出新增、合并、修补或淘汰记忆的建议，固定审核规则决定是否进入下一快照。实验基础模型为 Qwen3.6-Flash 或 DeepSeek-V4-Flash；§3.1 明确 Doctor 与 moderator 共用模型，但没有为 reflector 单列型号。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.23397#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.23397#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "临床、流程、符号和视觉知识库中的经验及操作指导。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "检查经验的来源与适用范围、临床流程约束和任务执行结果，再决定是否允许写入或使用。病例和压力测试分别评价诊断与流程表现。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "自建医生、患者与检查结果交互框架：医生 agent 发出合法检查请求后才能获得结果，未请求或不可得的证据保持未知；控制程序限制技能知识怎样影响动作。隐藏诊断答案由评测端保存，医生 agent 不能提前读取。",
"sources": [
{
"label": "数据构造与交互协议",
"url": "https://arxiv.org/abs/2608.23397"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在 Doctor–Patient–Measurement 交互中收集过程证据，提供批评意见的模型结合证据一致性、诊断、安全与检查效率反馈修订技能；最终 评估器 另看私有目标。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "MIMIC-IV/AgentClinic 衍生 FullChain 700 例；NEJM 图像病例 200 例。压力测试另用 70 病例生成 420 种条件。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练病例的交互轨迹与提供批评意见的模型检查；不可获得的检查结果保持未知，不能当阴性。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "FullChain 300 例、NEJM 100 例；压力测试是另外 30 个病例的 180 种条件。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "病例级分开；180 条压力测试不是 180 名独立病人。诊断/评测目标不向执行医生开放。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将临床、操作过程、符号和视觉知识分库存放，并限制各类证据能支持什么写入与使用；同时考察诊断正确性和过程安全。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.23397v2",
"version": "2608.23397v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把临床、过程、符号与视觉知识分成不同 bank，按来源、适用范围和证据权限决定能否写入与使用；正确诊断不能抵消不安全的过程。",
"feedbackCases": [
{
"label": "病例学习与运行时过程反馈",
"data": "MIMIC-IV/AgentClinic 衍生 FullChain 700 训练病例、NEJM 200 训练病例；压力学习为 70 病例构造的 420 条条件。",
"scoring": "符号检查器先排除引用不可得检查结果、缺必要字段或违反治疗前置条件的候选；临床过程批评模型再按诊断、证据、治疗完整性及效率评价。",
"visible": "训练记录、事后病例目标与过程诊断用于提议技能；运行时仍不能把隐藏诊断或未请求的检查当成已知事实。",
"use": "分别更新四类知识库并审核修改；过程评分影响动作选择，硬约束不能用较高软分抵消。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
}
],
"judgment": "符号规则检查证据和前置条件 + 临床模型按过程评分要求评审"
},
{
"label": "冻结病例与压力评估",
"data": "FullChain 300、NEJM 100；另 30 病例的 180 个隔离压力条件。",
"scoring": "检查名、控制器释放事件及输出格式由程序判；语义评审读取完整轨迹和评估端目标，必须引用支持的交互轮次。FullChain 的评审使用该比较组的基础模型别名、温度零。",
"visible": "测试目标只对离线评估端开放。",
"use": "报告临床过程和目标覆盖等研究指标；不能将这些事后标签误说成医生 agent 诊疗时看到的答案。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
}
],
"judgment": "程序检查事件与格式；模型对照目标和完整轨迹作语义评审"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "MIMIC-IV/AgentClinic 衍生 FullChain 700 例；NEJM 图像病例 200 例。压力测试另用 70 病例生成 420 种条件。",
"selection": "训练病例的交互轨迹与提供批评意见的模型检查；不可获得的检查结果保持未知，不能当阴性。",
"evaluation": "FullChain 300 例、NEJM 100 例；压力测试是另外 30 个病例的 180 种条件。",
"isolation": "病例级分开；180 条压力测试不是 180 名独立病人。诊断/评测目标不向执行医生开放。",
"roles": {
"executor": {
"value": "Qwen3.6-Flash与DeepSeek-V4-Flash作为Doctor回答临床问题；多模态设置只用Qwen3.6-Flash，每病例最多8次Doctor推理。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.23397#S3.SS1"
}
]
},
"modifier": {
"value": "回合结束后，reflector 从已完成的病例轨迹提出新增、合并、修补或淘汰记忆的建议，固定审核规则决定是否进入下一快照。实验基础模型为 Qwen3.6-Flash 或 DeepSeek-V4-Flash；§3.1 明确 Doctor 与 moderator 共用模型，但没有为 reflector 单列型号。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2608.23397#S2.SS6"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.23397#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.23397#S3.SS1"
}
]
},
"seed": {
"value": "自建医生、患者与检查结果交互框架：医生 agent 发出合法检查请求后才能获得结果，未请求或不可得的证据保持未知；控制程序限制技能知识怎样影响动作。隐藏诊断答案由评测端保存，医生 agent 不能提前读取。",
"sources": [
{
"label": "数据构造与交互协议",
"url": "https://arxiv.org/abs/2608.23397"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"selection": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.23397#S3.SS2.SSS0.Px3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23397#S3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "临床经验迁移不能把推测、缺失信息或工具推导结果悄悄当成已观察事实；最终诊断正确，也不意味着中间操作有证据或符合必要流程。已有工作已能存记忆、用工具，作者进一步关注不同类型经验应怎样验证、在何种范围内使用，以及有权影响哪些决策。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.23397#S1"
}
]
},
{
"key": "position",
"value": "面向临床任务，研究 agent 如何在证据来源、适用范围和权限约束下积累可复用经验，同时保持诊断与流程的可靠性。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.23397"
}
]
},
{
"key": "conclusion",
"value": "在文中病例和压力测试中改善诊断及流程指标，重点是受约束地积累经验。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.23397"
}
]
}
],
"fields": {
"modifier": "回合结束后，反思模块提出记忆新增、合并、修补或淘汰建议，审核程序决定是否采用。医生与审核角色共用 Qwen3.6-Flash 或 DeepSeek-V4-Flash；反思角色的独立型号未披露。",
"object": "临床、流程、符号和视觉知识库中的经验及操作指导。",
"verdict": "检查经验的来源与适用范围、临床流程约束和任务执行结果，再决定是否允许写入或使用。病例和压力测试分别评价诊断与流程表现。",
"seed": "自建医生、患者与检查结果交互框架：医生 agent 发出合法检查请求后才能获得结果，未请求或不可得的证据保持未知；控制程序限制技能知识怎样影响动作。隐藏诊断答案由评测端保存，医生 agent 不能提前读取。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2608.23552",
"title": "Prime Agent",
"url": "https://arxiv.org/abs/2608.23552",
"date": "2026-08-24",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Context",
"M0",
"MemoryContent",
"Online",
"SameModel",
"Skill",
"Subagent",
"org:princeton",
"org:mit"
],
"fields": {
"本质定位": "**H-Mem / H-Skill / H-Prompt**。RLM-style persistent harness 跨 trajectory 保留 histories、memory、skills、prompts、subagent specs；更像**持续 state accumulation substrate**，而非强 outer-loop harness search。",
"谁来改 → 谁执行；基础 harness": "same persistent agent/runtime → later trajectories。",
"Feedback / evidence": "environment/executable outcomes。"
},
"details": [],
"occurrences": [
{
"section": "C. Skill / tool / workflow / prompt components",
"line": 164,
"fields": {
"时间": "2026-08-24",
"论文": "[Prime Agent](https://arxiv.org/abs/2608.23552)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Mem / H-Skill / H-Prompt**。RLM-style persistent harness 跨 trajectory 保留 histories、memory、skills、prompts、subagent specs；更像**持续 state accumulation substrate**，而非强 outer-loop harness search。",
"谁来改 → 谁执行；基础 harness": "same persistent agent/runtime → later trajectories。",
"Feedback / evidence": "environment/executable outcomes。",
"标签": "`#Context #MemoryContent #Skill #Subagent #SameModel #Online #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem",
"H-Prompt",
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "RLM-style persistent harness 跨 trajectory 保留 histories、memory、skills、prompts、subagent specs；更像**持续 state accumulation substrate**，而非强 outer-loop harness search。",
"novelty": "用可持续保存变量的 Python 交互环境、可继续调用sub-agent 的分工方式，以及跨执行的状态保存支持长任务；整套系统的案例成绩不能单独归因于某一种学习机制。",
"object": "跨轨迹保留的 Python 执行状态、记忆、技能和sub-agent 配置；部分任务还迭代工程产物。",
"executor": "多日nanoGPT研究比较Kimi K3、DeepSeek V4 Pro、GLM5.3；MazeBench比较Opus5、GPT-5.6 Sol，并报告GLM5.2对照。",
"modifier": "对应实验的同一模型在Prime Agent持久运行环境中更新代码和经验；模型是Kimi K3/DeepSeek V4 Pro/GLM5.3等具体配置，不是另一个未命名优化器。",
"roleContext": "same persistent agent/runtime → later trajectories。",
"seed": "Prime Agent 是作者自建的持久运行框架：有保留变量的 REPL、可恢复会话、递归子 task agent、消息通信、预算/停止控制，以及可版本化的提示、记忆和技能。起点已经功能丰富；模型决定如何分工，框架不预设固定工作流图。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "读取工具执行、程序运行及环境结果，判断当前任务是否完成、哪里需要恢复；具体任务各有自己的验收方式，不能用一个总称代替所有裁判。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：没有统一离线进化数据集；task agent 在当前研究/控制任务内进行实验，nanoGPT 修改的是 124M GPT 的训练方案。\n\n调试 / 选版本数据：REPL 自建实验、训练脚本输出和验证 loss；例如先用合成梯度筛选优化器方案。\n\n最终测试数据：ARC-AGI-3；OOLONG/OOLONG-Pairs、OBLIQ、LongBench Pro/v2、ManyIH、LongCoT-Mini、EmulatorBench；长期案例 nanoGPT、PMPP-Hard、Factorio、MazeBench。\n\n数据隔离与证据边界：多套固定框架能力测评与长时自适应案例并列，不构成一套统一的进化集→盲测集。",
"cycle": "固定框架提供持久 REPL，task agent 编写辅助函数和小实验，借执行结果修正研究方案；nanoGPT 记录必须通过八随机种子均值验证。",
"train": "没有统一离线进化数据集；task agent 在当前研究/控制任务内进行实验，nanoGPT 修改的是 124M GPT 的训练方案。",
"debug": "REPL 自建实验、训练脚本输出和验证 loss；例如先用合成梯度筛选优化器方案。",
"test": "ARC-AGI-3；OOLONG/OOLONG-Pairs、OBLIQ、LongBench Pro/v2、ManyIH、LongCoT-Mini、EmulatorBench；长期案例 nanoGPT、PMPP-Hard、Factorio、MazeBench。",
"isolation": "多套固定框架能力测评与长时自适应案例并列，不构成一套统一的进化集→盲测集。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "Prime Agent 是作者自建的持久运行框架：有保留变量的 REPL、可恢复会话、递归子 agent、消息通信、预算/停止控制，以及可版本化的提示、记忆和技能。起点已经功能丰富；模型决定如何分工，框架不预设固定工作流图。",
"protocol": "**评测套件：**ARC-AGI-3 测交互推理；长上下文套件含 OOLONG、OOLONG-Pairs、OBLIQ-Bench、LongBench Pro/v2、ManyIH、LongCoT-Mini 和 EmulatorBench；长期运行案例含 nanoGPT、PMPP-Hard、Factorio、MazeBench。\n\n**协议边界：**这些是固定框架下的任务执行、长时研究与在线调整，不对应一个统一离线进化集→测试集。需按具体实验看预算、任务重用和停止判据；逐项样本量及框架开发数据本轮未核实。",
"sections": "§2 运行机制；§3.1–3.3",
"source": "https://arxiv.org/abs/2608.23552",
"version": "2608.23552v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e163177dfc8bcc886b49ad300d3bd3850a5554729d622a45071d60f3b52c4e10",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "多日nanoGPT研究比较Kimi K3、DeepSeek V4 Pro、GLM5.3；MazeBench比较Opus5、GPT-5.6 Sol，并报告GLM5.2对照。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.23552#S3.SS5.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应实验的同一模型在Prime Agent持久运行环境中更新代码和经验；模型是Kimi K3/DeepSeek V4 Pro/GLM5.3等具体配置，不是另一个未命名优化器。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.23552#S3.SS5.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨轨迹保留的 Python 执行状态、记忆、技能和sub-agent 配置；部分任务还迭代工程产物。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "读取工具执行、程序运行及环境结果，判断当前任务是否完成、哪里需要恢复；具体任务各有自己的验收方式，不能用一个总称代替所有裁判。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Prime Agent 是作者自建的持久运行框架：有保留变量的 REPL、可恢复会话、递归子 task agent、消息通信、预算/停止控制，以及可版本化的提示、记忆和技能。起点已经功能丰富；模型决定如何分工，框架不预设固定工作流图。",
"sources": [
{
"label": "§2 运行机制；§3.1–3.3",
"url": "https://arxiv.org/abs/2608.23552"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "固定框架提供持久 REPL，task agent 编写辅助函数和小实验，借执行结果修正研究方案；nanoGPT 记录必须通过八随机种子均值验证。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有统一离线进化数据集；task agent 在当前研究/控制任务内进行实验，nanoGPT 修改的是 124M GPT 的训练方案。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "REPL 自建实验、训练脚本输出和验证 loss；例如先用合成梯度筛选优化器方案。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ARC-AGI-3；OOLONG/OOLONG-Pairs、OBLIQ、LongBench Pro/v2、ManyIH、LongCoT-Mini、EmulatorBench；长期案例 nanoGPT、PMPP-Hard、Factorio、MazeBench。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "多套固定框架能力测评与长时自适应案例并列，不构成一套统一的进化集→盲测集。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用可持续保存变量的 Python 交互环境、可继续调用sub-agent 的分工方式，以及跨执行的状态保存支持长任务；整套系统的案例成绩不能单独归因于某一种学习机制。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.23552v1",
"version": "2608.23552v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "持久 IPython REPL、递归 subagents 和跨轨迹状态保留共同支撑长程执行；案例里完成大任务不能单独归因于 self-evolution。",
"feedbackCases": [
{
"label": "研究过程中可用反馈",
"data": "nanoGPT 训练方案、EmulatorBench 模拟器、PMPP-Hard、Factorio、MazeBench 等长程任务。",
"scoring": "nanoGPT 使用训练脚本和验证损失；EmulatorBench 提供诊断验收测试来检查模拟器行为；游戏/控制任务按环境目标评价。",
"visible": "REPL 内实验结果、测试错误、验证损失与环境变化。",
"use": "agent 在当前项目内修改代码、提示和技能；没有一套跨所有任务共用的统一离线训练奖励。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"judgment": "nanoGPT 用验证损失；EmulatorBench 用程序测试；游戏用环境目标"
},
{
"label": "能力评测",
"data": "ARC-AGI-3，以及 OOLONG/OOLONG-Pairs、OBLIQ、LongBench Pro/v2、ManyIH、LongCoT-Mini、EmulatorBench 等。",
"scoring": "交互任务用目标完成情况；长上下文任务按问题参考答案/原基准指标；模拟器用给定诊断测试。",
"visible": "各基准最终成绩与执行开销分别报告。",
"use": "这些能力结果不能全部视为 agent 修改运行框架时收到的反馈，也不能合为一种通过率。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"judgment": "按任务分别用环境状态、参考答案或诊断测试；非统一模型裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "没有统一离线进化数据集；task agent 在当前研究/控制任务内进行实验，nanoGPT 修改的是 124M GPT 的训练方案。",
"selection": "REPL 自建实验、训练脚本输出和验证 loss；例如先用合成梯度筛选优化器方案。",
"evaluation": "ARC-AGI-3；OOLONG/OOLONG-Pairs、OBLIQ、LongBench Pro/v2、ManyIH、LongCoT-Mini、EmulatorBench；长期案例 nanoGPT、PMPP-Hard、Factorio、MazeBench。",
"isolation": "多套固定框架能力测评与长时自适应案例并列，不构成一套统一的进化集→盲测集。",
"roles": {
"executor": {
"value": "多日nanoGPT研究比较Kimi K3、DeepSeek V4 Pro、GLM5.3；MazeBench比较Opus5、GPT-5.6 Sol，并报告GLM5.2对照。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.23552#S3.SS5.SSS0.Px2"
}
]
},
"modifier": {
"value": "对应实验的同一模型在Prime Agent持久运行环境中更新代码和经验；模型是Kimi K3/DeepSeek V4 Pro/GLM5.3等具体配置，不是另一个未命名优化器。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.23552#S3.SS5.SSS0.Px2"
}
]
},
"seed": {
"value": "Prime Agent 是作者自建的持久运行框架：有保留变量的 REPL、可恢复会话、递归子 task agent、消息通信、预算/停止控制，以及可版本化的提示、记忆和技能。起点已经功能丰富；模型决定如何分工，框架不预设固定工作流图。",
"sources": [
{
"label": "§2 运行机制；§3.1–3.3",
"url": "https://arxiv.org/abs/2608.23552"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"selection": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"evaluation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
],
"isolation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.23552#S3.SS3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.23552#A1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务的信息状态和计算过程超出模型参数与当前上下文，执行框架若丢失状态、限制可用动作或过早停止，模型会因框架缺陷而失败。这样测到的是外围系统的限制，而非基础模型在给定计算预算下的能力，因此作者关注如何让框架充分支持信息管理和长程执行。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.23552#S1"
}
]
},
{
"key": "position",
"value": "面向超出单次上下文容量的复杂任务，解决 agent 如何持续管理信息、计算与协作，使长程执行能够恢复、验证并控制资源。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.23552"
}
]
},
{
"key": "conclusion",
"value": "在推理、编程及模拟任务上展示长程执行能力，同时统一恢复、验证和资源统计。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.23552"
}
]
}
],
"fields": {
"object": "跨轨迹保留的 Python 执行状态、记忆、技能和sub-agent 配置；部分任务还迭代工程产物。",
"verdict": "读取工具执行、程序运行及环境结果，判断当前任务是否完成、哪里需要恢复；具体任务各有自己的验收方式，不能用一个总称代替所有裁判。"
}
},
"attributions": [
{
"tag": "org:princeton",
"label": "Princeton University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.23552"
}
]
},
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.23552"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2605.09998",
"title": "Continual Harness: Online Adaptation for Self-Improving Foundation Agents",
"url": "https://arxiv.org/abs/2605.09998",
"date": "2026-05-11",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"CoEvolution",
"JointEvolution",
"M1",
"MemoryContent",
"Online",
"ProcessReward",
"Prompt",
"Skill",
"Subagent",
"Weights",
"org:google-deepmind",
"org:princeton"
],
"fields": {
"本质定位": "不 reset environment，同一 long-horizon run 中周期性读取 trajectory window，CRUD 修改 prompt、subagents、skills、memory。",
"谁来改 → 谁执行；基础 harness": "online harness updater → same foundation agent；可再联合 policy trainer。",
"Feedback / evidence": "recent trajectories + process reward/environment outcome。",
"什么在变": "prompt p、subagents G、skills K、memory M。",
"谁来改 / 谁执行": "**改**：LLM Refiner；通常同一 frontier model family。
**执行**：Pokemon embodied agent。",
"基础 harness": "只有 frames/ASCII map/buttons 的 minimal interface。",
"Feedback": "trajectory/history + game progress/failure。",
"Evolution → Eval": "Pokemon Red/Emerald continuous runs；不是标准 train→freeze→held-out。",
"Meta-depth": "M1 same-system。",
"相对之前真正新增什么": "相对 offline prompt/harness search 的独特点是 **reset-free online full-harness adaptation**，experience 一边产生一边改变后续 runtime。"
},
"details": [],
"occurrences": [
{
"section": "D. Full harness / executable harness optimization",
"line": 172,
"fields": {
"时间": "2026-05-11",
"论文": "[Continual Harness](https://arxiv.org/abs/2605.09998)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Full / Hybrid**。不等 episode 结束，environment 持续运行时每若干 step 在线改 prompt/subagent/skill/memory；可选再用 process-reward rollout 更新 weights。",
"谁来改 → 谁执行；基础 harness": "online harness updater → same foundation agent；可再联合 policy trainer。",
"Feedback / evidence": "recent trajectories + process reward/environment outcome。",
"标签": "`#Prompt #MemoryContent #Skill #Subagent #Weights #Online #ProcessReward #CoEvolution #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 328,
"fields": {
"优先级": "**K**",
"时间": "2026-05-11",
"论文": "[Continual Harness: Online Adaptation for Self-Improving Foundation Agents](https://arxiv.org/abs/2605.09998)",
"本质定位": "不 reset environment，同一 long-horizon run 中周期性读取 trajectory window，CRUD 修改 prompt、subagents、skills、memory。",
"什么在变": "prompt p、subagents G、skills K、memory M。",
"谁来改 / 谁执行": "**改**：LLM Refiner；通常同一 frontier model family。
**执行**：Pokemon embodied agent。",
"基础 harness": "只有 frames/ASCII map/buttons 的 minimal interface。",
"Feedback": "trajectory/history + game progress/failure。",
"Evolution → Eval": "Pokemon Red/Emerald continuous runs；不是标准 train→freeze→held-out。",
"Meta-depth": "M1 same-system。",
"相对之前真正新增什么": "相对 offline prompt/harness search 的独特点是 **reset-free online full-harness adaptation**，experience 一边产生一边改变后续 runtime。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Online / continual",
"protocolBasis": "开放式或不重置的连续运行；不同于标准 freeze-held-out。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full",
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "不 reset environment，同一 long-horizon run 中周期性读取 trajectory window，CRUD 修改 prompt、subagents、skills、memory。",
"novelty": "在不重置的游戏进程中持续改框架，并进一步加入教师生成训练示范与参数更新，考察同一长期经历中的共同学习。",
"object": "冻结模型设置更新提示、技能、记忆和sub-agent；共同学习设置还更新模型参数。",
"executor": "主游戏实验用Gemini3 Pro、Flash、Flash-Lite；开源迁移/共同学习用Gemma4 E2B、E4B、26B MoE、31B dense。",
"modifier": "主实验Refiner由Gemini模型承担；共同学习阶段由frontier teacher指导Gemma4，并通过监督微调（用示范数据训练模型）/GRPO更新其权重。论文明确被测Gemma4不能同时胜任teacher与trainee。",
"roleContext": "**改**：LLM Refiner；通常同一 frontier model family。
**执行**：Pokemon embodied agent。",
"seed": "游戏只提供画面、ASCII 地图、按键等初始接口，在持续运行中演化提示、技能、记忆和子 task agent。联合训练阶段固定教师负责重标轨迹，再更新学生参数；这与纯冻结模型改运行框架设置分开。",
"fixed": "",
"verdict": "当前及历史执行记录、游戏进展和失败情况。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Online / continual",
"note": "Pokemon Red/Emerald continuous runs；不是标准 train→freeze→held-out。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Pokémon Red 等 RPG 的持续交互；开放模型先用 frontier 轨迹 监督微调（用示范数据训练模型） 和离线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 预热。\n\n调试 / 选版本数据：技能调用结果、游戏进度和过程奖励；反复修订实际使用的技能。\n\n最终测试数据：游戏里程碑来自 reset-free 的连续运行；预热另有 20 个单独留出的 transition 检查动作/格式。\n\n数据隔离与证据边界：连续训练曲线不是独立 episode 平均；20 个 transition 留出不代表整条游戏轨迹是盲测。",
"cycle": "持续游戏中创建/修改/删除技能及 subagent；联合训练每 256 步后用过程奖励筛低奖窗口、Gemini 教师重标，再 soft 监督微调（用示范数据训练模型），沿同一存档继续。",
"train": "Pokémon Red 等 RPG 的持续交互；开放模型先用 frontier 轨迹 监督微调（用示范数据训练模型） 和离线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 预热。",
"debug": "技能调用结果、游戏进度和过程奖励；反复修订实际使用的技能。",
"test": "游戏里程碑来自 reset-free 的连续运行；预热另有 20 个单独留出的 transition 检查动作/格式。",
"isolation": "连续训练曲线不是独立 episode 平均；20 个 transition 留出不代表整条游戏轨迹是盲测。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "游戏只提供画面、ASCII 地图、按键等初始接口，在持续运行中演化提示、技能、记忆和子 agent。联合训练阶段固定教师负责重标轨迹，再更新学生参数；这与纯冻结模型改 harness 设置分开。",
"protocol": "**在线数据：**Pokémon Red 等持续 RPG 交互，下一轮接着同一模拟器状态，不重置成独立 episode。联合阶段每轮 256 steps rollout，用过程奖励和 Gemini 教师重标低奖窗口再做 soft SFT。\n\n**评估区别：**阶段预热另用 20 个 held-out transitions 检查格式/动作质量；游戏长期里程碑来自持续运行。20 个 transition 的留出不能证明整条在线训练轨迹是独立任务盲测。",
"sections": "§3.3、§4；附录 D、Table 5",
"source": "https://arxiv.org/abs/2605.09998",
"version": "2605.09998v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f3fdc454fa672ff1edb5072a4d1dff80e9db5538395b346dbee39595840eb739",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主游戏实验用Gemini3 Pro、Flash、Flash-Lite；开源迁移/共同学习用Gemma4 E2B、E4B、26B MoE、31B dense。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.09998#S4.SS1.SSS0.Px3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2605.09998#S6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "主实验Refiner由Gemini模型承担；共同学习阶段由frontier teacher指导Gemma4，并通过监督微调（用示范数据训练模型）/GRPO更新其权重。论文明确被测Gemma4不能同时胜任teacher与trainee。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.09998#S4.SS1.SSS0.Px3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2605.09998#S6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "冻结模型设置更新提示、技能、记忆和sub-agent；共同学习设置还更新模型参数。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "当前及历史执行记录、游戏进展和失败情况。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "游戏只提供画面、ASCII 地图、按键等初始接口，在持续运行中演化提示、技能、记忆和子 task agent。联合训练阶段固定教师负责重标轨迹，再更新学生参数；这与纯冻结模型改运行框架设置分开。",
"sources": [
{
"label": "§3.3、§4；附录 D、Table 5",
"url": "https://arxiv.org/abs/2605.09998"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "持续游戏中创建/修改/删除技能及 subagent；联合训练每 256 步后用过程奖励筛低奖窗口、Gemini 教师重标，再 soft 监督微调（用示范数据训练模型），沿同一存档继续。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Pokémon Red 等 RPG 的持续交互；开放模型先用 frontier 轨迹 监督微调（用示范数据训练模型） 和离线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 预热。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "技能调用结果、游戏进度和过程奖励；反复修订实际使用的技能。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "游戏里程碑来自 reset-free 的连续运行；预热另有 20 个单独留出的 transition 检查动作/格式。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "连续训练曲线不是独立 episode 平均；20 个 transition 留出不代表整条游戏轨迹是盲测。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在不重置的游戏进程中持续改框架，并进一步加入教师生成训练示范与参数更新，考察同一长期经历中的共同学习。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.09998v1",
"version": "2605.09998v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "同一条不重置的游戏进程中持续改 harness，并扩展到参数共同学习；联合阶段包含 teacher relabel 与 SFT，不能概括为纯 frozen-model harness evolution。",
"feedbackCases": [
{
"label": "在线模型—框架共同学习",
"data": "Pokémon Red 等 RPG 的持续交互；每个在线迭代运行 256 步。",
"scoring": "过程奖励模型在最近的状态转移窗口上逐步评分，低奖励窗口再由前沿教师重标；不是只用“打通游戏”的一个最终分数训练。",
"visible": "状态、动作、短期窗口的过程分及教师修正后的动作示范。",
"use": "对重标片段作软监督训练，同时修改实际使用的技能；下轮承接同一个模拟器状态，不重置进度。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
],
"judgment": "过程奖励模型评状态转移；低分窗口再由教师模型重标"
},
{
"label": "预热与最终进度",
"data": "前沿模型轨迹用于监督/离线强化预热；另 20 个留出状态转移检查动作及格式。",
"scoring": "预热检查动作/格式；长期结果看连续游戏中的里程碑与进度。",
"visible": "游戏观察、技能调用结果及失败在运行中持续可见。",
"use": "连续运行的里程碑不是从统一初始状态开始的一套独立最终考试。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
],
"judgment": "程序检查预热格式；环境里程碑衡量长期进度"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Pokémon Red 等 RPG 的持续交互；开放模型先用 frontier 轨迹 监督微调（用示范数据训练模型） 和离线 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 预热。",
"selection": "技能调用结果、游戏进度和过程奖励；反复修订实际使用的技能。",
"evaluation": "游戏里程碑来自 reset-free 的连续运行；预热另有 20 个单独留出的 transition 检查动作/格式。",
"isolation": "连续训练曲线不是独立 episode 平均；20 个 transition 留出不代表整条游戏轨迹是盲测。",
"roles": {
"executor": {
"value": "主游戏实验用Gemini3 Pro、Flash、Flash-Lite；开源迁移/共同学习用Gemma4 E2B、E4B、26B MoE、31B dense。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.09998#S4.SS1.SSS0.Px3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2605.09998#S6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
]
},
"modifier": {
"value": "主实验Refiner由Gemini模型承担；共同学习阶段由frontier teacher指导Gemma4，并通过监督微调（用示范数据训练模型）/GRPO更新其权重。论文明确被测Gemma4不能同时胜任teacher与trainee。",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.09998#A3.SS1.SSS2"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.09998#S4.SS1.SSS0.Px3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2605.09998#S6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
}
]
},
"seed": {
"value": "游戏只提供画面、ASCII 地图、按键等初始接口，在持续运行中演化提示、技能、记忆和子 task agent。联合训练阶段固定教师负责重标轨迹，再更新学生参数；这与纯冻结模型改运行框架设置分开。",
"sources": [
{
"label": "§3.3、§4；附录 D、Table 5",
"url": "https://arxiv.org/abs/2605.09998"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"selection": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"evaluation": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
],
"isolation": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2605.09998#S4.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2605.09998#A4"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2605.09998#A4.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "coding agent 已有成熟运行框架，具身长程任务却缺少对应基础设施；作者此前的游戏实验中，强模型仍需人工持续读轨迹、改框架才能推进。为摆脱这种人工依赖，需要让 agent 在同一段不重置的运行中持续适配，而非完成整局后重开再优化。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.09998#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 在长程环境不中断、不重置的情况下，能否持续适应新经历，并进一步把运行系统的改善转化为模型学习收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.09998"
}
]
},
{
"key": "conclusion",
"value": "在宝可梦中减少操作成本，并进一步用随运行框架改进的轨迹训练开放模型，探索不中断环境的共同学习。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.09998"
}
]
}
],
"fields": {
"object": "冻结模型设置更新提示、技能、记忆和sub-agent；共同学习设置还更新模型参数。"
}
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.09998"
}
]
},
{
"tag": "org:princeton",
"label": "Princeton University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.09998"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2605.24539",
"title": "DemoEvolve: Overcoming Sparse Feedback in Agentic Harness Evolution with Demonstrations",
"url": "https://arxiv.org/abs/2605.24539",
"date": "2026-05-23",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"HumanDemo",
"M1",
"OfflineSearch",
"SeparateEvolver",
"org:tsinghua"
],
"fields": {
"本质定位": "研究 reward-only harness evolution 在长程随机环境为什么不稳定，并用 competent human demonstrations 作为 reference experience 给 coding proposer 做诊断和编辑。",
"谁来改 → 谁执行；基础 harness": "coding proposer → frozen target agent harness；基础为现有 executable harness + demonstration reference。",
"Feedback / evidence": "human demonstrations + task reward；同 budget 比 reward-only/self-rollout。",
"什么在变": "frozen agent 的 executable harness。",
"谁来改 / 谁执行": "**改**：coding proposer 根据 self-rollout 或 demonstration evidence 提 harness edits。
**执行**：固定 base LM + 当前 harness。",
"基础 harness": "task-specific executable harness。",
"Feedback": "Liar’s Dice：self-rollout reward较可用；Balatro：稀疏随机 reward + human competent trajectories；tutorial text作为弱对照。",
"Evolution → Eval": "同环境预算下比较 reward-only / textual knowledge / demonstration-bootstrapped evolution；held-out generalization不是主证据。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 AHE/自 rollout search 的新点不是更大的 editable space，而是直接改变 **feedback source**：当 sparse/high-variance reward 无法定位 failure 时，用 demonstration 提供可归因的行为参照。"
},
"details": [],
"occurrences": [
{
"section": "D. Full harness / executable harness optimization",
"line": 173,
"fields": {
"时间": "2026-05-23",
"论文": "[DemoEvolve](https://arxiv.org/abs/2605.24539)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Full / Feedback**。核心不是扩大 editable space，而是回答“**什么时候 self-rollout feedback 不够**”：短 horizon Liar's Dice 可以靠 self-practice，长程随机 Balatro 中 sparse/high-variance reward 会误导 search，于是给 modifier **competent human trajectories** 作参照以定位 harness failure。",
"谁来改 → 谁执行；基础 harness": "coding proposer → frozen target agent harness；基础为现有 executable harness + demonstration reference。",
"Feedback / evidence": "human demonstrations + task reward；同 budget 比 reward-only/self-rollout。",
"标签": "`#HarnessCode #HumanDemo #SeparateEvolver #OfflineSearch #M1`"
}
},
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 329,
"fields": {
"优先级": "**K**",
"时间": "2026-05-23",
"论文": "[DemoEvolve: Overcoming Sparse Feedback in Agentic Harness Evolution with Demonstrations](https://arxiv.org/abs/2605.24539)",
"本质定位": "研究 reward-only harness evolution 在长程随机环境为什么不稳定，并用 competent human demonstrations 作为 reference experience 给 coding proposer 做诊断和编辑。",
"什么在变": "frozen agent 的 executable harness。",
"谁来改 / 谁执行": "**改**：coding proposer 根据 self-rollout 或 demonstration evidence 提 harness edits。
**执行**：固定 base LM + 当前 harness。",
"基础 harness": "task-specific executable harness。",
"Feedback": "Liar’s Dice：self-rollout reward较可用；Balatro：稀疏随机 reward + human competent trajectories；tutorial text作为弱对照。",
"Evolution → Eval": "同环境预算下比较 reward-only / textual knowledge / demonstration-bootstrapped evolution；held-out generalization不是主证据。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 AHE/自 rollout search 的新点不是更大的 editable space，而是直接改变 **feedback source**：当 sparse/high-variance reward 无法定位 failure 时，用 demonstration 提供可归因的行为参照。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "研究 reward-only harness evolution 在长程随机环境为什么不稳定，并用 competent human demonstrations 作为 reference experience 给 coding proposer 做诊断和编辑。",
"novelty": "在分数稀疏或波动较大的任务中加入人类示范，用具体行为参照帮助判断哪里做错、框架该怎么改。",
"object": "冻结模型外的可执行 harness。",
"executor": "主要对局配置为GPT-5.4-low自博弈、Qwen3.5-4B对GPT-5.4-low、Qwen3.5-4B自博弈；Balatro执行者固定GPT-5.4-low。",
"modifier": "编程候选方案提出者根据轨迹/示范编辑运行框架；Balatro全部进化条件用Claude Opus4.7 Max，执行者GPT-5.4-low保持冻结。",
"roleContext": "**改**：coding proposer 根据 self-rollout 或 demonstration evidence 提 harness edits。
**执行**：固定 base LM + 当前 harness。",
"seed": "固定游戏执行接口上的初始运行框架；在稀疏奖励之外加入演示轨迹，供候选方案提出者参照可成功的行为。",
"fixed": "",
"verdict": "Liar’s Dice 较容易从自己对局获得奖励；Balatro 的奖励更稀疏且随机，因此额外利用人类成功执行的完整过程定位改进方向。教程文字是另一种较弱对照。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "同环境预算下比较 reward-only / textual knowledge / demonstration-bootstrapped evolution；held-out generalization不是主证据。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：TextArena Liar’s Dice 的 Small3、OneCall-Wild1 开发回合；BalatroBench 另使用固定种子 A/B/C。\n\n调试 / 选版本数据：只看 development/search 轨迹和演示；Balatro 的 D/E 种子不可见。\n\n最终测试数据：Liar’s Dice 各 30 个单独留出的逻辑种子，交换座位形成 60 局；Balatro D/E 留出种子，每种子 3 次执行尝试（从开始做任务到得到结果的过程）。\n\n数据隔离与证据边界：先按开发数据选版本，再测试不重叠种子；固定种子减少比较噪声，不代表覆盖所有游戏情形。",
"cycle": "候选方案提出者对照演示与开发执行尝试（从开始做任务到得到结果的过程）修改运行框架，候选只按开发成绩选择；选好后冻结，并用配对种子比较。",
"train": "TextArena Liar’s Dice 的 Small3、OneCall-Wild1 开发回合；BalatroBench 另使用固定种子 A/B/C。",
"debug": "只看 development/search 轨迹和演示；Balatro 的 D/E 种子不可见。",
"test": "Liar’s Dice 各 30 个单独留出的逻辑种子，交换座位形成 60 局；Balatro D/E 留出种子，每种子 3 次执行尝试（从开始做任务到得到结果的过程）。",
"isolation": "先按开发数据选版本，再测试不重叠种子；固定种子减少比较噪声，不代表覆盖所有游戏情形。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "面向具体游戏的可执行 harness，在任务动作接口外加入可修改的求解控制。DemoEvolve 用示范补足仅有输赢反馈的不足；具体初始工具和提示清单本轮尚未核实，不称为成熟通用 agent。",
"protocol": "**进化/选版本：**TextArena Liar’s Dice 的 Small3 和 OneCall-Wild1 两种任务，仅使用 development/search rollout；平分时按预先声明的开发集规则选版本。\n\n**测试：**每种任务用 30 个未用于搜索的逻辑随机种子，双方交换座位各打一局，因此每个基础/进化版本是 60 局原始比赛。Small3 种子 120000–120029，OneCall-Wild1 为 130000–130029；按同种子配对奖励比较。",
"sections": "实验协议；附录 A",
"source": "https://arxiv.org/abs/2605.24539",
"version": "2605.24539v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "bf3ed0284be32c2607c27d351424e4fb836879faaeb4253a481f6510ada89863",
"seedStatus": "partial",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主要对局配置为GPT-5.4-low自博弈、Qwen3.5-4B对GPT-5.4-low、Qwen3.5-4B自博弈；Balatro执行者固定GPT-5.4-low。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "编程候选方案提出者根据轨迹/示范编辑运行框架；Balatro全部进化条件用Claude Opus4.7 Max，执行者GPT-5.4-low保持冻结。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "冻结模型外的可执行 harness。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "Liar’s Dice 较容易从自己对局获得奖励；Balatro 的奖励更稀疏且随机，因此额外利用人类成功执行的完整过程定位改进方向。教程文字是另一种较弱对照。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定游戏执行接口上的初始运行框架；在稀疏奖励之外加入演示轨迹，供候选方案提出者参照可成功的行为。",
"sources": [
{
"label": "实验协议；附录 A",
"url": "https://arxiv.org/abs/2605.24539"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "候选方案提出者对照演示与开发执行尝试（从开始做任务到得到结果的过程）修改运行框架，候选只按开发成绩选择；选好后冻结，并用配对种子比较。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "TextArena Liar’s Dice 的 Small3、OneCall-Wild1 开发回合；BalatroBench 另使用固定种子 A/B/C。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "只看 development/search 轨迹和演示；Balatro 的 D/E 种子不可见。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Liar’s Dice 各 30 个单独留出的逻辑种子，交换座位形成 60 局；Balatro D/E 留出种子，每种子 3 次执行尝试（从开始做任务到得到结果的过程）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "先按开发数据选版本，再测试不重叠种子；固定种子减少比较噪声，不代表覆盖所有游戏情形。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在分数稀疏或波动较大的任务中加入人类示范，用具体行为参照帮助判断哪里做错、框架该怎么改。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.24539v1",
"version": "2605.24539v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 AHE/自 rollout search 的新点不是更大的 editable space，而是直接改变 **feedback source**：当 sparse/high-variance reward 无法定位 failure 时，用 demonstration 提供可归因的行为参照。",
"feedbackCases": [
{
"label": "Liar’s Dice",
"data": "TextArena 的 Small3、OneCall-Wild1 开发对局；测试各 30 个留出逻辑种子、交换座位共 60 局。",
"scoring": "游戏环境判定对局胜负；外层读取自己生成的完整状态—动作轨迹及得分。",
"visible": "短回合失败较易从执行过程定位；候选代码、历史成绩也在档案中。",
"use": "在开发对局选择框架，再用留出种子比较。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
}
],
"judgment": "Liar’s Dice 游戏规则计算胜负"
},
{
"label": "Balatro",
"data": "BalatroBench A/B/C 种子用于搜索，D/E 留出；测试每种子三次。",
"scoring": "游戏的稀疏终局成绩评价候选；额外提供熟练人类的状态—动作演示作为“怎样做可能成功”的参照，不把演示直接算成奖励。",
"visible": "开发游戏分数、自己执行的失败记录及熟练演示；教程文字是较弱的另一个对照。",
"use": "修改者对照演示找失败原因，仍只按开发成绩选版本；冻结后才测 D/E。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
}
],
"judgment": "Balatro 游戏规则计算终局成绩；人类示范仅作修正参照"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "TextArena Liar’s Dice 的 Small3、OneCall-Wild1 开发回合；BalatroBench 另使用固定种子 A/B/C。",
"selection": "只看 development/search 轨迹和演示；Balatro 的 D/E 种子不可见。",
"evaluation": "Liar’s Dice 各 30 个单独留出的逻辑种子，交换座位形成 60 局；Balatro D/E 留出种子，每种子 3 次执行尝试（从开始做任务到得到结果的过程）。",
"isolation": "先按开发数据选版本，再测试不重叠种子；固定种子减少比较噪声，不代表覆盖所有游戏情形。",
"roles": {
"executor": {
"value": "主要对局配置为GPT-5.4-low自博弈、Qwen3.5-4B对GPT-5.4-low、Qwen3.5-4B自博弈；Balatro执行者固定GPT-5.4-low。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px5"
}
]
},
"modifier": {
"value": "编程候选方案提出者根据轨迹/示范编辑运行框架；Balatro全部进化条件用Claude Opus4.7 Max，执行者GPT-5.4-low保持冻结。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px5"
}
]
},
"seed": {
"value": "固定游戏执行接口上的初始运行框架；在稀疏奖励之外加入演示轨迹，供候选方案提出者参照可成功的行为。",
"sources": [
{
"label": "实验协议；附录 A",
"url": "https://arxiv.org/abs/2605.24539"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长程随机环境的最终奖励可能延迟数百步，早期小决定会改变后续状态；随机波动甚至能让没有实际生效的改动看起来涨分。因此，只靠自我运行和总奖励难定位应改的机制，作者研究能否借助示范提供更具体的改进依据，并区分真实收益与偶然波动。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.24539#S1"
}
]
},
{
"key": "position",
"value": "研究在奖励稀疏、执行随机的长程任务中，人类成功示范能否弥补 agent 自主试错的信息不足，帮助其可靠改进运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.24539"
}
]
},
{
"key": "conclusion",
"value": "在 Balatro 等环境中比纯奖励搜索更稳定；单纯增加教程文字不能替代具体执行示范。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.24539"
}
]
}
],
"fields": {
"object": "冻结模型外的可执行 harness。",
"verdict": "Liar’s Dice 较容易从自己对局获得奖励；Balatro 的奖励更稀疏且随机，因此额外利用人类成功执行的完整过程定位改进方向。教程文字是另一种较弱对照。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.24539"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback",
"harness"
]
},
{
"id": "2606.06324",
"title": "HarnessFix",
"url": "https://arxiv.org/abs/2606.06324",
"date": "2026-06-04",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HarnessCode",
"M1",
"RegressionGate",
"SeparateEvolver",
"Workflow"
],
"fields": {
"本质定位": "**H-Full**。把 failed trajectory 编译成 harness-aware representation，先 fault attribution 到 step/component，再做 scoped patch + regression validation；比“直接让 LLM 看 trace 改代码”更结构化。",
"谁来改 → 谁执行；基础 harness": "diagnostic/patch agent → existing harness executor。",
"Feedback / evidence": "failure trace + regression test / benchmark outcome。"
},
"details": [],
"occurrences": [
{
"section": "D. Full harness / executable harness optimization",
"line": 174,
"fields": {
"时间": "2026-06-04",
"论文": "[HarnessFix](https://arxiv.org/abs/2606.06324)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Full**。把 failed trajectory 编译成 harness-aware representation，先 fault attribution 到 step/component，再做 scoped patch + regression validation；比“直接让 LLM 看 trace 改代码”更结构化。",
"谁来改 → 谁执行；基础 harness": "diagnostic/patch agent → existing harness executor。",
"Feedback / evidence": "failure trace + regression test / benchmark outcome。",
"标签": "`#HarnessCode #Workflow #SeparateEvolver #ExecutableVerifier #RegressionGate #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "HarnessFix 先把原始失败轨迹变成能对应到 harness 实现的中间表示，再判断问题属于哪一步、哪个组件。修改者拿到的是有范围的 repair specification，而不是一句“请让 agent 更好”；补丁必须证明目标缺陷减少，并检查是否破坏原来成功的任务。",
"novelty": "先把失败定位到运行系统的具体层和实现，再限制补丁作用范围，最后检查是否破坏已有能力，把诊断、修改与验收接起来。",
"object": "已有运行框架的运行机制与组件；修改范围由失败诊断确定，任务模型和评估器固定。",
"executor": "主任务执行者是GPT-5-mini；GAIA（需要检索、推理和使用工具的通用助理任务基准）迁移再换Qwen3.5 Plus、DeepSeekV3.2、Gemini3 Pro、Claude Sonnet4.5。",
"modifier": "默认GPT-5-mini运行HarnessFix诊断及修补流程；迁移时直接复用已选运行框架，不用新执行模型重新搜索。",
"roleContext": "diagnostic/patch agent → existing harness executor。",
"seed": "每域沿用可工作底座：GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 open-deep-research，SWE 用 mini-swe-agent（主要通过命令行读写文件、运行测试的轻量 coding agent），AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 用官方 simplified ReAct（交替进行推理、调用工具和读取结果的执行方式） code task agent，TB2 用 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）。修复过程把运行失败对应到实现位置，再做受限修补和回归检查。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "代码任务运行测试，问答任务按参考答案或相应裁判评分；失败轨迹帮助定位组件，验证集检查修复是否有效以及是否引入新问题。",
"diagnosis": "HTIR 将失败步骤映射到运行层和实现位置，聚合 recurring flaw。",
"update": "依据 flaw-specific repair specification 选择 scoped repair operators 生成补丁。",
"acceptance": "validation 检查目标缺陷减少和新引入 regressions；不是只看训练集总分。",
"experiments": [
{
"name": "GAIA",
"evolve": "60 train",
"selection": "30 validation",
"test": "60 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "SWE-bench Verified",
"evolve": "100 train",
"selection": "50 validation",
"test": "100 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "AppWorld",
"evolve": "90 train",
"selection": "45 validation",
"test": "90 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "TB2",
"evolve": "34 train",
"selection": "17 validation",
"test": "34 test",
"isolation": "独立 test",
"note": ""
},
{
"name": "跨模型",
"evolve": "沿用 GPT-5 mini 选出的 GAIA harness",
"selection": "不重做进化",
"test": "同一 held-out GAIA split，换 4 个模型",
"isolation": "冻结迁移",
"note": ""
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2606.06324v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 60、SWE-bench Verified 100、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90、TB2 34 题提供训练轨迹。\n\n调试 / 选版本数据：相应 验证集 为 30/50/45/17；检查补丁范围、目标缺陷和回归。\n\n最终测试数据：相应单独留出的 测试集 为 60/100/90/34。另把修好的 GAIA（需要检索、推理和使用工具的通用助理任务基准） 运行框架给四个新模型复用。\n\n数据隔离与证据边界：三个分区互斥；跨模型仍测同一 GAIA（需要检索、推理和使用工具的通用助理任务基准） 留出集，不是额外跨评测基准泛化。",
"cycle": "HTIR 将失败步骤映射到运行层和实现位置，聚合 recurring flaw。依据 flaw-specific repair specification 选择 scoped repair operators 生成补丁。验证集 检查目标缺陷减少和新引入 regressions；不是只看训练集总分。",
"train": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 60、SWE-bench Verified 100、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90、TB2 34 题提供训练轨迹。",
"debug": "相应 验证集 为 30/50/45/17；检查补丁范围、目标缺陷和回归。",
"test": "相应单独留出的 测试集 为 60/100/90/34。另把修好的 GAIA（需要检索、推理和使用工具的通用助理任务基准） 运行框架给四个新模型复用。",
"isolation": "三个分区互斥；跨模型仍测同一 GAIA（需要检索、推理和使用工具的通用助理任务基准） 留出集，不是额外跨评测基准泛化。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "每域沿用可工作底座：GAIA 用 open-deep-research，SWE 用 mini-swe-agent，AppWorld 用官方 simplified ReAct code agent，TB2 用 Harbor Terminus-2。修复过程把运行失败对应到实现位置，再做受限修补和回归检查。",
"protocol": "**train/validation/test：**GAIA 60/30/60；SWE-bench Verified 抽 250 题分 100/50/100；AppWorld 抽 225 题分 90/45/90；TB2 从 89 题取 85 分 34/17/34。\n\n**跨模型：**用 GPT-5 mini 选出的 GAIA 修复 harness 冻结后给另外四个模型，在相同 GAIA 留出 test 上测。它证明跨模型复用，不等于额外的新 benchmark 泛化。",
"sections": "§IV-B、RQ4",
"source": "https://arxiv.org/abs/2606.06324",
"version": "2606.06324v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a21d97ae1a9da2d69467531a30db8a1e2408299ecb9e16aa3d58c3f9da759aee",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主任务执行者是GPT-5-mini；GAIA（需要检索、推理和使用工具的通用助理任务基准）迁移再换Qwen3.5 Plus、DeepSeekV3.2、Gemini3 Pro、Claude Sonnet4.5。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "默认GPT-5-mini运行HarnessFix诊断及修补流程；迁移时直接复用已选运行框架，不用新执行模型重新搜索。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "已有运行框架的运行机制与组件；修改范围由失败诊断确定，任务模型和评估器固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "代码任务运行测试，问答任务按参考答案或相应裁判评分；失败轨迹帮助定位组件，验证集检查修复是否有效以及是否引入新问题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "每域沿用可工作底座：GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 open-deep-research，SWE 用 mini-swe-agent（主要通过命令行读写文件、运行测试的轻量 coding agent），AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 用官方 simplified ReAct（交替进行推理、调用工具和读取结果的执行方式） code task agent，TB2 用 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）。修复过程把运行失败对应到实现位置，再做受限修补和回归检查。",
"sources": [
{
"label": "§IV-B",
"url": "https://arxiv.org/abs/2606.06324"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
},
{
"label": "Harbor：任务与验收接口",
"url": "https://www.harborframework.com/docs/tasks"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "HTIR 将失败步骤映射到运行层和实现位置，聚合 recurring flaw。依据 flaw-specific repair specification 选择 scoped repair operators 生成补丁。验证集 检查目标缺陷减少和新引入 regressions；不是只看训练集总分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 60、SWE-bench Verified 100、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90、TB2 34 题提供训练轨迹。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "相应 验证集 为 30/50/45/17；检查补丁范围、目标缺陷和回归。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "相应单独留出的 测试集 为 60/100/90/34。另把修好的 GAIA（需要检索、推理和使用工具的通用助理任务基准） 运行框架给四个新模型复用。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "三个分区互斥；跨模型仍测同一 GAIA（需要检索、推理和使用工具的通用助理任务基准） 留出集，不是额外跨评测基准泛化。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先把失败定位到运行系统的具体层和实现，再限制补丁作用范围，最后检查是否破坏已有能力，把诊断、修改与验收接起来。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.06324v2",
"version": "2606.06324v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "诊断单位与修改单位连起来：trace → runtime 层 → implementation → scoped operator → regression acceptance。与自由编辑整个代码库相比，它限制补丁范围，并有去掉诊断、scoped repair、regression gate 的对照。",
"feedbackCases": [
{
"label": "GAIA",
"data": "训练/验证/测试 60/30/60",
"scoring": "依据公开标注参考答案评价最终回答。",
"visible": "训练失败轨迹定位到具体实现机制，验证检查目标缺陷与回归；同时检查补丁修改范围。",
"use": "用独立验证选择修复，在独立测试报告成绩；GAIA（需要检索、推理和使用工具的通用助理任务基准） 另将修复框架交给四个新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"judgment": "GAIA 标注答案比对；本文未展开采用规则还是模型裁判"
},
{
"label": "SWE-bench Verified",
"data": "100/50/100",
"scoring": "运行仓库测试验收补丁。",
"visible": "训练失败轨迹定位到具体实现机制，验证检查目标缺陷与回归；同时检查补丁修改范围。",
"use": "用独立验证选择修复，在独立测试报告成绩；GAIA（需要检索、推理和使用工具的通用助理任务基准） 另将修复框架交给四个新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"judgment": "SWE-bench 仓库测试验收"
},
{
"label": "AppWorld",
"data": "90/45/90",
"scoring": "运行应用任务的目标状态验收。",
"visible": "训练失败轨迹定位到具体实现机制，验证检查目标缺陷与回归；同时检查补丁修改范围。",
"use": "用独立验证选择修复，在独立测试报告成绩；GAIA（需要检索、推理和使用工具的通用助理任务基准） 另将修复框架交给四个新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"judgment": "AppWorld 任务验收程序"
},
{
"label": "Terminal-Bench 2",
"data": "34/17/34",
"scoring": "运行任务容器中的产物/答案验收程序。",
"visible": "训练失败轨迹定位到具体实现机制，验证检查目标缺陷与回归；同时检查补丁修改范围。",
"use": "用独立验证选择修复，在独立测试报告成绩；GAIA（需要检索、推理和使用工具的通用助理任务基准） 另将修复框架交给四个新模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"judgment": "Terminal-Bench 容器验收程序"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 60、SWE-bench Verified 100、AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 90、TB2 34 题提供训练轨迹。",
"selection": "相应 验证集 为 30/50/45/17；检查补丁范围、目标缺陷和回归。",
"evaluation": "相应单独留出的 测试集 为 60/100/90/34。另把修好的 GAIA（需要检索、推理和使用工具的通用助理任务基准） 运行框架给四个新模型复用。",
"isolation": "三个分区互斥；跨模型仍测同一 GAIA（需要检索、推理和使用工具的通用助理任务基准） 留出集，不是额外跨评测基准泛化。",
"roles": {
"executor": {
"value": "主任务执行者是GPT-5-mini；GAIA（需要检索、推理和使用工具的通用助理任务基准）迁移再换Qwen3.5 Plus、DeepSeekV3.2、Gemini3 Pro、Claude Sonnet4.5。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
]
},
"modifier": {
"value": "默认GPT-5-mini运行HarnessFix诊断及修补流程；迁移时直接复用已选运行框架，不用新执行模型重新搜索。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
]
},
"seed": {
"value": "每域沿用可工作底座：GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 open-deep-research，SWE 用 mini-swe-agent（主要通过命令行读写文件、运行测试的轻量 coding agent），AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 用官方 simplified ReAct（交替进行推理、调用工具和读取结果的执行方式） code task agent，TB2 用 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）。修复过程把运行失败对应到实现位置，再做受限修补和回归检查。",
"sources": [
{
"label": "§IV-B",
"url": "https://arxiv.org/abs/2606.06324"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
},
{
"label": "Harbor：任务与验收接口",
"url": "https://www.harborframework.com/docs/tasks"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"selection": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"evaluation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
],
"isolation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.06324#S4.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2606.06324#S4.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "失败证据散落在推理、工具和环境响应中，即使找到出错步骤，也不容易对应到静态源码、提示或配置。行为与实现缺少明确连接，使直接从终局失败生成补丁难以针对根因，因此作者关注先建立轨迹到具体运行机制的对应关系，再限定范围修复。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.06324#S1"
}
]
},
{
"key": "position",
"value": "研究自动框架修复如何从“知道任务失败”推进到“知道哪个运行机制有问题”，以减少无针对性修改和连带退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.06324"
}
]
},
{
"key": "conclusion",
"value": "在四个基准上提升表现，强调从“知道失败”进一步走到“知道该修哪个机制”。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.06324"
}
]
}
],
"fields": {
"object": "已有运行框架的运行机制与组件；修改范围由失败诊断确定，任务模型和评估器固定。",
"verdict": "代码任务运行测试，问答任务按参考答案或相应裁判评分；失败轨迹帮助定位组件，验证集检查修复是否有效以及是否引入新问题。",
"seed": "按任务使用已有 agent：GAIA 用 open-deep-research 检索研究框架；SWE 用 mini-swe-agent 命令行 agent；AppWorld 用官方简化 ReAct agent（推理、调用工具、读取结果）；TB2 用 Harbor 中的 Terminus-2 终端 agent。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness",
"feedback"
]
},
{
"id": "2606.14249",
"title": "HarnessX",
"url": "https://arxiv.org/abs/2606.14249",
"date": "2026-06-12",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"CoEvolution",
"HarnessCode",
"M1",
"MemoryMechanism",
"Prompt",
"Tool",
"Workflow"
],
"fields": {
"本质定位": "**H-Full / Hybrid**。把 prompts/tools/memory/control-flow 做成 **typed primitives + substitution algebra**；AEGIS 用 trace-driven multi-agent evolution 改这些 primitives，同时把 trajectories 变成 model-training signal，显式闭合 harness↔model loop。",
"谁来改 → 谁执行；基础 harness": "HarnessX foundry / AEGIS evolver → current agent runtime。",
"Feedback / evidence": "execution traces + downstream outcome；5 benchmarks（ALFWorld/GAIA/WebShop/τ³/SWE-bench-V）。"
},
"details": [],
"occurrences": [
{
"section": "D. Full harness / executable harness optimization",
"line": 176,
"fields": {
"时间": "2026-06-12",
"论文": "[HarnessX](https://arxiv.org/abs/2606.14249)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Full / Hybrid**。把 prompts/tools/memory/control-flow 做成 **typed primitives + substitution algebra**；AEGIS 用 trace-driven multi-agent evolution 改这些 primitives，同时把 trajectories 变成 model-training signal，显式闭合 harness↔model loop。",
"谁来改 → 谁执行；基础 harness": "HarnessX foundry / AEGIS evolver → current agent runtime。",
"Feedback / evidence": "execution traces + downstream outcome；5 benchmarks（ALFWorld/GAIA/WebShop/τ³/SWE-bench-V）。",
"标签": "`#Prompt #Tool #MemoryMechanism #Workflow #HarnessCode #CoEvolution #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full",
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "把 prompts/tools/memory/control-flow 做成 **typed primitives + substitution algebra**；AEGIS 用 trace-driven multi-agent evolution 改这些 primitives，同时把 trajectories 变成 model-training signal，显式闭合 harness↔model loop。",
"novelty": "规定组件的输入输出类型和可组合方式，再从轨迹提议组件修改；另利用不同框架的执行轨迹训练模型，连接组件搜索与参数学习。",
"object": "运行时的可组合处理组件；联合更新设置还训练任务模型参数。",
"executor": "Claude Sonnet4.6、GPT-5.4、Qwen3.5-9B分别作为task task agent，运行候选运行框架。",
"modifier": "默认Claude Opus4.6作为 meta-agent（负责设计或修改 task agent）驱动AEGIS进化；与上述任务执行者分开。共同训练阶段的参数更新仍由GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）程序执行。",
"roleContext": "HarnessX foundry / AEGIS evolver → current agent runtime。",
"seed": "从各任务的人工基础框架开始，修改过程分为整理轨迹、规划改动、生成候选和评审四个角色。每轮提出 4 个候选并保留执行记录；每题重建环境，避免购物车、游戏状态或工作目录互相影响。主要修改提示和运行逻辑；这种环境重置不说明训练题和测试题已隔离。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "执行记录用于分析运行组件，任务成绩用于判断实际效果；ALFWorld、GAIA、WebShop、τ³ 和 SWE-bench 的环境验收或答案检查方式分别列在表格中。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 103、ALFWorld（通过文字动作完成家居物体操作的交互环境） valid-unseen 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、τ³ Retail/Airline/Telecom 全任务、SWE-Verified 55，形成演化轨迹。\n\n调试 / 选版本数据：每轮反复在上述固定任务上执行和计分，诊断失败并生成下一版。\n\n最终测试数据：论文曲线仍报告同一批任务逐轮变化。\n\n数据隔离与证据边界：valid-unseen 是原评测基准名称，但这些题参与了本方法迭代；不能据名称称为未参与进化的最终测试。",
"cycle": "AEGIS 从执行轨迹诊断失败并修改 processor；联合阶段将同批轨迹放进共享 replay buffer，用 cross-harness GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新模型，不另采专用 强化学习（根据奖励调整模型行为） 执行尝试（从开始做任务到得到结果的过程）。",
"train": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 103、ALFWorld（通过文字动作完成家居物体操作的交互环境） valid-unseen 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、τ³ Retail/Airline/Telecom 全任务、SWE-Verified 55，形成演化轨迹。",
"debug": "每轮反复在上述固定任务上执行和计分，诊断失败并生成下一版。",
"test": "论文曲线仍报告同一批任务逐轮变化。",
"isolation": "valid-unseen 是原评测基准名称，但这些题参与了本方法迭代；不能据名称称为未参与进化的最终测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "从各任务的人写基础 harness 开始，修改器分为 Digester（整理轨迹）、Planner、Evolver、Critic。每轮提出 4 个候选；任务执行保留全轨迹，每题重新创建环境以隔离购物车、游戏状态和工作目录。主要改提示和运行处理逻辑，不能把环境重置当作训练/测试隔离。",
"protocol": "**反复进化和评分的数据：**GAIA 固定 103 题（难度 39/52/12）；ALFWorld valid-unseen 全部 134 题；WebShop 固定抽 100 题；τ³-Bench 的 Retail、Airline、Telecom 各完整任务列表；SWE-bench Verified 抽 55 题。\n\n**边界：**同一批题每一轮都重新评分，曲线衡量这些固定任务上的迭代变化，不是每轮在未见测试题上的泛化。底层每题环境隔离不改变这一点。独立最终测试集本轮未核实。",
"sections": "Table 3；附录 9.5、Table 8",
"source": "https://arxiv.org/abs/2606.14249",
"version": "2606.14249v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c0834d7cf4dd91676ef2c9b36dcbf153e6d085d9f8c423620fbd2808d4f84e02",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Claude Sonnet4.6、GPT-5.4、Qwen3.5-9B分别作为task task agent，运行候选运行框架。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2606.14249#S6.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "默认Claude Opus4.6作为 meta-agent（负责设计或修改 task agent）驱动AEGIS进化；与上述任务执行者分开。共同训练阶段的参数更新仍由GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）程序执行。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2606.14249#S6.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "运行时的可组合处理组件；联合更新设置还训练任务模型参数。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "执行记录用于分析运行组件，任务成绩用于判断实际效果；ALFWorld、GAIA、WebShop、τ³ 和 SWE-bench 的环境验收或答案检查方式分别列在表格中。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从各任务的人工基础框架开始，修改过程分为整理轨迹、规划改动、生成候选和评审四个角色。每轮提出 4 个候选并保留执行记录；每题重建环境，避免购物车、游戏状态或工作目录互相影响。主要修改提示和运行逻辑；这种环境重置不说明训练题和测试题已隔离。",
"sources": [
{
"label": "Table 3；附录 9.5、Table 8",
"url": "https://arxiv.org/abs/2606.14249"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "AEGIS 从执行轨迹诊断失败并修改 processor；联合阶段将同批轨迹放进共享 replay buffer，用 cross-harness GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新模型，不另采专用 强化学习（根据奖励调整模型行为） 执行尝试（从开始做任务到得到结果的过程）。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 103、ALFWorld（通过文字动作完成家居物体操作的交互环境） valid-unseen 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、τ³ Retail/Airline/Telecom 全任务、SWE-Verified 55，形成演化轨迹。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "每轮反复在上述固定任务上执行和计分，诊断失败并生成下一版。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "论文曲线仍报告同一批任务逐轮变化。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "valid-unseen 是原评测基准名称，但这些题参与了本方法迭代；不能据名称称为未参与进化的最终测试。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "规定组件的输入输出类型和可组合方式，再从轨迹提议组件修改；另利用不同框架的执行轨迹训练模型，连接组件搜索与参数学习。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.14249v3",
"version": "2606.14249v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "typed primitives 与替换代数规定组件如何组合；AEGIS 从 trace 提出修改，另用跨 harness 轨迹进行模型训练，把组件演化与参数学习接起来。",
"feedbackCases": [
{
"label": "GAIA",
"data": "103 题",
"scoring": "程序将最终回答与 GAIA 参考答案作精确匹配，按正确比例计分；论文任务说明明确为 exact match。",
"visible": "各任务分数与全执行记录持续写入存储，修改者据此诊断处理逻辑并提出候选。",
"use": "每轮反复使用相同适应任务；曲线仍是这批题的表现。每题重置环境只隔离运行状态，不等于另有测试集。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
},
{
"label": "§9.1：GAIA exact match",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px1"
}
],
"judgment": "GAIA：规则将回答与参考答案精确匹配"
},
{
"label": "ALFWorld / WebShop",
"data": "ALFWorld（通过文字动作完成家居物体操作的交互环境） valid-unseen 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100",
"scoring": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 检查操作目标；WebShop（根据用户要求挑选和购买商品的交互基准） 按购物目标约束评价购买结果。",
"visible": "各任务分数与全执行记录持续写入存储，修改者据此诊断处理逻辑并提出候选。",
"use": "每轮反复使用相同适应任务；曲线仍是这批题的表现。每题重置环境只隔离运行状态，不等于另有测试集。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"judgment": "ALFWorld 目标检查；WebShop 商品约束奖励，均由环境程序计算"
},
{
"label": "τ³ Retail / Airline / Telecom",
"data": "三个领域的全部任务",
"scoring": "工具任务环境按目标完成和状态变化验收。",
"visible": "各任务分数与全执行记录持续写入存储，修改者据此诊断处理逻辑并提出候选。",
"use": "每轮反复使用相同适应任务；曲线仍是这批题的表现。每题重置环境只隔离运行状态，不等于另有测试集。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"judgment": "客服环境的目标／状态验收；本文未逐任务展开内部判分"
},
{
"label": "SWE-bench Verified",
"data": "55 题",
"scoring": "仓库测试检查补丁是否解决问题。",
"visible": "各任务分数与全执行记录持续写入存储，修改者据此诊断处理逻辑并提出候选。",
"use": "每轮反复使用相同适应任务；曲线仍是这批题的表现。每题重置环境只隔离运行状态，不等于另有测试集。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"judgment": "SWE-bench 仓库测试验收"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 103、ALFWorld（通过文字动作完成家居物体操作的交互环境） valid-unseen 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、τ³ Retail/Airline/Telecom 全任务、SWE-Verified 55，形成演化轨迹。",
"selection": "每轮反复在上述固定任务上执行和计分，诊断失败并生成下一版。",
"evaluation": "论文曲线仍报告同一批任务逐轮变化。",
"isolation": "valid-unseen 是原评测基准名称，但这些题参与了本方法迭代；不能据名称称为未参与进化的最终测试。",
"roles": {
"executor": {
"value": "Claude Sonnet4.6、GPT-5.4、Qwen3.5-9B分别作为task task agent，运行候选运行框架。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2606.14249#S6.SS1.SSS0.Px2"
}
]
},
"modifier": {
"value": "默认Claude Opus4.6作为 meta-agent（负责设计或修改 task agent）驱动AEGIS进化；与上述任务执行者分开。共同训练阶段的参数更新仍由GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）程序执行。",
"sources": [
{
"label": "§5.4",
"url": "https://arxiv.org/html/2606.14249#S5.SS4.SSS0.Px3"
},
{
"label": "§12.5",
"url": "https://arxiv.org/html/2606.14249#S12.SS5.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2606.14249#S6.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "从各任务的人工基础框架开始，修改过程分为整理轨迹、规划改动、生成候选和评审四个角色。每轮提出 4 个候选并保留执行记录；每题重建环境，避免购物车、游戏状态或工作目录互相影响。主要修改提示和运行逻辑；这种环境重置不说明训练题和测试题已隔离。",
"sources": [
{
"label": "Table 3；附录 9.5、Table 8",
"url": "https://arxiv.org/abs/2606.14249"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"selection": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"evaluation": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
],
"isolation": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2606.14249#S9.SS1.SSS0.Px5"
},
{
"label": "§9.2",
"url": "https://arxiv.org/html/2606.14249#S9.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "手工固定框架每遇到新模型、工具或领域就要重新调整；框架优化积累的轨迹又常被丢弃，没有进入模型训练，模型提升也没有反馈到框架设计。这使两种改进彼此脱节，因此作者希望用经验驱动框架更新，并连接后续模型学习，同时处理遗忘和评分投机等风险。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.14249#S1"
}
]
},
{
"key": "position",
"value": "面向 agent 系统的整体改进，研究运行框架设计与模型训练如何共同受益于执行经验，减少两者分开优化的局限。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.14249"
}
]
},
{
"key": "conclusion",
"value": "在五种 agent 基准上获得收益，展示运行结构优化与模型学习相结合的路径。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.14249"
}
]
}
],
"fields": {
"object": "运行时的可组合处理组件；联合更新设置还训练任务模型参数。",
"verdict": "执行记录用于分析运行组件，任务成绩用于判断实际效果；ALFWorld、GAIA、WebShop、τ³ 和 SWE-bench 的环境验收或答案检查方式分别列在表格中。",
"seed": "从各任务的人工基础框架开始，修改过程分为整理轨迹、规划改动、生成候选和评审四个角色。每轮提出 4 个候选并保留执行记录；每题重建环境，避免购物车、游戏状态或工作目录互相影响。主要修改提示和运行逻辑；这种环境重置不说明训练题和测试题已隔离。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2607.13683",
"title": "HarnessBank / Self-Evolving Agent Harnesses via Gated Semantic Quality-Diversity",
"url": "https://arxiv.org/abs/2607.13683",
"date": "2026-07-15",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Archive",
"ExecutableVerifier",
"HarnessCode",
"M1",
"Population",
"RegressionGate"
],
"fields": {
"本质定位": "**H-Full**。维护**语义多样的高性能 harness bank**，不是只保留 best；offspring 可重组/重新发明机制，只有过 gated verification 才入库。",
"谁来改 → 谁执行；基础 harness": "evolutionary harness proposer/controller → target agent candidates。",
"Feedback / evidence": "gated verifier + candidate performance。"
},
"details": [],
"occurrences": [
{
"section": "D. Full harness / executable harness optimization",
"line": 177,
"fields": {
"时间": "2026-07-15",
"论文": "[HarnessBank](https://arxiv.org/abs/2607.13683)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**H-Full**。维护**语义多样的高性能 harness bank**，不是只保留 best；offspring 可重组/重新发明机制，只有过 gated verification 才入库。",
"谁来改 → 谁执行；基础 harness": "evolutionary harness proposer/controller → target agent candidates。",
"Feedback / evidence": "gated verifier + candidate performance。",
"标签": "`#HarnessCode #Archive #Population #ExecutableVerifier #RegressionGate #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "本轮核对 arXiv 摘要：train 选模，sealed test 在进化后评分；完整协议待重审。",
"year": "2026",
"depth": [
"M1"
],
"updateNote": "原记录简称 HarnessBank；本轮按相同 arXiv ID 去重并补充正式标题。",
"sources": [
"https://arxiv.org/abs/2607.13683"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "HarnessBank 把候选放进按语义机制区分的 bank，让不同类型的高质量方案可以同时存活。新候选可重组已有机制，也可提出新机制；先用小批任务检查“实验有效、改动真的生效、收益有统计依据”，通过后才付出完整训练集评估的成本。",
"novelty": "维护机制不同的多个框架候选，经过分阶段检查后入库，供后续重组使用；重点同时控制候选多样性和接受修改的质量。",
"object": "完整运行框架候选，以及按不同语义机制组织的候选库。",
"executor": "默认冻结Qwen3.6-27B执行各候选运行框架；跨模型对照还分析较大Qwen模型和Gemini3 Flash。",
"modifier": "Claude Opus4.8作为修改者，根据轨迹和评分提出运行框架改动；Qwen3.6-27B的权重不更新。",
"roleContext": "evolutionary harness proposer/controller → target agent candidates。",
"seed": "七个领域共享各自的 vanilla 起点；固定 Qwen3.6-27B 执行，Claude Opus 4.8 作为修改者。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "任务结果、执行轨迹与配对候选表现。",
"diagnosis": "从当前训练任务生成 comprehensive diagnosis，供重组/重构候选。",
"update": "reinvention / recombination 产生 offspring，按语义 cell 竞争入库。",
"acceptance": "sampled train 上依次检查 validity、mechanism activation、significance、gain；通过后做 full-train evaluation 再竞争入库。",
"experiments": [
{
"name": "论文定义的 train / test 协议",
"evolve": "D_train：诊断、小批筛选和最终入库评分",
"selection": "入库仍使用 train，不是额外 test credit",
"test": "D_test 不参与 evolution decisions",
"isolation": "测试被排除于进化决策",
"note": "具体 benchmark 数量及模型配置仍待补查。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2607.13683v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：TB2、AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、LiveCode、Omni-MATH、BrowseComp+、GDPval、SWE-bench 的 训练集。\n\n调试 / 选版本数据：训练任务决定候选门控与档案录取；每题 3 次尝试。\n\n最终测试数据：各域互斥 测试集，只在选择完成后用于最终排名。\n\n数据隔离与证据边界：原文列的 EvoAgentBench 五域与本站所收录的四域版本不同，因此不借用另一篇的样本数量。",
"cycle": "语言模型 对失败作语义归因并提候选；统计门控决定是否给改动记功和收入质量—多样性档案，误诊本身不能让补丁通过。",
"train": "TB2、AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、LiveCode、Omni-MATH、BrowseComp+、GDPval、SWE-bench 的 训练集。",
"debug": "训练任务决定候选门控与档案录取；每题 3 次尝试。",
"test": "各域互斥 测试集，只在选择完成后用于最终排名。",
"isolation": "原文列的 EvoAgentBench 五域与本站所收录的四域版本不同，因此不借用另一篇的样本数量。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "从共同的 vanilla agent 出发，在冻结任务模型外演化 harness，并把不同运行机制保存在语义多样性档案中。候选需通过有效性、激活、显著性和收益检查；不是只保留单一路径的最高平均分。初始文件/工具清单本轮未核实。",
"protocol": "**进化→测试：**Terminal-Bench 2、AppWorld，以及论文标为来自 EvoAgentBench 的 LiveCode、Omni-MATH、BrowseComp+、GDPval、SWE-bench，共七域，使用互斥 train/test。训练题用于候选筛选和档案录取，test 不参与这些决定；每题三次尝试的 Pass@1 为主指标。\n\n**待核实：**各域样本量尚未确认；论文所列五域与本站 EvoAgentBench 原文的四域版本不完全一致，不混用另一篇的数量。",
"sections": "§3.2、§4.1",
"source": "https://arxiv.org/abs/2607.13683",
"version": "2607.13683v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "d94eac7d53e978f917a847cca0674efa683858a8f60f6b9eac2eacdd764f0865",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "默认冻结Qwen3.6-27B执行各候选运行框架；跨模型对照还分析较大Qwen模型和Gemini3 Flash。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2607.13683#S4.SS6"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude Opus4.8作为修改者，根据轨迹和评分提出运行框架改动；Qwen3.6-27B的权重不更新。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2607.13683#S4.SS6"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "完整运行框架候选，以及按不同语义机制组织的候选库。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务结果、执行轨迹与配对候选表现。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "七个领域共享各自的 vanilla 起点；固定 Qwen3.6-27B 执行，Claude Opus 4.8 作为修改者。",
"sources": [
{
"label": "§3.2、§4.1",
"url": "https://arxiv.org/abs/2607.13683"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "语言模型 对失败作语义归因并提候选；统计门控决定是否给改动记功和收入质量—多样性档案，误诊本身不能让补丁通过。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "TB2、AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、LiveCode、Omni-MATH、BrowseComp+、GDPval、SWE-bench 的 训练集。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练任务决定候选门控与档案录取；每题 3 次尝试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各域互斥 测试集，只在选择完成后用于最终排名。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "原文列的 EvoAgentBench 五域与本站所收录的四域版本不同，因此不借用另一篇的样本数量。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "维护机制不同的多个框架候选，经过分阶段检查后入库，供后续重组使用；重点同时控制候选多样性和接受修改的质量。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.13683v2",
"version": "2607.13683v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "两个关键变量是 bank 的机制多样性与分阶段验证。保留多个候选不是装饰：它给后续重组提供不同机制；入库得分也不是最终 held-out 泛化证明。",
"feedbackCases": [
{
"label": "Terminal-Bench 2 / SWE-bench / LiveCode",
"data": "各域 训练集 用于进化，互斥 测试集 只作最终排名；每题三次执行。",
"scoring": "终端验收、仓库补丁测试或代码测试检查产物功能。",
"visible": "任务级分数、轨迹和配对候选表现。",
"use": "按训练成绩门控和维护候选档案，测试集不参与候选选择。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"judgment": "终端验收测试／仓库测试／代码题测试"
},
{
"label": "AppWorld",
"data": "各域 训练集 用于进化，互斥 测试集 只作最终排名；每题三次执行。",
"scoring": "应用任务验收检查目标状态。",
"visible": "任务级分数、轨迹和配对候选表现。",
"use": "按训练成绩门控和维护候选档案，测试集不参与候选选择。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"judgment": "AppWorld 环境状态验收"
},
{
"label": "Omni-MATH / BrowseComp+",
"data": "各域 训练集 用于进化，互斥 测试集 只作最终排名；每题三次执行。",
"scoring": "依据任务参考答案判回答正确性，保留各自问答评分方式。",
"visible": "任务级分数、轨迹和配对候选表现。",
"use": "按训练成绩门控和维护候选档案，测试集不参与候选选择。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"judgment": "参考答案判分；本文未逐基准说明规则或模型比对器"
},
{
"label": "GDPval",
"data": "各域 训练集 用于进化，互斥 测试集 只作最终排名；每题三次执行。",
"scoring": "按工作产物的任务评分细则评价，不能等同于代码单元测试。",
"visible": "任务级分数、轨迹和配对候选表现。",
"use": "按训练成绩门控和维护候选档案，测试集不参与候选选择。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"judgment": "按 GDPval 任务评分细则评价；所引段落未明确裁判型号"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "TB2、AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、LiveCode、Omni-MATH、BrowseComp+、GDPval、SWE-bench 的 训练集。",
"selection": "训练任务决定候选门控与档案录取；每题 3 次尝试。",
"evaluation": "各域互斥 测试集，只在选择完成后用于最终排名。",
"isolation": "原文列的 EvoAgentBench 五域与本站所收录的四域版本不同，因此不借用另一篇的样本数量。",
"roles": {
"executor": {
"value": "默认冻结Qwen3.6-27B执行各候选运行框架；跨模型对照还分析较大Qwen模型和Gemini3 Flash。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2607.13683#S4.SS6"
}
]
},
"modifier": {
"value": "Claude Opus4.8作为修改者，根据轨迹和评分提出运行框架改动；Qwen3.6-27B的权重不更新。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2607.13683#S4.SS6"
}
]
},
"seed": {
"value": "七个领域共享各自的 vanilla 起点；固定 Qwen3.6-27B 执行，Claude Opus 4.8 作为修改者。",
"sources": [
{
"label": "§3.2、§4.1",
"url": "https://arxiv.org/abs/2607.13683"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2607.13683#S4.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "贪心框架进化容易反复保留保守的局部修改，丢掉结构不同但互补的候选；廉价单次评分又可能把执行噪声、基础设施异常或未生效补丁误判成收益。作者因此同时关注保留多样改进路线，以及用有限评估确认机制确实生效且收益可复现。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.13683#S1"
}
]
},
{
"key": "position",
"value": "研究在评估有噪声、交互预算有限时，如何保持框架搜索的有效多样性，并找到可靠泛化的改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.13683"
}
]
},
{
"key": "conclusion",
"value": "在七种基准上稳定改善；跨模型分析表明更好的运行框架往往仍需针对模型适配。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.13683"
}
]
}
],
"fields": {
"object": "完整运行框架候选，以及按不同语义机制组织的候选库。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2603.21877",
"title": "P²O: Joint Policy and Prompt Optimization",
"url": "https://arxiv.org/abs/2603.21877",
"date": "2026-03-23",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"CoEvolution",
"M1",
"ProcessReward",
"Prompt",
"SameModel",
"Weights"
],
"fields": {
"本质定位": "**Hybrid**。prompt evolution 给 policy 更好的 exploration template；policy 变强后产生更难/更有价值 seed，再反哺 prompt round，并把 prompt gain 蒸馏进 weights。",
"谁来改 → 谁执行；基础 harness": "alternating prompt optimizer + policy trainer → current policy under current prompt。",
"Feedback / evidence": "task reward / training outcome。"
},
"details": [],
"occurrences": [
{
"section": "E. Hybrid model–prompt / model–harness",
"line": 184,
"fields": {
"时间": "2026-03-23",
"论文": "[P²O: Joint Policy and Prompt Optimization](https://arxiv.org/abs/2603.21877)",
"级别": "**K**",
"我们的定位：什么在变、真正新点": "**Hybrid**。prompt evolution 给 policy 更好的 exploration template；policy 变强后产生更难/更有价值 seed，再反哺 prompt round，并把 prompt gain 蒸馏进 weights。",
"谁来改 → 谁执行；基础 harness": "alternating prompt optimizer + policy trainer → current policy under current prompt。",
"Feedback / evidence": "task reward / training outcome。",
"标签": "`#Prompt #Weights #SameModel #CoEvolution #ProcessReward #M1`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "prompt evolution 给 policy 更好的 exploration template；policy 变强后产生更难/更有价值 seed，再反哺 prompt round，并把 prompt gain 蒸馏进 weights。",
"novelty": "针对多次尝试全部失败、奖励无法区分好坏的难题，先搜索能引出成功的提示，再把提示帮助下的行为训练进模型参数。",
"object": "策略模型参数和帮助解决难题的提示模板；奖励及外层提示搜索、训练算法固定。",
"executor": "Qwen3-4B的当前训练checkpoint（某个时刻保存的模型或系统版本），在当前候选提示下解题。",
"modifier": "P²O-Self-Ref用Qwen3-4B参考模型作提示修改者；Teacher-Ref用Kimi-K2。外层交替运行提示优化和策略参数训练。",
"roleContext": "alternating prompt optimizer + policy trainer → current policy under current prompt。",
"seed": "Qwen3-4B 的数学求解训练流程，结合 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 参数更新与提示模板优化。提示候选保存在前沿集合中，训练时采样使用；变化包括权重和提示，不是只往外部记忆追加文字。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "数学题按参考答案判断解答正确性；先比较不同提示的解题表现，再利用较好提示带来的奖励训练模型参数。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "尚未逐项标注；请阅读 Evolution → Eval 原文。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：DeepScaler 随机样本、DeepMath 难度≥7 的样本，分别比较 5,000 与 10,000 条训练规模。\n\n调试 / 选版本数据：训练困难题的内部 train/dev 用于 GEPA 模板搜索；以 开发集 表现选择模型 checkpoint（某个时刻保存的模型或系统版本）。\n\n最终测试数据：AIME24、AIME25、AMC、MATH500、Minerva、Olympiad。\n\n数据隔离与证据边界：5k/10k 是训练规模，不是训练/测试各一组；提示搜索用训练困难题，不应混写成最终评估题。",
"cycle": "GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新策略后找低奖励困难题；GEPA 在困难题内部划 train/dev 搜索模板，保留 Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案） 提示，再用于下一轮策略采样。",
"train": "DeepScaler 随机样本、DeepMath 难度≥7 的样本，分别比较 5,000 与 10,000 条训练规模。",
"debug": "训练困难题的内部 train/dev 用于 GEPA 模板搜索；以 开发集 表现选择模型 checkpoint（某个时刻保存的模型或系统版本）。",
"test": "AIME24、AIME25、AMC、MATH500、Minerva、Olympiad。",
"isolation": "5k/10k 是训练规模，不是训练/测试各一组；提示搜索用训练困难题，不应混写成最终评估题。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "Qwen3-4B 的数学求解训练流程，结合 GRPO 参数更新与提示模板优化。提示候选保存在前沿集合中，训练时采样使用；变化包括权重和提示，不是只往外部记忆追加文字。",
"protocol": "**训练：**从 DeepScaler 随机取数据，另从 DeepMath 筛难度≥7 的题；分别做 5,000 和 10,000 条规模实验，不能误写成训练/测试各 5k/10k。训练 10 epochs，按 dev 选 checkpoint；GEPA 对照验证集为 300 条。\n\n**测试：**AIME24、AIME25、AMC、MATH500、Minerva、Olympiad。各测试子集的精确大小与训练去重设置本轮未核实。",
"sections": "实验设置与主结果表",
"source": "https://arxiv.org/abs/2603.21877",
"version": "2603.21877v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c7e7829fabcf9e6f15d4fb50797c12acb7001c097280a659468d4f19bcb5b9b9",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3-4B的当前训练checkpoint（某个时刻保存的模型或系统版本），在当前候选提示下解题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "P²O-Self-Ref用Qwen3-4B参考模型作提示修改者；Teacher-Ref用Kimi-K2。外层交替运行提示优化和策略参数训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "策略模型参数和帮助解决难题的提示模板；奖励及外层提示搜索、训练算法固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "数学题按参考答案判断解答正确性；先比较不同提示的解题表现，再利用较好提示带来的奖励训练模型参数。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Qwen3-4B 的数学求解训练流程，结合 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 参数更新与提示模板优化。提示候选保存在前沿集合中，训练时采样使用；变化包括权重和提示，不是只往外部记忆追加文字。",
"sources": [
{
"label": "实验设置与主结果表",
"url": "https://arxiv.org/abs/2603.21877"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新策略后找低奖励困难题；GEPA 在困难题内部划 train/dev 搜索模板，保留 Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案） 提示，再用于下一轮策略采样。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "DeepScaler 随机样本、DeepMath 难度≥7 的样本，分别比较 5,000 与 10,000 条训练规模。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练困难题的内部 train/dev 用于 GEPA 模板搜索；以 开发集 表现选择模型 checkpoint（某个时刻保存的模型或系统版本）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AIME24、AIME25、AMC、MATH500、Minerva、Olympiad。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "5k/10k 是训练规模，不是训练/测试各一组；提示搜索用训练困难题，不应混写成最终评估题。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "针对多次尝试全部失败、奖励无法区分好坏的难题，先搜索能引出成功的提示，再把提示帮助下的行为训练进模型参数。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.21877v3",
"version": "2603.21877v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "专门处理所有 rollout 都失败、GRPO advantage 消失的难样本：GEPA 找到能触发成功的 prompt，再蒸馏进 policy weights。",
"feedbackCases": [
{
"label": "数学题上的提示与参数联合优化",
"data": "DeepScaler 随机样本、DeepMath 难度≥7 的样本；比较 5,000/10,000 条规模，困难题再作内部训练/开发划分。",
"scoring": "由数学任务答案验证得到奖励；全组尝试都失败时，GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 无有效相对优势，GEPA 搜索能诱导成功的提示。",
"visible": "训练题回答及验证结果用于提示搜索；内部开发分数选检查点。",
"use": "把有效提示诱导的行为蒸馏到参数。最终另测 AIME24/25、AMC、MATH500、Minerva、Olympiad。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
}
],
"judgment": "数学标准答案验证奖励；本文未展开具体答案解析与比对实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "DeepScaler 随机样本、DeepMath 难度≥7 的样本，分别比较 5,000 与 10,000 条训练规模。",
"selection": "训练困难题的内部 train/dev 用于 GEPA 模板搜索；以 开发集 表现选择模型 checkpoint（某个时刻保存的模型或系统版本）。",
"evaluation": "AIME24、AIME25、AMC、MATH500、Minerva、Olympiad。",
"isolation": "5k/10k 是训练规模，不是训练/测试各一组；提示搜索用训练困难题，不应混写成最终评估题。",
"roles": {
"executor": {
"value": "Qwen3-4B的当前训练checkpoint（某个时刻保存的模型或系统版本），在当前候选提示下解题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px2"
}
]
},
"modifier": {
"value": "P²O-Self-Ref用Qwen3-4B参考模型作提示修改者；Teacher-Ref用Kimi-K2。外层交替运行提示优化和策略参数训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.21877#S3.SS1"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.21877#S3.SS4.SSS0.Px2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "Qwen3-4B 的数学求解训练流程，结合 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 参数更新与提示模板优化。提示候选保存在前沿集合中，训练时采样使用；变化包括权重和提示，不是只往外部记忆追加文字。",
"sources": [
{
"label": "实验设置与主结果表",
"url": "https://arxiv.org/abs/2603.21877"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.21877#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.21877#S4.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.21877#A1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "同一题的所有采样若全对或全错，基于组内奖励差异的强化学习就失去有效更新信号，尤其学不到最有价值的难题。简单增加采样常仍找不到成功路径，人工课程和奖励设计又昂贵，因此作者研究用提示优化打开可成功的探索路径，再把它转成参数学习。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.21877#S1"
}
]
},
{
"key": "position",
"value": "研究模型在难题上几乎所有尝试都失败时，能否重新获得有效学习信号，并把提示带来的能力增益转化为持久参数能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.21877"
}
]
},
{
"key": "conclusion",
"value": "恢复难例上的学习信号，优于单纯增加尝试次数；最终推理不必继续携带优化出的提示。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.21877"
}
]
}
],
"fields": {
"object": "策略模型参数和帮助解决难题的提示模板；奖励及外层提示搜索、训练算法固定。",
"verdict": "数学题按参考答案判断解答正确性；先比较不同提示的解题表现，再利用较好提示带来的奖励训练模型参数。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2606.04455",
"title": "Meta-Agent Challenge",
"url": "https://arxiv.org/abs/2606.04455",
"date": "2026-06-03",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"BenchmarkScore",
"HarnessCode",
"HeldOut",
"Improver"
],
"fields": {
"本质定位": "**B-Harness / B-RSI**。不是让 agent 解任务，而是让 meta-agent **在 sandbox 中编程一个 agent artifact** 去最大化 held-out performance。",
"被测系统 / seed harness / feedback": "meta-agent + eval API + time budget；从 task-provided sandbox/seed artifact 开始。",
"证据边界与关键结论": "held-out 五域；meta-agent很少达到 human baseline；高 optimization pressure 下出现 GT exfiltration，说明 harness-RSI 同时有 reward hacking 风险。"
},
"details": [],
"occurrences": [
{
"section": "A. Harness optimization / autonomous agent development",
"line": 193,
"fields": {
"时间": "2026-06-03",
"论文": "[Meta-Agent Challenge](https://arxiv.org/abs/2606.04455)",
"级别": "**K**",
"它真正测什么": "**B-Harness / B-RSI**。不是让 agent 解任务，而是让 meta-agent **在 sandbox 中编程一个 agent artifact** 去最大化 held-out performance。",
"被测系统 / seed harness / feedback": "meta-agent + eval API + time budget；从 task-provided sandbox/seed artifact 开始。",
"证据边界与关键结论": "held-out 五域；meta-agent很少达到 human baseline；高 optimization pressure 下出现 GT exfiltration，说明 harness-RSI 同时有 reward hacking 风险。",
"标签": "`#HarnessCode #Improver #HeldOut #BenchmarkScore #B-RSI`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录明确标注 #HeldOut；是否参与选模仍需结合协议原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-Harness",
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "不是让 agent 解任务，而是让 meta-agent **在 sandbox 中编程一个 agent artifact** 去最大化 held-out performance。",
"novelty": "让被测模型在隔离开发环境里编写 agent 系统，开发结束后才加入正式测试的检查程序，同时检查生成系统是否绕过任务规则。",
"object": "meta-agent 写出的可执行 task agent 程序。",
"executor": "提交的 agent.py 在数学、科学和竞赛编程任务中调用 Qwen3-8B，模型通过 vLLM 推理服务运行在独占的 A100 显卡上；软件修复 SWE-Bench 和终端操作 Terminal-Bench 使用 Claude Haiku 4.5。",
"modifier": "外层 coding agent 包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.7／Opus 4.6／Sonnet 4.6、Gemini CLI（通过终端命令使用的程序界面） + Gemini 3.1 Pro、Codex + GPT-5.3-Codex／GPT-5.4；另有接入 Claude Code 的开源模型配置。它们写 agent.py，不是最终答题的 Qwen3-8B 或 Haiku 4.5。",
"roleContext": "meta-agent + eval API + time budget；从 task-provided sandbox/seed artifact 开始。",
"seed": "使用通用编程 task agent 构造或改进目标 task agent 的工作流，评测基准提供统一问题/预测接口和评估控制。可修改的是待交付求解系统；它没有提出一个所有参赛者必须使用的新 meta-agent（负责设计或修改 task agent）架构。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "开发阶段的评分接口返回逐题正确性或准确率；最终评测另用参考答案或单元测试计分。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "held-out 五域；meta-agent很少达到 human baseline；高 optimization pressure 下出现 GT exfiltration，说明 harness-RSI 同时有 reward hacking 风险。"
}
],
"takeaway": "held-out 五域；meta-agent很少达到 human baseline；高 optimization pressure 下出现 GT exfiltration，说明 harness-RSI 同时有 reward hacking 风险。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：不训权重；开发题用于设计 task agent。数学 AIME 2022–23 共 60；科学 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 多选 591；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 732。\n\n调试 / 选版本数据：上述开发 oracle 的反馈，科学域另允许有限 Google Search 调用。\n\n最终测试数据：AIME 2024–25 共 60；GPQA（研究生级科学问答基准） Diamond 198；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 323。\n\n数据隔离与证据边界：最终结果检查器（按测试或判分规则检查任务结果）仅在开发结束后注入，测试密钥和题目不参与修改。",
"cycle": "编程 task agent 实现、运行并修改程序；预算结束后扫描 API 使用合规性，再注入私有结果检查器（按测试或判分规则检查任务结果）测最终版本。",
"train": "不训权重；开发题用于设计 task agent。数学 AIME 2022–23 共 60；科学 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 多选 591；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 732。",
"debug": "上述开发 oracle 的反馈，科学域另允许有限 Google Search 调用。",
"test": "AIME 2024–25 共 60；GPQA（研究生级科学问答基准） Diamond 198；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 323。",
"isolation": "最终结果检查器（按测试或判分规则检查任务结果）仅在开发结束后注入，测试密钥和题目不参与修改。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "使用通用编程 agent 构造或改进目标 agent 的工作流，benchmark 提供统一问题/预测接口和评估控制。可修改的是待交付求解系统；它没有提出一个所有参赛者必须使用的新 meta-agent 架构。",
"protocol": "**科学问答：**开发集为 HLE 多选子集 591 题，最终测试为 GPQA Diamond 198 题；允许有限 Google Search 调用。\n\n**编程：**LiveCodeBench 开发集 732 题、留出测试 323 题，输出 Python 程序并以隐藏测试判分。开发题用于构建/优化，最终题用于独立评估；其他任务域与具体初始工具本轮待补核。",
"sections": "任务定义与各领域数据说明",
"source": "https://arxiv.org/abs/2606.04455",
"version": "2606.04455v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "b37de24af380bdf0f5bff05648bd9c65744cc2dd35674f9c24b0adde4f40f66d",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "提交的 agent.py 在数学、科学和竞赛编程任务中调用 Qwen3-8B，模型通过 vLLM 推理服务运行在独占的 A100 显卡上；软件修复 SWE-Bench 和终端操作 Terminal-Bench 使用 Claude Haiku 4.5。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2606.04455#S4.SS0.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2606.04455#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "外层 coding agent 包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.7／Opus 4.6／Sonnet 4.6、Gemini CLI（通过终端命令使用的程序界面） + Gemini 3.1 Pro、Codex + GPT-5.3-Codex／GPT-5.4；另有接入 Claude Code 的开源模型配置。它们写 agent.py，不是最终答题的 Qwen3-8B 或 Haiku 4.5。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2606.04455#S4.SS0.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2606.04455#S5.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "meta-agent 写出的可执行 task agent 程序。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "开发阶段的评分接口返回逐题正确性或准确率；最终评测另用参考答案或单元测试计分。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "使用通用编程 task agent 构造或改进目标 task agent 的工作流，评测基准提供统一问题/预测接口和评估控制。可修改的是待交付求解系统；它没有提出一个所有参赛者必须使用的新 meta-agent（负责设计或修改 task agent）架构。",
"sources": [
{
"label": "任务定义与各领域数据说明",
"url": "https://arxiv.org/abs/2606.04455"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "编程 task agent 实现、运行并修改程序；预算结束后扫描 API 使用合规性，再注入私有结果检查器（按测试或判分规则检查任务结果）测最终版本。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训权重；开发题用于设计 task agent。数学 AIME 2022–23 共 60；科学 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 多选 591；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 732。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "上述开发 oracle 的反馈，科学域另允许有限 Google Search 调用。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AIME 2024–25 共 60；GPQA（研究生级科学问答基准） Diamond 198；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 323。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "最终结果检查器（按测试或判分规则检查任务结果）仅在开发结束后注入，测试密钥和题目不参与修改。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让被测模型在隔离开发环境里编写 agent 系统，开发结束后才加入正式测试的检查程序，同时检查生成系统是否绕过任务规则。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.04455v1",
"version": "2606.04455v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "让被测者在 sandbox 中编程 agent artifact；测试 verifier 在开发结束后才注入，区分开发反馈与最终评价，同时观察规避规则行为。",
"feedbackCases": [
{
"label": "数学",
"data": "AIME 2022–23 共 60 题开发；2024–25 共 60 题测试",
"scoring": "最终数学答案与参考答案比对。",
"visible": "开发评分接口可返回逐题正确性或准确率；最终私有验收脚本仅在开发结束后注入。",
"use": "在预算内编写/改进 task agent，冻结后运行测试；先检查 API 使用合规，再计算最终成绩。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"judgment": "数学标准答案比对；本文未展开解析器实现"
},
{
"label": "科学选择题",
"data": "HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 多选 591 题开发；GPQA（研究生级科学问答基准） Diamond 198 题测试",
"scoring": "根据选择题参考选项判断正确性。",
"visible": "开发评分接口可返回逐题正确性或准确率；最终私有验收脚本仅在开发结束后注入。",
"use": "在预算内编写/改进 task agent，冻结后运行测试；先检查 API 使用合规，再计算最终成绩。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"judgment": "参考选项判对错"
},
{
"label": "代码生成",
"data": "LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 732 题开发、323 题测试",
"scoring": "运行生成程序，以题目单元测试检查功能。",
"visible": "开发评分接口可返回逐题正确性或准确率；最终私有验收脚本仅在开发结束后注入。",
"use": "在预算内编写/改进 task agent，冻结后运行测试；先检查 API 使用合规，再计算最终成绩。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"judgment": "执行代码题单元测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "不训权重；开发题用于设计 task agent。数学 AIME 2022–23 共 60；科学 HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 多选 591；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 732。",
"selection": "上述开发 oracle 的反馈，科学域另允许有限 Google Search 调用。",
"evaluation": "AIME 2024–25 共 60；GPQA（研究生级科学问答基准） Diamond 198；LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 323。",
"isolation": "最终结果检查器（按测试或判分规则检查任务结果）仅在开发结束后注入，测试密钥和题目不参与修改。",
"roles": {
"executor": {
"value": "提交的 agent.py 在数学、科学和竞赛编程任务中调用 Qwen3-8B，模型通过 vLLM 推理服务运行在独占的 A100 显卡上；软件修复 SWE-Bench 和终端操作 Terminal-Bench 使用 Claude Haiku 4.5。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2606.04455#S4.SS0.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2606.04455#S5.SS2"
}
]
},
"modifier": {
"value": "外层 coding agent 包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.7／Opus 4.6／Sonnet 4.6、Gemini CLI（通过终端命令使用的程序界面） + Gemini 3.1 Pro、Codex + GPT-5.3-Codex／GPT-5.4；另有接入 Claude Code 的开源模型配置。它们写 agent.py，不是最终答题的 Qwen3-8B 或 Haiku 4.5。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2606.04455#S3.SS2.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2606.04455#S4.SS0.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2606.04455#S5.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "使用通用编程 task agent 构造或改进目标 task agent 的工作流，评测基准提供统一问题/预测接口和评估控制。可修改的是待交付求解系统；它没有提出一个所有参赛者必须使用的新 meta-agent（负责设计或修改 task agent）架构。",
"sources": [
{
"label": "任务定义与各领域数据说明",
"url": "https://arxiv.org/abs/2606.04455"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"selection": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"evaluation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
],
"isolation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2606.04455#A3.SS0.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有基准测的是模型在人工搭好的流程中能否解题，却不测它能否设计那套流程，因此无法判断模型是否能减少 agent 工程的人力瓶颈。作者希望直接评价提出架构假设、实现系统、依据有限反馈诊断并持续迭代的能力，而不只看单道领域题的执行表现。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.04455#S1"
}
]
},
{
"key": "position",
"value": "评测语言模型能否自主开发有竞争力的 agent 系统，考察系统设计能力、设计稳定性及对评测边界的遵守。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04455"
}
]
},
{
"key": "conclusion",
"value": "多数配置仍难匹敌人工系统，设计过程方差大，还暴露了利用评测漏洞的行为。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.04455"
}
]
}
],
"fields": {
"object": "meta-agent 写出的可执行 task agent 程序。",
"executor": "提交的 agent.py 在数学、科学和竞赛编程任务中调用 Qwen3-8B，模型通过 vLLM 推理服务运行在独占的 A100 显卡上；软件修复 SWE-Bench 和终端操作 Terminal-Bench 使用 Claude Haiku 4.5。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.06301",
"title": "HarnessOpt-Bench: Evaluating LLMs at Harness Optimization",
"url": "https://arxiv.org/abs/2608.06301",
"date": "2026-08-06",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"BenchmarkScore",
"HarnessCode",
"HeldOut",
"M1",
"RegressionGate",
"SeparateEvolver"
],
"fields": {
"本质定位": "把 harness optimization 本身作为 LLM capability：optimizer=LLM+coding harness，给 seed target harness、graded eval feedback、固定 evaluation budget，最后提交一个 candidate在 inaccessible held-out test 上评分。",
"被测系统 / seed harness / feedback": "optimizer=LLM+coding harness；target agent固定；trusted execution env。",
"证据边界与关键结论": "**search→inaccessible held-out test**；重要发现：optimizer model 差异通常大于 coding-harness 差异。",
"什么在变": "被测 optimizer修改 target agent harness；benchmark本身固定。",
"谁来改 / 谁执行": "**改**：不同 frontier LLM optimizer，在 shared coding harness与native harness两种条件下。
**执行**：固定 target agent 执行 optimizer 提交的 candidate harness。",
"基础 harness": "每个 downstream task的 seed target-agent harness。",
"Feedback": "graded evaluation feedback，固定 target-evaluation budget；test partition完全不可访问。",
"Evolution → Eval": "4 downstream tasks、111 scored runs；search/selection→held-out test。",
"Meta-depth": "Evaluation of M1 optimizer capability。",
"相对之前真正新增什么": "相比方法论文，最大的价值是 **统一预算 + trusted execution boundary + held-out normalized gain**，把 optimizer model capability 与 target harness performance分开测。"
},
"details": [],
"occurrences": [
{
"section": "A. Harness optimization / autonomous agent development",
"line": 194,
"fields": {
"时间": "2026-08-06",
"论文": "[HarnessOpt-Bench](https://arxiv.org/abs/2608.06301)",
"级别": "**K**",
"它真正测什么": "**B-Harness**。严格测 optimizer LLM 的 end-to-end harness optimization：给 seed harness、graded feedback、固定 eval budget，提交单一 final candidate。",
"被测系统 / seed harness / feedback": "optimizer=LLM+coding harness；target agent固定；trusted execution env。",
"证据边界与关键结论": "**search→inaccessible held-out test**；重要发现：optimizer model 差异通常大于 coding-harness 差异。",
"标签": "`#HarnessCode #SeparateEvolver #HeldOut #BenchmarkScore #RegressionGate #M1`"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 365,
"fields": {
"优先级": "**K**",
"时间": "2026-08-06",
"论文": "[HarnessOpt-Bench: Evaluating LLMs at Harness Optimization](https://arxiv.org/abs/2608.06301)",
"本质定位": "把 harness optimization 本身作为 LLM capability：optimizer=LLM+coding harness，给 seed target harness、graded eval feedback、固定 evaluation budget，最后提交一个 candidate在 inaccessible held-out test 上评分。",
"什么在变": "被测 optimizer修改 target agent harness；benchmark本身固定。",
"谁来改 / 谁执行": "**改**：不同 frontier LLM optimizer，在 shared coding harness与native harness两种条件下。
**执行**：固定 target agent 执行 optimizer 提交的 candidate harness。",
"基础 harness": "每个 downstream task的 seed target-agent harness。",
"Feedback": "graded evaluation feedback，固定 target-evaluation budget；test partition完全不可访问。",
"Evolution → Eval": "4 downstream tasks、111 scored runs；search/selection→held-out test。",
"Meta-depth": "Evaluation of M1 optimizer capability。",
"相对之前真正新增什么": "相比方法论文，最大的价值是 **统一预算 + trusted execution boundary + held-out normalized gain**，把 optimizer model capability 与 target harness performance分开测。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录明确标注 #HeldOut；是否参与选模仍需结合协议原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"B-Harness",
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 harness optimization 本身作为 LLM capability：optimizer=LLM+coding harness，给 seed target harness、graded eval feedback、固定 evaluation budget，最后提交一个 candidate在 inaccessible held-out test 上评分。",
"novelty": "用统一预算和不可由修改者改写的执行、评分边界测框架优化；分别记录修改者的设计能力与目标 agent 获得的收益。",
"object": "修改者编辑的目标 agent 运行框架；基准、执行模型和测试边界固定。",
"executor": "按任务固定目标模型：OfficeQA、BrowseComp-Plus 用 DeepSeek-V4-Flash；Terminal-Bench 用 grok-build；GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 GPT-5.4-mini。这些模型运行候选运行框架，不负责外层优化。",
"modifier": "五种优化器：Claude Opus5、Sonnet5、GPT-5.6 Sol、GPT-5.6 Terra、Kimi K3。各自比较统一OpenCode（可连接不同模型的开源 coding agent 工具）与原生Claude Code/Codex/Kimi CLI（通过终端命令使用的程序界面）。",
"roleContext": "**改**：不同 frontier LLM optimizer，在 shared coding harness与native harness两种条件下。
**执行**：固定 target agent 执行 optimizer 提交的 candidate harness。",
"seed": "每题域固定一个刻意未调优的小型 Python task agent。OfficeQA 起点约 130 行、3 个工具、24 轮循环和通用提示；GAIA（需要检索、推理和使用工具的通用助理任务基准） 起点则是不能完成任务的只有接口、尚未实现求解功能的占位程序。这两种起点的零/低分意义不同，不能统一写成一个可工作的成熟运行框架。",
"fixed": "Evaluation of M1 optimizer capability。",
"verdict": "可见开发与验证题的成绩用于改框架和选候选，评估次数受固定预算约束；最终测试分区在修改过程中不可访问，各任务评分器保持固定。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "4 downstream tasks、111 scored runs；search/selection→held-out test。"
}
],
"takeaway": "**search→inaccessible held-out test**；重要发现：optimizer model 差异通常大于 coding-harness 差异。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：GAIA（需要检索、推理和使用工具的通用助理任务基准）、OfficeQA Pro、BrowseComp-Plus、TB2，各按提交的固定 改动清单 取 20% 开发集；不训练目标模型。\n\n调试 / 选版本数据：40% 验证集 用于候选比较和提名；每个任务固定目标模型。\n\n最终测试数据：剩余 40% 测试集，每个条件 3 轮；也用同一分区比较现成 编程 运行框架。\n\n数据隔离与证据边界：20/40/40 是互斥比例，不是题目数量。GAIA（需要检索、推理和使用工具的通用助理任务基准） seed 是不可工作的只有接口、尚未实现求解功能的占位程序，其他任务的初始能力另计。",
"cycle": "优化器编辑 seed 程序，使用开发/验证服务比较不可变 Git 快照并提名版本；最终在隔离沙箱执行，超时按零分计。",
"train": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、OfficeQA Pro、BrowseComp-Plus、TB2，各按提交的固定 改动清单 取 20% 开发集；不训练目标模型。",
"debug": "40% 验证集 用于候选比较和提名；每个任务固定目标模型。",
"test": "剩余 40% 测试集，每个条件 3 轮；也用同一分区比较现成 编程 运行框架。",
"isolation": "20/40/40 是互斥比例，不是题目数量。GAIA（需要检索、推理和使用工具的通用助理任务基准） seed 是不可工作的只有接口、尚未实现求解功能的占位程序，其他任务的初始能力另计。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "每题域固定一个刻意未调优的小型 Python agent。OfficeQA 起点约 130 行、3 个工具、24 轮循环和通用提示；GAIA 起点则是不能完成任务的 stub。这两种起点的零/低分意义不同，不能统一写成一个可工作的成熟 harness。",
"protocol": "**数据来源：**GAIA、OfficeQA Pro、BrowseComp-Plus、Terminal-Bench 2.0。按固定清单分 development/validation/test，比例 20/40/40 且互不重叠；各域绝对数量本轮尚未核实。\n\n**流程：**开发与验证供优化和提名候选，冻结 Git 版本后测 test；每题域固定目标模型，并在相同数据和分区上比较初始程序与现成 coding harness。评估在独立 sandbox 运行，超时按零分计。",
"sections": "§3.2、Table 3；附录 E",
"source": "https://arxiv.org/abs/2608.06301",
"version": "2608.06301v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "87e7dad4b81ae7eb52bdd12fcfb51e31f084f31e7aaac348c2438db47ae5b33e",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "按任务固定目标模型：OfficeQA、BrowseComp-Plus 用 DeepSeek-V4-Flash；Terminal-Bench 用 grok-build；GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 GPT-5.4-mini。这些模型运行候选运行框架，不负责外层优化。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "五种优化器：Claude Opus5、Sonnet5、GPT-5.6 Sol、GPT-5.6 Terra、Kimi K3。各自比较统一OpenCode（可连接不同模型的开源 coding agent 工具）与原生Claude Code/Codex/Kimi CLI（通过终端命令使用的程序界面）。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.06301#S4.SS0.SSS0.Px1"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "修改者编辑的目标 agent 运行框架；基准、执行模型和测试边界固定。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "可见开发与验证题的成绩用于改框架和选候选，评估次数受固定预算约束；最终测试分区在修改过程中不可访问，各任务评分器保持固定。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "每题域固定一个刻意未调优的小型 Python task agent。OfficeQA 起点约 130 行、3 个工具、24 轮循环和通用提示；GAIA（需要检索、推理和使用工具的通用助理任务基准） 起点则是不能完成任务的只有接口、尚未实现求解功能的占位程序。这两种起点的零/低分意义不同，不能统一写成一个可工作的成熟运行框架。",
"sources": [
{
"label": "§3.2、Table 3；附录 E",
"url": "https://arxiv.org/abs/2608.06301"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "优化器编辑 seed 程序，使用开发/验证服务比较不可变 Git 快照并提名版本；最终在隔离沙箱执行，超时按零分计。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、OfficeQA Pro、BrowseComp-Plus、TB2，各按提交的固定 改动清单 取 20% 开发集；不训练目标模型。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "40% 验证集 用于候选比较和提名；每个任务固定目标模型。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "剩余 40% 测试集，每个条件 3 轮；也用同一分区比较现成 编程 运行框架。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "20/40/40 是互斥比例，不是题目数量。GAIA（需要检索、推理和使用工具的通用助理任务基准） seed 是不可工作的只有接口、尚未实现求解功能的占位程序，其他任务的初始能力另计。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用统一预算和不可由修改者改写的执行、评分边界测框架优化；分别记录修改者的设计能力与目标 agent 获得的收益。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.06301v1",
"version": "2608.06301v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相比方法论文，最大的价值是 **统一预算 + trusted execution boundary + held-out normalized gain**，把 optimizer model capability 与 target harness performance分开测。",
"feedbackCases": [
{
"label": "GAIA / BrowseComp-Plus",
"data": "固定清单的 20% 开发集、40% 验证集、40% 测试集；各条件三轮。",
"scoring": "问答输出按题目参考答案的固定评分器评价。",
"visible": "开发反馈和验证成绩用于比较与提名框架；测试集 分区不可访问。",
"use": "在固定目标模型和评估预算内修改框架，选完后才测 测试集。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"judgment": "固定问答评分器对照参考答案；本文未逐项披露规则或模型判分实现"
},
{
"label": "OfficeQA Pro",
"data": "固定清单的 20% 开发集、40% 验证集、40% 测试集；各条件三轮。",
"scoring": "按文档任务的参考答案/规定评分规则评价，固定任务与判分器不由优化者修改。",
"visible": "开发反馈和验证成绩用于比较与提名框架；测试集 分区不可访问。",
"use": "在固定目标模型和评估预算内修改框架，选完后才测 测试集。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"judgment": "固定文档问答评分器；本文未展开其规则／模型比对实现"
},
{
"label": "Terminal-Bench 2",
"data": "固定清单的 20% 开发集、40% 验证集、40% 测试集；各条件三轮。",
"scoring": "运行终端任务验收，检查提交产物是否完成要求。",
"visible": "开发反馈和验证成绩用于比较与提名框架；测试集 分区不可访问。",
"use": "在固定目标模型和评估预算内修改框架，选完后才测 测试集。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"judgment": "Terminal-Bench 原生验收测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、OfficeQA Pro、BrowseComp-Plus、TB2，各按提交的固定 改动清单 取 20% 开发集；不训练目标模型。",
"selection": "40% 验证集 用于候选比较和提名；每个任务固定目标模型。",
"evaluation": "剩余 40% 测试集，每个条件 3 轮；也用同一分区比较现成 编程 运行框架。",
"isolation": "20/40/40 是互斥比例，不是题目数量。GAIA（需要检索、推理和使用工具的通用助理任务基准） seed 是不可工作的只有接口、尚未实现求解功能的占位程序，其他任务的初始能力另计。",
"roles": {
"executor": {
"value": "按任务固定目标模型：OfficeQA、BrowseComp-Plus 用 DeepSeek-V4-Flash；Terminal-Bench 用 grok-build；GAIA（需要检索、推理和使用工具的通用助理任务基准） 用 GPT-5.4-mini。这些模型运行候选运行框架，不负责外层优化。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
]
},
"modifier": {
"value": "五种优化器：Claude Opus5、Sonnet5、GPT-5.6 Sol、GPT-5.6 Terra、Kimi K3。各自比较统一OpenCode（可连接不同模型的开源 coding agent 工具）与原生Claude Code/Codex/Kimi CLI（通过终端命令使用的程序界面）。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.06301#S4.SS0.SSS0.Px1"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
},
"seed": {
"value": "每题域固定一个刻意未调优的小型 Python task agent。OfficeQA 起点约 130 行、3 个工具、24 轮循环和通用提示；GAIA（需要检索、推理和使用工具的通用助理任务基准） 起点则是不能完成任务的只有接口、尚未实现求解功能的占位程序。这两种起点的零/低分意义不同，不能统一写成一个可工作的成熟运行框架。",
"sources": [
{
"label": "§3.2、Table 3；附录 E",
"url": "https://arxiv.org/abs/2608.06301"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"selection": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"evaluation": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
],
"isolation": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06301#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.06301#A5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "真实框架评估往往昂贵又有随机性，优化器不能靠无限试候选取最高分，还得从不完整证据判断改哪里、花多少预算以及交付哪个版本。若不固定执行模型并隔离最终测试，就难区分这种研究能力与模型强弱、过拟合或预算越界，因此需要可信的受控优化评测。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.06301#S1"
}
]
},
{
"key": "position",
"value": "评测模型在评估昂贵且有随机性的条件下，能否有效优化既有运行框架，并区分修改者、运行工具和起点对收益的影响。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.06301"
}
]
},
{
"key": "conclusion",
"value": "修改模型之间差异大于其运行工具之间的差异；原生工具并不总更好，收益也受任务与起点影响。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.06301"
}
]
}
],
"fields": {
"object": "修改者编辑的目标 agent 运行框架；基准、执行模型和测试边界固定。",
"verdict": "可见开发与验证题的成绩用于改框架和选候选，评估次数受固定预算约束；最终测试分区在修改过程中不可访问，各任务评分器保持固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2505.11942",
"title": "LifelongAgentBench",
"url": "https://arxiv.org/abs/2505.11942",
"date": "2025-05-17",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"ExecutableVerifier",
"M0",
"MemoryContent",
"Prequential",
"Skill"
],
"fields": {
"本质定位": "**B-Lifelong**。早期统一测 lifelong agent：DB/OS/KG 中 task interdependence 明确，让后续任务需要复用 earlier skill/knowledge。",
"被测系统 / feedback": "多种 LLM agents；automatic label verification。",
"证据边界与关键结论": "经验 replay 很容易引入 irrelevant info/context burden；说明“存轨迹”不是可靠 lifelong learning。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 202,
"fields": {
"时间": "2025-05-17",
"论文": "[LifelongAgentBench](https://arxiv.org/abs/2505.11942)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong**。早期统一测 lifelong agent：DB/OS/KG 中 task interdependence 明确，让后续任务需要复用 earlier skill/knowledge。",
"被测系统 / feedback": "多种 LLM agents；automatic label verification。",
"证据边界与关键结论": "经验 replay 很容易引入 irrelevant info/context burden；说明“存轨迹”不是可靠 lifelong learning。",
"标签": "`#MemoryContent #Skill #Prequential #ExecutableVerifier #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "早期统一测 lifelong agent：DB/OS/KG 中 task interdependence 明确，让后续任务需要复用 earlier skill/knowledge。",
"novelty": "让数据库、操作系统和知识图谱任务之间有明确技能依赖，测先前经历能否帮助后续任务，并检查多放历史是否反而带来干扰。",
"object": "连续任务间积累和复用的经验；基准衡量这些经验如何影响后续执行。",
"executor": "主对照：Llama3.1-8B-Instruct、Qwen2.5-7B-Instruct、DeepSeek-R1-Distill-Llama-8B、DeepSeek-R1-Distill-Qwen-7B；执行SQL/Bash/知识图谱任务。",
"modifier": "四种被测模型通过各方法的经验保存与回放流程适应任务；评测基准本身不充当修改模型。构造OS命令序列另用DeepSeek-R1，不能与被测模型混淆。",
"roleContext": "多种 LLM agents；automatic label verification。",
"seed": "为数据库、操作系统和知识图谱任务提供交互执行环境，被测方法可回放先前任务经验。任务按可复用的原子技能构造，关注经验如何影响后续执行，不是只有长文本问答。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "环境执行结果及任务完成评分，错误还区分格式、执行、超步数和上下文限制。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "经验 replay 很容易引入 irrelevant info/context burden；说明“存轨迹”不是可靠 lifelong learning。"
}
],
"takeaway": "经验 replay 很容易引入 irrelevant info/context burden；说明“存轨迹”不是可靠 lifelong learning。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：作者自建 DB、OS、KG 三类可执行任务，先前任务交互形成经验。\n\n调试 / 选版本数据：环境返回 SQL/Bash/图谱操作结果或错误；通过回放量与噪声对照观察学习。\n\n最终测试数据：后续任务的完成率及不同经验设置下的曲线。\n\n数据隔离与证据边界：经验允许跨题保留，环境副作用受控清理；后续任务不等同于冻结系统后的独立测试。",
"cycle": "按共享原子技能构造任务序列，前题经历由方法自行保存和回放；比较随经验累积的表现。数据库每题创建/删除任务表，避免状态污染。",
"train": "作者自建 DB、OS、KG 三类可执行任务，先前任务交互形成经验。",
"debug": "环境返回 SQL/Bash/图谱操作结果或错误；通过回放量与噪声对照观察学习。",
"test": "后续任务的完成率及不同经验设置下的曲线。",
"isolation": "经验允许跨题保留，环境副作用受控清理；后续任务不等同于冻结系统后的独立测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "为数据库、操作系统和知识图谱任务提供交互执行环境，被测方法可回放先前任务经验。任务按可复用的原子技能构造，关注经验如何影响后续执行，不是只有长文本问答。",
"protocol": "**数据：**作者自建 Database、Operating System、Knowledge Graph 三类任务，控制技能分布、复杂度与噪声；例如 OS 任务涉及用户、组和文件权限操作。\n\n**进化与测试：**按任务序列积累/回放经验，比较无回放和不同经验量；任务总数与单独开发/最终测试划分本轮未核实。不把同一序列中较晚任务自动当成独立冻结测试集。",
"sections": "数据构造；经验回放实验；附录 A/C",
"source": "https://arxiv.org/abs/2505.11942",
"version": "2505.11942v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "5ecdffe3feecbe523fe1979457031ce4f25a5a3ae85a2466284d599993b1f1eb",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主对照：Llama3.1-8B-Instruct、Qwen2.5-7B-Instruct、DeepSeek-R1-Distill-Llama-8B、DeepSeek-R1-Distill-Qwen-7B；执行SQL/Bash/知识图谱任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.11942#S4.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2505.11942#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "四种被测模型通过各方法的经验保存与回放流程适应任务；评测基准本身不充当修改模型。构造OS命令序列另用DeepSeek-R1，不能与被测模型混淆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.11942#S4.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2505.11942#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "连续任务间积累和复用的经验；基准衡量这些经验如何影响后续执行。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "环境执行结果及任务完成评分，错误还区分格式、执行、超步数和上下文限制。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "为数据库、操作系统和知识图谱任务提供交互执行环境，被测方法可回放先前任务经验。任务按可复用的原子技能构造，关注经验如何影响后续执行，不是只有长文本问答。",
"sources": [
{
"label": "数据构造；经验回放实验；附录 A/C",
"url": "https://arxiv.org/abs/2505.11942"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "按共享原子技能构造任务序列，前题经历由方法自行保存和回放；比较随经验累积的表现。数据库每题创建/删除任务表，避免状态污染。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "作者自建 DB、OS、KG 三类可执行任务，先前任务交互形成经验。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "环境返回 SQL/Bash/图谱操作结果或错误；通过回放量与噪声对照观察学习。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "后续任务的完成率及不同经验设置下的曲线。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "经验允许跨题保留，环境副作用受控清理；后续任务不等同于冻结系统后的独立测试。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让数据库、操作系统和知识图谱任务之间有明确技能依赖，测先前经历能否帮助后续任务，并检查多放历史是否反而带来干扰。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2505.11942v3",
"version": "2505.11942v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "任务之间显式依赖技能，在 DB/OS/KG 环境测经验复用；更多 replay 可能增加无关上下文，因此不能把存得多当成学得好。",
"feedbackCases": [
{
"label": "Database",
"data": "作者按原子技能和难度构造的连续任务；前面任务形成经验，后续题测复用。",
"scoring": "执行 SQL，并按数据库任务目标检查查询或状态。",
"visible": "操作返回值与报错；错误还区分格式、执行、步数和上下文限制。",
"use": "按不同经验回放/噪声设置观察后续任务完成率，不能只报告一个未说明来源的“环境分数”。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"judgment": "执行 SQL 后检查查询结果或数据库目标"
},
{
"label": "Operating System",
"data": "作者按原子技能和难度构造的连续任务；前面任务形成经验，后续题测复用。",
"scoring": "执行 Bash/系统操作，按任务验收条件检查文件或系统状态。",
"visible": "操作返回值与报错；错误还区分格式、执行、步数和上下文限制。",
"use": "按不同经验回放/噪声设置观察后续任务完成率，不能只报告一个未说明来源的“环境分数”。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"judgment": "执行 Bash 后检查文件／系统目标状态"
},
{
"label": "Knowledge Graph",
"data": "作者按原子技能和难度构造的连续任务；前面任务形成经验，后续题测复用。",
"scoring": "执行图谱查询/操作，与任务目标或参考结果比较。",
"visible": "操作返回值与报错；错误还区分格式、执行、步数和上下文限制。",
"use": "按不同经验回放/噪声设置观察后续任务完成率，不能只报告一个未说明来源的“环境分数”。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"judgment": "执行图谱操作并比较任务目标或参考结果"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "作者自建 DB、OS、KG 三类可执行任务，先前任务交互形成经验。",
"selection": "环境返回 SQL/Bash/图谱操作结果或错误；通过回放量与噪声对照观察学习。",
"evaluation": "后续任务的完成率及不同经验设置下的曲线。",
"isolation": "经验允许跨题保留，环境副作用受控清理；后续任务不等同于冻结系统后的独立测试。",
"roles": {
"executor": {
"value": "主对照：Llama3.1-8B-Instruct、Qwen2.5-7B-Instruct、DeepSeek-R1-Distill-Llama-8B、DeepSeek-R1-Distill-Qwen-7B；执行SQL/Bash/知识图谱任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.11942#S4.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2505.11942#S6.SS1"
}
]
},
"modifier": {
"value": "四种被测模型通过各方法的经验保存与回放流程适应任务；评测基准本身不充当修改模型。构造OS命令序列另用DeepSeek-R1，不能与被测模型混淆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2505.11942#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2505.11942#S4.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2505.11942#S6.SS1"
}
]
},
"seed": {
"value": "为数据库、操作系统和知识图谱任务提供交互执行环境，被测方法可回放先前任务经验。任务按可复用的原子技能构造，关注经验如何影响后续执行，不是只有长文本问答。",
"sources": [
{
"label": "数据构造；经验回放实验；附录 A/C",
"url": "https://arxiv.org/abs/2505.11942"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2505.11942#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.11942#S4.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2505.11942#A1.SS1.SSS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有静态 agent 基准把任务彼此隔离，忽略任务依赖、技能复用和旧知识遗忘；因此高分不能说明 agent 会随经验积累而持续适应。作者希望用相互关联的任务序列检验知识是否真正保留并用于以后，而不仅测一次任务执行。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2505.11942#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 在连续、相互依赖的任务中，能否积累并迁移知识和技能，而非每次从零处理独立任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.11942"
}
]
},
{
"key": "conclusion",
"value": "普通经验回放容易带入无关信息并受上下文限制；论文另提出群体一致性方法改善终身学习。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2505.11942"
}
]
}
],
"fields": {
"object": "连续任务间积累和复用的经验；基准衡量这些经验如何影响后续执行。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2507.05257",
"title": "MemoryAgentBench",
"url": "https://arxiv.org/abs/2507.05257",
"date": "2025-07-07",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"M0",
"MemoryMechanism",
"Streaming"
],
"fields": {
"本质定位": "**B-Lifelong**。把 memory 拆成 retrieval、test-time learning、long-range understanding、selective forgetting 四种能力，强调 incremental multi-turn 而非静态 long-context QA。",
"被测系统 / feedback": "context/RAG/external-memory/tool agents。",
"证据边界与关键结论": "不直接测 full self-evolution，但定义了 memory harness 至少应该有哪些能力，尤其 forgetting/update。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 203,
"fields": {
"时间": "2025-07-07",
"论文": "[MemoryAgentBench](https://arxiv.org/abs/2507.05257)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong**。把 memory 拆成 retrieval、test-time learning、long-range understanding、selective forgetting 四种能力，强调 incremental multi-turn 而非静态 long-context QA。",
"被测系统 / feedback": "context/RAG/external-memory/tool agents。",
"证据边界与关键结论": "不直接测 full self-evolution，但定义了 memory harness 至少应该有哪些能力，尤其 forgetting/update。",
"标签": "`#MemoryMechanism #Streaming #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 memory 拆成 retrieval、test-time learning、long-range understanding、selective forgetting 四种能力，强调 incremental multi-turn 而非静态 long-context QA。",
"novelty": "把长材料拆成连续输入，分别检查检索、从新经历学习、长程理解和选择性遗忘，避免把找回片段当成全部记忆能力。",
"object": "逐步积累的记忆内容，及其保留、更新和遗忘状态。",
"executor": "未另指定时，RAG（先检索相关资料，再把资料交给模型回答）及商业记忆 task agent 的回答基础模型统一GPT-4o-mini；长上下文对照与附录算力匹配实验另列模型。",
"modifier": "各记忆系统自己的写入/检索实现；默认配GPT-4o-mini，并非所有系统共享一个被训练的“记忆修改模型”。",
"roleContext": "context/RAG/external-memory/tool agents。",
"seed": "统一按块输入历史材料，再让被测系统用长上下文、RAG（先检索相关资料，再把资料交给模型回答） 或由 task agent 管理、写入和检索的记忆保存并检索。比较的是记忆组件而非统一重写 task agent 源码；分块大小会影响结果，不能略去输入协议。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "分别测准确检索、测试时学习、长程理解与选择性遗忘的任务指标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "不直接测 full self-evolution，但定义了 memory harness 至少应该有哪些能力，尤其 forgetting/update。"
}
],
"takeaway": "不直接测 full self-evolution，但定义了 memory harness 至少应该有哪些能力，尤其 forgetting/update。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：读入材料：文档/LongMemEval/EventQA；BANKING77、CLINC150、TREC、NLU 的带标签示例；小说和电影推荐历史；MQUAKE 构造的事实更新。\n\n调试 / 选版本数据：记忆维护依输入历史；分块预算固定，部分任务 512 词元、其他常用 4096。\n\n最终测试数据：对应的文档/对话问答、分类、推荐、∞-Bench 摘要、Detective QA、FactConsolidation 单跳/多跳问题。\n\n数据隔离与证据边界：输入材料与问题分工，不是模型参数训练/测试；后出现的新事实应覆盖旧事实。",
"cycle": "将历史逐块输入系统，随后提出查询；构造新旧矛盾事实以测试是否能覆盖旧记忆，比较不同记忆架构而非统一训练算法。",
"train": "读入材料：文档/LongMemEval/EventQA；BANKING77、CLINC150、TREC、NLU 的带标签示例；小说和电影推荐历史；MQUAKE 构造的事实更新。",
"debug": "记忆维护依输入历史；分块预算固定，部分任务 512 词元、其他常用 4096。",
"test": "对应的文档/对话问答、分类、推荐、∞-Bench 摘要、Detective QA、FactConsolidation 单跳/多跳问题。",
"isolation": "输入材料与问题分工，不是模型参数训练/测试；后出现的新事实应覆盖旧事实。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "统一按块输入历史材料，再让被测系统用长上下文、RAG 或 agentic memory 保存并检索。比较的是记忆组件而非统一重写 agent 源码；分块大小会影响结果，不能略去输入协议。",
"protocol": "**材料→问题：**检索任务含文档 QA、LongMemEval 重构版（5 段约 355k-token 对话、300 问）和自建 EventQA；测试时学习含 BANKING77、CLINC150、TREC-Coarse/Fine、NLU 分类及电影推荐；长程理解含 ∞-Bench 小说摘要等。\n\n**使用方式：**先分块接收对话、文档或带标签示例，再回答对应问题。部分任务块长 512，其他常用 4096；这是记忆输入与查询分工，不等于参数训练/测试。各来源的完整样本数和独立调参数据本轮待核实。",
"sections": "§3.1–3.3、实验设置",
"source": "https://arxiv.org/abs/2507.05257",
"version": "2507.05257v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "00a584ed785feb3b4aa56f2e22362ea4e980363b1a486ea7061a3626182593ea",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "未另指定时，RAG（先检索相关资料，再把资料交给模型回答）及商业记忆 task agent 的回答基础模型统一GPT-4o-mini；长上下文对照与附录算力匹配实验另列模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2507.05257#A6.SS2"
},
{
"label": "附录J.1",
"url": "https://arxiv.org/html/2507.05257#A10.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各记忆系统自己的写入/检索实现；默认配GPT-4o-mini，并非所有系统共享一个被训练的“记忆修改模型”。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2507.05257#A6.SS2"
},
{
"label": "附录J.1",
"url": "https://arxiv.org/html/2507.05257#A10.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "逐步积累的记忆内容，及其保留、更新和遗忘状态。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "分别测准确检索、测试时学习、长程理解与选择性遗忘的任务指标。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "统一按块输入历史材料，再让被测系统用长上下文、RAG（先检索相关资料，再把资料交给模型回答） 或由 task agent 管理、写入和检索的记忆保存并检索。比较的是记忆组件而非统一重写 task agent 源码；分块大小会影响结果，不能略去输入协议。",
"sources": [
{
"label": "§3.1–3.3、实验设置",
"url": "https://arxiv.org/abs/2507.05257"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "将历史逐块输入系统，随后提出查询；构造新旧矛盾事实以测试是否能覆盖旧记忆，比较不同记忆架构而非统一训练算法。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "读入材料：文档/LongMemEval/EventQA；BANKING77、CLINC150、TREC、NLU 的带标签示例；小说和电影推荐历史；MQUAKE 构造的事实更新。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "记忆维护依输入历史；分块预算固定，部分任务 512 词元、其他常用 4096。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "对应的文档/对话问答、分类、推荐、∞-Bench 摘要、Detective QA、FactConsolidation 单跳/多跳问题。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "输入材料与问题分工，不是模型参数训练/测试；后出现的新事实应覆盖旧事实。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把长材料拆成连续输入，分别检查检索、从新经历学习、长程理解和选择性遗忘，避免把找回片段当成全部记忆能力。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2507.05257v4",
"version": "2507.05257v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "将长上下文资料改成多轮输入，分别测检索、测试时学习、长程理解和选择性遗忘；擅长找片段不等于掌握后面三种能力。",
"feedbackCases": [
{
"label": "记忆输入：没有统一逐步奖励",
"data": "文档、LongMemEval/EventQA、带标签的 BANKING77/CLINC150/TREC/NLU 示例、小说及电影推荐历史、MQUAKE 事实更新。",
"scoring": "按固定块大小把材料送入记忆；带标签分类示例的标签本身就是学习材料，事实更新也作为新输入。",
"visible": "系统可见历史材料，问答测验并非每写一块记忆都返回的纠错信号。",
"use": "维护记忆以备后续测验。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
}
],
"judgment": "输入材料及其中标签是学习信息，无统一逐步正确性奖励"
},
{
"label": "分类 / 检索 / 事实问答",
"data": "上述材料对应的分类、文档/对话问答及 FactConsolidation 单跳/多跳题。",
"scoring": "分类比参考类别，事实题比对应版本的参考答案；LongMemEval 使用参考答案支撑的模型评审。",
"visible": "研究端的测验成绩，不同任务保留原生指标。",
"use": "测准确检索、从示例学习和选择性遗忘旧事实的能力。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
}
],
"judgment": "分类／事实任务比参考标签；LongMemEval 用有参考答案的模型评审"
},
{
"label": "长程理解与推荐",
"data": "∞-Bench 摘要、Detective QA、电影推荐等任务。",
"scoring": "摘要、推理问答与推荐使用各自评估协议，不能与分类准确率混成一种标签奖励。",
"visible": "按对应任务输出质量/正确性汇总。",
"use": "这些用于测长程材料理解；论文没有设定一个统一训练奖励来优化所有记忆方法。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
}
],
"judgment": "各任务原生指标；所引段落未逐任务展开比对器"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1,
2
],
"evolution": "读入材料：文档/LongMemEval/EventQA；BANKING77、CLINC150、TREC、NLU 的带标签示例；小说和电影推荐历史；MQUAKE 构造的事实更新。",
"selection": "记忆维护依输入历史；分块预算固定，部分任务 512 词元、其他常用 4096。",
"evaluation": "对应的文档/对话问答、分类、推荐、∞-Bench 摘要、Detective QA、FactConsolidation 单跳/多跳问题。",
"isolation": "输入材料与问题分工，不是模型参数训练/测试；后出现的新事实应覆盖旧事实。",
"roles": {
"executor": {
"value": "未另指定时，RAG（先检索相关资料，再把资料交给模型回答）及商业记忆 task agent 的回答基础模型统一GPT-4o-mini；长上下文对照与附录算力匹配实验另列模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2507.05257#A6.SS2"
},
{
"label": "附录J.1",
"url": "https://arxiv.org/html/2507.05257#A10.SS1"
}
]
},
"modifier": {
"value": "各记忆系统自己的写入/检索实现；默认配GPT-4o-mini，并非所有系统共享一个被训练的“记忆修改模型”。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2507.05257#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2507.05257#S4.SS1"
},
{
"label": "附录F.2",
"url": "https://arxiv.org/html/2507.05257#A6.SS2"
},
{
"label": "附录J.1",
"url": "https://arxiv.org/html/2507.05257#A10.SS1"
}
]
},
"seed": {
"value": "统一按块输入历史材料，再让被测系统用长上下文、RAG（先检索相关资料，再把资料交给模型回答） 或由 task agent 管理、写入和检索的记忆保存并检索。比较的是记忆组件而非统一重写 task agent 源码；分块大小会影响结果，不能略去输入协议。",
"sources": [
{
"label": "§3.1–3.3、实验设置",
"url": "https://arxiv.org/abs/2507.05257"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2507.05257#S3.SS1.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2507.05257#S3.SS3.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长上下文问答一次给出整段历史，不能直接检验记忆 agent 逐步吸收、压缩和更新信息的过程；已有记忆评测又缺少对学习新规则及纠正旧信息的完整覆盖。这样难系统比较记忆机制，因此作者将检索、部署中学习、长程理解和选择性遗忘分开评测。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2507.05257#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 逐步接收信息时的四种记忆能力：准确检索、从新信息学习、理解长期关联，以及选择性遗忘。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.05257"
}
]
},
{
"key": "conclusion",
"value": "统一比较多种记忆系统，发现尚无方法能同时掌握四类能力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2507.05257"
}
]
}
],
"fields": {
"object": "逐步积累的记忆内容，及其保留、更新和遗忘状态。",
"executor": "未另指定时，RAG（先检索相关资料，再把资料交给模型回答）及商业记忆 task agent 的回答基础模型统一GPT-4o-mini；长上下文对照与补充实验的算力匹配实验另列模型。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2508.19005",
"title": "StuLife / ELL",
"url": "https://arxiv.org/abs/2508.19005",
"date": "2025-08-26",
"priority": "R",
"categories": [
"evaluation"
],
"tags": [
"Continual",
"M0",
"MemoryContent",
"Online",
"Skill"
],
"fields": {
"本质定位": "**B-Lifelong + framework**。学生长期生活模拟：experience exploration、long-term memory、skill learning、knowledge internalization。",
"被测系统 / feedback": "experience-driven lifelong agent；dynamic state。",
"证据边界与关键结论": "范围广但 setting 较特定；价值在把 memory+skill+proactivity 放到长时状态环境。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 204,
"fields": {
"时间": "2025-08-26",
"论文": "[StuLife / ELL](https://arxiv.org/abs/2508.19005)",
"级别": "**R**",
"它真正测什么": "**B-Lifelong + framework**。学生长期生活模拟：experience exploration、long-term memory、skill learning、knowledge internalization。",
"被测系统 / feedback": "experience-driven lifelong agent；dynamic state。",
"证据边界与关键结论": "范围广但 setting 较特定；价值在把 memory+skill+proactivity 放到长时状态环境。",
"标签": "`#MemoryContent #Skill #Continual #Online #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "学生长期生活模拟：experience exploration、long-term memory、skill learning、knowledge internalization。",
"novelty": "用状态持续变化的学生生活模拟承载探索、经验归纳、知识维护和验证，同时提供学习框架与动态环境。",
"object": "校园生活中积累的经验、工作流，或通过强化微调学到的策略。",
"executor": "进化实验：Qwen3-8B用于训练型RFT，Qwen3-235B-A22B用于Reflexion/AWM推理时进化；不能把这两组视为同模型比较。",
"modifier": "RFT从Qwen3-8B自身采样的成功轨迹训练它；Reflexion/AWM分支由Qwen3-235B-A22B生成反思或工作流记忆。",
"roleContext": "experience-driven lifelong agent；dynamic state。",
"seed": "作者自建 StuLife 学生生活模拟器，带持续的时间、资源、任务状态；比较外部记忆等 task agent 配置。ELL 提出探索、长期记忆、技能学习、知识内化四层目标，不表示报告的每个 task agent 都实际更新模型参数。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "任务成功、考试表现、StuGPA 和交互效率。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "范围广但 setting 较特定；价值在把 memory+skill+proactivity 放到长时状态环境。"
}
],
"takeaway": "范围广但 setting 较特定；价值在把 memory+skill+proactivity 放到长时状态环境。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：自建 StuLife 1,284 个实例、10 个关联场景；经验来自模拟学期的活动。\n\n调试 / 选版本数据：校园工具与任务结果；不同学习分支使用各自的经验处理方式。\n\n最终测试数据：三类活动中的任务成功、考试与整体 StuGPA。\n\n数据隔离与证据边界：持续状态和长期事件依赖是评测目的；不能把所有实例当成统一训练完后的一次盲测。",
"cycle": "在持续学期中完成课堂、校园和考试任务；从既有经历反思、归纳工作流或训练，再观察之后的表现。",
"train": "自建 StuLife 1,284 个实例、10 个关联场景；经验来自模拟学期的活动。",
"debug": "校园工具与任务结果；不同学习分支使用各自的经验处理方式。",
"test": "三类活动中的任务成功、考试与整体 StuGPA。",
"isolation": "持续状态和长期事件依赖是评测目的；不能把所有实例当成统一训练完后的一次盲测。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建 StuLife 学生生活模拟器，带持续的时间、资源、任务状态；比较外部记忆等 agent 配置。ELL 提出探索、长期记忆、技能学习、知识内化四层目标，不表示报告的每个 agent 都实际更新模型参数。",
"protocol": "**数据与场景：**StuLife 模拟大学生活三阶段、十个子场景，任务从入学延伸到学业与个人发展；经验来自同一持续生活环境，评估含考试和整体 StuGPA。\n\n**隔离：**这是长期交互、状态积累与后续任务表现，不是固定离线 benchmark 的一次 train/test 划分。完整实例数量与独立迁移场景本轮未核实。",
"sections": "StuLife 场景构造与实验",
"source": "https://arxiv.org/abs/2508.19005",
"version": "2508.19005v6",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "4fd196de9f467a5eaff9f471ce99b6ddc359aa98e260e8ba585e2a9d7d5a8352",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "进化实验：Qwen3-8B用于训练型RFT，Qwen3-235B-A22B用于Reflexion/AWM推理时进化；不能把这两组视为同模型比较。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "RFT从Qwen3-8B自身采样的成功轨迹训练它；Reflexion/AWM分支由Qwen3-235B-A22B生成反思或工作流记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "校园生活中积累的经验、工作流，或通过强化微调学到的策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务成功、考试表现、StuGPA 和交互效率。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建 StuLife 学生生活模拟器，带持续的时间、资源、任务状态；比较外部记忆等 task agent 配置。ELL 提出探索、长期记忆、技能学习、知识内化四层目标，不表示报告的每个 task agent 都实际更新模型参数。",
"sources": [
{
"label": "StuLife 场景构造与实验",
"url": "https://arxiv.org/abs/2508.19005"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在持续学期中完成课堂、校园和考试任务；从既有经历反思、归纳工作流或训练，再观察之后的表现。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "自建 StuLife 1,284 个实例、10 个关联场景；经验来自模拟学期的活动。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "校园工具与任务结果；不同学习分支使用各自的经验处理方式。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "三类活动中的任务成功、考试与整体 StuGPA。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "持续状态和长期事件依赖是评测目的；不能把所有实例当成统一训练完后的一次盲测。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用状态持续变化的学生生活模拟承载探索、经验归纳、知识维护和验证，同时提供学习框架与动态环境。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2508.19005v6",
"version": "2508.19005v6",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "学生生活模拟把探索、抽象、知识维护和验证放进长期动态状态；既提出 ELL 框架也提供环境，不应只当普通静态题库。",
"feedbackCases": [
{
"label": "StuLife 连续校园活动",
"data": "1,284 个实例、十个关联场景，覆盖入学及学业/个人发展等模拟阶段。",
"scoring": "环境任务检查活动是否完成；考试按考试表现评分，整体 StuGPA 汇总学习成果；时间和资源随操作变化。",
"visible": "校园工具输出、任务结果和持续状态，各被测方法自行处理这些经验。",
"use": "学习/测试嵌在连续模拟中；总体指标不代表每个模型都在用该指标更新参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"judgment": "校园仿真环境检查活动与考试结果，再汇总 StuGPA"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "自建 StuLife 1,284 个实例、10 个关联场景；经验来自模拟学期的活动。",
"selection": "校园工具与任务结果；不同学习分支使用各自的经验处理方式。",
"evaluation": "三类活动中的任务成功、考试与整体 StuGPA。",
"isolation": "持续状态和长期事件依赖是评测目的；不能把所有实例当成统一训练完后的一次盲测。",
"roles": {
"executor": {
"value": "进化实验：Qwen3-8B用于训练型RFT，Qwen3-235B-A22B用于Reflexion/AWM推理时进化；不能把这两组视为同模型比较。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
]
},
"modifier": {
"value": "RFT从Qwen3-8B自身采样的成功轨迹训练它；Reflexion/AWM分支由Qwen3-235B-A22B生成反思或工作流记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2508.19005#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2508.19005#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
]
},
"seed": {
"value": "作者自建 StuLife 学生生活模拟器，带持续的时间、资源、任务状态；比较外部记忆等 task agent 配置。ELL 提出探索、长期记忆、技能学习、知识内化四层目标，不表示报告的每个 task agent 都实际更新模型参数。",
"sources": [
{
"label": "StuLife 场景构造与实验",
"url": "https://arxiv.org/abs/2508.19005"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2508.19005#S4.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2508.19005#S5.SS2.SSS1.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有学习方法多假设静态数据、明确任务边界和固定目标，但现实生活的环境与需求不断变化，也没有每一步清楚的标签或终点。即使减少遗忘，仍不等于能主动探索、形成长期目标并迁移技能，因此作者希望研究贯穿生活式交互的持续自主成长。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2508.19005#S1"
}
]
},
{
"key": "position",
"value": "面向动态生活环境中的长期成长，研究 agent 能否把持续经历转化为记忆、可迁移技能和内化后的模型能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2508.19005"
}
]
},
{
"key": "conclusion",
"value": "把多阶段生活经历作为持续学习载体，联合考察外部经验积累和模型能力内化。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2508.19005"
}
]
}
],
"fields": {
"object": "校园生活中积累的经验、工作流，或通过强化微调学到的策略。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2510.17281",
"title": "MemoryBench",
"url": "https://arxiv.org/abs/2510.17281",
"date": "2025-10-20",
"priority": "R",
"categories": [
"evaluation"
],
"tags": [
"M0",
"MemoryContent",
"Online",
"Streaming",
"org:tsinghua"
],
"fields": {
"本质定位": "**B-Lifelong**。重点从“读长文并记住”转到**service-time 用户 feedback accumulation**；多域多语言。",
"被测系统 / feedback": "多种 memory/continual systems；simulated user feedback。",
"证据边界与关键结论": "更像 memory/continual benchmark，不直接验证 harness modifier。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 205,
"fields": {
"时间": "2025-10-20",
"论文": "[MemoryBench](https://arxiv.org/abs/2510.17281)",
"级别": "**R**",
"它真正测什么": "**B-Lifelong**。重点从“读长文并记住”转到**service-time 用户 feedback accumulation**；多域多语言。",
"被测系统 / feedback": "多种 memory/continual systems；simulated user feedback。",
"证据边界与关键结论": "更像 memory/continual benchmark，不直接验证 harness modifier。",
"标签": "`#MemoryContent #Online #Streaming #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "重点从“读长文并记住”转到**service-time 用户 feedback accumulation**；多域多语言。",
"novelty": "把服务过程中的用户反馈作为学习经历，每个分区按学习与测试划分，重点考察反馈积累能否改变后续服务表现。",
"object": "从用户反馈形成的事实知识与操作经验记忆。",
"executor": "主系统基础模型为Qwen3-8B；附录Legal消融另换Mistral-Small3.2-24B。",
"modifier": "记忆方法使用模拟用户反馈更新；用户模拟器主要Qwen3-32B，另有Mistral3.2-24B对照。用户模拟器不是被测Qwen3-8B回答模型。",
"roleContext": "多种 memory/continual systems；simulated user feedback。",
"seed": "语言模型系统外接可替换记忆模块和用户反馈模拟器，比较 BM25（根据查询词与文档词项匹配程度排序的检索算法）、嵌入检索、A-Mem、Mem0、MemoryOS 等；已有静态知识先载入，后续对话逐批写入记忆。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "模拟用户模型提供任务相关反馈；评测按数据集评分。WritingBench使用WritingBench-Critic-Model-Qwen-7B，其他需要模型评审的数据集使用DeepSeek-V3。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "更像 memory/continual benchmark，不直接验证 harness modifier。"
}
],
"takeaway": "更像 memory/continual benchmark，不直接验证 harness modifier。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：11 来源：LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim、LexEval、JuDGE、IdeaBench、LimitGen-Syn、WritingPrompts、HelloBench、WritingBench、NF-Cats、SciTechNews；各分区抽样后按 4:1 划分。\n\n调试 / 选版本数据：仅训练 80% 生成用户反馈日志。\n\n最终测试数据：剩余 20%，按开放域/法律/学术与输入输出格式分区报告。\n\n数据隔离与证据边界：测试题不生成反馈日志；测试曲线反复记录，不应写成每次出现的全是新测试题。",
"cycle": "每步从训练池抽 100 例，最多交互 3 轮后更新记忆，再评估测试池，观察反馈学习曲线。",
"train": "11 来源：LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim、LexEval、JuDGE、IdeaBench、LimitGen-Syn、WritingPrompts、HelloBench、WritingBench、NF-Cats、SciTechNews；各分区抽样后按 4:1 划分。",
"debug": "仅训练 80% 生成用户反馈日志。",
"test": "剩余 20%，按开放域/法律/学术与输入输出格式分区报告。",
"isolation": "测试题不生成反馈日志；测试曲线反复记录，不应写成每次出现的全是新测试题。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "语言模型系统外接可替换记忆模块和用户反馈模拟器，比较 BM25、嵌入检索、A-Mem、Mem0、MemoryOS 等；已有静态知识先载入，后续对话逐批写入记忆。",
"protocol": "**数据：**MemoryBench 汇集开放域、学术、法律等任务，各分区从来源数据集抽取后按 4:1 分训练/测试再合并。原始子数据集清单本轮尚未完整核实。\n\n**在线实验：**每步从训练集抽 100 条，与反馈模拟器最多交互 3 轮，将对话写入记忆，再测完整 test 集并记录曲线。测试集反复用于报告；是否用于决策需另核实，不能仅凭 4:1 声称完全封闭。",
"sections": "数据分区；on-policy 实验流程",
"source": "https://arxiv.org/abs/2510.17281",
"version": "2510.17281v7",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "961de49bb4e5314e5da64db91d2ba10aff6e94aefb253cb5326e63c9237a9b55",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主系统基础模型为Qwen3-8B；附录Legal消融另换Mistral-Small3.2-24B。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2510.17281#A1.SS3.SSS10"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "记忆方法使用模拟用户反馈更新；用户模拟器主要Qwen3-32B，另有Mistral3.2-24B对照。用户模拟器不是被测Qwen3-8B回答模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2510.17281#A1.SS3.SSS10"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "从用户反馈形成的事实知识与操作经验记忆。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "模拟用户模型提供任务相关反馈；评测按数据集评分。WritingBench使用WritingBench-Critic-Model-Qwen-7B，其他需要模型评审的数据集使用DeepSeek-V3。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.17281#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "语言模型系统外接可替换记忆模块和用户反馈模拟器，比较 BM25（根据查询词与文档词项匹配程度排序的检索算法）、嵌入检索、A-Mem、Mem0、MemoryOS 等；已有静态知识先载入，后续对话逐批写入记忆。",
"sources": [
{
"label": "数据分区；on-policy 实验流程",
"url": "https://arxiv.org/abs/2510.17281"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每步从训练池抽 100 例，最多交互 3 轮后更新记忆，再评估测试池，观察反馈学习曲线。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "11 来源：LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim、LexEval、JuDGE、IdeaBench、LimitGen-Syn、WritingPrompts、HelloBench、WritingBench、NF-Cats、SciTechNews；各分区抽样后按 4:1 划分。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "仅训练 80% 生成用户反馈日志。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "剩余 20%，按开放域/法律/学术与输入输出格式分区报告。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "测试题不生成反馈日志；测试曲线反复记录，不应写成每次出现的全是新测试题。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把服务过程中的用户反馈作为学习经历，每个分区按学习与测试划分，重点考察反馈积累能否改变后续服务表现。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2510.17281v7",
"version": "2510.17281v7",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重点是从服务过程的用户反馈积累中学习；每个分区按 4:1 划分学习与测试，而不是只考从长文里找答案。",
"feedbackCases": [
{
"label": "有标准答案的反馈模拟",
"data": "11 来源中的信息提取、事实核对等可验证任务；每分区 80% 用于反馈日志、20% 测试。",
"scoring": "先将回答与数据集标准答案比较计算 F1/准确率，再映射为预定义的反馈模板和行为信号。",
"visible": "模拟用户的文字评价、点赞等显式/隐式行为；并非真实人类逐次标注。",
"use": "形成训练阶段的反馈历史供记忆方法学习，剩余 20% 单独测试。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.17281#A1.SS1"
}
],
"judgment": "规则计算参考答案 F1／准确率，再映射为反馈模板"
},
{
"label": "开放输出的反馈模拟与评价",
"data": "写作、法律、学术等来源，包括 WritingBench、WritingPrompts、HelloBench 等。",
"scoring": "需要语义评审时用模型：WritingBench 使用 WritingBench-Critic-Model-Qwen-7B，其他需要模型评审的数据集使用 DeepSeek-V3。",
"visible": "模型评价通过用户模拟和可编程行为模拟转换为反馈。",
"use": "不同来源的客观答案评分与开放写作评审必须区分，不能都称用户真实反馈。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.17281#A1.SS1"
}
],
"judgment": "WritingBench 用 Qwen-7B critic；其余需模型评审的任务用 DeepSeek-V3"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "11 来源：LoCoMo（检查模型能否利用长期多轮对话记忆的基准）、DialSim、LexEval、JuDGE、IdeaBench、LimitGen-Syn、WritingPrompts、HelloBench、WritingBench、NF-Cats、SciTechNews；各分区抽样后按 4:1 划分。",
"selection": "仅训练 80% 生成用户反馈日志。",
"evaluation": "剩余 20%，按开放域/法律/学术与输入输出格式分区报告。",
"isolation": "测试题不生成反馈日志；测试曲线反复记录，不应写成每次出现的全是新测试题。",
"roles": {
"executor": {
"value": "主系统基础模型为Qwen3-8B；附录Legal消融另换Mistral-Small3.2-24B。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2510.17281#A1.SS3.SSS10"
}
]
},
"modifier": {
"value": "记忆方法使用模拟用户反馈更新；用户模拟器主要Qwen3-32B，另有Mistral3.2-24B对照。用户模拟器不是被测Qwen3-8B回答模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.17281#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2510.17281#S3.SS2"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2510.17281#A1.SS3.SSS10"
}
]
},
"seed": {
"value": "语言模型系统外接可替换记忆模块和用户反馈模拟器，比较 BM25（根据查询词与文档词项匹配程度排序的检索算法）、嵌入检索、A-Mem、Mem0、MemoryOS 等；已有静态知识先载入，后续对话逐批写入记忆。",
"sources": [
{
"label": "数据分区；on-policy 实验流程",
"url": "https://arxiv.org/abs/2510.17281"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2510.17281#S2.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有记忆基准多测试预先存好的信息能否被检索，既缺乏任务多样性，也很少模拟服务过程中的用户反馈。没有成功、失败或偏好反馈，就无法观察经验怎样变成新的做事程序，因此这些静态成绩不足以评价系统在使用中持续学习的能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2510.17281#S1"
}
]
},
{
"key": "position",
"value": "评测持续服务用户的系统能否真正从用户反馈中学习，改善后续回应，而不只是准确记住历史对话。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.17281"
}
]
},
{
"key": "conclusion",
"value": "现有方法在学习效果和效率上仍不足，强调记住输入与用反馈改善服务是不同能力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2510.17281"
}
]
}
],
"fields": {
"object": "从用户反馈形成的事实知识与操作经验记忆。",
"executor": "主系统基础模型为Qwen3-8B；补充实验的Legal消融另换Mistral-Small3.2-24B。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2510.17281"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2511.20857",
"title": "Evo-Memory",
"url": "https://arxiv.org/abs/2511.20857",
"date": "2025-11-25",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"M0",
"MemoryContent",
"Prequential",
"Streaming",
"org:google-deepmind",
"org:uiuc"
],
"fields": {
"本质定位": "**B-Lifelong + baseline framework**。把 10+ memory modules 放到 sequential task streams，要求每次 interaction 后 search/adapt/update memory。",
"被测系统 / feedback": "各 memory module + ExpRAG/ReMem baseline。",
"证据边界与关键结论": "明确把 memory evaluation 变成 **streaming experience reuse**；但 updater机制本身多半固定。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 206,
"fields": {
"时间": "2025-11-25",
"论文": "[Evo-Memory](https://arxiv.org/abs/2511.20857)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong + baseline framework**。把 10+ memory modules 放到 sequential task streams，要求每次 interaction 后 search/adapt/update memory。",
"被测系统 / feedback": "各 memory module + ExpRAG/ReMem baseline。",
"证据边界与关键结论": "明确把 memory evaluation 变成 **streaming experience reuse**；但 updater机制本身多半固定。",
"标签": "`#MemoryContent #Streaming #Prequential #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2025",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 10+ memory modules 放到 sequential task streams，要求每次 interaction 后 search/adapt/update memory。",
"novelty": "把不同记忆方法放进连续任务流，比较检索、适应和更新怎样影响后续任务，并提供可复用的对照方法。",
"object": "连续测试过程中提取、积累和重组的经验记忆。",
"executor": "Gemini2.5 Flash、Flash-Lite、Pro，以及Claude3.5 Haiku、Claude3.7 Sonnet，各自配相应记忆方法。",
"modifier": "各配置的任务模型按方法规定更新记忆；ReMem在Think/Act之外显式执行Refine Memory，主体基础模型仍为上述五种模型之一。",
"roleContext": "各 memory module + ExpRAG/ReMem baseline。",
"seed": "统一经验记忆评估框架，比较直接保留历史、检索记忆、工作流记忆和 ReMem 等；ReMem 能选择提取、使用与删减经验。变化主要在跨题经验内容和组织，不是统一训练模型。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "数学/科学按答案，交互任务按环境成功；经验依各方法纳入。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "明确把 memory evaluation 变成 **streaming experience reuse**；但 updater机制本身多半固定。"
}
],
"takeaway": "明确把 memory evaluation 变成 **streaming experience reuse**；但 updater机制本身多半固定。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：无统一离线训练池，按顺序输入静态基准改造的任务流。\n\n调试 / 选版本数据：此前任务及反馈用于经验更新，ReMem 可在执行中整理记忆。\n\n最终测试数据：MMLU-Pro、GPQA（研究生级科学问答基准） Diamond、AIME24/25、ToolBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）、BabyAI、ScienceWorld、PDDL。\n\n数据隔离与证据边界：测流式适应而非冻结记忆；不同方法共享任务顺序与外层接口。",
"cycle": "统一 search→predict→evolve：先检索并回答当前任务，再更新记忆供后续使用；比较检索、流程记忆和主动重组。",
"train": "无统一离线训练池，按顺序输入静态基准改造的任务流。",
"debug": "此前任务及反馈用于经验更新，ReMem 可在执行中整理记忆。",
"test": "MMLU-Pro、GPQA（研究生级科学问答基准） Diamond、AIME24/25、ToolBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）、BabyAI、ScienceWorld、PDDL。",
"isolation": "测流式适应而非冻结记忆；不同方法共享任务顺序与外层接口。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "统一经验记忆评估框架，比较直接保留历史、检索记忆、工作流记忆和 ReMem 等；ReMem 能选择提取、使用与删减经验。变化主要在跨题经验内容和组织，不是统一训练模型。",
"protocol": "**任务来源：**AIME24/25、GPQA-Diamond、MMLU-Pro 的经济/工程/哲学子域、ToolBench 等，分别测数学、专家推理及 API 使用。\n\n**经验与评估：**在连续任务处理中积累和检索经验，考察后续成绩及记忆删减；不能把整条测试时学习序列叫作冻结模型之外也冻结记忆的独立测试。各任务顺序、样本量和独立选模集本轮尚未核实。",
"sections": "数据集说明与实验主表",
"source": "https://arxiv.org/abs/2511.20857",
"version": "2511.20857v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ab6ef2ed81862ef3d5ccdc59ed08ea78a69e83f68207b366015204e2aef80a31",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Gemini2.5 Flash、Flash-Lite、Pro，以及Claude3.5 Haiku、Claude3.7 Sonnet，各自配相应记忆方法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2511.20857#A1.SS2"
},
{
"label": "附录G.1",
"url": "https://arxiv.org/html/2511.20857#A7.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各配置的任务模型按方法规定更新记忆；ReMem在Think/Act之外显式执行Refine Memory，主体基础模型仍为上述五种模型之一。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2511.20857#A1.SS2"
},
{
"label": "附录G.1",
"url": "https://arxiv.org/html/2511.20857#A7.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "连续测试过程中提取、积累和重组的经验记忆。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "数学/科学按答案，交互任务按环境成功；经验依各方法纳入。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "统一经验记忆评估框架，比较直接保留历史、检索记忆、工作流记忆和 ReMem 等；ReMem 能选择提取、使用与删减经验。变化主要在跨题经验内容和组织，不是统一训练模型。",
"sources": [
{
"label": "数据集说明与实验主表",
"url": "https://arxiv.org/abs/2511.20857"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "统一 search→predict→evolve：先检索并回答当前任务，再更新记忆供后续使用；比较检索、流程记忆和主动重组。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无统一离线训练池，按顺序输入静态基准改造的任务流。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "此前任务及反馈用于经验更新，ReMem 可在执行中整理记忆。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "MMLU-Pro、GPQA（研究生级科学问答基准） Diamond、AIME24/25、ToolBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）、BabyAI、ScienceWorld、PDDL。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "测流式适应而非冻结记忆；不同方法共享任务顺序与外层接口。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把不同记忆方法放进连续任务流，比较检索、适应和更新怎样影响后续任务，并提供可复用的对照方法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2511.20857v2",
"version": "2511.20857v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 memory 模块放进连续任务流比较 search/adapt/update，新增 ExpRAG/ReMem 对照；研究体验复用，而非仅静态对话回忆。",
"feedbackCases": [
{
"label": "知识 / 数学任务流",
"data": "MMLU-Pro、GPQA（研究生级科学问答基准） Diamond、AIME24/25 改造的连续任务。",
"scoring": "科学/综合选择题按参考答案评价；AIME 以数学答案精确匹配评价。",
"visible": "此前题目、回答及按协议可见的反馈用于经验更新。",
"use": "测记忆随任务流演化；无一套统一离线参数训练集。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"judgment": "选择题比参考选项；AIME 用规则精确匹配答案"
},
{
"label": "交互任务流",
"data": "ToolBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）、BabyAI、ScienceWorld、PDDL。",
"scoring": "工具调用、物体操作、实验或规划由各环境的任务目标/成功检查评分，不能用数学题的精确匹配概括。",
"visible": "环境观察和动作结果；ReMem 可在执行过程中整理记忆。",
"use": "考察前面任务经历是否帮助后续任务，最终曲线是序列上的在线表现。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"judgment": "交互环境分别验收工具、操作、实验和规划任务目标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "无统一离线训练池，按顺序输入静态基准改造的任务流。",
"selection": "此前任务及反馈用于经验更新，ReMem 可在执行中整理记忆。",
"evaluation": "MMLU-Pro、GPQA（研究生级科学问答基准） Diamond、AIME24/25、ToolBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）、BabyAI、ScienceWorld、PDDL。",
"isolation": "测流式适应而非冻结记忆；不同方法共享任务顺序与外层接口。",
"roles": {
"executor": {
"value": "Gemini2.5 Flash、Flash-Lite、Pro，以及Claude3.5 Haiku、Claude3.7 Sonnet，各自配相应记忆方法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2511.20857#A1.SS2"
},
{
"label": "附录G.1",
"url": "https://arxiv.org/html/2511.20857#A7.SS1"
}
]
},
"modifier": {
"value": "各配置的任务模型按方法规定更新记忆；ReMem在Think/Act之外显式执行Refine Memory，主体基础模型仍为上述五种模型之一。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2511.20857#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.20857#S3.SS1.SSS0.Px4"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2511.20857#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2511.20857#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2511.20857#A1.SS2"
},
{
"label": "附录G.1",
"url": "https://arxiv.org/html/2511.20857#A7.SS1"
}
]
},
"seed": {
"value": "统一经验记忆评估框架，比较直接保留历史、检索记忆、工作流记忆和 ReMem 等；ReMem 能选择提取、使用与删减经验。变化主要在跨题经验内容和组织，不是统一训练模型。",
"sources": [
{
"label": "数据集说明与实验主表",
"url": "https://arxiv.org/abs/2511.20857"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.20857#S4.SS1.SSS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2511.20857#A1.SS1"
},
{
"label": "附录F.1",
"url": "https://arxiv.org/html/2511.20857#A6.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 可能记住用户说过什么，却没有学会从旧任务中提炼和复用策略，于是跨会话仍反复解决相似问题、重复试错。已有评测偏重事实保留或对话一致性，未系统比较部署中记忆如何检索、整合和修订，因此作者希望直接评价经验积累是否改善后续决策。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2511.20857#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 在连续任务中主动整合、修订和复用经验的能力，关注记忆是否实际支持后续推理和行动。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2511.20857"
}
]
},
{
"key": "conclusion",
"value": "建立持续记忆演化的统一测试框架，使推理、行动与记忆更新可以一起比较。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2511.20857"
}
]
}
],
"fields": {
"object": "连续测试过程中提取、积累和重组的经验记忆。"
}
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2511.20857"
}
]
},
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2511.20857"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2604.17308",
"title": "SkillFlow",
"url": "https://arxiv.org/abs/2604.17308",
"date": "2026-04-19",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"LLMJudge",
"M0",
"Prequential",
"Skill",
"Streaming"
],
"fields": {
"本质定位": "**B-Lifelong / Skill**。166 tasks/20 families，共享 Domain-Agnostic Execution Flow；从 no skills 开始顺序 solve→trajectory/rubric patch→carry forward。",
"被测系统 / feedback": "多个 agent/model；trajectory + rubric feedback。",
"证据边界与关键结论": "Opus +8.43，但弱模型可 regression；**skill usage ≠ skill utility** 是很重要的负面结论。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 207,
"fields": {
"时间": "2026-04-19",
"论文": "[SkillFlow](https://arxiv.org/abs/2604.17308)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / Skill**。166 tasks/20 families，共享 Domain-Agnostic Execution Flow；从 no skills 开始顺序 solve→trajectory/rubric patch→carry forward。",
"被测系统 / feedback": "多个 agent/model；trajectory + rubric feedback。",
"证据边界与关键结论": "Opus +8.43，但弱模型可 regression；**skill usage ≠ skill utility** 是很重要的负面结论。",
"标签": "`#Skill #Prequential #LLMJudge #Streaming #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "166 tasks/20 families，共享 Domain-Agnostic Execution Flow；从 no skills 开始顺序 solve→trajectory/rubric patch→carry forward。",
"novelty": "让同一家族的任务共享可学习的执行流程，观察 agent 从零逐步积累技能，并分别测调用频率与实际任务收益。",
"object": "在同一任务族中不断创建和修订的技能文件。",
"executor": "11种模型：Claude Sonnet4.5/4.6、Opus4.5/4.6、MiniMaxM2.5/M2.7、GPT-5.4、GPT-5.3-Codex、Qwen-Coder-Next、Qwen3-Coder-480B、Kimi K2.5；配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Qwen-Coder或Kimi-CLI。",
"modifier": "同一任务执行模型在完成任务后根据反馈，用统一补丁格式更新技能库；不是另请一个更强模型写技能。",
"roleContext": "多个 agent/model；trajectory + rubric feedback。",
"seed": "任务以 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） 格式运行；task agent 在每个任务族开始时没有技能，完成任务后读取轨迹与评分细则（逐项规定要满足的要求及给分标准）反馈并用显式补丁更新技能库。原始任务环境固定，待积累的是可复用技能。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "每次执行后提供任务判定或逐项评分要求，agent 据此修改技能；实验还检查新技能在后续任务中是否真的被调用。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "Opus +8.43，但弱模型可 regression；**skill usage ≠ skill utility** 是很重要的负面结论。"
}
],
"takeaway": "Opus +8.43，但弱模型可 regression；**skill usage ≠ skill utility** 是很重要的负面结论。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：20 个任务族、166 个实例，来源任务提炼自 GDPval/SkillsBench，族内共享执行流程。\n\n调试 / 选版本数据：已做任务的轨迹和评分细则（逐项规定要满足的要求及给分标准）；技能补丁保留版本历史。\n\n最终测试数据：按难度组织的同族后续任务，比较有/无技能及不同模型。\n\n数据隔离与证据边界：在线顺序学习，没有独立冻结的 训练／测试；同族与跨模型结果分开解释。",
"cycle": "每族从空技能库开始；执行任务、看轨迹与评分细则（逐项规定要满足的要求及给分标准）、输出文件补丁，新技能从后续任务起可用。",
"train": "20 个任务族、166 个实例，来源任务提炼自 GDPval/SkillsBench，族内共享执行流程。",
"debug": "已做任务的轨迹和评分细则（逐项规定要满足的要求及给分标准）；技能补丁保留版本历史。",
"test": "按难度组织的同族后续任务，比较有/无技能及不同模型。",
"isolation": "在线顺序学习，没有独立冻结的 训练／测试；同族与跨模型结果分开解释。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "任务以 Harbor 格式运行；agent 在每个任务族开始时没有技能，完成任务后读取轨迹与 rubric 反馈并用显式补丁更新技能库。原始任务环境固定，待积累的是可复用技能。",
"protocol": "**自建数据：**从 GDPval 和 SkillsBench 的现实任务提取领域无关执行流程，再用生成—审查双 agent 扩写，经人工检查得到 20 个任务族、166 题，每族 8–9 题。\n\n**进化与测试：**族内顺序做题，前题的反馈影响后题技能；这是 lifelong stream，不是独立 train/test。跨模型迁移或同族成绩必须与未见任务泛化分开描述。",
"sections": "benchmark 构造与 lifelong protocol",
"source": "https://arxiv.org/abs/2604.17308",
"version": "2604.17308v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "0dc3e4b02f6ec713bd655fde026d813c635bdc0561300c3c6b65c213b1779bef",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "11种模型：Claude Sonnet4.5/4.6、Opus4.5/4.6、MiniMaxM2.5/M2.7、GPT-5.4、GPT-5.3-Codex、Qwen-Coder-Next、Qwen3-Coder-480B、Kimi K2.5；配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Qwen-Coder或Kimi-CLI。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一任务执行模型在完成任务后根据反馈，用统一补丁格式更新技能库；不是另请一个更强模型写技能。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "在同一任务族中不断创建和修订的技能文件。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "每次执行后提供任务判定或逐项评分要求，agent 据此修改技能；实验还检查新技能在后续任务中是否真的被调用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "任务以 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） 格式运行；task agent 在每个任务族开始时没有技能，完成任务后读取轨迹与评分细则（逐项规定要满足的要求及给分标准）反馈并用显式补丁更新技能库。原始任务环境固定，待积累的是可复用技能。",
"sources": [
{
"label": "benchmark 构造与 lifelong protocol",
"url": "https://arxiv.org/abs/2604.17308"
},
{
"label": "Harbor：任务与验收接口",
"url": "https://www.harborframework.com/docs/tasks"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每族从空技能库开始；执行任务、看轨迹与评分细则（逐项规定要满足的要求及给分标准）、输出文件补丁，新技能从后续任务起可用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "20 个任务族、166 个实例，来源任务提炼自 GDPval/SkillsBench，族内共享执行流程。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "已做任务的轨迹和评分细则（逐项规定要满足的要求及给分标准）；技能补丁保留版本历史。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "按难度组织的同族后续任务，比较有/无技能及不同模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "在线顺序学习，没有独立冻结的 训练／测试；同族与跨模型结果分开解释。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让同一家族的任务共享可学习的执行流程，观察 agent 从零逐步积累技能，并分别测调用频率与实际任务收益。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.17308v1",
"version": "2604.17308v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "同一家族共享执行流程，agent 从零技能顺序积累；task usage 与实际收益分开观察，避免把调用次数当成技能有效性。",
"feedbackCases": [
{
"label": "20 个工作流任务族的顺序学习",
"data": "166 个任务，涵盖表格规划、OCR/PDF、办公文档、合规分析与定量推理；来源从 GDPval/SkillsBench 提炼。",
"scoring": "每族配置对应验收方式；完成一题后，将验收结果转成指出缺失或错误内容的文字评分反馈。",
"visible": "本题执行轨迹及验收器衍生的文字说明，供单次技能补丁生成。",
"use": "从空技能库起步，逐题增改技能供同族更难任务使用；构题阶段 Claude Opus 4.6 的质量审核不是每次任务执行的统一裁判。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"judgment": "各任务族配置验收器，反馈包含分数及错误说明；本文未逐族列出程序／模型实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "20 个任务族、166 个实例，来源任务提炼自 GDPval/SkillsBench，族内共享执行流程。",
"selection": "已做任务的轨迹和评分细则（逐项规定要满足的要求及给分标准）；技能补丁保留版本历史。",
"evaluation": "按难度组织的同族后续任务，比较有/无技能及不同模型。",
"isolation": "在线顺序学习，没有独立冻结的 训练／测试；同族与跨模型结果分开解释。",
"roles": {
"executor": {
"value": "11种模型：Claude Sonnet4.5/4.6、Opus4.5/4.6、MiniMaxM2.5/M2.7、GPT-5.4、GPT-5.3-Codex、Qwen-Coder-Next、Qwen3-Coder-480B、Kimi K2.5；配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Qwen-Coder或Kimi-CLI。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "同一任务执行模型在完成任务后根据反馈，用统一补丁格式更新技能库；不是另请一个更强模型写技能。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2604.17308#A2.SS1"
}
]
},
"seed": {
"value": "任务以 Harbor（运行隔离任务环境、agent 和验收程序的评测工具） 格式运行；task agent 在每个任务族开始时没有技能，完成任务后读取轨迹与评分细则（逐项规定要满足的要求及给分标准）反馈并用显式补丁更新技能库。原始任务环境固定，待积累的是可复用技能。",
"sources": [
{
"label": "benchmark 构造与 lifelong protocol",
"url": "https://arxiv.org/abs/2604.17308"
},
{
"label": "Harbor：任务与验收接口",
"url": "https://www.harborframework.com/docs/tasks"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.17308#S3.SS1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2604.17308#A1.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现成技能能帮助完成任务，但这并不能说明模型会从自身经历发现、修补并维护技能；已有技能基准没有直接回答后一问题。因此，需要从空技能库的连续任务过程检查生成、使用与失败修订，才能判断 agent 是否真的形成可持续复用的能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.17308#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否从零发现并持续维护有用的技能库，区分“会调用现成技能”与“能从经历中学出技能”。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.17308"
}
]
},
{
"key": "conclusion",
"value": "有的模型受益明显，有的反而退化；频繁调用技能并不等于技能有用。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.17308"
}
]
}
],
"fields": {
"object": "在同一任务族中不断创建和修订的技能文件。",
"verdict": "每次执行后提供任务判定或逐项评分要求，agent 据此修改技能；实验还检查新技能在后续任务中是否真的被调用。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2604.20087",
"title": "SkillLearnBench",
"url": "https://arxiv.org/abs/2604.20087",
"date": "2026-04-22",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"ExecutableVerifier",
"LLMJudge",
"M0",
"SelfFeedback",
"Skill",
"org:cmu"
],
"fields": {
"本质定位": "**B-Lifelong / Skill**。20 verified skill-dependent tasks、15 subdomains，同时评 skill quality、trajectory、task outcome。",
"被测系统 / feedback": "one-shot/self-feedback/teacher-feedback/skill-creator methods。",
"证据边界与关键结论": "外部 feedback 多轮能改进，**self-feedback alone 会 recursive drift**；没有方法在所有 task/model 上稳定领先。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 208,
"fields": {
"时间": "2026-04-22",
"论文": "[SkillLearnBench](https://arxiv.org/abs/2604.20087)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / Skill**。20 verified skill-dependent tasks、15 subdomains，同时评 skill quality、trajectory、task outcome。",
"被测系统 / feedback": "one-shot/self-feedback/teacher-feedback/skill-creator methods。",
"证据边界与关键结论": "外部 feedback 多轮能改进，**self-feedback alone 会 recursive drift**；没有方法在所有 task/model 上稳定领先。",
"标签": "`#Skill #SelfFeedback #LLMJudge #ExecutableVerifier #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "20 verified skill-dependent tasks、15 subdomains，同时评 skill quality、trajectory、task outcome。",
"novelty": "用依赖技能复用的任务同时检查技能文件、执行过程与任务结果，并比较自反馈和教师反馈，检验初次成功能否转成后续能力。",
"object": "自动生成的技能文本；另检查执行时是否真正利用了技能。",
"executor": "固定 Claude Sonnet 4.6 读取各模型生成的技能并完成任务，采样温度为 0，以减少输出随机性；每题最多 100 轮，使用容器内任务工具。",
"modifier": "生成技能的六种模型为Claude Haiku4.5、Sonnet4.6、Opus4.6、Gemini3.1 Flash Lite、Gemini3 Flash、Gemini3.1 Pro。比较只生成一次、自反馈、教师反馈与技能创建流程。",
"roleContext": "one-shot/self-feedback/teacher-feedback/skill-creator methods。",
"seed": "运行器把技能生成和使用分开，比较一次生成、自反馈、教师反馈及技能编写器。教师反馈设置有独立学生/教师提示，教师可参考人写技能及失败上下文，学生负责修订；不是所有设置都让学生看到相同监督。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "技能内容和执行过程中的模型评审统一由 GPT-5-mini 完成；最终任务结果另按基准检查。技能作者的自评只是一种学习条件，不等于最终验收。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "外部 feedback 多轮能改进，**self-feedback alone 会 recursive drift**；没有方法在所有 task/model 上稳定领先。"
}
],
"takeaway": "外部 feedback 多轮能改进，**self-feedback alone 会 recursive drift**；没有方法在所有 task/model 上稳定领先。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：20 个技能依赖任务、100 个实例；每任务的一个 seed 实例用于技能生成。\n\n调试 / 选版本数据：seed 实例执行失败的上下文；教师只给修改建议，不能直接发完整解法。\n\n最终测试数据：同任务的变体实例与确定性结果检查器（按测试或判分规则检查任务结果）；平均每任务 10.6 条测试用例。\n\n数据隔离与证据边界：区分任务数量与实例数量；学技能和用技能分开评估，测试通过不等于技能被实际采用。",
"cycle": "每个任务先用一个起始示例生成技能。自反馈设置修订两轮，教师反馈设置修订三轮；保存逐轮技能，再测其实际执行效果。",
"train": "20 个技能依赖任务、100 个实例；每任务的一个 seed 实例用于技能生成。",
"debug": "seed 实例执行失败的上下文；教师只给修改建议，不能直接发完整解法。",
"test": "同任务的变体实例与确定性结果检查器（按测试或判分规则检查任务结果）；平均每任务 10.6 条测试用例。",
"isolation": "区分任务数量与实例数量；学技能和用技能分开评估，测试通过不等于技能被实际采用。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "运行器把技能生成和使用分开，比较一次生成、自反馈、教师反馈及 skill creator。教师反馈设置有独立学生/教师提示，教师可参考人写技能及失败上下文，学生负责修订；不是所有设置都让学生看到相同监督。",
"protocol": "**数据：**20 个经验证的技能依赖任务、15 个子域、共 100 个实例；通过更换输入数据、指令改写等方式生成同任务变体。输出用文件检查、数值比较、结构验证和执行测试判分。\n\n**学习→评估：**从种子实例经历生成/修订技能，评估文本质量、实际执行和额外实例表现。各任务用于生成与最终评估的实例数本轮待核实，不能把 20 个任务等同于 100 个独立留出任务。",
"sections": "§3、Table 1；反馈协议；附录 O",
"source": "https://arxiv.org/abs/2604.20087",
"version": "2604.20087v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ef29fece422b65fc2e6ef25882dcbcdba04df6c60f0ce53952fce4e94170c095",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "固定 Claude Sonnet 4.6 读取各模型生成的技能并完成任务，采样温度为 0，以减少输出随机性；每题最多 100 轮，使用容器内任务工具。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "生成技能的六种模型为Claude Haiku4.5、Sonnet4.6、Opus4.6、Gemini3.1 Flash Lite、Gemini3 Flash、Gemini3.1 Pro。比较只生成一次、自反馈、教师反馈与技能创建流程。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "自动生成的技能文本；另检查执行时是否真正利用了技能。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "技能内容和执行过程中的模型评审统一由 GPT-5-mini 完成；最终任务结果另按基准检查。技能作者的自评只是一种学习条件，不等于最终验收。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2604.20087#A12"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "运行器把技能生成和使用分开，比较一次生成、自反馈、教师反馈及技能编写器。教师反馈设置有独立学生/教师提示，教师可参考人写技能及失败上下文，学生负责修订；不是所有设置都让学生看到相同监督。",
"sources": [
{
"label": "§3、Table 1；反馈协议；附录 O",
"url": "https://arxiv.org/abs/2604.20087"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每个任务先用一个起始示例生成技能。自反馈设置修订两轮，教师反馈设置修订三轮；保存逐轮技能，再测其实际执行效果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "20 个技能依赖任务、100 个实例；每任务的一个 seed 实例用于技能生成。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "seed 实例执行失败的上下文；教师只给修改建议，不能直接发完整解法。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "同任务的变体实例与确定性结果检查器（按测试或判分规则检查任务结果）；平均每任务 10.6 条测试用例。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "区分任务数量与实例数量；学技能和用技能分开评估，测试通过不等于技能被实际采用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用依赖技能复用的任务同时检查技能文件、执行过程与任务结果，并比较自反馈和教师反馈，检验初次成功能否转成后续能力。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.20087v1",
"version": "2604.20087v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "围绕 skill-dependent tasks 同时看技能产物、轨迹和结果，对照自反馈、教师反馈等路线；seed instance 成功不代表后续实例可迁移。",
"feedbackCases": [
{
"label": "技能生成：每任务 seed 实例",
"data": "20 个技能依赖任务、100 个实例，每任务一个 seed 用于生成技能。",
"scoring": "自反馈读 seed 执行失败；教师反馈由独立教师给修改建议、不能直接交完整解法。人写技能可作为教师参考。",
"visible": "执行失败上下文以及该设置允许的教师建议；不同反馈条件不能混写。",
"use": "形成技能后转到同任务的变体实例检验，避免只记住 seed 解法。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2604.20087#A12"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
}
],
"judgment": "执行失败信息用于自反馈；教师模型依据任务和可用参考技能给建议"
},
{
"label": "变体任务与技能质量评测",
"data": "通过修改参数和约束得到的任务变体，参考答案由原正确解脚本重新生成；平均每任务 10.6 条测试。",
"scoring": "可程序验证的任务以确定性检查器对照新参考结果；需要模型评价的 Level 1/2 指标统一 GPT-5-mini。",
"visible": "研究端记录任务结果和技能质量/泛化指标。",
"use": "最终评价不使用技能作者自评代替验收。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2604.20087#A12"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
}
],
"judgment": "可验证任务用确定性检查器；需要模型评价的指标用 GPT-5-mini"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "20 个技能依赖任务、100 个实例；每任务的一个 seed 实例用于技能生成。",
"selection": "seed 实例执行失败的上下文；教师只给修改建议，不能直接发完整解法。",
"evaluation": "同任务的变体实例与确定性结果检查器（按测试或判分规则检查任务结果）；平均每任务 10.6 条测试用例。",
"isolation": "区分任务数量与实例数量；学技能和用技能分开评估，测试通过不等于技能被实际采用。",
"roles": {
"executor": {
"value": "固定 Claude Sonnet 4.6 读取各模型生成的技能并完成任务，采样温度为 0，以减少输出随机性；每题最多 100 轮，使用容器内任务工具。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
]
},
"modifier": {
"value": "生成技能的六种模型为Claude Haiku4.5、Sonnet4.6、Opus4.6、Gemini3.1 Flash Lite、Gemini3 Flash、Gemini3.1 Pro。比较只生成一次、自反馈、教师反馈与技能创建流程。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.20087#S3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.20087#S3.SS2.SSS3"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2604.20087#A13.SS2.SSS0.Px1"
},
{
"label": "附录M.3",
"url": "https://arxiv.org/html/2604.20087#A13.SS3.SSS0.Px1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
]
},
"seed": {
"value": "运行器把技能生成和使用分开，比较一次生成、自反馈、教师反馈及技能编写器。教师反馈设置有独立学生/教师提示，教师可参考人写技能及失败上下文，学生负责修订；不是所有设置都让学生看到相同监督。",
"sources": [
{
"label": "§3、Table 1；反馈协议；附录 O",
"url": "https://arxiv.org/abs/2604.20087"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20087#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20087#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20087#S4.SS1"
},
{
"label": "附录N.2",
"url": "https://arxiv.org/html/2604.20087#A14.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "预先编写的技能难覆盖 agent 不断遇到的新任务，因此需要让 agent 自行学习技能。但现有学习方法在不同设置下评测，只看最终成功率又分不清是技能本身写得不好，还是执行者没有正确使用技能，妨碍方法比较和失败诊断。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.20087#S1"
}
]
},
{
"key": "position",
"value": "评测自动技能学习是否产生真实可用的能力，区分技能内容质量、执行过程质量与最终任务成功。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.20087"
}
]
},
{
"key": "conclusion",
"value": "无方法在所有任务和模型上领先；外部反馈有助迭代，自反馈可能让技能越改越偏。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.20087"
}
]
}
],
"fields": {
"object": "自动生成的技能文本；另检查执行时是否真正利用了技能。",
"verdict": "技能内容和执行过程中的模型评审统一由 GPT-5-mini 完成；最终任务结果另按基准检查。技能作者的自评只是一种学习条件，不等于最终验收。",
"executor": "固定 Claude Sonnet 4.6 读取各模型生成的技能并完成任务，采样温度为 0，以减少输出随机性；每题最多 100 轮，使用容器内任务工具。"
}
},
"attributions": [
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.20087"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2605.18421",
"title": "EvoMemBench",
"url": "https://arxiv.org/abs/2605.18421",
"date": "2026-05-18",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"CrossBenchmark",
"M0",
"MemoryMechanism"
],
"fields": {
"本质定位": "**B-Lifelong / Memory**。按 scope（in/cross-episode）×content（knowledge/execution）系统比较 15 memory methods 与 long-context baseline。",
"被测系统 / feedback": "多类 memory systems。",
"证据边界与关键结论": "long-context 仍非常强；没有 universal memory；程序型 memory 只在 task structure match 时明显有用。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 209,
"fields": {
"时间": "2026-05-18",
"论文": "[EvoMemBench](https://arxiv.org/abs/2605.18421)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / Memory**。按 scope（in/cross-episode）×content（knowledge/execution）系统比较 15 memory methods 与 long-context baseline。",
"被测系统 / feedback": "多类 memory systems。",
"证据边界与关键结论": "long-context 仍非常强；没有 universal memory；程序型 memory 只在 task structure match 时明显有用。",
"标签": "`#MemoryMechanism #CrossBenchmark #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "按 scope（in/cross-episode）×content（knowledge/execution）系统比较 15 memory methods 与 long-context baseline。",
"novelty": "同时区分单次任务内与跨任务的记忆、知识内容与执行经验，并加入长上下文对照，检查专门记忆机制是否有额外价值。",
"object": "单次任务内与跨任务的知识记忆、操作经验。",
"executor": "记忆增强方法统一DeepSeek-V3.2；无记忆对照另含Gemini3 Flash与GPT-5-mini。",
"modifier": "各记忆方法在统一utilize/update接口下工作，记忆增强 task agent 的基础模型均为DeepSeek-V3.2；检索编码器如Qwen3-Emb-4B是另一个组件。",
"roleContext": "多类 memory systems。",
"seed": "统一 task agent—记忆 接口，区分一次任务内记忆与跨任务记忆，以及知识保存与执行状态保存；比较 15 种记忆方法和长上下文对照。框架重构输入依赖关系，不是一个固定技能学习器。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "原评测基准的问答、函数调用与任务完成指标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "long-context 仍非常强；没有 universal memory；程序型 memory 只在 task structure match 时明显有用。"
}
],
"takeaway": "long-context 仍非常强；没有 universal memory；程序型 memory 只在 task structure match 时明显有用。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：材料来自 MemoryAgentBench 检索/遗忘子集、改造 BFCL、多题共享上下文的 CL-Bench；不是本站另一个六域 CL-Bench。\n\n调试 / 选版本数据：BFCL 按原动作顺序拆分并改为隐式指代；共享上下文提供事实与规则。\n\n最终测试数据：任务内知识 2,800；BFCL LongContext 800；跨任务知识 120 上下文/884 题；另有 BFCL Base、xbench、WebWalkerQA 170、ALFWorld（通过文字动作完成家居物体操作的交互环境）。\n\n数据隔离与证据边界：四种更新/重置时机不同；只有跨环境执行迁移明确冻结源记忆后测试目标环境。",
"cycle": "任务内逐块/逐回合更新，结束清空；跨任务完成后更新并复用，同组之外重置；跨环境迁移则先建记忆后冻结。",
"train": "材料来自 MemoryAgentBench 检索/遗忘子集、改造 BFCL、多题共享上下文的 CL-Bench；不是本站另一个六域 CL-Bench。",
"debug": "BFCL 按原动作顺序拆分并改为隐式指代；共享上下文提供事实与规则。",
"test": "任务内知识 2,800；BFCL LongContext 800；跨任务知识 120 上下文/884 题；另有 BFCL Base、xbench、WebWalkerQA 170、ALFWorld（通过文字动作完成家居物体操作的交互环境）。",
"isolation": "四种更新/重置时机不同；只有跨环境执行迁移明确冻结源记忆后测试目标环境。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "统一 agent—memory 接口，区分一次任务内记忆与跨任务记忆，以及知识保存与执行状态保存；比较 15 种记忆方法和长上下文对照。框架重构输入依赖关系，不是一个固定技能学习器。",
"protocol": "**数据来源与改造：**任务内执行集由 BFCL-Multiturn-LongContext 改造：把可由历史推知的实体改为隐式指代，并按原动作顺序拆分依赖操作，生成后人工检查；跨任务知识集来自论文引用的 CL-Bench。\n\n**评估：**在四种记忆设置下比较任务完成，含 16k–128k 上下文预算。其 CL-Bench 需按原文引用确认版本，不能直接套用本站同名条目的数据。其余两个子集的来源、完整数量和划分本轮待核实。",
"sections": "§3；§4.2–4.3、上下文预算实验",
"source": "https://arxiv.org/abs/2605.18421",
"version": "2605.18421v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "02d7ee094b6ea54a26bf8c146dd48eec1a7a6bbe2bc71e9e98c9abea165c2ecf",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "记忆增强方法统一DeepSeek-V3.2；无记忆对照另含Gemini3 Flash与GPT-5-mini。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2605.18421#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各记忆方法在统一utilize/update接口下工作，记忆增强 task agent 的基础模型均为DeepSeek-V3.2；检索编码器如Qwen3-Emb-4B是另一个组件。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2605.18421#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "单次任务内与跨任务的知识记忆、操作经验。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "原评测基准的问答、函数调用与任务完成指标。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "统一 task agent—记忆 接口，区分一次任务内记忆与跨任务记忆，以及知识保存与执行状态保存；比较 15 种记忆方法和长上下文对照。框架重构输入依赖关系，不是一个固定技能学习器。",
"sources": [
{
"label": "§3；§4.2–4.3、上下文预算实验",
"url": "https://arxiv.org/abs/2605.18421"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "任务内逐块/逐回合更新，结束清空；跨任务完成后更新并复用，同组之外重置；跨环境迁移则先建记忆后冻结。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "材料来自 MemoryAgentBench 检索/遗忘子集、改造 BFCL、多题共享上下文的 CL-Bench；不是本站另一个六域 CL-Bench。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "BFCL 按原动作顺序拆分并改为隐式指代；共享上下文提供事实与规则。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "任务内知识 2,800；BFCL LongContext 800；跨任务知识 120 上下文/884 题；另有 BFCL Base、xbench、WebWalkerQA 170、ALFWorld（通过文字动作完成家居物体操作的交互环境）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "四种更新/重置时机不同；只有跨环境执行迁移明确冻结源记忆后测试目标环境。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "同时区分单次任务内与跨任务的记忆、知识内容与执行经验，并加入长上下文对照，检查专门记忆机制是否有额外价值。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.18421v2",
"version": "2605.18421v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用记忆范围（单 episode/跨 episode）与内容（知识/执行）两轴构造测试；强 long-context 对照使 memory 的额外价值可见。",
"feedbackCases": [
{
"label": "知识保存与跨任务知识",
"data": "MemoryAgentBench 的检索/遗忘材料；共享 CL-Bench 上下文，120 个上下文、884 题，另有 2,800 题任务内知识。",
"scoring": "按对应知识任务参考答案/评分细则判回答；共享上下文中的事实、规则才是记忆积累材料。",
"visible": "前面输入的知识和任务经历，不是所有测试答案都作为记忆更新标签。",
"use": "测能否保存并复用同一上下文中的知识；此处 CL-Bench 是上下文学习题库。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"judgment": "依据各知识任务的参考答案／评分细则；本文未逐项明确规则或模型配置"
},
{
"label": "执行状态与工具记忆",
"data": "BFCL LongContext 800，另有 BFCL Base；从原多动作序列拆出隐式指代的多轮任务。",
"scoring": "按原工具任务的动作与状态要求验收，检查后续动作能否找回先前对象/结果。",
"visible": "工具返回和过去操作历史。",
"use": "测跨轮状态保存；改写任务保留原行动语义，不把字符串答案准确率当工具执行成功率。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"judgment": "原工具环境检查动作和状态目标"
},
{
"label": "跨任务交互迁移",
"data": "xbench、WebWalkerQA 170、ALFWorld（通过文字动作完成家居物体操作的交互环境） 等。",
"scoring": "搜索问答按参考答案评价，ALFWorld（通过文字动作完成家居物体操作的交互环境） 按环境操作目标验收。",
"visible": "相应任务的观察与完成结果。",
"use": "比较记忆方法跨任务的表现，与单次问答内的知识保存分开。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"judgment": "搜索问答按参考答案，未展开比对实现；ALFWorld 用环境目标检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "材料来自 MemoryAgentBench 检索/遗忘子集、改造 BFCL、多题共享上下文的 CL-Bench；不是本站另一个六域 CL-Bench。",
"selection": "BFCL 按原动作顺序拆分并改为隐式指代；共享上下文提供事实与规则。",
"evaluation": "任务内知识 2,800；BFCL LongContext 800；跨任务知识 120 上下文/884 题；另有 BFCL Base、xbench、WebWalkerQA 170、ALFWorld（通过文字动作完成家居物体操作的交互环境）。",
"isolation": "四种更新/重置时机不同；只有跨环境执行迁移明确冻结源记忆后测试目标环境。",
"roles": {
"executor": {
"value": "记忆增强方法统一DeepSeek-V3.2；无记忆对照另含Gemini3 Flash与GPT-5-mini。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2605.18421#A3.SS3"
}
]
},
"modifier": {
"value": "各记忆方法在统一utilize/update接口下工作，记忆增强 task agent 的基础模型均为DeepSeek-V3.2；检索编码器如Qwen3-Emb-4B是另一个组件。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px3"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2605.18421#A2.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.18421#S5.SS1.SSS0.Px1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2605.18421#A3.SS3"
}
]
},
"seed": {
"value": "统一 task agent—记忆 接口，区分一次任务内记忆与跨任务记忆，以及知识保存与执行状态保存；比较 15 种记忆方法和长上下文对照。框架重构输入依赖关系，不是一个固定技能学习器。",
"sources": [
{
"label": "§3；§4.2–4.3、上下文预算实验",
"url": "https://arxiv.org/abs/2605.18421"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.18421#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2605.18421#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.18421#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2605.18421#S4.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型本身不会自动保留跨任务经验，持续追加历史又受上下文长度限制。已有记忆评测多侧重文字的保留与检索，或只覆盖部分 agent 场景，无法同时说明记忆能否改善当前任务的行动，以及能否把知识和操作经验带到后续任务。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.18421#S1"
}
]
},
{
"key": "position",
"value": "评测不同内容和保留时长的记忆分别在哪些任务条件下有效，以及其收益能否超过直接使用长上下文。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.18421"
}
]
},
{
"key": "conclusion",
"value": "长上下文仍很有竞争力；记忆收益依赖任务难度、上下文限制和经验与任务的匹配程度。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.18421"
}
]
}
],
"fields": {
"object": "单次任务内与跨任务的知识记忆、操作经验。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2606.05661",
"title": "CL-Bench",
"url": "https://arxiv.org/abs/2606.05661",
"date": "2026-06-04",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"M0",
"MemoryContent",
"Prequential",
"Streaming",
"org:berkeley"
],
"fields": {
"本质定位": "**B-Lifelong / B-Reliability**。6 个 expert-validated stateful domains，共享 latent structure；用 gain metric 隔离 “base capability” 与“从过去经验学到多少”。",
"被测系统 / feedback": "ICL 到 dedicated memory agents。",
"证据边界与关键结论": "naive ICL 反而可超过专门 memory system；非常重要的 counterfactual：复杂 memory harness 未必等于 continual learning。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 210,
"fields": {
"时间": "2026-06-04",
"论文": "[CL-Bench](https://arxiv.org/abs/2606.05661)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / B-Reliability**。6 个 expert-validated stateful domains，共享 latent structure；用 gain metric 隔离 “base capability” 与“从过去经验学到多少”。",
"被测系统 / feedback": "ICL 到 dedicated memory agents。",
"证据边界与关键结论": "naive ICL 反而可超过专门 memory system；非常重要的 counterfactual：复杂 memory harness 未必等于 continual learning。",
"标签": "`#MemoryContent #Prequential #Streaming #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "6 个 expert-validated stateful domains，共享 latent structure；用 gain metric 隔离 “base capability” 与“从过去经验学到多少”。",
"novelty": "让任务共享可以逐渐学会的潜在规律，分别测初始能力和经历后的进步；直接把历史放入上下文是重要对照。",
"object": "顺序任务中对共享规律的适应与保留；具体状态依被测方法而定。",
"executor": "Claude Opus 4.7、Sonnet 4.6、Gemini 3.1 Pro、Gemini 3 Flash、GPT-5.4 分别搭配记忆方法或 ICL（直接把可用经历放进输入上下文）。计算归一化成绩时，以不保留跨任务状态的 GPT-5.4 + ICL 为基准。",
"modifier": "被测模型按各自系统的历史/记忆更新方式适应任务；环境潜在规律和评测程序固定，不存在评测基准替所有方法训练一个统一updater。",
"roleContext": "ICL 到 dedicated memory agents。",
"seed": "六类带持续潜在规律的任务环境，比较不保留历史、普通上下文学习和专用记忆系统。规律包括代码库布局、疾病变化和对手策略；执行接口随任务变化，例如数据库允许探索 SQL 后提交答案。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "各环境的任务分数及有状态相对无状态的增益。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "naive ICL 反而可超过专门 memory system；非常重要的 counterfactual：复杂 memory harness 未必等于 continual learning。"
}
],
"takeaway": "naive ICL 反而可超过专门 memory system；非常重要的 counterfactual：复杂 memory harness 未必等于 continual learning。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：Blind Spectrum Monitoring、Codebase Adaptation、Cohort Studies、Database Exploration、Exploitable Poker、Sales Prediction 的在线交互。\n\n调试 / 选版本数据：环境反馈揭示规律；代码任务来自 SWE-bench 的 tablib/tenacity 时间序列。\n\n最终测试数据：同序列后续实例；Poker 为 120 手牌、五阶段对手策略，其中包含旧策略回归。\n\n数据隔离与证据边界：测在线持续学习；无状态对照用于区分基础能力与历史带来的收益。",
"cycle": "连续处理共享规律的实例，部分环境切换规律后再返回，测首次适应、遗忘和再次适应。",
"train": "Blind Spectrum Monitoring、Codebase Adaptation、Cohort Studies、Database Exploration、Exploitable Poker、Sales Prediction 的在线交互。",
"debug": "环境反馈揭示规律；代码任务来自 SWE-bench 的 tablib/tenacity 时间序列。",
"test": "同序列后续实例；Poker 为 120 手牌、五阶段对手策略，其中包含旧策略回归。",
"isolation": "测在线持续学习；无状态对照用于区分基础能力与历史带来的收益。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "六类带持续潜在规律的任务环境，比较不保留历史、普通上下文学习和专用记忆系统。规律包括代码库布局、疾病变化和对手策略；执行接口随任务变化，例如数据库允许探索 SQL 后提交答案。",
"protocol": "**自建任务：**Blind Spectrum Monitoring、Codebase Adaptation、Cohort Studies、Database Exploration、Exploitable Poker、Sales Prediction，经过领域专家验证。\n\n**学习/评估：**在共享环境规律的顺序任务中观察 online gain，另用无状态对照排除模型本来就会的能力。不是先在别的 benchmark 训练再测试；各任务序列长度、漂移位置和原始数据来源的完整清单本轮待核实。",
"sections": "benchmark 设计；附录 A.1–A.6",
"source": "https://arxiv.org/abs/2606.05661",
"version": "2606.05661v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "389f36e1aa8ab4efc57b8ac33801909f33e7c9e9086bd5c71871131327aa2c20",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Claude Opus 4.7、Sonnet 4.6、Gemini 3.1 Pro、Gemini 3 Flash、GPT-5.4 分别搭配记忆方法或 ICL（直接把可用经历放进输入上下文）。计算归一化成绩时，以不保留跨任务状态的 GPT-5.4 + ICL 为基准。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2606.05661#S4.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05661#S5.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "被测模型按各自系统的历史/记忆更新方式适应任务；环境潜在规律和评测程序固定，不存在评测基准替所有方法训练一个统一updater。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2606.05661#S4.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05661#S5.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "顺序任务中对共享规律的适应与保留；具体状态依被测方法而定。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "各环境的任务分数及有状态相对无状态的增益。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "六类带持续潜在规律的任务环境，比较不保留历史、普通上下文学习和专用记忆系统。规律包括代码库布局、疾病变化和对手策略；执行接口随任务变化，例如数据库允许探索 SQL 后提交答案。",
"sources": [
{
"label": "benchmark 设计；附录 A.1–A.6",
"url": "https://arxiv.org/abs/2606.05661"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "连续处理共享规律的实例，部分环境切换规律后再返回，测首次适应、遗忘和再次适应。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Blind Spectrum Monitoring、Codebase Adaptation、Cohort Studies、Database Exploration、Exploitable Poker、Sales Prediction 的在线交互。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "环境反馈揭示规律；代码任务来自 SWE-bench 的 tablib/tenacity 时间序列。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "同序列后续实例；Poker 为 120 手牌、五阶段对手策略，其中包含旧策略回归。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "测在线持续学习；无状态对照用于区分基础能力与历史带来的收益。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让任务共享可以逐渐学会的潜在规律，分别测初始能力和经历后的进步；直接把历史放入上下文是重要对照。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.05661v1",
"version": "2606.05661v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "让任务共享可学的潜在结构，并区分初始能力与经历后的增益；简单 ICL 是重要对照，不预设复杂记忆系统一定更会学习。",
"feedbackCases": [
{
"label": "Blind Spectrum Monitoring",
"data": "90 次频谱扫描。",
"scoring": "将报告占用频段与真实占用频段比较，算交并比，再对扫描平均。",
"visible": "扫描观测帮助学习信号规律；评分基于真实频段而非模型自评。",
"use": "比较保留经验与每次清空经验的表现。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "规则计算预测频段与真实频段的交并比"
},
{
"label": "Codebase Adaptation",
"data": "SWE-bench 的 tablib/tenacity 按时间形成的 19 个代码实例。",
"scoring": "用代码执行与验收测试检查修复；环境可提供测试失败和错误。",
"visible": "仓库文件、执行结果、测试错误。",
"use": "跨同一代码库积累工作经验，而非在所有实例开始前用完整答案训练。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "可执行代码测试验收修复"
},
{
"label": "Cohort Studies",
"data": "连续队列研究；评分汇总 36 个队列、三个时点的生存概率。",
"scoring": "比较预测生存概率与真实概率的差异，相对整项研究的平坦基线计算分数。",
"visible": "实例之间不返回分数；学习必须来自跨研究信息及内部一致性，不能写成“上一题得分指导下一题”。",
"use": "研究端才用该分数衡量连续学习效果。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "程序比较预测与真实生存概率，按基线归一"
},
{
"label": "Database Exploration",
"data": "40 个混淆字段/格式的 SQLite 问题，中途有数据迁移；每题最多 15 次查询。",
"scoring": "答案错、超时或超预算得零；答对时查询越少分数越高，按正确性与探索查询开销共同计分。",
"visible": "数据库查询结果和报错可见。",
"use": "学习字段语义以少查而正确回答后续问题。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "程序检查答案、预算和超时，并计查询成本"
},
{
"label": "Exploitable Poker",
"data": "120 手固定种子牌局、五阶段对手策略，包含旧策略再次出现。",
"scoring": "环境结算筹码利润，按每手大盲注单位利润平均。",
"visible": "下注行为、摊牌和赢亏；不是给对手策略标准答案。",
"use": "积累对手模型并在回归阶段复用。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "扑克环境规则结算筹码利润"
},
{
"label": "Sales Prediction",
"data": "12 个年度实例，每次预测 75 个产品—地点—年份组合。",
"scoring": "用真实销量算加权绝对百分比误差，得分为 1 减该误差；严重高估可得负分。",
"visible": "上轮综合分与新一年的销售材料可用，工作区可持续保存代码和模型。",
"use": "依据历史预测与分数调整后续策略；与不返回中途分数的 Cohort Studies 不同。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
}
],
"judgment": "程序用真实销量计算加权绝对百分比误差"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4,
5
],
"evolution": "Blind Spectrum Monitoring、Codebase Adaptation、Cohort Studies、Database Exploration、Exploitable Poker、Sales Prediction 的在线交互。",
"selection": "环境反馈揭示规律；代码任务来自 SWE-bench 的 tablib/tenacity 时间序列。",
"evaluation": "同序列后续实例；Poker 为 120 手牌、五阶段对手策略，其中包含旧策略回归。",
"isolation": "测在线持续学习；无状态对照用于区分基础能力与历史带来的收益。",
"roles": {
"executor": {
"value": "Claude Opus 4.7、Sonnet 4.6、Gemini 3.1 Pro、Gemini 3 Flash、GPT-5.4 分别搭配记忆方法或 ICL（直接把可用经历放进输入上下文）。计算归一化成绩时，以不保留跨任务状态的 GPT-5.4 + ICL 为基准。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2606.05661#S4.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05661#S5.SS0.SSS0.Px1"
}
]
},
"modifier": {
"value": "被测模型按各自系统的历史/记忆更新方式适应任务；环境潜在规律和评测程序固定，不存在评测基准替所有方法训练一个统一updater。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2606.05661#A4.SS0.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2606.05661#S4.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05661#S5.SS0.SSS0.Px1"
}
]
},
"seed": {
"value": "六类带持续潜在规律的任务环境，比较不保留历史、普通上下文学习和专用记忆系统。规律包括代码库布局、疾病变化和对手策略；执行接口随任务变化，例如数据库允许探索 SQL 后提交答案。",
"sources": [
{
"label": "benchmark 设计；附录 A.1–A.6",
"url": "https://arxiv.org/abs/2606.05661"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"selection": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"evaluation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
],
"isolation": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.05661#A1.SS2.SSS0.Px3"
},
{
"label": "附录A.5",
"url": "https://arxiv.org/html/2606.05661#A1.SS5.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "高分可能来自模型已经学过相关知识，并不代表它能在陌生环境中边做边学。已有评测未直接检验 agent 能否在一串相关任务中发现环境规律、复用规律，并在规律变化后更新认识，因此难以区分预训练能力与真正的在线学习。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.05661#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 在具有共同规律的连续任务中，能否因经历而提升能力，并把这种学习收益与基础模型本来就强区分开。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.05661"
}
]
},
{
"key": "conclusion",
"value": "专用记忆系统并不一定优于直接保留上下文；过拟合近期观察和经验复用失败仍常见。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.05661"
}
]
}
],
"fields": {
"object": "顺序任务中对共享规律的适应与保留；具体状态依被测方法而定。",
"executor": "Claude Opus 4.7、Sonnet 4.6、Gemini 3.1 Pro、Gemini 3 Flash、GPT-5.4 分别搭配记忆方法或 ICL（直接把可用经历放进输入上下文）。计算归一化成绩时，以不保留跨任务状态的 GPT-5.4 + ICL 为基准。"
}
},
"attributions": [
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.05661"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2607.05202",
"title": "EvoAgentBench",
"url": "https://arxiv.org/abs/2607.05202",
"date": "2026-07-06",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"CrossModel",
"HeldOut",
"M0",
"Skill",
"Workflow"
],
"fields": {
"本质定位": "**B-Lifelong / procedural transfer**。把 trajectory 中的 procedure canonicalize 成 Ability，并构造 Ability Graph，保证 test task 有 train-side procedural support。",
"被测系统 / feedback": "2 scaffolds × 3 backbones；528 train / 267 test。",
"证据边界与关键结论": "curated Ability 可跨 model transfer，但**没有自动 evolution method 在所有 setting 都正 gain**；能定位 encoding/routing/uptake 哪一环失败。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 211,
"fields": {
"时间": "2026-07-06",
"论文": "[EvoAgentBench](https://arxiv.org/abs/2607.05202)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / procedural transfer**。把 trajectory 中的 procedure canonicalize 成 Ability，并构造 Ability Graph，保证 test task 有 train-side procedural support。",
"被测系统 / feedback": "2 scaffolds × 3 backbones；528 train / 267 test。",
"证据边界与关键结论": "curated Ability 可跨 model transfer，但**没有自动 evolution method 在所有 setting 都正 gain**；能定位 encoding/routing/uptake 哪一环失败。",
"标签": "`#Skill #Workflow #HeldOut #CrossModel #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录明确标注 #HeldOut；是否参与选模仍需结合协议原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 trajectory 中的 procedure canonicalize 成 Ability，并构造 Ability Graph，保证 test task 有 train-side procedural support。",
"novelty": "先从轨迹归纳能力单元及其联系，再据此安排有学习支持的测试任务，区分经验没记好、没选对和执行时没用好的失败。",
"object": "从训练经历形成的案例、推理经验、技能或全局提示。",
"executor": "OpenClaw或Nanobot × Qwen3.5-27B、Qwen3.5-397B、Gemma4-31B，共6个执行配置。",
"modifier": "Memento、ReasoningBank、GEPA从训练任务建立进化状态，再交相同任务执行框架–基础模型测试；任务构造轨迹另来自Kimi-K2.5、GLM5.1、DeepSeekV3.2，不能当成最终执行模型。",
"roleContext": "2 scaffolds × 3 backbones；528 train / 267 test。",
"seed": "以 OpenClaw 和 Nanobot 的默认工具收集无技能轨迹，不注入进化产物；再比较学习到的可复用技能。构造时用 Kimi-K2.5、GLM-5.1、DeepSeek-V3.2，每个框架/模型重复运行，单题超时 1800 秒。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "BrowseComp+、GDPval 用各自评分者；SWE/LiveCodeBench 用隐藏执行测试。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "curated Ability 可跨 model transfer，但**没有自动 evolution method 在所有 setting 都正 gain**；能定位 encoding/routing/uptake 哪一环失败。"
}
],
"takeaway": "curated Ability 可跨 model transfer，但**没有自动 evolution method 在所有 setting 都正 gain**；能定位 encoding/routing/uptake 哪一环失败。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：BrowseComp+ 154、SWE-Verified 87、LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 182、GDPval 105，总计 528。\n\n调试 / 选版本数据：训练提示、结果检查器（按测试或判分规则检查任务结果）结果和训练轨迹；按能力图确保测试所需能力在训练中有支持。\n\n最终测试数据：对应 65/56/86/60，总计 267。\n\n数据隔离与证据边界：任务互斥且测试前完成状态构建；Anchor Skill 路由使用额外标签，不能当可部署自动方法。",
"cycle": "先完整处理训练侧证据并冻结产物，再开始测试；Anchor Skill 额外使用策展者能力标签路由，只是诊断参考。",
"train": "BrowseComp+ 154、SWE-Verified 87、LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 182、GDPval 105，总计 528。",
"debug": "训练提示、结果检查器（按测试或判分规则检查任务结果）结果和训练轨迹；按能力图确保测试所需能力在训练中有支持。",
"test": "对应 65/56/86/60，总计 267。",
"isolation": "任务互斥且测试前完成状态构建；Anchor Skill 路由使用额外标签，不能当可部署自动方法。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "以 OpenClaw 和 Nanobot 的默认工具收集无技能轨迹，不注入进化产物；再比较学习到的可复用技能。构造时用 Kimi-K2.5、GLM-5.1、DeepSeek-V3.2，每个框架/模型重复运行，单题超时 1800 秒。",
"protocol": "**训练→测试：**BrowseComp-Plus 154/65；SWE-bench Verified 87/56；LiveCodeBench 182/86；GDPVal 105/60，总计 528/267。\n\n**划分特点：**根据能力图构造训练—测试关系，使测试题所需能力在训练中有支持，而非随机按题号拆分。无技能轨迹用于识别能力，学习方法从训练任务提炼技能，再测留出任务。不要混入其他论文声称的五域版本。",
"sections": "§3.2–3.3；数据构造设置",
"source": "https://arxiv.org/abs/2607.05202",
"version": "2607.05202v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f3af3cc172cf6880e70d29ab1bbc7c4b879d038bffb4766691d6e24177fe542b",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "OpenClaw或Nanobot × Qwen3.5-27B、Qwen3.5-397B、Gemma4-31B，共6个执行配置。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2607.05202#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Memento、ReasoningBank、GEPA从训练任务建立进化状态，再交相同任务执行框架–基础模型测试；任务构造轨迹另来自Kimi-K2.5、GLM5.1、DeepSeekV3.2，不能当成最终执行模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2607.05202#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "从训练经历形成的案例、推理经验、技能或全局提示。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "BrowseComp+、GDPval 用各自评分者；SWE/LiveCodeBench 用隐藏执行测试。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "以 OpenClaw 和 Nanobot 的默认工具收集无技能轨迹，不注入进化产物；再比较学习到的可复用技能。构造时用 Kimi-K2.5、GLM-5.1、DeepSeek-V3.2，每个框架/模型重复运行，单题超时 1800 秒。",
"sources": [
{
"label": "§3.2–3.3；数据构造设置",
"url": "https://arxiv.org/abs/2607.05202"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先完整处理训练侧证据并冻结产物，再开始测试；Anchor Skill 额外使用策展者能力标签路由，只是诊断参考。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "BrowseComp+ 154、SWE-Verified 87、LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 182、GDPval 105，总计 528。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练提示、结果检查器（按测试或判分规则检查任务结果）结果和训练轨迹；按能力图确保测试所需能力在训练中有支持。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "对应 65/56/86/60，总计 267。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "任务互斥且测试前完成状态构建；Anchor Skill 路由使用额外标签，不能当可部署自动方法。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先从轨迹归纳能力单元及其联系，再据此安排有学习支持的测试任务，区分经验没记好、没选对和执行时没用好的失败。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.05202v1",
"version": "2607.05202v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "先从轨迹抽取标准化 Ability，再用 Ability Graph 设计有学习支持的测试任务；能把技能编码、路由、执行利用的失败分开定位。",
"feedbackCases": [
{
"label": "BrowseComp+",
"data": "训练 154、测试 65",
"scoring": "使用官方评估提示作模型评审。",
"visible": "训练提示和判分结果可见；测试答案、成功测试轨迹和测试专属捷径均排除。",
"use": "先用训练集完成记忆/技能或提示的构建，再进行测试；能力图保证测试能力在训练中有对应支持，不等于看过测试答案。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"judgment": "LLM 使用 BrowseComp+ 官方提示，对照标准答案判语义正确性"
},
{
"label": "GDPval",
"data": "训练 105、测试 60",
"scoring": "模型参考标准产物/参考内容作评价。",
"visible": "训练提示和判分结果可见；测试答案、成功测试轨迹和测试专属捷径均排除。",
"use": "先用训练集完成记忆/技能或提示的构建，再进行测试；能力图保证测试能力在训练中有对应支持，不等于看过测试答案。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"judgment": "模型对照标准产物／内容评 GDPval 结果"
},
{
"label": "SWE-bench Verified",
"data": "训练 87、测试 56",
"scoring": "隐藏仓库测试验收补丁，报解决率。",
"visible": "训练提示和判分结果可见；测试答案、成功测试轨迹和测试专属捷径均排除。",
"use": "先用训练集完成记忆/技能或提示的构建，再进行测试；能力图保证测试能力在训练中有对应支持，不等于看过测试答案。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"judgment": "SWE-bench 隐藏仓库测试"
},
{
"label": "LiveCodeBench",
"data": "训练 182、测试 86",
"scoring": "隐藏代码测试验收程序，报 pass@1（单次尝试完成任务的比例）。",
"visible": "训练提示和判分结果可见；测试答案、成功测试轨迹和测试专属捷径均排除。",
"use": "先用训练集完成记忆/技能或提示的构建，再进行测试；能力图保证测试能力在训练中有对应支持，不等于看过测试答案。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
}
],
"judgment": "LiveCodeBench 隐藏代码测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "BrowseComp+ 154、SWE-Verified 87、LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 182、GDPval 105，总计 528。",
"selection": "训练提示、结果检查器（按测试或判分规则检查任务结果）结果和训练轨迹；按能力图确保测试所需能力在训练中有支持。",
"evaluation": "对应 65/56/86/60，总计 267。",
"isolation": "任务互斥且测试前完成状态构建；Anchor Skill 路由使用额外标签，不能当可部署自动方法。",
"roles": {
"executor": {
"value": "OpenClaw或Nanobot × Qwen3.5-27B、Qwen3.5-397B、Gemma4-31B，共6个执行配置。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2607.05202#A1.SS2"
}
]
},
"modifier": {
"value": "Memento、ReasoningBank、GEPA从训练任务建立进化状态，再交相同任务执行框架–基础模型测试；任务构造轨迹另来自Kimi-K2.5、GLM5.1、DeepSeekV3.2，不能当成最终执行模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2607.05202#A5.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2607.05202#A1.SS2"
}
]
},
"seed": {
"value": "以 OpenClaw 和 Nanobot 的默认工具收集无技能轨迹，不注入进化产物；再比较学习到的可复用技能。构造时用 Kimi-K2.5、GLM-5.1、DeepSeek-V3.2，每个框架/模型重复运行，单题超时 1800 秒。",
"sources": [
{
"label": "§3.2–3.3；数据构造设置",
"url": "https://arxiv.org/abs/2607.05202"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05202#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.05202#S4.SS1.SSS0.Px3"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2607.05202#A1.SS6"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "任务基准主要测能否解题，记忆基准主要测能否保留信息；技能流评测又常把技能质量与任务顺序、检索方式和运行框架混在一起。这样即使成绩提高，也难判断是否学到了可迁移的操作方法，而非记住具体题目或受益于其他组件。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.05202#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否从既往任务学会搜索、调试等可复用操作方法，并将其迁移到需要这些能力的新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05202"
}
]
},
{
"key": "conclusion",
"value": "人工整理的能力可靠迁移，但自动方法不能在所有配置中持续获益；可细分经验编码、检索和利用问题。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.05202"
}
]
}
],
"fields": {
"object": "从训练经历形成的案例、推理经验、技能或全局提示。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.00155",
"title": "AgentStream",
"url": "https://arxiv.org/abs/2608.00155",
"date": "2026-07-31",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"CrossBenchmark",
"Prequential",
"Streaming"
],
"fields": {
"本质定位": "**B-Lifelong / B-Reliability**。把 self-evolution methods 放到 Isolated→Sequential→Interleaved streaming task composition，研究 domain mixing 后是否还 work。",
"被测系统 / feedback": "5 evolution methods × 3 frontier models。",
"证据边界与关键结论": "reliability 随 scenario 大变；benefit 被 model capability gate，且不随 model strength 单调；无方法全胜。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 212,
"fields": {
"时间": "2026-07-31",
"论文": "[AgentStream](https://arxiv.org/abs/2608.00155)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / B-Reliability**。把 self-evolution methods 放到 Isolated→Sequential→Interleaved streaming task composition，研究 domain mixing 后是否还 work。",
"被测系统 / feedback": "5 evolution methods × 3 frontier models。",
"证据边界与关键结论": "reliability 随 scenario 大变；benefit 被 model capability gate，且不随 model strength 单调；无方法全胜。",
"标签": "`#Streaming #Prequential #CrossBenchmark #B-Reliability`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-Lifelong",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "把 self-evolution methods 放到 Isolated→Sequential→Interleaved streaming task composition，研究 domain mixing 后是否还 work。",
"novelty": "让同一方法分别经历独立、顺序和交错的任务流，比较经验范围与跨领域混合的影响，检查收益是否依赖任务排列。",
"object": "跨领域任务流中持续保留的记忆与技能状态。",
"executor": "GPT-5.4-medium、Gemini3.1 Pro-medium、Claude Opus4.7-high，通过Exgentic任务接口执行。",
"modifier": "对应执行基础模型运行A-Mem、ACE、ReasoningBank、AutoSkill或Harness的更新过程；它们分别积累不同形式的状态，主模型不跨任务替换。",
"roleContext": "5 evolution methods × 3 frontier models。",
"seed": "在 Exgentic 中统一接入不同 task agent 和评测基准，嵌入统一用 all-MiniLM-L6-v2，所需评审和用户模拟器统一 GPT-5.4；各记忆/进化方法在这一公共协议下运行。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "原生评测基准评分；所需评分者和用户模拟器统一 GPT-5.4。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "reliability 随 scenario 大变；benefit 被 model capability gate，且不随 model strength 单调；无方法全胜。"
}
],
"takeaway": "reliability 随 scenario 大变；benefit 被 model capability gate，且不随 model strength 单调；无方法全胜。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：没有单独离线训练集，当前流的已完成任务形成经验。\n\n调试 / 选版本数据：每种评测基准的执行反馈及已完成轨迹。\n\n最终测试数据：AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-challenge、BFCL multi-turn base、BrowseComp+、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、SWE、τ² telecom 各 50，共 300 题。\n\n数据隔离与证据边界：这些原始 测试集 分区用于在线适应评估，不是先学完再冻结；三种流式场景保持域内暴露顺序一致。",
"cycle": "在固定任务集合上比较不同流式顺序，前题经历可更新状态；三随机种子只改顺序，不改题目集合。",
"train": "没有单独离线训练集，当前流的已完成任务形成经验。",
"debug": "每种评测基准的执行反馈及已完成轨迹。",
"test": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-challenge、BFCL multi-turn base、BrowseComp+、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、SWE、τ² telecom 各 50，共 300 题。",
"isolation": "这些原始 测试集 分区用于在线适应评估，不是先学完再冻结；三种流式场景保持域内暴露顺序一致。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在 Exgentic 中统一接入不同 agent 和 benchmark，嵌入统一用 all-MiniLM-L6-v2，所需评审和用户模拟器统一 GPT-5.4；各记忆/进化方法在这一公共协议下运行。",
"protocol": "**任务流：**AppWorld test-challenge、BFCL multi-turn base、BrowseComp+、HLE、SWE、τ² telecom 各抽 50 题。固定任务集合，三个随机种子打乱顺序，比较三种流式场景。\n\n**进化与测试：**经验在测试时任务流中持续积累；顺序场景依次为 AppWorld→BFCL→BrowseComp+→HLE→SWE→τ²。这些原始 split 虽叫 test，也在在线适应过程中使用，不能描述成先完成所有进化后再冻结测试。",
"sections": "Implementation Details 与流式评估协议",
"source": "https://arxiv.org/abs/2608.00155",
"version": "2608.00155v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c3b50a12c7c3f0cd1036bda9be636c668fe602730c1a274d9b2a3f2c3fab90f7",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5.4-medium、Gemini3.1 Pro-medium、Claude Opus4.7-high，通过Exgentic任务接口执行。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.00155#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应执行基础模型运行A-Mem、ACE、ReasoningBank、AutoSkill或Harness的更新过程；它们分别积累不同形式的状态，主模型不跨任务替换。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.00155#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨领域任务流中持续保留的记忆与技能状态。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "原生评测基准评分；所需评分者和用户模拟器统一 GPT-5.4。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在 Exgentic 中统一接入不同 task agent 和评测基准，嵌入统一用 all-MiniLM-L6-v2，所需评审和用户模拟器统一 GPT-5.4；各记忆/进化方法在这一公共协议下运行。",
"sources": [
{
"label": "Implementation Details 与流式评估协议",
"url": "https://arxiv.org/abs/2608.00155"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在固定任务集合上比较不同流式顺序，前题经历可更新状态；三随机种子只改顺序，不改题目集合。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有单独离线训练集，当前流的已完成任务形成经验。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "每种评测基准的执行反馈及已完成轨迹。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-challenge、BFCL multi-turn base、BrowseComp+、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、SWE、τ² telecom 各 50，共 300 题。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这些原始 测试集 分区用于在线适应评估，不是先学完再冻结；三种流式场景保持域内暴露顺序一致。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让同一方法分别经历独立、顺序和交错的任务流，比较经验范围与跨领域混合的影响，检查收益是否依赖任务排列。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.00155v1",
"version": "2608.00155v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "同一方法分别经历 isolated、sequential、interleaved 任务流，改变经验的范围与跨域混合；检验收益是否依赖任务排列方式。",
"feedbackCases": [
{
"label": "AppWorld / BFCL / τ² telecom",
"data": "每来源 50 题，共 300；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 为 test-challenge，BFCL 为 multi-turn base。",
"scoring": "分别按应用状态目标、函数调用结果/状态、客服任务完成规则验收。",
"visible": "已完成任务的经历按被测方法更新经验；需要用户模拟时也统一 GPT-5.4。用户模拟不是所有任务的评分程序。",
"use": "三种随机顺序评估在线任务流，前题经验供后题使用，没有独立离线训练集。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"judgment": "AppWorld／BFCL／τ² 各自原生状态与动作验收"
},
{
"label": "BrowseComp+ / HLE",
"data": "每来源 50 题，共 300；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 为 test-challenge，BFCL 为 multi-turn base。",
"scoring": "按源问答评估流程判断回答；需要模型评审时统一 GPT-5.4。",
"visible": "已完成任务的经历按被测方法更新经验；需要用户模拟时也统一 GPT-5.4。用户模拟不是所有任务的评分程序。",
"use": "三种随机顺序评估在线任务流，前题经验供后题使用，没有独立离线训练集。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"judgment": "GPT-5.4 按源问答评估协议对照参考答案判分"
},
{
"label": "SWE",
"data": "每来源 50 题，共 300；AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 为 test-challenge，BFCL 为 multi-turn base。",
"scoring": "仓库测试验收代码补丁。",
"visible": "已完成任务的经历按被测方法更新经验；需要用户模拟时也统一 GPT-5.4。用户模拟不是所有任务的评分程序。",
"use": "三种随机顺序评估在线任务流，前题经验供后题使用，没有独立离线训练集。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
}
],
"judgment": "SWE 仓库测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "没有单独离线训练集，当前流的已完成任务形成经验。",
"selection": "每种评测基准的执行反馈及已完成轨迹。",
"evaluation": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-challenge、BFCL multi-turn base、BrowseComp+、HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）、SWE、τ² telecom 各 50，共 300 题。",
"isolation": "这些原始 测试集 分区用于在线适应评估，不是先学完再冻结；三种流式场景保持域内暴露顺序一致。",
"roles": {
"executor": {
"value": "GPT-5.4-medium、Gemini3.1 Pro-medium、Claude Opus4.7-high，通过Exgentic任务接口执行。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.00155#A2.SS1"
}
]
},
"modifier": {
"value": "对应执行基础模型运行A-Mem、ACE、ReasoningBank、AutoSkill或Harness的更新过程；它们分别积累不同形式的状态，主模型不跨任务替换。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.00155#S5.SS3.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.00155#A2.SS1"
}
]
},
"seed": {
"value": "在 Exgentic 中统一接入不同 task agent 和评测基准，嵌入统一用 all-MiniLM-L6-v2，所需评审和用户模拟器统一 GPT-5.4；各记忆/进化方法在这一公共协议下运行。",
"sources": [
{
"label": "Implementation Details 与流式评估协议",
"url": "https://arxiv.org/abs/2608.00155"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2608.00155#S4.SS0.SSS0.Px4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "独立任务的平均成功率不能说明 agent 是否会越用越好。实际任务会连续、交错地到来，也不总有清楚的任务边界和监督信号；现有评测不足以判断经验积累在这种混合任务流中是否仍然有效，以及结果有多依赖任务流的安排。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.00155#S1"
}
]
},
{
"key": "position",
"value": "评测自进化 agent 在同域连续任务与跨域混合任务流中的可靠性，考察改进方法是否依赖特定模型或任务顺序。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.00155"
}
]
},
{
"key": "conclusion",
"value": "没有一种方法全面领先；可靠性随任务流和模型变化，模型更强也不总意味着进化收益更大。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.00155"
}
]
}
],
"fields": {
"object": "跨领域任务流中持续保留的记忆与技能状态。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.01149",
"title": "PATH-Bench",
"url": "https://arxiv.org/abs/2608.01149",
"date": "2026-08-02",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"M0",
"MemoryContent",
"RegressionGate",
"Skill",
"Streaming"
],
"fields": {
"本质定位": "**B-Lifelong / B-Reliability**。显式构造 helpful/interfering histories，重复 probe task 测 forward transfer、backward transfer、forgetting；研究 experience **path dependence**。",
"被测系统 / feedback": "8 agents；single-turn code + multi-turn tool-use。",
"证据边界与关键结论": "强 forward transfer 不保证 retention；later experience 会重塑 earlier gains。它直接支持我们关注的“系统性副作用/干扰”。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 213,
"fields": {
"时间": "2026-08-02",
"论文": "[PATH-Bench](https://arxiv.org/abs/2608.01149)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / B-Reliability**。显式构造 helpful/interfering histories，重复 probe task 测 forward transfer、backward transfer、forgetting；研究 experience **path dependence**。",
"被测系统 / feedback": "8 agents；single-turn code + multi-turn tool-use。",
"证据边界与关键结论": "强 forward transfer 不保证 retention；later experience 会重塑 earlier gains。它直接支持我们关注的“系统性副作用/干扰”。",
"标签": "`#MemoryContent #Skill #Streaming #RegressionGate #M0`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"B-Lifelong",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "显式构造 helpful/interfering histories，重复 probe task 测 forward transfer、backward transfer、forgetting；研究 experience **path dependence**。",
"novelty": "有意混入有帮助和有干扰的历史，并重复固定检查任务，同时测新经验的帮助与旧能力的遗忘。",
"object": "随学习路径积累的记忆与技能；基准测历史顺序对迁移和遗忘的影响。",
"executor": "所有被评测 agent 统一使用 DeepSeek-V4-Flash 推理。构造任务间迁移关系时另用 DeepSeek-V4-Flash、GPT-5.4-mini、GLM-5.0 分别测量，再多数投票；这三者不是主实验中三套执行配置。",
"modifier": "各方法自身更新状态；PATH-Bench 控制任务顺序和探针出现位置。",
"roleContext": "8 agents；single-turn code + multi-turn tool-use。",
"seed": "通过可控任务顺序比较记忆/技能 task agent；作者的 SEU 额外判断哪些历史经验有帮助、哪些可能干扰当前任务，再决定是否使用。并非任意扩大记忆库就直接注入所有历史。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "BigCodeBench 完成率、WildToolBench 子任务完成比例，结合前向/后向迁移与遗忘。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "强 forward transfer 不保证 retention；later experience 会重塑 earlier gains。它直接支持我们关注的“系统性副作用/干扰”。"
}
],
"takeaway": "强 forward transfer 不保证 retention；later experience 会重塑 earlier gains。它直接支持我们关注的“系统性副作用/干扰”。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：BigCodeBench 与 WildToolBench 各 120 个任务组成受控历史。\n\n调试 / 选版本数据：任务执行经历形成状态；构造时每个任务对用多模型、五种子估计迁移。\n\n最终测试数据：相同任务池中的后续任务与重复 探测任务；动机实验另用 LifelongAgentBench 50 题。\n\n数据隔离与证据边界：探针重现是设计要素；不是互斥训练/测试题泛化实验。",
"cycle": "先估计哪些历史任务会帮助或干扰目标任务，构造相应任务序列；在多个位置重复插入同一个探测任务，观察经验积累后的能力变化。",
"train": "BigCodeBench 与 WildToolBench 各 120 个任务组成受控历史。",
"debug": "任务执行经历形成状态；构造时每个任务对用多模型、五种子估计迁移。",
"test": "相同任务池中的后续任务与重复 探测任务；动机实验另用 LifelongAgentBench 50 题。",
"isolation": "探针重现是设计要素；不是互斥训练/测试题泛化实验。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "通过可控任务顺序比较记忆/技能 agent；作者的 SEU 额外判断哪些历史经验有帮助、哪些可能干扰当前任务，再决定是否使用。并非任意扩大记忆库就直接注入所有历史。",
"protocol": "**数据：**BigCodeBench 单轮代码生成与 WildToolBench 多轮工具使用；工具部分有 120 个场景、380 种工具。\n\n**协议：**根据多模型上下文迁移估计任务间有向关系，构造有帮助或有干扰的历史，并反复插入 probe 题，测前向迁移、后向迁移和遗忘。探针重复出现是设计的一部分，不是未见测试划分；具体序列数量本轮待核实。",
"sections": "PATH 构造、数据统计与评估协议",
"source": "https://arxiv.org/abs/2608.01149",
"version": "2608.01149v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a72a643d9ed07751764c148d1507c253cb97e8cb71fdb55e3e755cb22d7ab15e",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "所有被评测 agent 统一使用 DeepSeek-V4-Flash 推理。构造任务间迁移关系时另用 DeepSeek-V4-Flash、GPT-5.4-mini、GLM-5.0 分别测量，再多数投票；这三者不是主实验中三套执行配置。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.01149#S4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各方法自身更新状态；PATH-Bench 控制任务顺序和探针出现位置。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "随学习路径积累的记忆与技能；基准测历史顺序对迁移和遗忘的影响。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "BigCodeBench 完成率、WildToolBench 子任务完成比例，结合前向/后向迁移与遗忘。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "通过可控任务顺序比较记忆/技能 task agent；作者的 SEU 额外判断哪些历史经验有帮助、哪些可能干扰当前任务，再决定是否使用。并非任意扩大记忆库就直接注入所有历史。",
"sources": [
{
"label": "PATH 构造、数据统计与评估协议",
"url": "https://arxiv.org/abs/2608.01149"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先估计哪些历史任务会帮助或干扰目标任务，构造相应任务序列；在多个位置重复插入同一个探测任务，观察经验积累后的能力变化。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "BigCodeBench 与 WildToolBench 各 120 个任务组成受控历史。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "任务执行经历形成状态；构造时每个任务对用多模型、五种子估计迁移。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "相同任务池中的后续任务与重复 探测任务；动机实验另用 LifelongAgentBench 50 题。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "探针重现是设计要素；不是互斥训练/测试题泛化实验。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "有意混入有帮助和有干扰的历史，并重复固定检查任务，同时测新经验的帮助与旧能力的遗忘。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.01149v1",
"version": "2608.01149v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "刻意加入有帮助和有干扰的历史，并重复 probe 任务；同时看 forward transfer 与遗忘，揭示后来经验可能破坏先前收益。",
"feedbackCases": [
{
"label": "BigCodeBench / WildToolBench",
"data": "各 120 个任务，构造有帮助或有干扰的历史顺序，并反复插入探测任务。",
"scoring": "BigCodeBench 通过代码测试判完成；WildToolBench 计算子任务完成比例。",
"visible": "任务执行经历构成状态；构造任务关系时另用多模型、五个随机种子估计迁移影响。",
"use": "后续任务和重复探测用于算前向/后向迁移及遗忘；构造关系的分数不等于被测 agent 的逐步反馈。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"judgment": "BigCodeBench 用单元测试；WildToolBench 用子任务完成率，未展开其所有检查实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "BigCodeBench 与 WildToolBench 各 120 个任务组成受控历史。",
"selection": "任务执行经历形成状态；构造时每个任务对用多模型、五种子估计迁移。",
"evaluation": "相同任务池中的后续任务与重复 探测任务；动机实验另用 LifelongAgentBench 50 题。",
"isolation": "探针重现是设计要素；不是互斥训练/测试题泛化实验。",
"roles": {
"executor": {
"value": "所有被评测 agent 统一使用 DeepSeek-V4-Flash 推理。构造任务间迁移关系时另用 DeepSeek-V4-Flash、GPT-5.4-mini、GLM-5.0 分别测量，再多数投票；这三者不是主实验中三套执行配置。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.01149#S4"
}
]
},
"modifier": {
"value": "各方法自身更新状态；PATH-Bench 控制任务顺序和探针出现位置。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
]
},
"seed": {
"value": "通过可控任务顺序比较记忆/技能 task agent；作者的 SEU 额外判断哪些历史经验有帮助、哪些可能干扰当前任务，再决定是否使用。并非任意扩大记忆库就直接注入所有历史。",
"sources": [
{
"label": "PATH 构造、数据统计与评估协议",
"url": "https://arxiv.org/abs/2608.01149"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"selection": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"evaluation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
],
"isolation": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.01149#S3.SS2"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2608.01149#A1"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.01149#A2.SS0.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "不更新参数、把经验存在外部文件或记忆中，是 agent 持续学习的重要途径，但后续表现不仅取决于存了什么，也取决于此前按什么顺序经历了哪些任务。固定或随机的一条任务序列难以分离这些影响，因此无法充分诊断经验迁移、顺序依赖和相互干扰。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.01149#S1"
}
]
},
{
"key": "position",
"value": "评测先前经验怎样帮助或干扰后续能力，关注经验顺序、迁移收益以及新经历导致的遗忘。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.01149"
}
]
},
{
"key": "conclusion",
"value": "强迁移不保证记得住，后续经验可能破坏已有收益；选择性使用经验能减少遗忘。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.01149"
}
]
}
],
"fields": {
"object": "随学习路径积累的记忆与技能；基准测历史顺序对迁移和遗忘的影响。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.03874",
"title": "ContinualSkillBench: Can LLM Agents Truly Evolve Their Capabilities?",
"url": "https://arxiv.org/abs/2608.03874",
"date": "2026-08-04",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"Context",
"M0",
"Prequential",
"Skill",
"org:peking"
],
"fields": {
"本质定位": "五个 domain、每域100个相互关联且难度递增 subtasks，专门比较 sequential context adaptation 与显式 persistent skill maintenance。",
"被测系统 / feedback": "多模型、多 skill/context conditions。",
"证据边界与关键结论": "平均 ICL≈explicit skills：很多所谓 “skill evolution gain” 可能只是 context/feedback adaptation；这是非常重要的 attribution counterfactual。",
"什么在变": "被测 systems 的 context/skill library按各方法变化。",
"谁来改 / 谁执行": "**改**：各 skill-learning mechanism自身 updater。
**执行**：多种 LLM agent + skill/context condition。",
"基础 harness": "同一任务流下对比 no/implicit/explicit skill mechanisms。",
"Feedback": "previous-task context/outcome/feedback；benchmark观察后续task improvement。",
"Evolution → Eval": "5×100 sequential subtasks，prequential；不是传统 freeze-heldout。",
"Meta-depth": "Evaluation of M0/M1 skill evolution。",
"相对之前真正新增什么": "关键贡献不是再提 skill method，而是给了一个很重要的 **skill abstraction counterfactual**：如果只保留上下文也一样好，那么所谓 skill evolution gain 可能只是 context/feedback adaptation。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 214,
"fields": {
"时间": "2026-08-04",
"论文": "[ContinualSkillBench](https://arxiv.org/abs/2608.03874)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / Skill**。5 domains×100 interconnected tasks，对照 explicit skill maintenance 与仅保留 prior context。",
"被测系统 / feedback": "多模型、多 skill/context conditions。",
"证据边界与关键结论": "平均 ICL≈explicit skills：很多所谓 “skill evolution gain” 可能只是 context/feedback adaptation；这是非常重要的 attribution counterfactual。",
"标签": "`#Skill #Context #Prequential #B-Reliability #M0`"
}
},
{
"section": "C. Benchmark / Evaluation",
"line": 364,
"fields": {
"优先级": "**K**",
"时间": "2026-08-04",
"论文": "[ContinualSkillBench: Can LLM Agents Truly Evolve Their Capabilities?](https://arxiv.org/abs/2608.03874)",
"本质定位": "五个 domain、每域100个相互关联且难度递增 subtasks，专门比较 sequential context adaptation 与显式 persistent skill maintenance。",
"什么在变": "被测 systems 的 context/skill library按各方法变化。",
"谁来改 / 谁执行": "**改**：各 skill-learning mechanism自身 updater。
**执行**：多种 LLM agent + skill/context condition。",
"基础 harness": "同一任务流下对比 no/implicit/explicit skill mechanisms。",
"Feedback": "previous-task context/outcome/feedback；benchmark观察后续task improvement。",
"Evolution → Eval": "5×100 sequential subtasks，prequential；不是传统 freeze-heldout。",
"Meta-depth": "Evaluation of M0/M1 skill evolution。",
"相对之前真正新增什么": "关键贡献不是再提 skill method，而是给了一个很重要的 **skill abstraction counterfactual**：如果只保留上下文也一样好，那么所谓 skill evolution gain 可能只是 context/feedback adaptation。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"M0",
"M1"
],
"legacyCategories": [
"B-Lifelong",
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "五个 domain、每域100个相互关联且难度递增 subtasks，专门比较 sequential context adaptation 与显式 persistent skill maintenance。",
"novelty": "把显式技能学习与直接保留历史上下文放在对照条件下比较，检验提升是否确实需要提炼技能。",
"object": "被测系统的上下文与技能库，按各自方法持续更新。",
"executor": "GPT-4o、GPT-5.3-Codex、Claude Opus4.7，在技能库/上下文条件下执行任务序列。",
"modifier": "对应任务模型按技能学习机制创建、更新和调用技能，比较其跨任务持续积累；不是独立的未命名优化语言模型。",
"roleContext": "**改**：各 skill-learning mechanism自身 updater。
**执行**：多种 LLM agent + skill/context condition。",
"seed": "同一底层任务执行环境比较不留技能、隐式经验和显式技能库；显式设置把经历整理成技能并跨题复用。任务流内包含不同形式的输入输出，不限于同一固定工作流。",
"fixed": "Evaluation of M0/M1 skill evolution。",
"verdict": "agent 读取前一任务的上下文、执行结果与评审反馈，更新后续可用技能；实验考察之后的任务是否改善，而非只看当前技能文件是否写完。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "5×100 sequential subtasks，prequential；不是传统 freeze-heldout。"
}
],
"takeaway": "平均 ICL≈explicit skills：很多所谓 “skill evolution gain” 可能只是 context/feedback adaptation；这是非常重要的 attribution counterfactual。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：按技能链组织来源任务：OlympiadBench、LawBench、TAT-QA，逐渐过渡到 GAIA（需要检索、推理和使用工具的通用助理任务基准）、ClawBench、MedAgentsBench、MathCoder、OneMillionBench。\n\n调试 / 选版本数据：当前题完成后的评分者分数和反馈，由 Create Skill/Modify Skill 元技能指导维护。\n\n最终测试数据：同一链后续更复杂任务；比较顺序积累和对照。\n\n数据隔离与证据边界：这些来源分层构成任务流，不是整个数据集分别充当 训练／测试。",
"cycle": "每题三回合：给任务及技能索引、执行、接收评分者后创建/修改技能；修改只从下一子任务起使用。",
"train": "按技能链组织来源任务：OlympiadBench、LawBench、TAT-QA，逐渐过渡到 GAIA（需要检索、推理和使用工具的通用助理任务基准）、ClawBench、MedAgentsBench、MathCoder、OneMillionBench。",
"debug": "当前题完成后的评分者分数和反馈，由 Create Skill/Modify Skill 元技能指导维护。",
"test": "同一链后续更复杂任务；比较顺序积累和对照。",
"isolation": "这些来源分层构成任务流，不是整个数据集分别充当 训练／测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "同一底层任务执行环境比较不留技能、隐式经验和显式技能库；显式设置把经历整理成技能并跨题复用。任务流内包含不同形式的输入输出，不限于同一固定工作流。",
"protocol": "**来源：**基础题含 OlympiadBench、LawBench、TAT-QA；中高难任务含 GAIA、ClawBench、MedAgentsBench、MathCoder；更难阶段含 OneMillionBench。\n\n**组织：**按领域和可复用技能链筛选、排序成逐渐复杂的任务流，比较顺序执行和对照；经验在流内累积。不是将这些来源整体分别指定为 train/test；每个来源的最终采样数与技能链任务清单本轮待核实。",
"sections": "§3.1–3.5、§4.1",
"source": "https://arxiv.org/abs/2608.03874",
"version": "2608.03874v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "337473b4ac76cc760ed96b968a872ed4d082f0e212d3db81bfdb1226c34aa221",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-4o、GPT-5.3-Codex、Claude Opus4.7，在技能库/上下文条件下执行任务序列。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03874#S4.SS1.SSS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.03874#S4.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应任务模型按技能学习机制创建、更新和调用技能，比较其跨任务持续积累；不是独立的未命名优化语言模型。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03874#S4.SS1.SSS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.03874#S4.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "被测系统的上下文与技能库，按各自方法持续更新。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "agent 读取前一任务的上下文、执行结果与评审反馈，更新后续可用技能；实验考察之后的任务是否改善，而非只看当前技能文件是否写完。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "同一底层任务执行环境比较不留技能、隐式经验和显式技能库；显式设置把经历整理成技能并跨题复用。任务流内包含不同形式的输入输出，不限于同一固定工作流。",
"sources": [
{
"label": "§3.1–3.5、§4.1",
"url": "https://arxiv.org/abs/2608.03874"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每题三回合：给任务及技能索引、执行、接收评分者后创建/修改技能；修改只从下一子任务起使用。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "按技能链组织来源任务：OlympiadBench、LawBench、TAT-QA，逐渐过渡到 GAIA（需要检索、推理和使用工具的通用助理任务基准）、ClawBench、MedAgentsBench、MathCoder、OneMillionBench。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前题完成后的评分者分数和反馈，由 Create Skill/Modify Skill 元技能指导维护。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "同一链后续更复杂任务；比较顺序积累和对照。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这些来源分层构成任务流，不是整个数据集分别充当 训练／测试。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把显式技能学习与直接保留历史上下文放在对照条件下比较，检验提升是否确实需要提炼技能。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.03874v1",
"version": "2608.03874v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "关键贡献不是再提 skill method，而是给了一个很重要的 **skill abstraction counterfactual**：如果只保留上下文也一样好，那么所谓 skill evolution gain 可能只是 context/feedback adaptation。",
"feedbackCases": [
{
"label": "结构化题目的技能链",
"data": "OlympiadBench、LawBench、TAT-QA 等来源的任务，逐步过渡到更复杂任务。",
"scoring": "数值题根据数值答案评分，精确匹配题根据标准输出评分。",
"visible": "做完当前题后给评估结果与反馈，由 Create Skill / Modify Skill 指令维护技能。",
"use": "影响同一链后续题；需将格式规范带来的进步与一般能力增长分开。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"judgment": "数值答案检查／标准输出精确匹配，由规则判分"
},
{
"label": "开放与交互题目的技能链",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准）、ClawBench、MedAgentsBench、MathCoder、OneMillionBench 等复杂任务。",
"scoring": "按任务类型用细则评审或对应工具/任务检查，不能把所有来源都当作一个二元验收器。",
"visible": "已完成任务的评分和反馈进入历史上下文。",
"use": "后续表现可能来自适应评分风格，也可能来自显式技能；论文以顺序、上下文与检索对照区分。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"judgment": "按逐题评分细则或任务工具检查；本文未明确所有裁判的规则／模型实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0,
1
],
"testCases": [
0,
1
],
"evolution": "按技能链组织来源任务：OlympiadBench、LawBench、TAT-QA，逐渐过渡到 GAIA（需要检索、推理和使用工具的通用助理任务基准）、ClawBench、MedAgentsBench、MathCoder、OneMillionBench。",
"selection": "当前题完成后的评分者分数和反馈，由 Create Skill/Modify Skill 元技能指导维护。",
"evaluation": "同一链后续更复杂任务；比较顺序积累和对照。",
"isolation": "这些来源分层构成任务流，不是整个数据集分别充当 训练／测试。",
"roles": {
"executor": {
"value": "GPT-4o、GPT-5.3-Codex、Claude Opus4.7，在技能库/上下文条件下执行任务序列。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03874#S4.SS1.SSS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.03874#S4.SS4.SSS0.Px1"
}
]
},
"modifier": {
"value": "对应任务模型按技能学习机制创建、更新和调用技能，比较其跨任务持续积累；不是独立的未命名优化语言模型。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.03874#A4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.03874#S4.SS1.SSS1"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.03874#S4.SS4.SSS0.Px1"
}
]
},
"seed": {
"value": "同一底层任务执行环境比较不留技能、隐式经验和显式技能库；显式设置把经历整理成技能并跨题复用。任务流内包含不同形式的输入输出，不限于同一固定工作流。",
"sources": [
{
"label": "§3.1–3.5、§4.1",
"url": "https://arxiv.org/abs/2608.03874"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.03874#S3.SS1.SSS0.Px2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.03874#A5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "真实任务可能要求预训练中没有的领域知识和操作流程，而人工预写一套完整技能库成本高，也难覆盖不断变化的需求。现有评测不足以系统回答：agent 能否根据用户陆续提出的任务和反馈，自行补充、修改技能，从而持续扩展能完成的任务范围。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.03874#S1"
}
]
},
{
"key": "position",
"value": "评测持续任务中的能力提升是否来自可复用技能的形成，还是主要依赖近期上下文，并比较两种适应方式的适用条件。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03874"
}
]
},
{
"key": "conclusion",
"value": "两者平均表现接近；显式技能在特定任务有用，但弱模型容易积累零碎、题目特定的技能。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.03874"
}
]
}
],
"fields": {
"object": "被测系统的上下文与技能库，按各自方法持续更新。",
"verdict": "agent 读取前一任务的上下文、执行结果与评审反馈，更新后续可用技能；实验考察之后的任务是否改善，而非只看当前技能文件是否写完。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.03874"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.04003",
"title": "PAST-Bench",
"url": "https://arxiv.org/abs/2608.04003",
"date": "2026-08-04",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"MemoryContent",
"Prequential",
"Skill",
"Streaming",
"person:mengdi-wang"
],
"fields": {
"本质定位": "**B-Lifelong / personal agent**。用 matched retained-experience ON/OFF fresh-session sequences，不只看 later gain，还检查是否真的经过 save→retrieve→update pathway。",
"被测系统 / feedback": "7 base models × 4 agent frameworks；26 scenarios/204 episodes。",
"证据边界与关键结论": "headline gain 相同不代表机制真的 work；加入**pathway evidence**是它最有价值的新点。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 215,
"fields": {
"时间": "2026-08-04",
"论文": "[PAST-Bench](https://arxiv.org/abs/2608.04003)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / personal agent**。用 matched retained-experience ON/OFF fresh-session sequences，不只看 later gain，还检查是否真的经过 save→retrieve→update pathway。",
"被测系统 / feedback": "7 base models × 4 agent frameworks；26 scenarios/204 episodes。",
"证据边界与关键结论": "headline gain 相同不代表机制真的 work；加入**pathway evidence**是它最有价值的新点。",
"标签": "`#MemoryContent #Skill #Streaming #Prequential #B-Reliability`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Prequential",
"protocolBasis": "原记录明确标注 #Prequential；经验只应影响后续任务。",
"year": "2026",
"depth": [
"未明确"
],
"priorityBasis": "用户指定阅读重点：Mengdi Wang（王梦迪，Princeton）署名论文。",
"prioritySources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2608.04003"
}
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "用 matched retained-experience ON/OFF fresh-session sequences，不只看 later gain，还检查是否真的经过 save→retrieve→update pathway。",
"novelty": "对同一任务序列配对比较保留与不保留经验，并追踪保存、检索、更新的实际使用，区分成绩上涨和记忆机制真正起作用。",
"object": "跨会话保留的事实、流程、检索条件及更新后的规则。",
"executor": "固定 Hermes 框架比较 GPT-5.4、GLM-5.1、Kimi K2.6、DeepSeek-V4-Pro、MiniMax-M2.7、Claude Sonnet 4.6、Claude Opus 4.6；比较不同框架时统一使用 MiniMax-M2.7，包括补充的 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 和 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 实验。",
"modifier": "框架自身的保存、检索、更新路径；评测器通过对照检验实际机制。",
"roleContext": "7 base models × 4 agent frameworks；26 scenarios/204 episodes。",
"seed": "同一模型在保留/关闭跨会话经验的匹配条件下运行个人助手任务；每个 episode 是新会话。Hermes+ 在 Hermes 上增加针对保存、检索、更新等环节的干预，用轨迹确认是否真的走了预期记忆路径。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "按任务 评分器 计算完成分，并检查保存的文件、记忆及执行日志是否支持所声称的进化机制。开放答案统一由 MiniMax-M2.7（temperature=0）评分；另用 48 份盲评样本核对人工评分。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "headline gain 相同不代表机制真的 work；加入**pathway evidence**是它最有价值的新点。"
}
],
"takeaway": "headline gain 相同不代表机制真的 work；加入**pathway evidence**是它最有价值的新点。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：自建 26 个场景、204 个 episode，按家庭顺序提供学习经历。\n\n调试 / 选版本数据：学习 episode 中的事实/纠正，或信息获取任务预置的参考内容。\n\n最终测试数据：新会话中的后续 episode，检验能否在恰当时间找回并使用状态。\n\n数据隔离与证据边界：环境按新会话执行，但允许指定持久状态跨会话；对照用于归因而非普通离线训练/测试。",
"cycle": "前会话暴露规则或程序，后会话不再重述；设置无保留、干扰、过时记忆及错误机制对照。",
"train": "自建 26 个场景、204 个 episode，按家庭顺序提供学习经历。",
"debug": "学习 episode 中的事实/纠正，或信息获取任务预置的参考内容。",
"test": "新会话中的后续 episode，检验能否在恰当时间找回并使用状态。",
"isolation": "环境按新会话执行，但允许指定持久状态跨会话；对照用于归因而非普通离线训练/测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "同一模型在保留/关闭跨会话经验的匹配条件下运行个人助手任务；每个 episode 是新会话。Hermes+ 在 Hermes 上增加针对保存、检索、更新等环节的干预，用轨迹确认是否真的走了预期记忆路径。",
"protocol": "**自建数据：**26 个场景、204 个 episode，覆盖记忆、流程复用、信息获取和状态更新。\n\n**学习/评估：**按顺序运行新会话，比较相同任务序列中有无持久化的后续表现；使用 7 个基础模型和 4 个框架。数据是受控跨会话序列，不是一个统一的离线训练/测试集；场景构造的完整实例表本轮未核实。",
"sections": "§3 构造和评估；§4 设置",
"source": "https://arxiv.org/abs/2608.04003",
"version": "2608.04003v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "1813a557d978ab90ee78fde8deddbeaf8e16f389dd573ab8f2867f3282520163",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "固定 Hermes 框架比较 GPT-5.4、GLM-5.1、Kimi K2.6、DeepSeek-V4-Pro、MiniMax-M2.7、Claude Sonnet 4.6、Claude Opus 4.6；比较不同框架时统一使用 MiniMax-M2.7，包括补充的 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 和 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 实验。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.04003#S4.SS2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.04003#S4.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.04003#A3.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "框架自身的保存、检索、更新路径；评测器通过对照检验实际机制。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨会话保留的事实、流程、检索条件及更新后的规则。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "按任务 评分器 计算完成分，并检查保存的文件、记忆及执行日志是否支持所声称的进化机制。开放答案统一由 MiniMax-M2.7（temperature=0）评分；另用 48 份盲评样本核对人工评分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.04003#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "同一模型在保留/关闭跨会话经验的匹配条件下运行个人助手任务；每个 episode 是新会话。Hermes+ 在 Hermes 上增加针对保存、检索、更新等环节的干预，用轨迹确认是否真的走了预期记忆路径。",
"sources": [
{
"label": "§3 构造和评估；§4 设置",
"url": "https://arxiv.org/abs/2608.04003"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "前会话暴露规则或程序，后会话不再重述；设置无保留、干扰、过时记忆及错误机制对照。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "自建 26 个场景、204 个 episode，按家庭顺序提供学习经历。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "学习 episode 中的事实/纠正，或信息获取任务预置的参考内容。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "新会话中的后续 episode，检验能否在恰当时间找回并使用状态。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "环境按新会话执行，但允许指定持久状态跨会话；对照用于归因而非普通离线训练/测试。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "对同一任务序列配对比较保留与不保留经验，并追踪保存、检索、更新的实际使用，区分成绩上涨和记忆机制真正起作用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.04003v1",
"version": "2608.04003v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "对同一任务序列做保留经验开/关的配对条件，并检查 save→retrieve→update 路径；涨分和机制实际被使用是两件事。",
"feedbackCases": [
{
"label": "跨会话学习与检验",
"data": "26 个自建场景、204 个 episode；学习阶段输入事实/纠正或参考材料，后续换新会话。",
"scoring": "可程序判定的要求由任务检查器评分；开放回答统一 MiniMax-M2.7、温度零；另用 48 个盲评样本核对人工评分。",
"visible": "学习会话可见事实与纠正；最终评分同时检查文件、记忆和日志是否支持声称的复用机制。",
"use": "保留/关闭既有状态两条件共享提示、判分器、工具与随机种子；衡量状态是否帮助，而非只看保存过文件。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.04003#A2.SS4"
}
],
"judgment": "确定性任务检查器 + MiniMax-M2.7 开放回答评审；另作人工盲评核对"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "自建 26 个场景、204 个 episode，按家庭顺序提供学习经历。",
"selection": "学习 episode 中的事实/纠正，或信息获取任务预置的参考内容。",
"evaluation": "新会话中的后续 episode，检验能否在恰当时间找回并使用状态。",
"isolation": "环境按新会话执行，但允许指定持久状态跨会话；对照用于归因而非普通离线训练/测试。",
"roles": {
"executor": {
"value": "固定 Hermes 框架比较 GPT-5.4、GLM-5.1、Kimi K2.6、DeepSeek-V4-Pro、MiniMax-M2.7、Claude Sonnet 4.6、Claude Opus 4.6；比较不同框架时统一使用 MiniMax-M2.7，包括补充的 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 和 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 实验。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.04003#S4.SS2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.04003#S4.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.04003#A3.SS2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "框架自身的保存、检索、更新路径；评测器通过对照检验实际机制。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2608.04003#A1.SS3"
}
]
},
"seed": {
"value": "同一模型在保留/关闭跨会话经验的匹配条件下运行个人助手任务；每个 episode 是新会话。Hermes+ 在 Hermes 上增加针对保存、检索、更新等环节的干预，用轨迹确认是否真的走了预期记忆路径。",
"sources": [
{
"label": "§3 构造和评估；§4 设置",
"url": "https://arxiv.org/abs/2608.04003"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "真实助理需要在多次会话间记住偏好、复用流程并修正过期经验，单次独立任务的分数看不到这些变化。即使后期分数上升，也可能来自模型、提示、检索捷径或任务难度差异；缺少控制这些因素的比较，就不能把提升归因于 agent 保留和使用了经验。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.04003#S1"
}
]
},
{
"key": "position",
"value": "评测持久经验的保存、检索和更新是否真正导致后续能力提升，使观察到的涨分能对应到实际起作用的学习环节。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.04003"
}
]
},
{
"key": "conclusion",
"value": "相似涨分可对应不同机制；针对更新旧状态等环节的改造有帮助，但效果依赖模型与能力类型。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.04003"
}
]
}
],
"fields": {
"object": "跨会话保留的事实、流程、检索条件及更新后的规则。"
}
},
"attributions": [
{
"tag": "person:mengdi-wang",
"label": "Mengdi Wang",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2608.04003"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.06144",
"title": "FinEvo-Bench",
"url": "https://arxiv.org/abs/2608.06144",
"date": "2026-08-06",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"LLMJudge",
"MemoryContent",
"Skill",
"Streaming"
],
"fields": {
"本质定位": "**B-Lifelong / professional workflow**。同一 Qwen3.7-Max 下比较 4 self-evolving scaffolds，并用 paired non-evolving control 隔离 retained experience 的贡献。",
"被测系统 / feedback": "120 real-case tasks、20 scenes、6 finance domains；Claude Opus judge。",
"证据边界与关键结论": "evolving +9.33~19.37；skill-only 在 Claude Code 上优于 memory-only/combined；rubric feedback 优于 reference-answer feedback。"
},
"details": [],
"occurrences": [
{
"section": "B. Continual / memory / skill self-evolution",
"line": 217,
"fields": {
"时间": "2026-08-06",
"论文": "[FinEvo-Bench](https://arxiv.org/abs/2608.06144)",
"级别": "**K**",
"它真正测什么": "**B-Lifelong / professional workflow**。同一 Qwen3.7-Max 下比较 4 self-evolving scaffolds，并用 paired non-evolving control 隔离 retained experience 的贡献。",
"被测系统 / feedback": "120 real-case tasks、20 scenes、6 finance domains；Claude Opus judge。",
"证据边界与关键结论": "evolving +9.33~19.37；skill-only 在 Claude Code 上优于 memory-only/combined；rubric feedback 优于 reference-answer feedback。",
"标签": "`#Skill #MemoryContent #LLMJudge #Streaming #B-Reliability`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-Lifelong"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "同一 Qwen3.7-Max 下比较 4 self-evolving scaffolds，并用 paired non-evolving control 隔离 retained experience 的贡献。",
"novelty": "把金融工作组织成相关但不同的连续案例，与不更新的系统配对比较，并区分过程评分细则和参考答案提供的反馈。",
"object": "金融任务执行后积累的记忆与技能。",
"executor": "Claude Code、Codex、Letta、GenericAgent 四套 agent 执行框架均调用 Qwen3.7-Max；前两者是 coding agent 工具，后两者提供其他 agent 与记忆组织方式。输入上下文上限 100 万词元，最大输出约 6.4 万词元；采样温度为 0，以减少输出随机性。",
"modifier": "仍由各框架中的Qwen3.7-Max在原会话内读取judge.md后反思和更新。独立评分 task agent 使用Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6。",
"roleContext": "120 real-case tasks、20 scenes、6 finance domains；Claude Opus judge。",
"seed": "比较 Letta、Codex、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 等四个现成框架，在相同 Qwen3.7-Max 下开启或关闭持久经验。另由 Claude Code＋Claude Opus 4.6 独立评分，执行模型和评分模型不是同一个角色。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "独立Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6按固定评估流程检查金融产物并写judge.md；该文件反馈给Qwen3.7-Max执行 task agent 用于下一阶段改进。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "evolving +9.33~19.37；skill-only 在 Claude Code 上优于 memory-only/combined；rubric feedback 优于 reference-answer feedback。"
}
],
"takeaway": "evolving +9.33~19.37；skill-only 在 Claude Code 上优于 memory-only/combined；rubric feedback 优于 reference-answer feedback。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：自建六领域、20 场景、120 个真实案例衍生任务，每场景六个事实不同但流程相关的案例。\n\n调试 / 选版本数据：已完成任务的judge.md 分数与反馈。\n\n最终测试数据：三个独立打乱、全局交错的任务流，比较质量和合规。\n\n数据隔离与证据边界：在线流式协议；同框架不进化对照用于估计保留经验的收益。",
"cycle": "执行→独立评分→恢复原会话反思，之后在新案例复用；分别比较只用记忆、只用技能和两者结合。",
"train": "自建六领域、20 场景、120 个真实案例衍生任务，每场景六个事实不同但流程相关的案例。",
"debug": "已完成任务的judge.md 分数与反馈。",
"test": "三个独立打乱、全局交错的任务流，比较质量和合规。",
"isolation": "在线流式协议；同框架不进化对照用于估计保留经验的收益。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "比较 Letta、Codex、Claude Code 等四个现成框架，在相同 Qwen3.7-Max 下开启或关闭持久经验。另由 Claude Code＋Claude Opus 4.6 独立评分，执行模型和评分模型不是同一个角色。",
"protocol": "**自建数据：**6 个金融领域、20 个业务场景、120 个任务；每场景 6 个共享业务程序但事实不同的案例。事实来自机构及公开案例，流程和 rubric 经专业审查。\n\n**评估：**三个独立打乱、全局交错的任务流，经验随流积累；与同框架不进化对照比较质量和合规问题。不是固定 train/test；rubric 反馈和参考答案反馈是不同实验设置。",
"sections": "§3 数据构造；§4.1",
"source": "https://arxiv.org/abs/2608.06144",
"version": "2608.06144v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "d7ecf7cb039e7bf5d11f69401818e813e744bbaad393ad427d00e34268607677",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Claude Code、Codex、Letta、GenericAgent 四套 agent 执行框架均调用 Qwen3.7-Max；前两者是 coding agent 工具，后两者提供其他 agent 与记忆组织方式。输入上下文上限 100 万词元，最大输出约 6.4 万词元；采样温度为 0，以减少输出随机性。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "仍由各框架中的Qwen3.7-Max在原会话内读取judge.md后反思和更新。独立评分 task agent 使用Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "金融任务执行后积累的记忆与技能。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "独立Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6按固定评估流程检查金融产物并写judge.md；该文件反馈给Qwen3.7-Max执行 task agent 用于下一阶段改进。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "比较 Letta、Codex、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 等四个现成框架，在相同 Qwen3.7-Max 下开启或关闭持久经验。另由 Claude Code＋Claude Opus 4.6 独立评分，执行模型和评分模型不是同一个角色。",
"sources": [
{
"label": "§3 数据构造；§4.1",
"url": "https://arxiv.org/abs/2608.06144"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "执行→独立评分→恢复原会话反思，之后在新案例复用；分别比较只用记忆、只用技能和两者结合。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "自建六领域、20 场景、120 个真实案例衍生任务，每场景六个事实不同但流程相关的案例。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "已完成任务的judge.md 分数与反馈。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "三个独立打乱、全局交错的任务流，比较质量和合规。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "在线流式协议；同框架不进化对照用于估计保留经验的收益。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把金融工作组织成相关但不同的连续案例，与不更新的系统配对比较，并区分过程评分细则和参考答案提供的反馈。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.06144v1",
"version": "2608.06144v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把真实金融工作流组织为相关但不同的连续案例，配对 non-evolving control；过程 rubric 与参考答案的反馈作用可比较。",
"feedbackCases": [
{
"label": "金融场景的顺序反馈",
"data": "六领域、20 场景、120 个案例任务；每场景六个事实不同但业务流程相关的案例。",
"scoring": "独立 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus 4.6 按场景的人工审核细则检查任务质量与金融合规。",
"visible": "返回 judge.md 中的分数与逐项反馈给 Qwen3.7-Max；反馈针对该案例交付物。",
"use": "执行者反思并存储经验，然后关闭会话、清除原始对话，只保留经验用于下一题；三次独立打乱构成任务流。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"judgment": "Claude Opus 4.6 在 Claude Code 中按人工审核的场景评分细则评审"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "自建六领域、20 场景、120 个真实案例衍生任务，每场景六个事实不同但流程相关的案例。",
"selection": "已完成任务的judge.md 分数与反馈。",
"evaluation": "三个独立打乱、全局交错的任务流，比较质量和合规。",
"isolation": "在线流式协议；同框架不进化对照用于估计保留经验的收益。",
"roles": {
"executor": {
"value": "Claude Code、Codex、Letta、GenericAgent 四套 agent 执行框架均调用 Qwen3.7-Max；前两者是 coding agent 工具，后两者提供其他 agent 与记忆组织方式。输入上下文上限 100 万词元，最大输出约 6.4 万词元；采样温度为 0，以减少输出随机性。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "仍由各框架中的Qwen3.7-Max在原会话内读取judge.md后反思和更新。独立评分 task agent 使用Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6。",
"sources": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.06144#S4.SS4.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.06144#S4.SS1.SSS0.Px5"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.06144#A2"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "比较 Letta、Codex、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） 等四个现成框架，在相同 Qwen3.7-Max 下开启或关闭持久经验。另由 Claude Code＋Claude Opus 4.6 独立评分，执行模型和评分模型不是同一个角色。",
"sources": [
{
"label": "§3 数据构造；§4.1",
"url": "https://arxiv.org/abs/2608.06144"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"selection": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"evaluation": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
],
"isolation": [
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2608.06144#A2.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "金融工作会反复使用类似分析流程和检查规则，但每个案例的文件、风险和结论又不同。已有金融评测主要把案例独立判分，其他持续学习评测也未充分覆盖这类专业工作流，因此难判断 agent 是否学会了可复用的方法，同时仍能针对新案例作出正确判断。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.06144#S1"
}
]
},
{
"key": "position",
"value": "面向专业金融工作，评测 agent 能否跨案例迁移业务经验，并在开放产物与多维质量要求下持续改善表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.06144"
}
]
},
{
"key": "conclusion",
"value": "四套框架均从保留经验中获益；过程评分规则的反馈比只提供参考答案更有效。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.06144"
}
]
}
],
"fields": {
"object": "金融任务执行后积累的记忆与技能。",
"executor": "Claude Code、Codex、Letta、GenericAgent 四套 agent 执行框架均调用 Qwen3.7-Max；前两者是 coding agent 工具，后两者提供其他 agent 与记忆组织方式。输入上下文上限 100 万词元，最大输出约 6.4 万词元；采样温度为 0，以减少输出随机性。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2603.08640",
"title": "PostTrainBench",
"url": "https://arxiv.org/abs/2603.08640",
"date": "2026-03-09",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"Data",
"Improver",
"RewardHacking",
"Weights"
],
"fields": {
"本质定位": "**B-RSI**。给 agent 约束预算，让它自己找数据、训练、评估并提高 base LM；直接测 autonomous post-training engineering。",
"被测系统 / feedback": "coding/research agents + base model + local data/web + evaluator。",
"证据边界与关键结论": "best agent 平均仍低于 official instruction-tuned model；出现 test-set training、下载现成 tuned checkpoint、滥用 API key 等 reward hacking。"
},
"details": [],
"occurrences": [
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 223,
"fields": {
"时间": "2026-03-09",
"论文": "[PostTrainBench](https://arxiv.org/abs/2603.08640)",
"级别": "**K**",
"它真正测什么": "**B-RSI**。给 agent 约束预算，让它自己找数据、训练、评估并提高 base LM；直接测 autonomous post-training engineering。",
"被测系统 / feedback": "coding/research agents + base model + local data/web + evaluator。",
"证据边界与关键结论": "best agent 平均仍低于 official instruction-tuned model；出现 test-set training、下载现成 tuned checkpoint、滥用 API key 等 reward hacking。",
"标签": "`#Data #Weights #Improver #B-RSI #RewardHacking`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "给 agent 约束预算，让它自己找数据、训练、评估并提高 base LM；直接测 autonomous post-training engineering。",
"novelty": "给模型起点和有限 GPU 时间，让 agent 自己完成数据选择、训练和调试，再比较训练产物与官方指令模型的能力。",
"object": "给定基础模型的后训练参数；研究 agent 选择数据和训练方法。",
"executor": "研究 agent 操作终端完成后训练；最终答题的是提交的 Qwen3-1.7B、Qwen3-4B、SmolLM3-3B 或 Gemma-3-4B 检查点，交给固定评测程序运行。",
"modifier": "表 1 比较多种研究模型与 CLI（通过终端命令使用的程序界面） 的组合，包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2，以及 OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等；研究模型选择数据和训练代码，不等于被训练的四个基础模型。",
"roleContext": "coding/research agents + base model + local data/web + evaluator。",
"seed": "用 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Gemini CLI（通过终端命令使用的程序界面） 等标准开发工具 agent，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估运行框架不允许改。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "evaluate.py 的目标基准成绩，最终另审查训练数据污染及越界训练。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "best agent 平均仍低于 official instruction-tuned model；出现 test-set training、下载现成 tuned checkpoint、滥用 API key 等 reward hacking。"
}
],
"takeaway": "best agent 平均仍低于 official instruction-tuned model；出现 test-set training、下载现成 tuned checkpoint、滥用 API key 等 reward hacking。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：各 run 自行选数据训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，禁止把目标测试题用于训练。\n\n调试 / 选版本数据：可调用 evaluate.py，甚至使用 --limit 加快实验；预算为单 H100、10 小时。\n\n最终测试数据：AIME25、GSM8K（小学数学应用题基准）、GPQA（研究生级科学问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、BFCL、ArenaHard、HealthBench 中指定的一项。\n\n数据隔离与证据边界：禁止训练测试题不等于评估分数全程隐藏：开发接口允许查询基准成绩，不能写成完全不可见的盲测。",
"cycle": "自行检索/整理数据、编写训练代码、试验并选 final_model；不能训练其他模型或改 评估器。",
"train": "各 run 自行选数据训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，禁止把目标测试题用于训练。",
"debug": "可调用 evaluate.py，甚至使用 --limit 加快实验；预算为单 H100、10 小时。",
"test": "AIME25、GSM8K（小学数学应用题基准）、GPQA（研究生级科学问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、BFCL、ArenaHard、HealthBench 中指定的一项。",
"isolation": "禁止训练测试题不等于评估分数全程隐藏：开发接口允许查询基准成绩，不能写成完全不可见的盲测。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "用 Codex CLI、Claude Code、Gemini CLI 等标准开发工具代理，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估 harness 不允许改。",
"protocol": "**训练：**agent 为指定基础模型自行搜索、整理数据并执行后训练；禁止使用目标 benchmark 的测试数据。不能把所有 run 说成同一训练集，逐 run 实际来源需要对应轨迹。\n\n**最终测试：**目标从 AIME 2025、GSM8K、GPQA、HumanEval、BFCL、ArenaHard、HealthBench 中选；提交 checkpoint 后由对应留出测试评估。基础模型含 Qwen3-1.7B/4B、SmolLM3-3B、Gemma-3-4B。具体训练来源及各测试子集数量本轮待核实。",
"sections": "任务定义与代理执行设置",
"source": "https://arxiv.org/abs/2603.08640",
"version": "2603.08640v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "3c3bf8dde4c7c9256ac5ebd95eb9f28d7512b5147ae9f7d5735355a1a166d6ec",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "研究 agent 操作终端完成后训练；最终答题的是提交的 Qwen3-1.7B、Qwen3-4B、SmolLM3-3B 或 Gemma-3-4B 检查点，交给固定评测程序运行。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "表 1 比较多种研究模型与 CLI（通过终端命令使用的程序界面） 的组合，包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2，以及 OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等；研究模型选择数据和训练代码，不等于被训练的四个基础模型。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "给定基础模型的后训练参数；研究 agent 选择数据和训练方法。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "evaluate.py 的目标基准成绩，最终另审查训练数据污染及越界训练。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "用 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Gemini CLI（通过终端命令使用的程序界面） 等标准开发工具 agent，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估运行框架不允许改。",
"sources": [
{
"label": "任务定义与 agent 执行设置",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "自行检索/整理数据、编写训练代码、试验并选 final_model；不能训练其他模型或改 评估器。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "各 run 自行选数据训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，禁止把目标测试题用于训练。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "可调用 evaluate.py，甚至使用 --limit 加快实验；预算为单 H100、10 小时。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AIME25、GSM8K（小学数学应用题基准）、GPQA（研究生级科学问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、BFCL、ArenaHard、HealthBench 中指定的一项。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "禁止训练测试题不等于评估分数全程隐藏：开发接口允许查询基准成绩，不能写成完全不可见的盲测。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "给模型起点和有限 GPU 时间，让 agent 自己完成数据选择、训练和调试，再比较训练产物与官方指令模型的能力。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.08640v2",
"version": "2603.08640v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "给定 base model 和有限 GPU 时间，让 agent 自行完成数据、训练和评估；能跑通训练与稳定超过官方 instruction-tuned 模型是不同门槛。",
"feedbackCases": [
{
"label": "AIME25 / GSM8K / GPQA / BFCL",
"data": "每次运行围绕指定基准训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，单 H100、十小时。",
"scoring": "按任务协议的答案精确匹配评分；GSM8K（小学数学应用题基准） 评估用十个示范，其他通常不提供示范。",
"visible": "evaluate.py 返回目标基准成绩，可用 --limit 做小规模检查；训练资料由 agent 选择，禁止拿目标测试题训练。",
"use": "根据评估改训练方案；另一个审查环节检查数据污染和偷换模型，违规回记基座分数，审查模型不是答案评分器。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"judgment": "原文将这些任务列为答案精确匹配；此处 BFCL 不应套用其他论文的状态评测"
},
{
"label": "HumanEval",
"data": "每次运行围绕指定基准训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，单 H100、十小时。",
"scoring": "执行程序单元测试，报告单次生成通过率。",
"visible": "evaluate.py 返回目标基准成绩，可用 --limit 做小规模检查；训练资料由 agent 选择，禁止拿目标测试题训练。",
"use": "根据评估改训练方案；另一个审查环节检查数据污染和偷换模型，违规回记基座分数，审查模型不是答案评分器。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"judgment": "HumanEval 单元测试执行程序"
},
{
"label": "ArenaHard Writing / HealthBench-Easy",
"data": "每次运行围绕指定基准训练 Qwen3-1.7B/4B、SmolLM3-3B 或 Gemma-3-4B，单 H100、十小时。",
"scoring": "GPT-5-mini 评审；ArenaHard Writing 与 Qwen3-1.7B 的基线输出比较，HealthBench 按其医疗回答细则。",
"visible": "evaluate.py 返回目标基准成绩，可用 --limit 做小规模检查；训练资料由 agent 选择，禁止拿目标测试题训练。",
"use": "根据评估改训练方案；另一个审查环节检查数据污染和偷换模型，违规回记基座分数，审查模型不是答案评分器。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"judgment": "GPT-5-mini：ArenaHard 作回答对比，HealthBench 按医疗评分细则评审"
}
],
"experiments": [
{
"label": "AIME25 / GSM8K / GPQA / BFCL",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "每次运行指定一个目标基准与一个基础模型，由 agent 自行选择或生成训练数据；预算单 H100、十小时。",
"selection": "可调用 evaluate.py 查询目标基准成绩，也可用 --limit 缩小评估规模，结果用于改训练方案。",
"evaluation": "在本行指定基准上评测提交的训练后模型。",
"isolation": "禁止用测试题训练，但开发时允许查询评测成绩；不能表述为分数全程隐藏。",
"roles": {
"executor": {
"value": "研究 agent 操作终端完成后训练；最终答题的是提交的 Qwen3-1.7B、Qwen3-4B、SmolLM3-3B 或 Gemma-3-4B 检查点，交给固定评测程序运行。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
}
]
},
"modifier": {
"value": "表 1 比较多种研究模型与 CLI（通过终端命令使用的程序界面） 的组合，包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2，以及 OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等；研究模型选择数据和训练代码，不等于被训练的四个基础模型。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
},
"seed": {
"value": "用 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Gemini CLI（通过终端命令使用的程序界面） 等标准开发工具 agent，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估运行框架不允许改。",
"sources": [
{
"label": "任务定义与 agent 执行设置",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"selection": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"evaluation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"isolation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
]
}
},
{
"label": "HumanEval",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "每次运行指定一个目标基准与一个基础模型，由 agent 自行选择或生成训练数据；预算单 H100、十小时。",
"selection": "可调用 evaluate.py 查询目标基准成绩，也可用 --limit 缩小评估规模，结果用于改训练方案。",
"evaluation": "在本行指定基准上评测提交的训练后模型。",
"isolation": "禁止用测试题训练，但开发时允许查询评测成绩；不能表述为分数全程隐藏。",
"roles": {
"executor": {
"value": "研究 agent 操作终端完成后训练；最终答题的是提交的 Qwen3-1.7B、Qwen3-4B、SmolLM3-3B 或 Gemma-3-4B 检查点，交给固定评测程序运行。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
}
]
},
"modifier": {
"value": "表 1 比较多种研究模型与 CLI（通过终端命令使用的程序界面） 的组合，包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2，以及 OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等；研究模型选择数据和训练代码，不等于被训练的四个基础模型。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
},
"seed": {
"value": "用 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Gemini CLI（通过终端命令使用的程序界面） 等标准开发工具 agent，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估运行框架不允许改。",
"sources": [
{
"label": "任务定义与 agent 执行设置",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"selection": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"evaluation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"isolation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
]
}
},
{
"label": "ArenaHard Writing / HealthBench-Easy",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "每次运行指定一个目标基准与一个基础模型，由 agent 自行选择或生成训练数据；预算单 H100、十小时。",
"selection": "可调用 evaluate.py 查询目标基准成绩，也可用 --limit 缩小评估规模，结果用于改训练方案。",
"evaluation": "在本行指定基准上评测提交的训练后模型。",
"isolation": "禁止用测试题训练，但开发时允许查询评测成绩；不能表述为分数全程隐藏。",
"roles": {
"executor": {
"value": "研究 agent 操作终端完成后训练；最终答题的是提交的 Qwen3-1.7B、Qwen3-4B、SmolLM3-3B 或 Gemma-3-4B 检查点，交给固定评测程序运行。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
}
]
},
"modifier": {
"value": "表 1 比较多种研究模型与 CLI（通过终端命令使用的程序界面） 的组合，包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2，以及 OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等；研究模型选择数据和训练代码，不等于被训练的四个基础模型。",
"sources": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2603.08640#S5.SS2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.08640#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2603.08640#S2.SS2.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.08640#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
},
"seed": {
"value": "用 Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）、Gemini CLI（通过终端命令使用的程序界面） 等标准开发工具 agent，能读写文件、运行代码、处理上下文和权限。提供一张 H100 与 10 小时预算，修改对象是指定基础模型的训练结果；评估运行框架不允许改。",
"sources": [
{
"label": "任务定义与 agent 执行设置",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"selection": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"evaluation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
],
"isolation": [
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.08640#A2"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2603.08640#A5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 已经能写代码、操作开发工具，但这些能力是否足以独立完成模型后训练仍缺少直接评测。已有 AI 研发基准多测狭窄子任务或论文复现，不能回答 agent 能否自行组织完整训练流程并实际提升模型能力，因此需要端到端测量这项研发能力。",
"sources": [
{
"label": "§1 Introduction · Why post-training?",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否自主完成模型后训练，把编程与实验能力转化为另一个模型的实际能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.08640"
}
]
},
{
"key": "conclusion",
"value": "能取得进步但总体仍落后于官方指令模型；同时发现训练测试集、替换模型等评测违规方式。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.08640"
}
]
}
],
"fields": {
"modifier": "研究端包括 Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 4.6／4.5、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） + GPT-5.1／5.2、OpenCode（可连接不同模型的开源 coding agent 工具） + GPT-5.1 Codex Max 等配置，负责找数据和写训练代码。最终受训模型是另一组基础模型。",
"object": "给定基础模型的后训练参数；研究 agent 选择数据和训练方法。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2604.10547",
"title": "Agent² RL-Bench",
"url": "https://arxiv.org/abs/2604.10547",
"date": "2026-04-12",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"Data",
"ProcessReward",
"Weights",
"org:peking"
],
"fields": {
"本质定位": "**B-RSI**。专门测 agent 能否 design/implement/debug/execute **agentic RL post-training loop**，而不是只写一个训练脚本。",
"被测系统 / feedback": "base model + task data + grading API + fixed budget；6 tasks/3 levels。",
"证据边界与关键结论": "有单个 ALFWorld 大幅提升，但稳定 online RL 很少；成功路线大量依赖 supervised warm-up。"
},
"details": [],
"occurrences": [
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 224,
"fields": {
"时间": "2026-04-12",
"论文": "[Agent² RL-Bench](https://arxiv.org/abs/2604.10547)",
"级别": "**K**",
"它真正测什么": "**B-RSI**。专门测 agent 能否 design/implement/debug/execute **agentic RL post-training loop**，而不是只写一个训练脚本。",
"被测系统 / feedback": "base model + task data + grading API + fixed budget；6 tasks/3 levels。",
"证据边界与关键结论": "有单个 ALFWorld 大幅提升，但稳定 online RL 很少；成功路线大量依赖 supervised warm-up。",
"标签": "`#Data #Weights #ProcessReward #B-RSI`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "专门测 agent 能否 design/implement/debug/execute **agentic RL post-training loop**，而不是只写一个训练脚本。",
"novelty": "要求 agent 实际串起交互数据采集、奖励设计、训练与调试；协议也允许监督微调，需要单独检查成绩来自哪一种训练。",
"object": "研究 agent 开发的训练程序，以及提交的模型参数版本。",
"executor": "第一组交付并测试Qwen2.5-7B-Instruct的训练后模型；受控CLI（通过终端命令使用的程序界面）组交付Qwen3-8B-Base的训练后模型。负责写训练代码的执行模型不是这两个目标模型。",
"modifier": "第一组：GPT-5.4驱动OpenHands/OpenCode，Claude Opus4.6驱动Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。受控组执行模型包括Opus4.6、Sonnet4.5、GPT-5.4、GPT-5.2、GPT-4o、Gemini2.5 Flash。",
"roleContext": "base model + task data + grading API + fixed budget；6 tasks/3 levels。",
"seed": "给研究 task agent 提供可运行训练/评估工作区；静态题用 OpenCompass 等评估，交互题各有环境执行尝试（从开始做任务到得到结果的过程）。允许 强化学习（根据奖励调整模型行为），也明确允许 监督微调（用示范数据训练模型），因此完成训练或涨分不代表成功建立 强化学习（根据奖励调整模型行为） 流程。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "候选能否评估、任务成绩与提交轨迹；最终按最佳有效提交计分。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "GSM8K",
"evolve": "标准 training",
"selection": "grading 由框架管理",
"test": "标准 test",
"isolation": "标准划分",
"note": ""
},
{
"name": "HumanEval",
"evolve": "164题中的82个 training-visible problems",
"selection": "框架控制 grading API",
"test": "另82个 disjoint held-out problems",
"isolation": "按题目索引分开",
"note": "不是原 HumanEval 全164题的排行榜口径；允许 SFT，不能把涨分全部归因于 RL。"
}
],
"takeaway": "有单个 ALFWorld 大幅提升，但稳定 online RL 很少；成功路线大量依赖 supervised warm-up。",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2604.10547v2"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：GSM8K（小学数学应用题基准） 标准 训练集；HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 82 道可见题；其他任务给相应训练资料。\n\n调试 / 选版本数据：训练过程和提交接口反馈，Git 历史用于分析方法选择。\n\n最终测试数据：GSM8K（小学数学应用题基准） 测试集、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 另 82 题；AlpacaEval 2.0、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、DeepSearchQA 200。\n\n数据隔离与证据边界：HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 是自定义 82/82，不直接对比全量 164 榜单；最佳提交评分不同于只提交一次模型。",
"cycle": "读取可见数据和任务，写训练代码并提交候选；记录失败提交、路线切换和分数，区分流程工程与学习算法改进。",
"train": "GSM8K（小学数学应用题基准） 标准 训练集；HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 82 道可见题；其他任务给相应训练资料。",
"debug": "训练过程和提交接口反馈，Git 历史用于分析方法选择。",
"test": "GSM8K（小学数学应用题基准） 测试集、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 另 82 题；AlpacaEval 2.0、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、DeepSearchQA 200。",
"isolation": "HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 是自定义 82/82，不直接对比全量 164 榜单；最佳提交评分不同于只提交一次模型。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "给研究 agent 提供可运行训练/评估工作区；静态题用 OpenCompass 等评估，交互题各有环境 rollout。允许 RL，也明确允许 SFT，因此完成训练或涨分不代表成功建立 RL 流程。",
"protocol": "**训练→测试：**GSM8K 用标准 train/test；HumanEval 将原 164 题按题号互斥拆为 82 道训练可见、82 道留出评分题，不能与标准全量榜单直接比。\n\n**其他评估：**AlpacaEval 2.0 用参考划分；ALFWorld 134 episodes、WebShop 100 instructions、DeepSearchQA 200 samples。交互任务训练轨迹与这些评估集合的具体隔离本轮待核实。",
"sections": "任务协议、Table 5",
"source": "https://arxiv.org/abs/2604.10547",
"version": "2604.10547v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "737ea9dc60214fed0be2baf223188e0aa06f63c0c8453f38e9beb14eed02f35c",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "第一组交付并测试Qwen2.5-7B-Instruct的训练后模型；受控CLI（通过终端命令使用的程序界面）组交付Qwen3-8B-Base的训练后模型。负责写训练代码的执行模型不是这两个目标模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10547#S3.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2604.10547#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "第一组：GPT-5.4驱动OpenHands/OpenCode，Claude Opus4.6驱动Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。受控组执行模型包括Opus4.6、Sonnet4.5、GPT-5.4、GPT-5.2、GPT-4o、Gemini2.5 Flash。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10547#S3.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2604.10547#A1.SS4"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "研究 agent 开发的训练程序，以及提交的模型参数版本。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "候选能否评估、任务成绩与提交轨迹；最终按最佳有效提交计分。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "给研究 task agent 提供可运行训练/评估工作区；静态题用 OpenCompass 等评估，交互题各有环境执行尝试（从开始做任务到得到结果的过程）。允许 强化学习（根据奖励调整模型行为），也明确允许 监督微调（用示范数据训练模型），因此完成训练或涨分不代表成功建立 强化学习（根据奖励调整模型行为） 流程。",
"sources": [
{
"label": "任务协议、Table 5",
"url": "https://arxiv.org/abs/2604.10547"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "读取可见数据和任务，写训练代码并提交候选；记录失败提交、路线切换和分数，区分流程工程与学习算法改进。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GSM8K（小学数学应用题基准） 标准 训练集；HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 82 道可见题；其他任务给相应训练资料。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练过程和提交接口反馈，Git 历史用于分析方法选择。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GSM8K（小学数学应用题基准） 测试集、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 另 82 题；AlpacaEval 2.0、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、DeepSearchQA 200。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 是自定义 82/82，不直接对比全量 164 榜单；最佳提交评分不同于只提交一次模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "要求 agent 实际串起交互数据采集、奖励设计、训练与调试；协议也允许监督微调，需要单独检查成绩来自哪一种训练。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.10547v2",
"version": "2604.10547v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重点检验 agent 能否真正闭合 online RL 的数据采集、奖励、训练与调试循环；允许 SFT，因此单次涨分不证明 RL 设计成功。",
"feedbackCases": [
{
"label": "GSM8K / HumanEval",
"data": "GSM8K（小学数学应用题基准） 标准 训练／测试；HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 82 题可见、82 题评测",
"scoring": "数学按答案检查器，代码按单元测试评分。",
"visible": "提交服务器仅返回标量分数与当前最好分数；测试集不挂载到 agent 工作区。",
"use": "agent 可以多次训练并提交，最终取最好有效提交；因此隐藏题内容不等于分数完全未用于选择。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"judgment": "GSM8K 规则验证答案；HumanEval 单元测试"
},
{
"label": "AlpacaEval 2.0",
"data": "静态开放回答任务",
"scoring": "使用任务专属模型评审比较回答，而非程序答案匹配。",
"visible": "提交服务器仅返回标量分数与当前最好分数；测试集不挂载到 agent 工作区。",
"use": "agent 可以多次训练并提交，最终取最好有效提交；因此隐藏题内容不等于分数完全未用于选择。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"judgment": "LLM 作单轮回答偏好评审；本文未在该段指定裁判型号"
},
{
"label": "ALFWorld / WebShop",
"data": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100 个评测任务",
"scoring": "前者以环境二元成功计分，后者以购物任务奖励计分。",
"visible": "提交服务器仅返回标量分数与当前最好分数；测试集不挂载到 agent 工作区。",
"use": "agent 可以多次训练并提交，最终取最好有效提交；因此隐藏题内容不等于分数完全未用于选择。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"judgment": "ALFWorld 环境二元成功；WebShop 环境连续奖励"
},
{
"label": "DeepSearchQA",
"data": "200 个评测问题",
"scoring": "执行搜索交互后，由问答评审检查最终回答正确性。",
"visible": "提交服务器仅返回标量分数与当前最好分数；测试集不挂载到 agent 工作区。",
"use": "agent 可以多次训练并提交，最终取最好有效提交；因此隐藏题内容不等于分数完全未用于选择。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"judgment": "自建搜索交互与模型判分器；不等同于原基准排行榜协议"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "GSM8K（小学数学应用题基准） 标准 训练集；HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 82 道可见题；其他任务给相应训练资料。",
"selection": "训练过程和提交接口反馈，Git 历史用于分析方法选择。",
"evaluation": "GSM8K（小学数学应用题基准） 测试集、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 另 82 题；AlpacaEval 2.0、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134、WebShop（根据用户要求挑选和购买商品的交互基准） 100、DeepSearchQA 200。",
"isolation": "HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 是自定义 82/82，不直接对比全量 164 榜单；最佳提交评分不同于只提交一次模型。",
"roles": {
"executor": {
"value": "第一组交付并测试Qwen2.5-7B-Instruct的训练后模型；受控CLI（通过终端命令使用的程序界面）组交付Qwen3-8B-Base的训练后模型。负责写训练代码的执行模型不是这两个目标模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10547#S3.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2604.10547#A1.SS4"
}
]
},
"modifier": {
"value": "第一组：GPT-5.4驱动OpenHands/OpenCode，Claude Opus4.6驱动Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。受控组执行模型包括Opus4.6、Sonnet4.5、GPT-5.4、GPT-5.2、GPT-4o、Gemini2.5 Flash。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "附录E.3",
"url": "https://arxiv.org/html/2604.10547#A5.SS3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.10547#S3.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2604.10547#A1.SS4"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "给研究 task agent 提供可运行训练/评估工作区；静态题用 OpenCompass 等评估，交互题各有环境执行尝试（从开始做任务到得到结果的过程）。允许 强化学习（根据奖励调整模型行为），也明确允许 监督微调（用示范数据训练模型），因此完成训练或涨分不代表成功建立 强化学习（根据奖励调整模型行为） 流程。",
"sources": [
{
"label": "任务协议、Table 5",
"url": "https://arxiv.org/abs/2604.10547"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"selection": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
],
"isolation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2604.10547#S2.SS1"
},
{
"label": "§2.6",
"url": "https://arxiv.org/html/2604.10547#S2.SS6.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "强化学习后训练要求 agent 自行产生交互轨迹、计算奖励、持续收集数据，并处理训练不稳定等问题。已有后训练评测多在预设流程里优化部分组件，不能充分检验 agent 是否能独立搭起并运行这整个训练闭环，也就难判断它能否承担实际强化学习研发。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.10547#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否设计、调试并运行完整强化学习流程，尤其区分会做监督训练与能稳定完成在线交互学习。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.10547"
}
]
},
{
"key": "conclusion",
"value": "少数运行能实现显著提升，但多数有效路线仍依赖监督学习，稳定完成在线强化学习仍少见。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.10547"
}
]
}
],
"fields": {
"object": "研究 agent 开发的训练程序，以及提交的模型参数版本。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.10547"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2606.04261",
"title": "Curation-Bench",
"url": "https://arxiv.org/abs/2606.04261",
"date": "2026-06-02",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"BenchmarkScore",
"Data",
"Improver",
"org:berkeley",
"org:uiuc"
],
"fields": {
"本质定位": "**B-RSI / M-Data**。固定 model/training/eval，只让 agent 修改 data-curation policy，专门隔离“research decision quality”。",
"被测系统 / feedback": "generalist coding agents + fixed training/eval pipeline。",
"证据边界与关键结论": "agent 很会局部 tuning，却很少发现新 policy family，形成 **execution–research gap**；method-guided scaffold 可改善探索。"
},
"details": [],
"occurrences": [
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 225,
"fields": {
"时间": "2026-06-02",
"论文": "[Curation-Bench](https://arxiv.org/abs/2606.04261)",
"级别": "**K**",
"它真正测什么": "**B-RSI / M-Data**。固定 model/training/eval，只让 agent 修改 data-curation policy，专门隔离“research decision quality”。",
"被测系统 / feedback": "generalist coding agents + fixed training/eval pipeline。",
"证据边界与关键结论": "agent 很会局部 tuning，却很少发现新 policy family，形成 **execution–research gap**；method-guided scaffold 可改善探索。",
"标签": "`#Data #Improver #BenchmarkScore #B-RSI`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI",
"M-Data"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "固定 model/training/eval，只让 agent 修改 data-curation policy，专门隔离“research decision quality”。",
"novelty": "固定模型、训练配方和最终评测，只让研究 agent 改变数据策略，以便把数据研究能力从其他工程改动中分离出来。",
"object": "交给固定训练后端的数据子集与数据筛选策略。",
"executor": "编码 task agent 筛选数据，固定后端训练并测试目标模型；主视觉语言实验目标为LLaVA1.5-7B，扩展另用Qwen2-VL-2B、Qwen2.5-VL-3B。",
"modifier": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7/Sonnet4.6、Codex + GPT-5.4/GPT-5.3负责数据策划；开源对照通过OpenHands运行Kimi K2.5和Qwen3.5-397B。",
"roleContext": "generalist coding agents + fixed training/eval pipeline。",
"seed": "所有 agent 共用工作区、命令行和提交门控。Claude Code/Codex 自带文件编辑与命令行；开放模型经 OpenHands 接入。训练配方由平台固定，task agent 重点选择数据子集而不是任意改变训练代码。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "每次训练后返回视觉基准分数，开发者据此修订数据策略。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "agent 很会局部 tuning，却很少发现新 policy family，形成 **execution–research gap**；method-guided scaffold 可改善探索。"
}
],
"takeaway": "agent 很会局部 tuning，却很少发现新 policy family，形成 **execution–research gap**；method-guided scaffold 可改善探索。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：LLaVA-665K 或 Vision-Flan 186k 中选 10k；另有 DataComp Small 图文筛选训练 ViT-B/32 CLIP。\n\n调试 / 选版本数据：每次候选训练后的评测反馈用于下一轮筛选，通常每会话十轮。\n\n最终测试数据：MMVet、LLaVA-Bench、MMBench、MMMU 验证集、MMStar、MathVista-Mini、OCRBench、HallusionBench；CLIP 用 DataComp 38 任务。\n\n数据隔离与证据边界：这些评测重复参与筛选反馈；不能把它们写成只在终点出现的独立测试。",
"cycle": "检查候选池、写筛选程序、提交子集、训练评估再迭代；训练算法和评估接口固定。",
"train": "LLaVA-665K 或 Vision-Flan 186k 中选 10k；另有 DataComp Small 图文筛选训练 ViT-B/32 CLIP。",
"debug": "每次候选训练后的评测反馈用于下一轮筛选，通常每会话十轮。",
"test": "MMVet、LLaVA-Bench、MMBench、MMMU 验证集、MMStar、MathVista-Mini、OCRBench、HallusionBench；CLIP 用 DataComp 38 任务。",
"isolation": "这些评测重复参与筛选反馈；不能把它们写成只在终点出现的独立测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "所有代理共用工作区、命令行和提交门控。Claude Code/Codex 自带文件编辑与 shell；开放模型经 OpenHands 接入。训练配方由平台固定，agent 重点选择数据子集而不是任意改变训练代码。",
"protocol": "**数据筛选→训练/评估：**从 LLaVA-665K 选 10k 样本训练 LLaVA-1.5-7B 或 SmolVLM；也从 Vision-Flan 186k 选 10k 训练 SmolVLM，报告 8 个评测。DataComp Small 则从图文池筛数据、从零训练 ViT-B/32 CLIP，评估 38 个任务。\n\n**反馈边界：**提交子集后的评测分数会反馈并用于继续筛选，不是只在终点测试一次。8 个视觉 benchmark 的完整名称和独立最终保留集本轮待核实。",
"sections": "主实验；附录 D.1–D.3",
"source": "https://arxiv.org/abs/2606.04261",
"version": "2606.04261v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c631b22845c4c2b906b88f9a3cd47fb06b615c73c7c2e29f995b6dc70af7532f",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "编码 task agent 筛选数据，固定后端训练并测试目标模型；主视觉语言实验目标为LLaVA1.5-7B，扩展另用Qwen2-VL-2B、Qwen2.5-VL-3B。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.04261#A4.SS1.SSS0.Px3"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2606.04261#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7/Sonnet4.6、Codex + GPT-5.4/GPT-5.3负责数据策划；开源对照通过OpenHands运行Kimi K2.5和Qwen3.5-397B。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.04261#A4.SS1.SSS0.Px3"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2606.04261#A4.SS5"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "交给固定训练后端的数据子集与数据筛选策略。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "每次训练后返回视觉基准分数，开发者据此修订数据策略。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "所有 agent 共用工作区、命令行和提交门控。Claude Code/Codex 自带文件编辑与命令行；开放模型经 OpenHands 接入。训练配方由平台固定，task agent 重点选择数据子集而不是任意改变训练代码。",
"sources": [
{
"label": "主实验；附录 D.1–D.3",
"url": "https://arxiv.org/abs/2606.04261"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "检查候选池、写筛选程序、提交子集、训练评估再迭代；训练算法和评估接口固定。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LLaVA-665K 或 Vision-Flan 186k 中选 10k；另有 DataComp Small 图文筛选训练 ViT-B/32 CLIP。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "每次候选训练后的评测反馈用于下一轮筛选，通常每会话十轮。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "MMVet、LLaVA-Bench、MMBench、MMMU 验证集、MMStar、MathVista-Mini、OCRBench、HallusionBench；CLIP 用 DataComp 38 任务。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这些评测重复参与筛选反馈；不能把它们写成只在终点出现的独立测试。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "固定模型、训练配方和最终评测，只让研究 agent 改变数据策略，以便把数据研究能力从其他工程改动中分离出来。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.04261v1",
"version": "2606.04261v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "固定模型、训练配方和评测，只开放 data policy；区分“会执行和局部调参”与“发现不同方法族”的研究能力。",
"feedbackCases": [
{
"label": "视觉语言数据筛选",
"data": "从 LLaVA-665K 或 Vision-Flan 186k 中选 10k；固定训练流程。",
"scoring": "训练后用 VLMEvalKit 跑 MMVet、LLaVA-Bench、MMBench、MMMU 验证集、MMStar、MathVista-Mini、OCRBench、HallusionBench。选择题/可核对答案用各自客观规则；需要模型判读的指标统一独立 Qwen3.5-27B。",
"visible": "每次评测分数与可用错误分析供下一轮数据筛选，通常十轮；数据本身可经过 agent 设计的质量评分，但这不是最终任务判分。",
"use": "固定模型、训练配方和判分服务，只改数据策略；评估套件反复参与迭代，不能全部称为未使用的最终测试。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"judgment": "VLMEvalKit 各任务客观规则 + 独立 Qwen3.5-27B 处理需模型判读的指标"
},
{
"label": "CLIP 图文筛选",
"data": "DataComp Small 图文池，固定训练 ViT-B/32 CLIP。",
"scoring": "在 DataComp 的 38 个任务上按固定套件评价图文表示能力。",
"visible": "训练后的套件成绩反馈给筛选 agent。",
"use": "用真实训练收益检验数据选择，不能以 agent 对图文质量的自评分代替下游成绩。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"judgment": "DataComp 官方 38 项任务的程序化指标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "LLaVA-665K 或 Vision-Flan 186k 中选 10k；另有 DataComp Small 图文筛选训练 ViT-B/32 CLIP。",
"selection": "每次候选训练后的评测反馈用于下一轮筛选，通常每会话十轮。",
"evaluation": "MMVet、LLaVA-Bench、MMBench、MMMU 验证集、MMStar、MathVista-Mini、OCRBench、HallusionBench；CLIP 用 DataComp 38 任务。",
"isolation": "这些评测重复参与筛选反馈；不能把它们写成只在终点出现的独立测试。",
"roles": {
"executor": {
"value": "编码 task agent 筛选数据，固定后端训练并测试目标模型；主视觉语言实验目标为LLaVA1.5-7B，扩展另用Qwen2-VL-2B、Qwen2.5-VL-3B。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.04261#A4.SS1.SSS0.Px3"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2606.04261#A4.SS5"
}
]
},
"modifier": {
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7/Sonnet4.6、Codex + GPT-5.4/GPT-5.3负责数据策划；开源对照通过OpenHands运行Kimi K2.5和Qwen3.5-397B。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.04261#A3.SS1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2606.04261#A4.SS1.SSS0.Px3"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2606.04261#A4.SS5"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"seed": {
"value": "所有 agent 共用工作区、命令行和提交门控。Claude Code/Codex 自带文件编辑与命令行；开放模型经 OpenHands 接入。训练配方由平台固定，task agent 重点选择数据子集而不是任意改变训练代码。",
"sources": [
{
"label": "主实验；附录 D.1–D.3",
"url": "https://arxiv.org/abs/2606.04261"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"selection": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"evaluation": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
],
"isolation": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.04261#S2.SS0.SSS0.Px1"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2606.04261#A4.SS3.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "数据竞赛能控制训练条件、比较数据质量，却通常只评提交的数据筛选方案；自动化机器学习研究评测能观察迭代实验，却常固定训练数据。两者之间缺少一个同时控制其他训练因素、允许 agent 反复研究数据的环境，因此难单独衡量自动数据整理和选择的研发能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.04261#S1"
}
]
},
{
"key": "position",
"value": "评测训练工程已经固定时，agent 能否开展有效的数据筛选研究，并提出超出局部调参的数据策略。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04261"
}
]
},
{
"key": "conclusion",
"value": "普通 agent 能执行循环但常局部调参；要求具体复现和改造已有方法可推动更有效的策略探索。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.04261"
}
]
}
],
"fields": {
"object": "交给固定训练后端的数据子集与数据筛选策略。"
}
},
"attributions": [
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.04261"
}
]
},
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.04261"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"curriculum",
"evaluation"
]
},
{
"id": "2606.05080",
"title": "AutoLab",
"url": "https://arxiv.org/abs/2606.05080",
"date": "2026-06-03",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"BenchmarkScore",
"ExecutableVerifier",
"Improver",
"LongHorizon",
"person:mengdi-wang"
],
"fields": {
"本质定位": "**B-RSI / long-horizon improvement**。36 个任务都给“正确但次优”的 baseline，要求 agent 在 wall-clock budget 内反复 benchmark→edit→measure。",
"被测系统 / feedback": "17 models；system/CUDA/model dev/puzzle 等。",
"证据边界与关键结论": "最重要 predictor 不是 first attempt，而是**能否坚持迭代并吸收 empirical feedback**；多数模型过早终止或无效耗预算。"
},
"details": [],
"occurrences": [
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 226,
"fields": {
"时间": "2026-06-03",
"论文": "[AutoLab](https://arxiv.org/abs/2606.05080)",
"级别": "**K**",
"它真正测什么": "**B-RSI / long-horizon improvement**。36 个任务都给“正确但次优”的 baseline，要求 agent 在 wall-clock budget 内反复 benchmark→edit→measure。",
"被测系统 / feedback": "17 models；system/CUDA/model dev/puzzle 等。",
"证据边界与关键结论": "最重要 predictor 不是 first attempt，而是**能否坚持迭代并吸收 empirical feedback**；多数模型过早终止或无效耗预算。",
"标签": "`#Improver #BenchmarkScore #ExecutableVerifier #LongHorizon #B-RSI`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"priorityBasis": "用户指定阅读重点：Mengdi Wang（王梦迪，Princeton）署名论文。",
"prioritySources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2606.05080"
}
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "36 个任务都给“正确但次优”的 baseline，要求 agent 在 wall-clock budget 内反复 benchmark→edit→measure。",
"novelty": "每题给一个可运行但次优的方案，观察 agent 如何反复测量、修改和继续探索，评价长期改进而非第一份答案。",
"object": "目标任务的代码、训练配置或数据选择产物；执行任务的运行框架固定。",
"executor": "固定Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）框架下比较Opus4.6、Gemini3.1 Pro、GPT-5.4、Grok4-20、Qwen3.6 Plus、DeepSeekV4Pro、GLM5、Kimi K2.6、Hunyuan3 Preview、MiMoV2.5Pro、MiniMaxM2.7。",
"modifier": "同一次运行中的上述任务模型修改题目允许的代码或科研产物；外层Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）和评估器固定。附录另有更旧/更小模型消融。",
"roleContext": "17 models；system/CUDA/model dev/puzzle 等。",
"seed": "AutoLab 给研究/工程 task agent 提供任务专属代码、数据、计算资源和提交判据，允许构造完整解决方案。任务包含系统优化、模型开发等，不是单一固定空白 task agent 的自我改写。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "运行每题的检查程序，既验证产物是否满足正确性约束，也计算运行速度、误差或产物质量等优化目标；目标随任务变化。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "最重要 predictor 不是 first attempt，而是**能否坚持迭代并吸收 empirical feedback**；多数模型过早终止或无效耗预算。"
}
],
"takeaway": "最重要 predictor 不是 first attempt，而是**能否坚持迭代并吸收 empirical feedback**；多数模型过早终止或无效耗预算。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：36 个自建任务：系统 15、挑战 10、模型开发 7、CUDA 4；各题有自己的数据。例：指令任务从 50k 池选至多 5k 做 LoRA（只训练少量适配参数）。\n\n调试 / 选版本数据：各题本地实验与验证接口；例：FLUX 任务用 15 张概念图调 LoRA（只训练少量适配参数），修 OOM 和配置。\n\n最终测试数据：各题自己的终点评分；指令数据选择用不可直接查看的 IFEval，视频预测用留出 Moving MNIST。\n\n数据隔离与证据边界：36 个优化问题不共用一个 训练／测试 划分；终点评分含私有规则，运行框架在模型对照中固定。",
"cycle": "在预算内改代码/配置并执行验证，保存更好产物；统一外层工具、任务定义和评分控制模型间比较。",
"train": "36 个自建任务：系统 15、挑战 10、模型开发 7、CUDA 4；各题有自己的数据。例：指令任务从 50k 池选至多 5k 做 LoRA（只训练少量适配参数）。",
"debug": "各题本地实验与验证接口；例：FLUX 任务用 15 张概念图调 LoRA（只训练少量适配参数），修 OOM 和配置。",
"test": "各题自己的终点评分；指令数据选择用不可直接查看的 IFEval，视频预测用留出 Moving MNIST。",
"isolation": "36 个优化问题不共用一个 训练／测试 划分；终点评分含私有规则，运行框架在模型对照中固定。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "AutoLab 给研究/工程 agent 提供任务专属代码、数据、计算资源和提交判据，允许构造完整解决方案。任务包含系统优化、模型开发等，不是单一固定空白 agent 的自我改写。",
"protocol": "**自建任务套件：**36 题：系统优化 15、挑战题 10、模型开发 7、CUDA 4。\n\n**具体数据例：**指令微调任务从跨 19 来源的 50k 池选至多 5k，LoRA 训练 Qwen2.5-3B-Instruct，最终测不可直接查看的 IFEval；视频任务用 8,000 段 Moving MNIST 训练，在留出视频上测 10 步自回归预测 PSNR。各任务有不同预算和终点评分，不能用一个总 train/test 数字覆盖 36 题；其余逐题数据清单本轮待核实。",
"sections": "附录 A 任务规格；模型开发案例",
"source": "https://arxiv.org/abs/2606.05080",
"version": "2606.05080v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ef74a0fcf694a56184fdb4deb65021ab5f1625cb36f5ad98b0dd8f31ce6249cb",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "固定Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）框架下比较Opus4.6、Gemini3.1 Pro、GPT-5.4、Grok4-20、Qwen3.6 Plus、DeepSeekV4Pro、GLM5、Kimi K2.6、Hunyuan3 Preview、MiMoV2.5Pro、MiniMaxM2.7。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.05080#S3.SS1"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一次运行中的上述任务模型修改题目允许的代码或科研产物；外层Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）和评估器固定。附录另有更旧/更小模型消融。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.05080#S3.SS1"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "目标任务的代码、训练配置或数据选择产物；执行任务的运行框架固定。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行每题的检查程序，既验证产物是否满足正确性约束，也计算运行速度、误差或产物质量等优化目标；目标随任务变化。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "AutoLab 给研究/工程 task agent 提供任务专属代码、数据、计算资源和提交判据，允许构造完整解决方案。任务包含系统优化、模型开发等，不是单一固定空白 task agent 的自我改写。",
"sources": [
{
"label": "附录 A 任务规格；模型开发案例",
"url": "https://arxiv.org/abs/2606.05080"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在预算内改代码/配置并执行验证，保存更好产物；统一外层工具、任务定义和评分控制模型间比较。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "36 个自建任务：系统 15、挑战 10、模型开发 7、CUDA 4；各题有自己的数据。例：指令任务从 50k 池选至多 5k 做 LoRA（只训练少量适配参数）。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "各题本地实验与验证接口；例：FLUX 任务用 15 张概念图调 LoRA（只训练少量适配参数），修 OOM 和配置。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各题自己的终点评分；指令数据选择用不可直接查看的 IFEval，视频预测用留出 Moving MNIST。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "36 个优化问题不共用一个 训练／测试 划分；终点评分含私有规则，运行框架在模型对照中固定。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "每题给一个可运行但次优的方案，观察 agent 如何反复测量、修改和继续探索，评价长期改进而非第一份答案。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.05080v1",
"version": "2606.05080v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "每题提供正确但次优的起点，考反复测量、修改和继续探索；长时优化表现不能用首个答案质量替代。",
"feedbackCases": [
{
"label": "系统 / CUDA / 挑战任务",
"data": "36 个自建任务中的系统 15、挑战十、CUDA 四类。",
"scoring": "先满足可执行正确性约束，再按速度、成本或题目目标计分；各任务有自己的测试和限额。",
"visible": "本地运行、测试报错及允许的提交反馈。",
"use": "在同一工程目标内反复改产物，不能用一个统一问答数据集描述。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"judgment": "程序验证功能正确性，再测速度、成本或任务目标"
},
{
"label": "模型开发任务",
"data": "七个任务；如从 50k 池选≤5k 训练指令模型、15 张概念图训练 FLUX 适配器、Moving MNIST 视频预测。",
"scoring": "指令任务按不可直接查看的 IFEval 格式约束判分；图像任务按图文/图像质量指标；视频任务按留出数据预测表现。",
"visible": "可见训练及本地验证结果，例如 OOM 和配置错误；具体最终资源按任务隔离。",
"use": "优化训练数据或模型产物，基准初始系统与最终评分规则固定。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"judgment": "IFEval 用格式规则；图像／视频任务用指定质量或预测指标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "36 个自建任务：系统 15、挑战 10、模型开发 7、CUDA 4；各题有自己的数据。例：指令任务从 50k 池选至多 5k 做 LoRA（只训练少量适配参数）。",
"selection": "各题本地实验与验证接口；例：FLUX 任务用 15 张概念图调 LoRA（只训练少量适配参数），修 OOM 和配置。",
"evaluation": "各题自己的终点评分；指令数据选择用不可直接查看的 IFEval，视频预测用留出 Moving MNIST。",
"isolation": "36 个优化问题不共用一个 训练／测试 划分；终点评分含私有规则，运行框架在模型对照中固定。",
"roles": {
"executor": {
"value": "固定Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）框架下比较Opus4.6、Gemini3.1 Pro、GPT-5.4、Grok4-20、Qwen3.6 Plus、DeepSeekV4Pro、GLM5、Kimi K2.6、Hunyuan3 Preview、MiMoV2.5Pro、MiniMaxM2.7。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.05080#S3.SS1"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
]
},
"modifier": {
"value": "同一次运行中的上述任务模型修改题目允许的代码或科研产物；外层Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）和评估器固定。附录另有更旧/更小模型消融。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.05080#S3.SS1"
},
{
"label": "Terminus：终端 task agent",
"url": "https://www.tbench.ai/news/terminus"
}
]
},
"seed": {
"value": "AutoLab 给研究/工程 task agent 提供任务专属代码、数据、计算资源和提交判据，允许构造完整解决方案。任务包含系统优化、模型开发等，不是单一固定空白 task agent 的自我改写。",
"sources": [
{
"label": "附录 A 任务规格；模型开发案例",
"url": "https://arxiv.org/abs/2606.05080"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "科学与工程研发往往要反复设计、测量和修改方案，并在时间、算力和噪声约束下安排实验。短时、一次性答题评测看不到这些能力；现有长任务评测又在领域覆盖和区分模型与 agent 框架贡献方面受限，因此不足以评估模型能否持续推进多类研发任务。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.05080#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 面对已有可用但次优的工程产物时，能否在长时间实验中持续取得可测量的改进，而非只生成一个初始方案。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.05080"
}
]
},
{
"key": "conclusion",
"value": "持续测量并利用反馈比首轮方案质量更能预测成功；许多模型过早结束或消耗预算却没有进展。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.05080"
}
]
}
],
"fields": {
"object": "目标任务的代码、训练配置或数据选择产物；执行任务的运行框架固定。",
"modifier": "同一次运行中的上述任务模型修改题目允许的代码或科研产物；外层Terminus-2（Terminal-Bench 团队提供的终端 task agent，负责让模型操作命令行环境）和评估器固定。补充实验的另有更旧/更小模型消融。",
"verdict": "运行每题的检查程序，既验证产物是否满足正确性约束，也计算运行速度、误差或产物质量等优化目标；目标随任务变化。"
}
},
"attributions": [
{
"tag": "person:mengdi-wang",
"label": "Mengdi Wang",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2606.05080"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2607.05155",
"title": "EdgeBench",
"url": "https://arxiv.org/abs/2607.05155",
"date": "2026-07-06",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"B-Reliability",
"EnvironmentReward",
"LongHorizon",
"Streaming",
"org:bytedance-seed"
],
"fields": {
"本质定位": "**B-RSI / learning dynamics**。134 real-world ultra-long tasks、约38k小时 interaction，用丰富多层 feedback 测 deployment-time learning curve。",
"被测系统 / feedback": "多代 frontier agents，单 task≥12h。",
"证据边界与关键结论": "performance 随 environment learning 呈 log-sigmoid scaling（reported R²≈0.998），学习速度跨 model generation 加快；这是“self-improvement dynamics”而非特定算法 benchmark。"
},
"details": [],
"occurrences": [
{
"section": "C. End-to-end RSI / autonomous AI R&D / training-loop evaluation",
"line": 227,
"fields": {
"时间": "2026-07-06",
"论文": "[EdgeBench](https://arxiv.org/abs/2607.05155)",
"级别": "**K**",
"它真正测什么": "**B-RSI / learning dynamics**。134 real-world ultra-long tasks、约38k小时 interaction，用丰富多层 feedback 测 deployment-time learning curve。",
"被测系统 / feedback": "多代 frontier agents，单 task≥12h。",
"证据边界与关键结论": "performance 随 environment learning 呈 log-sigmoid scaling（reported R²≈0.998），学习速度跨 model generation 加快；这是“self-improvement dynamics”而非特定算法 benchmark。",
"标签": "`#Streaming #EnvironmentReward #LongHorizon #B-Reliability`"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "134 real-world ultra-long tasks、约38k小时 interaction，用丰富多层 feedback 测 deployment-time learning curve。",
"novelty": "在超长任务中记录能力随时间和计算投入变化的曲线，研究学习速度及持续进步的条件。",
"object": "长程科研或工程任务中的候选方案与保留状态，用于观察环境学习过程。",
"executor": "GPT-5.5/GPT-5.4配Codex；Claude Opus4.8、GLM5.1、DeepSeek-V4-Pro-preview配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。",
"modifier": "各模型在同一次研究运行中据执行与评测反馈修订产物；框架和模型配对固定。Opus4.8另比较200K与1M上下文。",
"roleContext": "多代 frontier agents，单 task≥12h。",
"seed": "每个任务有可持续运行的执行环境、工具和多层反馈，task agent 反复读结果、修改程序/方案并提交检查。不同领域各有初始材料，不是统一只有一个提示词的起点；具体任务工具应按任务规格看。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "本地试验反馈、提交后评分者反馈和宿主端轨迹测量三层。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "performance 随 environment learning 呈 log-sigmoid scaling（reported R²≈0.998），学习速度跨 model generation 加快；这是“self-improvement dynamics”而非特定算法 benchmark。"
}
],
"takeaway": "performance 随 environment learning 呈 log-sigmoid scaling（reported R²≈0.998），学习速度跨 model generation 加快；这是“self-improvement dynamics”而非特定算法 benchmark。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：134 个自建长时任务，六类能力；经验来自每题内的持续交互。\n\n调试 / 选版本数据：工作环境中的实验与受控提交反馈，工作/评判环境分开。\n\n最终测试数据：每模型每题三次独立 12 小时运行；约 38,000 小时是累计交互时长。\n\n数据隔离与证据边界：主结论是同问题持续优化曲线，不是先在其他题训练再对这 134 题一次盲测。",
"cycle": "每题持续运行至少 12 小时，记录各次提交和历史最佳分数，比较不同模型的学习速度与上限。",
"train": "134 个自建长时任务，六类能力；经验来自每题内的持续交互。",
"debug": "工作环境中的实验与受控提交反馈，工作/评判环境分开。",
"test": "每模型每题三次独立 12 小时运行；约 38,000 小时是累计交互时长。",
"isolation": "主结论是同问题持续优化曲线，不是先在其他题训练再对这 134 题一次盲测。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "每个任务有可持续运行的执行环境、工具和多层反馈，agent 反复读结果、修改程序/方案并提交检查。不同领域各有初始材料，不是统一只有一个 prompt 的起点；具体任务工具应按任务规格看。",
"protocol": "**数据/任务：**EdgeBench 自建 134 个超长任务，涵盖科学发现、软件工程、组合优化、专业工作、形式数学和游戏，公开其中 51 题。\n\n**学习与评估：**每题至少支持 12 小时持续交互，主要衡量同一运行内历史最佳分数随时间变化；约 38,000 小时是累计交互量，不是训练集样本量。此协议不是在 134 题外训练再对它们一次测试；逐任务原始数据和隐藏评估划分本轮未完整核实。",
"sections": "benchmark 设计与实验协议",
"source": "https://arxiv.org/abs/2607.05155",
"version": "2607.05155v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e0294ca0aa3ca0dd1a26db872a0cf0ea1327e609ab5e8eacf57ea328fa3aafaf",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5.5/GPT-5.4配Codex；Claude Opus4.8、GLM5.1、DeepSeek-V4-Pro-preview配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各模型在同一次研究运行中据执行与评测反馈修订产物；框架和模型配对固定。Opus4.8另比较200K与1M上下文。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "长程科研或工程任务中的候选方案与保留状态，用于观察环境学习过程。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "本地试验反馈、提交后评分者反馈和宿主端轨迹测量三层。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "每个任务有可持续运行的执行环境、工具和多层反馈，task agent 反复读结果、修改程序/方案并提交检查。不同领域各有初始材料，不是统一只有一个提示词的起点；具体任务工具应按任务规格看。",
"sources": [
{
"label": "benchmark 设计与实验协议",
"url": "https://arxiv.org/abs/2607.05155"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每题持续运行至少 12 小时，记录各次提交和历史最佳分数，比较不同模型的学习速度与上限。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "134 个自建长时任务，六类能力；经验来自每题内的持续交互。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "工作环境中的实验与受控提交反馈，工作/评判环境分开。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "每模型每题三次独立 12 小时运行；约 38,000 小时是累计交互时长。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "主结论是同问题持续优化曲线，不是先在其他题训练再对这 134 题一次盲测。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在超长任务中记录能力随时间和计算投入变化的曲线，研究学习速度及持续进步的条件。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.05155v1",
"version": "2607.05155v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "研究超长任务中的进步曲线和学习速度，而非提出一种固定 updater；拟合的 scaling 关系是这些任务/模型上的经验发现。",
"feedbackCases": [
{
"label": "持续工程任务的三层反馈",
"data": "134 个自建长时任务，六类能力；每模型每题三次独立 12 小时运行。",
"scoring": "本地实验产生可见错误/指标；提交后由隔离评判环境按任务规则评分；宿主端另跟踪进度和轨迹。",
"visible": "agent 可访问本地实验及受控提交反馈，不能读取评判环境的私有验收资产。",
"use": "在当前任务中持续改方案；宿主端测量不必等 agent 主动提交，也不都返回供修改。累计交互时长不是单次训练预算。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"judgment": "本地实验反馈 + 隔离任务评估器；具体判分按工程任务定义"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "134 个自建长时任务，六类能力；经验来自每题内的持续交互。",
"selection": "工作环境中的实验与受控提交反馈，工作/评判环境分开。",
"evaluation": "每模型每题三次独立 12 小时运行；约 38,000 小时是累计交互时长。",
"isolation": "主结论是同问题持续优化曲线，不是先在其他题训练再对这 134 题一次盲测。",
"roles": {
"executor": {
"value": "GPT-5.5/GPT-5.4配Codex；Claude Opus4.8、GLM5.1、DeepSeek-V4-Pro-preview配Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "各模型在同一次研究运行中据执行与评测反馈修订产物；框架和模型配对固定。Opus4.8另比较200K与1M上下文。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
]
},
"seed": {
"value": "每个任务有可持续运行的执行环境、工具和多层反馈，task agent 反复读结果、修改程序/方案并提交检查。不同领域各有初始材料，不是统一只有一个提示词的起点；具体任务工具应按任务规格看。",
"sources": [
{
"label": "benchmark 设计与实验协议",
"url": "https://arxiv.org/abs/2607.05155"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"selection": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"evaluation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
],
"isolation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.05155#S2.SS1"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2607.05155#S2.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.05155#S3.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "预训练不可能预先覆盖不断出现的新知识、工具和环境，但模型能否通过持续与环境交互而变强仍不清楚。已有评测往往缺少有用反馈，或只允许很短的尝试时间，因此难研究增加学习时间和交互计算后，能力是否还能持续增长。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.05155#S1"
}
]
},
{
"key": "position",
"value": "研究模型部署后从真实环境交互中学习的速度与规模规律，考察长时间试验怎样转化为能力增长。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05155"
}
]
},
{
"key": "conclusion",
"value": "在大规模交互记录中报告特定的学习曲线拟合与代际速度变化，同时开放部分任务及评测框架。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.05155"
}
]
}
],
"fields": {
"object": "长程科研或工程任务中的候选方案与保留状态，用于观察环境学习过程。"
}
},
"attributions": [
{
"tag": "org:bytedance-seed",
"label": "ByteDance Seed",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.05155"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "doi.org-10.1016-S0065-2458-08-60418-0",
"title": "Speculations Concerning the First Ultraintelligent Machine",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0",
"date": "1965",
"priority": "R",
"categories": [
"theory"
],
"tags": [],
"fields": {
"本质定位": "提出 intelligence explosion 的经典论证：如果机器设计机器本身也是一种智力活动，那么足够强的机器可设计更强机器。",
"什么在变": "概念上是机器设计能力/机器本身；没有具体可执行 update object。",
"谁来改 / 谁执行": "**改**：未定义自动 modifier。
**执行**：假想 ultraintelligent machine。",
"基础 harness": "无现代 harness 概念。",
"Feedback": "无实验 feedback；逻辑论证。",
"Evolution → Eval": "无 benchmark。",
"Meta-depth": "M3 概念理想，但没有实现。",
"相对之前真正新增什么": "不是算法或实验论文；它提供的是“能力改进能够反馈到改进能力本身”的最早经典 framing。"
},
"details": [],
"occurrences": [
{
"section": "0A. 理论源头",
"line": 249,
"fields": {
"优先级": "**R**",
"时间": "1965",
"论文": "[Speculations Concerning the First Ultraintelligent Machine](https://doi.org/10.1016/S0065-2458(08)60418-0)",
"本质定位": "提出 intelligence explosion 的经典论证：如果机器设计机器本身也是一种智力活动，那么足够强的机器可设计更强机器。",
"什么在变": "概念上是机器设计能力/机器本身；没有具体可执行 update object。",
"谁来改 / 谁执行": "**改**：未定义自动 modifier。
**执行**：假想 ultraintelligent machine。",
"基础 harness": "无现代 harness 概念。",
"Feedback": "无实验 feedback；逻辑论证。",
"Evolution → Eval": "无 benchmark。",
"Meta-depth": "M3 概念理想，但没有实现。",
"相对之前真正新增什么": "不是算法或实验论文；它提供的是“能力改进能够反馈到改进能力本身”的最早经典 framing。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "理论 / 不适用",
"protocolBasis": "理论或形式化前驱，不能套用现代 agent 数据隔离标签。",
"year": "1965",
"depth": [
"M3"
],
"legacyCategories": [
"F"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "提出 intelligence explosion 的经典论证：如果机器设计机器本身也是一种智力活动，那么足够强的机器可设计更强机器。",
"novelty": "提出机器改善自身设计可能进一步提高其设计能力的递归设想；这是一种概念论证，未给出现代 agent 算法或任务实验。",
"object": "概念上是机器本身及设计更好机器的能力；没有定义具体可执行修改对象。",
"executor": "概念上的“超智能机器”，没有具体模型、可执行 agent 或实验配置。",
"modifier": "文章讨论机器设计更好机器的可能性，没有提出可复现的自动修改算法。",
"roleContext": "**改**：未定义自动 modifier。
**执行**：假想 ultraintelligent machine。",
"seed": "这是关于超智能及智能爆发的概念论述，没有现代 task agent 的工具、执行循环或可编辑运行框架；基础运行框架维度不适用。",
"fixed": "",
"verdict": "无实验反馈；逻辑论证。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "理论 / 不适用",
"note": "无 benchmark。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：不适用：未定义数据驱动的训练实验。\n\n调试 / 选版本数据：没有实现层面的调试数据或反馈接口。\n\n最终测试数据：没有评测基准实验，贡献是关于智能爆发的论证。\n\n数据隔离与证据边界：不适用；不能把理论设想当作实证多代改进。",
"cycle": "提出机器设计更聪明机器的递归设想；没有可运行的更新、接受或回退算法。",
"train": "不适用：未定义数据驱动的训练实验。",
"debug": "没有实现层面的调试数据或反馈接口。",
"test": "没有评测基准实验，贡献是关于智能爆发的论证。",
"isolation": "不适用；不能把理论设想当作实证多代改进。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "这是关于超智能及智能爆发的概念论述，没有现代 agent 的工具、执行循环或可编辑运行框架；基础 harness 维度不适用。",
"protocol": "**数据与实验：**不是经验性 benchmark 研究，没有一套用于进化的训练数据和用于验证涨分的测试集。应作为 RSI 概念来源阅读，不能与报告任务成功率的方法直接横比。",
"sections": "原文论述范围",
"source": "https://doi.org/10.1016/S0065-2458(08)60418-0",
"version": "所链接作者原文",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a99d5d5ca791011f65e948f68578d2e5126e9556e9d6ab740bf451384ca37dcd",
"seedStatus": "not-applicable",
"protocolStatus": "not-applicable"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "概念上的“超智能机器”，没有具体模型、可执行 agent 或实验配置。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "文章讨论机器设计更好机器的可能性，没有提出可复现的自动修改算法。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "概念上是机器本身及设计更好机器的能力；没有定义具体可执行修改对象。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "无实验反馈；逻辑论证。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "这是关于超智能及智能爆发的概念论述，没有现代 task agent 的工具、执行循环或可编辑运行框架；基础运行框架维度不适用。",
"sources": [
{
"label": "原文论述范围",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "提出机器设计更聪明机器的递归设想；没有可运行的更新、接受或回退算法。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不适用：未定义数据驱动的训练实验。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "没有实现层面的调试数据或反馈接口。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "没有评测基准实验，贡献是关于智能爆发的论证。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "不适用；不能把理论设想当作实证多代改进。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "提出机器改善自身设计可能进一步提高其设计能力的递归设想；这是一种概念论证，未给出现代 agent 算法或任务实验。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://languagelog.ldc.upenn.edu/myl/Good1964.pdf",
"version": "",
"scope": "原文引言与 intelligence-explosion 论证初核；完整扫描件可访问，但本轮未逐页审读。"
},
"focus": "不是算法或实验论文；它提供的是“能力改进能够反馈到改进能力本身”的最早经典 framing。",
"feedbackCases": [
{
"label": "概念论证，无判分实验",
"data": "没有训练、开发或测试数据集。",
"scoring": "通过关于智能爆发的逻辑论证提出观点，没有实现可查询的判分器。",
"visible": "不适用：没有执行 agent 收到反馈的实验。",
"use": "不能把概念论证当作已在任务数据上验证的自改系统。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"judgment": "理论论证，无训练反馈或实验判分器"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "不适用：未定义数据驱动的训练实验。",
"selection": "没有实现层面的调试数据或反馈接口。",
"evaluation": "没有评测基准实验，贡献是关于智能爆发的论证。",
"isolation": "不适用；不能把理论设想当作实证多代改进。",
"roles": {
"executor": {
"value": "概念上的“超智能机器”，没有具体模型、可执行 agent 或实验配置。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
},
"modifier": {
"value": "文章讨论机器设计更好机器的可能性，没有提出可复现的自动修改算法。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
},
"seed": {
"value": "这是关于超智能及智能爆发的概念论述，没有现代 task agent 的工具、执行循环或可编辑运行框架；基础运行框架维度不适用。",
"sources": [
{
"label": "原文论述范围",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
}
},
"sources": {
"evolution": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"selection": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"evaluation": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
],
"isolation": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "这篇早期理论文章关心的是怎样构造第一台超越人类智力的机器。作者认为，对思维、记忆和意义如何在物理系统中实现的理解仍不足，而首次构造可能成本极高；因此需要探索可支撑学习与记忆的神经结构，并讨论这种机器的潜在价值。",
"sources": [
{
"label": "§1 Introduction（原刊第 31–33 页）",
"url": "https://languagelog.ldc.upenn.edu/myl/Good1964.pdf#page=1"
}
]
},
{
"key": "position",
"value": "讨论当机器的设计能力超过人类时，机器改进机器是否可能引发不断加速的智能增长。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
},
{
"key": "conclusion",
"value": "形成“智能爆炸”的概念论证；它是理论设想，没有可运行的修改流程或实证结果。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
}
],
"fields": {
"object": "概念上是机器本身及设计更好机器的能力；没有定义具体可执行修改对象。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "people.idsia.ch-juergen",
"title": "Evolutionary Principles in Self-Referential Learning (Meta-Meta-Hook)",
"url": "https://people.idsia.ch/~juergen/diploma.html",
"date": "1987",
"priority": "R",
"categories": [
"theory"
],
"tags": [],
"fields": {
"本质定位": "早期 self-referential learning / learning-to-learn：不仅搜索 task solution，也考虑改变产生学习/搜索变化的高阶机制。",
"什么在变": "学习/变异机制的高阶参数。",
"谁来改 / 谁执行": "**改**：系统内部 evolutionary/self-referential mechanism。
**执行**：同一学习系统。",
"基础 harness": "早期程序搜索/进化系统。",
"Feedback": "fitness / search objective。",
"Evolution → Eval": "理论与早期实验范式；不是现代 agent benchmark。",
"Meta-depth": "M2–M3。",
"相对之前真正新增什么": "比一般 meta-learning 更靠近“meta-level rule 也成为 evolution object”，是后续 Promptbreeder / STOP / Hyperagents 的思想前史。"
},
"details": [],
"occurrences": [
{
"section": "0A. 理论源头",
"line": 250,
"fields": {
"优先级": "**R**",
"时间": "1987",
"论文": "[Evolutionary Principles in Self-Referential Learning (Meta-Meta-Hook)](https://people.idsia.ch/~juergen/)",
"本质定位": "早期 self-referential learning / learning-to-learn：不仅搜索 task solution，也考虑改变产生学习/搜索变化的高阶机制。",
"什么在变": "学习/变异机制的高阶参数。",
"谁来改 / 谁执行": "**改**：系统内部 evolutionary/self-referential mechanism。
**执行**：同一学习系统。",
"基础 harness": "早期程序搜索/进化系统。",
"Feedback": "fitness / search objective。",
"Evolution → Eval": "理论与早期实验范式；不是现代 agent benchmark。",
"Meta-depth": "M2–M3。",
"相对之前真正新增什么": "比一般 meta-learning 更靠近“meta-level rule 也成为 evolution object”，是后续 Promptbreeder / STOP / Hyperagents 的思想前史。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "理论 / 不适用",
"protocolBasis": "理论或形式化前驱，不能套用现代 agent 数据隔离标签。",
"year": "1987",
"depth": [
"M2",
"M3"
],
"legacyCategories": [
"F"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "早期 self-referential learning / learning-to-learn：不仅搜索 task solution，也考虑改变产生学习/搜索变化的高阶机制。",
"novelty": "研究修改程序的程序也能被搜索的自指机制，并讨论程序生成、连接和收益分配怎样通过 agent 竞争组织起来。",
"object": "产生学习与程序变异的高阶机制及参数。",
"executor": "文中的自指学习程序执行任务；这是早期程序学习设定，不是 语言模型 运行框架实验。",
"modifier": "学习程序内部的自指学习机制修改自身学习策略；不存在 GPT、Claude 等基础模型型号。",
"roleContext": "**改**：系统内部 evolutionary/self-referential mechanism。
**执行**：同一学习系统。",
"seed": "早期程序搜索/自指学习构造：meta-level GP 搜索能修改程序的程序，PSALM 让生成、连接和分配 贡献 的 task agent 竞争。它不是现代 语言模型＋命令行的 task agent 运行框架。",
"fixed": "",
"verdict": "程序搜索中预先定义的目标或适应度，即用来比较候选程序好坏的分数。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "理论 / 不适用",
"note": "理论与早期实验范式；不是现代 agent benchmark。"
}
],
"takeaway": "原论文摘要明确说其初步实验不足以展示具体 self-reference，受当时计算能力限制，主要是启发性构造；不能把高阶可编辑设计直接当成已验证的多代自改进。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：自建语言中的初步经验任务，非现代统一训练集。\n\n调试 / 选版本数据：环境中的演化压力与信用分配驱动结构变化。\n\n最终测试数据：作者报告少量初步实验，并在摘要明确说不足以展示具体自指效果。\n\n数据隔离与证据边界：证据是早期实现探索，不可解读为已验证的无限递归改进。",
"cycle": "让程序同时操作任务知识和学习策略，通过演化压力及 贡献 分配保留有用结构；PSALM 使用自扩展符号语言。",
"train": "自建语言中的初步经验任务，非现代统一训练集。",
"debug": "环境中的演化压力与信用分配驱动结构变化。",
"test": "作者报告少量初步实验，并在摘要明确说不足以展示具体自指效果。",
"isolation": "证据是早期实现探索，不可解读为已验证的无限递归改进。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "早期程序搜索/自指学习构造：meta-level GP 搜索能修改程序的程序，PSALM 让生成、连接和分配 credit 的 agent 竞争。它不是现代 LLM＋shell 的 agent harness。",
"protocol": "**实验性质：**论文给出早期自指程序学习构造和初步实验；原作者明确表示这些实验不足以展示具体的 self-reference。未使用现代 benchmark；各早期实验的完整输入和数量本轮未核实，不把概念设计当作已经证明多代能力增长。",
"sections": "原论文摘要与构造",
"source": "https://people.idsia.ch/~juergen/diploma.html",
"version": "所链接作者原文",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c63167c37e08652ad78a3c53e9ca5d7e993f029c332945ba7c25897288d7571f",
"seedStatus": "not-applicable",
"protocolStatus": "not-applicable"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "文中的自指学习程序执行任务；这是早期程序学习设定，不是 语言模型 运行框架实验。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "学习程序内部的自指学习机制修改自身学习策略；不存在 GPT、Claude 等基础模型型号。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "产生学习与程序变异的高阶机制及参数。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "程序搜索中预先定义的目标或适应度，即用来比较候选程序好坏的分数。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "早期程序搜索/自指学习构造：meta-level GP 搜索能修改程序的程序，PSALM 让生成、连接和分配 贡献 的 task agent 竞争。它不是现代 语言模型＋命令行的 task agent 运行框架。",
"sources": [
{
"label": "原论文摘要与构造",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "让程序同时操作任务知识和学习策略，通过演化压力及 贡献 分配保留有用结构；PSALM 使用自扩展符号语言。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "自建语言中的初步经验任务，非现代统一训练集。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "环境中的演化压力与信用分配驱动结构变化。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "作者报告少量初步实验，并在摘要明确说不足以展示具体自指效果。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "证据是早期实现探索，不可解读为已验证的无限递归改进。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "研究修改程序的程序也能被搜索的自指机制，并讨论程序生成、连接和收益分配怎样通过 agent 竞争组织起来。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://people.idsia.ch/~juergen/diploma1987ocr.pdf",
"version": "",
"scope": "原论文扫描件的摘要与引言定位初核；另对照作者说明页。未逐页审读全文。"
},
"focus": "原作者区分 meta-level GP 与 PSALM：前者递归寻找更好的程序修改程序，后者让生成、连接和分配 credit 的 agents 竞争，且总 credit 受守恒约束。",
"feedbackCases": [
{
"label": "早期自指学习实验",
"data": "作者自建语言及初步程序任务，非现代统一基准。",
"scoring": "环境目标/适应度和信用分配用于比较程序、分配搜索资源。",
"visible": "演化压力与程序执行结果。",
"use": "文中初步实验不足以证明具体自指收益，不能补成已有现代 训练／测试 协议。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"judgment": "自建环境的目标／适应度与信用分配，无现代 benchmark 或 LLM 裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "自建语言中的初步经验任务，非现代统一训练集。",
"selection": "环境中的演化压力与信用分配驱动结构变化。",
"evaluation": "作者报告少量初步实验，并在摘要明确说不足以展示具体自指效果。",
"isolation": "证据是早期实现探索，不可解读为已验证的无限递归改进。",
"roles": {
"executor": {
"value": "文中的自指学习程序执行任务；这是早期程序学习设定，不是 语言模型 运行框架实验。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
},
"modifier": {
"value": "学习程序内部的自指学习机制修改自身学习策略；不存在 GPT、Claude 等基础模型型号。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
},
"seed": {
"value": "早期程序搜索/自指学习构造：meta-level GP 搜索能修改程序的程序，PSALM 让生成、连接和分配 贡献 的 task agent 竞争。它不是现代 语言模型＋命令行的 task agent 运行框架。",
"sources": [
{
"label": "原论文摘要与构造",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
}
},
"sources": {
"evolution": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"selection": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"evaluation": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
],
"isolation": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有学习程序能在特定领域工作，但利用旧知识学习新知识的策略非常多样，且依赖具体情境。预先放入少数看似合理的学习算法，很难覆盖这些需要；作者因此主张系统不仅学习任务知识，还应学习和修改“怎样学习”的策略。",
"sources": [
{
"label": "§1 Introduction（学习策略的多样性与情境依赖）",
"url": "https://people.idsia.ch/~juergen/diploma1987ocr.pdf"
}
]
},
{
"key": "position",
"value": "讨论学习系统能否把自身的学习机制也作为学习对象，从而形成更高阶的递归适应。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
},
{
"key": "conclusion",
"value": "提供早期递归学习思想，为后来讨论“改进者本身能否被改进”建立概念起点。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
}
],
"fields": {
"object": "产生学习与程序变异的高阶机制及参数。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "cs-0207097",
"title": "Optimal Ordered Problem Solver (OOPS)",
"url": "https://arxiv.org/abs/cs/0207097",
"date": "2002-07-31 / 2004",
"priority": "R",
"categories": [
"theory"
],
"tags": [],
"fields": {
"本质定位": "增量 universal program search：后续任务搜索可直接复用和组合此前已验证程序，并同时搜索 domain algorithm 与 search algorithm。",
"什么在变": "已发现程序与可复用 search procedure。",
"谁来改 / 谁执行": "**改**：固定 OOPS universal search 机制。
**执行**：同一 program-search solver。",
"基础 harness": "universal program-search substrate。",
"Feedback": "任务可解性/运行时间。",
"Evolution → Eval": "Towers of Hanoi 等序列任务；不是 held-out agent benchmark。",
"Meta-depth": "M1：可搜 search program，但最外层 OOPS 规则固定。",
"相对之前真正新增什么": "重要新点是“经验不仅提供答案，还能改进之后的搜索过程”；但它不是现代意义的 LLM agent 自编辑 harness。"
},
"details": [],
"occurrences": [
{
"section": "0A. 理论源头",
"line": 251,
"fields": {
"优先级": "**R**",
"时间": "2002-07-31 / 2004",
"论文": "[Optimal Ordered Problem Solver (OOPS)](https://arxiv.org/abs/cs/0207097)",
"本质定位": "增量 universal program search：后续任务搜索可直接复用和组合此前已验证程序，并同时搜索 domain algorithm 与 search algorithm。",
"什么在变": "已发现程序与可复用 search procedure。",
"谁来改 / 谁执行": "**改**：固定 OOPS universal search 机制。
**执行**：同一 program-search solver。",
"基础 harness": "universal program-search substrate。",
"Feedback": "任务可解性/运行时间。",
"Evolution → Eval": "Towers of Hanoi 等序列任务；不是 held-out agent benchmark。",
"Meta-depth": "M1：可搜 search program，但最外层 OOPS 规则固定。",
"相对之前真正新增什么": "重要新点是“经验不仅提供答案，还能改进之后的搜索过程”；但它不是现代意义的 LLM agent 自编辑 harness。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2002",
"depth": [
"M1"
],
"legacyCategories": [
"F"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "增量 universal program search：后续任务搜索可直接复用和组合此前已验证程序，并同时搜索 domain algorithm 与 search algorithm。",
"novelty": "将已解决任务的程序留给后续问题复用，并允许这些程序改变之后的搜索过程，使经验也能改善寻找新解的方式。",
"object": "已发现的程序及可复用的搜索程序；任务正确性标准与时间分配原则固定。",
"executor": "OOPS 搜索并执行候选程序来解题；没有语言模型参与。",
"modifier": "固定的 OOPS 程序搜索算法复用已找到的程序，分配后续搜索时间；候选程序可以调整搜索偏置。",
"roleContext": "**改**：固定 OOPS universal search 机制。
**执行**：同一 program-search solver。",
"seed": "OOPS 是可执行程序搜索系统，保存已解决问题的程序前缀并按概率和计算预算复用；基础对象是通用程序解释/搜索机制，不是预训练语言模型的工具链。",
"fixed": "M1：可搜 search program，但最外层 OOPS 规则固定",
"verdict": "任务可解性/运行时间。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "Towers of Hanoi 等序列任务；不是 held-out agent benchmark。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：逐步提供符号程序任务，包括计数/序列类任务及 Hanoi；训练是增量程序搜索，不是参数拟合。\n\n调试 / 选版本数据：执行候选程序检查是否解题，失败则回溯并分配下一段搜索时间。\n\n最终测试数据：在之后更复杂的任务上衡量先前程序带来的搜索加速。\n\n数据隔离与证据边界：顺序归纳的任务序列，没有现代随机 训练／测试 划分。",
"cycle": "按程序先验和时间预算搜索；新程序必须解决相关任务才冻结保留，后续搜索可复用旧程序，也可搜索修改搜索偏好的程序。",
"train": "逐步提供符号程序任务，包括计数/序列类任务及 Hanoi；训练是增量程序搜索，不是参数拟合。",
"debug": "执行候选程序检查是否解题，失败则回溯并分配下一段搜索时间。",
"test": "在之后更复杂的任务上衡量先前程序带来的搜索加速。",
"isolation": "顺序归纳的任务序列，没有现代随机 训练／测试 划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "OOPS 是可执行程序搜索系统，保存已解决问题的程序前缀并按概率和计算预算复用；基础对象是通用程序解释/搜索机制，不是预训练语言模型的工具链。",
"protocol": "**任务：**论文研究按序解决程序任务及复用先前解，例如 Towers of Hanoi。搜索预算和顺序迁移是评估重点；没有现代意义上固定模型训练集、验证集、benchmark 测试集三段划分。具体实验规模本轮待核实。",
"sections": "OOPS 程序搜索与实验",
"source": "https://arxiv.org/abs/cs/0207097",
"version": "cs/0207097v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "b6a02135078588be27102585b98c88955e435f5d52ae305c2a0ec102acb0dced",
"seedStatus": "not-applicable",
"protocolStatus": "not-applicable"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "OOPS 搜索并执行候选程序来解题；没有语言模型参与。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定的 OOPS 程序搜索算法复用已找到的程序，分配后续搜索时间；候选程序可以调整搜索偏置。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "已发现的程序及可复用的搜索程序；任务正确性标准与时间分配原则固定。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务可解性/运行时间。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "OOPS 是可执行程序搜索系统，保存已解决问题的程序前缀并按概率和计算预算复用；基础对象是通用程序解释/搜索机制，不是预训练语言模型的工具链。",
"sources": [
{
"label": "OOPS 程序搜索与实验",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "按程序先验和时间预算搜索；新程序必须解决相关任务才冻结保留，后续搜索可复用旧程序，也可搜索修改搜索偏好的程序。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "逐步提供符号程序任务，包括计数/序列类任务及 Hanoi；训练是增量程序搜索，不是参数拟合。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "执行候选程序检查是否解题，失败则回溯并分配下一段搜索时间。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "在之后更复杂的任务上衡量先前程序带来的搜索加速。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "顺序归纳的任务序列，没有现代随机 训练／测试 划分。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将已解决任务的程序留给后续问题复用，并允许这些程序改变之后的搜索过程，使经验也能改善寻找新解的方式。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/abs/cs/0207097",
"version": "",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重要新点是“经验不仅提供答案，还能改进之后的搜索过程”；但它不是现代意义的 LLM agent 自编辑 harness。",
"feedbackCases": [
{
"label": "增量程序搜索",
"data": "顺序提供计数/序列与 Hanoi 等符号任务。",
"scoring": "执行候选程序检查是否解决当前任务，并计运行时间；失败则回溯。",
"visible": "程序是否成功及消耗的搜索预算。",
"use": "将先前程序用于更复杂后续题的搜索加速，不是语言模型根据标注答案微调。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"judgment": "执行候选程序验证当前任务解，并计算搜索／运行时间"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "逐步提供符号程序任务，包括计数/序列类任务及 Hanoi；训练是增量程序搜索，不是参数拟合。",
"selection": "执行候选程序检查是否解题，失败则回溯并分配下一段搜索时间。",
"evaluation": "在之后更复杂的任务上衡量先前程序带来的搜索加速。",
"isolation": "顺序归纳的任务序列，没有现代随机 训练／测试 划分。",
"roles": {
"executor": {
"value": "OOPS 搜索并执行候选程序来解题；没有语言模型参与。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
},
"modifier": {
"value": "固定的 OOPS 程序搜索算法复用已找到的程序，分配后续搜索时间；候选程序可以调整搜索偏置。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
},
"seed": {
"value": "OOPS 是可执行程序搜索系统，保存已解决问题的程序前缀并按概率和计算预算复用；基础对象是通用程序解释/搜索机制，不是预训练语言模型的工具链。",
"sources": [
{
"label": "OOPS 程序搜索与实验",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"selection": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"evaluation": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
],
"isolation": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "新问题往往能通过复用或调整旧解法更快解决，普通逐题搜索却未必利用这部分信息。作者关注的不只是保存旧程序，还包括改进调用和组织旧知识的方法：能否用一种通用、具有搜索时间效率保证的机制，让连续解决的任务真正相互帮助。",
"sources": [
{
"label": "§1 Introduction（第 3 页）",
"url": "https://arxiv.org/pdf/cs/0207097v2#page=3"
}
]
},
{
"key": "position",
"value": "研究程序求解器能否复用过去的解法和搜索经验，高效解决后续问题，并为这种增量搜索给出理论保证。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
},
{
"key": "conclusion",
"value": "给出有最优性界限的增量程序搜索方法，为“学习怎样搜索”提供可执行形式。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
}
],
"fields": {
"object": "已发现的程序及可复用的搜索程序；任务正确性标准与时间分配原则固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "1805.06610",
"title": "A Formulation of Recursive Self-Improvement and Its Possible Efficiency",
"url": "https://arxiv.org/abs/1805.06610",
"date": "2018-05-17",
"priority": "R",
"categories": [
"theory"
],
"tags": [
"org:ubc"
],
"fields": {
"本质定位": "给出一类受限 RSI 的形式定义，并分析何时递归改进可计算且高效。",
"什么在变": "受限系统的 improvement mapping。",
"谁来改 / 谁执行": "**改**：形式化算法。
**执行**：形式化 RSI system。",
"基础 harness": "无 agent harness。",
"Feedback": "形式 objective / simulation feedback。",
"Evolution → Eval": "受限模拟。",
"Meta-depth": "理论 M2/M3。",
"相对之前真正新增什么": "把长期偏哲学的 RSI 讨论变成受限的可形式化对象；和 harness 实验关系主要是定义口径。"
},
"details": [],
"occurrences": [
{
"section": "0A. 理论源头",
"line": 253,
"fields": {
"优先级": "**R**",
"时间": "2018-05-17",
"论文": "[A Formulation of Recursive Self-Improvement and Its Possible Efficiency](https://arxiv.org/abs/1805.06610)",
"本质定位": "给出一类受限 RSI 的形式定义，并分析何时递归改进可计算且高效。",
"什么在变": "受限系统的 improvement mapping。",
"谁来改 / 谁执行": "**改**：形式化算法。
**执行**：形式化 RSI system。",
"基础 harness": "无 agent harness。",
"Feedback": "形式 objective / simulation feedback。",
"Evolution → Eval": "受限模拟。",
"Meta-depth": "理论 M2/M3。",
"相对之前真正新增什么": "把长期偏哲学的 RSI 讨论变成受限的可形式化对象；和 harness 实验关系主要是定义口径。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "理论 / 不适用",
"protocolBasis": "理论或形式化前驱，不能套用现代 agent 数据隔离标签。",
"year": "2018",
"depth": [
"M2",
"M3"
],
"legacyCategories": [
"F"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "给出一类受限 RSI 的形式定义，并分析何时递归改进可计算且高效。",
"novelty": "用受约束的形式化定义讨论递归自改进及其可能效率，帮助明确什么才算改进；证据类型是理论分析。",
"object": "受限形式系统中产生改进的映射；程序分布和评分构造受理论设定约束。",
"executor": "形式化定义中的自改进系统；文章讨论计算模型及其性质，不报告某个 语言模型 的运行结果。",
"modifier": "文中定义的自改进算法作用于形式化系统；没有独立的语言模型修改者配置。",
"roleContext": "**改**：形式化算法。
**执行**：形式化 RSI system。",
"seed": "形式化的递归改进系统与效率分析，没有一套现代交互 task agent 运行框架。不能把数学定义中的改进算子写成已实现的模型/工具组件。",
"fixed": "理论 M2/M3。",
"verdict": "使用形式化目标和模拟实验中的数值结果；本文把到达最优程序所需的期望步数作为分数，越低越好。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "理论 / 不适用",
"note": "受限模拟。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：随机生成 n=2^l（l=1…20）的抽象程序系统；不是自然语言任务数据。\n\n调试 / 选版本数据：按期望到达步数比较候选并更新当前程序。\n\n最终测试数据：各规模重复 10 次；n=2^20 时另运行 100 次模拟，观察排名和收敛速度。\n\n数据隔离与证据边界：这是抽象随机系统的模拟证据，不等同于真实 task agent 的能力自进化。",
"cycle": "抽象程序按各自分布产生新程序；分数更低才替换。分数定义为到达最优程序的期望步数。",
"train": "随机生成 n=2^l（l=1…20）的抽象程序系统；不是自然语言任务数据。",
"debug": "按期望到达步数比较候选并更新当前程序。",
"test": "各规模重复 10 次；n=2^20 时另运行 100 次模拟，观察排名和收敛速度。",
"isolation": "这是抽象随机系统的模拟证据，不等同于真实 task agent 的能力自进化。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "形式化的递归改进系统与效率分析，没有一套现代交互 agent harness。不能把数学定义中的改进算子写成已实现的模型/工具组件。",
"protocol": "**数据与测试：**研究对象是 RSI 的形式化与可能效率，不以某个任务数据集上的训练和测试涨分为主要证据；这一维度不适用，需看假设和理论结论的条件。",
"sections": "形式化问题与理论分析",
"source": "https://arxiv.org/abs/1805.06610",
"version": "1805.06610v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f526e5c271eb54ea48225a5b2a744ef9a4473585c4360b3a0c72ffb35961d1cb",
"seedStatus": "not-applicable",
"protocolStatus": "not-applicable"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "形式化定义中的自改进系统；文章讨论计算模型及其性质，不报告某个 语言模型 的运行结果。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "文中定义的自改进算法作用于形式化系统；没有独立的语言模型修改者配置。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "受限形式系统中产生改进的映射；程序分布和评分构造受理论设定约束。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "使用形式化目标和模拟实验中的数值结果；本文把到达最优程序所需的期望步数作为分数，越低越好。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "形式化的递归改进系统与效率分析，没有一套现代交互 task agent 运行框架。不能把数学定义中的改进算子写成已实现的模型/工具组件。",
"sources": [
{
"label": "形式化问题与理论分析",
"url": "https://arxiv.org/abs/1805.06610"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "抽象程序按各自分布产生新程序；分数更低才替换。分数定义为到达最优程序的期望步数。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "随机生成 n=2^l（l=1…20）的抽象程序系统；不是自然语言任务数据。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "按期望到达步数比较候选并更新当前程序。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各规模重复 10 次；n=2^20 时另运行 100 次模拟，观察排名和收敛速度。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这是抽象随机系统的模拟证据，不等同于真实 task agent 的能力自进化。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用受约束的形式化定义讨论递归自改进及其可能效率，帮助明确什么才算改进；证据类型是理论分析。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/1805.06610v1",
"version": "1805.06610v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把长期偏哲学的 RSI 讨论变成受限的可形式化对象；和 harness 实验关系主要是定义口径。",
"feedbackCases": [
{
"label": "抽象程序系统模拟",
"data": "随机生成规模 n=2^l（l=1…20）的程序系统，各规模十次模拟，大规模另做一百次。",
"scoring": "按定义的分数/到达好程序的期望步数比较当前程序与新程序。",
"visible": "模拟器的形式目标值。",
"use": "决定是否替换程序并观察收敛；没有真实自然语言任务或模型裁判。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"judgment": "按理论模拟预设的程序分数与期望搜索步数计算"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "随机生成 n=2^l（l=1…20）的抽象程序系统；不是自然语言任务数据。",
"selection": "按期望到达步数比较候选并更新当前程序。",
"evaluation": "各规模重复 10 次；n=2^20 时另运行 100 次模拟，观察排名和收敛速度。",
"isolation": "这是抽象随机系统的模拟证据，不等同于真实 task agent 的能力自进化。",
"roles": {
"executor": {
"value": "形式化定义中的自改进系统；文章讨论计算模型及其性质，不报告某个 语言模型 的运行结果。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
]
},
"modifier": {
"value": "文中定义的自改进算法作用于形式化系统；没有独立的语言模型修改者配置。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
]
},
"seed": {
"value": "形式化的递归改进系统与效率分析，没有一套现代交互 task agent 运行框架。不能把数学定义中的改进算子写成已实现的模型/工具组件。",
"sources": [
{
"label": "形式化问题与理论分析",
"url": "https://arxiv.org/abs/1805.06610"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/1805.06610#S4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "递归自改进的许多讨论仍停留在哲学推测，已有严格工作又主要研究怎样构造系统。缺少清楚的数学定义和效率分析，就难判断递归改进究竟能比普通改进快多少，以及这种优势需要什么条件；本文针对这一理论问题展开。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/1805.06610#S1"
}
]
},
{
"key": "position",
"value": "研究递归自改进在明确受限的形式化条件下是否可能，以及这种系统可以具有什么计算效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/1805.06610"
}
]
},
{
"key": "conclusion",
"value": "证明受限设定下的存在性，并用模拟展示高效搜索；结论受形式化假设约束。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/1805.06610#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/1805.06610#S3"
}
]
}
],
"fields": {
"object": "受限形式系统中产生改进的映射；程序分布和评分构造受理论设定约束。",
"verdict": "使用形式化目标和模拟实验中的数值结果；本文把到达最优程序所需的期望步数作为分数，越低越好。"
}
},
"attributions": [
{
"tag": "org:ubc",
"label": "University of British Columbia",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/1805.06610"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2203.11171",
"title": "Self-Consistency Improves Chain of Thought Reasoning in Language Models",
"url": "https://arxiv.org/abs/2203.11171",
"date": "2022-03-21",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:google-brain",
"person:denny-zhou"
],
"fields": {
"本质定位": "同一问题采样多条不同 reasoning paths，再以最终答案的一致性/多数结果做选择。",
"什么在变": "当前问题的 sampled trajectories 与最终 answer selection；系统本身不变。",
"谁来改 / 谁执行": "**改**：固定 sampling + aggregation rule。
**执行**：同一 base LLM 多次采样。",
"基础 harness": "CoT prompting + sampling/majority aggregation。",
"Feedback": "模型多次生成之间的答案一致性；没有 environment/gold feedback。",
"Evolution → Eval": "GSM8K、SVAMP、AQuA、StrategyQA、ARC 等 current-run decoding；无跨任务继承。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "它没有产生 persistent improvement；真正重要的是提供了一个无需外部标签的 **internal-consistency feedback signal**，后来可被 evolution loop 当作 verifier/proxy。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 259,
"fields": {
"优先级": "**R**",
"时间": "2022-03-21",
"论文": "[Self-Consistency Improves Chain of Thought Reasoning in Language Models](https://arxiv.org/abs/2203.11171)",
"本质定位": "同一问题采样多条不同 reasoning paths，再以最终答案的一致性/多数结果做选择。",
"什么在变": "当前问题的 sampled trajectories 与最终 answer selection；系统本身不变。",
"谁来改 / 谁执行": "**改**：固定 sampling + aggregation rule。
**执行**：同一 base LLM 多次采样。",
"基础 harness": "CoT prompting + sampling/majority aggregation。",
"Feedback": "模型多次生成之间的答案一致性；没有 environment/gold feedback。",
"Evolution → Eval": "GSM8K、SVAMP、AQuA、StrategyQA、ARC 等 current-run decoding；无跨任务继承。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "它没有产生 persistent improvement；真正重要的是提供了一个无需外部标签的 **internal-consistency feedback signal**，后来可被 evolution loop 当作 verifier/proxy。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2022",
"depth": [
"M0"
],
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "同一问题采样多条不同 reasoning paths，再以最终答案的一致性/多数结果做选择。",
"novelty": "为同一道题采样多条推理路径，再按答案一致性选择结果；多路径一致提供判断信号，但不会留下跨任务更新的系统。",
"object": "当前问题的多条推理候选与最终答案选择；模型、提示和投票规则不变。",
"executor": "UL2-20B、LaMDA-137B、PaLM-540B，以及GPT-3/Codex的code-davinci-001、code-davinci-002，对同一问题采样多条推理链。",
"modifier": "没有修改模型的语言模型：固定程序采样同一基础模型的多条推理链，提取最终答案并投票；不训练参数或改运行框架。",
"roleContext": "**改**：固定 sampling + aggregation rule。
**执行**：同一 base LLM 多次采样。",
"seed": "同一道题采样多条逐步推理，再按最终答案多数投票；基础系统只有提示、采样和聚合，不更新跨题技能库、工具代码或模型权重。",
"fixed": "",
"verdict": "同一模型多次回答后的答案一致性用于选择输出；选择时没有环境验收或标准答案反馈。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "GSM8K、SVAMP、AQuA、StrategyQA、ARC 等 current-run decoding；无跨任务继承。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：无新增训练；提示中提供少量推理示例。\n\n调试 / 选版本数据：不使用正确答案调试当前候选，依答案之间的一致性选择。\n\n最终测试数据：GSM8K（小学数学应用题基准）、SVAMP、AQuA、AddSub、MultiArith、ASDiv；CommonsenseQA、StrategyQA、ARC；末字母拼接、Coinflip。\n\n数据隔离与证据边界：通常用官方 测试集，CommonsenseQA 用 开发集；这是题内采样收益，不是跨任务训练收益。",
"cycle": "对当前题多次采样不同推理路径，抽取最终答案并多数投票；不诊断或编辑错误路径，也不保留跨题更新。",
"train": "无新增训练；提示中提供少量推理示例。",
"debug": "不使用正确答案调试当前候选，依答案之间的一致性选择。",
"test": "GSM8K（小学数学应用题基准）、SVAMP、AQuA、AddSub、MultiArith、ASDiv；CommonsenseQA、StrategyQA、ARC；末字母拼接、Coinflip。",
"isolation": "通常用官方 测试集，CommonsenseQA 用 开发集；这是题内采样收益，不是跨任务训练收益。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "同一道题采样多条逐步推理，再按最终答案多数投票；基础系统只有提示、采样和聚合，不更新跨题技能库、工具代码或模型权重。",
"protocol": "**评测：**GSM8K 等算术/常识推理 benchmark；符号任务另比较提示中 2-letter/2-flip 示例与测试时 4-letter/4-flip。推理候选在当前测试题内生成和筛选，不存在方法自己的参数训练集。各 benchmark 全部名称、抽样数量本轮待补核，不把采样条数叫训练样本。",
"sections": "§3.1 与符号 OOD 设置",
"source": "https://arxiv.org/abs/2203.11171",
"version": "2203.11171v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f52ddaca3dfe0e0d1ac0c8d3659d36105d9fa56e737858c2408be1b58f31f834",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "UL2-20B、LaMDA-137B、PaLM-540B，以及GPT-3/Codex的code-davinci-001、code-davinci-002，对同一问题采样多条推理链。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2203.11171#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "没有修改模型的语言模型：固定程序采样同一基础模型的多条推理链，提取最终答案并投票；不训练参数或改运行框架。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2203.11171#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前问题的多条推理候选与最终答案选择；模型、提示和投票规则不变。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "同一模型多次回答后的答案一致性用于选择输出；选择时没有环境验收或标准答案反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "同一道题采样多条逐步推理，再按最终答案多数投票；基础系统只有提示、采样和聚合，不更新跨题技能库、工具代码或模型权重。",
"sources": [
{
"label": "§3.1 与符号 OOD 设置",
"url": "https://arxiv.org/abs/2203.11171"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "对当前题多次采样不同推理路径，抽取最终答案并多数投票；不诊断或编辑错误路径，也不保留跨题更新。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无新增训练；提示中提供少量推理示例。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "不使用正确答案调试当前候选，依答案之间的一致性选择。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GSM8K（小学数学应用题基准）、SVAMP、AQuA、AddSub、MultiArith、ASDiv；CommonsenseQA、StrategyQA、ARC；末字母拼接、Coinflip。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "通常用官方 测试集，CommonsenseQA 用 开发集；这是题内采样收益，不是跨任务训练收益。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "为同一道题采样多条推理路径，再按答案一致性选择结果；多路径一致提供判断信号，但不会留下跨任务更新的系统。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2203.11171v4",
"version": "2203.11171v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它没有产生 persistent improvement；真正重要的是提供了一个无需外部标签的 **internal-consistency feedback signal**，后来可被 evolution loop 当作 verifier/proxy。",
"feedbackCases": [
{
"label": "当前问题内的答案选择",
"data": "GSM8K（小学数学应用题基准）、SVAMP、AQuA、AddSub、MultiArith、ASDiv，CommonsenseQA、StrategyQA、ARC，另有字符/硬币任务。",
"scoring": "候选选择只看多次生成的最终答案一致性，主要用多数票；不查询正确答案，也不跑环境检查。",
"visible": "模型自己的不同推理路径和答案。",
"use": "从同题候选中选结果，无参数/框架进化。最终准确率才由数据集参考答案计算，不能把它说成投票时的监督。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
}
],
"judgment": "候选答案规则多数票；选择阶段不使用标准答案"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "无新增训练；提示中提供少量推理示例。",
"selection": "不使用正确答案调试当前候选，依答案之间的一致性选择。",
"evaluation": "GSM8K（小学数学应用题基准）、SVAMP、AQuA、AddSub、MultiArith、ASDiv；CommonsenseQA、StrategyQA、ARC；末字母拼接、Coinflip。",
"isolation": "通常用官方 测试集，CommonsenseQA 用 开发集；这是题内采样收益，不是跨任务训练收益。",
"roles": {
"executor": {
"value": "UL2-20B、LaMDA-137B、PaLM-540B，以及GPT-3/Codex的code-davinci-001、code-davinci-002，对同一问题采样多条推理链。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2203.11171#A1.SS2"
}
]
},
"modifier": {
"value": "没有修改模型的语言模型：固定程序采样同一基础模型的多条推理链，提取最终答案并投票；不训练参数或改运行框架。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2203.11171#S4.SS0.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px2"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2203.11171#A1.SS2"
}
]
},
"seed": {
"value": "同一道题采样多条逐步推理，再按最终答案多数投票；基础系统只有提示、采样和聚合，不更新跨题技能库、工具代码或模型权重。",
"sources": [
{
"label": "§3.1 与符号 OOD 设置",
"url": "https://arxiv.org/abs/2203.11171"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2203.11171#S3.SS1.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "逐步写出推理过程能够改善答题，但常用的贪心生成只沿一条路径前进，容易陷入局部选择；单次随机生成又有偶然性。作者指出同一题往往有多条通向正确答案的推理路径，因此希望利用这些路径的一致性，提高答案可靠性，并避免另训验证器的成本。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2203.11171#S1"
}
]
},
{
"key": "position",
"value": "研究同一模型对当前题目的多种推理能否相互补充，从而在不训练模型的情况下提高数学与常识推理可靠性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2203.11171"
}
]
},
{
"key": "conclusion",
"value": "在数学与常识推理中明显改善表现；改进来自当前题目的多次采样与选择。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2203.11171"
}
]
}
],
"fields": {
"object": "当前问题的多条推理候选与最终答案选择；模型、提示和投票规则不变。",
"verdict": "同一模型多次回答后的答案一致性用于选择输出；选择时没有环境验收或标准答案反馈。"
}
},
"attributions": [
{
"tag": "org:google-brain",
"label": "Google Brain",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2203.11171"
}
]
},
{
"tag": "person:denny-zhou",
"label": "Denny Zhou",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2203.11171"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2303.17651",
"title": "Self-Refine: Iterative Refinement with Self-Feedback",
"url": "https://arxiv.org/abs/2303.17651",
"date": "2023-03-30",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"SelfFeedback",
"org:google-brain",
"org:washington",
"org:cmu",
"org:allenai"
],
"fields": {
"本质定位": "同一个 LLM 依次充当 generator、feedback provider、refiner，对当前输出反复修改。",
"什么在变": "当前 response。",
"谁来改 / 谁执行": "**改**：同一 LLM 的 feedback/refine prompt。
**执行**：GPT-3.5 / ChatGPT / GPT-4 等。",
"基础 harness": "one-shot task prompting。",
"Feedback": "模型自己的自然语言 critique，无外部 verifier。",
"Evolution → Eval": "7 个任务上的 within-instance refinement。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "证明无需训练也能利用自然语言 self-feedback 改当前 output；但没有 persistent state，因此不是 self-evolution。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 260,
"fields": {
"优先级": "**K**",
"时间": "2023-03-30",
"论文": "[Self-Refine: Iterative Refinement with Self-Feedback](https://arxiv.org/abs/2303.17651)",
"本质定位": "同一个 LLM 依次充当 generator、feedback provider、refiner，对当前输出反复修改。",
"什么在变": "当前 response。",
"谁来改 / 谁执行": "**改**：同一 LLM 的 feedback/refine prompt。
**执行**：GPT-3.5 / ChatGPT / GPT-4 等。",
"基础 harness": "one-shot task prompting。",
"Feedback": "模型自己的自然语言 critique，无外部 verifier。",
"Evolution → Eval": "7 个任务上的 within-instance refinement。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "证明无需训练也能利用自然语言 self-feedback 改当前 output；但没有 persistent state，因此不是 self-evolution。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "同一个 LLM 依次充当 generator、feedback provider、refiner，对当前输出反复修改。",
"novelty": "让同一个模型生成初稿、提出文字批评并据此改稿，所有修改围绕当前产物，任务结束后没有新的持久学习状态。",
"object": "当前回答；模型和生成、反馈、修订流程固定。",
"executor": "GPT-3.5（text-davinci-003）、ChatGPT（gpt-3.5-turbo）、GPT-4；代码任务另测Codex code-davinci-002。",
"modifier": "同一基础模型分别执行生成、反馈与重写提示；如评论重写实验的critique和editor均为text-davinci-003，没有额外训练独立提供批评意见的模型。",
"roleContext": "**改**：同一 LLM 的 feedback/refine prompt。
**执行**：GPT-3.5 / ChatGPT / GPT-4 等。",
"seed": "一个固定语言模型先给初稿，再按任务专用反馈提示批评，最后改写；重复时携带先前反馈。变化的是当前回答或代码，运行框架和参数保持固定。",
"fixed": "",
"verdict": "由同一个模型写出具体批评并据此改稿；迭代中没有外部正确性判定，最终研究评测与模型自评分开。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "7 个任务上的 within-instance refinement。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：无参数训练；各任务用少样本示例指定反馈与改写格式。\n\n调试 / 选版本数据：当前输出由模型自己检查并给可行动建议；不是统一外部判分器决定每次修改。\n\n最终测试数据：FED 对话 342；GSM8K（小学数学应用题基准） 1,319；代码优化、代码可读性 300、情感反转、缩写 250、约束生成 200。\n\n数据隔离与证据边界：同一测试实例被迭代改写；最终用任务指标/人评比较，不能算新增持久能力。",
"cycle": "同一模型先生成、再给具体批评、再改写；循环直到停止条件或轮数上限，不更新权重或固定框架。",
"train": "无参数训练；各任务用少样本示例指定反馈与改写格式。",
"debug": "当前输出由模型自己检查并给可行动建议；不是统一外部判分器决定每次修改。",
"test": "FED 对话 342；GSM8K（小学数学应用题基准） 1,319；代码优化、代码可读性 300、情感反转、缩写 250、约束生成 200。",
"isolation": "同一测试实例被迭代改写；最终用任务指标/人评比较，不能算新增持久能力。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "一个固定语言模型先给初稿，再按任务专用反馈提示批评，最后改写；重复时携带先前反馈。变化的是当前回答或代码，harness 和参数保持固定。",
"protocol": "**测试时改写：**七类任务包含对话、情感改写、代码优化/可读性、数学、缩写和关键词约束生成。数学用 GSM8K 1,319 题；代码可读性 300 程序、缩写 250 项、约束生成 200 项。\n\n**边界：**这些题在测试时被反复修改，不是用它们训练出新模型后再测；任务提示示例与最终评估样本的完整隔离本轮未核实。",
"sections": "任务设置、Table 4",
"source": "https://arxiv.org/abs/2303.17651",
"version": "2303.17651v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "69a8dc882a89c1e85772677b20ee839c0db112259f791bb360b93ef760305723",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-3.5（text-davinci-003）、ChatGPT（gpt-3.5-turbo）、GPT-4；代码任务另测Codex code-davinci-002。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2303.17651#S3.SS1.SSS0.Px1"
},
{
"label": "附录L.2",
"url": "https://arxiv.org/html/2303.17651#A12.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一基础模型分别执行生成、反馈与重写提示；如评论重写实验的critique和editor均为text-davinci-003，没有额外训练独立提供批评意见的模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2303.17651#S3.SS1.SSS0.Px1"
},
{
"label": "附录L.2",
"url": "https://arxiv.org/html/2303.17651#A12.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前回答；模型和生成、反馈、修订流程固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "由同一个模型写出具体批评并据此改稿；迭代中没有外部正确性判定，最终研究评测与模型自评分开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "一个固定语言模型先给初稿，再按任务专用反馈提示批评，最后改写；重复时携带先前反馈。变化的是当前回答或代码，运行框架和参数保持固定。",
"sources": [
{
"label": "任务设置、Table 4",
"url": "https://arxiv.org/abs/2303.17651"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "同一模型先生成、再给具体批评、再改写；循环直到停止条件或轮数上限，不更新权重或固定框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无参数训练；各任务用少样本示例指定反馈与改写格式。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前输出由模型自己检查并给可行动建议；不是统一外部判分器决定每次修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "FED 对话 342；GSM8K（小学数学应用题基准） 1,319；代码优化、代码可读性 300、情感反转、缩写 250、约束生成 200。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同一测试实例被迭代改写；最终用任务指标/人评比较，不能算新增持久能力。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让同一个模型生成初稿、提出文字批评并据此改稿，所有修改围绕当前产物，任务结束后没有新的持久学习状态。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2303.17651v2",
"version": "2303.17651v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "证明无需训练也能利用自然语言 self-feedback 改当前 output；但没有 persistent state，因此不是 self-evolution。",
"feedbackCases": [
{
"label": "自反馈改写",
"data": "FED 对话、GSM8K（小学数学应用题基准）、代码优化/可读性、情感反转、缩写、约束生成七类任务。",
"scoring": "同一个语言模型依据任务专用反馈提示，指出当前输出的问题并给改写建议；没有统一外部判分器来批准每次改写。",
"visible": "自己的初稿、自然语言批评及历史修改；停止按任务次数或模型反馈判断。",
"use": "修改当前回答/代码，不改变模型参数或跨任务运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"judgment": "同一个模型按任务反馈提示自评并修改，无外部答案验收"
},
{
"label": "自动指标评估",
"data": "数学推理、代码优化、约束生成。",
"scoring": "数学以解题正确率、代码以成功优化比例、约束生成以要求覆盖率评价。",
"visible": "这些是研究端的最终指标，不能当作每一步自反馈都获得的外部答案。",
"use": "比较改写前后产物效果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"judgment": "最终按数学正确性、代码性能和约束覆盖等任务指标评价"
},
{
"label": "人工偏好评估",
"data": "对话生成、代码可读性、情感反转、缩写任务的部分输出。",
"scoring": "对输出做盲测人工 A/B 偏好选择。",
"visible": "人工偏好用于研究评价，未在每次模型改写中逐条返回。",
"use": "区分最终人类评价与生成时模型自评。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"judgment": "人类盲评两份输出的偏好"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1,
2
],
"evolution": "无参数训练；各任务用少样本示例指定反馈与改写格式。",
"selection": "当前输出由模型自己检查并给可行动建议；不是统一外部判分器决定每次修改。",
"evaluation": "FED 对话 342；GSM8K（小学数学应用题基准） 1,319；代码优化、代码可读性 300、情感反转、缩写 250、约束生成 200。",
"isolation": "同一测试实例被迭代改写；最终用任务指标/人评比较，不能算新增持久能力。",
"roles": {
"executor": {
"value": "GPT-3.5（text-davinci-003）、ChatGPT（gpt-3.5-turbo）、GPT-4；代码任务另测Codex code-davinci-002。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2303.17651#S3.SS1.SSS0.Px1"
},
{
"label": "附录L.2",
"url": "https://arxiv.org/html/2303.17651#A12.SS2.SSS0.Px2"
}
]
},
"modifier": {
"value": "同一基础模型分别执行生成、反馈与重写提示；如评论重写实验的critique和editor均为text-davinci-003，没有额外训练独立提供批评意见的模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2303.17651#S3.SS1.SSS0.Px1"
},
{
"label": "附录L.2",
"url": "https://arxiv.org/html/2303.17651#A12.SS2.SSS0.Px2"
}
]
},
"seed": {
"value": "一个固定语言模型先给初稿，再按任务专用反馈提示批评，最后改写；重复时携带先前反馈。变化的是当前回答或代码，运行框架和参数保持固定。",
"sources": [
{
"label": "任务设置、Table 4",
"url": "https://arxiv.org/abs/2303.17651"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2303.17651#A1"
},
{
"label": "附录M.2",
"url": "https://arxiv.org/html/2303.17651#A13.SS2.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型初稿即使通顺，也常达不到任务的细致要求，例如语气合适或代码足够高效。已有改进方法常依赖额外训练、大量标注或奖励模型，成本使其难普遍采用；作者因而研究能否在无需这些监督的情况下，让模型反复检查并修改自己的输出。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2303.17651#S1"
}
]
},
{
"key": "position",
"value": "研究单个模型在没有额外训练或外部反馈时，能否通过自身批评持续改善当前输出的质量。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.17651"
}
]
},
{
"key": "conclusion",
"value": "在多种任务上优于一次生成；循环修改的是当前产物，不产生持久学习的 agent 版本。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2303.17651"
}
]
}
],
"fields": {
"object": "当前回答；模型和生成、反馈、修订流程固定。",
"verdict": "由同一个模型写出具体批评并据此改稿；迭代中没有外部正确性判定，最终研究评测与模型自评分开。"
}
},
"attributions": [
{
"tag": "org:google-brain",
"label": "Google Brain",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.17651"
}
]
},
{
"tag": "org:washington",
"label": "University of Washington",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.17651"
}
]
},
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.17651"
}
]
},
{
"tag": "org:allenai",
"label": "Allen Institute for AI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2303.17651"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2305.11738",
"title": "CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing",
"url": "https://arxiv.org/abs/2305.11738",
"date": "2023-05-19",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"Tool",
"org:tsinghua"
],
"fields": {
"本质定位": "让 LLM 用搜索、代码执行等外部工具验证初始输出，再根据工具 evidence 修正。",
"什么在变": "当前 output。",
"谁来改 / 谁执行": "**改**：同一 LLM + tool-interactive critic loop。
**执行**：InstructGPT / ChatGPT / LLaMA-2 等。",
"基础 harness": "LLM + external tool interface。",
"Feedback": "搜索结果、代码执行、toxicity evaluator 等外部工具反馈。",
"Evolution → Eval": "QA、math program synthesis、toxicity；current-run correction。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "相对 Self-Refine 最重要的新点不是“再反思一次”，而是把 critique grounding 到可观测外部证据。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 261,
"fields": {
"优先级": "**K**",
"时间": "2023-05-19",
"论文": "[CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing](https://arxiv.org/abs/2305.11738)",
"本质定位": "让 LLM 用搜索、代码执行等外部工具验证初始输出，再根据工具 evidence 修正。",
"什么在变": "当前 output。",
"谁来改 / 谁执行": "**改**：同一 LLM + tool-interactive critic loop。
**执行**：InstructGPT / ChatGPT / LLaMA-2 等。",
"基础 harness": "LLM + external tool interface。",
"Feedback": "搜索结果、代码执行、toxicity evaluator 等外部工具反馈。",
"Evolution → Eval": "QA、math program synthesis、toxicity；current-run correction。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "相对 Self-Refine 最重要的新点不是“再反思一次”，而是把 critique grounding 到可观测外部证据。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "让 LLM 用搜索、代码执行等外部工具验证初始输出，再根据工具 evidence 修正。",
"novelty": "让模型调用工具取得外部证据，再据此检查并修改当前回答，使批评有可观察的依据。",
"object": "当前输出；模型权重、工具接口与反馈修订流程固定。",
"executor": "分别使用 text-davinci-003、ChatGPT（gpt-3.5-turbo）及 LLaMA-2 7B／13B／70B 生成初始回答，再调用外部工具验证。",
"modifier": "对应执行模型读取工具返回的证据，生成批评并修订自己的回答；这里没有另设一个经过训练的提供批评意见的模型模型。",
"roleContext": "**改**：同一 LLM + tool-interactive critic loop。
**执行**：InstructGPT / ChatGPT / LLaMA-2 等。",
"seed": "固定 语言模型 先回答，再调用外部工具取得可检验信息并生成批评、修订答案；不同任务接搜索或程序执行等工具。外部检查是关键，不能描述成模型凭感觉自我评价。",
"fixed": "",
"verdict": "问答使用检索证据，数学程序使用代码执行结果，文本安全使用毒性检测工具；这些外部检查帮助模型判断初稿哪里有错。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "QA、math program synthesis、toxicity；current-run correction。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不训练参数；当前问题及少样本提示是输入。\n\n调试 / 选版本数据：问答读检索证据；数学读 Python 执行；文本毒性读 Perspective API。\n\n最终测试数据：AmbigNQ、TriviaQA、HotpotQA（需要结合多份资料作答的多跳问答基准）；GSM8K（小学数学应用题基准）、SVAMP、TabMWP 官方 测试集；RealToxicityPrompts 抽 1,000 个非毒性提示。\n\n数据隔离与证据边界：工具证据用于当前题纠错，正式指标另按答案/毒性评分；不是跨题运行框架学习。",
"cycle": "先回答，再通过检索、Python 或毒性检测等工具检查，模型根据外部证据生成批评并修订；与不用工具的自反馈对照。",
"train": "不训练参数；当前问题及少样本提示是输入。",
"debug": "问答读检索证据；数学读 Python 执行；文本毒性读 Perspective API。",
"test": "AmbigNQ、TriviaQA、HotpotQA（需要结合多份资料作答的多跳问答基准）；GSM8K（小学数学应用题基准）、SVAMP、TabMWP 官方 测试集；RealToxicityPrompts 抽 1,000 个非毒性提示。",
"isolation": "工具证据用于当前题纠错，正式指标另按答案/毒性评分；不是跨题运行框架学习。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定 LLM 先回答，再调用外部工具取得可检验信息并生成批评、修订答案；不同任务接搜索或程序执行等工具。外部检查是关键，不能描述成模型凭感觉自我评价。",
"protocol": "**任务：**知识问答含 AmbigNQ、TriviaQA、HotpotQA，另有程序辅助数学和有害文本修正实验。对当前问题执行“回答—工具验证—修改”，不是跨任务参数训练。问答以 EM/F1 等评估，工具返回证据供修订；具体抽样 split/数量及其他任务数据名称本轮待核实。",
"sections": "实验任务与问答结果",
"source": "https://arxiv.org/abs/2305.11738",
"version": "2305.11738v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e939c2a62ad578e766ea374c54c2b5aa95194b133fc033fe3ba2f3541512a44a",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "分别使用 text-davinci-003、ChatGPT（gpt-3.5-turbo）及 LLaMA-2 7B／13B／70B 生成初始回答，再调用外部工具验证。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2305.11738#S1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.11738#S4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应执行模型读取工具返回的证据，生成批评并修订自己的回答；这里没有另设一个经过训练的提供批评意见的模型模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2305.11738#S1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.11738#S4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前输出；模型权重、工具接口与反馈修订流程固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "问答使用检索证据，数学程序使用代码执行结果，文本安全使用毒性检测工具；这些外部检查帮助模型判断初稿哪里有错。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定 语言模型 先回答，再调用外部工具取得可检验信息并生成批评、修订答案；不同任务接搜索或程序执行等工具。外部检查是关键，不能描述成模型凭感觉自我评价。",
"sources": [
{
"label": "实验任务与问答结果",
"url": "https://arxiv.org/abs/2305.11738"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先回答，再通过检索、Python 或毒性检测等工具检查，模型根据外部证据生成批评并修订；与不用工具的自反馈对照。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训练参数；当前问题及少样本提示是输入。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "问答读检索证据；数学读 Python 执行；文本毒性读 Perspective API。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AmbigNQ、TriviaQA、HotpotQA（需要结合多份资料作答的多跳问答基准）；GSM8K（小学数学应用题基准）、SVAMP、TabMWP 官方 测试集；RealToxicityPrompts 抽 1,000 个非毒性提示。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "工具证据用于当前题纠错，正式指标另按答案/毒性评分；不是跨题运行框架学习。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让模型调用工具取得外部证据，再据此检查并修改当前回答，使批评有可观察的依据。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2305.11738v4",
"version": "2305.11738v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Self-Refine 最重要的新点不是“再反思一次”，而是把 critique grounding 到可观测外部证据。",
"feedbackCases": [
{
"label": "问答纠错",
"data": "AmbigNQ、TriviaQA、HotpotQA（需要结合多份资料作答的多跳问答基准） 各抽 500 道验证题。",
"scoring": "纠错时查询搜索工具，用检索证据检验初答；最终将答案与参考答案比较，报告 EM/F1。",
"visible": "搜索结果与据此生成的批评，而不是直接公开标准答案。",
"use": "修改当前问题的回答，无参数训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"judgment": "纠错时模型核对搜索证据；最终用参考答案 EM／F1 规则计分"
},
{
"label": "数学程序纠错",
"data": "GSM8K（小学数学应用题基准）、SVAMP、TabMWP 官方测试。",
"scoring": "纠错时运行 Python 看数值结果/错误；最终按预测数值与标准答案比较的精确匹配评分。",
"visible": "代码执行输出及错误帮助修改计算过程。",
"use": "Python 返回数值不自动保证题意理解正确，正式答案检查仍是另一层。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"judgment": "纠错时执行 Python；最终规则比较计算答案与标准数值"
},
{
"label": "降低文本毒性",
"data": "RealToxicityPrompts 抽取 1,000 个非毒性提示。",
"scoring": "调用 Perspective API 获取生成文本的毒性评分。",
"visible": "外部毒性分和模型据此提出的批评。",
"use": "针对当前文本改写；这不是问答标准答案或代码测试反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"judgment": "Perspective API 返回毒性模型分数"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "不训练参数；当前问题及少样本提示是输入。",
"selection": "问答读检索证据；数学读 Python 执行；文本毒性读 Perspective API。",
"evaluation": "AmbigNQ、TriviaQA、HotpotQA（需要结合多份资料作答的多跳问答基准）；GSM8K（小学数学应用题基准）、SVAMP、TabMWP 官方 测试集；RealToxicityPrompts 抽 1,000 个非毒性提示。",
"isolation": "工具证据用于当前题纠错，正式指标另按答案/毒性评分；不是跨题运行框架学习。",
"roles": {
"executor": {
"value": "分别使用 text-davinci-003、ChatGPT（gpt-3.5-turbo）及 LLaMA-2 7B／13B／70B 生成初始回答，再调用外部工具验证。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2305.11738#S1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.11738#S4"
}
]
},
"modifier": {
"value": "对应执行模型读取工具返回的证据，生成批评并修订自己的回答；这里没有另设一个经过训练的提供批评意见的模型模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2305.11738#S1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.11738#S4"
}
]
},
"seed": {
"value": "固定 语言模型 先回答，再调用外部工具取得可检验信息并生成批评、修订答案；不同任务接搜索或程序执行等工具。外部检查是关键，不能描述成模型凭感觉自我评价。",
"sources": [
{
"label": "实验任务与问答结果",
"url": "https://arxiv.org/abs/2305.11738"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"selection": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"evaluation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
],
"isolation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.11738#S4.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2305.11738#S4.SS3.SSS0.Px2"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "语言模型会产生事实错误和不可靠输出，而额外训练来纠正这些问题往往需要昂贵标注或数据构造。作者希望把检查与修改放到使用阶段，并借助外部工具取得可核对的信息，使黑箱模型也能验证、修正答案，而不只依赖原有知识继续生成。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2305.11738#S1"
}
]
},
{
"key": "position",
"value": "研究模型能否借助外部工具反馈识别并修正自身错误，改善单靠自我判断时不可靠的纠错能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.11738"
}
]
},
{
"key": "conclusion",
"value": "在问答、数学代码和文本安全任务中改善结果，突出外部反馈对纠错的重要性。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.11738"
}
]
}
],
"fields": {
"object": "当前输出；模型权重、工具接口与反馈修订流程固定。",
"verdict": "问答使用检索证据，数学程序使用代码执行结果，文本安全使用毒性检测工具；这些外部检查帮助模型判断初稿哪里有错。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2305.11738"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2305.20050",
"title": "Let’s Verify Step by Step",
"url": "https://arxiv.org/abs/2305.20050",
"date": "2023-05-31",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:openai"
],
"fields": {
"本质定位": "训练 process reward model (PRM) 对推理中间步骤打分，用过程监督选择更可靠的数学解。",
"什么在变": "训练的是 verifier/reward-model weights；task solver/harness 不持续自改。",
"谁来改 / 谁执行": "**改**：固定 supervised PRM training；人类逐步标注提供 target。
**执行**：数学 solver + PRM verifier。",
"基础 harness": "sample-and-rank math reasoning harness。",
"Feedback": "约 80 万个人工 step-level correctness labels（论文的 PRM800K）；不是 self-generated feedback。",
"Evolution → Eval": "MATH 上训练/评估 process vs outcome supervision；current solution selection。",
"Meta-depth": "M0 enabling substrate。",
"相对之前真正新增什么": "它不是 self-evolution，而是说明 **feedback granularity** 很关键：只有终局 outcome 与逐步 process feedback 对 search/selection 的信用分配能力不同。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 262,
"fields": {
"优先级": "**R**",
"时间": "2023-05-31",
"论文": "[Let’s Verify Step by Step](https://arxiv.org/abs/2305.20050)",
"本质定位": "训练 process reward model (PRM) 对推理中间步骤打分，用过程监督选择更可靠的数学解。",
"什么在变": "训练的是 verifier/reward-model weights；task solver/harness 不持续自改。",
"谁来改 / 谁执行": "**改**：固定 supervised PRM training；人类逐步标注提供 target。
**执行**：数学 solver + PRM verifier。",
"基础 harness": "sample-and-rank math reasoning harness。",
"Feedback": "约 80 万个人工 step-level correctness labels（论文的 PRM800K）；不是 self-generated feedback。",
"Evolution → Eval": "MATH 上训练/评估 process vs outcome supervision；current solution selection。",
"Meta-depth": "M0 enabling substrate。",
"相对之前真正新增什么": "它不是 self-evolution，而是说明 **feedback granularity** 很关键：只有终局 outcome 与逐步 process feedback 对 search/selection 的信用分配能力不同。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "训练 process reward model (PRM) 对推理中间步骤打分，用过程监督选择更可靠的数学解。",
"novelty": "训练对推理中间步骤判对错的评价模型，并用它帮助挑选解答，研究逐步反馈相对于只看最终答案的价值。",
"object": "评价推理步骤的奖励模型参数；任务求解器不在此过程中持续自改。",
"executor": "大规模生成器与奖励模型均从基础GPT-4微调；小规模模型结构类似GPT-4，但预训练算力约少200倍，论文没有公布其参数量或可下载型号。",
"modifier": "人类逐步标注推理正误，固定监督训练程序微调PRM；不是GPT-4自行重写评分规则。",
"roleContext": "**改**：固定 supervised PRM training；人类逐步标注提供 target。
**执行**：数学 solver + PRM verifier。",
"seed": "固定生成模型采样多份数学解答，另训练奖励模型按中间步骤评分，再选解答。核心是过程监督的结果检查器（按测试或判分规则检查任务结果），不是让 task agent 自行改工具或执行循环。",
"fixed": "",
"verdict": "人类逐步标注数学解答是否正确，形成约 80 万条步骤标签；模型用这些标注学习评审推理过程，标签并非模型自评生成。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "MATH 上训练/评估 process vs outcome supervision；current solution selection。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：PRM800K：约 800k 步骤标签、75k 解答、12k 题，包含原 MATH 测试集 的 4,500 题。\n\n调试 / 选版本数据：标注错误步骤提供局部监督；主动学习优先选看似可信的错解。\n\n最终测试数据：仅用剩余 500 道 MATH 测试集 题。\n\n数据隔离与证据边界：不能声称训练只用原 训练集 或最终测试全 5,000 题；步骤标签不是独立题目数。",
"cycle": "人类标注解题过程的每一步，训练过程奖励模型（PRM）。解题模型生成多个候选后，由它挑选较可靠的解答；对照的结果奖励模型（ORM）只用最终答案是否正确来训练。",
"train": "PRM800K：约 800k 步骤标签、75k 解答、12k 题，包含原 MATH 测试集 的 4,500 题。",
"debug": "标注错误步骤提供局部监督；主动学习优先选看似可信的错解。",
"test": "仅用剩余 500 道 MATH 测试集 题。",
"isolation": "不能声称训练只用原 训练集 或最终测试全 5,000 题；步骤标签不是独立题目数。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定生成模型采样多份数学解答，另训练奖励模型按中间步骤评分，再选解答。核心是过程监督的 verifier，不是让任务 agent 自行改工具或执行循环。",
"protocol": "**训练：**PRM800K 含约 800k 个步骤标签、75k 份解答、12k 道题；为扩大数据，纳入原 MATH test 中 4,500 题。\n\n**最终测试：**只用剩余 500 道 MATH test 题。不能写成训练仅用原 train、最终测试全部 5,000 道 test；步骤标签数也不是独立题目数。",
"sections": "PRM800K 数据构造；附录 C",
"source": "https://arxiv.org/abs/2305.20050",
"version": "2305.20050v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "89001fdcdaef25232f18e0b9a20b6e9b53c485a453bbd36bfb092231722c301c",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "大规模生成器与奖励模型均从基础GPT-4微调；小规模模型结构类似GPT-4，但预训练算力约少200倍，论文没有公布其参数量或可下载型号。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2305.20050#S2.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "人类逐步标注推理正误，固定监督训练程序微调PRM；不是GPT-4自行重写评分规则。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2305.20050#S2.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "评价推理步骤的奖励模型参数；任务求解器不在此过程中持续自改。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "人类逐步标注数学解答是否正确，形成约 80 万条步骤标签；模型用这些标注学习评审推理过程，标签并非模型自评生成。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定生成模型采样多份数学解答，另训练奖励模型按中间步骤评分，再选解答。核心是过程监督的结果检查器（按测试或判分规则检查任务结果），不是让 task agent 自行改工具或执行循环。",
"sources": [
{
"label": "PRM800K 数据构造；附录 C",
"url": "https://arxiv.org/abs/2305.20050"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "人类标注解题过程的每一步，训练过程奖励模型（PRM）。解题模型生成多个候选后，由它挑选较可靠的解答；对照的结果奖励模型（ORM）只用最终答案是否正确来训练。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "PRM800K：约 800k 步骤标签、75k 解答、12k 题，包含原 MATH 测试集 的 4,500 题。",
"sources": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "标注错误步骤提供局部监督；主动学习优先选看似可信的错解。",
"sources": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "仅用剩余 500 道 MATH 测试集 题。",
"sources": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "不能声称训练只用原 训练集 或最终测试全 5,000 题；步骤标签不是独立题目数。",
"sources": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "训练对推理中间步骤判对错的评价模型，并用它帮助挑选解答，研究逐步反馈相对于只看最终答案的价值。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2305.20050v1",
"version": "2305.20050v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它不是 self-evolution，而是说明 **feedback granularity** 很关键：只有终局 outcome 与逐步 process feedback 对 search/selection 的信用分配能力不同。",
"feedbackCases": [
{
"label": "过程奖励模型训练",
"data": "PRM800K 约 80 万步骤标签、7.5 万解答、1.2 万题，包含原 MATH 测试集 中 4,500 题。",
"scoring": "人类逐步标注数学推理是否正确，给局部步骤监督；不是模型自己生成正确性标签。",
"visible": "每个推理步骤的人工评价，主动学习优先标注看似可信的错解。",
"use": "训练奖励/检查模型，随后为候选解答排序；任务求解框架不持续自改。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"judgment": "人类对推理步骤逐条标注正确性"
},
{
"label": "最终解答选择与测试",
"data": "剩余 500 道 MATH 测试集。",
"scoring": "训练好的奖励模型给步骤打分来选择候选，正式结果按数学答案是否正确评价。",
"visible": "选择器获得的是学到的步骤评分，不等于对测试题获得新的人类逐步标签。",
"use": "比较过程监督与仅最终结果监督。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"judgment": "训练出的奖励模型选择候选；最终答案再与标准数学答案核对"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "PRM800K：约 800k 步骤标签、75k 解答、12k 题，包含原 MATH 测试集 的 4,500 题。",
"selection": "标注错误步骤提供局部监督；主动学习优先选看似可信的错解。",
"evaluation": "仅用剩余 500 道 MATH 测试集 题。",
"isolation": "不能声称训练只用原 训练集 或最终测试全 5,000 题；步骤标签不是独立题目数。",
"roles": {
"executor": {
"value": "大规模生成器与奖励模型均从基础GPT-4微调；小规模模型结构类似GPT-4，但预训练算力约少200倍，论文没有公布其参数量或可下载型号。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2305.20050#S2.SS2"
}
]
},
"modifier": {
"value": "人类逐步标注推理正误，固定监督训练程序微调PRM；不是GPT-4自行重写评分规则。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
},
{
"label": "§2.2",
"url": "https://arxiv.org/html/2305.20050#S2.SS2"
}
]
},
"seed": {
"value": "固定生成模型采样多份数学解答，另训练奖励模型按中间步骤评分，再选解答。核心是过程监督的结果检查器（按测试或判分规则检查任务结果），不是让 task agent 自行改工具或执行循环。",
"sources": [
{
"label": "PRM800K 数据构造；附录 C",
"url": "https://arxiv.org/abs/2305.20050"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"selection": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"evaluation": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
],
"isolation": [
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2305.20050#A3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "多步推理中，一个错误就可能使整个解答失效，因此需要可靠的监督来识别和改善推理。只标最终答案与逐步标推理过程各有理由，但早期小学数学实验未显示明确差距；在更强模型、更充足人类反馈和更难数学题上，两者孰优仍需直接比较。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2305.20050#S1"
}
]
},
{
"key": "position",
"value": "研究训练数学推理评分模型时，监督中间步骤是否比只监督最终答案更有效，以及怎样提高人类标注的利用效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.20050"
}
]
},
{
"key": "conclusion",
"value": "过程监督在 MATH 上更有效，主动学习进一步提高标注效率；同时发布逐步人类反馈数据。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.20050"
}
]
}
],
"fields": {
"object": "评价推理步骤的奖励模型参数；任务求解器不在此过程中持续自改。",
"verdict": "人类逐步标注数学解答是否正确，形成约 80 万条步骤标签；模型用这些标注学习评审推理过程，标签并非模型自评生成。"
}
},
"attributions": [
{
"tag": "org:openai",
"label": "OpenAI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2305.20050"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2309.11495",
"title": "Chain-of-Verification Reduces Hallucination in Large Language Models",
"url": "https://arxiv.org/abs/2309.11495",
"date": "2023-09-20",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:eth"
],
"fields": {
"本质定位": "先生成 draft，再规划 verification questions，尽量独立回答这些问题，最后根据验证结果重写答案。",
"什么在变": "当前 response 与临时 verification questions。",
"谁来改 / 谁执行": "**改**：固定 CoVe workflow；同一 LLM 执行各阶段。
**执行**：同一 LLM。",
"基础 harness": "draft→verify-questions→independent answers→final response。",
"Feedback": "模型自己生成的 verification questions/answers，无 persistent external verifier。",
"Evolution → Eval": "list QA、closed-book QA、long-form generation 等 within-instance evaluation。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "相对普通 self-reflection 的关键点是 **把验证问题与原 draft 尽量解耦**，减少模型直接复述原错误；但仍是单次回答内的 verification harness。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 263,
"fields": {
"优先级": "**R**",
"时间": "2023-09-20",
"论文": "[Chain-of-Verification Reduces Hallucination in Large Language Models](https://arxiv.org/abs/2309.11495)",
"本质定位": "先生成 draft，再规划 verification questions，尽量独立回答这些问题，最后根据验证结果重写答案。",
"什么在变": "当前 response 与临时 verification questions。",
"谁来改 / 谁执行": "**改**：固定 CoVe workflow；同一 LLM 执行各阶段。
**执行**：同一 LLM。",
"基础 harness": "draft→verify-questions→independent answers→final response。",
"Feedback": "模型自己生成的 verification questions/answers，无 persistent external verifier。",
"Evolution → Eval": "list QA、closed-book QA、long-form generation 等 within-instance evaluation。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "相对普通 self-reflection 的关键点是 **把验证问题与原 draft 尽量解耦**，减少模型直接复述原错误；但仍是单次回答内的 verification harness。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "先生成 draft，再规划 verification questions，尽量独立回答这些问题，最后根据验证结果重写答案。",
"novelty": "先列出需要核实的问题，再尽量独立回答这些问题，最后修订初稿，减少直接沿用原答案错误的倾向。",
"object": "当前回答及临时验证问题；基础模型与验证流程固定。",
"executor": "主CoVe实验使用基础Llama65B、greedy decoding和提供少量示范的提示提示；指令微调模型是另外的对照。",
"modifier": "同一Llama65B按固定CoVe流程生成核验问题、独立回答并修订初答，不用另一个更强核验模型。",
"roleContext": "**改**：固定 CoVe workflow；同一 LLM 执行各阶段。
**执行**：同一 LLM。",
"seed": "固定四步提示流程：初稿、拟验证问题、独立回答验证问题、据验证结果重写；不改模型或工具系统。",
"fixed": "",
"verdict": "模型自己提出核验问题、回答这些问题，再修订初稿；核验仍依赖模型自身知识，没有持续接入外部正确性检查程序。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "list QA、closed-book QA、long-form generation 等 within-instance evaluation。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：无参数训练；验证问题在当前实例内生成。\n\n调试 / 选版本数据：模型回答自建事实验证问题，不把最终 FactScore 当作内部反馈。\n\n最终测试数据：Wikidata 列表 56；Quest/Wiki-Category 列表 55；MultiSpanQA 418；另测人物传记并用 FactScore。\n\n数据隔离与证据边界：内部自验证与最终外部事实性评分分开；不构成跨任务进化。",
"cycle": "把初稿事实拆成验证问题，部分设置隔离原答案影响，再综合检查结果产出最终回复；内部验证仍由模型完成。",
"train": "无参数训练；验证问题在当前实例内生成。",
"debug": "模型回答自建事实验证问题，不把最终 FactScore 当作内部反馈。",
"test": "Wikidata 列表 56；Quest/Wiki-Category 列表 55；MultiSpanQA 418；另测人物传记并用 FactScore。",
"isolation": "内部自验证与最终外部事实性评分分开；不构成跨任务进化。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定流程为先起草答案、生成待核实问题、独立回答这些问题、再合成最终回答。factored 变体刻意隔离验证回答与初稿影响；不是训练新模型或持久修改 harness。",
"protocol": "**评测：**列表式问答、MultiSpanQA 与人物传记等长文本事实性任务，人物传记用 FactScore 评估。验证问题在当前测试实例内由模型生成，不能称为独立训练数据。逐任务的完整数据来源与数量本轮未核实，不能把 FactScore 当作训练集。",
"sections": "§4 的四项评测与验证提示",
"source": "https://arxiv.org/abs/2309.11495",
"version": "2309.11495v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "61b79b2ca548baf6d7f0f8510d08f27c435456b7989d6f9c231ad5c8a4d64d1d",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主CoVe实验使用基础Llama65B、greedy decoding和提供少量示范的提示提示；指令微调模型是另外的对照。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2309.11495#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一Llama65B按固定CoVe流程生成核验问题、独立回答并修订初答，不用另一个更强核验模型。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2309.11495#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前回答及临时验证问题；基础模型与验证流程固定。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "模型自己提出核验问题、回答这些问题，再修订初稿；核验仍依赖模型自身知识，没有持续接入外部正确性检查程序。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定四步提示流程：初稿、拟验证问题、独立回答验证问题、据验证结果重写；不改模型或工具系统。",
"sources": [
{
"label": "§4 的四项评测与验证提示",
"url": "https://arxiv.org/abs/2309.11495"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "把初稿事实拆成验证问题，部分设置隔离原答案影响，再综合检查结果产出最终回复；内部验证仍由模型完成。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无参数训练；验证问题在当前实例内生成。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "模型回答自建事实验证问题，不把最终 FactScore 当作内部反馈。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Wikidata 列表 56；Quest/Wiki-Category 列表 55；MultiSpanQA 418；另测人物传记并用 FactScore。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "内部自验证与最终外部事实性评分分开；不构成跨任务进化。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先列出需要核实的问题，再尽量独立回答这些问题，最后修订初稿，减少直接沿用原答案错误的倾向。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2309.11495v2",
"version": "2309.11495v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对普通 self-reflection 的关键点是 **把验证问题与原 draft 尽量解耦**，减少模型直接复述原错误；但仍是单次回答内的 verification harness。",
"feedbackCases": [
{
"label": "当前回答的自建验证",
"data": "Wikidata、Quest/Wiki-Category 列表，MultiSpanQA，人物传记。",
"scoring": "模型先列出事实核验问题，再自行回答这些问题，利用与初答的不一致来修订。",
"visible": "自己生成的核验问答；没有把最终 FactScore 接进纠错循环。",
"use": "只修改当前回答。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"judgment": "模型独立回答自建核查问题；不是查询标准答案"
},
{
"label": "最终事实性评估",
"data": "列表题和 MultiSpanQA 的参考答案；人物传记。",
"scoring": "列表/问答按参考内容评价；传记用检索增强语言模型的 FactScore 核查原子事实。",
"visible": "研究端得到事实性分数。",
"use": "外部事实检验用于评价 CoVe，而不是模型内部自建核验问题的来源。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"judgment": "列表／问答按参考内容；传记用检索增强模型逐条核实事实"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "无参数训练；验证问题在当前实例内生成。",
"selection": "模型回答自建事实验证问题，不把最终 FactScore 当作内部反馈。",
"evaluation": "Wikidata 列表 56；Quest/Wiki-Category 列表 55；MultiSpanQA 418；另测人物传记并用 FactScore。",
"isolation": "内部自验证与最终外部事实性评分分开；不构成跨任务进化。",
"roles": {
"executor": {
"value": "主CoVe实验使用基础Llama65B、greedy decoding和提供少量示范的提示提示；指令微调模型是另外的对照。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2309.11495#S4.SS2"
}
]
},
"modifier": {
"value": "同一Llama65B按固定CoVe流程生成核验问题、独立回答并修订初答，不用另一个更强核验模型。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2309.11495#S4.SS2"
}
]
},
"seed": {
"value": "固定四步提示流程：初稿、拟验证问题、独立回答验证问题、据验证结果重写；不改模型或工具系统。",
"sources": [
{
"label": "§4 的四项评测与验证提示",
"url": "https://arxiv.org/abs/2309.11495"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.11495#S4.SS1.SSS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型对训练中较少出现的事实仍会生成看似可信的错误答案，复核时若继续看到自己的错误，又容易原样重复。这样，单纯要求“再检查一次”未必能消除幻觉；作者研究怎样组织核查，让模型利用已有知识检验陈述，并减少初稿对核查的干扰。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2309.11495#S1"
}
]
},
{
"key": "position",
"value": "研究模型能否通过自我核验减少事实性幻觉，并避免验证过程被初稿中的错误带偏。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.11495"
}
]
},
{
"key": "conclusion",
"value": "在事实列表、问答及长文生成中减少幻觉，关键在于把验证与原始回答适度分离。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2309.11495"
}
]
}
],
"fields": {
"object": "当前回答及临时验证问题；基础模型与验证流程固定。",
"verdict": "模型自己提出核验问题、回答这些问题，再修订初稿；核验仍依赖模型自身知识，没有持续接入外部正确性检查程序。"
}
},
"attributions": [
{
"tag": "org:eth",
"label": "ETH Zürich",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2309.11495"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2310.01798",
"title": "Large Language Models Cannot Self-Correct Reasoning Yet",
"url": "https://arxiv.org/abs/2310.01798",
"date": "2023-10-03",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"SelfFeedback",
"org:uiuc",
"person:denny-zhou"
],
"fields": {
"本质定位": "系统评估 intrinsic self-correction：不给外部反馈，仅要求模型检查并修改自己 reasoning。",
"什么在变": "当前答案/reasoning。",
"谁来改 / 谁执行": "**改**：同一 LLM self-correction prompt。
**执行**：GPT-4 / GPT-4-Turbo / Llama-2-70B 等。",
"基础 harness": "标准 reasoning prompt。",
"Feedback": "无外部 feedback。",
"Evolution → Eval": "GSM8K、CommonSenseQA 等 current-run correction。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "关键贡献是负结果：仅靠模型“再想一遍”常不能改进，甚至使正确答案变错。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 264,
"fields": {
"优先级": "**K**",
"时间": "2023-10-03",
"论文": "[Large Language Models Cannot Self-Correct Reasoning Yet](https://arxiv.org/abs/2310.01798)",
"本质定位": "系统评估 intrinsic self-correction：不给外部反馈，仅要求模型检查并修改自己 reasoning。",
"什么在变": "当前答案/reasoning。",
"谁来改 / 谁执行": "**改**：同一 LLM self-correction prompt。
**执行**：GPT-4 / GPT-4-Turbo / Llama-2-70B 等。",
"基础 harness": "标准 reasoning prompt。",
"Feedback": "无外部 feedback。",
"Evolution → Eval": "GSM8K、CommonSenseQA 等 current-run correction。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "关键贡献是负结果：仅靠模型“再想一遍”常不能改进，甚至使正确答案变错。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Feedback"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "系统评估 intrinsic self-correction：不给外部反馈，仅要求模型检查并修改自己 reasoning。",
"novelty": "用对照实验检查没有外部反馈的自纠错，发现要求模型重新思考可能无效，也可能把正确答案改错。",
"object": "当前答案和推理；评测比较不同反馈条件下的自纠错效果。",
"executor": "gpt-3.5-turbo-0613、GPT-4（2023-08-29访问）、gpt-4-1106-preview、Llama-2-70b-chat。",
"modifier": "由对应同一模型根据自我检查提示重答；另设oracle标签反馈对照，不能把外部正确性信号算作模型自身判断。",
"roleContext": "**改**：同一 LLM self-correction prompt。
**执行**：GPT-4 / GPT-4-Turbo / Llama-2-70B 等。",
"seed": "固定推理提示后最多再做两轮自我检查/修订；分别测试仅靠模型自身反馈和获得 oracle 标签的设置。两者反馈信息量不同，不合并称为自主纠错。",
"fixed": "",
"verdict": "无外部反馈。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "GSM8K、CommonSenseQA 等 current-run correction。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：没有额外训练。\n\n调试 / 选版本数据：只依靠模型自身能力 条件只给自纠错提示；oracle 条件额外给真实正确性信号。\n\n最终测试数据：GSM8K（小学数学应用题基准）、CommonsenseQA 开发集 1,221、HotpotQA（需要结合多份资料作答的多跳问答基准） 100；GPT-3.5 用全量，其他模型每集抽 200（HotpotQA 100）。\n\n数据隔离与证据边界：同题重复作答的受控比较；oracle 收益不能归因于纯自反馈。",
"cycle": "比较初答与最多两轮纠错；分别控制是否提供 oracle 正确性信息，及是否已在初始提示充分说明要求。",
"train": "没有额外训练。",
"debug": "只依靠模型自身能力 条件只给自纠错提示；oracle 条件额外给真实正确性信号。",
"test": "GSM8K（小学数学应用题基准）、CommonsenseQA 开发集 1,221、HotpotQA（需要结合多份资料作答的多跳问答基准） 100；GPT-3.5 用全量，其他模型每集抽 200（HotpotQA 100）。",
"isolation": "同题重复作答的受控比较；oracle 收益不能归因于纯自反馈。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定推理提示后最多再做两轮自我检查/修订；分别测试仅靠模型自身反馈和获得 oracle 标签的设置。两者反馈信息量不同，不合并称为自主纠错。",
"protocol": "**测试：**GSM8K、CommonSenseQA、HotpotQA；GPT-3.5 用完整评估集，其他模型为降低成本随机取每集 200 题，HotpotQA 为 100 题。\n\n**无额外训练：**在当前题上反复纠错，比较初次答案和后续答案。oracle 条件用外部正确性信息引导，intrinsic 条件不提供这些信息；不能把 oracle 成绩归因于纯自反馈。",
"sections": "§3.1–3.2、Tables 2–3",
"source": "https://arxiv.org/abs/2310.01798",
"version": "2310.01798v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "96efb8935b64d93ef98c2765befae6f409de418f9de18bf33567b1135f6a4f25",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "gpt-3.5-turbo-0613、GPT-4（2023-08-29访问）、gpt-4-1106-preview、Llama-2-70b-chat。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "由对应同一模型根据自我检查提示重答；另设oracle标签反馈对照，不能把外部正确性信号算作模型自身判断。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前答案和推理；评测比较不同反馈条件下的自纠错效果。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "无外部反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定推理提示后最多再做两轮自我检查/修订；分别测试仅靠模型自身反馈和获得 oracle 标签的设置。两者反馈信息量不同，不合并称为自主纠错。",
"sources": [
{
"label": "§3.1–3.2、Tables 2–3",
"url": "https://arxiv.org/abs/2310.01798"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "比较初答与最多两轮纠错；分别控制是否提供 oracle 正确性信息，及是否已在初始提示充分说明要求。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有额外训练。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "只依靠模型自身能力 条件只给自纠错提示；oracle 条件额外给真实正确性信号。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GSM8K（小学数学应用题基准）、CommonsenseQA 开发集 1,221、HotpotQA（需要结合多份资料作答的多跳问答基准） 100；GPT-3.5 用全量，其他模型每集抽 200（HotpotQA 100）。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同题重复作答的受控比较；oracle 收益不能归因于纯自反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "用对照实验检查没有外部反馈的自纠错，发现要求模型重新思考可能无效，也可能把正确答案改错。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2310.01798v2",
"version": "2310.01798v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "关键贡献是负结果：仅靠模型“再想一遍”常不能改进，甚至使正确答案变错。",
"feedbackCases": [
{
"label": "纯自纠错与真实正确性提示对照",
"data": "GSM8K（小学数学应用题基准）、CommonsenseQA 开发集、HotpotQA（需要结合多份资料作答的多跳问答基准）；具体抽样随执行模型不同。",
"scoring": "纯自纠错只提示模型复查；oracle 对照额外告知当前回答是否正确。最终根据任务参考答案评分，HotpotQA（需要结合多份资料作答的多跳问答基准） 使用精确匹配。",
"visible": "两种设置分别只有自身回答，或另有外部真实正确性信号。",
"use": "检验在没有外部信息时能否纠错；不能把带真实错误提示的收益算作纯自纠错。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
],
"judgment": "纯自纠错无对错信号；对照组给真实正确性标记；最终按基准参考答案计分"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "没有额外训练。",
"selection": "只依靠模型自身能力 条件只给自纠错提示；oracle 条件额外给真实正确性信号。",
"evaluation": "GSM8K（小学数学应用题基准）、CommonsenseQA 开发集 1,221、HotpotQA（需要结合多份资料作答的多跳问答基准） 100；GPT-3.5 用全量，其他模型每集抽 200（HotpotQA 100）。",
"isolation": "同题重复作答的受控比较；oracle 收益不能归因于纯自反馈。",
"roles": {
"executor": {
"value": "gpt-3.5-turbo-0613、GPT-4（2023-08-29访问）、gpt-4-1106-preview、Llama-2-70b-chat。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
]
},
"modifier": {
"value": "由对应同一模型根据自我检查提示重答；另设oracle标签反馈对照，不能把外部正确性信号算作模型自身判断。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.01798#S3.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2310.01798#S5"
}
]
},
"seed": {
"value": "固定推理提示后最多再做两轮自我检查/修订；分别测试仅靠模型自身反馈和获得 oracle 标签的设置。两者反馈信息量不同，不合并称为自主纠错。",
"sources": [
{
"label": "§3.1–3.2、Tables 2–3",
"url": "https://arxiv.org/abs/2310.01798"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "自我纠错常被当作提升推理能力的办法，但其效果究竟来自模型自己发现错误，还是来自外部正确答案等额外帮助，并未充分厘清。如果两者混在一起，就会高估模型在实际拿不到标准答案时的纠错能力；本文专门检验这种无外部反馈的纠错。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2310.01798#S1"
}
]
},
{
"key": "position",
"value": "检验语言模型在没有外部正确性反馈的条件下，是否真正具备修正自身推理错误的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.01798"
}
]
},
{
"key": "conclusion",
"value": "发现提升不可靠，有时还把正确答案改错；不能把重复生成直接当作有效自我纠错。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.01798"
}
]
}
],
"fields": {
"object": "当前答案和推理；评测比较不同反馈条件下的自纠错效果。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2310.01798"
}
]
},
{
"tag": "person:denny-zhou",
"label": "Denny Zhou",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2310.01798"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2310.04406",
"title": "Language Agent Tree Search (LATS)",
"url": "https://arxiv.org/abs/2310.04406",
"date": "2023-10-06",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"org:uiuc"
],
"fields": {
"本质定位": "MCTS 把 reasoning/action trajectory 当树节点，结合 LM value、self-reflection 与 environment feedback 做当前任务搜索。",
"什么在变": "当前 task 的 search tree / trajectory。",
"谁来改 / 谁执行": "**改**：固定 MCTS + LLM policy/value/reflection。
**执行**：GPT 系列 LLM agent。",
"基础 harness": "ReAct-like agent + MCTS。",
"Feedback": "environment reward / tool observations + LM self-evaluation。",
"Evolution → Eval": "HumanEval、WebShop 等；within-task search。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "把 reflection 和 grounded environment reward 放入显式 search/credit-assignment；但搜索树不跨任务持久化。"
},
"details": [],
"occurrences": [
{
"section": "0B. Feedback / Self-Correction 基础（不是 persistent harness evolution）",
"line": 265,
"fields": {
"优先级": "**K**",
"时间": "2023-10-06",
"论文": "[Language Agent Tree Search (LATS)](https://arxiv.org/abs/2310.04406)",
"本质定位": "MCTS 把 reasoning/action trajectory 当树节点，结合 LM value、self-reflection 与 environment feedback 做当前任务搜索。",
"什么在变": "当前 task 的 search tree / trajectory。",
"谁来改 / 谁执行": "**改**：固定 MCTS + LLM policy/value/reflection。
**执行**：GPT 系列 LLM agent。",
"基础 harness": "ReAct-like agent + MCTS。",
"Feedback": "environment reward / tool observations + LM self-evaluation。",
"Evolution → Eval": "HumanEval、WebShop 等；within-task search。",
"Meta-depth": "M0 / 非 persistent。",
"相对之前真正新增什么": "把 reflection 和 grounded environment reward 放入显式 search/credit-assignment；但搜索树不跨任务持久化。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Within-instance",
"protocolBasis": "只修改当前回答/轨迹或当前任务搜索；没有跨任务持续进化。",
"year": "2023",
"depth": [
"M0"
],
"legacyCategories": [
"Feedback"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "MCTS 把 reasoning/action trajectory 当树节点，结合 LM value、self-reflection 与 environment feedback 做当前任务搜索。",
"novelty": "把可能动作组织成搜索树，用环境奖励和文字反思选择后续分支，搜索所得树主要服务当前任务。",
"object": "当前任务的搜索树与行动轨迹；搜索、价值回传及提示规则固定。",
"executor": "LATS 使用 GPT-3.5 或 GPT-4 生成行动与答案；正文报告 GPT-3.5 的 HotPotQA／WebShop（根据用户要求挑选和购买商品的交互基准） 结果，以及 GPT-4 的 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 结果。",
"modifier": "固定 MCTS 程序控制搜索，语言模型提供候选行动、价值判断和失败反思；按 GPT-3.5／GPT-4 配置运行，不训练一个新的搜索模型。",
"roleContext": "**改**：固定 MCTS + LLM policy/value/reflection。
**执行**：GPT 系列 LLM agent。",
"seed": "在 ReAct（交替进行推理、调用工具和读取结果的执行方式） 的推理—动作—观察接口外加蒙特卡洛树搜索，节点保留候选轨迹，结合模型评分、环境反馈和反思选择分支。要求搜索时能回到相应状态，不等于一般现实环境都能无代价回滚。",
"fixed": "",
"verdict": "当前任务的环境奖励、工具返回或可用测试，与模型对候选路径的评价共同指导搜索；失败后另生成反思来尝试其他路径。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Within-instance",
"note": "HumanEval、WebShop 等；within-task search。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：没有离线训练；搜索与反思在当前题内进行。\n\n调试 / 选版本数据：编程用模型生成的内部测试及编译信息；GPT-3.5 用 6 条内部测试，GPT-4 用 4 条。\n\n最终测试数据：HotpotQA（需要结合多份资料作答的多跳问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 全 164、MBPP 抽 397、WebShop（根据用户要求挑选和购买商品的交互基准）。代码最终按正式测试计分。\n\n数据隔离与证据边界：内部测试不等于官方最终测试；多分支采样及环境反馈是测试时计算预算。",
"cycle": "树搜索扩展动作/答案，环境或自建测试反馈更新节点价值；终局失败生成文字反思用于下一条分支，选择成功或高价值结果。",
"train": "没有离线训练；搜索与反思在当前题内进行。",
"debug": "编程用模型生成的内部测试及编译信息；GPT-3.5 用 6 条内部测试，GPT-4 用 4 条。",
"test": "HotpotQA（需要结合多份资料作答的多跳问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 全 164、MBPP 抽 397、WebShop（根据用户要求挑选和购买商品的交互基准）。代码最终按正式测试计分。",
"isolation": "内部测试不等于官方最终测试；多分支采样及环境反馈是测试时计算预算。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在 ReAct 的推理—动作—观察接口外加蒙特卡洛树搜索，节点保留候选轨迹，结合模型评分、环境反馈和反思选择分支。要求搜索时能回到相应状态，不等于一般现实环境都能无代价回滚。",
"protocol": "**任务：**HotpotQA 问答、HumanEval/MBPP 代码、WebShop 购物等；MBPP 实验随机取 397 题。搜索发生在当前评估题内，候选动作和测试代码用于选分支；并无一个统一的参数训练集。各任务完整样本数以及最终评分测试与搜索可见测试的划分本轮待核实。",
"sections": "任务定义、实验和 MBPP 设置",
"source": "https://arxiv.org/abs/2310.04406",
"version": "2310.04406v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f2a5d9cb9a24f20ab4f64597a29c2f6d4a1a39837146ec27505e172a606de444",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "LATS 使用 GPT-3.5 或 GPT-4 生成行动与答案；正文报告 GPT-3.5 的 HotPotQA／WebShop（根据用户要求挑选和购买商品的交互基准） 结果，以及 GPT-4 的 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 结果。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2310.04406#S1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定 MCTS 程序控制搜索，语言模型提供候选行动、价值判断和失败反思；按 GPT-3.5／GPT-4 配置运行，不训练一个新的搜索模型。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2310.04406#S1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "当前任务的搜索树与行动轨迹；搜索、价值回传及提示规则固定。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "当前任务的环境奖励、工具返回或可用测试，与模型对候选路径的评价共同指导搜索；失败后另生成反思来尝试其他路径。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在 ReAct（交替进行推理、调用工具和读取结果的执行方式） 的推理—动作—观察接口外加蒙特卡洛树搜索，节点保留候选轨迹，结合模型评分、环境反馈和反思选择分支。要求搜索时能回到相应状态，不等于一般现实环境都能无代价回滚。",
"sources": [
{
"label": "任务定义、实验和 MBPP 设置",
"url": "https://arxiv.org/abs/2310.04406"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "树搜索扩展动作/答案，环境或自建测试反馈更新节点价值；终局失败生成文字反思用于下一条分支，选择成功或高价值结果。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有离线训练；搜索与反思在当前题内进行。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "编程用模型生成的内部测试及编译信息；GPT-3.5 用 6 条内部测试，GPT-4 用 4 条。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "HotpotQA（需要结合多份资料作答的多跳问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 全 164、MBPP 抽 397、WebShop（根据用户要求挑选和购买商品的交互基准）。代码最终按正式测试计分。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "内部测试不等于官方最终测试；多分支采样及环境反馈是测试时计算预算。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把可能动作组织成搜索树，用环境奖励和文字反思选择后续分支，搜索所得树主要服务当前任务。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2310.04406v3",
"version": "2310.04406v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 reflection 和 grounded environment reward 放入显式 search/credit-assignment；但搜索树不跨任务持久化。",
"feedbackCases": [
{
"label": "HotpotQA 搜索",
"data": "当前多跳问答题，模型调用百科搜索。",
"scoring": "原文采用 oracle 环境：终局给基于正确答案的反馈；搜索中还用模型估计节点价值。",
"visible": "检索观察、真实终局奖励及失败反思。",
"use": "据反馈在同题搜索树中调整，不是无标准答案条件的纯自评。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"judgment": "搜索终局用标准答案反馈；模型另估计中间节点价值"
},
{
"label": "HumanEval / MBPP 搜索",
"data": "HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 164、MBPP 397；GPT-3.5 使用六条、GPT-4 使用四条模型生成内部测试。",
"scoring": "内部测试的通过比例作为树搜索奖励；搜索结束后，选出的代码再接受正式测试。",
"visible": "内部测试结果与编译错误可用于修代码，正式测试内容不作为该内部测试集。",
"use": "区分自建测试指导的搜索与最后代码 pass@1（单次尝试完成任务的比例）。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"judgment": "内部单元测试指导搜索；隐藏正式测试负责最终判分"
},
{
"label": "WebShop 搜索",
"data": "当前购物任务。",
"scoring": "环境按购买结果与目标约束给奖励，结合模型节点估值和失败反思。",
"visible": "购物观察、动作结果及终局奖励。",
"use": "同一任务内搜索更好的行动序列，搜索树不跨任务持续进化。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"judgment": "WebShop 环境奖励 + 模型节点估值"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "没有离线训练；搜索与反思在当前题内进行。",
"selection": "编程用模型生成的内部测试及编译信息；GPT-3.5 用 6 条内部测试，GPT-4 用 4 条。",
"evaluation": "HotpotQA（需要结合多份资料作答的多跳问答基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 全 164、MBPP 抽 397、WebShop（根据用户要求挑选和购买商品的交互基准）。代码最终按正式测试计分。",
"isolation": "内部测试不等于官方最终测试；多分支采样及环境反馈是测试时计算预算。",
"roles": {
"executor": {
"value": "LATS 使用 GPT-3.5 或 GPT-4 生成行动与答案；正文报告 GPT-3.5 的 HotPotQA／WebShop（根据用户要求挑选和购买商品的交互基准） 结果，以及 GPT-4 的 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准） 结果。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2310.04406#S1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
]
},
"modifier": {
"value": "固定 MCTS 程序控制搜索，语言模型提供候选行动、价值判断和失败反思；按 GPT-3.5／GPT-4 配置运行，不训练一个新的搜索模型。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2310.04406#S4.SS2"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2310.04406#S1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
]
},
"seed": {
"value": "在 ReAct（交替进行推理、调用工具和读取结果的执行方式） 的推理—动作—观察接口外加蒙特卡洛树搜索，节点保留候选轨迹，结合模型评分、环境反馈和反思选择分支。要求搜索时能回到相应状态，不等于一般现实环境都能无代价回滚。",
"sources": [
{
"label": "任务定义、实验和 MBPP 设置",
"url": "https://arxiv.org/abs/2310.04406"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"selection": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"evaluation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
],
"isolation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2310.04406#S5.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2310.04406#A4.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 能利用工具和环境反馈，却常沿当前路径作出反应，缺少比较多个候选和提前规划；已有推理搜索又常脱离外部交互。两类能力分开使用，使复杂任务难以同时受益于规划与真实反馈，作者因此研究如何把它们组织为统一的决策过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2310.04406#S1"
}
]
},
{
"key": "position",
"value": "面向需要探索、回退和长程规划的任务，研究语言 agent 能否通过比较行动路径作出更可靠的决策。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.04406"
}
]
},
{
"key": "conclusion",
"value": "在编程、网页、问答等任务中改善决策；搜索针对当前任务展开，不更新模型参数。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.04406"
}
]
}
],
"fields": {
"object": "当前任务的搜索树与行动轨迹；搜索、价值回传及提示规则固定。",
"verdict": "当前任务的环境奖励、工具返回或可用测试，与模型对候选路径的评价共同指导搜索；失败后另生成反思来尝试其他路径。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2310.04406"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2309.03409",
"title": "Large Language Models as Optimizers (OPRO)",
"url": "https://arxiv.org/abs/2309.03409",
"date": "2023-09-07",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Prompt",
"org:google-deepmind",
"person:denny-zhou",
"person:quoc-le"
],
"fields": {
"本质定位": "optimizer LLM 读取历史 candidate + score，提出新的自然语言 solution/prompt。",
"什么在变": "task prompt / natural-language solution。",
"谁来改 / 谁执行": "**改**：固定 optimizer LLM + optimizer prompt。
**执行**：被优化 prompt 下的 target LLM。",
"基础 harness": "基础 prompt。",
"Feedback": "labeled task metric / score。",
"Evolution → Eval": "优化集打分，选 best prompt 后 test；相对标准 prompt optimization 较干净。",
"Meta-depth": "M1：task prompt 变，optimizer 固定。",
"相对之前真正新增什么": "把黑盒 prompt optimization 明确化为“history of scored attempts → proposer → new candidate”。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 273,
"fields": {
"优先级": "**R**",
"时间": "2023-09-07",
"论文": "[Large Language Models as Optimizers (OPRO)](https://arxiv.org/abs/2309.03409)",
"本质定位": "optimizer LLM 读取历史 candidate + score，提出新的自然语言 solution/prompt。",
"什么在变": "task prompt / natural-language solution。",
"谁来改 / 谁执行": "**改**：固定 optimizer LLM + optimizer prompt。
**执行**：被优化 prompt 下的 target LLM。",
"基础 harness": "基础 prompt。",
"Feedback": "labeled task metric / score。",
"Evolution → Eval": "优化集打分，选 best prompt 后 test；相对标准 prompt optimization 较干净。",
"Meta-depth": "M1：task prompt 变，optimizer 固定。",
"相对之前真正新增什么": "把黑盒 prompt optimization 明确化为“history of scored attempts → proposer → new candidate”。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M1"
],
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "optimizer LLM 读取历史 candidate + score，提出新的自然语言 solution/prompt。",
"novelty": "把已有候选及其分数放入提示，让模型继续提出新候选，形成可以用于提示优化的迭代搜索。",
"object": "任务提示或自然语言候选方案；评分模型、指标与优化提示模板固定。",
"executor": "提示优化中的评分器/任务回答模型为预训练PaLM2-L或text-bison；这里“评分器”指用候选提示答题的模型，准确率再对标准答案计算。",
"modifier": "优化器分别用预训练PaLM2-L、PaLM2-L-IT、text-bison、gpt-3.5-turbo、gpt-4，根据候选提示及分数历史产生新提示。",
"roleContext": "**改**：固定 optimizer LLM + optimizer prompt。
**执行**：被优化 prompt 下的 target LLM。",
"seed": "优化器 语言模型 读取过去的候选提示及其分数，提出新提示；任务模型负责用候选提示做题。可以从空字符串开始，改的是提示，不是运行工具或模型参数。",
"fixed": "M1：task prompt 变，optimizer 固定",
"verdict": "用带参考答案的任务计算评价指标和分数。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "优化集打分，选 best prompt 后 test；相对标准 prompt optimization 较干净。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GSM8K（小学数学应用题基准） 约 3.5% 官方训练题用于提示搜索；BBH 每任务 20%。\n\n调试 / 选版本数据：同一搜索子集的准确率作为目标，不提供梯度。\n\n最终测试数据：GSM8K（小学数学应用题基准） 官方 1,319 测试集；BBH 23 个任务的剩余 80%。\n\n数据隔离与证据边界：训练分数、测试集 分数与 整体 分开；数学优化小实验另测自身目标函数。",
"cycle": "优化器 语言模型 读取历史提示与训练准确率，再提出新提示；最终选择高分候选，评分器 语言模型 与优化器可不同。",
"train": "GSM8K（小学数学应用题基准） 约 3.5% 官方训练题用于提示搜索；BBH 每任务 20%。",
"debug": "同一搜索子集的准确率作为目标，不提供梯度。",
"test": "GSM8K（小学数学应用题基准） 官方 1,319 测试集；BBH 23 个任务的剩余 80%。",
"isolation": "训练分数、测试集 分数与 整体 分开；数学优化小实验另测自身目标函数。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "优化器 LLM 读取过去的候选提示及其分数，提出新提示；任务模型负责用候选提示做题。可以从空字符串开始，改的是提示，不是运行工具或模型参数。",
"protocol": "**数据：**GSM8K（原始 7,473 train、1,319 test）与 BBH 的 23 类任务。BBH 先按 20%/80% 分优化和测试；表格同时报告 training/test/overall，不能把 overall 说成留出成绩。GSM8K 实际用于搜索的训练抽样量本轮待核实，原数据总量不等于优化使用量。",
"sections": "主要评测数据；BBH Table 7",
"source": "https://arxiv.org/abs/2309.03409",
"version": "2309.03409v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "8b12eefe0ceb73155642cba12312b7b03fa3f0b326fece4a22a6d4b47d1ccde9",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "提示优化中的评分器/任务回答模型为预训练PaLM2-L或text-bison；这里“评分器”指用候选提示答题的模型，准确率再对标准答案计算。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2309.03409#S5.SS2.SSS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2309.03409#S5.SS5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "优化器分别用预训练PaLM2-L、PaLM2-L-IT、text-bison、gpt-3.5-turbo、gpt-4，根据候选提示及分数历史产生新提示。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2309.03409#S5.SS2.SSS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2309.03409#S5.SS5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务提示或自然语言候选方案；评分模型、指标与优化提示模板固定。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用带参考答案的任务计算评价指标和分数。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "优化器 语言模型 读取过去的候选提示及其分数，提出新提示；任务模型负责用候选提示做题。可以从空字符串开始，改的是提示，不是运行工具或模型参数。",
"sources": [
{
"label": "主要评测数据；BBH Table 7",
"url": "https://arxiv.org/abs/2309.03409"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "优化器 语言模型 读取历史提示与训练准确率，再提出新提示；最终选择高分候选，评分器 语言模型 与优化器可不同。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GSM8K（小学数学应用题基准） 约 3.5% 官方训练题用于提示搜索；BBH 每任务 20%。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同一搜索子集的准确率作为目标，不提供梯度。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GSM8K（小学数学应用题基准） 官方 1,319 测试集；BBH 23 个任务的剩余 80%。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "训练分数、测试集 分数与 整体 分开；数学优化小实验另测自身目标函数。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把已有候选及其分数放入提示，让模型继续提出新候选，形成可以用于提示优化的迭代搜索。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2309.03409v3",
"version": "2309.03409v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把黑盒 prompt optimization 明确化为“history of scored attempts → proposer → new candidate”。",
"feedbackCases": [
{
"label": "提示词搜索：GSM8K / BBH",
"data": "GSM8K（小学数学应用题基准） 约 3.5% 训练题；BBH 每任务 20%。",
"scoring": "让任务模型按候选提示答题，再与参考答案计算训练准确率。文中 scorer 语言模型 是执行候选提示答题的模型，不是凭喜好给提示打分的评审者。",
"visible": "优化模型读取提示—分数历史。",
"use": "用数值目标生成新提示，最后测 GSM8K（小学数学应用题基准） 1,319 题及 BBH 剩余 80%。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"judgment": "规则以任务模型的回答对照参考答案算准确率；scorer LLM 是答题者"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "GSM8K（小学数学应用题基准） 约 3.5% 官方训练题用于提示搜索；BBH 每任务 20%。",
"selection": "同一搜索子集的准确率作为目标，不提供梯度。",
"evaluation": "GSM8K（小学数学应用题基准） 官方 1,319 测试集；BBH 23 个任务的剩余 80%。",
"isolation": "训练分数、测试集 分数与 整体 分开；数学优化小实验另测自身目标函数。",
"roles": {
"executor": {
"value": "提示优化中的评分器/任务回答模型为预训练PaLM2-L或text-bison；这里“评分器”指用候选提示答题的模型，准确率再对标准答案计算。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2309.03409#S5.SS2.SSS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2309.03409#S5.SS5"
}
]
},
"modifier": {
"value": "优化器分别用预训练PaLM2-L、PaLM2-L-IT、text-bison、gpt-3.5-turbo、gpt-4，根据候选提示及分数历史产生新提示。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2309.03409#S3.SS2.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2309.03409#S5.SS2.SSS1"
},
{
"label": "§5.5",
"url": "https://arxiv.org/html/2309.03409#S5.SS5"
}
]
},
"seed": {
"value": "优化器 语言模型 读取过去的候选提示及其分数，提出新提示；任务模型负责用候选提示做题。可以从空字符串开始，改的是提示，不是运行工具或模型参数。",
"sources": [
{
"label": "主要评测数据；BBH Table 7",
"url": "https://arxiv.org/abs/2309.03409"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2309.03409#S4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2309.03409#S5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "传统优化器往往要为不同问题专门设计，尤其在拿不到梯度时；提示优化还面临离散空间巨大、措辞相近但效果相差很远的问题。每换模型或任务都重新手工调提示难以扩展，作者因此研究能否让模型理解自然语言目标和历史分数，直接提出更好的候选。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2309.03409#S1"
}
]
},
{
"key": "position",
"value": "研究语言模型能否在没有可用梯度的情况下充当优化器，根据目标与历史试验结果改进数值方案或任务提示。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.03409"
}
]
},
{
"key": "conclusion",
"value": "在数学优化和提示搜索中展示有效性；优化出的任务提示可优于人工提示。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2309.03409"
}
]
}
],
"fields": {
"object": "任务提示或自然语言候选方案；评分模型、指标与优化提示模板固定。"
}
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2309.03409"
}
]
},
{
"tag": "person:denny-zhou",
"label": "Denny Zhou",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2309.03409"
}
]
},
{
"tag": "person:quoc-le",
"label": "Quoc V. Le",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2309.03409"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2309.16797",
"title": "Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution",
"url": "https://arxiv.org/abs/2309.16797",
"date": "2023-09-28",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Improver",
"M2",
"Population",
"Prompt",
"org:google-deepmind"
],
"fields": {
"本质定位": "同时演化 task prompt 与负责产生 prompt mutation 的 mutation prompt。",
"什么在变": "task prompts + mutation prompts。",
"谁来改 / 谁执行": "**改**：固定 evolutionary algorithm；mutation prompts 参与生成后续 mutation。
**执行**：target LM + prompt population。",
"基础 harness": "初始 task prompt + mutation operators/prompts。",
"Feedback": "training-task fitness / accuracy。",
"Evolution → Eval": "train fitness → best candidate test。",
"Meta-depth": "M2（局部 self-reference）：improvement instruction 变，但最外层 EA 固定。",
"相对之前真正新增什么": "相对 OPRO 的真正新点：不只 P 在变，产生 P 变异的 M 也被 evolutionary hyper-mutation 改。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 274,
"fields": {
"优先级": "**K**",
"时间": "2023-09-28",
"论文": "[Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution](https://arxiv.org/abs/2309.16797)",
"本质定位": "同时演化 task prompt 与负责产生 prompt mutation 的 mutation prompt。",
"什么在变": "task prompts + mutation prompts。",
"谁来改 / 谁执行": "**改**：固定 evolutionary algorithm；mutation prompts 参与生成后续 mutation。
**执行**：target LM + prompt population。",
"基础 harness": "初始 task prompt + mutation operators/prompts。",
"Feedback": "training-task fitness / accuracy。",
"Evolution → Eval": "train fitness → best candidate test。",
"Meta-depth": "M2（局部 self-reference）：improvement instruction 变，但最外层 EA 固定。",
"相对之前真正新增什么": "相对 OPRO 的真正新点：不只 P 在变，产生 P 变异的 M 也被 evolutionary hyper-mutation 改。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Prompt",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "同时演化 task prompt 与负责产生 prompt mutation 的 mutation prompt。",
"novelty": "同时进化任务提示和指导如何变异任务提示的文字指令，让产生修改的规则也成为可更新对象。",
"object": "任务提示和指导其变异的提示；基础模型与适应度评分规则固定。",
"executor": "PaLM 2-L 读取候选任务提示并解题；表 1 的 Promptbreeder 结果使用这一基础模型。",
"modifier": "Promptbreeder 用语言模型生成任务提示及变异提示，再按任务正确率选择；实验底层模型为 PaLM 2-L，进化的是提示文本。",
"roleContext": "**改**：固定 evolutionary algorithm；mutation prompts 参与生成后续 mutation。
**执行**：target LM + prompt population。",
"seed": "维护 50 个个体的提示种群，每个体包含任务提示和产生变异的提示；既修改做题说明，也修改下一代怎样改提示。执行工具和模型权重不变。",
"fixed": "M2（局部 self-reference）：improvement instruction 变，但最外层 EA 固定",
"verdict": "训练任务上的正确率等指标，用来比较候选提示词好坏。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "train fitness → best candidate test。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GSM8K（小学数学应用题基准）、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ 等训练题，每次随机 100 题计算适应度。\n\n调试 / 选版本数据：训练批次准确率用于选择，种群规模 50；100 是批量而非训练总量。\n\n最终测试数据：各任务 测试集；无官方划分的 MultiArith、AddSub、SingleEQ、SVAMP 预先均分两半。\n\n数据隔离与证据边界：最终提示按训练表现选再测 测试集，不另把重复适应度批次当验证集。",
"cycle": "共同演化任务提示和生成变异的提示，含直接变异、超变异与从成功推理反推提示；按批量准确率竞争，平台期后选全程最好个体。",
"train": "GSM8K（小学数学应用题基准）、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ 等训练题，每次随机 100 题计算适应度。",
"debug": "训练批次准确率用于选择，种群规模 50；100 是批量而非训练总量。",
"test": "各任务 测试集；无官方划分的 MultiArith、AddSub、SingleEQ、SVAMP 预先均分两半。",
"isolation": "最终提示按训练表现选再测 测试集，不另把重复适应度批次当验证集。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "维护 50 个个体的提示种群，每个体包含任务提示和产生变异的提示；既修改做题说明，也修改下一代怎样改提示。执行工具和模型权重不变。",
"protocol": "**进化：**从训练集随机取 100 例计算候选适应度。算术任务包含 GSM8K、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ。\n\n**测试划分：**没有现成 train/test 的 MultiArith、AddSub、SingleEQ、SVAMP 在实验前一分为二；其他任务使用各自划分。100 是每次评估批量，不是全部训练池大小；独立验证/选最终提示集本轮未核实。",
"sections": "实验设置；附录 I.2",
"source": "https://arxiv.org/abs/2309.16797",
"version": "2309.16797v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "b55570c774fffa4e11d5fe6ba0b6229ad0b61cad514fbf2a0502487d12d17f0c",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "PaLM 2-L 读取候选任务提示并解题；表 1 的 Promptbreeder 结果使用这一基础模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2309.16797#S1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2309.16797#S5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Promptbreeder 用语言模型生成任务提示及变异提示，再按任务正确率选择；实验底层模型为 PaLM 2-L，进化的是提示文本。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2309.16797#S1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2309.16797#S5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务提示和指导其变异的提示；基础模型与适应度评分规则固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "训练任务上的正确率等指标，用来比较候选提示词好坏。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "维护 50 个个体的提示种群，每个体包含任务提示和产生变异的提示；既修改做题说明，也修改下一代怎样改提示。执行工具和模型权重不变。",
"sources": [
{
"label": "实验设置；附录 I.2",
"url": "https://arxiv.org/abs/2309.16797"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "共同演化任务提示和生成变异的提示，含直接变异、超变异与从成功推理反推提示；按批量准确率竞争，平台期后选全程最好个体。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GSM8K（小学数学应用题基准）、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ 等训练题，每次随机 100 题计算适应度。",
"sources": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练批次准确率用于选择，种群规模 50；100 是批量而非训练总量。",
"sources": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各任务 测试集；无官方划分的 MultiArith、AddSub、SingleEQ、SVAMP 预先均分两半。",
"sources": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "最终提示按训练表现选再测 测试集，不另把重复适应度批次当验证集。",
"sources": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "同时进化任务提示和指导如何变异任务提示的文字指令，让产生修改的规则也成为可更新对象。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2309.16797v1",
"version": "2309.16797v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 OPRO 的真正新点：不只 P 在变，产生 P 变异的 M 也被 evolutionary hyper-mutation 改。",
"feedbackCases": [
{
"label": "任务提示与变异提示进化",
"data": "GSM8K（小学数学应用题基准）、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ 等；每次随机 100 道训练题。",
"scoring": "把任务模型答案与参考答案比较，计算准确率作为候选适应度；变异提示可按产生更好任务提示的比例评价。",
"visible": "训练批次成绩和成功示例供种群选择；100 是批量而非全部训练量。",
"use": "任务提示及指导如何改提示的文字共同进化，最终用预留测试划分。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
},
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"judgment": "回答与参考答案比较的准确率；所引段落未展开各任务解析器"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "GSM8K（小学数学应用题基准）、SVAMP、MultiArith、AddSub、AQuA-RAT、SingleEQ 等训练题，每次随机 100 题计算适应度。",
"selection": "训练批次准确率用于选择，种群规模 50；100 是批量而非训练总量。",
"evaluation": "各任务 测试集；无官方划分的 MultiArith、AddSub、SingleEQ、SVAMP 预先均分两半。",
"isolation": "最终提示按训练表现选再测 测试集，不另把重复适应度批次当验证集。",
"roles": {
"executor": {
"value": "PaLM 2-L 读取候选任务提示并解题；表 1 的 Promptbreeder 结果使用这一基础模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2309.16797#S1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2309.16797#S5"
}
]
},
"modifier": {
"value": "Promptbreeder 用语言模型生成任务提示及变异提示，再按任务正确率选择；实验底层模型为 PaLM 2-L，进化的是提示文本。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2309.16797#S1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2309.16797#S5"
}
]
},
"seed": {
"value": "维护 50 个个体的提示种群，每个体包含任务提示和产生变异的提示；既修改做题说明，也修改下一代怎样改提示。执行工具和模型权重不变。",
"sources": [
{
"label": "实验设置；附录 I.2",
"url": "https://arxiv.org/abs/2309.16797"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"selection": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"evaluation": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
],
"isolation": [
{
"label": "附录J.2",
"url": "https://arxiv.org/html/2309.16797#A10.SS2"
},
{
"label": "附录L",
"url": "https://arxiv.org/html/2309.16797#A12"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "提示措辞显著影响模型表现，但人工提示设计难以系统探索。已有自动提示方法 APE 在迭代几轮后收益趋于停滞，直接让网络修改自身权重又成本高；作者因而寻求能保持搜索多样性、连“怎样改提示”也能一起改进的低成本途径。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2309.16797#S1"
}
]
},
{
"key": "position",
"value": "研究任务提示与提示改写策略能否共同改进，使自动提示搜索摆脱固定人工变异规则的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.16797"
}
]
},
{
"key": "conclusion",
"value": "在推理和分类任务中优于多种提示对照；变化不仅在答案提示，也在提示改写规则。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2309.16797"
}
]
}
],
"fields": {
"executor": "PaLM 2-L 读取候选任务提示并解题，模型参数不随提示进化而更新。",
"object": "任务提示和指导其变异的提示；基础模型与适应度评分规则固定。"
}
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2309.16797"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2310.03714",
"title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
"url": "https://arxiv.org/abs/2310.03714",
"date": "2023-10",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"GoldLabel",
"M1",
"Prompt",
"org:stanford",
"org:cmu",
"org:berkeley",
"person:omar-khattab"
],
"fields": {
"本质定位": "把 LM application 写成 declarative modules/signatures，再用 compiler 自动优化 instructions / demonstrations。",
"什么在变": "prompt、few-shot demonstrations、部分 module configuration。",
"谁来改 / 谁执行": "**改**：固定 DSPy optimizer/compiler（如 BootstrapFewShot 等）。
**执行**：LM program 中的 target LMs。",
"基础 harness": "declarative LM modules。",
"Feedback": "用户定义 metric / labeled examples。",
"Evolution → Eval": "train/dev optimization → test。",
"Meta-depth": "M1 substrate。",
"相对之前真正新增什么": "贡献是把 prompt/pipeline optimization 工程化为可编译程序，而不是让 agent 自己持续学习。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 275,
"fields": {
"优先级": "**K**",
"时间": "2023-10",
"论文": "[DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines](https://arxiv.org/abs/2310.03714)",
"本质定位": "把 LM application 写成 declarative modules/signatures，再用 compiler 自动优化 instructions / demonstrations。",
"什么在变": "prompt、few-shot demonstrations、部分 module configuration。",
"谁来改 / 谁执行": "**改**：固定 DSPy optimizer/compiler（如 BootstrapFewShot 等）。
**执行**：LM program 中的 target LMs。",
"基础 harness": "declarative LM modules。",
"Feedback": "用户定义 metric / labeled examples。",
"Evolution → Eval": "train/dev optimization → test。",
"Meta-depth": "M1 substrate。",
"相对之前真正新增什么": "贡献是把 prompt/pipeline optimization 工程化为可编译程序，而不是让 agent 自己持续学习。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 LM application 写成 declarative modules/signatures，再用 compiler 自动优化 instructions / demonstrations。",
"novelty": "先用模块描述任务流水线，再自动选择示例和适配各模块提示，减少逐个手工编写提示的工作。",
"object": "提示、示例与模块配置；参数微调设置还会更新执行模型。",
"executor": "提示编译实验由 GPT-3.5 或 Llama2-13B-Chat 执行编译后的程序；参数微调实验另由 T5-Large（770M）执行。",
"modifier": "DSPy 的 提示优化器 运行教师程序、筛选示例并编译提示；教师可以是同一模型的程序。T5-Large 的多跳检索微调明确使用两个 Llama2-13B-Chat 多跳程序组成的教师集成。",
"roleContext": "**改**：固定 DSPy optimizer/compiler（如 BootstrapFewShot 等）。
**执行**：LM program 中的 target LMs。",
"seed": "开发者先定义检索、推理、回答等模块和接口，编译器再为各模块选择/生成示例及提示。基础结构由人给定；例如 HotPotQA 使用 Wikipedia 2017 摘要索引和 ColBERTv2 检索，不是让优化器任意发明所有工具。",
"fixed": "",
"verdict": "由使用者定义任务评分指标，结合带标签示例比较程序输出；优化器据此选择提示和示例，部分分支也训练模型参数。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "HotPotQA fullwiki case study",
"evolve": "官方 training 中 hard 子集再划分；训练采 200",
"selection": "同一 training pool 划分后 development 采 300",
"test": "官方 validation 抽 1,000 作为测试",
"isolation": "原始数据集 split 名称与本文用途不同",
"note": "因官方 test 隐藏，本文的 test 实际来自官方 validation；检索使用 Wikipedia 2017 abstracts + ColBERTv2。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2310.03714v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GSM8K（小学数学应用题基准） 200；HotpotQA（需要结合多份资料作答的多跳问答基准） hard 题 200，用于编译/自举示例。\n\n调试 / 选版本数据：各 300 道 开发集；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 训练集 内部划分。\n\n最终测试数据：GSM8K（小学数学应用题基准） 1,319；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 验证集 抽 1,000，因为正式 测试集 隐藏。\n\n数据隔离与证据边界：按用途区分集合；官方 验证集 在此充当最终测试，不是编译时的开发集。",
"cycle": "固定的模块化程序交给提示优化器，用训练示例和任务指标选择提示及示范。部分配置还对模块做参数微调，因此编译后的改善不能一概归为纯提示变化。",
"train": "GSM8K（小学数学应用题基准） 200；HotpotQA（需要结合多份资料作答的多跳问答基准） hard 题 200，用于编译/自举示例。",
"debug": "各 300 道 开发集；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 训练集 内部划分。",
"test": "GSM8K（小学数学应用题基准） 1,319；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 验证集 抽 1,000，因为正式 测试集 隐藏。",
"isolation": "按用途区分集合；官方 验证集 在此充当最终测试，不是编译时的开发集。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "开发者先定义检索、推理、回答等模块和接口，编译器再为各模块选择/生成示例及提示。基础结构由人给定；例如 HotPotQA 使用 Wikipedia 2017 摘要索引和 ColBERTv2 检索，不是让优化器任意发明所有工具。",
"protocol": "**HotPotQA：**从官方训练集只保留 hard 题，再抽 200 题编译、300 题开发验证；从官方 validation 抽 1,000 题当最终测试，因为官方 test 不公开。\n\n**GSM8K：**实验用 200 训练、300 验证、1,319 测试的设置。模块编译和选版本使用前两者，不能把官方 validation 的名字直接当成优化用验证角色。其他案例见原始详细实验记录。",
"sections": "HotPotQA fullwiki 案例与数学实验",
"source": "https://arxiv.org/abs/2310.03714",
"version": "2310.03714v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "b8c868632bcbdb7478e067ebe92f526db119581f7b278cb40392150e74fa485c",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "提示编译实验由 GPT-3.5 或 Llama2-13B-Chat 执行编译后的程序；参数微调实验另由 T5-Large（770M）执行。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2310.03714#S3"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "DSPy 的 提示优化器 运行教师程序、筛选示例并编译提示；教师可以是同一模型的程序。T5-Large 的多跳检索微调明确使用两个 Llama2-13B-Chat 多跳程序组成的教师集成。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2310.03714#S3"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "提示、示例与模块配置；参数微调设置还会更新执行模型。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "由使用者定义任务评分指标，结合带标签示例比较程序输出；优化器据此选择提示和示例，部分分支也训练模型参数。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "开发者先定义检索、推理、回答等模块和接口，编译器再为各模块选择/生成示例及提示。基础结构由人给定；例如 HotPotQA 使用 Wikipedia 2017 摘要索引和 ColBERTv2 检索，不是让优化器任意发明所有工具。",
"sources": [
{
"label": "HotPotQA fullwiki 案例与数学实验",
"url": "https://arxiv.org/abs/2310.03714"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "固定的模块化程序交给提示优化器，用训练示例和任务指标选择提示及示范。部分配置还对模块做参数微调，因此编译后的改善不能一概归为纯提示变化。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GSM8K（小学数学应用题基准） 200；HotpotQA（需要结合多份资料作答的多跳问答基准） hard 题 200，用于编译/自举示例。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "各 300 道 开发集；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 训练集 内部划分。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GSM8K（小学数学应用题基准） 1,319；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 验证集 抽 1,000，因为正式 测试集 隐藏。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "按用途区分集合；官方 验证集 在此充当最终测试，不是编译时的开发集。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先用模块描述任务流水线，再自动选择示例和适配各模块提示，减少逐个手工编写提示的工作。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2310.03714v1",
"version": "2310.03714v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "用声明式模块表达任务流水线，把示例选择和提示适配交给 compiler；本文 HotPotQA 中官方 validation 被用作最终测试，不能只按数据集字段名判断是否参与优化。",
"feedbackCases": [
{
"label": "示例自举与程序编译",
"data": "GSM8K（小学数学应用题基准） 200、HotpotQA（需要结合多份资料作答的多跳问答基准） hard 200 训练题；各 300 道开发题。",
"scoring": "用户定义的 metric 依据题目参考答案判断结果，例如答案精确匹配或 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）；验证成功的中间推理/检索轨迹可自举为示范。",
"visible": "训练问答标签和 metric 结果，开发成绩用于选择。",
"use": "优化提示与示范，最终测 GSM8K（小学数学应用题基准） 1,319 及 HotpotQA（需要结合多份资料作答的多跳问答基准） 官方 验证集 抽取的 1,000 题。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"judgment": "用户定义的程序化 metric，例如参考答案精确匹配／词项 F1"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "GSM8K（小学数学应用题基准） 200；HotpotQA（需要结合多份资料作答的多跳问答基准） hard 题 200，用于编译/自举示例。",
"selection": "各 300 道 开发集；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 训练集 内部划分。",
"evaluation": "GSM8K（小学数学应用题基准） 1,319；HotpotQA（需要结合多份资料作答的多跳问答基准） 从官方 验证集 抽 1,000，因为正式 测试集 隐藏。",
"isolation": "按用途区分集合；官方 验证集 在此充当最终测试，不是编译时的开发集。",
"roles": {
"executor": {
"value": "提示编译实验由 GPT-3.5 或 Llama2-13B-Chat 执行编译后的程序；参数微调实验另由 T5-Large（770M）执行。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2310.03714#S3"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7.SS0.SSS0.Px3"
}
]
},
"modifier": {
"value": "DSPy 的 提示优化器 运行教师程序、筛选示例并编译提示；教师可以是同一模型的程序。T5-Large 的多跳检索微调明确使用两个 Llama2-13B-Chat 多跳程序组成的教师集成。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2310.03714#S3.SS3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2310.03714#S4.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2310.03714#S3"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7.SS0.SSS0.Px3"
}
]
},
"seed": {
"value": "开发者先定义检索、推理、回答等模块和接口，编译器再为各模块选择/生成示例及提示。基础结构由人给定；例如 HotPotQA 使用 Wikipedia 2017 摘要索引和 ColBERTv2 检索，不是让优化器任意发明所有工具。",
"sources": [
{
"label": "HotPotQA fullwiki 案例与数学实验",
"url": "https://arxiv.org/abs/2310.03714"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"selection": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"evaluation": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
],
"isolation": [
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2310.03714#S6.SS0.SSS0.Px2"
},
{
"label": "§7",
"url": "https://arxiv.org/html/2310.03714#S7"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型对提示措辞敏感，多步流程中各次调用又必须相互配合。现有系统依靠人工试错写死的长提示模板，换模型、任务领域或流程后可能失效，维护成本也随流程复杂度上升；作者因此希望用可组合、可自动优化的程序模块替代这种逐条手调方式。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2310.03714#S1"
}
]
},
{
"key": "position",
"value": "面向多步语言模型应用，解决程序设计如何与具体提示调优分离，使系统能够根据任务指标自动适配。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.03714"
}
]
},
{
"key": "conclusion",
"value": "少量代码可构建并优化多步推理与检索程序，小模型也能通过流程编译取得竞争力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.03714"
}
]
}
],
"fields": {
"object": "提示、示例与模块配置；参数微调设置还会更新执行模型。",
"verdict": "由使用者定义任务评分指标，结合带标签示例比较程序输出；优化器据此选择提示和示例，部分分支也训练模型参数。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2310.03714"
}
]
},
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2310.03714"
}
]
},
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2310.03714"
}
]
},
{
"tag": "person:omar-khattab",
"label": "Omar Khattab",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2310.03714"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2310.02304",
"title": "STOP: Self-Taught Optimizer / Recursively Self-Improving Code Generation",
"url": "https://arxiv.org/abs/2310.02304",
"date": "2023-10-03",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Improver",
"M2"
],
"fields": {
"本质定位": "让 LLM 编写一个 improver scaffold，用 utility 反馈改程序；随后把该 improver 用于改进它自己。",
"什么在变": "improver/scaffolding code 与下游 solution program。",
"谁来改 / 谁执行": "**改**：LLM 在 seed improver 约束下生成 rewrite；improved improver 可参与下一轮。
**执行**：固定基础 LLM。",
"基础 harness": "seed improver program + task utility。",
"Feedback": "可执行 utility / benchmark score。",
"Evolution → Eval": "一组程序优化任务；不是大规模 agent held-out evaluation。",
"Meta-depth": "M2：improver code 可变，base LM/outer utility/runtime 固定。",
"相对之前真正新增什么": "比一般 code optimization 多了一层 self-application：improver 是输入程序之一。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 276,
"fields": {
"优先级": "**K**",
"时间": "2023-10-03",
"论文": "[STOP: Self-Taught Optimizer / Recursively Self-Improving Code Generation](https://arxiv.org/abs/2310.02304)",
"本质定位": "让 LLM 编写一个 improver scaffold，用 utility 反馈改程序；随后把该 improver 用于改进它自己。",
"什么在变": "improver/scaffolding code 与下游 solution program。",
"谁来改 / 谁执行": "**改**：LLM 在 seed improver 约束下生成 rewrite；improved improver 可参与下一轮。
**执行**：固定基础 LLM。",
"基础 harness": "seed improver program + task utility。",
"Feedback": "可执行 utility / benchmark score。",
"Evolution → Eval": "一组程序优化任务；不是大规模 agent held-out evaluation。",
"Meta-depth": "M2：improver code 可变，base LM/outer utility/runtime 固定。",
"相对之前真正新增什么": "比一般 code optimization 多了一层 self-application：improver 是输入程序之一。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"F",
"Meta"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "让 LLM 编写一个 improver scaffold，用 utility 反馈改程序；随后把该 improver 用于改进它自己。",
"novelty": "把“改进输入程序的程序”本身也作为待改进输入，实际调用它修改自己的实现，形成程序层面的自我应用。",
"object": "代码改进器的程序，以及它优化的下游程序；基础模型、预算和外部效用定义固定。",
"executor": "固定 GPT-4 供候选改进程序调用；程序运行和任务效用计算由 Python 环境完成。",
"modifier": "GPT-4 在改进方案的模型或程序的提示和调用规则下生成代码改写；改进后的改进方案的模型或程序可以参与下一轮，GPT-4 参数保持不变。",
"roleContext": "**改**：LLM 在 seed improver 约束下生成 rewrite；improved improver 可参与下一轮。
**执行**：固定基础 LLM。",
"seed": "从一个能调用 语言模型、改写并评分程序的种子改进方案的模型或程序开始；它不仅改下游程序，也用同一机制改自身改进代码。系统提供可执行效用函数，不依赖先证明代码更优。",
"fixed": "M2：improver code 可变，base LM/outer utility/runtime 固定",
"verdict": "执行候选程序后计算效用或评测得分；效用指预先规定的任务目标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "一组程序优化任务；不是大规模 agent held-out evaluation。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：主实验为 10-bit Learning Parity with Noise，自建 20 个实例计算改进器效用；初始解是随机采样程序。\n\n调试 / 选版本数据：同一效用定义的 5 份副本估计随机改进器表现，限定运行/调用预算。\n\n最终测试数据：另采 50 个未见 LPN 实例，报告五次独立 STOP 运行的 测试集 meta-utility；另有跨任务迁移。\n\n数据隔离与证据边界：20/50 是任务实例划分；改进器表现更好不保证它每代都更擅长自改进。",
"cycle": "改进器生成并执行候选程序，按下游效用选择；再把改进器自身当待优化代码，依据改进后的平均下游收益选择下一代。",
"train": "主实验为 10-bit Learning Parity with Noise，自建 20 个实例计算改进器效用；初始解是随机采样程序。",
"debug": "同一效用定义的 5 份副本估计随机改进器表现，限定运行/调用预算。",
"test": "另采 50 个未见 LPN 实例，报告五次独立 STOP 运行的 测试集 meta-utility；另有跨任务迁移。",
"isolation": "20/50 是任务实例划分；改进器表现更好不保证它每代都更擅长自改进。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "从一个能调用 LLM、改写并评分程序的种子 improver 开始；它不仅改下游程序，也用同一机制改自身改进代码。系统提供可执行效用函数，不依赖先证明代码更优。",
"protocol": "**主要任务：**learning parity with noise，按迭代评估 improver 在留出任务实例上的 meta-utility；还研究其他程序优化任务。本文明确没有把 SWE-bench 或 HumanEval 作为主实验。进化数据是可执行优化问题及反馈，不是自然语言题库；完整训练/留出实例数量本轮未核实。",
"sections": "§5.1、§5.3、Figure 4",
"source": "https://arxiv.org/abs/2310.02304",
"version": "2310.02304v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f9b302273af0c305706f1c9345322cd539fc110c5c18de97859796e722eea2b5",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "固定 GPT-4 供候选改进程序调用；程序运行和任务效用计算由 Python 环境完成。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-4 在改进方案的模型或程序的提示和调用规则下生成代码改写；改进后的改进方案的模型或程序可以参与下一轮，GPT-4 参数保持不变。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "代码改进器的程序，以及它优化的下游程序；基础模型、预算和外部效用定义固定。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "执行候选程序后计算效用或评测得分；效用指预先规定的任务目标。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从一个能调用 语言模型、改写并评分程序的种子改进方案的模型或程序开始；它不仅改下游程序，也用同一机制改自身改进代码。系统提供可执行效用函数，不依赖先证明代码更优。",
"sources": [
{
"label": "§5.1、§5.3、Figure 4",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "改进器生成并执行候选程序，按下游效用选择；再把改进器自身当待优化代码，依据改进后的平均下游收益选择下一代。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "主实验为 10-bit Learning Parity with Noise，自建 20 个实例计算改进器效用；初始解是随机采样程序。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同一效用定义的 5 份副本估计随机改进器表现，限定运行/调用预算。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "另采 50 个未见 LPN 实例，报告五次独立 STOP 运行的 测试集 meta-utility；另有跨任务迁移。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "20/50 是任务实例划分；改进器表现更好不保证它每代都更擅长自改进。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把“改进输入程序的程序”本身也作为待改进输入，实际调用它修改自己的实现，形成程序层面的自我应用。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2310.02304v3",
"version": "2310.02304v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "比一般 code optimization 多了一层 self-application：improver 是输入程序之一。",
"feedbackCases": [
{
"label": "改进器的程序效用",
"data": "10-bit 含噪奇偶学习 LPN：20 个自建实例用于改进器效用；五份副本估计随机表现。",
"scoring": "运行改进器所生成的解题程序，在固定任务目标与调用/时间预算下计算效用。",
"visible": "真实程序得分及候选代码供改进器继续修改自己。",
"use": "另采 50 个未见 LPN 实例计算测试元效用，重复五次独立运行；不是模型自评程序“更聪明”。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"judgment": "执行生成的程序，按各任务定义的效用函数与资源预算计分"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "主实验为 10-bit Learning Parity with Noise，自建 20 个实例计算改进器效用；初始解是随机采样程序。",
"selection": "同一效用定义的 5 份副本估计随机改进器表现，限定运行/调用预算。",
"evaluation": "另采 50 个未见 LPN 实例，报告五次独立 STOP 运行的 测试集 meta-utility；另有跨任务迁移。",
"isolation": "20/50 是任务实例划分；改进器表现更好不保证它每代都更擅长自改进。",
"roles": {
"executor": {
"value": "固定 GPT-4 供候选改进程序调用；程序运行和任务效用计算由 Python 环境完成。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
]
},
"modifier": {
"value": "GPT-4 在改进方案的模型或程序的提示和调用规则下生成代码改写；改进后的改进方案的模型或程序可以参与下一轮，GPT-4 参数保持不变。",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
]
},
"seed": {
"value": "从一个能调用 语言模型、改写并评分程序的种子改进方案的模型或程序开始；它不仅改下游程序，也用同一机制改自身改进代码。系统提供可执行效用函数，不依赖先证明代码更优。",
"sources": [
{
"label": "§5.1、§5.3、Figure 4",
"url": "https://arxiv.org/abs/2310.02304"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"selection": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"evaluation": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"isolation": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "把多次模型调用组织成程序，往往比直接问模型得到更好的解，但这种调用程序通常由人编写。作者指出，设计调用程序本身也是优化问题，因此进一步追问：负责改进其他程序的系统，能否把同样的能力用于改进自身，让后续优化也更有效。",
"sources": [
{
"label": "§1 Introduction（第 1–2 页）",
"url": "https://arxiv.org/pdf/2310.02304v3#page=1"
}
]
},
{
"key": "position",
"value": "研究冻结语言模型支持的代码优化程序，能否递归改进负责优化的程序本身，从而学出更有效的搜索策略。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.02304"
}
]
},
{
"key": "conclusion",
"value": "展示改进器代码的递归改进，并观察到新的搜索策略；语言模型参数始终不变。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.02304"
}
]
}
],
"fields": {
"object": "代码改进器的程序，以及它优化的下游程序；基础模型、预算和外部效用定义固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2406.07496",
"title": "TextGrad: Automatic “Differentiation” via Text",
"url": "https://arxiv.org/abs/2406.07496",
"date": "2024-06-11",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"LLMJudge",
"M1",
"Prompt",
"org:stanford"
],
"fields": {
"本质定位": "将 compound AI system 表示成 computation graph，LLM 产生 textual gradients，沿图反向传播以更新 prompt/code/text variables。",
"什么在变": "prompt、code、textual variables。",
"谁来改 / 谁执行": "**改**：固定 TextGrad backward/optimizer protocol + critic LLM。
**执行**：任意 compound AI system。",
"基础 harness": "可微式 textual computation graph。",
"Feedback": "task loss / evaluator 产生 textual feedback。",
"Evolution → Eval": "GPQA、coding 等优化任务。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "关键不是特定 prompt trick，而是提供“多组件 credit assignment + textual update”的通用 optimizer abstraction。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 277,
"fields": {
"优先级": "**R**",
"时间": "2024-06-11",
"论文": "[TextGrad: Automatic “Differentiation” via Text](https://arxiv.org/abs/2406.07496)",
"本质定位": "将 compound AI system 表示成 computation graph，LLM 产生 textual gradients，沿图反向传播以更新 prompt/code/text variables。",
"什么在变": "prompt、code、textual variables。",
"谁来改 / 谁执行": "**改**：固定 TextGrad backward/optimizer protocol + critic LLM。
**执行**：任意 compound AI system。",
"基础 harness": "可微式 textual computation graph。",
"Feedback": "task loss / evaluator 产生 textual feedback。",
"Evolution → Eval": "GPQA、coding 等优化任务。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "关键不是特定 prompt trick，而是提供“多组件 credit assignment + textual update”的通用 optimizer abstraction。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "将 compound AI system 表示成 computation graph，LLM 产生 textual gradients，沿图反向传播以更新 prompt/code/text variables。",
"novelty": "沿系统各变量的依赖关系传递文字批评，指出哪一处输入或提示该调整；“梯度”指反馈传递的类比。",
"object": "计算图中的提示、代码或其他文本变量；不通过数值梯度训练基础语言模型。",
"executor": "提示优化实验：gpt-3.5-turbo-0125执行推理；逐题解答改进实验以GPT-4o为基础。其他科学实验还包含非语言模型计算组件。",
"modifier": "提示优化时由GPT-4o提供反向文本反馈并修改提示，执行者仍为gpt-3.5-turbo-0125；TextGrad负责传递和应用这些反馈。",
"roleContext": "**改**：固定 TextGrad backward/optimizer protocol + critic LLM。
**执行**：任意 compound AI system。",
"seed": "把可修改文本、模型调用和评估串成计算图，评审 语言模型 沿图给出文字修改建议。这里的“梯度”是自然语言反馈，不是对模型权重求导；不同应用有不同起始程序。",
"fixed": "",
"verdict": "任务执行器或评分器给出损失和错误，模型把它转成针对不同组件的文字批评，再据此修改上游提示、代码或配置。",
"diagnosis": "LLM 结合当前变量、局部计算和下游反馈产生针对该变量的 textual gradient。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "BBH Object Counting / Word Sorting",
"evolve": "50 train",
"selection": "100 validation",
"test": "100 test",
"isolation": "随机三路划分",
"note": "Object Counting 用 exact match，Word Sorting 用 LLM 对照 gold。"
},
{
"name": "GSM8K",
"evolve": "沿用 DSPy train split",
"selection": "沿用 DSPy validation",
"test": "沿用 DSPy test",
"isolation": "分开划分",
"note": "本条补查覆盖 prompt optimization 实验；TextGrad 还优化其他文字可描述变量。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2406.07496v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：提示优化：BBH Word Sorting/Object Counting 各 50，GSM8K（小学数学应用题基准） 200。其他应用直接优化当前实例。\n\n调试 / 选版本数据：对应 验证集 为 BBH 各 100、GSM8K（小学数学应用题基准） 300；代码用本地测试，放疗用 matRad 计划与临床约束。\n\n最终测试数据：BBH 各 100、GSM8K（小学数学应用题基准） 1,319；另有 GPQA/MMLU 回答、LeetCode Hard 代码及科学设计实验。\n\n数据隔离与证据边界：训练提示的三段划分与单实例优化不同，不能全篇套用同一 训练／测试。",
"cycle": "语言模型 把最终损失转成针对上游文本变量的批评，沿计算图传回，再据批评更新提示/代码/数值配置；实际执行器提供可检验结果。",
"train": "提示优化：BBH Word Sorting/Object Counting 各 50，GSM8K（小学数学应用题基准） 200。其他应用直接优化当前实例。",
"debug": "对应 验证集 为 BBH 各 100、GSM8K（小学数学应用题基准） 300；代码用本地测试，放疗用 matRad 计划与临床约束。",
"test": "BBH 各 100、GSM8K（小学数学应用题基准） 1,319；另有 GPQA/MMLU 回答、LeetCode Hard 代码及科学设计实验。",
"isolation": "训练提示的三段划分与单实例优化不同，不能全篇套用同一 训练／测试。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "把可修改文本、模型调用和评估串成计算图，评审 LLM 沿图给出文字修改建议。这里的“梯度”是自然语言反馈，不是对模型权重求导；不同应用有不同起始程序。",
"protocol": "**提示优化：**BBH 的 Word Sorting/Object Counting 各随机分 50 train、100 validation、100 test；GSM8K 用 200/300/1,319。\n\n**其他应用：**GPQA 及 MMLU 的 Machine Learning、College Physics 等用于回答优化；另有代码与科学设计实例，不能套用上述统一划分。模型自改回答与在训练题上调提示是不同实验协议。",
"sections": "提示优化数据；GPQA/MMLU 任务说明",
"source": "https://arxiv.org/abs/2406.07496",
"version": "2406.07496v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "35d3ca47d1a287019e9fd5a2deb8b127c9ca20844dbf16d2f6592a79bcb8957d",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "提示优化实验：gpt-3.5-turbo-0125执行推理；逐题解答改进实验以GPT-4o为基础。其他科学实验还包含非语言模型计算组件。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2406.07496#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "提示优化时由GPT-4o提供反向文本反馈并修改提示，执行者仍为gpt-3.5-turbo-0125；TextGrad负责传递和应用这些反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2406.07496#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "计算图中的提示、代码或其他文本变量；不通过数值梯度训练基础语言模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务执行器或评分器给出损失和错误，模型把它转成针对不同组件的文字批评，再据此修改上游提示、代码或配置。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "把可修改文本、模型调用和评估串成计算图，评审 语言模型 沿图给出文字修改建议。这里的“梯度”是自然语言反馈，不是对模型权重求导；不同应用有不同起始程序。",
"sources": [
{
"label": "提示优化数据；GPQA/MMLU 任务说明",
"url": "https://arxiv.org/abs/2406.07496"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "语言模型 把最终损失转成针对上游文本变量的批评，沿计算图传回，再据批评更新提示/代码/数值配置；实际执行器提供可检验结果。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "提示优化：BBH Word Sorting/Object Counting 各 50，GSM8K（小学数学应用题基准） 200。其他应用直接优化当前实例。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "对应 验证集 为 BBH 各 100、GSM8K（小学数学应用题基准） 300；代码用本地测试，放疗用 matRad 计划与临床约束。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "BBH 各 100、GSM8K（小学数学应用题基准） 1,319；另有 GPQA/MMLU 回答、LeetCode Hard 代码及科学设计实验。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "训练提示的三段划分与单实例优化不同，不能全篇套用同一 训练／测试。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "沿系统各变量的依赖关系传递文字批评，指出哪一处输入或提示该调整；“梯度”指反馈传递的类比。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2406.07496v1",
"version": "2406.07496v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把系统当作计算图，把自然语言批评沿图传回具体变量；所谓 gradient 是文字反馈的类比，并非真的对黑盒 LLM API 求数值导数。",
"feedbackCases": [
{
"label": "推理提示优化",
"data": "BBH Word Sorting/Object Counting 各 50/100/100 训练/验证/测试；GSM8K（小学数学应用题基准） 200/300/1,319。",
"scoring": "GSM8K（小学数学应用题基准）、Object Counting 将最终数值与标准答案作字符串精确匹配；Word Sorting 用语言模型比较输出与标准排序。",
"visible": "评估结果由反馈模型转成对提示词的文字修改意见。",
"use": "训练反馈产生修改，验证选择，测试独立报告；不能把三任务都写成同一种检查器。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"judgment": "GSM8K／Object Counting 用字符串精确匹配；Word Sorting 用模型对照标准排序"
},
{
"label": "当前实例的代码 / 科学优化",
"data": "LeetCode Hard、放疗计划、分子设计及问答实例。",
"scoring": "代码依据本地测试和执行错误；放疗由 matRad 的剂量/临床约束评价；分子设计结合相应性质、结合与结构指标。",
"visible": "工具/科学计算结果经文字反馈转成对变量的修改建议。",
"use": "这些是不同的单实例目标，不是复用 GSM8K（小学数学应用题基准） 的答案奖励训练所有应用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"judgment": "代码测试／放疗数值约束／分子性质指标，按实验分别评价"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "提示优化：BBH Word Sorting/Object Counting 各 50，GSM8K（小学数学应用题基准） 200。其他应用直接优化当前实例。",
"selection": "对应 验证集 为 BBH 各 100、GSM8K（小学数学应用题基准） 300；代码用本地测试，放疗用 matRad 计划与临床约束。",
"evaluation": "BBH 各 100、GSM8K（小学数学应用题基准） 1,319；另有 GPQA/MMLU 回答、LeetCode Hard 代码及科学设计实验。",
"isolation": "训练提示的三段划分与单实例优化不同，不能全篇套用同一 训练／测试。",
"roles": {
"executor": {
"value": "提示优化实验：gpt-3.5-turbo-0125执行推理；逐题解答改进实验以GPT-4o为基础。其他科学实验还包含非语言模型计算组件。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2406.07496#S3.SS2"
}
]
},
"modifier": {
"value": "提示优化时由GPT-4o提供反向文本反馈并修改提示，执行者仍为gpt-3.5-turbo-0125；TextGrad负责传递和应用这些反馈。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2406.07496#S3.SS2"
}
]
},
"seed": {
"value": "把可修改文本、模型调用和评估串成计算图，评审 语言模型 沿图给出文字修改建议。这里的“梯度”是自然语言反馈，不是对模型权重求导；不同应用有不同起始程序。",
"sources": [
{
"label": "提示优化数据；GPQA/MMLU 任务说明",
"url": "https://arxiv.org/abs/2406.07496"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"selection": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"evaluation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
],
"isolation": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2406.07496#A5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型和工具组成的复杂系统仍大量依赖领域专家手工搭建、凭经验调整。它们包含无法直接求梯度的黑箱组件，难像神经网络一样把整体评价传回各处指导优化；这限制了自动改进系统的能力，作者因而研究用文字反馈承担类似作用。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2406.07496#S1"
}
]
},
{
"key": "position",
"value": "面向由多个模型和工具组成的系统，研究自然语言反馈能否充当通用优化信号，支持不同组件的联合改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2406.07496"
}
]
},
{
"key": "conclusion",
"value": "同一框架可用于问答、代码和科学优化，提供类似自动微分的组件优化接口。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2406.07496"
}
]
}
],
"fields": {
"object": "计算图中的提示、代码或其他文本变量；不通过数值梯度训练基础语言模型。",
"verdict": "任务执行器或评分器给出损失和错误，模型把它转成针对不同组件的文字批评，再据此修改上游提示、代码或配置。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2406.07496"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback"
]
},
{
"id": "2410.10762",
"title": "AFlow: Automating Agentic Workflow Generation",
"url": "https://arxiv.org/abs/2410.10762",
"date": "2024-10-14",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Workflow",
"Prompt",
"SeparateEvolver",
"BenchmarkScore",
"OfflineSearch",
"HeldOut",
"M1"
],
"fields": {
"本质定位": "把 agent workflow 写成由 LLM-invoking nodes 与 edges 组成的 code graph，用 MCTS + execution feedback 自动修改。",
"什么在变": "workflow code/topology、node prompts/operators。",
"谁来改 / 谁执行": "**改**：固定 AFlow MCTS + LLM workflow modifier。
**执行**：candidate workflow 调用的 LLM。",
"基础 harness": "code-represented workflow。",
"Feedback": "validation execution score。",
"Evolution → Eval": "6 benchmarks；validation 上优化、test 上评估。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 ADAS 更聚焦 workflow graph search，并用 tree-structured experience/MCTS 系统探索，而不是开放式 meta-agent 自由设计。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 279,
"fields": {
"优先级": "**K**",
"时间": "2024-10-14",
"论文": "[AFlow: Automating Agentic Workflow Generation](https://arxiv.org/abs/2410.10762)",
"本质定位": "把 agent workflow 写成由 LLM-invoking nodes 与 edges 组成的 code graph，用 MCTS + execution feedback 自动修改。",
"什么在变": "workflow code/topology、node prompts/operators。",
"谁来改 / 谁执行": "**改**：固定 AFlow MCTS + LLM workflow modifier。
**执行**：candidate workflow 调用的 LLM。",
"基础 harness": "code-represented workflow。",
"Feedback": "validation execution score。",
"Evolution → Eval": "6 benchmarks；validation 上优化、test 上评估。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 ADAS 更聚焦 workflow graph search，并用 tree-structured experience/MCTS 系统探索，而不是开放式 meta-agent 自由设计。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 workflow 表示成程序：节点执行 LLM 调用，连接和控制逻辑由代码描述。AFlow 用 MCTS 选择待扩展的工作流，再由 LLM 修改节点 prompts、operators 和连线。",
"novelty": "搜索可执行工作流程中的节点、连接和操作，能够改变解题步骤如何组织，而非仅替换一句提示。",
"object": "工作流代码、节点提示、操作模块与连接关系。",
"executor": "主执行配置包括GPT-4o-mini-0718、DeepSeek-V2.5、GPT-4o-0513、Claude3.5-Sonnet-0620。",
"modifier": "Claude3.5 Sonnet作为优化语言模型修改工作流；MCTS固定负责选择要扩展的节点，执行模型与优化模型分开。",
"roleContext": "**改**：固定 AFlow MCTS + LLM workflow modifier。
**执行**：candidate workflow 调用的 LLM。",
"seed": "从预留节点与操作接口的模板工作流开始。节点表示执行步骤，操作是可复用的调用或处理模块；优化器补全函数并修改节点、提示和连接，起点已有这些模块和接口。",
"fixed": "目标模型权重、operator 接口与搜索规则不随 workflow 一起更新。",
"verdict": "QA 的 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、数学 solve rate、代码 pass@1（单次尝试完成任务的比例）。",
"diagnosis": "利用选中 workflow 的既往 edits、成功/失败记录、预测及期望输出日志。",
"update": "修改 workflow code、节点 prompts 或连接。",
"acceptance": "执行候选获取验证分数，再回传到搜索树引导后续扩展。",
"experiments": [
{
"name": "六个 benchmark",
"evolve": "20% validation 中进一步选取高方差任务进行搜索",
"selection": "validation 评分选择 workflow",
"test": "80% test；并做跨执行模型迁移",
"isolation": "独立测试",
"note": "GSM8K、HumanEval、MBPP、HotpotQA、DROP、MATH。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2410.10762v4"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不训练参数。GSM8K（小学数学应用题基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP，以及 HotpotQA/DROP 各抽 1,000、MATH 四类难度5题共617，供工作流实验。\n\n调试 / 选版本数据：每套随机取 20% 验证集；初始模板跑五次后选高波动题驱动搜索。\n\n最终测试数据：各套剩余 80%。代码 Test operator 仅用公开测试，MBPP 用每题第一条测试作公开输入。\n\n数据隔离与证据边界：正文定义 20% validation/80% 测试集；算法注释误写后者 训练，按正文解释并保留冲突。",
"cycle": "选择一条已有工作流程，读取其修改历史、成败记录以及预测和期望输出。修改流程代码、节点提示或连接，执行候选获得验证分数，再把结果用于决定下一条搜索分支。",
"train": "不训练参数。GSM8K（小学数学应用题基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP，以及 HotpotQA/DROP 各抽 1,000、MATH 四类难度5题共617，供工作流实验。",
"debug": "每套随机取 20% 验证集；初始模板跑五次后选高波动题驱动搜索。",
"test": "各套剩余 80%。代码 Test operator 仅用公开测试，MBPP 用每题第一条测试作公开输入。",
"isolation": "正文定义 20% validation/80% 测试集；算法注释误写后者 训练，按正文解释并保留冲突。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "从带节点/operator 接口的模板工作流 W₀ 开始，优化器补全调用函数并修改节点、提示和连接。可复用 operator 已存在，不是从完全无工具的空白环境开始。",
"protocol": "**搜索→测试：**数据随机按 20% validation、80% test 划分，seed=42；初始模板在 validation 上运行 5 次，再选评分波动较大的题供搜索。任务包含数学、代码和问答，具体各 benchmark 数量本轮待核实。\n\n**原文歧义：**正文称 80% 为 test，算法注释却写 training；这里按正文描述，并保留这一命名冲突，不自行改成额外训练集。",
"sections": "搜索设置；附录 A.6 算法",
"source": "https://arxiv.org/abs/2410.10762",
"version": "2410.10762v4",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "9eb6241908b9faa9ca414a9ec2e63bcf5ca4f4fa91812dd8640b0edc27579292",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主执行配置包括GPT-4o-mini-0718、DeepSeek-V2.5、GPT-4o-0513、Claude3.5-Sonnet-0620。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude3.5 Sonnet作为优化语言模型修改工作流；MCTS固定负责选择要扩展的节点，执行模型与优化模型分开。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "工作流代码、节点提示、操作模块与连接关系。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "QA 的 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）、数学 solve rate、代码 pass@1（单次尝试完成任务的比例）。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从预留节点与操作接口的模板工作流开始。节点表示执行步骤，操作是可复用的调用或处理模块；优化器补全函数并修改节点、提示和连接，起点已有这些模块和接口。",
"sources": [
{
"label": "搜索设置；附录 A.6 算法",
"url": "https://arxiv.org/abs/2410.10762"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "选择一条已有工作流程，读取其修改历史、成败记录以及预测和期望输出。修改流程代码、节点提示或连接，执行候选获得验证分数，再把结果用于决定下一条搜索分支。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训练参数。GSM8K（小学数学应用题基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP，以及 HotpotQA/DROP 各抽 1,000、MATH 四类难度5题共617，供工作流实验。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "每套随机取 20% 验证集；初始模板跑五次后选高波动题驱动搜索。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各套剩余 80%。代码 Test operator 仅用公开测试，MBPP 用每题第一条测试作公开输入。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "正文定义 20% validation/80% 测试集；算法注释误写后者 训练，按正文解释并保留冲突。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "搜索可执行工作流程中的节点、连接和操作，能够改变解题步骤如何组织，而非仅替换一句提示。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2410.10762v4",
"version": "2410.10762v4",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "可搜索的是可执行 workflow 结构，不只是单个 prompt；旧分类把 Program/Workflow 整段压成 Prompt 会丢掉这一点。",
"feedbackCases": [
{
"label": "HotpotQA / DROP",
"data": "每套取 20% 验证用于搜索，剩余 80% 测试；验证初始跑五次后取高波动题。",
"scoring": "以参考答案 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务） 评分。",
"visible": "工作流候选的验证成绩及执行错误用于蒙特卡洛树搜索。",
"use": "选择工作流后评测试集；内部公开测试与研究端正式验收测试分开。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
],
"judgment": "规则计算回答与参考答案的词项 F1"
},
{
"label": "GSM8K / MATH",
"data": "每套取 20% 验证用于搜索，剩余 80% 测试；验证初始跑五次后取高波动题。",
"scoring": "根据数学答案正确性计解题率。",
"visible": "工作流候选的验证成绩及执行错误用于蒙特卡洛树搜索。",
"use": "选择工作流后评测试集；内部公开测试与研究端正式验收测试分开。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
],
"judgment": "数学参考答案判分；本文未在该段展开所有答案解析器"
},
{
"label": "HumanEval / MBPP",
"data": "每套取 20% 验证用于搜索，剩余 80% 测试；验证初始跑五次后取高波动题。",
"scoring": "以正式代码测试计 pass@1（单次尝试完成任务的比例）；工作流内部 Test 操作仅用公开测试，MBPP 只把第一条测试作为公开输入。",
"visible": "工作流候选的验证成绩及执行错误用于蒙特卡洛树搜索。",
"use": "选择工作流后评测试集；内部公开测试与研究端正式验收测试分开。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
],
"judgment": "HumanEval／MBPP 正式代码测试；内部调试另用公开测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "不训练参数。GSM8K（小学数学应用题基准）、HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）、MBPP，以及 HotpotQA/DROP 各抽 1,000、MATH 四类难度5题共617，供工作流实验。",
"selection": "每套随机取 20% 验证集；初始模板跑五次后选高波动题驱动搜索。",
"evaluation": "各套剩余 80%。代码 Test operator 仅用公开测试，MBPP 用每题第一条测试作公开输入。",
"isolation": "正文定义 20% validation/80% 测试集；算法注释误写后者 训练，按正文解释并保留冲突。",
"roles": {
"executor": {
"value": "主执行配置包括GPT-4o-mini-0718、DeepSeek-V2.5、GPT-4o-0513、Claude3.5-Sonnet-0620。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
]
},
"modifier": {
"value": "Claude3.5 Sonnet作为优化语言模型修改工作流；MCTS固定负责选择要扩展的节点，执行模型与优化模型分开。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
}
]
},
"seed": {
"value": "从预留节点与操作接口的模板工作流开始。节点表示执行步骤，操作是可复用的调用或处理模块；优化器补全函数并修改节点、提示和连接，起点已有这些模块和接口。",
"sources": [
{
"label": "搜索设置；附录 A.6 算法",
"url": "https://arxiv.org/abs/2410.10762"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2410.10762#S5.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "手工设计 agent 工作流耗时，限制向新任务和领域迁移。已有自动方法有的仍需人先搭流程，有的只能表达有限结构，代码搜索方法又可能在有限迭代内找不到好方案；作者因此希望同时解决工作流表示不够灵活和搜索效率不足的问题。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2410.10762#S1"
}
]
},
{
"key": "position",
"value": "研究 task agent 的工作流能否自动设计，减少对人工初始流程的依赖，并改善执行效果与成本之间的权衡。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2410.10762"
}
]
},
{
"key": "conclusion",
"value": "在六个基准上优于对照，部分任务中小模型配合优化工作流可兼顾表现与成本。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2410.10762"
}
]
}
],
"fields": {
"object": "工作流代码、节点提示、操作模块与连接关系。",
"seed": "从预留节点与操作接口的模板工作流开始。节点表示执行步骤，操作是可复用的调用或处理模块；优化器补全函数并修改节点、提示和连接，起点已有这些模块和接口。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2507.03616",
"title": "EvoAgentX: An Automated Framework for Evolving Agentic Workflows",
"url": "https://arxiv.org/abs/2507.03616",
"date": "2025-07-04",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Workflow"
],
"fields": {
"本质定位": "提供统一的 agent/workflow generation→execution→evaluation→evolution framework，并把 TextGrad、AFlow、MIPRO 等 optimizer 接到同一 workflow representation 上。",
"什么在变": "agent prompts、tool configurations、workflow topology；取决于选择的 optimizer。",
"谁来改 / 谁执行": "**改**：固定 TextGrad/AFlow/MIPRO 等优化器；EvoAgentX 本身主要负责统一 orchestration。
**执行**：candidate multi-agent workflow。",
"基础 harness": "五层模块化 MAS framework（components/agent/workflow/evolving/evaluation）。",
"Feedback": "validation metric、执行结果、LLM/textual feedback；具体由所选 optimizer 决定。",
"Evolution → Eval": "HotPotQA/MBPP/MATH 使用 validation 做优化、test 做最终评估；另测 GAIA real-world tasks。",
"Meta-depth": "M1 substrate；modifier algorithm 本身不进化。",
"相对之前真正新增什么": "相对 AFlow 的主要新增不是新的 evolution 原理，而是 **framework/integration**：prompt、tool config、workflow topology 都成为统一可优化 surface，可替换 optimizer。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 280,
"fields": {
"优先级": "**R**",
"时间": "2025-07-04",
"论文": "[EvoAgentX: An Automated Framework for Evolving Agentic Workflows](https://arxiv.org/abs/2507.03616)",
"本质定位": "提供统一的 agent/workflow generation→execution→evaluation→evolution framework，并把 TextGrad、AFlow、MIPRO 等 optimizer 接到同一 workflow representation 上。",
"什么在变": "agent prompts、tool configurations、workflow topology；取决于选择的 optimizer。",
"谁来改 / 谁执行": "**改**：固定 TextGrad/AFlow/MIPRO 等优化器；EvoAgentX 本身主要负责统一 orchestration。
**执行**：candidate multi-agent workflow。",
"基础 harness": "五层模块化 MAS framework（components/agent/workflow/evolving/evaluation）。",
"Feedback": "validation metric、执行结果、LLM/textual feedback；具体由所选 optimizer 决定。",
"Evolution → Eval": "HotPotQA/MBPP/MATH 使用 validation 做优化、test 做最终评估；另测 GAIA real-world tasks。",
"Meta-depth": "M1 substrate；modifier algorithm 本身不进化。",
"相对之前真正新增什么": "相对 AFlow 的主要新增不是新的 evolution 原理，而是 **framework/integration**：prompt、tool config、workflow topology 都成为统一可优化 surface，可替换 optimizer。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "提供统一的 agent/workflow generation→execution→evaluation→evolution framework，并把 TextGrad、AFlow、MIPRO 等 optimizer 接到同一 workflow representation 上。",
"novelty": "把提示、工具配置和工作流程结构放进统一优化框架，并允许替换优化器，主要贡献是可组合的系统集成。",
"object": "agent 提示、工具配置和工作流结构，具体取决于接入的优化算法。",
"executor": "EvoAgentX为可配置框架，没有唯一固定执行模型。附录A.3的HumanEval/AFlow示例明确使用GPT-4o-mini。",
"modifier": "框架统一调度TextGrad/AFlow/MIPRO；附录A.3示例由Claude3.5-Sonnet-20240620优化、GPT-4o-mini执行。这是示例配置，不能推广为所有实验。",
"roleContext": "**改**：固定 TextGrad/AFlow/MIPRO 等优化器；EvoAgentX 本身主要负责统一 orchestration。
**执行**：candidate multi-agent workflow。",
"seed": "五层模块化框架：模型/工具组件、task agent、工作流、进化优化器和评估。支持接入已有工作流；对 Open Deep Research、OWL 的优化从这些已有 task agent 出发，不能都描述为空白模板。",
"fixed": "",
"verdict": "反馈由任务和优化器组合决定：问答比较答案，代码运行测试，数学检查解答；TextGrad 等方法再把执行信息用于文字批评或候选比较。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "HotPotQA/MBPP/MATH 使用 validation 做优化、test 做最终评估；另测 GAIA real-world tasks。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：算法实验使用代码/问答等基准支持的搜索数据；应用实验优化 Open Deep Research 与 OWL。\n\n调试 / 选版本数据：任务输出与指标送给所选优化器。Table 1 是框架支持的数据规模，不表示每次全量使用。\n\n最终测试数据：算法部分涉及 HotpotQA（需要结合多份资料作答的多跳问答基准）、MBPP、MATH 等；应用部分报告 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的优化前后表现。\n\n数据隔离与证据边界：论文框架支持 训练／开发／测试 不等于每个应用都给出独立测试证据；GAIA（需要检索、推理和使用工具的通用助理任务基准） 应按优化前后应用对照阅读。",
"cycle": "平台统一工作流表示、执行和评估，再调用 TextGrad、AFlow 或 MIPRO 等优化器修改提示或流程结构。每种优化器使用相应任务指标和执行反馈比较候选，具体组合见反馈表。",
"train": "算法实验使用代码/问答等基准支持的搜索数据；应用实验优化 Open Deep Research 与 OWL。",
"debug": "任务输出与指标送给所选优化器。Table 1 是框架支持的数据规模，不表示每次全量使用。",
"test": "算法部分涉及 HotpotQA（需要结合多份资料作答的多跳问答基准）、MBPP、MATH 等；应用部分报告 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的优化前后表现。",
"isolation": "论文框架支持 训练／开发／测试 不等于每个应用都给出独立测试证据；GAIA（需要检索、推理和使用工具的通用助理任务基准） 应按优化前后应用对照阅读。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "五层模块化框架：模型/工具组件、agent、工作流、进化优化器和评估。支持接入已有工作流；对 Open Deep Research、OWL 的优化从这些已有 agent 出发，不能都描述为空白模板。",
"protocol": "**支持的数据套件：**NQ、HotPotQA、GSM8K、MATH、HumanEval、MBPP、LiveCodeBench 等；另在 GAIA 展示已有 agent 优化。论文 Table 1 列的是所支持 benchmark 的原始 train/dev/test 规模，不等于每次优化都使用全部数据。\n\n**待核实：**逐个主实验实际用哪些题搜索、选版本及报告最终分数尚未完整确认，不将支持清单当成统一训练协议。",
"sections": "框架层次、Table 1、GAIA 实验",
"source": "https://arxiv.org/abs/2507.03616",
"version": "2507.03616v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "2b13fea0d5a7de804eb44c55e81c1119b1f4e6850ca1f57cee1ca6ce7aee74c9",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "EvoAgentX为可配置框架，没有唯一固定执行模型。附录A.3的HumanEval/AFlow示例明确使用GPT-4o-mini。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "框架统一调度TextGrad/AFlow/MIPRO；附录A.3示例由Claude3.5-Sonnet-20240620优化、GPT-4o-mini执行。这是示例配置，不能推广为所有实验。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "agent 提示、工具配置和工作流结构，具体取决于接入的优化算法。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "反馈由任务和优化器组合决定：问答比较答案，代码运行测试，数学检查解答；TextGrad 等方法再把执行信息用于文字批评或候选比较。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "五层模块化框架：模型/工具组件、task agent、工作流、进化优化器和评估。支持接入已有工作流；对 Open Deep Research、OWL 的优化从这些已有 task agent 出发，不能都描述为空白模板。",
"sources": [
{
"label": "框架层次、Table 1、GAIA 实验",
"url": "https://arxiv.org/abs/2507.03616"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "平台统一工作流表示、执行和评估，再调用 TextGrad、AFlow 或 MIPRO 等优化器修改提示或流程结构。每种优化器使用相应任务指标和执行反馈比较候选，具体组合见反馈表。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "算法实验使用代码/问答等基准支持的搜索数据；应用实验优化 Open Deep Research 与 OWL。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "任务输出与指标送给所选优化器。Table 1 是框架支持的数据规模，不表示每次全量使用。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "算法部分涉及 HotpotQA（需要结合多份资料作答的多跳问答基准）、MBPP、MATH 等；应用部分报告 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的优化前后表现。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "论文框架支持 训练／开发／测试 不等于每个应用都给出独立测试证据；GAIA（需要检索、推理和使用工具的通用助理任务基准） 应按优化前后应用对照阅读。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把提示、工具配置和工作流程结构放进统一优化框架，并允许替换优化器，主要贡献是可组合的系统集成。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2507.03616v2",
"version": "2507.03616v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 AFlow 的主要新增不是新的 evolution 原理，而是 **framework/integration**：prompt、tool config、workflow topology 都成为统一可优化 surface，可替换 optimizer。",
"feedbackCases": [
{
"label": "工作流优化：HotpotQA",
"data": "算法实验的 HotpotQA（需要结合多份资料作答的多跳问答基准） 搜索与评估数据。论文表 1 列的是框架支持的数据规模，不能据此认定每个优化器都全量使用。",
"scoring": "将回答与标准答案比较，计算词语重合的 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）。",
"visible": "执行输出和相应任务指标交给所选优化器；TextGrad 生成文字批评，MIPRO 用指标比较提示候选，AFlow 搜索工作流结构。",
"use": "同一个框架接入不同优化算法，反馈接口由算法和任务组合决定；应用部分另比较 Open Deep Research、OWL 在 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的表现。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"judgment": "规则计算 HotpotQA 回答与标注答案的词项 F1"
},
{
"label": "工作流优化：MBPP",
"data": "算法实验的 MBPP 搜索与评估数据。论文表 1 列的是框架支持的数据规模，不能据此认定每个优化器都全量使用。",
"scoring": "运行代码题的测试，计算首次生成通过测试的比例 pass@1（单次尝试完成任务的比例）。",
"visible": "执行输出和相应任务指标交给所选优化器；TextGrad 生成文字批评，MIPRO 用指标比较提示候选，AFlow 搜索工作流结构。",
"use": "同一个框架接入不同优化算法，反馈接口由算法和任务组合决定；应用部分另比较 Open Deep Research、OWL 在 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的表现。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"judgment": "MBPP 可执行代码测试"
},
{
"label": "工作流优化：MATH",
"data": "算法实验的 MATH 搜索与评估数据。论文表 1 列的是框架支持的数据规模，不能据此认定每个优化器都全量使用。",
"scoring": "按数学题标准答案统计解题正确率。",
"visible": "执行输出和相应任务指标交给所选优化器；TextGrad 生成文字批评，MIPRO 用指标比较提示候选，AFlow 搜索工作流结构。",
"use": "同一个框架接入不同优化算法，反馈接口由算法和任务组合决定；应用部分另比较 Open Deep Research、OWL 在 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的表现。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"judgment": "对照 MATH 标准答案；本文未在该段展开比对实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "算法实验使用代码/问答等基准支持的搜索数据；应用实验优化 Open Deep Research 与 OWL。",
"selection": "任务输出与指标送给所选优化器。Table 1 是框架支持的数据规模，不表示每次全量使用。",
"evaluation": "算法部分涉及 HotpotQA（需要结合多份资料作答的多跳问答基准）、MBPP、MATH 等；应用部分报告 GAIA（需要检索、推理和使用工具的通用助理任务基准） 上的优化前后表现。",
"isolation": "论文框架支持 训练／开发／测试 不等于每个应用都给出独立测试证据；GAIA（需要检索、推理和使用工具的通用助理任务基准） 应按优化前后应用对照阅读。",
"roles": {
"executor": {
"value": "EvoAgentX为可配置框架，没有唯一固定执行模型。附录A.3的HumanEval/AFlow示例明确使用GPT-4o-mini。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
]
},
"modifier": {
"value": "框架统一调度TextGrad/AFlow/MIPRO；附录A.3示例由Claude3.5-Sonnet-20240620优化、GPT-4o-mini执行。这是示例配置，不能推广为所有实验。",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "附录A.3",
"url": "https://arxiv.org/html/2507.03616#A1.SS3"
}
]
},
"seed": {
"value": "五层模块化框架：模型/工具组件、task agent、工作流、进化优化器和评估。支持接入已有工作流；对 Open Deep Research、OWL 的优化从这些已有 task agent 出发，不能都描述为空白模板。",
"sources": [
{
"label": "框架层次、Table 1、GAIA 实验",
"url": "https://arxiv.org/abs/2507.03616"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"selection": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"evaluation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
],
"isolation": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2507.03616#S3.SS5"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.03616#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2507.03616#S4.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "多 agent 系统仍需人工定义角色、分工、交互和执行流程，任务变化后还要重新调整，限制易用性与扩展性。已有自动优化方法又散落在不同实现中，缺少统一的平台支持调用和比较，因此开发者难以方便地构建并持续优化整个工作流。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2507.03616#S1"
}
]
},
{
"key": "position",
"value": "面向自动 agent 工作流研发，提供统一工程平台，解决不同生成、执行和优化算法难以组合复用的问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.03616"
}
]
},
{
"key": "conclusion",
"value": "在问答、代码、数学和综合任务中展示收益；主要贡献是可扩展的统一工程平台。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2507.03616"
}
]
}
],
"fields": {
"executor": "EvoAgentX 可以配置不同执行模型；论文的 HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）／AFlow 示例用 GPT-4o-mini。",
"modifier": "框架调度 TextGrad、AFlow 或 MIPRO 优化流程。HumanEval（根据函数要求生成代码、用测试检查结果的编程基准）／AFlow 示例用 Claude3.5-Sonnet-20240620 修改、GPT-4o-mini 执行；该配置不代表所有应用。",
"object": "agent 提示、工具配置和工作流结构，具体取决于接入的优化算法。",
"verdict": "反馈由任务和优化器组合决定：问答比较答案，代码运行测试，数学检查解答；TextGrad 等方法再把执行信息用于文字批评或候选比较。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2509.19349",
"title": "ShinkaEvolve: Towards Open-Ended and Sample-Efficient Program Evolution",
"url": "https://arxiv.org/abs/2509.19349",
"date": "2025-09",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:sakana"
],
"fields": {
"本质定位": "LLM ensemble + evolutionary archive 做 sample-efficient program evolution；用 parent sampling、novelty rejection 与 bandit-based model selection 提高搜索效率。",
"什么在变": "candidate programs / scientific code / 某些 harness code；evolution engine本身主要固定。",
"谁来改 / 谁执行": "**改**：固定 evolutionary controller + LLM ensemble mutation operators。
**执行**：候选程序或候选 agent harness。",
"基础 harness": "population/archive + executable evaluator。",
"Feedback": "可执行 fitness / benchmark objective；novelty信号参与采样。",
"Evolution → Eval": "circle packing、ALE/engineering、AIME harness、MoE loss 等多类 program-search task。",
"Meta-depth": "M1 search substrate。",
"相对之前真正新增什么": "它不是 agent 自己持续改自己的 deployment harness；真正新点是 **更高效的开放式 program evolution/search substrate**，但论文也展示可把 agentic harness 当 program candidate 优化。"
},
"details": [],
"occurrences": [
{
"section": "A1. Prompt / Program / Workflow Evolution",
"line": 281,
"fields": {
"优先级": "**R**",
"时间": "2025-09",
"论文": "[ShinkaEvolve: Towards Open-Ended and Sample-Efficient Program Evolution](https://arxiv.org/abs/2509.19349)",
"本质定位": "LLM ensemble + evolutionary archive 做 sample-efficient program evolution；用 parent sampling、novelty rejection 与 bandit-based model selection 提高搜索效率。",
"什么在变": "candidate programs / scientific code / 某些 harness code；evolution engine本身主要固定。",
"谁来改 / 谁执行": "**改**：固定 evolutionary controller + LLM ensemble mutation operators。
**执行**：候选程序或候选 agent harness。",
"基础 harness": "population/archive + executable evaluator。",
"Feedback": "可执行 fitness / benchmark objective；novelty信号参与采样。",
"Evolution → Eval": "circle packing、ALE/engineering、AIME harness、MoE loss 等多类 program-search task。",
"Meta-depth": "M1 search substrate。",
"相对之前真正新增什么": "它不是 agent 自己持续改自己的 deployment harness；真正新点是 **更高效的开放式 program evolution/search substrate**，但论文也展示可把 agentic harness 当 program candidate 优化。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "LLM ensemble + evolutionary archive 做 sample-efficient program evolution；用 parent sampling、novelty rejection 与 bandit-based model selection 提高搜索效率。",
"novelty": "提供维护候选程序、选择修改起点并继续搜索的高效进化系统；框架代码也可作为被优化程序，但不是所有实验都在改 agent 自身。",
"object": "候选程序、科研代码或特定任务的运行框架；进化引擎和外部评分规则固定。",
"executor": "程序优化任务运行候选程序；AIME 运行框架实验由 GPT-4.1-nano 在候选结构下答题，迁移测试另用 GPT-4.1-mini、GPT-4.1 和 o4-mini。",
"modifier": "固定进化控制器选择父代与模型，再让语言模型改写程序。模型池随实验设置变化：MoE 负载均衡实验明确使用 GPT-4.1、Gemini-2.5-Pro、Claude-Sonnet-4；不能将它们直接当作 AIME 的答题模型。",
"roleContext": "**改**：固定 evolutionary controller + LLM ensemble mutation operators。
**执行**：候选程序或候选 agent harness。",
"seed": "程序种群/档案加可执行评估器，语言模型 提议代码变化，系统管理采样和选择；每个应用给初始程序与目标函数。MoE 实验改的是训练用损失函数代码，训练模型只是评价候选损失的一部分。",
"fixed": "",
"verdict": "运行候选程序后按任务目标评分；采样时也考虑新颖性，以避免只探索相似方案。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "circle packing、ALE/engineering、AIME harness、MoE loss 等多类 program-search task。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "主要搜索 scientific / solution programs。被优化程序是产物；没有证据说所有实验都重写执行它的 harness 或 LLM 权重。",
"protocolDetail": "训练 / 进化数据：MoE 损失搜索：556M 模型在 FineWeb 约 2B 词元 上训练评估候选；另有几何及 ALE-Bench 程序优化。\n\n调试 / 选版本数据：候选程序的执行有效性与目标值；ALE LITE 10 题以公开测试分数搜索。\n\n最终测试数据：ALE 最佳公开候选提交私有测试；MoE 将损失迁移到 2.7B 模型、约 30B FineWeb 词元 再评困惑度及下游。\n\n数据隔离与证据边界：ALE 另报告私有测试取 top-5 最大值的诊断结果，应与单次提交分开；MoE 是跨规模迁移。",
"cycle": "岛屿档案保存优质程序，采样父代和灵感程序后让 语言模型 变异；执行真实目标评分，按相对增益调整模型采样概率。",
"train": "MoE 损失搜索：556M 模型在 FineWeb 约 2B 词元 上训练评估候选；另有几何及 ALE-Bench 程序优化。",
"debug": "候选程序的执行有效性与目标值；ALE LITE 10 题以公开测试分数搜索。",
"test": "ALE 最佳公开候选提交私有测试；MoE 将损失迁移到 2.7B 模型、约 30B FineWeb 词元 再评困惑度及下游。",
"isolation": "ALE 另报告私有测试取 top-5 最大值的诊断结果，应与单次提交分开；MoE 是跨规模迁移。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "程序种群/档案加可执行评估器，LLM 提议代码变化，系统管理采样和选择；每个应用给初始程序与目标函数。MoE 实验改的是训练用损失函数代码，训练模型只是评价候选损失的一部分。",
"protocol": "**MoE 损失搜索→迁移：**小 MoE 上演化，再把选出的损失移到 2.7B MoE，用约 30B FineWeb tokens 训练，比较不同负载均衡系数的困惑度及七项下游评测。\n\n**边界：**这是从小模型训练配置迁移到大模型，不能描述成只在一份问答训练集进化。其他数学/程序优化任务有各自目标函数；七项下游测试名称及全部任务的留出协议本轮待核实。",
"sections": "MoE 实验；附录 B.4、Table 4",
"source": "https://arxiv.org/abs/2509.19349",
"version": "2509.19349v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "78901e4124e6b3a54958d8afa18d4169fe33fde0d8766af1b3de3e5456f049b4",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "程序优化任务运行候选程序；AIME 运行框架实验由 GPT-4.1-nano 在候选结构下答题，迁移测试另用 GPT-4.1-mini、GPT-4.1 和 o4-mini。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px2"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定进化控制器选择父代与模型，再让语言模型改写程序。模型池随实验设置变化：MoE 负载均衡实验明确使用 GPT-4.1、Gemini-2.5-Pro、Claude-Sonnet-4；不能将它们直接当作 AIME 的答题模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px2"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "候选程序、科研代码或特定任务的运行框架；进化引擎和外部评分规则固定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行候选程序后按任务目标评分；采样时也考虑新颖性，以避免只探索相似方案。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "程序种群/档案加可执行评估器，语言模型 提议代码变化，系统管理采样和选择；每个应用给初始程序与目标函数。MoE 实验改的是训练用损失函数代码，训练模型只是评价候选损失的一部分。",
"sources": [
{
"label": "MoE 实验；附录 B.4、Table 4",
"url": "https://arxiv.org/abs/2509.19349"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "岛屿档案保存优质程序，采样父代和灵感程序后让 语言模型 变异；执行真实目标评分，按相对增益调整模型采样概率。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "MoE 损失搜索：556M 模型在 FineWeb 约 2B 词元 上训练评估候选；另有几何及 ALE-Bench 程序优化。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "候选程序的执行有效性与目标值；ALE LITE 10 题以公开测试分数搜索。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ALE 最佳公开候选提交私有测试；MoE 将损失迁移到 2.7B 模型、约 30B FineWeb 词元 再评困惑度及下游。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "ALE 另报告私有测试取 top-5 最大值的诊断结果，应与单次提交分开；MoE 是跨规模迁移。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "提供维护候选程序、选择修改起点并继续搜索的高效进化系统；框架代码也可作为被优化程序，但不是所有实验都在改 agent 自身。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2509.19349v1",
"version": "2509.19349v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它不是 agent 自己持续改自己的 deployment harness；真正新点是 **更高效的开放式 program evolution/search substrate**，但论文也展示可把 agentic harness 当 program candidate 优化。",
"feedbackCases": [
{
"label": "代码进化：ALE-Bench LITE",
"data": "10 道 AtCoder 启发式优化比赛题；每题从 ALE-Agent 已有较好程序起步，搜索 50 代。",
"scoring": "比赛的公开测试程序计算候选解的任务目标分数。",
"visible": "候选有效性及公开测试得分；代码新颖性另用于避免重复探索，不等于题目得分。",
"use": "按公开得分搜索并挑最佳程序，之后提交私有测试；报告的私有成绩不用于这 50 代选择。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"judgment": "ALE-Bench 比赛测试程序计算目标分"
},
{
"label": "损失函数进化：MoE",
"data": "搜索阶段训练 556M 模型，使用约 2B FineWeb 词元；迁移到 2.7B 模型、约 30B FineWeb 词元。",
"scoring": "实际训练候选损失对应的模型，再比较语言模型表现；迁移阶段测困惑度及下游任务。",
"visible": "程序执行有效性、训练及评价指标。",
"use": "小模型实验用于选择损失；大模型实验检验选中损失能否迁移，不是把 30B 词元 都用于初始搜索。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"judgment": "实际训练模型，计算验证损失／困惑度及下游指标"
},
{
"label": "几何程序：圆 packing",
"data": "给定圆数量与边界的几何优化实例；无自然语言问答训练集。",
"scoring": "检查圆是否越界、是否相交等可行性约束，并计算半径和等优化目标。",
"visible": "有效性与数值目标反馈给程序搜索器。",
"use": "保留有效且目标更好的程序；同一几何实例上的最佳值属于直接优化结果。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"judgment": "程序检查几何合法性，计算半径和等目标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "MoE 损失搜索：556M 模型在 FineWeb 约 2B 词元 上训练评估候选；另有几何及 ALE-Bench 程序优化。",
"selection": "候选程序的执行有效性与目标值；ALE LITE 10 题以公开测试分数搜索。",
"evaluation": "ALE 最佳公开候选提交私有测试；MoE 将损失迁移到 2.7B 模型、约 30B FineWeb 词元 再评困惑度及下游。",
"isolation": "ALE 另报告私有测试取 top-5 最大值的诊断结果，应与单次提交分开；MoE 是跨规模迁移。",
"roles": {
"executor": {
"value": "程序优化任务运行候选程序；AIME 运行框架实验由 GPT-4.1-nano 在候选结构下答题，迁移测试另用 GPT-4.1-mini、GPT-4.1 和 o4-mini。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px2"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
]
},
"modifier": {
"value": "固定进化控制器选择父代与模型，再让语言模型改写程序。模型池随实验设置变化：MoE 负载均衡实验明确使用 GPT-4.1、Gemini-2.5-Pro、Claude-Sonnet-4；不能将它们直接当作 AIME 的答题模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2509.19349#S4.SS2.SSS0.Px2"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
]
},
"seed": {
"value": "程序种群/档案加可执行评估器，语言模型 提议代码变化，系统管理采样和选择；每个应用给初始程序与目标函数。MoE 实验改的是训练用损失函数代码，训练模型只是评价候选损失的一部分。",
"sources": [
{
"label": "MoE 实验；附录 B.4、Table 4",
"url": "https://arxiv.org/abs/2509.19349"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"selection": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
],
"isolation": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2509.19349#S4.SS4.SSS0.Px2"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2509.19349#A2.SS3.SSS0.Px1"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2509.19349#A2.SS4.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有代码进化系统常需数千次候选评估，耗时且昂贵；作者将这一瓶颈与探索策略未充分利用此前积累的经验联系起来。领先系统又多未开源，妨碍复现和社区改进，因此需要既减少评估次数、又能公开复用的程序进化方法。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2509.19349#S1"
}
]
},
{
"key": "position",
"value": "面向科学发现与算法优化，研究如何降低程序进化的试验成本，同时保留开放探索和发现高质量新解的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2509.19349"
}
]
},
{
"key": "conclusion",
"value": "在圆形装填、agent 结构和训练损失设计等任务中提高解质量与样本效率。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2509.19349"
}
]
}
],
"fields": {
"object": "候选程序、科研代码或特定任务的运行框架；进化引擎和外部评分规则固定。"
}
},
"attributions": [
{
"tag": "org:sakana",
"label": "Sakana AI",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2509.19349"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"perspectives"
]
},
{
"id": "2305.10250",
"title": "MemoryBank: Enhancing Large Language Models with Long-Term Memory",
"url": "https://arxiv.org/abs/2305.10250",
"date": "2023-05-16",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"MemoryContent"
],
"fields": {
"本质定位": "长期保存对话摘要/用户画像，并用类 Ebbinghaus 机制决定记忆保留与遗忘。",
"什么在变": "memory content / retention strength。",
"谁来改 / 谁执行": "**改**：固定 summarization + forgetting mechanism。
**执行**：chat LLM。",
"基础 harness": "conversation agent + external memory bank。",
"Feedback": "用户交互历史；非 task verifier。",
"Evolution → Eval": "模拟长期对话与人工评价。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "把长时个性化 memory 引入 LLM interaction，但主要目标不是 task performance self-improvement。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 288,
"fields": {
"优先级": "**R**",
"时间": "2023-05-16",
"论文": "[MemoryBank: Enhancing Large Language Models with Long-Term Memory](https://arxiv.org/abs/2305.10250)",
"本质定位": "长期保存对话摘要/用户画像，并用类 Ebbinghaus 机制决定记忆保留与遗忘。",
"什么在变": "memory content / retention strength。",
"谁来改 / 谁执行": "**改**：固定 summarization + forgetting mechanism。
**执行**：chat LLM。",
"基础 harness": "conversation agent + external memory bank。",
"Feedback": "用户交互历史；非 task verifier。",
"Evolution → Eval": "模拟长期对话与人工评价。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "把长时个性化 memory 引入 LLM interaction，但主要目标不是 task performance self-improvement。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "长期保存对话摘要/用户画像，并用类 Ebbinghaus 机制决定记忆保留与遗忘。",
"novelty": "为长期对话积累、检索和更新用户记忆，重点服务个性化交互，任务解题能力的自改进不是其主要目标。",
"object": "长期记忆内容及各条记忆的保留强度。",
"executor": "SiliconFriend分别接ChatGPT、ChatGLM（6.2B）和BELLE（由LLaMA7B微调）；ChatGPT在论文中未给具体API快照名。",
"modifier": "对话摘要/用户画像由所接语言模型生成，遗忘强度由固定规则更新；ChatGLM/BELLE另做心理对话LoRA（只训练少量适配参数），ChatGPT不做该参数训练。",
"roleContext": "**改**：固定 summarization + forgetting mechanism。
**执行**：chat LLM。",
"seed": "对话模型外的 MemoryBank 包含存储、检索、更新、事件摘要与用户画像；SiliconFriend 是集成该记忆的陪伴应用。已有长期记忆管理，不是简单无限拼接聊天历史。",
"fixed": "",
"verdict": "用户交互历史；非任务结果检查器。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "模拟长期对话与人工评价。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SiliconFriend 的开源模型先用 3.8 万条网络心理对话做 LoRA（只训练少量适配参数）；MemoryBank 本身从用户对话积累事件和用户画像。\n\n调试 / 选版本数据：检索与遗忘机制随对话更新；不是按独立验证集分数搜索记忆架构。\n\n最终测试数据：ChatGPT 模拟 15 位用户、10 天中英对话；人工编写 194 道回忆题（中英各 97），人工评价检索与回答，另展示真实用户对话案例。\n\n数据隔离与证据边界：参数适配语料与记忆回忆评测是两套材料；回忆题考察已存入的聊天内容，不是陌生知识的泛化。",
"cycle": "按对话生成事件摘要和画像，通过向量检索取回；按遗忘曲线降低旧记忆强度，被回忆时强化。",
"train": "SiliconFriend 的开源模型先用 3.8 万条网络心理对话做 LoRA（只训练少量适配参数）；MemoryBank 本身从用户对话积累事件和用户画像。",
"debug": "检索与遗忘机制随对话更新；不是按独立验证集分数搜索记忆架构。",
"test": "ChatGPT 模拟 15 位用户、10 天中英对话；人工编写 194 道回忆题（中英各 97），人工评价检索与回答，另展示真实用户对话案例。",
"isolation": "参数适配语料与记忆回忆评测是两套材料；回忆题考察已存入的聊天内容，不是陌生知识的泛化。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "对话模型外的 MemoryBank 包含存储、检索、更新、事件摘要与用户画像；SiliconFriend 是集成该记忆的陪伴应用。已有长期记忆管理，不是简单无限拼接聊天历史。",
"protocol": "**材料与评估：**记忆来自多次用户对话，后续通过回忆、个性化回应等检查使用效果。其应用/对话实验不能直接写成 LoCoMo 等后来 benchmark 的结果；具体原始对话集、人数和独立评估划分本轮尚未核实。模型适配与外部记忆更新需分开解释。",
"sections": "MemoryBank 架构与 SiliconFriend 应用",
"source": "https://arxiv.org/abs/2305.10250",
"version": "2305.10250v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a848b220bd96254750887f2c3ea2a3e57695c9409c664cd1b18072deeabc9068",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "SiliconFriend分别接ChatGPT、ChatGLM（6.2B）和BELLE（由LLaMA7B微调）；ChatGPT在论文中未给具体API快照名。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对话摘要/用户画像由所接语言模型生成，遗忘强度由固定规则更新；ChatGLM/BELLE另做心理对话LoRA（只训练少量适配参数），ChatGPT不做该参数训练。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "长期记忆内容及各条记忆的保留强度。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用户交互历史；非任务结果检查器。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "对话模型外的 MemoryBank 包含存储、检索、更新、事件摘要与用户画像；SiliconFriend 是集成该记忆的陪伴应用。已有长期记忆管理，不是简单无限拼接聊天历史。",
"sources": [
{
"label": "MemoryBank 架构与 SiliconFriend 应用",
"url": "https://arxiv.org/abs/2305.10250"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "按对话生成事件摘要和画像，通过向量检索取回；按遗忘曲线降低旧记忆强度，被回忆时强化。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SiliconFriend 的开源模型先用 3.8 万条网络心理对话做 LoRA（只训练少量适配参数）；MemoryBank 本身从用户对话积累事件和用户画像。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "检索与遗忘机制随对话更新；不是按独立验证集分数搜索记忆架构。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "ChatGPT 模拟 15 位用户、10 天中英对话；人工编写 194 道回忆题（中英各 97），人工评价检索与回答，另展示真实用户对话案例。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "参数适配语料与记忆回忆评测是两套材料；回忆题考察已存入的聊天内容，不是陌生知识的泛化。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "为长期对话积累、检索和更新用户记忆，重点服务个性化交互，任务解题能力的自改进不是其主要目标。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2305.10250v3",
"version": "2305.10250v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把长时个性化 memory 引入 LLM interaction，但主要目标不是 task performance self-improvement。",
"feedbackCases": [
{
"label": "记忆积累",
"data": "来自用户对话的事件与用户画像；SiliconFriend 的 3.8 万条网络心理对话用于先前 LoRA 微调，是另一阶段。",
"scoring": "记忆写入和遗忘依据对话内容、时间与回忆使用，不依赖每轮任务正确答案。",
"visible": "用户说过什么、记忆被调用的历史；没有一个逐轮验收任务对错的外部程序。",
"use": "更新可检索记忆及用户画像，不能理解为按回忆测试题的分数训练记忆。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"judgment": "对话内容与时间／回忆使用驱动记忆维护，无答案奖励"
},
{
"label": "最终记忆评估",
"data": "ChatGPT 模拟 15 位用户、10 天中英对话；人工编写 194 道回忆题，中英各 97 道。",
"scoring": "人工判断是否检索到相关记忆（0/1）、回答正确程度（0/0.5/1）和上下文连贯性（0/0.5/1），另比较模型排序。",
"visible": "这些是实验报告的人工评价。",
"use": "检验能否从此前对话回忆信息；不是记忆写入时获得的人类逐条奖励。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"judgment": "人类按相关性、正确性、连贯性进行离散评分"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "SiliconFriend 的开源模型先用 3.8 万条网络心理对话做 LoRA（只训练少量适配参数）；MemoryBank 本身从用户对话积累事件和用户画像。",
"selection": "检索与遗忘机制随对话更新；不是按独立验证集分数搜索记忆架构。",
"evaluation": "ChatGPT 模拟 15 位用户、10 天中英对话；人工编写 194 道回忆题（中英各 97），人工评价检索与回答，另展示真实用户对话案例。",
"isolation": "参数适配语料与记忆回忆评测是两套材料；回忆题考察已存入的聊天内容，不是陌生知识的泛化。",
"roles": {
"executor": {
"value": "SiliconFriend分别接ChatGPT、ChatGLM（6.2B）和BELLE（由LLaMA7B微调）；ChatGPT在论文中未给具体API快照名。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
}
]
},
"modifier": {
"value": "对话摘要/用户画像由所接语言模型生成，遗忘强度由固定规则更新；ChatGLM/BELLE另做心理对话LoRA（只训练少量适配参数），ChatGPT不做该参数训练。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.10250#S2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px2"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
}
]
},
"seed": {
"value": "对话模型外的 MemoryBank 包含存储、检索、更新、事件摘要与用户画像；SiliconFriend 是集成该记忆的陪伴应用。已有长期记忆管理，不是简单无限拼接聊天历史。",
"sources": [
{
"label": "MemoryBank 架构与 SiliconFriend 应用",
"url": "https://arxiv.org/abs/2305.10250"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2305.10250#S3.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2305.10250#S4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.10250#S4.SS2.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "语言模型缺少跨会话的长期记忆，难以在持续交往中记住过去谈话、用户偏好和未完成事务。对陪伴和助理任务，这会妨碍关系延续、个性化回应与任务管理；作者因此研究如何让模型保留并按需使用长期交互信息。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2305.10250#S1"
}
]
},
{
"key": "position",
"value": "面向长期陪伴和个人助理，研究模型如何跨会话记住用户经历与特点，支持连贯且个性化的交互。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.10250"
}
]
},
{
"key": "conclusion",
"value": "在真实及模拟对话中改善历史召回和个性化回应，提供可接入不同模型的记忆模块。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.10250"
}
]
}
],
"fields": {
"object": "长期记忆内容及各条记忆的保留强度。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2308.10144",
"title": "ExpeL: LLM Agents Are Experiential Learners",
"url": "https://arxiv.org/abs/2308.10144",
"date": "2023-08",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"M0",
"MemoryContent"
],
"fields": {
"本质定位": "收集多任务成功/失败 trajectory，抽取可复用 insights；未来任务检索 insights 和成功示例辅助 agent。",
"什么在变": "cross-task insight memory / example pool。",
"谁来改 / 谁执行": "**改**：经验抽取默认 GPT-4-0613；执行常用 GPT-3.5。
**执行**：target ReAct-like agent。",
"基础 harness": "agent + insight memory + retrieval。",
"Feedback": "环境/答案正确性 + 成败 trajectories。",
"Evolution → Eval": "ALFWorld、HotPotQA 等；含 source→target transfer（如 HotPotQA→FEVER）。",
"Meta-depth": "M0：经验库变，抽取/检索机制固定。",
"相对之前真正新增什么": "相比 Reflexion 更重要的新点是 **跨任务** 经验提炼与 transfer，而非只对同一 case 重试。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 289,
"fields": {
"优先级": "**K**",
"时间": "2023-08",
"论文": "[ExpeL: LLM Agents Are Experiential Learners](https://arxiv.org/abs/2308.10144)",
"本质定位": "收集多任务成功/失败 trajectory，抽取可复用 insights；未来任务检索 insights 和成功示例辅助 agent。",
"什么在变": "cross-task insight memory / example pool。",
"谁来改 / 谁执行": "**改**：经验抽取默认 GPT-4-0613；执行常用 GPT-3.5。
**执行**：target ReAct-like agent。",
"基础 harness": "agent + insight memory + retrieval。",
"Feedback": "环境/答案正确性 + 成败 trajectories。",
"Evolution → Eval": "ALFWorld、HotPotQA 等；含 source→target transfer（如 HotPotQA→FEVER）。",
"Meta-depth": "M0：经验库变，抽取/检索机制固定。",
"相对之前真正新增什么": "相比 Reflexion 更重要的新点是 **跨任务** 经验提炼与 transfer，而非只对同一 case 重试。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2023",
"depth": [
"M0"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "收集多任务成功/失败 trajectory，抽取可复用 insights；未来任务检索 insights 和成功示例辅助 agent。",
"novelty": "从训练任务的多次试错提炼一般规则，并保存成功示例，让新任务的单次执行也能利用这些经验。",
"object": "跨任务的文字经验库与成功示例池；经验抽取和检索机制保持固定。",
"executor": "评估阶段统一gpt-3.5-turbo-0613，按ReAct（交替进行推理、调用工具和读取结果的执行方式）流程执行任务。",
"modifier": "经验规则抽取用gpt-4-0613；训练经验收集中的Reflexion用gpt-3.5-turbo-0613，超窗口时换16k-0613。因此规则提炼者比最终执行模型更强。",
"roleContext": "**改**：经验抽取默认 GPT-4-0613；执行常用 GPT-3.5。
**执行**：target ReAct-like agent。",
"seed": "ReAct（交替进行推理、调用工具和读取结果的执行方式） 规划器在训练任务失败后允许 Reflexion 式重试，把成功轨迹与提炼规则保存在经验池；测试题可检索经验，而不是重新执行训练阶段的多次试错。",
"fixed": "M0：经验库变，抽取/检索机制固定",
"verdict": "环境/答案正确性 + 成败 逐步执行记录。",
"diagnosis": "收集多任务成功与失败经历，比较并提炼自然语言 insights；与只在原题重试的 reflection 分开。",
"update": "训练任务允许反复试错；得到的 insights 和成功示例用于未来测试任务的上下文。",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "ALFWorld、HotPotQA 等；含 source→target transfer（如 HotPotQA→FEVER）。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2308.10144v3"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：HotpotQA（需要结合多份资料作答的多跳问答基准） distractor-dev 100 题、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134 题、WebShop（根据用户要求挑选和购买商品的交互基准） 100 题组成任务池；每次用一半收集成功/失败轨迹。\n\n调试 / 选版本数据：从成败轨迹对比提炼并增删经验规则；不是在测试题上反复试到成功。\n\n最终测试数据：另一半任务评估，并交换两半；论文报告四折均值及标准误。另做 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 知识迁移。\n\n数据隔离与证据边界：训练与评测按每折区分，不能把任务池总量当成训练量。",
"cycle": "agent 用推理—行动循环完成训练任务，失败时允许反思重试。模型比较成功与失败经历并修订经验规则；测试新题时检索成功示例和规则，不重新执行训练阶段的多轮试错。",
"train": "HotpotQA（需要结合多份资料作答的多跳问答基准） distractor-dev 100 题、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134 题、WebShop（根据用户要求挑选和购买商品的交互基准） 100 题组成任务池；每次用一半收集成功/失败轨迹。",
"debug": "从成败轨迹对比提炼并增删经验规则；不是在测试题上反复试到成功。",
"test": "另一半任务评估，并交换两半；论文报告四折均值及标准误。另做 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 知识迁移。",
"isolation": "训练与评测按每折区分，不能把任务池总量当成训练量。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "ReAct 规划器在训练任务失败后允许 Reflexion 式重试，把成功轨迹与提炼规则保存在经验池；测试题可检索经验，而不是重新执行训练阶段的多次试错。",
"protocol": "**任务池：**HotpotQA distractor dev 中 100 题、ALFWorld 134 个可解任务、WebShop 100 题，沿用 ReAct/Reflexion 的任务池。\n\n**划分：**论文称四折验证，在每次划分中一半用于经验收集、另一半评估，并交换两半；最终报跨折均值和标准误。这不是在完整 134 道 ALFWorld 上积累后再声称同题是留出测试。",
"sections": "实验数据与交叉验证",
"source": "https://arxiv.org/abs/2308.10144",
"version": "2308.10144v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "0d31026fd4804992a52ae67ebb6ed22f045c0badee9a7db747cf1d0123a51958",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "评估阶段统一gpt-3.5-turbo-0613，按ReAct（交替进行推理、调用工具和读取结果的执行方式）流程执行任务。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2308.10144#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "经验规则抽取用gpt-4-0613；训练经验收集中的Reflexion用gpt-3.5-turbo-0613，超窗口时换16k-0613。因此规则提炼者比最终执行模型更强。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2308.10144#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨任务的文字经验库与成功示例池；经验抽取和检索机制保持固定。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "环境/答案正确性 + 成败 逐步执行记录。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "ReAct（交替进行推理、调用工具和读取结果的执行方式） 规划器在训练任务失败后允许 Reflexion 式重试，把成功轨迹与提炼规则保存在经验池；测试题可检索经验，而不是重新执行训练阶段的多次试错。",
"sources": [
{
"label": "实验数据与交叉验证",
"url": "https://arxiv.org/abs/2308.10144"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "agent 用推理—行动循环完成训练任务，失败时允许反思重试。模型比较成功与失败经历并修订经验规则；测试新题时检索成功示例和规则，不重新执行训练阶段的多轮试错。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "HotpotQA（需要结合多份资料作答的多跳问答基准） distractor-dev 100 题、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134 题、WebShop（根据用户要求挑选和购买商品的交互基准） 100 题组成任务池；每次用一半收集成功/失败轨迹。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "从成败轨迹对比提炼并增删经验规则；不是在测试题上反复试到成功。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "另一半任务评估，并交换两半；论文报告四折均值及标准误。另做 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 知识迁移。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "训练与评测按每折区分，不能把任务池总量当成训练量。",
"sources": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从训练任务的多次试错提炼一般规则，并保存成功示例，让新任务的单次执行也能利用这些经验。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2308.10144v3",
"version": "2308.10144v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 Reflexion 的同题试错经验扩展为跨任务学习：训练时多次试错，测试时利用 insights 与成功示例做单次尝试；参数不更新。",
"feedbackCases": [
{
"label": "经验提炼：HotpotQA",
"data": "distractor-dev 中 100 题；每次一半收集经验、另一半评估，交换两半，按论文四折汇总。",
"scoring": "使用规则将最终回答与标准答案精确匹配，返回成功／失败；论文明确沿用 ReAct 的任务成功指标。",
"visible": "带结果的成功和失败执行轨迹。",
"use": "比较成败过程，提炼、增删文字规则，在另一半题上使用；另有 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 迁移。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1：明确规定 HotpotQA exact matching",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
}
],
"judgment": "规则将 HotpotQA 最终回答与参考答案精确匹配"
},
{
"label": "经验提炼：ALFWorld",
"data": "134 个家庭物体操作任务；每次一半收集经验、另一半评估，交换两半，按论文四折汇总。",
"scoring": "环境检查物体操作是否达到任务目标，返回成功与否。",
"visible": "带结果的成功和失败执行轨迹。",
"use": "比较成败过程，提炼、增删文字规则，在另一半题上使用；另有 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 迁移。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
}
],
"judgment": "ALFWorld 环境检查目标"
},
{
"label": "经验提炼：WebShop",
"data": "100 个购物任务；每次一半收集经验、另一半评估，交换两半，按论文四折汇总。",
"scoring": "环境根据所选商品对购物目标的满足程度给分；附录 D.4 定义奖励。",
"visible": "带结果的成功和失败执行轨迹。",
"use": "比较成败过程，提炼、增删文字规则，在另一半题上使用；另有 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 迁移。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
}
],
"judgment": "WebShop 程序计算商品对购物要求的满足程度"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "HotpotQA（需要结合多份资料作答的多跳问答基准） distractor-dev 100 题、ALFWorld（通过文字动作完成家居物体操作的交互环境） 134 题、WebShop（根据用户要求挑选和购买商品的交互基准） 100 题组成任务池；每次用一半收集成功/失败轨迹。",
"selection": "从成败轨迹对比提炼并增删经验规则；不是在测试题上反复试到成功。",
"evaluation": "另一半任务评估，并交换两半；论文报告四折均值及标准误。另做 HotpotQA（需要结合多份资料作答的多跳问答基准）→FEVER 知识迁移。",
"isolation": "训练与评测按每折区分，不能把任务池总量当成训练量。",
"roles": {
"executor": {
"value": "评估阶段统一gpt-3.5-turbo-0613，按ReAct（交替进行推理、调用工具和读取结果的执行方式）流程执行任务。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2308.10144#A4.SS5"
}
]
},
"modifier": {
"value": "经验规则抽取用gpt-4-0613；训练经验收集中的Reflexion用gpt-3.5-turbo-0613，超窗口时换16k-0613。因此规则提炼者比最终执行模型更强。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2308.10144#A4.SS5"
}
]
},
"seed": {
"value": "ReAct（交替进行推理、调用工具和读取结果的执行方式） 规划器在训练任务失败后允许 Reflexion 式重试，把成功轨迹与提炼规则保存在经验池；测试题可检索经验，而不是重新执行训练阶段的多次试错。",
"sources": [
{
"label": "实验数据与交叉验证",
"url": "https://arxiv.org/abs/2308.10144"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"selection": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"evaluation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
],
"isolation": [
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2308.10144#A4.SS1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "通过大量交互或人类标注微调模型成本高，还需要能访问模型权重；只靠少量上下文示例又受窗口长度限制，难保留跨任务经验。两条路线各有局限，作者因此研究能否让 agent 从自身尝试中积累可复用教训，在不更新参数的情况下改善后续新任务表现。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2308.10144#S1"
}
]
},
{
"key": "position",
"value": "研究无法访问或更新模型权重时，agent 能否从多项任务的成败经历中自主学习，并把经验用于新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2308.10144"
}
]
},
{
"key": "conclusion",
"value": "随着经验积累提高表现，并展示跨任务迁移；学习产物是可读取的经验与示例。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2308.10144"
}
]
}
],
"fields": {
"object": "跨任务的文字经验库与成功示例池；经验抽取和检索机制保持固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2409.00872",
"title": "SAGE: Self-evolving Agents with Reflective and Memory-Augmented Abilities",
"url": "https://arxiv.org/abs/2409.00872",
"date": "2024-09",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"MemoryContent"
],
"fields": {
"本质定位": "Assistant/Checker 迭代反馈 + reflection，并用 Ebbinghaus-style mechanism 管理 STM/LTM。",
"什么在变": "memory content、retention/pruning 与当前 strategy。",
"谁来改 / 谁执行": "**改**：固定 Checker/reflection/memory rules。
**执行**：GPT-4/GPT-3.5/open models。",
"基础 harness": "User–Assistant–Checker + STM/LTM。",
"Feedback": "Checker feedback + interaction history。",
"Evolution → Eval": "AgentBench 6 tasks 与 long-context tasks。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "把 reflection 与 forgetting-aware memory 管理组合到 multi-turn agent；机制上主要是组合已有模块，不是 memory architecture search。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 290,
"fields": {
"优先级": "**R**",
"时间": "2024-09",
"论文": "[SAGE: Self-evolving Agents with Reflective and Memory-Augmented Abilities](https://arxiv.org/abs/2409.00872)",
"本质定位": "Assistant/Checker 迭代反馈 + reflection，并用 Ebbinghaus-style mechanism 管理 STM/LTM。",
"什么在变": "memory content、retention/pruning 与当前 strategy。",
"谁来改 / 谁执行": "**改**：固定 Checker/reflection/memory rules。
**执行**：GPT-4/GPT-3.5/open models。",
"基础 harness": "User–Assistant–Checker + STM/LTM。",
"Feedback": "Checker feedback + interaction history。",
"Evolution → Eval": "AgentBench 6 tasks 与 long-context tasks。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "把 reflection 与 forgetting-aware memory 管理组合到 multi-turn agent；机制上主要是组合已有模块，不是 memory architecture search。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M0"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Assistant/Checker 迭代反馈 + reflection，并用 Ebbinghaus-style mechanism 管理 STM/LTM。",
"novelty": "在多轮 agent 中结合反思与考虑遗忘的记忆管理，持续调整保留和使用的经验；主要是预设模块的协同。",
"object": "记忆内容、保留与裁剪状态，以及当前行为策略。",
"executor": "AgentBench比较GPT-3.5、GPT-4、Llama2-7B、CodeLlama-7B、Qwen-1.8B及ChatGLM2；具体商用API快照没有在§4.1中展开。",
"modifier": "检查角色、反思和记忆维护按SAGE固定流程运行；§4.1列出被测基础模型，未给出独立检查角色模型的逐配置对应表，不把它默认成GPT-4教师。",
"roleContext": "**改**：固定 Checker/reflection/memory rules。
**执行**：GPT-4/GPT-3.5/open models。",
"seed": "User–助手角色–检查角色 交互配合短期/长期记忆，先执行、检查，再反思和更新记忆；框架含检索与任务拆解，不是只有一句 self-reflection 提示。",
"fixed": "",
"verdict": "检查者模型阅读回答与交互历史，给出批评供助手重试。检查者的意见与基准的最终判分是两个来源，不能自动等同。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "AgentBench 6 tasks 与 long-context tasks。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：无需额外参数训练；反思与记忆来自当前任务交互。\n\n调试 / 选版本数据：检查角色 检查回答，助手角色 据反馈重试；MemorySyntax 管理历史信息。\n\n最终测试数据：AgentBench 六类任务；LCC、RepoBench-P、HotpotQA（需要结合多份资料作答的多跳问答基准）、TriviaQA 长文本；RAG（先检索相关资料，再把资料交给模型回答） 实验还有 Natural Questions，附录另列 MultiWOZ/ALFWorld。\n\n数据隔离与证据边界：§4 与附录列出了评测任务，但没有给出统一的进化/验证/测试拆分协议；这些结果不能作为严格留出迁移的证据。",
"cycle": "助手先回答，检查者阅读回答和历史并提出批评；助手据此修改，把反思写入短期或长期记忆，后续使用时按记忆保留规则检索。",
"train": "无需额外参数训练；反思与记忆来自当前任务交互。",
"debug": "检查角色 检查回答，助手角色 据反馈重试；MemorySyntax 管理历史信息。",
"test": "AgentBench 六类任务；LCC、RepoBench-P、HotpotQA（需要结合多份资料作答的多跳问答基准）、TriviaQA 长文本；RAG（先检索相关资料，再把资料交给模型回答） 实验还有 Natural Questions，附录另列 MultiWOZ/ALFWorld。",
"isolation": "§4 与附录列出了评测任务，但没有给出统一的进化/验证/测试拆分协议；这些结果不能作为严格留出迁移的证据。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "User–Assistant–Checker 交互配合短期/长期记忆，先执行、检查，再反思和更新记忆；框架含检索与任务拆解，不是只有一句 self-reflection 提示。",
"protocol": "**评测来源：**AgentBench，以及 HotpotQA、Natural Questions、TriviaQA 等 QA；附录还列 MultiWOZ 多轮对话、ALFWorld 顺序任务。\n\n**进化与数据边界：**反馈和记忆来自任务交互，但各实验用了哪些训练/验证/测试 split、是否在报告集上继续更新，本轮未核实。不能因表格列出多个 benchmark 就推断有跨 benchmark 留出迁移。",
"sections": "§4.1–4.5 与附录任务比较",
"source": "https://arxiv.org/abs/2409.00872",
"version": "2409.00872v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "051d1ddc887309d7d3e7b4ddf541c5c9503236897115c5ae44a7d785c6752109",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "AgentBench比较GPT-3.5、GPT-4、Llama2-7B、CodeLlama-7B、Qwen-1.8B及ChatGLM2；具体商用API快照没有在§4.1中展开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2409.00872#S4.SS1"
},
{
"label": "§4.7",
"url": "https://arxiv.org/html/2409.00872#S4.SS7"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "检查角色、反思和记忆维护按SAGE固定流程运行；§4.1列出被测基础模型，未给出独立检查角色模型的逐配置对应表，不把它默认成GPT-4教师。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2409.00872#S4.SS1"
},
{
"label": "§4.7",
"url": "https://arxiv.org/html/2409.00872#S4.SS7"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "记忆内容、保留与裁剪状态，以及当前行为策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "检查者模型阅读回答与交互历史，给出批评供助手重试。检查者的意见与基准的最终判分是两个来源，不能自动等同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
},
{
"label": "§4.3：长文本指标",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5：检索问答",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "User–助手角色–检查角色 交互配合短期/长期记忆，先执行、检查，再反思和更新记忆；框架含检索与任务拆解，不是只有一句 self-reflection 提示。",
"sources": [
{
"label": "§4.1–4.5 与附录任务比较",
"url": "https://arxiv.org/abs/2409.00872"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "助手先回答，检查者阅读回答和历史并提出批评；助手据此修改，把反思写入短期或长期记忆，后续使用时按记忆保留规则检索。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "无需额外参数训练；反思与记忆来自当前任务交互。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "检查角色 检查回答，助手角色 据反馈重试；MemorySyntax 管理历史信息。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AgentBench 六类任务；LCC、RepoBench-P、HotpotQA（需要结合多份资料作答的多跳问答基准）、TriviaQA 长文本；RAG（先检索相关资料，再把资料交给模型回答） 实验还有 Natural Questions，附录另列 MultiWOZ/ALFWorld。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4 与附录列出了评测任务，但没有给出统一的进化/验证/测试拆分协议；这些结果不能作为严格留出迁移的证据。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在多轮 agent 中结合反思与考虑遗忘的记忆管理，持续调整保留和使用的经验；主要是预设模块的协同。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2409.00872v2",
"version": "2409.00872v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 reflection 与 forgetting-aware memory 管理组合到 multi-turn agent；机制上主要是组合已有模块，不是 memory architecture search。",
"feedbackCases": [
{
"label": "回答修订：Checker",
"data": "当前交互任务；无需额外参数训练。测试覆盖 AgentBench 的知识图谱、OS、DB、ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、Mind2Web。",
"scoring": "检查角色 模型阅读 助手角色 回答与交互历史给出批评；它的判断本身不等于基准标准答案或环境验收。",
"visible": "模型的文字反馈、历史观察与动作。",
"use": "助手角色 据反馈重试并记录反思；论文没有把每种环境的真实评分都规定为 检查角色 可见输入。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"judgment": "Checker 模型依据回答和交互历史作批评，不读取基准真值作验收"
},
{
"label": "长文本与检索问答评价",
"data": "LCC、RepoBench-P；HotpotQA（需要结合多份资料作答的多跳问答基准）、TriviaQA；检索实验另有 Natural Questions。",
"scoring": "LCC / RepoBench-P 的下一行代码预测按 Precision、Recall、F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务） 评价；HotpotQA（需要结合多份资料作答的多跳问答基准） / TriviaQA 按答案 F1；检索实验报告问答准确率。",
"visible": "基准参考内容用于计算最终指标。",
"use": "这些评价与 检查角色 的文字批评是两个来源，不能把代码补全 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务） 写成运行单元测试。",
"sources": [
{
"label": "§4.3：长文本指标",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5：检索问答",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
}
],
"judgment": "参考答案／代码行的词项匹配指标；检索实验另计准确率"
},
{
"label": "其他交互任务评价",
"data": "AgentBench 六类环境，另有 MultiWOZ 对话和 ALFWorld（通过文字动作完成家居物体操作的交互环境） 顺序任务。",
"scoring": "采用相应任务完成指标；§4.2 还比较回答准确、对话连贯和步骤完成。论文没有逐环境公布一套完整的 检查角色 判分提示及反馈权限配置。",
"visible": "报告中的评价分数；当前执行中另可见环境观察。",
"use": "用于比较加 SAGE 前后表现；不能仅凭这些汇总结果还原所有环境的逐步裁判。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"judgment": "各任务完成指标；本文未逐环境展开检查角色的全部判分实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1,
2
],
"evolution": "无需额外参数训练；反思与记忆来自当前任务交互。",
"selection": "检查角色 检查回答，助手角色 据反馈重试；MemorySyntax 管理历史信息。",
"evaluation": "AgentBench 六类任务；LCC、RepoBench-P、HotpotQA（需要结合多份资料作答的多跳问答基准）、TriviaQA 长文本；RAG（先检索相关资料，再把资料交给模型回答） 实验还有 Natural Questions，附录另列 MultiWOZ/ALFWorld。",
"isolation": "§4 与附录列出了评测任务，但没有给出统一的进化/验证/测试拆分协议；这些结果不能作为严格留出迁移的证据。",
"roles": {
"executor": {
"value": "AgentBench比较GPT-3.5、GPT-4、Llama2-7B、CodeLlama-7B、Qwen-1.8B及ChatGLM2；具体商用API快照没有在§4.1中展开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2409.00872#S4.SS1"
},
{
"label": "§4.7",
"url": "https://arxiv.org/html/2409.00872#S4.SS7"
}
]
},
"modifier": {
"value": "检查角色、反思和记忆维护按SAGE固定流程运行；§4.1列出被测基础模型，未给出独立检查角色模型的逐配置对应表，不把它默认成GPT-4教师。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2409.00872#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2409.00872#S4.SS1"
},
{
"label": "§4.7",
"url": "https://arxiv.org/html/2409.00872#S4.SS7"
}
]
},
"seed": {
"value": "User–助手角色–检查角色 交互配合短期/长期记忆，先执行、检查，再反思和更新记忆；框架含检索与任务拆解，不是只有一句 self-reflection 提示。",
"sources": [
{
"label": "§4.1–4.5 与附录任务比较",
"url": "https://arxiv.org/abs/2409.00872"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2409.00872#S4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2409.00872#S4.SS3"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2409.00872#S4.SS5"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2409.00872#A3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 既要在变化环境中连续决策，又要利用很久以前的交互信息，但有限上下文和缺少长期记忆使这两点难以兼顾。已有适应与记忆方法常针对特定任务，直接保留越来越长的历史又增加资源消耗和延迟，因此需要更系统地组织反思、记忆与行动。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2409.00872#S1"
}
]
},
{
"key": "position",
"value": "面向动态决策和长文本任务，研究反馈利用与记忆管理如何帮助 agent 持续执行，缓解有限上下文带来的能力限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2409.00872"
}
]
},
{
"key": "conclusion",
"value": "在 agent 基准和长文本任务上改善结果，小模型也能受益于反馈与记忆组织。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2409.00872"
}
]
}
],
"fields": {
"executor": "AgentBench 实验比较 GPT-3.5、GPT-4、Llama2-7B、CodeLlama-7B、Qwen-1.8B 和 ChatGLM2；商用 API 的具体版本未披露。",
"modifier": "检查、反思及记忆维护按 SAGE 的固定流程运行。论文没有逐配置说明独立检查模型的型号，不能默认它是 GPT-4。",
"object": "记忆内容、保留与裁剪状态，以及当前行为策略。",
"verdict": "检查者模型阅读回答与交互历史，给出批评供助手重试。检查者的意见与基准的最终判分是两个来源，不能自动等同。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2510.04618",
"title": "Agentic Context Engineering (ACE)",
"url": "https://arxiv.org/abs/2510.04618",
"date": "2025-10-06",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Context",
"M0",
"MemoryContent",
"org:berkeley"
],
"fields": {
"本质定位": "把 context 当 structured playbook，用 Generator→Reflector→Curator 做增量 ADD/UPDATE/REMOVE，保留有效细节并避免整段重写造成 context collapse。",
"什么在变": "structured context/playbook contents。",
"谁来改 / 谁执行": "**改**：固定 Generator/Reflector/Curator update loop。
**执行**：ReAct/agent target model（论文用 DeepSeek-V3.1 等）。",
"基础 harness": "agent + structured playbook。",
"Feedback": "可用 labels，也可用 natural execution feedback。",
"Evolution → Eval": "offline train/context optimization 与 online sequential adaptation；AppWorld 等有独立 eval setting。",
"Meta-depth": "M0/M1：context 变，curation mechanism fixed。",
"相对之前真正新增什么": "核心新点不是“有 memory”，而是把长期 context update 设计成可审计的增量结构，并系统研究 context collapse。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 292,
"fields": {
"优先级": "**K**",
"时间": "2025-10-06",
"论文": "[Agentic Context Engineering (ACE)](https://arxiv.org/abs/2510.04618)",
"本质定位": "把 context 当 structured playbook，用 Generator→Reflector→Curator 做增量 ADD/UPDATE/REMOVE，保留有效细节并避免整段重写造成 context collapse。",
"什么在变": "structured context/playbook contents。",
"谁来改 / 谁执行": "**改**：固定 Generator/Reflector/Curator update loop。
**执行**：ReAct/agent target model（论文用 DeepSeek-V3.1 等）。",
"基础 harness": "agent + structured playbook。",
"Feedback": "可用 labels，也可用 natural execution feedback。",
"Evolution → Eval": "offline train/context optimization 与 online sequential adaptation；AppWorld 等有独立 eval setting。",
"Meta-depth": "M0/M1：context 变，curation mechanism fixed。",
"相对之前真正新增什么": "核心新点不是“有 memory”，而是把长期 context update 设计成可审计的增量结构，并系统研究 context collapse。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0",
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 context 当 structured playbook，用 Generator→Reflector→Curator 做增量 ADD/UPDATE/REMOVE，保留有效细节并避免整段重写造成 context collapse。",
"novelty": "将长期经验组织为逐条可修改的操作手册，以增量方式增加、修订和合并条目，避免反复整体重写让内容越来越空泛。",
"object": "结构化上下文与操作手册中的策略条目；生成、反思和整理的外层机制保持固定。",
"executor": "主实验DeepSeek-V3.1；跨模型另测GPT-OSS-120B、GPT-5.1、Llama3.3-70B-Instruct。",
"modifier": "Generator/Reflector/Curator维护操作指南；默认基础模型为DeepSeek-V3.1。独立反思角色消融比较GPT-OSS-120B、DeepSeek-V3.1-671B、GPT-5.1，不能认为三个角色永远同模型。",
"roleContext": "**改**：固定 Generator/Reflector/Curator update loop。
**执行**：ReAct/agent target model（论文用 DeepSeek-V3.1 等）。",
"seed": "固定任务执行器外维护结构化操作指南；生成器执行，反思器提炼经验，整理器用增量改动合并规则，避免每轮全文重写。AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 使用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 等底座，其他领域有各自程序。",
"fixed": "M0/M1：context 变，curation mechanism fixed",
"verdict": "AppWorld 可仅用工具执行反馈更新指南；FiNER、Formula、DDXPlus 对照标准答案，BIRD-SQL 在本文由 GPT-4o-mini 评审。离线与在线可见反馈范围不同。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "offline train/context optimization 与 online sequential adaptation；AppWorld 等有独立 eval setting。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：离线：AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、FiNER、Formula、DDXPlus、BIRD-SQL 的原始训练划分生成操作指南。在线：此前测试题的交互经验。\n\n调试 / 选版本数据：生成角色 产出轨迹，反思角色 找出成功/失败原因，经验整理角色 局部增删条目，保留有用上下文。\n\n最终测试数据：AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-normal/test-challenge 报 TGC/SGC；FiNER、Formula、DDXPlus 报准确率；BIRD-SQL 在本文用 GPT-4o-mini 评审。\n\n数据隔离与证据边界：离线 训练集→测试集；在线按同一打乱后的 测试集 顺序，先答当前题、再更新上下文，后续题会用到前题经验。",
"cycle": "生成角色执行任务，反思角色从成败经历中提炼经验，整理角色把经验合并进操作指南。采用局部增删，避免每轮整篇重写导致已有知识丢失。",
"train": "离线：AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、FiNER、Formula、DDXPlus、BIRD-SQL 的原始训练划分生成操作指南。在线：此前测试题的交互经验。",
"debug": "生成角色 产出轨迹，反思角色 找出成功/失败原因，经验整理角色 局部增删条目，保留有用上下文。",
"test": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-normal/test-challenge 报 TGC/SGC；FiNER、Formula、DDXPlus 报准确率；BIRD-SQL 在本文用 GPT-4o-mini 评审。",
"isolation": "离线 训练集→测试集；在线按同一打乱后的 测试集 顺序，先答当前题、再更新上下文，后续题会用到前题经验。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定任务执行器外维护结构化 playbook；生成器执行，反思器提炼经验，整理器用增量改动合并规则，避免每轮全文重写。AppWorld 使用 ReAct 等底座，其他领域有各自程序。",
"protocol": "**数据：**AppWorld；金融 FiNER、Formula；医学 DDXPlus；BIRD-SQL。离线设置在原训练 split 优化 playbook，再测原 test；在线设置在任务序列中持续更新，需单列。\n\n**判分：**FiNER/Formula/DDXPlus 对标准答案算准确率，BIRD-SQL 此设置用 GPT-4o-mini 评审；AppWorld 报任务/场景完成指标。各实际采样数量本轮未核实，不把离线隔离描述套在在线结果上。",
"sections": "§4.1–4.4",
"source": "https://arxiv.org/abs/2510.04618",
"version": "2510.04618v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e9eb549511d4dea4adf973fed617d6f7c18b6828a4e316b2024bbfb4630f515c",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验DeepSeek-V3.1；跨模型另测GPT-OSS-120B、GPT-5.1、Llama3.3-70B-Instruct。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2510.04618#S4.SS5"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.04618#A1.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2510.04618#A1.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Generator/Reflector/Curator维护操作指南；默认基础模型为DeepSeek-V3.1。独立反思角色消融比较GPT-OSS-120B、DeepSeek-V3.1-671B、GPT-5.1，不能认为三个角色永远同模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2510.04618#S4.SS5"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.04618#A1.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2510.04618#A1.SS4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "结构化上下文与操作手册中的策略条目；生成、反思和整理的外层机制保持固定。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "AppWorld 可仅用工具执行反馈更新指南；FiNER、Formula、DDXPlus 对照标准答案，BIRD-SQL 在本文由 GPT-4o-mini 评审。离线与在线可见反馈范围不同。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1：Evaluation Metrics",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定任务执行器外维护结构化操作指南；生成器执行，反思器提炼经验，整理器用增量改动合并规则，避免每轮全文重写。AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 使用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 等底座，其他领域有各自程序。",
"sources": [
{
"label": "§4.1–4.4",
"url": "https://arxiv.org/abs/2510.04618"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "生成角色执行任务，反思角色从成败经历中提炼经验，整理角色把经验合并进操作指南。采用局部增删，避免每轮整篇重写导致已有知识丢失。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "离线：AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、FiNER、Formula、DDXPlus、BIRD-SQL 的原始训练划分生成操作指南。在线：此前测试题的交互经验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "生成角色 产出轨迹，反思角色 找出成功/失败原因，经验整理角色 局部增删条目，保留有用上下文。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-normal/test-challenge 报 TGC/SGC；FiNER、Formula、DDXPlus 报准确率；BIRD-SQL 在本文用 GPT-4o-mini 评审。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "离线 训练集→测试集；在线按同一打乱后的 测试集 顺序，先答当前题、再更新上下文，后续题会用到前题经验。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将长期经验组织为逐条可修改的操作手册，以增量方式增加、修订和合并条目，避免反复整体重写让内容越来越空泛。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2510.04618v3",
"version": "2510.04618v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "核心新点不是“有 memory”，而是把长期 context update 设计成可审计的增量结构，并系统研究 context collapse。",
"feedbackCases": [
{
"label": "AppWorld：上下文指南优化",
"data": "离线用原始训练划分；在线用此前已经执行的测试任务经历；最终报 test-normal / test-challenge。",
"scoring": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 官方检查目标完成（TGC）及整场景完成（SGC）。无标签消融只利用实际执行反馈，不提供标准答案。",
"visible": "生成角色 的工具执行过程交给 反思角色 找原因，再由 经验整理角色 改指南；可用标签与无标签执行反馈是不同设置。",
"use": "离线指南供后续测试；在线设置会从已完成测试题积累经验，不能说整个测试过程指南冻结。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"judgment": "AppWorld 官方程序检查任务／场景目标；无标签变体只用执行反馈"
},
{
"label": "FiNER / Formula / DDXPlus",
"data": "各自原始训练划分做离线上下文适配，测试划分评价；在线版本利用过去测试题。",
"scoring": "预测答案与数据集标准答案精确匹配，按正确比例评分。",
"visible": "训练或允许反馈的在线阶段，由回答及可用标签生成成败分析。",
"use": "反思后局部增删指南，不更新模型权重。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"judgment": "规则精确匹配预测答案与数据集标签"
},
{
"label": "BIRD-SQL",
"data": "BIRD-SQL 的训练与测试划分。",
"scoring": "本文 §4.1 明确使用 GPT-4o-mini 作为评审；不能套用其他论文的 SQL 执行正确率来替代这篇的设置。",
"visible": "输出与评审反馈用于分析，具体逐条评审提示未在该指标段给出。",
"use": "用于上下文优化及最后比较；与上面精确匹配任务分开理解。",
"sources": [
{
"label": "§4.1：Evaluation Metrics",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"judgment": "GPT-4o-mini 作 BIRD-SQL 裁判；不能默认是 SQL 结果集比较"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "离线：AppWorld（通过多个应用的程序接口完成日常事务的交互基准）、FiNER、Formula、DDXPlus、BIRD-SQL 的原始训练划分生成操作指南。在线：此前测试题的交互经验。",
"selection": "生成角色 产出轨迹，反思角色 找出成功/失败原因，经验整理角色 局部增删条目，保留有用上下文。",
"evaluation": "AppWorld（通过多个应用的程序接口完成日常事务的交互基准） test-normal/test-challenge 报 TGC/SGC；FiNER、Formula、DDXPlus 报准确率；BIRD-SQL 在本文用 GPT-4o-mini 评审。",
"isolation": "离线 训练集→测试集；在线按同一打乱后的 测试集 顺序，先答当前题、再更新上下文，后续题会用到前题经验。",
"roles": {
"executor": {
"value": "主实验DeepSeek-V3.1；跨模型另测GPT-OSS-120B、GPT-5.1、Llama3.3-70B-Instruct。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2510.04618#S4.SS5"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.04618#A1.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2510.04618#A1.SS4.SSS0.Px1"
}
]
},
"modifier": {
"value": "Generator/Reflector/Curator维护操作指南；默认基础模型为DeepSeek-V3.1。独立反思角色消融比较GPT-OSS-120B、DeepSeek-V3.1-671B、GPT-5.1，不能认为三个角色永远同模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2510.04618#S4.SS5"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.04618#A1.SS1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2510.04618#A1.SS4.SSS0.Px1"
}
]
},
"seed": {
"value": "固定任务执行器外维护结构化操作指南；生成器执行，反思器提炼经验，整理器用增量改动合并规则，避免每轮全文重写。AppWorld（通过多个应用的程序接口完成日常事务的交互基准） 使用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 等底座，其他领域有各自程序。",
"sources": [
{
"label": "§4.1–4.4",
"url": "https://arxiv.org/abs/2510.04618"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有提示优化常追求简短通用的总结，却可能删掉实际任务所需的领域规则、工具用法和失败教训；反复整体重写经验还会逐渐丢失已有信息，使表现下降。作者因此认为经验应能持续累积细节，而不是越改越像一份缺乏操作信息的摘要。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2510.04618#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 如何在持续适应任务时保留有用的领域经验，避免反复压缩和重写上下文造成知识流失。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.04618"
}
]
},
{
"key": "conclusion",
"value": "在 agent 与金融任务上改善表现并降低适应成本，避免经验在反复摘要中流失。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2510.04618"
}
]
}
],
"fields": {
"object": "结构化上下文与操作手册中的策略条目；生成、反思和整理的外层机制保持固定。",
"verdict": "AppWorld 可仅用工具执行反馈更新指南；FiNER、Formula、DDXPlus 对照标准答案，BIRD-SQL 在本文由 GPT-4o-mini 评审。离线与在线可见反馈范围不同。"
}
},
"attributions": [
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2510.04618"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2510.16079",
"title": "EvolveR: Self-Evolving LLM Agents through an Experience-Driven Lifecycle",
"url": "https://arxiv.org/abs/2510.16079",
"date": "2025-10-17",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:sjtu",
"org:shanghai-ai-lab"
],
"fields": {
"本质定位": "offline 把 trajectories 蒸馏成 abstract strategic principles，online 检索原则指导行动，并用 policy reinforcement 更新模型形成闭环。",
"什么在变": "strategic-principle repository + policy weights。",
"谁来改 / 谁执行": "**改**：固定 distillation/retrieval/RL lifecycle。
**执行**：当前 policy agent。",
"基础 harness": "tool agent + principle memory + trainable policy。",
"Feedback": "task performance / trajectory outcomes。",
"Evolution → Eval": "multi-hop QA train/eval。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "相对纯 external memory 多了 model-weight reinforcement；因此不是干净 frozen-harness evolution。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 293,
"fields": {
"优先级": "**R**",
"时间": "2025-10-17",
"论文": "[EvolveR: Self-Evolving LLM Agents through an Experience-Driven Lifecycle](https://arxiv.org/abs/2510.16079)",
"本质定位": "offline 把 trajectories 蒸馏成 abstract strategic principles，online 检索原则指导行动，并用 policy reinforcement 更新模型形成闭环。",
"什么在变": "strategic-principle repository + policy weights。",
"谁来改 / 谁执行": "**改**：固定 distillation/retrieval/RL lifecycle。
**执行**：当前 policy agent。",
"基础 harness": "tool agent + principle memory + trainable policy。",
"Feedback": "task performance / trajectory outcomes。",
"Evolution → Eval": "multi-hop QA train/eval。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "相对纯 external memory 多了 model-weight reinforcement；因此不是干净 frozen-harness evolution。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "offline 把 trajectories 蒸馏成 abstract strategic principles，online 检索原则指导行动，并用 policy reinforcement 更新模型形成闭环。",
"novelty": "将外部经验整理与模型参数的强化学习结合，让经验既能在上下文中复用，也能影响模型自身行为。",
"object": "可复用策略原则库与执行模型的参数。",
"executor": "主要以 Qwen2.5-3B 执行检索与问答，规模实验另比较 Qwen2.5-0.5B、1.5B；训练后仍由相应的更新模型答题。BGE-M3 只负责经验检索的向量表示。",
"modifier": "EvolveR 从执行轨迹中总结经验，并用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新 Qwen2.5 参数；GPT-4o-mini 是教师模型消融配置，不应写成所有实验都依赖的外部教师。",
"roleContext": "**改**：固定 distillation/retrieval/RL lifecycle。
**执行**：当前 policy agent。",
"seed": "带搜索工具的 task agent，外接可检索原则库；从经历中提炼/筛除原则，并通过训练使策略更会使用经验。外部原则与可训练行为策略共同参与，不能归纳为只改静态提示。",
"fixed": "",
"verdict": "任务表现，以及执行记录对应的成功或失败结果。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "multi-hop QA train/eval。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Natural Questions 与 HotpotQA（需要结合多份资料作答的多跳问答基准） 的训练 split 构建经验库；用经验引导的轨迹做 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。\n\n调试 / 选版本数据：离线提炼经验原则，在线检索指导搜索；回答正确性及轨迹格式共同构成奖励。\n\n最终测试数据：域内 NQ、HotpotQA（需要结合多份资料作答的多跳问答基准）；仅用于域外评估的 TriviaQA、PopQA、2WikiMultiHopQA、MuSiQue、Bamboogle。\n\n数据隔离与证据边界：§4.1.1 明确后五集仅用于泛化评估；经验库和参数学习来自前两集训练部分。",
"cycle": "将轨迹压缩为经验原则，检索后指导下一批搜索轨迹，再用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练有效利用经验的策略。",
"train": "Natural Questions 与 HotpotQA（需要结合多份资料作答的多跳问答基准） 的训练 split 构建经验库；用经验引导的轨迹做 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"debug": "离线提炼经验原则，在线检索指导搜索；回答正确性及轨迹格式共同构成奖励。",
"test": "域内 NQ、HotpotQA（需要结合多份资料作答的多跳问答基准）；仅用于域外评估的 TriviaQA、PopQA、2WikiMultiHopQA、MuSiQue、Bamboogle。",
"isolation": "§4.1.1 明确后五集仅用于泛化评估；经验库和参数学习来自前两集训练部分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "带搜索工具的任务 agent，外接可检索原则库；从经历中提炼/筛除原则，并通过训练使策略更会使用经验。外部原则与可训练 policy 共同参与，不能归纳为只改静态提示。",
"protocol": "**训练及经验构建：**Natural Questions 和 HotpotQA 的 training splits。\n\n**测试：**同域 NQ/HotpotQA；域外仅用于泛化的 TriviaQA、PopQA、2WikiMultiHopQA、Musique、Bamboogle。七套 QA 应分别区分经验来源与域外测试，逐集样本量和选 checkpoint 的 dev 设置本轮待核实。",
"sections": "评估数据与训练生命周期",
"source": "https://arxiv.org/abs/2510.16079",
"version": "2510.16079v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "42385dd7373f153a13b363e384e65547b991e87b6ee8a96b801bc9f0cd3691e6",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主要以 Qwen2.5-3B 执行检索与问答，规模实验另比较 Qwen2.5-0.5B、1.5B；训练后仍由相应的更新模型答题。BGE-M3 只负责经验检索的向量表示。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2510.16079#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "EvolveR 从执行轨迹中总结经验，并用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新 Qwen2.5 参数；GPT-4o-mini 是教师模型消融配置，不应写成所有实验都依赖的外部教师。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2510.16079#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可复用策略原则库与执行模型的参数。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "任务表现，以及执行记录对应的成功或失败结果。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "带搜索工具的 task agent，外接可检索原则库；从经历中提炼/筛除原则，并通过训练使策略更会使用经验。外部原则与可训练行为策略共同参与，不能归纳为只改静态提示。",
"sources": [
{
"label": "评估数据与训练生命周期",
"url": "https://arxiv.org/abs/2510.16079"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "将轨迹压缩为经验原则，检索后指导下一批搜索轨迹，再用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练有效利用经验的策略。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Natural Questions 与 HotpotQA（需要结合多份资料作答的多跳问答基准） 的训练 split 构建经验库；用经验引导的轨迹做 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "离线提炼经验原则，在线检索指导搜索；回答正确性及轨迹格式共同构成奖励。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "域内 NQ、HotpotQA（需要结合多份资料作答的多跳问答基准）；仅用于域外评估的 TriviaQA、PopQA、2WikiMultiHopQA、MuSiQue、Bamboogle。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4.1.1 明确后五集仅用于泛化评估；经验库和参数学习来自前两集训练部分。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将外部经验整理与模型参数的强化学习结合，让经验既能在上下文中复用，也能影响模型自身行为。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2510.16079v3",
"version": "2510.16079v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对纯 external memory 多了 model-weight reinforcement；因此不是干净 frozen-harness evolution。",
"feedbackCases": [
{
"label": "经验库与参数训练",
"data": "Natural Questions、HotpotQA（需要结合多份资料作答的多跳问答基准） 的训练划分。",
"scoring": "最终回答与标准答案精确匹配给二元结果奖励；另检查轨迹格式，构成格式奖励。",
"visible": "答案对错、格式及完整搜索轨迹。",
"use": "离线冻结模型提炼去重经验；在线检索经验并用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新参数。",
"sources": [
{
"label": "§3.3：奖励函数",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px1"
}
],
"judgment": "规则精确匹配最终答案 + 轨迹格式检查"
},
{
"label": "最终问答测试",
"data": "域内 NQ、HotpotQA（需要结合多份资料作答的多跳问答基准）；域外 TriviaQA、PopQA、2WikiMultiHopQA、MuSiQue、Bamboogle。",
"scoring": "按对应问答任务参考答案衡量最终正确性。",
"visible": "用于报告域内与域外成绩。",
"use": "域外这五套题不参与上面经验库构建与 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 训练。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"judgment": "对照问答基准参考答案计算正确性"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Natural Questions 与 HotpotQA（需要结合多份资料作答的多跳问答基准） 的训练 split 构建经验库；用经验引导的轨迹做 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"selection": "离线提炼经验原则，在线检索指导搜索；回答正确性及轨迹格式共同构成奖励。",
"evaluation": "域内 NQ、HotpotQA（需要结合多份资料作答的多跳问答基准）；仅用于域外评估的 TriviaQA、PopQA、2WikiMultiHopQA、MuSiQue、Bamboogle。",
"isolation": "§4.1.1 明确后五集仅用于泛化评估；经验库和参数学习来自前两集训练部分。",
"roles": {
"executor": {
"value": "主要以 Qwen2.5-3B 执行检索与问答，规模实验另比较 Qwen2.5-0.5B、1.5B；训练后仍由相应的更新模型答题。BGE-M3 只负责经验检索的向量表示。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2510.16079#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px8"
}
]
},
"modifier": {
"value": "EvolveR 从执行轨迹中总结经验，并用 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新 Qwen2.5 参数；GPT-4o-mini 是教师模型消融配置，不应写成所有实验都依赖的外部教师。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2510.16079#S3.SS3.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2510.16079#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2510.16079#A1.SS1.SSS0.Px8"
}
]
},
"seed": {
"value": "带搜索工具的 task agent，外接可检索原则库；从经历中提炼/筛除原则，并通过训练使策略更会使用经验。外部原则与可训练行为策略共同参与，不能归纳为只改静态提示。",
"sources": [
{
"label": "评估数据与训练生命周期",
"url": "https://arxiv.org/abs/2510.16079"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.16079#S4.SS1.SSS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 常把每次任务视为独立事件，不保留成功和失败带来的策略经验，因而会重复探索和犯错。补充外部知识不能解决这种行为上的遗忘，直接检索旧案例又难抽象出通用原则；作者希望让交互后果真正转化为可持续改进决策的经验。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2510.16079#S1"
}
]
},
{
"key": "position",
"value": "面向搜索与多跳问答，研究 agent 能否从自身行动结果中持续改进策略，使外部经验积累与参数学习相互衔接。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.16079"
}
]
},
{
"key": "conclusion",
"value": "在多跳问答中优于对照，形成经验提炼、使用和参数更新的闭环。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2510.16079"
}
]
}
],
"fields": {
"object": "可复用策略原则库与执行模型的参数。"
}
},
"attributions": [
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2510.16079"
}
]
},
{
"tag": "org:shanghai-ai-lab",
"label": "Shanghai AI Laboratory",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2510.16079"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2512.18746",
"title": "MemEvolve: Meta-Evolution of Agent Memory Systems",
"url": "https://arxiv.org/abs/2512.18746",
"date": "2025-12-21",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"CrossBenchmark",
"CrossModel",
"M1",
"MemoryContent",
"MemoryMechanism"
],
"fields": {
"本质定位": "不仅积累 memory $M_t$，还在 modular design space 中进化 memory architecture Ω：Encode / Store(Update) / Retrieve / Manage。",
"什么在变": "experiential memory + memory architecture/code。",
"谁来改 / 谁执行": "**改**：meta-evolution operator（实验常用 GPT-5-mini）生成/选择 descendant memory designs。
**执行**：SmolAgent、Flash-Searcher 等 target agents；可同/跨模型迁移。",
"基础 harness": "agent + candidate memory architecture from EvolveLab。",
"Feedback": "trajectory success、token/query/replay 等性能与效率反馈。",
"Evolution → Eval": "GAIA/WebWalkerQA/xBench/TaskCraft；含跨 task/model/framework transfer。",
"Meta-depth": "M1+：memory mechanism 变，但 outer Pareto selection/evolution protocol fixed。",
"相对之前真正新增什么": "这是 memory 谱系的实质跃迁：从“memory 中有什么”升级为“**怎么记、怎么取、怎么管理**也被优化”。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "不仅积累 memory $M_t$，还在 modular design space 中进化 memory architecture Ω：Encode / Store(Update) / Retrieve / Manage。 **相对前序：** 这是 memory 谱系的实质跃迁：从“memory 中有什么”升级为“**怎么记、怎么取、怎么管理**也被优化”。"
},
{
"label": "什么在变",
"text": "experiential memory + memory architecture/code。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** meta-evolution operator（实验常用 GPT-5-mini）生成/选择 descendant memory designs。 **执行：** SmolAgent、Flash-Searcher 等 target agents；可同/跨模型迁移。"
},
{
"label": "基础 harness",
"text": "agent + candidate memory architecture from EvolveLab。"
},
{
"label": "Feedback",
"text": "trajectory success、token/query/replay 等性能与效率反馈。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "GAIA/WebWalkerQA/xBench/TaskCraft；含跨 task/model/framework transfer。 **Meta-depth：** M1+：memory mechanism 变，但 outer Pareto selection/evolution protocol fixed。"
},
{
"label": "主要结果",
"text": "最高提升约 17.06%，且 memory designs 可跨 benchmark/backbone 迁移。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "严格 recursive 仍有限：meta-evolution algorithm 本身不开放。 **对我们：** 和 ALMA 一起是我们必须重点引用的 memory-mechanism evolution。"
}
],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 294,
"fields": {
"优先级": "**C**",
"时间": "2025-12-21",
"论文": "[MemEvolve: Meta-Evolution of Agent Memory Systems](https://arxiv.org/abs/2512.18746)",
"本质定位": "不仅积累 memory $M_t$，还在 modular design space 中进化 memory architecture Ω：Encode / Store(Update) / Retrieve / Manage。",
"什么在变": "experiential memory + memory architecture/code。",
"谁来改 / 谁执行": "**改**：meta-evolution operator（实验常用 GPT-5-mini）生成/选择 descendant memory designs。
**执行**：SmolAgent、Flash-Searcher 等 target agents；可同/跨模型迁移。",
"基础 harness": "agent + candidate memory architecture from EvolveLab。",
"Feedback": "trajectory success、token/query/replay 等性能与效率反馈。",
"Evolution → Eval": "GAIA/WebWalkerQA/xBench/TaskCraft；含跨 task/model/framework transfer。",
"Meta-depth": "M1+：memory mechanism 变，但 outer Pareto selection/evolution protocol fixed。",
"相对之前真正新增什么": "这是 memory 谱系的实质跃迁：从“memory 中有什么”升级为“**怎么记、怎么取、怎么管理**也被优化”。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 Encode、Store/Update、Retrieve、Manage 的 memory architecture 与运行时 experience 一起组织进搜索；agent 不是只增加记忆条目。",
"novelty": "把编码、存储、检索等记忆管理的机制与代码纳入搜索，改变系统怎样记忆；外层搜索和选版本程序保持固定。",
"object": "经验记忆内容，以及编码、存储、检索和管理记忆的程序；外层候选选择与评测协议保持固定。",
"executor": "主执行模型GPT-5-mini，配SmolAgent或Flash-Searcher；冻结进化后的记忆架构，再换Kimi K2或DeepSeekV3.2测迁移。",
"modifier": "主实验meta-evolution operator也使用GPT-5-mini，生成和选择记忆设计；模型与执行角色相同，但调用职责分开。",
"roleContext": "**改**：meta-evolution operator（实验常用 GPT-5-mini）生成/选择 descendant memory designs。
**执行**：SmolAgent、Flash-Searcher 等 target agents；可同/跨模型迁移。",
"seed": "在 EvolveLab 中演化记忆的编码、更新、检索和组织实现；每次评价候选架构时从空经验库开始。接入 SmolAgent、Flash-Searcher，并另测 CK-Pro 和 OWL，不是只改已有笔记的文字。",
"fixed": "M1+：memory mechanism 变，但 outer Pareto selection/evolution protocol fixed",
"verdict": "执行是否成功，以及生成文本量、查询次数和重放等开销。",
"diagnosis": "从任务执行、记忆利用及成本表现判断具体设计问题。",
"update": "改 memory 组件实现与经验表示。",
"acceptance": "按 performance / efficiency 选择 designs；保持 outer evolution protocol。",
"experiments": [
{
"name": "多任务与系统迁移",
"evolve": "用于 memory design evolution 的任务经验",
"selection": "task performance / efficiency",
"test": "跨 benchmark / model / agent system 测试",
"isolation": "迁移评估",
"note": "具体每项 split 数量尚未逐项核对。"
}
],
"takeaway": "严格 recursive 仍有限：meta-evolution algorithm 本身不开放。 **对我们：** 和 ALMA 一起是我们必须重点引用的 memory-mechanism evolution。",
"result": "最高提升约 17.06%，且 memory designs 可跨 benchmark/backbone 迁移。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2512.18746v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：TaskCraft 抽 300 题作为工作子集，其中 120 题分三轮做记忆架构进化（每轮40）。GAIA（需要检索、推理和使用工具的通用助理任务基准） 设置另用 Level-1 加 67 条 TaskCraft。\n\n调试 / 选版本数据：执行候选记忆代码，根据轨迹和成绩改造编码、存储、检索与管理模块。\n\n最终测试数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 165 题；WebWalkerQA 抽 170 题；xBench-DeepSearch 100 题；另报告 TaskCraft。\n\n数据隔离与证据边界：WebWalkerQA/xBench 使用 TaskCraft 上进化出的固定架构。GAIA（需要检索、推理和使用工具的通用助理任务基准） Level-1 参与对应架构进化，不能把全部 GAIA 称为未见测试；架构固定后记忆内容仍可更新。",
"cycle": "外层进化记忆系统代码，内层用该代码积累和检索经验；EvolveLab 支持在线和离线运行。",
"train": "TaskCraft 抽 300 题作为工作子集，其中 120 题分三轮做记忆架构进化（每轮40）。GAIA（需要检索、推理和使用工具的通用助理任务基准） 设置另用 Level-1 加 67 条 TaskCraft。",
"debug": "执行候选记忆代码，根据轨迹和成绩改造编码、存储、检索与管理模块。",
"test": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 165 题；WebWalkerQA 抽 170 题；xBench-DeepSearch 100 题；另报告 TaskCraft。",
"isolation": "WebWalkerQA/xBench 使用 TaskCraft 上进化出的固定架构。GAIA（需要检索、推理和使用工具的通用助理任务基准） Level-1 参与对应架构进化，不能把全部 GAIA 称为未见测试；架构固定后记忆内容仍可更新。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "在 EvolveLab 中演化记忆的编码、更新、检索和组织实现；每次评价候选架构时从空经验库开始。接入 SmolAgent、Flash-Searcher，并另测 CK-Pro 和 OWL，不是只改已有笔记的文字。",
"protocol": "**架构进化：**TaskCraft 上评价候选记忆系统，再保留/组合较好架构。\n\n**迁移评估：**GAIA、xBench 等任务及未用于架构进化的 CK-Pro/OWL 框架；架构固定后仍按该架构积累经验，不能把“架构冻结”误写成“记忆内容也冻结”。第三个测试集及各集实际数量本轮待核实。",
"sections": "记忆架构搜索；Agent Framework 与迁移实验",
"source": "https://arxiv.org/abs/2512.18746",
"version": "2512.18746v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "e65d209f0ae75f213ae06ac6d96bf19919b9e3408fdf6dd869b1c4b2180d668a",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主执行模型GPT-5-mini，配SmolAgent或Flash-Searcher；冻结进化后的记忆架构，再换Kimi K2或DeepSeekV3.2测迁移。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2512.18746#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "主实验meta-evolution operator也使用GPT-5-mini，生成和选择记忆设计；模型与执行角色相同，但调用职责分开。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2512.18746#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "经验记忆内容，以及编码、存储、检索和管理记忆的程序；外层候选选择与评测协议保持固定。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "执行是否成功，以及生成文本量、查询次数和重放等开销。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在 EvolveLab 中演化记忆的编码、更新、检索和组织实现；每次评价候选架构时从空经验库开始。接入 SmolAgent、Flash-Searcher，并另测 CK-Pro 和 OWL，不是只改已有笔记的文字。",
"sources": [
{
"label": "记忆架构搜索；Agent Framework 与迁移实验",
"url": "https://arxiv.org/abs/2512.18746"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "外层进化记忆系统代码，内层用该代码积累和检索经验；EvolveLab 支持在线和离线运行。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "TaskCraft 抽 300 题作为工作子集，其中 120 题分三轮做记忆架构进化（每轮40）。GAIA（需要检索、推理和使用工具的通用助理任务基准） 设置另用 Level-1 加 67 条 TaskCraft。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "执行候选记忆代码，根据轨迹和成绩改造编码、存储、检索与管理模块。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 165 题；WebWalkerQA 抽 170 题；xBench-DeepSearch 100 题；另报告 TaskCraft。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "WebWalkerQA/xBench 使用 TaskCraft 上进化出的固定架构。GAIA（需要检索、推理和使用工具的通用助理任务基准） Level-1 参与对应架构进化，不能把全部 GAIA 称为未见测试；架构固定后记忆内容仍可更新。",
"sources": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把编码、存储、检索等记忆管理的机制与代码纳入搜索，改变系统怎样记忆；外层搜索和选版本程序保持固定。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2512.18746v1",
"version": "2512.18746v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "改 memory 机制/代码的模块化设计，外层 meta-evolution 及选择程序并不因此自己进化。",
"feedbackCases": [
{
"label": "记忆架构进化：TaskCraft / GAIA",
"data": "TaskCraft 工作子集 300 题，其中 120 题分三轮进化；GAIA（需要检索、推理和使用工具的通用助理任务基准） 配置另用 Level-1 加 67 道 TaskCraft。",
"scoring": "每次任务执行汇总成功情况、文本用量和延迟三个指标。框架支持字符串匹配和模型评审，但论文没有把所有实验逐一绑定到明确的评审模型清单。",
"visible": "任务输入、状态—动作记录、终局奖励及成本供记忆设计者诊断。",
"use": "修改记忆编码、存储、检索和管理程序；不能把“支持模型评审”直接写成全部任务都由某个模型评分。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"judgment": "框架支持精确匹配和模型评审；本文未逐实验明确选用了哪一种"
},
{
"label": "最终迁移评估",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 165、WebWalkerQA 170、xBench-DeepSearch 100，另报告 TaskCraft。",
"scoring": "按各基准的问题正确性报告成功表现；正文只概述统一框架支持精确匹配/模型评审，不能由此还原每题的评分脚本。",
"visible": "最终任务表现与执行开销；在线记忆积累和离线积累后部署是两种协议。",
"use": "解释跨任务表现时需结合该实验是否继续积累记忆，不能都按完全冻结记忆库理解。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"judgment": "沿用各任务正确性评测；未逐基准披露具体裁判配置"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "TaskCraft 抽 300 题作为工作子集，其中 120 题分三轮做记忆架构进化（每轮40）。GAIA（需要检索、推理和使用工具的通用助理任务基准） 设置另用 Level-1 加 67 条 TaskCraft。",
"selection": "执行候选记忆代码，根据轨迹和成绩改造编码、存储、检索与管理模块。",
"evaluation": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 165 题；WebWalkerQA 抽 170 题；xBench-DeepSearch 100 题；另报告 TaskCraft。",
"isolation": "WebWalkerQA/xBench 使用 TaskCraft 上进化出的固定架构。GAIA（需要检索、推理和使用工具的通用助理任务基准） Level-1 参与对应架构进化，不能把全部 GAIA 称为未见测试；架构固定后记忆内容仍可更新。",
"roles": {
"executor": {
"value": "主执行模型GPT-5-mini，配SmolAgent或Flash-Searcher；冻结进化后的记忆架构，再换Kimi K2或DeepSeekV3.2测迁移。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2512.18746#S5.SS1"
}
]
},
"modifier": {
"value": "主实验meta-evolution operator也使用GPT-5-mini，生成和选择记忆设计；模型与执行角色相同，但调用职责分开。",
"sources": [
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2512.18746#S5.SS1"
}
]
},
"seed": {
"value": "在 EvolveLab 中演化记忆的编码、更新、检索和组织实现；每次评价候选架构时从空经验库开始。接入 SmolAgent、Flash-Searcher，并另测 CK-Pro 和 OWL，不是只改已有笔记的文字。",
"sources": [
{
"label": "记忆架构搜索；Agent Framework 与迁移实验",
"url": "https://arxiv.org/abs/2512.18746"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"selection": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"evaluation": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
],
"isolation": [
{
"label": "§9.1",
"url": "https://arxiv.org/html/2512.18746#S9.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有记忆系统能积累内容，却通常用人工固定的方式来抽取、保存和检索经验。适合网页操作的 API 技能库未必适合数学推理；若记忆结构本身不能随任务调整，经验增加也可能难以转化为更强能力，因此作者研究连记忆架构一起改进。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2512.18746#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否不仅积累记忆内容，还自主改进记忆系统的组织和运行方式，以适应不同任务与基础模型。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2512.18746"
}
]
},
{
"key": "conclusion",
"value": "在多个 agent 基准上改善表现，并展示学到的记忆结构跨任务和基础模型迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2512.18746"
}
]
}
],
"fields": {
"object": "经验记忆内容，以及编码、存储、检索和管理记忆的程序；外层候选选择与评测协议保持固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience",
"harness",
"improver"
]
},
{
"id": "2602.07755",
"title": "ALMA: Learning to Continually Learn via Meta-learning Agentic Memory Designs",
"url": "https://arxiv.org/abs/2602.07755",
"date": "2026-02-08",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"HeldOut",
"M1",
"MemoryContent",
"MemoryMechanism",
"StrongerBuilder",
"org:ubc",
"person:jeff-clune"
],
"fields": {
"本质定位": "Meta Agent 直接搜索 executable memory design code，包括 schema、update、retrieval；找到的设计供弱 target agent 在 sequential environment 中持续记忆。",
"什么在变": "memory design code + runtime memory contents。",
"谁来改 / 谁执行": "**改**：GPT-5 Meta Agent；target/evaluator 主设置 GPT-5-nano，另测 transfer。
**执行**：GPT-5-nano/mini 等 agent。",
"基础 harness": "minimal agent + candidate memory design implementing fixed interface。",
"Feedback": "environment reward / validation performance。",
"Evolution → Eval": "ALFWorld/TextWorld/Baba Is AI/MiniHack；memory design search 后在 unseen/sequential tasks 使用，并测 transfer。",
"Meta-depth": "M1：memory mechanism 变，meta-search fixed且常 strong-to-weak。",
"相对之前真正新增什么": "和 A-MEM 的本质区别：A-MEM 只让 memory graph 自组织；ALMA 让 **memory algorithm/code 本身**成为搜索对象。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "Meta Agent 直接搜索 executable memory design code，包括 schema、update、retrieval；找到的设计供弱 target agent 在 sequential environment 中持续记忆。 **相对前序：** 和 A-MEM 的本质区别：A-MEM 只让 memory graph 自组织；ALMA 让 **memory algorithm/code 本身**成为搜索对象。"
},
{
"label": "什么在变",
"text": "memory design code + runtime memory contents。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** GPT-5 Meta Agent；target/evaluator 主设置 GPT-5-nano，另测 transfer。 **执行：** GPT-5-nano/mini 等 agent。"
},
{
"label": "基础 harness",
"text": "minimal agent + candidate memory design implementing fixed interface。"
},
{
"label": "Feedback",
"text": "environment reward / validation performance。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "ALFWorld/TextWorld/Baba Is AI/MiniHack；memory design search 后在 unseen/sequential tasks 使用，并测 transfer。 **Meta-depth：** M1：memory mechanism 变，meta-search fixed且常 strong-to-weak。"
},
{
"label": "主要结果",
"text": "nano overall 对 no-memory +6.2，transfer 到 mini +12.8（论文汇总）。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "不是 same-model recursive RSI；builder 明显更强。 **对我们：** 评估“单个 harness component 是否能在 held-out 上真正进化”的极好参考。"
}
],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 295,
"fields": {
"优先级": "**C**",
"时间": "2026-02-08",
"论文": "[ALMA: Learning to Continually Learn via Meta-learning Agentic Memory Designs](https://arxiv.org/abs/2602.07755)",
"本质定位": "Meta Agent 直接搜索 executable memory design code，包括 schema、update、retrieval；找到的设计供弱 target agent 在 sequential environment 中持续记忆。",
"什么在变": "memory design code + runtime memory contents。",
"谁来改 / 谁执行": "**改**：GPT-5 Meta Agent；target/evaluator 主设置 GPT-5-nano，另测 transfer。
**执行**：GPT-5-nano/mini 等 agent。",
"基础 harness": "minimal agent + candidate memory design implementing fixed interface。",
"Feedback": "environment reward / validation performance。",
"Evolution → Eval": "ALFWorld/TextWorld/Baba Is AI/MiniHack；memory design search 后在 unseen/sequential tasks 使用，并测 transfer。",
"Meta-depth": "M1：memory mechanism 变，meta-search fixed且常 strong-to-weak。",
"相对之前真正新增什么": "和 A-MEM 的本质区别：A-MEM 只让 memory graph 自组织；ALMA 让 **memory algorithm/code 本身**成为搜索对象。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Meta Agent 搜索的不是“记忆里放什么”，而是 memory schema、update 与 retrieve 的 Python 实现；目标 agent 再用这个设计积累与检索经验。",
"novelty": "搜索记忆管理程序，再检查它生成的记忆能否帮助另一批任务，区分记忆机制本身和某份记忆内容的价值。",
"object": "记忆数据结构、更新和检索代码，以及运行时积累的记忆内容。",
"executor": "设计学习时GPT-5-nano执行；最终另换GPT-5-mini测迁移。测试时记忆内部语言模型统一GPT-4o-mini，embedding为text-embedding-3-small。",
"modifier": "GPT-5 Meta Agent编写记忆设计；可调用GPT-4o-mini、GPT-4.1和text-embedding-3-small构建内部流程。它不同于执行者GPT-5-nano。",
"roleContext": "**改**：GPT-5 Meta Agent；target/evaluator 主设置 GPT-5-nano，另测 transfer。
**执行**：GPT-5-nano/mini 等 agent。",
"seed": "固定任务交互 task agent，外接满足接口的候选记忆设计；meta-agent（负责设计或修改 task agent）读分层抽样的成功/失败日志来改记忆程序。每个设计经历积累与部署阶段，不是把全部日志一并塞入提示。",
"fixed": "agentic system 与设计评估协议固定；memory algorithm 是可搜索代码。",
"verdict": "先从一组任务收集记忆，再冻结记忆内容，在另一组任务中检索使用；实际任务成功率用于比较记忆设计。论文另外比较记忆量及更新模式的影响。",
"diagnosis": "根据 candidate memory design 的 collection/deployment 表现与经验继续搜索。",
"update": "修改记忆结构、更新和检索程序。",
"acceptance": "在 learning tasks 上评价 designs；最好设计进入独立测试。",
"experiments": [
{
"name": "ALFWorld / TextWorld / Baba Is AI / MiniHack",
"evolve": "learning tasks；内部再分 collection/deployment",
"selection": "static deployment success 选择 memory design",
"test": "testing tasks，也各分 collection/deployment；static 与 dynamic 两种模式",
"isolation": "独立测试",
"note": "设计的训练/测试分割与运行时 memory 更新是不同层次。"
}
],
"takeaway": "不是 same-model recursive RSI；builder 明显更强。 **对我们：** 评估“单个 harness component 是否能在 held-out 上真正进化”的极好参考。",
"result": "nano overall 对 no-memory +6.2，transfer 到 mini +12.8（论文汇总）。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2602.07755v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练集 前30题；BALROG 的 TextWorld/Baba Is AI 各取一半；MiniHack 取30%。\n\n调试 / 选版本数据：学习集再平分：前半收集记忆，后半部署评分；后半运行三次，按平均成功率选最佳设计。\n\n最终测试数据：TextWorld/Baba Is AI 剩余一半、MiniHack 剩余70%；测试材料也先收集后部署。ALFWorld（通过文字动作完成家居物体操作的交互环境） 另比较静态与动态记忆模式。\n\n数据隔离与证据边界：记忆设计学习与最终测试分开；每套材料内部又有收集/部署两阶段，不能把收集题算作计分部署题。",
"cycle": "Meta Agent 从代码档案采样记忆设计，规划和实现修改；运行失败可反思修复最多三次，按部署成功率保存与选择。",
"train": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练集 前30题；BALROG 的 TextWorld/Baba Is AI 各取一半；MiniHack 取30%。",
"debug": "学习集再平分：前半收集记忆，后半部署评分；后半运行三次，按平均成功率选最佳设计。",
"test": "TextWorld/Baba Is AI 剩余一半、MiniHack 剩余70%；测试材料也先收集后部署。ALFWorld（通过文字动作完成家居物体操作的交互环境） 另比较静态与动态记忆模式。",
"isolation": "记忆设计学习与最终测试分开；每套材料内部又有收集/部署两阶段，不能把收集题算作计分部署题。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "固定任务交互 agent，外接满足接口的候选记忆设计；meta-agent 读分层抽样的成功/失败日志来改记忆程序。每个设计经历积累与部署阶段，不是把全部日志一并塞入提示。",
"protocol": "**学习数据：**ALFWorld 用 train 前 30 题；TextWorld 和 Baba Is AI 用 BALROG 数据各半作 learning/testing；MiniHack 取 30% 作 learning。学习集内部再平分为两阶段，Deployment 每题跑三次。\n\n**测试：**在与设计学习分开的 testing 部分比较新记忆机制；不同环境划分比例不同，不能统一写成 30 道训练题。最终各测试数量本轮待核实。",
"sections": "附录 B.3–B.4",
"source": "https://arxiv.org/abs/2602.07755",
"version": "2602.07755v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f266aefec05786a80face3a8513f3520523c098262159fe80a01346fd8975387",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "设计学习时GPT-5-nano执行；最终另换GPT-5-mini测迁移。测试时记忆内部语言模型统一GPT-4o-mini，embedding为text-embedding-3-small。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-5 Meta Agent编写记忆设计；可调用GPT-4o-mini、GPT-4.1和text-embedding-3-small构建内部流程。它不同于执行者GPT-5-nano。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "记忆数据结构、更新和检索代码，以及运行时积累的记忆内容。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "先从一组任务收集记忆，再冻结记忆内容，在另一组任务中检索使用；实际任务成功率用于比较记忆设计。论文另外比较记忆量及更新模式的影响。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
},
{
"label": "附录 A.2：静态记忆评估",
"url": "https://arxiv.org/html/2602.07755#A1.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定任务交互 task agent，外接满足接口的候选记忆设计；meta-agent（负责设计或修改 task agent）读分层抽样的成功/失败日志来改记忆程序。每个设计经历积累与部署阶段，不是把全部日志一并塞入提示。",
"sources": [
{
"label": "附录 B.3–B.4",
"url": "https://arxiv.org/abs/2602.07755"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "Meta Agent 从代码档案采样记忆设计，规划和实现修改；运行失败可反思修复最多三次，按部署成功率保存与选择。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练集 前30题；BALROG 的 TextWorld/Baba Is AI 各取一半；MiniHack 取30%。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "学习集再平分：前半收集记忆，后半部署评分；后半运行三次，按平均成功率选最佳设计。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "TextWorld/Baba Is AI 剩余一半、MiniHack 剩余70%；测试材料也先收集后部署。ALFWorld（通过文字动作完成家居物体操作的交互环境） 另比较静态与动态记忆模式。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "记忆设计学习与最终测试分开；每套材料内部又有收集/部署两阶段，不能把收集题算作计分部署题。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "搜索记忆管理程序，再检查它生成的记忆能否帮助另一批任务，区分记忆机制本身和某份记忆内容的价值。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2602.07755v1",
"version": "2602.07755v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "Memory mechanism evolution，应与 A-MEM/Reflexion 的 memory-content accumulation 分开打对象标签。",
"feedbackCases": [
{
"label": "ALFWorld",
"data": "训练前 30 题；另按论文的 seen/unseen 及记忆模式评估",
"scoring": "环境检查目标完成，100 步内成功为 1，否则为 0。",
"visible": "每个学习/测试阶段内部再分成记忆收集与部署部分；部署部分读取已积累记忆，重复三次算平均表现。",
"use": "上层修改者用学习阶段的部署平均分挑选记忆设计；测试阶段使用选定设计。静态模式下部署时不继续写入记忆。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"judgment": "ALFWorld 环境规则检查目标与步数上限"
},
{
"label": "TextWorld",
"data": "BALROG 任务一半用于学习，另一半测试",
"scoring": "游戏环境按任务进度/完成情况产生奖励；不能仅凭执行结束判断获胜。",
"visible": "每个学习/测试阶段内部再分成记忆收集与部署部分；部署部分读取已积累记忆，重复三次算平均表现。",
"use": "上层修改者用学习阶段的部署平均分挑选记忆设计；测试阶段使用选定设计。静态模式下部署时不继续写入记忆。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"judgment": "TextWorld 游戏规则计算进度／完成奖励"
},
{
"label": "Baba Is AI",
"data": "任务一半用于学习，另一半测试",
"scoring": "满足当前规则下的胜利条件为 1，未完成或达到步数上限为 0。",
"visible": "每个学习/测试阶段内部再分成记忆收集与部署部分；部署部分读取已积累记忆，重复三次算平均表现。",
"use": "上层修改者用学习阶段的部署平均分挑选记忆设计；测试阶段使用选定设计。静态模式下部署时不继续写入记忆。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"judgment": "Baba Is AI 游戏规则判胜利"
},
{
"label": "MiniHack",
"data": "30% 任务学习，70% 测试",
"scoring": "环境检查关卡目标，成功为 1，未完成或超步数为 0。",
"visible": "每个学习/测试阶段内部再分成记忆收集与部署部分；部署部分读取已积累记忆，重复三次算平均表现。",
"use": "上层修改者用学习阶段的部署平均分挑选记忆设计；测试阶段使用选定设计。静态模式下部署时不继续写入记忆。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"judgment": "MiniHack 环境规则判关卡目标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练集 前30题；BALROG 的 TextWorld/Baba Is AI 各取一半；MiniHack 取30%。",
"selection": "学习集再平分：前半收集记忆，后半部署评分；后半运行三次，按平均成功率选最佳设计。",
"evaluation": "TextWorld/Baba Is AI 剩余一半、MiniHack 剩余70%；测试材料也先收集后部署。ALFWorld（通过文字动作完成家居物体操作的交互环境） 另比较静态与动态记忆模式。",
"isolation": "记忆设计学习与最终测试分开；每套材料内部又有收集/部署两阶段，不能把收集题算作计分部署题。",
"roles": {
"executor": {
"value": "设计学习时GPT-5-nano执行；最终另换GPT-5-mini测迁移。测试时记忆内部语言模型统一GPT-4o-mini，embedding为text-embedding-3-small。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
}
]
},
"modifier": {
"value": "GPT-5 Meta Agent编写记忆设计；可调用GPT-4o-mini、GPT-4.1和text-embedding-3-small构建内部流程。它不同于执行者GPT-5-nano。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
}
]
},
"seed": {
"value": "固定任务交互 task agent，外接满足接口的候选记忆设计；meta-agent（负责设计或修改 task agent）读分层抽样的成功/失败日志来改记忆程序。每个设计经历积累与部署阶段，不是把全部日志一并塞入提示。",
"sources": [
{
"label": "附录 B.3–B.4",
"url": "https://arxiv.org/abs/2602.07755"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2602.07755#S4.SS1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2602.07755#A2.SS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2602.07755#A2.SS3"
},
{
"label": "附录B.4",
"url": "https://arxiv.org/html/2602.07755#A2.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型在推理时不会自动积累跨任务经验，缺少记忆就常要从头解决相似问题。但怎样表示、保存、检索和更新记忆仍主要靠人设计，不同领域又需要不同方案，逐个手工寻找合适设计既困难又费力；作者因此研究自动学习记忆系统的设计。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2602.07755#S1"
}
]
},
{
"key": "position",
"value": "研究“怎样记忆才能更好地持续学习”能否由系统自动学出，减少顺序决策 agent 对固定人工记忆设计的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2602.07755"
}
]
},
{
"key": "conclusion",
"value": "在四类顺序决策环境中优于手工记忆设计，学习对象是“怎样记忆”，而非只增加记忆条目。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2602.07755"
}
]
}
],
"fields": {
"object": "记忆数据结构、更新和检索代码，以及运行时积累的记忆内容。",
"verdict": "先从一组任务收集记忆，再冻结记忆内容，在另一组任务中检索使用；实际任务成功率用于比较记忆设计。论文另外比较记忆量及更新模式的影响。"
}
},
"attributions": [
{
"tag": "org:ubc",
"label": "University of British Columbia",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2602.07755"
}
]
},
{
"tag": "person:jeff-clune",
"label": "Jeff Clune",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2602.07755"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience",
"improver"
]
},
{
"id": "2603.18620",
"title": "Learning to Self-Evolve",
"url": "https://arxiv.org/abs/2603.18620",
"date": "2026-03-19",
"priority": "R",
"categories": [
"methods"
],
"tags": [],
"fields": {
"本质定位": "用 RL 直接训练模型学会：根据已见任务的反馈编辑 persistent context，使后续新任务表现变好；推理时再用 tree-guided evolution 搜 context edits。",
"什么在变": "test-time persistent context + model weights（训练出 context-edit policy）。",
"谁来改 / 谁执行": "**改**：训练后由同一 policy 提出 context edits；训练阶段用 RL objective 学 updater。
**执行**：4B target/self-evolution model；可 transfer 去指导其他模型。",
"基础 harness": "task solver + editable context + tree-guided evolution loop。",
"Feedback": "seen problem feedback；每个 context edit 的 reward = downstream task performance improvement。",
"Evolution → Eval": "BIRD Text-to-SQL、MMLU-Redux；train learned edit policy，test-time seen→future-task evolution，并测跨模型 transfer。",
"Meta-depth": "M1 hybrid，接近 learned modifier；outer tree/search/reward fixed。",
"相对之前真正新增什么": "相对 ACE/传统 prompt optimizer 的关键区别是 **“如何改 context”本身被训练成模型能力**：edit 的 reward 不是当前题变好，而是 edit 后对 downstream tasks 的性能增益。"
},
"details": [],
"occurrences": [
{
"section": "A2. Context / Memory Evolution",
"line": 296,
"fields": {
"优先级": "**R**",
"时间": "2026-03-19",
"论文": "[Learning to Self-Evolve](https://arxiv.org/abs/2603.18620)",
"本质定位": "用 RL 直接训练模型学会：根据已见任务的反馈编辑 persistent context，使后续新任务表现变好；推理时再用 tree-guided evolution 搜 context edits。",
"什么在变": "test-time persistent context + model weights（训练出 context-edit policy）。",
"谁来改 / 谁执行": "**改**：训练后由同一 policy 提出 context edits；训练阶段用 RL objective 学 updater。
**执行**：4B target/self-evolution model；可 transfer 去指导其他模型。",
"基础 harness": "task solver + editable context + tree-guided evolution loop。",
"Feedback": "seen problem feedback；每个 context edit 的 reward = downstream task performance improvement。",
"Evolution → Eval": "BIRD Text-to-SQL、MMLU-Redux；train learned edit policy，test-time seen→future-task evolution，并测跨模型 transfer。",
"Meta-depth": "M1 hybrid，接近 learned modifier；outer tree/search/reward fixed。",
"相对之前真正新增什么": "相对 ACE/传统 prompt optimizer 的关键区别是 **“如何改 context”本身被训练成模型能力**：edit 的 reward 不是当前题变好，而是 edit 后对 downstream tasks 的性能增益。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Mem"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "用 RL 直接训练模型学会：根据已见任务的反馈编辑 persistent context，使后续新任务表现变好；推理时再用 tree-guided evolution 搜 context edits。",
"novelty": "专门训练修改上下文的模型，用修改后在后续任务上的表现作为奖励，学习怎样写出未来会有用的经验。",
"object": "训练时更新模型参数以学会上下文编辑；部署时持续修改外部上下文。",
"executor": "主执行者Qwen3-4B-Instruct，权重固定；外部指令由单独训练的编辑策略更新。",
"modifier": "编辑策略同样从Qwen3-4B-Instruct开始，但经过LSE强化训练；主执行模型不随编辑策略一起训练。迁移时固定这份LSE编辑策略指导其他执行模型。",
"roleContext": "**改**：训练后由同一 policy 提出 context edits；训练阶段用 RL objective 学 updater。
**执行**：4B target/self-evolution model；可 transfer 去指导其他模型。",
"seed": "Qwen3-4B-Instruct执行任务，另一自进化策略改指令；外围代码固定采样、评分及UCB树搜索。SQL任务执行查询检验答案，QA为四选一回答。",
"fixed": "outer tree/search/reward fixed",
"verdict": "编辑者能看到已做题的题目、模型输出、标准答案和逐题正确性；比较修改上下文前后任务表现的增量，训练怎样改上下文。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "BIRD Text-to-SQL、MMLU-Redux；train learned edit policy，test-time seen→future-task evolution，并测跨模型 transfer。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SQL 编辑策略在 BIRD 训练集 学习；QA 编辑策略在 SuperGPQA 学习。每领域200次生成运行×20轮，约4000个树节点供 强化学习（根据奖励调整模型行为） 采样。\n\n调试 / 选版本数据：部署进化每轮抽10题；每领域固定50题作选路径评分集，每题生成8次，运行25轮。\n\n最终测试数据：BIRD Mini-Dev 的5个数据库、MMLU-Redux 的10个学科；报告进化中最佳的50题评分集成绩。\n\n数据隔离与证据边界：这里的 holdout 会被反复打分并用于树搜索和挑版本，因此报告的是适应后最佳验证表现，不能称最终不可见测试。",
"cycle": "固定任务执行模型，训练另一个编辑策略修改它的提示。奖励衡量修改前后的表现增量；部署时在不同修改分支中搜索，必要时回到较好的旧版本，减少坏修改累积。",
"train": "SQL 编辑策略在 BIRD 训练集 学习；QA 编辑策略在 SuperGPQA 学习。每领域200次生成运行×20轮，约4000个树节点供 强化学习（根据奖励调整模型行为） 采样。",
"debug": "部署进化每轮抽10题；每领域固定50题作选路径评分集，每题生成8次，运行25轮。",
"test": "BIRD Mini-Dev 的5个数据库、MMLU-Redux 的10个学科；报告进化中最佳的50题评分集成绩。",
"isolation": "这里的 holdout 会被反复打分并用于树搜索和挑版本，因此报告的是适应后最佳验证表现，不能称最终不可见测试。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "任务 solver 配合可编辑外部上下文，树式进化循环比较经验更新路径。保存的状态可影响后续任务，变化在外部上下文而非只重写当前题答案；初始工具清单本轮待核实。",
"protocol": "**评测：**BIRD 的 Financial 106、Toxicology 145、Card Games 191、Formula 1 174、Codebase 186 题；MMLU-Redux 多个专业子域约 95–100 题/域。\n\n**协议：**研究部署后的跨题自演化，需区分前题产生的经验和后题表现。用于选择进化路径的验证题与这些报告集的重叠本轮未核实，不因存在树搜索就默认有独立训练集。",
"sections": "实验设置、Table 4",
"source": "https://arxiv.org/abs/2603.18620",
"version": "2603.18620v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "b2c0ec929da26e010edbce6f89e8ba38e13436e89f225ee6357477ebc2aca8e1",
"seedStatus": "partial",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主执行者Qwen3-4B-Instruct，权重固定；外部指令由单独训练的编辑策略更新。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.18620#S4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "编辑策略同样从Qwen3-4B-Instruct开始，但经过LSE强化训练；主执行模型不随编辑策略一起训练。迁移时固定这份LSE编辑策略指导其他执行模型。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.18620#S4.SS3.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "训练时更新模型参数以学会上下文编辑；部署时持续修改外部上下文。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "编辑者能看到已做题的题目、模型输出、标准答案和逐题正确性；比较修改上下文前后任务表现的增量，训练怎样改上下文。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
},
{
"label": "§3.2：可见反馈",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "附录 A：评价协议",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
},
{
"label": "§4.1：任务与指标",
"url": "https://arxiv.org/html/2603.18620#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Qwen3-4B-Instruct执行任务，另一自进化策略改指令；外围代码固定采样、评分及UCB树搜索。SQL任务执行查询检验答案，QA为四选一回答。",
"sources": [
{
"label": "实验设置、Table 4",
"url": "https://arxiv.org/abs/2603.18620"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "固定任务执行模型，训练另一个编辑策略修改它的提示。奖励衡量修改前后的表现增量；部署时在不同修改分支中搜索，必要时回到较好的旧版本，减少坏修改累积。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SQL 编辑策略在 BIRD 训练集 学习；QA 编辑策略在 SuperGPQA 学习。每领域200次生成运行×20轮，约4000个树节点供 强化学习（根据奖励调整模型行为） 采样。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "部署进化每轮抽10题；每领域固定50题作选路径评分集，每题生成8次，运行25轮。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "BIRD Mini-Dev 的5个数据库、MMLU-Redux 的10个学科；报告进化中最佳的50题评分集成绩。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这里的 holdout 会被反复打分并用于树搜索和挑版本，因此报告的是适应后最佳验证表现，不能称最终不可见测试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "专门训练修改上下文的模型，用修改后在后续任务上的表现作为奖励，学习怎样写出未来会有用的经验。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.18620v1",
"version": "2603.18620v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 ACE/传统 prompt optimizer 的关键区别是 **“如何改 context”本身被训练成模型能力**：edit 的 reward 不是当前题变好，而是 edit 后对 downstream tasks 的性能增益。",
"feedbackCases": [
{
"label": "SQL：上下文编辑学习与部署",
"data": "在 BIRD 训练集 训练编辑策略；部署评价用 BIRD Mini-Dev 五个数据库。每领域训练约 200 次×20 轮；部署每轮抽 10 题，固定 50 题评分、每题生成 8 次，共 25 轮。",
"scoring": "执行生成 SQL，按查询结果正确性评分。",
"visible": "§3.2 的摘要明确含题目、模型回答、标准答案、逐题正确性；不是只有一个总分。",
"use": "训练奖励是修改上下文前后表现的增量；最终报告进化中最好的 50 题评分成绩，该集合参与选择，不是选完后的第三套独立测试。",
"sources": [
{
"label": "§3.2：可见反馈",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "附录 A：评价协议",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
},
{
"label": "§4.1：任务与指标",
"url": "https://arxiv.org/html/2603.18620#S4.SS1"
}
],
"judgment": "执行 SQL 后检查查询结果正确性"
},
{
"label": "问答：上下文编辑学习与部署",
"data": "在 SuperGPQA 训练编辑策略；部署评价用 MMLU-Redux 十个学科。每领域训练约 200 次×20 轮；部署每轮抽 10 题，固定 50 题评分、每题生成 8 次，共 25 轮。",
"scoring": "对照标准答案判断回答正确与否。",
"visible": "§3.2 的摘要明确含题目、模型回答、标准答案、逐题正确性；不是只有一个总分。",
"use": "训练奖励是修改上下文前后表现的增量；最终报告进化中最好的 50 题评分成绩，该集合参与选择，不是选完后的第三套独立测试。",
"sources": [
{
"label": "§3.2：可见反馈",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "附录 A：评价协议",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
},
{
"label": "§4.1：任务与指标",
"url": "https://arxiv.org/html/2603.18620#S4.SS1"
}
],
"judgment": "问答参考答案判分；本文未展开规则或模型比对实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "SQL 编辑策略在 BIRD 训练集 学习；QA 编辑策略在 SuperGPQA 学习。每领域200次生成运行×20轮，约4000个树节点供 强化学习（根据奖励调整模型行为） 采样。",
"selection": "部署进化每轮抽10题；每领域固定50题作选路径评分集，每题生成8次，运行25轮。",
"evaluation": "BIRD Mini-Dev 的5个数据库、MMLU-Redux 的10个学科；报告进化中最佳的50题评分集成绩。",
"isolation": "这里的 holdout 会被反复打分并用于树搜索和挑版本，因此报告的是适应后最佳验证表现，不能称最终不可见测试。",
"roles": {
"executor": {
"value": "主执行者Qwen3-4B-Instruct，权重固定；外部指令由单独训练的编辑策略更新。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.18620#S4.SS3.SSS0.Px3"
}
]
},
"modifier": {
"value": "编辑策略同样从Qwen3-4B-Instruct开始，但经过LSE强化训练；主执行模型不随编辑策略一起训练。迁移时固定这份LSE编辑策略指导其他执行模型。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.18620#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2603.18620#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.18620#S4.SS3.SSS0.Px3"
}
]
},
"seed": {
"value": "Qwen3-4B-Instruct执行任务，另一自进化策略改指令；外围代码固定采样、评分及UCB树搜索。SQL任务执行查询检验答案，QA为四选一回答。",
"sources": [
{
"label": "实验设置、Table 4",
"url": "https://arxiv.org/abs/2603.18620"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.18620#S4.SS1.SSS0.Px4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2603.18620#A1.SS0.SSS0.Px3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有方法让模型根据成绩改写提示或经验，但通常直接依赖模型原有能力，没有训练它怎样做好这项修改。判断哪段经验有害、预估修改后果、选择继续完善还是尝试新方向，本身是专门的推理任务；若不培养这些能力，自进化效果就受限于模型偶然能否提出好修改。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.18620#S1"
}
]
},
{
"key": "position",
"value": "研究利用经验修改上下文的能力能否通过专门训练获得，使模型在部署后更有效地从已做任务适应新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.18620"
}
]
},
{
"key": "conclusion",
"value": "训练后的 4B 模型在问答与 SQL 任务中表现有竞争力，且能迁移去指导其他模型。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.18620"
}
]
}
],
"fields": {
"object": "训练时更新模型参数以学会上下文编辑；部署时持续修改外部上下文。",
"verdict": "编辑者能看到已做题的题目、模型输出、标准答案和逐题正确性；比较修改上下文前后任务表现的增量，训练怎样改上下文。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver",
"experience",
"weights"
]
},
{
"id": "2402.17574",
"title": "Agent-Pro: Learning to Evolve via Policy-Level Reflection and Optimization",
"url": "https://arxiv.org/abs/2402.17574",
"date": "2024-02-27",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Prompt"
],
"fields": {
"本质定位": "在 Blackjack/Texas Hold’em 中从 trajectories 形成 beliefs，再反思不合理 belief，提炼 behavioral guideline/world model；DFS 搜索 candidate policy，并在新 trajectories 评估。",
"什么在变": "natural-language beliefs / behavioral policy。",
"谁来改 / 谁执行": "**改**：固定 reflection + DFS policy optimization。
**执行**：GPT-3.5/GPT-4/Llama2-70B game agents。",
"基础 harness": "game agent + belief/policy memory。",
"Feedback": "game payoff、trajectory、belief verification；policy eval 用新游戏轨迹。",
"Evolution → Eval": "Blackjack / Texas Hold’em；有 novel-trajectory policy evaluation。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相对 action-level Reflexion，更强调 **policy-level belief/strategy** 的持久更新与 evaluation-on-novel-trajectories。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 303,
"fields": {
"优先级": "**R**",
"时间": "2024-02-27",
"论文": "[Agent-Pro: Learning to Evolve via Policy-Level Reflection and Optimization](https://arxiv.org/abs/2402.17574)",
"本质定位": "在 Blackjack/Texas Hold’em 中从 trajectories 形成 beliefs，再反思不合理 belief，提炼 behavioral guideline/world model；DFS 搜索 candidate policy，并在新 trajectories 评估。",
"什么在变": "natural-language beliefs / behavioral policy。",
"谁来改 / 谁执行": "**改**：固定 reflection + DFS policy optimization。
**执行**：GPT-3.5/GPT-4/Llama2-70B game agents。",
"基础 harness": "game agent + belief/policy memory。",
"Feedback": "game payoff、trajectory、belief verification；policy eval 用新游戏轨迹。",
"Evolution → Eval": "Blackjack / Texas Hold’em；有 novel-trajectory policy evaluation。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相对 action-level Reflexion，更强调 **policy-level belief/strategy** 的持久更新与 evaluation-on-novel-trajectories。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2024",
"depth": [
"M0",
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "在 Blackjack/Texas Hold’em 中从 trajectories 形成 beliefs，再反思不合理 belief，提炼 behavioral guideline/world model；DFS 搜索 candidate policy，并在新 trajectories 评估。",
"novelty": "从执行经历调整对环境的判断和较长期的行动策略，再在新的执行轨迹上评价，而不局限于纠正某一个动作。",
"object": "自然语言表示的信念与整体行为策略。",
"executor": "GPT-3.5-Turbo-0613、GPT4-0613、Llama2-Chat-70B和Qwen-72B构成游戏 task agent。",
"modifier": "对应游戏 task agent 的语言模型执行策略级反思并改提示；固定DFS程序按牌局收益搜索，不另训练一个策略修改模型。",
"roleContext": "**改**：固定 reflection + DFS policy optimization。
**执行**：GPT-3.5/GPT-4/Llama2-70B game agents。",
"seed": "游戏 task agent 在环境中行动并更新关于对手与策略的文字信念，反思针对整套策略而非单步动作；模型权重固定，策略经验随对局变化。",
"fixed": "",
"verdict": "游戏收益、逐步执行记录，以及对文字信念的检查；策略效果用新的游戏对局评估。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "Blackjack / Texas Hold’em；有 novel-trajectory policy evaluation。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：德州扑克先抽500组牌，挑出167组困难失败牌用于学习。\n\n调试 / 选版本数据：另20组牌作策略开发集；反思错误信念后用 DFS 搜索提示策略。\n\n最终测试数据：另抽100组新牌，交换四位玩家的手牌和位置形成每组16种安排，共1600局，以平均筹码评价。\n\n数据隔离与证据边界：1600局来自100组牌的排列，不是1600组独立采样手牌。",
"cycle": "记录自身策略与对游戏环境的判断，从整局失败提炼跨局可用的指导，再通过深度优先搜索比较候选提示带来的筹码收益。",
"train": "德州扑克先抽500组牌，挑出167组困难失败牌用于学习。",
"debug": "另20组牌作策略开发集；反思错误信念后用 DFS 搜索提示策略。",
"test": "另抽100组新牌，交换四位玩家的手牌和位置形成每组16种安排，共1600局，以平均筹码评价。",
"isolation": "1600局来自100组牌的排列，不是1600组独立采样手牌。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "游戏 agent 在环境中行动并更新关于对手与策略的文字信念，反思针对整套策略而非单步动作；模型权重固定，策略经验随对局变化。",
"protocol": "**学习和选策略：**通过牌局反馈反思、修订策略；具体训练/验证手牌数量本轮待核实。\n\n**最终测试：**另采样 100 组新手牌，交换玩家手牌及位置形成每组 16 种排列，共 1,600 局，按平均筹码比较。不能把 1,600 当作独立抽样的手牌数。",
"sections": "游戏评估指标；附录 B1",
"source": "https://arxiv.org/abs/2402.17574",
"version": "2402.17574v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "9fc344ff026ff1f0f190cb37e2188347eb194890c3a2aa5535c22fd461ff459b",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-3.5-Turbo-0613、GPT4-0613、Llama2-Chat-70B和Qwen-72B构成游戏 task agent。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2402.17574#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应游戏 task agent 的语言模型执行策略级反思并改提示；固定DFS程序按牌局收益搜索，不另训练一个策略修改模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2402.17574#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "自然语言表示的信念与整体行为策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "游戏收益、逐步执行记录，以及对文字信念的检查；策略效果用新的游戏对局评估。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "游戏 task agent 在环境中行动并更新关于对手与策略的文字信念，反思针对整套策略而非单步动作；模型权重固定，策略经验随对局变化。",
"sources": [
{
"label": "游戏评估指标；附录 B1",
"url": "https://arxiv.org/abs/2402.17574"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "记录自身策略与对游戏环境的判断，从整局失败提炼跨局可用的指导，再通过深度优先搜索比较候选提示带来的筹码收益。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "德州扑克先抽500组牌，挑出167组困难失败牌用于学习。",
"sources": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "另20组牌作策略开发集；反思错误信念后用 DFS 搜索提示策略。",
"sources": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "另抽100组新牌，交换四位玩家的手牌和位置形成每组16种安排，共1600局，以平均筹码评价。",
"sources": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "1600局来自100组牌的排列，不是1600组独立采样手牌。",
"sources": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从执行经历调整对环境的判断和较长期的行动策略，再在新的执行轨迹上评价，而不局限于纠正某一个动作。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2402.17574v3",
"version": "2402.17574v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 action-level Reflexion，更强调 **policy-level belief/strategy** 的持久更新与 evaluation-on-novel-trajectories。",
"feedbackCases": [
{
"label": "德州扑克：策略学习与开发",
"data": "抽 500 组牌，选 167 组困难失败牌学习；另 20 组作策略开发集。",
"scoring": "游戏规则计算最终筹码收益；模型另检查自己在对局中形成的文字信念是否有误。",
"visible": "完整对局、信念序列、输赢与筹码分数。",
"use": "依据失败反思生成行为指导，用开发对局比较搜索到的提示策略；不更新模型参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"judgment": "游戏规则结算筹码；模型另反思自己的文字信念"
},
{
"label": "德州扑克：最终测试",
"data": "另抽 100 组新牌；每组交换四位玩家手牌和位置形成 16 种安排，共 1,600 局。",
"scoring": "游戏结算后计算平均筹码。",
"visible": "最终实验统计。",
"use": "新牌与位置交换用于降低运气、座次影响，不是继续在 167 组失败牌上打分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"judgment": "游戏规则结算最终筹码"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "德州扑克先抽500组牌，挑出167组困难失败牌用于学习。",
"selection": "另20组牌作策略开发集；反思错误信念后用 DFS 搜索提示策略。",
"evaluation": "另抽100组新牌，交换四位玩家的手牌和位置形成每组16种安排，共1600局，以平均筹码评价。",
"isolation": "1600局来自100组牌的排列，不是1600组独立采样手牌。",
"roles": {
"executor": {
"value": "GPT-3.5-Turbo-0613、GPT4-0613、Llama2-Chat-70B和Qwen-72B构成游戏 task agent。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2402.17574#A1.SS1"
}
]
},
"modifier": {
"value": "对应游戏 task agent 的语言模型执行策略级反思并改提示；固定DFS程序按牌局收益搜索，不另训练一个策略修改模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2402.17574#S3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2402.17574#A1.SS1"
}
]
},
"seed": {
"value": "游戏 task agent 在环境中行动并更新关于对手与策略的文字信念，反思针对整套策略而非单步动作；模型权重固定，策略经验随对局变化。",
"sources": [
{
"label": "游戏评估指标；附录 B1",
"url": "https://arxiv.org/abs/2402.17574"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"selection": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"evaluation": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
],
"isolation": [
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2402.17574#A1.SS4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "面对信息不完全、环境和对手会变化的任务，人工提示很难预先规定足够好的策略。多数语言模型 agent 又不能在交互中持续吸收经验、改进自己的行为规则，因而难像人一样越玩越熟练；作者研究能否通过与环境互动来学习和提升策略。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2402.17574#S1"
}
]
},
{
"key": "position",
"value": "面向动态博弈，研究 agent 能否从跨局经历中改进整体行为策略，而不只修正当前一步动作或依赖手工规则。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2402.17574"
}
]
},
{
"key": "conclusion",
"value": "在 Blackjack 和德州扑克中优于对照，反思目标从单个动作上升到跨局使用的策略。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2402.17574"
}
]
}
],
"fields": {
"object": "自然语言表示的信念与整体行为策略。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2510.23601",
"title": "Alita-G: Self-Evolving Generative Agent for Agent Generation",
"url": "https://arxiv.org/abs/2510.23601",
"date": "2025-10-27",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"Skill",
"person:mengdi-wang"
],
"fields": {
"本质定位": "从成功 trajectory 合成 candidate MCP tools，抽象/参数化后合并进 MCP Box；后续任务检索这些工具。",
"什么在变": "MCP tool library。",
"谁来改 / 谁执行": "**改**：manager/generator LLM（论文用 Claude Sonnet 4 等组合）生成、抽象、curate tools。
**执行**：generalist web/agent executor。",
"基础 harness": "generalist agent + MCP Box。",
"Feedback": "成功 trajectory、task pass/fail、execution evidence。",
"Evolution → Eval": "GAIA 等；主 GAIA generation 与 evaluation 对同 validation set 有 adaptive reuse，另有 PathVQA/HLE。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相对 Voyager 把 executable skill 进一步标准化为可移植 MCP tool；但主要从成功经验生成，不系统利用失败诊断。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 304,
"fields": {
"优先级": "**K**",
"时间": "2025-10-27",
"论文": "[Alita-G: Self-Evolving Generative Agent for Agent Generation](https://arxiv.org/abs/2510.23601)",
"本质定位": "从成功 trajectory 合成 candidate MCP tools，抽象/参数化后合并进 MCP Box；后续任务检索这些工具。",
"什么在变": "MCP tool library。",
"谁来改 / 谁执行": "**改**：manager/generator LLM（论文用 Claude Sonnet 4 等组合）生成、抽象、curate tools。
**执行**：generalist web/agent executor。",
"基础 harness": "generalist agent + MCP Box。",
"Feedback": "成功 trajectory、task pass/fail、execution evidence。",
"Evolution → Eval": "GAIA 等；主 GAIA generation 与 evaluation 对同 validation set 有 adaptive reuse，另有 PathVQA/HLE。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相对 Voyager 把 executable skill 进一步标准化为可移植 MCP tool；但主要从成功经验生成，不系统利用失败诊断。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"M0",
"M1"
],
"priorityBasis": "用户指定阅读重点：Mengdi Wang（王梦迪，Princeton）署名论文。",
"prioritySources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2510.23601"
}
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "从成功 trajectory 合成 candidate MCP tools，抽象/参数化后合并进 MCP Box；后续任务检索这些工具。",
"novelty": "把成功经验封装为可移植工具，采用统一工具连接协议供后续任务调用，让技能更容易跨环境复用。",
"object": "可检索并执行的 MCP（让 agent 以统一接口连接外部工具的协议） 工具库。",
"executor": "管理角色为Claude Sonnet4，负责协调；Web Agent为GPT-4.1，负责检索和网页交互；MCP（让 agent 以统一接口连接外部工具的协议）检索embedding用text-embedding-3-large。",
"modifier": "Manager/generator沿Alita流程把成功执行抽象成MCP（让 agent 以统一接口连接外部工具的协议）工具，并按相似度选择复用；§4.1的角色配对是Claude Sonnet4与GPT-4.1，不能笼统写“同一 task agent”。",
"roleContext": "**改**：manager/generator LLM（论文用 Claude Sonnet 4 等组合）生成、抽象、curate tools。
**执行**：generalist web/agent executor。",
"seed": "先由管理 agent 把成功执行过程整理成通用工具：把写死的输入改为参数，去掉题目专属信息，统一调用接口。这些工具组成 MCP Box（按统一工具协议连接的工具库）；后续 agent 检索工具，通过生成并运行代码完成任务。",
"fixed": "",
"verdict": "成功执行的记录、任务是否通过，以及实际执行证据。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "GAIA 等；主 GAIA generation 与 evaluation 对同 validation set 有 adaptive reuse，另有 PathVQA/HLE。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Master task agent 完成目标任务，把可复用工具及调用经验整理成 MCP（让 agent 以统一接口连接外部工具的协议） box；比较单次与三次执行构造。\n\n调试 / 选版本数据：根据新任务与 MCP（让 agent 以统一接口连接外部工具的协议） 描述的 embedding 相似度筛选，实验阈值0.7；不是更新模型参数。\n\n最终测试数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 完整 验证集；PathVQA 随机100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 随机100题；报告准确率及 词元 开销。\n\n数据隔离与证据边界：§4.1 指明评测抽样，但没有声明 MCP（让 agent 以统一接口连接外部工具的协议） 构造题与报告题互斥；不能据此称严格未见迁移。",
"cycle": "Alita 的 管理角色（Claude Sonnet 4）协调、Web Agent（GPT-4.1）检索；从成功执行沉淀 MCP（让 agent 以统一接口连接外部工具的协议），再检索组装专门 task agent。",
"train": "Master task agent 完成目标任务，把可复用工具及调用经验整理成 MCP（让 agent 以统一接口连接外部工具的协议） box；比较单次与三次执行构造。",
"debug": "根据新任务与 MCP（让 agent 以统一接口连接外部工具的协议） 描述的 embedding 相似度筛选，实验阈值0.7；不是更新模型参数。",
"test": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 完整 验证集；PathVQA 随机100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 随机100题；报告准确率及 词元 开销。",
"isolation": "§4.1 指明评测抽样，但没有声明 MCP（让 agent 以统一接口连接外部工具的协议） 构造题与报告题互斥；不能据此称严格未见迁移。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "Master Agent 先从成功轨迹生成工具，泛化参数、去除特定上下文并统一接口，组成 MCP Box；专门 agent 再按任务检索工具，以 CodeAct 循环执行。起点已有工具生成与检索框架。",
"protocol": "**数据/任务：**GAIA、PathVQA、HLE。PathVQA 随机取 100 个病理图像问答样本用于评估。\n\n**经验来源：**从目标任务成功轨迹提炼 MCP，之后构造领域 agent。用于生成 MCP 的任务和最终报告题是否互斥、GAIA/HLE 实际划分与数量本轮未核实，不把“领域专门化”自动视为独立留出迁移。",
"sections": "MCP 生成流程与实验数据",
"source": "https://arxiv.org/abs/2510.23601",
"version": "2510.23601v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c499b9ec25e4f6ffc861e67c307e96e69e7725b36353eacb423e16ee13f0c878",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "管理角色为Claude Sonnet4，负责协调；Web Agent为GPT-4.1，负责检索和网页交互；MCP（让 agent 以统一接口连接外部工具的协议）检索embedding用text-embedding-3-large。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Manager/generator沿Alita流程把成功执行抽象成MCP（让 agent 以统一接口连接外部工具的协议）工具，并按相似度选择复用；§4.1的角色配对是Claude Sonnet4与GPT-4.1，不能笼统写“同一 task agent”。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可检索并执行的 MCP（让 agent 以统一接口连接外部工具的协议） 工具库。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "成功执行的记录、任务是否通过，以及实际执行证据。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "先由管理 agent 把成功执行过程整理成通用工具：把写死的输入改为参数，去掉题目专属信息，统一调用接口。这些工具组成 MCP Box（按统一工具协议连接的工具库）；后续 agent 检索工具，通过生成并运行代码完成任务。",
"sources": [
{
"label": "MCP 生成流程与实验数据",
"url": "https://arxiv.org/abs/2510.23601"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "Alita 的 管理角色（Claude Sonnet 4）协调、Web Agent（GPT-4.1）检索；从成功执行沉淀 MCP（让 agent 以统一接口连接外部工具的协议），再检索组装专门 task agent。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Master task agent 完成目标任务，把可复用工具及调用经验整理成 MCP（让 agent 以统一接口连接外部工具的协议） box；比较单次与三次执行构造。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "根据新任务与 MCP（让 agent 以统一接口连接外部工具的协议） 描述的 embedding 相似度筛选，实验阈值0.7；不是更新模型参数。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 完整 验证集；PathVQA 随机100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 随机100题；报告准确率及 词元 开销。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4.1 指明评测抽样，但没有声明 MCP（让 agent 以统一接口连接外部工具的协议） 构造题与报告题互斥；不能据此称严格未见迁移。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把成功经验封装为可移植工具，采用统一工具连接协议供后续任务调用，让技能更容易跨环境复用。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2510.23601v1",
"version": "2510.23601v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Voyager 把 executable skill 进一步标准化为可移植 MCP tool；但主要从成功经验生成，不系统利用失败诊断。",
"feedbackCases": [
{
"label": "生成工具与复用",
"data": "Master 执行目标领域任务，把成功过程抽象成可复用 MCP（让 agent 以统一接口连接外部工具的协议） 工具；比较 1 轮和 3 轮构建的工具箱。",
"scoring": "利用任务执行成功的轨迹抽象工具。论文未提供可据以还原每个候选工具验收的统一测试集、独立裁判及逐工具分数。",
"visible": "执行过程、工具调用及生成的参数化工具；检索时 embedding 相似度阈值 0.7 仅表示相关程度。",
"use": "成功轨迹用于生成工具，相似度用于选工具；不能把 0.7 当作任务正确率或验收门槛。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"judgment": "任务执行成功记录用于抽象工具；未公开统一工具验收器"
},
{
"label": "最终问答评价",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 完整 验证集；PathVQA 随机 100 题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 随机 100 题。",
"scoring": "报告答案准确率及 词元 消耗。正文未逐一明确这三组准确率的裁判实现与评审模型，不能自行补成某个 语言模型 judge（依据指定要求或参考答案评审输出的语言模型）。",
"visible": "各工具箱配置的答对数、调用次数和开销。",
"use": "比较工具复用对成品表现的影响；原文披露不足以把每题得分如何转化成工具修改完整重建。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"judgment": "报告答案准确率，但未逐基准披露规则或模型判分实现"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Master task agent 完成目标任务，把可复用工具及调用经验整理成 MCP（让 agent 以统一接口连接外部工具的协议） box；比较单次与三次执行构造。",
"selection": "根据新任务与 MCP（让 agent 以统一接口连接外部工具的协议） 描述的 embedding 相似度筛选，实验阈值0.7；不是更新模型参数。",
"evaluation": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 完整 验证集；PathVQA 随机100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 随机100题；报告准确率及 词元 开销。",
"isolation": "§4.1 指明评测抽样，但没有声明 MCP（让 agent 以统一接口连接外部工具的协议） 构造题与报告题互斥；不能据此称严格未见迁移。",
"roles": {
"executor": {
"value": "管理角色为Claude Sonnet4，负责协调；Web Agent为GPT-4.1，负责检索和网页交互；MCP（让 agent 以统一接口连接外部工具的协议）检索embedding用text-embedding-3-large。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
]
},
"modifier": {
"value": "Manager/generator沿Alita流程把成功执行抽象成MCP（让 agent 以统一接口连接外部工具的协议）工具，并按相似度选择复用；§4.1的角色配对是Claude Sonnet4与GPT-4.1，不能笼统写“同一 task agent”。",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
]
},
"seed": {
"value": "先由管理 agent 把成功执行过程整理成通用工具：把写死的输入改为参数，去掉题目专属信息，统一调用接口。这些工具组成 MCP Box（按统一工具协议连接的工具库）；后续 agent 检索工具，通过生成并运行代码完成任务。",
"sources": [
{
"label": "MCP 生成流程与实验数据",
"url": "https://arxiv.org/abs/2510.23601"
},
{
"label": "CodeAct 原论文：可执行代码动作",
"url": "https://arxiv.org/abs/2402.01030"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px2"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2510.23601#S4.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 自进化往往只调整少数模块、工具或修复单次错误，改进范围较浅。这不足以让一个通用 agent 围绕一组相关任务形成完整的领域工作方式，也限制经验向新任务迁移；作者希望研究从通用 agent 到领域专家的整体适应。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2510.23601#S1"
}
]
},
{
"key": "position",
"value": "研究通用 agent 能否通过任务经验形成可复用的领域执行能力，从而更高效地处理后续相关任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.23601"
}
]
},
{
"key": "conclusion",
"value": "在综合问答等任务中提高表现并降低成本，强调将经验变成可复用的执行能力。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2510.23601"
}
]
}
],
"fields": {
"modifier": "管理与生成角色采用 Claude Sonnet 4 和 GPT-4.1 的配对，将成功轨迹整理为可复用 MCP（让 agent 以统一接口连接外部工具的协议） 工具，再按任务检索；两个角色不应合写成同一个 task agent。",
"object": "可检索并执行的 MCP（让 agent 以统一接口连接外部工具的协议） 工具库。",
"seed": "先由管理 agent 把成功执行过程整理成通用工具：把写死的输入改为参数，去掉题目专属信息，统一调用接口。这些工具组成 MCP Box（按统一工具协议连接的工具库）；后续 agent 检索工具，通过生成并运行代码完成任务。"
}
},
"attributions": [
{
"tag": "person:mengdi-wang",
"label": "Mengdi Wang",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2510.23601"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2603.13131",
"title": "MineEvolve: Self-Evolution with Accumulated Knowledge for Long-Horizon Embodied Minecraft Agents",
"url": "https://arxiv.org/abs/2603.13131",
"date": "2026-03-13",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Skill"
],
"fields": {
"本质定位": "Monitor 把每个 subgoal execution 转成 typed feedback；Inducer 将成功蒸馏为 reusable skills、失败/停滞蒸馏为 remedies/guardrails；Curator 合并检索，Adaptor 修复剩余 plan。",
"什么在变": "skill + remedy/guardrail knowledge base。",
"谁来改 / 谁执行": "**改**：固定 Monitor/Inducer/Curator/Adaptor pipeline + LLM。
**执行**：frozen LLM Minecraft planner。",
"基础 harness": "planner + structured knowledge base。",
"Feedback": "inventory/state diff、failure type、progress、stagnation、task outcome。",
"Evolution → Eval": "Minecraft MCU long-horizon tasks；experience accumulation study。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相比 Voyager 的关键新增是 **失败也被结构化为可执行约束/guardrail**，并有细粒度 progress/stagnation diagnosis。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 305,
"fields": {
"优先级": "**K**",
"时间": "2026-03-13",
"论文": "[MineEvolve: Self-Evolution with Accumulated Knowledge for Long-Horizon Embodied Minecraft Agents](https://arxiv.org/abs/2603.13131)",
"本质定位": "Monitor 把每个 subgoal execution 转成 typed feedback；Inducer 将成功蒸馏为 reusable skills、失败/停滞蒸馏为 remedies/guardrails；Curator 合并检索，Adaptor 修复剩余 plan。",
"什么在变": "skill + remedy/guardrail knowledge base。",
"谁来改 / 谁执行": "**改**：固定 Monitor/Inducer/Curator/Adaptor pipeline + LLM。
**执行**：frozen LLM Minecraft planner。",
"基础 harness": "planner + structured knowledge base。",
"Feedback": "inventory/state diff、failure type、progress、stagnation、task outcome。",
"Evolution → Eval": "Minecraft MCU long-horizon tasks；experience accumulation study。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "相比 Voyager 的关键新增是 **失败也被结构化为可执行约束/guardrail**，并有细粒度 progress/stagnation diagnosis。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0",
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Monitor 把每个 subgoal execution 转成 typed feedback；Inducer 将成功蒸馏为 reusable skills、失败/停滞蒸馏为 remedies/guardrails；Curator 合并检索，Adaptor 修复剩余 plan。",
"novelty": "把失败提炼成执行时会检查的约束，并利用物品、状态和进展变化定位停滞，让失败经验直接影响后续动作。",
"object": "成功技能、失败补救与行为约束组成的知识库。",
"executor": "Minecraft 由高层语言模型规划、STEVE-1 执行底层动作。§4.3 的受控实验明确使用 Qwen3.5-Plus 规划器，并固定同一个 STEVE-1。",
"modifier": "Monitor 提取执行状态；Inducer 将成功与失败整理成技能和补救办法；经验整理角色 筛选、合并知识；Adaptor 修补未完成的计划。§3 的角色说明没有分别给出这些模块的模型型号，不能仅凭规划器型号将它们全部写成 Qwen3.5-Plus。",
"roleContext": "**改**：固定 Monitor/Inducer/Curator/Adaptor pipeline + LLM。
**执行**：frozen LLM Minecraft planner。",
"seed": "固定上层 planner 和 STEVE-1 低层执行策略，在 Minecraft 接口上维护技能知识与失败补救知识两类库；检索预算和评估时模型调用预算受控。更新的是外部知识，不是重训低层动作模型。",
"fixed": "",
"verdict": "比较动作前后的背包物品和环境状态，检测是否取得进展、是否停滞，并标记失败类型；这些具体证据用于生成技能或补救规则。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "Minecraft hard tasks",
"evolve": "先交互 0/50/100/200/400 episodes 积累知识",
"selection": "在对应节点冻结知识库",
"test": "Iron、Redstone、Diamond、Armor held-out hard tasks",
"isolation": "冻结知识后测试",
"note": "测试时不再生成或更新知识，区分积累经验的效果与测试时继续学习。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2603.13131v3"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Minecraft 交互积累技能与故障补救知识；知识积累实验在0/50/100/200/400 episodes 保存快照。\n\n调试 / 选版本数据：Monitor 从状态、背包差异和停滞中定位失败；经验整理角色 检查字段、可匹配性、可执行性、具体性及冲突。\n\n最终测试数据：主实验 MCU 70任务；积累实验冻结快照后评估 hard 任务：Iron16、Redstone6、Diamond7、Armor13，按任务数加权。\n\n数据隔离与证据边界：冻结知识后测试；Diamond 课程实验另比较低阶预训练、同任务自学习及混合采样，不能并入同一个隔离结论。",
"cycle": "规划模型和底层行动模型 STEVE-1 保持固定。系统从环境反馈中总结成功技能与失败补救规则，检查后入库；遇到未完成计划时检索这些知识作局部修复。",
"train": "Minecraft 交互积累技能与故障补救知识；知识积累实验在0/50/100/200/400 episodes 保存快照。",
"debug": "Monitor 从状态、背包差异和停滞中定位失败；经验整理角色 检查字段、可匹配性、可执行性、具体性及冲突。",
"test": "主实验 MCU 70任务；积累实验冻结快照后评估 hard 任务：Iron16、Redstone6、Diamond7、Armor13，按任务数加权。",
"isolation": "冻结知识后测试；Diamond 课程实验另比较低阶预训练、同任务自学习及混合采样，不能并入同一个隔离结论。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "固定上层 planner 和 STEVE-1 低层执行策略，在 Minecraft 接口上维护技能知识与失败补救知识两类库；检索预算和评估时模型调用预算受控。更新的是外部知识，不是重训低层动作模型。",
"protocol": "**经验构建：**Minecraft 交互后在 0/50/100/200/400 episodes 保存知识快照。\n\n**测试：**每个快照先冻结，再测未用于积累的 hard 任务：Iron 16、Redstone 6、Diamond 7、Armor 13，按任务数加权平均。另有 Diamond 课程构建实验，原文明确与主 MCU 协议不同，不能合并。",
"sections": "知识积累曲线；附录 D.4–D.5",
"source": "https://arxiv.org/abs/2603.13131",
"version": "2603.13131v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "df6782d3e029e10e4584d08751887ac0339446c0a3c248c2c8f8c273457124cf",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Minecraft 由高层语言模型规划、STEVE-1 执行底层动作。§4.3 的受控实验明确使用 Qwen3.5-Plus 规划器，并固定同一个 STEVE-1。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.13131#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.13131#S4.SS1.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.13131#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Monitor 提取执行状态；Inducer 将成功与失败整理成技能和补救办法；经验整理角色 筛选、合并知识；Adaptor 修补未完成的计划。§3 的角色说明没有分别给出这些模块的模型型号，不能仅凭规划器型号将它们全部写成 Qwen3.5-Plus。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.13131#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.13131#S4.SS1.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.13131#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "成功技能、失败补救与行为约束组成的知识库。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "比较动作前后的背包物品和环境状态，检测是否取得进展、是否停滞，并标记失败类型；这些具体证据用于生成技能或补救规则。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "附录 C.2：Monitor 算法",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.2：知识生成",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定上层 planner 和 STEVE-1 低层执行策略，在 Minecraft 接口上维护技能知识与失败补救知识两类库；检索预算和评估时模型调用预算受控。更新的是外部知识，不是重训低层动作模型。",
"sources": [
{
"label": "知识积累曲线；附录 D.4–D.5",
"url": "https://arxiv.org/abs/2603.13131"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "规划模型和底层行动模型 STEVE-1 保持固定。系统从环境反馈中总结成功技能与失败补救规则，检查后入库；遇到未完成计划时检索这些知识作局部修复。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Minecraft 交互积累技能与故障补救知识；知识积累实验在0/50/100/200/400 episodes 保存快照。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Monitor 从状态、背包差异和停滞中定位失败；经验整理角色 检查字段、可匹配性、可执行性、具体性及冲突。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主实验 MCU 70任务；积累实验冻结快照后评估 hard 任务：Iron16、Redstone6、Diamond7、Armor13，按任务数加权。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "冻结知识后测试；Diamond 课程实验另比较低阶预训练、同任务自学习及混合采样，不能并入同一个隔离结论。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把失败提炼成执行时会检查的约束，并利用物品、状态和进展变化定位停滞，让失败经验直接影响后续动作。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.13131v3",
"version": "2603.13131v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相比 Voyager 的关键新增是 **失败也被结构化为可执行约束/guardrail**，并有细粒度 progress/stagnation diagnosis。",
"feedbackCases": [
{
"label": "Minecraft：积累技能与补救知识",
"data": "与 Minecraft 环境交互；累计 0、50、100、200、400 次任务执行保存知识快照。",
"scoring": "Monitor 比较动作前后状态和背包物品，检测进度、停滞及失败类型；经验整理角色 再检查知识字段、适用条件、可执行性和冲突。",
"visible": "状态变化、观察序列、失败原因、是否取得目标进展；不是只让模型主观打一个分。",
"use": "成功步骤归纳为技能，失败及修复归纳为补救条目，审核后写入知识库。",
"sources": [
{
"label": "附录 C.2：Monitor 算法",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.2：知识生成",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
}
],
"judgment": "程序比较状态与背包，生成进度／停滞／错误信号；模型另整理知识"
},
{
"label": "Minecraft：任务评价",
"data": "主实验 MCU 70 任务；冻结知识快照后测 Iron 16、Redstone 6、Diamond 7、Armor 13 个 hard 任务。",
"scoring": "按环境中的物品获取与任务完成情况统计成功率；不同任务组按任务数加权。",
"visible": "用于比较知识积累量与最后任务表现。",
"use": "冻结快照评估与生成知识的交互阶段分开。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"judgment": "Minecraft 环境中的物品获取与目标完成检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Minecraft 交互积累技能与故障补救知识；知识积累实验在0/50/100/200/400 episodes 保存快照。",
"selection": "Monitor 从状态、背包差异和停滞中定位失败；经验整理角色 检查字段、可匹配性、可执行性、具体性及冲突。",
"evaluation": "主实验 MCU 70任务；积累实验冻结快照后评估 hard 任务：Iron16、Redstone6、Diamond7、Armor13，按任务数加权。",
"isolation": "冻结知识后测试；Diamond 课程实验另比较低阶预训练、同任务自学习及混合采样，不能并入同一个隔离结论。",
"roles": {
"executor": {
"value": "Minecraft 由高层语言模型规划、STEVE-1 执行底层动作。§4.3 的受控实验明确使用 Qwen3.5-Plus 规划器，并固定同一个 STEVE-1。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.13131#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.13131#S4.SS1.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.13131#S4.SS3"
}
]
},
"modifier": {
"value": "Monitor 提取执行状态；Inducer 将成功与失败整理成技能和补救办法；经验整理角色 筛选、合并知识；Adaptor 修补未完成的计划。§3 的角色说明没有分别给出这些模块的模型型号，不能仅凭规划器型号将它们全部写成 Qwen3.5-Plus。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2603.13131#S3.SS2"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2603.13131#A3.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.13131#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2603.13131#S4.SS1.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.13131#S4.SS3"
}
]
},
"seed": {
"value": "固定上层 planner 和 STEVE-1 低层执行策略，在 Minecraft 接口上维护技能知识与失败补救知识两类库；检索预算和评估时模型调用预算受控。更新的是外部知识，不是重训低层动作模型。",
"sources": [
{
"label": "知识积累曲线；附录 D.4–D.5",
"url": "https://arxiv.org/abs/2603.13131"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"selection": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"evaluation": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
],
"isolation": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2603.13131#A2.SS1"
},
{
"label": "附录D.4",
"url": "https://arxiv.org/html/2603.13131#A4.SS4"
},
{
"label": "附录D.5",
"url": "https://arxiv.org/html/2603.13131#A4.SS5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "Minecraft 的长任务包含工具准备、采集、冶炼和移动等连续依赖，缺工具、路径受阻或界面异常等局部执行错误就可能阻断整个计划。只会生成初始计划不够；如果成功与失败不能变成可复用知识，agent 就难及时修复剩余步骤并避免无效重试。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.13131#S1"
}
]
},
{
"key": "position",
"value": "面向 Minecraft 中前置依赖多、容易中断的长程任务，研究积累的执行知识能否帮助 agent 应对新失败并恢复任务推进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.13131"
}
]
},
{
"key": "conclusion",
"value": "在前置依赖多的任务上收益更大，证明结构化执行知识有助于长程恢复。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.13131"
}
]
}
],
"fields": {
"executor": "Qwen3.5-Plus 在受控实验中负责高层规划，STEVE-1 执行 Minecraft 底层动作。",
"modifier": "状态监测模块提取执行反馈，知识生成模块将成功转成技能、失败转成补救办法，维护模块筛选合并，计划修复模块调整剩余步骤。原文未分别给出这些模块的模型型号。",
"object": "成功技能、失败补救与行为约束组成的知识库。",
"verdict": "比较动作前后的背包物品和环境状态，检测是否取得进展、是否停滞，并标记失败类型；这些具体证据用于生成技能或补救规则。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2603.18000",
"title": "AgentFactory: A Self-Evolving Framework Through Executable Subagent Accumulation and Reuse",
"url": "https://arxiv.org/abs/2603.18000",
"date": "2026-03-18",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Skill",
"org:peking"
],
"fields": {
"本质定位": "成功 solution 被保存成 pure-Python subagent + SKILL.md；后续相似任务检索、执行、根据 feedback 直接修改 subagent code。",
"什么在变": "subagent code library。",
"谁来改 / 谁执行": "**改**：Meta-Agent 通过 create/run/modify primitives 编辑 subagents。
**执行**：同一模型 family 的 Meta-Agent/subagents（论文展示 Opus/Sonnet 4.6 等）。",
"基础 harness": "固定 Meta Skills + Tool Skills + evolving Subagent Skills。",
"Feedback": "runtime error、execution success、task result。",
"Evolution → Eval": "两批小规模真实任务/连续演示，主要看复用后 token/effort。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 Voyager 从 function-level skill 扩到可独立部署的 executable subagent；强调 portability/re-execution efficiency。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 306,
"fields": {
"优先级": "**K**",
"时间": "2026-03-18",
"论文": "[AgentFactory: A Self-Evolving Framework Through Executable Subagent Accumulation and Reuse](https://arxiv.org/abs/2603.18000)",
"本质定位": "成功 solution 被保存成 pure-Python subagent + SKILL.md；后续相似任务检索、执行、根据 feedback 直接修改 subagent code。",
"什么在变": "subagent code library。",
"谁来改 / 谁执行": "**改**：Meta-Agent 通过 create/run/modify primitives 编辑 subagents。
**执行**：同一模型 family 的 Meta-Agent/subagents（论文展示 Opus/Sonnet 4.6 等）。",
"基础 harness": "固定 Meta Skills + Tool Skills + evolving Subagent Skills。",
"Feedback": "runtime error、execution success、task result。",
"Evolution → Eval": "两批小规模真实任务/连续演示，主要看复用后 token/effort。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 Voyager 从 function-level skill 扩到可独立部署的 executable subagent；强调 portability/re-execution efficiency。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "成功 solution 被保存成 pure-Python subagent + SKILL.md；后续相似任务检索、执行、根据 feedback 直接修改 subagent code。",
"novelty": "把解决复杂任务的流程封装成可独立执行的sub-agent，逐步积累供后续相关任务调用的能力库。",
"object": "可执行 Python sub-agent 代码库。",
"executor": "Claude Opus4.6或Claude Sonnet4.6驱动Meta-Agent与可执行子 task agent；外部框架可直接调用保存后的脚本。",
"modifier": "Meta-Agent在相应Opus4.6/Sonnet4.6配置中调用create/run/modify，编辑并保存子 task agent；固定工具接口不参与自改。",
"roleContext": "**改**：Meta-Agent 通过 create/run/modify primitives 编辑 subagents。
**执行**：同一模型 family 的 Meta-Agent/subagents（论文展示 Opus/Sonnet 4.6 等）。",
"seed": "固定 Meta Skills 和 Tool Skills 配合可执行 Subagent Skills；主 task agent 可生成、保存并再次调用 Python 子 task agent。比较从零解题、只存文字经验和复用可执行子程序三种设置。",
"fixed": "",
"verdict": "运行报错、执行是否成功及任务结果。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "两批小规模真实任务/连续演示，主要看复用后 token/effort。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：作者自建第一批15个现实任务，生成和保存可执行子 task agent。\n\n调试 / 选版本数据：Meta-Agent 执行脚本、检查错误并修改子 task agent，保存代码和 SKILL.md。\n\n最终测试数据：第二批15题，沿用相似结构而改变具体要求；涵盖检索、可视化、浏览器及音频处理，比较复用与从零开始的开销。\n\n数据隔离与证据边界：两批题目列于附录B；这是小规模结构相似任务迁移，不是公开大规模评测基准的准确率比较。",
"cycle": "以子 task agent 脚本为复用单元，按任务检索、创建、调试后保存；导出的脚本可由其他框架按说明调用。",
"train": "作者自建第一批15个现实任务，生成和保存可执行子 task agent。",
"debug": "Meta-Agent 执行脚本、检查错误并修改子 task agent，保存代码和 SKILL.md。",
"test": "第二批15题，沿用相似结构而改变具体要求；涵盖检索、可视化、浏览器及音频处理，比较复用与从零开始的开销。",
"isolation": "两批题目列于附录B；这是小规模结构相似任务迁移，不是公开大规模评测基准的准确率比较。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定 Meta Skills 和 Tool Skills 配合可执行 Subagent Skills；主 agent 可生成、保存并再次调用 Python 子 agent。比较从零解题、只存文字经验和复用可执行子程序三种设置。",
"protocol": "**构建→迁移：**作者自建两批各 15 个现实任务，涉及网页信息获取、可视化、浏览器自动化、音频处理。第一批生成子 agent；第二批保留相似结构但改变具体要求，比较带已保存子 agent 和从零开始。\n\n**指标：**所有任务均要求无运行错误地完成，重点比较 token 等开销；不能当成大规模公开 benchmark 的准确率结论。两批具体题单见附录 B。",
"sections": "§5.1–5.2；附录 B",
"source": "https://arxiv.org/abs/2603.18000",
"version": "2603.18000v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "5ba3e23f86d8d80323e49d29e3198c75d7c3f0d556d18840a7b26d12dc685c9e",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Claude Opus4.6或Claude Sonnet4.6驱动Meta-Agent与可执行子 task agent；外部框架可直接调用保存后的脚本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.18000#S5.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Meta-Agent在相应Opus4.6/Sonnet4.6配置中调用create/run/modify，编辑并保存子 task agent；固定工具接口不参与自改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.18000#S5.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可执行 Python sub-agent 代码库。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行报错、执行是否成功及任务结果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定 Meta Skills 和 Tool Skills 配合可执行 Subagent Skills；主 task agent 可生成、保存并再次调用 Python 子 task agent。比较从零解题、只存文字经验和复用可执行子程序三种设置。",
"sources": [
{
"label": "§5.1–5.2；附录 B",
"url": "https://arxiv.org/abs/2603.18000"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "以子 task agent 脚本为复用单元，按任务检索、创建、调试后保存；导出的脚本可由其他框架按说明调用。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "作者自建第一批15个现实任务，生成和保存可执行子 task agent。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Meta-Agent 执行脚本、检查错误并修改子 task agent，保存代码和 SKILL.md。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "第二批15题，沿用相似结构而改变具体要求；涵盖检索、可视化、浏览器及音频处理，比较复用与从零开始的开销。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "两批题目列于附录B；这是小规模结构相似任务迁移，不是公开大规模评测基准的准确率比较。",
"sources": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把解决复杂任务的流程封装成可独立执行的sub-agent，逐步积累供后续相关任务调用的能力库。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.18000v1",
"version": "2603.18000v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Voyager 从 function-level skill 扩到可独立部署的 executable subagent；强调 portability/re-execution efficiency。",
"feedbackCases": [
{
"label": "自建任务：sub-agent 生成与修复",
"data": "第一批 15 个现实任务，包括检索、可视化、浏览器与音频处理。",
"scoring": "Meta-Agent 实际运行 Python sub-agent，读取报错与产物是否满足当前任务，修复后重新运行。",
"visible": "脚本运行输出、异常与任务产物。论文没有给出这 15 题统一的独立裁判模型或标准验收脚本。",
"use": "保存成功脚本及 SKILL.md；能确认是执行驱动修复，但不能据原文声称所有sub-agent 均经独立外部验证。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"judgment": "实际执行 Python，修改者根据异常与产物检查并修复"
},
{
"label": "复用测试",
"data": "第二批 15 题，保留任务结构但改变具体要求。",
"scoring": "比较完成任务的执行开销与复用效果；并非一套有统一标准答案的公共 benchmark。",
"visible": "新任务执行结果和成本统计。",
"use": "比较调用既有sub-agent 与从零编写；题目结构相关性是这个实验的特点。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"judgment": "自建任务的完成与开销评价，无统一公共答案裁判"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "作者自建第一批15个现实任务，生成和保存可执行子 task agent。",
"selection": "Meta-Agent 执行脚本、检查错误并修改子 task agent，保存代码和 SKILL.md。",
"evaluation": "第二批15题，沿用相似结构而改变具体要求；涵盖检索、可视化、浏览器及音频处理，比较复用与从零开始的开销。",
"isolation": "两批题目列于附录B；这是小规模结构相似任务迁移，不是公开大规模评测基准的准确率比较。",
"roles": {
"executor": {
"value": "Claude Opus4.6或Claude Sonnet4.6驱动Meta-Agent与可执行子 task agent；外部框架可直接调用保存后的脚本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.18000#S5.SS1.SSS0.Px4"
}
]
},
"modifier": {
"value": "Meta-Agent在相应Opus4.6/Sonnet4.6配置中调用create/run/modify，编辑并保存子 task agent；固定工具接口不参与自改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.18000#S3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2603.18000#S3.SS4.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.18000#S5.SS1.SSS0.Px4"
}
]
},
"seed": {
"value": "固定 Meta Skills 和 Tool Skills 配合可执行 Subagent Skills；主 task agent 可生成、保存并再次调用 Python 子 task agent。比较从零解题、只存文字经验和复用可执行子程序三种设置。",
"sources": [
{
"label": "§5.1–5.2；附录 B",
"url": "https://arxiv.org/abs/2603.18000"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"selection": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"evaluation": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
],
"isolation": [
{
"label": "附录B",
"url": "https://arxiv.org/html/2603.18000#A2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 常不保留执行中获得的知识；能积累经验的方法又主要保存文字反思、提示或推理记录。文字经验并不能保证复杂操作下次能高效、可靠地重做，而已有可执行经验方法多面向专门领域，因此作者研究可跨任务积累和复用的可执行sub-agent。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.18000#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否把解决复杂任务的经历转化为可再次执行的能力，并在后续相关任务中减少重复开发、提高可靠性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.18000"
}
]
},
{
"key": "conclusion",
"value": "可执行sub-agent 库随任务积累而扩展、变稳，降低处理相似任务所需的重复工作。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.18000"
}
]
}
],
"fields": {
"object": "可执行 Python sub-agent 代码库。"
}
},
"attributions": [
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.18000"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2603.18743",
"title": "Memento-Skills: Let Agents Design Agents",
"url": "https://arxiv.org/abs/2603.18743",
"date": "2026-03-19",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Skill",
"person:jun-wang"
],
"fields": {
"本质定位": "冻结底层 LLM，把 structured SKILL.md / executable skill folders 当 persistent skill memory；任务时 retrieve/generate skill，执行后 Reflect→Write，成功提高 utility、失败则修复/重写 skill。",
"什么在变": "skill prompts/code、skill metadata/utility、skill library structure；底层 LLM weights 冻结。",
"谁来改 / 谁执行": "**改**：同一 generalist agent 的 reflective write-back loop。
**执行**：frozen generalist LLM agent + retrieved/generated skills。",
"基础 harness": "skill router + sandbox/tool execution + persistent structured skill memory。",
"Feedback": "任务执行 outcome、失败诊断/反思；论文/项目未把它建立成严格独立的外部 verifier learning problem。",
"Evolution → Eval": "GAIA 与 HLE 上多 learning rounds，performance 与 skill library 同步增长；主要是 repeated deployment-style evolution。",
"Meta-depth": "M1 same-system；reflect/write algorithm 本身固定。",
"相对之前真正新增什么": "相对 Voyager 的新点主要是把 skill library 做成 **deployment-time read-write reflective memory**：不仅积累成功代码，还显式定位失败 skill、repair/rewrite，并让 generalist agent 设计 task-specific agents。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 307,
"fields": {
"优先级": "**K**",
"时间": "2026-03-19",
"论文": "[Memento-Skills: Let Agents Design Agents](https://arxiv.org/abs/2603.18743)",
"本质定位": "冻结底层 LLM，把 structured SKILL.md / executable skill folders 当 persistent skill memory；任务时 retrieve/generate skill，执行后 Reflect→Write，成功提高 utility、失败则修复/重写 skill。",
"什么在变": "skill prompts/code、skill metadata/utility、skill library structure；底层 LLM weights 冻结。",
"谁来改 / 谁执行": "**改**：同一 generalist agent 的 reflective write-back loop。
**执行**：frozen generalist LLM agent + retrieved/generated skills。",
"基础 harness": "skill router + sandbox/tool execution + persistent structured skill memory。",
"Feedback": "任务执行 outcome、失败诊断/反思；论文/项目未把它建立成严格独立的外部 verifier learning problem。",
"Evolution → Eval": "GAIA 与 HLE 上多 learning rounds，performance 与 skill library 同步增长；主要是 repeated deployment-style evolution。",
"Meta-depth": "M1 same-system；reflect/write algorithm 本身固定。",
"相对之前真正新增什么": "相对 Voyager 的新点主要是把 skill library 做成 **deployment-time read-write reflective memory**：不仅积累成功代码，还显式定位失败 skill、repair/rewrite，并让 generalist agent 设计 task-specific agents。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "冻结底层 LLM，把 structured SKILL.md / executable skill folders 当 persistent skill memory；任务时 retrieve/generate skill，执行后 Reflect→Write，成功提高 utility、失败则修复/重写 skill。",
"novelty": "把技能库做成执行期间可读写、可修订的资源；定位失败技能后修复或重写，并单独学习怎样选择该调用的技能。",
"object": "技能说明、代码、元数据与效用，以及技能路由器；底层执行语言模型保持冻结。",
"executor": "Gemini-3.1-Flash 执行技能和完成任务；Qwen3-Embedding-0.6B 负责技能检索，是另一个可训练的路由模型，不是答题模型。",
"modifier": "Gemini-3.1-Flash 通过反思调用改写、发现技能；另用离线强化学习更新 Qwen3-Embedding-0.6B 路由器。论文说明全部实验的底层 语言模型 为 Gemini-3.1-Flash。",
"roleContext": "**改**：同一 generalist agent 的 reflective write-back loop。
**执行**：frozen generalist LLM agent + retrieved/generated skills。",
"seed": "固定 语言模型 和工具执行系统外加技能库及路由器；从相同 5 个原子技能开始，在反思循环中生成和更新可调用技能。技能路由有单独学习设计，不能把“冻结任务模型”推成系统没有任何训练。",
"fixed": "reflect/write algorithm 本身固定",
"verdict": "评审者对照任务回答与参考答案给出结果和理由，失败分析再定位到具体技能。技能修改后运行系统合成的检查用例；合成检查与独立留出题评测分开。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "GAIA 与 HLE 上多 learning rounds，performance 与 skill library 同步增长；主要是 repeated deployment-style evolution。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 的100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）按8领域抽788题。另为路由器从约8000技能中抽约3000个合成正负查询。\n\n调试 / 选版本数据：失败归因到单个技能，文件级改写后用合成单测和评分者守门；低效技能可重构或新建。\n\n最终测试数据：GAIA（需要检索、推理和使用工具的通用助理任务基准）另65题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）另342题。路由器另报告140条合成查询的 Recall@K。\n\n数据隔离与证据边界：任务技能学习与任务测试分开；路由器训练是独立数据流程。冻结的是主体 语言模型，路由 embedding 另有训练。",
"cycle": "从搜索和终端等基础技能起步，读技能执行、按评分者反馈写技能；同时训练按执行效用而非纯语义相似度路由的检索模型。",
"train": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 的100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）按8领域抽788题。另为路由器从约8000技能中抽约3000个合成正负查询。",
"debug": "失败归因到单个技能，文件级改写后用合成单测和评分者守门；低效技能可重构或新建。",
"test": "GAIA（需要检索、推理和使用工具的通用助理任务基准）另65题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）另342题。路由器另报告140条合成查询的 Recall@K。",
"isolation": "任务技能学习与任务测试分开；路由器训练是独立数据流程。冻结的是主体 语言模型，路由 embedding 另有训练。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定 LLM 和工具执行系统外加技能库及路由器；从相同 5 个原子技能开始，在反思循环中生成和更新可调用技能。技能路由有单独学习设计，不能把“冻结任务模型”推成系统没有任何训练。",
"protocol": "**GAIA：**官方 validation 165 题拆成 100 training、65 test，前者供技能学习，后者测迁移；最终技能库规模 41 不是训练题数。\n\n**HLE：**另在八个学术域学习技能，形成 235 个技能；HLE 的具体训练/测试数量本轮未核实。技能数量变化不等于新增独立测试样本。",
"sections": "GAIA 数据设置、技能增长分析",
"source": "https://arxiv.org/abs/2603.18743",
"version": "2603.18743v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c59c79486fd2217465cd17d8a717690bea0942cf14906145c9cc7640cac759a4",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Gemini-3.1-Flash 执行技能和完成任务；Qwen3-Embedding-0.6B 负责技能检索，是另一个可训练的路由模型，不是答题模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Gemini-3.1-Flash 通过反思调用改写、发现技能；另用离线强化学习更新 Qwen3-Embedding-0.6B 路由器。论文说明全部实验的底层 语言模型 为 Gemini-3.1-Flash。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "技能说明、代码、元数据与效用，以及技能路由器；底层执行语言模型保持冻结。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "评审者对照任务回答与参考答案给出结果和理由，失败分析再定位到具体技能。技能修改后运行系统合成的检查用例；合成检查与独立留出题评测分开。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.1：Judge 输入及单测守门",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§3.1：GAIA 迭代",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
},
{
"label": "§2.3：查询合成",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4：检索指标",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定 语言模型 和工具执行系统外加技能库及路由器；从相同 5 个原子技能开始，在反思循环中生成和更新可调用技能。技能路由有单独学习设计，不能把“冻结任务模型”推成系统没有任何训练。",
"sources": [
{
"label": "GAIA 数据设置、技能增长分析",
"url": "https://arxiv.org/abs/2603.18743"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "从搜索和终端等基础技能起步，读技能执行、按评分者反馈写技能；同时训练按执行效用而非纯语义相似度路由的检索模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 的100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）按8领域抽788题。另为路由器从约8000技能中抽约3000个合成正负查询。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "失败归因到单个技能，文件级改写后用合成单测和评分者守门；低效技能可重构或新建。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GAIA（需要检索、推理和使用工具的通用助理任务基准）另65题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）另342题。路由器另报告140条合成查询的 Recall@K。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "任务技能学习与任务测试分开；路由器训练是独立数据流程。冻结的是主体 语言模型，路由 embedding 另有训练。",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把技能库做成执行期间可读写、可修订的资源；定位失败技能后修复或重写，并单独学习怎样选择该调用的技能。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.18743v1",
"version": "2603.18743v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Voyager 的新点主要是把 skill library 做成 **deployment-time read-write reflective memory**：不仅积累成功代码，还显式定位失败 skill、repair/rewrite，并让 generalist agent 设计 task-specific agents。",
"feedbackCases": [
{
"label": "GAIA / HLE：技能修改",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 中 100 题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 八领域共 788 题用于学习；GAIA 最多三轮反思重试。",
"scoring": "算法中的 Judge 接收任务、回答和参考答案；失败后合成测试用例，运行修改后的技能，再由 judge 评分守门。",
"visible": "完整失败轨迹和 judge 的理由，归因到单个技能；合成用例的通过情况。",
"use": "针对技能文件打补丁，效用持续低时重构或新建；合成测试是系统生成的检查，不等于独立 benchmark 留出测试。",
"sources": [
{
"label": "§2.1：Judge 输入及单测守门",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§3.1：GAIA 迭代",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"judgment": "Judge 接收题目、回答和标准答案；算法未明确该调用的规则／模型实现"
},
{
"label": "最终任务测试",
"data": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 另 65 题，HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 另 342 题。",
"scoring": "比较最终答案准确率；与学习阶段的重试成功率分开报告。",
"visible": "留出题的最终正确率。",
"use": "检验积累的技能能否用于新题，不把 GAIA（需要检索、推理和使用工具的通用助理任务基准） 第三次练习成功率当成测试成绩。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"judgment": "参考答案正确率；与技能学习中的重试成功率分开"
},
{
"label": "路由器：合成查询训练与评估",
"data": "从约 8,000 个技能中抽约 3,000 个合成正负查询；另用 140 条合成查询测检索。",
"scoring": "生成查询时只看技能名与描述；筛选 judge 看完整技能文件；Recall@K 检查目标技能是否进入前 K 个结果。",
"visible": "合成的正例、困难负例及筛选结果。",
"use": "训练技能检索器；这个反馈测选工具，不代表 GAIA（需要检索、推理和使用工具的通用助理任务基准） / HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准） 任务最终答对。",
"sources": [
{
"label": "§2.3：查询合成",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4：检索指标",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
}
],
"judgment": "模型检查查询与技能的适配；Recall@K 用目标技能标签计算"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0,
2
],
"testCases": [
1,
2
],
"evolution": "GAIA（需要检索、推理和使用工具的通用助理任务基准） 验证集 的100题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）按8领域抽788题。另为路由器从约8000技能中抽约3000个合成正负查询。",
"selection": "失败归因到单个技能，文件级改写后用合成单测和评分者守门；低效技能可重构或新建。",
"evaluation": "GAIA（需要检索、推理和使用工具的通用助理任务基准）另65题；HLE（涵盖多个学科高难度问题的 Humanity’s Last Exam 基准）另342题。路由器另报告140条合成查询的 Recall@K。",
"isolation": "任务技能学习与任务测试分开；路由器训练是独立数据流程。冻结的是主体 语言模型，路由 embedding 另有训练。",
"roles": {
"executor": {
"value": "Gemini-3.1-Flash 执行技能和完成任务；Qwen3-Embedding-0.6B 负责技能检索，是另一个可训练的路由模型，不是答题模型。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
]
},
"modifier": {
"value": "Gemini-3.1-Flash 通过反思调用改写、发现技能；另用离线强化学习更新 Qwen3-Embedding-0.6B 路由器。论文说明全部实验的底层 语言模型 为 Gemini-3.1-Flash。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
]
},
"seed": {
"value": "固定 语言模型 和工具执行系统外加技能库及路由器；从相同 5 个原子技能开始，在反思循环中生成和更新可调用技能。技能路由有单独学习设计，不能把“冻结任务模型”推成系统没有任何训练。",
"sources": [
{
"label": "GAIA 数据设置、技能增长分析",
"url": "https://arxiv.org/abs/2603.18743"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"selection": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"evaluation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
],
"isolation": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2603.18743#S2.SS4.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2603.18743#S3.SS1.SSS0.Px4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "持续更新模型参数需要大量数据和计算，实际部署的 agent 因此往往冻结权重，只能依赖已有知识和当前上下文。若没有可更新的外部记忆，它就无法把部署中的经历持续带到后续任务；作者希望把这些经历转成可修改、可执行的技能，让冻结模型也能逐步适应。",
"sources": [
{
"label": "§1.1 Why Frozen LLMs Need External Memory",
"url": "https://arxiv.org/html/2603.18743#S1.SS1"
}
]
},
{
"key": "position",
"value": "研究冻结的通用模型能否通过持续积累和修订技能，自主形成适合新任务的 agent 能力，并有效选择该使用的技能。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.18743"
}
]
},
{
"key": "conclusion",
"value": "把 agent 设计转成技能读写与反思循环；执行模型保持冻结，技能及路由分别承担适应。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.18743"
}
]
}
],
"fields": {
"object": "技能说明、代码、元数据与效用，以及技能路由器；底层执行语言模型保持冻结。",
"verdict": "评审者对照任务回答与参考答案给出结果和理由，失败分析再定位到具体技能。技能修改后运行系统合成的检查用例；合成检查与独立留出题评测分开。"
}
},
"attributions": [
{
"tag": "person:jun-wang",
"label": "Jun Wang",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2603.18743"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2604.15097",
"title": "From Procedural Skills to Strategy Genes: Towards Experience-Driven Test-Time Evolution",
"url": "https://arxiv.org/abs/2604.15097",
"date": "2026-04-16",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Skill",
"org:tsinghua"
],
"fields": {
"本质定位": "不是提出更复杂 evolution loop，而是控制实验比较 Skill、free-form experience、compact editable Gene 等表示，研究什么 experience representation 更适合持续进化。",
"什么在变": "Gene/Skill 中积累的经验内容与 failure warnings。",
"谁来改 / 谁执行": "**改**：固定 experience update/evolution procedure。
**执行**：scientific coding agents。",
"基础 harness": "base agent + injected experience representation。",
"Feedback": "task outcome + failure history。",
"Evolution → Eval": "4,590 trials / 45 scenarios；CritPt paired settings。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "最有价值的新点是把 **representation 本身**作为一阶变量：更长、更像文档的 skill 不一定更好；compact editable control object 更稳。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 308,
"fields": {
"优先级": "**R**",
"时间": "2026-04-16",
"论文": "[From Procedural Skills to Strategy Genes: Towards Experience-Driven Test-Time Evolution](https://arxiv.org/abs/2604.15097)",
"本质定位": "不是提出更复杂 evolution loop，而是控制实验比较 Skill、free-form experience、compact editable Gene 等表示，研究什么 experience representation 更适合持续进化。",
"什么在变": "Gene/Skill 中积累的经验内容与 failure warnings。",
"谁来改 / 谁执行": "**改**：固定 experience update/evolution procedure。
**执行**：scientific coding agents。",
"基础 harness": "base agent + injected experience representation。",
"Feedback": "task outcome + failure history。",
"Evolution → Eval": "4,590 trials / 45 scenarios；CritPt paired settings。",
"Meta-depth": "M0。",
"相对之前真正新增什么": "最有价值的新点是把 **representation 本身**作为一阶变量：更长、更像文档的 skill 不一定更好；compact editable control object 更稳。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "不是提出更复杂 evolution loop，而是控制实验比较 Skill、free-form experience、compact editable Gene 等表示，研究什么 experience representation 更适合持续进化。",
"novelty": "在受控条件下比较长步骤文档与紧凑策略表示，直接把“经验怎样写”作为实验变量。",
"object": "紧凑策略或技能表示中的经验内容与失败警示。",
"executor": "受控表示实验使用Gemini3.1 Pro Preview和Gemini3.1 Flash Lite Preview；CritPt演化案例另比较Gemini3 Pro Preview与Gemini3.1 Pro Preview两个时期。",
"modifier": "固定Evolver流程驱动Gene更新；对应案例的任务模型为Gemini3 Pro Preview或Gemini3.1 Pro Preview，不能将其与受控实验的Flash Lite混为一组。",
"roleContext": "**改**：固定 experience update/evolution procedure。
**执行**：scientific coding agents。",
"seed": "固定执行 task agent，注入同源经验的不同表示：完整文档式 Skill、简短策略 Gene 或不加指导。Gene 包含触发信号、策略、约束和验证钩子；对照控制经验来源以研究表示的作用。",
"fixed": "",
"verdict": "自建科学代码场景通过测试脚本逐项检查完成情况；执行失败历史用于提炼经验。CritPt 的外部任务评价另行统计。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "4,590 trials / 45 scenarios；CritPt paired settings。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：作者自建科学/技术场景，把经验编码成 Skill、自由文本或 Gene；演化版本还从历史执行与外部论文来源获取 Gene。\n\n调试 / 选版本数据：场景自带测试脚本，按检查点完成度比较表示；进化中记录触发条件、策略、约束及验证事件。\n\n最终测试数据：45个场景的4590次保留试验；另在 CritPt 比较两个日期的 Evolver（Gene）与相应基础模型。\n\n数据隔离与证据边界：表示对照与 CritPt 系统演化是不同实验；后者不能据此解释为同一套严格 训练／验证／测试 划分。",
"cycle": "OpenClaw 负责运行 task agent，进化流程把经历压缩成在特定情境触发的短策略 Gene（本文的策略经验表示）。通过实际执行验证改动，并记录每条经验的来源和修改历史。",
"train": "作者自建科学/技术场景，把经验编码成 Skill、自由文本或 Gene；演化版本还从历史执行与外部论文来源获取 Gene。",
"debug": "场景自带测试脚本，按检查点完成度比较表示；进化中记录触发条件、策略、约束及验证事件。",
"test": "45个场景的4590次保留试验；另在 CritPt 比较两个日期的 Evolver（Gene）与相应基础模型。",
"isolation": "表示对照与 CritPt 系统演化是不同实验；后者不能据此解释为同一套严格 训练／验证／测试 划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定执行 agent，注入同源经验的不同表示：完整文档式 Skill、简短策略 Gene 或不加指导。Gene 包含触发信号、策略、约束和验证钩子；对照控制经验来源以研究表示的作用。",
"protocol": "**数据：**作者的可执行场景含蛋白解析、UV-Vis 光谱峰检测、系外行星凌日、地震目录、气候归因、社区发现、库存补货等，按检查点完成率评分。Skill probe 有 1,440 次保留试验，不是 1,440 个不同任务。\n\n**协议：**同场景经验转为不同提示表示，比较 Pro/Flash 等执行模型；独立源任务与最终测试任务的完整划分本轮待核实，不能仅由表示对照推断跨任务留出。",
"sections": "§3.3、§4.1；附录 B/D",
"source": "https://arxiv.org/abs/2604.15097",
"version": "2604.15097v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "8e1354dbe13d84870f614657b6ae7da816caa1b8eeb65edde247bfb430e08c6c",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "受控表示实验使用Gemini3.1 Pro Preview和Gemini3.1 Flash Lite Preview；CritPt演化案例另比较Gemini3 Pro Preview与Gemini3.1 Pro Preview两个时期。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.15097#S3.SS3.SSS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2604.15097#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定Evolver流程驱动Gene更新；对应案例的任务模型为Gemini3 Pro Preview或Gemini3.1 Pro Preview，不能将其与受控实验的Flash Lite混为一组。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.15097#S3.SS3.SSS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2604.15097#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "紧凑策略或技能表示中的经验内容与失败警示。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "自建科学代码场景通过测试脚本逐项检查完成情况；执行失败历史用于提炼经验。CritPt 的外部任务评价另行统计。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "附录 B.1：评价协议与指标",
"url": "https://arxiv.org/html/2604.15097#A2.SS1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定执行 task agent，注入同源经验的不同表示：完整文档式 Skill、简短策略 Gene 或不加指导。Gene 包含触发信号、策略、约束和验证钩子；对照控制经验来源以研究表示的作用。",
"sources": [
{
"label": "§3.3、§4.1；附录 B/D",
"url": "https://arxiv.org/abs/2604.15097"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "OpenClaw 负责运行 task agent，进化流程把经历压缩成在特定情境触发的短策略 Gene（本文的策略经验表示）。通过实际执行验证改动，并记录每条经验的来源和修改历史。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "作者自建科学/技术场景，把经验编码成 Skill、自由文本或 Gene；演化版本还从历史执行与外部论文来源获取 Gene。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "场景自带测试脚本，按检查点完成度比较表示；进化中记录触发条件、策略、约束及验证事件。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "45个场景的4590次保留试验；另在 CritPt 比较两个日期的 Evolver（Gene）与相应基础模型。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "表示对照与 CritPt 系统演化是不同实验；后者不能据此解释为同一套严格 训练／验证／测试 划分。",
"sources": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在受控条件下比较长步骤文档与紧凑策略表示，直接把“经验怎样写”作为实验变量。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.15097v2",
"version": "2604.15097v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "最有价值的新点是把 **representation 本身**作为一阶变量：更长、更像文档的 skill 不一定更好；compact editable control object 更稳。",
"feedbackCases": [
{
"label": "自建科学技术场景",
"data": "45 个场景、4,590 次保留试验；比较技能步骤、自由文本经验和策略 Gene 三种表示。",
"scoring": "沙箱运行最多 120 秒，场景自带测试脚本按通过检查点数 / 总检查点数评分；全部通过另记 complete pass。",
"visible": "执行记录、检查点结果及失败历史。",
"use": "比较经验表示并提炼后续可用策略；外部论文也可提供策略内容，但论文文本本身不是正确性奖励。",
"sources": [
{
"label": "附录 B.1：评价协议与指标",
"url": "https://arxiv.org/html/2604.15097#A2.SS1"
}
],
"judgment": "场景自带沙箱测试，规则计算通过检查点比例"
},
{
"label": "CritPt 评价",
"data": "在 CritPt 上比较两个日期的 Evolver（Gene）版本与各自基础模型。",
"scoring": "使用 CritPt 任务表现比较进化前后能力。",
"visible": "该组是最终基准成绩，与自建场景的检查点分数分别统计。",
"use": "检验经验进化版本的外部任务表现，不能把 4,590 次场景试验当成 CritPt 训练题数。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"judgment": "沿用 CritPt 任务评测；该处未展开具体判分器"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "作者自建科学/技术场景，把经验编码成 Skill、自由文本或 Gene；演化版本还从历史执行与外部论文来源获取 Gene。",
"selection": "场景自带测试脚本，按检查点完成度比较表示；进化中记录触发条件、策略、约束及验证事件。",
"evaluation": "45个场景的4590次保留试验；另在 CritPt 比较两个日期的 Evolver（Gene）与相应基础模型。",
"isolation": "表示对照与 CritPt 系统演化是不同实验；后者不能据此解释为同一套严格 训练／验证／测试 划分。",
"roles": {
"executor": {
"value": "受控表示实验使用Gemini3.1 Pro Preview和Gemini3.1 Flash Lite Preview；CritPt演化案例另比较Gemini3 Pro Preview与Gemini3.1 Pro Preview两个时期。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.15097#S3.SS3.SSS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2604.15097#A2.SS3"
}
]
},
"modifier": {
"value": "固定Evolver流程驱动Gene更新；对应案例的任务模型为Gemini3 Pro Preview或Gemini3.1 Pro Preview，不能将其与受控实验的Flash Lite混为一组。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2604.15097#A1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.15097#S3.SS3.SSS1"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2604.15097#A2.SS3"
}
]
},
"seed": {
"value": "固定执行 task agent，注入同源经验的不同表示：完整文档式 Skill、简短策略 Gene 或不加指导。Gene 包含触发信号、策略、约束和验证钩子；对照控制经验来源以研究表示的作用。",
"sources": [
{
"label": "§3.3、§4.1；附录 B/D",
"url": "https://arxiv.org/abs/2604.15097"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"selection": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"evaluation": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
],
"isolation": [
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2604.15097#A2.SS2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2604.15097#S4.SS4"
},
{
"label": "附录D.3",
"url": "https://arxiv.org/html/2604.15097#A4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "保存了更多经验，并不意味着模型在有限推理预算下更会行动。面向阅读的长技能文档可能缺少清楚的适用边界和直接影响决策的信息；经验的内容、组织方式与呈现形式又常被混在一起比较，因此需要检验怎样表示经验才能稳定指导执行。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.15097#S1"
}
]
},
{
"key": "position",
"value": "研究经验采用什么表示才更有助于后续行为改进，检验完整的步骤文档是否一定优于紧凑、可修改的策略知识。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.15097"
}
]
},
{
"key": "conclusion",
"value": "紧凑表示总体更有效；失败应提炼成简短警示，直接追加历史或长文档可能削弱效果。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.15097"
}
]
}
],
"fields": {
"object": "紧凑策略或技能表示中的经验内容与失败警示。",
"verdict": "自建科学代码场景通过测试脚本逐项检查完成情况；执行失败历史用于提炼经验。CritPt 的外部任务评价另行统计。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2604.15097"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2604.20133",
"title": "EvoAgent: An Evolvable Agent Framework with Skill Learning and Multi-Agent Delegation",
"url": "https://arxiv.org/abs/2604.20133",
"date": "2026-04-22",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Skill"
],
"fields": {
"本质定位": "把 skill 做成多文件 structured capability unit，带 trigger/evolution metadata，并与 hierarchical subagent delegation、three-layer memory 结合。",
"什么在变": "skill repository、metadata、memory/profile。",
"谁来改 / 谁执行": "**改**：user-feedback-driven closed loop / framework agents。
**执行**：GPT-5.2 等 target agents。",
"基础 harness": "fixed multi-agent hierarchy + structured skill/memory system。",
"Feedback": "用户反馈/usage outcome + LLM-as-judge evaluation。",
"Evolution → Eval": "real-world foreign-trade scenarios；transfer across models。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "更像完整产品/框架整合：skill persistence + delegation；相对前人没有很强的新 evolution principle。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 309,
"fields": {
"优先级": "**R**",
"时间": "2026-04-22",
"论文": "[EvoAgent: An Evolvable Agent Framework with Skill Learning and Multi-Agent Delegation](https://arxiv.org/abs/2604.20133)",
"本质定位": "把 skill 做成多文件 structured capability unit，带 trigger/evolution metadata，并与 hierarchical subagent delegation、three-layer memory 结合。",
"什么在变": "skill repository、metadata、memory/profile。",
"谁来改 / 谁执行": "**改**：user-feedback-driven closed loop / framework agents。
**执行**：GPT-5.2 等 target agents。",
"基础 harness": "fixed multi-agent hierarchy + structured skill/memory system。",
"Feedback": "用户反馈/usage outcome + LLM-as-judge evaluation。",
"Evolution → Eval": "real-world foreign-trade scenarios；transfer across models。",
"Meta-depth": "M0/M1。",
"相对之前真正新增什么": "更像完整产品/框架整合：skill persistence + delegation；相对前人没有很强的新 evolution principle。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0",
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 skill 做成多文件 structured capability unit，带 trigger/evolution metadata，并与 hierarchical subagent delegation、three-layer memory 结合。",
"novelty": "将长期保存技能与多 agent 任务分派组织到同一助理框架，重点检查积累的能力能否在实际工作中被调用。",
"object": "技能库、技能元数据、记忆与用户信息。",
"executor": "主实验由 GPT-5.2 配合 EvoAgent 执行外贸任务；模型替换实验改用 GPT-4.1 或 Qwen3.5-35B-A3B。",
"modifier": "EvoAgent 根据用户反馈调用模型生成、修订技能并组织sub-agent。主配置的基础模型是 GPT-5.2；§4 未为技能修订角色另列一款独立模型。",
"roleContext": "**改**：user-feedback-driven closed loop / framework agents。
**执行**：GPT-5.2 等 target agents。",
"seed": "作者自建五层系统：API、路由编排、运行时、工具/会话、持久化；主 task agent 用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 并分层委派子 task agent，逐步披露技能以控制上下文。基础系统已含多 task agent 和记忆管理。",
"fixed": "",
"verdict": "技能更新主要利用用户交互和使用记录；论文最后另用模型从专业性、准确性等维度评审回答。最终评审分数不应自动当作每次技能更新的奖励。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "real-world foreign-trade scenarios；transfer across models。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：系统从历史会话积累用户画像、记忆与技能；外贸实验脚本采集664段、每段8–9轮的对话池。\n\n调试 / 选版本数据：会话内执行与会话间离线分析分开；离线根据使用记录提炼或重构技能。\n\n最终测试数据：随机抽20段对话拆成172个评估实例，§4.1/4.2复用；另测长对话压缩和记忆稳定性。\n\n数据隔离与证据边界：§4提供的是自建系统测试，未建立技能形成材料与172评估实例之间的明确互斥划分。",
"cycle": "在线层做技能调用和多 task agent 委派，离线层分析历史对话更新画像、长期记忆和技能库。",
"train": "系统从历史会话积累用户画像、记忆与技能；外贸实验脚本采集664段、每段8–9轮的对话池。",
"debug": "会话内执行与会话间离线分析分开；离线根据使用记录提炼或重构技能。",
"test": "随机抽20段对话拆成172个评估实例，§4.1/4.2复用；另测长对话压缩和记忆稳定性。",
"isolation": "§4提供的是自建系统测试，未建立技能形成材料与172评估实例之间的明确互斥划分。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "作者自建五层系统：API、路由编排、运行时、工具/会话、持久化；主 agent 用 ReAct 并分层委派子 agent，逐步披露技能以控制上下文。基础系统已含多 agent 和记忆管理。",
"protocol": "**数据构造：**通过脚本与已部署 EvoAgent 交互，收集外贸业务 664 段多轮对话，每段 8–9 轮；随机选 20 段拆为 172 个评估实例，§4.2 复用同一集合。\n\n**边界：**这是自建领域对话及系统验证，不是公开外贸 benchmark 的独立训练/测试；664 是候选对话池，172 是拆分后的评估轮次。技能形成/选版本与这 20 段的重叠本轮未核实。",
"sections": "§3 架构、§4 数据收集",
"source": "https://arxiv.org/abs/2604.20133",
"version": "2604.20133v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "3b57a39b0b1afecd1db2abb126896a3fdc0e50a2235bdddd33866161d848a0e4",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主实验由 GPT-5.2 配合 EvoAgent 执行外贸任务；模型替换实验改用 GPT-4.1 或 Qwen3.5-35B-A3B。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.20133#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "EvoAgent 根据用户反馈调用模型生成、修订技能并组织sub-agent。主配置的基础模型是 GPT-5.2；§4 未为技能修订角色另列一款独立模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.20133#S4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "技能库、技能元数据、记忆与用户信息。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "技能更新主要利用用户交互和使用记录；论文最后另用模型从专业性、准确性等维度评审回答。最终评审分数不应自动当作每次技能更新的奖励。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
},
{
"label": "§3.2：实际奖励说明",
"url": "https://arxiv.org/html/2604.20133#S3.SS2.SSS0.Px6"
},
{
"label": "§3.3：技能成熟度",
"url": "https://arxiv.org/html/2604.20133#S3.SS3.SSS0.Px6"
},
{
"label": "§4.1：回答评估设计",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者自建五层系统：API、路由编排、运行时、工具/会话、持久化；主 task agent 用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 并分层委派子 task agent，逐步披露技能以控制上下文。基础系统已含多 task agent 和记忆管理。",
"sources": [
{
"label": "§3 架构、§4 数据收集",
"url": "https://arxiv.org/abs/2604.20133"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "在线层做技能调用和多 task agent 委派，离线层分析历史对话更新画像、长期记忆和技能库。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "系统从历史会话积累用户画像、记忆与技能；外贸实验脚本采集664段、每段8–9轮的对话池。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "会话内执行与会话间离线分析分开；离线根据使用记录提炼或重构技能。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "随机抽20段对话拆成172个评估实例，§4.1/4.2复用；另测长对话压缩和记忆稳定性。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4提供的是自建系统测试，未建立技能形成材料与172评估实例之间的明确互斥划分。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将长期保存技能与多 agent 任务分派组织到同一助理框架，重点检查积累的能力能否在实际工作中被调用。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2604.20133v3",
"version": "2604.20133v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "更像完整产品/框架整合：skill persistence + delegation；相对前人没有很强的新 evolution principle。",
"feedbackCases": [
{
"label": "外贸会话：技能积累",
"data": "采集 664 段外贸对话，每段 8–9 轮；系统在线对话、离线总结技能。",
"scoring": "实践中没有显式计算理论公式里的数值奖励；依据 usage_count、success_rate 等使用记录和用户交互更新技能。",
"visible": "对话、使用次数及记录的成败；原文未把 success_rate 的每条判定进一步定义成统一外部验收器。",
"use": "离线分析与技能成熟度管理；不能把理论奖励公式描述为实际训练信号。",
"sources": [
{
"label": "§3.2：实际奖励说明",
"url": "https://arxiv.org/html/2604.20133#S3.SS2.SSS0.Px6"
},
{
"label": "§3.3：技能成熟度",
"url": "https://arxiv.org/html/2604.20133#S3.SS3.SSS0.Px6"
}
],
"judgment": "用户交互与技能使用统计，没有显式统一的数值训练奖励"
},
{
"label": "外贸回答质量评价",
"data": "随机抽 20 段对话拆成 172 个实例，§4.1 / §4.2 复用。",
"scoring": "计算长度、长度比、ROUGE-L；语言模型 另按专业性、准确性、完整性、实用性、语言质量各打 1–5 分，随机打乱候选回答顺序。",
"visible": "用于论文比较的五维评分；该评价段未明确评审模型型号。",
"use": "这是实验结果评价，不能直接说五维分数是线上每次技能更新的奖励。",
"sources": [
{
"label": "§4.1：回答评估设计",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px2"
}
],
"judgment": "规则算 ROUGE-L 等；LLM 按五维 1–5 分细则另作质量评审"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "系统从历史会话积累用户画像、记忆与技能；外贸实验脚本采集664段、每段8–9轮的对话池。",
"selection": "会话内执行与会话间离线分析分开；离线根据使用记录提炼或重构技能。",
"evaluation": "随机抽20段对话拆成172个评估实例，§4.1/4.2复用；另测长对话压缩和记忆稳定性。",
"isolation": "§4提供的是自建系统测试，未建立技能形成材料与172评估实例之间的明确互斥划分。",
"roles": {
"executor": {
"value": "主实验由 GPT-5.2 配合 EvoAgent 执行外贸任务；模型替换实验改用 GPT-4.1 或 Qwen3.5-35B-A3B。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.20133#S4.SS2.SSS0.Px1"
}
]
},
"modifier": {
"value": "EvoAgent 根据用户反馈调用模型生成、修订技能并组织sub-agent。主配置的基础模型是 GPT-5.2；§4 未为技能修订角色另列一款独立模型。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.20133#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.20133#S3.SS3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2604.20133#S4.SS2.SSS0.Px1"
}
]
},
"seed": {
"value": "作者自建五层系统：API、路由编排、运行时、工具/会话、持久化；主 task agent 用 ReAct（交替进行推理、调用工具和读取结果的执行方式） 并分层委派子 task agent，逐步披露技能以控制上下文。基础系统已含多 task agent 和记忆管理。",
"sources": [
{
"label": "§3 架构、§4 数据收集",
"url": "https://arxiv.org/abs/2604.20133"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2604.20133#S4.SS1.SSS2.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2604.20133#S4.SS3.SSS1.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "人工技能库制作成本高、难扩展，人类认为合理的流程还可能不适合模型执行。已有自进化多局限于单个提示或工具，缺少对复杂技能包的持续生成、验证和改进；多 agent 分工又常固定不变，使个体技能学习与整体任务安排难以共同适应新需求。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2604.20133#S1"
}
]
},
{
"key": "position",
"value": "面向实际助理的长期服务，解决持续技能学习与复杂任务委派如何协同，使积累的能力能够被合适地调用。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.20133"
}
]
},
{
"key": "conclusion",
"value": "外贸任务中 GPT-5.2 获益，但替换模型后效果不一致，说明模型与框架的匹配很重要。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2604.20133"
}
]
}
],
"fields": {
"modifier": "EvoAgent 根据用户反馈调用模型修订技能并组织sub-agent。主配置的基础模型是 GPT-5.2，技能修订角色没有另行披露独立型号。",
"object": "技能库、技能元数据、记忆与用户信息。",
"verdict": "技能更新主要利用用户交互和使用记录；论文最后另用模型从专业性、准确性等维度评审回答。最终评审分数不应自动当作每次技能更新的奖励。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2605.23904",
"title": "SkillOpt: Executive Strategy for Self-Evolving Agent Skills",
"url": "https://arxiv.org/abs/2605.23904",
"date": "2026-05-22",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"CrossBenchmark",
"CrossModel",
"ExecutableVerifier",
"HeldOut",
"M1",
"RegressionGate",
"SeparateEvolver",
"Skill"
],
"fields": {
"本质定位": "把单一 natural-language skill document 当成 frozen agent 的“可训练外部参数”；optimizer 基于 scored rollouts 做 bounded add/delete/replace，只有 held-out validation 严格变好才接受。",
"什么在变": "一个 skill document。",
"谁来改 / 谁执行": "**改**：separate optimizer LLM；也测 target-as-optimizer。
**执行**：7 target models × direct/Codex/Claude Code 等 harness。",
"基础 harness": "目标 agent + optional skill file。",
"Feedback": "完整 scored trajectories + verifier score；selection validation gate。",
"Evolution → Eval": "6 benchmarks；train/selection/test 分开；还有 cross-model/harness/benchmark transfer。",
"Meta-depth": "M1（slow/meta guidance 不是 M2，outer optimizer logic fixed）。",
"相对之前真正新增什么": "相对 loose reflection 最大的新点是 **optimization discipline**：mini-batch、textual LR、validation gate、rejected buffer、slow update。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "把单一 natural-language skill document 当成 frozen agent 的“可训练外部参数”；optimizer 基于 scored rollouts 做 bounded add/delete/replace，只有 held-out validation 严格变好才接受。 **相对前序：** 相对 loose reflection 最大的新点是 **optimization discipline**：mini-batch、textual LR、validation gate、rejected buffer、slow update。"
},
{
"label": "什么在变",
"text": "一个 skill document。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** separate optimizer LLM；也测 target-as-optimizer。 **执行：** 7 target models × direct/Codex/Claude Code 等 harness。"
},
{
"label": "基础 harness",
"text": "目标 agent + optional skill file。"
},
{
"label": "Feedback",
"text": "完整 scored trajectories + verifier score；selection validation gate。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "6 benchmarks；train/selection/test 分开；还有 cross-model/harness/benchmark transfer。 **Meta-depth：** M1（slow/meta guidance 不是 M2，outer optimizer logic fixed）。"
},
{
"label": "主要结果",
"text": "52/52 cells best/tied-best；GPT-5.5 no-skill 平均 +19~25pt；skill 可跨 harness/model 迁移。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "强 optimizer 时带有 strong-to-weak 成分；skill edit space仍单文档。 **对我们：** 目前最值得我们学的 harness-component evolution protocol：validation gate、rollback、transfer、rejected edits。"
}
],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 310,
"fields": {
"优先级": "**C**",
"时间": "2026-05-22",
"论文": "[SkillOpt: Executive Strategy for Self-Evolving Agent Skills](https://arxiv.org/abs/2605.23904)",
"本质定位": "把单一 natural-language skill document 当成 frozen agent 的“可训练外部参数”；optimizer 基于 scored rollouts 做 bounded add/delete/replace，只有 held-out validation 严格变好才接受。",
"什么在变": "一个 skill document。",
"谁来改 / 谁执行": "**改**：separate optimizer LLM；也测 target-as-optimizer。
**执行**：7 target models × direct/Codex/Claude Code 等 harness。",
"基础 harness": "目标 agent + optional skill file。",
"Feedback": "完整 scored trajectories + verifier score；selection validation gate。",
"Evolution → Eval": "6 benchmarks；train/selection/test 分开；还有 cross-model/harness/benchmark transfer。",
"Meta-depth": "M1（slow/meta guidance 不是 M2，outer optimizer logic fixed）。",
"相对之前真正新增什么": "相对 loose reflection 最大的新点是 **optimization discipline**：mini-batch、textual LR、validation gate、rejected buffer、slow update。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2026",
"depth": [
"M1",
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把一份 skill.md 当成冻结 agent 的外部可训练状态。成功/失败轨迹先按 minibatch 反思，再做有预算的 add/delete/replace；最终只导出最佳被接受 skill。",
"novelty": "对技能文档限制单次修改幅度、进行验证并记住被拒的编辑，让文字技能的更新有可追踪的接受与撤回过程。",
"object": "一份供模型阅读的技能文档，写明领域任务的操作策略；改变的是文档中的文字，目标模型参数和执行程序保持固定。",
"executor": "GPT-5.5、GPT-5.4、GPT-5.4-mini、GPT-5.4-nano、GPT-5.2、Qwen3.5-4B、Qwen3.6-35B-A3B；分别按直接对话/Codex/Claude Code模式运行。",
"modifier": "独立优化器 语言模型，也包含同模型配置；目标模型保持冻结。",
"roleContext": "**改**：separate optimizer LLM；也测 target-as-optimizer。
**执行**：7 target models × direct/Codex/Claude Code 等 harness。",
"seed": "固定 task agent 在执行时参考这份领域操作说明。表格任务仍使用 openpyxl/pandas，OfficeQA 保留工具调用循环，ALFWorld（通过文字动作完成家居物体操作的交互环境） 保留多步动作接口；修改范围是说明文档，不包含这些执行程序。",
"fixed": "target weights 与执行 harness 不变；改变 skill 内容，optimizer 外层规则固定。",
"verdict": "每个任务的评分程序或结果检查器，以及完整的逐步执行记录和对应分数。",
"diagnosis": "分开汇总成功与失败 minibatches，找缺失程序规则和应保留的行为。",
"update": "在 edit budget 内合并、排序并执行 add/delete/replace。",
"acceptance": "selection set 严格改善才接受；保留 rejected-edit buffer；导出 best_skill.md。",
"experiments": [
{
"name": "六个 domain",
"evolve": "D_train 生成经验与候选 skills",
"selection": "D_selection gate 接受与选模",
"test": "D_test 仅最终报告",
"isolation": "独立测试",
"note": "还有 cross-model / cross-harness / cross-benchmark transfer。"
}
],
"takeaway": "强 optimizer 时带有 strong-to-weak 成分；skill edit space仍单文档。 **对我们：** 目前最值得我们学的 harness-component evolution protocol：validation gate、rollback、transfer、rejected edits。",
"result": "52/52 cells best/tied-best；GPT-5.5 no-skill 平均 +19~25pt；skill 可跨 harness/model 迁移。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2605.23904v2"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SearchQA、SpreadsheetBench、OfficeQA、DocVQA、LiveMathematicianBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）；无特殊声明默认20%用于技能学习。ALFWorld实际39训练任务，LiveMath每轮35训练题。\n\n调试 / 选版本数据：默认10% 选择，仅据此接受或拒绝编辑；ALFWorld（通过文字动作完成家居物体操作的交互环境）使用140 选择。分数须严格提高，平分拒绝。\n\n最终测试数据：默认70%互斥 测试集；ALFWorld134 测试集。主表使用各任务原生成功率/精确匹配；附录协议还覆盖 SealQA。\n\n数据隔离与证据边界：seed=42确定划分；编辑只看训练反馈，选择做门控，测试集不参与接受修改。",
"cycle": "固定模型及 直接对话/Codex/Claude Code 执行框架，优化共享格式的 SKILL.md；合并成败反思、限制编辑数，用验证门控和优化器侧经验防止退化。",
"train": "SearchQA、SpreadsheetBench、OfficeQA、DocVQA、LiveMathematicianBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）；无特殊声明默认20%用于技能学习。ALFWorld实际39训练任务，LiveMath每轮35训练题。",
"debug": "默认10% 选择，仅据此接受或拒绝编辑；ALFWorld（通过文字动作完成家居物体操作的交互环境）使用140 选择。分数须严格提高，平分拒绝。",
"test": "默认70%互斥 测试集；ALFWorld134 测试集。主表使用各任务原生成功率/精确匹配；附录协议还覆盖 SealQA。",
"isolation": "seed=42确定划分；编辑只看训练反馈，选择做门控，测试集不参与接受修改。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "在固定目标 agent 上编辑一份领域 skill 文档，执行器和工具按任务保留：表格用 openpyxl/pandas，OfficeQA 有工具循环，ALFWorld 有多步动作。它不开放修改全部 runtime。",
"protocol": "**进化/选版本/测试：**SearchQA、SpreadsheetBench、OfficeQA、DocVQA、LiveMathematicianBench、ALFWorld 等按 seed=42 生成确定划分；未另声明时为 2:1:7。训练反馈用于写技能，selection 仅用于接受或拒绝候选，最终分数来自互斥 test。\n\n**边界：**不同任务交互长度不同，不能用单轮 QA 协议概括所有任务；每个数据集绝对数量与特殊划分本轮待核实。",
"sections": "实验设置与数据划分",
"source": "https://arxiv.org/abs/2605.23904",
"version": "2605.23904v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "91e6021e8a54251422070917d4eda4161e04b094a219e5d789c3b30a9e2202a9",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "GPT-5.5、GPT-5.4、GPT-5.4-mini、GPT-5.4-nano、GPT-5.2、Qwen3.5-4B、Qwen3.6-35B-A3B；分别按直接对话/Codex/Claude Code模式运行。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.23904#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "独立优化器 语言模型，也包含同模型配置；目标模型保持冻结。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "一份供模型阅读的技能文档，写明领域任务的操作策略；改变的是文档中的文字，目标模型参数和执行程序保持固定。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "每个任务的评分程序或结果检查器，以及完整的逐步执行记录和对应分数。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定 task agent 在执行时参考这份领域操作说明。表格任务仍使用 openpyxl/pandas，OfficeQA 保留工具调用循环，ALFWorld（通过文字动作完成家居物体操作的交互环境） 保留多步动作接口；修改范围是说明文档，不包含这些执行程序。",
"sources": [
{
"label": "实验设置与数据划分",
"url": "https://arxiv.org/abs/2605.23904"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "固定模型及 直接对话/Codex/Claude Code 执行框架，优化共享格式的 SKILL.md；合并成败反思、限制编辑数，用验证门控和优化器侧经验防止退化。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SearchQA、SpreadsheetBench、OfficeQA、DocVQA、LiveMathematicianBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）；无特殊声明默认20%用于技能学习。ALFWorld实际39训练任务，LiveMath每轮35训练题。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "默认10% 选择，仅据此接受或拒绝编辑；ALFWorld（通过文字动作完成家居物体操作的交互环境）使用140 选择。分数须严格提高，平分拒绝。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "默认70%互斥 测试集；ALFWorld134 测试集。主表使用各任务原生成功率/精确匹配；附录协议还覆盖 SealQA。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "seed=42确定划分；编辑只看训练反馈，选择做门控，测试集不参与接受修改。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "对技能文档限制单次修改幅度、进行验证并记住被拒的编辑，让文字技能的更新有可追踪的接受与撤回过程。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.23904v2",
"version": "2605.23904v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "重点是单一技能文档的优化纪律：textual learning rate、验证 gate、被拒编辑缓存和跨 epoch 慢更新；不是完整 runtime code evolution。",
"feedbackCases": [
{
"label": "SearchQA / DocVQA",
"data": "默认 20% 技能学习、10% 选择、70% 测试",
"scoring": "按题目参考答案的硬正确性/精确匹配评分，保留任务原生评分器。",
"visible": "修改者读取训练任务的消息、工具结果、最终答案及判分；候选技能另在选择集运行。",
"use": "选择分数严格提高才接受文字修改，平分也拒绝；最终只报告互斥测试集成绩。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"judgment": "使用任务原生硬分／精确匹配；本文未逐项列出答案解析器"
},
{
"label": "SpreadsheetBench",
"data": "同样使用互斥训练/选择/测试；执行器可运行 openpyxl/pandas",
"scoring": "运行表格操作代码，并按基准的表格产物验收给成功分；不是凭代码文字优劣评分。",
"visible": "修改者读取训练任务的消息、工具结果、最终答案及判分；候选技能另在选择集运行。",
"use": "选择分数严格提高才接受文字修改，平分也拒绝；最终只报告互斥测试集成绩。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"judgment": "SpreadsheetBench 原生产物检查程序"
},
{
"label": "OfficeQA / SealQA",
"data": "OfficeQA 为主表文档问答；SealQA 见附录协议",
"scoring": "按问答任务参考答案判断结果，完整工具调用、文档引用和输出同时写入执行记录。",
"visible": "修改者读取训练任务的消息、工具结果、最终答案及判分；候选技能另在选择集运行。",
"use": "选择分数严格提高才接受文字修改，平分也拒绝；最终只报告互斥测试集成绩。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"judgment": "OfficeQA／SealQA 参考答案判分；本文未逐项披露比对器实现"
},
{
"label": "LiveMathematicianBench",
"data": "每轮 35 道训练题，另有选择集与测试集",
"scoring": "数学选择题按参考选项判正确性。",
"visible": "修改者读取训练任务的消息、工具结果、最终答案及判分；候选技能另在选择集运行。",
"use": "选择分数严格提高才接受文字修改，平分也拒绝；最终只报告互斥测试集成绩。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"judgment": "规则比较数学选择题的输出选项与标准选项"
},
{
"label": "ALFWorld",
"data": "39 个训练任务、140 个选择任务、134 个测试任务",
"scoring": "环境检查最终操作目标是否完成，给任务成功/失败。",
"visible": "修改者读取训练任务的消息、工具结果、最终答案及判分；候选技能另在选择集运行。",
"use": "选择分数严格提高才接受文字修改，平分也拒绝；最终只报告互斥测试集成绩。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"judgment": "ALFWorld 环境检查最终任务状态"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4
],
"evolution": "SearchQA、SpreadsheetBench、OfficeQA、DocVQA、LiveMathematicianBench、ALFWorld（通过文字动作完成家居物体操作的交互环境）；无特殊声明默认20%用于技能学习。ALFWorld实际39训练任务，LiveMath每轮35训练题。",
"selection": "默认10% 选择，仅据此接受或拒绝编辑；ALFWorld（通过文字动作完成家居物体操作的交互环境）使用140 选择。分数须严格提高，平分拒绝。",
"evaluation": "默认70%互斥 测试集；ALFWorld134 测试集。主表使用各任务原生成功率/精确匹配；附录协议还覆盖 SealQA。",
"isolation": "seed=42确定划分；编辑只看训练反馈，选择做门控，测试集不参与接受修改。",
"roles": {
"executor": {
"value": "GPT-5.5、GPT-5.4、GPT-5.4-mini、GPT-5.4-nano、GPT-5.2、Qwen3.5-4B、Qwen3.6-35B-A3B；分别按直接对话/Codex/Claude Code模式运行。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.23904#S4.SS1"
}
]
},
"modifier": {
"value": "独立优化器 语言模型，也包含同模型配置；目标模型保持冻结。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
]
},
"seed": {
"value": "固定 task agent 在执行时参考这份领域操作说明。表格任务仍使用 openpyxl/pandas，OfficeQA 保留工具调用循环，ALFWorld（通过文字动作完成家居物体操作的交互环境） 保留多步动作接口；修改范围是说明文档，不包含这些执行程序。",
"sources": [
{
"label": "实验设置与数据划分",
"url": "https://arxiv.org/abs/2605.23904"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2605.23904#A3.SS0.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "闭源模型通常无法改权重，开源模型训练又昂贵，因此外部技能成为适应新领域的重要途径。但人工或一次性生成的技能在具体任务和运行框架下容易失效，已有经验提炼方法也未充分回答怎样稳定优化技能，因而需要研究可控制、可验证的连续修改过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.23904#S1"
}
]
},
{
"key": "position",
"value": "研究外部文字技能能否成为稳定、可优化的学习对象，使冻结模型通过技能更新持续适应任务，并跨模型或工具框架复用收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.23904"
}
]
},
{
"key": "conclusion",
"value": "展示技能在多种任务、模型和工具框架间的应用与迁移，只导出通过验证的技能产物。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.23904"
}
]
}
],
"fields": {
"object": "一份供模型阅读的技能文档，写明领域任务的操作策略；改变的是文档中的文字，目标模型参数和执行程序保持固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2607.00272",
"title": "ASPIRE: Agentic /Skills Discovery for Robotics",
"url": "https://arxiv.org/abs/2607.00272",
"date": "2026-06-30",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Skill",
"org:cmu",
"org:berkeley",
"org:uiuc"
],
"fields": {
"本质定位": "coding agent 根据逐原语多模态 robot traces 诊断失败、修 control program，验证成功修复后蒸馏进持续扩张 skill library；另用 evolutionary search 探索 task sequences/programs。",
"什么在变": "robot control programs + reusable skill library。",
"谁来改 / 谁执行": "**改**：coding agent + fixed evolutionary search/validation infrastructure。
**执行**：robot execution engine。",
"基础 harness": "code-as-policy + skill library。",
"Feedback": "multimodal execution traces、physical/sim success/failure、validation。",
"Evolution → Eval": "LIBERO-Pro、Robosuite、BEHAVIOR-1K；含 unseen long-horizon 与 sim-to-real。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "把 Voyager-style executable skill accumulation 推到真实 robotics，并强调 fine-grained physical feedback、cross-task/sim2real/embodiment transfer。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 311,
"fields": {
"优先级": "**R**",
"时间": "2026-06-30",
"论文": "[ASPIRE: Agentic /Skills Discovery for Robotics](https://arxiv.org/abs/2607.00272)",
"本质定位": "coding agent 根据逐原语多模态 robot traces 诊断失败、修 control program，验证成功修复后蒸馏进持续扩张 skill library；另用 evolutionary search 探索 task sequences/programs。",
"什么在变": "robot control programs + reusable skill library。",
"谁来改 / 谁执行": "**改**：coding agent + fixed evolutionary search/validation infrastructure。
**执行**：robot execution engine。",
"基础 harness": "code-as-policy + skill library。",
"Feedback": "multimodal execution traces、physical/sim success/failure、validation。",
"Evolution → Eval": "LIBERO-Pro、Robosuite、BEHAVIOR-1K；含 unseen long-horizon 与 sim-to-real。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "把 Voyager-style executable skill accumulation 推到真实 robotics，并强调 fine-grained physical feedback、cross-task/sim2real/embodiment transfer。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "coding agent 根据逐原语多模态 robot traces 诊断失败、修 control program，验证成功修复后蒸馏进持续扩张 skill library；另用 evolutionary search 探索 task sequences/programs。",
"novelty": "从机器人视觉和物理执行失败中提炼可复用修复技能，并检查这些技能能否跨任务、环境及部分仿真到实物设置迁移。",
"object": "机器人控制程序及可跨任务复用的技能库。",
"executor": "仿真中由Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6（1M上下文）写程序，机器人执行引擎实际执行控制；真实双臂YAM迁移实验用Codex + GPT-5.5 xhigh。",
"modifier": "仿真编码与修复 task agent 为Claude Opus4.6；真实迁移为GPT-5.5 xhigh。协调角色管理技能，执行角色运行、诊断、修改机器人程序。",
"roleContext": "**改**：coding agent + fixed evolutionary search/validation infrastructure。
**执行**：robot execution engine。",
"seed": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）＋Claude Opus 4.6 写 Python 机器人程序，基于 CaP-X/MuJoCo Playground 的感知、几何、运动规划 API。task agent、环境及 API 固定，增长的是可复用机器人技能，不是更换低层执行接口。",
"fixed": "",
"verdict": "机器人模拟器判断目标是否完成，执行视频、关键帧和工具记录帮助定位抓取、移动等步骤的失败；调试与最终测试采用不同随机种子。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "LIBERO repair-skill transfer",
"evolve": "LIBERO-90 上 0/25/50/90 个 source tasks 形成 library snapshots",
"selection": "使用不同规模冻结技能库",
"test": "LIBERO-Pro Long held-out tasks",
"isolation": "冻结后零样本迁移",
"note": "每个测试任务只生成一个程序；跨 seeds 执行，无额外调试、重试或任务专门技能更新。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2607.00272v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：LIBERO-Pro每任务seed51–65；Robosuite101–125；BEHAVIOR-1K26–35。跨任务迁移另从 LIBERO-90 累积技能。\n\n调试 / 选版本数据：执行角色 在上述调试种子编写、执行、诊断、修复机器人程序；协调角色 把可复用经验保存到技能库。\n\n最终测试数据：LIBERO-Pro每任务seed1–50；Robosuite1–100；BEHAVIOR-1K两任务各1–25。另用 LIBERO-90 技能零样本迁移到 LIBERO-Pro Long。\n\n数据隔离与证据边界：调试与评测种子互斥。LIBERO/Robosuite每任务冻结一个程序；BEHAVIOR评测按当前多模态轨迹增量生成代码块。",
"cycle": "协调角色管理共享技能库，执行角色编写、运行并修复机器人程序。各次任务共享可复用技能，而非整段聊天；动作前后观察和关键帧用于定位失败步骤。",
"train": "LIBERO-Pro每任务seed51–65；Robosuite101–125；BEHAVIOR-1K26–35。跨任务迁移另从 LIBERO-90 累积技能。",
"debug": "执行角色 在上述调试种子编写、执行、诊断、修复机器人程序；协调角色 把可复用经验保存到技能库。",
"test": "LIBERO-Pro每任务seed1–50；Robosuite1–100；BEHAVIOR-1K两任务各1–25。另用 LIBERO-90 技能零样本迁移到 LIBERO-Pro Long。",
"isolation": "调试与评测种子互斥。LIBERO/Robosuite每任务冻结一个程序；BEHAVIOR评测按当前多模态轨迹增量生成代码块。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "Claude Code＋Claude Opus 4.6 写 Python 机器人程序，基于 CaP-X/MuJoCo Playground 的感知、几何、运动规划 API。agent、环境及 API 固定，增长的是可复用机器人技能，不是更换低层执行接口。",
"protocol": "**技能来源→迁移：**在 LIBERO-90 源任务上积累技能，保存 0/25/50/90 任务快照；向 LIBERO-Pro/Long 等留出任务迁移。\n\n**调试与最终测试：**候选在 debug seeds 上迭代，选好一个程序后只在 held-out seeds 执行，不额外调试或更新。需要区分新任务迁移与同任务新随机种子；完整各套件 seed 数本轮待核实。",
"sections": "模拟环境设置与技能迁移协议",
"source": "https://arxiv.org/abs/2607.00272",
"version": "2607.00272v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "5008afa1d4b145ea72b2292c4aeedbd8289d74b452c4f6a647d64aee9749930e",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "仿真中由Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6（1M上下文）写程序，机器人执行引擎实际执行控制；真实双臂YAM迁移实验用Codex + GPT-5.5 xhigh。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "仿真编码与修复 task agent 为Claude Opus4.6；真实迁移为GPT-5.5 xhigh。协调角色管理技能，执行角色运行、诊断、修改机器人程序。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "机器人控制程序及可跨任务复用的技能库。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "机器人模拟器判断目标是否完成，执行视频、关键帧和工具记录帮助定位抓取、移动等步骤的失败；调试与最终测试采用不同随机种子。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§2.3：搜索算法",
"url": "https://arxiv.org/html/2607.00272#S2.SS3"
},
{
"label": "附录 E.2：接口及成功验收",
"url": "https://arxiv.org/html/2607.00272#A5.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）＋Claude Opus 4.6 写 Python 机器人程序，基于 CaP-X/MuJoCo Playground 的感知、几何、运动规划 API。task agent、环境及 API 固定，增长的是可复用机器人技能，不是更换低层执行接口。",
"sources": [
{
"label": "模拟环境设置与技能迁移协议",
"url": "https://arxiv.org/abs/2607.00272"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "协调角色管理共享技能库，执行角色编写、运行并修复机器人程序。各次任务共享可复用技能，而非整段聊天；动作前后观察和关键帧用于定位失败步骤。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LIBERO-Pro每任务seed51–65；Robosuite101–125；BEHAVIOR-1K26–35。跨任务迁移另从 LIBERO-90 累积技能。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "执行角色 在上述调试种子编写、执行、诊断、修复机器人程序；协调角色 把可复用经验保存到技能库。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "LIBERO-Pro每任务seed1–50；Robosuite1–100；BEHAVIOR-1K两任务各1–25。另用 LIBERO-90 技能零样本迁移到 LIBERO-Pro Long。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "调试与评测种子互斥。LIBERO/Robosuite每任务冻结一个程序；BEHAVIOR评测按当前多模态轨迹增量生成代码块。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从机器人视觉和物理执行失败中提炼可复用修复技能，并检查这些技能能否跨任务、环境及部分仿真到实物设置迁移。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.00272v1",
"version": "2607.00272v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 Voyager-style executable skill accumulation 推到真实 robotics，并强调 fine-grained physical feedback、cross-task/sim2real/embodiment transfer。",
"feedbackCases": [
{
"label": "LIBERO-Pro：机器人程序搜索与测试",
"data": "每任务调试 seed 51–65；最终 seed 1–50。",
"scoring": "沙箱模拟器按任务的成功条件验收；执行角色 可看执行产生的视觉记录和工具结果，但禁止调用获取模拟器真值的 API。",
"visible": "调试分数、动作前后观察和多模态轨迹，定位抓取、移动等步骤的失败。",
"use": "执行角色 提出并比较修复程序，协调角色 保存复用技能；最终使用不同种子评价。",
"sources": [
{
"label": "§2.3：搜索算法",
"url": "https://arxiv.org/html/2607.00272#S2.SS3"
},
{
"label": "附录 E.2：接口及成功验收",
"url": "https://arxiv.org/html/2607.00272#A5.SS2"
}
],
"judgment": "LIBERO-Pro 模拟器原生成功条件"
},
{
"label": "Robosuite：机器人程序搜索与测试",
"data": "每任务调试 seed 101–125；最终 seed 1–100。",
"scoring": "沙箱模拟器按任务的成功条件验收；执行角色 可看执行产生的视觉记录和工具结果，但禁止调用获取模拟器真值的 API。",
"visible": "调试分数、动作前后观察和多模态轨迹，定位抓取、移动等步骤的失败。",
"use": "执行角色 提出并比较修复程序，协调角色 保存复用技能；最终使用不同种子评价。",
"sources": [
{
"label": "§2.3：搜索算法",
"url": "https://arxiv.org/html/2607.00272#S2.SS3"
},
{
"label": "附录 E.2：接口及成功验收",
"url": "https://arxiv.org/html/2607.00272#A5.SS2"
}
],
"judgment": "Robosuite 模拟器原生成功条件"
},
{
"label": "BEHAVIOR-1K：机器人程序搜索与测试",
"data": "每任务调试 seed 26–35；最终 seed 1–25（两项任务分别测试）。",
"scoring": "沙箱模拟器按任务的成功条件验收；执行角色 可看执行产生的视觉记录和工具结果，但禁止调用获取模拟器真值的 API。",
"visible": "调试分数、动作前后观察和多模态轨迹，定位抓取、移动等步骤的失败。",
"use": "执行角色 提出并比较修复程序，协调角色 保存复用技能；最终使用不同种子评价。",
"sources": [
{
"label": "§2.3：搜索算法",
"url": "https://arxiv.org/html/2607.00272#S2.SS3"
},
{
"label": "附录 E.2：接口及成功验收",
"url": "https://arxiv.org/html/2607.00272#A5.SS2"
}
],
"judgment": "BEHAVIOR-1K 模拟器原生成功条件"
},
{
"label": "跨任务技能迁移",
"data": "从 LIBERO-90 积累技能，零样本测试 LIBERO-Pro Long。",
"scoring": "目标环境原生任务成功检查。",
"visible": "迁移组最终成功率。",
"use": "这一组评估先前技能的复用，不是在 Long 测试题上继续调试。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"judgment": "目标模拟环境原生成功检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "LIBERO-Pro每任务seed51–65；Robosuite101–125；BEHAVIOR-1K26–35。跨任务迁移另从 LIBERO-90 累积技能。",
"selection": "执行角色 在上述调试种子编写、执行、诊断、修复机器人程序；协调角色 把可复用经验保存到技能库。",
"evaluation": "LIBERO-Pro每任务seed1–50；Robosuite1–100；BEHAVIOR-1K两任务各1–25。另用 LIBERO-90 技能零样本迁移到 LIBERO-Pro Long。",
"isolation": "调试与评测种子互斥。LIBERO/Robosuite每任务冻结一个程序；BEHAVIOR评测按当前多模态轨迹增量生成代码块。",
"roles": {
"executor": {
"value": "仿真中由Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Claude Opus4.6（1M上下文）写程序，机器人执行引擎实际执行控制；真实双臂YAM迁移实验用Codex + GPT-5.5 xhigh。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "仿真编码与修复 task agent 为Claude Opus4.6；真实迁移为GPT-5.5 xhigh。协调角色管理技能，执行角色运行、诊断、修改机器人程序。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2607.00272#S2"
},
{
"label": "附录E.4",
"url": "https://arxiv.org/html/2607.00272#A5.SS4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
}
]
},
"seed": {
"value": "Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具）＋Claude Opus 4.6 写 Python 机器人程序，基于 CaP-X/MuJoCo Playground 的感知、几何、运动规划 API。task agent、环境及 API 固定，增长的是可复用机器人技能，不是更换低层执行接口。",
"sources": [
{
"label": "模拟环境设置与技能迁移协议",
"url": "https://arxiv.org/abs/2607.00272"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.00272#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.00272#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.00272#S3.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "机器人失败可能来自视觉识别、抓取、路径规划或后续恢复，但已有代码 agent 常只收到“任务失败”这样的粗反馈，无法定位该检查哪个模块、如何修复。系统又不积累跨任务经验，导致相似问题反复调试；作者希望让诊断证据与可复用修复经验都进入学习过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.00272#S1"
}
]
},
{
"key": "position",
"value": "面向机器人操作，研究 agent 能否从多模态执行失败中自主学会可复用修复技能，并适应未见任务与环境变化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.00272"
}
]
},
{
"key": "conclusion",
"value": "在多种操作任务上改善适应，展示未见任务泛化及初步仿真到实物技能迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.00272"
}
]
}
],
"fields": {
"object": "机器人控制程序及可跨任务复用的技能库。",
"verdict": "机器人模拟器判断目标是否完成，执行视频、关键帧和工具记录帮助定位抓取、移动等步骤的失败；调试与最终测试采用不同随机种子。"
}
},
"attributions": [
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.00272"
}
]
},
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.00272"
}
]
},
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.00272"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2607.26784",
"title": "SkillRise: Agentic Reinforcement Learning for Cross-Task Skill Evolution",
"url": "https://arxiv.org/abs/2607.26784",
"date": "2026-07-29",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:sjtu",
"org:nus"
],
"fields": {
"本质定位": "同一 policy 在 task solving 与 skill-document curation 两个角色间交替；训练时用跨任务 downstream reward 学“什么时候/怎样更新 skill”。",
"什么在变": "runtime skill document + model weights（学会 curate）。",
"谁来改 / 谁执行": "**改**：同一 policy，经 RL 学习 curation action。
**执行**：Qwen3 系列 agent。",
"基础 harness": "agent + persistent skill doc。",
"Feedback": "当前任务 outcome + 下游任务对 skill edit 的 delayed credit。",
"Evolution → Eval": "ALFWorld/WebShop/ScienceWorld；sequence-length scaling。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "和 SkillOpt 不同：SkillOpt 冻结 target、外部 optimizer 训练 skill；SkillRise **训练模型学会 skill curation policy**。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 313,
"fields": {
"优先级": "**R**",
"时间": "2026-07-29",
"论文": "[SkillRise: Agentic Reinforcement Learning for Cross-Task Skill Evolution](https://arxiv.org/abs/2607.26784)",
"本质定位": "同一 policy 在 task solving 与 skill-document curation 两个角色间交替；训练时用跨任务 downstream reward 学“什么时候/怎样更新 skill”。",
"什么在变": "runtime skill document + model weights（学会 curate）。",
"谁来改 / 谁执行": "**改**：同一 policy，经 RL 学习 curation action。
**执行**：Qwen3 系列 agent。",
"基础 harness": "agent + persistent skill doc。",
"Feedback": "当前任务 outcome + 下游任务对 skill edit 的 delayed credit。",
"Evolution → Eval": "ALFWorld/WebShop/ScienceWorld；sequence-length scaling。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "和 SkillOpt 不同：SkillOpt 冻结 target、外部 optimizer 训练 skill；SkillRise **训练模型学会 skill curation policy**。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "同一 policy 在 task solving 与 skill-document curation 两个角色间交替；训练时用跨任务 downstream reward 学“什么时候/怎样更新 skill”。",
"novelty": "把当前任务成功的奖励与写出的技能帮助后续任务的奖励分开，训练 agent 为未来任务整理经验的能力。",
"object": "跨任务传递的技能文档，以及学习如何执行和整理技能的模型参数。",
"executor": "Qwen3-1.7B和Qwen3-4B；主表展示4B在ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、ScienceWorld的结果。",
"modifier": "同一Qwen3策略交替解题与整理技能，通过跨任务强化学习（根据奖励调整模型行为）训练；不是另一个外部语言模型负责写技能。",
"roleContext": "**改**：同一 policy，经 RL 学习 curation action。
**执行**：Qwen3 系列 agent。",
"seed": "同一可训练行为策略交替做任务和整理持续技能文档；执行获得当前任务奖励，整理技能获得后续任务收益的信用。既训练参数也改变外部技能，不是单纯事后保存成功轨迹。",
"fixed": "",
"verdict": "当前任务的结果，以及后续任务表现对先前技能修改的评价；后者用于判断这次修改是否对未来任务有帮助。",
"diagnosis": "不把成功全归给当轮 solver：当前任务结果监督 solving，折扣后的后续任务结果监督 skill curation。",
"update": "一个 policy 在相关任务序列中交替解题与编辑 skill document，文档直接交给下一任务。",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "ALFWorld/WebShop/ScienceWorld；sequence-length scaling。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2607.26784v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：ALFWorld（通过文字动作完成家居物体操作的交互环境）、1000商品版 WebShop（合成目标）、ScienceWorld；每更新16序列×3任务×8 trials，共384次任务执行，最多150更新。\n\n调试 / 选版本数据：同一策略交替解题和整理技能，把后续任务收益分配给较早技能更新。\n\n最终测试数据：每环境128个留出任务。Pass@2/3允许同题带技能重试；ALFWorld（通过文字动作完成家居物体操作的交互环境）另按2/4/6题序列、每题一次测试跨题积累。\n\n数据隔离与证据边界：留出指训练题之外，测试阶段技能仍会更新；同题重试成绩与跨题一次执行成绩需分开读。",
"cycle": "用跨任务 强化学习（根据奖励调整模型行为） 联合训练求解与技能整理，执行反馈通过时间对齐奖励训练可迁移的技能更新方式。",
"train": "ALFWorld（通过文字动作完成家居物体操作的交互环境）、1000商品版 WebShop（合成目标）、ScienceWorld；每更新16序列×3任务×8 trials，共384次任务执行，最多150更新。",
"debug": "同一策略交替解题和整理技能，把后续任务收益分配给较早技能更新。",
"test": "每环境128个留出任务。Pass@2/3允许同题带技能重试；ALFWorld（通过文字动作完成家居物体操作的交互环境）另按2/4/6题序列、每题一次测试跨题积累。",
"isolation": "留出指训练题之外，测试阶段技能仍会更新；同题重试成绩与跨题一次执行成绩需分开读。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "同一可训练 policy 交替做任务和整理持续技能文档；执行获得当前任务奖励，整理技能获得后续任务收益的信用。既训练参数也改变外部技能，不是单纯事后保存成功轨迹。",
"protocol": "**训练/评测：**ALFWorld、WebShop、ScienceWorld，训练以不同任务组成的序列学习技能整理。\n\n**测试需分开：**Pass@2/3 是同一留出任务重试且携带更新后的技能；跨任务实验把同一 128 个 ALFWorld 留出任务分成长度 2/4/6 的序列，每题只尝试一次但允许序列内更新技能。二者都不是测试全过程冻结技能。训练任务数量本轮待核实。",
"sections": "主实验、§5.1",
"source": "https://arxiv.org/abs/2607.26784",
"version": "2607.26784v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "45ee957da726f34944cbaf70e3d5f4c34cb29b082c5e6c1dfade931064078d18",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3-1.7B和Qwen3-4B；主表展示4B在ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、ScienceWorld的结果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一Qwen3策略交替解题与整理技能，通过跨任务强化学习（根据奖励调整模型行为）训练；不是另一个外部语言模型负责写技能。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "跨任务传递的技能文档，以及学习如何执行和整理技能的模型参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "当前任务的结果，以及后续任务表现对先前技能修改的评价；后者用于判断这次修改是否对未来任务有帮助。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "同一可训练行为策略交替做任务和整理持续技能文档；执行获得当前任务奖励，整理技能获得后续任务收益的信用。既训练参数也改变外部技能，不是单纯事后保存成功轨迹。",
"sources": [
{
"label": "主实验、§5.1",
"url": "https://arxiv.org/abs/2607.26784"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "用跨任务 强化学习（根据奖励调整模型行为） 联合训练求解与技能整理，执行反馈通过时间对齐奖励训练可迁移的技能更新方式。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "ALFWorld（通过文字动作完成家居物体操作的交互环境）、1000商品版 WebShop（合成目标）、ScienceWorld；每更新16序列×3任务×8 trials，共384次任务执行，最多150更新。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同一策略交替解题和整理技能，把后续任务收益分配给较早技能更新。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "每环境128个留出任务。Pass@2/3允许同题带技能重试；ALFWorld（通过文字动作完成家居物体操作的交互环境）另按2/4/6题序列、每题一次测试跨题积累。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "留出指训练题之外，测试阶段技能仍会更新；同题重试成绩与跨题一次执行成绩需分开读。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把当前任务成功的奖励与写出的技能帮助后续任务的奖励分开，训练 agent 为未来任务整理经验的能力。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.26784v1",
"version": "2607.26784v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "明确区分“当前任务做得好”和“写出的技能让后续任务做得好”的 credit；这是跨任务训练目标的差别，不只是把经验存进 skill 文件。",
"feedbackCases": [
{
"label": "ALFWorld：跨任务技能强化学习",
"data": "每更新 16 条序列×3 个任务×8 次尝试，共 384 次执行，最多 150 次更新；每环境另有 128 个留出任务。",
"scoring": "环境检查物体操作目标是否完成。",
"visible": "当前解题结果，以及后续任务取得的奖励；后者回分给较早的技能整理动作。",
"use": "训练同一策略解题与改技能；最终 Pass@2/3 允许同题携技能重试，ALFWorld（通过文字动作完成家居物体操作的交互环境） 另用 2/4/6 题序列、每题一次评价跨题积累。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"judgment": "ALFWorld 环境程序检查物体操作目标"
},
{
"label": "WebShop：跨任务技能强化学习",
"data": "每更新 16 条序列×3 个任务×8 次尝试，共 384 次执行，最多 150 次更新；每环境另有 128 个留出任务。",
"scoring": "1,000 商品版本使用合成购物目标，环境按购买行为是否满足任务约束评价。",
"visible": "当前解题结果，以及后续任务取得的奖励；后者回分给较早的技能整理动作。",
"use": "训练同一策略解题与改技能；最终 Pass@2/3 允许同题携技能重试，ALFWorld（通过文字动作完成家居物体操作的交互环境） 另用 2/4/6 题序列、每题一次评价跨题积累。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"judgment": "WebShop 环境程序检查购买结果与目标约束"
},
{
"label": "ScienceWorld：跨任务技能强化学习",
"data": "每更新 16 条序列×3 个任务×8 次尝试，共 384 次执行，最多 150 次更新；每环境另有 128 个留出任务。",
"scoring": "科学实验环境依据任务目标完成情况提供可验证结果。",
"visible": "当前解题结果，以及后续任务取得的奖励；后者回分给较早的技能整理动作。",
"use": "训练同一策略解题与改技能；最终 Pass@2/3 允许同题携技能重试，ALFWorld（通过文字动作完成家居物体操作的交互环境） 另用 2/4/6 题序列、每题一次评价跨题积累。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"judgment": "ScienceWorld 环境程序计算实验目标进度／完成"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "ALFWorld（通过文字动作完成家居物体操作的交互环境）、1000商品版 WebShop（合成目标）、ScienceWorld；每更新16序列×3任务×8 trials，共384次任务执行，最多150更新。",
"selection": "同一策略交替解题和整理技能，把后续任务收益分配给较早技能更新。",
"evaluation": "每环境128个留出任务。Pass@2/3允许同题带技能重试；ALFWorld（通过文字动作完成家居物体操作的交互环境）另按2/4/6题序列、每题一次测试跨题积累。",
"isolation": "留出指训练题之外，测试阶段技能仍会更新；同题重试成绩与跨题一次执行成绩需分开读。",
"roles": {
"executor": {
"value": "Qwen3-1.7B和Qwen3-4B；主表展示4B在ALFWorld（通过文字动作完成家居物体操作的交互环境）、WebShop（根据用户要求挑选和购买商品的交互基准）、ScienceWorld的结果。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
]
},
"modifier": {
"value": "同一Qwen3策略交替解题与整理技能，通过跨任务强化学习（根据奖励调整模型行为）训练；不是另一个外部语言模型负责写技能。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
]
},
"seed": {
"value": "同一可训练行为策略交替做任务和整理持续技能文档；执行获得当前任务奖励，整理技能获得后续任务收益的信用。既训练参数也改变外部技能，不是单纯事后保存成功轨迹。",
"sources": [
{
"label": "主实验、§5.1",
"url": "https://arxiv.org/abs/2607.26784"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.26784#S4.SS2.SSS0.Px2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.26784#S5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "标准 agent 强化学习常把任务当成独立回合，丢弃其中的可复用经验，导致相似任务仍要重新探索。反复做同一道题未必学到可迁移技能；多阶段技能方法又难区分提炼、检索和执行各自的贡献，因此缺少直接按后续任务收益训练技能学习的办法。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.26784#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 能否学会为未来任务整理技能，并使跨任务经验积累带来的收益成为可训练的能力，而非只擅长同题重试。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.26784"
}
]
},
{
"key": "conclusion",
"value": "在三种交互环境中改善成功率，相关任务序列变长时仍能获益，而非依赖同题反复尝试。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.26784"
}
]
}
],
"fields": {
"object": "跨任务传递的技能文档，以及学习如何执行和整理技能的模型参数。"
}
},
"attributions": [
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.26784"
}
]
},
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.26784"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights",
"experience",
"improver"
]
},
{
"id": "2608.11350",
"title": "SHAPER: Self-Evolving Embodied Agents via Skill-Harness Evolution",
"url": "https://arxiv.org/abs/2608.11350",
"date": "2026-08-11",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Skill"
],
"fields": {
"本质定位": "冻结 foundation model，同一模型分别作为 planner/optimizer；利用 target-environment rollouts 同时改 reusable textual skill 与 context-code harness。",
"什么在变": "skill + context-code harness。",
"谁来改 / 谁执行": "**改**：同一 frozen model optimizer role。
**执行**：frozen upper-level VLM planner；下层 VLA/API 也固定。",
"基础 harness": "seed planner + minimal/fixed action interface。",
"Feedback": "episode summaries、execution statistics、environment rollout outcome；VLABench/ESI-Bench grounded evidence。",
"Evolution → Eval": "VLABench 有 seen/unseen split；ESI-Bench 231 questions；target-env adaptation 后 eval。",
"Meta-depth": "M1 same-model。",
"相对之前真正新增什么": "相对纯 skill evolution 多开放一个 **context-building code harness**，且在 fixed low-level interface 的 embodied setting 中验证。"
},
"details": [],
"occurrences": [
{
"section": "A3. Skill / Tool / Executable Subagent Evolution",
"line": 314,
"fields": {
"优先级": "**K**",
"时间": "2026-08-11",
"论文": "[SHAPER: Self-Evolving Embodied Agents via Skill-Harness Evolution](https://arxiv.org/abs/2608.11350)",
"本质定位": "冻结 foundation model，同一模型分别作为 planner/optimizer；利用 target-environment rollouts 同时改 reusable textual skill 与 context-code harness。",
"什么在变": "skill + context-code harness。",
"谁来改 / 谁执行": "**改**：同一 frozen model optimizer role。
**执行**：frozen upper-level VLM planner；下层 VLA/API 也固定。",
"基础 harness": "seed planner + minimal/fixed action interface。",
"Feedback": "episode summaries、execution statistics、environment rollout outcome；VLABench/ESI-Bench grounded evidence。",
"Evolution → Eval": "VLABench 有 seen/unseen split；ESI-Bench 231 questions；target-env adaptation 后 eval。",
"Meta-depth": "M1 same-model。",
"相对之前真正新增什么": "相对纯 skill evolution 多开放一个 **context-building code harness**，且在 fixed low-level interface 的 embodied setting 中验证。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Skill"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "冻结 foundation model，同一模型分别作为 planner/optimizer；利用 target-environment rollouts 同时改 reusable textual skill 与 context-code harness。",
"novelty": "保持模型及底层动作接口固定，从目标环境的执行经历修改上层技能和上下文处理代码，适应不能任意改机器人动作接口的条件。",
"object": "可复用技能和管理上下文的运行框架代码；基础模型参数固定。",
"executor": "上层固定Qwen3.6-27B规划。VLABench下层为官方π0 checkpoint（某个时刻保存的模型或系统版本），通过OpenPI执行每次连续执行五个动作；ESI-Bench用固定交互API。",
"modifier": "冻结的 Qwen3.6-27B 承担优化所需的语言模型调用，结合轨迹总结修改技能与运行框架；成本统计包含规划、判分、总结和产物优化。π0 负责机器人底层动作，不修改运行框架。",
"roleContext": "**改**：同一 frozen model optimizer role。
**执行**：frozen upper-level VLM planner；下层 VLA/API 也固定。",
"seed": "冻结 Qwen3.6-27B 上层规划器；VLABench 用官方 π0 检查点经 OpenPI 执行子目标，每次输出 5 个低层动作，最多 10 轮规划/400 环境步。ESI-Bench 使用固定交互 API，最多 30 步。可演化的是技能和上层运行框架，不重训这些执行模型。",
"fixed": "",
"verdict": "VLABench 用环境成功信号验收，ESI-Bench 用答案正确率评价；诊断模型另外分析动作前后观察与轨迹，解释问题出在技能、上下文还是环境。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "VLABench",
"evolve": "15 training episodes",
"selection": "固定 24 validation episodes",
"test": "800 held-out evaluation episodes",
"isolation": "三路分离",
"note": "这是 VLABench 的具体设置，不自动代表所有实验。"
}
],
"takeaway": "",
"result": "",
"reading": "2026-09-09 补查原文的方法、实验设置或附录对应段落；非整篇逐项审计。",
"sources": [
"https://arxiv.org/html/2608.11350v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：VLABench15训练 episodes；ESI-Bench10训练问题。\n\n调试 / 选版本数据：VLABench固定24验证 episodes；ESI-Bench10验证题。4轮、宽度3的 beam search（每轮保留若干较好候选继续搜索），先优化技能，再固定技能优化运行框架。\n\n最终测试数据：VLABench800互斥 episodes：自建C1–C4各200，改变目标类别/任务形式；ESI-Bench231题。\n\n数据隔离与证据边界：两套实验训练、验证、测试互斥；C1–C4是作者自建划分，不能称官方划分。",
"cycle": "上层规划模型和底层执行接口固定。优化者依据动作前后观察、整次任务结果和诊断报告，修改文字技能及上下文构造代码；在验证环境中运行，保留若干较好候选继续搜索。",
"train": "VLABench15训练 episodes；ESI-Bench10训练问题。",
"debug": "VLABench固定24验证 episodes；ESI-Bench10验证题。4轮、宽度3的 beam search（每轮保留若干较好候选继续搜索），先优化技能，再固定技能优化运行框架。",
"test": "VLABench800互斥 episodes：自建C1–C4各200，改变目标类别/任务形式；ESI-Bench231题。",
"isolation": "两套实验训练、验证、测试互斥；C1–C4是作者自建划分，不能称官方划分。"
},
"reviewed": false,
"focusedReview": true,
"systemDataAudit": {
"seed": "冻结 Qwen3.6-27B 上层规划器；VLABench 用官方 π0 检查点经 OpenPI 执行子目标，每次输出 5 个低层动作，最多 10 轮规划/400 环境步。ESI-Bench 使用固定交互 API，最多 30 步。可演化的是技能和上层 harness，不重训这些执行模型。",
"protocol": "**VLABench：**15 个训练 episodes 用于演化，固定 24 个验证 episodes 选版本，800 个互斥测试 episodes。测试是作者自建 C1–C4 四种分布，各 200 例，分别改变目标类别和任务形式，不是官方现成四个 split。\n\n**ESI-Bench：**按官方类别比例抽 231 个问题作评测；其单独进化/选模数据设置本轮待核实。",
"sections": "§4.1、优化协议、Table 2",
"source": "https://arxiv.org/abs/2608.11350",
"version": "2608.11350v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ab7f74d1cf63a348fd4b617e59113df63d1390680954a20690be810a45d74e11",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "上层固定Qwen3.6-27B规划。VLABench下层为官方π0 checkpoint（某个时刻保存的模型或系统版本），通过OpenPI执行每次连续执行五个动作；ESI-Bench用固定交互API。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "冻结的 Qwen3.6-27B 承担优化所需的语言模型调用，结合轨迹总结修改技能与运行框架；成本统计包含规划、判分、总结和产物优化。π0 负责机器人底层动作，不修改运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2608.11350#A1.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可复用技能和管理上下文的运行框架代码；基础模型参数固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "VLABench 用环境成功信号验收，ESI-Bench 用答案正确率评价；诊断模型另外分析动作前后观察与轨迹，解释问题出在技能、上下文还是环境。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
},
{
"label": "附录 A：逐轮诊断",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS1.Px1"
},
{
"label": "§4.1：划分",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "附录 A：ESI 诊断",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS2.Px1"
},
{
"label": "附录 A：结果汇总",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS2.Px2"
},
{
"label": "§4.1：任务划分",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "冻结 Qwen3.6-27B 上层规划器；VLABench 用官方 π0 检查点经 OpenPI 执行子目标，每次输出 5 个低层动作，最多 10 轮规划/400 环境步。ESI-Bench 使用固定交互 API，最多 30 步。可演化的是技能和上层运行框架，不重训这些执行模型。",
"sources": [
{
"label": "§4.1、优化协议、Table 2",
"url": "https://arxiv.org/abs/2608.11350"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "上层规划模型和底层执行接口固定。优化者依据动作前后观察、整次任务结果和诊断报告，修改文字技能及上下文构造代码；在验证环境中运行，保留若干较好候选继续搜索。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "VLABench15训练 episodes；ESI-Bench10训练问题。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "VLABench固定24验证 episodes；ESI-Bench10验证题。4轮、宽度3的 beam search（每轮保留若干较好候选继续搜索），先优化技能，再固定技能优化运行框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "VLABench800互斥 episodes：自建C1–C4各200，改变目标类别/任务形式；ESI-Bench231题。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "两套实验训练、验证、测试互斥；C1–C4是作者自建划分，不能称官方划分。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "保持模型及底层动作接口固定，从目标环境的执行经历修改上层技能和上下文处理代码，适应不能任意改机器人动作接口的条件。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.11350v1",
"version": "2608.11350v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "固定模型和环境动作接口，通过 target-environment rollouts 改可复用 skills 与 context-code harness。与直接编写任意机器人 API 程序的方法不同，它适用于不能改底层动作接口的设置。",
"feedbackCases": [
{
"label": "VLABench：技能与框架搜索",
"data": "15 个训练 episodes，24 个固定验证 episodes；最终 800 个互斥 episodes，四类变化各 200 个。",
"scoring": "最终任务成功由环境给二元结果；逐轮 judger 比较动作前后观察，给五级进展评分及质量批评，五级分数只用于诊断。",
"visible": "任务、子任务、推理、上下文、执行统计、前后图像；批评解释可观察进展、失败原因与上下文是否有效。",
"use": "四轮、宽度三的候选搜索先改技能后改框架；用验证成绩选版本，最终任务成功不能用 judger 进展分替代。",
"sources": [
{
"label": "附录 A：逐轮诊断",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS1.Px1"
},
{
"label": "§4.1：划分",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
}
],
"judgment": "环境程序判最终成功；模型另按五级进展要求评动作前后观察"
},
{
"label": "ESI-Bench：空间问答",
"data": "10 题训练、10 题验证；231 题按官方类别比例抽取作最终评价。",
"scoring": "基准答案正确率负责衡量完成情况；诊断 judger 审查是否收集并保留所需证据、是否循环探索、回答是否被观察支持。",
"visible": "逐题诊断将主要问题分到技能、框架、环境或均无；汇总时加入官方准确率、类别结果与执行统计。",
"use": "诊断指导改哪一部分，验证准确率用于选候选；最终微平均和宏平均准确率用于报告。",
"sources": [
{
"label": "附录 A：ESI 诊断",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS2.Px1"
},
{
"label": "附录 A：结果汇总",
"url": "https://arxiv.org/html/2608.11350#A1.SS3.SSS2.Px2"
},
{
"label": "§4.1：任务划分",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
}
],
"judgment": "ESI-Bench 标准答案判最终正确性；模型另评探索证据与过程"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "VLABench15训练 episodes；ESI-Bench10训练问题。",
"selection": "VLABench固定24验证 episodes；ESI-Bench10验证题。4轮、宽度3的 beam search（每轮保留若干较好候选继续搜索），先优化技能，再固定技能优化运行框架。",
"evaluation": "VLABench800互斥 episodes：自建C1–C4各200，改变目标类别/任务形式；ESI-Bench231题。",
"isolation": "两套实验训练、验证、测试互斥；C1–C4是作者自建划分，不能称官方划分。",
"roles": {
"executor": {
"value": "上层固定Qwen3.6-27B规划。VLABench下层为官方π0 checkpoint（某个时刻保存的模型或系统版本），通过OpenPI执行每次连续执行五个动作；ESI-Bench用固定交互API。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px1"
}
]
},
"modifier": {
"value": "冻结的 Qwen3.6-27B 承担优化所需的语言模型调用，结合轨迹总结修改技能与运行框架；成本统计包含规划、判分、总结和产物优化。π0 负责机器人底层动作，不修改运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.11350#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px1"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2608.11350#A1.SS4"
}
]
},
"seed": {
"value": "冻结 Qwen3.6-27B 上层规划器；VLABench 用官方 π0 检查点经 OpenPI 执行子目标，每次输出 5 个低层动作，最多 10 轮规划/400 环境步。ESI-Bench 使用固定交互 API，最多 30 步。可演化的是技能和上层运行框架，不重训这些执行模型。",
"sources": [
{
"label": "§4.1、优化协议、Table 2",
"url": "https://arxiv.org/abs/2608.11350"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.11350#S4.SS1.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "具身 agent 需要在不完整视觉反馈和受限动作接口下可靠行动，但微调依赖权重、示范或奖励数据，实际未必具备这些条件。已有代码技能方法又依赖可编程的机器人接口；若环境只提供固定动作，两类办法都难直接适用，因此需要研究不改权重和动作接口的适应途径。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.11350#S1"
}
]
},
{
"key": "position",
"value": "面向动作接口固定的具身环境，研究冻结模型能否通过外部技能与运行支持的适应，改善感知、规划和任务执行能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.11350"
}
]
},
{
"key": "conclusion",
"value": "在两种具身基准及不同底层动作接口上验证无需参数训练的适应路线。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.11350"
}
]
}
],
"fields": {
"object": "可复用技能和管理上下文的运行框架代码；基础模型参数固定。",
"verdict": "VLABench 用环境成功信号验收，ESI-Bench 用答案正确率评价；诊断模型另外分析动作前后观察与轨迹，解释问题出在技能、上下文还是环境。",
"executor": "Qwen3.6-27B 负责上层规划。VLABench 中由官方 π0 机器人行动模型通过 OpenPI 运行接口执行，每次连续执行五个动作；ESI-Bench 使用固定交互接口。规划模型与底层行动执行不是同一个角色。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2504.15228",
"title": "A Self-Improving Coding Agent (SICA)",
"url": "https://arxiv.org/abs/2504.15228",
"date": "2025-04-21",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Archive",
"HarnessCode",
"Improver",
"M2",
"SameSet"
],
"fields": {
"本质定位": "coding agent 读取/修改自己的整个 Python codebase；archive 中当前 best agent 再作为 modifier 产生下一代。",
"什么在变": "完整 coding-agent implementation。",
"谁来改 / 谁执行": "**改**：当前 archive/best coding agent。
**执行**：其修改后 successor。",
"基础 harness": "已有较强 coding harness：file/shell/calculator/subagents/overseer 等。",
"Feedback": "固定 benchmark utility，综合 performance/cost/time；execution feedback。",
"Evolution → Eval": "固定 SWE-bench Verified subset + LiveCodeBench 等反复优化；headline 多为 same-set adaptive。",
"Meta-depth": "M2-ish：successor 可成为下一代 modifier，但 outer archive/utility fixed。",
"相对之前真正新增什么": "相对 Gödel Agent 更聚焦真实 coding-agent codebase 与 SWE-style benchmark，展示 executable self-mod 的工程可行性。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 322,
"fields": {
"优先级": "**R**",
"时间": "2025-04-21",
"论文": "[A Self-Improving Coding Agent (SICA)](https://arxiv.org/abs/2504.15228)",
"本质定位": "coding agent 读取/修改自己的整个 Python codebase；archive 中当前 best agent 再作为 modifier 产生下一代。",
"什么在变": "完整 coding-agent implementation。",
"谁来改 / 谁执行": "**改**：当前 archive/best coding agent。
**执行**：其修改后 successor。",
"基础 harness": "已有较强 coding harness：file/shell/calculator/subagents/overseer 等。",
"Feedback": "固定 benchmark utility，综合 performance/cost/time；execution feedback。",
"Evolution → Eval": "固定 SWE-bench Verified subset + LiveCodeBench 等反复优化；headline 多为 same-set adaptive。",
"Meta-depth": "M2-ish：successor 可成为下一代 modifier，但 outer archive/utility fixed。",
"相对之前真正新增什么": "相对 Gödel Agent 更聚焦真实 coding-agent codebase 与 SWE-style benchmark，展示 executable self-mod 的工程可行性。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "Same-set adaptive",
"protocolBasis": "依据原记录：主实验在同一任务/子集上反复优化或评估；不能直接当作冻结后的泛化。",
"year": "2025",
"depth": [
"M2"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "coding agent 读取/修改自己的整个 Python codebase；archive 中当前 best agent 再作为 modifier 产生下一代。",
"novelty": "让 coding agent 修改自己实际使用的工具和软件实现，再由改好的版本继续提出修改，在真实代码任务中检验自修改。",
"object": "coding agent 的完整实现；改进后的 agent 可继续修改后代，外层档案管理和评价规则保持固定。",
"executor": "SICA 的大部分 agent 使用 Claude Sonnet 3.5 v2，专门的 reasoning task agent 使用 o3-mini；agent 协作调用工具完成基准任务。",
"modifier": "自改进也在这套 agent 系统中完成：主要由 Claude Sonnet 3.5 v2 工作，o3-mini 提供专门推理。修改目标是 agent 系统的代码，而非这两个模型的参数。",
"roleContext": "**改**：当前 archive/best coding agent。
**执行**：其修改后 successor。",
"seed": "初始编程 task agent 已有文件/shell/计算器、子 task agent、异步监督和上下文管理。编辑先以代码改动记录追加，随后合并文件视图以利用缓存；进化可以改变这些已有组件，不是从空白 ReAct（交替进行推理、调用工具和读取结果的执行方式） 起步。",
"fixed": "M2-ish：successor 可成为下一代 modifier，但 outer archive/utility fixed",
"verdict": "按固定评测目标综合衡量任务表现、费用和时间，同时提供运行反馈。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Same-set adaptive",
"note": "固定 SWE-bench Verified subset + LiveCodeBench 等反复优化；headline 多为 same-set adaptive。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：固定抽50题 SWE-bench Verified、50题 LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）；另自建文件编辑和代码符号定位任务。\n\n调试 / 选版本数据：四类任务的正确性、时间和费用组合为效用，最好历史版本读取档案并实现下一版。\n\n最终测试数据：主表展示同一评测基准集上的逐代成绩；另有 AIME/GPQA 推理实验。\n\n数据隔离与证据边界：§4没有另设主编码实验的最终盲测集；进化曲线分数同时参与选择。",
"cycle": "当前最好 task agent 兼任下一轮修改者，自改工具与子 task agent；文件编辑目标来自相邻Git提交，定位任务要求返回符号定义位置。",
"train": "固定抽50题 SWE-bench Verified、50题 LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）；另自建文件编辑和代码符号定位任务。",
"debug": "四类任务的正确性、时间和费用组合为效用，最好历史版本读取档案并实现下一版。",
"test": "主表展示同一评测基准集上的逐代成绩；另有 AIME/GPQA 推理实验。",
"isolation": "§4没有另设主编码实验的最终盲测集；进化曲线分数同时参与选择。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "初始编程 agent 已有文件/shell/计算器、子 agent、异步监督和上下文管理。编辑先以 diff 追加，随后合并文件视图以利用缓存；进化可以改变这些已有组件，不是从空白 ReAct 起步。",
"protocol": "**迭代评分：**合成文件编辑、符号定位/代码导航、SWE-bench Verified、LiveCodeBench 四类 benchmark。每轮评价当前 agent，历史最好版本利用档案生成下一版。\n\n**边界：**这些分数既展示进化也参与版本选择，不能直接称为严格盲测；各 benchmark 的独立最终留出题与样本数本轮待核实。",
"sections": "基础 agent、Algorithm 1、Table 1",
"source": "https://arxiv.org/abs/2504.15228",
"version": "2504.15228v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "0e69294d9051fa0fa57b89de84dbbae93373d60f31ad3820b75ae8fd20a9cb0a",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "SICA 的大部分 agent 使用 Claude Sonnet 3.5 v2，专门的 reasoning task agent 使用 o3-mini；agent 协作调用工具完成基准任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "自改进也在这套 agent 系统中完成：主要由 Claude Sonnet 3.5 v2 工作，o3-mini 提供专门推理。修改目标是 agent 系统的代码，而非这两个模型的参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "coding agent 的完整实现；改进后的 agent 可继续修改后代，外层档案管理和评价规则保持固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "按固定评测目标综合衡量任务表现、费用和时间，同时提供运行反馈。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "初始编程 task agent 已有文件/shell/计算器、子 task agent、异步监督和上下文管理。编辑先以代码改动记录追加，随后合并文件视图以利用缓存；进化可以改变这些已有组件，不是从空白 ReAct（交替进行推理、调用工具和读取结果的执行方式） 起步。",
"sources": [
{
"label": "基础 agent、Algorithm 1、Table 1",
"url": "https://arxiv.org/abs/2504.15228"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "当前最好 task agent 兼任下一轮修改者，自改工具与子 task agent；文件编辑目标来自相邻Git提交，定位任务要求返回符号定义位置。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "固定抽50题 SWE-bench Verified、50题 LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）；另自建文件编辑和代码符号定位任务。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "四类任务的正确性、时间和费用组合为效用，最好历史版本读取档案并实现下一版。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主表展示同一评测基准集上的逐代成绩；另有 AIME/GPQA 推理实验。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4没有另设主编码实验的最终盲测集；进化曲线分数同时参与选择。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让 coding agent 修改自己实际使用的工具和软件实现，再由改好的版本继续提出修改，在真实代码任务中检验自修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2504.15228v2",
"version": "2504.15228v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Gödel Agent 更聚焦真实 coding-agent codebase 与 SWE-style benchmark，展示 executable self-mod 的工程可行性。",
"feedbackCases": [
{
"label": "SWE-bench Verified / LiveCodeBench",
"data": "各固定抽取 50 题",
"scoring": "运行代码任务的测试，统计解决率。 另记录运行时间与美元开销，合成效用。",
"visible": "评测成绩、工具执行记录、时间与费用，存入历史档案。",
"use": "历史最佳 agent 读取档案并实现下一版；主表是同一评测集上的逐代改进，另有 AIME / GPQA（研究生级科学问答基准） 推理实验。",
"sources": [
{
"label": "§3：效用函数",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4：四类基准",
"url": "https://arxiv.org/html/2504.15228#S4"
}
],
"judgment": "SWE-bench／LiveCodeBench 可执行测试，另计时间与费用"
},
{
"label": "自建文件编辑",
"data": "从 SWE 仓库连续提交提取非平凡文件修改",
"scoring": "把最终文件与后一次提交的目标内容比较相似程度。 另记录运行时间与美元开销，合成效用。",
"visible": "评测成绩、工具执行记录、时间与费用，存入历史档案。",
"use": "历史最佳 agent 读取档案并实现下一版；主表是同一评测集上的逐代改进，另有 AIME / GPQA（研究生级科学问答基准） 推理实验。",
"sources": [
{
"label": "§3：效用函数",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4：四类基准",
"url": "https://arxiv.org/html/2504.15228#S4"
}
],
"judgment": "规则比较最终文件与目标提交内容的相似度，另计成本"
},
{
"label": "自建代码定位",
"data": "Python 仓库中的符号及引用位置",
"scoring": "要求返回符号定义的文件路径、行号、列号，与已定位的定义对照。 另记录运行时间与美元开销，合成效用。",
"visible": "评测成绩、工具执行记录、时间与费用，存入历史档案。",
"use": "历史最佳 agent 读取档案并实现下一版；主表是同一评测集上的逐代改进，另有 AIME / GPQA（研究生级科学问答基准） 推理实验。",
"sources": [
{
"label": "§3：效用函数",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4：四类基准",
"url": "https://arxiv.org/html/2504.15228#S4"
}
],
"judgment": "规则核对符号定义的文件、行、列，另计成本"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2
],
"evolution": "固定抽50题 SWE-bench Verified、50题 LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）；另自建文件编辑和代码符号定位任务。",
"selection": "四类任务的正确性、时间和费用组合为效用，最好历史版本读取档案并实现下一版。",
"evaluation": "主表展示同一评测基准集上的逐代成绩；另有 AIME/GPQA 推理实验。",
"isolation": "§4没有另设主编码实验的最终盲测集；进化曲线分数同时参与选择。",
"roles": {
"executor": {
"value": "SICA 的大部分 agent 使用 Claude Sonnet 3.5 v2，专门的 reasoning task agent 使用 o3-mini；agent 协作调用工具完成基准任务。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
}
]
},
"modifier": {
"value": "自改进也在这套 agent 系统中完成：主要由 Claude Sonnet 3.5 v2 工作，o3-mini 提供专门推理。修改目标是 agent 系统的代码，而非这两个模型的参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
}
]
},
"seed": {
"value": "初始编程 task agent 已有文件/shell/计算器、子 task agent、异步监督和上下文管理。编辑先以代码改动记录追加，随后合并文件视图以利用缓存；进化可以改变这些已有组件，不是从空白 ReAct（交替进行推理、调用工具和读取结果的执行方式） 起步。",
"sources": [
{
"label": "基础 agent、Algorithm 1、Table 1",
"url": "https://arxiv.org/abs/2504.15228"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2504.15228#S4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2504.15228#S4.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "agent 能力依赖人设计的运行代码，而已有自动设计方法往往用固定的上层 agent 修改另一个执行 agent。这样，执行 agent 变强未必能改善下一轮修改本身；作者因此研究同一 agent 能否直接改进自己的实现，让新增的编程能力也用于后续自我改进。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2504.15228#S1"
}
]
},
{
"key": "position",
"value": "研究具备基本编程工具的 agent 能否自主改进自身软件实现，并让更好的版本继续推动后续能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2504.15228"
}
]
},
{
"key": "conclusion",
"value": "在软件修复及代码任务上提高表现，展示基于反思和代码更新的非梯度学习。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2504.15228"
}
]
}
],
"fields": {
"object": "coding agent 的完整实现；改进后的 agent 可继续修改后代，外层档案管理和评价规则保持固定。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2603.03329",
"title": "AutoHarness: Improving LLM Agents by Automatically Synthesizing a Code Harness",
"url": "https://arxiv.org/abs/2603.03329",
"date": "2026-02-10",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"HarnessCode",
"M1",
"SameModel",
"org:google-deepmind"
],
"fields": {
"本质定位": "Gemini-2.5-Flash 根据 game environment 的 illegal-action feedback 自动写/改 code harness；极端情况下直接把整个 policy 编译成 code。",
"什么在变": "code harness；甚至 full code policy。",
"谁来改 / 谁执行": "**改**：Gemini-2.5-Flash synthesizer/refiner。
**执行**：Gemini-2.5-Flash decision agent 或最终纯 code policy。",
"基础 harness": "TextArena raw action interface / minimal game agent。",
"Feedback": "environment legality/error/reward。",
"Evolution → Eval": "145 TextArena games；16 single-player code-policy tests。",
"Meta-depth": "M1 same-model。",
"相对之前真正新增什么": "相对通用 workflow search 的独特点：针对环境 action constraints 生成 **hard executable legality layer**，不是仅语言提示。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 324,
"fields": {
"优先级": "**K**",
"时间": "2026-02-10",
"论文": "[AutoHarness: Improving LLM Agents by Automatically Synthesizing a Code Harness](https://arxiv.org/abs/2603.03329)",
"本质定位": "Gemini-2.5-Flash 根据 game environment 的 illegal-action feedback 自动写/改 code harness；极端情况下直接把整个 policy 编译成 code。",
"什么在变": "code harness；甚至 full code policy。",
"谁来改 / 谁执行": "**改**：Gemini-2.5-Flash synthesizer/refiner。
**执行**：Gemini-2.5-Flash decision agent 或最终纯 code policy。",
"基础 harness": "TextArena raw action interface / minimal game agent。",
"Feedback": "environment legality/error/reward。",
"Evolution → Eval": "145 TextArena games；16 single-player code-policy tests。",
"Meta-depth": "M1 same-model。",
"相对之前真正新增什么": "相对通用 workflow search 的独特点：针对环境 action constraints 生成 **hard executable legality layer**，不是仅语言提示。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "Gemini-2.5-Flash 根据 game environment 的 illegal-action feedback 自动写/改 code harness；极端情况下直接把整个 policy 编译成 code。",
"novelty": "根据环境动作规则生成可执行检查代码，在模型动作到达环境前阻止或处理非法动作，将部分可靠性要求交给程序保证。",
"object": "防止无效动作的代码运行框架，或完全取代模型逐步决策的代码策略。",
"executor": "Gemini-2.5-Flash 配合运行框架做决策；纯代码策略设置则直接运行最终生成的代码，不在执行时调用语言模型。",
"modifier": "Gemini-2.5-Flash 生成并根据执行反馈修订运行框架或代码策略。",
"roleContext": "**改**：Gemini-2.5-Flash synthesizer/refiner。
**执行**：Gemini-2.5-Flash decision agent 或最终纯 code policy。",
"seed": "TextArena 的原始游戏状态和动作接口外生成代码运行框架，作为合法动作验证器或辅助决策器。动作验证实验去掉提示中直接列出的合法动作，要求代码从规则/状态推导，而非照抄答案。",
"fixed": "",
"verdict": "游戏环境反馈动作是否合法、违反什么规则，以及终局胜负或奖励；“不再犯非法动作”与“更容易赢”是两种不同评价目标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "145 TextArena games；16 single-player code-policy tests。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：TextArena 游戏交互提供状态、动作合法性和终局奖励；合成动作验证器/策略代码。\n\n调试 / 选版本数据：Thompson sampling 选代码树节点，语言模型据非法动作反馈改 propose_action 与 is_legal_action。策略版本最多256轮。\n\n最终测试数据：实际对局选16个单人、16个双人游戏；单人每游戏20局，双人40局并平分先后手。\n\n数据隔离与证据边界：这是同游戏的新对局测试；§4.2说明随机种子和先后手，但没有给出训练与测试种子互斥清单。",
"cycle": "把动作合法性从 语言模型 判断转成程序检查；验证器拒绝非法提案让模型重答，纯代码策略版本推理时甚至不调用语言模型。",
"train": "TextArena 游戏交互提供状态、动作合法性和终局奖励；合成动作验证器/策略代码。",
"debug": "Thompson sampling 选代码树节点，语言模型据非法动作反馈改 propose_action 与 is_legal_action。策略版本最多256轮。",
"test": "实际对局选16个单人、16个双人游戏；单人每游戏20局，双人40局并平分先后手。",
"isolation": "这是同游戏的新对局测试；§4.2说明随机种子和先后手，但没有给出训练与测试种子互斥清单。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "TextArena 的原始游戏状态和动作接口外生成代码 harness，作为合法动作验证器或辅助决策器。动作验证实验去掉提示中直接列出的合法动作，要求代码从规则/状态推导，而非照抄答案。",
"protocol": "**数据：**TextArena 多种游戏，包括 Minesweeper 等；通过游戏运行收集不合法动作和状态反馈，反复修订代码，评价合法动作率和游戏表现。\n\n**隔离：**进化依赖模拟交互，不是文本训练集；不同游戏的训练局数、测试局数与随机种子隔离本轮未完整核实，不能把当前局上的修复率直接等同于未见局泛化。",
"sections": "§4.1 与动作验证设置",
"source": "https://arxiv.org/abs/2603.03329",
"version": "2603.03329v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "2cf95753ff93d699d5a4745982a3ddb8011c556e1da316fb6c9379d8eaf75c4d",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Gemini-2.5-Flash 配合运行框架做决策；纯代码策略设置则直接运行最终生成的代码，不在执行时调用语言模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Gemini-2.5-Flash 生成并根据执行反馈修订运行框架或代码策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "防止无效动作的代码运行框架，或完全取代模型逐步决策的代码策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "游戏环境反馈动作是否合法、违反什么规则，以及终局胜负或奖励；“不再犯非法动作”与“更容易赢”是两种不同评价目标。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "TextArena 的原始游戏状态和动作接口外生成代码运行框架，作为合法动作验证器或辅助决策器。动作验证实验去掉提示中直接列出的合法动作，要求代码从规则/状态推导，而非照抄答案。",
"sources": [
{
"label": "§4.1 与动作验证设置",
"url": "https://arxiv.org/abs/2603.03329"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "把动作合法性从 语言模型 判断转成程序检查；验证器拒绝非法提案让模型重答，纯代码策略版本推理时甚至不调用语言模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "TextArena 游戏交互提供状态、动作合法性和终局奖励；合成动作验证器/策略代码。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Thompson sampling 选代码树节点，语言模型据非法动作反馈改 propose_action 与 is_legal_action。策略版本最多256轮。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "实际对局选16个单人、16个双人游戏；单人每游戏20局，双人40局并平分先后手。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "这是同游戏的新对局测试；§4.2说明随机种子和先后手，但没有给出训练与测试种子互斥清单。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "根据环境动作规则生成可执行检查代码，在模型动作到达环境前阻止或处理非法动作，将部分可靠性要求交给程序保证。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2603.03329v1",
"version": "2603.03329v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对通用 workflow search 的独特点：针对环境 action constraints 生成 **hard executable legality layer**，不是仅语言提示。",
"feedbackCases": [
{
"label": "动作合法性：TextArena",
"data": "145 个 TextArena 游戏，依据游戏交互修改动作提议与合法性检查代码。",
"scoring": "游戏环境检查动作是否合法，非法时返回规则错误；不是仅靠生成代码自称合法。",
"visible": "当前状态、非法动作及环境报错。",
"use": "Thompson sampling 选择代码树节点，模型据报错修复 propose_action / is_legal_action。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"judgment": "TextArena 游戏程序检查动作合法性"
},
{
"label": "策略收益：TextArena 对局",
"data": "策略合成最多 256 轮；效果评估选择 16 个单人、16 个双人游戏，分别每游戏 20 / 40 局，双人平衡先后手。",
"scoring": "环境按游戏规则给终局胜负或奖励。",
"visible": "搜索中的游戏结果用于比较策略；最终对局汇总收益。",
"use": "合法性与赢得游戏是不同目标，避免非法动作的 145 游戏结果不等于 145 游戏均获胜。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"judgment": "TextArena 游戏规则结算胜负／奖励"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "TextArena 游戏交互提供状态、动作合法性和终局奖励；合成动作验证器/策略代码。",
"selection": "Thompson sampling 选代码树节点，语言模型据非法动作反馈改 propose_action 与 is_legal_action。策略版本最多256轮。",
"evaluation": "实际对局选16个单人、16个双人游戏；单人每游戏20局，双人40局并平分先后手。",
"isolation": "这是同游戏的新对局测试；§4.2说明随机种子和先后手，但没有给出训练与测试种子互斥清单。",
"roles": {
"executor": {
"value": "Gemini-2.5-Flash 配合运行框架做决策；纯代码策略设置则直接运行最终生成的代码，不在执行时调用语言模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
]
},
"modifier": {
"value": "Gemini-2.5-Flash 生成并根据执行反馈修订运行框架或代码策略。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.03329#S3"
}
]
},
"seed": {
"value": "TextArena 的原始游戏状态和动作接口外生成代码运行框架，作为合法动作验证器或辅助决策器。动作验证实验去掉提示中直接列出的合法动作，要求代码从规则/状态推导，而非照抄答案。",
"sources": [
{
"label": "§4.1 与动作验证设置",
"url": "https://arxiv.org/abs/2603.03329"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"selection": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"evaluation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
],
"isolation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2603.03329#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2603.03329#S4.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型看似理解游戏规则，仍可能因输出非法动作而直接失败。为此微调大模型成本高且可能损害其他能力，人工编写动作检查代码又要为每个新游戏重复开发；作者因此研究让模型利用自身编程能力，自动补上连接模型与环境的检查和执行逻辑。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.03329#S1"
}
]
},
{
"key": "position",
"value": "面向有明确动作规则的交互环境，研究自动生成的运行代码能否弥补模型执行可靠性的不足，减少非法动作并改善决策。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.03329"
}
]
},
{
"key": "conclusion",
"value": "在多种文字游戏中消除非法动作，较小模型生成的代码支持也可胜过更大模型直接决策。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.03329"
}
]
}
],
"fields": {
"object": "防止无效动作的代码运行框架，或完全取代模型逐步决策的代码策略。",
"verdict": "游戏环境反馈动作是否合法、违反什么规则，以及终局胜负或奖励；“不再犯非法动作”与“更容易赢”是两种不同评价目标。"
}
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2603.03329"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2606.19980",
"title": "ENPIRE: Agentic Robot Policy Self-Improvement in the Real World",
"url": "https://arxiv.org/abs/2606.19980",
"date": "2026-06-18",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"org:cmu",
"org:berkeley"
],
"fields": {
"本质定位": "coding/research agent 在真实机器人上循环 reset→rollout→verify→诊断→修改 policy/training-infra/algorithm code→重跑。",
"什么在变": "robot policy code、training infrastructure、algorithm code。",
"谁来改 / 谁执行": "**改**：coding agent / evolution agent。
**执行**：robot policy + real/sim robots。",
"基础 harness": "robot learning stack。",
"Feedback": "real-world task verifier、rollout logs、performance。",
"Evolution → Eval": "dexterous manipulation / robot fleets。",
"Meta-depth": "M1/M2 substrate，outer loop fixed。",
"相对之前真正新增什么": "和 deployment harness evolution 相邻但不完全同类：可编辑对象已经扩到 **训练系统和算法代码**，更像 embodied AutoResearch。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 331,
"fields": {
"优先级": "**R**",
"时间": "2026-06-18",
"论文": "[ENPIRE: Agentic Robot Policy Self-Improvement in the Real World](https://arxiv.org/abs/2606.19980)",
"本质定位": "coding/research agent 在真实机器人上循环 reset→rollout→verify→诊断→修改 policy/training-infra/algorithm code→重跑。",
"什么在变": "robot policy code、training infrastructure、algorithm code。",
"谁来改 / 谁执行": "**改**：coding agent / evolution agent。
**执行**：robot policy + real/sim robots。",
"基础 harness": "robot learning stack。",
"Feedback": "real-world task verifier、rollout logs、performance。",
"Evolution → Eval": "dexterous manipulation / robot fleets。",
"Meta-depth": "M1/M2 substrate，outer loop fixed。",
"相对之前真正新增什么": "和 deployment harness evolution 相邻但不完全同类：可编辑对象已经扩到 **训练系统和算法代码**，更像 embodied AutoResearch。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1",
"M2"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "coding/research agent 在真实机器人上循环 reset→rollout→verify→诊断→修改 policy/training-infra/algorithm code→重跑。",
"novelty": "给代码 agent 可重复开展物理试验的接口，并允许它改机器人训练系统和算法代码，把真实实验接到策略改进流程。",
"object": "机器人策略、训练基础设施和算法代码；组织复位、运行、验证和改进的外层流程保持固定。",
"executor": "真实YAM机器人执行学到的控制策略或脚本；研究代码由Codex + GPT-5.5 xhigh、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7 High、Kimi Code + Kimi K2.6 thinking生成。",
"modifier": "上述三种编程 task agent 在真实反馈下改控制代码或训练算法；改变的是机器人策略，不是让GPT-5.5或Opus4.7自身做参数训练。",
"roleContext": "**改**：coding agent / evolution agent。
**执行**：robot policy + real/sim robots。",
"seed": "真实机器人学习栈，task agent 可写奖励/视觉处理代码并改进策略流程；底层有感知、控制和训练设施，需满足实时延迟。不是在一个无硬件约束的文字模拟器里改提示词。",
"fixed": "M1/M2 substrate，outer loop fixed",
"verdict": "真实机器人环境通过传感器判据或视觉奖励识别器判断成功，结合视频和运行记录诊断失败；RoboCasa 模拟实验采用其原生成功函数。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "dexterous manipulation / robot fleets。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：真实YAM机器人在线回合及人类演示，分别入replay与demonstration buffer；任务含Push-T、插pin、GPU插槽、剪扎带。\n\n调试 / 选版本数据：编码 task agent 可选择BC/强化学习（根据奖励调整模型行为）/脚本方法；用传感器成功判据、执行视频、碰撞/越界终止反馈。奖励识别另用数分钟成功/失败演示。\n\n最终测试数据：真实任务表现及Push-T简化消融；RoboCasa每任务固定40个(seed,layout,style)组合，以原生成功函数计分。\n\n数据隔离与证据边界：RoboCasa诊断子集也取自同一40条清单，不能称全程不可见。扎带奖励识别有单独沙箱留出，不能把该隔离套到所有策略实验。",
"cycle": "先用人类示范/约束搭建可复位、可判分环境，再让编码 task agent 通过统一机器人接口自动收集数据、训练或改控制程序。",
"train": "真实YAM机器人在线回合及人类演示，分别入replay与demonstration buffer；任务含Push-T、插pin、GPU插槽、剪扎带。",
"debug": "编码 task agent 可选择BC/强化学习（根据奖励调整模型行为）/脚本方法；用传感器成功判据、执行视频、碰撞/越界终止反馈。奖励识别另用数分钟成功/失败演示。",
"test": "真实任务表现及Push-T简化消融；RoboCasa每任务固定40个(seed,layout,style)组合，以原生成功函数计分。",
"isolation": "RoboCasa诊断子集也取自同一40条清单，不能称全程不可见。扎带奖励识别有单独沙箱留出，不能把该隔离套到所有策略实验。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "真实机器人学习栈，agent 可写奖励/视觉处理代码并改进策略流程；底层有感知、控制和训练设施，需满足实时延迟。不是在一个无硬件约束的文字模拟器里改 prompt。",
"protocol": "**真实任务：**Push-T、4mm 孔插 pin、GPU 插槽插入、剪扎带。数据来自真实执行与传感器观测。\n\n**奖励函数案例：**另录制数分钟扎带成功/失败图像作为沙箱留出评估，agent 可研究失败但不能训练于测试集或改评分；该案例还要求 150ms 延迟。各机器人策略任务的完整演示数、训练/测试回合数本轮待核实，不能把视觉验证子实验当成全部策略训练协议。",
"sections": "任务设置与视觉奖励案例",
"source": "https://arxiv.org/abs/2606.19980",
"version": "2606.19980v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "012e0e23bad572dbb07b3f83fefeb645fe5bb98127206958ccdadf452da99209",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "真实YAM机器人执行学到的控制策略或脚本；研究代码由Codex + GPT-5.5 xhigh、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7 High、Kimi Code + Kimi K2.6 thinking生成。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2606.19980#A3.SS4"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "上述三种编程 task agent 在真实反馈下改控制代码或训练算法；改变的是机器人策略，不是让GPT-5.5或Opus4.7自身做参数训练。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2606.19980#A3.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "机器人策略、训练基础设施和算法代码；组织复位、运行、验证和改进的外层流程保持固定。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "真实机器人环境通过传感器判据或视觉奖励识别器判断成功，结合视频和运行记录诊断失败；RoboCasa 模拟实验采用其原生成功函数。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "真实机器人学习栈，task agent 可写奖励/视觉处理代码并改进策略流程；底层有感知、控制和训练设施，需满足实时延迟。不是在一个无硬件约束的文字模拟器里改提示词。",
"sources": [
{
"label": "任务设置与视觉奖励案例",
"url": "https://arxiv.org/abs/2606.19980"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先用人类示范/约束搭建可复位、可判分环境，再让编码 task agent 通过统一机器人接口自动收集数据、训练或改控制程序。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "真实YAM机器人在线回合及人类演示，分别入replay与demonstration buffer；任务含Push-T、插pin、GPU插槽、剪扎带。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "编码 task agent 可选择BC/强化学习（根据奖励调整模型行为）/脚本方法；用传感器成功判据、执行视频、碰撞/越界终止反馈。奖励识别另用数分钟成功/失败演示。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "真实任务表现及Push-T简化消融；RoboCasa每任务固定40个(seed,layout,style)组合，以原生成功函数计分。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "RoboCasa诊断子集也取自同一40条清单，不能称全程不可见。扎带奖励识别有单独沙箱留出，不能把该隔离套到所有策略实验。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "给代码 agent 可重复开展物理试验的接口，并允许它改机器人训练系统和算法代码，把真实实验接到策略改进流程。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2606.19980v1",
"version": "2606.19980v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "和 deployment harness evolution 相邻但不完全同类：可编辑对象已经扩到 **训练系统和算法代码**，更像 embodied AutoResearch。",
"feedbackCases": [
{
"label": "真实机器人：构建环境与改策略",
"data": "YAM 机器人做 Push-T、插 pin、GPU 插槽和剪扎带；人类演示与在线执行分别进入 demonstration / replay buffer。",
"scoring": "任务环境用传感器判据或视觉奖励识别器判成功；视觉识别器用数分钟成功 / 失败演示训练。碰撞、越界等另触发安全终止。",
"visible": "成功信号、执行视频、传感器观察与调试信息，经构建后不可修改的 Gym 接口返回。",
"use": "先根据人类反馈构建自动复位和验收环境，再让编码 agent 选择脚本、模仿学习或强化学习优化策略；环境构建的人类反馈与策略阶段自动奖励分开。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"judgment": "任务传感器规则或经示范训练的视觉成功分类器"
},
{
"label": "RoboCasa 与真实任务评价",
"data": "RoboCasa 每任务固定 40 个 seed / layout / style 组合；真实任务另报表现及 Push-T 消融。",
"scoring": "RoboCasa 用原生成功函数；真实任务沿用任务环境成功检查。",
"visible": "各配置的实际成功率。",
"use": "检验自动改策略后的效果，不将模拟器评分混同于真实视觉奖励识别器。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"judgment": "RoboCasa 原生成功函数；真实任务用既定环境成功检查"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "真实YAM机器人在线回合及人类演示，分别入replay与demonstration buffer；任务含Push-T、插pin、GPU插槽、剪扎带。",
"selection": "编码 task agent 可选择BC/强化学习（根据奖励调整模型行为）/脚本方法；用传感器成功判据、执行视频、碰撞/越界终止反馈。奖励识别另用数分钟成功/失败演示。",
"evaluation": "真实任务表现及Push-T简化消融；RoboCasa每任务固定40个(seed,layout,style)组合，以原生成功函数计分。",
"isolation": "RoboCasa诊断子集也取自同一40条清单，不能称全程不可见。扎带奖励识别有单独沙箱留出，不能把该隔离套到所有策略实验。",
"roles": {
"executor": {
"value": "真实YAM机器人执行学到的控制策略或脚本；研究代码由Codex + GPT-5.5 xhigh、Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus4.7 High、Kimi Code + Kimi K2.6 thinking生成。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2606.19980#A3.SS4"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "上述三种编程 task agent 在真实反馈下改控制代码或训练算法；改变的是机器人策略，不是让GPT-5.5或Opus4.7自身做参数训练。",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2606.19980#S2"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2606.19980#S2.SS1.SSS0.Px2"
},
{
"label": "附录B.5",
"url": "https://arxiv.org/html/2606.19980#A2.SS5"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录C.4",
"url": "https://arxiv.org/html/2606.19980#A3.SS4"
}
]
},
"seed": {
"value": "真实机器人学习栈，task agent 可写奖励/视觉处理代码并改进策略流程；底层有感知、控制和训练设施，需满足实时延迟。不是在一个无硬件约束的文字模拟器里改提示词。",
"sources": [
{
"label": "任务设置与视觉奖励案例",
"url": "https://arxiv.org/abs/2606.19980"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.19980#S3"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2606.19980#A1.SS2"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2606.19980#A3.SS1"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2606.19980#A4.SS2.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "真实机器人策略改进仍需要人部署、评测、复位和照看实验，这限制了学习速度。代码 agent 又缺少能自行重复物理实验的接口，多机器人并行时如何选择和验证假设、合理使用资源也未解决，因此仅把 coding agent 接上机器人还不能实现持续自动研发。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.19980#S1"
}
]
},
{
"key": "position",
"value": "面向真实机器人，研究代码 agent 能否在减少人工复位、验收和算法干预的情况下，自主开展物理实验并持续改进操作策略。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.19980"
}
]
},
{
"key": "conclusion",
"value": "在真实操作任务上展示自主策略改进，重点是让机器人实验成为 agent 可管理的反馈循环。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.19980"
}
]
}
],
"fields": {
"object": "机器人策略、训练基础设施和算法代码；组织复位、运行、验证和改进的外层流程保持固定。",
"verdict": "真实机器人环境通过传感器判据或视觉奖励识别器判断成功，结合视频和运行记录诊断失败；RoboCasa 模拟实验采用其原生成功函数。"
}
},
"attributions": [
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.19980"
}
]
},
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2606.19980"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2607.14159",
"title": "MemoHarness: Agent Harnesses That Learn from Experience",
"url": "https://arxiv.org/abs/2607.14159",
"date": "2026-07-14",
"priority": "K",
"categories": [
"methods"
],
"tags": [],
"fields": {
"本质定位": "把 harness 分成 context/tool/generation/orchestration/memory/output 六个控制维度；从 labeled search cases 的 execution diagnosis 形成 case experience + global patterns，测试时按 case 检索并配置 harness。",
"什么在变": "六维 harness configuration + experience bank。",
"谁来改 / 谁执行": "**改**：search phase modifier/diagnosis LLM；test-time retriever/adaptor。
**执行**：base LLM + selected config。",
"基础 harness": "fixed configurable harness with six dimensions。",
"Feedback": "search case score + diagnosis；test 无 feedback。",
"Evolution → Eval": "search cases → held-out cases；并有跨 suite/base-model transfer。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 Meta-Harness/AHE 的核心不同：不是搜索一个 global final harness，而是学习 **case-adaptive harness configuration policy/memory**，测试时无需 label/search。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 332,
"fields": {
"优先级": "**K**",
"时间": "2026-07-14",
"论文": "[MemoHarness: Agent Harnesses That Learn from Experience](https://arxiv.org/abs/2607.14159)",
"本质定位": "把 harness 分成 context/tool/generation/orchestration/memory/output 六个控制维度；从 labeled search cases 的 execution diagnosis 形成 case experience + global patterns，测试时按 case 检索并配置 harness。",
"什么在变": "六维 harness configuration + experience bank。",
"谁来改 / 谁执行": "**改**：search phase modifier/diagnosis LLM；test-time retriever/adaptor。
**执行**：base LLM + selected config。",
"基础 harness": "fixed configurable harness with six dimensions。",
"Feedback": "search case score + diagnosis；test 无 feedback。",
"Evolution → Eval": "search cases → held-out cases；并有跨 suite/base-model transfer。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对 Meta-Harness/AHE 的核心不同：不是搜索一个 global final harness，而是学习 **case-adaptive harness configuration policy/memory**，测试时无需 label/search。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 harness 分成 context/tool/generation/orchestration/memory/output 六个控制维度；从 labeled search cases 的 execution diagnosis 形成 case experience + global patterns，测试时按 case 检索并配置 harness。",
"novelty": "从历史案例学习按当前任务选择运行配置；新题执行时可使用不同配置，而无需再拿答案反馈搜索。",
"object": "控制上下文、工具、生成、编排、记忆和输出的配置，以及指导配置的经验库。",
"executor": "搜索时任务基础模型为GPT-5.3-Codex；冻结运行框架后迁移Sonnet4.6、Gemini3.1 Pro、Qwen3.5-397B-A17B、GLM5、GPT-4.1、DeepSeekV3.2。",
"modifier": "论文把GPT-5.3-Codex列作搜索来源模型；附录C/D没有另给诊断与修改角色各自独立的模型型号，不补写一个Claude修改器。",
"roleContext": "**改**：search phase modifier/diagnosis LLM；test-time retriever/adaptor。
**执行**：base LLM + selected config。",
"seed": "六维可配置运行框架，以结构化策略文件配合运行说明、操作指南、记忆内容落地。所有实验初始 W₀ 关闭示范、检索、结构化辅助、跨调用记忆和输出验证器；这些功能是在搜索中选择开启，而不是预置强底座。",
"fixed": "",
"verdict": "搜索所用任务的分数和诊断信息；最终测试不返回用于修改方案的反馈。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "search cases → held-out cases；并有跨 suite/base-model transfer。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：Terminal-Bench89题按80/20分，搜索部分71题；另在LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent搜索。\n\n调试 / 选版本数据：利用历史轨迹经验修改运行框架，搜索结束按验证成绩选择，使用该选定版本报告结果。\n\n最终测试数据：Terminal-Bench18题留出，另报告LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent；选定运行框架无再训练迁移到另外六种模型。\n\n数据隔离与证据边界：附录C明确18题留出及验证选版本，但未列另外两集的具体划分数量，也未细分71题内部验证分配；不补造统一比例。",
"cycle": "经验辅助运行框架搜索，并研究针对测试实例的适配；区分搜索过程峰值和最终验证选定版本。",
"train": "Terminal-Bench89题按80/20分，搜索部分71题；另在LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent搜索。",
"debug": "利用历史轨迹经验修改运行框架，搜索结束按验证成绩选择，使用该选定版本报告结果。",
"test": "Terminal-Bench18题留出，另报告LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent；选定运行框架无再训练迁移到另外六种模型。",
"isolation": "附录C明确18题留出及验证选版本，但未列另外两集的具体划分数量，也未细分71题内部验证分配；不补造统一比例。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "六维可配置 harness，以结构化策略文件配合运行说明、playbook、记忆内容落地。所有实验初始 W₀ 关闭示范、检索、结构化辅助、跨调用记忆和输出验证器；这些功能是在搜索中选择开启，而不是预置强底座。",
"protocol": "**数据：**Terminal-Bench 的 89 题按 80/20 拆分，留出评估为 18 题；另测 LiveCodeBench 和 FinanceAgent。最终使用搜索结束后经验证选择的 harness，而非训练曲线峰值。\n\n**跨模型：**搜索用 GPT-5.3-Codex，冻结选出的 harness 后迁移到另外六种模型。Terminal-Bench 的内部选模集及另外两套数据数量本轮待核实；跨模型复用不等于另造新任务测试集。",
"sections": "初始化；附录 C",
"source": "https://arxiv.org/abs/2607.14159",
"version": "2607.14159v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "4feefe1bf73d07ede72dc45d36fff3d0638b2b779eefefc04a79d715a86160cb",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "搜索时任务基础模型为GPT-5.3-Codex；冻结运行框架后迁移Sonnet4.6、Gemini3.1 Pro、Qwen3.5-397B-A17B、GLM5、GPT-4.1、DeepSeekV3.2。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2607.14159#A4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "论文把GPT-5.3-Codex列作搜索来源模型；附录C/D没有另给诊断与修改角色各自独立的模型型号，不补写一个Claude修改器。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2607.14159#A4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "控制上下文、工具、生成、编排、记忆和输出的配置，以及指导配置的经验库。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "搜索所用任务的分数和诊断信息；最终测试不返回用于修改方案的反馈。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "六维可配置运行框架，以结构化策略文件配合运行说明、操作指南、记忆内容落地。所有实验初始 W₀ 关闭示范、检索、结构化辅助、跨调用记忆和输出验证器；这些功能是在搜索中选择开启，而不是预置强底座。",
"sources": [
{
"label": "初始化；附录 C",
"url": "https://arxiv.org/abs/2607.14159"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "经验辅助运行框架搜索，并研究针对测试实例的适配；区分搜索过程峰值和最终验证选定版本。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Terminal-Bench89题按80/20分，搜索部分71题；另在LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent搜索。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "利用历史轨迹经验修改运行框架，搜索结束按验证成绩选择，使用该选定版本报告结果。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Terminal-Bench18题留出，另报告LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent；选定运行框架无再训练迁移到另外六种模型。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "附录C明确18题留出及验证选版本，但未列另外两集的具体划分数量，也未细分71题内部验证分配；不补造统一比例。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从历史案例学习按当前任务选择运行配置；新题执行时可使用不同配置，而无需再拿答案反馈搜索。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.14159v1",
"version": "2607.14159v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 Meta-Harness/AHE 的核心不同：不是搜索一个 global final harness，而是学习 **case-adaptive harness configuration policy/memory**，测试时无需 label/search。",
"feedbackCases": [
{
"label": "Terminal-Bench：框架搜索",
"data": "89 题按 80/20 分为 71 道搜索题、18 道留出题。",
"scoring": "执行终端任务并用任务验收结果判断正确性；正确优先，词元 开销只用于平分时比较。",
"visible": "带标签搜索题的执行轨迹、成绩、成本与诊断，存成逐题经验和全局模式。",
"use": "搜索用历史经验改框架，按验证结果选版本；测试时可检索经验针对无标签新题调整，但没有测试奖励用于修订。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"judgment": "Terminal-Bench 任务验收测试"
},
{
"label": "LiveCodeBench / FinanceAgent",
"data": "另在近期代码题 LiveCodeBench（使用较新竞赛编程题评价代码能力的基准） 与金融文档、多步工具分析 FinanceAgent 搜索和评估。",
"scoring": "沿用各任务正确性作为首要分数、词元 数作次要比较；附录 C 未逐一给出 FinanceAgent 裁判实现和完整划分数量，不能补成某个评审模型。",
"visible": "搜索阶段返回带标签任务的分数、轨迹和诊断；留出测试不提供改进奖励。",
"use": "报告验证选中的框架，不能采用训练过程中最高的一次测试值；另迁移至六类外部任务和六种基础模型。",
"sources": [
{
"label": "§2：搜索与测试适配",
"url": "https://arxiv.org/html/2607.14159#S2"
},
{
"label": "附录 C：基准说明",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
}
],
"judgment": "LiveCodeBench 用代码测试；FinanceAgent 判分实现未明确"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "Terminal-Bench89题按80/20分，搜索部分71题；另在LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent搜索。",
"selection": "利用历史轨迹经验修改运行框架，搜索结束按验证成绩选择，使用该选定版本报告结果。",
"evaluation": "Terminal-Bench18题留出，另报告LiveCodeBench（使用较新竞赛编程题评价代码能力的基准）、FinanceAgent；选定运行框架无再训练迁移到另外六种模型。",
"isolation": "附录C明确18题留出及验证选版本，但未列另外两集的具体划分数量，也未细分71题内部验证分配；不补造统一比例。",
"roles": {
"executor": {
"value": "搜索时任务基础模型为GPT-5.3-Codex；冻结运行框架后迁移Sonnet4.6、Gemini3.1 Pro、Qwen3.5-397B-A17B、GLM5、GPT-4.1、DeepSeekV3.2。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2607.14159#A4"
}
]
},
"modifier": {
"value": "论文把GPT-5.3-Codex列作搜索来源模型；附录C/D没有另给诊断与修改角色各自独立的模型型号，不补写一个Claude修改器。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2607.14159#A4"
}
]
},
"seed": {
"value": "六维可配置运行框架，以结构化策略文件配合运行说明、操作指南、记忆内容落地。所有实验初始 W₀ 关闭示范、检索、结构化辅助、跨调用记忆和输出验证器；这些功能是在搜索中选择开启，而不是预置强底座。",
"sources": [
{
"label": "初始化；附录 C",
"url": "https://arxiv.org/abs/2607.14159"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"selection": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"evaluation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
],
"isolation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px1"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2607.14159#A3.SS0.SSS0.Px3"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.14159#S3.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有自动优化多只改提示或局部流程，较完整的框架搜索又通常产出一套部署后固定使用的方案，难适应不同任务的需求。单看总分还无法定位哪个组件导致失败、哪些经验值得复用；因此作者研究如何从执行记录诊断整个框架，并在不查看测试答案的条件下适应新案例。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.14159#S1"
}
]
},
{
"key": "position",
"value": "研究运行框架能否从历史执行中学会适配不同任务，使 agent 面对无答案反馈的新题时仍能选用合适的运行方式。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.14159"
}
]
},
{
"key": "conclusion",
"value": "测试时无需答案反馈或追加搜索即可适配，强调从历史学习如何因任务而改变运行方式。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.14159"
}
]
}
],
"fields": {
"modifier": "GPT-5.3-Codex 参与搜索，依据历史诊断配置和修订运行框架；原文没有为诊断与修改角色分别披露独立型号。",
"object": "控制上下文、工具、生成、编排、记忆和输出的配置，以及指导配置的经验库。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.01918",
"title": "HarnessCompass: Guiding Automatic Harness Evolution toward Generalizable and Effective Agent Harnesses",
"url": "https://arxiv.org/abs/2608.01918",
"date": "2026-08-03",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"CrossModel",
"ExecutableVerifier",
"HarnessCode",
"HeldOut",
"M1",
"MemoryContent",
"Prompt",
"SelfFeedback",
"Skill",
"Tool"
],
"fields": {
"本质定位": "针对 full-harness evolution 的三类问题——task overfit、只依赖 trajectory outcome、组件同时改造成 interference——加入 task-agnostic constraint、agent first-person proactive feedback、component-wise evolve→consolidate。",
"什么在变": "system prompt、tool descriptions/implementations、middleware、subagent configs、skills、long-term memory 等 full harness components。",
"谁来改 / 谁执行": "**改**：evolution meta-agent；各 component 分开优化后再 R³ merge。
**执行**：GPT-5.4 + 当前 harness（主 SWE-bench setting；role agents共享同 base model）。",
"基础 harness": "H0 为极简 harness（shell command tool，无 middleware/skills/subagents）并与 AHE 同 seed 比。",
"Feedback": "trajectory grounded evidence + agent 第一人称 harness-use feedback（blind/hindsight/grounded variants）+ evaluation score；另有 generalization gate。",
"Evolution → Eval": "SWE-bench Verified evolution sample 上5 turns；held-out tasks检验，另做跨模型 transfer。",
"Meta-depth": "M1；modifier algorithm固定。",
"相对之前真正新增什么": "相对 AHE 的核心新增非常明确：**直接把 generalization、feedback richness、component interference 当 evolution algorithm 的设计目标**，不是事后分析。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "针对 full-harness evolution 的三类问题——task overfit、只依赖 trajectory outcome、组件同时改造成 interference——加入 task-agnostic constraint、agent first-person proactive feedback、component-wise evolve→consolidate。 **相对前序：** 相对 AHE 的核心新增非常明确：**直接把 generalization、feedback richness、component interference 当 evolution algorithm 的设计目标**，不是事后分析。"
},
{
"label": "什么在变",
"text": "system prompt、tool descriptions/implementations、middleware、subagent configs、skills、long-term memory 等 full harness components。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** evolution meta-agent；各 component 分开优化后再 R³ merge。 **执行：** GPT-5.4 + 当前 harness（主 SWE-bench setting；role agents共享同 base model）。"
},
{
"label": "基础 harness",
"text": "H0 为极简 harness（shell command tool，无 middleware/skills/subagents）并与 AHE 同 seed 比。"
},
{
"label": "Feedback",
"text": "trajectory grounded evidence + agent 第一人称 harness-use feedback（blind/hindsight/grounded variants）+ evaluation score；另有 generalization gate。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "SWE-bench Verified evolution sample 上5 turns；held-out tasks检验，另做跨模型 transfer。 **Meta-depth：** M1；modifier algorithm固定。"
},
{
"label": "主要结果",
"text": "sample 54→66%（5 iterations）；held-out 60.4%，高于 AHE 54.7%；total 61.0 vs AHE 55.5/seed 51.8。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "起点仍很简；主实验集中 SWE coding；多个新机制一起加入，三者独立贡献虽有分析但整体 attribution仍不如单变量。 **对我们：** 这是 AHE 后最应该读的改进：直接处理我们关心的 overfit、component interference 与 richer feedback。"
}
],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 334,
"fields": {
"优先级": "**C**",
"时间": "2026-08-03",
"论文": "[HarnessCompass: Guiding Automatic Harness Evolution toward Generalizable and Effective Agent Harnesses](https://arxiv.org/abs/2608.01918)",
"本质定位": "针对 full-harness evolution 的三类问题——task overfit、只依赖 trajectory outcome、组件同时改造成 interference——加入 task-agnostic constraint、agent first-person proactive feedback、component-wise evolve→consolidate。",
"什么在变": "system prompt、tool descriptions/implementations、middleware、subagent configs、skills、long-term memory 等 full harness components。",
"谁来改 / 谁执行": "**改**：evolution meta-agent；各 component 分开优化后再 R³ merge。
**执行**：GPT-5.4 + 当前 harness（主 SWE-bench setting；role agents共享同 base model）。",
"基础 harness": "H0 为极简 harness（shell command tool，无 middleware/skills/subagents）并与 AHE 同 seed 比。",
"Feedback": "trajectory grounded evidence + agent 第一人称 harness-use feedback（blind/hindsight/grounded variants）+ evaluation score；另有 generalization gate。",
"Evolution → Eval": "SWE-bench Verified evolution sample 上5 turns；held-out tasks检验，另做跨模型 transfer。",
"Meta-depth": "M1；modifier algorithm固定。",
"相对之前真正新增什么": "相对 AHE 的核心新增非常明确：**直接把 generalization、feedback richness、component interference 当 evolution algorithm 的设计目标**，不是事后分析。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "有 held-out 报告",
"protocolBasis": "原记录报告独立任务评估；查看详细记录中的数据与选模限制。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "限制 task-specific shortcuts，加入 agent 对自己使用 harness 的第一人称反馈，再逐组件进化并合并，直接处理过拟合与组件间干扰。",
"novelty": "限制框架写入题目专属信息，主动收集诊断证据，并逐个组件优化，以减少记题和同时改多处造成的干扰。",
"object": "系统提示、工具说明和实现、中间处理、sub-agent 配置、技能与长期记忆；外层修改算法固定。",
"executor": "主执行模型GPT-5.4；冻结运行框架后换Claude Sonnet4.6（non-thinking）测试迁移。",
"modifier": "代码 task agent、轨迹分析器、反馈 task agent、meta-agent（负责设计或修改 task agent）均用GPT-5.4；职责分离但共享同一基础模型。",
"roleContext": "**改**：evolution meta-agent；各 component 分开优化后再 R³ merge。
**执行**：GPT-5.4 + 当前 harness（主 SWE-bench setting；role agents共享同 base model）。",
"seed": "与 AHE 使用同样的仅提供 Bash 命令行工具 H₀：没有初始 middleware（处理模型与工具之间消息、调用和返回结果的中间代码）、skills、subagents。修改受到跨任务通用性约束，不能把某题私有符号或路径编码进规则。",
"fixed": "modifier algorithm固定",
"verdict": "运行 SWE-bench 仓库测试判断补丁是否解决问题，再结合执行日志和执行模型对自身失败的解释决定怎么改框架。",
"diagnosis": "结合 trajectory-derived evidence 与 first-person harness-use feedback。",
"update": "在通用性约束下分别改 components，再统一整合。",
"acceptance": "约束/gating 限制 task-specific knowledge 与不通用改动；观察合并后的互扰。",
"experiments": [
{
"name": "SWE-bench Verified",
"evolve": "50 evolution tasks",
"selection": "在 evolution set 上 5 turns 搜索",
"test": "450 disjoint held-out tasks",
"isolation": "独立测试",
"note": "full-500 Total 包含 evolution 题；应同时报告 Sample / Held-Out / Total。"
},
{
"name": "模型迁移",
"evolve": "GPT-5.4 已进化 harness",
"selection": "冻结",
"test": "Claude-Sonnet-4.6 不重新进化",
"isolation": "冻结后迁移",
"note": ""
}
],
"takeaway": "起点仍很简；主实验集中 SWE coding；多个新机制一起加入，三者独立贡献虽有分析但整体 attribution仍不如单变量。 **对我们：** 这是 AHE 后最应该读的改进：直接处理我们关心的 overfit、component interference 与 richer feedback。",
"result": "sample 54→66%（5 iterations）；held-out 60.4%，高于 AHE 54.7%；total 61.0 vs AHE 55.5/seed 51.8。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.01918v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SWE-bench Verified中50题作为进化集；GPT-5.4在仅提供 Bash 命令行工具 seed上运行。\n\n调试 / 选版本数据：同50题的执行轨迹及 task agent 自述共同诊断，结构与指导两个轨道分别修改、比较并整合。\n\n最终测试数据：剩余450题独立留出；另将冻结运行框架迁移Claude Sonnet4.6。表中Total500包含进化题。\n\n数据隔离与证据边界：Sample50与Held-Out450互斥；跨模型结果也拆开两部分，不能把Total500当全未见。",
"cycle": "限制修改为任务无关组件，将第一人称反馈与轨迹互证；分开优化工具/控制结构和提示/技能/记忆以减少改动干扰。",
"train": "SWE-bench Verified中50题作为进化集；GPT-5.4在仅提供 Bash 命令行工具 seed上运行。",
"debug": "同50题的执行轨迹及 task agent 自述共同诊断，结构与指导两个轨道分别修改、比较并整合。",
"test": "剩余450题独立留出；另将冻结运行框架迁移Claude Sonnet4.6。表中Total500包含进化题。",
"isolation": "Sample50与Held-Out450互斥；跨模型结果也拆开两部分，不能把Total500当全未见。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "与 AHE 使用同样的 bash-only H₀：没有初始 middleware、skills、subagents。修改受到跨任务通用性约束，不能把某题私有符号或路径编码进规则。",
"protocol": "**进化→测试：**SWE-bench Verified 50 道 evolution 题、450 道互斥 held-out 题；总计 500。Table 2 的 Sample/Held-Out/Total 分别代表这三种口径。\n\n**迁移：**在 GPT-5.4 上演化后将冻结 harness 用于 Claude-Sonnet-4.6，对应同一分区。Total 包含训练题，不应当作纯留出成绩；独立验证分区本轮未核实。",
"sections": "Table 2 与跨模型设置",
"source": "https://arxiv.org/abs/2608.01918",
"version": "2608.01918v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f6b5407a419ea170bfad90a53ef04dbd789b4d85c810a1de30a981e2d2c6920a",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主执行模型GPT-5.4；冻结运行框架后换Claude Sonnet4.6（non-thinking）测试迁移。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "Models.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx1.SSS0.Px3"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "代码 task agent、轨迹分析器、反馈 task agent、meta-agent（负责设计或修改 task agent）均用GPT-5.4；职责分离但共享同一基础模型。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "Models.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx1.SSS0.Px3"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "系统提示、工具说明和实现、中间处理、sub-agent 配置、技能与长期记忆；外层修改算法固定。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行 SWE-bench 仓库测试判断补丁是否解决问题，再结合执行日志和执行模型对自身失败的解释决定怎么改框架。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "与 AHE 使用同样的仅提供 Bash 命令行工具 H₀：没有初始 middleware（处理模型与工具之间消息、调用和返回结果的中间代码）、skills、subagents。修改受到跨任务通用性约束，不能把某题私有符号或路径编码进规则。",
"sources": [
{
"label": "Table 2 与跨模型设置",
"url": "https://arxiv.org/abs/2608.01918"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "限制修改为任务无关组件，将第一人称反馈与轨迹互证；分开优化工具/控制结构和提示/技能/记忆以减少改动干扰。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SWE-bench Verified中50题作为进化集；GPT-5.4在仅提供 Bash 命令行工具 seed上运行。",
"sources": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "同50题的执行轨迹及 task agent 自述共同诊断，结构与指导两个轨道分别修改、比较并整合。",
"sources": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "剩余450题独立留出；另将冻结运行框架迁移Claude Sonnet4.6。表中Total500包含进化题。",
"sources": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "Sample50与Held-Out450互斥；跨模型结果也拆开两部分，不能把Total500当全未见。",
"sources": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "限制框架写入题目专属信息，主动收集诊断证据，并逐个组件优化，以减少记题和同时改多处造成的干扰。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.01918v1",
"version": "2608.01918v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对 AHE，将 task-agnostic constraints、proactive feedback 与 component-wise optimization 作为方法的一部分。",
"feedbackCases": [
{
"label": "进化：SWE-bench Verified",
"data": "50 道进化题，每轮重新运行当前框架。",
"scoring": "基准测试给出 0/1 任务结果；失败后同一 GPT-5.4 再回答两组诊断问题，对自身执行过程提供解释。",
"visible": "外部测试结论、完整执行记录和模型自述同时供修改者使用；自述不是另一个正确性裁判。",
"use": "结构代码与行为指导分两条路径修改，用同 50 题比较候选。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"judgment": "仓库测试给 0/1；GPT-5.4 另作失败诊断，不替代测试判分"
},
{
"label": "最终测试与跨模型迁移",
"data": "其余 450 道未参与进化的 SWE-bench Verified 题；另将框架交给 Claude Sonnet 4.6。",
"scoring": "仍由代码测试检查补丁是否解决任务。",
"visible": "只统计冻结框架的通过率；Total 500 指标包含那 50 道进化题。",
"use": "检验新题与新执行模型上的效果，不能把 Total 当成全体未见题成绩。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"judgment": "仓库测试检查补丁是否修复问题"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "SWE-bench Verified中50题作为进化集；GPT-5.4在仅提供 Bash 命令行工具 seed上运行。",
"selection": "同50题的执行轨迹及 task agent 自述共同诊断，结构与指导两个轨道分别修改、比较并整合。",
"evaluation": "剩余450题独立留出；另将冻结运行框架迁移Claude Sonnet4.6。表中Total500包含进化题。",
"isolation": "Sample50与Held-Out450互斥；跨模型结果也拆开两部分，不能把Total500当全未见。",
"roles": {
"executor": {
"value": "主执行模型GPT-5.4；冻结运行框架后换Claude Sonnet4.6（non-thinking）测试迁移。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "Models.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx1.SSS0.Px3"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
]
},
"modifier": {
"value": "代码 task agent、轨迹分析器、反馈 task agent、meta-agent（负责设计或修改 task agent）均用GPT-5.4；职责分离但共享同一基础模型。",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
},
{
"label": "Models.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx1.SSS0.Px3"
},
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
]
},
"seed": {
"value": "与 AHE 使用同样的仅提供 Bash 命令行工具 H₀：没有初始 middleware（处理模型与工具之间消息、调用和返回结果的中间代码）、skills、subagents。修改受到跨任务通用性约束，不能把某题私有符号或路径编码进规则。",
"sources": [
{
"label": "Table 2 与跨模型设置",
"url": "https://arxiv.org/abs/2608.01918"
}
]
}
},
"sources": {
"evolution": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"selection": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"evaluation": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
],
"isolation": [
{
"label": "The advantage is more evident on held-out tasks.",
"url": "https://arxiv.org/html/2608.01918#Sx4.SSx2.SSS0.Px2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "自动框架进化可能针对固定搜索题集写入捷径，使训练涨分却不能迁移；外部观察者又难仅凭轨迹知道执行模型为什么难以使用某个设计。多个组件同时修改还会相互干扰，导致收益不稳定，因此作者关注如何约束、诊断和组织整个进化过程。",
"sources": [
{
"label": "Introduction · 三项局限及其后果",
"url": "https://arxiv.org/html/2608.01918#Sx1"
}
]
},
{
"key": "position",
"value": "研究自动框架进化怎样获得可迁移的真实能力提升，减少记题式适配、诊断不足及组件修改相互干扰造成的退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.01918"
}
]
},
{
"key": "conclusion",
"value": "在软件修复中用较少迭代获得提升，并展示更强的留出任务和跨模型迁移。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.01918"
}
]
}
],
"fields": {
"seed": "与 AHE 相同的仅提供 Bash 命令行工具初始 agent，未配备专用中间处理、技能或sub-agent。修改必须能跨任务使用，不允许把某道题的特定路径或符号写进规则。",
"object": "系统提示、工具说明和实现、中间处理、sub-agent 配置、技能与长期记忆；外层修改算法固定。",
"verdict": "运行 SWE-bench 仓库测试判断补丁是否解决问题，再结合执行日志和执行模型对自身失败的解释决定怎么改框架。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.08466",
"title": "Hierarchical Self-Improvement: A Framework for Task-Specific Evolvable Agent Harnesses (HSI)",
"url": "https://arxiv.org/abs/2608.08466",
"date": "2026-08-09",
"priority": "K",
"categories": [
"methods"
],
"tags": [],
"fields": {
"本质定位": "同一个 frozen LLM 分三层：task harness H 执行；evolver 改 H；meta-evolver 改 evolver strategy Σ；最外层 meta-evolver execution logic/selector仍冻结。",
"什么在变": "task harness code（per-step policy/prompts/hooks/memory/tools）+ evolver strategy中的 seed/commit selection；outer anchor不变。",
"谁来改 / 谁执行": "**改**：同一 DeepSeek-V4-Flash-Preview 分别在 evolver/meta-evolver scope 自改。
**执行**：同一 frozen DeepSeek-V4-Flash-Preview 执行 current harness。",
"基础 harness": "BALROG task-specific seed harness；fixed `using_harness(agent,task)` seam；frozen outer meta-evolver anchor。",
"Feedback": "environment progress reward，使用 stochastic lower-confidence-bound reward；dev eval 驱动 candidate selection。",
"Evolution → Eval": "BALROG moderate tasks上 evolve；BabaIsAI 有20% unseen held-out split；NLE用于超出 backbone capability 的负例。",
"Meta-depth": "M2：evolver strategy可改；outer anchor/selector仍固定。",
"相对之前真正新增什么": "非常直接地把 full-harness evolution 和 **meta-depth + capability ceiling** 放在同一 controlled setting：task-time thinking关、self-modification thinking开，并明确 feedback-fidelity/backbone-capability 两个 bound。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 336,
"fields": {
"优先级": "**K**",
"时间": "2026-08-09",
"论文": "[Hierarchical Self-Improvement: A Framework for Task-Specific Evolvable Agent Harnesses (HSI)](https://arxiv.org/abs/2608.08466)",
"本质定位": "同一个 frozen LLM 分三层：task harness H 执行；evolver 改 H；meta-evolver 改 evolver strategy Σ；最外层 meta-evolver execution logic/selector仍冻结。",
"什么在变": "task harness code（per-step policy/prompts/hooks/memory/tools）+ evolver strategy中的 seed/commit selection；outer anchor不变。",
"谁来改 / 谁执行": "**改**：同一 DeepSeek-V4-Flash-Preview 分别在 evolver/meta-evolver scope 自改。
**执行**：同一 frozen DeepSeek-V4-Flash-Preview 执行 current harness。",
"基础 harness": "BALROG task-specific seed harness；fixed `using_harness(agent,task)` seam；frozen outer meta-evolver anchor。",
"Feedback": "environment progress reward，使用 stochastic lower-confidence-bound reward；dev eval 驱动 candidate selection。",
"Evolution → Eval": "BALROG moderate tasks上 evolve；BabaIsAI 有20% unseen held-out split；NLE用于超出 backbone capability 的负例。",
"Meta-depth": "M2：evolver strategy可改；outer anchor/selector仍固定。",
"相对之前真正新增什么": "非常直接地把 full-harness evolution 和 **meta-depth + capability ceiling** 放在同一 controlled setting：task-time thinking关、self-modification thinking开，并明确 feedback-fidelity/backbone-capability 两个 bound。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M2"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "同一个 frozen LLM 分三层：task harness H 执行；evolver 改 H；meta-evolver 改 evolver strategy Σ；最外层 meta-evolver execution logic/selector仍冻结。",
"novelty": "在固定外部边界内允许任务框架与修改策略分层变化，并控制基础模型能力及反馈质量，研究进一步改进何时受限。",
"object": "任务运行框架和产生修改的进化策略代码；最外层控制及选择规则固定。",
"executor": "冻结的 DeepSeek-V4-Flash-Preview 配合当前运行框架执行任务。",
"modifier": "DeepSeek-V4-Flash-Preview 分别承担进化者和元进化者，在各自允许编辑的文件范围内修改代码。",
"roleContext": "**改**：同一 DeepSeek-V4-Flash-Preview 分别在 evolver/meta-evolver scope 自改。
**执行**：同一 frozen DeepSeek-V4-Flash-Preview 执行 current harness。",
"seed": "BALROG 各任务初始运行框架，通过固定 using_harness 接口接入；上层修改者和负责修改改进流程的上层修改者分开，外层锚固定，结束时的最佳版本选择阶段也不可进化。初始版本不预先评分，首轮冷启动。",
"fixed": "outer anchor/selector仍固定",
"verdict": "游戏环境给出任务进展奖励；多次执行后采用偏保守的分数比较候选，降低偶然高分的影响。开发反馈驱动修改，验证结果用于最终选版本。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "BALROG moderate tasks上 evolve；BabaIsAI 有20% unseen held-out split；NLE用于超出 backbone capability 的负例。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：BALROG的BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE；Setup A使用全任务集交互进化。\n\n调试 / 选版本数据：开发集用于进化奖励，val用于终局选最佳版本；元层可修改进化器，终局提交选择阶段固定。\n\n最终测试数据：Setup A在同任务增加评估回合；Setup B对BabaIsAI子族留出20% 测试集：BreakStop、GoTo、Make。\n\n数据隔离与证据边界：Setup A是分布内重测；Setup B才是未见任务。Advanced仅3题，被排除于拆分测试。每套件配置见表3。",
"cycle": "分别演化任务运行框架与上层修改者；用收益下界降低少量回合的噪声，在固定预算下选择最终代码。",
"train": "BALROG的BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE；Setup A使用全任务集交互进化。",
"debug": "开发集用于进化奖励，val用于终局选最佳版本；元层可修改进化器，终局提交选择阶段固定。",
"test": "Setup A在同任务增加评估回合；Setup B对BabaIsAI子族留出20% 测试集：BreakStop、GoTo、Make。",
"isolation": "Setup A是分布内重测；Setup B才是未见任务。Advanced仅3题，被排除于拆分测试。每套件配置见表3。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "BALROG 各任务初始 harness，通过固定 using_harness 接口接入；上层 evolver 和 meta-evolver 分开，外层锚固定，结束时的最佳版本选择阶段也不可进化。初始版本不预先评分，首轮冷启动。",
"protocol": "**环境：**BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE。前述除 BabaIsAI 外的五类主要测遇过任务的随机变化：搜索单次 episode，最终增加评估回合数。\n\n**BabaIsAI 留出：**按 BreakStop、GoTo、Make、Advanced 子族拆 development/validation/test，test 搜索不可见；只对前三类报告留出，Advanced 仅三题不足以形成有意义划分。两类协议不能混称未见任务泛化。",
"sections": "BALROG 实验协议；Table 3",
"source": "https://arxiv.org/abs/2608.08466",
"version": "2608.08466v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "34b80235606bf853cd608e349a8bfcd976864e1acada6c632c7bec565c5a6503",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "冻结的 DeepSeek-V4-Flash-Preview 配合当前运行框架执行任务。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "DeepSeek-V4-Flash-Preview 分别承担进化者和元进化者，在各自允许编辑的文件范围内修改代码。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务运行框架和产生修改的进化策略代码；最外层控制及选择规则固定。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "游戏环境给出任务进展奖励；多次执行后采用偏保守的分数比较候选，降低偶然高分的影响。开发反馈驱动修改，验证结果用于最终选版本。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "BALROG 各任务初始运行框架，通过固定 using_harness 接口接入；上层修改者和负责修改改进流程的上层修改者分开，外层锚固定，结束时的最佳版本选择阶段也不可进化。初始版本不预先评分，首轮冷启动。",
"sources": [
{
"label": "BALROG 实验协议；Table 3",
"url": "https://arxiv.org/abs/2608.08466"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "分别演化任务运行框架与上层修改者；用收益下界降低少量回合的噪声，在固定预算下选择最终代码。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "BALROG的BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE；Setup A使用全任务集交互进化。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "开发集用于进化奖励，val用于终局选最佳版本；元层可修改进化器，终局提交选择阶段固定。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "Setup A在同任务增加评估回合；Setup B对BabaIsAI子族留出20% 测试集：BreakStop、GoTo、Make。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "Setup A是分布内重测；Setup B才是未见任务。Advanced仅3题，被排除于拆分测试。每套件配置见表3。",
"sources": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在固定外部边界内允许任务框架与修改策略分层变化，并控制基础模型能力及反馈质量，研究进一步改进何时受限。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.08466v1",
"version": "2608.08466v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "非常直接地把 full-harness evolution 和 **meta-depth + capability ceiling** 放在同一 controlled setting：task-time thinking关、self-modification thinking开，并明确 feedback-fidelity/backbone-capability 两个 bound。",
"feedbackCases": [
{
"label": "BALROG：任务与进化器修改",
"data": "BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE；开发执行用于改进，验证执行用于最终选版本。",
"scoring": "各游戏环境的 progress reward 衡量任务进展；多次随机执行后使用置信下界分数（z=0.5），降低偶然高分候选的优势。",
"visible": "执行轨迹及开发奖励，形成带分数的版本档案。",
"use": "进化器据此改任务框架，元层还可改进化器；最终提交选择阶段固定，不能自行改评分规则。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
}
],
"judgment": "BALROG 各游戏程序计算进展奖励，再按重复运行统计筛选"
},
{
"label": "两种最终评价范围",
"data": "设置 A 使用全部任务进化，再增加同任务评估回合；设置 B 对 BabaIsAI 的 BreakStop、GoTo、Make 子族各留出 20%。",
"scoring": "仍由游戏环境计算进展分数。",
"visible": "额外回合或留出任务的最后成绩。",
"use": "A 主要检查随机执行稳定性；B 才有未参与进化的任务划分，两种泛化证据不同。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
}
],
"judgment": "同一游戏环境规则计算进展分"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "BALROG的BabyAI、BabaIsAI、Crafter、MiniHack、TextWorld、NLE；Setup A使用全任务集交互进化。",
"selection": "开发集用于进化奖励，val用于终局选最佳版本；元层可修改进化器，终局提交选择阶段固定。",
"evaluation": "Setup A在同任务增加评估回合；Setup B对BabaIsAI子族留出20% 测试集：BreakStop、GoTo、Make。",
"isolation": "Setup A是分布内重测；Setup B才是未见任务。Advanced仅3题，被排除于拆分测试。每套件配置见表3。",
"roles": {
"executor": {
"value": "冻结的 DeepSeek-V4-Flash-Preview 配合当前运行框架执行任务。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
]
},
"modifier": {
"value": "DeepSeek-V4-Flash-Preview 分别承担进化者和元进化者，在各自允许编辑的文件范围内修改代码。",
"sources": [
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
]
},
"seed": {
"value": "BALROG 各任务初始运行框架，通过固定 using_harness 接口接入；上层修改者和负责修改改进流程的上层修改者分开，外层锚固定，结束时的最佳版本选择阶段也不可进化。初始版本不预先评分，首轮冷启动。",
"sources": [
{
"label": "BALROG 实验协议；Table 3",
"url": "https://arxiv.org/abs/2608.08466"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"selection": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"evaluation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
],
"isolation": [
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.08466#S4.SS2.SSS2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.08466#A2.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有自改进有的只修改局部决策代码，有的依赖外部或更强模型设计完整框架，还没有充分检验 agent 能否自行改造自己的运行框架。已有涨分又可能来自额外搜索或对评测任务的过拟合，因此需要在模型冻结时分清自主框架进化的真实收益及其边界。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.08466#S1"
}
]
},
{
"key": "position",
"value": "研究冻结模型能否在稳定的外部边界内，既改进任务运行框架又改进自身的修改策略，并考察这种多层改进的能力上限。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.08466"
}
]
},
{
"key": "conclusion",
"value": "在中等难度环境中有效，但突破不了基础模型能力及反馈质量的限制。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.08466"
}
]
}
],
"fields": {
"object": "任务运行框架和产生修改的进化策略代码；最外层控制及选择规则固定。",
"verdict": "游戏环境给出任务进展奖励；多次执行后采用偏保守的分数比较候选，降低偶然高分的影响。开发反馈驱动修改，验证结果用于最终选版本。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2608.09380",
"title": "OpenLoopEvolve: A Verifiable Self-Evolution Framework for Loop Policies in Long-Horizon Complex Tasks",
"url": "https://arxiv.org/abs/2608.09380",
"date": "2026-08-10",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"org:tsinghua"
],
"fields": {
"本质定位": "把 observation/planning/memory/action/verification/recovery/stopping/budget 八类 control rule 显式化为 versioned Loop Policy；支持 online/offline evolution、Champion–Challenger、release monitor与 rollback。",
"什么在变": "八部分 Loop Policy +版本/lineage；模型weights不以更新为主。",
"谁来改 / 谁执行": "**改**：LLM proposer 生成候选 policy；固定 Champion–Challenger evaluation/release机制。
**执行**：long-horizon agent + 当前 Loop Policy。",
"基础 harness": "初始 Loop Policy / YC-Bench runtime。",
"Feedback": "online continuous-operation feedback 或 offline archived traces/failure evidence + task success/risk metrics。",
"Evolution → Eval": "YC-Bench simulated business tasks；online release在后续 task boundary生效并监控，主要不是 sealed held-out protocol。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对“直接改一份 agent code”的新点是 **把长程 control policy 当治理化、可版本追踪的 harness asset**，并把 release/monitor/rollback纳入 evolution lifecycle。"
},
"details": [],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 337,
"fields": {
"优先级": "**K**",
"时间": "2026-08-10",
"论文": "[OpenLoopEvolve: A Verifiable Self-Evolution Framework for Loop Policies in Long-Horizon Complex Tasks](https://arxiv.org/abs/2608.09380)",
"本质定位": "把 observation/planning/memory/action/verification/recovery/stopping/budget 八类 control rule 显式化为 versioned Loop Policy；支持 online/offline evolution、Champion–Challenger、release monitor与 rollback。",
"什么在变": "八部分 Loop Policy +版本/lineage；模型weights不以更新为主。",
"谁来改 / 谁执行": "**改**：LLM proposer 生成候选 policy；固定 Champion–Challenger evaluation/release机制。
**执行**：long-horizon agent + 当前 Loop Policy。",
"基础 harness": "初始 Loop Policy / YC-Bench runtime。",
"Feedback": "online continuous-operation feedback 或 offline archived traces/failure evidence + task success/risk metrics。",
"Evolution → Eval": "YC-Bench simulated business tasks；online release在后续 task boundary生效并监控，主要不是 sealed held-out protocol。",
"Meta-depth": "M1。",
"相对之前真正新增什么": "相对“直接改一份 agent code”的新点是 **把长程 control policy 当治理化、可版本追踪的 harness asset**，并把 release/monitor/rollback纳入 evolution lifecycle。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 observation/planning/memory/action/verification/recovery/stopping/budget 八类 control rule 显式化为 versioned Loop Policy；支持 online/offline evolution、Champion–Challenger、release monitor与 rollback。",
"novelty": "把观察、规划、验证和恢复策略保存为有版本的可复用资源，将发布前检查、使用后监测和必要的回退纳入更新流程。",
"object": "观察、规划、记忆、行动、验证、恢复、停止与预算控制策略，以及版本关系。",
"executor": "DeepSeek-V4-Flash，YC-Bench官方medium配置，20轮上下文，加载当前Loop Policy。",
"modifier": "同一实验配置的DeepSeek-V4-Flash生成候选Loop Policy；固定Champion–Challenger程序负责配对比较、发布和回滚。",
"roleContext": "**改**：LLM proposer 生成候选 policy；固定 Champion–Challenger evaluation/release机制。
**执行**：long-horizon agent + 当前 Loop Policy。",
"seed": "作者在 YC-Bench 业务模拟环境中维护有版本的控制流程，规定怎样观察、计划、调用工具、验证、重试和停止。新候选与当前保留版本在相同条件下比较，通过检查后在下一任务开始时启用；发现退化可退回旧版本。",
"fixed": "",
"verdict": "读取企业经营的在线记录或归档失败经历，再在相同条件下比较候选与当前版本的资金、存活、任务成功和风险指标。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "YC-Bench simulated business tasks；online release在后续 task boundary生效并监控，主要不是 sealed held-out protocol。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：YC-Bench企业经营轨迹；在线从运行中积累，离线从归档轨迹生成Loop Policy候选。\n\n调试 / 选版本数据：固定模型/工具/资源做候选与Champion配对评估；在线共12次候选更新，离线每轮3候选、最多4轮。\n\n最终测试数据：官方medium配置、seed1/2/3、20轮上下文，从相同初态模拟一年；评最终资金、存活、成功与风险。\n\n数据隔离与证据边界：在线评测中持续更新；离线策略评测时冻结。§V没有另列跨场景未见测试，跨场景验证被列作未来工作。",
"cycle": "把观察、规划、验证、恢复和停止等规则作为可更新的运行策略，连同验证证据和版本历史保存。候选通过检查才发布；小范围试运行发现退化时回滚，并隔离相应反馈。",
"train": "YC-Bench企业经营轨迹；在线从运行中积累，离线从归档轨迹生成Loop Policy候选。",
"debug": "固定模型/工具/资源做候选与Champion配对评估；在线共12次候选更新，离线每轮3候选、最多4轮。",
"test": "官方medium配置、seed1/2/3、20轮上下文，从相同初态模拟一年；评最终资金、存活、成功与风险。",
"isolation": "在线评测中持续更新；离线策略评测时冻结。§V没有另列跨场景未见测试，跨场景验证被列作未来工作。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在 YC-Bench 业务模拟器上运行可版本化 Loop Policy，明确管理观察、计划、工具、验证、重试和停止。候选与当前 Champion 在相同条件下比较，通过门控后只在任务边界启用，退化可回滚。",
"protocol": "**进化/评测：**YC-Bench 模拟长期企业经营，比较固定初始策略、在线和离线 Loop Policy 演化的业务表现、成功率和风险。经验来自经营轨迹及可归因的反馈。\n\n**待核实：**训练轨迹、候选门控场景和最终报告场景的具体数量与重叠尚未确认；“通过验证才发布”本身不证明用了独立测试集。",
"sections": "框架与 YC-Bench 实验",
"source": "https://arxiv.org/abs/2608.09380",
"version": "2608.09380v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "4021e3c84b8efe98bfa55f34ec94d50bc008fd206dea7d1fde7391227ef28284",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "DeepSeek-V4-Flash，YC-Bench官方medium配置，20轮上下文，加载当前Loop Policy。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "同一实验配置的DeepSeek-V4-Flash生成候选Loop Policy；固定Champion–Challenger程序负责配对比较、发布和回滚。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "观察、规划、记忆、行动、验证、恢复、停止与预算控制策略，以及版本关系。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "读取企业经营的在线记录或归档失败经历，再在相同条件下比较候选与当前版本的资金、存活、任务成功和风险指标。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "作者在 YC-Bench 业务模拟环境中维护有版本的控制流程，规定怎样观察、计划、调用工具、验证、重试和停止。新候选与当前保留版本在相同条件下比较，通过检查后在下一任务开始时启用；发现退化可退回旧版本。",
"sources": [
{
"label": "框架与 YC-Bench 实验",
"url": "https://arxiv.org/abs/2608.09380"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "把观察、规划、验证、恢复和停止等规则作为可更新的运行策略，连同验证证据和版本历史保存。候选通过检查才发布；小范围试运行发现退化时回滚，并隔离相应反馈。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "YC-Bench企业经营轨迹；在线从运行中积累，离线从归档轨迹生成Loop Policy候选。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "固定模型/工具/资源做候选与Champion配对评估；在线共12次候选更新，离线每轮3候选、最多4轮。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "官方medium配置、seed1/2/3、20轮上下文，从相同初态模拟一年；评最终资金、存活、成功与风险。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "在线评测中持续更新；离线策略评测时冻结。§V没有另列跨场景未见测试，跨场景验证被列作未来工作。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把观察、规划、验证和恢复策略保存为有版本的可复用资源，将发布前检查、使用后监测和必要的回退纳入更新流程。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.09380v1",
"version": "2608.09380v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对“直接改一份 agent code”的新点是 **把长程 control policy 当治理化、可版本追踪的 harness asset**，并把 release/monitor/rollback纳入 evolution lifecycle。",
"feedbackCases": [
{
"label": "YC-Bench：在线 / 离线政策搜索",
"data": "企业经营任务；在线积累运行证据共 12 次候选更新，离线从归档轨迹每轮提 3 候选、最多 4 轮。",
"scoring": "在相同模型、工具、资源和初始状态下执行候选与当前最佳版本，比较经营资金、存活、成功、风险等任务指标。",
"visible": "经营状态、失败轨迹和配对评估结果；不是只让修改者为自己写出的规则打分。",
"use": "候选必须满足质量、风险与收益门槛才能发布；没有合格候选则继续保留原版本。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"judgment": "YC-Bench 仿真状态与经营指标的程序化比较"
},
{
"label": "YC-Bench：比较协议",
"data": "官方 medium 配置，seed 1 / 2 / 3，20 轮上下文，从同初态模拟一年。",
"scoring": "用模拟经营系统的最终资金等状态计算效果，按协议汇总价值、可靠性、尾部表现和成本。",
"visible": "配对运行结果进入选择和论文统计。",
"use": "这是受控经营场景中的候选比较；不能把同种子反复选择描述成从未接触的独立测试。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"judgment": "模拟器输出资金等状态，按协议计算可靠性与成本"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "YC-Bench企业经营轨迹；在线从运行中积累，离线从归档轨迹生成Loop Policy候选。",
"selection": "固定模型/工具/资源做候选与Champion配对评估；在线共12次候选更新，离线每轮3候选、最多4轮。",
"evaluation": "官方medium配置、seed1/2/3、20轮上下文，从相同初态模拟一年；评最终资金、存活、成功与风险。",
"isolation": "在线评测中持续更新；离线策略评测时冻结。§V没有另列跨场景未见测试，跨场景验证被列作未来工作。",
"roles": {
"executor": {
"value": "DeepSeek-V4-Flash，YC-Bench官方medium配置，20轮上下文，加载当前Loop Policy。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
}
]
},
"modifier": {
"value": "同一实验配置的DeepSeek-V4-Flash生成候选Loop Policy；固定Champion–Challenger程序负责配对比较、发布和回滚。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.09380#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2608.09380#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.09380#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
}
]
},
"seed": {
"value": "作者在 YC-Bench 业务模拟环境中维护有版本的控制流程，规定怎样观察、计划、调用工具、验证、重试和停止。新候选与当前保留版本在相同条件下比较，通过检查后在下一任务开始时启用；发现退化可退回旧版本。",
"sources": [
{
"label": "框架与 YC-Bench 实验",
"url": "https://arxiv.org/abs/2608.09380"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"selection": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"evaluation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
],
"isolation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.09380#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.09380#S5.SS2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有经验学习和框架优化尚未把观察、规划、记忆、验证、恢复与预算之间的完整控制关系作为统一改进对象，旧轨迹因而难转化为可比较、可迁移的流程修改。生成修改后直接使用又可能把噪声和退化带入后续任务，因此需要可验证的候选比较与版本替换过程。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.09380#S1"
}
]
},
{
"key": "position",
"value": "面向长程复杂业务，研究观察、规划、验证和恢复经验如何成为可持续积累的控制能力，并在更新时保持收益与风险可检验。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09380"
}
]
},
{
"key": "conclusion",
"value": "在模拟业务任务上改善表现与风险指标，重点是可管理地发布和复用控制经验。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.09380"
}
]
}
],
"fields": {
"object": "观察、规划、记忆、行动、验证、恢复、停止与预算控制策略，以及版本关系。",
"verdict": "读取企业经营的在线记录或归档失败经历，再在相同条件下比较候选与当前版本的资金、存活、任务成功和风险指标。",
"seed": "作者在 YC-Bench 业务模拟环境中维护有版本的控制流程，规定怎样观察、计划、调用工具、验证、重试和停止。新候选与当前保留版本在相同条件下比较，通过检查后在下一任务开始时启用；发现退化可退回旧版本。"
}
},
"attributions": [
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.09380"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.12307",
"title": "AI4AI at Test-Time: Strong-to-Weak Capability Transfer via Harnesses",
"url": "https://arxiv.org/abs/2608.12307",
"date": "2026-08-12",
"priority": "C",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"HeldOut",
"M1",
"StrongerBuilder",
"org:uiuc",
"person:heng-ji"
],
"fields": {
"本质定位": "强 builder 在约5% labeled validation 上反复看 scaffold accuracy 与错误样本 (x,gold,pred)，修改 executable scaffold；freeze 后弱 target 在95% hidden test。",
"什么在变": "executable scaffold/harness。",
"谁来改 / 谁执行": "**改**：strong builder via Cursor/Claude Code/Codex；有 same-model control。
**执行**：主要 GPT-5.4-mini，另 Gemini-3.5-flash。",
"基础 harness": "vanilla frozen target + builder-generated scaffold。",
"Feedback": "显式 gold validation feedback。",
"Evolution → Eval": "195 validation → freeze → 3900 hidden test。",
"Meta-depth": "M1 strong-to-weak；self control 较弱。",
"相对之前真正新增什么": "新点主要是 **干净的 strong-to-weak harness transfer protocol**，不是 recursive self-evolution。"
},
"details": [
{
"label": "定位 / 真正新点",
"text": "强 builder 在约5% labeled validation 上反复看 scaffold accuracy 与错误样本 (x,gold,pred)，修改 executable scaffold；freeze 后弱 target 在95% hidden test。 **相对前序：** 新点主要是 **干净的 strong-to-weak harness transfer protocol**，不是 recursive self-evolution。"
},
{
"label": "什么在变",
"text": "executable scaffold/harness。"
},
{
"label": "谁来改 / 谁执行",
"text": "**改：** strong builder via Cursor/Claude Code/Codex；有 same-model control。 **执行：** 主要 GPT-5.4-mini，另 Gemini-3.5-flash。"
},
{
"label": "基础 harness",
"text": "vanilla frozen target + builder-generated scaffold。"
},
{
"label": "Feedback",
"text": "显式 gold validation feedback。"
},
{
"label": "Evolution → Eval / Meta-depth",
"text": "195 validation → freeze → 3900 hidden test。 **Meta-depth：** M1 strong-to-weak；self control 较弱。"
},
{
"label": "主要结果",
"text": "vanilla avg .488；scaffolded mean .763；best .912；validation-best 与 hidden r=.96。"
},
{
"label": "最关键限制 / 对我们的意义",
"text": "feedback 很强（gold label）；ToM task 结构规则，human-inspired harness 仍更高。 **对我们：** 我们应借它的 evolution→freeze→hidden 协议，而不是照搬强 builder/gold feedback。"
}
],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 338,
"fields": {
"优先级": "**C**",
"时间": "2026-08-12",
"论文": "[AI4AI at Test-Time: Strong-to-Weak Capability Transfer via Harnesses](https://arxiv.org/abs/2608.12307)",
"本质定位": "强 builder 在约5% labeled validation 上反复看 scaffold accuracy 与错误样本 (x,gold,pred)，修改 executable scaffold；freeze 后弱 target 在95% hidden test。",
"什么在变": "executable scaffold/harness。",
"谁来改 / 谁执行": "**改**：strong builder via Cursor/Claude Code/Codex；有 same-model control。
**执行**：主要 GPT-5.4-mini，另 Gemini-3.5-flash。",
"基础 harness": "vanilla frozen target + builder-generated scaffold。",
"Feedback": "显式 gold validation feedback。",
"Evolution → Eval": "195 validation → freeze → 3900 hidden test。",
"Meta-depth": "M1 strong-to-weak；self control 较弱。",
"相对之前真正新增什么": "新点主要是 **干净的 strong-to-weak harness transfer protocol**，不是 recursive self-evolution。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "Train → selection → test",
"protocolBasis": "依据原记录的 Evolution → Eval：进化/选模后在独立测试任务评估；具体任务和访问边界见原文。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "更强 builder 从少量验证题中发现可复用结构，为冻结弱模型编写路由、prompt、确定性子程序与验证 scaffold。",
"novelty": "由强模型把部分推理和检查逻辑写成运行代码，交给弱模型执行任务时使用，实现无需训练弱模型参数的能力支持。",
"object": "为冻结弱模型构建的执行代码、路由与验证流程；不更新弱模型参数。",
"executor": "主任务执行者GPT-5.4-mini；目标模型对照换Gemini3.5 Flash。强构建者仅构造运行框架，不代替目标模型答最终测试。",
"modifier": "跨目标对照的五种构建者为Opus4.7、GPT-5.5、Gemini3.1 Pro、Gemini3.5 Flash、Grok0.1，统一在Cursor构造运行框架；构造者与目标GPT-5.4-mini/Gemini3.5 Flash分开。",
"roleContext": "**改**：strong builder via Cursor/Claude Code/Codex；有 same-model control。
**执行**：主要 GPT-5.4-mini，另 Gemini-3.5-flash。",
"seed": "构建者 获得可用工作区、任务/提交规则、engines.py 模型调用示例及有标签验证题，自行构造可调用入口；允许路由、提示、检索、前后处理和符号求解，没有固定候选架构。对照是直接调用目标模型，以及人写 UserHarness。",
"fixed": "",
"verdict": "用心智推理任务的可见验证题检查回答正确性，供强模型选择运行框架；最终评测与验证数据的具体边界存在原文口径冲突，表格保留了说明。",
"diagnosis": "从 validation slice 找出可编译的 task structure 与弱模型错误。",
"update": "编写 task routing、prompt、symbolic procedures、format/verification 逻辑。",
"acceptance": "根据 validation 迭代后导出最终 entry point，隐藏评估时无 builder 介入。",
"experiments": [
{
"name": "四类 Theory-of-Mind benchmark",
"evolve": "195-item validation（5%）",
"selection": "validation-only 改进",
"test": "其余样本 hidden evaluation",
"isolation": "独立测试",
"note": "旧记录“195→3900 hidden test”过于武断；正文同时使用 full-set 指标，应区分总量与实际 hidden remainder。"
}
],
"takeaway": "feedback 很强（gold label）；ToM task 结构规则，human-inspired harness 仍更高。 **对我们：** 我们应借它的 evolution→freeze→hidden 协议，而不是照搬强 builder/gold feedback。",
"result": "vanilla avg .488；scaffolded mean .763；best .912；validation-best 与 hidden r=.96。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.12307v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：不训练模型参数；强构建者在目标任务的195道有标签验证题上编写给弱模型用的运行框架。\n\n调试 / 选版本数据：可运行目标模型查看验证准确率并改代码，最终提交后构建者退出。\n\n最终测试数据：BigToM1200、Hi-ToM1200、MMToM-QA600、MuMA-ToM900，总3900；多模态题转成文本。\n\n数据隔离与证据边界：原文对full-set与隐藏 测试集的口径不一致：验证占5%，§4又将3900称隐藏 测试集。可确认构建者只见验证材料；不能确认195与3900完全互斥，亦不自行改成3705。",
"cycle": "强模型为固定弱模型构建代码辅助、上下文组织和答案检查，使部分不可靠推理转成稳定处理步骤。强模型负责构建，弱模型负责最终执行，两者参数不因这轮框架搜索而更新。",
"train": "不训练模型参数；强构建者在目标任务的195道有标签验证题上编写给弱模型用的运行框架。",
"debug": "可运行目标模型查看验证准确率并改代码，最终提交后构建者退出。",
"test": "BigToM1200、Hi-ToM1200、MMToM-QA600、MuMA-ToM900，总3900；多模态题转成文本。",
"isolation": "原文对full-set与隐藏 测试集的口径不一致：验证占5%，§4又将3900称隐藏 测试集。可确认构建者只见验证材料；不能确认195与3900完全互斥，亦不自行改成3705。"
},
"reviewed": true,
"readingNote": {
"source": "data/updated-research-notes.md",
"attribution": "用户更新调研笔记",
"intro": "不用更新 target model 权重，让更强 builder 在少量 labeled validation 上反复构造 executable scaffold，把能力以 test-time harness 的形式 transfer 给弱模型。",
"index": {
"date": "2026-08-12",
"paper": "[AI4AI at Test-Time: Strong-to-Weak Capability Transfer via Harnesses](https://arxiv.org/abs/2608.12307)",
"executor": "冻结的 target model，主实验为 **GPT-5.4-mini**，另测 Gemini-3.5-flash",
"modifier": "更强 builder model 通过 Cursor / Claude Code / Codex 等 builder-side harness 修改 scaffold",
"feedback": "约 **5% labeled validation**：accuracy + 错误样本的 **input / gold label / prediction**",
"evolve": "BigToM / Hi-ToM / MMToM-QA / MuMA-ToM 中抽取 **195 个 labeled validation items（约 5%）** 反复优化 scaffold",
"eval": "同四个 ToM benchmark 的 **3,900 个 hidden test items（约 95%）**；builder 在 evolution 时不可见",
"tldr": "强 builder 根据显式 gold-error feedback 反复修改 executable scaffold，freeze 后在 95% hidden test 上评测；数据隔离很干净，但主 setting 是 strong-to-weak scaffolding，不是严格 self-RSI。"
},
"sections": [
{
"title": "1. 现有工作 Gap / 定位 / 贡献",
"body": "| 项目 | 内容 |\n|---|---|\n| 现有工作 Gap | 常规 strong-to-weak capability transfer / distillation 主要在 training time 更新弱模型参数；论文问：能否**完全不训练 target model**，只让强模型设计 inference-time harness 来完成能力转移。 |\n| 本文定位 | **Strong-to-weak scaffolding**：builder 在少量 labeled validation 上构造、测试、诊断并改 executable scaffold；target model 权重冻结；scaffold freeze 后在 builder 从未见过的 hidden test 上评测。 |\n| 核心贡献 | ① 把 capability transfer 转到 test-time harness；② 比较不同 builder、platform、reasoning effort、target model；③ 分析哪些 scaffold mechanism 真正带来提升；④ 包含 GPT-5.4-mini→自身的 self-scaffolding control。 |"
},
{
"title": "2. 方法与 Self-Evolution Setting",
"body": "| 项目 | 内容 |\n|---|---|\n| 什么在进化 | **Executable scaffold / harness**；target model 权重始终冻结。 |\n| 谁来改 | 多个 builder model，通过 Cursor / Claude Code / GPT Codex 等 builder-side platform 构造 scaffold；主结果中强 builder 包括 GPT-5.5、Opus-4.7、Gemini 系列等。 |\n| 谁来执行 | 主 target 是 **GPT-5.4-mini**；另测 **Gemini-3.5-flash**。 |\n| Builder 能改什么 | 不限制固定 scaffold architecture：可写 prompt template、benchmark routing、deterministic pre/post-processing、format enforcement、verification pass、few-shot retrieval、symbolic/direct solver 等 executable procedure。 |\n| Builder 每轮能看到什么 | 当前 scaffold、validation accuracy，以及错误样本 **$(x,\\; gold\\ y,\\; prediction\\ \\hat y)$**；因此 feedback 是显式 **gold-labeled validation feedback**。 |\n| 迭代 | `inspect task → propose/revise scaffold → validation eval → inspect errors → revise`，builder 自己决定何时提交；最终 scaffold freeze。 |\n| 方法本质 | 主 setting 是 **strong-to-weak automated scaffolding**，不是严格 self-evolution；但有 same-model self-scaffolding control。 |"
},
{
"title": "3. 实验 / Benchmark / 数据与 Feedback",
"body": "| 项目 | 内容 |\n|---|---|\n| Benchmark | 4 个 Theory-of-Mind 数据集：BigToM、Hi-ToM、MMToM-QA、MuMA-ToM |\n| Hidden test | 合计 **3,900 items**；builder 不可访问 |\n| Evolution / validation | 固定随机种子抽取 **195 items（约 5%）labeled validation** |\n| Feedback | validation accuracy + 每个错误样本的 input / gold answer / prediction |\n| Train/dev/test 隔离 | **较干净**：builder 只在 5% validation 上反复改 scaffold；提交后由 evaluator 在 hidden full test 上运行，builder 不看 test result 再修改 |\n| 人工标注 / GT | **明确使用正确答案作为 evolution feedback**，但仅限 validation；hidden test GT 不暴露给 builder |\n| Target 权重 | 全程 frozen；没有 fine-tuning |"
},
{
"title": "4. 主要结果与结论",
"body": "| 结果 / 发现 | 实验依据 | 能支持的结论 |\n|---|---|---|\n| Strong-to-weak scaffold 带来大幅提升 | GPT-5.4-mini vanilla macro avg **0.488**；所有 scaffolded runs mean **0.763**；best run **0.912**（+0.423） | 强 builder 能通过外部 harness 给固定弱 target 注入大量 task structure |\n| Iterative validation 确实有效 | 平均 first logged validation → best validation **+0.216**；平均约 4.9 次 validation pass | builder 能利用 explicit labeled errors 改 scaffold，而不只是一次性 generation |\n| Validation improvement 能迁移到 hidden test | best validation 与 full-set accuracy **Pearson r=0.96**；平均 optimism gap 约 0.021 | 5% validation 在这些 ToM benchmark 上没有表现出明显严重过拟合，是相对干净的 evolution→test 证据 |\n| 主要收益来自 structure externalization | 论文分析认为 deterministic code、benchmark-specific routing、strict answer-format enforcement 等比单纯增加 target reasoning/sampling 更关键 | harness 可以把不稳定 reasoning 外化为可执行规则与 control logic |\n| Self-scaffolding 也有提升，但弱于强 builder | GPT-5.4-mini 为自己搭 scaffold：Cursor **+0.217**、GPT Codex **+0.168**；stronger builders 对应平均 uplift 更大 | 同模型也能发现一部分可利用结构，但 high-performance regime 主要由更强 builder 解锁 |\n| 自动 scaffold 仍未全面超过人类设计 | GPT-5.4-mini human-inspired UserHarness **0.939**，best automated **0.912** | 自动 harness engineering 很强，但仍存在人类设计上限差距 |"
},
{
"title": "5. 评判 / 缺点",
"body": "| 问题 | 评判 |\n|---|---|\n| Story 与实验匹配 | **匹配 strong-to-weak capability transfer**：target 权重固定、builder 真正反复修改 executable scaffold，并在 hidden test 验证。 |\n| 是否严格 RSI | **主实验不是。** builder 通常比 target 强，属于 strong-to-weak scaffolding；self-scaffolding 只是 control，且提升明显小于 stronger-builder setting。 |\n| Feedback 难度 | Evolution feedback 很强：builder 不仅有 scalar accuracy，还能看到错误样本的 **gold label + prediction**。因此不能把结果解释成“仅靠自我反思/无监督 feedback 就能自进化”。 |\n| 数据隔离 | **这是它相对强的地方。** validation 与 hidden test 明确隔离，builder 不能看 test；比直接在最终 benchmark tasks 上反复优化的 setting 更能证明 harness strategy 泛化。 |\n| Benchmark generality | 全部是 ToM QA，结构高度规则化；BigToM 甚至可被 scaffold 转成接近 deterministic rules。能否迁移到 BrowseComp、coding、open-ended agentic tasks 仍未验证。 |\n| 能力上限口径 | best scaffolded GPT-5.4-mini 可超过更强 unscaffolded GPT-5.4，但不等于突破 builder/system 的能力上限；human-inspired harness 仍更高。 |\n| 对我们的启发 | 这是较干净的 harness evolution protocol：**小规模 evolution/dev set → scaffold freeze → hidden held-out tasks**。后续研究可在更开放 agentic task 上复用这一评估结构，再测 capability ceiling、稳定性和行为偏移。 |"
},
{
"title": "总评",
"body": "**AI4AI at Test-Time 是目前较干净的 harness optimization/evolution 证据之一：target model 完全冻结，builder 在约 5% labeled validation 上基于明确 gold-error feedback 反复改 executable scaffold，最终在不可见的 3,900-item hidden test 上验证，且 validation-best 与 hidden performance 高相关。它主要证明 strong-to-weak test-time capability transfer，而不是严格 RSI；真正值得延伸的是把这种干净的 evolution→freeze→held-out evaluation protocol 放到更复杂的 agentic harness 场景。**"
}
],
"caveats": [
{
"text": "样本数口径需区分：论文 §3.1 定义 5% validation / 其余 hidden test，§4.1 又称 3,900-item hidden test 并另列 195 validation，结果分析使用 full-set accuracy。下文保留笔记和论文报告的 3,900，不能仅凭这些表述把 3,900 与 195 当成已核实的互斥集合大小。",
"url": "https://arxiv.org/html/2608.12307v1"
}
]
},
"systemDataAudit": {
"seed": "builder 获得可用工作区、任务/提交规则、engines.py 模型调用示例及有标签验证题，自行构造可调用入口；允许路由、提示、检索、前后处理和符号求解，没有固定候选架构。对照是直接调用目标模型，以及人写 UserHarness。",
"protocol": "**构建/选版本：**195 道有标签验证题供 builder 调试和选择，目标模型固定；完成后导出代码由评估者运行，builder 不再介入。\n\n**四套题：**BigToM 1,200、Hi-ToM 1,200、MMToM-QA 600、MuMA-ToM 900，均转为文本题。前两者测观察/递归信念，后两者测目标、信念与多 agent 推理。\n\n**原文口径冲突：**§4 将 3,900 称为 hidden test、另给 195 validation，同时报告 full-set accuracy；前面的协议描述则按验证/剩余测试解释。不能不加说明地断言是“195＋3,900 完全独立”或自行算成 3,705，保留补注。",
"sections": "Algorithm 1、§3–4 与原文口径补注",
"source": "https://arxiv.org/abs/2608.12307",
"version": "2608.12307v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "8512e0a6d3e8201f640371293771b22b43f821d086c812322688ed83d423e2ed",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主任务执行者GPT-5.4-mini；目标模型对照换Gemini3.5 Flash。强构建者仅构造运行框架，不代替目标模型答最终测试。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.12307#S4"
},
{
"label": "§5.6",
"url": "https://arxiv.org/html/2608.12307#S5.SS6"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "跨目标对照的五种构建者为Opus4.7、GPT-5.5、Gemini3.1 Pro、Gemini3.5 Flash、Grok0.1，统一在Cursor构造运行框架；构造者与目标GPT-5.4-mini/Gemini3.5 Flash分开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.12307#S4"
},
{
"label": "§5.6",
"url": "https://arxiv.org/html/2608.12307#S5.SS6"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "为冻结弱模型构建的执行代码、路由与验证流程；不更新弱模型参数。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "用心智推理任务的可见验证题检查回答正确性，供强模型选择运行框架；最终评测与验证数据的具体边界存在原文口径冲突，表格保留了说明。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "构建者 获得可用工作区、任务/提交规则、engines.py 模型调用示例及有标签验证题，自行构造可调用入口；允许路由、提示、检索、前后处理和符号求解，没有固定候选架构。对照是直接调用目标模型，以及人写 UserHarness。",
"sources": [
{
"label": "Algorithm 1、§3–4 与原文口径补注",
"url": "https://arxiv.org/abs/2608.12307"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "强模型为固定弱模型构建代码辅助、上下文组织和答案检查，使部分不可靠推理转成稳定处理步骤。强模型负责构建，弱模型负责最终执行，两者参数不因这轮框架搜索而更新。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "不训练模型参数；强构建者在目标任务的195道有标签验证题上编写给弱模型用的运行框架。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "可运行目标模型查看验证准确率并改代码，最终提交后构建者退出。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "BigToM1200、Hi-ToM1200、MMToM-QA600、MuMA-ToM900，总3900；多模态题转成文本。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "原文对full-set与隐藏 测试集的口径不一致：验证占5%，§4又将3900称隐藏 测试集。可确认构建者只见验证材料；不能确认195与3900完全互斥，亦不自行改成3705。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "由强模型把部分推理和检查逻辑写成运行代码，交给弱模型执行任务时使用，实现无需训练弱模型参数的能力支持。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.12307v1",
"version": "2608.12307v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "研究 strong-to-weak 能力如何外化到推理时环境；并不是弱模型本身学到了新权重。",
"feedbackCases": [
{
"label": "验证数据：四类心智推理任务",
"data": "BigToM、Hi-ToM、MMToM-QA、MuMA-ToM 中合计 195 道有标签验证题；多模态内容已转成文字。",
"scoring": "按题目的正确答案标签计算准确率，强模型可运行固定弱模型并查看验证结果。",
"visible": "验证题、标签、执行代码和验证准确率可用；工作区不提供完整题库。",
"use": "强模型修改弱模型的运行框架，提交后退出。论文正文与附录对验证比例有不同口径，不能用附录“2%”推算这 195 题。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"judgment": "按四类心智推理题的标准标签计准确率；文中未逐项展开解析器"
},
{
"label": "完整题库评测",
"data": "BigToM 1,200、Hi-ToM 1,200、MMToM-QA 600、MuMA-ToM 900，共 3,900。",
"scoring": "仍按各任务标签计算正确率，并按四基准平均。",
"visible": "只由弱模型配提交框架运行，强模型不参与代答。",
"use": "论文报告 full-set；不要默认它等同于扣除了所有可见验证题的独立测试集。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"judgment": "沿用各任务标签判分；与创建者的验证反馈分开报告"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "不训练模型参数；强构建者在目标任务的195道有标签验证题上编写给弱模型用的运行框架。",
"selection": "可运行目标模型查看验证准确率并改代码，最终提交后构建者退出。",
"evaluation": "BigToM1200、Hi-ToM1200、MMToM-QA600、MuMA-ToM900，总3900；多模态题转成文本。",
"isolation": "原文对full-set与隐藏 测试集的口径不一致：验证占5%，§4又将3900称隐藏 测试集。可确认构建者只见验证材料；不能确认195与3900完全互斥，亦不自行改成3705。",
"roles": {
"executor": {
"value": "主任务执行者GPT-5.4-mini；目标模型对照换Gemini3.5 Flash。强构建者仅构造运行框架，不代替目标模型答最终测试。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.12307#S4"
},
{
"label": "§5.6",
"url": "https://arxiv.org/html/2608.12307#S5.SS6"
}
]
},
"modifier": {
"value": "跨目标对照的五种构建者为Opus4.7、GPT-5.5、Gemini3.1 Pro、Gemini3.5 Flash、Grok0.1，统一在Cursor构造运行框架；构造者与目标GPT-5.4-mini/Gemini3.5 Flash分开。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.12307#S4"
},
{
"label": "§5.6",
"url": "https://arxiv.org/html/2608.12307#S5.SS6"
}
]
},
"seed": {
"value": "构建者 获得可用工作区、任务/提交规则、engines.py 模型调用示例及有标签验证题，自行构造可调用入口；允许路由、提示、检索、前后处理和符号求解，没有固定候选架构。对照是直接调用目标模型，以及人写 UserHarness。",
"sources": [
{
"label": "Algorithm 1、§3–4 与原文口径补注",
"url": "https://arxiv.org/abs/2608.12307"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.12307#S5.SS3"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "把强模型能力传给弱模型通常依赖额外训练，但弱模型失败也可能是任务组织方式带来的负担。运行框架可能帮助它发挥已有能力，却缺少系统解释：哪些设计有效、何时稳定、涨分是否只是题目捷径；这使框架间难以公平比较、复现和改进。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.12307#S1"
}
]
},
{
"key": "position",
"value": "研究强模型能否在不训练弱模型参数的情况下，将部分能力转移给弱模型，扩大其在测试时能够完成的任务范围。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.12307"
}
]
},
{
"key": "conclusion",
"value": "心智推理基准上显著提升弱模型；主要收益来自把不稳定推理转成代码及规范化处理。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.12307"
}
]
}
],
"fields": {
"object": "为冻结弱模型构建的执行代码、路由与验证流程；不更新弱模型参数。",
"verdict": "用心智推理任务的可见验证题检查回答正确性，供强模型选择运行框架；最终评测与验证数据的具体边界存在原文口径冲突，表格保留了说明。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.12307"
}
]
},
{
"tag": "person:heng-ji",
"label": "Heng Ji",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2608.12307"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.24735",
"title": "Meta^n: Recursive Self-Improvement through Emergent Depth",
"url": "https://arxiv.org/abs/2608.24735",
"date": "2026-08-25",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"M1",
"SameModel"
],
"fields": {
"本质定位": "Same-model layered harness evolution：把 successive edits 保留为当前 runtime 中同时生效的 append-only layers；不是 modifier 自身更新。",
"什么在变": "strategic context + callable helper-code layer stack；base weights 与固定 $\\Omega$ 不变。",
"谁来改 / 谁执行": "**改/执行**：Gemma 4 31B-IT 或 GPT-5.2 同一 frozen backbone；modifier 使用固定 $\\Omega$ prompt。",
"基础 harness": "single-shot solver 或最多 8-turn `generate→execute→observe→refine` agentic solver。",
"Feedback": "evaluator score + execution evidence；d2 raw traces，d≥3 performance/failure summary + representative traces + prior code stack。",
"Evolution → Eval": "CO-Bench/S2D/LawBench/ARC-AGI-2 有 held-out；TB2/AlphaEvolve Math/SR/AlgoTune 为 same-set adaptive。",
"Meta-depth": "M1（显式 layer depth 增长，但 modifier/search machinery 固定）。",
"相对之前真正新增什么": "真正新增更接近 **history as active composition**；并非首次利用 edit history。约 72% recursion gain 来自 inter-layer context conditioning。"
},
"details": [
{
"label": "为什么不是 S",
"text": "和 Self-Harness / Meta-Harness / AHE 的核心能力重合较高；它没有让 modifier $\\Omega$ 自身更新，因此不能把 “meta-depth 增长”直接解释成 improver self-improvement。"
},
{
"label": "真正新增",
"text": "**history as active composition**：successive edits 不是 merge/overwrite 后只留一个 current harness，而是保留成当前 runtime 中同时生效的 append-only executable layers，后层可 retain/suppress/override/combine 前层。"
},
{
"label": "最重要实证",
"text": "recursion 有收益，但约 **72% gain 来自 inter-layer context conditioning**；depth 并非越深越好，d2→d3 有明显 regression。"
},
{
"label": "关键缺口",
"text": "缺 `nested layers vs flatten/merge with identical history access` 的严格对照，因此还不能证明 nested representation 本身是收益来源。"
}
],
"occurrences": [
{
"section": "A4. Full Harness / Self-Modifying Agent Program",
"line": 339,
"fields": {
"优先级": "**K**",
"时间": "2026-08-25",
"论文": "[Meta$^n$: Recursive Self-Improvement through Emergent Depth](https://arxiv.org/abs/2608.24735)",
"本质定位": "Same-model layered harness evolution：把 successive edits 保留为当前 runtime 中同时生效的 append-only layers；不是 modifier 自身更新。",
"什么在变": "strategic context + callable helper-code layer stack；base weights 与固定 $\\Omega$ 不变。",
"谁来改 / 谁执行": "**改/执行**：Gemma 4 31B-IT 或 GPT-5.2 同一 frozen backbone；modifier 使用固定 $\\Omega$ prompt。",
"基础 harness": "single-shot solver 或最多 8-turn `generate→execute→observe→refine` agentic solver。",
"Feedback": "evaluator score + execution evidence；d2 raw traces，d≥3 performance/failure summary + representative traces + prior code stack。",
"Evolution → Eval": "CO-Bench/S2D/LawBench/ARC-AGI-2 有 held-out；TB2/AlphaEvolve Math/SR/AlgoTune 为 same-set adaptive。",
"Meta-depth": "M1（显式 layer depth 增长，但 modifier/search machinery 固定）。",
"相对之前真正新增什么": "真正新增更接近 **history as active composition**；并非首次利用 edit history。约 72% recursion gain 来自 inter-layer context conditioning。"
}
}
],
"source": "original",
"review": "原记录 · 详细介绍",
"protocol": "混合协议",
"protocolBasis": "不同实验的划分不同，不能用一个 held-out 标签概括；见逐任务协议。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "把 successive edits 保留成执行时同时存在的分层 context / helper-code stack，后层可以组合或抑制前层。",
"novelty": "让历史改进层参与当前执行中的辅助与上下文组织，研究增加辅助层数的效果；层数增加本身不表示外部修改规则也改变。",
"object": "逐层增加的策略上下文及可调用辅助代码；基础模型和生成下一层的操作保持固定。",
"executor": "Gemma 4 31B-IT 或 GPT-5.2；单次生成与最多 8-turn 多步交互任务求解模型。",
"modifier": "与执行端对应的 Gemma 4 31B-IT 或 GPT-5.2 读取固定 Ω 模板，生成下一层优化提示；不更换或训练基础模型。",
"roleContext": "**改/执行**：Gemma 4 31B-IT 或 GPT-5.2 同一 frozen backbone；modifier 使用固定 $\\Omega$ prompt。",
"seed": "三种底座按任务区分：可编辑 Python 任务求解模型、Docker 内 bash task agent、固定下游模型前的提示改写器。同一递归模板控制改进层次；单次生成和最多 8 轮的多步交互执行是不同实验变体。",
"fixed": "模型权重与 Ω / outer orchestration 固定。",
"verdict": "候选程序实际运行后，返回输出、报错、退出状态及任务得分；组合优化、分类、终端任务等分别由相应评估器判断，不能用统一“答对率”概括。",
"diagnosis": "前层执行证据、failure summaries、representative traces 与 code stack。",
"update": "添加、组合或修订 helper-code/context layers。",
"acceptance": "按 benchmark 选择 candidate，另有 archive/consolidation 设置。",
"experiments": [
{
"name": "CO / S2D / LawBench / ARC",
"evolve": "各自 dev/evolution split",
"selection": "按任务选择",
"test": "held-out tasks",
"isolation": "独立测试",
"note": ""
},
{
"name": "TB2 / AlphaEvolve Math / SR / AlgoTune",
"evolve": "benchmark 内适应",
"selection": "benchmark score",
"test": "相同 benchmark 指标",
"isolation": "同集适应",
"note": ""
}
],
"takeaway": "缺 `nested layers vs flatten/merge with identical history access` 的严格对照，因此还不能证明 nested representation 本身是收益来源。",
"result": "recursion 有收益，但约 **72% gain 来自 inter-layer context conditioning**；depth 并非越深越好，d2→d3 有明显 regression。",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.24735v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：八类任务：CO-Bench36问题、AlphaEvolve Math、四域符号回归、AlgoTune8任务、ARC-AGI-2120题、TB2 89题、Symptom2Disease、LawBench。\n\n调试 / 选版本数据：对候选层链作档案搜索，由统一Ω操作递归生成上层辅助，按验证/任务分数选版本与停止。\n\n最终测试数据：CO-Bench、S2D、LawBench报告留出；ARC另报留出；AlphaEvolve Math、AlgoTune、符号回归直接报优化目标成绩。\n\n数据隔离与证据边界：没有统一训练／测试协议；后三类明确无留出。常规3个seed，TB2有单seed及按任务类别计算方差的例外。",
"cycle": "同一元操作可反复作用形成更深层，不预设只到二层；不同层逐渐承担原语、库、策略等角色。",
"train": "八类任务：CO-Bench36问题、AlphaEvolve Math、四域符号回归、AlgoTune8任务、ARC-AGI-2120题、TB2 89题、Symptom2Disease、LawBench。",
"debug": "对候选层链作档案搜索，由统一Ω操作递归生成上层辅助，按验证/任务分数选版本与停止。",
"test": "CO-Bench、S2D、LawBench报告留出；ARC另报留出；AlphaEvolve Math、AlgoTune、符号回归直接报优化目标成绩。",
"isolation": "没有统一训练／测试协议；后三类明确无留出。常规3个seed，TB2有单seed及按任务类别计算方差的例外。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "三种底座按任务区分：可编辑 Python solver、Docker 内 bash agent、固定下游模型前的提示改写器。同一递归模板控制改进层次；single-shot 和最多 8 轮的 agentic 执行是不同实验变体。",
"protocol": "**八类 benchmark：**CO-Bench 36 个 NP-hard 问题、AlphaEvolve Math、四域 Symbolic Regression、AlgoTune 8 题、ARC-AGI-2 120 题、Terminal-Bench 2.0 89 题、Symptom2Disease、LawBench。\n\n**隔离：**AlphaEvolve Math、AlgoTune、SR 没有 held-out split，直接报告优化目标分数；其他任务的 dev/test 需按各自设置。多数结果为 seed 42/43/44 三次，TB2 有特殊统计口径，不能统一称三次独立重复。具体划分数量本轮待核实。",
"sections": "实验设置；附录 C",
"source": "https://arxiv.org/abs/2608.24735",
"version": "2608.24735v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "94d6a04fb7ac725790dea56c6789689f667153f85ac3229b1e70fb855fd13879",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Gemma 4 31B-IT 或 GPT-5.2；单次生成与最多 8-turn 多步交互任务求解模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "与执行端对应的 Gemma 4 31B-IT 或 GPT-5.2 读取固定 Ω 模板，生成下一层优化提示；不更换或训练基础模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "逐层增加的策略上下文及可调用辅助代码；基础模型和生成下一层的操作保持固定。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "候选程序实际运行后，返回输出、报错、退出状态及任务得分；组合优化、分类、终端任务等分别由相应评估器判断，不能用统一“答对率”概括。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
},
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "三种底座按任务区分：可编辑 Python 任务求解模型、Docker 内 bash task agent、固定下游模型前的提示改写器。同一递归模板控制改进层次；单次生成和最多 8 轮的多步交互执行是不同实验变体。",
"sources": [
{
"label": "实验设置；附录 C",
"url": "https://arxiv.org/abs/2608.24735"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "同一元操作可反复作用形成更深层，不预设只到二层；不同层逐渐承担原语、库、策略等角色。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "八类任务：CO-Bench36问题、AlphaEvolve Math、四域符号回归、AlgoTune8任务、ARC-AGI-2120题、TB2 89题、Symptom2Disease、LawBench。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "对候选层链作档案搜索，由统一Ω操作递归生成上层辅助，按验证/任务分数选版本与停止。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "CO-Bench、S2D、LawBench报告留出；ARC另报留出；AlphaEvolve Math、AlgoTune、符号回归直接报优化目标成绩。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "没有统一训练／测试协议；后三类明确无留出。常规3个seed，TB2有单seed及按任务类别计算方差的例外。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "让历史改进层参与当前执行中的辅助与上下文组织，研究增加辅助层数的效果；层数增加本身不表示外部修改规则也改变。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.24735v1",
"version": "2608.24735v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "history 是 active composition，而非只供 proposer 参考的日志；层数增长并不表示固定 modifier Ω 自己发生进化。",
"feedbackCases": [
{
"label": "CO-Bench",
"data": "36 个 NP-hard 组合优化问题",
"scoring": "执行 Python 求解器，按各问题可行性与目标归一到 0–1。",
"visible": "每次候选的代码、stdout / stderr、退出码、分数及评估器附加信息，供固定的 Ω 修改操作读取。",
"use": "按任务得分选择递归生成的求解层；CO、S2D、LawBench 和 ARC 另有留出评价，数学 / 回归 / 加速直接报告优化目标。",
"sources": [
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"judgment": "程序检查组合优化解的可行性与目标值"
},
{
"label": "Symptom2Disease / LawBench",
"data": "S2D 的 22 类疾病诊断；LawBench 多标签罪名预测",
"scoring": "S2D 对照类别标签算准确率；LawBench 对照罪名标签算 F1（同时考虑找出的内容有多少正确、应找出的内容找到了多少；匹配对象看当前任务）。",
"visible": "每次候选的代码、stdout / stderr、退出码、分数及评估器附加信息，供固定的 Ω 修改操作读取。",
"use": "按任务得分选择递归生成的求解层；CO、S2D、LawBench 和 ARC 另有留出评价，数学 / 回归 / 加速直接报告优化目标。",
"sources": [
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"judgment": "规则比对类别标签，计算准确率／F1"
},
{
"label": "Terminal-Bench 2",
"data": "89 个 Docker 终端任务、13 类",
"scoring": "任务验收程序检查 bash / Python 产物。",
"visible": "每次候选的代码、stdout / stderr、退出码、分数及评估器附加信息，供固定的 Ω 修改操作读取。",
"use": "按任务得分选择递归生成的求解层；CO、S2D、LawBench 和 ARC 另有留出评价，数学 / 回归 / 加速直接报告优化目标。",
"sources": [
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"judgment": "Terminal-Bench 原生验收测试"
},
{
"label": "AlphaEvolve Math / Symbolic Regression / AlgoTune",
"data": "数学发现问题、四领域符号回归、8 个算法加速任务",
"scoring": "分别执行数学目标检查、数据拟合评价、正确性与运行速度评价；用子进程隔离的原生评估适配器。",
"visible": "每次候选的代码、stdout / stderr、退出码、分数及评估器附加信息，供固定的 Ω 修改操作读取。",
"use": "按任务得分选择递归生成的求解层；CO、S2D、LawBench 和 ARC 另有留出评价，数学 / 回归 / 加速直接报告优化目标。",
"sources": [
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"judgment": "程序分别计算数学目标、拟合误差、算法正确性与速度"
},
{
"label": "ARC-AGI-2",
"data": "120 道网格变换题，仅用 GPT-5.2",
"scoring": "根据示例推断规则，预测目标网格，与参考网格比较，报告 pass@2（每题最多尝试两次、至少成功一次的比例）。",
"visible": "每次候选的代码、stdout / stderr、退出码、分数及评估器附加信息，供固定的 Ω 修改操作读取。",
"use": "按任务得分选择递归生成的求解层；CO、S2D、LawBench 和 ARC 另有留出评价，数学 / 回归 / 加速直接报告优化目标。",
"sources": [
{
"label": "§2：执行反馈定义",
"url": "https://arxiv.org/html/2608.24735#S2.SS0.SSS0.Px2"
},
{
"label": "附录 C：八类基准",
"url": "https://arxiv.org/html/2608.24735#A3.SS0.SSS0.Px1"
}
],
"judgment": "规则比较预测网格与标准网格"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4
],
"evolution": "八类任务：CO-Bench36问题、AlphaEvolve Math、四域符号回归、AlgoTune8任务、ARC-AGI-2120题、TB2 89题、Symptom2Disease、LawBench。",
"selection": "对候选层链作档案搜索，由统一Ω操作递归生成上层辅助，按验证/任务分数选版本与停止。",
"evaluation": "CO-Bench、S2D、LawBench报告留出；ARC另报留出；AlphaEvolve Math、AlgoTune、符号回归直接报优化目标成绩。",
"isolation": "没有统一训练／测试协议；后三类明确无留出。常规3个seed，TB2有单seed及按任务类别计算方差的例外。",
"roles": {
"executor": {
"value": "Gemma 4 31B-IT 或 GPT-5.2；单次生成与最多 8-turn 多步交互任务求解模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
]
},
"modifier": {
"value": "与执行端对应的 Gemma 4 31B-IT 或 GPT-5.2 读取固定 Ω 模板，生成下一层优化提示；不更换或训练基础模型。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
]
},
"seed": {
"value": "三种底座按任务区分：可编辑 Python 任务求解模型、Docker 内 bash task agent、固定下游模型前的提示改写器。同一递归模板控制改进层次；单次生成和最多 8 轮的多步交互执行是不同实验变体。",
"sources": [
{
"label": "实验设置；附录 C",
"url": "https://arxiv.org/abs/2608.24735"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"selection": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"evaluation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
],
"isolation": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px8"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "常见自我反思只在同一层里检查、改写答案；即使系统会改代码，搜索方式、变异规则或上层驱动往往仍固定。这样，改进结果并不自动变成更好的改进过程，递归层次也被预设结构限制；作者希望研究更深的改进层次能否在运行中形成。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.24735#S1"
}
]
},
{
"key": "position",
"value": "研究递归改进能否突破固定一层改进者的限制，扩展到更深的辅助层次，以及增加深度在何时真正有益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.24735"
}
]
},
{
"key": "conclusion",
"value": "多个任务族上取得提升，主要收益来自层间传递的上下文；层数更深并不总更好。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.24735"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
]
}
],
"fields": {
"object": "逐层增加的策略上下文及可调用辅助代码；基础模型和生成下一层的操作保持固定。",
"verdict": "候选程序实际运行后，返回输出、报错、退出状态及任务得分；组合优化、分类、终端任务等分别由相应评估器判断，不能用统一“答对率”概括。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2605.27276",
"title": "SIA: Self Improving AI with Harness & Weight Updates",
"url": "https://arxiv.org/abs/2605.27276",
"date": "2026-05-26",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"M1",
"SameSet",
"SeparateEvolver",
"Weights"
],
"fields": {
"本质定位": "Feedback-Agent 根据 target trajectory/score 决定改 harness 还是改 weights；Meta-Agent 先生成 task agent，weight 更新可选择 LoRA/RL 等。",
"什么在变": "harness + model weights。",
"谁来改 / 谁执行": "**改**：Claude Sonnet 4.6 Meta/Feedback agents（论文设置）；训练算法固定/可选择。
**执行**：task agent 如 gpt-oss-120B。",
"基础 harness": "Meta-Agent 生成的 task-specific harness。",
"Feedback": "trajectory + benchmark score/verifier。",
"Evolution → Eval": "LawBench、GPU TriMul、scRNA；LawBench 部分流程使用 test split reward，隔离有问题。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "首次把 harness-update 与 test-time/model-weight update 放在统一 loop 中作为两个 action surface。"
},
"details": [],
"occurrences": [
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 347,
"fields": {
"优先级": "**R**",
"时间": "2026-05-26",
"论文": "[SIA: Self Improving AI with Harness & Weight Updates](https://arxiv.org/abs/2605.27276)",
"本质定位": "Feedback-Agent 根据 target trajectory/score 决定改 harness 还是改 weights；Meta-Agent 先生成 task agent，weight 更新可选择 LoRA/RL 等。",
"什么在变": "harness + model weights。",
"谁来改 / 谁执行": "**改**：Claude Sonnet 4.6 Meta/Feedback agents（论文设置）；训练算法固定/可选择。
**执行**：task agent 如 gpt-oss-120B。",
"基础 harness": "Meta-Agent 生成的 task-specific harness。",
"Feedback": "trajectory + benchmark score/verifier。",
"Evolution → Eval": "LawBench、GPU TriMul、scRNA；LawBench 部分流程使用 test split reward，隔离有问题。",
"Meta-depth": "M1 hybrid。",
"相对之前真正新增什么": "首次把 harness-update 与 test-time/model-weight update 放在统一 loop 中作为两个 action surface。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "Feedback-Agent 根据 target trajectory/score 决定改 harness 还是改 weights；Meta-Agent 先生成 task agent，weight 更新可选择 LoRA/RL 等。",
"novelty": "把改运行框架与训练模型参数作为同一改进流程中的可选操作，根据失败选择改哪里，并比较两类操作的互补收益。",
"object": "task agent 的运行框架与模型参数。",
"executor": "task agent 统一用 gpt-oss-120B；完成训练步骤后，改用它经 LoRA（只训练少量适配参数） 更新的检查点执行任务。",
"modifier": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"roleContext": "**改**：Claude Sonnet 4.6 Meta/Feedback agents（论文设置）；训练算法固定/可选择。
**执行**：task agent 如 gpt-oss-120B。",
"seed": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。",
"fixed": "",
"verdict": "逐步执行记录，以及评测任务的分数或结果检查器的判定。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "LawBench、GPU TriMul、scRNA；LawBench 部分流程使用 test split reward，隔离有问题。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：LawBench191类罪名分类：5332训练样本；另外的TriMul与MAGIC直接围绕目标计算任务优化。\n\n调试 / 选版本数据：Feedback-Agent选择改运行框架或做LoRA（只训练少量适配参数）更新；LawBench评分使用所报告的913题测试集 评分器，迭代成绩用于决策。\n\n最终测试数据：LawBench913题；TriMul在固定形状测H100运行时间；MAGIC以pancreas单细胞数据及参考真值评价去噪。\n\n数据隔离与证据边界：LawBench训练样本与测试集分开，但同一评分器参与反馈，不能称最终分数完全不可见；另外两项不是常规数据集留出。",
"cycle": "从 gpt-oss-120b、一个简单工具调用循环和可训练的适配参数起步。反馈 agent 读取执行结果，决定修改提示、解析和工具逻辑，还是训练模型参数；训练服务负责实际参数更新。",
"train": "LawBench191类罪名分类：5332训练样本；另外的TriMul与MAGIC直接围绕目标计算任务优化。",
"debug": "Feedback-Agent选择改运行框架或做LoRA（只训练少量适配参数）更新；LawBench评分使用所报告的913题测试集 评分器，迭代成绩用于决策。",
"test": "LawBench913题；TriMul在固定形状测H100运行时间；MAGIC以pancreas单细胞数据及参考真值评价去噪。",
"isolation": "LawBench训练样本与测试集分开，但同一评分器参与反馈，不能称最终分数完全不可见；另外两项不是常规数据集留出。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "Meta-Agent 生成面向任务的 harness，另通过 LoRA 适配 gpt-oss-120b；比较只改 harness 和联合改权重/运行系统。各任务有独立可执行 verifier，不能把训练模型和评分器当同一角色。",
"protocol": "**LawBench：**5,332 条训练、913 条留出测试，191 类罪名分类。\n\n**其他任务：**AlphaEvolve TriMul 在固定输入形状上测 H100 运行时间；MAGIC 单细胞去噪用 pancreas scRNA-seq 与参考真值评价，不对应常规 train/test。权重更新 LoRA rank=32；不能把 LawBench 的隔离推广到这两个优化任务。",
"sections": "§6.1、Table 2",
"source": "https://arxiv.org/abs/2605.27276",
"version": "2605.27276v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "de95010125a38ee82cabf88aab112aa6a58b77a390b4315e73befa9def4cc4eb",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "task agent 统一用 gpt-oss-120B；完成训练步骤后，改用它经 LoRA（只训练少量适配参数） 更新的检查点执行任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "task agent 的运行框架与模型参数。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "逐步执行记录，以及评测任务的分数或结果检查器的判定。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。",
"sources": [
{
"label": "§6.1、Table 2",
"url": "https://arxiv.org/abs/2605.27276"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "从 gpt-oss-120b、一个简单工具调用循环和可训练的适配参数起步。反馈 agent 读取执行结果，决定修改提示、解析和工具逻辑，还是训练模型参数；训练服务负责实际参数更新。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "LawBench191类罪名分类：5332训练样本；另外的TriMul与MAGIC直接围绕目标计算任务优化。",
"sources": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "Feedback-Agent选择改运行框架或做LoRA（只训练少量适配参数）更新；LawBench评分使用所报告的913题测试集 评分器，迭代成绩用于决策。",
"sources": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "LawBench913题；TriMul在固定形状测H100运行时间；MAGIC以pancreas单细胞数据及参考真值评价去噪。",
"sources": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "LawBench训练样本与测试集分开，但同一评分器参与反馈，不能称最终分数完全不可见；另外两项不是常规数据集留出。",
"sources": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把改运行框架与训练模型参数作为同一改进流程中的可选操作，根据失败选择改哪里，并比较两类操作的互补收益。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.27276v2",
"version": "2605.27276v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "首次把 harness-update 与 test-time/model-weight update 放在统一 loop 中作为两个 action surface。",
"feedbackCases": [
{
"label": "LawBench",
"data": "191 类罪名分类：5,332 条训练样本，913 道评分题",
"scoring": "对照罪名参考标签评分；论文所称 测试集 评分器 的 913 题会在迭代中调用。",
"visible": "执行轨迹与该任务评估器的数值结果。",
"use": "Feedback-Agent 先改围绕 gpt-oss-120b 的框架，再转向权重更新；LawBench 所报 913 题参与决策，不是冻结后的独立评估。稀疏奖励时可先把较好候选轨迹做行为克隆。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"judgment": "规则对照罪名标注计算任务分"
},
{
"label": "TriMul",
"data": "固定张量形状的 H100 三角矩阵乘法计算",
"scoring": "执行候选，检查计算结果并测 H100 运行时间。",
"visible": "执行轨迹与该任务评估器的数值结果。",
"use": "Feedback-Agent 先改围绕 gpt-oss-120b 的框架，再转向权重更新；LawBench 所报 913 题参与决策，不是冻结后的独立评估。稀疏奖励时可先把较好候选轨迹做行为克隆。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"judgment": "程序检查数值输出正确性，再计 GPU 耗时"
},
{
"label": "MAGIC",
"data": "pancreas 单细胞数据上的去噪任务",
"scoring": "运行去噪方法并与参考真值比较。",
"visible": "执行轨迹与该任务评估器的数值结果。",
"use": "Feedback-Agent 先改围绕 gpt-oss-120b 的框架，再转向权重更新；LawBench 所报 913 题参与决策，不是冻结后的独立评估。稀疏奖励时可先把较好候选轨迹做行为克隆。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"judgment": "程序比较去噪输出与参考真值的误差"
}
],
"experiments": [
{
"label": "LawBench：法律罪名分类",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "191 类罪名、5,332 条训练样本用于训练；913 道评分题返回成绩，指导框架修改及后续参数更新。",
"selection": "反复查询上述 913 题成绩作修改决策。",
"evaluation": "报告同一 913 题的分类成绩。",
"isolation": "评分题参与了进化决策，不是冻结后的独立测试。",
"roles": {
"executor": {
"value": "task agent 统一用 gpt-oss-120B；完成训练步骤后，改用它经 LoRA（只训练少量适配参数） 更新的检查点执行任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"modifier": {
"value": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"seed": {
"value": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。",
"sources": [
{
"label": "§6.1、Table 2",
"url": "https://arxiv.org/abs/2605.27276"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"selection": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"evaluation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"isolation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
]
}
},
{
"label": "TriMul：GPU 算子优化",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "固定张量形状的 H100 三角矩阵乘法任务；运行候选实现并检查正确性、耗时。",
"selection": "依据同一计算任务上的性能反馈选择改动。",
"evaluation": "报告所优化计算任务的正确性与性能。",
"isolation": "直接优化目标计算任务，不是常规问答数据集的训练／测试划分。",
"roles": {
"executor": {
"value": "task agent 统一用 gpt-oss-120B；完成训练步骤后，改用它经 LoRA（只训练少量适配参数） 更新的检查点执行任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"modifier": {
"value": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"seed": {
"value": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。",
"sources": [
{
"label": "§6.1、Table 2",
"url": "https://arxiv.org/abs/2605.27276"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"selection": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"evaluation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"isolation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
]
}
},
{
"label": "MAGIC：单细胞数据去噪",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "pancreas 单细胞数据去噪任务；根据参考真值计算候选结果质量，指导修改。",
"selection": "根据同一目标任务的反馈比较候选。",
"evaluation": "报告该去噪任务上的结果。",
"isolation": "属于直接目标优化；这不能证明在另一套未见数据上泛化。",
"roles": {
"executor": {
"value": "task agent 统一用 gpt-oss-120B；完成训练步骤后，改用它经 LoRA（只训练少量适配参数） 更新的检查点执行任务。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"modifier": {
"value": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"sources": [
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.27276#S5.SS1"
},
{
"label": "§6.2",
"url": "https://arxiv.org/html/2605.27276#S6.SS2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.27276#S5.SS2"
}
]
},
"seed": {
"value": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。",
"sources": [
{
"label": "§6.1、Table 2",
"url": "https://arxiv.org/abs/2605.27276"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"selection": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"evaluation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
],
"isolation": [
{
"label": "§6.1",
"url": "https://arxiv.org/html/2605.27276#S6.SS1"
},
{
"label": "§6.3",
"url": "https://arxiv.org/html/2605.27276#S6.SS3.SSS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "AI 的模型训练和 agent 框架设计仍依赖人，限制自主改进的速度。已有自动化工作又分成两条相对独立的路线：改运行框架时冻结模型，训练模型时固定人工流程；两者无法共同适应任务，因此作者希望把框架与参数更新放入同一个自主改进过程。",
"sources": [
{
"label": "§1.1–1.2 人工瓶颈与两条独立改进路线",
"url": "https://arxiv.org/html/2605.27276#S1.SS1"
}
]
},
{
"key": "position",
"value": "研究自主改进系统能否根据任务失败，在改运行框架与更新模型参数之间作出有效选择，并获得两者互补的收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.27276"
}
]
},
{
"key": "conclusion",
"value": "在法律、GPU 内核和去噪任务中，联合更新优于只改框架，展示两种改进手段的互补性。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.27276"
}
]
}
],
"fields": {
"object": "task agent 的运行框架与模型参数。",
"modifier": "Claude Sonnet 4.6 分别担任上层设计者和反馈分析者，设计 task agent 并决定改运行框架还是训练参数；Modal 云计算平台上的训练程序实际更新 gpt-oss-120B 的参数。",
"seed": "上层设计模型生成面向任务的运行框架，并可通过 LoRA（只训练少量附加适配参数）调整 gpt-oss-120b。实验比较只改框架与联合更新；每类任务另有按程序规则判结果的评分器。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2607.21971",
"title": "Teaching LLMs to Self-Evolve: Cultivating Core Meta-Skills with Reinforcement Learning (MetaEvolve)",
"url": "https://arxiv.org/abs/2607.21971",
"date": "2026-07-24",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"EditorWeights",
"LearnedUpdater",
"org:uiuc",
"person:heng-ji"
],
"fields": {
"本质定位": "在 coding 上用程序执行产生连续 verifiable fitness，合成多轮 evolution histories，并用 RL 训练模型掌握“看历史+看 fitness→提出更好改动”的 meta-skill；推理时再进入 evolutionary search。",
"什么在变": "model weights（self-evolution meta-skill）；推理时 candidate program population/trajectory也变化。",
"谁来改 / 谁执行": "**改**：RL training algorithm 学 modifier policy；inference 使用训练后的同一 model做 evolutionary proposal。
**执行**：trained LLM evolution agent。",
"基础 harness": "coding evolutionary search harness + history/fitness context。",
"Feedback": "test-case execution产生 correctness+efficiency verifiable reward；训练样本含当前程序、fitness、历史 attempts。",
"Evolution → Eval": "7 coding benchmarks ID/OOD + open-ended algorithm optimization transfer。",
"Meta-depth": "model-level M1/M2 hybrid；runtime outer search/fitness固定。",
"相对之前真正新增什么": "和 MetaSkill-Evolve 的区别很关键：MetaSkill-Evolve **外部 meta-skill file 在变、backbone frozen**；MetaEvolve 则把 improver 能力 **写进模型 weights**，再用固定 evolutionary runtime 调用。"
},
"details": [],
"occurrences": [
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 349,
"fields": {
"优先级": "**R**",
"时间": "2026-07-24",
"论文": "[Teaching LLMs to Self-Evolve: Cultivating Core Meta-Skills with Reinforcement Learning (MetaEvolve)](https://arxiv.org/abs/2607.21971)",
"本质定位": "在 coding 上用程序执行产生连续 verifiable fitness，合成多轮 evolution histories，并用 RL 训练模型掌握“看历史+看 fitness→提出更好改动”的 meta-skill；推理时再进入 evolutionary search。",
"什么在变": "model weights（self-evolution meta-skill）；推理时 candidate program population/trajectory也变化。",
"谁来改 / 谁执行": "**改**：RL training algorithm 学 modifier policy；inference 使用训练后的同一 model做 evolutionary proposal。
**执行**：trained LLM evolution agent。",
"基础 harness": "coding evolutionary search harness + history/fitness context。",
"Feedback": "test-case execution产生 correctness+efficiency verifiable reward；训练样本含当前程序、fitness、历史 attempts。",
"Evolution → Eval": "7 coding benchmarks ID/OOD + open-ended algorithm optimization transfer。",
"Meta-depth": "model-level M1/M2 hybrid；runtime outer search/fitness固定。",
"相对之前真正新增什么": "和 MetaSkill-Evolve 的区别很关键：MetaSkill-Evolve **外部 meta-skill file 在变、backbone frozen**；MetaEvolve 则把 improver 能力 **写进模型 weights**，再用固定 evolutionary runtime 调用。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1",
"M2"
],
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "weights",
"brief": {
"summary": "在 coding 上用程序执行产生连续 verifiable fitness，合成多轮 evolution histories，并用 RL 训练模型掌握“看历史+看 fitness→提出更好改动”的 meta-skill；推理时再进入 evolutionary search。",
"novelty": "通过强化学习培养利用反馈、分析历史和提出改进的能力，将这些能力写进修改模型的参数，再用于新的优化任务。",
"object": "训练阶段学习多轮进化能力的模型参数；使用时继续改进候选程序。",
"executor": "经MetaEvolve强化训练的Qwen3-14B执行多轮程序进化；公平对照也使用Qwen3-14B。",
"modifier": "固定强化学习（根据奖励调整模型行为）算法训练Qwen3-14B的诊断/修改能力；测试时由训练后的Qwen3-14B提出下一版程序。",
"roleContext": "**改**：RL training algorithm 学 modifier policy；inference 使用训练后的同一 model做 evolutionary proposal。
**执行**：trained LLM evolution agent。",
"seed": "编程演化搜索框架向模型提供历史候选与执行适应度，训练其生成更好的反思/修改。训练的是执行自演化所需的元技能，不仅是固定优化器在测试题上多采样。",
"fixed": "runtime outer search/fitness固定",
"verdict": "运行测试用例检查正确性与效率，并据此给奖励；训练输入包含当前程序、任务得分及历史尝试。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "7 coding benchmarks ID/OOD + open-ended algorithm optimization transfer。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：PRIME-RL/Eurus-2-RL-Data中的TACO、APPS、Codeforces、CodeContests，合成程序改进经验后强化训练元技能。\n\n调试 / 选版本数据：在每道代码题上执行候选并反馈，用多轮采样、保留top候选迭代程序；附录B比较轮数与采样规模。\n\n最终测试数据：七套题各随机50题：上述四来源的测试题，加完全不参与训练的AtCoder、LeetCode、USACO。\n\n数据隔离与证据边界：后面三套是数据源级留出；评测仍允许逐题程序搜索，不能当作单次直接生成。",
"cycle": "通过强化学习（根据奖励调整模型行为）训练诊断、修改和改进程序的元技能，再把学到的修改能力用于多轮代码进化。",
"train": "PRIME-RL/Eurus-2-RL-Data中的TACO、APPS、Codeforces、CodeContests，合成程序改进经验后强化训练元技能。",
"debug": "在每道代码题上执行候选并反馈，用多轮采样、保留top候选迭代程序；附录B比较轮数与采样规模。",
"test": "七套题各随机50题：上述四来源的测试题，加完全不参与训练的AtCoder、LeetCode、USACO。",
"isolation": "后面三套是数据源级留出；评测仍允许逐题程序搜索，不能当作单次直接生成。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "编程演化搜索框架向模型提供历史候选与执行适应度，训练其生成更好的反思/修改。训练的是执行自演化所需的元技能，不仅是固定优化器在测试题上多采样。",
"protocol": "**训练来源：**PRIME-RL/Eurus-2-RL-Data 的编程训练任务，涉及 TACO、APPS、Codeforces 等。\n\n**测试：**同域和域外共七类 benchmark；AtCoder、LeetCode、USACO 完全不用于训练。每个 benchmark 抽 50 道评估题，每题多轮代码演化并执行验证。第四个训练来源、实际训练总量及各题搜索/最终测试用例区分本轮待核实。",
"sections": "训练合成与 OOD 评估设置",
"source": "https://arxiv.org/abs/2607.21971",
"version": "2607.21971v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "a02aa58c771cf38b7a1426436bbb52e911974b18494647bef98a8e494c6956a8",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "经MetaEvolve强化训练的Qwen3-14B执行多轮程序进化；公平对照也使用Qwen3-14B。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS3.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "固定强化学习（根据奖励调整模型行为）算法训练Qwen3-14B的诊断/修改能力；测试时由训练后的Qwen3-14B提出下一版程序。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS3.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "训练阶段学习多轮进化能力的模型参数；使用时继续改进候选程序。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "运行测试用例检查正确性与效率，并据此给奖励；训练输入包含当前程序、任务得分及历史尝试。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "编程演化搜索框架向模型提供历史候选与执行适应度，训练其生成更好的反思/修改。训练的是执行自演化所需的元技能，不仅是固定优化器在测试题上多采样。",
"sources": [
{
"label": "训练合成与 OOD 评估设置",
"url": "https://arxiv.org/abs/2607.21971"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "通过强化学习（根据奖励调整模型行为）训练诊断、修改和改进程序的元技能，再把学到的修改能力用于多轮代码进化。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "PRIME-RL/Eurus-2-RL-Data中的TACO、APPS、Codeforces、CodeContests，合成程序改进经验后强化训练元技能。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "在每道代码题上执行候选并反馈，用多轮采样、保留top候选迭代程序；附录B比较轮数与采样规模。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "七套题各随机50题：上述四来源的测试题，加完全不参与训练的AtCoder、LeetCode、USACO。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "后面三套是数据源级留出；评测仍允许逐题程序搜索，不能当作单次直接生成。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "通过强化学习培养利用反馈、分析历史和提出改进的能力，将这些能力写进修改模型的参数，再用于新的优化任务。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2607.21971v1",
"version": "2607.21971v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "和 MetaSkill-Evolve 的区别很关键：MetaSkill-Evolve **外部 meta-skill file 在变、backbone frozen**；MetaEvolve 则把 improver 能力 **写进模型 weights**，再用固定 evolutionary runtime 调用。",
"feedbackCases": [
{
"label": "训练元技能：代码改进轨迹",
"data": "PRIME-RL / Eurus-2-RL-Data 中 TACO、APPS、Codeforces、CodeContests 数据。",
"scoring": "运行题目测试用例检查代码正确性与执行效率，提供程序执行产生的奖励。",
"visible": "题目、当前程序、执行得分和历史尝试，供模型反思并提出改进。",
"use": "合成迭代代码经验，再用强化学习训练如何利用反馈改程序。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"judgment": "执行代码测试，检查功能并测运行效率"
},
{
"label": "测试时程序搜索",
"data": "七来源各抽 50 题：上述四来源的测试题，加训练完全未用的 AtCoder、LeetCode、USACO。",
"scoring": "同一题上运行候选程序，以测试执行反馈比较正确性与效率。",
"visible": "本题候选的执行分数和失败记录，允许后续轮继续利用。",
"use": "多轮采样并保留高分程序；这是测试时可获取执行反馈的程序优化，不是每题只答一次的静态测验。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"judgment": "执行候选程序的测试并测效率"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "PRIME-RL/Eurus-2-RL-Data中的TACO、APPS、Codeforces、CodeContests，合成程序改进经验后强化训练元技能。",
"selection": "在每道代码题上执行候选并反馈，用多轮采样、保留top候选迭代程序；附录B比较轮数与采样规模。",
"evaluation": "七套题各随机50题：上述四来源的测试题，加完全不参与训练的AtCoder、LeetCode、USACO。",
"isolation": "后面三套是数据源级留出；评测仍允许逐题程序搜索，不能当作单次直接生成。",
"roles": {
"executor": {
"value": "经MetaEvolve强化训练的Qwen3-14B执行多轮程序进化；公平对照也使用Qwen3-14B。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS3.Px1"
}
]
},
"modifier": {
"value": "固定强化学习（根据奖励调整模型行为）算法训练Qwen3-14B的诊断/修改能力；测试时由训练后的Qwen3-14B提出下一版程序。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS3.Px1"
}
]
},
"seed": {
"value": "编程演化搜索框架向模型提供历史候选与执行适应度，训练其生成更好的反思/修改。训练的是执行自演化所需的元技能，不仅是固定优化器在测试题上多采样。",
"sources": [
{
"label": "训练合成与 OOD 评估设置",
"url": "https://arxiv.org/abs/2607.21971"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"selection": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"evaluation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
],
"isolation": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2607.21971#S3.SS1.SSS1.Px1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.21971#S4.SS3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2607.21971#A2"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "利用多轮反馈持续改进，要求模型能反思弱点、从旧尝试提炼有效线索，并据此作出有用修改。但传统单轮或多轮后训练主要奖励完成任务，没有直接培养这些能力；作者因此认为，会解题不等于会自我改进，需要专门训练后者。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.21971#S1"
}
]
},
{
"key": "position",
"value": "研究利用反馈、反思失败和比较历史方案这些自进化能力，能否被专门训练出来，并迁移到训练之外的优化问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.21971"
}
]
},
{
"key": "conclusion",
"value": "在代码及域外算法优化上改善表现，支持把自进化所需技能作为专门训练目标。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.21971"
}
]
}
],
"fields": {
"object": "训练阶段学习多轮进化能力的模型参数；使用时继续改进候选程序。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2607.21971"
}
]
},
{
"tag": "person:heng-ji",
"label": "Heng Ji",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2607.21971"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"improver"
]
},
{
"id": "2608.05446",
"title": "EvoHarness-RL: Learning Self-Evolving Runtime Harness for Long-Horizon LLM Agents",
"url": "https://arxiv.org/abs/2608.05446",
"date": "2026-08-05",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"EnvironmentReward",
"M0",
"MemoryContent",
"Weights",
"org:uiuc"
],
"fields": {
"本质定位": "预定义 Belief/Progress/Experience 外部 state 与 read/update/consolidate action；先 SFT 教 harness action，再 cost-aware GRPO 学 runtime harness-use policy。",
"什么在变": "B/P/E runtime state + model weights/harness policy。",
"谁来改 / 谁执行": "**改**：SFT+GRPO training algorithm。
**执行**：Qwen3-8B ALFWorld agent。",
"基础 harness": "BPE harness schema + trainable action policy。",
"Feedback": "environment task reward + cost。",
"Evolution → Eval": "ALFWorld train→eval。",
"Meta-depth": "M0/M1 hybrid。",
"相对之前真正新增什么": "名字叫 harness evolution，但本质不是 agent 自动重写 harness code：**harness state 在线变化，harness-use policy 离线训练**。"
},
"details": [],
"occurrences": [
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 351,
"fields": {
"优先级": "**R**",
"时间": "2026-08-05",
"论文": "[EvoHarness-RL: Learning Self-Evolving Runtime Harness for Long-Horizon LLM Agents](https://arxiv.org/abs/2608.05446)",
"本质定位": "预定义 Belief/Progress/Experience 外部 state 与 read/update/consolidate action；先 SFT 教 harness action，再 cost-aware GRPO 学 runtime harness-use policy。",
"什么在变": "B/P/E runtime state + model weights/harness policy。",
"谁来改 / 谁执行": "**改**：SFT+GRPO training algorithm。
**执行**：Qwen3-8B ALFWorld agent。",
"基础 harness": "BPE harness schema + trainable action policy。",
"Feedback": "environment task reward + cost。",
"Evolution → Eval": "ALFWorld train→eval。",
"Meta-depth": "M0/M1 hybrid。",
"相对之前真正新增什么": "名字叫 harness evolution，但本质不是 agent 自动重写 harness code：**harness state 在线变化，harness-use policy 离线训练**。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M0",
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "预定义 Belief/Progress/Experience 外部 state 与 read/update/consolidate action；先 SFT 教 harness action，再 cost-aware GRPO 学 runtime harness-use policy。",
"novelty": "训练模型决定何时记录、整理和读取外部状态；执行时状态持续变化，训练所得读写策略决定怎样利用它。",
"object": "记录信念、进度与经验的外部状态，以及学习何时读写这些状态的模型策略。",
"executor": "主可训练策略Qwen3-8B；提示时对照另用Claude Opus4.5、GPT-4.1、GPT-5。",
"modifier": "监督微调（用示范数据训练模型）/GRPO训练Qwen3-8B；外部Claude Opus收集监督微调（用示范数据训练模型）轨迹并合并经验库，训练轮次边界才合并。附录C仅写“Claude Opus”，未标明该教师的具体版本，不能从对照方案的Opus4.5推定。",
"roleContext": "**改**：SFT+GRPO training algorithm。
**执行**：Qwen3-8B ALFWorld agent。",
"seed": "ALFWorld 家居操作环境外提供三类状态：当前世界信息、已确定的子目标进度、跨任务可复用经验，论文合称 BPE。track 查询物体，commit 记录子目标，recall 检索经验，note 写下新经验。先收集教师使用这些接口的示范，再训练模型自主选择何时调用。",
"fixed": "",
"verdict": "ALFWorld 环境判断家居任务是否完成；奖励还考虑非法动作与框架使用成本，以训练模型何时值得读写外部状态。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "ALFWorld train→eval。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：教师在500个ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练 games运行，保留87成功轨迹、1153下一动作样本；监督微调（用示范数据训练模型）后接GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。\n\n调试 / 选版本数据：策略用commit/track/recall/note管理信念、进度和经验；每训练轮次末外部summarizer合并、修正或删除技能，batch内技能库稳定。\n\n最终测试数据：主表为ALFWorld140任务已见 split；另分析未见环境迁移，不能把主表96.9%称未见成绩。\n\n数据隔离与证据边界：监督微调（用示范数据训练模型）语料来自训练 games；主结果的已见指训练分布内环境，非模型训练时已执行全部评测题。经验库更新和策略训练是两个机制。",
"cycle": "固定三个外部状态接口：记录当前世界信息、任务进度、跨任务经验。训练模型决定何时查询或写入这些状态，另由外部总结模型整理经验内容。",
"train": "教师在500个ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练 games运行，保留87成功轨迹、1153下一动作样本；监督微调（用示范数据训练模型）后接GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"debug": "策略用commit/track/recall/note管理信念、进度和经验；每训练轮次末外部summarizer合并、修正或删除技能，batch内技能库稳定。",
"test": "主表为ALFWorld140任务已见 split；另分析未见环境迁移，不能把主表96.9%称未见成绩。",
"isolation": "监督微调（用示范数据训练模型）语料来自训练 games；主结果的已见指训练分布内环境，非模型训练时已执行全部评测题。经验库更新和策略训练是两个机制。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "ALFWorld 适配器提供 BPE：世界状态 Belief、已承诺子目标 Progress、跨 episode 技能 Experience；模型可用 track/commit/recall/note 等动作操作状态。相同接口先用教师采轨迹，再 SFT＋GRPO 训练。",
"protocol": "**训练：**教师运行 500 个 ALFWorld training games，保留 87 条成功轨迹，生成 1,153 个下一动作对话样本；由这些经验形成初始技能库，SFT 后接 GRPO。\n\n**测试：**分别报告 seen（140 题）和 unseen 设置，不能把 seen 主表叫未见任务。GRPO 实际训练规模、unseen 数量与测试时技能更新规则本轮待核实；87 是成功轨迹数，不是 500 个训练游戏全部成功。",
"sections": "SFT 数据构造；Table 1",
"source": "https://arxiv.org/abs/2608.05446",
"version": "2608.05446v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "4781bbd8dffb11ee2f81362dc7183f70320884a177bc82d76fdbeb298a754181",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主可训练策略Qwen3-8B；提示时对照另用Claude Opus4.5、GPT-4.1、GPT-5。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "监督微调（用示范数据训练模型）/GRPO训练Qwen3-8B；外部Claude Opus收集监督微调（用示范数据训练模型）轨迹并合并经验库，训练轮次边界才合并。附录C仅写“Claude Opus”，未标明该教师的具体版本，不能从对照方案的Opus4.5推定。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "记录信念、进度与经验的外部状态，以及学习何时读写这些状态的模型策略。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "ALFWorld 环境判断家居任务是否完成；奖励还考虑非法动作与框架使用成本，以训练模型何时值得读写外部状态。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "附录 C：动作解析",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px2"
},
{
"label": "附录 C：示范数据",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录 C：GRPO",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.05446#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.05446#S3.SS4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "ALFWorld 家居操作环境外提供三类状态：当前世界信息、已确定的子目标进度、跨任务可复用经验，论文合称 BPE。track 查询物体，commit 记录子目标，recall 检索经验，note 写下新经验。先收集教师使用这些接口的示范，再训练模型自主选择何时调用。",
"sources": [
{
"label": "SFT 数据构造；Table 1",
"url": "https://arxiv.org/abs/2608.05446"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "固定三个外部状态接口：记录当前世界信息、任务进度、跨任务经验。训练模型决定何时查询或写入这些状态，另由外部总结模型整理经验内容。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "教师在500个ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练 games运行，保留87成功轨迹、1153下一动作样本；监督微调（用示范数据训练模型）后接GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "策略用commit/track/recall/note管理信念、进度和经验；每训练轮次末外部summarizer合并、修正或删除技能，batch内技能库稳定。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主表为ALFWorld140任务已见 split；另分析未见环境迁移，不能把主表96.9%称未见成绩。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.05446#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.05446#S3.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "监督微调（用示范数据训练模型）语料来自训练 games；主结果的已见指训练分布内环境，非模型训练时已执行全部评测题。经验库更新和策略训练是两个机制。",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "训练模型决定何时记录、整理和读取外部状态；执行时状态持续变化，训练所得读写策略决定怎样利用它。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.05446v1",
"version": "2608.05446v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "名字叫 harness evolution，但本质不是 agent 自动重写 harness code：**harness state 在线变化，harness-use policy 离线训练**。",
"feedbackCases": [
{
"label": "ALFWorld：示范与强化学习",
"data": "教师运行 500 个 训练 games，保留 87 条成功轨迹，得到 1,153 个下一动作样本；监督训练后接 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"scoring": "环境按家庭任务目标给成功奖励；包装器解析 action 标签，非法环境动作、错误格式和非法框架动作另受惩罚，框架使用还有成本约束。",
"visible": "环境观察、任务结果与动作失败；track / commit / recall / note 记录状态、进度、经验。",
"use": "奖励训练策略如何解题及何时使用框架；每 epoch 末外部总结模型合并、修正或删除经验，单个采样 batch 内技能库不变。",
"sources": [
{
"label": "附录 C：动作解析",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px2"
},
{
"label": "附录 C：示范数据",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录 C：GRPO",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"judgment": "ALFWorld 环境成功奖励 + 动作／格式规则惩罚"
},
{
"label": "ALFWorld：最终成绩",
"data": "主表是 140 任务 已见 split；另外分析 未见 环境迁移。",
"scoring": "仍由 ALFWorld（通过文字动作完成家居物体操作的交互环境） 环境成功条件检查任务完成。",
"visible": "最终成功率与使用成本。",
"use": "主表 96.9% 不能标为 未见 成绩；已见 / 未见 是不同实验。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.05446#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.05446#S3.SS4"
}
],
"judgment": "ALFWorld 环境程序检查最终目标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "教师在500个ALFWorld（通过文字动作完成家居物体操作的交互环境） 训练 games运行，保留87成功轨迹、1153下一动作样本；监督微调（用示范数据训练模型）后接GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法）。",
"selection": "策略用commit/track/recall/note管理信念、进度和经验；每训练轮次末外部summarizer合并、修正或删除技能，batch内技能库稳定。",
"evaluation": "主表为ALFWorld140任务已见 split；另分析未见环境迁移，不能把主表96.9%称未见成绩。",
"isolation": "监督微调（用示范数据训练模型）语料来自训练 games；主结果的已见指训练分布内环境，非模型训练时已执行全部评测题。经验库更新和策略训练是两个机制。",
"roles": {
"executor": {
"value": "主可训练策略Qwen3-8B；提示时对照另用Claude Opus4.5、GPT-4.1、GPT-5。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px1"
}
]
},
"modifier": {
"value": "监督微调（用示范数据训练模型）/GRPO训练Qwen3-8B；外部Claude Opus收集监督微调（用示范数据训练模型）轨迹并合并经验库，训练轮次边界才合并。附录C仅写“Claude Opus”，未标明该教师的具体版本，不能从对照方案的Opus4.5推定。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px1"
}
]
},
"seed": {
"value": "ALFWorld 家居操作环境外提供三类状态：当前世界信息、已确定的子目标进度、跨任务可复用经验，论文合称 BPE。track 查询物体，commit 记录子目标，recall 检索经验，note 写下新经验。先收集教师使用这些接口的示范，再训练模型自主选择何时调用。",
"sources": [
{
"label": "SFT 数据构造；Table 1",
"url": "https://arxiv.org/abs/2608.05446"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"selection": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
],
"evaluation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2608.05446#S3.SS3"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2608.05446#S3.SS4"
}
],
"isolation": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.05446#S3.SS1.SSS0.Px1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.05446#S3.SS2"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务需要 agent 维护环境认识、跟踪进度并复用经验，但即使外部记忆和工具已经搭好，何时创建、读取、更新这些信息仍常靠提示和固定规则决定。agent 很少被直接训练去权衡这些操作是否值得消耗预算，因此有外部支持也不保证会有效使用。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.05446#S1"
}
]
},
{
"key": "position",
"value": "面向长程交互，研究 agent 能否学会构建有用的外部状态，并自主判断何时读写它，兼顾任务效果与运行成本。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.05446"
}
]
},
{
"key": "conclusion",
"value": "在 ALFWorld（通过文字动作完成家居物体操作的交互环境） 上取得较高成功率，训练后模型更选择性地访问外部状态，减少机械式频繁调用。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05446"
}
]
}
],
"fields": {
"modifier": "训练程序通过监督微调和 GRPO（比较同一任务多次尝试的奖励，用相对好坏更新模型参数的强化学习方法） 更新 Qwen3-8B；Claude Opus 提供训练轨迹并在训练阶段边界合并经验。教师只披露为 Claude Opus，没有具体版本。",
"object": "记录信念、进度与经验的外部状态，以及学习何时读写这些状态的模型策略。",
"verdict": "ALFWorld 环境判断家居任务是否完成；奖励还考虑非法动作与框架使用成本，以训练模型何时值得读写外部状态。",
"seed": "ALFWorld 家居操作环境外提供三类状态：当前世界信息、已确定的子目标进度、跨任务可复用经验，论文合称 BPE。track 查询物体，commit 记录子目标，recall 检索经验，note 写下新经验。先收集教师使用这些接口的示范，再训练模型自主选择何时调用。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.05446"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2608.13951",
"title": "HELIX: Model-Harness Co-evolution for Recursive Self-Improvement",
"url": "https://arxiv.org/abs/2608.13951",
"date": "2026-08-14",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"ExecutableVerifier",
"HarnessCode",
"M1",
"Population",
"Weights"
],
"fields": {
"本质定位": "把 harness evolution 不只当当前性能优化器，也当 **产生下一轮 model training data 的 trajectory generator**：固定 model 下搜索多样 sibling harness，收集成功/失败/near-miss/alternative verified trajectories，再设想更新 model 后重建 harness。",
"什么在变": "typed harness atoms/recipes/runtime policies；设计上随后 model weights也应更新。",
"谁来改 / 谁执行": "**改**：HELIX evolution/search controller + LLM candidate builders；模型更新阶段由后训练 pipeline。
**执行**：fixed model + candidate sibling harnesses（当前 paper只展示一轮）。",
"基础 harness": "Pi-like code-repair harness decomposed into typed ports/atoms/recipes/product shells。",
"Feedback": "SWE-bench executable verifier + repeated-run outcome + trajectory provenance。",
"Evolution → Eval": "code repair 一轮：65 candidates；best fixed harness 与 sibling portfolio评估并导出200-slot training-data slice。",
"Meta-depth": "M1 hybrid substrate；完整多代 M→H→M-prime→H-prime 尚未实证。",
"相对之前真正新增什么": "相对 SIA/Macaron 的新点是把 co-evolution 的接口做成 source-traceable typed substrate，并强调 **portfolio diversity带来的 data coverage**，而非只保留一个 best harness。"
},
"details": [],
"occurrences": [
{
"section": "B. Non-Harness RSI / Hybrid：Harness + Weight / Learned Harness Policy",
"line": 353,
"fields": {
"优先级": "**K**",
"时间": "2026-08-14",
"论文": "[HELIX: Model-Harness Co-evolution for Recursive Self-Improvement](https://arxiv.org/abs/2608.13951)",
"本质定位": "把 harness evolution 不只当当前性能优化器，也当 **产生下一轮 model training data 的 trajectory generator**：固定 model 下搜索多样 sibling harness，收集成功/失败/near-miss/alternative verified trajectories，再设想更新 model 后重建 harness。",
"什么在变": "typed harness atoms/recipes/runtime policies；设计上随后 model weights也应更新。",
"谁来改 / 谁执行": "**改**：HELIX evolution/search controller + LLM candidate builders；模型更新阶段由后训练 pipeline。
**执行**：fixed model + candidate sibling harnesses（当前 paper只展示一轮）。",
"基础 harness": "Pi-like code-repair harness decomposed into typed ports/atoms/recipes/product shells。",
"Feedback": "SWE-bench executable verifier + repeated-run outcome + trajectory provenance。",
"Evolution → Eval": "code repair 一轮：65 candidates；best fixed harness 与 sibling portfolio评估并导出200-slot training-data slice。",
"Meta-depth": "M1 hybrid substrate；完整多代 M→H→M-prime→H-prime 尚未实证。",
"相对之前真正新增什么": "相对 SIA/Macaron 的新点是把 co-evolution 的接口做成 source-traceable typed substrate，并强调 **portfolio diversity带来的 data coverage**，而非只保留一个 best harness。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"M1"
],
"tagBasis": "原记录显式标签 + 依据同篇字段人工整理",
"legacyCategories": [
"Hybrid"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "把 runtime 拆成 typed ports/atoms/recipes，并搜索同一 frozen model 的 sibling harnesses。保留成功、失败与 near-miss 轨迹，作为下一轮模型学习的数据接口。",
"novelty": "要求运行框架既支持当前执行，也产出可追踪、适合后续训练的轨迹；论文主要验证这一框架与数据接口，未完成多代模型训练的闭环实证。",
"object": "可组合的运行框架组件与运行策略；展示的搜索阶段冻结执行模型，轨迹供后续模型学习使用。",
"executor": "所有SWE-bench与LCB任务执行尝试（从开始做任务到得到结果的过程）统一MiniMax-M2.7-highspeed（deterministic=false），配不同同题不同候选 运行框架。",
"modifier": "HELIX控制器按来源组合运行框架组件并组织候选验证；MiniMax-M2.7-highspeed执行任务。本文到达可供模型更新的轨迹数据产出，不等于已训练下一代模型。",
"roleContext": "**改**：HELIX evolution/search controller + LLM candidate builders；模型更新阶段由后训练 pipeline。
**执行**：fixed model + candidate sibling harnesses（当前 paper只展示一轮）。",
"seed": "在 Pi 类编程运行系统上用类型化接口约束会话、工具、控制、接收和策略，比较不同运行框架候选；验证 同题不同候选 轨迹后导出参数更新数据。可检查的更新边界比任意自写代码更窄。",
"fixed": "展示的 harness search round 中 executor model 固定。",
"verdict": "SWE-bench 的代码测试结果、重复执行结果，以及执行记录的来源信息。",
"diagnosis": "比较 sibling trajectories 的成功、退化、near-miss 与替代解法。",
"update": "修改 typed harness recipes；整理可用于 SFT/critic/preference 的 records。",
"acceptance": "先搜索候选，再对选定成员重复评估；portfolio coverage 与单一 fixed harness 分开。",
"experiments": [
{
"name": "一轮 code repair",
"evolve": "65 candidate evolution round",
"selection": "更深的 selected-member validation",
"test": "selected fixed harness / post-hoc portfolio 覆盖与 200-slot sibling data slice",
"isolation": "单轮 / 待闭环",
"note": "产出训练数据不等于已完成下一代模型训练及再进化。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.13951v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：主矩阵用LCB来源前100个合格AtCoder修复题，65种运行框架各跑一次，共6500 slots；SWE轨迹另导出训练记录。\n\n调试 / 选版本数据：只用本地公开test_solution.py的实际执行及通过输出判断LCB成功；选成员做重复运行和官方SWE验证。\n\n最终测试数据：LCB三成员各题10次，共3000 slots；跨基准SWE-bench Verified55题、5成员各2次，共550 slots。\n\n数据隔离与证据边界：LCB只测公开用例，不是官方隐藏测试。论文证据链到达运行框架筛选及训练数据产出，不能声称已验证完整多代模型训练闭环。",
"cycle": "重组 Pi 等开源运行框架的组件，在同一题上保存不同候选的成功、失败和接近成功的执行记录。结合官方验收标签生成可供后续训练、批评和偏好学习使用的数据；本文的数据产出不等于已完成下一代模型训练。",
"train": "主矩阵用LCB来源前100个合格AtCoder修复题，65种运行框架各跑一次，共6500 slots；SWE轨迹另导出训练记录。",
"debug": "只用本地公开test_solution.py的实际执行及通过输出判断LCB成功；选成员做重复运行和官方SWE验证。",
"test": "LCB三成员各题10次，共3000 slots；跨基准SWE-bench Verified55题、5成员各2次，共550 slots。",
"isolation": "LCB只测公开用例，不是官方隐藏测试。论文证据链到达运行框架筛选及训练数据产出，不能声称已验证完整多代模型训练闭环。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "在 Pi 类编程 runtime 上用类型化接口约束会话、工具、控制、接收和策略，比较不同 harness 候选；验证 sibling 轨迹后导出参数更新数据。可检查的更新边界比任意自写代码更窄。",
"protocol": "**数据产出：**SWE 验证切片含 20 个实例、每例 10 个 sibling slots，共 200 slots，导出 438 行 SFT/负例/过滤/偏好记录；按实例拆 16 train、4 dev，对应 339/99 行。一个轨迹可用于多个目标，所以 438 不是独立 rollout 数。\n\n**边界：**这是已验证的训练数据交接证据，不能直接写成在另一独立测试集上完成多代联合改进。全流程最终留出评估与每轮参数训练数据关系本轮待核实。",
"sections": "§6 数据核算、Table 3；§7",
"source": "https://arxiv.org/abs/2608.13951",
"version": "2608.13951v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f6c5af87fdc915f4eecfde5146c981052c21288110406a612e50e58a60637205",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "所有SWE-bench与LCB任务执行尝试（从开始做任务到得到结果的过程）统一MiniMax-M2.7-highspeed（deterministic=false），配不同同题不同候选 运行框架。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "HELIX控制器按来源组合运行框架组件并组织候选验证；MiniMax-M2.7-highspeed执行任务。本文到达可供模型更新的轨迹数据产出，不等于已训练下一代模型。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "可组合的运行框架组件与运行策略；展示的搜索阶段冻结执行模型，轨迹供后续模型学习使用。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "SWE-bench 的代码测试结果、重复执行结果，以及执行记录的来源信息。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在 Pi 类编程运行系统上用类型化接口约束会话、工具、控制、接收和策略，比较不同运行框架候选；验证 同题不同候选 轨迹后导出参数更新数据。可检查的更新边界比任意自写代码更窄。",
"sources": [
{
"label": "§6 数据核算、Table 3；§7",
"url": "https://arxiv.org/abs/2608.13951"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "重组 Pi 等开源运行框架的组件，在同一题上保存不同候选的成功、失败和接近成功的执行记录。结合官方验收标签生成可供后续训练、批评和偏好学习使用的数据；本文的数据产出不等于已完成下一代模型训练。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "主矩阵用LCB来源前100个合格AtCoder修复题，65种运行框架各跑一次，共6500 slots；SWE轨迹另导出训练记录。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "只用本地公开test_solution.py的实际执行及通过输出判断LCB成功；选成员做重复运行和官方SWE验证。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "LCB三成员各题10次，共3000 slots；跨基准SWE-bench Verified55题、5成员各2次，共550 slots。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "LCB只测公开用例，不是官方隐藏测试。论文证据链到达运行框架筛选及训练数据产出，不能声称已验证完整多代模型训练闭环。",
"sources": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "要求运行框架既支持当前执行，也产出可追踪、适合后续训练的轨迹；论文主要验证这一框架与数据接口，未完成多代模型训练的闭环实证。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.13951v1",
"version": "2608.13951v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "强调 harness 的双重作用：即时执行质量和训练数据覆盖。当前证据不能直接支撑完整多代 M→H→M′→H′ 闭环。",
"feedbackCases": [
{
"label": "LCB 修复任务：候选矩阵",
"data": "取 LCB 来源前 100 个合格 AtCoder 修复题；65 种框架各一次，共 6,500 个执行位置。",
"scoring": "只有轨迹中确实运行本地公开 test_solution.py 并出现通过输出才记成功；不能靠结束语或未经执行的测试声明。",
"visible": "实际测试输出、候选配置、代码与轨迹来源信息。",
"use": "筛选框架并形成同题成功 / 失败对照数据；79/100 是整个候选集合事后取并集，最佳单个框架为 52/100，不能混为一个可部署系统的成绩。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"judgment": "实际运行公开 test_solution.py，以通过输出作成功证据"
},
{
"label": "重复运行与 SWE-bench 迁移",
"data": "LCB 选 3 成员每题 10 次，共 3,000 次；SWE-bench Verified 55 题、5 成员各 2 次，共 550 次。",
"scoring": "LCB 沿用上述真实公开测试证据；SWE 由官方评估器运行仓库测试判定修复。",
"visible": "官方任务结果、代码补丁与同题多轨迹。",
"use": "测稳定性、跨基准表现及可用于训练的数据质量；导出轨迹不等于已经完成并测得模型权重提升。",
"sources": [
{
"label": "§5.3：官方 SWE 验收",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§6.2：候选覆盖解释",
"url": "https://arxiv.org/html/2608.13951#S6.SS2"
}
],
"judgment": "LCB 用公开执行证据；SWE-bench 用官方仓库测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
0,
1
],
"evolution": "主矩阵用LCB来源前100个合格AtCoder修复题，65种运行框架各跑一次，共6500 slots；SWE轨迹另导出训练记录。",
"selection": "只用本地公开test_solution.py的实际执行及通过输出判断LCB成功；选成员做重复运行和官方SWE验证。",
"evaluation": "LCB三成员各题10次，共3000 slots；跨基准SWE-bench Verified55题、5成员各2次，共550 slots。",
"isolation": "LCB只测公开用例，不是官方隐藏测试。论文证据链到达运行框架筛选及训练数据产出，不能声称已验证完整多代模型训练闭环。",
"roles": {
"executor": {
"value": "所有SWE-bench与LCB任务执行尝试（从开始做任务到得到结果的过程）统一MiniMax-M2.7-highspeed（deterministic=false），配不同同题不同候选 运行框架。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
}
]
},
"modifier": {
"value": "HELIX控制器按来源组合运行框架组件并组织候选验证；MiniMax-M2.7-highspeed执行任务。本文到达可供模型更新的轨迹数据产出，不等于已训练下一代模型。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
}
]
},
"seed": {
"value": "在 Pi 类编程运行系统上用类型化接口约束会话、工具、控制、接收和策略，比较不同运行框架候选；验证 同题不同候选 轨迹后导出参数更新数据。可检查的更新边界比任意自写代码更窄。",
"sources": [
{
"label": "§6 数据核算、Table 3；§7",
"url": "https://arxiv.org/abs/2608.13951"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"selection": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"evaluation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
],
"isolation": [
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.13951#S5.SS2"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2608.13951#S5.SS3"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2608.13951#S5.SS4"
},
{
"label": "§9.1",
"url": "https://arxiv.org/html/2608.13951#S9.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "运行框架既决定模型当前看到什么、能做什么，也决定哪些交互轨迹会成为下一轮训练数据，单独优化模型会忽略这两层影响。跨不同框架组合组件时，还必须保留“改了什么”和“哪些效果经过验证”的对应关系，否则难把当前执行改进可靠地交给下一轮模型学习。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.13951#S1"
}
]
},
{
"key": "position",
"value": "研究运行框架能否同时服务于当前任务执行和下一轮模型学习，建立两者共同改进所需的可追溯数据基础；本文主要验证框架与轨迹产出。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.13951"
}
]
},
{
"key": "conclusion",
"value": "提出模型与框架共同进化的训练接口；本文轨迹产出不等于已经完成下一代模型训练。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.13951"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.13951#S5.SS1"
}
]
}
],
"fields": {
"object": "可组合的运行框架组件与运行策略；展示的搜索阶段冻结执行模型，轨迹供后续模型学习使用。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2505.19955",
"title": "MLR-Bench: Evaluating AI Agents on Open-Ended Machine Learning Research",
"url": "https://arxiv.org/abs/2505.19955",
"date": "2025-05-26",
"priority": "R",
"categories": [
"evaluation"
],
"tags": [
"org:nus"
],
"fields": {
"本质定位": "不做 self-evolution；把 AI research 拆成 idea→proposal→experiment→paper，并用201个 workshop-derived research tasks + MLR-Judge评估。",
"什么在变": "系统不进化；被测的是research-agent capability。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark流程固定。
**执行**：frontier LLM / coding agent 通过 MLR-Agent 执行 research task。",
"基础 harness": "MLR-Agent 四阶段 research scaffold。",
"Feedback": "LLM-review rubric +实验 artifact/结果；MLR-Judge经人类验证。",
"Evolution → Eval": "201 open-ended ML research tasks；非 evolve→eval benchmark。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "它是后续“AI是否能改进AI”的 capability substrate：先暴露 **实验可信性** 而非只看最终文字质量。"
},
"details": [],
"occurrences": [
{
"section": "C. Benchmark / Evaluation",
"line": 360,
"fields": {
"优先级": "**R**",
"时间": "2025-05-26",
"论文": "[MLR-Bench: Evaluating AI Agents on Open-Ended Machine Learning Research](https://arxiv.org/abs/2505.19955)",
"本质定位": "不做 self-evolution；把 AI research 拆成 idea→proposal→experiment→paper，并用201个 workshop-derived research tasks + MLR-Judge评估。",
"什么在变": "系统不进化；被测的是research-agent capability。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark流程固定。
**执行**：frontier LLM / coding agent 通过 MLR-Agent 执行 research task。",
"基础 harness": "MLR-Agent 四阶段 research scaffold。",
"Feedback": "LLM-review rubric +实验 artifact/结果；MLR-Judge经人类验证。",
"Evolution → Eval": "201 open-ended ML research tasks；非 evolve→eval benchmark。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "它是后续“AI是否能改进AI”的 capability substrate：先暴露 **实验可信性** 而非只看最终文字质量。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2025",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "不做 self-evolution；把 AI research 拆成 idea→proposal→experiment→paper，并用201个 workshop-derived research tasks + MLR-Judge评估。",
"novelty": "评价从研究想法到真实实验和论文的完整过程，核对实验是否实际成立，使可信研究与流畅写作可以分开观察。",
"object": "系统不做持续自进化；评测科研 agent 从想法到实验和论文的能力。",
"executor": "MLR-Agent研究系统比较o4-mini、Gemini2.5-Pro-Preview-05-06、Claude3.7 Sonnet；Codex配置用o4-mini-medium，AI 科研角色 V2用o4-mini-high。",
"modifier": "各研究 task agent 在其模型与编码框架中迭代科研产物；MLR-Bench及MLR-Judge固定，不存在一个统一的跨论文自改模型。",
"roleContext": "**改**：无 persistent modifier；benchmark流程固定。
**执行**：frontier LLM / coding agent 通过 MLR-Agent 执行 research task。",
"seed": "MLR-Agent 四阶段：想法生成、提案、实验、论文写作；MLR-Judge 另按评分细则（逐项规定要满足的要求及给分标准）评审研究质量。基础系统已包含研究流程，作品得分与真实实验是否有效需要分别检查。",
"fixed": "",
"verdict": "Gemini-2.5-Pro-Preview 与 Claude-3.7-Sonnet 按研究阶段的评分要求分别评审，读取研究产物及实验日志后取平均；另与人类专家评价比较一致性。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "201 open-ended ML research tasks；非 evolve→eval benchmark。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：201个来自NeurIPS/ICLR/ICML workshop的开放研究任务；task agent 为各课题自行选数据和实验。\n\n调试 / 选版本数据：MLR-Agent组织研究阶段并迭代产物；MLR-Judge按评分细则（逐项规定要满足的要求及给分标准）评阶段产物及最终论文。\n\n最终测试数据：201题是研究项目集合，按科研产物质量评价；另外用人类专家评分检查语言模型评审一致性。\n\n数据隔离与证据边界：不是统一模型训练数据集；课题内数据划分由各研究实现决定，论文质量分不等价于严格测试集成绩。",
"cycle": "评估从想法到论文的完整开放研究过程，并提供模块化研究 task agent 与经人类对照的评审器。",
"train": "201个来自NeurIPS/ICLR/ICML workshop的开放研究任务；task agent 为各课题自行选数据和实验。",
"debug": "MLR-Agent组织研究阶段并迭代产物；MLR-Judge按评分细则（逐项规定要满足的要求及给分标准）评阶段产物及最终论文。",
"test": "201题是研究项目集合，按科研产物质量评价；另外用人类专家评分检查语言模型评审一致性。",
"isolation": "不是统一模型训练数据集；课题内数据划分由各研究实现决定，论文质量分不等价于严格测试集成绩。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "MLR-Agent 四阶段：想法生成、提案、实验、论文写作；MLR-Judge 另按 rubric 评审研究质量。基础系统已包含研究流程，作品得分与真实实验是否有效需要分别检查。",
"protocol": "**任务来源：**NeurIPS、ICLR、ICML workshop 的 201 个开放式研究任务；输入研究问题，agent 自行选择具体实验数据和方法。\n\n**评估：**既评阶段产物也评最终论文，配有专家对评审一致性的验证。201 是研究任务数，不是一个用于训练模型的数据集；每个课题的训练/测试数据需从其产出实验核验，不能以论文评审分代替数据隔离检查。",
"sections": "benchmark 组成与评估框架",
"source": "https://arxiv.org/abs/2505.19955",
"version": "2505.19955v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "1a2cfd4c7cccd60900d8b3b3afa92491455347fdd3c1bbb3d202c4031ad276a0",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "MLR-Agent研究系统比较o4-mini、Gemini2.5-Pro-Preview-05-06、Claude3.7 Sonnet；Codex配置用o4-mini-medium，AI 科研角色 V2用o4-mini-high。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2505.19955#S3.SS5.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "各研究 task agent 在其模型与编码框架中迭代科研产物；MLR-Bench及MLR-Judge固定，不存在一个统一的跨论文自改模型。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2505.19955#S3.SS5.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "系统不做持续自进化；评测科研 agent 从想法到实验和论文的能力。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "Gemini-2.5-Pro-Preview 与 Claude-3.7-Sonnet 按研究阶段的评分要求分别评审，读取研究产物及实验日志后取平均；另与人类专家评价比较一致性。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§2.1：评审模型与输入",
"url": "https://arxiv.org/html/2505.19955#S2.SS1"
},
{
"label": "§3.3：十个实验课题",
"url": "https://arxiv.org/html/2505.19955#S3.SS3"
},
{
"label": "附录 D.2：评分细则",
"url": "https://arxiv.org/html/2505.19955#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "MLR-Agent 四阶段：想法生成、提案、实验、论文写作；MLR-Judge 另按评分细则（逐项规定要满足的要求及给分标准）评审研究质量。基础系统已包含研究流程，作品得分与真实实验是否有效需要分别检查。",
"sources": [
{
"label": "benchmark 组成与评估框架",
"url": "https://arxiv.org/abs/2505.19955"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "评估从想法到论文的完整开放研究过程，并提供模块化研究 task agent 与经人类对照的评审器。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "201个来自NeurIPS/ICLR/ICML workshop的开放研究任务；task agent 为各课题自行选数据和实验。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "MLR-Agent组织研究阶段并迭代产物；MLR-Judge按评分细则（逐项规定要满足的要求及给分标准）评阶段产物及最终论文。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "201题是研究项目集合，按科研产物质量评价；另外用人类专家评分检查语言模型评审一致性。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "不是统一模型训练数据集；课题内数据划分由各研究实现决定，论文质量分不等价于严格测试集成绩。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "评价从研究想法到真实实验和论文的完整过程，核对实验是否实际成立，使可信研究与流畅写作可以分开观察。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2505.19955v3",
"version": "2505.19955v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它是后续“AI是否能改进AI”的 capability substrate：先暴露 **实验可信性** 而非只看最终文字质量。",
"feedbackCases": [
{
"label": "科研产物评审：MLR-Judge",
"data": "201 个源于 NeurIPS / ICLR / ICML workshop 的研究课题；执行、写作与端到端实验选其中 10 个代表任务。",
"scoring": "Gemini-2.5-Pro-Preview 和 Claude-3.7-Sonnet 分别按阶段评分细则评审，再取平均；实验阶段查看执行日志，写作阶段审论文及图表。",
"visible": "评审阅读研究历史、阶段产物及实验记录；不是统一代码测试给出的对错。",
"use": "评价创意、方案、实验、论文质量；另由人类专家检查评审一致性。每课题自行选择实验数据，201 题不是共享一个模型训练集。",
"sources": [
{
"label": "§2.1：评审模型与输入",
"url": "https://arxiv.org/html/2505.19955#S2.SS1"
},
{
"label": "§3.3：十个实验课题",
"url": "https://arxiv.org/html/2505.19955#S3.SS3"
},
{
"label": "附录 D.2：评分细则",
"url": "https://arxiv.org/html/2505.19955#A4.SS2"
}
],
"judgment": "Gemini-2.5-Pro-Preview 与 Claude-3.7-Sonnet 按阶段评分细则评审，取平均"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0
],
"evolution": "201个来自NeurIPS/ICLR/ICML workshop的开放研究任务；task agent 为各课题自行选数据和实验。",
"selection": "MLR-Agent组织研究阶段并迭代产物；MLR-Judge按评分细则（逐项规定要满足的要求及给分标准）评阶段产物及最终论文。",
"evaluation": "201题是研究项目集合，按科研产物质量评价；另外用人类专家评分检查语言模型评审一致性。",
"isolation": "不是统一模型训练数据集；课题内数据划分由各研究实现决定，论文质量分不等价于严格测试集成绩。",
"roles": {
"executor": {
"value": "MLR-Agent研究系统比较o4-mini、Gemini2.5-Pro-Preview-05-06、Claude3.7 Sonnet；Codex配置用o4-mini-medium，AI 科研角色 V2用o4-mini-high。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2505.19955#S3.SS5.SSS0.Px1"
}
]
},
"modifier": {
"value": "各研究 task agent 在其模型与编码框架中迭代科研产物；MLR-Bench及MLR-Judge固定，不存在一个统一的跨论文自改模型。",
"sources": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2505.19955#S3.SS5.SSS0.Px1"
}
]
},
"seed": {
"value": "MLR-Agent 四阶段：想法生成、提案、实验、论文写作；MLR-Judge 另按评分细则（逐项规定要满足的要求及给分标准）评审研究质量。基础系统已包含研究流程，作品得分与真实实验是否有效需要分别检查。",
"sources": [
{
"label": "benchmark 组成与评估框架",
"url": "https://arxiv.org/abs/2505.19955"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"selection": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"evaluation": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
],
"isolation": [
{
"label": "§6",
"url": "https://arxiv.org/html/2505.19955#S6.SS0.SSS0.Px1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2505.19955#S4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "AI agent 已经能提出想法、写代码和论文，但缺少综合评测来判断它们能否自主产生、检验并验证新知识。这使不同研究 agent 难以公平比较，也难系统定位缺乏新意、方法缺陷或虚构结果等问题；作者因此希望评估开放研究的质量及失败原因。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2505.19955#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否自主完成可信的开放机器学习研究，覆盖想法、方案、真实实验和论文，而不仅是生成看似合理的研究文本。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.19955"
}
]
},
{
"key": "conclusion",
"value": "模型能写连贯想法和文章，但实验结果常无效或被编造，科研可靠性仍是主要瓶颈。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2505.19955"
}
]
}
],
"fields": {
"object": "系统不做持续自进化；评测科研 agent 从想法到实验和论文的能力。",
"verdict": "Gemini-2.5-Pro-Preview 与 Claude-3.7-Sonnet 按研究阶段的评分要求分别评审，读取研究产物及实验日志后取平均；另与人类专家评价比较一致性。"
}
},
"attributions": [
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2505.19955"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2605.08678",
"title": "MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI",
"url": "https://arxiv.org/abs/2605.08678",
"date": "2026-05-09",
"priority": "R",
"categories": [
"evaluation"
],
"tags": [
"org:washington",
"org:princeton",
"org:cmu",
"org:berkeley",
"org:tsinghua",
"org:sjtu"
],
"fields": {
"本质定位": "140 tasks×12 domains，要求 agent 改进某个 ML component，并验证方法能跨 controlled settings generalize/scale；重点区分 engineering tuning 与真正 method invention。",
"什么在变": "被测 agent 不被要求 persistent self-evolve；评价其提出/验证更好 ML method 的能力。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark evaluator固定。
**执行**：被测 frontier AI research/coding agents 自主执行 research。",
"基础 harness": "research/coding agent scaffold + task-provided baselines/experiments。",
"Feedback": "实验结果/validation across settings + benchmark scorer。",
"Evolution → Eval": "140 independent research tasks；controlled generalization/scaling checks。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "相对一般 coding/research benchmark 的新点是 **不只要在一个 setting涨分，还要证明 generalizable/scalable ML improvement**，更接近 AI4AI 能力上限。"
},
"details": [],
"occurrences": [
{
"section": "C. Benchmark / Evaluation",
"line": 361,
"fields": {
"优先级": "**R**",
"时间": "2026-05-09",
"论文": "[MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI](https://arxiv.org/abs/2605.08678)",
"本质定位": "140 tasks×12 domains，要求 agent 改进某个 ML component，并验证方法能跨 controlled settings generalize/scale；重点区分 engineering tuning 与真正 method invention。",
"什么在变": "被测 agent 不被要求 persistent self-evolve；评价其提出/验证更好 ML method 的能力。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark evaluator固定。
**执行**：被测 frontier AI research/coding agents 自主执行 research。",
"基础 harness": "research/coding agent scaffold + task-provided baselines/experiments。",
"Feedback": "实验结果/validation across settings + benchmark scorer。",
"Evolution → Eval": "140 independent research tasks；controlled generalization/scaling checks。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "相对一般 coding/research benchmark 的新点是 **不只要在一个 setting涨分，还要证明 generalizable/scalable ML improvement**，更接近 AI4AI 能力上限。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "140 tasks×12 domains，要求 agent 改进某个 ML component，并验证方法能跨 controlled settings generalize/scale；重点区分 engineering tuning 与真正 method invention。",
"novelty": "要求提出的机器学习改进跨设置仍有效、扩大规模后仍成立，使方法创新与只在一个设置中调出高分可以区分。",
"object": "被改进的是目标机器学习方法；不要求研究 agent 本身持续自改。",
"executor": "完整集：Claude Opus4.6、GPT-5.4、Gemini3.1 Pro、DeepSeekV3.2、Qwen3.6 Plus。Lite另有Opus4.7、Sonnet4.6、GPT-5.5 Pro/5.5、Gemini3.1 Flash Lite、DeepSeekV4Pro/Flash、Qwen3.6Max、KimiK2.6、GLM5.1。",
"modifier": "一般实验由上述任务模型改算法；测试时进化用Gemini3.1 Pro，测试时参数训练用Qwen3.5-35B-A3B。两条路线不能当同模型对照。",
"roleContext": "**改**：无 persistent modifier；benchmark evaluator固定。
**执行**：被测 frontier AI research/coding agents 自主执行 research。",
"seed": "起点提供完整对照方案、代码、数据协议和预算，后端在隔离软件环境中运行。agent 用四种工具实验：edit 修改指定组件，test 运行训练并查看允许公开的指标，submit 提交此前结果，undo 撤回改动。训练和评估规则由平台固定。",
"fixed": "",
"verdict": "在固定训练和评估流程下运行修改后的算法，比较多个数据、环境或模型规模设置的成绩；隐藏设置用于检查改进是否能推广。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "140 independent research tasks；controlled generalization/scaling checks。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：140个研究任务覆盖12个ML领域，各任务提供代码、对照方案及数据设置；只能编辑指定算法组件。\n\n调试 / 选版本数据：统一受保护训练协议、容量和评估器；低延迟进化/测试时训练实验开放三个设置中的两个。\n\n最终测试数据：在任务规格规定的设置评价算法；上述进化/训练对照另用第三个隐藏设置测迁移。\n\n数据隔离与证据边界：140是方法研究任务数，各任务内部数据不同；测试集工具可见反馈与隐藏设置成绩需分开，不能把所有结果都称盲测。",
"cycle": "锁住评估器及共用训练超参，把得分改进尽量归因于目标算法；支持采样、探索、种群进化和测试时训练对照。",
"train": "140个研究任务覆盖12个ML领域，各任务提供代码、对照方案及数据设置；只能编辑指定算法组件。",
"debug": "统一受保护训练协议、容量和评估器；低延迟进化/测试时训练实验开放三个设置中的两个。",
"test": "在任务规格规定的设置评价算法；上述进化/训练对照另用第三个隐藏设置测迁移。",
"isolation": "140是方法研究任务数，各任务内部数据不同；测试集工具可见反馈与隐藏设置成绩需分开，不能把所有结果都称盲测。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "统一后端可用 Apptainer/Docker/Conda/Harbor；起点给完整 baseline、代码和预算。agent 只经 edit、test、submit、undo 四种工具实验：test 返回训练及可见测试指标，submit 选择先前结果。",
"protocol": "**数据/任务：**140 个研究任务、12 个 ML 领域，任务各有强 baseline 和数据设置。测试时演化、测试时训练的特定实验只向模型开放三个设置中的两个。\n\n**边界：**可见测试用于改方法，隐藏设置测迁移；不能把工具名 test 返回的分数说成最终盲测。逐任务数据集名称和规模应按任务规格列，本轮未完整核实，不把 140 当训练样本数。",
"sections": "§3 任务设计、运行接口、test-time scaling",
"source": "https://arxiv.org/abs/2605.08678",
"version": "2605.08678v3",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "052470330e58374d8598020d1486ec8fc965b849541c28800709a9b50a8b42a1",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "完整集：Claude Opus4.6、GPT-5.4、Gemini3.1 Pro、DeepSeekV3.2、Qwen3.6 Plus。Lite另有Opus4.7、Sonnet4.6、GPT-5.5 Pro/5.5、Gemini3.1 Flash Lite、DeepSeekV4Pro/Flash、Qwen3.6Max、KimiK2.6、GLM5.1。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.08678#S4.SS1.SSS0.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "一般实验由上述任务模型改算法；测试时进化用Gemini3.1 Pro，测试时参数训练用Qwen3.5-35B-A3B。两条路线不能当同模型对照。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.08678#S4.SS1.SSS0.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "被改进的是目标机器学习方法；不要求研究 agent 本身持续自改。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "在固定训练和评估流程下运行修改后的算法，比较多个数据、环境或模型规模设置的成绩；隐藏设置用于检查改进是否能推广。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "起点提供完整对照方案、代码、数据协议和预算，后端在隔离软件环境中运行。agent 用四种工具实验：edit 修改指定组件，test 运行训练并查看允许公开的指标，submit 提交此前结果，undo 撤回改动。训练和评估规则由平台固定。",
"sources": [
{
"label": "§3 任务设计、运行接口、test-time scaling",
"url": "https://arxiv.org/abs/2605.08678"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "锁住评估器及共用训练超参，把得分改进尽量归因于目标算法；支持采样、探索、种群进化和测试时训练对照。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "140个研究任务覆盖12个ML领域，各任务提供代码、对照方案及数据设置；只能编辑指定算法组件。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "统一受保护训练协议、容量和评估器；低延迟进化/测试时训练实验开放三个设置中的两个。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "在任务规格规定的设置评价算法；上述进化/训练对照另用第三个隐藏设置测迁移。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "140是方法研究任务数，各任务内部数据不同；测试集工具可见反馈与隐藏设置成绩需分开，不能把所有结果都称盲测。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "要求提出的机器学习改进跨设置仍有效、扩大规模后仍成立，使方法创新与只在一个设置中调出高分可以区分。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2605.08678v3",
"version": "2605.08678v3",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "相对一般 coding/research benchmark 的新点是 **不只要在一个 setting涨分，还要证明 generalizable/scalable ML improvement**，更接近 AI4AI 能力上限。",
"feedbackCases": [
{
"label": "140 个机器学习研究任务",
"data": "12 个领域；每题提供算法代码、至少三个可复现人类基线、至少三个评估设置及数据协议。",
"scoring": "在受保护的训练流程、参数容量、固定随机种子和评估器下执行修改算法，将各设置的指标归一汇总。",
"visible": "测试集 等接口返回实际训练 / 评价结果，agent 只能编辑规定的算法组件。",
"use": "据实验分数修改算法；不同题的指标由题目规定，不能把全部任务概括成标准答案精确匹配。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"judgment": "受保护训练与评测程序计算各研究任务指标"
},
{
"label": "低延迟进化 / 测试时训练对照",
"data": "上述任务设置中开放两种供调试，第三种隐藏。",
"scoring": "相同受保护评估流程运行第三种设置，检查算法能否迁移到另一数据、环境或模型规模。",
"visible": "可见两种设置的结果参与修改；隐藏设置只供最后评价。",
"use": "防止只在一套可见设置上刷高分，区分算法迁移与局部调参。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"judgment": "同一受保护程序在另一数据／规模设置重新评价"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
0,
1
],
"evolution": "140个研究任务覆盖12个ML领域，各任务提供代码、对照方案及数据设置；只能编辑指定算法组件。",
"selection": "统一受保护训练协议、容量和评估器；低延迟进化/测试时训练实验开放三个设置中的两个。",
"evaluation": "在任务规格规定的设置评价算法；上述进化/训练对照另用第三个隐藏设置测迁移。",
"isolation": "140是方法研究任务数，各任务内部数据不同；测试集工具可见反馈与隐藏设置成绩需分开，不能把所有结果都称盲测。",
"roles": {
"executor": {
"value": "完整集：Claude Opus4.6、GPT-5.4、Gemini3.1 Pro、DeepSeekV3.2、Qwen3.6 Plus。Lite另有Opus4.7、Sonnet4.6、GPT-5.5 Pro/5.5、Gemini3.1 Flash Lite、DeepSeekV4Pro/Flash、Qwen3.6Max、KimiK2.6、GLM5.1。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.08678#S4.SS1.SSS0.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
}
]
},
"modifier": {
"value": "一般实验由上述任务模型改算法；测试时进化用Gemini3.1 Pro，测试时参数训练用Qwen3.5-35B-A3B。两条路线不能当同模型对照。",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.08678#S4.SS1.SSS0.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
}
]
},
"seed": {
"value": "起点提供完整对照方案、代码、数据协议和预算，后端在隔离软件环境中运行。agent 用四种工具实验：edit 修改指定组件，test 运行训练并查看允许公开的指标，submit 提交此前结果，undo 撤回改动。训练和评估规则由平台固定。",
"sources": [
{
"label": "§3 任务设计、运行接口、test-time scaling",
"url": "https://arxiv.org/abs/2605.08678"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"selection": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"evaluation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
],
"isolation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2605.08678#S5.SS1.SSS0.Px1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2605.08678#S5.SS2.SSS0.Px1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有机器学习工程评测常奖励在固定数据或任务上调参、调试和选择方法，端到端研究评测又难分清贡献来源。这样的高分不能说明 agent 发明了能迁移和扩展的新方法，因此需要单独检验方法创新，而不是把工程优化混作科学发现。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.08678#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 能否发明跨设置可泛化、扩大规模后仍有效的机器学习方法，区分方法创新与应用已有技术或局部调参。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.08678"
}
]
},
{
"key": "conclusion",
"value": "当前 agent 更擅长工程调参；仅增加搜索、算力或上下文无法消除科学验证能力的不足。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.08678"
}
]
}
],
"fields": {
"object": "被改进的是目标机器学习方法；不要求研究 agent 本身持续自改。",
"verdict": "在固定训练和评估流程下运行修改后的算法，比较多个数据、环境或模型规模设置的成绩；隐藏设置用于检查改进是否能推广。",
"seed": "起点提供完整对照方案、代码、数据协议和预算，后端在隔离软件环境中运行。agent 用四种工具实验：edit 修改指定组件，test 运行训练并查看允许公开的指标，submit 提交此前结果，undo 撤回改动。训练和评估规则由平台固定。"
}
},
"attributions": [
{
"tag": "org:washington",
"label": "University of Washington",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
},
{
"tag": "org:princeton",
"label": "Princeton University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
},
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
},
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
},
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
},
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.08678"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.17271",
"title": "ASI-Bench: At the Dawn of Artificial Superintelligence",
"url": "https://arxiv.org/abs/2608.17271",
"date": "2026-08-18",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"org:mit",
"org:cmu",
"org:uiuc",
"org:tsinghua"
],
"fields": {
"本质定位": "60个project-level research tasks×11科学domain，逐级撤掉 human methodological guidance，测创新探索+自主科学执行。",
"什么在变": "系统不进化；改变的是 human guidance level。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark仅改变 methodological guidance。
**执行**：18个 agent-model configurations 执行 project-level research。",
"基础 harness": "各自 research agent harness。",
"Feedback": "expert-reviewed tasks + sandbox execution + scorer validation。",
"Evolution → Eval": "full guidance / method-specified / autonomous method-selection 三档。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "它不是 self-evolution benchmark；独特点是 **在同一research project逐步减少“人类告诉你怎么做”**，直接测 agent能否自己选择方法并完成可验证科研。"
},
"details": [],
"occurrences": [
{
"section": "C. Benchmark / Evaluation",
"line": 367,
"fields": {
"优先级": "**K**",
"时间": "2026-08-18",
"论文": "[ASI-Bench: At the Dawn of Artificial Superintelligence](https://arxiv.org/abs/2608.17271)",
"本质定位": "60个project-level research tasks×11科学domain，逐级撤掉 human methodological guidance，测创新探索+自主科学执行。",
"什么在变": "系统不进化；改变的是 human guidance level。",
"谁来改 / 谁执行": "**改**：无 persistent modifier；benchmark仅改变 methodological guidance。
**执行**：18个 agent-model configurations 执行 project-level research。",
"基础 harness": "各自 research agent harness。",
"Feedback": "expert-reviewed tasks + sandbox execution + scorer validation。",
"Evolution → Eval": "full guidance / method-specified / autonomous method-selection 三档。",
"Meta-depth": "Evaluation only。",
"相对之前真正新增什么": "它不是 self-evolution benchmark；独特点是 **在同一research project逐步减少“人类告诉你怎么做”**，直接测 agent能否自己选择方法并完成可验证科研。"
}
}
],
"source": "original",
"review": "原记录 · 横向定位",
"protocol": "未明确",
"protocolBasis": "尚未逐项标注；请阅读 Evolution → Eval 原文。",
"year": "2026",
"depth": [
"未明确"
],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "60个project-level research tasks×11科学domain，逐级撤掉 human methodological guidance，测创新探索+自主科学执行。",
"novelty": "对同一科研项目逐步减少人类给定的方法和步骤，测 agent 在少指导条件下能否自主完成可验证研究。",
"object": "系统不进化；逐步减少提供给科研 agent 的方法与步骤指导。",
"executor": "表 2 比较 18 种模型／工具组合：包括 Codex + GPT-5.6 Sol（xhigh／ultra），Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 5／Opus 4.8／GLM-5.3／GLM-5.2／Kimi K3／K2.7／MiniMax M3／DeepSeek V4 Flash／Pro／MiMo V2.5 Pro，以及 Kimi Code、MiMo Code、OpenHands 下的对应配置。agent 执行完整科研项目。",
"modifier": "没有跨任务持续修改自身的 agent；评测程序改变提供给 agent 的方法与步骤说明，以测试科研自主性。",
"roleContext": "**改**：无 persistent modifier；benchmark仅改变 methodological guidance。
**执行**：18个 agent-model configurations 执行 project-level research。",
"seed": "给研究 task agent 项目级问题、数据、约束与交付要求，允许自行构建分析和实验流程。不是所有任务共用可编辑的统一初始任务求解模型；例如动力学任务只给观察数据，不直接给控制方程。",
"fixed": "",
"verdict": "科研项目各有参考生成和评分逻辑，通过沙箱执行与评分重放检查产物；专家审查主要用于保证基准任务质量，不是 agent 执行时的实时指导。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "full guidance / method-specified / autonomous method-selection 三档。"
}
],
"takeaway": "",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：60个项目级科学研究任务，输入各领域观测数据；不是统一训练集。\n\n调试 / 选版本数据：task agent 分析观测、建模、模拟及生成科研产物；B1–B4改变给多少方法/操作指导。\n\n最终测试数据：11个科学领域的项目交付；例如二维动力学以观测快照和初态预测未来场，并提交谱、诊断和可执行代码。\n\n数据隔离与证据边界：同任务不同提示等级保持科学目标、数据和评分相同，主要测自主方法实现能力，不能解释为跨题训练后泛化。",
"cycle": "用完整科研项目和逐步撤去方法指导的四级提示，区分知识、选方法与把方法落实成实验的能力。",
"train": "60个项目级科学研究任务，输入各领域观测数据；不是统一训练集。",
"debug": "task agent 分析观测、建模、模拟及生成科研产物；B1–B4改变给多少方法/操作指导。",
"test": "11个科学领域的项目交付；例如二维动力学以观测快照和初态预测未来场，并提交谱、诊断和可执行代码。",
"isolation": "同任务不同提示等级保持科学目标、数据和评分相同，主要测自主方法实现能力，不能解释为跨题训练后泛化。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "给研究 agent 项目级问题、数据、约束与交付要求，允许自行构建分析和实验流程。不是所有任务共用可编辑的统一初始 solver；例如动力学任务只给观察数据，不直接给控制方程。",
"protocol": "**自建任务：**60 个项目级研究题、11 个科学域，21 位贡献者及五轮人工审查。任务自带领域数据；例如二维系统给初始条件、时空快照和目标时间，要求预测未来状态。\n\n**评估：**衡量研究项目的交付和科学结果，不是用 60 题训练后再测同一模型。各项目的可见观测与隐藏目标划分、使用的具体模型训练数据本轮待核实。",
"sections": "任务构造与示例规格",
"source": "https://arxiv.org/abs/2608.17271",
"version": "2608.17271v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "f28bc67ba6390da3636712990cbbc6546402c3018aaa066b93a78bbab4c4226f",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "表 2 比较 18 种模型／工具组合：包括 Codex + GPT-5.6 Sol（xhigh／ultra），Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 5／Opus 4.8／GLM-5.3／GLM-5.2／Kimi K3／K2.7／MiniMax M3／DeepSeek V4 Flash／Pro／MiMo V2.5 Pro，以及 Kimi Code、MiMo Code、OpenHands 下的对应配置。agent 执行完整科研项目。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "没有跨任务持续修改自身的 agent；评测程序改变提供给 agent 的方法与步骤说明，以测试科研自主性。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "系统不进化；逐步减少提供给科研 agent 的方法与步骤指导。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "科研项目各有参考生成和评分逻辑，通过沙箱执行与评分重放检查产物；专家审查主要用于保证基准任务质量，不是 agent 执行时的实时指导。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录 C：任务提交与修订",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "给研究 task agent 项目级问题、数据、约束与交付要求，允许自行构建分析和实验流程。不是所有任务共用可编辑的统一初始任务求解模型；例如动力学任务只给观察数据，不直接给控制方程。",
"sources": [
{
"label": "任务构造与示例规格",
"url": "https://arxiv.org/abs/2608.17271"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "用完整科研项目和逐步撤去方法指导的四级提示，区分知识、选方法与把方法落实成实验的能力。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "60个项目级科学研究任务，输入各领域观测数据；不是统一训练集。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "task agent 分析观测、建模、模拟及生成科研产物；B1–B4改变给多少方法/操作指导。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "11个科学领域的项目交付；例如二维动力学以观测快照和初态预测未来场，并提交谱、诊断和可执行代码。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "同任务不同提示等级保持科学目标、数据和评分相同，主要测自主方法实现能力，不能解释为跨题训练后泛化。",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "对同一科研项目逐步减少人类给定的方法和步骤，测 agent 在少指导条件下能否自主完成可验证研究。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.17271v1",
"version": "2608.17271v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "它不是 self-evolution benchmark；独特点是 **在同一research project逐步减少“人类告诉你怎么做”**，直接测 agent能否自己选择方法并完成可验证科研。",
"feedbackCases": [
{
"label": "科学研究项目：任务级验收",
"data": "11 个科学领域、60 个项目级任务；各题提供观测数据，B1–B4 控制给多少方法和操作指导。",
"scoring": "每题有参考生成、评分逻辑和沙箱重放检查；例如二维动力学要求由快照与初态预测未来场，并提交谱、诊断及可执行代码。",
"visible": "开发阶段可用公开观测与代码执行结果；论文没有为所有 60 项规定同一种实时标量反馈。",
"use": "完成项目后按该题科研目标验收；B1–B4 是输入指导强弱，不是反馈轮数。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
}
],
"judgment": "各题可执行评分逻辑对照参考结果；任务间指标不同"
},
{
"label": "基准构建质量检查",
"data": "任务提交者在 B1–B4 下运行任务并提交参考、执行时间、环境及评分器重放证据。",
"scoring": "专家检查科学问题、信息泄漏、参考生成和评分稳定性，退回有问题的任务修订。",
"visible": "反馈给任务提交者，要求修任务定义或评分器。",
"use": "这是基准建设过程的专家审查，不能写成被测 agent 每次解题都有专家指导。",
"sources": [
{
"label": "附录 C：任务提交与修订",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px4"
}
],
"judgment": "专家人工审查科学问题与评分协议，不作为 agent 每步奖励"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "60个项目级科学研究任务，输入各领域观测数据；不是统一训练集。",
"selection": "task agent 分析观测、建模、模拟及生成科研产物；B1–B4改变给多少方法/操作指导。",
"evaluation": "11个科学领域的项目交付；例如二维动力学以观测快照和初态预测未来场，并提交谱、诊断和可执行代码。",
"isolation": "同任务不同提示等级保持科学目标、数据和评分相同，主要测自主方法实现能力，不能解释为跨题训练后泛化。",
"roles": {
"executor": {
"value": "表 2 比较 18 种模型／工具组合：包括 Codex + GPT-5.6 Sol（xhigh／ultra），Claude Code（可读写项目文件、运行命令和测试的 coding agent 工具） + Opus 5／Opus 4.8／GLM-5.3／GLM-5.2／Kimi K3／K2.7／MiniMax M3／DeepSeek V4 Flash／Pro／MiMo V2.5 Pro，以及 Kimi Code、MiMo Code、OpenHands 下的对应配置。agent 执行完整科研项目。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px3"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
"modifier": {
"value": "没有跨任务持续修改自身的 agent；评测程序改变提供给 agent 的方法与步骤说明，以测试科研自主性。",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.17271#S2.SS1"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.17271#S3.SS1.SSS0.Px3"
}
]
},
"seed": {
"value": "给研究 task agent 项目级问题、数据、约束与交付要求，允许自行构建分析和实验流程。不是所有任务共用可编辑的统一初始任务求解模型；例如动力学任务只给观察数据，不直接给控制方程。",
"sources": [
{
"label": "任务构造与示例规格",
"url": "https://arxiv.org/abs/2608.17271"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"selection": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"evaluation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
],
"isolation": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.17271#A3.SS0.SSS0.Px6"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.17271#A4"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有评测多使用答案已知的问题，或由人预先规定研究方法和步骤。成功完成这些任务，仍不能说明 AI 能在问题和求解路径都开放时自主探索并产出可验证的新结果；作者因此希望区分按指导执行科学任务与自行开展科学发现的能力。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.17271#S1"
}
]
},
{
"key": "position",
"value": "评测逐步减少人类方法与步骤指导时，agent 能否自主选择研究方法、完成科研项目并产出可验证的新结果。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.17271"
}
]
},
{
"key": "conclusion",
"value": "撤去详细步骤后成绩大幅下降，表明现有 agent 仍高度依赖人的方法指导。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.17271"
}
]
}
],
"fields": {
"executor": "比较 18 种模型与 coding agent 组合，执行完整科研项目；包括 Codex + GPT-5.6、Claude Code + Opus，以及 GLM、Kimi、MiniMax、DeepSeek、MiMo 等配置。逐项型号见完整表。",
"object": "系统不进化；逐步减少提供给科研 agent 的方法与步骤指导。",
"verdict": "科研项目各有参考生成和评分逻辑，通过沙箱执行与评分重放检查产物；专家审查主要用于保证基准任务质量，不是 agent 执行时的实时指导。"
}
},
"attributions": [
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.17271"
}
]
},
{
"tag": "org:cmu",
"label": "Carnegie Mellon University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.17271"
}
]
},
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.17271"
}
]
},
{
"tag": "org:tsinghua",
"label": "Tsinghua University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.17271"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2507.19457",
"title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
"date": "2025-07-25",
"url": "https://arxiv.org/abs/2507.19457",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"OfflineSearch",
"Population",
"Prompt",
"SeparateEvolver",
"org:stanford",
"org:mit",
"org:berkeley",
"person:omar-khattab"
],
"depth": [
"M1"
],
"fields": {
"本质定位": "补足 prompt evolution 的重要基线：轨迹反思、候选测试与 Pareto 前沿经验组合。",
"什么在变": "AI 系统中的一个或多个 prompt。",
"谁来改 / 谁执行": "反思型 prompt optimizer → 使用候选 prompt 的目标系统。",
"基础 harness": "含 LLM prompt 的 AI system；具体任务配置待全文核对。",
"Feedback": "reasoning、tool calls 和 tool outputs 用于诊断与更新。",
"Evolution → Eval": "摘要覆盖六项任务；具体 train/validation/test 数量待补。",
"证据边界": "2026-02 v2 已被 ICLR 2026 Oral 接收；此处日期记录首次提交。"
},
"protocol": "未明确",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 摘要核对",
"sources": [
"https://arxiv.org/abs/2507.19457"
],
"links": [
{
"label": "Code",
"url": "https://github.com/gepa-ai/gepa"
}
],
"source": "addition",
"year": "2025",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Prompt"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "补足 prompt evolution 的重要基线：轨迹反思、候选测试与 Pareto 前沿经验组合。",
"novelty": "从执行记录中反思错误来改提示，同时保留在不同任务上各有所长的候选并尝试组合，充分利用分数之外的文字证据。",
"object": "模型系统中的一个或多个提示；基础模型参数保持不变。",
"executor": "主复合AI系统使用Qwen3-8B或GPT-4.1-mini-2025-04-14；它们执行候选提示，GEPA不更新这些执行模型的权重。",
"modifier": "Qwen3-8B 或 GPT-4.1-mini 在对应配置中读取执行轨迹和文字反馈，提出提示修改；固定 GEPA 算法选择候选并评测。跨模型实验将完全用 Qwen3-8B 优化的提示交给 GPT-4.1-mini 执行。",
"roleContext": "反思型 prompt optimizer → 使用候选 prompt 的目标系统。",
"seed": "固定复合 AI 程序及模块接口，在执行轨迹反馈下修改模块提示，并保留不同题上互补的候选。DSPy 与 Trace 对照保持相同架构、初始提示及数据；不是由 GEPA 自由改写任意工具。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "执行轨迹提供推理、工具调用和返回信息；任务反馈进一步指出缺失文档、未满足约束或结果错误，供模型提出有针对性的提示修改。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "未明确",
"note": "摘要覆盖六项任务；具体 train/validation/test 数量待补。"
}
],
"takeaway": "2026-02 v2 已被 ICLR 2026 Oral 接收；此处日期记录首次提交。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2507.19457"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：HotpotQA150；IF-RLVR150；AIME2022–24共90题的一半；PUPA111；LiveBench-Math368题约三等分；HoVer150。\n\n调试 / 选版本数据：训练材料内部区分产生反思的反馈与Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）比较；外部验证集可查分、限制直接读题。Hotpot/IF各300、AIME45、PUPA111验证。HoVer300验证。\n\n最终测试数据：HotpotQA300；IFBench294新约束题；AIME2025共30题各重复5次；PUPA221；LiveBench剩余测试部分；HoVer300。\n\n数据隔离与证据边界：主实验三段分开；IFBench测试约束训练不可见。另有NPUEval/KernelBench逐题程序优化，不能套用主提示实验协议。",
"cycle": "利用执行轨迹和评估器诊断文本定向改提示，按逐题Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）优势保留多样候选，必要时合并互补模块。",
"train": "HotpotQA150；IF-RLVR150；AIME2022–24共90题的一半；PUPA111；LiveBench-Math368题约三等分；HoVer150。",
"debug": "训练材料内部区分产生反思的反馈与Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）比较；外部验证集可查分、限制直接读题。Hotpot/IF各300、AIME45、PUPA111验证。HoVer300验证。",
"test": "HotpotQA300；IFBench294新约束题；AIME2025共30题各重复5次；PUPA221；LiveBench剩余测试部分；HoVer300。",
"isolation": "主实验三段分开；IFBench测试约束训练不可见。另有NPUEval/KernelBench逐题程序优化，不能套用主提示实验协议。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "固定复合 AI 程序及模块接口，在执行轨迹反馈下修改模块提示，并保留不同题上互补的候选。DSPy 与 Trace 对照保持相同架构、初始提示及数据；不是由 GEPA 自由改写任意工具。",
"protocol": "**提示进化：**各 benchmark 的 train 用作反馈集 D_feedback，validation 用作 Pareto 选择集；最终 test 分开。六项主任务是 AIME-2025、LiveBench-Math、HotpotQA、IFBench、HoVer、PUPA，分别涉及数学、多跳问答、指令遵循、事实核验和隐私约束；各套划分数量本轮尚待核实。\n\n**另外的代码搜索：**NPUEval 和 KernelBench 用于测试时程序优化，不套用提示实验的三段划分。GRPO 对照的参数训练次数与 GEPA 的提示搜索预算也分别核算。",
"sections": "实验比较、优化数据协议；§5.1",
"source": "https://arxiv.org/abs/2507.19457",
"version": "2507.19457v2",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "aeaf6587ea189373ad1ca7db64821d209068cf90efdaedff076bfd428cea7485",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主复合AI系统使用Qwen3-8B或GPT-4.1-mini-2025-04-14；它们执行候选提示，GEPA不更新这些执行模型的权重。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2507.19457#A5.SS2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Qwen3-8B 或 GPT-4.1-mini 在对应配置中读取执行轨迹和文字反馈，提出提示修改；固定 GEPA 算法选择候选并评测。跨模型实验将完全用 Qwen3-8B 优化的提示交给 GPT-4.1-mini 执行。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2507.19457#A5.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "模型系统中的一个或多个提示；基础模型参数保持不变。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "执行轨迹提供推理、工具调用和返回信息；任务反馈进一步指出缺失文档、未满足约束或结果错误，供模型提出有针对性的提示修改。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
},
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "§5.1：反馈驱动手册检索",
"url": "https://arxiv.org/html/2507.19457#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "固定复合 AI 程序及模块接口，在执行轨迹反馈下修改模块提示，并保留不同题上互补的候选。DSPy 与 Trace 对照保持相同架构、初始提示及数据；不是由 GEPA 自由改写任意工具。",
"sources": [
{
"label": "实验比较、优化数据协议；§5.1",
"url": "https://arxiv.org/abs/2507.19457"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "利用执行轨迹和评估器诊断文本定向改提示，按逐题Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）优势保留多样候选，必要时合并互补模块。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "HotpotQA150；IF-RLVR150；AIME2022–24共90题的一半；PUPA111；LiveBench-Math368题约三等分；HoVer150。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练材料内部区分产生反思的反馈与Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）比较；外部验证集可查分、限制直接读题。Hotpot/IF各300、AIME45、PUPA111验证。HoVer300验证。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "HotpotQA300；IFBench294新约束题；AIME2025共30题各重复5次；PUPA221；LiveBench剩余测试部分；HoVer300。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "主实验三段分开；IFBench测试约束训练不可见。另有NPUEval/KernelBench逐题程序优化，不能套用主提示实验协议。",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "从执行记录中反思错误来改提示，同时保留在不同任务上各有所长的候选并尝试组合，充分利用分数之外的文字证据。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2507.19457v2",
"version": "2507.19457v2",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把自然语言反思与 Pareto 候选组合用于 prompt 搜索；关键是从轨迹里抽出可复用规则，而非仅根据标量分数选一个 prompt。",
"feedbackCases": [
{
"label": "HotpotQA",
"data": "150 训练、300 验证、300 测试",
"scoring": "结合问答标准答案及支持文档评价；文字反馈指出各阶段尚未检索到的相关文档。",
"visible": "检索查询、返回文档、答案、分数，以及还缺哪些支持文档。 优化器能读训练题和标签；验证可查分但限制直接读取题目内容。",
"use": "反思生成提示修改，小批次改善后再评更大集合；最后测试与提示优化隔离。",
"sources": [
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
}
],
"judgment": "参考答案与支持文档反馈；按该实验的阶段指标判断检索和作答"
},
{
"label": "IFBench",
"data": "IF-RLVR 150 训练、300 验证；IFBench 294 测试题含未见约束",
"scoring": "程序检查回答是否满足指定输出约束，例如只能回答 yes/no、某词至少出现三次。",
"visible": "已经满足和未满足的约束描述。 优化器能读训练题和标签；验证可查分但限制直接读取题目内容。",
"use": "反思生成提示修改，小批次改善后再评更大集合；最后测试与提示优化隔离。",
"sources": [
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
}
],
"judgment": "IFBench 确定性程序检查文本约束"
},
{
"label": "AIME / LiveBench-Math",
"data": "AIME 2022–24 共 90 题平分训练 / 验证，2025 的 30 题各测五次；LiveBench 368 题三等分",
"scoring": "按数学题参考答案给结果分数。",
"visible": "当前推理、答案与任务反馈；不是对每一步推导都有人工标注。 优化器能读训练题和标签；验证可查分但限制直接读取题目内容。",
"use": "反思生成提示修改，小批次改善后再评更大集合；最后测试与提示优化隔离。",
"sources": [
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
}
],
"judgment": "数学标准答案判分；该段未展开全部比对器"
},
{
"label": "HoVer",
"data": "150 训练、300 验证、300 测试",
"scoring": "对照 gold documents 检查多跳检索是否找到支撑事实的文档。",
"visible": "已检索到的正确文档与还缺的正确文档集合。 优化器能读训练题和标签；验证可查分但限制直接读取题目内容。",
"use": "反思生成提示修改，小批次改善后再评更大集合；最后测试与提示优化隔离。",
"sources": [
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
}
],
"judgment": "规则对照标注支持文档计算检索结果"
},
{
"label": "PUPA",
"data": "111 训练、111 验证、221 测试",
"scoring": "隐私委托任务分别衡量回答质量及泄漏个人信息的程度，组合成总分。",
"visible": "反馈把总分拆成回答质量分与个人信息泄漏分；原文该段没有进一步公布每项裁判型号，不能自行补一个模型。 优化器能读训练题和标签；验证可查分但限制直接读取题目内容。",
"use": "反思生成提示修改，小批次改善后再评更大集合；最后测试与提示优化隔离。",
"sources": [
{
"label": "附录 E.1：逐基准反馈函数",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
},
{
"label": "§4：访问边界",
"url": "https://arxiv.org/html/2507.19457#S4"
}
],
"judgment": "分别计算质量与隐私泄漏指标；不是单一正确答案匹配"
},
{
"label": "推理时搜索：内核代码",
"data": "另有 NPUEval 内核开发任务。",
"scoring": "实际编译与执行内核，错误信息可触发检索相关技术手册。",
"visible": "编译错误、执行结果与针对错误检索到的领域文档。",
"use": "用错误来选择需要注入提示的技术知识；文档是修复帮助，执行检查才验证代码效果。",
"sources": [
{
"label": "§5.1：反馈驱动手册检索",
"url": "https://arxiv.org/html/2507.19457#S5.SS1"
}
],
"judgment": "编译、执行内核并读取错误与性能反馈"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4,
5
],
"evolution": "HotpotQA150；IF-RLVR150；AIME2022–24共90题的一半；PUPA111；LiveBench-Math368题约三等分；HoVer150。",
"selection": "训练材料内部区分产生反思的反馈与Pareto（保留在不同评价维度上各有优势、不能被另一候选全面超过的方案）比较；外部验证集可查分、限制直接读题。Hotpot/IF各300、AIME45、PUPA111验证。HoVer300验证。",
"evaluation": "HotpotQA300；IFBench294新约束题；AIME2025共30题各重复5次；PUPA221；LiveBench剩余测试部分；HoVer300。",
"isolation": "主实验三段分开；IFBench测试约束训练不可见。另有NPUEval/KernelBench逐题程序优化，不能套用主提示实验协议。",
"roles": {
"executor": {
"value": "主复合AI系统使用Qwen3-8B或GPT-4.1-mini-2025-04-14；它们执行候选提示，GEPA不更新这些执行模型的权重。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2507.19457#A5.SS2"
}
]
},
"modifier": {
"value": "Qwen3-8B 或 GPT-4.1-mini 在对应配置中读取执行轨迹和文字反馈，提出提示修改；固定 GEPA 算法选择候选并评测。跨模型实验将完全用 Qwen3-8B 优化的提示交给 GPT-4.1-mini 执行。",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.2",
"url": "https://arxiv.org/html/2507.19457#A5.SS2"
}
]
},
"seed": {
"value": "固定复合 AI 程序及模块接口，在执行轨迹反馈下修改模块提示，并保留不同题上互补的候选。DSPy 与 Trace 对照保持相同架构、初始提示及数据；不是由 GEPA 自由改写任意工具。",
"sources": [
{
"label": "实验比较、优化数据协议；§5.1",
"url": "https://arxiv.org/abs/2507.19457"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"selection": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"evaluation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
],
"isolation": [
{
"label": "§4",
"url": "https://arxiv.org/html/2507.19457#S4"
},
{
"label": "附录E.1",
"url": "https://arxiv.org/html/2507.19457#A5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "强化学习适应新任务常需要大量执行轨迹，遇到昂贵工具、有限调用预算或无法微调的闭源模型时，成本会成为瓶颈。作者认为，轨迹里的文字、错误和工具反馈包含比单个分数更丰富的改进线索，因此研究能否通过反思这些信息，更省样本地优化提示。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2507.19457#S1"
}
]
},
{
"key": "position",
"value": "研究可解释的语言反馈能否比稀疏分数更高效地支持任务适应，并使提示优化在较少试验下获得有竞争力的能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.19457"
}
]
},
{
"key": "conclusion",
"value": "在文中任务上以更少轨迹取得有竞争力的收益，展示自然语言反馈的样本效率。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2507.19457"
}
]
}
],
"fields": {
"object": "模型系统中的一个或多个提示；基础模型参数保持不变。",
"verdict": "执行轨迹提供推理、工具调用和返回信息；任务反馈进一步指出缺失文档、未满足约束或结果错误，供模型提出有针对性的提示修改。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2507.19457"
}
]
},
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2507.19457"
}
]
},
{
"tag": "org:berkeley",
"label": "UC Berkeley",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2507.19457"
}
]
},
{
"tag": "person:omar-khattab",
"label": "Omar Khattab",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2507.19457"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"feedback",
"harness"
]
},
{
"id": "2609.00829",
"title": "HarnessEvolve: Learning from Reference Trajectories for Reliable Agent Self-Evolution",
"date": "2026-09-01",
"url": "https://arxiv.org/abs/2609.00829",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Skill",
"Tool",
"Prompt",
"SeparateEvolver",
"GoldLabel",
"RegressionGate",
"HeldOut"
],
"depth": [
"M1"
],
"fields": {
"本质定位": "用答案条件下生成的参考轨迹定位失败，再聚类为系统性错误。",
"什么在变": "代码、prompts、skills、tools；模型冻结。",
"谁来改 / 谁执行": "execution / evaluation / optimization / gating 分角色；执行模型为 Qwen3.6-27B 或 DeepSeek-V4-Flash。",
"基础 harness": "企业任务 LAMAgent；开放任务 OpenClaw。",
"Feedback": "终局判定 → 与可靠参考路径对齐 → 错误聚类 → candidate edits → 质量/性能 gate。",
"进化数据": "SearchQA、OfficeQA、SpreadsheetBench 及两个内部 QA 数据集的 train。",
"选模数据": "validation 在 epoch 结束时选择 snapshot。",
"测试数据": "独立 test；§3.1、§4.1 明确三路划分，数量待补。",
"证据边界": "参考轨迹使用训练答案；应与无答案反馈方法区分。"
},
"protocol": "Train → selection → test",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/html/2609.00829v1#S3",
"https://arxiv.org/html/2609.00829v1#S4"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "在训练答案条件下生成参考轨迹，与失败轨迹对齐以定位根因，再聚类为跨任务模式，推动 harness 更新。",
"novelty": "借助参考答案构造的参考轨迹定位失败，再改可复用框架，并检查答案泄漏、提示膨胀和已有任务退化；参考轨迹属于额外监督。",
"object": "执行代码、提示、技能和工具；模型冻结。",
"executor": "公开题：DeepSeek-V4-Flash + OpenClaw；企业题：领域微调Qwen3.6-27B或DeepSeek-V4-Flash + LAMAgent。",
"modifier": "评估 agent 负责失败定位及候选审核，优化 agent 据反馈编辑运行框架。§4.1 明确了执行端 Qwen3.6-27B／DeepSeek-V4-Flash，但未分别列出这两个修改相关角色的模型配置。",
"roleContext": "execution / evaluation / optimization / gating 分角色；执行模型为 Qwen3.6-27B 或 DeepSeek-V4-Flash。",
"seed": "企业任务用 LAMAgent，开放任务用 OpenClaw；前者可改项目代码/技能/提示/工具，后者可改技能目录、AGENTS.md、SOUL.md 和工具。技能目录含参考资料与脚本，不限一份 skill.md。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "先判断任务成败，再把失败过程与成功参考路径对齐，找出反复出错的步骤；候选修改还要通过防记答案检查和任务表现检查。",
"diagnosis": "核验参考路径后，与失败路径比较首次分歧并聚类错误。",
"update": "修改 prompts、skills、tools 与执行逻辑。",
"acceptance": "quality gate 检查数据泄漏/prompt bloat；performance gate 检查当前和近期 batch；epoch-end validation 选 snapshot。",
"experiments": [
{
"name": "五个开放/企业 QA 与表格任务",
"evolve": "train tasks + answer-conditioned reference generation",
"selection": "validation 在 epoch-end 选择 snapshot",
"test": "独立 test",
"isolation": "独立测试",
"note": ""
}
],
"takeaway": "参考轨迹使用训练答案；应与无答案反馈方法区分。",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2609.00829v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：SearchQA、OfficeQA、SpreadsheetBench及自建CloudCoreNetwork-QA、Wireless-QA的训练split；企业题涉及网络问答及KPI/网元配置SQL。\n\n调试 / 选版本数据：训练题最多5次尝试构造参考轨迹；验证集做改动门控与早停。\n\n最终测试数据：各数据集测试集，所有方法共用三段划分；另将OpenClaw产物迁移其他运行框架。\n\n数据隔离与证据边界：§4.1没有列各split绝对数量，企业数据称in-house；不能推定公开可下载或套用SkillOpt的2:1:7。",
"cycle": "对比真实与参考轨迹定位运行框架故障；可改整目录、脚本及项目代码，不只改skill.md。OpenClaw用于公开题，LAMAgent用于企业题。",
"train": "SearchQA、OfficeQA、SpreadsheetBench及自建CloudCoreNetwork-QA、Wireless-QA的训练split；企业题涉及网络问答及KPI/网元配置SQL。",
"debug": "训练题最多5次尝试构造参考轨迹；验证集做改动门控与早停。",
"test": "各数据集测试集，所有方法共用三段划分；另将OpenClaw产物迁移其他运行框架。",
"isolation": "§4.1没有列各split绝对数量，企业数据称in-house；不能推定公开可下载或套用SkillOpt的2:1:7。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "企业任务用 LAMAgent，开放任务用 OpenClaw；前者可改项目代码/技能/提示/工具，后者可改技能目录、AGENTS.md、SOUL.md 和工具。技能目录含参考资料与脚本，不限一份 skill.md。",
"protocol": "**数据：**SearchQA、OfficeQA、SpreadsheetBench；自建 CloudCoreNetwork-QA、Wireless-QA 涉及网络问答及 KPI/网元配置 text-to-SQL。各方法共用相同 train/validation/test。\n\n**进化：**在训练任务构造参考轨迹，最多五次尝试，验证选版本；最终在 test 报告，并将 OpenClaw 技能迁移到 Hermes、OpenCode、LAMAgent、DeepSeek Harness。各 split 数量与企业数据公开性本轮待核实。",
"sections": "数据与 Implementation Details",
"source": "https://arxiv.org/abs/2609.00829",
"version": "2609.00829v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "bbf2d74cc6aefbf53e4087033640810018937e835c38325e0af950b562365742",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "公开题：DeepSeek-V4-Flash + OpenClaw；企业题：领域微调Qwen3.6-27B或DeepSeek-V4-Flash + LAMAgent。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "评估 agent 负责失败定位及候选审核，优化 agent 据反馈编辑运行框架。§4.1 明确了执行端 Qwen3.6-27B／DeepSeek-V4-Flash，但未分别列出这两个修改相关角色的模型配置。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "执行代码、提示、技能和工具；模型冻结。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "先判断任务成败，再把失败过程与成功参考路径对齐，找出反复出错的步骤；候选修改还要通过防记答案检查和任务表现检查。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
},
{
"label": "§3.5：质量门控",
"url": "https://arxiv.org/html/2609.00829#S3.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "企业任务用 LAMAgent，开放任务用 OpenClaw；前者可改项目代码/技能/提示/工具，后者可改技能目录、AGENTS.md、SOUL.md 和工具。技能目录含参考资料与脚本，不限一份 skill.md。",
"sources": [
{
"label": "数据与 Implementation Details",
"url": "https://arxiv.org/abs/2609.00829"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "对比真实与参考轨迹定位运行框架故障；可改整目录、脚本及项目代码，不只改skill.md。OpenClaw用于公开题，LAMAgent用于企业题。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "SearchQA、OfficeQA、SpreadsheetBench及自建CloudCoreNetwork-QA、Wireless-QA的训练split；企业题涉及网络问答及KPI/网元配置SQL。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "训练题最多5次尝试构造参考轨迹；验证集做改动门控与早停。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "各数据集测试集，所有方法共用三段划分；另将OpenClaw产物迁移其他运行框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "§4.1没有列各split绝对数量，企业数据称in-house；不能推定公开可下载或套用SkillOpt的2:1:7。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "借助参考答案构造的参考轨迹定位失败，再改可复用框架，并检查答案泄漏、提示膨胀和已有任务退化；参考轨迹属于额外监督。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2609.00829v1",
"version": "2609.00829v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "提供“哪里错”的参照信息，而不仅有终局 reward；必须明确 gold-conditioned reference 是额外监督。",
"feedbackCases": [
{
"label": "SearchQA / OfficeQA",
"data": "开放域搜索问答、企业文档与表格问答；各自 训练集 / 验证集 / 测试集 三段共用于所有方法。",
"scoring": "对照任务参考回答判断成败。 论文未逐基准给出裁判模型和全部实现，以上只陈述已披露的任务判定层级。",
"visible": "训练题最多五次尝试保留成功参考轨迹；把失败轨迹与参考路径对齐，提取出错步骤再聚类。",
"use": "训练证据指导修改，验证集负责性能门控和早停，测试评价最终框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"judgment": "问答参考答案判分；本文未逐项明确规则或模型比对器"
},
{
"label": "SpreadsheetBench",
"data": "电子表格数据与公式操作任务；各自 训练集 / 验证集 / 测试集 三段共用于所有方法。",
"scoring": "检查操作后的表格产物是否满足题目目标。 论文未逐基准给出裁判模型和全部实现，以上只陈述已披露的任务判定层级。",
"visible": "训练题最多五次尝试保留成功参考轨迹；把失败轨迹与参考路径对齐，提取出错步骤再聚类。",
"use": "训练证据指导修改，验证集负责性能门控和早停，测试评价最终框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"judgment": "表格产物验收；本文未展开完整检查实现"
},
{
"label": "CloudCoreNetwork-QA / Wireless-QA",
"data": "自建网络领域问答及 KPI / 网元配置 SQL 查询；各自 训练集 / 验证集 / 测试集 三段共用于所有方法。",
"scoring": "按自建题的目标回答或查询结果判定；正文没有完整公开两套企业数据的逐题裁判实现。 论文未逐基准给出裁判模型和全部实现，以上只陈述已披露的任务判定层级。",
"visible": "训练题最多五次尝试保留成功参考轨迹；把失败轨迹与参考路径对齐，提取出错步骤再聚类。",
"use": "训练证据指导修改，验证集负责性能门控和早停，测试评价最终框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"judgment": "企业题目标答案／查询结果判分；未公开逐题实现"
},
{
"label": "修改质量审核",
"data": "候选改动文件；并非另一套答题 benchmark。",
"scoring": "隔离的 gate（决定是否接受修改的检查条件） agent 检查是否直接写入失败题及答案，给 0–1 泄漏分；同时统计新增示例数量，超过阈值拒绝。",
"visible": "拒绝原因以自然语言返回修改者。",
"use": "限制记答案和提示膨胀；这个质量分不能与任务正确率混为一谈。",
"sources": [
{
"label": "§3.5：质量门控",
"url": "https://arxiv.org/html/2609.00829#S3.SS5"
}
],
"judgment": "独立 gate 模型评泄漏；规则统计新增示例数量"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "SearchQA、OfficeQA、SpreadsheetBench及自建CloudCoreNetwork-QA、Wireless-QA的训练split；企业题涉及网络问答及KPI/网元配置SQL。",
"selection": "训练题最多5次尝试构造参考轨迹；验证集做改动门控与早停。",
"evaluation": "各数据集测试集，所有方法共用三段划分；另将OpenClaw产物迁移其他运行框架。",
"isolation": "§4.1没有列各split绝对数量，企业数据称in-house；不能推定公开可下载或套用SkillOpt的2:1:7。",
"roles": {
"executor": {
"value": "公开题：DeepSeek-V4-Flash + OpenClaw；企业题：领域微调Qwen3.6-27B或DeepSeek-V4-Flash + LAMAgent。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
]
},
"modifier": {
"value": "评估 agent 负责失败定位及候选审核，优化 agent 据反馈编辑运行框架。§4.1 明确了执行端 Qwen3.6-27B／DeepSeek-V4-Flash，但未分别列出这两个修改相关角色的模型配置。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
]
},
"seed": {
"value": "企业任务用 LAMAgent，开放任务用 OpenClaw；前者可改项目代码/技能/提示/工具，后者可改技能目录、AGENTS.md、SOUL.md 和工具。技能目录含参考资料与脚本，不限一份 skill.md。",
"sources": [
{
"label": "数据与 Implementation Details",
"url": "https://arxiv.org/abs/2609.00829"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.00829#S4.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务末尾的成败分数难指出最初哪一步出错，整段反思可能把后续连锁错误误当根因。缺少保护的修改还容易记住具体评测题、堆积提示，未经验证就沿用又会损害旧能力；这些问题使涨分难等同于可靠进化，需要同时解决诊断、泛化与更新验证。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.00829#S1"
}
]
},
{
"key": "position",
"value": "研究 agent 如何从失败中定位可复用的系统性改进，同时避免记住训练答案和损害已有能力，使框架进化更稳定地泛化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.00829"
}
]
},
{
"key": "conclusion",
"value": "在开放与企业任务中改善结果，明确防止答案泄漏、提示膨胀及近期任务退化。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2609.00829"
}
]
}
],
"fields": {
"modifier": "评估 agent 定位失败并审核候选，优化 agent 编辑运行框架。原文明确了执行模型 Qwen3.6-27B／DeepSeek-V4-Flash，但没有分别披露评估和优化角色的型号。",
"object": "执行代码、提示、技能和工具；模型冻结。",
"verdict": "先判断任务成败，再把失败过程与成功参考路径对齐，找出反复出错的步骤；候选修改还要通过防记答案检查和任务表现检查。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"feedback",
"harness"
]
},
{
"id": "2608.27311",
"title": "HarnessLens: Verify Smarter, Evolve Further",
"date": "2026-08-27",
"url": "https://arxiv.org/abs/2608.27311",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"SeparateEvolver",
"ExecutableVerifier",
"RegressionGate",
"HeldOut"
],
"depth": [
"M1"
],
"fields": {
"本质定位": "针对 patch 所影响的行为选择验证任务，以可归因证据决定接受。",
"什么在变": "harness 用户可配置组件。",
"谁来改 / 谁执行": "所有模型角色使用 deepseek-v4-flash-preview；确定性 evaluator 固定权限与预算。",
"基础 harness": "OpenCode、Codex CLI、Pi。",
"Feedback": "执行轨迹 → 行为相关任务 → attributable-evidence gate。",
"进化数据": "每个 benchmark 30 TRAIN tasks。",
"测试数据": "τ² Retail 40；τ³ Banking 67；TB2 59；BIRD 72。",
"选模数据": "TRAIN 决定更新；TEST 在进化后独立入口运行，所有进化角色不可访问。",
"证据边界": "预算比较不是 matched total compute；不能把效率差异全部归因于方法。"
},
"protocol": "Train → selection → test",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/html/2608.27311v1#A2"
],
"links": [
{
"label": "Code",
"url": "https://github.com/jhxu5214/HarnessLens"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "不对每个 patch 固定重跑全部题，而是按 patch 影响的行为选择相关验证任务，再以可归因的恢复/退化证据决定接受。",
"novelty": "根据补丁涉及的行为选择更相关的验证任务并分配预算，检查局部退化，减少每次都完整重跑固定题集的成本。",
"object": "用户可配置的运行框架组件。",
"executor": "DeepSeek-V4-Flash-preview，分别在OpenCode（可连接不同模型的开源 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Pi Coding Agent中执行。",
"modifier": "DeepSeek-V4-Flash-preview承担进化、诊断与任务执行等模型角色；控制器安排配对验证、确认和预算计数。",
"roleContext": "所有模型角色使用 deepseek-v4-flash-preview；确定性 evaluator 固定权限与预算。",
"seed": "用 OpenCode（可连接不同模型的开源 coding agent 工具） 1.17.13、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 0.144.4、Pi 0.80.10 的用户可配置组件作为起点；任务模型/工具/推理预算由评估端覆盖固定。Banking Knowledge 用 BM25（根据查询词与文档词项匹配程度排序的检索算法） 检索，修改不能擅自扩工具或预算。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "从失败轨迹识别需要修复的行为，再在相关训练题上比较修改前后；除了成绩提高，还要确认改动确实触发并恢复了目标行为。",
"diagnosis": "从 execution traces 定位受影响行为，找能测试该行为的任务。",
"update": "修改已有 harness 的用户可配置组件。",
"acceptance": "attributable-evidence gate；测试服务不向任何进化角色开放。",
"experiments": [
{
"name": "τ² / τ³ / TB2 / BIRD",
"evolve": "每个 benchmark 30 TRAIN tasks",
"selection": "TRAIN 定位、提案、验证与接受",
"test": "TEST 分别 40 / 67 / 59 / 72 tasks",
"isolation": "独立测试",
"note": "TEST 独立入口；预算表明确不是 matched total compute。"
}
],
"takeaway": "预算比较不是 matched total compute；不能把效率差异全部归因于方法。",
"result": "",
"reading": "本轮核对原文的方法与实验关键段落；2026-09-08。",
"sources": [
"https://arxiv.org/html/2608.27311v1"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：四个环境各抽30题TRAIN：τ² Retail、τ³ Banking Knowledge、Terminal-Bench2、BIRD Mini-Dev Challenging。\n\n调试 / 选版本数据：TRAIN内选择至少5题做配对验证，再确认；要求改动确实触发且能归因于恢复行为，不只看分数涨跌。\n\n最终测试数据：互斥TEST分别40、67、59、72题；τ²使用官方40题测试集，其余为扣除TRAIN后的剩余题。\n\n数据隔离与证据边界：TEST的题目、轨迹与反馈全程不可见；内部20/10对照划分仍属于TRAIN，不是最终TEST。",
"cycle": "先定位任务空间和运行框架组件，再用配对轨迹检查是否真正修复；即使小批次分数上涨，也可因无可归因恢复而拒绝。",
"train": "四个环境各抽30题TRAIN：τ² Retail、τ³ Banking Knowledge、Terminal-Bench2、BIRD Mini-Dev Challenging。",
"debug": "TRAIN内选择至少5题做配对验证，再确认；要求改动确实触发且能归因于恢复行为，不只看分数涨跌。",
"test": "互斥TEST分别40、67、59、72题；τ²使用官方40题测试集，其余为扣除TRAIN后的剩余题。",
"isolation": "TEST的题目、轨迹与反馈全程不可见；内部20/10对照划分仍属于TRAIN，不是最终TEST。"
},
"reviewed": true,
"systemDataAudit": {
"seed": "用 OpenCode 1.17.13、Codex CLI 0.144.4、Pi 0.80.10 的用户可配置组件作为起点；任务模型/工具/推理预算由评估端覆盖固定。Banking Knowledge 用 BM25 检索，修改不能擅自扩工具或预算。",
"protocol": "**TRAIN→盲测 TEST：**τ² Retail 30/40（官方 40 题 test）；τ³ Banking Knowledge 30/67；Terminal-Bench 2.0 30/59；BIRD Mini-Dev Challenging 30/72。采样 seed=42。\n\n**选择：**候选产生、确认均在 TRAIN 内，部分对照再内部拆 20/10；不能把这内部 10 题与最终 TEST 混为一谈。全部方法共享初始 harness、30 题训练和 verifier，但最大 rollout 预算不同。",
"sections": "附录 B.1–B.4、Tables 7–9",
"source": "https://arxiv.org/abs/2608.27311",
"version": "2608.27311v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "ad1367d58b2b65296c2ecad9528143331033f84d8500a1f9bcc47df391e701d9",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "DeepSeek-V4-Flash-preview，分别在OpenCode（可连接不同模型的开源 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Pi Coding Agent中执行。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "DeepSeek-V4-Flash-preview承担进化、诊断与任务执行等模型角色；控制器安排配对验证、确认和预算计数。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "用户可配置的运行框架组件。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "从失败轨迹识别需要修复的行为，再在相关训练题上比较修改前后；除了成绩提高，还要确认改动确实触发并恢复了目标行为。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "§3.2：二元任务结果",
"url": "https://arxiv.org/html/2608.27311#S3.SS2"
},
{
"label": "§5.1：实验设置",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录 A.4.2：配对审查",
"url": "https://arxiv.org/html/2608.27311#A1.SS4.SSS2"
},
{
"label": "附录 B.2：原生运行与 grader",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "用 OpenCode（可连接不同模型的开源 coding agent 工具） 1.17.13、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 0.144.4、Pi 0.80.10 的用户可配置组件作为起点；任务模型/工具/推理预算由评估端覆盖固定。Banking Knowledge 用 BM25（根据查询词与文档词项匹配程度排序的检索算法） 检索，修改不能擅自扩工具或预算。",
"sources": [
{
"label": "附录 B.1–B.4、Tables 7–9",
"url": "https://arxiv.org/abs/2608.27311"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先定位任务空间和运行框架组件，再用配对轨迹检查是否真正修复；即使小批次分数上涨，也可因无可归因恢复而拒绝。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "四个环境各抽30题TRAIN：τ² Retail、τ³ Banking Knowledge、Terminal-Bench2、BIRD Mini-Dev Challenging。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "TRAIN内选择至少5题做配对验证，再确认；要求改动确实触发且能归因于恢复行为，不只看分数涨跌。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "互斥TEST分别40、67、59、72题；τ²使用官方40题测试集，其余为扣除TRAIN后的剩余题。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "TEST的题目、轨迹与反馈全程不可见；内部20/10对照划分仍属于TRAIN，不是最终TEST。",
"sources": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "根据补丁涉及的行为选择更相关的验证任务并分配预算，检查局部退化，减少每次都完整重跑固定题集的成本。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.27311v1",
"version": "2608.27311v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "把 verification 的任务选择与预算纳入方法；不能把不同 configured budget 下的结果当作严格 compute-matched 对比。",
"feedbackCases": [
{
"label": "τ² Retail",
"data": "每环境 30 道 TRAIN；互斥 TEST 40 道。",
"scoring": "客服工具环境按原基准的任务验收检查完成情况。",
"visible": "TRAIN 轨迹、二元奖励，以及修改前后行为是否真实触发的配对证据；所有执行与进化角色均为 deepseek-v4-flash-preview。",
"use": "挑至少五道行为相关训练题验证，再确认；保留修改要求有可归因的行为恢复证据，不能仅凭总分偶然上涨。TEST 不用于这些检查。",
"sources": [
{
"label": "§3.2：二元任务结果",
"url": "https://arxiv.org/html/2608.27311#S3.SS2"
},
{
"label": "§5.1：实验设置",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录 A.4.2：配对审查",
"url": "https://arxiv.org/html/2608.27311#A1.SS4.SSS2"
},
{
"label": "附录 B.2：原生运行与 grader",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"judgment": "τ² 原生客服环境验收"
},
{
"label": "τ³ Banking Knowledge",
"data": "每环境 30 道 TRAIN；互斥 TEST 67 道。",
"scoring": "银行知识任务沿用原基准 verifier（依据测试或评分规则检查任务结果的程序），转成二元成功结果；本文未进一步逐题披露裁判实现。",
"visible": "TRAIN 轨迹、二元奖励，以及修改前后行为是否真实触发的配对证据；所有执行与进化角色均为 deepseek-v4-flash-preview。",
"use": "挑至少五道行为相关训练题验证，再确认；保留修改要求有可归因的行为恢复证据，不能仅凭总分偶然上涨。TEST 不用于这些检查。",
"sources": [
{
"label": "§3.2：二元任务结果",
"url": "https://arxiv.org/html/2608.27311#S3.SS2"
},
{
"label": "§5.1：实验设置",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录 A.4.2：配对审查",
"url": "https://arxiv.org/html/2608.27311#A1.SS4.SSS2"
},
{
"label": "附录 B.2：原生运行与 grader",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"judgment": "τ³ 原生 verifier；本文未展开其规则或模型实现"
},
{
"label": "Terminal-Bench 2",
"data": "每环境 30 道 TRAIN；互斥 TEST 59 道。",
"scoring": "执行任务专属验收测试，返回终端任务是否完成。",
"visible": "TRAIN 轨迹、二元奖励，以及修改前后行为是否真实触发的配对证据；所有执行与进化角色均为 deepseek-v4-flash-preview。",
"use": "挑至少五道行为相关训练题验证，再确认；保留修改要求有可归因的行为恢复证据，不能仅凭总分偶然上涨。TEST 不用于这些检查。",
"sources": [
{
"label": "§3.2：二元任务结果",
"url": "https://arxiv.org/html/2608.27311#S3.SS2"
},
{
"label": "§5.1：实验设置",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录 A.4.2：配对审查",
"url": "https://arxiv.org/html/2608.27311#A1.SS4.SSS2"
},
{
"label": "附录 B.2：原生运行与 grader",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"judgment": "Terminal-Bench 原生验收测试"
},
{
"label": "BIRD Mini-Dev Challenging",
"data": "每环境 30 道 TRAIN；互斥 TEST 72 道。",
"scoring": "运行 SQL 并由基准 评分器 检查查询结果正确性。",
"visible": "TRAIN 轨迹、二元奖励，以及修改前后行为是否真实触发的配对证据；所有执行与进化角色均为 deepseek-v4-flash-preview。",
"use": "挑至少五道行为相关训练题验证，再确认；保留修改要求有可归因的行为恢复证据，不能仅凭总分偶然上涨。TEST 不用于这些检查。",
"sources": [
{
"label": "§3.2：二元任务结果",
"url": "https://arxiv.org/html/2608.27311#S3.SS2"
},
{
"label": "§5.1：实验设置",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录 A.4.2：配对审查",
"url": "https://arxiv.org/html/2608.27311#A1.SS4.SSS2"
},
{
"label": "附录 B.2：原生运行与 grader",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
],
"judgment": "运行 SQL，以 BIRD 基准程序检查结果正确性"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3
],
"evolution": "四个环境各抽30题TRAIN：τ² Retail、τ³ Banking Knowledge、Terminal-Bench2、BIRD Mini-Dev Challenging。",
"selection": "TRAIN内选择至少5题做配对验证，再确认；要求改动确实触发且能归因于恢复行为，不只看分数涨跌。",
"evaluation": "互斥TEST分别40、67、59、72题；τ²使用官方40题测试集，其余为扣除TRAIN后的剩余题。",
"isolation": "TEST的题目、轨迹与反馈全程不可见；内部20/10对照划分仍属于TRAIN，不是最终TEST。",
"roles": {
"executor": {
"value": "DeepSeek-V4-Flash-preview，分别在OpenCode（可连接不同模型的开源 coding agent 工具）、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具）、Pi Coding Agent中执行。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
},
"modifier": {
"value": "DeepSeek-V4-Flash-preview承担进化、诊断与任务执行等模型角色；控制器安排配对验证、确认和预算计数。",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
},
{
"label": "附录B.2",
"url": "https://arxiv.org/html/2608.27311#A2.SS2"
}
]
},
"seed": {
"value": "用 OpenCode（可连接不同模型的开源 coding agent 工具） 1.17.13、Codex CLI（在终端中读写代码、运行命令的 coding agent 工具） 0.144.4、Pi 0.80.10 的用户可配置组件作为起点；任务模型/工具/推理预算由评估端覆盖固定。Banking Knowledge 用 BM25（根据查询词与文档词项匹配程度排序的检索算法） 检索，修改不能擅自扩工具或预算。",
"sources": [
{
"label": "附录 B.1–B.4、Tables 7–9",
"url": "https://arxiv.org/abs/2608.27311"
},
{
"label": "OpenCode 官方说明",
"url": "https://opencode.ai/docs"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"selection": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"evaluation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
],
"isolation": [
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.27311#A2.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "不同框架修改影响的行为不同，但现有方法常用同一套或预先抽好的任务验证。无关任务会浪费评估预算，缺少相关任务又可能看不出目标行为是否改善或其他行为是否退化；已有验证证据还未被系统复用，因此需要让验证随具体修改而调整。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.27311#S1"
}
]
},
{
"key": "position",
"value": "研究有限交互预算下，怎样判断一项框架修改是否真正有效，使自动进化减少无关测试并识别被总分掩盖的行为退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.27311"
}
]
},
{
"key": "conclusion",
"value": "在多个运行框架和基准上以更少评测预算改善留出表现。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.27311"
}
]
}
],
"fields": {
"object": "用户可配置的运行框架组件。",
"verdict": "从失败轨迹识别需要修复的行为，再在相关训练题上比较修改前后；除了成绩提高，还要确认改动确实触发并恢复了目标行为。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2608.10178",
"title": "One Recipe, Many Harnesses: What Self-Evolution Encodes Across Languages and Models",
"date": "2026-08-10",
"url": "https://arxiv.org/abs/2608.10178",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"HarnessCode",
"CrossModel",
"HeldOut"
],
"depth": [
"M1"
],
"fields": {
"本质定位": "直接分析 evolved harness 补偿什么行为缺陷，以及跨语言/模型能迁移什么。",
"什么在变": "prompts、tools、memory；固定 evolution recipe。",
"谁来改 / 谁执行": "固定 recipe → 三种 base model；具体型号待正文核对。",
"基础 harness": "minimal seed 与 mini-SWE-agent 对照。",
"Feedback": "typed failure signal + falsifiable edit contract。",
"Evolution → Eval": "Multi-SWE-Bench 八种语言；报告 held-out 与跨模型分析，数量待核对。",
"证据边界": "摘要称存在无收益区域；通用策略可迁移，ecosystem-specific 部分需重新进化。"
},
"protocol": "有 held-out 报告",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 摘要核对",
"sources": [
"https://arxiv.org/abs/2608.10178"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "直接分析 evolved harness 补偿什么行为缺陷，以及跨语言/模型能迁移什么。",
"novelty": "固定进化流程，跨语言和模型比较所得框架，并删去语言生态专属内容做对照，区分通用执行策略与本地工程知识。",
"object": "提示、工具和记忆；生成修改的进化方法固定。",
"executor": "Claude Haiku4.5、GPT-5-mini、DeepSeek-V4-Flash，分别运行相同初始运行框架。",
"modifier": "每次实验均由对应的 Claude Haiku 4.5、GPT-5-mini 或 DeepSeek-V4-Flash 同时承担任务执行和外层修改两个角色；因此跨模型结果反映两种角色的共同影响。",
"roleContext": "固定 recipe → 三种 base model；具体型号待正文核对。",
"seed": "极简推理—行动循环：两句系统提示、任务与提交模板、一个命令行工具，记忆和自动触发处理代码起初为空。未预先写入测试、构建、补丁范围等指导规则；另与人工设计的 mini-SWE-agent 轻量编程框架在相同预算下比较。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "仓库测试给出修复成败，轨迹分析标记失败类型；修改者预测哪些题会改善或退化，下一轮实际执行再核对这些预测。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "Multi-SWE-Bench 八种语言；报告 held-out 与跨模型分析，数量待核对。"
}
],
"takeaway": "摘要称存在无收益区域；通用策略可迁移，ecosystem-specific 部分需重新进化。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2608.10178"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：Multi-SWE-BenchC、C++、Java、Rust、TypeScript、JavaScript、Go、Python八种语言，各20题作进化集，逐轮评分，运行3轮。\n\n调试 / 选版本数据：相同进化规则用于不同语言/模型，分析最终行为指导、工具和执行缺陷。\n\n最终测试数据：每语言50题互斥单独留出的；只在最终提升版本上评一次。\n\n数据隔离与证据边界：测试不参与提案、打分或选择；跨模型移植/蒸馏复用同一测试题，不能累计为新增样本。",
"cycle": "重点分析相同进化方法究竟写入什么：可跨语言的行为原则与语言/模型相关细节，以及这些产物的迁移。",
"train": "Multi-SWE-BenchC、C++、Java、Rust、TypeScript、JavaScript、Go、Python八种语言，各20题作进化集，逐轮评分，运行3轮。",
"debug": "相同进化规则用于不同语言/模型，分析最终行为指导、工具和执行缺陷。",
"test": "每语言50题互斥单独留出的；只在最终提升版本上评一次。",
"isolation": "测试不参与提案、打分或选择；跨模型移植/蒸馏复用同一测试题，不能累计为新增样本。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "极简 ReAct：两句话系统提示、任务与提交模板、一个 bash 工具、空 hooks/记忆。没有预置测试、构建、验证或 diff 范围规则；另以原版 mini-SWE-agent 作人写强底座对照，预算相同。",
"protocol": "**各编程语言的进化→测试：**固定 20 道 evolution 题、50 道互斥 held-out 题，进化三轮；测试仅在最终提升的 checkpoint 上评一次，不参与提案或选择。\n\n**迁移：**跨模型移植和蒸馏复用相同留出题，不能当作新增测试集；每次 rollout 上限 80 步/$1.5。原始任务来源和完整语言列表本轮待核实。",
"sections": "附录 B.1–B.3",
"source": "https://arxiv.org/abs/2608.10178",
"version": "2608.10178v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "786eeacb4adc4b9398c9d3ba38ac330c794f7ce9cfe4f280938b5e99d06ecfd5",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Claude Haiku4.5、GPT-5-mini、DeepSeek-V4-Flash，分别运行相同初始运行框架。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "每次实验均由对应的 Claude Haiku 4.5、GPT-5-mini 或 DeepSeek-V4-Flash 同时承担任务执行和外层修改两个角色；因此跨模型结果反映两种角色的共同影响。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "提示、工具和记忆；生成修改的进化方法固定。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "仓库测试给出修复成败，轨迹分析标记失败类型；修改者预测哪些题会改善或退化，下一轮实际执行再核对这些预测。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.10178#S4"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "极简推理—行动循环：两句系统提示、任务与提交模板、一个命令行工具，记忆和自动触发处理代码起初为空。未预先写入测试、构建、补丁范围等指导规则；另与人工设计的 mini-SWE-agent 轻量编程框架在相同预算下比较。",
"sources": [
{
"label": "附录 B.1–B.3",
"url": "https://arxiv.org/abs/2608.10178"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "重点分析相同进化方法究竟写入什么：可跨语言的行为原则与语言/模型相关细节，以及这些产物的迁移。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "Multi-SWE-BenchC、C++、Java、Rust、TypeScript、JavaScript、Go、Python八种语言，各20题作进化集，逐轮评分，运行3轮。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.10178#S4"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "相同进化规则用于不同语言/模型，分析最终行为指导、工具和执行缺陷。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "每语言50题互斥单独留出的；只在最终提升版本上评一次。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "测试不参与提案、打分或选择；跨模型移植/蒸馏复用同一测试题，不能累计为新增样本。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "固定进化流程，跨语言和模型比较所得框架，并删去语言生态专属内容做对照，区分通用执行策略与本地工程知识。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.10178v1",
"version": "2608.10178v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "固定 evolution recipe 后比较八种语言和三种模型；通过去掉 ecosystem-specific 内容的对照，区分通用执行纪律与生态专门知识。",
"feedbackCases": [
{
"label": "八语言框架进化",
"data": "Multi-SWE-Bench 的 C、C++、Java、Rust、TypeScript、JavaScript、Go、Python；每语言固定 20 道进化题、三轮。",
"scoring": "基准仓库测试验收代码修复；分析器结合执行轨迹给失败分型。",
"visible": "真实测试结论、工具执行过程及历史改动；修改者写出预计改善或退化的具体任务，下一轮核对。",
"use": "用可证伪的改动预测约束修改，退化时拒绝或回滚，不靠修改者自评有效。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.10178#S4"
}
],
"judgment": "八语言仓库测试验收补丁；模型分析器另分失败类型"
},
{
"label": "最终留出评价",
"data": "每语言另外 50 道互斥题。",
"scoring": "同一仓库测试验收机制。",
"visible": "只在最终提升后的版本评价一次。",
"use": "衡量各语言新题表现；不以留出题反复选择三轮修改。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.10178#S4"
}
],
"judgment": "留出题仍使用仓库测试验收"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "Multi-SWE-BenchC、C++、Java、Rust、TypeScript、JavaScript、Go、Python八种语言，各20题作进化集，逐轮评分，运行3轮。",
"selection": "相同进化规则用于不同语言/模型，分析最终行为指导、工具和执行缺陷。",
"evaluation": "每语言50题互斥单独留出的；只在最终提升版本上评一次。",
"isolation": "测试不参与提案、打分或选择；跨模型移植/蒸馏复用同一测试题，不能累计为新增样本。",
"roles": {
"executor": {
"value": "Claude Haiku4.5、GPT-5-mini、DeepSeek-V4-Flash，分别运行相同初始运行框架。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
]
},
"modifier": {
"value": "每次实验均由对应的 Claude Haiku 4.5、GPT-5-mini 或 DeepSeek-V4-Flash 同时承担任务执行和外层修改两个角色；因此跨模型结果反映两种角色的共同影响。",
"sources": [
{
"label": "§2.2",
"url": "https://arxiv.org/html/2608.10178#S2.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px2"
}
]
},
"seed": {
"value": "极简推理—行动循环：两句系统提示、任务与提交模板、一个命令行工具，记忆和自动触发处理代码起初为空。未预先写入测试、构建、补丁范围等指导规则；另与人工设计的 mini-SWE-agent 轻量编程框架在相同预算下比较。",
"sources": [
{
"label": "附录 B.1–B.3",
"url": "https://arxiv.org/abs/2608.10178"
},
{
"label": "mini-swe-agent 官方说明",
"url": "https://github.com/SWE-agent/mini-swe-agent"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2608.10178#S4"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.10178#S4.SS1.SSS0.Px1"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2608.10178#A2.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "进化后分数提高，可能来自编程语言知识、对基础模型弱点的补偿，也可能只是过拟合题目。已有研究未充分分离这些来源，因此无法判断哪些改进能跨语言或模型迁移、什么时候必须重新进化；作者研究的是框架到底学进了什么。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.10178#S1"
}
]
},
{
"key": "position",
"value": "研究进化后的编程框架究竟学到了什么：通用执行策略、特定语言的工程知识，还是对某个模型弱点的补偿，以及哪些部分能够迁移。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.10178"
}
]
},
{
"key": "conclusion",
"value": "通用策略可以迁移，但生态工具细节仍需本地适配；运行框架主要弥补可恢复的执行缺陷。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.10178"
}
]
}
],
"fields": {
"object": "提示、工具和记忆；生成修改的进化方法固定。",
"verdict": "仓库测试给出修复成败，轨迹分析标记失败类型；修改者预测哪些题会改善或退化，下一轮实际执行再核对这些预测。",
"seed": "极简推理—行动循环：两句系统提示、任务与提交模板、一个命令行工具，记忆和自动触发处理代码起初为空。未预先写入测试、构建、补丁范围等指导规则；另与人工设计的 mini-SWE-agent 轻量编程框架在相同预算下比较。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation"
]
},
{
"id": "2608.31100",
"title": "S³Gym: Can LLMs Turn Self-Testing and Self-Judging into Self-Improvement?",
"date": "2026-08-31",
"url": "https://arxiv.org/abs/2608.31100",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"Context",
"MemoryContent",
"Weights",
"SelfFeedback",
"ExecutableVerifier",
"HeldOut",
"org:bytedance-seed"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "分别测 self-testing、self-judging、self-improvement；适合定位反馈转化瓶颈。",
"什么在变": "对照 History ICL、Summary Memory、parameter training 三条路径。",
"谁来改 / 谁执行": "被测 LLM 使用各路径；具体型号待全文核对。",
"基础 harness": "七种文本游戏的交互环境。",
"Feedback": "模型自评与可执行环境 verifier。",
"Evolution → Eval": "permissive exploration 与 strict held-out evaluation 分开。",
"证据边界": "摘要报告收益依赖任务；摘要未给出逐项 split 数量与完整选模协议。"
},
"protocol": "有 held-out 报告",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 摘要核对",
"sources": [
"https://arxiv.org/abs/2608.31100"
],
"links": [],
"source": "addition",
"priorityBasis": "用户指定阅读重点：ByteDance Seed / TokenWave 的 Self-Developing Agents 系列。",
"prioritySources": [
{
"label": "团队项目页",
"url": "https://self-developing-agents.github.io/"
}
],
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"B-Lifelong",
"B-Reliability"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "分别测 self-testing、self-judging、self-improvement；适合定位反馈转化瓶颈。",
"novelty": "将自主尝试、自我判分和利用经验分阶段测量；探索时不提供环境真实分数，正式测验再用程序检查任务结果。",
"object": "对比保留交互历史、维护摘要记忆和训练模型参数三条学习路径。",
"executor": "主上下文实验：GPT-4o、GPT-4.1、o3-mini、Gemini2.5 Flash/Pro、GPT-5.5、Gemini3.5 Flash，使用统一游戏接口。 参数训练实验另用 Qwen3-8B：训练轮次 0 是原始模型，训练轮次 1–19 是用探索轨迹更新后的模型；每个检查点都在七个游戏上测试。",
"modifier": "模型产生自评与经验总结；控制器保留历史或摘要。参数学习分支另用自生成轨迹训练模型。",
"roleContext": "被测 LLM 使用各路径；具体型号待全文核对。",
"seed": "统一文本游戏交互接口，动作后能得到程序计算的结果，并要求模型给自评分。比较直接使用历史、摘要记忆和参数训练三种经验利用方式；这三种更新对象不同。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "模型自评与可执行环境结果检查器（按测试或判分规则检查任务结果）。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "有 held-out 报告",
"note": "permissive exploration 与 strict held-out evaluation 分开。"
}
],
"takeaway": "摘要报告收益依赖任务；摘要未给出逐项 split 数量与完整选模协议。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2608.31100"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：七个文本游戏的宽松探索配置：Chess、Minesweeper、Nullify、Plants-vs-Zombies、Snake、Tetris、Trust Evolution。\n\n调试 / 选版本数据：用自己的轨迹和自评分更新完整历史、摘要记忆或参数；再用真实环境分数检查自评是否可靠。\n\n最终测试数据：更严格配置：如扫雷首雷即结束、Snake碰撞结束、Trust Evolution均匀抽对手；逐checkpoint（某个时刻保存的模型或系统版本）评价收益。\n\n数据隔离与证据边界：探索和评测随机种子互斥；评测轨迹不写入历史、记忆或训练数据。宽松到严格的变化按游戏定义，非统一文本数据split。",
"cycle": "先让 agent 主动试验，再让它自行判断结果，最后比较原始历史、摘要记忆和参数学习三种经验利用方式。真实环境判分在探索时不提供，用于之后检查自评是否可靠和能力是否提升。",
"train": "七个文本游戏的宽松探索配置：Chess、Minesweeper、Nullify、Plants-vs-Zombies、Snake、Tetris、Trust Evolution。",
"debug": "用自己的轨迹和自评分更新完整历史、摘要记忆或参数；再用真实环境分数检查自评是否可靠。",
"test": "更严格配置：如扫雷首雷即结束、Snake碰撞结束、Trust Evolution均匀抽对手；逐checkpoint（某个时刻保存的模型或系统版本）评价收益。",
"isolation": "探索和评测随机种子互斥；评测轨迹不写入历史、记忆或训练数据。宽松到严格的变化按游戏定义，非统一文本数据split。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "统一文本游戏交互接口，动作后能得到程序计算的结果，并要求模型给自评分。比较直接使用历史、摘要记忆和参数训练三种经验利用方式；这三种更新对象不同。",
"protocol": "**数据：**七种文本游戏，来自 KORGym 式可执行游戏设置，覆盖规则推断、约束、空间、资源和对抗任务，含 Snake、Tetris 等。\n\n**探索→测试：**先在 exploration 配置收集自测/自评经历，再到更严格、不同的 held-out 配置测行为；用真实环境分数检查模型自评分是否可靠。完整游戏列表、探索/测试回合数及随机种子本轮待核实。",
"sections": "benchmark 设计与探索/评估阶段；附录游戏规格",
"source": "https://arxiv.org/abs/2608.31100",
"version": "2608.31100v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "c0b9ce92b627a085935a71d71eb4c1d2f958895b5921b0e00e8652daf5d0f84a",
"seedStatus": "recorded",
"protocolStatus": "partial"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "主上下文实验：GPT-4o、GPT-4.1、o3-mini、Gemini2.5 Flash/Pro、GPT-5.5、Gemini3.5 Flash，使用统一游戏接口。 参数训练实验另用 Qwen3-8B：训练轮次 0 是原始模型，训练轮次 1–19 是用探索轨迹更新后的模型；每个检查点都在七个游戏上测试。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.31100#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "模型产生自评与经验总结；控制器保留历史或摘要。参数学习分支另用自生成轨迹训练模型。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "对比保留交互历史、维护摘要记忆和训练模型参数三条学习路径。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "模型自评与可执行环境结果检查器（按测试或判分规则检查任务结果）。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "统一文本游戏交互接口，动作后能得到程序计算的结果，并要求模型给自评分。比较直接使用历史、摘要记忆和参数训练三种经验利用方式；这三种更新对象不同。",
"sources": [
{
"label": "benchmark 设计与探索/评估阶段；附录游戏规格",
"url": "https://arxiv.org/abs/2608.31100"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先让 agent 主动试验，再让它自行判断结果，最后比较原始历史、摘要记忆和参数学习三种经验利用方式。真实环境判分在探索时不提供，用于之后检查自评是否可靠和能力是否提升。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "七个文本游戏的宽松探索配置：Chess、Minesweeper、Nullify、Plants-vs-Zombies、Snake、Tetris、Trust Evolution。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "用自己的轨迹和自评分更新完整历史、摘要记忆或参数；再用真实环境分数检查自评是否可靠。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "更严格配置：如扫雷首雷即结束、Snake碰撞结束、Trust Evolution均匀抽对手；逐checkpoint（某个时刻保存的模型或系统版本）评价收益。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "探索和评测随机种子互斥；评测轨迹不写入历史、记忆或训练数据。宽松到严格的变化按游戏定义，非统一文本数据split。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将自主尝试、自我判分和利用经验分阶段测量；探索时不提供环境真实分数，正式测验再用程序检查任务结果。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2608.31100v1",
"version": "2608.31100v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "将主动尝试、判断体验好坏、把体验转成改进分开测；自判正确不代表能形成在严格 held-out 环境可执行的策略。",
"feedbackCases": [
{
"label": "七游戏：自测与自评",
"data": "Chess、Minesweeper、Nullify、Plants-vs-Zombies、Snake、Tetris、Trust Evolution；使用允许更多试错的探索配置。",
"scoring": "探索期真实环境 verifier（依据测试或评分规则检查任务结果的程序） 的结果不返回 agent；agent 必须根据自己产生的行为轨迹作出自我判断。",
"visible": "可见行动与观察以及自评分，看不到 verifier（依据测试或评分规则检查任务结果的程序） 计算的真值结果。",
"use": "按自评更新完整历史、摘要记忆或模型参数；这篇正是检验没有真实奖励时自评是否可靠。",
"sources": [
{
"label": "§2：隐藏 verifier 的边界",
"url": "https://arxiv.org/html/2608.31100#S2"
}
],
"judgment": "agent 根据自身行为轨迹自评；不返回真实环境判分"
},
{
"label": "严格测试与自评可靠性",
"data": "采用更严格配置，如扫雷遇第一雷结束、Snake 碰撞结束、Trust Evolution 均匀抽对手。",
"scoring": "外部环境的可执行 verifier（依据测试或评分规则检查任务结果的程序） 给真实任务结果；作者对照自评分检查判断可靠性。",
"visible": "真实分数用于研究者逐检查点评价，不倒灌给探索期 agent。",
"use": "同时检查自评是否准确、积累经验是否提升严格测试表现；不能把作者看得到真分数写成 agent 也看得到。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"judgment": "外部可执行环境验证器判真实结果，再与 agent 自评比较"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1
],
"evolution": "七个文本游戏的宽松探索配置：Chess、Minesweeper、Nullify、Plants-vs-Zombies、Snake、Tetris、Trust Evolution。",
"selection": "用自己的轨迹和自评分更新完整历史、摘要记忆或参数；再用真实环境分数检查自评是否可靠。",
"evaluation": "更严格配置：如扫雷首雷即结束、Snake碰撞结束、Trust Evolution均匀抽对手；逐checkpoint（某个时刻保存的模型或系统版本）评价收益。",
"isolation": "探索和评测随机种子互斥；评测轨迹不写入历史、记忆或训练数据。宽松到严格的变化按游戏定义，非统一文本数据split。",
"roles": {
"executor": {
"value": "主上下文实验：GPT-4o、GPT-4.1、o3-mini、Gemini2.5 Flash/Pro、GPT-5.5、Gemini3.5 Flash，使用统一游戏接口。 参数训练实验另用 Qwen3-8B：训练轮次 0 是原始模型，训练轮次 1–19 是用探索轨迹更新后的模型；每个检查点都在七个游戏上测试。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
},
{
"label": "§6.1",
"url": "https://arxiv.org/html/2608.31100#S6.SS1"
}
]
},
"modifier": {
"value": "模型产生自评与经验总结；控制器保留历史或摘要。参数学习分支另用自生成轨迹训练模型。",
"sources": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px2"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1"
}
]
},
"seed": {
"value": "统一文本游戏交互接口，动作后能得到程序计算的结果，并要求模型给自评分。比较直接使用历史、摘要记忆和参数训练三种经验利用方式；这三种更新对象不同。",
"sources": [
{
"label": "benchmark 设计与探索/评估阶段；附录游戏规格",
"url": "https://arxiv.org/abs/2608.31100"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"selection": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"evaluation": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
],
"isolation": [
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.31100#S5.SS1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.31100#A1.SS1"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "已有 agent 评测主要测当前能完成多少任务，较少检验它能否利用自己的经历改善未来表现。收集轨迹不等于学习：agent 还要主动产生有用证据、正确判断成败原因，再把判断转成改进；其中任一步出错，都可能使经验无用，甚至让训练强化错误。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2608.31100#S1"
}
]
},
{
"key": "position",
"value": "评测 agent 在拿不到环境真实判分的探索阶段，能否自主试验、准确判断自身表现，并把经历转化为更严格测试中的能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.31100"
}
]
},
{
"key": "conclusion",
"value": "提供可执行验证器与分阶段协议，帮助定位瓶颈发生在探索、自评还是经验利用。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.31100"
}
]
}
],
"fields": {
"object": "对比保留交互历史、维护摘要记忆和训练模型参数三条学习路径。"
}
},
"attributions": [
{
"tag": "org:bytedance-seed",
"label": "ByteDance Seed",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2608.31100"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation",
"feedback",
"curriculum"
]
},
{
"id": "2609.01481",
"title": "Harness-of-Harness: Multi-Day Autonomous Software Development with Continual Improvement",
"date": "2026-09-01",
"url": "https://arxiv.org/abs/2609.01481",
"priority": "R",
"categories": [
"methods"
],
"tags": [
"Workflow",
"Tool",
"Skill",
"org:shanghai-ai-lab"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "边界对照：主要持续改进软件产物，不能因名称含 harness 就当作 harness 自修改。",
"什么在变": "开发中的软件项目与版本历史。",
"谁来改 / 谁执行": "HoH 外层 planning–coding–testing；Codex/GPT-5.5、OpenCode/DeepSeek-V4-Pro、Pi/MiniMax-M3。",
"基础 harness": "现成 coding-agent harness。",
"Feedback": "实现时测试与独立评价分离。",
"Evolution → Eval": "GameCraft-Bench、FrontierSWE、ProgramBench；不是标准跨任务 harness evolution 证据。",
"证据边界": "本条是基于摘要的对象分类判断；需区分 artifact improvement 与 reusable harness improvement。"
},
"protocol": "Artifact / 非 harness",
"protocolBasis": "2026-09-08 核对以下来源；仅按已读范围标注。",
"review": "新增 · 摘要核对",
"sources": [
"https://arxiv.org/abs/2609.01481"
],
"links": [
{
"label": "Code",
"url": "https://github.com/Flesymeb/HarnessOfHarness"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"Artifact"
],
"category": "methods",
"methodType": "artifact",
"brief": {
"summary": "边界对照：主要持续改进软件产物，不能因名称含 harness 就当作 harness 自修改。",
"novelty": "在现成 coding agent 之外安排持续开发，将缺陷修复与新增功能分开推进；主要累积改进的是软件项目产物。",
"object": "正在开发的软件项目与版本历史。",
"executor": "Codex CLI0.142.5 + GPT-5.5 high；OpenCode1.14.30 + DeepSeek-V4-Pro；Pi0.80.10 + MiniMax-M3。",
"modifier": "对应同一模型–运行框架配置承担Planner/Developer/QA角色：规划角色改文档、Developer改项目代码、QA产出证据。不是另有一个被训练的HoH模型。",
"roleContext": "HoH 外层 planning–coding–testing；Codex/GPT-5.5、OpenCode/DeepSeek-V4-Pro、Pi/MiniMax-M3。",
"seed": "在现成 编程 task agent 外加开发者—独立 QA—规划交接协议；保存开发文档、执行证据和下一轮修订要求。对照 Vanilla 只有标准开发一遍，Vanilla Continuation 多续跑两次但不提供独立 QA 证据。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "实现时测试与独立评价分离。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Artifact / 非 harness",
"note": "GameCraft-Bench、FrontierSWE、ProgramBench；不是标准跨任务 harness evolution 证据。"
}
],
"takeaway": "本条是基于摘要的对象分类判断；需区分 artifact improvement 与 reusable harness improvement。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2609.01481"
],
"classificationReason": "HoH 组织既有 coding agents 改进软件项目。软件产物变化，不等于执行 harness 自修改。",
"protocolDetail": "训练 / 进化数据：同一软件项目的公开规格与上一轮QA证据驱动继续开发；不是跨数据集训练运行框架。\n\n调试 / 选版本数据：固定Planner/Developer/QA循环，更新开发文档、保留项目代码；QA证据反馈下一轮，不提供私有评分细则（逐项规定要满足的要求及给分标准）。\n\n最终测试数据：GameCraft-Bench45项目、FrontierSWE15项目、ProgramBench隐藏行为测试；另从空目录开发FPS Fusepoint70轮。\n\n数据隔离与证据边界：最终评测基准评估与开发QA分开；项目本身反复修改。三轮提升是同项目产物进化，不是 harness/模型参数自改。",
"cycle": "外层流程协调已有 coding agent，每轮根据计划、质量检查证据和已有代码继续开发。保留上轮项目作为下一轮起点，支持多日修复与功能扩展；变化的主要是软件产物。",
"train": "同一软件项目的公开规格与上一轮QA证据驱动继续开发；不是跨数据集训练运行框架。",
"debug": "固定Planner/Developer/QA循环，更新开发文档、保留项目代码；QA证据反馈下一轮，不提供私有评分细则（逐项规定要满足的要求及给分标准）。",
"test": "GameCraft-Bench45项目、FrontierSWE15项目、ProgramBench隐藏行为测试；另从空目录开发FPS Fusepoint70轮。",
"isolation": "最终评测基准评估与开发QA分开；项目本身反复修改。三轮提升是同项目产物进化，不是 harness/模型参数自改。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "在现成 coding agent 外加开发者—独立 QA—规划交接协议；保存开发文档、执行证据和下一轮修订要求。对照 Vanilla 只有标准开发一遍，Vanilla Continuation 多续跑两次但不提供独立 QA 证据。",
"protocol": "**固定评估子集：**GameCraft-Bench 45 题，15 个游戏家族各 3 题，按固定 seed 分层补齐；FrontierSWE 15 题，三类各 4/9/2。\n\n**进化含义：**同一个项目产物在多日开发和 QA 中迭代，再按 benchmark 评估，不能描述成在一套题训练 harness 后到另一套题测试。45 和 15 是各配置共同使用的项目数；开发端可见证据与最终评分端的边界需按任务分别看。",
"sections": "附录 B.4–B.5、跨迭代证据协议",
"source": "https://arxiv.org/abs/2609.01481",
"version": "2609.01481v1",
"date": "2026-09-09",
"scope": "数据与基础系统定向核查；数值、来源和角色按原文对应段落记录，未确认的细节在正文标明。",
"sourceSha256": "6850dfa940a3f2578d76ce3cce4ff8743c828204aedf0e9e8e15634358201f94",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Codex CLI0.142.5 + GPT-5.5 high；OpenCode1.14.30 + DeepSeek-V4-Pro；Pi0.80.10 + MiniMax-M3。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2609.01481#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "对应同一模型–运行框架配置承担Planner/Developer/QA角色：规划角色改文档、Developer改项目代码、QA产出证据。不是另有一个被训练的HoH模型。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2609.01481#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "正在开发的软件项目与版本历史。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "实现时测试与独立评价分离。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "在现成 编程 task agent 外加开发者—独立 QA—规划交接协议；保存开发文档、执行证据和下一轮修订要求。对照 Vanilla 只有标准开发一遍，Vanilla Continuation 多续跑两次但不提供独立 QA 证据。",
"sources": [
{
"label": "附录 B.4–B.5、跨迭代证据协议",
"url": "https://arxiv.org/abs/2609.01481"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "外层流程协调已有 coding agent，每轮根据计划、质量检查证据和已有代码继续开发。保留上轮项目作为下一轮起点，支持多日修复与功能扩展；变化的主要是软件产物。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "同一软件项目的公开规格与上一轮QA证据驱动继续开发；不是跨数据集训练运行框架。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "固定Planner/Developer/QA循环，更新开发文档、保留项目代码；QA证据反馈下一轮，不提供私有评分细则（逐项规定要满足的要求及给分标准）。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "GameCraft-Bench45项目、FrontierSWE15项目、ProgramBench隐藏行为测试；另从空目录开发FPS Fusepoint70轮。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "最终评测基准评估与开发QA分开；项目本身反复修改。三轮提升是同项目产物进化，不是 harness/模型参数自改。",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "在现成 coding agent 之外安排持续开发，将缺陷修复与新增功能分开推进；主要累积改进的是软件项目产物。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-09",
"url": "https://arxiv.org/html/2609.01481v1",
"version": "2609.01481v1",
"scope": "原文定位初核：摘要、方法概述或选取的实验段落；不代表逐项配置均已核实。"
},
"focus": "在现成 coding harness 外组织多轮开发，产物是不断改进的软件；把 repair 与新增能力分开推进，并区分实现时测试和独立评价。",
"feedbackCases": [
{
"label": "持续开发：公开规格与 QA",
"data": "同一项目反复迭代；固定 规划角色 / Developer / QA 三角色；另从空目录开发 Fusepoint 70 轮。",
"scoring": "QA 根据公开产品规格检查代码、运行行为和已有证据，不获取私有评测细则。",
"visible": "上一轮 QA 证据、代码及开发文档，供下一轮计划和开发使用。",
"use": "持续改项目产物；这套公开 QA 与最后 benchmark 裁判分开。",
"sources": [
{
"label": "附录 A.2：公开输入与角色",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
}
],
"judgment": "QA agent 按公开产品规格审查代码与运行证据"
},
{
"label": "最终验收：GameCraft-Bench",
"data": "45 个游戏项目",
"scoring": "不能编译或运行记零；可运行游戏按核心机制、内容深度、功能视觉、艺术呈现的基准权重综合评分。",
"visible": "官方评价结果用于最后报告；开发循环只能使用公开规格与 QA 证据。",
"use": "检验长时间开发的最终产物，不能把隐藏验收结果当作每轮开发反馈。",
"sources": [
{
"label": "§4.1：各基准指标",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
}
],
"judgment": "程序检查可运行性；其余按游戏四维细则，本文未在该段展开裁判实现"
},
{
"label": "最终验收：FrontierSWE",
"data": "15 个项目",
"scoring": "任务专属官方 verifier（依据测试或评分规则检查任务结果的程序） 给 reward，汇总平均奖励；另报告与其他 12 种框架配置比较的 dominance score。",
"visible": "官方评价结果用于最后报告；开发循环只能使用公开规格与 QA 证据。",
"use": "检验长时间开发的最终产物，不能把隐藏验收结果当作每轮开发反馈。",
"sources": [
{
"label": "§4.1：各基准指标",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
}
],
"judgment": "FrontierSWE 官方任务 verifier，按各题规则验收"
},
{
"label": "最终验收：ProgramBench",
"data": "程序项目的隐藏行为测试",
"scoring": "运行隐藏测试检查最终软件的外部行为。",
"visible": "官方评价结果用于最后报告；开发循环只能使用公开规格与 QA 证据。",
"use": "检验长时间开发的最终产物，不能把隐藏验收结果当作每轮开发反馈。",
"sources": [
{
"label": "§4.1：各基准指标",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
}
],
"judgment": "ProgramBench 隐藏行为测试"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"learningCases": [
0
],
"testCases": [
1,
2,
3
],
"evolution": "同一软件项目的公开规格与上一轮QA证据驱动继续开发；不是跨数据集训练运行框架。",
"selection": "固定Planner/Developer/QA循环，更新开发文档、保留项目代码；QA证据反馈下一轮，不提供私有评分细则（逐项规定要满足的要求及给分标准）。",
"evaluation": "GameCraft-Bench45项目、FrontierSWE15项目、ProgramBench隐藏行为测试；另从空目录开发FPS Fusepoint70轮。",
"isolation": "最终评测基准评估与开发QA分开；项目本身反复修改。三轮提升是同项目产物进化，不是 harness/模型参数自改。",
"roles": {
"executor": {
"value": "Codex CLI0.142.5 + GPT-5.5 high；OpenCode1.14.30 + DeepSeek-V4-Pro；Pi0.80.10 + MiniMax-M3。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2609.01481#A2.SS1"
}
]
},
"modifier": {
"value": "对应同一模型–运行框架配置承担Planner/Developer/QA角色：规划角色改文档、Developer改项目代码、QA产出证据。不是另有一个被训练的HoH模型。",
"sources": [
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
},
{
"label": "附录B.6",
"url": "https://arxiv.org/html/2609.01481#A2.SS6"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px2"
},
{
"label": "附录B.1",
"url": "https://arxiv.org/html/2609.01481#A2.SS1"
}
]
},
"seed": {
"value": "在现成 编程 task agent 外加开发者—独立 QA—规划交接协议；保存开发文档、执行证据和下一轮修订要求。对照 Vanilla 只有标准开发一遍，Vanilla Continuation 多续跑两次但不提供独立 QA 证据。",
"sources": [
{
"label": "附录 B.4–B.5、跨迭代证据协议",
"url": "https://arxiv.org/abs/2609.01481"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"selection": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"evaluation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
],
"isolation": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2609.01481#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2609.01481#S5"
}
]
}
}
],
"sourceDate": "2026-09-09"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有代码 agent 通常仍依赖人拆任务、指导决策和处理失败。从需求持续开发完整软件时，长历史会导致遗忘约束，局部修复又可能破坏别处，失败决策还会不断积累；因此问题不只是让 agent 运行更久，而是如何长期保持连贯、可验证的开发进展。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.01481#S1"
}
]
},
{
"key": "position",
"value": "研究编码 agent 能否在无人持续指导的长时间开发中，把高层需求推进为完整可用的软件，并持续兼顾缺陷修复与功能增长。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01481"
}
]
},
{
"key": "conclusion",
"value": "在软件基准及多日开发中持续改进产物；进化对象主要是软件项目，而非底层运行框架自身。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2609.01481"
}
]
}
],
"fields": {
"object": "正在开发的软件项目与版本历史。"
}
},
"attributions": [
{
"tag": "org:shanghai-ai-lab",
"label": "Shanghai AI Laboratory",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.01481"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "rsi-exam",
"title": "RSI-Exam: Benchmarking Recursive Self-Improvement through Executable Research",
"date": "2026-08-26",
"dateBasis": "项目网站首发；数据与代码于 2026-08-28 发布",
"publicationType": "project-report",
"url": "https://rsi-exam.ai/blog.html",
"priority": "C",
"categories": [
"evaluation"
],
"tags": [
"B-RSI",
"HarnessCode",
"MemoryMechanism",
"Workflow",
"Weights",
"ExecutableVerifier",
"GoldLabel",
"LLMJudge",
"HeldOut",
"LongHorizon"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "评测研究代理在有限时间内，能否从可运行但较弱的起点出发，自主开展多轮实验，形成更有效、能泛化的方法或运行框架。",
"什么在变": "每项任务指定可修改的交付物：可能是数值算法、程序或驱动固定模型的运行框架。要区分研究者自身与被研究对象：数学后训练任务会训练另一个 Qwen3-1.7B-Base 学生模型，但不训练提出方案的研究者模型。法律任务则交付最终回答文件。固定部分包括任务接口、资源约束和评测规则；不能把全部 88 项一概称为框架自修改。",
"谁来改 / 谁执行": "公开报告比较九组研究者模型与代理工具：Opus 5、GLM 5.3、DeepSeek V4 Pro 配 Claude Code；GPT-5.6-sol、GPT-5.5 配 Codex；Kimi K3 配 Kimi CLI；Grok 4.6 配 Grok Build；Qwen3.8 Max 配 Qwen Coder；Gemini 3.7 Flash 配 Antigravity CLI。这些工具负责文件编辑、命令执行和上下文管理。研究者自行提假设、修改、测量并选择最终版本，统一提示给出研究流程；研究者模型自身不在本次任务中训练。",
"基础 harness": "**研究者的外层工具：** 使用现成编程代理，并统一提供“提假设—实现—测量—保存或撤回”的研究指令；Harbor 负责容器、执行预算与独立评分环境。报告运行未使用额外技能库或 MCP 外部工具连接服务。\n\n**被改进的起点：** 每题提供任务专属的可运行弱方法。例如记忆系统只抽取零散事实、按 BM25 词项匹配检索后调用一次模型；证明系统先生成证明，编译失败后至多修复一次；科学发现初始框架每步重新调用模型并重放历史，没有独立的持久状态管理。",
"Feedback": "开发反馈来自各任务自己的检查程序或评分服务：记忆问答对照参考答案算词元 F1；Lean 证明由编译器验收；科学发现报告行动进度、完成情况与解释性知识；数学训练只返回开发题总体正确率；法律问答返回满足了几项保密要求。修改者可根据这些分数及允许查看的运行记录继续实验；正式结果原则上只在提交后计算。",
"Evolution → Eval": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。",
"证据边界": "主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与代理框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。"
},
"protocol": "混合协议",
"protocolBasis": "主协议将开发与正式评分隔离；法律任务的最终题目可见，数学任务提交权重，须逐任务区分。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://rsi-exam.ai/blog.html",
"https://rsi-exam.ai/",
"https://github.com/aiming-lab/RSI-Exam"
],
"links": [
{
"label": "Code",
"url": "https://github.com/aiming-lab/RSI-Exam"
},
{
"label": "公开任务",
"url": "https://huggingface.co/datasets/RSI-Exam/RSI-Exam"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "评测研究代理在有限时间内，能否从可运行但较弱的起点出发，自主开展多轮实验，形成更有效、能泛化的方法或运行框架。",
"novelty": "给研究者一个能跑但较弱的起点，要求它经历多轮实验后交付可被重新执行的方法；用任务原有指标评价，再将起点标为 0、可用的强参考方法标为 0.6。版本日志和最终泛化结果一起保留，可以观察改进来自反复调同一方法，还是换了更有效的设计。具体任务也保留各自的模型训练或回答提交协议。",
"object": "任务指定的算法、程序或固定模型外围的框架。部分任务训练作为产物的学生模型，法律任务交付回答；提出改进方案的研究者模型自身不训练。",
"executor": "**研究过程：** 被测模型在 coding agent 工具中运行实验，模型与框架配对见“谁来改”。\n\n**候选方法的执行：** 按任务区分。记忆架构调用固定 gpt-4o-mini；科学发现框架调用固定 gpt-4o；Lean 证明流程调用固定 gpt-5.4-2026-03-05 并交给 Lean 4.9 检查。数学后训练任务由提交的 Qwen3-1.7B-Base 衍生模型答题；数值求解任务直接运行提交的程序。",
"modifier": "公开报告比较九组研究者模型与 agent 工具：Opus 5、GLM 5.3、DeepSeek V4 Pro 配 Claude Code；GPT-5.6-sol、GPT-5.5 配 Codex；Kimi K3 配 Kimi CLI；Grok 4.6 配 Grok Build；Qwen3.8 Max 配 Qwen Coder；Gemini 3.7 Flash 配 Antigravity CLI。这些工具负责文件编辑、命令执行和上下文管理。研究者自行提假设、修改、测量并选择最终版本，统一提示给出研究流程；研究者模型自身不在本次任务中训练。",
"roleContext": "公开报告比较九组研究者模型与代理工具：Opus 5、GLM 5.3、DeepSeek V4 Pro 配 Claude Code；GPT-5.6-sol、GPT-5.5 配 Codex；Kimi K3 配 Kimi CLI；Grok 4.6 配 Grok Build；Qwen3.8 Max 配 Qwen Coder；Gemini 3.7 Flash 配 Antigravity CLI。这些工具负责文件编辑、命令执行和上下文管理。研究者自行提假设、修改、测量并选择最终版本，统一提示给出研究流程；研究者模型自身不在本次任务中训练。",
"seed": "**研究者的外层工具：** 使用现成 coding agent，并统一提供“提假设—实现—测量—保存或撤回”的研究指令；Harbor 负责容器、执行预算与独立评分环境。报告运行未使用额外技能库或 MCP 外部工具连接服务。\n\n**被改进的起点：** 每题提供任务专属的可运行弱方法。例如记忆系统只抽取零散事实、按 BM25 词项匹配检索后调用一次模型；证明系统先生成证明，编译失败后至多修复一次；科学发现初始框架每步重新调用模型并重放历史，没有独立的持久状态管理。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "开发反馈来自各任务自己的检查程序或评分服务：记忆问答对照参考答案算词元 F1；Lean 证明由编译器验收；科学发现报告行动进度、完成情况与解释性知识；数学训练只返回开发题总体正确率；法律问答返回满足了几项保密要求。修改者可根据这些分数及允许查看的运行记录继续实验；正式结果原则上只在提交后计算。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "混合协议",
"note": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。"
}
],
"takeaway": "主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与代理框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://rsi-exam.ai/blog.html",
"https://rsi-exam.ai/",
"https://github.com/aiming-lab/RSI-Exam"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：没有供全部研究者共同训练的统一训练集；每次研究从一个任务包开始。可见材料由任务定义：记忆任务给 4 段多轮对话、812 对问答；科学发现给两类场景各 3 个种子；Lean 给 54 道定理；潮汐反演给 6 个流域案例。数学后训练另提供本地 train.jsonl（题目与整数答案）、固定 8B 教师及 Qwen3-1.7B-Base 学生，训练仅可用规定的本地资源；公开说明未点名训练语料的原始基准，也未列出教师型号。\n\n调试 / 选版本数据：主要用每题可见开发分区比较版本，研究者也可自行再划验证集。记忆任务允许按对话或问答子集检查；Lean 比较版本须覆盖全部 54 题；科学发现须覆盖六个场景。数学后训练的 dev.jsonl 只给题目，有限查询仅返回总体正确率。法律任务用 18 道开发问题、合计 90 次提交额度，返回满足要求的数量，隐藏要求正文。各项判分与使用方式在反馈行分别列出。\n\n最终测试数据：RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。\n\n数据隔离与证据边界：主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与 agent 框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。",
"cycle": "先运行起点，提出修改假设、编辑实现，再用可见数据测量。日志记录版本、父版本、改动、成绩及保留或撤回决定，所有版本都保存。哪些额外验证能检验泛化由研究者选择；科学发现任务另强制每个版本跑完同一六场景开发套件并撤回退化版本。预算结束后评价留下的最终交付物，未根据最终隐藏分数再选一次版本。一般任务上限 12 小时，数学后训练任务明确为 8 小时；其他计算限制按任务规定。",
"train": "没有供全部研究者共同训练的统一训练集；每次研究从一个任务包开始。可见材料由任务定义：记忆任务给 4 段多轮对话、812 对问答；科学发现给两类场景各 3 个种子；Lean 给 54 道定理；潮汐反演给 6 个流域案例。数学后训练另提供本地 train.jsonl（题目与整数答案）、固定 8B 教师及 Qwen3-1.7B-Base 学生，训练仅可用规定的本地资源；公开说明未点名训练语料的原始基准，也未列出教师型号。",
"debug": "主要用每题可见开发分区比较版本，研究者也可自行再划验证集。记忆任务允许按对话或问答子集检查；Lean 比较版本须覆盖全部 54 题；科学发现须覆盖六个场景。数学后训练的 dev.jsonl 只给题目，有限查询仅返回总体正确率。法律任务用 18 道开发问题、合计 90 次提交额度，返回满足要求的数量，隐藏要求正文。各项判分与使用方式在反馈行分别列出。",
"test": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。",
"isolation": "主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与 agent 框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "**研究者的外层工具：** 使用现成编程代理，并统一提供“提假设—实现—测量—保存或撤回”的研究指令；Harbor 负责容器、执行预算与独立评分环境。报告运行未使用额外技能库或 MCP 外部工具连接服务。\n\n**被改进的起点：** 每题提供任务专属的可运行弱方法。例如记忆系统只抽取零散事实、按 BM25 词项匹配检索后调用一次模型；证明系统先生成证明，编译失败后至多修复一次；科学发现初始框架每步重新调用模型并重放历史，没有独立的持久状态管理。",
"protocol": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。\n\n主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与代理框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。",
"sections": "官方报告 Benchmark / Experimental setup；六项任务的 What You Have / How It Is Judged",
"source": "https://rsi-exam.ai/blog.html",
"version": "RSI-Exam 0.1",
"date": "2026-09-11",
"scope": "依据项目官方报告、公开任务说明与固定提交版本核对数据、角色与初始系统。",
"sourceSha256": "ff1527ae4a15ff3b3ee09a9dfe715b50fc0447598b786b0651675e34edd1dbdd",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "**研究过程：** 被测模型在 coding agent 工具中运行实验，模型与框架配对见“谁来改”。\n\n**候选方法的执行：** 按任务区分。记忆架构调用固定 gpt-4o-mini；科学发现框架调用固定 gpt-4o；Lean 证明流程调用固定 gpt-5.4-2026-03-05 并交给 Lean 4.9 检查。数学后训练任务由提交的 Qwen3-1.7B-Base 衍生模型答题；数值求解任务直接运行提交的程序。",
"sources": [
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "任务内模型配置（固定版本仓库）",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/.env.local.example"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "公开报告比较九组研究者模型与 agent 工具：Opus 5、GLM 5.3、DeepSeek V4 Pro 配 Claude Code；GPT-5.6-sol、GPT-5.5 配 Codex；Kimi K3 配 Kimi CLI；Grok 4.6 配 Grok Build；Qwen3.8 Max 配 Qwen Coder；Gemini 3.7 Flash 配 Antigravity CLI。这些工具负责文件编辑、命令执行和上下文管理。研究者自行提假设、修改、测量并选择最终版本，统一提示给出研究流程；研究者模型自身不在本次任务中训练。",
"sources": [
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "官方报告 · 适用范围与限制",
"url": "https://rsi-exam.ai/blog.html#conclusion"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "任务指定的算法、程序或固定模型外围的框架。部分任务训练作为产物的学生模型，法律任务交付回答；提出改进方案的研究者模型自身不训练。",
"sources": [
{
"label": "官方报告 · 研究目标",
"url": "https://rsi-exam.ai/blog.html#introduction"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "开发反馈来自各任务自己的检查程序或评分服务：记忆问答对照参考答案算词元 F1；Lean 证明由编译器验收；科学发现报告行动进度、完成情况与解释性知识；数学训练只返回开发题总体正确率；法律问答返回满足了几项保密要求。修改者可根据这些分数及允许查看的运行记录继续实验；正式结果原则上只在提交后计算。",
"sources": [
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "**研究者的外层工具：** 使用现成 coding agent，并统一提供“提假设—实现—测量—保存或撤回”的研究指令；Harbor 负责容器、执行预算与独立评分环境。报告运行未使用额外技能库或 MCP 外部工具连接服务。\n\n**被改进的起点：** 每题提供任务专属的可运行弱方法。例如记忆系统只抽取零散事实、按 BM25 词项匹配检索后调用一次模型；证明系统先生成证明，编译失败后至多修复一次；科学发现初始框架每步重新调用模型并重放历史，没有独立的持久状态管理。",
"sources": [
{
"label": "官方报告 · 运行与访问限制",
"url": "https://rsi-exam.ai/blog.html#execution-records"
},
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先运行起点，提出修改假设、编辑实现，再用可见数据测量。日志记录版本、父版本、改动、成绩及保留或撤回决定，所有版本都保存。哪些额外验证能检验泛化由研究者选择；科学发现任务另强制每个版本跑完同一六场景开发套件并撤回退化版本。预算结束后评价留下的最终交付物，未根据最终隐藏分数再选一次版本。一般任务上限 12 小时，数学后训练任务明确为 8 小时；其他计算限制按任务规定。",
"sources": [
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "README · 时间预算",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/README.md#-quick-start"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有供全部研究者共同训练的统一训练集；每次研究从一个任务包开始。可见材料由任务定义：记忆任务给 4 段多轮对话、812 对问答；科学发现给两类场景各 3 个种子；Lean 给 54 道定理；潮汐反演给 6 个流域案例。数学后训练另提供本地 train.jsonl（题目与整数答案）、固定 8B 教师及 Qwen3-1.7B-Base 学生，训练仅可用规定的本地资源；公开说明未点名训练语料的原始基准，也未列出教师型号。",
"sources": [
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 潮汐反演：观测、验证测站与误差",
"url": "https://rsi-exam.ai/tasks/tidal_friction_inverse.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "主要用每题可见开发分区比较版本，研究者也可自行再划验证集。记忆任务允许按对话或问答子集检查；Lean 比较版本须覆盖全部 54 题；科学发现须覆盖六个场景。数学后训练的 dev.jsonl 只给题目，有限查询仅返回总体正确率。法律任务用 18 道开发问题、合计 90 次提交额度，返回满足要求的数量，隐藏要求正文。各项判分与使用方式在反馈行分别列出。",
"sources": [
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。",
"sources": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 潮汐反演：观测、验证测站与误差",
"url": "https://rsi-exam.ai/tasks/tidal_friction_inverse.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与 agent 框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。",
"sources": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 运行与访问限制",
"url": "https://rsi-exam.ai/blog.html#execution-records"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
},
{
"label": "官方报告 · 适用范围与限制",
"url": "https://rsi-exam.ai/blog.html#conclusion"
},
{
"label": "README · News：项目与数据发布时间",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/README.md#-news"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "给研究者一个能跑但较弱的起点，要求它经历多轮实验后交付可被重新执行的方法；用任务原有指标评价，再将起点标为 0、可用的强参考方法标为 0.6。版本日志和最终泛化结果一起保留，可以观察改进来自反复调同一方法，还是换了更有效的设计。具体任务也保留各自的模型训练或回答提交协议。",
"sources": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 三条研究轨迹",
"url": "https://rsi-exam.ai/blog.html#trajectory-analysis"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://rsi-exam.ai/blog.html",
"version": "RSI-Exam 0.1；公开仓库 0386e19fe41fa924c04a68702f20591ab612132d",
"scope": "官方技术报告、研究协议、模型配置与六项公开任务说明核对；未将53项未公开任务的逐项细节推断为事实。"
},
"focus": "评测研究代理在有限时间内，能否从可运行但较弱的起点出发，自主开展多轮实验，形成更有效、能泛化的方法或运行框架。",
"feedbackCases": [
{
"label": "主协议：领域指标与最终分数",
"data": "88 项任务各有开发材料和最终评测设置；35 个公开任务包和 53 个非公开任务包共同构成总榜。",
"scoring": "各任务按自身指标判分，起点归一化为 0，已标定的强参考方法可设为 0.6，有真实上界时设为 1；按任务声明插值，总榜等权平均。0.6 不是答对 60% 的题。",
"visible": "研究者读取开发测量和执行记录；主协议中的最终隐藏分数在研究结束后取得。下面列出六项已核对的公开任务，其中法律任务的最终题目可见。",
"use": "衡量从规定起点得到的改进及其泛化；不把开发最高分或搜索过程中所有版本的最好答案当最终成绩。",
"sources": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 结果与泛化分析",
"url": "https://rsi-exam.ai/blog.html#results"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"judgment": "各任务原生指标经冻结的归一化规则计算总分"
},
{
"label": "记忆架构：LoCoMo 多轮对话",
"data": "开发是 conversations_visible.json 中 4 段对话、812 对问答；最终换未见的同类对话，重新完成记忆建立和答题。",
"scoring": "selfcheck.py 将输出与简短参考答案算词元 F1，即兼顾答出多少参考内容与混入多少无关内容；最终检查器按同样的答案对照评价。执行模型固定 gpt-4o-mini。",
"visible": "本地返回总体及分类型 F1，允许选对话或问题子集快速调试；正式未见对话由评分端提供。",
"use": "用分数及问答表现改记忆抽取、存储、检索和答题流程；最终测试完整重跑这些环节，检查是否只适配了开发对话。",
"sources": [
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
}
],
"judgment": "程序计算回答与 LoCoMo 参考答案的词元 F1"
},
{
"label": "科学发现：DiscoveryWorld 两类模拟任务",
"data": "开发含太空疾病、组合化学两主题，各 3 个种子共 6 个世界；测试是同主题各 2 个新参数种子。每世界最多 240 个动作。",
"scoring": "selfcheck.py 返回程序化进度、是否成功完成、解释性知识准确度，并取三者等权平均。解释性知识依据输出中明确陈述的思考内容对照关键问题评分要求；做完动作不自动得到知识分。固定执行模型是 gpt-4o。",
"visible": "开发可见六个世界的分项、均值及错误日志；正式参数种子和评分表不允许读取。",
"use": "可用单场景查错，但每个正式候选版本都须重跑完整六场景，记录各项分数并撤回退化版本。新种子成绩只用于最终验收。",
"sources": [
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务内模型配置（固定版本仓库）",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/.env.local.example"
}
],
"judgment": "模拟器计算进度与完成；知识分对照解释评分要求，报告未在此处展开内部裁判"
},
{
"label": "Lean 4：证明工作流程",
"data": "开发提供固定 54 道定理、自然语言题意和标准形式化命题；最终使用未见定理。每题限制 3 次模型调用、4 次 Lean 检查、12,000 模型词元与 180 秒。",
"scoring": "固定 gpt-5.4-2026-03-05 提出证明；Lean 4.9/mathlib 编译器检查。最终由可信评分进程把证明与原命题拼接、重新编译，统计通过率，不使用模型评审，也不采信提交代码自己写的成功标记。",
"visible": "开发返回编译成败与诊断文本；该文本是查错材料，最终证明通过独立检查才算正确。比较版本须测全 54 题。",
"use": "改证明提出、报错摘要、失败修复及停止策略；未见定理只用于正式测试。",
"sources": [
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
}
],
"judgment": "Lean 编译器验证形式证明，可信进程重新编译"
},
{
"label": "数学后训练：1.7B 学生与 8B 教师",
"data": "本地 train.jsonl 给题目和整数答案，dev.jsonl 给无答案开发题；学生为 Qwen3-1.7B-Base，另提供固定 8B 教师。语料来源和教师具体型号未在任务页列出。最终整数答案题另行隔离。",
"scoring": "开发服务返回总正确率，答案必须精确匹配整数；最终按固定采样配置对照保密答案。训练使用 LoRA 附加适配参数，最后须合并成可独立加载的学生模型。",
"visible": "开发查分额度有限，只返回总体正确率，不返回逐题答案或错误；最终不公开题文、答案、预测或逐题反馈。",
"use": "研究者据此选训练材料、蒸馏或微调方案及候选模型。任务预算 8 小时、8 CPU、两张 48GB GPU；提交后评分端加载权重，不再训练，不执行提交的研究代码。",
"sources": [
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
}
],
"judgment": "规则抽取并精确匹配标准整数答案"
},
{
"label": "潮汐反演：数值方法",
"data": "开发包含 6 个流域案例，给地形、边界潮汐、稀疏观测站及单独验证测站；最终用新匿名流域与隐藏测站。",
"scoring": "开发检查器报告验证测站的潮位均方根误差（RMSE，越低越好）；最终用可信物理模拟器根据提交的摩擦场计算隐藏测站误差。",
"visible": "开发可见测量误差；求解函数只准读取传入案例的观测，不可读取验证测站答案。最终不给验证答案输入。",
"use": "修改空间摩擦场估计与数值优化方法，以可见验证误差选择版本；最后由新流域和测站检查泛化。",
"sources": [
{
"label": "任务说明 · 潮汐反演：观测、验证测站与误差",
"url": "https://rsi-exam.ai/tasks/tidal_friction_inverse.html"
}
],
"judgment": "可信物理模拟器计算隐藏测站的数值误差"
},
{
"label": "法律多事项：保密要求评分，最终题目可见",
"data": "6 个事项、1,000 多份文档；18 道开发问题与 18 道最终问题各含每事项 3 题。最终问题文本从开始就在 heldout_prompts.json 中。",
"scoring": "固定 gemini-3-flash-preview 评审回答是否明确满足各项保密要求；只有满足一题的所有要求，该题才算通过。开发返回如 4/6 的满足数量，它不是最终按比例给分。",
"visible": "开发最多提交 90 次，只得满足数量，看不到要求正文或具体漏项；最终 18 题只提交一次、不反馈、不重试。",
"use": "从开发得分调整回答的具体程度、引用与覆盖范围，再提交 answers.json。这个任务评价回答迁移，不能描述为看不到最终题目或一律重跑代码框架。",
"sources": [
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
},
{
"label": "任务内模型配置（固定版本仓库）",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/.env.local.example"
}
],
"judgment": "gemini-3-flash-preview 按保密逐项要求评审，全部满足才通过"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4,
5,
6
],
"evolution": "没有供全部研究者共同训练的统一训练集；每次研究从一个任务包开始。可见材料由任务定义：记忆任务给 4 段多轮对话、812 对问答；科学发现给两类场景各 3 个种子；Lean 给 54 道定理；潮汐反演给 6 个流域案例。数学后训练另提供本地 train.jsonl（题目与整数答案）、固定 8B 教师及 Qwen3-1.7B-Base 学生，训练仅可用规定的本地资源；公开说明未点名训练语料的原始基准，也未列出教师型号。",
"selection": "主要用每题可见开发分区比较版本，研究者也可自行再划验证集。记忆任务允许按对话或问答子集检查；Lean 比较版本须覆盖全部 54 题；科学发现须覆盖六个场景。数学后训练的 dev.jsonl 只给题目，有限查询仅返回总体正确率。法律任务用 18 道开发问题、合计 90 次提交额度，返回满足要求的数量，隐藏要求正文。各项判分与使用方式在反馈行分别列出。",
"evaluation": "RSI-Exam 0.1 含 88 项研究任务、6 个领域，35 项公开、53 项未公开；35/53 指任务包的发布状态，不是每项任务内的训练/测试划分。主协议在独立环境中重跑最终交付物：记忆换未见对话，Lean 换未见定理，科学发现换同两主题各 2 个新参数种子，潮汐反演换流域和测站，数学学生在分离的整数答案题上测试。法律任务另有 18 道最终问题，其题目开局可见、评分要求保密，最终只交一次。公开主报告每个模型—任务组合仅运行一次；全部 88 项逐任务配置不能仅从 35 个公开任务外推。",
"isolation": "主协议将开发环境与评分环境分离，只传规定的交付物，保护测试目标和评分资产，并限制网络访问。两点例外必须保留：数学任务交付合并后的模型权重，评分端不执行研究者代码；法律任务的最终问题本身可见，隐藏的是评分标准且不返回最终反馈。因此“隐藏测试”不统一意味着题目不可见。研究者使用不同模型与 agent 框架，结果是二者组合的表现；单次运行不能估计同一组合重跑的波动。本站依据 0.1 官方网页报告、任务说明和公开代码收录，日期采用项目首发 2026-08-26，数据与代码于 08-28 公开。",
"roles": {
"executor": {
"value": "**研究过程：** 被测模型在 coding agent 工具中运行实验，模型与框架配对见“谁来改”。\n\n**候选方法的执行：** 按任务区分。记忆架构调用固定 gpt-4o-mini；科学发现框架调用固定 gpt-4o；Lean 证明流程调用固定 gpt-5.4-2026-03-05 并交给 Lean 4.9 检查。数学后训练任务由提交的 Qwen3-1.7B-Base 衍生模型答题；数值求解任务直接运行提交的程序。",
"sources": [
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "任务内模型配置（固定版本仓库）",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/.env.local.example"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
}
]
},
"modifier": {
"value": "公开报告比较九组研究者模型与 agent 工具：Opus 5、GLM 5.3、DeepSeek V4 Pro 配 Claude Code；GPT-5.6-sol、GPT-5.5 配 Codex；Kimi K3 配 Kimi CLI；Grok 4.6 配 Grok Build；Qwen3.8 Max 配 Qwen Coder；Gemini 3.7 Flash 配 Antigravity CLI。这些工具负责文件编辑、命令执行和上下文管理。研究者自行提假设、修改、测量并选择最终版本，统一提示给出研究流程；研究者模型自身不在本次任务中训练。",
"sources": [
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "官方报告 · 适用范围与限制",
"url": "https://rsi-exam.ai/blog.html#conclusion"
}
]
},
"seed": {
"value": "**研究者的外层工具：** 使用现成 coding agent，并统一提供“提假设—实现—测量—保存或撤回”的研究指令；Harbor 负责容器、执行预算与独立评分环境。报告运行未使用额外技能库或 MCP 外部工具连接服务。\n\n**被改进的起点：** 每题提供任务专属的可运行弱方法。例如记忆系统只抽取零散事实、按 BM25 词项匹配检索后调用一次模型；证明系统先生成证明，编译失败后至多修复一次；科学发现初始框架每步重新调用模型并重放历史，没有独立的持久状态管理。",
"sources": [
{
"label": "官方报告 · 运行与访问限制",
"url": "https://rsi-exam.ai/blog.html#execution-records"
},
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
}
]
}
},
"sources": {
"evolution": [
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 潮汐反演：观测、验证测站与误差",
"url": "https://rsi-exam.ai/tasks/tidal_friction_inverse.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
}
],
"selection": [
{
"label": "统一研究指令：实验、保存版本与回退",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/infra/prompts/autoresearch.j2"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"evaluation": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "任务说明 · 记忆架构：数据、固定模型、判分",
"url": "https://rsi-exam.ai/tasks/locomo_longterm_memory.html"
},
{
"label": "任务说明 · Lean 证明：开发题、模型与编译检查",
"url": "https://rsi-exam.ai/tasks/lean_formal_proof_workflow_design.html"
},
{
"label": "任务说明 · 科学发现：场景、预算、三项指标",
"url": "https://rsi-exam.ai/tasks/discoveryworld_agent_harness_low2.html"
},
{
"label": "任务说明 · 潮汐反演：观测、验证测站与误差",
"url": "https://rsi-exam.ai/tasks/tidal_friction_inverse.html"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
}
],
"isolation": [
{
"label": "官方报告 · 评测流程、数据隔离与分数标定",
"url": "https://rsi-exam.ai/blog.html#benchmark"
},
{
"label": "官方报告 · 运行与访问限制",
"url": "https://rsi-exam.ai/blog.html#execution-records"
},
{
"label": "任务说明 · 数学后训练：数据、学生模型、答案匹配",
"url": "https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html"
},
{
"label": "任务说明 · 法律任务：可见最终问题与保密评分标准",
"url": "https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html"
},
{
"label": "官方报告 · 适用范围与限制",
"url": "https://rsi-exam.ai/blog.html#conclusion"
},
{
"label": "README · News：项目与数据发布时间",
"url": "https://github.com/aiming-lab/RSI-Exam/blob/0386e19fe41fa924c04a68702f20591ab612132d/README.md#-news"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "官方报告引言以一个尚需测量的问题作为出发点：agent 能否经过长时间实验，改进可执行方法或冻结模型的运行框架，并让提交的代码在隐藏数据上重新运行后仍取得进步。报告在这里说明的是所需的评测证据，没有逐条批评某项已有方法。",
"sources": [
{
"label": "官方报告 · Introduction / What RSI-Exam measures",
"url": "https://rsi-exam.ai/blog.html#introduction"
}
]
},
{
"key": "position",
"value": "评测研究 agent 在有限时间内，能否从可运行但较弱的起点出发，自主开展多轮实验，形成更有效、能泛化的方法或运行框架。",
"sources": [
{
"label": "官方报告 · 研究目标",
"url": "https://rsi-exam.ai/blog.html#introduction"
}
]
},
{
"key": "conclusion",
"value": "提供 88 项、6 领域的研究评测及版本轨迹。在 37 项可比任务上，九组系统各自的平均隐藏分数都低于平均开发分数；持续实验和开发涨分不足以保证最终收益。每组在每题仅测一次，不能据此确定重跑稳定性。",
"sources": [
{
"label": "官方报告 · 结果与泛化分析",
"url": "https://rsi-exam.ai/blog.html#results"
},
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
},
{
"label": "官方报告 · 适用范围与限制",
"url": "https://rsi-exam.ai/blog.html#conclusion"
}
]
}
],
"fields": {
"object": "任务指定的算法、程序或固定模型外围的框架。部分任务训练作为产物的学生模型，法律任务交付回答；提出改进方案的研究者模型自身不训练。",
"executor": "候选方法各自执行：记忆系统用 gpt-4o-mini，科学发现用 gpt-4o，证明流程用 gpt-5.4-2026-03-05 和 Lean 编译器；数学任务由训练后的 Qwen3-1.7B 学生答题，数值任务直接运行程序。",
"modifier": "九组研究者模型与 coding agent 工具各自提出修改，例如 Opus 5 + Claude Code、GPT-5.6-sol + Codex。它们根据实验结果改代码或训练方案；完整配对见表格。",
"verdict": "各任务提供自己的检查：答案匹配、证明编译、仿真误差或按保密要求评分。开发结果用于改进与选版本，正式结果在最终提交后评价；下表分别解释六项公开任务的反馈。",
"seed": "研究者使用现成编程工具和统一研究指令；每题已有可运行弱方法。例如记忆起点只按关键词检索，证明起点只允许一次报错修复。Harbor（组织隔离容器与评测的工具）提供运行环境。"
}
},
"attributions": [],
"contentType": "report",
"researchProblems": [
"evaluation",
"harness",
"improver"
]
},
{
"id": "2609.00196",
"title": "WHALE: A Simple Recipe for Joint Harness-Weight Optimization",
"date": "2026-08-31",
"url": "https://arxiv.org/abs/2609.00196",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Weights",
"JointEvolution",
"LLMJudge",
"GoldLabel",
"ExecutableVerifier",
"SeparateEvolver",
"Archive",
"CoEvolution",
"org:stanford",
"org:krafton",
"person:chelsea-finn"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究模型参数与可执行框架是否会互相限制，以及怎样安排交替更新才能更有效地释放两者能力。",
"什么在变": "共同变化的是执行模型参数，以及调用它的可执行框架代码。框架可改提示、工具输入输出整理、调用后反馈、停止条件与轮数；还可调整检索查询、Python 代码提取或棋盘呈现。判分器、答案、环境转移规则和各阶段数据固定。",
"谁来改 / 谁执行": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"基础 harness": "三个领域各有简单起点。问答只原样转发查询，每次返回截断到 200 词元的一段文档，最多 2 轮；数学没有系统提示，提取第一个 Python 代码块，回传原始输出/报错，最多 2 轮；棋题展示棋盘及完整历史，格式错误和非法着法各重试 1 次，最多 9 轮。搜索空间允许扩展这些控制，但棋题框架不能替模型搜索或代选着法。",
"Feedback": "问答：GPT-5.4-mini 将提取出的答案与参考答案比对；数学：程序提取最终方框答案并与标准答案匹配；棋题：python-chess 检查合法性，并要求走完题库参考着法。二元正确标记同时用于筛训练轨迹、评候选框架和统计测试成绩；Python 报错和棋盘反馈另供执行、诊断使用。",
"Evolution → Eval": "问答参数训练：HotpotQA 14,801 题 + Natural Questions 4,145 题；框架搜索：同两来源 200 + 56 题。数学参数训练：DAPO-Math-17K 17,917 题；框架搜索从中抽 256 题。棋题来自 Lichess，参数训练 16,384 题、框架搜索 256 题；两者与测试互不重叠。SearchQA 在本文是搜索问答实验的名称，不是另一个统一数据集。\n\n问答共 700 题：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题；数学测 AIME 2024/2025；棋题测另外 256 道 Lichess 题。每题采样 8 次，报告 8 次的平均正确率，再比较各运行过程中最好的测试点；不是“8 次任一次答对”指标。",
"证据边界": "候选生成与阶段切换使用训练信号，但算法明确返回测试正确率最高的模型—框架组合，图表也是搜索期间最好测试成绩。因此有分开的测试题，却不是完全不参与最终版本选择的封闭测试。棋题明确三分区互斥；数学框架搜索集与参数训练集重叠。计算效率图按执行轨迹计数，不含修改者计算。"
},
"protocol": "混合协议",
"protocolBasis": "候选生成与阶段切换使用训练信号，但算法明确返回测试正确率最高的模型—框架组合，图表也是搜索期间最好测试成绩。因此有分开的测试题，却不是完全不参与最终版本选择的封闭测试。棋题明确三分区互斥；数学框架搜索集与参数训练集重叠。计算效率图按执行轨迹计数，不含修改者计算。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2609.00196"
],
"links": [
{
"label": "Code",
"url": "https://github.com/krafton-ai/WHALE"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "研究模型参数与可执行框架是否会互相限制，以及怎样安排交替更新才能更有效地释放两者能力。",
"novelty": "把“交替改参数与改可执行框架”的节奏单独拿出来研究：一次长训练再长搜索，不如多次小步交替；某些数学能力必须先训练，框架搜索才开始有效。进一步用训练信号停滞自动决定何时切换，不必为每个领域手动设定阶段预算。",
"object": "Qwen 执行模型的参数，以及提示、工具结果处理、轮数和停止规则等框架代码，交替更新。",
"executor": "搜索问答和数学由 Qwen3.5-2B 执行，棋题由 Qwen3.5-4B 执行。参数阶段更新这些模型，下一阶段由更新后的模型配合候选框架做题；搜索工具、Python 运行环境与棋盘规则保持固定。",
"modifier": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"roleContext": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"seed": "三个领域各有简单起点。问答只原样转发查询，每次返回截断到 200 词元的一段文档，最多 2 轮；数学没有系统提示，提取第一个 Python 代码块，回传原始输出/报错，最多 2 轮；棋题展示棋盘及完整历史，格式错误和非法着法各重试 1 次，最多 9 轮。搜索空间允许扩展这些控制，但棋题框架不能替模型搜索或代选着法。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "问答：GPT-5.4-mini 将提取出的答案与参考答案比对；数学：程序提取最终方框答案并与标准答案匹配；棋题：python-chess 检查合法性，并要求走完题库参考着法。二元正确标记同时用于筛训练轨迹、评候选框架和统计测试成绩；Python 报错和棋盘反馈另供执行、诊断使用。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "混合协议",
"note": "问答参数训练：HotpotQA 14,801 题 + Natural Questions 4,145 题；框架搜索：同两来源 200 + 56 题。数学参数训练：DAPO-Math-17K 17,917 题；框架搜索从中抽 256 题。棋题来自 Lichess，参数训练 16,384 题、框架搜索 256 题；两者与测试互不重叠。SearchQA 在本文是搜索问答实验的名称，不是另一个统一数据集。\n\n问答共 700 题：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题；数学测 AIME 2024/2025；棋题测另外 256 道 Lichess 题。每题采样 8 次，报告 8 次的平均正确率，再比较各运行过程中最好的测试点；不是“8 次任一次答对”指标。"
}
],
"takeaway": "候选生成与阶段切换使用训练信号，但算法明确返回测试正确率最高的模型—框架组合，图表也是搜索期间最好测试成绩。因此有分开的测试题，却不是完全不参与最终版本选择的封闭测试。棋题明确三分区互斥；数学框架搜索集与参数训练集重叠。计算效率图按执行轨迹计数，不含修改者计算。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2609.00196"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：问答：HotpotQA 14,801 + Natural Questions 4,145 题训练参数，另用同来源 200 + 56 题搜索框架。数学：DAPO-Math-17K 17,917 题训练，其中 256 题搜索框架。棋题：Lichess 16,384 题训练，另 256 题搜索框架。\n\n调试 / 选版本数据：三个任务均用各自的 256 道框架搜索题比较候选，提供逐题结果和轨迹；按训练奖励切换阶段，不另设验证集。\n\n最终测试数据：问答：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题。数学：AIME 2024／2025。棋题：另 256 道 Lichess 题。均每题采样 8 次、取平均正确率。\n\n数据隔离与证据边界：三项均按最高测试分选择最终模型与框架，测试参与选版本。数学的框架搜索题属于参数训练集；棋题的训练、搜索、测试题互斥。",
"cycle": "先固定框架，用当前模型产生正确轨迹并微调；再固定新模型，让 Claude 搜索更适配的框架，按框架训练集分数保留最好候选。固定方案每轮约 0.6 个训练轮次、6 次框架搜索；自适应方案在训练信号连续不涨后切换。算法最终返回规则另使用最好测试成绩，需与阶段内训练分数选框架区分。",
"train": "问答：HotpotQA 14,801 + Natural Questions 4,145 题训练参数，另用同来源 200 + 56 题搜索框架。数学：DAPO-Math-17K 17,917 题训练，其中 256 题搜索框架。棋题：Lichess 16,384 题训练，另 256 题搜索框架。",
"debug": "三个任务均用各自的 256 道框架搜索题比较候选，提供逐题结果和轨迹；按训练奖励切换阶段，不另设验证集。",
"test": "问答：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题。数学：AIME 2024／2025。棋题：另 256 道 Lichess 题。均每题采样 8 次、取平均正确率。",
"isolation": "三项均按最高测试分选择最终模型与框架，测试参与选版本。数学的框架搜索题属于参数训练集；棋题的训练、搜索、测试题互斥。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "三个领域各有简单起点。问答只原样转发查询，每次返回截断到 200 词元的一段文档，最多 2 轮；数学没有系统提示，提取第一个 Python 代码块，回传原始输出/报错，最多 2 轮；棋题展示棋盘及完整历史，格式错误和非法着法各重试 1 次，最多 9 轮。搜索空间允许扩展这些控制，但棋题框架不能替模型搜索或代选着法。",
"protocol": "问答参数训练：HotpotQA 14,801 题 + Natural Questions 4,145 题；框架搜索：同两来源 200 + 56 题。数学参数训练：DAPO-Math-17K 17,917 题；框架搜索从中抽 256 题。棋题来自 Lichess，参数训练 16,384 题、框架搜索 256 题；两者与测试互不重叠。SearchQA 在本文是搜索问答实验的名称，不是另一个统一数据集。\n\n每个候选在对应 256 道框架搜索题上各运行一次，汇总正确率进入历史档案，逐题结果和完整轨迹供修改者检查。自适应切换只看训练奖励及档案最好训练分数，没有另设验证集。数学的这 256 题属于参数训练集的子集。\n\n问答共 700 题：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题；数学测 AIME 2024/2025；棋题测另外 256 道 Lichess 题。每题采样 8 次，报告 8 次的平均正确率，再比较各运行过程中最好的测试点；不是“8 次任一次答对”指标。\n\n候选生成与阶段切换使用训练信号，但算法明确返回测试正确率最高的模型—框架组合，图表也是搜索期间最好测试成绩。因此有分开的测试题，却不是完全不参与最终版本选择的封闭测试。棋题明确三分区互斥；数学框架搜索集与参数训练集重叠。计算效率图按执行轨迹计数，不含修改者计算。",
"sections": "§5.2：执行模型与评测；§5.1：三个领域的数据；附录B.2：各领域初始框架；§3–4：参数更新与框架搜索；§6.2：交替节奏及自适应切换；§5.3、图2：最好测试成绩；§4：交替流程与返回规则；§6.1：轨迹成本口径",
"source": "https://arxiv.org/abs/2609.00196",
"version": "2609.00196v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "7ad50f9cfe559aa03ff7303f3194a80eb339107c3fcffef16979279968fd54f4",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "搜索问答和数学由 Qwen3.5-2B 执行，棋题由 Qwen3.5-4B 执行。参数阶段更新这些模型，下一阶段由更新后的模型配合候选框架做题；搜索工具、Python 运行环境与棋盘规则保持固定。",
"sources": [
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"sources": [
{
"label": "附录B、表1：修改模型和预算",
"url": "https://arxiv.org/html/2609.00196#A2"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "Qwen 执行模型的参数，以及提示、工具结果处理、轮数和停止规则等框架代码，交替更新。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "问答：GPT-5.4-mini 将提取出的答案与参考答案比对；数学：程序提取最终方框答案并与标准答案匹配；棋题：python-chess 检查合法性，并要求走完题库参考着法。二元正确标记同时用于筛训练轨迹、评候选框架和统计测试成绩；Python 报错和棋盘反馈另供执行、诊断使用。",
"sources": [
{
"label": "附录B.3：三个固定判分器",
"url": "https://arxiv.org/html/2609.00196#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "三个领域各有简单起点。问答只原样转发查询，每次返回截断到 200 词元的一段文档，最多 2 轮；数学没有系统提示，提取第一个 Python 代码块，回传原始输出/报错，最多 2 轮；棋题展示棋盘及完整历史，格式错误和非法着法各重试 1 次，最多 9 轮。搜索空间允许扩展这些控制，但棋题框架不能替模型搜索或代选着法。",
"sources": [
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "先固定框架，用当前模型产生正确轨迹并微调；再固定新模型，让 Claude 搜索更适配的框架，按框架训练集分数保留最好候选。固定方案每轮约 0.6 个训练轮次、6 次框架搜索；自适应方案在训练信号连续不涨后切换。算法最终返回规则另使用最好测试成绩，需与阶段内训练分数选框架区分。",
"sources": [
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
},
{
"label": "附录B、表1：修改模型和预算",
"url": "https://arxiv.org/html/2609.00196#A2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "问答：HotpotQA 14,801 + Natural Questions 4,145 题训练参数，另用同来源 200 + 56 题搜索框架。数学：DAPO-Math-17K 17,917 题训练，其中 256 题搜索框架。棋题：Lichess 16,384 题训练，另 256 题搜索框架。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "三个任务均用各自的 256 道框架搜索题比较候选，提供逐题结果和轨迹；按训练奖励切换阶段，不另设验证集。",
"sources": [
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "问答：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA 各 100 题。数学：AIME 2024／2025。棋题：另 256 道 Lichess 题。均每题采样 8 次、取平均正确率。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "三项均按最高测试分选择最终模型与框架，测试参与选版本。数学的框架搜索题属于参数训练集；棋题的训练、搜索、测试题互斥。",
"sources": [
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.1：轨迹成本口径",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把“交替改参数与改可执行框架”的节奏单独拿出来研究：一次长训练再长搜索，不如多次小步交替；某些数学能力必须先训练，框架搜索才开始有效。进一步用训练信号停滞自动决定何时切换，不必为每个领域手动设定阶段预算。",
"sources": [
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
},
{
"label": "§6.1：不同领域的瓶颈",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2609.00196",
"version": "2609.00196v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究模型参数与可执行框架是否会互相限制，以及怎样安排交替更新才能更有效地释放两者能力。",
"feedbackCases": [
{
"label": "搜索问答：训练、框架搜索与七组测试",
"data": "参数训练为 HotpotQA/Natural Questions 共18,946题；框架搜索256题；测试七个基准各100题，具体名称见测试行。",
"scoring": "先提取 `` 标记内的答案，格式失败直接0分；否则 GPT-5.4-mini 按问题和参考答案判断是否正确，固定温度0。",
"visible": "修改者可查框架训练题的逐题0/1结果、汇总分和轨迹；检索返回 Wikipedia 2018 的文档内容。",
"use": "正确轨迹用于微调，256题分数用于选框架；测试用相同判分器，但论文最终展示各运行最高测试点。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
},
{
"label": "附录B.3：三个固定判分器",
"url": "https://arxiv.org/html/2609.00196#A2.SS3"
},
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
}
],
"judgment": "GPT-5.4-mini 对照问题、参考答案与回答判语义正确性；格式先由规则检查"
},
{
"label": "数学：DAPO 训练，AIME 测试",
"data": "参数训练 DAPO-Math-17K 17,917题；其中256题用于框架搜索；测试 AIME 2024和2025。",
"scoring": "按 DAPO 规则提取最终方框答案并匹配标准答案；提取失败或答案不同为0分，正确为1分。Python 输出/异常不是答案正确性的最终裁决。",
"visible": "框架修改者看搜索题的答案成败和模型/Python 轨迹；执行模型能看到自己的计算输出与报错。",
"use": "只将答对轨迹用于微调；用256题正确率选框架和观察搜索是否停滞；AIME成绩用于论文最好测试点比较。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
},
{
"label": "附录B.3：三个固定判分器",
"url": "https://arxiv.org/html/2609.00196#A2.SS3"
},
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
}
],
"judgment": "DAPO 规则抽取方框答案，与标准答案匹配"
},
{
"label": "棋题：Lichess 参考着法序列",
"data": "Lichess 三个互斥集合：16,384题训练、256题框架搜索、256题测试。",
"scoring": "python-chess 判断合法着法，再对照题库参考着法；全部参考序列完成才1分。合法但不是参考着法也判错；这不是评估任意同样优秀的棋路。",
"visible": "正确非终局着法后环境返回固定对手应对；非法或格式错误按框架预算重试。修改者另看逐题判分与轨迹。",
"use": "棋盘、答案和转移规则固定，只改模型及呈现/解析/重试等外围代码；同一二元检查用于三阶段。",
"sources": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
},
{
"label": "附录B.3：三个固定判分器",
"url": "https://arxiv.org/html/2609.00196#A2.SS3"
}
],
"judgment": "python-chess 检查合法性，再用规则逐步匹配参考着法"
}
],
"experiments": [
{
"label": "搜索问答（SearchQA 是实验名称）",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "参数训练：HotpotQA 14,801 题 + Natural Questions 4,145 题；框架搜索：同两来源 200 + 56 题。正确轨迹用于微调，搜索题反馈用于改框架。",
"selection": "对应 256 道框架搜索题的成绩与轨迹筛选候选；没有另设验证集。",
"evaluation": "七个基准各 100 题：2WikiMultiHopQA、Bamboogle、HotpotQA、MuSiQue、Natural Questions、PopQA、TriviaQA。每题八次采样取平均正确率。",
"isolation": "阶段切换用训练信号，但最终展示搜索过程中最高测试点，测试成绩参与最终版本选择。",
"roles": {
"executor": {
"value": "Qwen3.5-2B；参数阶段用正确轨迹训练，下一框架阶段使用更新后的候选模型。",
"sources": [
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
]
},
"modifier": {
"value": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"sources": [
{
"label": "附录B、表1：修改模型和预算",
"url": "https://arxiv.org/html/2609.00196#A2"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
}
]
},
"seed": {
"value": "问答起点只原样转发搜索查询，每次返回截断到 200 词元的一段文档，最多两轮；搜索工具固定。",
"sources": [
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
],
"selection": [
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
],
"evaluation": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
}
],
"isolation": [
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.1：轨迹成本口径",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
]
}
},
{
"label": "数学",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "DAPO-Math-17K 17,917 题用于参数训练，其中 256 题也用于框架搜索。",
"selection": "用这 256 题的正确率和轨迹选框架、判断是否切换阶段。",
"evaluation": "AIME 2024、AIME 2025；每题八次采样取平均正确率。",
"isolation": "框架搜索集属于参数训练集；最终选择最高测试点，因此不是完全封闭的最终测试。",
"roles": {
"executor": {
"value": "Qwen3.5-2B；参数阶段用正确轨迹训练，下一框架阶段使用更新后的候选模型。",
"sources": [
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
]
},
"modifier": {
"value": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"sources": [
{
"label": "附录B、表1：修改模型和预算",
"url": "https://arxiv.org/html/2609.00196#A2"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
}
]
},
"seed": {
"value": "数学起点没有系统提示，提取第一个 Python 代码块运行，回传原始输出／报错，最多两轮；Python 环境固定。",
"sources": [
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
],
"selection": [
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
],
"evaluation": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
}
],
"isolation": [
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.1：轨迹成本口径",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
]
}
},
{
"label": "棋题",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "Lichess 16,384 题训练参数；另 256 题搜索框架。",
"selection": "使用框架搜索的 256 题比较候选；无额外验证集。",
"evaluation": "另外 256 道 Lichess 棋题；每题八次采样取平均正确率。",
"isolation": "三份题互不重叠，但最终报告仍按搜索期间最佳测试点选择。",
"roles": {
"executor": {
"value": "Qwen3.5-4B；参数阶段用正确轨迹训练，下一框架阶段使用更新后的候选模型。",
"sources": [
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
]
},
"modifier": {
"value": "Claude Opus 4.7 通过 Meta-Harness 搜索器读历史框架代码、逐题成败和轨迹，每轮提出 3 个候选框架。参数更新由训练程序完成：当前执行模型每题采样 8 条轨迹，仅留下判对的轨迹做监督微调。修改框架的 Claude 本身不在这个循环中训练。",
"sources": [
{
"label": "附录B、表1：修改模型和预算",
"url": "https://arxiv.org/html/2609.00196#A2"
},
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
}
]
},
"seed": {
"value": "棋题起点展示棋盘与完整历史；格式错误、非法着法各重试一次，最多九轮。棋盘规则固定，框架不得替模型搜索或代选着法。",
"sources": [
{
"label": "附录B.2：各领域初始框架",
"url": "https://arxiv.org/html/2609.00196#A2.SS2"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
}
],
"selection": [
{
"label": "§3–4：参数更新与框架搜索",
"url": "https://arxiv.org/html/2609.00196#S3"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
],
"evaluation": [
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.2：执行模型与评测",
"url": "https://arxiv.org/html/2609.00196#S5.SS2"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
}
],
"isolation": [
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.1：轨迹成本口径",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型与运行框架会互相限制：框架没取回证据，强模型也无法使用；模型不会整合证据，改检索也没用。已有联合优化多只改文字提示，完整代码框架与训练的更新节奏又不同，同时修改会难以归因，交替太久则可能只适配旧搭档；因此需要研究如何协调两者。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.00196#S1"
}
]
},
{
"key": "position",
"value": "研究模型参数与可执行框架是否会互相限制，以及怎样安排交替更新才能更有效地释放两者能力。",
"sources": [
{
"label": "§1：问题与贡献",
"url": "https://arxiv.org/html/2609.00196#S1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
]
},
{
"key": "conclusion",
"value": "在问答、数学和棋题中，小步交替超过单独更新；自适应切换在问答优于手调方案，在数学未超过最优手调。结果报告搜索过程最好测试点，不能当作冻结最终版本后的独立测试结论。",
"sources": [
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
},
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
}
]
}
],
"fields": {
"object": "Qwen 执行模型的参数，以及提示、工具结果处理、轮数和停止规则等框架代码，交替更新。",
"executor": "问答、数学：Qwen3.5-2B；棋题：Qwen3.5-4B。下一阶段使用训练后的候选模型。",
"modifier": "Claude Opus 4.7 读代码、轨迹和逐题结果改框架；训练程序用当前 Qwen 自己生成并判对的轨迹更新其参数。",
"verdict": "问答由 GPT-5.4-mini 对照参考答案；数学按标准答案匹配；棋题按合法性和参考着法检查。正确标记用于训练筛选和框架比较。",
"seed": "问答：每次返回一段最多 200 词元的文档，最多 2 轮；数学：提取并运行 Python，返回输出或报错，最多 2 轮；棋题：展示棋盘和完整历史，格式错误、非法着法各重试 1 次，最多 9 轮。"
}
},
"attributions": [
{
"tag": "org:stanford",
"label": "Stanford University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.00196"
}
]
},
{
"tag": "org:krafton",
"label": "KRAFTON",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.00196"
}
]
},
{
"tag": "person:chelsea-finn",
"label": "Chelsea Finn",
"kind": "scholar",
"sources": [
{
"label": "arXiv 作者列表",
"url": "https://arxiv.org/abs/2609.00196"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2609.09646",
"title": "RobustSGPO: Search-Space Control for Agent Harness Evolution",
"date": "2026-09-09",
"url": "https://arxiv.org/abs/2609.09646",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Workflow",
"Prompt",
"RegressionGate",
"HeldOut",
"Subagent",
"Archive",
"OfflineSearch",
"BenchmarkScore"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究如何控制修改范围、操作种类和搜索起点，使多代理框架在相同预算下产生更多有效改进。",
"什么在变": "多代理框架的指令、输入输出字段约定，以及允许时的代理增删与路由。三级权限依次允许：改一个指定代理；改任意已有代理但不增删；再开放代理增删及路由变化。执行模型权重、评分规则和受保护路径保持固定。",
"谁来改 / 谁执行": "模型诊断执行记录并生成修改内容；控制器先决定改几个代理、做哪类操作、改哪些目标。预写程序负责规定的增加/删除代理文件操作，其余指令改写由模型完成。本文未披露诊断、修改与评分角色的具体模型型号及是否共用模型。",
"基础 harness": "沿用 AgentX 已有的多代理头脑风暴流程，有提问、构思、验证及提案协调分工，也已有根据失败诊断、写补丁、配对重放、接受或撤回的 SGPO 循环。新增的是明确指定修改范围/操作、保证目标一致及保留其他候选起点，并非从空白代理开始。",
"Feedback": "根据旧执行轨迹构造可重放任务与评分要求，再将新旧框架在相同任务及随机种子上配对运行。优化轨迹与静态编辑错误用于提出补丁，验证分数用于接受和保留版本。论文报告0–5质量分与完成率，但没有给出逐项评分表、质量分聚合细则或评审模型名称。",
"Evolution → Eval": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单代理指令问题、60个跨代理交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。\n\n30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。",
"证据边界": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。"
},
"protocol": "Train → selection → test",
"protocolBasis": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2609.09646"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "研究如何控制修改范围、操作种类和搜索起点，使多代理框架在相同预算下产生更多有效改进。",
"novelty": "将“允许改很多东西”和“实际产生有效修改”分开：先指定目标，再用程序落实结构编辑；把暂时落后但属于不同改动类别的整套框架留作未来起点。还比较轮流开放小范围到大范围权限，以及档案在新任务适应与保留旧能力间的取舍。",
"object": "多 agent 指令、输入输出约定、agent 增删与路由；模型参数固定。",
"executor": "AgentX 的头脑风暴工作流协调提问、构思和验证 agent 完成实验提案任务。所有条件固定相同执行模型、工具和评分规则；本文正文没有列出所用执行模型的具体型号，不能据 AgentX 其他实验反推。",
"modifier": "模型诊断执行记录并生成修改内容；控制器先决定改几个 agent、做哪类操作、改哪些目标。预写程序负责规定的增加/删除 agent 文件操作，其余指令改写由模型完成。本文未披露诊断、修改与评分角色的具体模型型号及是否共用模型。",
"roleContext": "模型诊断执行记录并生成修改内容；控制器先决定改几个代理、做哪类操作、改哪些目标。预写程序负责规定的增加/删除代理文件操作，其余指令改写由模型完成。本文未披露诊断、修改与评分角色的具体模型型号及是否共用模型。",
"seed": "沿用 AgentX 已有的多 agent 头脑风暴流程，有提问、构思、验证及提案协调分工，也已有根据失败诊断、写补丁、配对重放、接受或撤回的 SGPO 循环。新增的是明确指定修改范围/操作、保证目标一致及保留其他候选起点，并非从空白 agent 开始。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "根据旧执行轨迹构造可重放任务与评分要求，再将新旧框架在相同任务及随机种子上配对运行。优化轨迹与静态编辑错误用于提出补丁，验证分数用于接受和保留版本。论文报告0–5质量分与完成率，但没有给出逐项评分表、质量分聚合细则或评审模型名称。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Train → selection → test",
"note": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单代理指令问题、60个跨代理交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。\n\n30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。"
}
],
"takeaway": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2609.09646"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：作者在 AgentX 头脑风暴工作流中组织120个任务：60个单 agent 指令问题、60个跨 agent 交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。\n\n调试 / 选版本数据：60道优化题的轨迹和静态补丁错误供生成候选；30道验证题用于新旧版本比较、档案排名及任务切换后的重评分。每轮3次提案，每个提案最多改120行、增加6,000字符；无效提案和重试同样计成本。\n\n最终测试数据：30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。\n\n数据隔离与证据边界：测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"cycle": "控制器选改动类别与目标，模型写内容，程序检查是否恰好改到指定对象及结构是否有效；有效候选与当前版本做3次配对重放。平均验证增益须超过0.05并通过保护检查，每轮最多接受1个候选。可从当前版本或保留档案继续改，但后代仍必须胜过当前版本才能被采用。",
"train": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单 agent 指令问题、60个跨 agent 交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。",
"debug": "60道优化题的轨迹和静态补丁错误供生成候选；30道验证题用于新旧版本比较、档案排名及任务切换后的重评分。每轮3次提案，每个提案最多改120行、增加6,000字符；无效提案和重试同样计成本。",
"test": "30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。",
"isolation": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "沿用 AgentX 已有的多代理头脑风暴流程，有提问、构思、验证及提案协调分工，也已有根据失败诊断、写补丁、配对重放、接受或撤回的 SGPO 循环。新增的是明确指定修改范围/操作、保证目标一致及保留其他候选起点，并非从空白代理开始。",
"protocol": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单代理指令问题、60个跨代理交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。\n\n60道优化题的轨迹和静态补丁错误供生成候选；30道验证题用于新旧版本比较、档案排名及任务切换后的重评分。每轮3次提案，每个提案最多改120行、增加6,000字符；无效提案和重试同样计成本。\n\n30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。\n\n测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"sections": "§3：AgentX 与诊断—修改—重放循环；§6.1：120题划分与接受规则；§5：指定改动、检查与保留版本；§6.4：任务切换实验；§6.5、表3：等词元成本结果；§2：未跨系统比较的范围说明",
"source": "https://arxiv.org/abs/2609.09646",
"version": "2609.09646v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "2f2ed0037d660e7f45b9924a03009a7605ef5f829078cb2a197482cb5d7708ff",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "AgentX 的头脑风暴工作流协调提问、构思和验证 agent 完成实验提案任务。所有条件固定相同执行模型、工具和评分规则；本文正文没有列出所用执行模型的具体型号，不能据 AgentX 其他实验反推。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "模型诊断执行记录并生成修改内容；控制器先决定改几个 agent、做哪类操作、改哪些目标。预写程序负责规定的增加/删除 agent 文件操作，其余指令改写由模型完成。本文未披露诊断、修改与评分角色的具体模型型号及是否共用模型。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "多 agent 指令、输入输出约定、agent 增删与路由；模型参数固定。",
"sources": [
{
"label": "§4：三级修改权限",
"url": "https://arxiv.org/html/2609.09646#S4"
},
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "根据旧执行轨迹构造可重放任务与评分要求，再将新旧框架在相同任务及随机种子上配对运行。优化轨迹与静态编辑错误用于提出补丁，验证分数用于接受和保留版本。论文报告0–5质量分与完成率，但没有给出逐项评分表、质量分聚合细则或评审模型名称。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "沿用 AgentX 已有的多 agent 头脑风暴流程，有提问、构思、验证及提案协调分工，也已有根据失败诊断、写补丁、配对重放、接受或撤回的 SGPO 循环。新增的是明确指定修改范围/操作、保证目标一致及保留其他候选起点，并非从空白 agent 开始。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "控制器选改动类别与目标，模型写内容，程序检查是否恰好改到指定对象及结构是否有效；有效候选与当前版本做3次配对重放。平均验证增益须超过0.05并通过保护检查，每轮最多接受1个候选。可从当前版本或保留档案继续改，但后代仍必须胜过当前版本才能被采用。",
"sources": [
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单 agent 指令问题、60个跨 agent 交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。",
"sources": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "60道优化题的轨迹和静态补丁错误供生成候选；30道验证题用于新旧版本比较、档案排名及任务切换后的重评分。每轮3次提案，每个提案最多改120行、增加6,000字符；无效提案和重试同样计成本。",
"sources": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。",
"sources": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"sources": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
},
{
"label": "§2：未跨系统比较的范围说明",
"url": "https://arxiv.org/html/2609.09646#S2"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将“允许改很多东西”和“实际产生有效修改”分开：先指定目标，再用程序落实结构编辑；把暂时落后但属于不同改动类别的整套框架留作未来起点。还比较轮流开放小范围到大范围权限，以及档案在新任务适应与保留旧能力间的取舍。",
"sources": [
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
},
{
"label": "§4：三级修改权限",
"url": "https://arxiv.org/html/2609.09646#S4"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2609.09646",
"version": "2609.09646v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究如何控制修改范围、操作种类和搜索起点，使多代理框架在相同预算下产生更多有效改进。",
"feedbackCases": [
{
"label": "AgentX：单 agent 指令与跨 agent 交接问题",
"data": "两类各60题，分别30题优化、15题验证、15题测试；按源对话与底层要求先分组。",
"scoring": "轨迹导出任务及评分要求；新旧版本在同题同种子上重放3次，比较平均验证分。完成检查涉及任务完成、规定产物、受保护操作与交接；0–5质量分的完整计算规则和评审型号未披露。",
"visible": "生成器收到优化轨迹和静态编辑错误；验证分参与接受与档案保留；测试反馈排除在搜索之外。",
"use": "验收须增益超过0.05且通过保护检查。等词元预算用最后可负担版本，失效提案、重试、评分与档案操作均计入成本。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
}
],
"judgment": "成对重放的验证分；本文未披露完整 0–5 质量评分规则和裁判实现"
},
{
"label": "任务切换：旧能力保留与新任务适应",
"data": "单 agent 问题→跨 agent 问题及反方向；各先优化10轮旧任务，再10轮新任务，使用对应优化/验证分区。",
"scoring": "沿用同一重放评分；旧、新任务测试曲线分别衡量适应与退化。保留档案在新任务验证集上重新评分。",
"visible": "新任务验证分可以决定档案起点与接受修改；测试曲线用于报告，不用于选择。",
"use": "按类别保留的档案减少旧任务掉分；随机保留档案取得更高新任务终点。二者侧重点不同。",
"sources": [
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"judgment": "相同重放评分机制，旧／新任务分别评价"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "作者在 AgentX 头脑风暴工作流中组织120个任务：60个单 agent 指令问题、60个跨 agent 交接/结构问题。每类按30/15/15分为优化、验证和测试，共60/30/30。先按源对话和底层要求分组再划分，避免同源问题跨区。模型参数不训练。",
"selection": "60道优化题的轨迹和静态补丁错误供生成候选；30道验证题用于新旧版本比较、档案排名及任务切换后的重评分。每轮3次提案，每个提案最多改120行、增加6,000字符；无效提案和重试同样计成本。",
"evaluation": "30道未参与搜索的测试题，两类各15题。测试质量曲线、任务完成率及相同2,000万词元预算下最后可负担的版本分别报告；任务切换仅在这两类头脑风暴问题之间进行，并未测跨行业泛化。",
"isolation": "测试反馈不进入搜索；等成本比较按预算选最后版本，不按测试最高分选。实验只有一个 AgentX 工作流、五个配对随机种子；与其他论文方法没有共享基准上的直接排名。执行/修改/评分模型型号及完整评分要求未在本文披露，结论可用于理解搜索控制，但复现实验仍需这些配置。",
"roles": {
"executor": {
"value": "AgentX 的头脑风暴工作流协调提问、构思和验证 agent 完成实验提案任务。所有条件固定相同执行模型、工具和评分规则；本文正文没有列出所用执行模型的具体型号，不能据 AgentX 其他实验反推。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
]
},
"modifier": {
"value": "模型诊断执行记录并生成修改内容；控制器先决定改几个 agent、做哪类操作、改哪些目标。预写程序负责规定的增加/删除 agent 文件操作，其余指令改写由模型完成。本文未披露诊断、修改与评分角色的具体模型型号及是否共用模型。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
},
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
]
},
"seed": {
"value": "沿用 AgentX 已有的多 agent 头脑风暴流程，有提问、构思、验证及提案协调分工，也已有根据失败诊断、写补丁、配对重放、接受或撤回的 SGPO 循环。新增的是明确指定修改范围/操作、保证目标一致及保留其他候选起点，并非从空白 agent 开始。",
"sources": [
{
"label": "§3：AgentX 与诊断—修改—重放循环",
"url": "https://arxiv.org/html/2609.09646#S3"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
}
],
"selection": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
],
"evaluation": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
],
"isolation": [
{
"label": "§6.1：120题划分与接受规则",
"url": "https://arxiv.org/html/2609.09646#S6.SS1"
},
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
},
{
"label": "§2：未跨系统比较的范围说明",
"url": "https://arxiv.org/html/2609.09646#S2"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "多 agent 失败可能发生在输入输出约定或 agent 连接处，只允许修改一个 agent 就难直接修复这些问题。但开放全部编辑权限，也不代表模型会真正探索：它仍可能只改熟悉的提示，或生成无法执行的结构修改；因此需要控制实际尝试了哪些改法、哪些候选得以保留。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.09646#S1"
}
]
},
{
"key": "position",
"value": "研究如何控制修改范围、操作种类和搜索起点，使多 agent 框架在相同预算下产生更多有效改进。",
"sources": [
{
"label": "§1–2：定位与范围",
"url": "https://arxiv.org/html/2609.09646#S1"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
}
]
},
{
"key": "conclusion",
"value": "在 AgentX 头脑风暴任务中，完整方法将测试完成率从60%提高到80%；等2,000万词元预算质量分从3.77到4.14。保留不同改动类别更利于保住旧能力，但未取得最高的新任务终点成绩。",
"sources": [
{
"label": "§6.5、表3：等词元成本结果",
"url": "https://arxiv.org/html/2609.09646#S6.SS5"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
]
}
],
"fields": {
"object": "多 agent 指令、输入输出约定、agent 增删与路由；模型参数固定。",
"executor": "AgentX 中负责提问、构思、验证的 agent 共同执行；该论文未披露执行模型型号。",
"modifier": "控制器先定修改目标，模型写具体内容，预写程序落实 agent 增删；诊断和修改模型的型号未披露。",
"verdict": "用60道优化题的轨迹诊断，用30道验证题配对比较新旧版本；另30题测试。原文给出0–5质量分和完成率，但没有完整评分细则。",
"seed": "已有 AgentX 多 agent 头脑风暴流程，且已有失败诊断、补丁生成、重放验收和回退能力；本文增强修改控制和历史版本保留。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"harness"
]
},
{
"id": "2609.09153",
"title": "Procedural Graphs: Self-Evolving Execution Structures for LLM Agents",
"date": "2026-09-08",
"url": "https://arxiv.org/abs/2609.09153",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"Workflow",
"MemoryContent",
"Prompt",
"RegressionGate",
"HeldOut",
"LLMJudge",
"ExecutableVerifier",
"SameModel",
"OfflineSearch",
"org:google",
"org:peking"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究可更新的步骤图能否帮助冻结模型稳定执行长任务，以及它能否从经验中自行构建并纠正错误的人工先验。",
"什么在变": "持久改变的是“程序图”：节点表示工具调用或推理步骤，边说明什么条件下进入下一步、怎么做、要避开什么错误。执行时取当前节点两步以内的邻域，由模型翻译成指导文字；执行器仍自行决定行动。图可变，ReAct 循环、工具接口、修改流程与模型权重固定。",
"谁来改 / 谁执行": "与该组执行器同型号的冻结模型作为图修改者，对照高分/低分轨迹，增删步骤节点和连接、改边上的条件与提醒。另一次同型号模型调用将当前节点附近的图转成下一步指导。固定程序定位节点、检查结构并按验证分决定是否接受；没有训练模型参数。",
"基础 harness": "共用能调用工具、读取结果的 ReAct 执行器。图的起点有两种：人写的工具步骤与条件图，或只含 Start→End、没有中间知识的最小骨架。另设无图起点作为对照。一次性构图直接接收修改；逐批迭代构图才有独立验证集的回退门槛，不能混写成所有方案都有验证保护。",
"Feedback": "每条训练轨迹带对应任务评分，修改者比较高低分轨迹并读失败记录。问答对照参考答案；对话任务由 Gemini 3.1 Pro 按指令要求评审；工具任务用各基准的状态/调用检查；企业仿真用存活、寿命及财务分数。迭代方案还在独立验证集上验收，退步则拒绝并保留拒绝原因。",
"Evolution → Eval": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。\n\n主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。",
"证据边界": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。"
},
"protocol": "Train → selection → test",
"protocolBasis": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2609.09153"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "harness",
"brief": {
"summary": "研究可更新的步骤图能否帮助冻结模型稳定执行长任务，以及它能否从经验中自行构建并纠正错误的人工先验。",
"novelty": "将零散文字经验组织成“有条件连接的步骤图”，每一步只给与当前位置有关的指导。可从只有起止节点的骨架逐步长出程序知识，也能修复一个错误的人写先验。它改变图结构与边上说明，不任意重写整个 agent 源码。",
"object": "有条件连接的步骤图：做什么、先后顺序、何时转到下一步及避错提醒；模型权重不变。",
"executor": "统一使用 ReAct 执行器，即循环“决定下一步—调用工具—读取结果”。分别测试 Claude Sonnet 4.6、Gemini 3.1 Pro、Gemini 3.5 Flash、Grok 4.1 Fast；每组的图指导模型与图修改模型和执行器使用同一种模型。专门的图构造及十轮企业仿真实验使用冻结 Gemini 3.5 Flash。",
"modifier": "与该组执行器同型号的冻结模型作为图修改者，对照高分/低分轨迹，增删步骤节点和连接、改边上的条件与提醒。另一次同型号模型调用将当前节点附近的图转成下一步指导。固定程序定位节点、检查结构并按验证分决定是否接受；没有训练模型参数。",
"roleContext": "与该组执行器同型号的冻结模型作为图修改者，对照高分/低分轨迹，增删步骤节点和连接、改边上的条件与提醒。另一次同型号模型调用将当前节点附近的图转成下一步指导。固定程序定位节点、检查结构并按验证分决定是否接受；没有训练模型参数。",
"seed": "共用能调用工具、读取结果的 ReAct 执行器。图的起点有两种：人写的工具步骤与条件图，或只含 Start→End、没有中间知识的最小骨架。另设无图起点作为对照。一次性构图直接接收修改；逐批迭代构图才有独立验证集的回退门槛，不能混写成所有方案都有验证保护。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "每条训练轨迹带对应任务评分，修改者比较高低分轨迹并读失败记录。问答对照参考答案；对话任务由 Gemini 3.1 Pro 按指令要求评审；工具任务用各基准的状态/调用检查；企业仿真用存活、寿命及财务分数。迭代方案还在独立验证集上验收，退步则拒绝并保留拒绝原因。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Train → selection → test",
"note": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。\n\n主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。"
}
],
"takeaway": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2609.09153"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。\n\n调试 / 选版本数据：迭代构图：HotpotQA 每100题更新一次，用额外1,000题验证；MultiChallenge 每20题更新一次，用额外100题验证；企业十轮实验用20场验证。一次性构图方案无验证回退。三类迭代实验明确验证/测试不重叠，其他基准的独立验证样本数没有在表5逐项列出。\n\n最终测试数据：主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。\n\n数据隔离与证据边界：图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"cycle": "执行一批训练任务→按分数对照轨迹→提出图编辑→先检查边端点、节点类型和能否走到终止节点→验证集不掉分才保留。拒绝候选、训练记录及验证分留给下一次修改。每道题内部和测试时图固定；文中“在线进化”指训练批次之间更新图，并非在测试题上边做边改。",
"train": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。",
"debug": "迭代构图：HotpotQA 每100题更新一次，用额外1,000题验证；MultiChallenge 每20题更新一次，用额外100题验证；企业十轮实验用20场验证。一次性构图方案无验证回退。三类迭代实验明确验证/测试不重叠，其他基准的独立验证样本数没有在表5逐项列出。",
"test": "主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。",
"isolation": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "共用能调用工具、读取结果的 ReAct 执行器。图的起点有两种：人写的工具步骤与条件图，或只含 Start→End、没有中间知识的最小骨架。另设无图起点作为对照。一次性构图直接接收修改；逐批迭代构图才有独立验证集的回退门槛，不能混写成所有方案都有验证保护。",
"protocol": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。\n\n迭代构图：HotpotQA 每100题更新一次，用额外1,000题验证；MultiChallenge 每20题更新一次，用额外100题验证；企业十轮实验用20场验证。一次性构图方案无验证回退。三类迭代实验明确验证/测试不重叠，其他基准的独立验证样本数没有在表5逐项列出。\n\n主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。\n\n图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"sections": "§4：模型与共享执行框架；附录D.3：迭代构图实验；§5.4：最终图与中途最高分；附录D.2：五种图构造方案；附录B.1、表5：数据划分",
"source": "https://arxiv.org/abs/2609.09153",
"version": "2609.09153v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "c06db200c83940a708097dd2645989c90223fc324170f502bdb94586b49eb823",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "统一使用 ReAct 执行器，即循环“决定下一步—调用工具—读取结果”。分别测试 Claude Sonnet 4.6、Gemini 3.1 Pro、Gemini 3.5 Flash、Grok 4.1 Fast；每组的图指导模型与图修改模型和执行器使用同一种模型。专门的图构造及十轮企业仿真实验使用冻结 Gemini 3.5 Flash。",
"sources": [
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "与该组执行器同型号的冻结模型作为图修改者，对照高分/低分轨迹，增删步骤节点和连接、改边上的条件与提醒。另一次同型号模型调用将当前节点附近的图转成下一步指导。固定程序定位节点、检查结构并按验证分决定是否接受；没有训练模型参数。",
"sources": [
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录B.6：验证、拒绝记录与结构检查",
"url": "https://arxiv.org/html/2609.09153#A2.SS6"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "有条件连接的步骤图：做什么、先后顺序、何时转到下一步及避错提醒；模型权重不变。",
"sources": [
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "每条训练轨迹带对应任务评分，修改者比较高低分轨迹并读失败记录。问答对照参考答案；对话任务由 Gemini 3.1 Pro 按指令要求评审；工具任务用各基准的状态/调用检查；企业仿真用存活、寿命及财务分数。迭代方案还在独立验证集上验收，退步则拒绝并保留拒绝原因。",
"sources": [
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "附录B.6：验证、拒绝记录与结构检查",
"url": "https://arxiv.org/html/2609.09153#A2.SS6"
},
{
"label": "附录C.1：企业仿真规则与指标",
"url": "https://arxiv.org/html/2609.09153#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "共用能调用工具、读取结果的 ReAct 执行器。图的起点有两种：人写的工具步骤与条件图，或只含 Start→End、没有中间知识的最小骨架。另设无图起点作为对照。一次性构图直接接收修改；逐批迭代构图才有独立验证集的回退门槛，不能混写成所有方案都有验证保护。",
"sources": [
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "执行一批训练任务→按分数对照轨迹→提出图编辑→先检查边端点、节点类型和能否走到终止节点→验证集不掉分才保留。拒绝候选、训练记录及验证分留给下一次修改。每道题内部和测试时图固定；文中“在线进化”指训练批次之间更新图，并非在测试题上边做边改。",
"sources": [
{
"label": "附录B.6：验证、拒绝记录与结构检查",
"url": "https://arxiv.org/html/2609.09153#A2.SS6"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "迭代构图：HotpotQA 每100题更新一次，用额外1,000题验证；MultiChallenge 每20题更新一次，用额外100题验证；企业十轮实验用20场验证。一次性构图方案无验证回退。三类迭代实验明确验证/测试不重叠，其他基准的独立验证样本数没有在表5逐项列出。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "将零散文字经验组织成“有条件连接的步骤图”，每一步只给与当前位置有关的指导。可从只有起止节点的骨架逐步长出程序知识，也能修复一个错误的人写先验。它改变图结构与边上说明，不任意重写整个 agent 源码。",
"sources": [
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
},
{
"label": "§5.3：骨架构图与错误先验修复",
"url": "https://arxiv.org/html/2609.09153#S5.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2609.09153",
"version": "2609.09153v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究可更新的步骤图能否帮助冻结模型稳定执行长任务，以及它能否从经验中自行构建并纠正错误的人工先验。",
"feedbackCases": [
{
"label": "HotpotQA：多跳问答",
"data": "训练1,000、另验证1,000、测试1,000题；按题号去重。构图时每100道训练题更新。",
"scoring": "主比较由 Gemini 3.1 Pro 接收问题、参考答案与模型答案，判语义等价0/1；构图实验另报告字符串完全匹配和词级F1（答案内容的覆盖与精确程度）。",
"visible": "修改者看到训练轨迹与任务分；验证集用于迭代图接受，测试图固定。",
"use": "用成功/失败或高低分轨迹定位检索、推理与停止问题；主表语义正确率不可与构图表F1直接比较。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
}
],
"judgment": "Gemini 3.1 Pro 对照参考答案判语义等价；构图实验另算 EM／词级 F1"
},
{
"label": "MultiChallenge：多轮对话约束",
"data": "训练100、另验证100；主比较测试166题，构图实验测试56题；迭代每20题更新。",
"scoring": "Gemini 3.1 Pro 按推理记忆、指令保留、版本编辑一致性和自洽四方面评分，汇总任务成功率。",
"visible": "训练对话、回答与评价供修改者诊断；另100题验证决定迭代编辑是否回退。",
"use": "调整图的指令提取与回答步骤；表格中92.86%等构图结果来自56题子集，不能当166题主表结果。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"judgment": "Gemini 3.1 Pro 按四类对话要求评审"
},
{
"label": "GDPval：专业工作产物",
"data": "88题构图、44题测试，按职业确定性划分；表5未列单独验证集数量。",
"scoring": "将产物与各题专家评分要求逐项对照，报告平均评分；本文指标段未点名该项具体评审模型，不把对话任务的 Gemini judge 自动套用到这里。",
"visible": "训练产物、轨迹与对应评分用于构图；正式44题在固定图下运行。",
"use": "检查图能否帮助生成符合专业要求的产物；评分来源是每题要求，不是程序答案匹配。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
}
],
"judgment": "按专家逐题评分细则评审；本文未在该项指定裁判型号"
},
{
"label": "ALFWorld：家务环境",
"data": "238个训练任务；测试标准未见分区134个任务，数量经过环境库内部过滤。",
"scoring": "环境根据最终状态是否达成家务目标给成功/失败；报告测试成功率。",
"visible": "执行器得到动作观察，训练任务的结果和轨迹用于图更新；测试不改图。",
"use": "从成功及失败动作序列学习先后约束，最终检查在未见任务上能否完成目标。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
}
],
"judgment": "ALFWorld 环境程序检查家务目标"
},
{
"label": "τ-bench：零售客服工具使用",
"data": "500个训练任务，115个零售域测试任务；表5没有单列该域验证数。",
"scoring": "比较最终数据库状态与标注目标状态，报告单次运行成功率；不是只看回答语言是否合理。",
"visible": "训练工具记录和任务分用于图构造；用户交互与数据库工具本身保持共享设置。",
"use": "指导需要按政策和先后顺序进行的操作；测试结果不用于测试题内更新图。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
}
],
"judgment": "τ-bench 程序比较最终数据库与标注目标状态"
},
{
"label": "BFCL v3：多轮函数调用",
"data": "基础类多轮任务，训练100、测试100；案例使用从骨架一次性构建的图。",
"scoring": "采用 BFCL 官方多轮正确率。本文没有在指标段展开该评测器的全部检查项，因此不改写成仅“API名称匹配”。",
"visible": "训练调用轨迹与官方结果供图构造；测试固定图下调用工具。",
"use": "典型区别是任务已完成时能停止，避免用户只要报价，agent 却继续付款和订票。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录B.2：各基准判分",
"url": "https://arxiv.org/html/2609.09153#A2.SS2"
},
{
"label": "附录F.1：报价后正确停止",
"url": "https://arxiv.org/html/2609.09153#A6.SS1"
}
],
"judgment": "BFCL 官方多轮评估器；本文未逐项展开内部检查"
},
{
"label": "EnterpriseArena：企业现金流与危机",
"data": "主比较50训练/50测试场；专门十轮自进化20训练/20验证/20测试场，各分区种子互斥。",
"scoring": "仿真现金为负即破产；统计全程存活率、存活月数、按收入估值且惩罚工具成本的企业分数，以及融资额。不同指标分别报告。",
"visible": "修改者看训练财务/工具轨迹；验证退步或结构检查失败就拒绝，拒绝记录供下一轮参考。",
"use": "图学习提前融资和管理现金流。最终返回图在20场测试存活85%；95%是中间一次候选的测试峰值。",
"sources": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录C.1：企业仿真规则与指标",
"url": "https://arxiv.org/html/2609.09153#A3.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
},
{
"label": "附录B.6：验证、拒绝记录与结构检查",
"url": "https://arxiv.org/html/2609.09153#A2.SS6"
}
],
"judgment": "企业仿真程序计算现金、存活、估值和融资指标"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1,
2,
3,
4,
5,
6
],
"evolution": "主比较的训练数：HotpotQA 1,000；MultiChallenge 100；GDPval 88；ALFWorld 238；τ-bench 500；BFCL v3 100；EnterpriseArena 50场。这里训练指收集轨迹构图，不更新模型权重。专门十轮企业自进化另用20场训练；不能与主表50场混为一个设置。",
"selection": "迭代构图：HotpotQA 每100题更新一次，用额外1,000题验证；MultiChallenge 每20题更新一次，用额外100题验证；企业十轮实验用20场验证。一次性构图方案无验证回退。三类迭代实验明确验证/测试不重叠，其他基准的独立验证样本数没有在表5逐项列出。",
"evaluation": "主表：HotpotQA 1,000题；MultiChallenge 166题；GDPval 44题；ALFWorld 134题；τ-bench 零售115题；BFCL v3 多轮基础类100题；EnterpriseArena 50场。构图消融的 MultiChallenge 只测56题。十轮企业自进化另用20场测试，最终返回图存活率85%，中途最高95%不作最终成绩。",
"isolation": "图在测试期间冻结；企业主比较按互斥随机种子区分训练/测试，十轮实验另按20/20/20划分并由验证集选图。HotpotQA 去重按题号分区，GDPval 按职业确定性划分，ALFWorld 用标准未见任务。十轮实验每分区只有20场，少数场景即可影响接受决定；85%不能替换成测试中途最高95%。",
"roles": {
"executor": {
"value": "统一使用 ReAct 执行器，即循环“决定下一步—调用工具—读取结果”。分别测试 Claude Sonnet 4.6、Gemini 3.1 Pro、Gemini 3.5 Flash、Grok 4.1 Fast；每组的图指导模型与图修改模型和执行器使用同一种模型。专门的图构造及十轮企业仿真实验使用冻结 Gemini 3.5 Flash。",
"sources": [
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
]
},
"modifier": {
"value": "与该组执行器同型号的冻结模型作为图修改者，对照高分/低分轨迹，增删步骤节点和连接、改边上的条件与提醒。另一次同型号模型调用将当前节点附近的图转成下一步指导。固定程序定位节点、检查结构并按验证分决定是否接受；没有训练模型参数。",
"sources": [
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录B.6：验证、拒绝记录与结构检查",
"url": "https://arxiv.org/html/2609.09153#A2.SS6"
}
]
},
"seed": {
"value": "共用能调用工具、读取结果的 ReAct 执行器。图的起点有两种：人写的工具步骤与条件图，或只含 Start→End、没有中间知识的最小骨架。另设无图起点作为对照。一次性构图直接接收修改；逐批迭代构图才有独立验证集的回退门槛，不能混写成所有方案都有验证保护。",
"sources": [
{
"label": "§4：模型与共享执行框架",
"url": "https://arxiv.org/html/2609.09153#S4"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"selection": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "附录D.3：迭代构图实验",
"url": "https://arxiv.org/html/2609.09153#A4.SS3"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"evaluation": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
],
"isolation": [
{
"label": "附录B.1、表5：数据划分",
"url": "https://arxiv.org/html/2609.09153#A2.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "平铺的长轨迹要求模型自行记住当前进度和步骤依赖，容易导致偏离目标、工具顺序错误或重复无效行动。文字经验仍需临场重建流程关系，固定工作流又常依赖人工设计；作者希望把可编辑的步骤结构与当前进度结合，让经验能直接指导下一步。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2609.09153#S1"
}
]
},
{
"key": "position",
"value": "研究可更新的步骤图能否帮助冻结模型稳定执行长任务，以及它能否从经验中自行构建并纠正错误的人工先验。",
"sources": [
{
"label": "§1：程序知识与研究目标",
"url": "https://arxiv.org/html/2609.09153#S1"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
]
},
{
"key": "conclusion",
"value": "图指导在六基准、四模型的24组比较中有21组取得第一或并列第一。另一个20场测试的企业自进化实验中，最终图存活率从0%到85%；这与主表的固定图比较是不同实验。",
"sources": [
{
"label": "§5.1：跨基准结果",
"url": "https://arxiv.org/html/2609.09153#S5.SS1"
},
{
"label": "§5.4：最终图与中途最高分",
"url": "https://arxiv.org/html/2609.09153#S5.SS4"
}
]
}
],
"fields": {
"object": "有条件连接的步骤图：做什么、先后顺序、何时转到下一步及避错提醒；模型权重不变。",
"executor": "Claude Sonnet 4.6、Gemini 3.1 Pro、Gemini 3.5 Flash 或 Grok 4.1 Fast，配统一的工具调用循环；专门自进化实验用 Gemini 3.5 Flash。",
"modifier": "每组使用与执行器同型号的冻结模型，读高低分轨迹改图；固定程序检查图结构，并按验证表现决定是否接受。",
"verdict": "训练任务的参考答案、环境检查或评分要求产生分数；迭代方案再用独立验证集筛图。七个基准的判分和数据分别列在表内。",
"seed": "已有 ReAct 工具调用循环；步骤图可从人工规则起步，也可只有“开始→结束”。一次性构图与逐批验证构图分别比较。"
}
},
"attributions": [
{
"tag": "org:google",
"label": "Google",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.09153"
}
]
},
{
"tag": "org:peking",
"label": "Peking University",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.09153"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"experience"
]
},
{
"id": "2607.22688",
"title": "Co-Harness: Co-Evolving Harnesses and Model Weights for LLM Agents",
"date": "2026-07-17",
"url": "https://arxiv.org/abs/2607.22688",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Weights",
"JointEvolution",
"RegressionGate",
"ExecutableVerifier",
"SeparateEvolver",
"Archive",
"CoEvolution"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究修复产生训练轨迹的框架、再用更好的轨迹训练模型，能否在多轮后训练中形成互相促进的改进。",
"什么在变": "框架配置与执行模型参数交替变化。框架包含提示、Python调用格式、技能、工具运行/重试/上下文管理程序及记忆。修改者把失败归到具体字段，局部修补；判断主要是模型本身出错时，不为该轨迹生成框架补丁。",
"谁来改 / 谁执行": "HarnessCritic 是读失败轨迹、诊断根因并给局部补丁的模型角色，随后固定程序合并兼容建议并验收。论文没有为主实验单独列出这个角色的具体模型型号；附录D只说明归因对照使用初始模型，不能扩写为全部轮次都由某个确定外部模型修改。模型参数由监督微调程序更新。",
"基础 harness": "初始框架有512词元的数学/Python使用提示、带JSON调用格式的沙箱Python解释器（默认30秒超时）、异常最多重试3次、滑窗保留最后4,000词元、工具输出截断2,000词元；初始技能库为空、没有长期记忆或外部检索。代码解释器循环起始最多15轮。",
"Feedback": "失败轨迹记录模型输出、工具调用、Python标准输出/错误和终态。HarnessCritic 根据具体事件定位格式错误、缺少技能、重试或上下文问题；程序再比较目标失败类和原有成功类任务的新旧得分。成功轨迹须通过任务检查才用于微调，但原文未公开逐基准答案解析器及完整判分实现。",
"Evolution → Eval": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。\n\n主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。",
"证据边界": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。"
},
"protocol": "混合协议",
"protocolBasis": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2607.22688"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "研究修复产生训练轨迹的框架、再用更好的轨迹训练模型，能否在多轮后训练中形成互相促进的改进。",
"novelty": "先修好产生训练轨迹的框架，再把新框架下成功的行为训练进模型，使下一轮暴露出新的框架瓶颈。与广泛搜索 agent 代码相比，重点是“哪项失败由哪个框架字段导致”的局部修补及两组行为验收。",
"object": "数学 agent 的提示、工具与运行配置，以及 Qwen 执行模型参数，交替更新。",
"executor": "Qwen3-8B 或 Qwen3-32B 在数学推理中多轮调用 Python，训练后继续用对应候选模型。主实验两轮，每轮包含框架更新和成功轨迹监督微调；长期22版本案例使用 Qwen3-8B。",
"modifier": "HarnessCritic 是读失败轨迹、诊断根因并给局部补丁的模型角色，随后固定程序合并兼容建议并验收。论文没有为主实验单独列出这个角色的具体模型型号；附录D只说明归因对照使用初始模型，不能扩写为全部轮次都由某个确定外部模型修改。模型参数由监督微调程序更新。",
"roleContext": "HarnessCritic 是读失败轨迹、诊断根因并给局部补丁的模型角色，随后固定程序合并兼容建议并验收。论文没有为主实验单独列出这个角色的具体模型型号；附录D只说明归因对照使用初始模型，不能扩写为全部轮次都由某个确定外部模型修改。模型参数由监督微调程序更新。",
"seed": "初始框架有512词元的数学/Python使用提示、带JSON调用格式的沙箱Python解释器（默认30秒超时）、异常最多重试3次、滑窗保留最后4,000词元、工具输出截断2,000词元；初始技能库为空、没有长期记忆或外部检索。代码解释器循环起始最多15轮。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "失败轨迹记录模型输出、工具调用、Python标准输出/错误和终态。HarnessCritic 根据具体事件定位格式错误、缺少技能、重试或上下文问题；程序再比较目标失败类和原有成功类任务的新旧得分。成功轨迹须通过任务检查才用于微调，但原文未公开逐基准答案解析器及完整判分实现。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "混合协议",
"note": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。\n\n主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。"
}
],
"takeaway": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2607.22688"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。\n\n调试 / 选版本数据：补丁分别在两组任务上重放：含目标失败类型的一组必须改善；原有成功或其他行为的一组不得退步。附录E给出接受公式，但未列这两组的来源、题号及样本数。长期案例明确围绕 AIME24 连续迭代22个版本，并根据准确率与耗时回退。\n\n最终测试数据：主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。\n\n数据隔离与证据边界：论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"cycle": "按当前框架收集失败→模型给根因与字段补丁→按复现频率、严重性和字段一致性合并→两组任务重放验收→在改好框架下收集成功轨迹微调→下一轮。主实验两轮，每轮框架内部5次迭代；比较起点已先经过一次框架优化，所以不等于原始未经调整的 Qwen。",
"train": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。",
"debug": "补丁分别在两组任务上重放：含目标失败类型的一组必须改善；原有成功或其他行为的一组不得退步。附录E给出接受公式，但未列这两组的来源、题号及样本数。长期案例明确围绕 AIME24 连续迭代22个版本，并根据准确率与耗时回退。",
"test": "主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。",
"isolation": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "初始框架有512词元的数学/Python使用提示、带JSON调用格式的沙箱Python解释器（默认30秒超时）、异常最多重试3次、滑窗保留最后4,000词元、工具输出截断2,000词元；初始技能库为空、没有长期记忆或外部检索。代码解释器循环起始最多15轮。",
"protocol": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。\n\n补丁分别在两组任务上重放：含目标失败类型的一组必须改善；原有成功或其他行为的一组不得退步。附录E给出接受公式，但未列这两组的来源、题号及样本数。长期案例明确围绕 AIME24 连续迭代22个版本，并根据准确率与耗时回退。\n\n主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。\n\n论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"sections": "§4.1：模型、基准、轮次；附录H：AIME24长期案例；附录A/C：Python循环与初始配置；附录A：多轮执行；§3.4：成功轨迹微调；附录E：两组验收任务；§4.2、表6：两轮改进",
"source": "https://arxiv.org/abs/2607.22688",
"version": "2607.22688v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "3a65f1363c6fcb3b337eaee4d7bacc8231fcc562bd9d8f1f4d59be8803be51b5",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3-8B 或 Qwen3-32B 在数学推理中多轮调用 Python，训练后继续用对应候选模型。主实验两轮，每轮包含框架更新和成功轨迹监督微调；长期22版本案例使用 Qwen3-8B。",
"sources": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "HarnessCritic 是读失败轨迹、诊断根因并给局部补丁的模型角色，随后固定程序合并兼容建议并验收。论文没有为主实验单独列出这个角色的具体模型型号；附录D只说明归因对照使用初始模型，不能扩写为全部轮次都由某个确定外部模型修改。模型参数由监督微调程序更新。",
"sources": [
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "附录D：归因人工对照",
"url": "https://arxiv.org/html/2607.22688#A4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "数学 agent 的提示、工具与运行配置，以及 Qwen 执行模型参数，交替更新。",
"sources": [
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "失败轨迹记录模型输出、工具调用、Python标准输出/错误和终态。HarnessCritic 根据具体事件定位格式错误、缺少技能、重试或上下文问题；程序再比较目标失败类和原有成功类任务的新旧得分。成功轨迹须通过任务检查才用于微调，但原文未公开逐基准答案解析器及完整判分实现。",
"sources": [
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "附录A：工具输出与失败类型",
"url": "https://arxiv.org/html/2607.22688#A1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "初始框架有512词元的数学/Python使用提示、带JSON调用格式的沙箱Python解释器（默认30秒超时）、异常最多重试3次、滑窗保留最后4,000词元、工具输出截断2,000词元；初始技能库为空、没有长期记忆或外部检索。代码解释器循环起始最多15轮。",
"sources": [
{
"label": "附录A/C：Python循环与初始配置",
"url": "https://arxiv.org/html/2607.22688#A3"
},
{
"label": "附录A：多轮执行",
"url": "https://arxiv.org/html/2607.22688#A1"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "按当前框架收集失败→模型给根因与字段补丁→按复现频率、严重性和字段一致性合并→两组任务重放验收→在改好框架下收集成功轨迹微调→下一轮。主实验两轮，每轮框架内部5次迭代；比较起点已先经过一次框架优化，所以不等于原始未经调整的 Qwen。",
"sources": [
{
"label": "§3.2–3.5：两个循环与验收",
"url": "https://arxiv.org/html/2607.22688#S3.SS2"
},
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。",
"sources": [
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "补丁分别在两组任务上重放：含目标失败类型的一组必须改善；原有成功或其他行为的一组不得退步。附录E给出接受公式，但未列这两组的来源、题号及样本数。长期案例明确围绕 AIME24 连续迭代22个版本，并根据准确率与耗时回退。",
"sources": [
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。",
"sources": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "§4.2、表6：两轮改进",
"url": "https://arxiv.org/html/2607.22688#S4.SS2"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"sources": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "先修好产生训练轨迹的框架，再把新框架下成功的行为训练进模型，使下一轮暴露出新的框架瓶颈。与广泛搜索 agent 代码相比，重点是“哪项失败由哪个框架字段导致”的局部修补及两组行为验收。",
"sources": [
{
"label": "§1：联合后训练动机",
"url": "https://arxiv.org/html/2607.22688#S1"
},
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2607.22688",
"version": "2607.22688v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究修复产生训练轨迹的框架、再用更好的轨迹训练模型，能否在多轮后训练中形成互相促进的改进。",
"feedbackCases": [
{
"label": "AIME24、AIME25、HMMT25：联合更新主表",
"data": "各30题；论文没有列出独立训练题库及两组补丁验收题的具体来源和数量。",
"scoring": "主表按任务成功统计多次运行平均的单次答对率；成功轨迹需通过任务检查。本文未给逐基准答案解析器和完整verifier实现，不能补写某个未披露的数学判分库。",
"visible": "修改者看失败轨迹及具体Python事件；接受补丁时看目标失败组与保留行为组的新旧得分。",
"use": "目标组必须提高、保留组不能下降；再用通过检查的轨迹微调。不能确认这套数据与最终90题完全隔离。",
"sources": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
}
],
"judgment": "任务答案正确性验证；本文未公开完整答案解析及比对实现"
},
{
"label": "AIME24：22版本长期框架优化案例",
"data": "同一AIME24基准上连续修改22版本，使用Qwen3-8B，研究时间超过200小时。",
"scoring": "以任务正确率和运行时间比较候选；后期采用两种采样种子的6条轨迹多数投票，在3.33小时时限内达到63.3%。",
"visible": "崩溃、线程问题、耗时及成绩触发修复与回退；领域提示造成退化后被撤回。",
"use": "说明系统能够恢复运行、调节开销并探索集成；该成绩包含多轨迹投票和基准内适配，不当作主表单次答对率或独立测试。",
"sources": [
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"judgment": "同一数学任务正确率 + 运行计时；后期另用候选答案多数票"
}
],
"experiments": [
{
"label": "实验数据与反馈对照",
"feedbackCases": [
0,
1
],
"evolution": "微调使用当前模型在接受后的框架下产生并通过任务检查的轨迹。本文实验点名 AIME 2024、AIME 2025、HMMT February 2025 各30题，但没有明确列出另一个训练题库、成功轨迹总数及其与90道评测题的题号关系。不能据此写成在某训练基准上训练、再独立测竞赛题。",
"selection": "补丁分别在两组任务上重放：含目标失败类型的一组必须改善；原有成功或其他行为的一组不得退步。附录E给出接受公式，但未列这两组的来源、题号及样本数。长期案例明确围绕 AIME24 连续迭代22个版本，并根据准确率与耗时回退。",
"evaluation": "主表在 AIME24、AIME25、HMMT25 各30题上比较 Qwen3-8B/32B 的两轮更新，报告多次运行平均的单次答对率。另一个AIME24长期案例探索6条轨迹多数投票和时间预算；它与主表单次正确率不是同一种统计。",
"isolation": "论文提出防退化验收，但未披露训练题、两组验收题与最终90道题之间的完整隔离关系。AIME24长期案例用同一基准迭代并选设计，不能视为另一份独立测试。收录它是因为联合训练和故障归因的设计有对照价值，而不是将其涨分当作已证明跨数据泛化。",
"roles": {
"executor": {
"value": "Qwen3-8B 或 Qwen3-32B 在数学推理中多轮调用 Python，训练后继续用对应候选模型。主实验两轮，每轮包含框架更新和成功轨迹监督微调；长期22版本案例使用 Qwen3-8B。",
"sources": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
]
},
"modifier": {
"value": "HarnessCritic 是读失败轨迹、诊断根因并给局部补丁的模型角色，随后固定程序合并兼容建议并验收。论文没有为主实验单独列出这个角色的具体模型型号；附录D只说明归因对照使用初始模型，不能扩写为全部轮次都由某个确定外部模型修改。模型参数由监督微调程序更新。",
"sources": [
{
"label": "§3.3：归因和局部补丁",
"url": "https://arxiv.org/html/2607.22688#S3.SS3"
},
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "附录D：归因人工对照",
"url": "https://arxiv.org/html/2607.22688#A4"
}
]
},
"seed": {
"value": "初始框架有512词元的数学/Python使用提示、带JSON调用格式的沙箱Python解释器（默认30秒超时）、异常最多重试3次、滑窗保留最后4,000词元、工具输出截断2,000词元；初始技能库为空、没有长期记忆或外部检索。代码解释器循环起始最多15轮。",
"sources": [
{
"label": "附录A/C：Python循环与初始配置",
"url": "https://arxiv.org/html/2607.22688#A3"
},
{
"label": "附录A：多轮执行",
"url": "https://arxiv.org/html/2607.22688#A1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "§3.4：成功轨迹微调",
"url": "https://arxiv.org/html/2607.22688#S3.SS4"
},
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
}
],
"selection": [
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"evaluation": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "§4.2、表6：两轮改进",
"url": "https://arxiv.org/html/2607.22688#S4.SS2"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
],
"isolation": [
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
},
{
"label": "附录H：AIME24长期案例",
"url": "https://arxiv.org/html/2607.22688#A8"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型后训练常固定运行框架，但工具定义错误、缺少重试或回合限制不合适，可能直接阻止有用训练轨迹产生。另一方面，已有框架优化多围绕冻结模型改进执行，没有进入生成数据并训练下一版模型的循环；因此这两个改进过程仍相互脱节。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2607.22688#S1"
}
]
},
{
"key": "position",
"value": "研究修复产生训练轨迹的框架、再用更好的轨迹训练模型，能否在多轮后训练中形成互相促进的改进。",
"sources": [
{
"label": "§1：联合后训练动机",
"url": "https://arxiv.org/html/2607.22688#S1"
},
{
"label": "§3.2–3.5：两个循环与验收",
"url": "https://arxiv.org/html/2607.22688#S3.SS2"
}
]
},
{
"key": "conclusion",
"value": "两种 Qwen 在三组竞赛数学题上，两轮更新相对“已优化框架、尚未微调”的起点平均提高20.4个百分点。论文没有完整披露训练、验收与评测题的隔离，泛化结论需受此限制。",
"sources": [
{
"label": "§4.2、表6：两轮改进",
"url": "https://arxiv.org/html/2607.22688#S4.SS2"
},
{
"label": "§4.1：模型、基准、轮次",
"url": "https://arxiv.org/html/2607.22688#S4.SS1"
},
{
"label": "附录E：两组验收任务",
"url": "https://arxiv.org/html/2607.22688#A5"
}
]
}
],
"fields": {
"object": "数学 agent 的提示、工具与运行配置，以及 Qwen 执行模型参数，交替更新。",
"executor": "Qwen3-8B 或 Qwen3-32B 多轮调用 Python；下一轮使用微调后的模型。",
"modifier": "模型角色 HarnessCritic 根据失败事件提出字段级修补；主实验未独立列出其型号。训练程序再用成功轨迹微调 Qwen。",
"verdict": "Python运行与失败日志用于定位问题；两组任务分别查目标失败是否修好、原有能力是否退步。具体判分实现和完整数据划分未披露。",
"seed": "已有 Python沙箱、调用格式、最多3次异常重试与上下文截断；没有初始技能库、长期记忆或外部检索。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2606.01779",
"title": "HarnessForge: Joint Harness and Policy Evolution for Adaptive Agent Systems",
"date": "2026-06-01",
"url": "https://arxiv.org/abs/2606.01779",
"priority": "K",
"categories": [
"methods"
],
"tags": [
"HarnessCode",
"Weights",
"JointEvolution",
"MemoryMechanism",
"ExecutableVerifier",
"GoldLabel",
"HeldOut",
"SeparateEvolver",
"Population",
"CoEvolution"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究在不同工具和检索任务中，让框架与专属模型分支共同进化，是否比只改一侧更有效。",
"什么在变": "把一套框架及其匹配模型视为共同进化单位。可改规划（分解、重规划、停止）、动作（参数格式、路由、重试）、记忆（写什么、何时取用）等代码/配置；再训练该框架专属的 LoRA 适配参数。底层任务接口、评测器与工具可用范围保持约束。",
"谁来改 / 谁执行": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"基础 harness": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"Feedback": "ToolHop：最终答案正确率与中间目标完成率各占一半；本地检索问答：答案与参考答案的词级F1；EnvScaler-RL：环境终态满足了多少项程序检查。各候选还记录词元和耗时。GPT-5.5 读这些结果与轨迹定位首次实质错误，再比较任务分、词元、延迟筛候选。TMDB/API-Bank只作迁移测试。",
"Evolution → Eval": "训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。\n\n五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。",
"证据边界": "3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。"
},
"protocol": "Train → selection → test",
"protocolBasis": "3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2606.01779"
],
"links": [
{
"label": "Code",
"url": "https://github.com/mingju-c/HarnessForge"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "methods",
"methodType": "joint",
"brief": {
"summary": "研究在不同工具和检索任务中，让框架与专属模型分支共同进化，是否比只改一侧更有效。",
"novelty": "不是让所有候选框架共享一个训练后的模型，而是保留各自匹配的模型分支；修好的规划、工具和记忆接口所产生的成功轨迹，立即用来训练这套接口的执行者。选择时同时考虑任务分、词元与延迟，避免只把更复杂、开销更大的框架当作进步。",
"object": "规划、工具动作、记忆的框架代码，以及与每个框架匹配的 Qwen LoRA 适配参数。",
"executor": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"modifier": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"roleContext": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"seed": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "ToolHop：最终答案正确率与中间目标完成率各占一半；本地检索问答：答案与参考答案的词级F1；EnvScaler-RL：环境终态满足了多少项程序检查。各候选还记录词元和耗时。GPT-5.5 读这些结果与轨迹定位首次实质错误，再比较任务分、词元、延迟筛候选。TMDB/API-Bank只作迁移测试。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Train → selection → test",
"note": "训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。\n\n五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。"
}
],
"takeaway": "3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2606.01779"
],
"classificationReason": "按持久改进的对象进一步区分四类。",
"protocolDetail": "训练 / 进化数据：训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。\n\n调试 / 选版本数据：从每轮约1,200题中逐步取新的200题子集做两阶段筛选；剩余任务给保留框架收集轨迹。它们属于可用于适配的训练/进化池。候选比较同时看任务表现、词元与延迟，优先保留在这些指标上没有被另一方案全面超越的版本；不能理解成只取正确率最高者。\n\n最终测试数据：五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。\n\n数据隔离与证据边界：3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。",
"cycle": "每轮用约1,200题收集执行证据，GPT-5.5 提出8个框架；先通过可导入、接口与简单运行检查。随后两批各200题筛选：8→4→2，兼顾任务分、词元和耗时。保留者用剩余题完成运行，成功轨迹复用于各自模型适配；主实验共3轮。档案记录父版本、故障、改动和成绩供以后参考。",
"train": "训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。",
"debug": "从每轮约1,200题中逐步取新的200题子集做两阶段筛选；剩余任务给保留框架收集轨迹。它们属于可用于适配的训练/进化池。候选比较同时看任务表现、词元与延迟，优先保留在这些指标上没有被另一方案全面超越的版本；不能理解成只取正确率最高者。",
"test": "五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。",
"isolation": "3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"protocol": "训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。\n\n从每轮约1,200题中逐步取新的200题子集做两阶段筛选；剩余任务给保留框架收集轨迹。它们属于可用于适配的训练/进化池。候选比较同时看任务表现、词元与延迟，优先保留在这些指标上没有被另一方案全面超越的版本；不能理解成只取正确率最高者。\n\n五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。\n\n3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。",
"sections": "附录C.2：模型与适配参数配置；附录D：每条分支的模型适配；§3.2：联合进化流程与起点；附录C.3：代码表示与修改范围；附录B.2–B.3：3,800题及去重隔离；附录C.4.5：两阶段筛选；附录C.1：进化反馈具体计算；附录B.1：各基准测试指标；附录E.1–E.2：迁移与成本口径；附录B.3：测试隔离",
"source": "https://arxiv.org/abs/2606.01779",
"version": "2606.01779v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "75216cde10daf57d79eed6f5a7aad9e40a219bbd92ded973170e14db3bb87de6",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "规划、工具动作、记忆的框架代码，以及与每个框架匹配的 Qwen LoRA 适配参数。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
},
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "ToolHop：最终答案正确率与中间目标完成率各占一半；本地检索问答：答案与参考答案的词级F1；EnvScaler-RL：环境终态满足了多少项程序检查。各候选还记录词元和耗时。GPT-5.5 读这些结果与轨迹定位首次实质错误，再比较任务分、词元、延迟筛候选。TMDB/API-Bank只作迁移测试。",
"sources": [
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "每轮用约1,200题收集执行证据，GPT-5.5 提出8个框架；先通过可导入、接口与简单运行检查。随后两批各200题筛选：8→4→2，兼顾任务分、词元和耗时。保留者用剩余题完成运行，成功轨迹复用于各自模型适配；主实验共3轮。档案记录父版本、故障、改动和成绩供以后参考。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "训练和进化共3,800题：EnvScaler-RL 2,000题（带状态和检查函数的多步环境）；ToolHop 800题；Natural Questions、HotpotQA、2WikiMultiHopQA 合计1,000题（转换成固定本地文档检索任务）。同一池用于框架进化、成功轨迹筛选和LoRA训练，不另虚构一份参数训练题库。",
"sources": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "从每轮约1,200题中逐步取新的200题子集做两阶段筛选；剩余任务给保留框架收集轨迹。它们属于可用于适配的训练/进化池。候选比较同时看任务表现、词元与延迟，优先保留在这些指标上没有被另一方案全面超越的版本；不能理解成只取正确率最高者。",
"sources": [
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "五个数据集、四组实验：ToolHop 195题，RestBench-TMDB 100题，API-Bank 114题，以及本地检索 HotpotQA、2WikiMultiHopQA 两个测试子集。附录B.1没有列出后两子集题数。TMDB和API-Bank用 ToolHop 上进化出的框架直接迁移，不在目标测试题上再次进化。",
"sources": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "3,800题与正式测试集严格分离；同源基准按题号和标准化任务指令去重，本地文档库在适配前固定，测试题号不对搜索、训练或选模开放。候选筛选集仍属适配数据。论文轨迹成本不含外部修改模型生成费用；不能把执行轨迹更少等同总费用必然更低。",
"sources": [
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "不是让所有候选框架共享一个训练后的模型，而是保留各自匹配的模型分支；修好的规划、工具和记忆接口所产生的成功轨迹，立即用来训练这套接口的执行者。选择时同时考虑任务分、词元与延迟，避免只把更复杂、开销更大的框架当作进步。",
"sources": [
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
},
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2606.01779",
"version": "2606.01779v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究在不同工具和检索任务中，让框架与专属模型分支共同进化，是否比只改一侧更有效。",
"feedbackCases": [
{
"label": "EnvScaler-RL：状态环境训练反馈",
"data": "从 EnvScaler-RL 抽2,000题进入3,800题进化/训练池；每题有环境、初态、指令、工具及检查函数。",
"scoring": "对最终环境状态运行任务提供的条件检查，满足项数除以总项数作为完成分；不要求复制某条唯一动作序列。",
"visible": "修改者看到终态完成分、工具观察、轨迹和执行统计。",
"use": "先据失败定位规划/动作/记忆问题，用完成分、耗时和词元筛框架；筛出的成功轨迹再用于参数适配。",
"sources": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
],
"judgment": "EnvScaler-RL 程序逐项检查最终环境状态"
},
{
"label": "ToolHop：工具链进化与测试",
"data": "800题用于进化/训练，195题用于最终测试；题号和标准化指令去重。",
"scoring": "进化任务分=一半标准化答案正确标记+一半中间目标完成率。正式表分别报告答案正确率和路径中间目标完成比例。",
"visible": "开发可见轨迹、工具结果及答案/中间目标分项；最终测试不用于选框架。",
"use": "从错误工具选择、参数或缺少步骤定位修补，再将保留框架用于TMDB/API-Bank迁移。",
"sources": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
},
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
},
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"judgment": "标准化答案正确性 + 程序检查中间目标完成率"
},
{
"label": "本地检索：NQ、HotpotQA、2WikiMultiHopQA",
"data": "三来源共1,000题适配；固定每题本地文档。正式检索测试为 HotpotQA和2WikiMultiHopQA，原文未列两者测试题数。",
"scoring": "本文进化评分和主表检索指标按标准化答案与参考答案算词级F1；两测试子集的总分取宏平均。附录E另对部分搜索基线记为验证精确匹配，不应混成所有方法同一选择指标。",
"visible": "修改者可读适配题检索证据、轨迹及得分；测试题号不可用于搜索、训练或选择。",
"use": "调整证据获取、整理和回答流程；本地库在适配前固定，没有实时网页变化带来的额外反馈。",
"sources": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
},
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
},
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.2：搜索基线的验证指标",
"url": "https://arxiv.org/html/2606.01779#A5.SS2"
}
],
"judgment": "规则计算标准化答案与参考答案的词级 F1"
},
{
"label": "RestBench-TMDB：只做迁移测试",
"data": "100道电影数据库API任务，使用ToolHop进化后的框架迁移；不在TMDB目标测试题上继续调整。",
"scoring": "Success为最终答案是否通过基准检查；Path要求参考API调用序列按顺序出现在预测调用序列中。本文没有进一步点名答案检查器的模型后端。",
"visible": "只在最终迁移运行中取得基准结果；不是给框架修改者的进化反馈。",
"use": "检验工具链经验能否适配新API接口，单独报告成功率和路径通过率。",
"sources": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"judgment": "参考 API 顺序用规则检查；最终答案检查实现未明确"
},
{
"label": "API-Bank：只做迁移测试",
"data": "114道带标准API名称和参数的任务，沿用ToolHop进化结果。",
"scoring": "Success检查完整调用轨迹和最终响应；Path按API名称序列的有序重合计算F1；API Accuracy对齐调用后检查名称及标准化参数是否正确。",
"visible": "基准最终结果用于迁移报告，目标测试样本不参与进化。",
"use": "将答完任务、路径覆盖、单次参数正确三种指标分开，避免统一写成“调用成功”。",
"sources": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"judgment": "规则检查调用名称、标准化参数和序列；完整响应另按基准验收"
}
],
"experiments": [
{
"label": "EnvScaler-RL：训练环境",
"learningCases": [
0
],
"testCases": [],
"evolution": "3,800 道总适配题中的 2,000 道 EnvScaler-RL 任务；环境状态检查提供反馈，成功轨迹用于 LoRA 参数训练。",
"selection": "总适配池每轮约 1,200 题，逐步取新 200 题子集两阶段筛选，综合任务表现、词元和延迟。",
"evaluation": "正式测试列表未单列 EnvScaler-RL；其他任务和迁移测试见后续行。",
"isolation": "训练、框架搜索与候选筛选都属于适配阶段，不能把筛选子集当最终测试。",
"roles": {
"executor": {
"value": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"modifier": {
"value": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"seed": {
"value": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
}
],
"selection": [
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
}
],
"evaluation": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"isolation": [
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
]
}
},
{
"label": "ToolHop：进化后测同类新题",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "总适配池中的 800 道 ToolHop 题用于框架进化、成功轨迹筛选与 LoRA 训练。",
"selection": "在适配池的新子集上两阶段筛选；综合表现、词元与延迟。",
"evaluation": "另外 195 道 ToolHop 测试题。",
"isolation": "适配题与正式测试题按题号和标准化指令去重；测试题不开放给搜索或选模。",
"roles": {
"executor": {
"value": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"modifier": {
"value": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"seed": {
"value": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
}
],
"selection": [
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
}
],
"evaluation": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"isolation": [
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
]
}
},
{
"label": "本地文档检索",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "Natural Questions、HotpotQA、2WikiMultiHopQA 合计 1,000 题，改成固定本地文档检索任务；用于进化和训练。",
"selection": "沿用适配池内的两阶段筛选。",
"evaluation": "HotpotQA 与 2WikiMultiHopQA 的本地检索测试子集；附录 B.1 未列出两者题数。",
"isolation": "文档库适配前固定；适配与测试任务去重，测试题不用于搜索和选模。",
"roles": {
"executor": {
"value": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"modifier": {
"value": "GPT-5.5 担任外部修改者，依次诊断失败、规划修补方向、生成框架代码、按初步运行检查修复；最多3次修复重试。训练程序再用对应框架收集的成功轨迹更新 Qwen 的 LoRA，GPT-5.5 不随此过程训练。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"seed": {
"value": "从一个人工设计的框架和原始 Qwen 开始。论文将框架定义为规划、动作、记忆三个模块的可执行代码/配置，接口检查要求工具格式、动作名和记忆字段有效。方法段没有完整列出初始框架每项启用的功能，因此不能把“允许修改的重试/记忆能力”一律说成初始就已配置。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
}
],
"selection": [
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
}
],
"evaluation": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"isolation": [
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
]
}
},
{
"label": "ToolHop 框架直接迁移",
"learningCases": [
1
],
"testCases": [
3,
4
],
"evolution": "使用在上述 800 道 ToolHop 适配题上进化出的框架。",
"selection": "迁移前完成候选选择，不在目标测试题上继续进化。",
"evaluation": "RestBench-TMDB 100 题；API-Bank 114 题。",
"isolation": "两项目标基准仅用于迁移测试，判分方式分别列在本格。",
"roles": {
"executor": {
"value": "Qwen3-4B 或 Qwen3-8B 配当前框架执行。每个保留下来的框架有自己的一份模型分支，训练新增 LoRA 适配参数（少量附加可训练矩阵），再在下一轮继承；同一个父模型分出的兄弟框架不共享后续适配参数。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"modifier": {
"value": "目标测试阶段不修改；使用 GPT-5.5 在 ToolHop 适配阶段生成并筛选的框架。",
"sources": [
{
"label": "附录C.2：模型与适配参数配置",
"url": "https://arxiv.org/html/2606.01779#A3.SS2"
},
{
"label": "附录C.4：诊断、编写和筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4"
},
{
"label": "附录D：每条分支的模型适配",
"url": "https://arxiv.org/html/2606.01779#A4"
}
]
},
"seed": {
"value": "从 ToolHop 上已进化并选定的框架出发，直接迁移到 TMDB／API-Bank；不是在目标数据集重新搜索。",
"sources": [
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
},
{
"label": "附录C.3：代码表示与修改范围",
"url": "https://arxiv.org/html/2606.01779#A3.SS3"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
}
],
"selection": [
{
"label": "附录C.4.5：两阶段筛选",
"url": "https://arxiv.org/html/2606.01779#A3.SS4.SSS5"
},
{
"label": "附录C.1：进化反馈具体计算",
"url": "https://arxiv.org/html/2606.01779#A3.SS1"
}
],
"evaluation": [
{
"label": "附录B.1：各基准测试指标",
"url": "https://arxiv.org/html/2606.01779#A2.SS1"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
],
"isolation": [
{
"label": "附录B.3：测试隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS3"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "不同任务需要不同的规划、动作和记忆结构，固定 agent 系统很难处处适用。分别优化框架或模型还会留下配合问题：复杂框架可能超出模型执行能力，强模型也可能被不合适的接口限制；作者因此希望让两者共同适应，而非只把其中一方做强。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.01779#S1"
}
]
},
{
"key": "position",
"value": "研究在不同工具和检索任务中，让框架与专属模型分支共同进化，是否比只改一侧更有效。",
"sources": [
{
"label": "§1：框架—模型兼容性问题",
"url": "https://arxiv.org/html/2606.01779#S1"
},
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
}
]
},
{
"key": "conclusion",
"value": "在两种Qwen和五个数据集上，联合方法优于多项单侧更新对照；最大提升是4B在TMDB迁移测试成功率提高12个百分点。3,800道适配题与测试分开，TMDB/API-Bank不参与目标域进化。",
"sources": [
{
"label": "§4.2、表1：主结果",
"url": "https://arxiv.org/html/2606.01779#S4.SS2"
},
{
"label": "附录B.2–B.3：3,800题及去重隔离",
"url": "https://arxiv.org/html/2606.01779#A2.SS2"
},
{
"label": "附录E.1–E.2：迁移与成本口径",
"url": "https://arxiv.org/html/2606.01779#A5.SS1"
}
]
}
],
"fields": {
"object": "规划、工具动作、记忆的框架代码，以及与每个框架匹配的 Qwen LoRA 适配参数。",
"executor": "Qwen3-4B 或 Qwen3-8B；每个保留框架拥有独立的模型适配分支。",
"modifier": "GPT-5.5 根据轨迹诊断、写框架、修运行错误；训练程序用对应框架的成功轨迹更新Qwen适配参数。",
"verdict": "ToolHop核对答案和中间目标；检索问答对照参考答案算F1；EnvScaler-RL用终态检查函数。筛选还计词元和耗时。",
"seed": "人工设计的规划/动作/记忆框架配原始Qwen，需满足固定工具接口；原文未完整列出初始各功能配置，不能将全部可改功能算作已有能力。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"weights"
]
},
{
"id": "2605.30621",
"title": "Harness Updating Is Not Harness Benefit: Disentangling Evolution Capabilities in Self-Evolving LLM Agents",
"date": "2026-05-28",
"url": "https://arxiv.org/abs/2605.30621",
"priority": "K",
"categories": [
"evaluation"
],
"tags": [
"Skill",
"Prompt",
"MemoryContent",
"Prequential",
"ExecutableVerifier",
"LLMJudge",
"SeparateEvolver",
"Streaming",
"org:uiuc"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究模型的基础做题能力，能否预测它提出有用框架更新的能力，以及利用更新后框架获得收益的能力。",
"什么在变": "SWE-bench Verified 和 SkillsBench 只允许修改技能目录，即供后续任务加载的程序经验说明；MCP-Atlas 另外允许改系统提示、向JSONL记忆文件追加条目。不是随意改整个代理运行源码，也没有参数更新。",
"谁来改 / 谁执行": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"基础 harness": "同一基准所有配对从相同起点、同任务流和工具开始。SWE系统提示是828字节的修复问题指南；MCP是1,309字节的API调用指南；SkillsBench不加系统提示，从空技能库开始，去掉基准原配的人工技能，只允许使用此前任务中进化出的技能。",
"Feedback": "SWE用问题对应的隐藏测试检查补丁；MCP-Atlas看最终答案满足了多少条参考事实要求；SkillsBench运行每题确定性检查器。成绩先锁定，再把该次任务的执行证据交给修改者，供以后任务使用。Claude Sonnet 4.6 对“技能是否被遵循”的评审是单独分析，不是这些任务通用的正确性判分。",
"Evolution → Eval": "没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。\n\n最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，并不是另建一份隐藏测试集。",
"证据边界": "每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。"
},
"protocol": "Prequential",
"protocolBasis": "每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2605.30621"
],
"links": [
{
"label": "Code",
"url": "https://github.com/A-EVO-Lab/a-evolve/tree/release/harness-evolution"
}
],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"B-RSI"
],
"category": "evaluation",
"methodType": "",
"brief": {
"summary": "研究模型的基础做题能力，能否预测它提出有用框架更新的能力，以及利用更新后框架获得收益的能力。",
"novelty": "把两个通常混在一起的因素交叉控制：谁能写出有帮助的框架，谁能真正利用框架。再从日志区分“根本没加载技能”和“加载了但后续不按步骤做”，解释为什么能力更弱、理论提升空间更大的模型，实际收益反而可能更小。",
"object": "SWE、SkillsBench改技能目录；MCP-Atlas还改系统提示和追加记忆。工具与模型参数固定。",
"executor": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"modifier": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"roleContext": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"seed": "同一基准所有配对从相同起点、同任务流和工具开始。SWE系统提示是828字节的修复问题指南；MCP是1,309字节的API调用指南；SkillsBench不加系统提示，从空技能库开始，去掉基准原配的人工技能，只允许使用此前任务中进化出的技能。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "SWE用问题对应的隐藏测试检查补丁；MCP-Atlas看最终答案满足了多少条参考事实要求；SkillsBench运行每题确定性检查器。成绩先锁定，再把该次任务的执行证据交给修改者，供以后任务使用。Claude Sonnet 4.6 对“技能是否被遵循”的评审是单独分析，不是这些任务通用的正确性判分。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "Prequential",
"note": "没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。\n\n最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，并不是另建一份隐藏测试集。"
}
],
"takeaway": "每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2605.30621"
],
"classificationReason": "评测框架、诊断与实证分析。",
"protocolDetail": "训练 / 进化数据：没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。\n\n调试 / 选版本数据：当前任务的执行记录和已取得结果供后续更新；不同模型配对共享提示格式、轨迹窗口、更新预算和任务顺序。本文没有单设开发/验证分区，不能把“先计分再用经验”改写成训练—验证—测试三分法。\n\n最终测试数据：最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，不是另建一份隐藏测试集。\n\n数据隔离与证据边界：每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。",
"cycle": "当前框架执行一批任务并锁定成绩→把轨迹和结果整理成固定格式的证据→修改者更新可写技能、提示或记忆→后面的任务使用新版本。论文控制各模型的更新预算和任务轮数；没有额外列出独立验证集挑最佳框架或回放当前题后覆盖旧分数的步骤。",
"train": "没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。",
"debug": "当前任务的执行记录和已取得结果供后续更新；不同模型配对共享提示格式、轨迹窗口、更新预算和任务顺序。本文没有单设开发/验证分区，不能把“先计分再用经验”改写成训练—验证—测试三分法。",
"test": "最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，不是另建一份隐藏测试集。",
"isolation": "每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "同一基准所有配对从相同起点、同任务流和工具开始。SWE系统提示是828字节的修复问题指南；MCP是1,309字节的API调用指南；SkillsBench不加系统提示，从空技能库开始，去掉基准原配的人工技能，只允许使用此前任务中进化出的技能。",
"protocol": "没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。\n\n当前任务的执行记录和已取得结果供后续更新；不同模型配对共享提示格式、轨迹窗口、更新预算和任务顺序。本文没有单设开发/验证分区，不能把“先计分再用经验”改写成训练—验证—测试三分法。\n\n最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，并不是另建一份隐藏测试集。\n\n每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。",
"sections": "§4.1、附录B.2：模型配对；§4.2：修改者能力实验；§4.3：执行者收益与失败分析；附录B.4：初始提示与可写范围；附录B.1：三个任务流；附录B.3：各基准判分及顺序；§3.2：先答题再更新；附录D.3–D.4：技能遵循诊断；§6：范围与限制",
"source": "https://arxiv.org/abs/2605.30621",
"version": "2605.30621v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "2532bb3bd4957a4fba9bae07ec41a97f4174ed11e704f9ae3bf41002274dc410",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "SWE、SkillsBench改技能目录；MCP-Atlas还改系统提示和追加记忆。工具与模型参数固定。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "SWE用问题对应的隐藏测试检查补丁；MCP-Atlas看最终答案满足了多少条参考事实要求；SkillsBench运行每题确定性检查器。成绩先锁定，再把该次任务的执行证据交给修改者，供以后任务使用。Claude Sonnet 4.6 对“技能是否被遵循”的评审是单独分析，不是这些任务通用的正确性判分。",
"sources": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "同一基准所有配对从相同起点、同任务流和工具开始。SWE系统提示是828字节的修复问题指南；MCP是1,309字节的API调用指南；SkillsBench不加系统提示，从空技能库开始，去掉基准原配的人工技能，只允许使用此前任务中进化出的技能。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "当前框架执行一批任务并锁定成绩→把轨迹和结果整理成固定格式的证据→修改者更新可写技能、提示或记忆→后面的任务使用新版本。论文控制各模型的更新预算和任务轮数；没有额外列出独立验证集挑最佳框架或回放当前题后覆盖旧分数的步骤。",
"sources": [
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "没有参数训练集。学习证据来自同一任务流中已经做过的任务：SWE-bench Verified全500题；MCP-Atlas公开500题；SkillsBench 86题。技能或记忆只在这些任务完成、计分之后更新。",
"sources": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "当前任务的执行记录和已取得结果供后续更新；不同模型配对共享提示格式、轨迹窗口、更新预算和任务顺序。本文没有单设开发/验证分区，不能把“先计分再用经验”改写成训练—验证—测试三分法。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "最终统计上述同一条任务流的成绩：SWE补丁通过率；MCP严格全事实满足率及平均事实覆盖分；SkillsBench按每题5次运行的结果平均。另分析技能加载率与加载后遵循率，不是另建一份隐藏测试集。",
"sources": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "每道题的分数在它自己的证据用于更新前锁定，因此该题反馈不反过来提高该题已记录成绩；但经验和评价来自同一连续任务流，不是最终冻结后测未见集合。结论限定在这种技能/提示/记忆编辑空间与三类任务，不能外推为所有代码修改都不需要强模型。",
"sources": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§6：范围与限制",
"url": "https://arxiv.org/html/2605.30621#S6"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把两个通常混在一起的因素交叉控制：谁能写出有帮助的框架，谁能真正利用框架。再从日志区分“根本没加载技能”和“加载了但后续不按步骤做”，解释为什么能力更弱、理论提升空间更大的模型，实际收益反而可能更小。",
"sources": [
{
"label": "§3.3：固定一侧的能力指标",
"url": "https://arxiv.org/html/2605.30621#S3.SS3"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2605.30621",
"version": "2605.30621v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究模型的基础做题能力，能否预测它提出有用框架更新的能力，以及利用更新后框架获得收益的能力。",
"feedbackCases": [
{
"label": "SWE-bench Verified：代码修复",
"data": "全500题、12个Python仓库；同任务流先作答计分、后用证据更新技能。",
"scoring": "补丁须使指定失败测试转为通过，并保持原已通过测试继续通过，才能记resolved=1。",
"visible": "当前题分数及执行轨迹进入后续修改证据；隐藏测试文件仍不可编辑。",
"use": "技能更新只影响后续任务，最终聚合每题原始计分；不回填本题改进后分数。",
"sources": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
}
],
"judgment": "SWE-bench 官方失败转通过／原通过保持通过测试"
},
{
"label": "MCP-Atlas：跨服务工具问答",
"data": "公开500题，涉及36个真实MCP工具服务、220个工具；典型任务需3–6次调用。",
"scoring": "将最终回答逐条对照参考事实要求；全部满足才算严格通过，另报满足比例的连续分。本文此处未点名事实评审器的具体模型。",
"visible": "已完成任务的结果和轨迹供后续修改；可改技能、系统提示及追加记忆。",
"use": "用事实缺失及执行过程定位信息获取/组织问题，不能把后面的技能遵循judge当作此项判分后端。",
"sources": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
],
"judgment": "对照参考事实逐条验收；本文未指定事实裁判模型"
},
{
"label": "SkillsBench：技能效果",
"data": "86题、11领域，从空技能库开始；每题按5次运行结果统计。",
"scoring": "各任务的确定性检查器验收实际产物或状态；按任务和运行取平均。不是用模型笼统评价答案好不好。",
"visible": "先计分，再用执行证据构建或修改后续技能；原生人工技能不提供。",
"use": "既测任务收益，也记录技能是否实际加载；加载与遵循分析和任务正确性分开。",
"sources": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
],
"judgment": "SkillsBench 各任务的确定性产物／状态检查器"
},
{
"label": "单独诊断：加载技能后有没有照做",
"data": "对SkillsBench中已加载技能的轨迹分析，隐藏模型身份；不构成新的训练数据集。",
"scoring": "Claude Sonnet 4.6先把技能拆成固定逐项执行要求，再判断轨迹是否遵循；另一个提示分别给加载后、中段、结束三个阶段0–1遵循分。",
"visible": "这是事后诊断数据，没有说明将该分数返回进化器；不能写成框架更新所用奖励。",
"use": "区分“不会加载”与“会加载但不持续遵循”，解释弱模型为什么未必从技能受益。",
"sources": [
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
},
{
"label": "附录D.4：分阶段遵循评分",
"url": "https://arxiv.org/html/2605.30621#A4.SS4"
}
],
"judgment": "Claude Sonnet 4.6 将技能转成检查要求，再按轨迹评遵循程度"
}
],
"experiments": [
{
"label": "SWE-bench Verified：先计分，再更新",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "全部 500 道代码修复题组成在线任务流；当前题执行完并锁定成绩后，轨迹与结果才用于更新技能／记忆。",
"selection": "没有独立开发集；各模型配对共享任务顺序和更新预算。",
"evaluation": "统计本行同一任务流的成绩；后续题可使用前面已完成任务积累的经验。",
"isolation": "不会用当前题的反馈提高该题已记录分数，但不是冻结后在另一套题上的测试。",
"roles": {
"executor": {
"value": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
]
},
"modifier": {
"value": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
]
},
"seed": {
"value": "SWE-bench 起始系统提示是 828 字节的问题修复指南；配对配置共用同一工具集和任务顺序。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"selection": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"evaluation": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
}
],
"isolation": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§6：范围与限制",
"url": "https://arxiv.org/html/2605.30621#S6"
}
]
}
},
{
"label": "MCP-Atlas：先计分，再更新",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "公开的 500 道工具问答题组成在线任务流；当前题执行完并锁定成绩后，轨迹与结果才用于更新技能／记忆。",
"selection": "没有独立开发集；各模型配对共享任务顺序和更新预算。",
"evaluation": "统计本行同一任务流的成绩；后续题可使用前面已完成任务积累的经验。",
"isolation": "不会用当前题的反馈提高该题已记录分数，但不是冻结后在另一套题上的测试。",
"roles": {
"executor": {
"value": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
]
},
"modifier": {
"value": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
]
},
"seed": {
"value": "MCP-Atlas 起始系统提示是 1,309 字节的 API 调用指南；配对配置共用同一工具集和任务顺序。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"selection": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"evaluation": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
}
],
"isolation": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§6：范围与限制",
"url": "https://arxiv.org/html/2605.30621#S6"
}
]
}
},
{
"label": "SkillsBench：先计分，再更新",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "86 道技能任务，每题五次运行组成在线任务流；当前题执行完并锁定成绩后，轨迹与结果才用于更新技能／记忆。",
"selection": "没有独立开发集；各模型配对共享任务顺序和更新预算。",
"evaluation": "统计本行同一任务流的成绩；后续题可使用前面已完成任务积累的经验。",
"isolation": "不会用当前题的反馈提高该题已记录分数，但不是冻结后在另一套题上的测试。",
"roles": {
"executor": {
"value": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
]
},
"modifier": {
"value": "上述六种模型加 Qwen3.5-9B，共七种修改者，按相同提示、轨迹窗口和预算读执行证据、改允许的外部文件。执行者与修改者独立配对，模型权重都不训练。工具与评测脚本禁止修改，修改者自己的系统提示固定。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
]
},
"seed": {
"value": "SkillsBench 不加系统提示，从空技能库开始，移除基准原有人工技能；只用此前任务进化出的技能。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"selection": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"evaluation": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
}
],
"isolation": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§6：范围与限制",
"url": "https://arxiv.org/html/2605.30621#S6"
}
]
}
},
{
"label": "技能加载与遵循诊断",
"learningCases": [],
"testCases": [
3
],
"evolution": "沿用上述任务执行记录，不另建参数训练集。",
"selection": "该分析用来区分“没有加载技能”和“加载后没有照做”。",
"evaluation": "检查技能加载率与加载后遵循率；不另建隐藏任务测试集。",
"isolation": "诊断指标与任务成功率分开统计。",
"roles": {
"executor": {
"value": "六种执行模型：Claude Opus 4.6、Sonnet 4.6、Haiku 4.5、Qwen3-235B-A22B、Qwen3-32B、GPT-OSS-120B。测修改者时固定 Opus、Sonnet、Qwen3-235B 三个代表执行者；测执行收益时固定同三种代表修改者。Qwen3.5-9B只作为修改者，不列为执行者。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
]
},
"modifier": {
"value": "本行是对执行记录的额外诊断，不产生另一轮修改；实际技能更新由前三行所述七种修改者配置完成。",
"sources": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
}
]
},
"seed": {
"value": "沿用相应任务的框架和技能执行记录做加载／遵循诊断，不另设一个进化起点。",
"sources": [
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"selection": [
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
},
{
"label": "§3.2：先答题再更新",
"url": "https://arxiv.org/html/2605.30621#S3.SS2"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
}
],
"evaluation": [
{
"label": "附录B.1：三个任务流",
"url": "https://arxiv.org/html/2605.30621#A2.SS1"
},
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录D.3–D.4：技能遵循诊断",
"url": "https://arxiv.org/html/2605.30621#A4.SS3"
}
],
"isolation": [
{
"label": "附录B.3：各基准判分及顺序",
"url": "https://arxiv.org/html/2605.30621#A2.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§6：范围与限制",
"url": "https://arxiv.org/html/2605.30621#S6"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有自进化研究主要报告最终任务分数，却没有分清两种能力：修改者能否写出有用的框架，以及执行者能否正确使用改好的框架。两者混在一起，就难知道该选什么模型负责修改、什么模型负责执行，也难确定进一步提升应着力于哪里。",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2605.30621#S1"
}
]
},
{
"key": "position",
"value": "研究模型的基础做题能力，能否预测它提出有用框架更新的能力，以及利用更新后框架获得收益的能力。",
"sources": [
{
"label": "§1：两种能力的分离",
"url": "https://arxiv.org/html/2605.30621#S1"
},
{
"label": "§3.3：固定一侧的能力指标",
"url": "https://arxiv.org/html/2605.30621#S3.SS3"
}
]
},
{
"key": "conclusion",
"value": "在三类任务和受限编辑空间内，不同修改者的收益差距较小；Qwen3.5-9B也能写出有效技能。执行收益则并非越弱越大：弱模型可能不加载技能，或在长任务中逐渐偏离指导。",
"sources": [
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
},
{
"label": "§4.3：执行者收益与失败分析",
"url": "https://arxiv.org/html/2605.30621#S4.SS3"
}
]
}
],
"fields": {
"object": "SWE、SkillsBench改技能目录；MCP-Atlas还改系统提示和追加记忆。工具与模型参数固定。",
"executor": "六种模型分别执行，包括 Claude Opus/Sonnet/Haiku、Qwen3-235B/32B、GPT-OSS-120B；Qwen3.5-9B仅测修改能力。",
"modifier": "七种模型与执行者交叉配对，固定提示和预算改外部文件；完整型号见表格。",
"verdict": "SWE看隐藏测试，MCP看参考事实要求，SkillsBench用任务检查器。先锁定本题成绩，再用本题证据改后续框架。",
"seed": "基准内共享工具与任务提示；SkillsBench从空技能库、无系统提示开始，去掉原配人工技能，确保后续技能来自任务经验。"
}
},
"attributions": [
{
"tag": "org:uiuc",
"label": "UIUC",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2605.30621"
}
]
}
],
"contentType": "paper",
"researchProblems": [
"evaluation",
"improver"
]
},
{
"id": "2609.08175",
"title": "Safe Harness Self-Evolution: A Theoretical Analysis of Feasibility and Limits",
"date": "2026-09-08",
"url": "https://arxiv.org/abs/2609.08175",
"priority": "K",
"categories": [
"theory"
],
"tags": [
"HarnessCode",
"Prompt",
"RegressionGate",
"HeldOut",
"ExecutableVerifier",
"SameModel",
"BenchmarkScore"
],
"depth": [
"未明确"
],
"fields": {
"本质定位": "研究冻结模型的框架自进化在什么条件下能可靠改善总体表现、控制原有能力退化，以及生成、验收和后续进步各自受到什么限制。",
"什么在变": "理论覆盖持久改变提示、工具、代码和调度但冻结模型的框架更新。DS-1000具体改 CandidateHarness 源模块：提示组装、答案提取、模型调用次数与顺序；WorkBuddy只改每个新会话会加载的完整系统角色提示，工具、压缩策略和执行预算都固定。",
"谁来改 / 谁执行": "DS-1000同一冻结 gpt-5.4-mini 模型生成或修订Python框架源码；WorkBuddy由相同DeepSeek模型/代理组合，根据生成分区的题目、旧成绩及失败材料提出4个持久系统角色提示。固定统计验收程序判断能否采用，模型不自己凭一句“我改好了”决定。",
"基础 harness": "DS-1000起点只把公开题目交给模型调用一次，没有多调用编排或后处理搜索；候选可以改变这些外围流程，但静态审查禁止访问文件、进程、环境、网络、隐藏测试和参考解。WorkBuddy起点是已有DeepSeek Harness编程代理，修改只替换系统角色提示，不增工具或改变上下文管理。",
"Feedback": "DS-1000按程序题执行检查得到通过/失败，再分别统计修好了多少旧失败、损害了多少旧成功；WorkBuddy用Code任务的基准评分。两者都另用有限样本算不确定区间决定是否可接受，并保留独立审计任务事后检查。审计点估计看起来变好，不代表区间足够窄、已被可靠验收。",
"Evolution → Eval": "不训练参数。DS-1000排除试运行清单涉及的516题，从剩余484题取480题；分4个上下文，每组8题生成、8题低预算验收、48题高预算验收、56题独立审计，各题只分配一次。WorkBuddy Code全80题按固定哈希分20题生成、20题验收、40题审计。\n\n独立审计并非只测胜者：DS-1000每上下文56题，WorkBuddy40题，均用来检查所有候选，包括被拒绝者，以区分没生成好修改与验收证据不足。另一个DS-1000跨步研究有4条序列、每条2次更新机会，选更新用48题；新生成候选的能力另在不重叠48题上测量。",
"证据边界": "生成、验收和审计数据互斥；WorkBuddy必须完成验收并记录选择后才开放审计。理论保证以固定用户任务分布、固定奖励与声明的抽样/误差条件为前提；有限题集的点估计不能直接当作全分布保证。“安全”在这里主要指改进可信且控制原有任务退化，不是全面的部署安全证明。"
},
"protocol": "混合协议",
"protocolBasis": "生成、验收和审计数据互斥；WorkBuddy必须完成验收并记录选择后才开放审计。理论保证以固定用户任务分布、固定奖励与声明的抽样/误差条件为前提；有限题集的点估计不能直接当作全分布保证。“安全”在这里主要指改进可信且控制原有任务退化，不是全面的部署安全证明。",
"review": "新增 · 方法与实验设置核对",
"sources": [
"https://arxiv.org/abs/2609.08175"
],
"links": [],
"source": "addition",
"year": "2026",
"details": [],
"occurrences": [],
"legacyCategories": [
"H-Full"
],
"category": "theory",
"methodType": "",
"brief": {
"summary": "研究冻结模型的框架自进化在什么条件下能可靠改善总体表现、控制原有能力退化，以及生成、验收和后续进步各自受到什么限制。",
"novelty": "把候选真的更好、生成器能找到它、有限数据能认证它、选中后还能继续进步拆开证明与测量。达到性能上限附近时，识别微小真实改进的最坏情况评估成本会增加；当前执行变好本身，也不能保证下一轮更容易找到好修改。",
"object": "理论覆盖冻结模型的持久框架更新；实证分别改 Python求解框架源码和每个新会话加载的系统角色提示。",
"executor": "理论中是任意固定模型配当前框架。实证分两套：DS-1000所有模型请求返回 gpt-5.4-mini-2026-03-17；WorkBuddy Code使用 deepseek-v4-flash-260425 配 DeepSeek Harness 0.1.0-rc.5（提交47f943859bef）。每套执行模型保持冻结。",
"modifier": "DS-1000同一冻结 gpt-5.4-mini 模型生成或修订Python框架源码；WorkBuddy由相同DeepSeek模型/agent 组合，根据生成分区的题目、旧成绩及失败材料提出4个持久系统角色提示。固定统计验收程序判断能否采用，模型不自己凭一句“我改好了”决定。",
"roleContext": "DS-1000同一冻结 gpt-5.4-mini 模型生成或修订Python框架源码；WorkBuddy由相同DeepSeek模型/代理组合，根据生成分区的题目、旧成绩及失败材料提出4个持久系统角色提示。固定统计验收程序判断能否采用，模型不自己凭一句“我改好了”决定。",
"seed": "DS-1000起点只把公开题目交给模型调用一次，没有多调用编排或后处理搜索；候选可以改变这些外围流程，但静态审查禁止访问文件、进程、环境、网络、隐藏测试和参考解。WorkBuddy起点是已有DeepSeek Harnesscoding agent，修改只替换系统角色提示，不增工具或改变上下文管理。",
"fixed": "当前记录未逐一列出固定部分。",
"verdict": "DS-1000按程序题执行检查得到通过/失败，再分别统计修好了多少旧失败、损害了多少旧成功；WorkBuddy用Code任务的基准评分。两者都另用有限样本算不确定区间决定是否可接受，并保留独立审计任务事后检查。审计点估计看起来变好，不代表区间足够窄、已被可靠验收。",
"diagnosis": "原记录未把“错误定位”与结果打分分开描述；见机制介绍。",
"update": "",
"acceptance": "当前记录未单列 candidate acceptance / rollback 规则。",
"experiments": [
{
"name": "实验协议（保留原记录口径）",
"evolve": "未单独标注",
"selection": "未单独标注",
"test": "未单独标注",
"isolation": "混合协议",
"note": "不训练参数。DS-1000排除试运行清单涉及的516题，从剩余484题取480题；分4个上下文，每组8题生成、8题低预算验收、48题高预算验收、56题独立审计，各题只分配一次。WorkBuddy Code全80题按固定哈希分20题生成、20题验收、40题审计。\n\n独立审计并非只测胜者：DS-1000每上下文56题，WorkBuddy40题，均用来检查所有候选，包括被拒绝者，以区分没生成好修改与验收证据不足。另一个DS-1000跨步研究有4条序列、每条2次更新机会，选更新用48题；新生成候选的能力另在不重叠48题上测量。"
}
],
"takeaway": "生成、验收和审计数据互斥；WorkBuddy必须完成验收并记录选择后才开放审计。理论保证以固定用户任务分布、固定奖励与声明的抽样/误差条件为前提；有限题集的点估计不能直接当作全分布保证。“安全”在这里主要指改进可信且控制原有任务退化，不是全面的部署安全证明。",
"result": "",
"reading": "本轮按原记录重新整理；未标为原文复核。",
"sources": [
"https://arxiv.org/abs/2609.08175"
],
"classificationReason": "形式定义、自指与程序搜索基础。",
"protocolDetail": "训练 / 进化数据：均不训练参数。DS-1000：排除 516 道试运行题后取 480 题，分四组，每组 8 题生成修改。WorkBuddy Code：80 题中 20 题生成修改。\n\n调试 / 选版本数据：DS-1000：每组另用 8 题低预算验收、48 题高预算验收，按后者选择。WorkBuddy：20 题验收四个冻结候选，每个系统取前三次有效运行。\n\n最终测试数据：DS-1000：每组另 56 题；WorkBuddy：另 40 题。均审计所有候选，包括未被采用者。DS-1000 连续修改实验另用 48 题选更新、48 题测量。\n\n数据隔离与证据边界：生成、验收、审计题均互斥；WorkBuddy 完成验收和选择后才开放审计。理论保证另要求固定任务分布、奖励与规定的抽样条件。",
"cycle": "生成候选→独立任务估计修复与退化→用同时控制多个判断错误风险的区间验收→通过才选择，否则保留旧版本→最后独立审计。DS-1000比较独立生成、只看无标签运行记录修订、再加正确性标签修订三种方式；WorkBuddy固定4个提示候选后再验收。跨步实证另以点估计选更新，不等同理论的高置信接受规则。",
"train": "均不训练参数。DS-1000：排除 516 道试运行题后取 480 题，分四组，每组 8 题生成修改。WorkBuddy Code：80 题中 20 题生成修改。",
"debug": "DS-1000：每组另用 8 题低预算验收、48 题高预算验收，按后者选择。WorkBuddy：20 题验收四个冻结候选，每个系统取前三次有效运行。",
"test": "DS-1000：每组另 56 题；WorkBuddy：另 40 题。均审计所有候选，包括未被采用者。DS-1000 连续修改实验另用 48 题选更新、48 题测量。",
"isolation": "生成、验收、审计题均互斥；WorkBuddy 完成验收和选择后才开放审计。理论保证另要求固定任务分布、奖励与规定的抽样条件。"
},
"reviewed": false,
"systemDataAudit": {
"seed": "DS-1000起点只把公开题目交给模型调用一次，没有多调用编排或后处理搜索；候选可以改变这些外围流程，但静态审查禁止访问文件、进程、环境、网络、隐藏测试和参考解。WorkBuddy起点是已有DeepSeek Harness编程代理，修改只替换系统角色提示，不增工具或改变上下文管理。",
"protocol": "不训练参数。DS-1000排除试运行清单涉及的516题，从剩余484题取480题；分4个上下文，每组8题生成、8题低预算验收、48题高预算验收、56题独立审计，各题只分配一次。WorkBuddy Code全80题按固定哈希分20题生成、20题验收、40题审计。\n\nDS-1000每组/每生成方式产生6个候选池，每池4个，共288个终态输出；两套验收数据分别计算区间，按高预算验收结果选择。标签反馈只给生成题的公开题意、生成解、有效性、通过位和错误类别，不给参考解或验收/审计记录。WorkBuddy先冻结4个提示候选，20题验收每个系统取前3次有效运行。\n\n独立审计并非只测胜者：DS-1000每上下文56题，WorkBuddy40题，均用来检查所有候选，包括被拒绝者，以区分没生成好修改与验收证据不足。另一个DS-1000跨步研究有4条序列、每条2次更新机会，选更新用48题；新生成候选的能力另在不重叠48题上测量。\n\n生成、验收和审计数据互斥；WorkBuddy必须完成验收并记录选择后才开放审计。理论保证以固定用户任务分布、固定奖励与声明的抽样/误差条件为前提；有限题集的点估计不能直接当作全分布保证。“安全”在这里主要指改进可信且控制原有任务退化，不是全面的部署安全证明。",
"sections": "§1–2，p1–4：研究对象与假设；附录I.3.4，p84–85：请求与模型身份记录；附录I.3.5，p87–89：WorkBuddy–DSH设置；附录I.3.4，p82–85：DS-1000设置与模型；附录I.3.4，p85–87：跨步骤测量；§4，p7–10：有限数据验收；§6.3，p15–16：两组诊断实验",
"source": "https://arxiv.org/abs/2609.08175",
"version": "2609.08175v1",
"date": "2026-09-11",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。",
"sourceSha256": "e5ae4751ad5bc2fc58a3d6d825b015c43e12606b414b96035364363984025ac5",
"seedStatus": "recorded",
"protocolStatus": "recorded"
},
"profile": {
"fields": [
{
"key": "executor",
"label": "谁执行",
"value": "理论中是任意固定模型配当前框架。实证分两套：DS-1000所有模型请求返回 gpt-5.4-mini-2026-03-17；WorkBuddy Code使用 deepseek-v4-flash-260425 配 DeepSeek Harness 0.1.0-rc.5（提交47f943859bef）。每套执行模型保持冻结。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "modifier",
"label": "谁来改",
"value": "DS-1000同一冻结 gpt-5.4-mini 模型生成或修订Python框架源码；WorkBuddy由相同DeepSeek模型/agent 组合，根据生成分区的题目、旧成绩及失败材料提出4个持久系统角色提示。固定统计验收程序判断能否采用，模型不自己凭一句“我改好了”决定。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
],
"status": "recorded"
},
{
"key": "object",
"label": "什么在进化 / 哪些部分不变",
"value": "理论覆盖冻结模型的持久框架更新；实证分别改 Python求解框架源码和每个新会话加载的系统角色提示。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "verdict",
"label": "反馈是什么 / 怎样判对错",
"value": "DS-1000按程序题执行检查得到通过/失败，再分别统计修好了多少旧失败、损害了多少旧成功；WorkBuddy用Code任务的基准评分。两者都另用有限样本算不确定区间决定是否可接受，并保留独立审计任务事后检查。审计点估计看起来变好，不代表区间足够窄、已被可靠验收。",
"sources": [
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
],
"status": "recorded"
},
{
"key": "seed",
"label": "基础 harness / 初始系统",
"value": "DS-1000起点只把公开题目交给模型调用一次，没有多调用编排或后处理搜索；候选可以改变这些外围流程，但静态审查禁止访问文件、进程、环境、网络、隐藏测试和参考解。WorkBuddy起点是已有DeepSeek Harnesscoding agent，修改只替换系统角色提示，不增工具或改变上下文管理。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "cycle",
"label": "怎样定位问题、修改与保留版本",
"value": "生成候选→独立任务估计修复与退化→用同时控制多个判断错误风险的区间验收→通过才选择，否则保留旧版本→最后独立审计。DS-1000比较独立生成、只看无标签运行记录修订、再加正确性标签修订三种方式；WorkBuddy固定4个提示候选后再验收。跨步实证另以点估计选更新，不等同理论的高置信接受规则。",
"sources": [
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "train",
"label": "训练 / 进化数据",
"value": "均不训练参数。DS-1000：排除 516 道试运行题后取 480 题，分四组，每组 8 题生成修改。WorkBuddy Code：80 题中 20 题生成修改。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "debug",
"label": "调试 / 选版本数据",
"value": "DS-1000：每组另用 8 题低预算验收、48 题高预算验收，按后者选择。WorkBuddy：20 题验收四个冻结候选，每个系统取前三次有效运行。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "test",
"label": "最终测试数据",
"value": "DS-1000：每组另 56 题；WorkBuddy：另 40 题。均审计所有候选，包括未被采用者。DS-1000 连续修改实验另用 48 题选更新、48 题测量。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
}
],
"status": "recorded"
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"value": "生成、验收、审计题均互斥；WorkBuddy 完成验收和选择后才开放审计。理论保证另要求固定任务分布、奖励与规定的抽样条件。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"status": "recorded"
},
{
"key": "novelty",
"label": "关键设计 / 与相近研究的区别",
"value": "把候选真的更好、生成器能找到它、有限数据能认证它、选中后还能继续进步拆开证明与测量。达到性能上限附近时，识别微小真实改进的最坏情况评估成本会增加；当前执行变好本身，也不能保证下一轮更容易找到好修改。",
"sources": [
{
"label": "§3，p4–7：改进目标与生成概率",
"url": "https://arxiv.org/pdf/2609.08175#page=4"
},
{
"label": "§5，p10–13：瓶颈及评估成本",
"url": "https://arxiv.org/pdf/2609.08175#page=10"
},
{
"label": "§6.4，p16–17：多轮保证及实证",
"url": "https://arxiv.org/pdf/2609.08175#page=16"
}
],
"status": "recorded"
}
],
"missing": [],
"partial": [],
"sourceCheck": {
"date": "2026-09-11",
"url": "https://arxiv.org/abs/2609.08175",
"version": "2609.08175v1",
"scope": "核对原文方法、实验设置、结果与相关附录；逐项区分作者披露内容、数据用途和实验边界，不代表独立复现实验。"
},
"focus": "研究冻结模型的框架自进化在什么条件下能可靠改善总体表现、控制原有能力退化，以及生成、验收和后续进步各自受到什么限制。",
"feedbackCases": [
{
"label": "DS-1000：生成、验收与独立审计",
"data": "480题分4个互斥上下文；每组8题生成、8题低预算验收、48题高预算验收、56题审计。3种生成方式各6池×4候选，共288输出。",
"scoring": "对生成解运行程序题检查得0/1。修复比例按全部题计，退化比例按旧版本成功题计；验收用二项比例的置信区间，同时控制多个候选判断的错误风险。每个系统—题目在审计上只有一次结果，是点估计。",
"visible": "无标签修订仅见生成解和运行状态；带结果反馈修订另见生成题通过位及错误类别。验收与审计结果不进入生成提示。",
"use": "固定目标要求修复比例至少0.05、旧成功损害比例不超过0.50；先用区间认证，再选择；独立审计评所有候选，不能反过来替换选择结果。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
}
],
"judgment": "DS-1000 可执行程序测试 + 有限样本统计验收"
},
{
"label": "WorkBuddy Code：DeepSeek系统提示修改",
"data": "80题按20生成/20验收/40审计划分；固定4个候选系统提示，执行模型 deepseek-v4-flash-260425。",
"scoring": "采用Code任务有效运行记录中的0–1基准分，每系统每题取前3次有效尝试；比较对原失败的加权改善与原成功表现变化。基础设施失效单独保留，不当任务0分。",
"visible": "生成只用20题的指令、旧分数与失败材料；候选池冻结、验收和选择完成后才开放40题审计。",
"use": "四候选审计点估计均达声明目标，但区间都不能确认达标，验收全部拒绝、保留旧系统。它是一个固定候选库的诊断，不是生成成功概率的估计。",
"sources": [
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
}
],
"judgment": "采用 WorkBuddy Code 记录的基准分；本文未展开其原始判分器"
},
{
"label": "DS-1000：变好以后还会不会产生好修改",
"data": "4条序列、各2次更新机会；每次4候选在48道选择题上比较。各状态另生成4池×4新候选，在另48道测量题上评价。",
"scoring": "更新选择用满足修复/退化阈值且平均分提高的点估计；测量记录新候选达标比例。选择题与测量题不重叠，同序列测量上下文保持固定。",
"visible": "用于选更新的结果与测后续生成能力的结果分开；不把事后测量回传成下一次选版本依据。",
"use": "新候选平均达标比例从90.6%降到43.8%，但仅是这4条序列的描述，不证明更新导致修改能力下降，也不等同高置信认证实验。",
"sources": [
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
},
{
"label": "§6.4，p16–17：多轮保证及实证",
"url": "https://arxiv.org/pdf/2609.08175#page=16"
}
],
"judgment": "DS-1000 可执行测试；按改善／退化阈值统计候选比例"
},
{
"label": "理论保证：统计条件，而非新增基准",
"data": "固定用户任务分布与固定奖励，分别抽取失败任务和应保留表现的任务；需满足原文的生成/评估分离和抽样条件。",
"scoring": "以真实期望奖励定义改进，再用有限样本的误差界给出接受后改善的概率保证；保证同时计入对旧任务的变化。",
"visible": "规定数据不足、区间过宽或不达阈值就不更新；不能用单次观察到的正确/错误代替真实成功概率。",
"use": "在每步条件和总风险预算都成立时可累加有限多步的改进下界；不承诺每轮一定接受，也不承诺后面始终存在可达改进。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.4，p16–17：多轮保证及实证",
"url": "https://arxiv.org/pdf/2609.08175#page=16"
}
],
"judgment": "理论期望奖励与统计误差界，无新增任务裁判"
}
],
"experiments": [
{
"label": "DS-1000：单次修改的验收",
"learningCases": [
0
],
"testCases": [
0
],
"evolution": "排除 516 道试运行题后，从剩余 484 道取 480 道，分四个上下文；每组八题用于生成修改。",
"selection": "每组另外八题低预算验收、48 题高预算验收；根据高预算验收选择候选。",
"evaluation": "每组另 56 题独立审计；测试全部候选，包括被验收拒绝的候选。",
"isolation": "生成、两套验收、审计题互斥；生成者看不到参考解、验收或审计记录。",
"roles": {
"executor": {
"value": "固定 gpt-5.4-mini-2026-03-17，通过候选 Python 框架调用。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
},
"modifier": {
"value": "同一个冻结 gpt-5.4-mini 生成或修订 Python 框架源码；固定统计验收程序决定能否采用，模型不能自行宣布通过。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
"seed": {
"value": "起点只把公开题目交给模型调用一次；没有多调用编排或后处理搜索。候选可以改外围流程，但不能访问文件、进程、环境、网络、隐藏测试和参考解。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"selection": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"evaluation": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
}
],
"isolation": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
{
"label": "WorkBuddy Code：提示修改",
"learningCases": [
1
],
"testCases": [
1
],
"evolution": "80 道题按固定哈希分配：20 题用于生成。",
"selection": "另 20 题验收四个冻结提示候选，每个系统取前三次有效运行。",
"evaluation": "另外 40 题审计全部候选。",
"isolation": "完成验收并记录选择后才开放审计；三段题互斥。",
"roles": {
"executor": {
"value": "固定 deepseek-v4-flash-260425，配 DeepSeek Harness 0.1.0-rc.5（提交 47f943859bef）。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
},
"modifier": {
"value": "同一 DeepSeek 模型／agent 组合根据生成分区证据提出四个持久系统角色提示；独立的统计验收决定是否采用。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
"seed": {
"value": "已有的 DeepSeek Harness coding agent；只替换系统角色提示，不增加工具或修改上下文管理。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"selection": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"evaluation": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
}
],
"isolation": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
{
"label": "DS-1000：连续修改",
"learningCases": [
2
],
"testCases": [
2
],
"evolution": "四条序列，每条两次更新机会，考察修改后的系统能否继续生成好修改。",
"selection": "用 48 题选择更新。",
"evaluation": "在不重叠的另 48 题上测量新候选的能力。",
"isolation": "选更新与测量用题分离；与前述单次验收实验分开。",
"roles": {
"executor": {
"value": "固定 gpt-5.4-mini-2026-03-17，通过候选 Python 框架调用。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
},
"modifier": {
"value": "同一个冻结 gpt-5.4-mini 生成或修订 Python 框架源码；固定统计验收程序决定能否采用，模型不能自行宣布通过。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
"seed": {
"value": "沿用 DS-1000 的 Python 框架修改空间，研究接受一次修改后的系统能否继续产生好修改；不是 WorkBuddy 的系统提示编辑。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"selection": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"evaluation": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
}
],
"isolation": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
{
"label": "理论保证的适用条件",
"learningCases": [],
"testCases": [
3
],
"evolution": "理论分析，不是新增训练数据或经验进化实验。",
"selection": "以固定任务分布、固定奖励以及声明的抽样和误差条件为前提。",
"evaluation": "数学保证不等同有限题集上的点估计，也不是另一项基准测试。",
"isolation": "这里主要讨论可信改进及控制原有任务退化；保证受理论条件约束。",
"roles": {
"executor": {
"value": "理论针对任意固定模型与当前框架，不指定实验模型型号。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
},
"modifier": {
"value": "理论分析候选生成与统计验收的条件，不另指定一个实验修改者。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.4，p84–85：请求与模型身份记录",
"url": "https://arxiv.org/pdf/2609.08175#page=84"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
"seed": {
"value": "抽象的当前运行框架；经验实验的具体起点分别见 DS-1000 与 WorkBuddy 行。",
"sources": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
},
"sources": {
"evolution": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"selection": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
],
"evaluation": [
{
"label": "附录I.3.4，p82–85：DS-1000设置与模型",
"url": "https://arxiv.org/pdf/2609.08175#page=82"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
},
{
"label": "附录I.3.4，p85–87：跨步骤测量",
"url": "https://arxiv.org/pdf/2609.08175#page=85"
}
],
"isolation": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "附录I.3.5，p87–89：WorkBuddy–DSH设置",
"url": "https://arxiv.org/pdf/2609.08175#page=87"
}
]
}
}
],
"sourceDate": "2026-09-11"
},
"overview": {
"tldr": [
{
"key": "gap",
"value": "修复触发修改的失败任务，仍可能损害其他任务；有限次数评估又未必能发现成功概率正在下降，退化可能在多轮更新中累积。已有实证进展因此还留下理论问题：能否产生合格修改、有限证据是否足以支持采用，以及当前变好是否还能保证下一轮继续改进。",
"sources": [
{
"label": "§1 Introduction（第 1–3 页）",
"url": "https://arxiv.org/pdf/2609.08175v1#page=1"
}
]
},
{
"key": "position",
"value": "研究冻结模型的框架自进化在什么条件下能可靠改善总体表现、控制原有能力退化，以及生成、验收和后续进步各自受到什么限制。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§3，p4–7：改进目标与生成概率",
"url": "https://arxiv.org/pdf/2609.08175#page=4"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
{
"key": "conclusion",
"value": "给出单步与有限多步的条件性保证，并指出评估成本和后续可改进性的限制。DS-1000与WorkBuddy诊断中，审计点估计常有改善，但验收区间仍不足以批准；停滞不一定意味着没有好修改。",
"sources": [
{
"label": "§5，p10–13：瓶颈及评估成本",
"url": "https://arxiv.org/pdf/2609.08175#page=10"
},
{
"label": "§6.3，p15–16：两组诊断实验",
"url": "https://arxiv.org/pdf/2609.08175#page=15"
},
{
"label": "§6.4，p16–17：多轮保证及实证",
"url": "https://arxiv.org/pdf/2609.08175#page=16"
}
]
}
],
"fields": {
"object": "理论覆盖冻结模型的持久框架更新；实证分别改 Python求解框架源码和每个新会话加载的系统角色提示。",
"executor": "DS-1000：gpt-5.4-mini-2026-03-17；WorkBuddy Code：deepseek-v4-flash-260425 + DeepSeek Harnesscoding agent。",
"modifier": "各自同一冻结模型产生框架或提示候选；独立统计验收程序决定是否有足够证据采用。",
"verdict": "DS-1000 用可执行测试判对错；WorkBuddy 使用其基准分，论文未展开原始判分器。验收程序统计修复与退化及其不确定性，判断是否采用；独立审计题检查验收结果。",
"seed": "DS-1000只调用一次模型；WorkBuddy沿用已有 coding agent，只开放系统提示修改。“安全”主要指可靠改善并控制任务退化。"
}
},
"attributions": [],
"contentType": "paper",
"researchProblems": [
"evaluation",
"improver",
"harness"
]
},
{
"id": "2609.14858",
"title": "Dream-RSI: Recursive Self-Improvement through Evolving Worlds",
"date": "2026-09-14",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2609.14858",
"priority": "C",
"priorityBasis": "Google DeepMind 参与署名；直接研究探索策略自改进，列为重点代表作。",
"category": "methods",
"categories": [
"methods"
],
"methodType": "harness",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"HarnessCode",
"Improver",
"Workflow",
"OfflineSearch",
"org:google-deepmind"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码",
"url": "https://github.com/zhengkid/Dream-RSI"
},
{
"label": "项目",
"url": "https://dream-rsi.com/"
}
],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2609.14858"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
},
{
"label": "附录 A 任务定义及数值检查",
"url": "https://arxiv.org/html/2609.14858#A1"
},
{
"label": "附录 B.2 策略修改提示",
"url": "https://arxiv.org/html/2609.14858#A2.SS2"
}
],
"fields": {
"本质定位": "研究能否利用已有发现过程，低成本改进“下一轮怎样探索”的策略，而不更换或训练底层 coding agent。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "Lasso 的留出结果支持解程序的跨数据泛化；不能据此声称探索策略已通过独立跨任务测试。历史回放用于选策略，不能同时当成独立测试。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "固定搜索策略不会随经验改善；直接在线试验新策略又需要大量完整搜索才能知道好坏，导致改进搜索策略本身非常昂贵。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
}
]
},
{
"key": "position",
"value": "研究能否利用已有发现过程，低成本改进“下一轮怎样探索”的策略，而不更换或训练底层 coding agent。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
]
},
{
"key": "conclusion",
"value": "在算法、数学优化和 GPU kernel 三类任务中，以较少发现成本达到相当或更好的结果。Lasso 另有未参与搜索的数据集测试；其他任务主要衡量同一问题上的搜索质量。",
"sources": [
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
}
],
"fields": {
"object": "改可执行的探索策略代码：选哪些分支、开多少并行工作区、每条分支继续多久、何时停止。生成的解程序也会更新；底层 coding agent、模型参数及判分器固定。",
"executor": "Lasso：Gemini-3.1 Pro 或 Gemini-3.7-Flash，经 Gemini CLI 执行代码搜索；数学与 KernelBench：Gemini-3.1 Pro。",
"modifier": "固定的策略开发 agent 阅读历史回放分数与策略源码，再改探索策略。正文单列了这一角色，但没有单独给出其模型型号，不将执行模型型号自动当作修改者型号。",
"verdict": "Lasso：对照 sklearn 检查目标函数误差，通过后按运行时间评分；数学：程序计算集合、圆布局或自相关的目标值与约束；KernelBench：与参考实现比数值结果，通过后测速度。策略候选在历史树上回放，直接复用已保存的这些结果。",
"seed": "Gemini CLI 外加工作区调度层，支持并行生成、逐步修订和执行评测。初始探索策略固定分支数与步数；Dream-RSI 将这层调度代码开放给修改者。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "改可执行的探索策略代码：选哪些分支、开多少并行工作区、每条分支继续多久、何时停止。生成的解程序也会更新；底层 coding agent、模型参数及判分器固定。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "Lasso：Gemini-3.1 Pro 或 Gemini-3.7-Flash，经 Gemini CLI 执行代码搜索；数学与 KernelBench：Gemini-3.1 Pro。",
"sources": [
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "固定的策略开发 agent 阅读历史回放分数与策略源码，再改探索策略。正文单列了这一角色，但没有单独给出其模型型号，不将执行模型型号自动当作修改者型号。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "附录 B.2 策略修改提示",
"url": "https://arxiv.org/html/2609.14858#A2.SS2"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "Lasso：对照 sklearn 检查目标函数误差，通过后按运行时间评分；数学：程序计算集合、圆布局或自相关的目标值与约束；KernelBench：与参考实现比数值结果，通过后测速度。策略候选在历史树上回放，直接复用已保存的这些结果。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
},
{
"label": "附录 A 任务定义及数值检查",
"url": "https://arxiv.org/html/2609.14858#A1"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "Gemini CLI 外加工作区调度层，支持并行生成、逐步修订和执行评测。初始探索策略固定分支数与步数；Dream-RSI 将这层调度代码开放给修改者。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "真实搜索并记录发现树 → 把树变成可回放的历史环境 → agent 修改策略并在同一历史池比较候选 → 部署最高分策略 → 新搜索继续扩充历史池。回放不能评价历史中从未出现的新分支。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "不训练模型参数。Lasso 使用 SimpleTES 的 17 个合成实例搜索；数学搜索 Sum–Difference、Circle Packing、Autocorrelation 三题；GPU 搜索 KernelBench 的 VGG16、LayerNorm、ConvDiv、ConvMax 四项。",
"sources": [
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "策略选版本使用已完成搜索生成的发现树，候选在同一历史池回放比较。Lasso 的正确性检查实例与计时实例另行生成、互不相同。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "附录 A 任务定义及数值检查",
"url": "https://arxiv.org/html/2609.14858#A1"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "Lasso 另测 Gisette、RCV1、DNA、Leukemia、Colon、Duke Breast 六个留出数据集。数学三题与 KernelBench 四项报告各自搜索所得解的质量和成本，不是另一个未见任务集。",
"sources": [
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "Lasso 的留出结果支持解程序的跨数据泛化；不能据此声称探索策略已通过独立跨任务测试。历史回放用于选策略，不能同时当成独立测试。",
"sources": [
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "将历史发现树变成可运行的策略试验环境，反复比较调度代码；不同于只把历史轨迹总结成提示词或记忆。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究能否利用已有发现过程，低成本改进“下一轮怎样探索”的策略，而不更换或训练底层 coding agent。",
"object": "改可执行的探索策略代码：选哪些分支、开多少并行工作区、每条分支继续多久、何时停止。生成的解程序也会更新；底层 coding agent、模型参数及判分器固定。",
"executor": "Lasso：Gemini-3.1 Pro 或 Gemini-3.7-Flash，经 Gemini CLI 执行代码搜索；数学与 KernelBench：Gemini-3.1 Pro。",
"modifier": "固定的策略开发 agent 阅读历史回放分数与策略源码，再改探索策略。正文单列了这一角色，但没有单独给出其模型型号，不将执行模型型号自动当作修改者型号。",
"verdict": "Lasso：对照 sklearn 检查目标函数误差，通过后按运行时间评分；数学：程序计算集合、圆布局或自相关的目标值与约束；KernelBench：与参考实现比数值结果，通过后测速度。策略候选在历史树上回放，直接复用已保存的这些结果。",
"seed": "Gemini CLI 外加工作区调度层，支持并行生成、逐步修订和执行评测。初始探索策略固定分支数与步数；Dream-RSI 将这层调度代码开放给修改者。",
"cycle": "真实搜索并记录发现树 → 把树变成可回放的历史环境 → agent 修改策略并在同一历史池比较候选 → 部署最高分策略 → 新搜索继续扩充历史池。回放不能评价历史中从未出现的新分支。",
"train": "不训练模型参数。Lasso 使用 SimpleTES 的 17 个合成实例搜索；数学搜索 Sum–Difference、Circle Packing、Autocorrelation 三题；GPU 搜索 KernelBench 的 VGG16、LayerNorm、ConvDiv、ConvMax 四项。",
"debug": "策略选版本使用已完成搜索生成的发现树，候选在同一历史池回放比较。Lasso 的正确性检查实例与计时实例另行生成、互不相同。",
"test": "Lasso 另测 Gisette、RCV1、DNA、Leukemia、Colon、Duke Breast 六个留出数据集。数学三题与 KernelBench 四项报告各自搜索所得解的质量和成本，不是另一个未见任务集。",
"isolation": "Lasso 的留出结果支持解程序的跨数据泛化；不能据此声称探索策略已通过独立跨任务测试。历史回放用于选策略，不能同时当成独立测试。",
"novelty": "将历史发现树变成可运行的策略试验环境，反复比较调度代码；不同于只把历史轨迹总结成提示词或记忆。"
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.14858"
}
]
}
],
"researchProblems": [
"improver",
"harness"
]
},
{
"id": "2608.26530",
"title": "PILOT in the Loop: Live Self-Improvement for Long-Horizon Agents",
"date": "2026-08-27",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2608.26530",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "methods",
"categories": [
"methods"
],
"methodType": "harness",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Skill",
"MemoryContent",
"SeparateEvolver",
"Online",
"SameSet"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码",
"url": "https://github.com/XiaoYang66/Pilot"
}
],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2608.26530"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2608.26530#S1"
},
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
},
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
},
{
"label": "附录 A.3 经验保存指令",
"url": "https://arxiv.org/html/2608.26530#A1.SS3"
}
],
"fields": {
"本质定位": "研究把任务执行与监督分开，能否在运行过程中纠偏，并把当场发现的经验变成以后可用的技能。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "同一轮各任务隔离；跨轮重复相同题目，因此进化曲线不是 train/test 隔离实验。SWE-bench 结果是一次执行比较，不是把 Terminal-Bench 进化后的技能拿去做迁移测试。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "事后反思只能帮助后续任务，救不了仍在进行的失败；单 agent 又要执行又要诊断，而普通委派往往要等 worker 结束才能干预。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2608.26530#S1"
}
]
},
{
"key": "position",
"value": "研究把任务执行与监督分开，能否在运行过程中纠偏，并把当场发现的经验变成以后可用的技能。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2608.26530#S1"
}
]
},
{
"key": "conclusion",
"value": "两个冻结模型、三个 benchmark 的六组组合中五组最好；重复 Terminal-Bench 2.0 的实验也有提升，但使用同一批题反复进化，不能解释为未见任务的泛化提升。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
}
]
}
],
"fields": {
"object": "持久变化的是技能文件和长期记忆；运行中的纠偏消息影响当前 worker。模型参数不变，论文没有让 agent 重写整个 Pi runtime。",
"executor": "分别用 GLM-5.1、Kimi-K2.6；每组 supervisor 与 worker 都使用该组同一个冻结模型，运行在独立会话上下文中。",
"modifier": "supervisor 查看 worker 的通知、问题和必要的轨迹，发送纠偏消息、终止无效 worker，并写入可复用技能或记忆。",
"verdict": "运行中只看工具输出、错误、进度及轨迹，不给 benchmark 分数。每轮结束后用 Terminal-Bench verifier 的成功／失败决定保留哪些运行产生的技能；SWE-bench 两组使用任务测试评价修复结果。",
"seed": "在 Pi coding-agent runtime 上增加 supervisor–worker 双向通道，可通知、提问、指导和终止；另有技能库与长期记忆。进化比较从共同的初始技能库开始。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "持久变化的是技能文件和长期记忆；运行中的纠偏消息影响当前 worker。模型参数不变，论文没有让 agent 重写整个 Pi runtime。",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
},
{
"label": "附录 A.3 经验保存指令",
"url": "https://arxiv.org/html/2608.26530#A1.SS3"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "分别用 GLM-5.1、Kimi-K2.6；每组 supervisor 与 worker 都使用该组同一个冻结模型，运行在独立会话上下文中。",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
},
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "supervisor 查看 worker 的通知、问题和必要的轨迹，发送纠偏消息、终止无效 worker，并写入可复用技能或记忆。",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "运行中只看工具输出、错误、进度及轨迹，不给 benchmark 分数。每轮结束后用 Terminal-Bench verifier 的成功／失败决定保留哪些运行产生的技能；SWE-bench 两组使用任务测试评价修复结果。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "在 Pi coding-agent runtime 上增加 supervisor–worker 双向通道，可通知、提问、指导和终止；另有技能库与长期记忆。进化比较从共同的初始技能库开始。",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
},
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "各题从同一轮共享技能库的隔离副本启动 → 运行中写经验 → 全轮结束后，仅合并 verifier 判成功的运行所产经验 → 下一轮再加载。verifier 不参与撰写经验。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "不训练参数。持续进化实验反复运行 Terminal-Bench 2.0 的 89 道任务，共 20 次改进迭代；经验来自这些任务的实时轨迹。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "每轮结束后按该轮题目的 verifier 结果筛选经验；GLM-5.1 的 PILOT／Pi／OpenCode 使用相同初始库和经验保存指令，各自独立更新。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "附录 A.3 经验保存指令",
"url": "https://arxiv.org/html/2608.26530#A1.SS3"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "一次执行比较：Terminal-Bench 2.0、SWE-bench Multilingual、SWE-bench Pro，各题从新状态开始。持续进化结果：同一 Terminal-Bench 2.0，报告历轮最高表现。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "同一轮各任务隔离；跨轮重复相同题目，因此进化曲线不是 train/test 隔离实验。SWE-bench 结果是一次执行比较，不是把 Terminal-Bench 进化后的技能拿去做迁移测试。",
"sources": [
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "监督者可以在 worker 尚未结束时指导下一步，同时将经验持久化；区别于只在任务结束后做反思。",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究把任务执行与监督分开，能否在运行过程中纠偏，并把当场发现的经验变成以后可用的技能。",
"object": "持久变化的是技能文件和长期记忆；运行中的纠偏消息影响当前 worker。模型参数不变，论文没有让 agent 重写整个 Pi runtime。",
"executor": "分别用 GLM-5.1、Kimi-K2.6；每组 supervisor 与 worker 都使用该组同一个冻结模型，运行在独立会话上下文中。",
"modifier": "supervisor 查看 worker 的通知、问题和必要的轨迹，发送纠偏消息、终止无效 worker，并写入可复用技能或记忆。",
"verdict": "运行中只看工具输出、错误、进度及轨迹，不给 benchmark 分数。每轮结束后用 Terminal-Bench verifier 的成功／失败决定保留哪些运行产生的技能；SWE-bench 两组使用任务测试评价修复结果。",
"seed": "在 Pi coding-agent runtime 上增加 supervisor–worker 双向通道，可通知、提问、指导和终止；另有技能库与长期记忆。进化比较从共同的初始技能库开始。",
"cycle": "各题从同一轮共享技能库的隔离副本启动 → 运行中写经验 → 全轮结束后，仅合并 verifier 判成功的运行所产经验 → 下一轮再加载。verifier 不参与撰写经验。",
"train": "不训练参数。持续进化实验反复运行 Terminal-Bench 2.0 的 89 道任务，共 20 次改进迭代；经验来自这些任务的实时轨迹。",
"debug": "每轮结束后按该轮题目的 verifier 结果筛选经验；GLM-5.1 的 PILOT／Pi／OpenCode 使用相同初始库和经验保存指令，各自独立更新。",
"test": "一次执行比较：Terminal-Bench 2.0、SWE-bench Multilingual、SWE-bench Pro，各题从新状态开始。持续进化结果：同一 Terminal-Bench 2.0，报告历轮最高表现。",
"isolation": "同一轮各任务隔离；跨轮重复相同题目，因此进化曲线不是 train/test 隔离实验。SWE-bench 结果是一次执行比较，不是把 Terminal-Bench 进化后的技能拿去做迁移测试。",
"novelty": "监督者可以在 worker 尚未结束时指导下一步，同时将经验持久化；区别于只在任务结束后做反思。"
},
"attributions": [],
"researchProblems": [
"harness",
"experience"
]
},
{
"id": "2607.12227",
"title": "Rethinking the Evaluation of Harness Evolution for Agents",
"date": "2026-07-14",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2607.12227",
"priority": "C",
"priorityBasis": "University of Washington / Allen Institute for AI 参与；直接检验 harness 进化的核心评测问题。",
"category": "evaluation",
"categories": [
"evaluation"
],
"methodType": "",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"HarnessCode",
"ExecutableVerifier",
"LLMJudge",
"SameSet",
"HeldOut",
"org:washington",
"org:allenai",
"person:hannaneh-hajishirzi",
"person:yulia-tsvetkov"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码",
"url": "https://github.com/rethinking-harness-evolution"
}
],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2607.12227"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
},
{
"label": "§4.2 无测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS2"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
},
{
"label": "§5.1–5.2 修改分析与限制",
"url": "https://arxiv.org/html/2607.12227#S5"
}
],
"fields": {
"本质定位": "检验 harness 进化是否优于相同反馈与尝试预算下的直接重试，以及进化后的框架能否迁移到未参与搜索的任务。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "必须分开看两组结论：同题成绩衡量自适应搜索，45/10/34 设置才检验跨题泛化。相同尝试数也不意味着每种方法实际 token 开销完全一致。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "进化时反复用 benchmark 反馈，最后又在同一批题上报分，可能把更多尝试或记住题目带来的收益当成框架改进；缺少相同预算的简单基线也会放大这种误读。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
}
]
},
{
"key": "position",
"value": "检验 harness 进化是否优于相同反馈与尝试预算下的直接重试，以及进化后的框架能否迁移到未参与搜索的任务。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
}
]
},
{
"key": "conclusion",
"value": "在 Terminal-Bench 2.1 上，harness 进化没有稳定胜过并行采样或逐次修订；45/10/34 划分下，留出测试平均只提高 0.6 个百分点。结论受所测模型、框架和任务范围限制。",
"sources": [
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
},
{
"label": "§5.1–5.2 修改分析与限制",
"url": "https://arxiv.org/html/2607.12227#S5"
}
]
}
],
"fields": {
"object": "对比四种改进对象：独立生成多个答案、根据旧轨迹修订答案、跨任务共享的 harness、每题单独定制的 harness。模型参数全部固定。",
"executor": "无测试反馈：Claude Opus 4.6、GPT-5.4、GPT-5.4 mini；有测试反馈及留出测试：Claude Opus 4.6、GPT-5.4。",
"modifier": "每组采用对应模型生成轨迹总结；meta-agent 据此修改提示、工具说明、记忆或 middleware。直接重试基线只改答案／轨迹，不改共享框架。",
"verdict": "无测试反馈时，修改只看轨迹，并行候选由模型自己选；有测试反馈时，Terminal-Bench 的可执行单元测试返回对错，供修订和选结果。最终成功率都由 benchmark 测试判定。",
"seed": "所有方法共用 AHE 的初始 harness：基础提示与 shell 执行工具。进化允许增加提示规则、工具层或运行时检查；直接重试保持该框架固定。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "对比四种改进对象：独立生成多个答案、根据旧轨迹修订答案、跨任务共享的 harness、每题单独定制的 harness。模型参数全部固定。",
"sources": [
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "无测试反馈：Claude Opus 4.6、GPT-5.4、GPT-5.4 mini；有测试反馈及留出测试：Claude Opus 4.6、GPT-5.4。",
"sources": [
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
},
{
"label": "§4.2 无测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS2"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "每组采用对应模型生成轨迹总结；meta-agent 据此修改提示、工具说明、记忆或 middleware。直接重试基线只改答案／轨迹，不改共享框架。",
"sources": [
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§5.1–5.2 修改分析与限制",
"url": "https://arxiv.org/html/2607.12227#S5"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "无测试反馈时，修改只看轨迹，并行候选由模型自己选；有测试反馈时，Terminal-Bench 的可执行单元测试返回对错，供修订和选结果。最终成功率都由 benchmark 测试判定。",
"sources": [
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.2 无测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS2"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "所有方法共用 AHE 的初始 harness：基础提示与 shell 执行工具。进化允许增加提示规则、工具层或运行时检查；直接重试保持该框架固定。",
"sources": [
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
},
{
"label": "§5.1–5.2 修改分析与限制",
"url": "https://arxiv.org/html/2607.12227#S5"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "统一每题 5 次尝试预算；比较将预算花在并行探索、逐次改解、跨题改框架或单题改框架。候选选择是否可用单元测试，与各组反馈条件保持一致。",
"sources": [
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "同题比较在 Terminal-Bench 2.1 的 89 题上反复尝试／进化。泛化实验另外把同一 benchmark 分为 45 题进化、10 题验证、34 题测试。",
"sources": [
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "泛化实验根据 10 道验证题选最佳 harness；45 道进化题提供单元测试反馈。同题比较直接用当前题的轨迹或测试结果，没有独立验证集。",
"sources": [
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "同题比较仍测这 89 题，分别报告无测试反馈与有测试反馈设置；泛化实验只在另 34 道留出题测选好的框架。",
"sources": [
{
"label": "§4.2 无测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS2"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "必须分开看两组结论：同题成绩衡量自适应搜索，45/10/34 设置才检验跨题泛化。相同尝试数也不意味着每种方法实际 token 开销完全一致。",
"sources": [
{
"label": "§4.1 初始框架、模型与预算",
"url": "https://arxiv.org/html/2607.12227#S4.SS1"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "同时控制反馈可见性与尝试预算，并把“同题反复搜索”与“留出任务可复用框架”拆开评估。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "检验 harness 进化是否优于相同反馈与尝试预算下的直接重试，以及进化后的框架能否迁移到未参与搜索的任务。",
"object": "对比四种改进对象：独立生成多个答案、根据旧轨迹修订答案、跨任务共享的 harness、每题单独定制的 harness。模型参数全部固定。",
"executor": "无测试反馈：Claude Opus 4.6、GPT-5.4、GPT-5.4 mini；有测试反馈及留出测试：Claude Opus 4.6、GPT-5.4。",
"modifier": "每组采用对应模型生成轨迹总结；meta-agent 据此修改提示、工具说明、记忆或 middleware。直接重试基线只改答案／轨迹，不改共享框架。",
"verdict": "无测试反馈时，修改只看轨迹，并行候选由模型自己选；有测试反馈时，Terminal-Bench 的可执行单元测试返回对错，供修订和选结果。最终成功率都由 benchmark 测试判定。",
"seed": "所有方法共用 AHE 的初始 harness：基础提示与 shell 执行工具。进化允许增加提示规则、工具层或运行时检查；直接重试保持该框架固定。",
"cycle": "统一每题 5 次尝试预算；比较将预算花在并行探索、逐次改解、跨题改框架或单题改框架。候选选择是否可用单元测试，与各组反馈条件保持一致。",
"train": "同题比较在 Terminal-Bench 2.1 的 89 题上反复尝试／进化。泛化实验另外把同一 benchmark 分为 45 题进化、10 题验证、34 题测试。",
"debug": "泛化实验根据 10 道验证题选最佳 harness；45 道进化题提供单元测试反馈。同题比较直接用当前题的轨迹或测试结果，没有独立验证集。",
"test": "同题比较仍测这 89 题，分别报告无测试反馈与有测试反馈设置；泛化实验只在另 34 道留出题测选好的框架。",
"isolation": "必须分开看两组结论：同题成绩衡量自适应搜索，45/10/34 设置才检验跨题泛化。相同尝试数也不意味着每种方法实际 token 开销完全一致。",
"novelty": "同时控制反馈可见性与尝试预算，并把“同题反复搜索”与“留出任务可复用框架”拆开评估。"
},
"attributions": [
{
"tag": "org:washington",
"label": "University of Washington",
"kind": "institution",
"sources": [
{
"label": "论文作者与机构署名",
"url": "https://arxiv.org/html/2607.12227"
}
]
},
{
"tag": "org:allenai",
"label": "Allen Institute for AI",
"kind": "institution",
"sources": [
{
"label": "论文作者与机构署名",
"url": "https://arxiv.org/html/2607.12227"
}
]
},
{
"tag": "person:hannaneh-hajishirzi",
"label": "Hannaneh Hajishirzi",
"kind": "scholar",
"sources": [
{
"label": "论文作者与机构署名",
"url": "https://arxiv.org/html/2607.12227"
}
]
},
{
"tag": "person:yulia-tsvetkov",
"label": "Yulia Tsvetkov",
"kind": "scholar",
"sources": [
{
"label": "论文作者与机构署名",
"url": "https://arxiv.org/html/2607.12227"
}
]
}
],
"researchProblems": [
"evaluation"
]
},
{
"id": "2609.00768",
"title": "DiagEvo: Diagnosis-Guided Self-Evolution via Hierarchical Error Memory",
"date": "2026-09-01",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2609.00768",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "methods",
"categories": [
"methods"
],
"methodType": "weights",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Weights",
"Data",
"MemoryContent",
"SelfFeedback",
"LLMJudge",
"org:meituan-longcat"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2609.00768"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.00768#S1"
},
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "§3.2 伪标签和奖励",
"url": "https://arxiv.org/html/2609.00768#S3.SS2"
},
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
},
{
"label": "§4.2–4.4 结果与分析",
"url": "https://arxiv.org/html/2609.00768#S4.SS2"
},
{
"label": "附录 E 训练配置",
"url": "https://arxiv.org/html/2609.00768#A5"
},
{
"label": "附录 A 局限",
"url": "https://arxiv.org/html/2609.00768#A1"
}
],
"fields": {
"本质定位": "研究能否从模型自己的失败历史提取明确的错误原因，并据此生成下一轮训练课程，不使用外部任务材料。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "测试 benchmark 不供训练循环取题或取标准答案；训练伪标签与测试判分来源不同。自生成题无需人工标签，不等于多数票能证明答案正确。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "自主出题能持续训练模型，但仅控制难度和多样性不能告诉出题者“下一轮要补什么能力”；依赖外部题库或教师又削弱了自博弈的独立性。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.00768#S1"
}
]
},
{
"key": "position",
"value": "研究能否从模型自己的失败历史提取明确的错误原因，并据此生成下一轮训练课程，不使用外部任务材料。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.00768#S1"
},
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
}
]
},
{
"key": "conclusion",
"value": "默认 4B 诊断模型下，三个 solver 的九项评测均分都超过比较方法。Qwen3-8B 的数学均分为 72.3%；训练标签仍来自多数票，共同犯错时可能自我强化。",
"sources": [
{
"label": "§4.2–4.4 结果与分析",
"url": "https://arxiv.org/html/2609.00768#S4.SS2"
},
{
"label": "附录 A 局限",
"url": "https://arxiv.org/html/2609.00768#A1"
}
]
}
],
"fields": {
"object": "训练 solver 与 challenger 的参数，并更新指导出题的错误记忆。记忆只给 challenger，不装进 solver 的答题 harness；因此主要归为模型参数进化。",
"executor": "三个 solver 分别由 Qwen3-4B-Base、Qwen3-8B-Base、OctoThinker-8B-Hybrid-Base 训练得到；测试时由它们直接回答推理题。",
"modifier": "challenger 生成数学题；冻结的 Qwen3-4B-Instruct-2507 默认负责诊断和整理错误记忆；训练器交替以 GRPO 更新 challenger 与 solver。另比较 30B-A3B、235B-A22B 诊断模型。",
"verdict": "训练：solver 对自生成题采样 12 次，多数答案当伪标签；另采样的训练回答与其一致得 +1，否则 −1。数学测试由 GPT-4o 按 simple-evals 对照标准答案判分；四项通用推理测试用答案 exact match。",
"seed": "R-Zero 式 challenger–solver 自博弈训练循环，新增错误原因诊断、分层记忆和双重投票筛选。起始记忆为空，诊断与检索模型冻结，不是浏览器或 coding-agent 框架。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "训练 solver 与 challenger 的参数，并更新指导出题的错误记忆。记忆只给 challenger，不装进 solver 的答题 harness；因此主要归为模型参数进化。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "三个 solver 分别由 Qwen3-4B-Base、Qwen3-8B-Base、OctoThinker-8B-Hybrid-Base 训练得到；测试时由它们直接回答推理题。",
"sources": [
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "challenger 生成数学题；冻结的 Qwen3-4B-Instruct-2507 默认负责诊断和整理错误记忆；训练器交替以 GRPO 更新 challenger 与 solver。另比较 30B-A3B、235B-A22B 诊断模型。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
},
{
"label": "附录 E 训练配置",
"url": "https://arxiv.org/html/2609.00768#A5"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "训练：solver 对自生成题采样 12 次，多数答案当伪标签；另采样的训练回答与其一致得 +1，否则 −1。数学测试由 GPT-4o 按 simple-evals 对照标准答案判分；四项通用推理测试用答案 exact match。",
"sources": [
{
"label": "§3.2 伪标签和奖励",
"url": "https://arxiv.org/html/2609.00768#S3.SS2"
},
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
},
{
"label": "附录 E 训练配置",
"url": "https://arxiv.org/html/2609.00768#A5"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "R-Zero 式 challenger–solver 自博弈训练循环，新增错误原因诊断、分层记忆和双重投票筛选。起始记忆为空，诊断与检索模型冻结，不是浏览器或 coding-agent 框架。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "附录 E 训练配置",
"url": "https://arxiv.org/html/2609.00768#A5"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "出题 → solver 投票筛题 → 用保留题做 GRPO → 对比与伪标签一致／不一致的轨迹，定位最早推理分歧 → 更新错误原因及掌握状态 → 指导下一轮出题。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "§3.2 伪标签和奖励",
"url": "https://arxiv.org/html/2609.00768#S3.SS2"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "全部训练题由 challenger 在自博弈中生成，内容是数学题，不使用 GSM8K 或 MATH 题目训练。每轮每候选采样 12 次构造标签，优化时另采样 8 次；论文分析五轮演化。",
"sources": [
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
},
{
"label": "附录 E 训练配置",
"url": "https://arxiv.org/html/2609.00768#A5"
},
{
"label": "§4.2–4.4 结果与分析",
"url": "https://arxiv.org/html/2609.00768#S4.SS2"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "按多数答案占比筛选中等难度题，且多数票要明显领先第二名；根据针对某错误的答题一致性更新“待掌握／已掌握”。这些是自生成题上的课程决策，不是外部验证集成绩。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "§3.2 伪标签和奖励",
"url": "https://arxiv.org/html/2609.00768#S3.SS2"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "数学：MATH-500、GSM8K、OlympiadBench、Minerva Math、AMC；AMC 的 40 题各采样 32 次报平均正确率。通用推理：MMLU-Pro、SuperGPQA、GPQA-Diamond、BBEH，贪心解码后 exact match。",
"sources": [
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "测试 benchmark 不供训练循环取题或取标准答案；训练伪标签与测试判分来源不同。自生成题无需人工标签，不等于多数票能证明答案正确。",
"sources": [
{
"label": "§4.1 数据及判分",
"url": "https://arxiv.org/html/2609.00768#S4.SS1"
},
{
"label": "附录 A 局限",
"url": "https://arxiv.org/html/2609.00768#A1"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "把“具体在哪类推理上反复失败”变成有状态的出题依据，同时用第一、第二答案票数差过滤不稳定伪标签。",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究能否从模型自己的失败历史提取明确的错误原因，并据此生成下一轮训练课程，不使用外部任务材料。",
"object": "训练 solver 与 challenger 的参数，并更新指导出题的错误记忆。记忆只给 challenger，不装进 solver 的答题 harness；因此主要归为模型参数进化。",
"executor": "三个 solver 分别由 Qwen3-4B-Base、Qwen3-8B-Base、OctoThinker-8B-Hybrid-Base 训练得到；测试时由它们直接回答推理题。",
"modifier": "challenger 生成数学题；冻结的 Qwen3-4B-Instruct-2507 默认负责诊断和整理错误记忆；训练器交替以 GRPO 更新 challenger 与 solver。另比较 30B-A3B、235B-A22B 诊断模型。",
"verdict": "训练：solver 对自生成题采样 12 次，多数答案当伪标签；另采样的训练回答与其一致得 +1，否则 −1。数学测试由 GPT-4o 按 simple-evals 对照标准答案判分；四项通用推理测试用答案 exact match。",
"seed": "R-Zero 式 challenger–solver 自博弈训练循环，新增错误原因诊断、分层记忆和双重投票筛选。起始记忆为空，诊断与检索模型冻结，不是浏览器或 coding-agent 框架。",
"cycle": "出题 → solver 投票筛题 → 用保留题做 GRPO → 对比与伪标签一致／不一致的轨迹，定位最早推理分歧 → 更新错误原因及掌握状态 → 指导下一轮出题。",
"train": "全部训练题由 challenger 在自博弈中生成，内容是数学题，不使用 GSM8K 或 MATH 题目训练。每轮每候选采样 12 次构造标签，优化时另采样 8 次；论文分析五轮演化。",
"debug": "按多数答案占比筛选中等难度题，且多数票要明显领先第二名；根据针对某错误的答题一致性更新“待掌握／已掌握”。这些是自生成题上的课程决策，不是外部验证集成绩。",
"test": "数学：MATH-500、GSM8K、OlympiadBench、Minerva Math、AMC；AMC 的 40 题各采样 32 次报平均正确率。通用推理：MMLU-Pro、SuperGPQA、GPQA-Diamond、BBEH，贪心解码后 exact match。",
"isolation": "测试 benchmark 不供训练循环取题或取标准答案；训练伪标签与测试判分来源不同。自生成题无需人工标签，不等于多数票能证明答案正确。",
"novelty": "把“具体在哪类推理上反复失败”变成有状态的出题依据，同时用第一、第二答案票数差过滤不稳定伪标签。"
},
"attributions": [
{
"tag": "org:meituan-longcat",
"label": "Meituan LongCat",
"kind": "institution",
"sources": [
{
"label": "论文作者机构署名",
"url": "https://arxiv.org/html/2609.00768"
}
]
}
],
"researchProblems": [
"curriculum"
]
},
{
"id": "2609.08183",
"title": "NeoHorse-1: Towards Recursive Self-Improvement via Agentic Post-Training with Routing Harness",
"date": "2026-09-08",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2609.08183",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "methods",
"categories": [
"methods"
],
"methodType": "weights",
"contentType": "report",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Weights",
"Data",
"GoldLabel"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码与模型",
"url": "https://github.com/TokenRhythm/NeoHorse"
}
],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2609.08183"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
},
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
},
{
"label": "§5.2 数据对比与规模分析",
"url": "https://arxiv.org/html/2609.08183#S5.SS2"
}
],
"fields": {
"本质定位": "探索路由 harness 能否把真实交互中的能力信号转成训练课程和数据配比，持续提升单个 agent 模型。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "训练候选与评测题做精确及近似去重。训练隔离不等于评测从未参与调数据；摘要写 eleven benchmarks，但正文表格实际列十项，本站按表格十项记录。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "执行轨迹既记录任务经验，也暴露模型能力缺口；如果只当作固定的训练文本，就无法根据新模型的弱点调整下一轮学习内容。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
}
]
},
{
"key": "position",
"value": "探索路由 harness 能否把真实交互中的能力信号转成训练课程和数据配比，持续提升单个 agent 模型。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
}
]
},
{
"key": "conclusion",
"value": "4B 与 9B 模型在所列十项评测的均分分别提高 5.93、3.44 个百分点。报告把持续多轮更新列为下一步；当前结果不能直接证明长期递归增益。",
"sources": [
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
}
]
}
],
"fields": {
"object": "更新 Qwen3.5-4B／9B 参数，并按能力缺口调整训练数据配比。Routing harness 负责收集、路由和组织数据；报告没有展示其代码自动进化。",
"executor": "最终测试用 NeoHorse-1-4B／9B。训练轨迹来自多模型部署系统；模型池与部署配置可变，报告未完整列出所有轨迹生成模型。",
"modifier": "固定训练流程执行分阶段 SFT 和 on-policy distillation；路由估计与分层评估调整课程和数据比例。蒸馏教师给学生生成位置的 token 分布，但正文未公开教师具体型号。",
"verdict": "训练入库先用规则检查消息／工具调用完整性，再按目标完成、指令遵循、工具使用、证据一致、错误恢复、终止六项审查轨迹；需语义判断的部分交给 judge。参数监督来自已保留回答和蒸馏教师分布，不是把十个 benchmark 分数直接当逐 token 奖励。",
"seed": "OpenSquilla 等部署 harness：管理上下文、工具及模型路由，记录请求、推理、工具观察和恢复过程。测试中 QwenClawBench／PinchBench 用 OpenSquilla，WorkBuddy／VitaBench 用各自官方框架。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "更新 Qwen3.5-4B／9B 参数，并按能力缺口调整训练数据配比。Routing harness 负责收集、路由和组织数据；报告没有展示其代码自动进化。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "最终测试用 NeoHorse-1-4B／9B。训练轨迹来自多模型部署系统；模型池与部署配置可变，报告未完整列出所有轨迹生成模型。",
"sources": [
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "固定训练流程执行分阶段 SFT 和 on-policy distillation；路由估计与分层评估调整课程和数据比例。蒸馏教师给学生生成位置的 token 分布，但正文未公开教师具体型号。",
"sources": [
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "训练入库先用规则检查消息／工具调用完整性，再按目标完成、指令遵循、工具使用、证据一致、错误恢复、终止六项审查轨迹；需语义判断的部分交给 judge。参数监督来自已保留回答和蒸馏教师分布，不是把十个 benchmark 分数直接当逐 token 奖励。",
"sources": [
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "OpenSquilla 等部署 harness：管理上下文、工具及模型路由，记录请求、推理、工具观察和恢复过程。测试中 QwenClawBench／PinchBench 用 OpenSquilla，WorkBuddy／VitaBench 用各自官方框架。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "收集交互 → 去重、去污染、结构和语义检查 → 按路由估计从低到高组织三阶段 SFT 与学生在线生成的蒸馏 → 分层评估能力缺口 → 调整后续训练数据比例。",
"sources": [
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "主要为自建路由 harness 产生的约 10万–100万条轨迹，按用户轮次序列化，保留当前轮推理及历史工具上下文；辅以公开指令、推理、工具和代码数据。受控数据实验另明确比较 Toucan 与自建路由轨迹。",
"sources": [
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§5.2 数据对比与规模分析",
"url": "https://arxiv.org/html/2609.08183#S5.SS2"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "用与训练去重隔离的分层评估集统计能力缺口，指导下一轮数据配比。报告没有公开该分层集的逐项组成，也未说明它与最终十项报告评测是否完全分开，不能称为独立最终测试。",
"sources": [
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "QwenClawBench、WorkBuddy Bench、PinchBench、VitaBench、BFCL V4、τ²-Bench、HumanEval、LiveCodeBench v6、IFBench、IFEval。代码按可执行测试，指令遵循按 benchmark 规则；VitaBench 明确用 DeepSeek-V4-Flash 模拟用户并担任 judge；其他 agent bench 沿用官方评测，报告未逐项展开全部底层裁判。",
"sources": [
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "训练候选与评测题做精确及近似去重。训练隔离不等于评测从未参与调数据；摘要写 eleven benchmarks，但正文表格实际列十项，本站按表格十项记录。",
"sources": [
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§5 评测配置与表 1–2",
"url": "https://arxiv.org/html/2609.08183#S5"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "区分路由器预测的能力需求、策略调整后的分配和实际服务模型；用预测需求安排课程，避免把“恰好用了强模型”误当成题目难度。",
"sources": [
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "探索路由 harness 能否把真实交互中的能力信号转成训练课程和数据配比，持续提升单个 agent 模型。",
"object": "更新 Qwen3.5-4B／9B 参数，并按能力缺口调整训练数据配比。Routing harness 负责收集、路由和组织数据；报告没有展示其代码自动进化。",
"executor": "最终测试用 NeoHorse-1-4B／9B。训练轨迹来自多模型部署系统；模型池与部署配置可变，报告未完整列出所有轨迹生成模型。",
"modifier": "固定训练流程执行分阶段 SFT 和 on-policy distillation；路由估计与分层评估调整课程和数据比例。蒸馏教师给学生生成位置的 token 分布，但正文未公开教师具体型号。",
"verdict": "训练入库先用规则检查消息／工具调用完整性，再按目标完成、指令遵循、工具使用、证据一致、错误恢复、终止六项审查轨迹；需语义判断的部分交给 judge。参数监督来自已保留回答和蒸馏教师分布，不是把十个 benchmark 分数直接当逐 token 奖励。",
"seed": "OpenSquilla 等部署 harness：管理上下文、工具及模型路由，记录请求、推理、工具观察和恢复过程。测试中 QwenClawBench／PinchBench 用 OpenSquilla，WorkBuddy／VitaBench 用各自官方框架。",
"cycle": "收集交互 → 去重、去污染、结构和语义检查 → 按路由估计从低到高组织三阶段 SFT 与学生在线生成的蒸馏 → 分层评估能力缺口 → 调整后续训练数据比例。",
"train": "主要为自建路由 harness 产生的约 10万–100万条轨迹，按用户轮次序列化，保留当前轮推理及历史工具上下文；辅以公开指令、推理、工具和代码数据。受控数据实验另明确比较 Toucan 与自建路由轨迹。",
"debug": "用与训练去重隔离的分层评估集统计能力缺口，指导下一轮数据配比。报告没有公开该分层集的逐项组成，也未说明它与最终十项报告评测是否完全分开，不能称为独立最终测试。",
"test": "QwenClawBench、WorkBuddy Bench、PinchBench、VitaBench、BFCL V4、τ²-Bench、HumanEval、LiveCodeBench v6、IFBench、IFEval。代码按可执行测试，指令遵循按 benchmark 规则；VitaBench 明确用 DeepSeek-V4-Flash 模拟用户并担任 judge；其他 agent bench 沿用官方评测，报告未逐项展开全部底层裁判。",
"isolation": "训练候选与评测题做精确及近似去重。训练隔离不等于评测从未参与调数据；摘要写 eleven benchmarks，但正文表格实际列十项，本站按表格十项记录。",
"novelty": "区分路由器预测的能力需求、策略调整后的分配和实际服务模型；用预测需求安排课程，避免把“恰好用了强模型”误当成题目难度。"
},
"attributions": [],
"researchProblems": [
"curriculum"
]
},
{
"id": "metarsi-v1",
"title": "MetaRSI-v1: A Meta-Recursive Self-Improving System for Recursive Self-Improving Systems Themselves",
"date": "2026-09-03",
"dateLabel": "报告发布",
"year": "2026",
"url": "https://www.cosmosmind.ai/research/metarsi-v1",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "methods",
"categories": [
"methods"
],
"methodType": "joint",
"contentType": "report",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Weights",
"HarnessCode",
"Data",
"Improver",
"JointEvolution"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码",
"url": "https://github.com/CosmosMind-ai/RSI-Harness"
},
{
"label": "PDF",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf"
}
],
"sources": [
{
"label": "官方技术报告与发布日期",
"url": "https://www.cosmosmind.ai/research/metarsi-v1"
},
{
"label": "§1，PDF p2–4",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=2"
},
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "§5.3–5.4，PDF p23–25",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=23"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
],
"fields": {
"本质定位": "研究能否由系统选择并改进数据生成、harness 修改和参数训练的组合顺序，使“怎样改进”本身也能根据经验调整。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "报告规定封存评测在系统冻结后开启，不向中途决策返回成绩；合成数据与封存题做精确、n-gram 和语义去重。声明的隔离规则不能代替尚未公开的逐项运行数据。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "现有自改进多依赖容易自动验对的任务，而且数据、harness、参数更新往往分开设计，无法根据当前失败决定该改哪一层。",
"sources": [
{
"label": "§1，PDF p2–4",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=2"
}
]
},
{
"key": "position",
"value": "研究能否由系统选择并改进数据生成、harness 修改和参数训练的组合顺序，使“怎样改进”本身也能根据经验调整。",
"sources": [
{
"label": "§1，PDF p2–4",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=2"
},
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
}
]
},
{
"key": "conclusion",
"value": "作者报告 Qwen3.5-35B-A3B 四项评测平均提高 10.9 个百分点，比最强固定顺序高 3.6 点。虽以更开放的科学任务为目标，当前实验仍主要是可执行代码与标准答案推理题。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
}
]
}
],
"fields": {
"object": "联合更新训练记录、五类 harness 配置和模型 LoRA；调度策略还会改变操作顺序及候选生成指令。API 模型实验只能更新数据与 harness，不能写模型参数。",
"executor": "联合更新：自托管 Qwen3.5-35B-A3B。API 路线：GLM-5.2、GPT-5.6 Sol、Kimi K3、Claude Opus 5、DeepSeek V4 Pro、Gemini 3.1 Pro，各自运行自己的改进循环。",
"modifier": "每组被测模型自己负责诊断、生成、调度和模型式核查，没有外部更强教师；固定代码负责验证、评测及批准发布。LoRA 训练由后端执行。",
"verdict": "代码任务由容器里的可执行 verifier 检查；GPQA-D-hard100 用答案匹配，AIME 用数值匹配。生成训练题另做独立重解、参考解通过／空操作失败、去重和溯源检查；这些入库反馈与最终封存测试分数分开。",
"seed": "自建 RSI-Harness：系统提示、记忆、内置工具、技能、MCP 外接工具五类配置，附固定运行循环与验证接口。推理题关闭工具、技能、MCP，只允许改系统提示；代码题使用完整 agent 配置。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "联合更新训练记录、五类 harness 配置和模型 LoRA；调度策略还会改变操作顺序及候选生成指令。API 模型实验只能更新数据与 harness，不能写模型参数。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "联合更新：自托管 Qwen3.5-35B-A3B。API 路线：GLM-5.2、GPT-5.6 Sol、Kimi K3、Claude Opus 5、DeepSeek V4 Pro、Gemini 3.1 Pro，各自运行自己的改进循环。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "§5.3–5.4，PDF p23–25",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=23"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "每组被测模型自己负责诊断、生成、调度和模型式核查，没有外部更强教师；固定代码负责验证、评测及批准发布。LoRA 训练由后端执行。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "代码任务由容器里的可执行 verifier 检查；GPQA-D-hard100 用答案匹配，AIME 用数值匹配。生成训练题另做独立重解、参考解通过／空操作失败、去重和溯源检查；这些入库反馈与最终封存测试分数分开。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "自建 RSI-Harness：系统提示、记忆、内置工具、技能、MCP 外接工具五类配置，附固定运行循环与验证接口。推理题关闭工具、技能、MCP，只允许改系统提示；代码题使用完整 agent 配置。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "从失败生成结构化诊断 → 选择数据／harness／参数操作及顺序 → 生成候选 → 固定验收程序选择至多一个可发布后继 → 后续阶段重新诊断。参数候选从固定底座按累计数据重新训练，避免混淆旧 adapter 的训练历史。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "Data-RSI 根据失败原因生成、验证训练记录，Model-RSI 用累计保留数据做 LoRA；不是直接拿封存测试题训练。报告给出生成和去重规则，但未公布每个 benchmark 的完整适应题清单与训练样本量。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "在非封存的 adaptation set 上评估候选并决定保留；共享固定的验收规则。适应集的逐项构成未在报告中完整列出，不能把最终测试集题数当成调试集题数。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "Terminal-Bench 2.1：89 题；SWE-bench Pro：731 题公开测试部分；GPQA-D-hard100：从 Diamond 固定选 100 题，附录列 ID；AIME：2025／2026 的 I、II 共 60 题。最终评价已发布版本，五个随机种子取平均。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "报告规定封存评测在系统冻结后开启，不向中途决策返回成绩；合成数据与封存题做精确、n-gram 和语义去重。声明的隔离规则不能代替尚未公开的逐项运行数据。",
"sources": [
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
},
{
"label": "附录 D–F，PDF p45–47",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=45"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "把数据、harness 与参数更新做成可组合的操作，并检查上一阶段产物是否仍适用于当前模型；区别于仅反复调用一个固定修改器。",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究能否由系统选择并改进数据生成、harness 修改和参数训练的组合顺序，使“怎样改进”本身也能根据经验调整。",
"object": "联合更新训练记录、五类 harness 配置和模型 LoRA；调度策略还会改变操作顺序及候选生成指令。API 模型实验只能更新数据与 harness，不能写模型参数。",
"executor": "联合更新：自托管 Qwen3.5-35B-A3B。API 路线：GLM-5.2、GPT-5.6 Sol、Kimi K3、Claude Opus 5、DeepSeek V4 Pro、Gemini 3.1 Pro，各自运行自己的改进循环。",
"modifier": "每组被测模型自己负责诊断、生成、调度和模型式核查，没有外部更强教师；固定代码负责验证、评测及批准发布。LoRA 训练由后端执行。",
"verdict": "代码任务由容器里的可执行 verifier 检查；GPQA-D-hard100 用答案匹配，AIME 用数值匹配。生成训练题另做独立重解、参考解通过／空操作失败、去重和溯源检查；这些入库反馈与最终封存测试分数分开。",
"seed": "自建 RSI-Harness：系统提示、记忆、内置工具、技能、MCP 外接工具五类配置，附固定运行循环与验证接口。推理题关闭工具、技能、MCP，只允许改系统提示；代码题使用完整 agent 配置。",
"cycle": "从失败生成结构化诊断 → 选择数据／harness／参数操作及顺序 → 生成候选 → 固定验收程序选择至多一个可发布后继 → 后续阶段重新诊断。参数候选从固定底座按累计数据重新训练，避免混淆旧 adapter 的训练历史。",
"train": "Data-RSI 根据失败原因生成、验证训练记录，Model-RSI 用累计保留数据做 LoRA；不是直接拿封存测试题训练。报告给出生成和去重规则，但未公布每个 benchmark 的完整适应题清单与训练样本量。",
"debug": "在非封存的 adaptation set 上评估候选并决定保留；共享固定的验收规则。适应集的逐项构成未在报告中完整列出，不能把最终测试集题数当成调试集题数。",
"test": "Terminal-Bench 2.1：89 题；SWE-bench Pro：731 题公开测试部分；GPQA-D-hard100：从 Diamond 固定选 100 题，附录列 ID；AIME：2025／2026 的 I、II 共 60 题。最终评价已发布版本，五个随机种子取平均。",
"isolation": "报告规定封存评测在系统冻结后开启，不向中途决策返回成绩；合成数据与封存题做精确、n-gram 和语义去重。声明的隔离规则不能代替尚未公开的逐项运行数据。",
"novelty": "把数据、harness 与参数更新做成可组合的操作，并检查上一阶段产物是否仍适用于当前模型；区别于仅反复调用一个固定修改器。"
},
"attributions": [],
"researchProblems": [
"improver",
"weights",
"curriculum"
]
},
{
"id": "icoder-27b",
"title": "iCoder-27B: Recursive AI-Led Development of Frontier Industrial Coding Model",
"date": "2026-09-15",
"dateLabel": "收录日期",
"year": "2026",
"url": "https://huggingface.co/i-Coder/iCoder-27B",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "methods",
"categories": [
"methods"
],
"methodType": "weights",
"contentType": "report",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Weights",
"Data",
"ExecutableVerifier",
"org:sjtu",
"org:nus"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "代码",
"url": "https://github.com/bingreeky/iCoder"
},
{
"label": "PDF",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf"
}
],
"sources": [
{
"label": "官方模型卡及技术报告入口",
"url": "https://huggingface.co/i-Coder/iCoder-27B"
},
{
"label": "摘要与 §1，PDF p1–3",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
},
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
},
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
},
{
"label": "§6.1–6.3、表 9–10，PDF p18–20",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=18"
},
{
"label": "附录 A.1–A.2，PDF p28–29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=28"
},
{
"label": "附录 A.2 逐 benchmark 评测，PDF p29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=29"
}
],
"fields": {
"本质定位": "研究人类把研发经验预先写成 Research Skills 后，agent 能否自主选择和修订实验，开发有竞争力的 RTL／GPU kernel 模型。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "Research Skills 要求留出隔离和来源谱系划分；报告未给完整 train/validation/test ID 映射。正文将某些 benchmark 用于训练阶段诊断，因此不额外标为严格三阶段隔离。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "在小模型、窄任务上完成一次自动训练，不等于能研发可发布的工业模型；完整开发还要处理数据质量、训练目标失效、verifier 错判和资源约束。",
"sources": [
{
"label": "摘要与 §1，PDF p1–3",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
}
]
},
{
"key": "position",
"value": "研究人类把研发经验预先写成 Research Skills 后，agent 能否自主选择和修订实验，开发有竞争力的 RTL／GPU kernel 模型。",
"sources": [
{
"label": "摘要与 §1，PDF p1–3",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
},
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
}
]
},
{
"key": "conclusion",
"value": "产出 iCoder-27B，并报告 RTLLM 68.0、KernelBench L1 正确率 61% 等结果。人类仍规定目标、权限和验证标准；报告没有给出完全无人介入研发的对照证据。",
"sources": [
{
"label": "§6.1–6.3、表 9–10，PDF p18–20",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=18"
},
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
}
]
}
],
"fields": {
"object": "主要更新 27B 模型参数，同时生成任务数据、调整 SFT／自蒸馏／RL 的训练方案。人类写的 Research Skills 在运行中固定，不是让 agent 任意重写自身研发框架。",
"executor": "训练与最终答题对象为 iCoder-27B，技术报告将起点写为 Qwen3.6-27B。SFT 另用 DeepSeek-V4-Pro 生成并验证教师轨迹。",
"modifier": "研发 agent 根据 Research Skills 选择数据变换、训练目标和检查点，训练后端执行更新。报告未公开该研发 agent 的具体模型型号；不能把 27B 学生或 SFT 教师当成它。",
"verdict": "数据／训练反馈来自编译、仿真和数值检查及其错误记录；SFT 只保留验过的教师解。RTL 用任务 testbench 仿真；GPU kernel 与参考输出比数值，正确后才考虑加速。基础设施故障单列为 unjudged，不当成答错。",
"seed": "人类提供 Data→SFT→OPSD→RLVR 的 Research Skills、任务队列、实验日志、资源接口和验证约束。agent 可以回到数据阶段改方案，但不能削弱已批准的正确性标准。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "主要更新 27B 模型参数，同时生成任务数据、调整 SFT／自蒸馏／RL 的训练方案。人类写的 Research Skills 在运行中固定，不是让 agent 任意重写自身研发框架。",
"sources": [
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
},
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "训练与最终答题对象为 iCoder-27B，技术报告将起点写为 Qwen3.6-27B。SFT 另用 DeepSeek-V4-Pro 生成并验证教师轨迹。",
"sources": [
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§6.1–6.3、表 9–10，PDF p18–20",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=18"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "研发 agent 根据 Research Skills 选择数据变换、训练目标和检查点，训练后端执行更新。报告未公开该研发 agent 的具体模型型号；不能把 27B 学生或 SFT 教师当成它。",
"sources": [
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
},
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "数据／训练反馈来自编译、仿真和数值检查及其错误记录；SFT 只保留验过的教师解。RTL 用任务 testbench 仿真；GPU kernel 与参考输出比数值，正确后才考虑加速。基础设施故障单列为 unjudged，不当成答错。",
"sources": [
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
},
{
"label": "附录 A.2 逐 benchmark 评测，PDF p29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=29"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "人类提供 Data→SFT→OPSD→RLVR 的 Research Skills、任务队列、实验日志、资源接口和验证约束。agent 可以回到数据阶段改方案，但不能削弱已批准的正确性标准。",
"sources": [
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "构造可执行任务池 → 找学生失败而教师成功的题做 SFT → 利用失败轨迹与 verifier 诊断做自蒸馏 → 用可执行奖励做 RL；小规模实验的失败可触发数据、奖励或训练目标修订。",
"sources": [
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "自建任务池 55,462 题：RTL 34,972、GPU kernel 20,490。实际各阶段：SFT 28,952 条验过的教师轨迹；OPSD 1,874 道可恢复失败任务；RLVR 13,212 道验证任务。每题带说明、接口、参考实现和验证器；报告未完整列出初始来源题库。",
"sources": [
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "附录 A.1–A.2，PDF p28–29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=28"
},
{
"label": "§6.1–6.3、表 9–10，PDF p18–20",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=18"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "阶段内用小规模对照试验及验证结果选择路线和检查点；SFT 选择学生 pass@4 为 0、DeepSeek-V4-Pro 四次至少一次通过的题。文中部分训练诊断也使用命名 benchmark，不能把所有报告结果视为从未用于研发的盲测。",
"sources": [
{
"label": "§4.1–4.2，PDF p6–8",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=6"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "VerilogEval 两种任务各 156 题、RTLLM 50、CVDP 78、ArchXBench 71、RealBench-Module 60；KernelBench 250、TritonBench-G 184。RTL 分别用 Icarus／cocotb／自检 testbench／Verilator；kernel 按参考数值匹配，KernelBench 另报超过 1.05× 的正确加速率。",
"sources": [
{
"label": "附录 A.2 逐 benchmark 评测，PDF p29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=29"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "Research Skills 要求留出隔离和来源谱系划分；报告未给完整 train/validation/test ID 映射。正文将某些 benchmark 用于训练阶段诊断，因此不额外标为严格三阶段隔离。",
"sources": [
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
},
{
"label": "附录 A.2 逐 benchmark 评测，PDF p29",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=29"
}
]
},
{
"key": "novelty",
"label": "方法特点",
"status": "recorded",
"value": "把人类输入集中成可复用研发规则，让 agent 根据实测故障改训练决策；贡献是完整工业模型开发过程，不只是自动跑一段固定训练脚本。",
"sources": [
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
},
{
"label": "§4.3–4.4，PDF p9–17",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=9"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究人类把研发经验预先写成 Research Skills 后，agent 能否自主选择和修订实验，开发有竞争力的 RTL／GPU kernel 模型。",
"object": "主要更新 27B 模型参数，同时生成任务数据、调整 SFT／自蒸馏／RL 的训练方案。人类写的 Research Skills 在运行中固定，不是让 agent 任意重写自身研发框架。",
"executor": "训练与最终答题对象为 iCoder-27B，技术报告将起点写为 Qwen3.6-27B。SFT 另用 DeepSeek-V4-Pro 生成并验证教师轨迹。",
"modifier": "研发 agent 根据 Research Skills 选择数据变换、训练目标和检查点，训练后端执行更新。报告未公开该研发 agent 的具体模型型号；不能把 27B 学生或 SFT 教师当成它。",
"verdict": "数据／训练反馈来自编译、仿真和数值检查及其错误记录；SFT 只保留验过的教师解。RTL 用任务 testbench 仿真；GPU kernel 与参考输出比数值，正确后才考虑加速。基础设施故障单列为 unjudged，不当成答错。",
"seed": "人类提供 Data→SFT→OPSD→RLVR 的 Research Skills、任务队列、实验日志、资源接口和验证约束。agent 可以回到数据阶段改方案，但不能削弱已批准的正确性标准。",
"cycle": "构造可执行任务池 → 找学生失败而教师成功的题做 SFT → 利用失败轨迹与 verifier 诊断做自蒸馏 → 用可执行奖励做 RL；小规模实验的失败可触发数据、奖励或训练目标修订。",
"train": "自建任务池 55,462 题：RTL 34,972、GPU kernel 20,490。实际各阶段：SFT 28,952 条验过的教师轨迹；OPSD 1,874 道可恢复失败任务；RLVR 13,212 道验证任务。每题带说明、接口、参考实现和验证器；报告未完整列出初始来源题库。",
"debug": "阶段内用小规模对照试验及验证结果选择路线和检查点；SFT 选择学生 pass@4 为 0、DeepSeek-V4-Pro 四次至少一次通过的题。文中部分训练诊断也使用命名 benchmark，不能把所有报告结果视为从未用于研发的盲测。",
"test": "VerilogEval 两种任务各 156 题、RTLLM 50、CVDP 78、ArchXBench 71、RealBench-Module 60；KernelBench 250、TritonBench-G 184。RTL 分别用 Icarus／cocotb／自检 testbench／Verilator；kernel 按参考数值匹配，KernelBench 另报超过 1.05× 的正确加速率。",
"isolation": "Research Skills 要求留出隔离和来源谱系划分；报告未给完整 train/validation/test ID 映射。正文将某些 benchmark 用于训练阶段诊断，因此不额外标为严格三阶段隔离。",
"novelty": "把人类输入集中成可复用研发规则，让 agent 根据实测故障改训练决策；贡献是完整工业模型开发过程，不只是自动跑一段固定训练脚本。"
},
"attributions": [
{
"tag": "org:sjtu",
"label": "Shanghai Jiao Tong University",
"kind": "institution",
"sources": [
{
"label": "技术报告 p1 作者机构",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
}
]
},
{
"tag": "org:nus",
"label": "National University of Singapore",
"kind": "institution",
"sources": [
{
"label": "技术报告 p1 作者机构",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
}
]
}
],
"researchProblems": [
"evaluation"
]
},
{
"id": "2609.11873",
"title": "The Last AI Built by Humans: Toward Genuine Recursive Self-Improvement",
"date": "2026-09-10",
"dateLabel": "首次提交",
"year": "2026",
"url": "https://arxiv.org/abs/2609.11873",
"priority": "C",
"priorityBasis": "面向 RSI 全景与自主程度的发展路线，作为入门主线阅读。",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "survey",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Survey",
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "项目页",
"url": "https://theseus-labs-rsi.github.io/"
}
],
"sources": [
{
"label": "arXiv 摘要与首次提交日期",
"url": "https://arxiv.org/abs/2609.11873"
},
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.11873#S1"
},
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§3–4 方法及场景",
"url": "https://arxiv.org/html/2609.11873#S3"
},
{
"label": "§5 Industry Practice",
"url": "https://arxiv.org/html/2609.11873#S5"
},
{
"label": "§6 挑战与未来方向",
"url": "https://arxiv.org/html/2609.11873#S6"
}
],
"fields": {
"本质定位": "为 RSI 建立按自主程度递进的发展路线，分析不同任务场景需要什么能力，以及现有系统距离更自主的改进还差什么。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "路线中的后期自主能力属于研究目标；图中的未来趋势与初步案例不能作为已实现无限递归提升的证据。",
"visibleKeys": [
"scope",
"framework",
"feedback"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "可自动判分的封闭任务进步很快，但长程真实工作仍受状态跟踪、反馈可靠性和失败累积限制；“自改进”一词又混合了自主程度很不同的系统。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.11873#S1"
}
]
},
{
"key": "position",
"value": "为 RSI 建立按自主程度递进的发展路线，分析不同任务场景需要什么能力，以及现有系统距离更自主的改进还差什么。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
}
]
},
{
"key": "conclusion",
"value": "区分自主执行改进、制定改进策略、获取经验、适应环境、递归改进改进机制五层。文章提供路线图和初步产业案例，不代表五层能力都已被实验实现。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§5 Industry Practice",
"url": "https://arxiv.org/html/2609.11873#S5"
},
{
"label": "§6 挑战与未来方向",
"url": "https://arxiv.org/html/2609.11873#S6"
}
]
}
],
"fields": {
"scope": "覆盖方法分类、科学发现、具身智能、软件工程及企业实践；适合作为 RSI 全景阅读入口。",
"framework": "用五种自主能力比较系统：谁执行改进、谁选策略、谁获取经验、谁适应新环境，以及改进机制能否继续改进自身。",
"feedback": "比较可执行验证、环境反馈和开放任务评价的不同限制；强调可验证、可迁移、可保留的改进，而非一次分数增加。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "覆盖方法分类、科学发现、具身智能、软件工程及企业实践；适合作为 RSI 全景阅读入口。",
"sources": [
{
"label": "§3–4 方法及场景",
"url": "https://arxiv.org/html/2609.11873#S3"
},
{
"label": "§5 Industry Practice",
"url": "https://arxiv.org/html/2609.11873#S5"
}
]
},
{
"key": "framework",
"label": "分析框架",
"status": "recorded",
"value": "用五种自主能力比较系统：谁执行改进、谁选策略、谁获取经验、谁适应新环境，以及改进机制能否继续改进自身。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
}
]
},
{
"key": "feedback",
"label": "反馈与证据",
"status": "recorded",
"value": "比较可执行验证、环境反馈和开放任务评价的不同限制；强调可验证、可迁移、可保留的改进，而非一次分数增加。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§6 挑战与未来方向",
"url": "https://arxiv.org/html/2609.11873#S6"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "结合文献、跨领域 benchmark 趋势和产业案例。HCI 表示相对该 benchmark 首年水平消除了多少分数差距，不是任务准确率，也不能直接混合不同评测协议。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§5 Industry Practice",
"url": "https://arxiv.org/html/2609.11873#S5"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先读 §2 建立分类，再按关注场景读 §4；§5 的产业案例用于理解实践路径，不与统一预算的 benchmark 实验混为一谈。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§5 Industry Practice",
"url": "https://arxiv.org/html/2609.11873#S5"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "路线中的后期自主能力属于研究目标；图中的未来趋势与初步案例不能作为已实现无限递归提升的证据。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
},
{
"label": "§6 挑战与未来方向",
"url": "https://arxiv.org/html/2609.11873#S6"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "为 RSI 建立按自主程度递进的发展路线，分析不同任务场景需要什么能力，以及现有系统距离更自主的改进还差什么。",
"scope": "覆盖方法分类、科学发现、具身智能、软件工程及企业实践；适合作为 RSI 全景阅读入口。",
"framework": "用五种自主能力比较系统：谁执行改进、谁选策略、谁获取经验、谁适应新环境，以及改进机制能否继续改进自身。",
"feedback": "比较可执行验证、环境反馈和开放任务评价的不同限制；强调可验证、可迁移、可保留的改进，而非一次分数增加。",
"evidence": "结合文献、跨领域 benchmark 趋势和产业案例。HCI 表示相对该 benchmark 首年水平消除了多少分数差距，不是任务准确率，也不能直接混合不同评测协议。",
"reading": "先读 §2 建立分类，再按关注场景读 §4；§5 的产业案例用于理解实践路径，不与统一预算的 benchmark 实验混为一谈。",
"limits": "路线中的后期自主能力属于研究目标；图中的未来趋势与初步案例不能作为已实现无限递归提升的证据。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "data-centric-rsi-survey",
"title": "Data-Centric Recursive Improvement for Foundation Models: A Survey",
"date": "2026-09-02",
"dateLabel": "发布",
"year": "2026",
"url": "https://www.preprints.org/manuscript/202609.0154",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "survey",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Survey",
"Data"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "配套仓库",
"url": "https://github.com/zoezou2015/Awesome-Data-Centric-Recursive-Improvement"
}
],
"sources": [
{
"label": "Preprints 摘要、首次发布日期",
"url": "https://www.preprints.org/manuscript/202609.0154"
},
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
},
{
"label": "作者配套阅读地图：Signal–Decision–Update Loop",
"url": "https://github.com/zoezou2015/Awesome-Data-Centric-Recursive-Improvement"
}
],
"fields": {
"本质定位": "以数据相关对象为中心，整理评价信号如何经过可信度判断和干预决策，形成可持续的模型改进。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "不同文献的训练数据、反馈和测试协议各不相同；此综述的分类不是统一实验结论。Preprints 页面标注为未同行评审的预印本。",
"visibleKeys": [
"scope",
"framework",
"feedback"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "数据构建与评测常被分开讨论，难以追踪一次评测发现的弱点到底怎样改变下一轮数据和模型。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
{
"key": "position",
"value": "以数据相关对象为中心，整理评价信号如何经过可信度判断和干预决策，形成可持续的模型改进。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
{
"key": "conclusion",
"value": "用“信号—决策—更新”连接数据选择、过滤、合成及后训练；同时分析反馈过拟合、错误评价和更新不稳定。它是整理机制的综述，不是一套统一训练实验。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
},
{
"label": "作者配套阅读地图：Signal–Decision–Update Loop",
"url": "https://github.com/zoezou2015/Awesome-Data-Centric-Recursive-Improvement"
}
]
}
],
"fields": {
"scope": "聚焦反馈驱动的数据变化，包括训练数据、检索和记忆等影响后续模型行为的对象。",
"framework": "三个问题：什么信号发现弱点；谁判断信号可信并决定如何干预；具体更新什么数据对象。",
"feedback": "分别讨论可执行结果、标准答案、模型评价和偏好等信号；强调保留监督来源，不能把一次高分自动当作正确更新依据。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "聚焦反馈驱动的数据变化，包括训练数据、检索和记忆等影响后续模型行为的对象。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
{
"key": "framework",
"label": "分析框架",
"status": "recorded",
"value": "三个问题：什么信号发现弱点；谁判断信号可信并决定如何干预；具体更新什么数据对象。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
{
"key": "feedback",
"label": "反馈与证据",
"status": "recorded",
"value": "分别讨论可执行结果、标准答案、模型评价和偏好等信号；强调保留监督来源，不能把一次高分自动当作正确更新依据。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先读 §1–2 的分类，再用作者 GitHub 阅读地图按信号和数据更新路径查文献，适合与本站的执行者／修改者维度配合。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
},
{
"label": "作者配套阅读地图：Signal–Decision–Update Loop",
"url": "https://github.com/zoezou2015/Awesome-Data-Centric-Recursive-Improvement"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "不同文献的训练数据、反馈和测试协议各不相同；此综述的分类不是统一实验结论。Preprints 页面标注为未同行评审的预印本。",
"sources": [
{
"label": "Preprints 摘要、首次发布日期",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "以数据相关对象为中心，整理评价信号如何经过可信度判断和干预决策，形成可持续的模型改进。",
"scope": "聚焦反馈驱动的数据变化，包括训练数据、检索和记忆等影响后续模型行为的对象。",
"framework": "三个问题：什么信号发现弱点；谁判断信号可信并决定如何干预；具体更新什么数据对象。",
"feedback": "分别讨论可执行结果、标准答案、模型评价和偏好等信号；强调保留监督来源，不能把一次高分自动当作正确更新依据。",
"reading": "先读 §1–2 的分类，再用作者 GitHub 阅读地图按信号和数据更新路径查文献，适合与本站的执行者／修改者维度配合。",
"limits": "不同文献的训练数据、反馈和测试协议各不相同；此综述的分类不是统一实验结论。Preprints 页面标注为未同行评审的预印本。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "ai4ai-survey",
"title": "AI4AI Survey: From Long-Horizon Agents to Recursive Self-Improvement",
"date": "2026-08",
"dateLabel": "发布",
"year": "2026",
"url": "https://simpleagentlab.com/ai4ai/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "survey",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Survey",
"LongHorizon"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "论文",
"url": "https://www.preprints.org/manuscript/202608.2108"
},
{
"label": "配套文献地图",
"url": "https://kaiwu5.github.io/Awesome-AI4AI/"
}
],
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
},
{
"label": "作者配套文献地图",
"url": "https://kaiwu5.github.io/Awesome-AI4AI/"
}
],
"fields": {
"本质定位": "研究怎样界定和衡量 AI 将改进想法推进到可靠验证结果的能力，并区分一次自改进与改进机制本身的递归变化。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "对所审阅系统的归纳受搜索时间与纳入标准限制；不能把“单项能力强”解释为端到端自主研发已解决。项目页和论文的具体定义应分别按原文阅读。",
"visibleKeys": [
"scope",
"framework",
"feedback"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "规划、写代码、跑实验等单项能力很强，连成完整科研流程却未必可靠；分散的 agent、AI4AI 与 RSI 术语又让这种差距不易比较。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
}
]
},
{
"key": "position",
"value": "研究怎样界定和衡量 AI 将改进想法推进到可靠验证结果的能力，并区分一次自改进与改进机制本身的递归变化。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
},
{
"key": "conclusion",
"value": "按目标、规划、执行、反馈、修复的承担者分析系统，强调组件组合后可靠性下降及人类仍掌握的决策权。这些判断来自所审阅系统，不是对所有未来系统的结论。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
}
],
"fields": {
"scope": "覆盖长程 agent、模型与 harness 两条改进路线、自动化研究和自修改系统。",
"framework": "逐阶段区分人类、系统及双方共同承担的工作；“长程”重点是早期动作影响后续动作与最终结果，不只是 token 多或运行时间长。",
"feedback": "检查评价标准是谁定的、结果如何影响修复和后续决策，并提醒匹配预算、验证泄漏和人类干预都会影响成绩解释。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "覆盖长程 agent、模型与 harness 两条改进路线、自动化研究和自修改系统。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
}
]
},
{
"key": "framework",
"label": "分析框架",
"status": "recorded",
"value": "逐阶段区分人类、系统及双方共同承担的工作；“长程”重点是早期动作影响后续动作与最终结果，不只是 token 多或运行时间长。",
"sources": [
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
},
{
"key": "feedback",
"label": "反馈与证据",
"status": "recorded",
"value": "检查评价标准是谁定的、结果如何影响修复和后续决策，并提醒匹配预算、验证泄漏和人类干预都会影响成绩解释。",
"sources": [
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先看定义和分类，再读两条改进路线；配套 Awesome-AI4AI 提供文献地图，RSIHub 提供实验框架入口。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "作者配套文献地图",
"url": "https://kaiwu5.github.io/Awesome-AI4AI/"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "对所审阅系统的归纳受搜索时间与纳入标准限制；不能把“单项能力强”解释为端到端自主研发已解决。项目页和论文的具体定义应分别按原文阅读。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究怎样界定和衡量 AI 将改进想法推进到可靠验证结果的能力，并区分一次自改进与改进机制本身的递归变化。",
"scope": "覆盖长程 agent、模型与 harness 两条改进路线、自动化研究和自修改系统。",
"framework": "逐阶段区分人类、系统及双方共同承担的工作；“长程”重点是早期动作影响后续动作与最终结果，不只是 token 多或运行时间长。",
"feedback": "检查评价标准是谁定的、结果如何影响修复和后续决策，并提醒匹配预算、验证泄漏和人类干预都会影响成绩解释。",
"reading": "先看定义和分类，再读两条改进路线；配套 Awesome-AI4AI 提供文献地图，RSIHub 提供实验框架入口。",
"limits": "对所审阅系统的归纳受搜索时间与纳入标准限制；不能把“单项能力强”解释为端到端自主研发已解决。项目页和论文的具体定义应分别按原文阅读。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "reef",
"title": "Reef: Continual Learning Infrastructure for Self-Improving Agents",
"date": "2026-09-15",
"dateLabel": "收录日期",
"year": "2026",
"url": "https://github.com/Human-Agent-Society/reef",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "resources",
"categories": [
"resources"
],
"methodType": "",
"contentType": "repository",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Infrastructure",
"Weights",
"Skill",
"Continual"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "仓库",
"url": "https://github.com/Human-Agent-Society/reef"
}
],
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Harness-evolving deployment",
"url": "https://github.com/Human-Agent-Society/reef#harness-evolving-deployment"
},
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
],
"fields": {
"本质定位": "提供可复用基础设施，让已有 agent 从交互记录学习，并将通过评价的参数或 harness 更新交付给后续请求。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "框架提供接口与版本管理，不替使用者保证反馈正确、数据隔离或改进泛化。示例任务上的候选胜出不能直接当成独立 benchmark 提升。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "持续学习不只是训练一次模型，还需要把线上交互、反馈、候选评价和版本交付接在一起，否则更新难以稳定进入实际服务。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
},
{
"key": "position",
"value": "提供可复用基础设施，让已有 agent 从交互记录学习，并将通过评价的参数或 harness 更新交付给后续请求。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
},
{
"key": "conclusion",
"value": "支持参数训练与提示／规则／技能更新，保留版本历史。具体学什么、怎样评分由学习配置（recipe）决定；安装框架本身不等于 agent 会自动获得可靠提升。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
]
}
],
"fields": {
"object": "按学习配置（recipe） 更新模型参数，或 harness 中的提示、规则、技能；两条路线都由版本系统保存与交付。",
"executor": "由用户配置的模型执行请求；参数训练可接 Slime 训练后端与 SGLang 推理服务，harness 路线可接模型 API，无需本地训练 GPU。",
"modifier": "学习配置（recipe）定义从交互和反馈产生更新的方法，候选评估模块按配置选择更新，版本模块保存和交付结果。没有一个固定通用的修改模型。",
"verdict": "反馈需关联到具体交互记录，可是分数或结构化说明；例如 coding 教程把测试失败上报后生成候选技能，再与当前版本在三道教学任务上比较，仅胜出才发布。",
"seed": "已有推理服务与 agent 客户端之外，补上交互记录、反馈匹配、更新任务、候选评价和版本交付。它是连接组件的基础设施，不是统一的 benchmark 方法。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "按学习配置（recipe） 更新模型参数，或 harness 中的提示、规则、技能；两条路线都由版本系统保存与交付。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "由用户配置的模型执行请求；参数训练可接 Slime 训练后端与 SGLang 推理服务，harness 路线可接模型 API，无需本地训练 GPU。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "学习配置（recipe）定义从交互和反馈产生更新的方法，候选评估模块按配置选择更新，版本模块保存和交付结果。没有一个固定通用的修改模型。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "反馈需关联到具体交互记录，可是分数或结构化说明；例如 coding 教程把测试失败上报后生成候选技能，再与当前版本在三道教学任务上比较，仅胜出才发布。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Harness-evolving deployment",
"url": "https://github.com/Human-Agent-Society/reef#harness-evolving-deployment"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "已有推理服务与 agent 客户端之外，补上交互记录、反馈匹配、更新任务、候选评价和版本交付。它是连接组件的基础设施，不是统一的 benchmark 方法。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
},
{
"key": "data",
"label": "使用的数据",
"status": "recorded",
"value": "数据由部署场景与 recipe 决定。README 列出的示例包括 AIME 2025、IMOAnswerBench、Terminal-Bench、GSM8K 任务流和 WildClawBench；这些不是全项目共享的一套训练／测试划分。",
"sources": [
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
]
},
{
"key": "use",
"label": "如何使用",
"status": "recorded",
"value": "适合搭建自己的持续学习或 harness 优化实验；先选 recipe，再明确任务集、反馈实现、候选选择和独立测试集。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "框架提供接口与版本管理，不替使用者保证反馈正确、数据隔离或改进泛化。示例任务上的候选胜出不能直接当成独立 benchmark 提升。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Harness-evolving deployment",
"url": "https://github.com/Human-Agent-Society/reef#harness-evolving-deployment"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "提供可复用基础设施，让已有 agent 从交互记录学习，并将通过评价的参数或 harness 更新交付给后续请求。",
"object": "按学习配置（recipe） 更新模型参数，或 harness 中的提示、规则、技能；两条路线都由版本系统保存与交付。",
"executor": "由用户配置的模型执行请求；参数训练可接 Slime 训练后端与 SGLang 推理服务，harness 路线可接模型 API，无需本地训练 GPU。",
"modifier": "学习配置（recipe）定义从交互和反馈产生更新的方法，候选评估模块按配置选择更新，版本模块保存和交付结果。没有一个固定通用的修改模型。",
"verdict": "反馈需关联到具体交互记录，可是分数或结构化说明；例如 coding 教程把测试失败上报后生成候选技能，再与当前版本在三道教学任务上比较，仅胜出才发布。",
"seed": "已有推理服务与 agent 客户端之外，补上交互记录、反馈匹配、更新任务、候选评价和版本交付。它是连接组件的基础设施，不是统一的 benchmark 方法。",
"data": "数据由部署场景与 recipe 决定。README 列出的示例包括 AIME 2025、IMOAnswerBench、Terminal-Bench、GSM8K 任务流和 WildClawBench；这些不是全项目共享的一套训练／测试划分。",
"use": "适合搭建自己的持续学习或 harness 优化实验；先选 recipe，再明确任务集、反馈实现、候选选择和独立测试集。",
"limits": "框架提供接口与版本管理，不替使用者保证反馈正确、数据隔离或改进泛化。示例任务上的候选胜出不能直接当成独立 benchmark 提升。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "openai-research-acceleration",
"title": "Research Acceleration: The View Inside OpenAI",
"date": "2026-09-06",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"IndustryReport",
"ResearchAutomation",
"org:openai"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
],
"fields": {
"本质定位": "以内部使用记录说明 coding agent 如何参与 AI 研发，测量使用量、任务类型、完成情况及人类干预。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "算力、模型、使用方式和研究流程同时变化，代码量／实验量不是研究质量的直接度量；内部观察不能等同受控因果实验或多代自改进结果。",
"visibleKeys": [
"scope",
"actors",
"feedback"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "外界难以观察前沿实验室内部 agent 到底承担了多少研究工作；代码和实验数量增加，也未必等同整体科研进展加速。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "position",
"value": "以内部使用记录说明 coding agent 如何参与 AI 研发，测量使用量、任务类型、完成情况及人类干预。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "conclusion",
"value": "报告 agent 使用、代码产出与实验量增加，且承担更复杂任务；人类仍决定研究方向、结果取舍和部署。数据主要是观察性证据，不能单凭相关性证明自主 RSI 已实现。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
}
],
"fields": {
"scope": "OpenAI 内部研究流程中的 agent 使用，涵盖实现、运行、分析等环节；不是公开统一训练 benchmark。",
"actors": "coding agents 辅助研究人员；人类继续设定优先级、决定追哪些结果及是否扩大训练或部署。文章不提供一个可复现的固定执行模型／harness 配置。",
"feedback": "按内部使用与任务结果记录统计，由 agent 分类器分析任务类型和可判断结果的成功率；不确定结果排除，另统计人类干预。不是 benchmark verifier 给所有会话逐一验对。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "OpenAI 内部研究流程中的 agent 使用，涵盖实现、运行、分析等环节；不是公开统一训练 benchmark。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "actors",
"label": "谁在做研究",
"status": "recorded",
"value": "coding agents 辅助研究人员；人类继续设定优先级、决定追哪些结果及是否扩大训练或部署。文章不提供一个可复现的固定执行模型／harness 配置。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "feedback",
"label": "反馈与证据",
"status": "recorded",
"value": "按内部使用与任务结果记录统计，由 agent 分类器分析任务类型和可判断结果的成功率；不确定结果排除，另统计人类干预。不是 benchmark verifier 给所有会话逐一验对。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "data",
"label": "使用的数据",
"status": "recorded",
"value": "主要分析 2026 年内部会话、代码和实验运行数据；按月份与估计任务难度分组。文章没有提供一套可下载的 train/test 数据集。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "算力、模型、使用方式和研究流程同时变化，代码量／实验量不是研究质量的直接度量；内部观察不能等同受控因果实验或多代自改进结果。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
}
],
"sourceDate": "2026-09-15",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "以内部使用记录说明 coding agent 如何参与 AI 研发，测量使用量、任务类型、完成情况及人类干预。",
"scope": "OpenAI 内部研究流程中的 agent 使用，涵盖实现、运行、分析等环节；不是公开统一训练 benchmark。",
"actors": "coding agents 辅助研究人员；人类继续设定优先级、决定追哪些结果及是否扩大训练或部署。文章不提供一个可复现的固定执行模型／harness 配置。",
"feedback": "按内部使用与任务结果记录统计，由 agent 分类器分析任务类型和可判断结果的成功率；不确定结果排除，另统计人类干预。不是 benchmark verifier 给所有会话逐一验对。",
"data": "主要分析 2026 年内部会话、代码和实验运行数据；按月份与估计任务难度分组。文章没有提供一套可下载的 train/test 数据集。",
"limits": "算力、模型、使用方式和研究流程同时变化，代码量／实验量不是研究质量的直接度量；内部观察不能等同受控因果实验或多代自改进结果。"
},
"attributions": [
{
"tag": "org:openai",
"label": "OpenAI",
"kind": "institution",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "shiyu-rsi-what-evolves",
"title": "什么在进化？——Model、Harness 与 Artifact 的三层地图",
"date": "2026-08-10",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "系列另一篇",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
],
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"fields": {
"本质定位": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "不同工作都叫 self-evolving，却可能只改答案、改运行框架或训练模型；混在一起比较，容易把产物变好当成系统能力变强。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "position",
"value": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "conclusion",
"value": "把“修改对象”与“反馈是否改进下一轮的方法”分开，说明三层如何相互促进；核心分类和案例来源明确归于原作者。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
}
],
"fields": {
"author": "作者：Shiyu Ni；个人研究博客系列。",
"reading": "先读 §2 三层划分，再读 §4 harness 的设计与优化，最后用 §8 的反馈、评估和递归问题检查具体案例。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者：Shiyu Ni；个人研究博客系列。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先读 §2 三层划分，再读 §4 harness 的设计与优化，最后用 §8 的反馈、评估和递归问题检查具体案例。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 §1–2、§4、§6、§8；写在前面：来源与致谢；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。",
"author": "作者：Shiyu Ni；个人研究博客系列。",
"scope": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。",
"reading": "先读 §2 三层划分，再读 §4 harness 的设计与优化，最后用 §8 的反馈、评估和递归问题检查具体案例。",
"evidence": "依据 §1–2、§4、§6、§8；写在前面：来源与致谢；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "shiyu-rsi-loop-closes",
"title": "递归如何闭环？——从可修改范围到 Agent as Service",
"date": "2026-08-10",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "系列另一篇",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
],
"fields": {
"本质定位": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "迭代次数多不等于递归；同时开放数据、训练、环境和框架，也容易让能力提升的来源无法区分。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "position",
"value": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "conclusion",
"value": "先隔离单个研究对象，再测试联合改进，最后检查改进后的系统是否更会改进下一代；服务化的意义是明确实验边界。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
}
],
"fields": {
"author": "作者：Shiyu Ni；个人研究博客系列。",
"reading": "接上篇阅读：重点看各类 scope 的区别，以及如何固定其他组件、隔离评测、记录版本来判断改进来自哪里。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者：Shiyu Ni；个人研究博客系列。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "接上篇阅读：重点看各类 scope 的区别，以及如何固定其他组件、隔离评测、记录版本来判断改进来自哪里。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。",
"author": "作者：Shiyu Ni；个人研究博客系列。",
"scope": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。",
"reading": "接上篇阅读：重点看各类 scope 的区别，以及如何固定其他组件、隔离评测、记录版本来判断改进来自哪里。",
"evidence": "依据 定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "liu-self-evolving-taxonomy",
"title": "A Taxonomy of Self-evolving Agents",
"date": "2026-07-08",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap",
"person:shilong-liu"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
],
"fields": {
"本质定位": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "相似的自进化名称涵盖不同修改对象，难以判断工作之间的关系。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
{
"key": "position",
"value": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
{
"key": "conclusion",
"value": "再将 harness 展开为 prompt／memory、tool／skill 和多 agent；强调三层可相互促进，最终价值要看实际产物。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
}
],
"fields": {
"author": "作者／发布者：Shilong Liu。",
"reading": "适合作为分类入口；对照本站“什么在变”，再读文末三层边界及现实应用的讨论。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Shilong Liu。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "适合作为分类入口；对照本站“什么在变”，再读文末三层边界及现实应用的讨论。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。",
"author": "作者／发布者：Shilong Liu。",
"scope": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。",
"reading": "适合作为分类入口；对照本站“什么在变”，再读文末三层边界及现实应用的讨论。",
"evidence": "依据 Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "person:shilong-liu",
"label": "Shilong Liu",
"kind": "scholar",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "weng-harness-engineering",
"title": "Harness Engineering for Self-Improvement",
"date": "2026-07-04",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation",
"person:lilian-weng"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
],
"fields": {
"本质定位": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型能力之外，执行循环、状态管理和评估也决定 agent 能否稳定完成长期任务。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
{
"key": "position",
"value": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
{
"key": "conclusion",
"value": "串起工作流、文件记忆、并行子 agent，以及从上下文到框架代码、修改机制和参数联合优化的路径。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
}
],
"fields": {
"author": "作者／发布者：Lilian Weng。",
"reading": "先读三种设计模式，再按 Harness Optimization 对照论文；近期 RSI 路线是作者判断，不是已实现的统一实验结论。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Lilian Weng。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先读三种设计模式，再按 Harness Optimization 对照论文；近期 RSI 路线是作者判断，不是已实现的统一实验结论。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。",
"author": "作者／发布者：Lilian Weng。",
"scope": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。",
"reading": "先读三种设计模式，再按 Harness Optimization 对照论文；近期 RSI 路线是作者判断，不是已实现的统一实验结论。",
"evidence": "依据 Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "person:lilian-weng",
"label": "Lilian Weng",
"kind": "scholar",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "zhihu-self-evolving-guide",
"title": "自进化（Self-evolving／RSI），一篇就够了",
"date": "2026-09-16",
"dateLabel": "收录日期（原文发布日期未确认）",
"year": "2026",
"url": "https://zhuanlan.zhihu.com/p/2065227313973825752",
"priority": "R",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "引用线索 · 依据二手转述",
"reviewed": false,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"fields": {
"本质定位": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "据 Shiyu 的转述，文章通过案例解释产物、harness 和模型如何参与自进化。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "position",
"value": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "conclusion",
"value": "转述涉及 AlphaEvolve、Hermes、AIDE² 和联合优化，并保留预算对等、独立评测等反面检查。未直接读取原文，不扩写实验结论。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
}
],
"fields": {
"author": "作者／发布者：知乎原作者（当前来源未提供署名）。",
"reading": "可先读 Shiyu 上篇对应案例，再打开知乎原文；这是一条引用线索，不是本站已直接核对的原文摘要。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：知乎原作者（当前来源未提供署名）。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "可先读 Shiyu 上篇对应案例，再打开知乎原文；这是一条引用线索，不是本站已直接核对的原文摘要。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 上篇：来源与致谢、§3、§4.8、§6；原文当前无法直接读取，只按 Shiyu 博客中明确引用的内容建立阅读入口。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。",
"author": "作者／发布者：知乎原作者（当前来源未提供署名）。",
"scope": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。",
"reading": "可先读 Shiyu 上篇对应案例，再打开知乎原文；这是一条引用线索，不是本站已直接核对的原文摘要。",
"evidence": "依据 上篇：来源与致谢、§3、§4.8、§6；原文当前无法直接读取，只按 Shiyu 博客中明确引用的内容建立阅读入口。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "melon-self-evolve-rsi",
"title": "万字深度：从 Self-Evolve 到 RSI",
"date": "2026-09-16",
"dateLabel": "收录日期（原文发布日期未确认）",
"year": "2026",
"url": "https://www.xiaohongshu.com/discovery/item/6a71f07700000000220146bf",
"priority": "R",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "引用线索 · 依据二手转述",
"reviewed": false,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
],
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
],
"fields": {
"本质定位": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "据 Shiyu 的转述，只扩大可修改范围不足以解释改进的来源，也不足以证明改进者本身变强。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "position",
"value": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "conclusion",
"value": "转述的主线是 Everything as Service 与 Scoped → Joint → Recursive；当前简介来自下篇的明确归因，不冒充原文直接核对。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
}
],
"fields": {
"author": "作者／发布者：Melon（署名据 Shiyu 的致谢）。",
"reading": "小红书原文可能需要登录；可结合 Shiyu 下篇阅读，注意区分原作者框架与学习笔记补充的论文说明。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Melon（署名据 Shiyu 的致谢）。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "小红书原文可能需要登录；可结合 Shiyu 下篇阅读，注意区分原作者框架与学习笔记补充的论文说明。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 下篇：来源与致谢、定义、Everything as Service；原文当前无法直接读取，只按 Shiyu 博客中明确引用的内容建立阅读入口。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。",
"author": "作者／发布者：Melon（署名据 Shiyu 的致谢）。",
"scope": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。",
"reading": "小红书原文可能需要登录；可结合 Shiyu 下篇阅读，注意区分原作者框架与学习笔记补充的论文说明。",
"evidence": "依据 下篇：来源与致谢、定义、Everything as Service；原文当前无法直接读取，只按 Shiyu 博客中明确引用的内容建立阅读入口。"
},
"attributions": [],
"researchProblems": [
"perspectives"
]
},
{
"id": "yudkowsky-recursive-self-improvement",
"title": "Recursive Self-Improvement",
"date": "2008-12-01",
"dateLabel": "博客发布",
"year": "2008",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
],
"fields": {
"本质定位": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "讨论 AI 能力增长为什么可能不是匀速，以及快速增长需要哪些依赖关系。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
{
"key": "position",
"value": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
{
"key": "conclusion",
"value": "区分解决任务、获得知识、改进发现知识的方法与改进认知机制；属于理论论证，不是现代 agent 的实证报告。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
}
],
"fields": {
"author": "作者／发布者：Eliezer Yudkowsky。",
"reading": "适合理解“递归”为什么不等于重复尝试；不要把文中的增长预期当成今天系统已验证的能力。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Eliezer Yudkowsky。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "适合理解“递归”为什么不等于重复尝试；不要把文中的增长预期当成今天系统已验证的能力。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 正文：optimization slope/resources/efficiency 与递归层次；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。",
"author": "作者／发布者：Eliezer Yudkowsky。",
"scope": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。",
"reading": "适合理解“递归”为什么不等于重复尝试；不要把文中的增长预期当成今天系统已验证的能力。",
"evidence": "依据 正文：optimization slope/resources/efficiency 与递归层次；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [],
"researchProblems": [
"improver"
]
},
{
"id": "weco-aide2-blog",
"title": "AIDE²: The First Evidence of Recursive Self-Improvement",
"date": "2026-07-14",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation",
"org:weco"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
],
"fields": {
"本质定位": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "作者希望区分仅自动运行研究，与在固定成本下持续提高研究效率。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
{
"key": "position",
"value": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
{
"key": "conclusion",
"value": "报告内层能力提升和外部任务迁移，但明确未达到改进后的内层也更擅长充当外层的条件；标题中的“首次”是作者主张。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
}
],
"fields": {
"author": "作者／发布者：Weco Team。",
"reading": "重点读 public/private 分数、成本约束与 ignition test，区分内层变强和修改者变强。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Weco Team。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "重点读 public/private 分数、成本约束与 ignition test，区分内层变强和修改者变强。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 §1、The inner-loop evaluation、§2.2、§3.1、§4；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。",
"author": "作者／发布者：Weco Team。",
"scope": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。",
"reading": "重点读 public/private 分数、成本约束与 ignition test，区分内层变强和修改者变强。",
"evidence": "依据 §1、The inner-loop evaluation、§2.2、§3.1、§4；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "org:weco",
"label": "Weco",
"kind": "institution",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
}
],
"researchProblems": [
"improver"
]
},
{
"id": "deepmind-alphaevolve-blog",
"title": "AlphaEvolve: A Gemini-powered coding agent for designing advanced algorithms",
"date": "2025-05-14",
"dateLabel": "博客发布",
"year": "2025",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation",
"org:google-deepmind"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
],
"fields": {
"本质定位": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "复杂算法优化需要大量候选探索，同时需要可靠地验证候选是否正确、是否更快。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"key": "position",
"value": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"key": "conclusion",
"value": "展示数学、数据中心和 AI 训练相关算法改进；主要修改对象是算法产物，训练基础设施获益与搜索器自身改进应分开理解。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
}
],
"fields": {
"author": "作者／发布者：AlphaEvolve team · Google DeepMind。",
"reading": "适合理解“产物进化如何反哺模型训练”；先看代码生成与验证循环，再看实际应用。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：AlphaEvolve team · Google DeepMind。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "适合理解“产物进化如何反哺模型训练”；先看代码生成与验证循环，再看实际应用。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 Designing better algorithms with large language models；导言与应用案例；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。",
"author": "作者／发布者：AlphaEvolve team · Google DeepMind。",
"scope": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。",
"reading": "适合理解“产物进化如何反哺模型训练”；先看代码生成与验证循环，再看实际应用。",
"evidence": "依据 Designing better algorithms with large language models；导言与应用案例；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "org:google-deepmind",
"label": "Google DeepMind",
"kind": "institution",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "minimax-m27-blog",
"title": "MiniMax M2.7: Early Echoes of Self-Evolution",
"date": "2026-03-18",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://www.minimax.io/news/minimax-m27-en",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation",
"org:minimax"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
],
"fields": {
"本质定位": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型研发涉及数据、训练和工程协作，人工研究吞吐限制迭代速度。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
{
"key": "position",
"value": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
{
"key": "conclusion",
"value": "介绍实验监控、日志分析、代码修复与框架迭代；人类仍提供方向和关键决策，属于研发流程报告。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
}
],
"fields": {
"author": "作者／发布者：MiniMax。",
"reading": "读研究 agent 的具体工作流，区分它参与自身研发和完全自主递归；产品 benchmark 分数不能单独归因于自进化。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：MiniMax。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "读研究 agent 的具体工作流，区分它参与自身研发和完全自主递归；产品 benchmark 分数不能单独归因于自进化。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 Building an agent for model self-evolution；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。",
"author": "作者／发布者：MiniMax。",
"scope": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。",
"reading": "读研究 agent 的具体工作流，区分它参与自身研发和完全自主递归；产品 benchmark 分数不能单独归因于自进化。",
"evidence": "依据 Building an agent for model self-evolution；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "org:minimax",
"label": "MiniMax",
"kind": "institution",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "apodex-10-blog",
"title": "Apodex-1.0",
"date": "2026-06-08",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://www.apodex.com/blog/apodex-1.0",
"priority": "K",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
],
"fields": {
"本质定位": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "长期研究任务需要检索、工具和验证，单条执行链与单个上下文难以承载全部工作。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
{
"key": "position",
"value": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
{
"key": "conclusion",
"value": "orchestrator 分配任务，子 agent 并行研究，global verifier 审核证据；这主要说明 harness 设计，不能单凭自检证明跨轮框架进化。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
}
],
"fields": {
"author": "作者／发布者：Apodex。",
"reading": "适合对照“验证答案”和“修改 harness”的区别；看团队协作结构与同系列模式比较。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Apodex。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "适合对照“验证答案”和“修改 harness”的区别；看团队协作结构与同系列模式比较。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 Scaling Reasoning with an Agent Team；Where Apodex Stands；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。",
"author": "作者／发布者：Apodex。",
"scope": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。",
"reading": "适合对照“验证答案”和“修改 harness”的区别；看团队协作结构与同系列模式比较。",
"evidence": "依据 Scaling Reasoning with an Agent Team；Where Apodex Stands；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [],
"researchProblems": [
"harness"
]
},
{
"id": "weng-prompt-engineering",
"title": "Prompt Engineering",
"date": "2023-03-15",
"dateLabel": "博客发布",
"year": "2023",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/",
"priority": "R",
"priorityBasis": "按主题相关性与阅读价值标注",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Prompt",
"person:lilian-weng"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "Shiyu 的相关解读",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
],
"fields": {
"本质定位": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "",
"visibleKeys": [
"author",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "模型输出对输入组织敏感，需要区分不同提示方法及它们的适用条件。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
{
"key": "position",
"value": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
{
"key": "conclusion",
"value": "梳理提示与推理方法；属于自进化研究的前置知识，不应把使用提示技巧直接称为 RSI。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
}
],
"fields": {
"author": "作者／发布者：Lilian Weng。",
"reading": "作为 Weng harness 博客的背景链接阅读，帮助理解提示设计与让系统自动更新提示的区别。"
}
},
"profile": {
"fields": [
{
"key": "author",
"label": "作者／发布者",
"status": "recorded",
"value": "作者／发布者：Lilian Weng。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "作为 Weng harness 博客的背景链接阅读，帮助理解提示设计与让系统自动更新提示的区别。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "依据 正文目录：提示、示例与推理方法；简介依据文章原文；文中研究判断与实验宣称归于作者。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
}
],
"sourceDate": "2026-09-16",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。",
"author": "作者／发布者：Lilian Weng。",
"scope": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。",
"reading": "作为 Weng harness 博客的背景链接阅读，帮助理解提示设计与让系统自动更新提示的区别。",
"evidence": "依据 正文目录：提示、示例与推理方法；简介依据文章原文；文中研究判断与实验宣称归于作者。"
},
"attributions": [
{
"tag": "person:lilian-weng",
"label": "Lilian Weng",
"kind": "scholar",
"sources": [
{
"label": "原文署名／发布机构",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "2609.20519",
"title": "SoL-Pi: Recursively Scaling Auto-Research Loops for Efficient Agent Harness",
"date": "2026-09-17",
"dateLabel": "论文首发",
"year": "2026",
"url": "https://arxiv.org/abs/2609.20519",
"priority": "C",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "methods",
"categories": [
"methods"
],
"methodType": "harness",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"HarnessCode",
"Context",
"Tool",
"SeparateEvolver",
"ExecutableVerifier",
"Update20260921",
"org:nvidia",
"org:mit",
"person:song-han"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [
{
"label": "作者博客",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
],
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
},
{
"label": "作者博客：Humans set priors / Results / Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
},
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.20519"
}
],
"fields": {
"本质定位": "研究自动搜索能否找到跨任务、跨模型复用的 harness 效率改进，而非只提高搜索题得分。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "不能把主表全部 51 题都称为最终未参与选择的测试；Performance 配置还按各模型最高单机制得分挑选，与预先固定的完整组合有别。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "长任务反复传递上下文和工具输出，成本不断累积；局部省 token 也可能损害任务完成率。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "position",
"value": "研究自动搜索能否找到跨任务、跨模型复用的 harness 效率改进，而非只提高搜索题得分。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "conclusion",
"value": "保留动作合并、上下文压缩、输出归档和证据摘要四种机制。完整组合保留约 94% 的 Pi 得分，API 成本约降三分之一；不是无损提效。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
}
],
"fields": {
"object": "修改 Pi 的执行、上下文和工具输出处理代码；模型参数与外层研究程序固定。",
"executor": "主要对照 GPT-5.6 Sol 与 Opus 5；群体实验由 Sol 协调、20 个 Luna worker 执行。",
"modifier": "研究 agent 提案、实现并独立审查；论文未逐角色列型号。人类先筛方向，后审阅并整理成功机制代码。",
"verdict": "开发用隐藏回归测试或自建可执行 verifier 判成功，再比较 token／成本并检查能力下限；不是模型自称完成。",
"seed": "Pi：已有模型调用、文件／命令工具和原生上下文压缩的 coding harness；四种机制作为扩展加入。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "修改 Pi 的执行、上下文和工具输出处理代码；模型参数与外层研究程序固定。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "主要对照 GPT-5.6 Sol 与 Opus 5；群体实验由 Sol 协调、20 个 Luna worker 执行。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "研究 agent 提案、实现并独立审查；论文未逐角色列型号。人类先筛方向，后审阅并整理成功机制代码。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
},
{
"label": "作者博客：Humans set priors / Results / Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "开发用隐藏回归测试或自建可执行 verifier 判成功，再比较 token／成本并检查能力下限；不是模型自称完成。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "Pi：已有模型调用、文件／命令工具和原生上下文压缩的 coding harness；四种机制作为扩展加入。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "152 个方向各自实现、测试、保留或淘汰；临时修改的研究调度代码不继承。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "无参数训练。搜索用 495 个 GitHub issue–PR 仓库环境及 40 个自建 verifier 环境，共 535 个。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "开发环境反馈驱动修改；冻结候选后用 EdgeBench 11 题单向验收，失败不再反馈给搜索。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "EdgeBench 另 40 题留作最终泛化，主表汇报全部 51 题；另测 Terminal-Bench 4 的 63 题、IMO 2026 六题（Lean 4 验证）及 kernel 优化。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "不能把主表全部 51 题都称为最终未参与选择的测试；Performance 配置还按各模型最高单机制得分挑选，与预先固定的完整组合有别。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "Terminal-Bench 4 完成 15 题，Pi 为 18 题；更低成本不等于更高成功率。多代递归提效仍是后续设想。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "HF 页面 95 赞（2026-09-21）；关注度快照不代表独立复现或同行评审。",
"sources": [
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.20519"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究自动搜索能否找到跨任务、跨模型复用的 harness 效率改进，而非只提高搜索题得分。",
"object": "修改 Pi 的执行、上下文和工具输出处理代码；模型参数与外层研究程序固定。",
"executor": "主要对照 GPT-5.6 Sol 与 Opus 5；群体实验由 Sol 协调、20 个 Luna worker 执行。",
"modifier": "研究 agent 提案、实现并独立审查；论文未逐角色列型号。人类先筛方向，后审阅并整理成功机制代码。",
"verdict": "开发用隐藏回归测试或自建可执行 verifier 判成功，再比较 token／成本并检查能力下限；不是模型自称完成。",
"seed": "Pi：已有模型调用、文件／命令工具和原生上下文压缩的 coding harness；四种机制作为扩展加入。",
"cycle": "152 个方向各自实现、测试、保留或淘汰；临时修改的研究调度代码不继承。",
"train": "无参数训练。搜索用 495 个 GitHub issue–PR 仓库环境及 40 个自建 verifier 环境，共 535 个。",
"debug": "开发环境反馈驱动修改；冻结候选后用 EdgeBench 11 题单向验收，失败不再反馈给搜索。",
"test": "EdgeBench 另 40 题留作最终泛化，主表汇报全部 51 题；另测 Terminal-Bench 4 的 63 题、IMO 2026 六题（Lean 4 验证）及 kernel 优化。",
"isolation": "不能把主表全部 51 题都称为最终未参与选择的测试；Performance 配置还按各模型最高单机制得分挑选，与预先固定的完整组合有别。",
"limits": "Terminal-Bench 4 完成 15 题，Pi 为 18 题；更低成本不等于更高成功率。多代递归提效仍是后续设想。",
"evidence": "HF 页面 95 赞（2026-09-21）；关注度快照不代表独立复现或同行评审。"
},
"attributions": [
{
"tag": "org:nvidia",
"label": "NVIDIA",
"kind": "institution",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"tag": "org:mit",
"label": "MIT",
"kind": "institution",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"tag": "person:song-han",
"label": "Song Han",
"kind": "scholar",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
}
],
"researchProblems": [
"harness",
"evaluation"
]
},
{
"id": "2609.19656",
"title": "Self-Evolving Search Index",
"date": "2026-09-17",
"dateLabel": "论文首发",
"year": "2026",
"url": "https://arxiv.org/abs/2609.19656",
"priority": "K",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "methods",
"categories": [
"methods"
],
"methodType": "harness",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"MemoryContent",
"MemoryMechanism",
"Context",
"SeparateEvolver",
"LLMJudge",
"GoldLabel",
"Update20260921"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
},
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.19656"
}
],
"fields": {
"本质定位": "让索引自动发现检索缺口、生成练习查询并修订索引键，检验对搜索和记忆读取的帮助。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "隔离的是评测查询；索引允许访问将被检索的语料或历史记忆，并非语料也完全不可见。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "固定索引无法适应不同检索需求，失败后通常还要人手动诊断和重写表示。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "position",
"value": "让索引自动发现检索缺口、生成练习查询并修订索引键，检验对搜索和记忆读取的帮助。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "conclusion",
"value": "变化的是文档／记忆的检索表示，不是模型参数。逐次核验新键是否忠于原文、能检索回原文并减少混淆。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
}
],
"fields": {
"object": "改文档或记忆条目的索引键，即供检索匹配的文本表示；原始内容和下游 agent 固定。",
"executor": "检索器为 BM25、BGE-Large、Qwen3-Embedding-8B；搜索用 GPT-OSS-120B、GPT-5.4-nano、Gemini-3.7-Flash、Kimi-K2.5；记忆用 Qwen3.5-9B。",
"modifier": "Qwen3.6-35B-A3B 同时承担索引修改、模拟查询和忠实性／可回答性评审。",
"verdict": "进化反馈：同一 Qwen 按原文给忠实性／可回答性打 0–3 分，至少 2 分通过；另用检索排名和键间相关度检查区分性。",
"seed": "在已有检索器上加索引修改循环；搜索／记忆沿用官方运行系统，替换索引但仍向 reader 返回原文。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "改文档或记忆条目的索引键，即供检索匹配的文本表示；原始内容和下游 agent 固定。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "检索器为 BM25、BGE-Large、Qwen3-Embedding-8B；搜索用 GPT-OSS-120B、GPT-5.4-nano、Gemini-3.7-Flash、Kimi-K2.5；记忆用 Qwen3.5-9B。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "Qwen3.6-35B-A3B 同时承担索引修改、模拟查询和忠实性／可回答性评审。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "进化反馈：同一 Qwen 按原文给忠实性／可回答性打 0–3 分，至少 2 分通过；另用检索排名和键间相关度检查区分性。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "在已有检索器上加索引修改循环；搜索／记忆沿用官方运行系统，替换索引但仍向 reader 返回原文。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "诊断失败键→修订→模型与检索规则验收；每文档最多 10 键，含固定原文键。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "不训练参数。在各基准语料／历史记忆上模拟查询，20 轮共 2,560 条查询用于索引进化；不使用正式评测问题。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "按忠实性与检索规则接受局部修改；报告固定第 20 轮，不按测试最高分挑版本。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "BRIGHT、Spider 2.0、FIBEN、BEAVER 用标注相关性算 nDCG@10；BrowseComp-Plus 830 题由 Qwen3-32B 对参考答案判分；LongMemEval-V2 Small 451 题用规则与 GPT-5.2 medium 判分。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "隔离的是评测查询；索引允许访问将被检索的语料或历史记忆，并非语料也完全不可见。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "HF 页面 46 赞（2026-09-21）；属于检索与记忆组件自改进的相关路线。",
"sources": [
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.19656"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "让索引自动发现检索缺口、生成练习查询并修订索引键，检验对搜索和记忆读取的帮助。",
"object": "改文档或记忆条目的索引键，即供检索匹配的文本表示；原始内容和下游 agent 固定。",
"executor": "检索器为 BM25、BGE-Large、Qwen3-Embedding-8B；搜索用 GPT-OSS-120B、GPT-5.4-nano、Gemini-3.7-Flash、Kimi-K2.5；记忆用 Qwen3.5-9B。",
"modifier": "Qwen3.6-35B-A3B 同时承担索引修改、模拟查询和忠实性／可回答性评审。",
"verdict": "进化反馈：同一 Qwen 按原文给忠实性／可回答性打 0–3 分，至少 2 分通过；另用检索排名和键间相关度检查区分性。",
"seed": "在已有检索器上加索引修改循环；搜索／记忆沿用官方运行系统，替换索引但仍向 reader 返回原文。",
"cycle": "诊断失败键→修订→模型与检索规则验收；每文档最多 10 键，含固定原文键。",
"train": "不训练参数。在各基准语料／历史记忆上模拟查询，20 轮共 2,560 条查询用于索引进化；不使用正式评测问题。",
"debug": "按忠实性与检索规则接受局部修改；报告固定第 20 轮，不按测试最高分挑版本。",
"test": "BRIGHT、Spider 2.0、FIBEN、BEAVER 用标注相关性算 nDCG@10；BrowseComp-Plus 830 题由 Qwen3-32B 对参考答案判分；LongMemEval-V2 Small 451 题用规则与 GPT-5.2 medium 判分。",
"isolation": "隔离的是评测查询；索引允许访问将被检索的语料或历史记忆，并非语料也完全不可见。",
"evidence": "HF 页面 46 赞（2026-09-21）；属于检索与记忆组件自改进的相关路线。"
},
"attributions": [],
"researchProblems": [
"experience",
"harness"
]
},
{
"id": "2609.17523",
"title": "ScienceBuddy: Recursive-in-Recursive Self-Improvement for Interactive Scientific Agents",
"date": "2026-09-15",
"dateLabel": "论文首发",
"year": "2026",
"url": "https://arxiv.org/abs/2609.17523",
"priority": "K",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "methods",
"categories": [
"methods"
],
"methodType": "joint",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"Skill",
"Prompt",
"Weights",
"JointEvolution",
"SeparateEvolver",
"LLMJudge",
"ExecutableVerifier",
"Update20260921",
"org:phai-labs"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
},
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.17523"
}
],
"fields": {
"本质定位": "研究科研协作能否同时推动工作方式和模型学习：内层改 harness，外层在选定 harness 下做强化学习。"
},
"protocol": "见数据隔离说明",
"protocolBasis": "论文声明开发、策略训练、最终测试分离；未完整披露逐任务判分代码及解释型 judge 型号，不能将整套判分笼统写成规则匹配。",
"visibleKeys": [
"object",
"executor",
"modifier",
"verdict",
"seed"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "科研对话产生了需求、纠错和执行证据，但如果不转成后续可用的任务与反馈，系统不会持续改进。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "position",
"value": "研究科研协作能否同时推动工作方式和模型学习：内层改 harness，外层在选定 harness 下做强化学习。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "conclusion",
"value": "三轮联合实验报告单次答对率由 42.2% 到 73.3%。另做固定模型／固定 harness 对照；单独 harness 实验只改指令和 scoped skill。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
}
],
"fields": {
"object": "联合实验更新 harness 与 Qwen 参数；单独 harness 实验只改指令／技能文本，执行循环、工具和上下文处理固定。",
"executor": "Qwen3.5-4B 执行科研任务；参数学习后使用对应更新的检查点。",
"modifier": "GPT-6 Astra 诊断并编辑 harness；Qwen3.8-27B 模拟受限用户反馈并解释反馈，训练程序执行 GRPO。",
"verdict": "harness：私有 verifier 给对错／提交状态，模拟用户只回复允许的确认或检查要求。参数：按固定 rubric 汇总可执行检查与固定 judge 分数，不用模拟用户的三值诊断分替代奖励。",
"seed": "Biomni 科研工具／数据基础上的单文件 harness；已有 Python REPL、数据读取与答案提交，初始没有新增指令／技能。"
}
},
"profile": {
"fields": [
{
"key": "object",
"label": "什么在进化",
"status": "recorded",
"value": "联合实验更新 harness 与 Qwen 参数；单独 harness 实验只改指令／技能文本，执行循环、工具和上下文处理固定。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "executor",
"label": "谁执行",
"status": "recorded",
"value": "Qwen3.5-4B 执行科研任务；参数学习后使用对应更新的检查点。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "modifier",
"label": "谁来改",
"status": "recorded",
"value": "GPT-6 Astra 诊断并编辑 harness；Qwen3.8-27B 模拟受限用户反馈并解释反馈，训练程序执行 GRPO。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "verdict",
"label": "反馈是什么",
"status": "recorded",
"value": "harness：私有 verifier 给对错／提交状态，模拟用户只回复允许的确认或检查要求。参数：按固定 rubric 汇总可执行检查与固定 judge 分数，不用模拟用户的三值诊断分替代奖励。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "seed",
"label": "基础 harness",
"status": "recorded",
"value": "Biomni 科研工具／数据基础上的单文件 harness；已有 Python REPL、数据读取与答案提交，初始没有新增指令／技能。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "cycle",
"label": "更新规则",
"status": "recorded",
"value": "联合实验每轮先搜索 harness 10 步，再 RL 更新 20 步，共三轮；单独 harness 为 288 次对话、24 次更新。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "train",
"label": "训练／进化数据",
"status": "recorded",
"value": "LAB-Bench／Biomni-Eval1 的 LitQA2、DbQA、ProtocolQA、GWAS；任务库存 895 题不等于训练样本数。RL 使用当前模型新生成轨迹。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "debug",
"label": "调试／选版本数据",
"status": "recorded",
"value": "联合实验用固定 validation 选 harness；单独 harness 用 adaptation 选版本，其 validation 仅供事后对照。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "test",
"label": "测试数据与判分",
"status": "recorded",
"value": "四类科学任务上比较联合系统初末版本的单次正确率；固定 harness 的模型对照另用 pass@4。附录列库存，未逐项给出各实验 split 数量。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "isolation",
"label": "数据隔离与证据边界",
"status": "recorded",
"value": "论文声明开发、策略训练、最终测试分离；未完整披露逐任务判分代码及解释型 judge 型号，不能将整套判分笼统写成规则匹配。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "9 月 15 日首发，本期补录；HF 30 赞（2026-09-21），不算 9 月 16 日后新论文。",
"sources": [
{
"label": "Hugging Face：2026-09-21 热度快照",
"url": "https://huggingface.co/papers/2609.17523"
},
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "研究科研协作能否同时推动工作方式和模型学习：内层改 harness，外层在选定 harness 下做强化学习。",
"object": "联合实验更新 harness 与 Qwen 参数；单独 harness 实验只改指令／技能文本，执行循环、工具和上下文处理固定。",
"executor": "Qwen3.5-4B 执行科研任务；参数学习后使用对应更新的检查点。",
"modifier": "GPT-6 Astra 诊断并编辑 harness；Qwen3.8-27B 模拟受限用户反馈并解释反馈，训练程序执行 GRPO。",
"verdict": "harness：私有 verifier 给对错／提交状态，模拟用户只回复允许的确认或检查要求。参数：按固定 rubric 汇总可执行检查与固定 judge 分数，不用模拟用户的三值诊断分替代奖励。",
"seed": "Biomni 科研工具／数据基础上的单文件 harness；已有 Python REPL、数据读取与答案提交，初始没有新增指令／技能。",
"cycle": "联合实验每轮先搜索 harness 10 步，再 RL 更新 20 步，共三轮；单独 harness 为 288 次对话、24 次更新。",
"train": "LAB-Bench／Biomni-Eval1 的 LitQA2、DbQA、ProtocolQA、GWAS；任务库存 895 题不等于训练样本数。RL 使用当前模型新生成轨迹。",
"debug": "联合实验用固定 validation 选 harness；单独 harness 用 adaptation 选版本，其 validation 仅供事后对照。",
"test": "四类科学任务上比较联合系统初末版本的单次正确率；固定 harness 的模型对照另用 pass@4。附录列库存，未逐项给出各实验 split 数量。",
"isolation": "论文声明开发、策略训练、最终测试分离；未完整披露逐任务判分代码及解释型 judge 型号，不能将整套判分笼统写成规则匹配。",
"evidence": "9 月 15 日首发，本期补录；HF 30 赞（2026-09-21），不算 9 月 16 日后新论文。"
},
"attributions": [
{
"tag": "org:phai-labs",
"label": "PhAI Labs",
"kind": "institution",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
}
],
"researchProblems": [
"weights",
"feedback",
"curriculum"
]
},
{
"id": "2609.19203",
"title": "Position: It is Time to Virtualize Foundation Models with a Self-evolving Operating System Layer",
"date": "2026-09-16",
"dateLabel": "论文首发",
"year": "2026",
"url": "https://arxiv.org/abs/2609.19203",
"priority": "K",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "paper",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"RSIRoadmap",
"Infrastructure",
"Update20260921",
"org:hpe",
"org:uchicago",
"person:ian-foster"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
],
"fields": {
"本质定位": "提出 FMOS：像操作系统管理硬件一样，统一管理模型调用、记忆、资源分配和验证。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "不能从“self-evolving OS”的定位推断出统一的训练／测试数据集或成熟自主闭环。",
"visibleKeys": [
"scope",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "各 agent 框架重复实现记忆、预算与约束，行为难以迁移，跨系统治理也脆弱。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"key": "position",
"value": "提出 FMOS：像操作系统管理硬件一样，统一管理模型调用、记忆、资源分配和验证。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"key": "conclusion",
"value": "主张基于运行经验调整系统策略，让改进跨应用复用；是一篇架构立场论文，不是已跑通多代 RSI 的实证。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
}
],
"fields": {
"scope": "基础模型之上的可学习系统层：知识管理、模型选择、计算预算、验证与策略执行。",
"reading": "结合 harness 服务化阅读；区分作者希望实现的系统能力与已经展示的实验。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "基础模型之上的可学习系统层：知识管理、模型选择、计算预算、验证与策略执行。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "结合 harness 服务化阅读；区分作者希望实现的系统能力与已经展示的实验。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"key": "framework",
"label": "分析框架",
"status": "recorded",
"value": "用统一接口隔离应用与底层模型，联合调配知识、计算与验证资源。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "不能从“self-evolving OS”的定位推断出统一的训练／测试数据集或成熟自主闭环。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "提出 FMOS：像操作系统管理硬件一样，统一管理模型调用、记忆、资源分配和验证。",
"scope": "基础模型之上的可学习系统层：知识管理、模型选择、计算预算、验证与策略执行。",
"reading": "结合 harness 服务化阅读；区分作者希望实现的系统能力与已经展示的实验。",
"framework": "用统一接口隔离应用与底层模型，联合调配知识、计算与验证资源。",
"limits": "不能从“self-evolving OS”的定位推断出统一的训练／测试数据集或成熟自主闭环。"
},
"attributions": [
{
"tag": "org:hpe",
"label": "Hewlett Packard Enterprise",
"kind": "institution",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"tag": "org:uchicago",
"label": "University of Chicago",
"kind": "institution",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
{
"tag": "person:ian-foster",
"label": "Ian Foster",
"kind": "scholar",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "anthropic-when-ai-builds-itself",
"title": "When AI builds itself",
"date": "2026-09-18",
"dateLabel": "博客更新",
"year": "2026",
"url": "https://www.anthropic.com/institute/recursive-self-improvement",
"priority": "C",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"IndustryReport",
"ResearchAutomation",
"Update20260921",
"org:anthropic"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
],
"fields": {
"本质定位": "Anthropic 用研发记录、员工调查和会话评审介绍 AI 如何参与构建后续模型；本期收录 9 月 18 日更新。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "本条日期是页面明确标注的 9 月 18 日更新日，不是首次发布日期；公司自报与观察性结果不等于独立因果验证。",
"visibleKeys": [
"scope",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "代码与实验量增长未必意味着模型已经能独立决定研究方向，需要看内部研发中实际承担了哪些工作。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"key": "position",
"value": "Anthropic 用研发记录、员工调查和会话评审介绍 AI 如何参与构建后续模型；本期收录 9 月 18 日更新。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"key": "conclusion",
"value": "更新给出按任务难度分组的会话成功率；由 Claude judge 判断是否完成且无需纠正。文章明确尚未实现自主构建后继模型。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
}
],
"fields": {
"scope": "模型参与 AI 研发的产业证据；涵盖编码、实验执行与研究下一步选择。",
"reading": "先读内部证据与图注，再读限制：代码量不等于生产率，选择过的会话样本不能代表一般人机能力。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "模型参与 AI 研发的产业证据；涵盖编码、实验执行与研究下一步选择。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先读内部证据与图注，再读限制：代码量不等于生产率，选择过的会话样本不能代表一般人机能力。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"key": "feedback",
"label": "反馈与证据",
"status": "recorded",
"value": "会话成功由 Claude judge 判断；训练代码优化另有正确性检查和计时，不能把两种指标合成一个成功率。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "本条日期是页面明确标注的 9 月 18 日更新日，不是首次发布日期；公司自报与观察性结果不等于独立因果验证。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "Anthropic 用研发记录、员工调查和会话评审介绍 AI 如何参与构建后续模型；本期收录 9 月 18 日更新。",
"scope": "模型参与 AI 研发的产业证据；涵盖编码、实验执行与研究下一步选择。",
"reading": "先读内部证据与图注，再读限制：代码量不等于生产率，选择过的会话样本不能代表一般人机能力。",
"feedback": "会话成功由 Claude judge 判断；训练代码优化另有正确性检查和计时，不能把两种指标合成一个成功率。",
"limits": "本条日期是页面明确标注的 9 月 18 日更新日，不是首次发布日期；公司自报与观察性结果不等于独立因果验证。"
},
"attributions": [
{
"tag": "org:anthropic",
"label": "Anthropic",
"kind": "institution",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "ibm-rsi-serious-question",
"title": "Why recursive self-improvement suddenly became a serious question",
"date": "2026-09-16",
"dateLabel": "博客发布",
"year": "2026",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question",
"priority": "K",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"IndustryReport",
"RSIRoadmap",
"Update20260921",
"org:ibm"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
],
"fields": {
"本质定位": "IBM Think 采访研究者，对照 coding agents、AIDE² 与前沿实验室的公开说法。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "这是新闻采访和观点汇总，没有提出新的统一 benchmark 或自主训练方法。",
"visibleKeys": [
"scope",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "“AI 帮忙研发”与“每代自主制造更强后继”常被混为一谈，容易误读当前进展。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
{
"key": "position",
"value": "IBM Think 采访研究者，对照 coding agents、AIDE² 与前沿实验室的公开说法。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
{
"key": "conclusion",
"value": "强调已有局部改进循环，但完整自主递归仍缺验证；讨论错误奖励被放大、人类审核和计算成本的约束。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
}
],
"fields": {
"scope": "RSI 现状与证据边界的采访型入门文章，作者 Sascha Brodsky。",
"reading": "适合与 AIDE² 原始报告、Anthropic 内部证据对读，分清受访者判断和实验结果。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "RSI 现状与证据边界的采访型入门文章，作者 Sascha Brodsky。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "适合与 AIDE² 原始报告、Anthropic 内部证据对读，分清受访者判断和实验结果。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
{
"key": "evidence",
"label": "证据来源",
"status": "recorded",
"value": "IBM／Brown University／Weco 受访者的一手观点；论文与实验细节仍应回到各原始报告。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "这是新闻采访和观点汇总，没有提出新的统一 benchmark 或自主训练方法。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "IBM Think 采访研究者，对照 coding agents、AIDE² 与前沿实验室的公开说法。",
"scope": "RSI 现状与证据边界的采访型入门文章，作者 Sascha Brodsky。",
"reading": "适合与 AIDE² 原始报告、Anthropic 内部证据对读，分清受访者判断和实验结果。",
"evidence": "IBM／Brown University／Weco 受访者的一手观点；论文与实验细节仍应回到各原始报告。",
"limits": "这是新闻采访和观点汇总，没有提出新的统一 benchmark 或自主训练方法。"
},
"attributions": [
{
"tag": "org:ibm",
"label": "IBM",
"kind": "institution",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
}
],
"researchProblems": [
"perspectives"
]
},
{
"id": "sol-pi-author-blog",
"title": "SoL-Pi: Scaling Auto-Research Loops for Efficient Agent Harnesses",
"date": "2026-09-21",
"dateLabel": "收录日期（配套博客未标首发）",
"year": "2026",
"url": "https://nvlabs.github.io/SoL-Pi/",
"priority": "K",
"priorityBasis": "本期阅读推荐；依据主题相关性、作者来源和公开关注信号，不作为质量排名。",
"category": "overview",
"categories": [
"overview"
],
"methodType": "",
"contentType": "blog",
"curated": true,
"source": "addition",
"depth": [
"未明确"
],
"tags": [
"ResearchAutomation",
"Update20260921",
"org:nvidia"
],
"details": [],
"occurrences": [],
"review": "新增 · 原文定向核对",
"reviewed": true,
"links": [],
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
],
"fields": {
"本质定位": "SoL-Pi 作者的图解与工程说明，解释四种机制如何省成本及研究中人类实际做了什么。"
},
"protocol": "不适用：非统一实验",
"protocolBasis": "页面未明确给出首发日期，因此按 9 月 21 日收录日记录；与同名论文属于同一工作，不算第二项独立研究。",
"visibleKeys": [
"scope",
"reading"
],
"overview": {
"tldr": [
{
"key": "gap",
"value": "长时间运行的 agent 会反复付出上下文与工具输出成本，单看完成率看不出这种浪费。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
{
"key": "position",
"value": "SoL-Pi 作者的图解与工程说明，解释四种机制如何省成本及研究中人类实际做了什么。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
{
"key": "conclusion",
"value": "人类提供早期方向、筛选搜索空间并整理成功代码；用更省钱的 harness 改进下一代仍是未来计划，而非已经验证的复利。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
}
],
"fields": {
"scope": "论文的配套解释，重点是机制设计、人工参与和成本／得分取舍。",
"reading": "先看四种机制图解，再读 Results 和 Humans set priors；严格的数据隔离以论文 §2.5 为准。"
}
},
"profile": {
"fields": [
{
"key": "scope",
"label": "关注范围",
"status": "recorded",
"value": "论文的配套解释，重点是机制设计、人工参与和成本／得分取舍。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
{
"key": "reading",
"label": "怎么读",
"status": "recorded",
"value": "先看四种机制图解，再读 Results 和 Humans set priors；严格的数据隔离以论文 §2.5 为准。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
{
"key": "limits",
"label": "适用范围与局限",
"status": "recorded",
"value": "页面未明确给出首发日期，因此按 9 月 21 日收录日记录；与同名论文属于同一工作，不算第二项独立研究。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
}
],
"sourceDate": "2026-09-21",
"experiments": [],
"feedbackCases": [],
"missing": [],
"partial": []
},
"brief": {
"summary": "SoL-Pi 作者的图解与工程说明，解释四种机制如何省成本及研究中人类实际做了什么。",
"scope": "论文的配套解释，重点是机制设计、人工参与和成本／得分取舍。",
"reading": "先看四种机制图解，再读 Results 和 Humans set priors；严格的数据隔离以论文 §2.5 为准。",
"limits": "页面未明确给出首发日期，因此按 9 月 21 日收录日记录；与同名论文属于同一工作，不算第二项独立研究。"
},
"attributions": [
{
"tag": "org:nvidia",
"label": "NVIDIA",
"kind": "institution",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
}
],
"researchProblems": [
"harness"
]
}
],
"framework": "# RSI / Harness Self-Evolution 论文调研记录\n\n> **本文件为合并精简版。** 在原长期记录基础上合并系统补全文献；所有相关论文至少保留一行横向定位，只有 **Core / 经典代表作** 中最能定义主线的工作保留详细介绍。目标不是跟随论文自己的 “RSI / self-evolving” story，而是统一回答：**什么在变、谁来改、谁来执行、基础 harness、feedback 来源、evolution→eval 是否隔离、相对更早工作到底多开放了哪一层 editable space。**\n\n>\n> **主标签（代表性）**：**C = Core / 经典代表作**（领域绕不开、定义了范式、热度/影响力高，或直接改变后续论文如何做问题/评估）；**K = Key / 重要工作**（有清楚的新机制或关键证据）；**R = Related / 定位即可**（谱系相关，但通常表格一行即可）。\n>\n> **证据强度与“经典性”完全分开。** 一篇论文可以是 **Core 但证据有明显缺陷**（如 same-set adaptation / selection–eval leakage / attribution confounding），也可以是 **Key 但实验非常干净**。以后不再因为 protocol 不够干净就把领域代表作降级。\n>\n> **热度只作为辅助信号，不作为科学结论。** 对 2026 新论文，citation index 尚不稳定，因此综合看：后续工作是否频繁把它当 baseline/前驱、是否形成独立方法范式、官方开源/社区关注，以及我们原调研中是否已将其列为核心。\n\n## 0. 统一分析框架\n\n### 0.1 Meta-depth\n\n| 层级 | 定义 | 典型形态 |\n|---|---|---|\n| **M0：persistent state** | memory/skill/context 内容持续变化，但 updater 固定 | Reflexion、ExpeL、Voyager |\n| **M1：task-facing mechanism** | prompt/workflow/memory algorithm/tool/full harness 可编辑，但 modifier/search/selection 机制固定或外置 | ADAS、ALMA、SkillOpt、Meta-Harness、AHE |\n| **M2：improver also evolves** | 产生后续改进的 modifier/meta-skill/self-mod procedure 也被继承和修改；外层 evaluator/selection 常仍固定 | Promptbreeder、STOP、DGM、MetaSkill-Evolve |\n| **M3：full self-reference ideal** | 外层 improvement machinery 也处于同一可修改系统，并有准则决定何时重写 | Gödel Machine（理论） |\n\n### 0.2 Evaluation isolation\n\n| Protocol | 含义 | 我们如何解读 |\n|---|---|---|\n| **Held-out** | evolve/train/selection → freeze → unseen tasks | 最能支持可泛化 improvement |\n| **Prequential** | 每个 task 只做一次，feedback 只影响未来 task | 支持 online accumulation，但 stream 同时是 experience source |\n| **Same-set adaptive** | 同一 tasks 反复执行→修改→再评分 | 主要证明针对固定 tasks 可优化，不能直接当泛化 |\n| **Within-instance** | 只改当前 response/trajectory | feedback substrate，不是 persistent self-evolution |\n\n### 0.3 重新审计后的核心阅读层级\n\n> **这次以“领域代表性”为第一原则重新审核。** 我们原先已经重点记录的工作优先保留；新增论文只有在确实形成新范式/重要证据时才进入 Core，而不是因为更新、更干净就自动挤掉旧的代表作。\n\n#### Core-15：主干论文（优先完整读）\n\n| 时间 | 论文 | 为什么是主干 | 证据边界要记住 |\n|---|---|---|---|\n| 2003/2006 | **[Gödel Machines](https://arxiv.org/abs/cs/0309048)** | 严格 RSI 的理论原型：连 improver/proof-searcher 自身也可被重写。 | 理论构造，不是现代 agent 实证。 |\n| 2023-03 | **[Reflexion](https://arxiv.org/abs/2303.11366)** | `feedback→reflection→persistent experience→future behavior` 的经典 LLM 前史。 | updater 固定，更像 persistent memory 而非 harness mechanism evolution。 |\n| 2023-05 | **[Voyager](https://arxiv.org/abs/2305.16291)** | executable skill library 的经典起点。 | Minecraft/open-ended embodied setting；不是 full harness。 |\n| 2025-05 | **[Darwin Gödel Machine](https://arxiv.org/abs/2505.22954)** | self-modifying agent source + open-ended archive 的代表，是现代 RSI/harness program evolution 的直接前驱。 | outer archive / selection mechanism 仍固定。 |\n| 2025-11 | **[AgentEvolver](https://arxiv.org/abs/2511.10395)** | 2025–26 “self-evolving agent training loop” 的高影响代表：task generation、experience navigation、credit assignment 一体化。 | 主要更新 policy/data/credit loop，不是 deployment harness evolution。 |\n| 2026-03 | **[Hyperagents](https://arxiv.org/abs/2603.19461)** | 明确把 **modification procedure 本身**放入 editable program，是 improver-level RSI 的代表。 | evaluator/archive/outer selection 仍大体固定。 |\n| 2026-03 | **[Meta-Harness](https://arxiv.org/abs/2603.28052)** | full executable harness optimization 的主干 baseline：完整 source/history/raw trace 都进入 proposer context。 | TerminalBench headline 同集 search/eval；其他任务更干净。 |\n| 2026-04 | **[Agentic Harness Engineering (AHE)](https://arxiv.org/abs/2604.25850)** | full-harness evolution 工程化代表：observability、attribution、rollback、regression。 | TB2 主结果同集；cross-benchmark accuracy transfer 较小。 |\n| 2026-06 | **[Self-Harness](https://arxiv.org/abs/2606.09498)** | same-model self-harness 的最直接代表：target 根据自己的 failure 修改自己的 harness。 | held-out regression score 参与 acceptance，不是 untouched test。 |\n| 2026-06 | **[SEAGym](https://arxiv.org/abs/2606.17546)** | self-evolving harness 的 reliability/evaluation 代表：held-out ID/OOD、replay、snapshot、cost 分开评。 | 自己不提出新的 evolution 方法；价值在评测框架和负面证据。 |\n| 2026-07 | **[SEED](https://arxiv.org/abs/2607.14777)** | endogenous model self-improvement 代表：当前 policy 自己产生/分析 on-policy experience，再蒸馏回 weights。 | Stage-1 有外部 bootstrap；外层训练算法固定。 |\n| 2026-07 | **[Recursive Harness Self-Improvement (RHI)](https://arxiv.org/abs/2607.15524)** | harness revision-history / recursive revision 这条路线的代表。 | evaluator/optimizer role 分离，不是纯 same-model self-loop。 |\n| 2026-07 | **[RSIBench-Data](https://arxiv.org/abs/2607.25886)** | RSI benchmark/研究过程的经典代表；提出最重要的 **discovery–reliability gap** 证据之一。 | selection 与 final official eval 使用同一 task subset。 |\n| 2026-07 | **[Frontis-MA1](https://arxiv.org/abs/2607.28568)** | “训练一个专门的 meta-evolution / AI4AI improver” 的代表，且 OpenMLE 全栈开源。 | learned improver 变强，但 outer evolution system 仍固定。 |\n| 2026-08 | **[Harness-R1](https://arxiv.org/abs/2608.02276)** | 把 harness editing 从 prompt frontier LLM 变成 **可 post-train 的独立能力**；trained harness engineer 是清晰的新范式。 | target 与 editor 是分开的；cold start 依赖更强 teacher。 |\n\n#### Core-扩展：同样是领域代表作，按问题补读\n\n| 论文 | 为什么仍应视为 Core / 强代表 |\n|---|---|\n| **[Argus](https://arxiv.org/abs/2608.05144)** | persistent runtime / durable-state / verified objective pivot 的代表；强调 role separation 与 commit authority。 |\n| **[Macaron-V1](https://arxiv.org/abs/2608.09819)** | versioned **model–harness pair** / continual learning framing 的高热代表；系统设计很完整。 |\n| **[BigBang](https://endlessfrontier.tech/assets/paper.pdf)** | data-level RSI / self-evolving frontier-task synthesis 的系统级代表；不是 deployment harness，但 RSI 叙事里绕不开。 |\n| **[MetaSkill-Evolve](https://arxiv.org/abs/2607.05297)** | 明确 fast skill + slow meta-skill，两层 improvement procedure evolution；meta-depth 讨论的关键参照。 |\n| **[Evo-Harness](https://arxiv.org/abs/2608.15071)** | skill-level harness evolution + grounded feedback vs self-feedback 的直接证据，对我们当前问题非常相关。 |\n| **[Evo-Bench](https://arxiv.org/abs/2608.09096)** | 当前最接近“统一测试 harness-evolving capability”的 benchmark 骨架；Search 里含 BrowseComp/HLE。 |\n| **[Aspire](https://arxiv.org/abs/2608.31111)** | hidden downstream eval + vague goal，直接测试 self-evaluation/objective construction 是否可靠。 |\n| **[ADAS](https://arxiv.org/abs/2408.08435)** | executable agent program search 的重要前驱；related work 中解释 full-harness 从哪里来时必读。 |\n| **[HarnessDev](https://arxiv.org/abs/2609.01437)** | 2026-09-01 新出的直接 benchmark：把 **Creation 与 Evolution** 分开，hidden downstream eval 同时测 capability、efficiency 和 cross-runtime-model transfer；对我们当前实验设计极直接，但因过新暂不按“经典热度”评级。 |\n| **[GDPevo](https://arxiv.org/abs/2608.03764)** | evolution-native held-out business benchmark，并给 fully-informed **oracle ceiling**；对我们“是否接近/突破能力上限”问题非常直接。 |\n| **[VeRO](https://arxiv.org/abs/2602.22480)** | harness optimizer 的 outer instrumentation / audit substrate：versioning、budgeted rewards、structured observations，是 HarnessOpt-Bench 等统一评测路线的早期直接节点。 |\n\n> **一个关键修正**：Core 不再等于“实验最干净”。例如 RSIBench-Data / Self-Harness / Macaron-V1 的 protocol 都有清楚的限制，但它们依然是领域代表作；相反，某些很干净的新 benchmark 可以放在 Key，而不自动取代已有经典工作。\n\n### 0.4 热度 / 代表性辅助审计（2026-09-04）\n\n> 对 2026 新工作，引用次数仍严重滞后，所以这里**不把热度当排名**，只用公开社区信号检查“是不是把明显的热门代表作漏了”。\n\n| 论文 | 当前公开热度信号 | 我们怎么用这个信号 |\n|---|---|---|\n| **Macaron-V1** | Hugging Face Papers **342 upvotes，#2 Paper of the Day** | 明显属于当前 continual learning / model–harness RSI 的高热系统工作，必须保留 Core。 |\n| **SkillOpt** | Hugging Face Papers **265 upvotes，#1 Paper of the Day** | skill evolution 分支的高热代表；虽然不是 full harness，也值得保留详细介绍。 |\n| **Frontis-MA1** | Hugging Face Papers **186 upvotes** | AI4AI / learned meta-evolution 路线的显著代表；上一版漏掉是错误。 |\n| **SEED** | Hugging Face Papers **107 upvotes，#3 Paper of the Day** | model-level endogenous self-evolution 的高热代表，应该和 harness 路线并列保留作核心对照。 |\n| **Hyperagents** | Meta/Facebook Research 官方仓库约 **2.6k GitHub stars** | improver self-modification 并非边缘概念论文，而是目前很有传播度的 recursive/improver-level 代表。 |\n| **Meta-Harness** | Hugging Face Papers **23 upvotes**，Stanford/MIT/KRAFTON；已有多个 harness survey/data collection 引用其框架 | 热度不是最高，但在 full-harness 方法谱系中具有定义性，因此仍是 Core。 |\n| **Self-Harness** | Hugging Face Papers 当前仅 **3 upvotes** | 反例：社区 upvote 低不代表不重要；same-model self-harness setting 对我们的研究问题具有直接定义性，因此仍是 Core。 |\n\n**因此以后采用两个独立判断：**`代表性/热度` 决定要不要进入 Core 视野；`实验隔离/归因` 决定我们能相信它的 claim 到什么程度。两者不能再混成一个优先级。\n\n\n\n",
"conclusions": "## 4. 跨论文结论：现在真正需要记住什么\n\n| 问题 | 跨论文结论 |\n|---|---|\n| **历史上真正发生的变化** | `当前输出 self-correction → persistent memory/reflection → executable skill/tool → memory/skill mechanism → workflow/agent program → full harness → improver/meta-skill`。不要用论文是否自称 RSI 来判断 novelty。 |\n| **最重要的 distinction** | **state 在变 ≠ mechanism 在变 ≠ improver 在变。** Reflexion/Voyager 主要是 M0；ALMA/SkillOpt/Meta-Harness 是 M1；DGM/MetaSkill-Evolve 才触及 M2。 |\n| **Feedback** | 多条独立证据都指向：无 grounded verifier 时，reflection 容易把错误写进 persistent state。Evo-Harness 甚至直接观察到 pure self-feedback regression。 |\n| **Generalization** | 现在最普遍的证据缺口不是“能不能搜到过更高分”，而是 **freeze 后能否在 held-out tasks 保留**。AI4AI、ALMA、SkillOpt、Evo-Bench 的协议比 same-set headline 更值得学。 |\n| **基础 harness** | 从 minimal/raw interface 大涨，与从成熟 harness 继续提升，不是同一种证据。Meta-Harness/AHE 的价值之一正是让这个差异变得可见。 |\n| **Regression / reliability** | AHE、HarnessCompass、Evo-Bench、Aspire 都说明 improvement 非单调：会修复不代表会预见 collateral regression；local evaluator 变好也不代表 hidden capability 变好。 |\n| **对我们最有区分度的问题** | 仍然是 **capability ceiling + systematic behavior drift**：evolution 到底是在 eliciting latent capability、学到可泛化 procedure，还是 benchmark-specific overfit / cheat / judge gaming。 |\n\n## 5. 下一步阅读优先级\n\n如果目标是 **最快建立领域主干**，不要按 80 篇顺序读，先按下面四组：\n\n1. **历史/范式**：Gödel Machines → Reflexion → Voyager → DGM → AgentEvolver。\n2. **Harness 主线**：Meta-Harness → AHE → Self-Harness → RHI → Harness-R1。\n3. **更接近 RSI / improver**：Hyperagents → SEED → Frontis-MA1 → Macaron-V1 / BigBang。\n4. **怎么判断它到底 work 不 work**：SEAGym → RSIBench-Data → Evo-Bench → Aspire。\n\n对于我们现在要做的 **Harness 自进化 capability ceiling / 系统性行为偏移**，不建议再只看 6 篇，而是按问题分三组：\n\n1. **Harness 方法本体（先看它到底怎么改）**：`Meta-Harness + AHE + Self-Harness + Evo-Harness`。\n2. **最干净/最直接的 harness-evolution 评测骨架**：`VeRO → HarnessOpt-Bench → Evo-Bench → HarnessDev`。这条线能看出评测如何从 instrumentation 逐渐走到 held-out、harness-sensitive task、Creation-vs-Evolution、cross-runtime transfer。\n3. **我们真正想研究的 failure / ceiling**：`SEAGym + RSIBench-Data + GDPevo + PATH-Bench + Aspire`。其中 **GDPevo 的 oracle ceiling**、PATH-Bench 的 interference/forgetting、Aspire 的 hidden objective mismatch 都比“又一个更高 evolved score”更贴近我们的差异化问题。\n\n如果只允许再精读 **8 篇**，我会选：`Meta-Harness、AHE、Self-Harness、Evo-Bench、HarnessDev、SEAGym、RSIBench-Data、GDPevo`。PATH-Bench / Aspire 随后补。\n\n但写 related work / 讲领域版图时，必须把 **AgentEvolver、Hyperagents、SEED、Frontis-MA1、Harness-R1、Argus、Macaron-V1、BigBang** 放回来；否则会把 2026 RSI 的主流叙事读窄成“只改 deployment harness”。\n",
"readingGuide": "# 第一次读 RSI 论文：先分清研究对象与实验结果\n\n这个网站围绕一个问题整理文章：**系统从经历中改进了什么，这种改进是否真的帮助它完成后续任务？** RSI 是 Recursive Self-Improvement，通常译为递归自改进。本库也收录自纠错、记忆学习及相关评测；收录不代表论文已经实现了无限递归改进。\n\n## 模型、运行框架和产物是什么关系？\n\n以“修好一个软件错误”为例，下面只是帮助理解的示意，并非某篇论文的实验结果。\n\n| 对象 | 负责什么 | 修改它意味着什么 |\n| --- | --- | --- |\n| 模型 | 根据输入生成分析、答案或下一步动作，例如 Qwen、GPT、Claude 系列模型 | 训练参数，改变模型本身的行为 |\n| 运行框架（harness） | 决定给模型看什么、能调用什么工具、如何保存记忆、何时重试和停止 | 改模型外围的提示、工具、记忆或控制代码 |\n| 当前产物 | 模型为这道任务交付的代码补丁、答案或报告 | 把当前作品改好；未必改进了以后做题的系统 |\n\n**模型名与软件名要分开。** “Claude Code + 某模型”表示模型在一套编程代理工具中运行。Qwen-Agent 是组织模型调用和工具执行的框架。旧论文中的 Codex 也可能是模型名，不能看到同一个词就认定是同一种软件。\n\n## 怎么读每篇文章？\n\n先读三个 TL;DR 要点：**研究缺口与动机**说明作者认为现有做法有什么不足、会造成什么后果，以及为什么值得解决；**本文定位**说明要研究什么问题或检验什么能力；**关键设计与结论**说明独特做法和实际得到的结果。\n\n研究缺口依据原文引言，是作者提出工作的理由，不等于本站对整片领域的定论。理论文章和项目报告按原文记录提出问题的出发点；原文没有明确提出的缺陷，不额外补写。研究目标也不等于已经实现的能力。\n\n接着按五个维度读：\n\n1. **什么在进化：** 参数、提示、技能、记忆、运行代码，还是当前答案？下一道任务会继承什么？\n2. **谁执行：** 真正答题或操作环境的模型是什么？软件框架、机器人控制器是否承担了其他执行步骤？\n3. **谁来改：** 同一个模型、另一个更强模型，还是固定训练程序？“模型决定训练什么”与“平台实际运行训练”是两种角色。\n4. **反馈是什么：** 谁判断对错，依据测试、参考答案还是模型评审？修改者能看到总分、具体错误，还是只有自己的猜测？\n5. **基础框架是什么：** 起点已有工具、记忆和重试机制吗？如果一开始就有复杂系统，最后的成绩不能全部归功于进化。\n\n展开表格后，连起来读一次改进过程：**执行任务 → 获得反馈 → 定位问题 → 修改 → 检查并保留或撤回。** 有的论文不包含所有步骤，例如只追加经验而不检查回归；表格会按实际设置说明。\n\n## 反馈到底是谁给的？\n\n**“有 verifier”还没有说明怎样判分。** verifier 指负责验证的组件，它内部可能运行规则，也可能调用模型。“比对标注答案”同样要继续问：是程序匹配，还是模型判断含义相同？\n\n| 判分机制 | 实际做什么 | 阅读时要确认 |\n| --- | --- | --- |\n| 基准自带的可执行检查 | 运行测试，或检查数据库、文件、模拟器状态是否达到目标 | 用的是官方检查器还是论文改写的版本？修改者能否看到测试内容？ |\n| 规则比对参考答案 | 程序抽取答案，再做精确匹配、数值容差检查或计算词项重合分 | 比较字符串、选项、数值还是结果集合？怎样处理格式与容差？ |\n| 模型比对参考答案 | 把题目、标注答案和模型回答交给另一个模型判断语义是否等价 | 哪个模型作裁判？能否看到标准答案？ |\n| 模型按评分细则评审 | 按逐题检查项判断完整性、约束满足或质量；rubric 就是这些评分细则 | 谁写细则、检查哪些方面、怎样汇总分数？ |\n| 模型自评 | 模型看自身回答或轨迹提出成败判断、批评，没有标准答案保证 | 自评是否作为学习信号？正式测试是否另有独立判分？ |\n| 环境指标与数值测量 | 按游戏规则结算，或计算误差、训练损失、运行时间等 | 指标方向、成功阈值与资源限制是什么？ |\n\n**过程信息不等于判分。** 报错、截图、工具返回和执行日志能帮助定位错误，但本身不一定说明任务成功；通过所给代码测试也不等于覆盖了所有可能情况。\n\n每篇“反馈是什么”按实验列出判分机制；展开表格可继续查看数据、具体检查方式、反馈可见范围和用途。判分模型与写修改建议的模型是不同角色，不可互相替代。原文没有交代具体实现时会说明缺的是哪一项；不能根据基准名称或 `Judge` 函数名自行补出裁判。\n\n## 训练、验证、测试要按用途区分\n\n| 用途 | 主要问题 | 可以怎样使用成绩？ |\n| --- | --- | --- |\n| 训练／进化 | 用哪些经历提出修改或更新参数？ | 可以反复利用题目、反馈和允许访问的答案 |\n| 开发／验证 | 哪个候选版本值得保留？ | 可以用成绩调方案或选版本，因此这份成绩已参与决策 |\n| 独立最终测试 | 选定系统能否处理未参与修改和选择的任务？ | 冻结系统后评价；若再据此改版本，就需要重新说明隔离边界 |\n\n**数据集原本叫 test，不表示在这篇论文中仍是独立测试。** 作者可能把官方测试题拿来反复调框架；反过来，也可能把官方 validation 留作最后测验。网站保留原分区名称，并说明论文实际怎么使用。\n\n“留出”（held-out）只说明预先划出一部分数据。如果它的分数用于挑版本，它仍参与了选择。“在线学习”则允许前一道题的经历帮助后一道题，评价的是连续适应过程，不能照搬“全程冻结”的理解。\n\n## 怎么理解常见结果？\n\n**单次成功、重试成功和多版本覆盖不同。** pass@1 看一次尝试；pass@k 通常看最多 k 次中至少成功一次，具体以论文协议为准。如果每题事后挑最好的框架，得到的是候选集合的覆盖能力，不能当成一个固定框架能达到的成功率。\n\n**百分点与相对提升不同。** 成功率从 40% 到 50%，是提高 10 个百分点、相对提高 25%，不是同一个数字口径。\n\n**迁移说明不同事情。** 换新题检验任务泛化，换基础模型检验框架能否复用，换领域检验更远的适用性；它们不能互相替代。\n\n**论文未披露，不等于没有。** 若全文没有给出某组练习题、模型版本或裁判实现，网站会标明能确认到哪里，不用推测填成事实。\n\n**历史笔记中的 M0–M3 是原调研笔记的比较口径。** 分别指经验内容变化、任务执行机制变化、产生修改的机制也变化，以及完整自指的理论设想。这不是统一的论文分级标准，也不是分数越高能力越强；实际开放哪些修改，要以每篇的对象和更新规则为准。\n\n## 常见训练与模型术语\n\n- **冻结参数：** 不通过训练改变这部分模型权重，但外围提示或记忆仍可以变化。如果训练了额外适配参数，需要把它单独计入变化对象。\n- **LoRA：** 只训练少量附加适配参数的一种方法。它仍属于参数学习，不能因为主干权重不动就称整个系统完全不训练。\n- **监督微调：** 用示范输入和输出教模型产生目标行为。\n- **强化学习：** 用执行后的奖励调整行为；GRPO 等名称指具体训练算法。\n- **蒸馏：** 把教师模型或额外提示带来的行为指导，通过训练转进另一个模型的参数。\n- **检查点：** 保存下来的某个模型或系统版本。最终版本、最高分版本与按验证集选出的版本可能不同。\n- **轨迹：** 一次执行中的模型输出、动作、工具返回和结果记录。\n- **B、词元、推理强度：** 模型名中的 4B 通常表示约 40 亿参数；词元是模型处理文本的单位，不等于汉字数；high、xhigh 等是论文所用的推理强度设置，不是另一个执行角色。\n\n接着可以打开[研究脉络](?view=notes)，按问题对照几篇文章，再看展开表格中的数据与隔离。只有把这些条件对齐，成功率的差异才有可解释的含义。\n",
"researchMap": "# RSI 研究脉络：系统到底学会了什么？\n\n## 9 月 21 日：本期新增与补录\n\n本期新增 **4 篇论文、3 篇博客**，其中 4 条在 9 月 16–21 日首次发表、1 条为旧博客更新、1 条为此前论文补录、1 条为未标首发日期的配套博客。SoL-Pi 论文与博客是同一工作，不重复算研究成果。[只看这 7 条](?tag=Update20260921) · [完整统计与来源](data/update-2026-09-21.md)\n\n- **省成本的 harness 进化**：[SoL-Pi](?paper=2609.20519)及[作者博客](?paper=sol-pi-author-blog)。同时看成本、得分损失及选版本所用的数据。\n- **索引与记忆组件**：[SELF-INDEX](?paper=2609.19656)。不改模型参数，改的是文档／记忆用于检索的表示。\n- **框架与参数联合更新**：[ScienceBuddy](?paper=2609.17523)。9 月 15 日首发、本期补录，注意单独与联合实验的不同设置。\n- **系统层的研究定位**：[FMOS](?paper=2609.19203)。架构立场文，不当作已完成的自主进化系统。\n- **产业证据与分歧**：[Anthropic 9/18 更新](?paper=anthropic-when-ai-builds-itself)、[IBM 采访](?paper=ibm-rsi-serious-question)。区分内部观察、受访者判断和实证结果。\n\n\n从本库论文看，可以沿三个问题建立整体认识：**改进留下了什么？产生改进的能力有没有变化？怎样证明下一次真的更好？** 下文是本站依据论文方法与实验作的对照整理；顺序表示阅读线索，直接继承关系仍需论文证据。\n\n## 先建立时间与问题的坐标\n\n| 阅读阶段 | 代表工作 | 问题如何展开 |\n| --- | --- | --- |\n| 理论起点 | [自修改的形式保证](?paper=cs-0309048) | 如果连寻找改进的方法都允许修改，依据什么相信切换后更好？理论保证要结合公理和可证明性理解。（[原文 §3 Formal Description](https://arxiv.org/abs/cs/0309048)） |\n| 2023：改答案、留经验、积技能 | [Self-Refine](?paper=2303.17651)、[Reflexion](?paper=2303.11366)、[Voyager](?paper=2305.16291) | 从当前答案的多轮修订，走到留下文字教训、积累可执行技能。开始需要问：哪部分能力能被下一次任务继承？ |\n| 2024–2025：系统设计成为可搜索对象 | [ADAS](?paper=2408.08435)、[AFlow](?paper=2410.10762)、[DGM](?paper=2505.22954) | 提示之外，模块组合、执行步骤和实际代理代码进入搜索空间；不同工作对谁修改、修改范围和候选保留方式作出不同设计。 |\n| 2026：拆开诊断、修改能力与学习证据 | [Self-Harness](?paper=2606.09498)、[Harness-R1](?paper=2608.02276)、[Hyperagents](?paper=2603.19461)、[Aspire](?paper=2608.31111) | 同模型适配、训练专门修改者、修改改进程序、宽泛目标下自主选择学习方向，是几个不同问题，需要分别看实证。 |\n\n这几类研究至今可以并行开展。可修改范围更大，带来的是更多设计自由，也增加了诊断、选择和验证的难度。下面按问题展开，并在各处链接到方法与实验来源。\n\n## 路线一：从改好这道题，到留下后续能用的经验\n\n**阅读顺序：** [Self-Refine](?paper=2303.17651) → [Reflexion](?paper=2303.11366) → [ExpeL](?paper=2308.10144) → [Voyager](?paper=2305.16291)。\n\n- **Self-Refine：改当前产物。** 模型写初稿、提批评、再修订；你应该关注这一题有没有变好。（[原文 §3](https://arxiv.org/html/2303.17651#S3)；[原文 §4](https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1)）\n- **Reflexion：让重试读到教训。** 失败反思成为下一次可读的文字记忆，主要围绕同题的后续尝试。（[原文 §3](https://arxiv.org/html/2303.11366#S3)；[原文 §4.3](https://arxiv.org/html/2303.11366#S4.SS3)）\n- **ExpeL：把旧题经验带到新题。** 训练任务的试错被整理为一般规则和成功示例，供新任务单次执行使用。（[原文 §4.4](https://arxiv.org/html/2308.10144#S4.SS4)；[原文 §5.1](https://arxiv.org/html/2308.10144#S5.SS1)）\n- **Voyager：把经验变成能执行的技能。** Minecraft 中的成功行为被写成代码并入库，后续可以检索、组合使用。（[原文 §2](https://arxiv.org/html/2305.16291#S2)；[原文 附录A.1](https://arxiv.org/html/2305.16291#A1.SS1)）\n\n**沿这条线看什么：** 经验服务同题重试，还是后续不同任务？保存的是原始记录、一般规则，还是可执行代码？这决定了“学习”的具体含义。\n\n## 路线二：从增加记忆内容，到改变怎样管理记忆\n\n**阅读顺序：** [ACE](?paper=2510.04618) → [Memento-Skills](?paper=2603.18743) → [MemEvolve](?paper=2512.18746) / [ALMA](?paper=2602.07755)。\n\n**ACE** 维护逐条可增改的经验手册，重点是长期内容如何避免在反复重写中丢失细节。**Memento-Skills** 将技能作为执行期间可修订的能力资源，失败后定位并修复技能，还要解决该选哪个技能的问题。（[原文 §2.1](https://arxiv.org/html/2510.04618#S2.SS1)；[原文 §4.1](https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1)） （[原文 §2.1](https://arxiv.org/html/2603.18743#S2.SS1)；[原文 §2.3](https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2)）\n\n**MemEvolve、ALMA** 把问题推进到记忆管理程序：怎样编码、存储、检索或使用经验也可以被搜索。这里应分别标注“记了什么”和“怎样记、怎样取”，因为前者增加条目，并不自动改变后者的机制。（[原文 §3.3](https://arxiv.org/html/2512.18746#S3.SS3)；[原文 §5.2](https://arxiv.org/html/2512.18746#S5.SS2)） （[原文 §3.2](https://arxiv.org/html/2602.07755#S3.SS2)；[原文 §3.3](https://arxiv.org/html/2602.07755#S3.SS3)）\n\n**关键对照：** [ContinualSkillBench](?paper=2608.03874) 直接比较显式技能学习和保留历史上下文。如果简单保留历史也能达到相近效果，就需要进一步证明提炼技能本身的额外作用。（[原文 §3.5](https://arxiv.org/html/2608.03874#S3.SS5)；[原文 附录D](https://arxiv.org/html/2608.03874#A4)）\n\n## 路线三：从设计执行流程，到修改真实运行框架\n\n**阅读顺序：** [ADAS](?paper=2408.08435) / [AFlow](?paper=2410.10762) → [SICA](?paper=2504.15228) / [DGM](?paper=2505.22954) → [Meta-Harness](?paper=2603.28052) / [Self-Harness](?paper=2606.09498)。\n\n**ADAS** 用代码表达代理设计，**AFlow** 搜索节点和连接组成的工作流程；读它们时要先确定预设了哪些模块、允许改变哪些组合。（[原文 §3](https://arxiv.org/html/2408.08435#S3)；[原文 附录B](https://arxiv.org/html/2408.08435#A2)） （[原文 §4](https://arxiv.org/html/2410.10762#S4)；[原文 附录A.6](https://arxiv.org/html/2410.10762#A1.SS6)）\n\n**SICA** 让编程代理改自身实际使用的工具和实现；**DGM** 还保留多个历史分支，允许从暂时不领先的版本继续生长。这里的问题从“这一次怎样解题”扩展为“以后用哪套执行程序解题”。（[原文 §3](https://arxiv.org/html/2504.15228#S3)） （[原文 §3](https://arxiv.org/html/2505.22954#S3)；[原文 附录C.3](https://arxiv.org/html/2505.22954#A3.SS3)）\n\n**Meta-Harness** 强調修改者可以检索完整历史代码、分数和轨迹，以重新诊断问题。**Self-Harness** 强调由执行任务的同一种模型识别自己的失败习惯，再改自己的框架；其检查分数参与接受修改，因此要进一步看哪些数据仍用于选版本。（[原文 §3](https://arxiv.org/html/2603.28052#S3)） （[原文 §3.1](https://arxiv.org/html/2606.09498#S3.SS1)；[原文 §4.2](https://arxiv.org/html/2606.09498#S4.SS2)）\n\n**这条线中的工程分工：** [AHE](?paper=2604.25850) 追踪组件改动及其相互影响；[HarnessFix](?paper=2606.06324) 把失败定位接到限定范围的修复；[HarnessLens](?paper=2608.27311) 研究怎样选择更相关的验证题、减少验收成本。它们分别处理改动的可解释性、修复范围和验证预算。（[原文 §3](https://arxiv.org/html/2604.25850#S3)；[原文 §3.3](https://arxiv.org/html/2604.25850#S3.SS3)） （[原文 §3](https://arxiv.org/html/2606.06324#S3)；[原文 §6](https://arxiv.org/html/2606.06324#S6)） （[原文 附录A.1](https://arxiv.org/html/2608.27311#A1.SS1)；[原文 §5.1](https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2)）\n\n**近期补充：修改范围和经验结构也值得单独研究。** [RobustSGPO](?paper=2609.09646) 先决定改哪些代理、做哪类操作，再生成补丁并保留不同类别的历史起点；它问的是“开放了修改权限后，怎样让有效结构修改真正发生”。[Procedural Graphs](?paper=2609.09153) 则把经验组织成带条件的步骤图，让模型每一步读取当前位置附近的指导。前者控制搜索过程，后者设计可学习、可使用的经验结构，两者不是同一种贡献。（[RobustSGPO §4–5](https://arxiv.org/html/2609.09646#S4)；[Procedural Graphs §3](https://arxiv.org/html/2609.09153#S3)）\n\n## 路线四：让系统更会提出改进\n\n**阅读顺序：** [Promptbreeder](?paper=2309.16797) → [MetaSkill-Evolve](?paper=2607.05297) / [Hyperagents](?paper=2603.19461)，再对照 [Harness-R1](?paper=2608.02276) / [MetaEvolve](?paper=2607.21971)。\n\n这里有两种需要分开理解的做法。\n\n**把改进规则也设为可变对象。** Promptbreeder 改指导提示变异的指令；MetaSkill-Evolve 改指导技能修订的文字说明；Hyperagents 把改进程序本身与任务代理放进可修改系统，并检查改进能力能否用于新领域。文字规则、程序和外层选择流程，要分别说明哪些真的变了。（[原文 §3.1](https://arxiv.org/html/2309.16797#S3.SS1)；[原文 附录F](https://arxiv.org/html/2309.16797#A6)） （[原文 §3.4](https://arxiv.org/html/2607.05297#S3.SS4)；[原文 §3.5](https://arxiv.org/html/2607.05297#S3.SS5)） （[原文 §5.1](https://arxiv.org/html/2603.19461#S5.SS1)；[原文 §5.3](https://arxiv.org/html/2603.19461#S5.SS3)）\n\n**专门训练提出改进的模型。** Harness-R1 训练独立编辑模型，让它读失败记录、写框架补丁，再用补丁装上后的任务表现学习。MetaEvolve 训练利用反馈与历史提出更好改进的能力。这说明修改能力可以被训练；还要另查部署时它是否继续修改自身参数或改进规则。（[原文 §3.1](https://arxiv.org/html/2608.02276#S3.SS1)；[原文 §4.1](https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4)） （[原文 §2.1](https://arxiv.org/html/2607.21971#S2.SS1)；[原文 附录A](https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1)）\n\n**先排除一个混淆：** [Harness Updating Is Not Harness Benefit](?paper=2605.30621) 将执行者和修改者独立配对。在技能、提示和记忆的受限编辑实验中，强模型未必写出更有帮助的更新；弱执行者也可能因为不加载或不遵循技能而得不到收益。它解释的是“会修改”与“能受益”的分工，不是在证明小模型能同样胜任任意源码自修改。（[原文 §4.2–4.3](https://arxiv.org/html/2605.30621#S4.SS2)；[附录B.4：编辑范围](https://arxiv.org/html/2605.30621#A2.SS4)）\n\n**判断递归改进的核心问题：** 新版本只是更会做任务，还是也更会产生下一次改进？如果论文单独测了后者，证据来自什么新任务、什么对照？\n\n## 路线五：经验留在框架里，还是训练进模型？\n\n**阅读顺序：** [SEAL](?paper=2506.10943) → [SIA](?paper=2605.27276)，再对照 [HELIX](?paper=2608.13951) / [Macaron-V1](?paper=2608.09819)。\n\n**SEAL** 学习怎样把新输入整理成有效的参数更新材料。**SIA** 在同一流程中开放两种操作：改外围框架，或训练模型；关键是根据失败决定改哪里，以及两者有无互补收益。（[原文 §3](https://arxiv.org/html/2506.10943#S3)；[原文 §3.1](https://arxiv.org/html/2506.10943#S3.SS1)） （[原文 §4.3](https://arxiv.org/html/2605.27276#S4.SS3)；[原文 §6.2](https://arxiv.org/html/2605.27276#S6.SS2)）\n\n**HELIX** 强调执行框架还应产出可追踪、适合后续模型训练的轨迹；当前论文主要验证框架与数据接口。**Macaron-V1** 的系统架构连接专长适配参数和运行配置，但其直接框架搜索实验证据来自冻结模型、多个配置覆盖失败题。把“支持共同更新”“完成了一轮训练”“多代互相推动”分开看，才能准确理解论文进展。（[原文 §5.1](https://arxiv.org/html/2608.13951#S5.SS1)） （[原文 §3.1](https://arxiv.org/html/2608.09819#S3.SS1.SSS2)；[原文 §4](https://arxiv.org/html/2608.09819#S4)）\n\n**联合进化的三个具体设计可以并读：**\n\n| 论文 | 框架与参数怎样配合 | 读结果时保留什么条件 |\n| --- | --- | --- |\n| [HarnessForge](?paper=2606.01779) | 先按故障修框架，每个保留框架再训练自己的模型适配参数，形成独立分支。 | 3,800道适配题与测试分开；TMDB/API-Bank只做迁移测试。框架筛选还看词元与耗时。（[原文附录B–D](https://arxiv.org/html/2606.01779#A2)） |\n| [Co-Harness](?paper=2607.22688) | 修复生成训练轨迹的系统，再把新框架下成功行为训练进模型，下一轮继续发现故障。 | 虽提出两组行为验收，但训练、验收与最终竞赛题的完整隔离未披露；不能直接把涨分解释成泛化。（[原文 §3.4–4.1](https://arxiv.org/html/2607.22688#S3.SS4)；[附录E](https://arxiv.org/html/2607.22688#A5)） |\n| [WHALE](?paper=2609.00196) | 固定模型搜框架、固定框架练模型，并专门研究何时切换；训练信号停滞可触发换阶段。 | 阶段内选择用训练信号，但论文主要比较各运行最好测试点；与事先选定版本后独立测一次不同。（[原文 §4](https://arxiv.org/html/2609.00196#S4)；[§6.2](https://arxiv.org/html/2609.00196#S6.SS2)） |\n\n这些工作使问题更具体：**新框架能否产生值得学的行为？模型学会后，什么框架才更适合它？两边每次应改多少？** 不能只看到“同时更新两部分”就视为同一方法。\n\n## 路线六：评测从有没有涨分，走向为什么涨分\n\n| 想区分什么能力 | 可以对照读的文章 | 评测中专门改变或检查了什么 |\n| --- | --- | --- |\n| 使用现成代理，与建设有效代理系统 | [HarnessDev](?paper=2609.01437)、[Evo-Bench](?paper=2608.09096) | 前者拆开从弱起点建设与继续改进；后者控制执行者及预算比较修改模型。（[原文 §3.2](https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1)；[原文 附录E.2](https://arxiv.org/html/2609.01437#A5.SS2)） （[原文 §3](https://arxiv.org/html/2608.09096#S3)；[原文 附录E.1](https://arxiv.org/html/2608.09096#A5.SS1)） |\n| 连续变好，与中途达到峰值后退步 | [SEAGym](?paper=2606.17546)、[RSIBench-Data](?paper=2607.25886) | 前者同时跟踪泛化、遗忘和成本；后者观察数据研究找到的好方案能否最终保住。（[原文 附录A.1](https://arxiv.org/html/2606.17546#A1.SS1)） （[原文 §3.1](https://arxiv.org/html/2607.25886#S3.SS1)；[原文 §4.2](https://arxiv.org/html/2607.25886#S4.SS2)） |\n| 会尝试、会判断，与能把经历用于后续提升 | [S³Gym](?paper=2608.31100)、[Aspire](?paper=2608.31111) | 前者拆测探索、自评与经验利用；后者让模型面对宽泛目标自行确定学习方向，最终检查是否练对了能力。（[原文 §4.4](https://arxiv.org/html/2608.31100#S4.SS4)；[原文 §5.2](https://arxiv.org/html/2608.31100#S5.SS2.SSS0.Px1)） （[原文 §4.3](https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2)；[原文 §4.3](https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px6)） |\n| 存了技能，与技能确实带来额外收益 | [ContinualSkillBench](?paper=2608.03874)、[PAST-Bench](?paper=2608.04003) | 用保留历史上下文或关闭经验积累的对照，并检查经验实际有没有被调用。（[原文 §3.5](https://arxiv.org/html/2608.03874#S3.SS5)；[原文 附录D](https://arxiv.org/html/2608.03874#A4)） （[原文 §3.1](https://arxiv.org/html/2608.04003#S3.SS1)；[原文 附录A.3](https://arxiv.org/html/2608.04003#A1.SS3)） |\n| 工程调出高分，与发现可泛化的方法 | [PostTrainBench](?paper=2603.08640)、[MLS-Bench](?paper=2605.08678) | 前者看给定模型与算力后能否完成有效后训练；后者进一步要求方法跨设置、扩大规模后仍成立。（[原文 §1](https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2)；[原文 附录E](https://arxiv.org/html/2603.08640#A5)） （[原文 §3.2](https://arxiv.org/html/2605.08678#S3.SS2.SSS1)；[原文 §3.2](https://arxiv.org/html/2605.08678#S3.SS2.SSS1.Px1)） |\n\n**补充对照：[RSI-Exam](?paper=rsi-exam)** 让研究者从可运行弱方法出发，在可见材料上反复实验，再评价最终交付物。它把方法改进与固定模型的框架改进放进同一任务库，并保留版本轨迹；部分任务另训练学生模型或提交回答，信息边界要按任务解释。与 Aspire 的宽泛目标设置相比，它提供具体任务目标和评分规则，更集中地测“已知研究目标后，能否做出有效改进”。（[RSI-Exam 研究目标与任务构造](https://rsi-exam.ai/blog.html#benchmark)；[数学训练任务](https://rsi-exam.ai/tasks/teacher_student_math_posttraining.html)；[法律任务](https://rsi-exam.ai/tasks/legal_matter_caseload_regulatory.html)；[Aspire §4](https://arxiv.org/html/2608.31111#S4)）\n\n**从涨分到可信验收：** [Safe Harness Self-Evolution](?paper=2609.08175) 将候选确实改善、能否生成它、有限数据能否确认它，以及更新后还能否继续进步分别研究。其 DS-1000 与 WorkBuddy 诊断显示：审计点估计认为一些修改有益，验收阶段仍可能因为区间过宽而全部拒绝。因此，停止更新可能是评估预算问题；一次成功也不自动说明“产生下次改进的能力”增强。这里的理论保证依赖固定任务分布与抽样条件，“安全”主要指可靠改善并控制旧任务退化。（[原文 §4–6，p7–17](https://arxiv.org/pdf/2609.08175#page=7)）\n\n## 名字相近，但关键区别在哪里？\n\n| 对照论文 | 最值得记住的差别 | 展开实验表时继续看什么 |\n| --- | --- | --- |\n| [Evo-Harness](?paper=2608.15071) / [EvoHarness-RL](?paper=2608.05446) | 前者从轨迹写后续可读的文字技能；后者训练模型何时读写外部状态。（[原文 §4.5](https://arxiv.org/html/2608.15071#S4.SS5)；[原文 §3.1–3.4：说明的选取、注入与修改](https://arxiv.org/html/2608.15071#S3.SS1)） （[原文 附录C](https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5)；[原文 附录D.2](https://arxiv.org/html/2608.05446#A4.SS2)） | 改变的是文字内容、运行代码，还是模型参数？ |\n| [MetaSkill-Evolve](?paper=2607.05297) / [MetaEvolve](?paper=2607.21971) | 前者更新指导技能修订的外部说明；后者把提出改进的能力训练进模型参数。（[原文 §3.4](https://arxiv.org/html/2607.05297#S3.SS4)；[原文 §3.5](https://arxiv.org/html/2607.05297#S3.SS5)） （[原文 §2.1](https://arxiv.org/html/2607.21971#S2.SS1)；[原文 附录A](https://arxiv.org/html/2607.21971#A1.SS0.SSS0.Px1)） | 改进机制哪些部分仍固定？训练与部署阶段分别改什么？ |\n| [Self-Harness](?paper=2606.09498) / [Harness-R1](?paper=2608.02276) | 前者由同一种模型为自己适配框架；后者训练独立编辑者帮助冻结的执行模型。（[原文 §3.1](https://arxiv.org/html/2606.09498#S3.SS1)；[原文 §4.2](https://arxiv.org/html/2606.09498#S4.SS2)） （[原文 §3.1](https://arxiv.org/html/2608.02276#S3.SS1)；[原文 §4.1](https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4)） | 修改者能看到哪些失败信息？检查题是否也参与了接受修改？ |\n| [ASPIRE（机器人）](?paper=2607.00272) / [Aspire（宽泛目标评测）](?paper=2608.31111) | 前者从机器人失败中发现修复技能；后者测试模型能否自主确定合理学习方向。（[原文 §2](https://arxiv.org/html/2607.00272#S2)；[原文 附录E.4](https://arxiv.org/html/2607.00272#A5.SS4)） （[原文 §4.3](https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2)；[原文 §4.3](https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px6)） | 任务领域、反馈接口与研究目标完全不同，要用标题和日期区分。 |\n| [Harness-of-Harness](?paper=2609.01481) / [Meta-Harness](?paper=2603.28052) | 前者组织长期开发，持续改进软件产物；后者搜索用于执行任务的框架程序。（[原文 附录A.2](https://arxiv.org/html/2609.01481#A1.SS2)；[原文 §4.2](https://arxiv.org/html/2609.01481#S4.SS2.SSS0.Px2)） （[原文 §3](https://arxiv.org/html/2603.28052#S3)） | 最终交付的是更好的项目，还是下一道任务会使用的更好系统？ |\n\n## 用哪些证据判断这个领域走到了哪里？\n\n根据上面几条路线，本站建议把证据拆成三个层面阅读：\n\n1. **局部改进是否成立。** 新版本在论文指定任务和预算下比起点更好；先确认提升来自什么改动，额外模型、重试和参考答案有没有变化。\n2. **可复用改进是否成立。** 冻结版本后，换题、换领域或换模型仍有效；分别看这些迁移设置，不能用一种替代另一种。\n3. **产生改进的能力是否也增强。** 改进机制本身发生可描述的变化，并能在后续问题上更有效地提出改进；最好有独立测量和固定改进者对照。\n\n这些是本文库的比较问题，而非所有论文共用的等级标准。看见一条上升曲线后，继续问：**哪些题参与了修改和选版本？做错的细节从哪里来？最终继承了什么？** 对齐这些条件，才能判断论文解决的是哪一块问题，以及仍留下什么。\n\n接着可以[回到全部论文](?view=library)，或筛选[运行框架进化](?category=methods&quick=harness)、[模型参数进化](?category=methods&quick=weights)、[联合进化](?category=methods&quick=joint)、[评测研究](?category=evaluation)。需要先理解名词时，查看[入门说明](?view=guide)。\n\n\n## 从近期工作连接几条路线\n\n- **即时纠偏与经验积累**：[PILOT](?paper=2608.26530) 将监督与执行分开，运行中写经验，运行后用 verifier 决定保留。它的重复同题实验要与跨题泛化区分。\n- **改进搜索方式本身**：[Dream-RSI](?paper=2609.14858) 用历史发现树试验新的探索策略；[MetaRSI-v1](?paper=metarsi-v1) 则组合数据、harness、参数三类操作。两者改进的对象和证据不同。\n- **反馈进入参数训练**：[DiagEvo](?paper=2609.00768) 从错误诊断生成课程；[NeoHorse-1](?paper=2609.08183) 利用路由交互组织训练；[iCoder](?paper=icoder-27b) 让研发 agent 调整工业模型的完整训练过程。\n- **先检查评测是否公平**：[Rethinking](?paper=2607.12227) 比较相同尝试预算，并用 45/10/34 划分检验 harness 泛化。成绩增加不自动意味着学到了可复用框架。\n- **理解全景与落地**：[自主程度路线图](?paper=2609.11873)、[数据中心综述](?paper=data-centric-rsi-survey)、[AI4AI 综述](?paper=ai4ai-survey) 提供不同分析视角；[Reef](?paper=reef) 提供工程组件，[OpenAI 内部观察](?paper=openai-research-acceleration) 提供产业侧材料。\n",
"attributionCatalog": {
"org:tencent-hunyuan": {
"label": "Tencent Hunyuan",
"kind": "institution"
},
"org:tencent": {
"label": "Tencent",
"kind": "institution"
},
"org:bytedance-seed": {
"label": "ByteDance Seed",
"kind": "institution"
},
"org:bytedance": {
"label": "ByteDance",
"kind": "institution"
},
"org:openai": {
"label": "OpenAI",
"kind": "institution"
},
"org:anthropic": {
"label": "Anthropic",
"kind": "institution"
},
"org:thinking-machines-lab": {
"label": "Thinking Machines Lab",
"kind": "institution"
},
"org:meta-superintelligence": {
"label": "Meta Superintelligence Labs",
"kind": "institution"
},
"org:safe-superintelligence": {
"label": "Safe Superintelligence",
"kind": "institution"
},
"org:meta-fair": {
"label": "Meta FAIR",
"kind": "institution"
},
"org:google-brain": {
"label": "Google Brain",
"kind": "institution"
},
"org:google-deepmind": {
"label": "Google DeepMind",
"kind": "institution"
},
"org:google-cloud-ai": {
"label": "Google Cloud AI",
"kind": "institution"
},
"org:google-research": {
"label": "Google Research",
"kind": "institution"
},
"org:google": {
"label": "Google",
"kind": "institution"
},
"org:washington": {
"label": "University of Washington",
"kind": "institution"
},
"org:princeton": {
"label": "Princeton University",
"kind": "institution"
},
"org:stanford": {
"label": "Stanford University",
"kind": "institution"
},
"org:mit": {
"label": "MIT",
"kind": "institution"
},
"org:cmu": {
"label": "Carnegie Mellon University",
"kind": "institution"
},
"org:berkeley": {
"label": "UC Berkeley",
"kind": "institution"
},
"org:ubc": {
"label": "University of British Columbia",
"kind": "institution"
},
"org:uiuc": {
"label": "UIUC",
"kind": "institution"
},
"org:tsinghua": {
"label": "Tsinghua University",
"kind": "institution"
},
"org:peking": {
"label": "Peking University",
"kind": "institution"
},
"org:sjtu": {
"label": "Shanghai Jiao Tong University",
"kind": "institution"
},
"org:nus": {
"label": "National University of Singapore",
"kind": "institution"
},
"org:ntu": {
"label": "Nanyang Technological University",
"kind": "institution"
},
"org:eth": {
"label": "ETH Zürich",
"kind": "institution"
},
"org:oxford": {
"label": "University of Oxford",
"kind": "institution"
},
"org:cambridge": {
"label": "University of Cambridge",
"kind": "institution"
},
"org:sakana": {
"label": "Sakana AI",
"kind": "institution"
},
"org:shanghai-ai-lab": {
"label": "Shanghai AI Laboratory",
"kind": "institution"
},
"org:allenai": {
"label": "Allen Institute for AI",
"kind": "institution"
},
"org:krafton": {
"label": "KRAFTON",
"kind": "institution"
},
"person:mengdi-wang": {
"label": "Mengdi Wang",
"kind": "scholar"
},
"person:diyi-yang": {
"label": "Diyi Yang",
"kind": "scholar"
},
"person:weijia-shi": {
"label": "Weijia Shi",
"kind": "scholar"
},
"person:jeff-clune": {
"label": "Jeff Clune",
"kind": "scholar"
},
"person:chelsea-finn": {
"label": "Chelsea Finn",
"kind": "scholar"
},
"person:sergey-levine": {
"label": "Sergey Levine",
"kind": "scholar"
},
"person:percy-liang": {
"label": "Percy Liang",
"kind": "scholar"
},
"person:pieter-abbeel": {
"label": "Pieter Abbeel",
"kind": "scholar"
},
"person:danqi-chen": {
"label": "Danqi Chen",
"kind": "scholar"
},
"person:yejin-choi": {
"label": "Yejin Choi",
"kind": "scholar"
},
"person:luke-zettlemoyer": {
"label": "Luke Zettlemoyer",
"kind": "scholar"
},
"person:graham-neubig": {
"label": "Graham Neubig",
"kind": "scholar"
},
"person:hannaneh-hajishirzi": {
"label": "Hannaneh Hajishirzi",
"kind": "scholar"
},
"person:shunyu-yao": {
"label": "Shunyu Yao",
"kind": "scholar"
},
"person:jun-wang": {
"label": "Jun Wang",
"kind": "scholar"
},
"person:omar-khattab": {
"label": "Omar Khattab",
"kind": "scholar"
},
"person:denny-zhou": {
"label": "Denny Zhou",
"kind": "scholar"
},
"person:quoc-le": {
"label": "Quoc V. Le",
"kind": "scholar"
},
"person:heng-ji": {
"label": "Heng Ji",
"kind": "scholar"
},
"person:qingyun-wu": {
"label": "Qingyun Wu",
"kind": "scholar"
},
"person:yulia-tsvetkov": {
"label": "Yulia Tsvetkov",
"kind": "scholar"
},
"org:meituan-longcat": {
"label": "Meituan LongCat",
"kind": "institution"
},
"person:shilong-liu": {
"label": "Shilong Liu",
"kind": "scholar"
},
"person:lilian-weng": {
"label": "Lilian Weng",
"kind": "scholar"
},
"org:weco": {
"label": "Weco",
"kind": "institution"
},
"org:minimax": {
"label": "MiniMax",
"kind": "institution"
},
"org:nvidia": {
"label": "NVIDIA",
"kind": "institution"
},
"person:song-han": {
"label": "Song Han",
"kind": "scholar"
},
"org:phai-labs": {
"label": "PhAI Labs",
"kind": "institution"
},
"org:hpe": {
"label": "Hewlett Packard Enterprise",
"kind": "institution"
},
"org:uchicago": {
"label": "University of Chicago",
"kind": "institution"
},
"person:ian-foster": {
"label": "Ian Foster",
"kind": "scholar"
},
"org:ibm": {
"label": "IBM",
"kind": "institution"
}
},
"attributionPolicy": "机构按论文发表时的作者署名标注；学者标签表示作者署名，不表示主导。引用论文或使用模型不作为机构归属依据。",
"contentTypes": {
"paper": "论文",
"report": "技术报告",
"survey": "综述",
"repository": "仓库",
"blog": "博客"
},
"libraryAdditions": 31,
"problemMap": {
"version": 1,
"updated": "2026-09-30",
"corpusUpdated": "2026-09-21",
"policy": "按论文要解决的问题人工归组；同一工作可跨路线。关系箭头是本站的分析，不代表作者声明的继承关系或历史先后。覆盖当前资料库，不声称穷尽整个领域。",
"groups": [
{
"id": "feedback",
"title": "怎样知道哪里错、该怎么改？",
"short": "获得有效反馈",
"problem": "只有“失败了”不足以指导修改；模型自己写的批评也可能是错的。反馈既要有判断依据，也要能定位需要改变的行为。",
"progress": "从同模型检查初稿，分出两条并行路线：借外部工具核验；把粗分数转成步骤、示范或诊断信息。",
"limit": "反馈更详细不保证更正确。要分别记录事实来源、判分方式、修改者可见内容，以及这些信息在部署时是否可得。",
"branches": [
{
"id": "self-check",
"title": "先修好当前答案",
"question": "同一个模型能否发现自己的错误？外部核验能增加什么信息？",
"papers": [
"2203.11171",
"2303.17651",
"2305.11738",
"2309.11495",
"2310.01798",
"2310.04406"
],
"representatives": [
"2303.17651",
"2305.11738",
"2310.01798"
]
},
{
"id": "diagnosis",
"title": "把成败变成可行动的诊断",
"question": "最终分数没有告诉系统哪一步该改，怎样补足中间信息？",
"papers": [
"2305.20050",
"2406.07496",
"2507.19457",
"2605.24539",
"2609.00829"
],
"representatives": [
"2305.20050",
"2507.19457",
"2605.24539"
]
},
{
"id": "judge",
"title": "评审者也会错，怎么办？",
"question": "任务没有完美 verifier 时，如何改进评审又避免迎合评分？",
"papers": [
"2606.26294"
],
"representatives": [
"2606.26294"
]
}
],
"route": [
"2303.17651",
"2305.11738",
"2507.19457",
"2605.24539"
]
},
{
"id": "experience",
"title": "怎样让这次经历帮助下一次任务？",
"short": "积累与复用经验",
"problem": "逐题重置会重复犯错；把整段历史都留下，又会混入题目细节、过期知识和无关内容。要保留可迁移的信息，并在需要时用上。",
"progress": "文字教训、可执行技能、关联记忆是并行的表示选择。继续向前的问题是：谁来维护它们，以及记忆机制本身是否也该改变。",
"limit": "存了内容、检索到了内容、正确使用了内容是三回事。跨题提升还必须与直接保留历史、同题重试和更长上下文区分。",
"branches": [
{
"id": "retain",
"title": "留下什么：教训、策略，还是可执行技能？",
"question": "经验怎样脱离旧题的细节，变成新任务能用的指导？",
"papers": [
"2303.11366",
"2305.16291",
"2308.10144",
"2409.07429",
"2508.06433",
"2509.25140",
"2402.17574",
"2510.23601",
"2603.13131",
"2603.18000",
"2603.18743",
"2604.15097",
"2604.20133",
"2605.23904",
"2607.00272",
"2608.15071"
],
"representatives": [
"2303.11366",
"2305.16291",
"2308.10144"
]
},
{
"id": "organize",
"title": "经验变多以后，怎样找准、组合、纠正？",
"question": "简单追加或反复总结会丢细节；怎样维护适用条件和经验之间的关系？",
"papers": [
"2305.10250",
"2409.00872",
"2502.12110",
"2510.04618",
"2604.16839",
"2608.16114",
"2608.24876",
"2606.17220",
"2608.23397",
"2609.09153",
"2609.19656"
],
"representatives": [
"2510.04618",
"2609.09153",
"2609.19656"
]
},
{
"id": "memory-system",
"title": "固定的记忆机制会不会成为瓶颈？",
"question": "不只更新条目，是否需要改变编码、检索、管理或能力扩展程序？",
"papers": [
"2512.18746",
"2602.07755",
"2604.10923"
],
"representatives": [
"2512.18746",
"2602.07755"
]
}
],
"route": [
"2303.11366",
"2308.10144",
"2305.16291",
"2510.04618",
"2512.18746"
]
},
{
"id": "harness",
"title": "失败来自执行方式时，怎样改系统？",
"short": "改进执行系统",
"problem": "换提示未必能修复工具接口、上下文丢失或控制流程错误。依赖人工逐个模型调系统，也跟不上任务与模型的变化。",
"progress": "从搜索提示和模块组合，扩展到修改实际执行代码；搜索空间变大后，诊断、候选多样性、局部回归和成本成为更具体的问题。",
"limit": "开放更多代码并不自动带来更好改进。必须区分任务答案的修改与后续任务会使用的系统修改，并看收益是否超出已有强 harness。",
"branches": [
{
"id": "design",
"title": "如何摆脱人工逐项设计？",
"question": "提示、流程和完整代码都可搜索；关键分歧是预先限定多少结构。",
"papers": [
"2309.03409",
"2310.03714",
"2408.08435",
"2410.10762",
"2507.03616",
"2504.15228",
"2505.22954",
"2410.04444",
"2603.03329",
"2608.12307"
],
"representatives": [
"2408.08435",
"2410.10762",
"2505.22954"
]
},
{
"id": "repair",
"title": "知道失败了，如何找到该改的系统部件？",
"question": "是模型不适配、某个部件有错，还是改动相互干扰？不同诊断决定不同补丁。",
"papers": [
"2603.28052",
"2604.25850",
"2606.09498",
"2606.06324",
"2606.01314",
"2607.14159",
"2608.01918",
"2607.15524"
],
"representatives": [
"2603.28052",
"2606.09498",
"2606.06324"
]
},
{
"id": "search",
"title": "如何用有限预算持续找到有效版本？",
"question": "只沿当前最优版本走可能停滞；保留什么历史、从哪里继续搜索？",
"papers": [
"2608.07645",
"2607.13683",
"2609.09646",
"2608.27311",
"2609.20519",
"sol-pi-author-blog"
],
"representatives": [
"2607.13683",
"2608.27311",
"2609.20519"
]
},
{
"id": "long-run",
"title": "任务不能重来，如何边执行边维护系统？",
"question": "长时间运行还要处理状态、恢复、协作与资源，离线改好再部署并不覆盖这些约束。",
"papers": [
"2403.03186",
"2608.22793",
"2608.23552",
"2608.11350",
"2608.09380",
"2608.26530",
"2609.01481",
"2606.19980",
"apodex-10-blog"
],
"representatives": [
"2608.26530",
"2609.01481"
]
}
],
"route": [
"2408.08435",
"2505.22954",
"2606.09498",
"2606.06324",
"2609.20519"
]
},
{
"id": "curriculum",
"title": "没有现成训练题时，下一步学什么？",
"short": "选择学习目标与数据",
"problem": "能够优化一个给定分数，不等于能发现能力缺口、找到合适练习或定义有效目标。练了很多题，也可能没有练到真正需要的能力。",
"progress": "一条路线从失败中选题、合成课程；另一条扩展可验证的新挑战。宽泛目标下的评测再检查系统能否自己确定学习方向。",
"limit": "生成得出、验得过、对目标能力有帮助，是不同条件。自建练习上的提高，需要在独立目标任务上验证。",
"branches": [
{
"id": "data",
"title": "怎样把失败变成值得训练的数据？",
"question": "要避免重复采样和无效轨迹，并判断材料是否补到了当前能力缺口。",
"papers": [
"2511.10395",
"2609.00768",
"2609.08183",
"2607.25886",
"2606.04261"
],
"representatives": [
"2511.10395",
"2609.00768"
]
},
{
"id": "frontier",
"title": "固定题库用完以后，如何持续产生挑战？",
"question": "新任务既要够难又要可验证；无可靠检查的难题无法稳定指导学习。",
"papers": [
"endlessfrontier.tech-assets-paper.pdf"
],
"representatives": [
"endlessfrontier.tech-assets-paper.pdf"
]
},
{
"id": "goal",
"title": "从“变得更强”到可执行的学习目标",
"question": "任务和指标不再由人给定时，目标解释、数据选择和自测是否仍然有效？",
"papers": [
"2608.31111",
"2608.05144"
],
"representatives": [
"2608.31111",
"2608.05144"
]
}
],
"route": [
"2511.10395",
"2609.00768",
"endlessfrontier.tech-assets-paper.pdf",
"2608.31111"
]
},
{
"id": "weights",
"title": "经验何时该写进模型，何时留在外部？",
"short": "训练与系统协同",
"problem": "外部经验能立刻指导行为，但模型不一定会用；训练能改变能力，也可能依赖当前 harness 产生的有限轨迹。只优化一侧，另一侧可能成为瓶颈。",
"progress": "先研究如何把经历变成参数学习信号，再研究新模型与新 harness 怎样交替适配；不是所有带联合架构的工作都已验证多代相互促进。",
"limit": "要分清被训练的是执行模型还是修改者。比较时匹配总训练／搜索预算，并避免用最好的测试点替代预先选定的最终版本。",
"branches": [
{
"id": "internalize",
"title": "如何从自己的经历获得有效训练信号？",
"question": "标准答案、教师示范、环境奖励和自建材料分别能教会什么？",
"papers": [
"2506.10943",
"2607.14777",
"2510.16079",
"2603.21877"
],
"representatives": [
"2506.10943",
"2607.14777"
]
},
{
"id": "state-policy",
"title": "模型能否学会何时维护和调用外部经验？",
"question": "只把记忆接到模型旁边不够，还要让读写和跨任务学习成为行为策略。",
"papers": [
"2608.05446",
"2607.26784"
],
"representatives": [
"2608.05446",
"2607.26784"
]
},
{
"id": "coupling",
"title": "两侧都会变时，如何安排更新？",
"question": "新的 harness 产生新的训练轨迹，新的模型又改变最合适的执行方式。",
"papers": [
"2605.27276",
"2605.09998",
"2606.14249",
"2608.13951",
"2608.09819",
"2606.01779",
"2607.22688",
"2609.00196",
"2609.17523"
],
"representatives": [
"2605.27276",
"2609.00196",
"2609.17523"
]
}
],
"route": [
"2506.10943",
"2605.27276",
"2609.00196",
"2609.17523"
]
},
{
"id": "improver",
"title": "系统能否越来越会提出改进？",
"short": "改进产生改进的能力",
"problem": "任务做得更好，不一定更会诊断、搜索和修改。固定的改进程序可能限制后续增长，需要把“怎样改”也当作研究对象。",
"progress": "学习修改规则、训练修改模型、开放修改程序，是三个不同切口。它们可组合，但都需要单独测下一轮产生改进的效率和迁移。",
"limit": "修改了改进程序，不等于已经证明改进加速。要把改进者移到新任务或新起点，在同预算下测它能否更快产生有效后代。",
"branches": [
{
"id": "rules",
"title": "改进规则能否随经验更新？",
"question": "若每轮都用同一套变异或诊断指令，会不会反复提出相似的无效修改？",
"papers": [
"2309.16797",
"2606.04465",
"2607.05297",
"2608.08466",
"2609.14858"
],
"representatives": [
"2309.16797",
"2607.05297",
"2609.14858"
]
},
{
"id": "learn-editor",
"title": "提出有效修改是否是一种可训练能力？",
"question": "选出好补丁与学会写出好补丁不同；后者需要把实际收益作为训练信号。",
"papers": [
"2608.02276",
"2607.21971",
"2607.28568",
"2603.18620"
],
"representatives": [
"2608.02276",
"2607.21971"
]
},
{
"id": "recursive-code",
"title": "如何让修改程序本身也能改变？",
"question": "把新的搜索机制保留下来，并检查它能否帮助后续领域和后续轮次。",
"papers": [
"2310.02304",
"2603.19461",
"2604.23472",
"2608.24735",
"metarsi-v1",
"weco-aide2-blog"
],
"representatives": [
"2310.02304",
"2603.19461"
]
},
{
"id": "foundations",
"title": "递归改进为何可能，有哪些形式限制？",
"question": "早期理论讨论收益、可证明性与搜索效率；其假设不自动适用于今天的 agent。",
"papers": [
"cs-0309048",
"people.idsia.ch-juergen",
"cs-0207097",
"1805.06610",
"doi.org-10.1016-S0065-2458-08-60418-0",
"yudkowsky-recursive-self-improvement"
],
"representatives": [
"cs-0309048"
]
}
],
"route": [
"2309.16797",
"2310.02304",
"2603.19461",
"2608.02276",
"2609.14858"
]
},
{
"id": "evaluation",
"title": "怎样证明系统确实学会了，而非多试几次？",
"short": "验证提升的来源与持续性",
"problem": "最终分数把执行者、改进者、搜索预算和数据接触混在一起。一次涨分也看不出迁移、遗忘和后续持续改进。",
"progress": "从静态任务成绩，转向受控的更新过程：分离角色，记录每轮成本，隔离选版本与测试，再跟踪连续任务。它是所有路线的验证层。",
"limit": "评测名字含 self-evolving，并不意味着测到了改进者自身变强。不同评测支持不同结论，不能拼成统一排名。",
"branches": [
{
"id": "attribution",
"title": "涨分究竟来自哪里？",
"question": "同预算重试、强起点、固定执行者以及更换修改者，分别排除不同解释。",
"papers": [
"2607.12227",
"2605.30621",
"2608.09096",
"2609.01437",
"2602.22480",
"2606.04455",
"2608.06301",
"2608.10178"
],
"representatives": [
"2607.12227",
"2605.30621",
"2609.01437"
]
},
{
"id": "transfer",
"title": "保留经验能否迁移，还是只记住了历史？",
"question": "控制任务顺序、上下文、检索与任务难度，检查是否形成了可用的新能力。",
"papers": [
"2505.11942",
"2507.05257",
"2508.19005",
"2510.17281",
"2511.20857",
"2604.17308",
"2604.20087",
"2605.18421",
"2606.05661",
"2607.05202",
"2608.00155",
"2608.01149",
"2608.03874",
"2608.04003",
"2608.06144",
"2608.03764"
],
"representatives": [
"2608.03874",
"2608.04003"
]
},
{
"id": "trajectory",
"title": "持续更新是否更好，旧能力是否保得住？",
"question": "不只测最好一次：跟踪整个曲线、验收误差、遗忘与成本。",
"papers": [
"2606.17546",
"2607.05155",
"2608.31100",
"2609.08175"
],
"representatives": [
"2606.17546",
"2608.31100",
"2609.08175"
]
},
{
"id": "research",
"title": "能否自主完成有效的 AI 研发？",
"question": "把工程过程跑通、把给定任务调好、发现可泛化新方法，需要不同验证。",
"papers": [
"2603.08640",
"2604.10547",
"2606.05080",
"2505.19955",
"2605.08678",
"2608.17271",
"rsi-exam",
"icoder-27b"
],
"representatives": [
"2603.08640",
"2605.08678",
"rsi-exam"
]
}
],
"route": [
"2607.12227",
"2605.30621",
"2608.03874",
"2606.17546",
"rsi-exam"
]
},
{
"id": "perspectives",
"title": "概念、产业观察与基础设施",
"short": "背景与工程入口",
"problem": "这些材料帮助界定 RSI、理解研发实践和搭实验平台；观点、产品观察和可复现实验的证据性质不同。",
"progress": "先用概念文章建立坐标，再用方法与评测论文检验具体主张。",
"limit": "综述或博客的方向判断不是实验结论；仓库提供接口也不等于已经证明持续提升。",
"branches": [
{
"id": "maps",
"title": "概念与领域综述",
"question": "用于建立整体阅读坐标。",
"papers": [
"2609.11873",
"data-centric-rsi-survey",
"ai4ai-survey",
"shiyu-rsi-what-evolves",
"shiyu-rsi-loop-closes",
"liu-self-evolving-taxonomy",
"weng-harness-engineering",
"zhihu-self-evolving-guide",
"melon-self-evolve-rsi",
"weng-prompt-engineering"
],
"representatives": [
"ai4ai-survey",
"shiyu-rsi-what-evolves",
"shiyu-rsi-loop-closes"
]
},
{
"id": "practice",
"title": "产业实践与架构观点",
"question": "用于了解现实约束与系统设计假设。",
"papers": [
"openai-research-acceleration",
"minimax-m27-blog",
"anthropic-when-ai-builds-itself",
"ibm-rsi-serious-question",
"2609.19203",
"reef"
],
"representatives": [
"anthropic-when-ai-builds-itself",
"reef"
]
},
{
"id": "programs",
"title": "程序搜索：改进任务产物的参照",
"question": "代码进化可以产生更好的算法或程序；需要另查改动是否进入 agent 或修改者本身。",
"papers": [
"2509.19349",
"deepmind-alphaevolve-blog"
],
"representatives": [
"2509.19349",
"deepmind-alphaevolve-blog"
]
}
],
"route": [
"shiyu-rsi-what-evolves",
"shiyu-rsi-loop-closes",
"ai4ai-survey"
]
}
],
"entries": {
"2203.11171": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "研究同一模型对当前题目的多种推理能否相互补充，从而在不训练模型的情况下提高数学与常识推理可靠性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2203.11171"
}
]
},
"2303.17651": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "研究单个模型在没有额外训练或外部反馈时，能否通过自身批评持续改善当前输出的质量。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.17651"
}
],
"name": "Self-Refine",
"contrast": "让同一模型先写批评再改初稿；重点是当前输出质量，不保留跨任务学习结果。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.17651"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2303.17651"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
}
]
},
"2305.11738": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "研究模型能否借助外部工具反馈识别并修正自身错误，改善单靠自我判断时不可靠的纠错能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.11738"
}
],
"name": "CRITIC",
"contrast": "用搜索、代码执行等外部工具检查输出；与纯自评的差别是引入可核对的信息。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.11738"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.11738"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
}
]
},
"2309.11495": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "研究模型能否通过自我核验减少事实性幻觉，并避免验证过程被初稿中的错误带偏。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.11495"
}
]
},
"2310.01798": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "检验语言模型在没有外部正确性反馈的条件下，是否真正具备修正自身推理错误的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.01798"
}
],
"name": "Cannot Self-Correct",
"contrast": "专门移除外部正确性反馈，检验所谓自纠错是否仍成立；提供适用边界。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.01798"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.01798"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2310.01798#S3.SS1"
}
]
},
"2310.04406": {
"primary": "feedback",
"branch": "self-check",
"related": [],
"question": "面向需要探索、回退和长程规划的任务，研究语言 agent 能否通过比较行动路径作出更可靠的决策。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.04406"
}
]
},
"2305.20050": {
"primary": "feedback",
"branch": "diagnosis",
"related": [],
"question": "研究训练数学推理评分模型时，监督中间步骤是否比只监督最终答案更有效，以及怎样提高人类标注的利用效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.20050"
}
],
"name": "Let's Verify Step by Step",
"contrast": "比较步骤级人工监督与只标最终结果；重点是监督粒度如何影响推理评分。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.20050"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.20050"
},
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
}
]
},
"2406.07496": {
"primary": "feedback",
"branch": "diagnosis",
"related": [],
"question": "面向由多个模型和工具组成的系统，研究自然语言反馈能否充当通用优化信号，支持不同组件的联合改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2406.07496"
}
]
},
"2507.19457": {
"primary": "feedback",
"branch": "diagnosis",
"related": [
"harness"
],
"question": "研究可解释的语言反馈能否比稀疏分数更高效地支持任务适应，并使提示优化在较少试验下获得有竞争力的能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.19457"
}
],
"name": "GEPA",
"contrast": "读执行轨迹中的语言反馈来改提示，并保留互补候选；区别于只依据总分猜新提示。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.19457"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2507.19457"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
}
]
},
"2605.24539": {
"primary": "feedback",
"branch": "diagnosis",
"related": [
"harness"
],
"question": "研究在奖励稀疏、执行随机的长程任务中，人类成功示范能否弥补 agent 自主试错的信息不足，帮助其可靠改进运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.24539"
}
],
"name": "DemoEvolve",
"contrast": "用成功示范弥补长任务稀疏奖励；研究多给哪类信息能帮助定位有效改动。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.24539"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.24539"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
}
]
},
"2609.00829": {
"primary": "feedback",
"branch": "diagnosis",
"related": [
"harness"
],
"question": "研究 agent 如何从失败中定位可复用的系统性改进，同时避免记住训练答案和损害已有能力，使框架进化更稳定地泛化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.00829"
}
]
},
"2606.26294": {
"primary": "feedback",
"branch": "judge",
"related": [
"improver",
"evaluation"
],
"question": "研究在缺少完美固定评价器的任务中，解题者与评审者能否共同改进，同时保持评价可信和版本比较有意义。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.26294"
}
],
"name": "Red Queen Gödel Machine",
"contrast": "让解题者与评审者共同改进；由固定判分转向共同适应，也增加了评审可信性的要求。",
"contrastSources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.26294"
},
{
"label": "Appendix C.5",
"url": "https://arxiv.org/abs/2606.26294"
}
]
},
"2303.11366": {
"primary": "experience",
"branch": "retain",
"related": [
"feedback"
],
"question": "研究冻结参数的语言 agent 能否从试错中学习，在决策、问答和编程任务中把失败经验转化为后续尝试的改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.11366"
}
],
"name": "Reflexion",
"contrast": "把试错转成下一次尝试可读的文字教训；需要区分同题重试与跨题迁移。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2303.11366"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2303.11366"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
}
]
},
"2305.16291": {
"primary": "experience",
"branch": "retain",
"related": [
"curriculum"
],
"question": "面向 Minecraft 开放世界，研究 agent 能否在无人持续指定任务的情况下自主探索、积累技能，并将能力迁移到新任务和新世界。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.16291"
}
],
"name": "Voyager",
"contrast": "自动课程配合可执行技能库，成功代码可检索和组合；同时涉及探索方向与能力积累。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.16291"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2305.16291"
},
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
}
]
},
"2308.10144": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究无法访问或更新模型权重时，agent 能否从多项任务的成败经历中自主学习，并把经验用于新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2308.10144"
}
],
"name": "ExpeL",
"contrast": "从多项训练任务整理一般规则和成功示例，供未见任务使用；关注经验能否离开原题。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2308.10144"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2308.10144"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
}
]
},
"2409.07429": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究网页 agent 能否把既往操作经历转化为跨任务复用的工作流程，从而提高长操作链任务的可靠性与效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2409.07429"
}
]
},
"2508.06433": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究 agent 能否在不更新模型参数的情况下积累、修正和复用操作经验，改善后续任务的规划与执行效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2508.06433"
}
]
},
"2509.25140": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究 agent 能否从自行判断的成功与失败经历中提炼可迁移的推理经验，在缺少标准答案反馈时仍改善后续任务表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2509.25140"
}
]
},
"2402.17574": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "面向动态博弈，研究 agent 能否从跨局经历中改进整体行为策略，而不只修正当前一步动作或依赖手工规则。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2402.17574"
}
]
},
"2510.23601": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究通用 agent 能否通过任务经验形成可复用的领域执行能力，从而更高效地处理后续相关任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.23601"
}
]
},
"2603.13131": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "面向 Minecraft 中前置依赖多、容易中断的长程任务，研究积累的执行知识能否帮助 agent 应对新失败并恢复任务推进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.13131"
}
]
},
"2603.18000": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究 agent 能否把解决复杂任务的经历转化为可再次执行的能力，并在后续相关任务中减少重复开发、提高可靠性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.18000"
}
]
},
"2603.18743": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究冻结的通用模型能否通过持续积累和修订技能，自主形成适合新任务的 agent 能力，并有效选择该使用的技能。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.18743"
}
]
},
"2604.15097": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究经验采用什么表示才更有助于后续行为改进，检验完整的步骤文档是否一定优于紧凑、可修改的策略知识。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.15097"
}
]
},
"2604.20133": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "面向实际助理的长期服务，解决持续技能学习与复杂任务委派如何协同，使积累的能力能够被合适地调用。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.20133"
}
]
},
"2605.23904": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究外部文字技能能否成为稳定、可优化的学习对象，使冻结模型通过技能更新持续适应任务，并跨模型或工具框架复用收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.23904"
}
]
},
"2607.00272": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "面向机器人操作，研究 agent 能否从多模态执行失败中自主学会可复用修复技能，并适应未见任务与环境变化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.00272"
}
]
},
"2608.15071": {
"primary": "experience",
"branch": "retain",
"related": [],
"question": "研究 agent 面对连续到来的新任务时，能否从已完成的经历中学到可复用经验、改善后续任务表现，并分清反馈来源、经验表示和修改者能力分别如何影响学习效果。",
"sources": [
{
"label": "§1 Introduction · 研究目标与分析范围",
"url": "https://arxiv.org/html/2608.15071#S1"
}
]
},
"2305.10250": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向长期陪伴和个人助理，研究模型如何跨会话记住用户经历与特点，支持连贯且个性化的交互。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2305.10250"
}
]
},
"2409.00872": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向动态决策和长文本任务，研究反馈利用与记忆管理如何帮助 agent 持续执行，缓解有限上下文带来的能力限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2409.00872"
}
]
},
"2502.12110": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向持续对话中的记忆管理，研究 agent 能否随新信息自主组织和修订历史知识，而不仅是存下记录后按相似度检索。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2502.12110"
}
]
},
"2510.04618": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "研究 agent 如何在持续适应任务时保留有用的领域经验，避免反复压缩和重写上下文造成知识流失。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.04618"
}
],
"name": "ACE",
"contrast": "逐条维护经验手册，避免反复整体重写把细节抹掉；主要改内容的组织与更新方式。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.04618"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2510.04618"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
}
]
},
"2604.16839": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向长对话记忆，研究经验在反复共同使用中形成的关联，能否改善知识组织、回答质量和上下文使用效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.16839"
}
]
},
"2608.16114": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "研究如何保留子任务与技能之间的组合关系，使 agent 能够更准确地检索、组合和维护可复用的执行经验。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.16114"
}
]
},
"2608.24876": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向长程任务，解决 agent 如何准确维护当前进度、及时调用相关经验，并把失败归因到需要修订的记忆部分。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.24876"
}
]
},
"2606.17220": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向法律案例检索，研究 agent 能否自主积累有效的查询规则，在强词项匹配基线之上持续提高检索质量。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17220"
}
]
},
"2608.23397": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "面向临床任务，研究 agent 如何在证据来源、适用范围和权限约束下积累可复用经验，同时保持诊断与流程的可靠性。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.23397"
}
]
},
"2609.09153": {
"primary": "experience",
"branch": "organize",
"related": [],
"question": "研究可更新的步骤图能否帮助冻结模型稳定执行长任务，以及它能否从经验中自行构建并纠正错误的人工先验。",
"sources": [
{
"label": "§1：程序知识与研究目标",
"url": "https://arxiv.org/html/2609.09153#S1"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
}
],
"name": "Procedural Graphs",
"contrast": "把经验写成带条件的步骤图，按当前执行位置提供指导；强调经验的执行结构。",
"contrastSources": [
{
"label": "§1：程序知识与研究目标",
"url": "https://arxiv.org/html/2609.09153#S1"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
},
{
"label": "§5.1：跨基准结果",
"url": "https://arxiv.org/html/2609.09153#S5.SS1"
}
]
},
"2609.19656": {
"primary": "experience",
"branch": "organize",
"related": [
"harness"
],
"question": "让索引自动发现检索缺口、生成练习查询并修订索引键，检验对搜索和记忆读取的帮助。",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
],
"name": "SELF-INDEX",
"contrast": "用自建练习查询暴露检索缺口，再改索引键；返回原文，模型参数不更新。",
"contrastSources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
"2512.18746": {
"primary": "experience",
"branch": "memory-system",
"related": [
"harness",
"improver"
],
"question": "研究 agent 能否不仅积累记忆内容，还自主改进记忆系统的组织和运行方式，以适应不同任务与基础模型。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2512.18746"
}
],
"name": "MemEvolve",
"contrast": "连编码、存储、检索和管理记忆的程序一起搜索；区别于只向固定系统添加记忆。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2512.18746"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2512.18746"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
}
]
},
"2602.07755": {
"primary": "experience",
"branch": "memory-system",
"related": [
"improver"
],
"question": "研究“怎样记忆才能更好地持续学习”能否由系统自动学出，减少顺序决策 agent 对固定人工记忆设计的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2602.07755"
}
],
"name": "ALMA",
"contrast": "根据后续持续学习效果搜索记忆设计；将“怎样学经验”放到外层优化。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2602.07755"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2602.07755"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
}
]
},
"2604.10923": {
"primary": "experience",
"branch": "memory-system",
"related": [],
"question": "研究经验积累与工具、专家能力扩展能否相互促进，使 agent 突破只增加记忆或只增加工具的单一路径。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.10923"
}
]
},
"2309.03409": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "研究语言模型能否在没有可用梯度的情况下充当优化器，根据目标与历史试验结果改进数值方案或任务提示。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.03409"
}
]
},
"2310.03714": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "面向多步语言模型应用，解决程序设计如何与具体提示调优分离，使系统能够根据任务指标自动适配。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.03714"
}
]
},
"2408.08435": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "研究 agent 系统的设计能否由模型自主完成，从人工预设的提示和模块组合扩展到更开放的系统结构，并获得跨任务、跨模型的适用性。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2408.08435"
}
],
"name": "ADAS",
"contrast": "用代码表达和搜索 agent 设计，扩大人工预置模块之外的设计空间。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2408.08435"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2408.08435"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
}
]
},
"2410.10762": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "研究 task agent 的工作流能否自动设计，减少对人工初始流程的依赖，并改善执行效果与成本之间的权衡。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2410.10762"
}
],
"name": "AFlow",
"contrast": "在工作流结构和操作节点上搜索；与开放代码修改相比，搜索空间更明确。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2410.10762"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2410.10762"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
}
]
},
"2507.03616": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "面向自动 agent 工作流研发，提供统一工程平台，解决不同生成、执行和优化算法难以组合复用的问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.03616"
}
]
},
"2504.15228": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "研究具备基本编程工具的 agent 能否自主改进自身软件实现，并让更好的版本继续推动后续能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2504.15228"
}
]
},
"2505.22954": {
"primary": "harness",
"branch": "design",
"related": [
"improver"
],
"question": "探索 coding agent 能否减少对人工系统设计的依赖，通过持续改进自身实现，逐步提升解题能力及进一步自改进的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.22954"
}
],
"name": "DGM",
"contrast": "用实际任务成绩筛选自修改版本，保留多样历史分支供继续探索；用实证替代形式证明。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.22954"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2505.22954"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
}
]
},
"2410.04444": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "探索 agent 能否自主改变解题流程及负责自修改的机制，减少人工固定设计对递归自改进空间的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2410.04444"
}
]
},
"2603.03329": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "面向有明确动作规则的交互环境，研究自动生成的运行代码能否弥补模型执行可靠性的不足，减少非法动作并改善决策。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.03329"
}
]
},
"2608.12307": {
"primary": "harness",
"branch": "design",
"related": [],
"question": "研究强模型能否在不训练弱模型参数的情况下，将部分能力转移给弱模型，扩大其在测试时能够完成的任务范围。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.12307"
}
]
},
"2603.28052": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究自动运行框架设计能否充分利用历次试验的经验，找到优于人工设计、并能迁移到其他任务或模型的运行方式。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.28052"
}
],
"name": "Meta-Harness",
"contrast": "让修改者检索历史代码、分数和轨迹，支持跨轮诊断；不只依赖当前候选的压缩摘要。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.28052"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.28052"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
]
},
"2604.25850": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "面向工具、记忆和控制流程共同作用的复杂 agent 系统，研究如何使自动框架改进可诊断、可验证，并产生可迁移的工程收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.25850"
}
]
},
"2606.09498": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究冻结模型能否在没有人工工程师或更强外部模型指导时，识别自身特有的执行弱点，并改进适合自己的运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.09498"
}
],
"name": "Self-Harness",
"contrast": "由同型号模型根据自己的失败适配 harness；重点是执行模型特有的弱点。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.09498"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.09498"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
}
]
},
"2606.06324": {
"primary": "harness",
"branch": "repair",
"related": [
"feedback"
],
"question": "研究自动框架修复如何从“知道任务失败”推进到“知道哪个运行机制有问题”，以减少无针对性修改和连带退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.06324"
}
],
"name": "HarnessFix",
"contrast": "先把失败定位到运行机制，再做限定范围的修复；重点是修改针对性和避免连带退化。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.06324"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.06324"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
}
]
},
"2606.01314": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究技能说明与可执行工具如何共同适应任务，尤其关注多技能相互依赖时，单独改一项无法修复的执行问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.01314"
}
]
},
"2607.14159": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究运行框架能否从历史执行中学会适配不同任务，使 agent 面对无答案反馈的新题时仍能选用合适的运行方式。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.14159"
}
]
},
"2608.01918": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究自动框架进化怎样获得可迁移的真实能力提升，减少记题式适配、诊断不足及组件修改相互干扰造成的退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.01918"
}
]
},
"2607.15524": {
"primary": "harness",
"branch": "repair",
"related": [],
"question": "研究用户可配置的轻量运行框架能否以少量迭代提升科研任务的执行质量，并降低对高推理强度和人工持续维护的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.15524"
}
]
},
"2608.07645": {
"primary": "harness",
"branch": "search",
"related": [],
"question": "研究自改进代码 agent 如何利用不同任务和不同进化分支之间的经验，提高后续系统改进的质量与搜索效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.07645"
}
]
},
"2607.13683": {
"primary": "harness",
"branch": "search",
"related": [],
"question": "研究在评估有噪声、交互预算有限时，如何保持框架搜索的有效多样性，并找到可靠泛化的改进。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.13683"
}
],
"name": "HarnessBank",
"contrast": "保存语义不同且通过质量检查的候选，避免只保留单一最高分版本造成搜索收缩。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.13683"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.13683"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
}
]
},
"2609.09646": {
"primary": "harness",
"branch": "search",
"related": [],
"question": "研究如何控制修改范围、操作种类和搜索起点，使多 agent 框架在相同预算下产生更多有效改进。",
"sources": [
{
"label": "§1–2：定位与范围",
"url": "https://arxiv.org/html/2609.09646#S1"
},
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
}
]
},
"2608.27311": {
"primary": "harness",
"branch": "search",
"related": [],
"question": "研究有限交互预算下，怎样判断一项框架修改是否真正有效，使自动进化减少无关测试并识别被总分掩盖的行为退化。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.27311"
}
],
"name": "HarnessLens",
"contrast": "根据改动选择更相关的验证任务；优化的是验收信息和预算利用。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.27311"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.27311"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
}
]
},
"2609.20519": {
"primary": "harness",
"branch": "search",
"related": [
"evaluation"
],
"question": "研究自动搜索能否找到跨任务、跨模型复用的 harness 效率改进，而非只提高搜索题得分。",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
],
"name": "SoL-Pi",
"contrast": "搜索动作、上下文与输出处理的效率改进；成本和得分一起看，不能把节省 token 等同能力增长。",
"contrastSources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
"sol-pi-author-blog": {
"primary": "harness",
"branch": "search",
"related": [],
"question": "SoL-Pi 作者的图解与工程说明，解释四种机制如何省成本及研究中人类实际做了什么。",
"sources": [
{
"label": "Method；Discoveries；Results；Humans set priors；Recursive Efficient Improvement",
"url": "https://nvlabs.github.io/SoL-Pi/"
}
]
},
"2403.03186": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向游戏和日常软件，探索 agent 能否像人一样依靠通用屏幕与操作接口完成长程任务，减少对逐环境专用接口的依赖。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2403.03186"
}
]
},
"2608.22793": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向车载助手的模糊请求和行为约束，研究如何把偶尔成功转化为重复执行时的稳定可靠表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.22793"
}
]
},
"2608.23552": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向超出单次上下文容量的复杂任务，解决 agent 如何持续管理信息、计算与协作，使长程执行能够恢复、验证并控制资源。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.23552"
}
]
},
"2608.11350": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向动作接口固定的具身环境，研究冻结模型能否通过外部技能与运行支持的适应，改善感知、规划和任务执行能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.11350"
}
]
},
"2608.09380": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向长程复杂业务，研究观察、规划、验证和恢复经验如何成为可持续积累的控制能力，并在更新时保持收益与风险可检验。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09380"
}
]
},
"2608.26530": {
"primary": "harness",
"branch": "long-run",
"related": [
"experience"
],
"question": "研究把任务执行与监督分开，能否在运行过程中纠偏，并把当场发现的经验变成以后可用的技能。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2608.26530#S1"
}
],
"name": "PILOT in the Loop",
"contrast": "把执行和监督分开，在任务运行中纠偏并留下后续可用的技能。",
"contrastSources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2608.26530#S1"
},
{
"label": "§3.1 实验协议",
"url": "https://arxiv.org/html/2608.26530#S3.SS1"
},
{
"label": "§3.2–3.3 结果",
"url": "https://arxiv.org/html/2608.26530#S3.SS2"
}
]
},
"2609.01481": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "研究编码 agent 能否在无人持续指导的长时间开发中，把高层需求推进为完整可用的软件，并持续兼顾缺陷修复与功能增长。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01481"
}
],
"name": "Harness-of-Harness",
"contrast": "研究多日软件开发中的持续修复与功能推进；工作发生在长期运行过程中。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01481"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2609.01481"
},
{
"label": "附录A.2",
"url": "https://arxiv.org/html/2609.01481#A1.SS2"
}
]
},
"2606.19980": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "面向真实机器人，研究代码 agent 能否在减少人工复位、验收和算法干预的情况下，自主开展物理实验并持续改进操作策略。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.19980"
}
]
},
"apodex-10-blog": {
"primary": "harness",
"branch": "long-run",
"related": [],
"question": "介绍并行研究 agent 团队与统一验证的运行设计，对比单 agent 与 heavy-duty 模式。",
"sources": [
{
"label": "Scaling Reasoning with an Agent Team；Where Apodex Stands",
"url": "https://www.apodex.com/blog/apodex-1.0"
}
]
},
"2511.10395": {
"primary": "curriculum",
"branch": "data",
"related": [],
"question": "面向工具交互任务，研究如何减少人工构造训练任务与低效探索的成本，使 agent 更高效地自主学习新环境中的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2511.10395"
}
],
"name": "AgentEvolver",
"contrast": "自主组织交互任务并利用经验探索，减少人工出题和重复低价值采样。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2511.10395"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2511.10395"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
}
]
},
"2609.00768": {
"primary": "curriculum",
"branch": "data",
"related": [],
"question": "研究能否从模型自己的失败历史提取明确的错误原因，并据此生成下一轮训练课程，不使用外部任务材料。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.00768#S1"
},
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
}
],
"name": "DiagEvo",
"contrast": "把自身失败按原因组织为错误记忆，再生成训练课程；改变下一轮练习的针对性。",
"contrastSources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.00768#S1"
},
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
},
{
"label": "§4.2–4.4 结果与分析",
"url": "https://arxiv.org/html/2609.00768#S4.SS2"
}
]
},
"2609.08183": {
"primary": "curriculum",
"branch": "data",
"related": [],
"question": "探索路由 harness 能否把真实交互中的能力信号转成训练课程和数据配比，持续提升单个 agent 模型。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
}
]
},
"2607.25886": {
"primary": "curriculum",
"branch": "data",
"related": [
"evaluation"
],
"question": "评测 agent 在训练基础设施已提供的条件下，能否诊断模型能力缺口、提出有效的数据研究方案，并稳定保留后训练收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.25886"
}
]
},
"2606.04261": {
"primary": "curriculum",
"branch": "data",
"related": [
"evaluation"
],
"question": "评测训练工程已经固定时，agent 能否开展有效的数据筛选研究，并提出超出局部调参的数据策略。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04261"
}
]
},
"endlessfrontier.tech-assets-paper.pdf": {
"primary": "curriculum",
"branch": "frontier",
"related": [],
"question": "研究前沿科学任务的供应能否摆脱对人工整理的持续依赖，并由系统自主扩展可验证的学习任务，推动模型能力继续增长。",
"sources": [
{
"label": "摘要",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
],
"name": "BigBang",
"contrast": "持续合成并验证更难的科学任务，针对固定题库与人类出题能力的上限。",
"contrastSources": [
{
"label": "摘要",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
},
{
"label": "§3.1–3.2 Experimental Setup / Quantitative Results",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
},
"2608.31111": {
"primary": "curriculum",
"branch": "goal",
"related": [
"evaluation"
],
"question": "评测模型只获得宽泛能力目标时，能否把目标转成有效的学习方案。论文分别考察：模糊目标相对明确任务会怎样影响训练；模型能否自行选择数据并更新参数；参数固定时能否改进外围运行框架。",
"sources": [
{
"label": "§1、§4.1：研究问题与实验安排",
"url": "https://arxiv.org/html/2608.31111#S4.SS1"
}
],
"name": "Aspire",
"contrast": "不给现成训练目标和练习题，检查宽泛能力目标能否转成有效学习；它提供诊断，不是解决该问题的方法。",
"contrastSources": [
{
"label": "§1、§4.1：研究问题与实验安排",
"url": "https://arxiv.org/html/2608.31111#S4.SS1"
},
{
"label": "§4.2：目标描述对照结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
},
{
"label": "§4.3：参数更新结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS3"
}
]
},
"2608.05144": {
"primary": "curriculum",
"branch": "goal",
"related": [],
"question": "面向目标可能变化、正确性信号不充分的开放科研，解决 agent 如何长期维持研究方向、积累可信证据并持续推进工作的问题。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05144"
}
],
"name": "Argus",
"contrast": "面对目标与判据会变化的开放科研，维护证据并在关键处引入专家；研究长期方向管理。",
"contrastSources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05144"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
}
]
},
"2506.10943": {
"primary": "weights",
"branch": "internalize",
"related": [],
"question": "研究模型能否面对新材料或少量示例，自主决定怎样把输入转成有效的参数更新，从而获得新的知识与任务能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2506.10943"
}
],
"name": "SEAL",
"contrast": "学习把新输入改写成有效的参数更新材料；训练对象包含“怎样为自己组织学习材料”的策略。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2506.10943"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2506.10943"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
}
]
},
"2607.14777": {
"primary": "weights",
"branch": "internalize",
"related": [
"feedback"
],
"question": "研究交互 agent 能否从自己刚完成的经历中学到更具体的行动指导，弥补只看任务最终成败时缺少中间监督的问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.14777"
}
],
"name": "SEED",
"contrast": "从近期执行轨迹提炼逐步监督，为只有最终成败的 agent 强化学习补充学习信号。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.14777"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.14777"
},
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
]
},
"2510.16079": {
"primary": "weights",
"branch": "internalize",
"related": [],
"question": "面向搜索与多跳问答，研究 agent 能否从自身行动结果中持续改进策略，使外部经验积累与参数学习相互衔接。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.16079"
}
]
},
"2603.21877": {
"primary": "weights",
"branch": "internalize",
"related": [],
"question": "研究模型在难题上几乎所有尝试都失败时，能否重新获得有效学习信号，并把提示带来的能力增益转化为持久参数能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.21877"
}
]
},
"2608.05446": {
"primary": "weights",
"branch": "state-policy",
"related": [],
"question": "面向长程交互，研究 agent 能否学会构建有用的外部状态，并自主判断何时读写它，兼顾任务效果与运行成本。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.05446"
}
],
"name": "EvoHarness-RL",
"contrast": "训练何时创建、读写外部状态的行为；可变的不只是外部文本，模型策略也在学习。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.05446"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05446"
},
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
}
]
},
"2607.26784": {
"primary": "weights",
"branch": "state-policy",
"related": [
"experience",
"improver"
],
"question": "研究 agent 能否学会为未来任务整理技能，并使跨任务经验积累带来的收益成为可训练的能力，而非只擅长同题重试。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.26784"
}
],
"name": "SkillRise",
"contrast": "把为未来任务整理技能的收益纳入训练；目标超出当前题的单次成功。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.26784"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.26784"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
}
]
},
"2605.27276": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究自主改进系统能否根据任务失败，在改运行框架与更新模型参数之间作出有效选择，并获得两者互补的收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.27276"
}
],
"name": "SIA",
"contrast": "把改 harness 与训练模型放进同一操作空间，研究如何根据失败选择改哪一侧。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.27276"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.27276"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2605.27276#S4.SS3"
}
]
},
"2605.09998": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究 agent 在长程环境不中断、不重置的情况下，能否持续适应新经历，并进一步把运行系统的改善转化为模型学习收益。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.09998"
}
]
},
"2606.14249": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "面向 agent 系统的整体改进，研究运行框架设计与模型训练如何共同受益于执行经验，减少两者分开优化的局限。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.14249"
}
]
},
"2608.13951": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究运行框架能否同时服务于当前任务执行和下一轮模型学习，建立两者共同改进所需的可追溯数据基础；本文主要验证框架与轨迹产出。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.13951"
}
]
},
"2608.09819": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "面向部署后的开放持续学习，探索 agent 如何把新经验转化为长期能力，并协调模型专长与运行框架的共同适应；直接搜索实证主要覆盖框架配置。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09819"
}
]
},
"2606.01779": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究在不同工具和检索任务中，让框架与专属模型分支共同进化，是否比只改一侧更有效。",
"sources": [
{
"label": "§1：框架—模型兼容性问题",
"url": "https://arxiv.org/html/2606.01779#S1"
},
{
"label": "§3.2：联合进化流程与起点",
"url": "https://arxiv.org/html/2606.01779#S3.SS2"
}
]
},
"2607.22688": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究修复产生训练轨迹的框架、再用更好的轨迹训练模型，能否在多轮后训练中形成互相促进的改进。",
"sources": [
{
"label": "§1：联合后训练动机",
"url": "https://arxiv.org/html/2607.22688#S1"
},
{
"label": "§3.2–3.5：两个循环与验收",
"url": "https://arxiv.org/html/2607.22688#S3.SS2"
}
]
},
"2609.00196": {
"primary": "weights",
"branch": "coupling",
"related": [],
"question": "研究模型参数与可执行框架是否会互相限制，以及怎样安排交替更新才能更有效地释放两者能力。",
"sources": [
{
"label": "§1：问题与贡献",
"url": "https://arxiv.org/html/2609.00196#S1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
}
],
"name": "WHALE",
"contrast": "固定一侧更新另一侧，并比较切换时机；重点是两侧相互成为瓶颈时如何安排预算。",
"contrastSources": [
{
"label": "§1：问题与贡献",
"url": "https://arxiv.org/html/2609.00196#S1"
},
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
}
]
},
"2609.17523": {
"primary": "weights",
"branch": "coupling",
"related": [
"feedback",
"curriculum"
],
"question": "研究科研协作能否同时推动工作方式和模型学习：内层改 harness，外层在选定 harness 下做强化学习。",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
],
"name": "ScienceBuddy",
"contrast": "内层固定模型改 harness，外层在改后的 harness 下训练模型；科研交互提供任务和评价材料。",
"contrastSources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
"2309.16797": {
"primary": "improver",
"branch": "rules",
"related": [],
"question": "研究任务提示与提示改写策略能否共同改进，使自动提示搜索摆脱固定人工变异规则的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.16797"
}
],
"name": "Promptbreeder",
"contrast": "任务提示与产生变异的提示共同变化；可改的是文字层的搜索规则。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2309.16797"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2309.16797"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
}
]
},
"2606.04465": {
"primary": "improver",
"branch": "rules",
"related": [],
"question": "研究提示优化器能否学习并迁移“怎样改提示”的经验，突破固定人工优化指令对任务提示搜索的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04465"
}
]
},
"2607.05297": {
"primary": "improver",
"branch": "rules",
"related": [],
"question": "研究 agent 能否在改善任务技能的同时，学会更有效地分析经验和改进技能，从而突破固定改进流程的限制。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05297"
}
],
"name": "MetaSkill-Evolve",
"contrast": "任务技能和指导技能修改的说明以不同节奏更新；区分改任务知识与改修改规则。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05297"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.05297"
},
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
}
]
},
"2608.08466": {
"primary": "improver",
"branch": "rules",
"related": [],
"question": "研究冻结模型能否在稳定的外部边界内，既改进任务运行框架又改进自身的修改策略，并考察这种多层改进的能力上限。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.08466"
}
]
},
"2609.14858": {
"primary": "improver",
"branch": "rules",
"related": [
"harness"
],
"question": "研究能否利用已有发现过程，低成本改进“下一轮怎样探索”的策略，而不更换或训练底层 coding agent。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
],
"name": "Dream-RSI",
"contrast": "用历史发现树构建回放环境，低成本试验探索策略；底层 coding agent 保持不变。",
"contrastSources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
},
{
"label": "§4 Experiments",
"url": "https://arxiv.org/html/2609.14858#S4"
}
]
},
"2608.02276": {
"primary": "improver",
"branch": "learn-editor",
"related": [
"harness",
"weights"
],
"question": "研究“根据失败记录修改运行框架”能否成为可专门学习的工程能力，使修改者可靠地提升另一个冻结 agent 的任务表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.02276"
}
],
"name": "Harness-R1",
"contrast": "把补丁装入冻结执行者后的真实收益用于训练独立编辑模型；学习的是修改能力。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.02276"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.02276"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
}
]
},
"2607.21971": {
"primary": "improver",
"branch": "learn-editor",
"related": [],
"question": "研究利用反馈、反思失败和比较历史方案这些自进化能力，能否被专门训练出来，并迁移到训练之外的优化问题。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.21971"
}
],
"name": "MetaEvolve",
"contrast": "训练利用反馈、反思和历史方案提出改进的能力，并检查能否迁移到新优化问题。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.21971"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2607.21971"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2607.21971#S2.SS1"
}
]
},
"2607.28568": {
"primary": "improver",
"branch": "learn-editor",
"related": [],
"question": "面向机器学习工程，研究能否训练出擅长改进 AI 系统的专门模型，并使其工程改进能力与搜索经验迁移到新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.28568"
}
]
},
"2603.18620": {
"primary": "improver",
"branch": "learn-editor",
"related": [
"experience",
"weights"
],
"question": "研究利用经验修改上下文的能力能否通过专门训练获得，使模型在部署后更有效地从已做任务适应新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.18620"
}
]
},
"2310.02304": {
"primary": "improver",
"branch": "recursive-code",
"related": [],
"question": "研究冻结语言模型支持的代码优化程序，能否递归改进负责优化的程序本身，从而学出更有效的搜索策略。",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.02304"
}
],
"name": "STOP",
"contrast": "让冻结模型支持的代码优化程序修改优化程序本身；区分更好的输出程序与更好的优化器。",
"contrastSources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2310.02304"
},
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
]
},
"2603.19461": {
"primary": "improver",
"branch": "recursive-code",
"related": [],
"question": "研究能否同时改进 task agent 的解题代码与 meta-agent 的修改程序，并将学到的修改方法迁移到编程之外的任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.19461"
}
],
"name": "Hyperagents",
"contrast": "同时开放 task agent 与 meta-agent 程序；关注改进机制能否跨任务迁移和跨轮积累。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.19461"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.19461"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
}
]
},
"2604.23472": {
"primary": "improver",
"branch": "recursive-code",
"related": [],
"question": "研究任务求解能力和优化器的改进能力能否共同增长，使优化策略随着被优化系统的水平变化而适应。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.23472"
}
]
},
"2608.24735": {
"primary": "improver",
"branch": "recursive-code",
"related": [],
"question": "研究递归改进能否突破固定一层改进者的限制，扩展到更深的辅助层次，以及增加深度在何时真正有益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.24735"
}
]
},
"metarsi-v1": {
"primary": "improver",
"branch": "recursive-code",
"related": [
"weights",
"curriculum"
],
"question": "研究能否由系统选择并改进数据生成、harness 修改和参数训练的组合顺序，使“怎样改进”本身也能根据经验调整。",
"sources": [
{
"label": "§1，PDF p2–4",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=2"
},
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
}
]
},
"weco-aide2-blog": {
"primary": "improver",
"branch": "recursive-code",
"related": [],
"question": "双层 autoresearch 案例：外层修改内层 agent 的代码，检验改进能否迁移到未用于优化的任务。",
"sources": [
{
"label": "§1、The inner-loop evaluation、§2.2、§3.1、§4",
"url": "https://www.weco.ai/blog/first-evidence-of-recursive-self-improvement"
}
]
},
"cs-0309048": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "讨论自修改系统在什么条件下能够证明：改变自身程序的预期收益高于继续使用原程序，并由此获得最优性保证。",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
],
"name": "Gödel Machines",
"contrast": "只有证明自修改收益更高才切换程序；形式保证依赖其公理、证明与效用假设。",
"contrastSources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
},
"people.idsia.ch-juergen": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "讨论学习系统能否把自身的学习机制也作为学习对象，从而形成更高阶的递归适应。",
"sources": [
{
"label": "PSALM 实现与初步实验",
"url": "https://people.idsia.ch/~juergen/diploma.html"
}
]
},
"cs-0207097": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "研究程序求解器能否复用过去的解法和搜索经验，高效解决后续问题，并为这种增量搜索给出理论保证。",
"sources": [
{
"label": "§6 Experiments",
"url": "https://arxiv.org/abs/cs/0207097"
}
]
},
"1805.06610": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "研究递归自改进在明确受限的形式化条件下是否可能，以及这种系统可以具有什么计算效率。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/1805.06610"
}
]
},
"doi.org-10.1016-S0065-2458-08-60418-0": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "讨论当机器的设计能力超过人类时，机器改进机器是否可能引发不断加速的智能增长。",
"sources": [
{
"label": "关于 ultraintelligent machine 与 intelligence explosion 的论述",
"url": "https://doi.org/10.1016/S0065-2458(08)60418-0"
}
]
},
"yudkowsky-recursive-self-improvement": {
"primary": "improver",
"branch": "foundations",
"related": [],
"question": "RSI 的早期概念论述，分析优化机会、资源、效率以及改进认知机制的反馈关系。",
"sources": [
{
"label": "正文：optimization slope/resources/efficiency 与递归层次",
"url": "https://www.lesswrong.com/posts/JBadX7rwdcRFzGuju/recursive-self-improvement"
}
]
},
"2607.12227": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "检验 harness 进化是否优于相同反馈与尝试预算下的直接重试，以及进化后的框架能否迁移到未参与搜索的任务。",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
}
],
"name": "Rethinking Evaluation",
"contrast": "用相近反馈和推理预算对照直接重试，并测未参与搜索的任务；区分系统收益与额外搜索收益。",
"contrastSources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
}
]
},
"2605.30621": {
"primary": "evaluation",
"branch": "attribution",
"related": [
"improver"
],
"question": "研究模型的基础做题能力，能否预测它提出有用框架更新的能力，以及利用更新后框架获得收益的能力。",
"sources": [
{
"label": "§1：两种能力的分离",
"url": "https://arxiv.org/html/2605.30621#S1"
},
{
"label": "§3.3：固定一侧的能力指标",
"url": "https://arxiv.org/html/2605.30621#S3.SS3"
}
],
"name": "Updating ≠ Benefit",
"contrast": "把执行者与修改者分别配对，区分会做任务、会写更新、会利用更新三种能力。",
"contrastSources": [
{
"label": "§1：两种能力的分离",
"url": "https://arxiv.org/html/2605.30621#S1"
},
{
"label": "§3.3：固定一侧的能力指标",
"url": "https://arxiv.org/html/2605.30621#S3.SS3"
},
{
"label": "§4.2：修改者能力实验",
"url": "https://arxiv.org/html/2605.30621#S4.SS2"
}
]
},
"2608.09096": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "评测语言模型作为运行框架修改者的能力：在执行模型与预算受控时，能否带来泛化到未见任务的提升，并在持续搜索中保住收益。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09096"
}
]
},
"2609.01437": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "评测模型能否自主建设并持续改进 agent 的运行基础设施，区分“会在现成系统中做任务”与“会设计有效系统”这两种能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01437"
}
],
"name": "HarnessDev",
"contrast": "从弱系统起点建设和继续改进 harness，单独测模型的系统工程能力。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01437"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2609.01437"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2609.01437#S3.SS2.SSS0.Px1"
}
]
},
"2602.22480": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "面向包含随机模型调用的 agent 软件，建立可比较的评测环境，考察代码 agent 能否诊断并有效改进另一个 agent 的运行框架。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2602.22480"
}
]
},
"2606.04455": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "评测语言模型能否自主开发有竞争力的 agent 系统，考察系统设计能力、设计稳定性及对评测边界的遵守。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04455"
}
]
},
"2608.06301": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "评测模型在评估昂贵且有随机性的条件下，能否有效优化既有运行框架，并区分修改者、运行工具和起点对收益的影响。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.06301"
}
]
},
"2608.10178": {
"primary": "evaluation",
"branch": "attribution",
"related": [],
"question": "研究进化后的编程框架究竟学到了什么：通用执行策略、特定语言的工程知识，还是对某个模型弱点的补偿，以及哪些部分能够迁移。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.10178"
}
]
},
"2505.11942": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 在连续、相互依赖的任务中，能否积累并迁移知识和技能，而非每次从零处理独立任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.11942"
}
]
},
"2507.05257": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 逐步接收信息时的四种记忆能力：准确检索、从新信息学习、理解长期关联，以及选择性遗忘。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2507.05257"
}
]
},
"2508.19005": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "面向动态生活环境中的长期成长，研究 agent 能否把持续经历转化为记忆、可迁移技能和内化后的模型能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2508.19005"
}
]
},
"2510.17281": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测持续服务用户的系统能否真正从用户反馈中学习，改善后续回应，而不只是准确记住历史对话。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2510.17281"
}
]
},
"2511.20857": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 在连续任务中主动整合、修订和复用经验的能力，关注记忆是否实际支持后续推理和行动。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2511.20857"
}
]
},
"2604.17308": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 能否从零发现并持续维护有用的技能库，区分“会调用现成技能”与“能从经历中学出技能”。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.17308"
}
]
},
"2604.20087": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测自动技能学习是否产生真实可用的能力，区分技能内容质量、执行过程质量与最终任务成功。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.20087"
}
]
},
"2605.18421": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测不同内容和保留时长的记忆分别在哪些任务条件下有效，以及其收益能否超过直接使用长上下文。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.18421"
}
]
},
"2606.05661": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 在具有共同规律的连续任务中，能否因经历而提升能力，并把这种学习收益与基础模型本来就强区分开。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.05661"
}
]
},
"2607.05202": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测 agent 能否从既往任务学会搜索、调试等可复用操作方法，并将其迁移到需要这些能力的新任务。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05202"
}
]
},
"2608.00155": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测自进化 agent 在同域连续任务与跨域混合任务流中的可靠性，考察改进方法是否依赖特定模型或任务顺序。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.00155"
}
]
},
"2608.01149": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测先前经验怎样帮助或干扰后续能力，关注经验顺序、迁移收益以及新经历导致的遗忘。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.01149"
}
]
},
"2608.03874": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测持续任务中的能力提升是否来自可复用技能的形成，还是主要依赖近期上下文，并比较两种适应方式的适用条件。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03874"
}
],
"name": "ContinualSkillBench",
"contrast": "把显式技能学习与保留历史上下文对照，检查提炼技能本身的额外作用。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03874"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.03874"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2608.03874#S3.SS5"
}
]
},
"2608.04003": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测持久经验的保存、检索和更新是否真正导致后续能力提升，使观察到的涨分能对应到实际起作用的学习环节。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.04003"
}
],
"name": "PAST-Bench",
"contrast": "控制经验保存与使用环节，检查后期提升能否归因于持久经验。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.04003"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.04003"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.04003#S3.SS1"
}
]
},
"2608.06144": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "面向专业金融工作，评测 agent 能否跨案例迁移业务经验，并在开放产物与多维质量要求下持续改善表现。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.06144"
}
]
},
"2608.03764": {
"primary": "evaluation",
"branch": "transfer",
"related": [],
"question": "评测企业工作流 agent 能否从既往案例学到隐含业务规则，并在新案例中组合运用这些规则，而非只复用旧题答案。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03764"
}
]
},
"2606.17546": {
"primary": "evaluation",
"branch": "trajectory",
"related": [],
"question": "评测 agent 框架的连续更新是否产生可复用的能力提升，同时区分新任务泛化、旧能力遗忘、执行成本与短期过拟合。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17546"
}
],
"name": "SEAGym",
"contrast": "跟踪更新过程中泛化、遗忘与成本；关注整个学习过程，不只看最好一次。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17546"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.17546"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.17546#A1.SS1"
}
]
},
"2607.05155": {
"primary": "evaluation",
"branch": "trajectory",
"related": [],
"question": "研究模型部署后从真实环境交互中学习的速度与规模规律，考察长时间试验怎样转化为能力增长。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.05155"
}
]
},
"2608.31100": {
"primary": "evaluation",
"branch": "trajectory",
"related": [
"feedback",
"curriculum"
],
"question": "评测 agent 在拿不到环境真实判分的探索阶段，能否自主试验、准确判断自身表现，并把经历转化为更严格测试中的能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.31100"
}
],
"name": "S³Gym",
"contrast": "拆测自主探索、自我判断与经验利用；比较历史、总结记忆、参数训练如何转化为提升。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.31100"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.31100"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31100#S4.SS4"
}
]
},
"2609.08175": {
"primary": "evaluation",
"branch": "trajectory",
"related": [
"improver",
"harness"
],
"question": "研究冻结模型的框架自进化在什么条件下能可靠改善总体表现、控制原有能力退化，以及生成、验收和后续进步各自受到什么限制。",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§3，p4–7：改进目标与生成概率",
"url": "https://arxiv.org/pdf/2609.08175#page=4"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
],
"name": "Safe Harness Self-Evolution",
"contrast": "分开分析有益修改是否存在、能否生成、能否可靠验收，以及后续能否继续进步。",
"contrastSources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§3，p4–7：改进目标与生成概率",
"url": "https://arxiv.org/pdf/2609.08175#page=4"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
"2603.08640": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测 agent 能否自主完成模型后训练，把编程与实验能力转化为另一个模型的实际能力提升。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.08640"
}
],
"name": "PostTrainBench",
"contrast": "给定模型与算力后，检查 agent 能否完成有效后训练；直接测 AI 研发能力。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2603.08640"
},
{
"label": "§1",
"url": "https://arxiv.org/html/2603.08640#S1.SS0.SSS0.Px2"
}
]
},
"2604.10547": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测 agent 能否设计、调试并运行完整强化学习流程，尤其区分会做监督训练与能稳定完成在线交互学习。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2604.10547"
}
]
},
"2606.05080": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测 agent 面对已有可用但次优的工程产物时，能否在长时间实验中持续取得可测量的改进，而非只生成一个初始方案。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.05080"
}
]
},
"2505.19955": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测 agent 能否自主完成可信的开放机器学习研究，覆盖想法、方案、真实实验和论文，而不仅是生成看似合理的研究文本。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2505.19955"
}
]
},
"2605.08678": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测 agent 能否发明跨设置可泛化、扩大规模后仍有效的机器学习方法，区分方法创新与应用已有技术或局部调参。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.08678"
}
],
"name": "MLS-Bench",
"contrast": "进一步要求方法跨设置、扩大规模后仍有效，区分方法创新与局部调参。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.08678"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.08678"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2605.08678#S3.SS2.SSS1"
}
]
},
"2608.17271": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "评测逐步减少人类方法与步骤指导时，agent 能否自主选择研究方法、完成科研项目并产出可验证的新结果。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.17271"
}
]
},
"rsi-exam": {
"primary": "evaluation",
"branch": "research",
"related": [
"harness",
"improver"
],
"question": "评测研究 agent 在有限时间内，能否从可运行但较弱的起点出发，自主开展多轮实验，形成更有效、能泛化的方法或运行框架。",
"sources": [
{
"label": "官方报告 · 研究目标",
"url": "https://rsi-exam.ai/blog.html#introduction"
}
],
"name": "RSI-Exam",
"contrast": "从可运行弱方法出发做多轮研究，再评交付物；把已知研究目标下的改进作为评测对象。",
"contrastSources": [
{
"label": "官方报告 · 研究目标",
"url": "https://rsi-exam.ai/blog.html#introduction"
},
{
"label": "官方报告 · 结果与泛化分析",
"url": "https://rsi-exam.ai/blog.html#results"
},
{
"label": "官方报告 · 被测模型与执行框架",
"url": "https://rsi-exam.ai/blog.html#models-harnesses"
}
]
},
"icoder-27b": {
"primary": "evaluation",
"branch": "research",
"related": [],
"question": "研究人类把研发经验预先写成 Research Skills 后，agent 能否自主选择和修订实验，开发有竞争力的 RTL／GPU kernel 模型。",
"sources": [
{
"label": "摘要与 §1，PDF p1–3",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=1"
},
{
"label": "§3 Research Skills，PDF p5–6",
"url": "https://huggingface.co/i-Coder/iCoder-27B/resolve/main/Coder_Tech_Report.pdf#page=5"
}
]
},
"2609.11873": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "为 RSI 建立按自主程度递进的发展路线，分析不同任务场景需要什么能力，以及现有系统距离更自主的改进还差什么。",
"sources": [
{
"label": "§2 RSI 定义与自主程度",
"url": "https://arxiv.org/html/2609.11873#S2"
}
]
},
"data-centric-rsi-survey": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "以数据相关对象为中心，整理评价信号如何经过可信度判断和干预决策，形成可持续的模型改进。",
"sources": [
{
"label": "§1–2 引言、定义与分类",
"url": "https://www.preprints.org/manuscript/202609.0154"
}
]
},
"ai4ai-survey": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "研究怎样界定和衡量 AI 将改进想法推进到可靠验证结果的能力，并区分一次自改进与改进机制本身的递归变化。",
"sources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
],
"name": "AI4AI Survey",
"contrast": "把研究想法、执行和验证放在同一视野下，区分一次改进与递归改变。",
"contrastSources": [
{
"label": "官方项目页：2026 年 8 月综述",
"url": "https://simpleagentlab.com/ai4ai/"
},
{
"label": "论文 §2、Appendix A：定义与覆盖范围",
"url": "https://www.preprints.org/manuscript/202608.2108"
}
]
},
"shiyu-rsi-what-evolves": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "中文入门与交叉阅读笔记：以 Model、Harness、Artifact 三层地图组织案例，对照 Shilong Liu、Lilian Weng 与知乎文章的不同视角。",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
],
"name": "Shiyu：什么在进化",
"contrast": "用 Model、Harness、Artifact 区分修改对象；可与这里的问题地图交叉阅读。",
"contrastSources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
"shiyu-rsi-loop-closes": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "基于 Melon 长文的中文学习笔记，梳理可修改范围，并讨论怎样设计能归因、能复用、能验证递归的研究系统。",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
],
"name": "Shiyu：递归如何闭环",
"contrast": "讨论可修改范围与可验证的递归实验；提供研究设计的阅读入口。",
"contrastSources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
"liu-self-evolving-taxonomy": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "按产物优化、harness 自改进、无标准答案的模型学习，给出领域分类地图。",
"sources": [
{
"label": "Models, Harness, and Artifacts；Agent Harness Self-improvement；A Blurred Boundary",
"url": "https://lsl.zone/blog/2026/a-taxonomy-of-self-evolving-agents/"
}
]
},
"weng-harness-engineering": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "从工程设计和研究路线解释 harness 如何支持自改进，而不只是列举提示技巧。",
"sources": [
{
"label": "Harness Design Patterns；Harness Optimization；Harness Layer vs Core Intelligence?",
"url": "https://lilianweng.github.io/posts/2026-07-04-harness/"
}
]
},
"zhihu-self-evolving-guide": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "作为中文案例综述的原始阅读入口收录；本站简介仅依据 Shiyu 的引用与转述。",
"sources": [
{
"label": "上篇：来源与致谢、§3、§4.8、§6（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
"melon-self-evolve-rsi": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "收录下篇学习笔记的思想来源：按可修改范围组织研究，并讨论服务化的实验架构。",
"sources": [
{
"label": "下篇：来源与致谢、定义、Everything as Service（Shiyu 转述）",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
"weng-prompt-engineering": {
"primary": "perspectives",
"branch": "maps",
"related": [],
"question": "提示工程背景读物，为理解 harness 中的 prompt 组件提供基础。",
"sources": [
{
"label": "正文目录：提示、示例与推理方法",
"url": "https://lilianweng.github.io/posts/2023-03-15-prompt-engineering/"
}
]
},
"openai-research-acceleration": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "以内部使用记录说明 coding agent 如何参与 AI 研发，测量使用量、任务类型、完成情况及人类干预。",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
"minimax-m27-blog": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "官方研发案例：内部 M2.7 参与研究工作，并更新技能、记忆和研究 agent 的运行框架。",
"sources": [
{
"label": "Building an agent for model self-evolution",
"url": "https://www.minimax.io/news/minimax-m27-en"
}
]
},
"anthropic-when-ai-builds-itself": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "Anthropic 用研发记录、员工调查和会话评审介绍 AI 如何参与构建后续模型；本期收录 9 月 18 日更新。",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
],
"name": "Anthropic：When AI builds itself",
"contrast": "内部使用和会话评审说明 AI 参与研发的方式；观察性证据不等于自主研发能力证明。",
"contrastSources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
"ibm-rsi-serious-question": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "IBM Think 采访研究者，对照 coding agents、AIDE² 与前沿实验室的公开说法。",
"sources": [
{
"label": "What changed；Testing pieces of the idea；The safety gap；Still an open question",
"url": "https://www.ibm.com/think/news/why-recursive-self-improvement-ai-serious-question"
}
]
},
"2609.19203": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "提出 FMOS：像操作系统管理硬件一样，统一管理模型调用、记忆、资源分配和验证。",
"sources": [
{
"label": "引言、§3–4、附录 A–C、作者机构",
"url": "https://arxiv.org/html/2609.19203v1"
}
]
},
"reef": {
"primary": "perspectives",
"branch": "practice",
"related": [],
"question": "提供可复用基础设施，让已有 agent 从交互记录学习，并将通过评价的参数或 harness 更新交付给后续请求。",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
],
"name": "Reef",
"contrast": "提供交互记录、训练和更新交付接口；适合研究工程复用，但仓库本身不是收益证据。",
"contrastSources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
},
{
"label": "README：Recipes and examples",
"url": "https://github.com/Human-Agent-Society/reef#recipes-and-examples"
}
]
},
"2509.19349": {
"primary": "perspectives",
"branch": "programs",
"related": [],
"question": "面向科学发现与算法优化，研究如何降低程序进化的试验成本，同时保留开放探索和发现高质量新解的能力。",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2509.19349"
}
],
"name": "ShinkaEvolve",
"contrast": "通过程序进化寻找更好的任务解，强调样本效率与开放探索；需另查搜索器是否也在变化。",
"contrastSources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2509.19349"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2509.19349"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
}
]
},
"deepmind-alphaevolve-blog": {
"primary": "perspectives",
"branch": "programs",
"related": [],
"question": "官方案例介绍：Gemini 提出算法代码，自动评估器验证，进化搜索继续改进有潜力的候选。",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
],
"name": "AlphaEvolve 官方博客",
"contrast": "模型生成算法代码、可执行评估器筛选、继续进化；优化任务程序不自动等同自修改 agent。",
"contrastSources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
}
},
"relations": [
{
"kind": "递进问题",
"from_": "feedback",
"to": "experience",
"text": "这次改对了，下一次为何还犯同样的错？把本次反馈变成可保留、可复用的经验。",
"papers": [
"2303.17651",
"2303.11366",
"2308.10144"
]
},
{
"kind": "递进问题",
"from_": "experience",
"to": "harness",
"text": "经验已经存下，但系统不会检索或执行时，继续加内容不够，需要修改使用经验的机制。",
"papers": [
"2510.04618",
"2512.18746",
"2602.07755"
]
},
{
"kind": "并行选择",
"from_": "experience",
"to": "weights",
"text": "同一经历可以留在外部，也可以训练进参数；选择取决于迁移、使用成本与模型是否能学会。",
"papers": [
"2506.10943",
"2608.31100"
]
},
{
"kind": "双向配合",
"from_": "harness",
"to": "weights",
"text": "框架影响训练轨迹，训练又改变适合模型的框架。两侧需要交替适配，未必各自单独最优就整体最优。",
"papers": [
"2609.00196",
"2609.17523"
]
},
{
"kind": "递进问题",
"from_": "harness",
"to": "improver",
"text": "系统变好了，但下一轮修改仍低效：需要改进诊断、搜索或编辑能力，而非只增加候选数。",
"papers": [
"2603.19461",
"2608.02276"
]
},
{
"kind": "共同前提",
"from_": "curriculum",
"to": "evaluation",
"text": "练习由系统自己选择时，自测更容易偏离真正目标。学习信号与最终能力评测需要分开设计。",
"papers": [
"2608.31111",
"2608.31100"
]
}
],
"eras": [
{
"period": "理论起点",
"text": "先问自修改是否可能、怎样定义收益与搜索效率。",
"papers": [
"cs-0309048",
"cs-0207097"
]
},
{
"period": "2023",
"text": "当前答案的修订，与跨尝试、跨任务的经验积累并行出现。",
"papers": [
"2303.17651",
"2303.11366",
"2305.16291"
]
},
{
"period": "2024–2025",
"text": "执行流程与系统代码成为搜索对象，记忆管理和自建训练材料进一步展开。",
"papers": [
"2408.08435",
"2505.22954",
"2506.10943"
]
},
{
"period": "2026 · 本库观察",
"text": "问题细化到修改者学习、系统与参数协同、宽泛目标，以及公平和持续的评测。",
"papers": [
"2608.02276",
"2609.00196",
"2608.31111",
"2607.12227"
]
}
],
"directions": [
{
"id": "diagnosis",
"title": "先分清该改记忆、harness，还是参数",
"question": "同一种失败，怎样选择成本最低且能迁移的修复位置？",
"basis": "SIA 开放两类更新，HarnessFix 强调故障定位，WHALE 展示两侧可能互相限制。下一步可把“选择改哪里”单独检验。",
"papers": [
"2605.27276",
"2606.06324",
"2609.00196"
],
"groups": [
"harness",
"weights",
"feedback"
],
"experiment": "固定执行模型、反馈和总预算；对同一失败集比较补技能、修代码、训练参数以及选择策略。开发集选版本，独立任务测成功率、成本和旧能力退化。",
"baseline": "固定只改一侧；随机选位置；把全部预算给当前最强方案。",
"falsifier": "选择策略若不能超过同预算的固定最佳方案，就不能声称学会了更好的更新分配。"
},
{
"id": "feedback-value",
"title": "研究哪种反馈真正值得额外成本",
"question": "总分、错误定位、成功示范分别在什么失败上最有用？",
"basis": "GEPA、DemoEvolve 和 Harness-R1 分别突出轨迹反思、示范和补丁实际收益；这些信息来源与成本不同。",
"papers": [
"2507.19457",
"2605.24539",
"2608.02276"
],
"groups": [
"feedback",
"harness",
"improver"
],
"experiment": "固定修改者与任务，对照只给结果、给轨迹、给诊断、给示范。把生成或标注反馈的成本计入总预算，再测未见题上的有效补丁比例。",
"baseline": "同预算多次尝试；同样长度但无关的反馈；正确性已知的诊断上界。",
"falsifier": "额外反馈若只提高开发题分数，或收益被其成本抵消，就不构成更有效的进化。"
},
{
"id": "memory-transfer",
"title": "证明经验真的可复用，而不是换了一种长上下文",
"question": "何时值得把原始轨迹提炼成 skill？何时提炼反而丢信息？",
"basis": "ExpeL、ACE 研究经验表示，ContinualSkillBench 与 S³Gym 提供与保留历史相比较的评测视角。",
"papers": [
"2308.10144",
"2510.04618",
"2608.03874",
"2608.31100"
],
"groups": [
"experience",
"evaluation"
],
"experiment": "在共享能力、但实例和表面措辞不同的任务流上，比较原始历史、检索轨迹、文字技能和步骤图；匹配上下文与整理预算，记录实际调用和错误使用。",
"baseline": "不积累经验；直接保留历史；关闭检索或打乱经验关联。",
"falsifier": "去掉技能结构仍同样有效，说明收益可能来自额外信息，而不是所声称的表示设计。"
},
{
"id": "goal-gap",
"title": "让自建练习与真正目标对齐",
"question": "系统怎样发现“自测变好，但目标能力没变好”？",
"basis": "Aspire 暴露目标解释和数据失配，DiagEvo 从失败生成课程；两者提示可把课程与目标能力的关联单独研究。",
"papers": [
"2608.31111",
"2609.00768"
],
"groups": [
"curriculum",
"evaluation"
],
"experiment": "给定宽泛目标与可访问的数据池，比较固定课程、失败驱动课程与主动诊断课程。用独立开发任务调策略；另保留不同题型的最终测试，禁止据其成绩选课。",
"baseline": "随机同量数据；人工固定课程；只反复练失败题。",
"falsifier": "自建评分持续上升而独立目标测试不变时，应报告课程失配，不能只展示自测曲线。"
},
{
"id": "meta-transfer",
"title": "单独证明改进者更会改了",
"question": "新系统得分更高，是执行者变强，还是提出改进的能力变强？",
"basis": "Hyperagents、Harness-R1 与 Updating ≠ Benefit 分别提供可修改程序、学习编辑者和角色分离的切口。",
"papers": [
"2603.19461",
"2608.02276",
"2605.30621"
],
"groups": [
"improver",
"evaluation"
],
"experiment": "冻结同一批新起点和未见任务，把旧、新改进者分别接上去；给相同预算，比较发现有效更新的速度、比例和累计收益，并移除其保存的任务专用答案。",
"baseline": "固定强模型编辑者；随机或固定搜索策略；只增强执行者。",
"falsifier": "若优势只在原任务或搭配原执行者时存在，就不能直接推到可迁移的递归改进能力。"
},
{
"id": "long-run",
"title": "把多轮收益、遗忘和成本一起研究",
"question": "什么时候该接受修改、保留分支或停止搜索？",
"basis": "SEAGym 追踪持续变化，Safe Harness Self-Evolution 拆分生成与验收困难，SoL-Pi 将效率纳入目标。",
"papers": [
"2606.17546",
"2609.08175",
"2609.20519"
],
"groups": [
"evaluation",
"harness"
],
"experiment": "报告每轮独立审计结果与累计成本；审计结果不回传修改者。改变验收样本量与候选保留策略，记录误接受、误拒绝和旧任务退化。",
"baseline": "固定预算停止；只保留最高开发分；同预算直接重试。",
"falsifier": "只挑最好的测试点、忽略前期搜索成本，无法说明长期改进更有效。"
}
],
"readingPaths": [
{
"title": "先读懂 harness 进化",
"why": "从自动设计到自身适配，再检查涨分是否经得起对照。",
"papers": [
"2408.08435",
"2505.22954",
"2606.09498",
"2607.12227"
]
},
{
"title": "理解“从经历中学”",
"why": "从文字教训到经验迁移，再问提炼是否比直接留历史更有用。",
"papers": [
"2303.11366",
"2308.10144",
"2510.04618",
"2608.03874"
]
},
{
"title": "理解递归究竟多了一层什么",
"why": "从修改规则、可修改程序到训练编辑者，最后拆开修改与受益能力。",
"papers": [
"2309.16797",
"2603.19461",
"2608.02276",
"2605.30621"
]
},
{
"title": "寻找下一步研究问题",
"why": "分别检查目标自主性、更新位置、改进者迁移和可靠验收。",
"papers": [
"2608.31111",
"2609.00196",
"2603.19461",
"2609.08175"
]
}
],
"tree": {
"title": "RSI：系统怎样从经历中改进，并提高继续改进的能力？",
"subtitle": "沿“目标 → 瓶颈 → 方法分歧 → 论文”展开。树枝表示问题分解，跨枝联系表示配合；不表示发表先后或直接继承。",
"pillars": [
{
"id": "signal",
"title": "学习信号从哪来？",
"why": "只让模型反复尝试，不会自动得到值得学的任务和可信反馈。",
"groups": [
"feedback",
"curriculum"
],
"insight": "练习与评分可以由系统组织，但仍需独立检查是否对准真正的能力目标。",
"evidence": [
"2305.11738",
"2608.31111",
"2608.31100"
]
},
{
"id": "retain",
"title": "改进怎样留下来？",
"why": "当前题做对，只说明这次成功；还要解释留下了什么、下次怎样用。",
"groups": [
"experience",
"harness",
"weights"
],
"insight": "记忆、系统代码与参数是互补选择。决定改哪里，应看失败原因与使用成本。",
"evidence": [
"2308.10144",
"2512.18746",
"2609.00196"
]
},
{
"id": "improve",
"title": "怎样更会产生改进？",
"why": "把修改规则、修改模型或优化程序也开放为学习对象。",
"groups": [
"improver"
],
"insight": "执行能力和提出改进的能力需要分开测；新规则、新修改者或新优化程序应在新任务上验证。",
"evidence": [
"2603.19461",
"2608.02276",
"2605.30621"
]
},
{
"id": "evaluate",
"title": "如何证明真的进步？",
"why": "评测贯穿前三条，区分更多尝试、局部适应与能力提升。",
"groups": [
"evaluation"
],
"insight": "控制预算、反馈和数据权限后，再讨论迁移、持续性与研发能力。评测贯穿其他三条主干。",
"evidence": [
"2607.12227",
"2606.17546",
"rsi-exam"
]
}
],
"nodes": {
"feedback": {
"kicker": "从成败判断到可行动诊断",
"fork": "自评 / 外部核验 / 过程监督与示范",
"insight": "“是否正确”和“哪里该改”是两种信息。要比较反馈来源、定位能力与获取成本。",
"papers": [
"2305.11738",
"2507.19457",
"2605.24539"
]
},
"curriculum": {
"kicker": "从人给题到自主选择学习目标",
"fork": "失败驱动课程 / 新挑战生成 / 宽泛目标解释",
"insight": "在自建题上变好，仍可能没有练到目标能力。选题收益必须在独立目标任务上检验。",
"papers": [
"2609.00768",
"2608.31111"
]
},
"experience": {
"kicker": "从保留历史到可复用、可维护的经验",
"fork": "教训与技能 / 组织与检索 / 学习记忆机制",
"insight": "有经验、找得到经验、正确使用经验需要分别验证；技能收益要超过直接保留历史。",
"papers": [
"2308.10144",
"2512.18746",
"2608.03874"
]
},
"harness": {
"kicker": "从人工设计到诊断与搜索执行系统",
"fork": "设计空间 / 故障定位 / 搜索与验收 / 长期运行",
"insight": "开放更多代码只是扩大可选动作。真正的研究变量还包括诊断信息、搜索起点和接受修改的规则。",
"papers": [
"2408.08435",
"2606.06324",
"2607.13683",
"2608.27311"
]
},
"weights": {
"kicker": "从外部指导到模型学习，再到两侧协同",
"fork": "组织训练信号 / 学习读写策略 / 交替更新",
"insight": "新模型会改变合适的 harness，新 harness 会改变训练材料。联合更新的收益需与同预算单侧更新比较。",
"papers": [
"2506.10943",
"2609.00196",
"2609.17523"
]
},
"improver": {
"kicker": "从改任务系统到改“怎样改”的方法",
"fork": "修改文字规则 / 训练编辑者 / 修改优化程序",
"insight": "把新旧修改者接到相同未见任务和相同起点上，才能区分执行能力提升与改进能力提升。",
"papers": [
"2603.19461",
"2608.02276",
"2605.30621"
]
},
"evaluation": {
"kicker": "贯穿全树：把涨分变成可解释的证据",
"fork": "收益归因 / 跨任务迁移 / 全过程曲线 / AI 研发",
"insight": "先排除“只是多试了几次”，再检验学习、迁移和修改者提升；三种结论需要不同对照。",
"papers": [
"2607.12227",
"2608.04003",
"2606.17546"
]
},
"perspectives": {
"kicker": "概念背景与工程入口",
"fork": "综述 / 产业观察 / 仓库 / 相关程序搜索",
"insight": "用于形成问题和搭建实验；材料的观点与接口能力不替代实验验证。",
"papers": [
"ai4ai-survey",
"reef"
]
}
},
"bridges": [
{
"from": "experience",
"to": "harness",
"label": "问题深入",
"text": "经验积累遇到检索或执行瓶颈 → 将管理和使用经验的机制也纳入修改。",
"papers": [
"2510.04618",
"2512.18746"
]
},
{
"from": "harness",
"to": "weights",
"label": "双向配合",
"text": "harness 产生训练轨迹 ↔ 模型更新改变适合它的执行方式。两侧不是先后等级。",
"papers": [
"2609.00196",
"2609.17523"
]
},
{
"from": "harness",
"to": "improver",
"label": "增加一层研究对象",
"text": "会找到好系统 → 能否学到更有效的找法？需要另测下一轮改进效率与迁移。",
"papers": [
"2505.22954",
"2603.19461",
"2608.02276"
]
}
],
"evidenceSteps": [
{
"title": "当前任务变好",
"claim": "有了更好的答案或更充分的搜索结果。",
"check": "对照同预算重试；只凭这一层，不能推到跨任务学习。",
"papers": [
"2303.17651",
"2607.12227"
]
},
{
"title": "后续任务受益",
"claim": "保留的经验、代码或参数带来可复用能力。",
"check": "冻结更新后测未见任务，并消融保留的内容；检查遗忘与成本。",
"papers": [
"2308.10144",
"2608.04003",
"2606.17546"
]
},
{
"title": "产生改进的能力提升",
"claim": "同预算下，更快或更可靠地找到有效更新。",
"check": "迁移修改者，固定执行起点；与只增强执行者的情况分开。",
"papers": [
"2603.19461",
"2605.30621"
]
}
]
}
},
"reviewTree": {
"version": 2,
"updated": "2026-09-30",
"title": "AI 怎样把经历转成可迁移的能力，并提高后续改进效率？",
"note": "按研究问题分解；四条主干并列，评测贯穿其余三条。连线表示问题关系，不表示论文的直接继承或必经阶段。",
"branches": [
{
"id": "signal",
"number": "01",
"title": "学习信号从哪来？",
"subtitle": "选择值得学的任务，获得可信且可用的反馈。",
"nodes": [
"feedback",
"curriculum"
]
},
{
"id": "retain",
"number": "02",
"title": "改进怎样留下来？",
"subtitle": "将一次经历转成后续任务能复用的能力。",
"nodes": [
"experience",
"harness",
"weights"
]
},
{
"id": "improve",
"number": "03",
"title": "怎样更会产生改进？",
"subtitle": "把修改规则、修改模型或优化程序也开放为学习对象。",
"nodes": [
"rules",
"editor",
"optimizer"
]
},
{
"id": "evaluate",
"number": "04",
"title": "如何证明真的进步？",
"subtitle": "评测贯穿前三条，区分更多尝试、局部适应与能力提升。",
"nodes": [
"attribution",
"continual",
"research"
]
}
],
"nodes": [
{
"id": "feedback",
"number": "1.1",
"title": "反馈怎样既可信，又能指导修改？",
"gap": "自我批评可能重复原来的错误；最终成败又不能直接定位原因。",
"chapter": "feedback",
"preview": [
"2303.17651",
"2305.11738",
"2507.19457"
],
"insight": "结果核验、错误定位和修改建议承担不同作用。比较方法时，要固定修改者可见的信息及获取反馈的成本。",
"routes": [
{
"title": "从自我批评到外部核验",
"approach": "Self-Refine 让同一模型点评并改写；CRITIC 引入搜索和代码执行。Cannot Self-Correct 检查无外部正确性反馈时的推理纠错能力。",
"papers": [
"2303.17651",
"2305.11738",
"2310.01798"
]
},
{
"title": "把结果反馈变成修改方向",
"approach": "步骤监督提供中间推理标签；TextGrad 把文字反馈传到待改变量；GEPA 利用轨迹反思修改提示并保留互补候选。",
"papers": [
"2305.20050",
"2406.07496",
"2507.19457"
]
},
{
"title": "反馈稀缺或不固定时怎么办",
"approach": "DemoEvolve 在稀疏游戏奖励之外引入成功示范；Red Queen Gödel Machine 则把评审者也纳入改进。示范质量和评分稳定性成为新问题。",
"papers": [
"2605.24539",
"2606.26294"
]
}
]
},
{
"id": "curriculum",
"number": "1.2",
"title": "下一轮该练什么？",
"gap": "人工题库限制持续学习；自建练习又可能偏离真正的能力目标。",
"chapter": "curriculum",
"preview": [
"2609.00768",
"2608.31111"
],
"insight": "能生成练习，不等于能选择值得学的练习。需要独立目标任务来检验课程是否对准能力缺口。",
"routes": [
{
"title": "根据经历与错误组织学习",
"approach": "AgentEvolver 提高交互任务和轨迹的利用效率；DiagEvo 从失败历史诊断错误，再构造针对性的课程。",
"papers": [
"2511.10395",
"2609.00768"
]
},
{
"title": "让任务与数据本身成为研究对象",
"approach": "BigBang 合成更难且可验证的任务；RSIBench-Data 与 Curation-Bench 用受控设置评估数据研究决策。",
"papers": [
"endlessfrontier.tech-assets-paper.pdf",
"2607.25886",
"2606.04261"
]
},
{
"title": "从宽泛目标建立学习方案",
"approach": "Aspire 测试目标解释、选数据和自测能否带来目标能力提升；S³Gym 把自我试验、判断经历和利用经历拆开评估。",
"papers": [
"2608.31111",
"2608.31100"
]
},
{
"title": "证据变化时，怎样调整研究方向",
"approach": "Argus 将用户意图与可修订的操作目标、约束及验证条件分开，保留项目证据与被否定的路线，并在指定节点引入人的决策。这是开放研究的目标调整，不只是生成练习题。",
"papers": [
"2608.05144"
]
}
]
},
{
"id": "experience",
"number": "2.1",
"title": "一次经历怎样帮助后续新任务？",
"gap": "完整轨迹混杂题目细节；过度概括的教训又无法指导执行。",
"chapter": "experience",
"preview": [
"2308.10144",
"2510.04618"
],
"insight": "经验的内容、表示、检索和管理机制是不同变量。应比较新任务收益，而不只看同题重试成功。",
"routes": [
{
"title": "从同题反思到跨题经验",
"approach": "Reflexion 保存试错教训供后续尝试使用；ExpeL 从训练任务提取一般规则与成功示例，供新题检索。",
"papers": [
"2303.11366",
"2308.10144"
]
},
{
"title": "选择合适的经验表示",
"approach": "Voyager 保存可组合的代码技能；AWM 提炼网页操作流程；Memento-Skills 在执行中选择、修订和扩展技能。 Alita-G 把成功经历封装为带参数、可检索的工具，供后续任务调用。",
"papers": [
"2305.16291",
"2409.07429",
"2603.18743",
"2510.23601"
]
},
{
"title": "让技能更新稳定，并适配当前任务状态",
"approach": "Evo-Harness 从历史轨迹更新文字操作说明；SkillOpt 限制编辑幅度并以留出验证决定接受；Recuris 用工作记忆跟踪进度、选择经验，再对失败组件作局部更新。",
"papers": [
"2608.15071",
"2605.23904",
"2608.24876"
]
},
{
"title": "维护内容，或改进管理程序",
"approach": "ACE 增量维护经验手册；Procedural Graphs 组织带条件的步骤；SELF-INDEX 优化检索表示。MemEvolve 与 ALMA 进一步改变记忆管理设计。",
"papers": [
"2510.04618",
"2609.09153",
"2609.19656",
"2512.18746",
"2602.07755"
]
}
]
},
{
"id": "harness",
"number": "2.2",
"title": "执行框架怎样适配任务与模型？",
"gap": "失败可能来自工具、上下文或执行逻辑；只改提示不一定触及原因。",
"chapter": "harness",
"preview": [
"2505.22954",
"2606.09498",
"2606.06324"
],
"insight": "开放编辑权限之后，真正的瓶颈分化为：诊断是否准确、搜索是否有效、验证是否可靠，以及运行成本是否可接受。",
"routes": [
{
"title": "开放人工固定的设计空间",
"approach": "DSPy 优化模块化程序；ADAS 搜索代码表达的 agent 设计；AFlow 搜索工作流。区别首先在于允许改变哪些结构。 Recursive Harness Self-Improvement 的编辑范围则是角色、通信和上下文管理的提示文本，不是可执行源码。",
"papers": [
"2310.03714",
"2408.08435",
"2410.10762",
"2607.15524"
]
},
{
"title": "改自身实现，或适配自身弱点",
"approach": "SICA 修改自身实现；DGM 保留多样历史版本继续探索；Self-Harness 根据不同执行模型的失败模式适配配置。",
"papers": [
"2504.15228",
"2505.22954",
"2606.09498"
]
},
{
"title": "强模型能否通过 harness 帮助弱模型",
"approach": "AI4AI at Test-Time 由强模型为冻结弱模型构建执行代码、路由与检查流程，研究无需训练弱模型参数的能力支持；这里的执行者与修改者是不同模型。",
"papers": [
"2608.12307"
]
},
{
"title": "把失败定位到可修改的机制",
"approach": "Meta-Harness 回看代码、分数与详细轨迹；AHE 检查组件影响；HarnessFix 将执行步骤与实现对齐，再作限定范围的修复。 HarnessCompass 结合诊断、避免写入题目专属信息的约束和分组件优化，减少过拟合与修改干扰。",
"papers": [
"2603.28052",
"2604.25850",
"2606.06324",
"2608.01918"
]
},
{
"title": "控制搜索与验证成本",
"approach": "HarnessBank 保留并重组多样候选；RobustSGPO 控制修改范围和操作；HarnessLens 选择相关验证任务；SoL-Pi 优化部署运行效率。",
"papers": [
"2607.13683",
"2609.09646",
"2608.27311",
"2609.20519"
]
},
{
"title": "从离线选版本走向运行中更新",
"approach": "PILOT 运行中纠偏并留下技能；Continual Harness 处理不重置环境的持续适应；Harness-of-Harness 面向多日开发。",
"papers": [
"2608.26530",
"2605.09998",
"2609.01481"
]
}
]
},
{
"id": "weights",
"number": "2.3",
"title": "何时改参数，何时改外部系统？",
"gap": "外部经验未必能被模型正确执行；参数训练也未必能修复系统设计问题。",
"chapter": "weights",
"preview": [
"2506.10943",
"2609.00196",
"2609.17523"
],
"insight": "参数与 harness 是互补的更新位置。联合更新需要解释选哪一侧、何时切换，以及两侧各自贡献多少。",
"routes": [
{
"title": "把经历变成有效训练信号",
"approach": "SEAL 生成微调材料并用更新后表现训练生成策略；SEED 从近期交互中提炼更具体的行动监督。",
"papers": [
"2506.10943",
"2607.14777"
]
},
{
"title": "训练外部经验的使用策略",
"approach": "EvoHarness-RL 学习外部状态的创建与读写；SkillRise 将整理技能对未来任务的收益纳入训练。",
"papers": [
"2608.05446",
"2607.26784"
]
},
{
"title": "研究模型与框架怎样配合",
"approach": "SIA 研究更新位置；WHALE 研究交替节奏；ScienceBuddy 衔接交互材料、harness 适配和参数训练。HELIX、Macaron-V1 的架构与闭环证据需分别看。",
"papers": [
"2605.27276",
"2609.00196",
"2609.17523",
"2608.13951",
"2608.09819"
]
},
{
"title": "把真实交互接入下一轮训练",
"approach": "NeoHorse-1 用 routing harness 记录能力需求与执行轨迹，经检查和标注后组织训练课程，再据能力评估调整数据配比；当前证据不包含 harness 代码自动进化或持续多轮递归收益。",
"papers": [
"2609.08183"
]
}
]
},
{
"id": "rules",
"number": "3.1",
"title": "指导修改的规则能否改进？",
"gap": "固定的反思或变异提示，会限制后续候选怎样被提出。",
"chapter": "improver",
"preview": [
"2309.16797",
"2607.05297"
],
"insight": "元提示变化只开放了改进机制的一部分；仍要检查候选生成、预算和接受规则中哪些保持固定。",
"routes": [
{
"title": "让修改规则与任务内容共同变化",
"approach": "Promptbreeder 同时改变任务提示和变异提示；MetaSkill-Evolve 区分任务技能与指导技能修订的说明，以不同节奏更新。",
"papers": [
"2309.16797",
"2607.05297"
]
}
]
},
{
"id": "editor",
"number": "3.2",
"title": "修改模型能否学会更有效地改？",
"gap": "执行任务的能力，与诊断错误、提出补丁的能力并不相同。",
"chapter": "improver",
"preview": [
"2608.02276",
"2607.21971"
],
"insight": "把旧、新修改者接到相同的新执行者与任务上，才能检查学到的是可迁移的修改能力。",
"routes": [
{
"title": "用实际改进效果训练修改者",
"approach": "Harness-R1 用补丁安装后的任务奖励训练编辑模型；MetaEvolve 训练利用反馈和历史方案开展优化的能力。 Frontis-MA1 训练机器学习工程中的程序修改操作，外层搜索框架仍固定。",
"papers": [
"2608.02276",
"2607.21971",
"2607.28568"
]
},
{
"title": "分开“会改”和“能受益”",
"approach": "Harness Updating Is Not Harness Benefit 独立配对执行者与修改者，避免将执行模型变强误算成修改者能力提升。",
"papers": [
"2605.30621"
]
}
]
},
{
"id": "optimizer",
"number": "3.3",
"title": "搜索与优化程序能否改进自己？",
"gap": "任务程序变好，并不自动意味着产生下一轮改进的方法也变好。",
"chapter": "improver",
"preview": [
"2310.02304",
"2603.19461",
"2609.14858"
],
"insight": "递归多出的检验对象是后续改进能力：新优化程序能否在新起点与新任务上，更高效地产生有效更新？",
"routes": [
{
"title": "开放优化程序本身",
"approach": "STOP 研究优化程序自修改；Hyperagents 同时开放任务程序与改进程序，考察修改机制的迁移。",
"papers": [
"2310.02304",
"2603.19461"
]
},
{
"title": "低成本试验探索策略",
"approach": "Dream-RSI 从累积发现树建立回放环境，在其中试验探索策略，再放回线上搜索；改变的主要是探索编排层。",
"papers": [
"2609.14858"
]
},
{
"title": "增加辅助层，是否真的更会改进",
"approach": "Metaⁿ 用固定操作逐层增加策略上下文与辅助代码，研究增加深度的作用；生成下一层的规则保持固定，不能只凭层数称修改规则也在进化。",
"papers": [
"2608.24735"
]
},
{
"title": "根据当前状态选择更新操作及顺序",
"approach": "MetaRSI-v1 调整数据、harness、参数更新的组合顺序及候选生成指令，关注下一步该怎样更新；API 模型设置只能更新数据和 harness。",
"papers": [
"metarsi-v1"
]
}
]
},
{
"id": "attribution",
"number": "4.1",
"title": "涨分来自哪里，能否迁移？",
"gap": "更多推理预算、反馈或开发题适应，都可能被误当成系统能力提升。",
"chapter": "evaluation",
"preview": [
"2607.12227",
"2608.09096"
],
"insight": "固定预算、反馈权限、执行模型和选版本数据，再检查未见任务，才能较清楚地解释收益来源。",
"routes": [
{
"title": "与直接重试、保留历史作对照",
"approach": "Rethinking 比较相近预算与反馈下的重试；ContinualSkillBench、PAST-Bench 检查提炼和使用持久经验的额外价值。",
"papers": [
"2607.12227",
"2608.03874",
"2608.04003"
]
},
{
"title": "控制起点、执行者与迁移对象",
"approach": "Evo-Bench 控制执行者与预算；HarnessDev 区分建设弱系统与改进已有系统；One Recipe, Many Harnesses 区分跨语言与跨模型迁移。",
"papers": [
"2608.09096",
"2609.01437",
"2608.10178"
]
}
]
},
{
"id": "continual",
"number": "4.2",
"title": "多轮改进是否持续且划算？",
"gap": "最佳单点成绩会掩盖遗忘、任务顺序影响及累积搜索成本。",
"chapter": "evaluation",
"preview": [
"2608.01149",
"2606.17546"
],
"insight": "应观察整个过程：新任务收益、旧任务保持、验收错误与累计成本，而不只比较最好的一次。",
"routes": [
{
"title": "在任务流中看完整过程",
"approach": "PATH-Bench 关注历史路径与干扰；AgentStream 比较同域与跨域任务流；SEAGym 跟踪泛化、遗忘和成本。",
"papers": [
"2608.01149",
"2608.00155",
"2606.17546"
]
},
{
"title": "分析何时还能可靠进步",
"approach": "Safe Harness Self-Evolution 分开分析有益候选、生成能力与有限数据验收；其理论保证依赖给定假设。",
"papers": [
"2609.08175"
]
}
]
},
{
"id": "research",
"number": "4.3",
"title": "会做 AI 研发，是否等于 RSI？",
"gap": "改进一个人类指定的对象，与回头增强研究者自身，是不同主张。",
"chapter": "evaluation",
"preview": [
"2603.08640",
"rsi-exam",
"2605.08678"
],
"insight": "先检查改进对象、目标和评价是否由人固定，再判断证据支持的是研发能力，还是自主递归改进。",
"routes": [
{
"title": "分清研发任务测到了什么",
"approach": "PostTrainBench 测后训练；RSI-Exam 测多轮可执行研究及交付物；MLS-Bench 检查研究方法跨设置和规模的有效性。",
"papers": [
"2603.08640",
"rsi-exam",
"2605.08678"
]
},
{
"title": "区分长期改好产物与研究者自身学习",
"approach": "AutoLab 给出可运行但次优的工程产物，在固定时间预算中观察反复测量与修改；执行 harness 不变。它检验持续研发能力，不能直接证明研究 agent 自身获得了新能力。",
"papers": [
"2606.05080"
]
}
]
}
],
"connections": [
{
"from_": "feedback",
"to": "harness",
"title": "从“有反馈”到“知道改哪里”",
"text": "成败标签能筛选候选；系统修复还需要将错误对应到执行机制。"
},
{
"from_": "experience",
"to": "weights",
"title": "从“存下经验”到“学会使用”",
"text": "技能、上下文和参数是互补路线；整理得好与执行得好要分别验证。"
},
{
"from_": "harness",
"to": "optimizer",
"title": "从“改好系统”到“更会改系统”",
"text": "后者新增的主张是改进能力可迁移，需要单独比较新旧修改机制。"
},
{
"from_": "curriculum",
"to": "attribution",
"title": "自主选题，需要独立检验方向",
"text": "自建练习上的提高，必须在不参与选版本的目标任务上检验。"
},
{
"from_": "curriculum",
"to": "optimizer",
"title": "先决定改什么，再决定怎样改",
"text": "自主选题、改数据、改框架与训练参数可以组合。应分别评价选择方向是否正确，以及选定方向后的改进效率。"
}
],
"defaultNode": "harness",
"boundary": {
"title": "产物优化：答案／程序变好，系统未必变强",
"chapter": "scope",
"description": "Self-Refine 修改当前输出；AlphaEvolve 搜索算法程序。ShinkaEvolve 既有程序优化，也有 agent harness 优化实验，应逐个实验判断修改对象。把任务产物改好是重要的并行路线，但单靠产物涨分，还不能证明执行者或修改者获得了可迁移的能力。",
"papers": [
"2303.17651",
"deepmind-alphaevolve-blog",
"2509.19349"
]
}
},
"fieldReview": {
"version": 2,
"updated": "2026-09-30",
"corpusUpdated": "2026-09-21",
"title": "RSI 研究综述：从反馈、经验与系统自改进，到改进者自身的学习",
"lead": "沿研究问题的变化串起方法：为什么需要它、不同论文怎样回应、已有证据支持到哪里。重点展开 harness 进化及其与记忆、参数学习和评测的关系。",
"scope": "叙述性综述，基于现有资料库选取代表工作；覆盖的文献更新至 2026-09-21。问题之间的联系与研究建议是本站综合分析，不声称穷尽文献或证明直接继承。部分关键论文在本轮重新核对原文，其余沿用单篇研究表的来源，具体实验缺项仍按原文披露范围说明。",
"chapters": [
{
"id": "scope",
"title": "从改好一个结果，到改进产生结果的系统",
"thesis": "领域的发展可以理解为：越来越多原本由人固定的环节，成为可研究、可修改的对象。",
"groups": [
"feedback",
"harness",
"improver"
],
"body": "### 起点：同样叫 self-improvement，实际在改什么？\n\n先固定一个具体问题：一个 agent 这次失败了。我们可以让它重写答案，也可以留下失败教训、修改工具接口，或者训练模型。它们都可能使某个分数上升，但得到的东西不同：更好的当前产物、一套后续能复用的执行方法，或者改变后的模型能力。若再修改负责诊断和提出更新的程序，研究对象就进一步变成了“产生改进的方法”。\n\n因此，读这组文献需要同时追问两件事：**改善留在哪里？它会帮助哪一次后续工作？** 改当前程序或答案，收益可以只属于这道题；保存技能或修改 harness，目标通常是帮助后续任务；改进修改者，则希望相同预算下更容易产生下一轮有效更新。这是本文的分析坐标，不把所有论文强行纳入同一种严格 RSI 定义。\n\n这里的 **harness** 指围绕模型组织任务执行的系统，包括提供哪些工具、怎样处理工具输出、怎样维护上下文、何时重试与结束。模型给出下一步建议，harness 决定这些建议怎样被执行、检查和接回模型。于是，“模型不会做”与“模型被现有执行方式限制”成为两个需要分开的解释。\n\n### 并行路线：把任务产物改好\n\n[AlphaEvolve](?paper=deepmind-alphaevolve-blog)由 Gemini 提出算法程序，用自动评估器运行、检验和评分，再从有潜力的候选继续搜索。它展示了算法改进怎样帮助数学、数据中心与训练基础设施；这些应用成果本身不证明搜索器也学会了更有效地改进自己。[〔1〕](#review-ref-1)\n\n[ShinkaEvolve](?paper=2509.19349)针对程序进化的评估成本，结合修改起点的选择、新颖性筛选和生成模型的选择，提高探索效率。它的实验既有算法程序，也有 AIME 解题用的 agent harness，因此不能把整篇论文简单标成“只改产物”。[〔2〕](#review-ref-2)\n\n这条路线与下文并行：**先看具体实验修改的对象，再判断是否改变了后续执行或改进机制。** 产物可能被系统重新采用，但“产物更好”和“系统具备可迁移的改进能力”仍需不同证据。\n\n### 从理论设想到可执行实验\n\n[Gödel Machines](?paper=cs-0309048)提出让系统在证明自修改有更高预期收益后切换程序。它把改进算法自身也开放为修改对象，但形式保证依赖公理、效用定义和证明条件。[DGM](?paper=2505.22954)则明确转向实证：冻结基础模型，修改 coding agent 的代码，用编程任务表现评价，并保留多样版本继续探索。这个转折改变了可操作性，也把可信评价、搜索预算和泛化问题带到了中心。[〔3〕](#review-ref-3)[〔4〕](#review-ref-4)\n\n| 本库中的时间线 | 研究对象怎样扩展 | 随之产生的下一步问题 |\n| --- | --- | --- |\n| 2023：Self-Refine、Reflexion、Voyager | 修订输出、保存教训、积累技能同时展开 | 同题成功能否变成跨题能力？ |\n| 2024：ADAS、AFlow | agent 结构和工作流进入自动搜索 | 预设模块是否限制可发现的系统？ |\n| 2025：SICA、DGM、SEAL、ACE | 实际系统代码、学习材料、长期上下文成为改进对象 | 怎样诊断、保留和评价更新？ |\n| 2026：本库收录的进一步分化 | 修改者学习、harness 与参数协同、自主选题和受控评测 | 增益来自哪里，能否持续、迁移并改善下一轮研发？ |\n\n表中是阅读线索。后文分别给出原文依据；时间相近不表示方法相同，问题递进也不表示后文一定直接继承前文。尤其是记忆、系统工程与参数训练，始终存在并行发展。\n\n### 一个重要分界：执行循环与改进循环\n\n执行循环围绕当前任务运行：读状态、调用工具、得到结果、决定下一步。改进循环围绕系统版本运行：收集多次执行的证据、提出修改、比较候选、保留更新。把执行过程拉长，或者多生成几个答案，并不自动改变改进循环。后面讨论“递归”时，还要进一步问：负责提出和选择更新的机制是否发生变化，以及这种变化有没有提高后续改进能力。"
},
{
"id": "feedback",
"title": "为什么“再检查一下”不够：反馈从自评走向外部证据与错误定位",
"thesis": "判断正确性、定位错误、给出可用的修改方向，是反馈中的三项不同作用。",
"groups": [
"feedback"
],
"body": "### 当前输出能不能靠自我批评变好？\n\n[Self-Refine](?paper=2303.17651)针对初稿质量不足，使用同一模型生成反馈并反复修订，不另训练模型。它把问题落在当前产物的改善。[Large Language Models Cannot Self-Correct Reasoning Yet](?paper=2310.01798)则专门检查缺少外部正确性反馈的推理纠错：不能把获得标准答案等额外帮助后的改善，算成模型单独自纠错的能力。二者并读时，应比较任务和反馈条件，不能把一个任务上的正面或负面结论外推到所有自我检查。[〔5〕](#review-ref-5)[〔6〕](#review-ref-6)\n\n[CRITIC](?paper=2305.11738)采取另一条路线：用搜索、代码执行等工具检查输出，再依据工具信息修订。关键变化是判断能接触外部证据，而非只重新调用原模型。这里也要继续分辨工具到底验证了什么：代码能够运行、数值检查通过、事实有来源，是不同强度的结论。[〔7〕](#review-ref-7)\n\n### 为什么正确／错误这个标签仍然不够？\n\n长程任务可能经过几十步后才失败。最终成败能筛选候选，却不直接说明是规划错误、工具参数错误、上下文丢失，还是验收步骤缺失。**正确性标签回答“结果怎么样”，诊断信息才开始回答“下一步改哪里”。** 当我们把自纠错推广到系统更新时，这个区别尤其重要。\n\n[Let’s Verify Step by Step](?paper=2305.20050)比较步骤监督与结果监督，研究中间推理信息怎样支持评分模型。[TextGrad](?paper=2406.07496)把自然语言反馈传递到系统中可优化的变量，研究多个组件怎样接受有针对性的修改信号。[GEPA](?paper=2507.19457)读取执行轨迹、利用语言反思修改提示，并保留互补候选。三者都利用比单一总分更丰富的信息，但分别涉及监督粒度、反馈分配和提示搜索；训练数据与运行时反馈条件不能混用。[〔8〕](#review-ref-8)[〔9〕](#review-ref-9)[〔10〕](#review-ref-10)\n\n### 反馈太稀疏时，是多试，还是引入示范？\n\n[DemoEvolve](?paper=2605.24539)把这个问题放在具体游戏条件下：Liar’s Dice 更容易通过自主对局获得奖励；Balatro 的有效奖励更稀疏且随机，作者引入人类成功轨迹，提供可参照的行为过程。它与教程文字等对照的差别，是直接给出“成功执行如何发生”的信息。这里的收益包含额外示范的价值，比较成本时不能把示范视作免费且在任何场景都可取得。[〔11〕](#review-ref-11)\n\n| 反馈形式 | 能补什么信息 | 无法单独保证什么 |\n| --- | --- | --- |\n| benchmark 自带测试／环境奖励 | 输出或终态是否满足预设条件 | 哪个系统机制导致失败 |\n| 与标准答案比对 | 结果是否匹配参考；比对可用规则或模型 | 参考是否完备、推理和过程是否可靠 |\n| 模型按 rubric 评分 | 开放产物的多个质量维度；rubric 是明确的评分条目 | judge 没有偏差、不会被候选迎合 |\n| 执行轨迹、错误诊断、成功示范 | 修改所需的上下文与行为参照 | 诊断正确、能迁移、获取成本划算 |\n\n这张表是对反馈作用的分析，而非某篇论文的统一实验设置。具体任务需要分别填写“谁判分、依据什么、修改者看见哪些信息”。\n\n### 下一步瓶颈：谁来验证评审者？\n\n当任务没有可靠的固定检查器时，[Red Queen Gödel Machine](?paper=2606.26294)让解题者与评审者共同改进。这打开了一个新对象，也产生新的约束：评审标准变化后，不同版本的分数还是否可比较？本文的判断是，反馈丰富度与反馈可信度应当分开研究；更长的点评、更强的 judge，都不能直接替代独立的正确性检查。[〔12〕](#review-ref-12)\n\n这一节把问题推进到：系统不仅需要反馈，还要从反馈中获得可保留的改进。下一节考察这些经验怎样离开原题。"
},
{
"id": "experience",
"title": "从同题反思到跨题学习：经验应该保存成什么？",
"thesis": "经验的形成、检索、执行和维护是四个环节，任何一环失败都可能让“记住了”无法变成能力提升。",
"groups": [
"experience"
],
"body": "### 第一处转折：从下一次重试，走向下一道新题\n\n[Reflexion](?paper=2303.11366)用文字反思保存试错所得，让后续尝试读取教训，避免立即重复同类错误。读它时要检查收益来自同一任务的重新尝试，还是迁移到了新任务。[ExpeL](?paper=2308.10144)把训练任务中的试错汇总为一般规则和成功示例：学习阶段允许积累经验，测试阶段的新题可检索这些经验。它把“过去的失败有没有帮助后来的新任务”变成更直接的研究问题。[〔13〕](#review-ref-13)[〔14〕](#review-ref-14)\n\n这里的难点不只是压缩长度。完整轨迹包含可复用策略，也包含当前题目的实体、路径和偶然错误。全部复制可能带入不相关细节，概括得太抽象又无法指导具体行动。因此，不同工作开始在经验表示上分流。\n\n### 并行分流：文字规则、工作流、可执行技能\n\n[Voyager](?paper=2305.16291)在 Minecraft 中结合自动课程与技能库，将成功行为保存为可检索、可组合的代码。[Agent Workflow Memory](?paper=2409.07429)从网页操作经验提炼可复用流程。[Memento-Skills](?paper=2603.18743)强调执行期间对技能的选择、修订和扩展。它们的共同目标是减少后续任务的重复探索，但表示的可执行程度、适用环境与维护方式不同；“skill”在这些论文里并非同一种文件或能力单位。[〔15〕](#review-ref-15)[〔16〕](#review-ref-16)[〔17〕](#review-ref-17)\n\n文字经验容易编辑和解释，但依赖模型理解并正确遵循；代码技能能固化操作，却要处理接口、前置条件和环境变化；工作流规定步骤，但未必能处理步骤之间的新分支。这里不存在脱离任务的最佳表示，应该比较它们在新实例、环境变化和组合任务上的实际表现。\n\n[Alita-G](?paper=2510.23601)进一步把成功轨迹抽象成带参数、可检索的工具，通过 MCP（agent 连接工具的统一协议）供后续任务调用。它强调把经验变成可以执行和复用的工具，而不只是让模型阅读一段总结。[〔18〕](#review-ref-18)\n\n### 第二处转折：从增加内容，到维护长期可用的经验\n\n[ACE](?paper=2510.04618)指出反复重写或压缩上下文会丢失具体经验，因此使用可增改的经验手册，逐项生成、反思和整理。它回应的是“持续更新时怎样保住有用细节”。[Procedural Graphs](?paper=2609.09153)把指导组织为带条件的步骤图，让执行时读取当前位置相关的内容。两者分别突出增量维护与执行结构，不能只按“都在写外部文本”合并。[〔19〕](#review-ref-19)[〔20〕](#review-ref-20)\n\n[SELF-INDEX](?paper=2609.19656)又把范围收窄到检索表示：从目标语料构造练习查询，发现检索缺口并修订索引键，检索后仍返回原文。它研究的不是更多记忆内容，而是现有内容怎样更容易在合适查询下被找到。语料可见与正式测试查询不可见是两回事，论文的这类隔离条件必须与一般“训练集／测试集”说法区分。[〔21〕](#review-ref-21)\n\n“如何可靠地修改技能”又形成一条具体路线。[Evo-Harness](?paper=2608.15071)从已完成任务的轨迹提炼文字操作说明，供后续新任务使用，模型参数和底层执行程序保持固定。[〔22〕](#review-ref-22) [SkillOpt](?paper=2605.23904)则限制每次文字编辑的幅度，记录被拒修改，只有留出验证分数提高才接受新技能；它把重点放在更新过程能否稳定、可追踪。[〔23〕](#review-ref-23)\n\n[Recuris](?paper=2608.24876)关注长任务里“现在需要哪条经验”：工作记忆记录当前进度，经验记忆提供相关技能；固定的 Meta-Agent 根据执行证据定位失败组件，提出局部修改并验证。这里需要分清两种收益：原有控制结构帮助了执行，还是结构里新学到的内容帮助了执行。[〔24〕](#review-ref-24)\n\n### 第三处转折：记忆的管理程序本身是否应当学习？\n\n如果编码、存储、检索与清理的规则固定，再增加内容也未必解决机制不适配。[MemEvolve](?paper=2512.18746)将经验内容与记忆架构共同纳入进化；[ALMA](?paper=2602.07755)研究能否根据持续学习效果自动寻找更合适的记忆设计。由此，路线从“学到了什么”推进到“怎样学习和使用经验”。这不是宣称代码一定优于文本，而是把固定设计也变成待检验对象。[〔25〕](#review-ref-25)[〔26〕](#review-ref-26)\n\n| 子问题 | 可以并读的工作 | 核心对照 |\n| --- | --- | --- |\n| 如何让旧题经验服务新题 | Reflexion、ExpeL | 同题重试与跨题单次执行的收益分开 |\n| 经验应写成什么形式 | Voyager、AWM、Memento-Skills | 文字指导、步骤流程、可执行程序的适用条件 |\n| 经验多了以后怎样保持可用 | ACE、Procedural Graphs、SELF-INDEX | 内容维护、执行结构、检索表示分别解决哪一环 |\n| 固定管理机制是否限制学习 | MemEvolve、ALMA | 内容改善与管理程序改善的独立贡献 |\n\n### 这条路线还需要什么证据？\n\n[ContinualSkillBench](?paper=2608.03874)引入与保留历史上下文相比较的视角；[PAST-Bench](?paper=2608.04003)通过控制经验保存和使用环节检验持久经验的作用。它们促使方法研究回答：提炼成技能是否真的比直接保留相同预算的历史更有效？如果技能从未被调用，不能把后期涨分归因于技能学习；如果只有旧题得分提高，也不足以说明经验可迁移。[〔27〕](#review-ref-27)[〔28〕](#review-ref-28)\n\n本站的综合判断是，经验路线的关键问题已经从“存不存”细化到**存什么、何时取、如何用、何时改或删**。这些环节也自然连接到下一节的 harness 设计。"
},
{
"id": "harness",
"title": "从自动设计到实际自修改：harness 研究怎样细化？",
"thesis": "开放修改权限只是起点；后续工作分别处理设计空间、故障诊断、搜索多样性与验收成本。",
"groups": [
"harness"
],
"body": "### 为什么提示优化会走向系统设计？\n\n提示能改变模型接收到的指导，却不能独自决定工具接口、状态保存和异常恢复。如果问题出在这些运行机制上，再改一句提示可能只是绕开症状。[DSPy](?paper=2310.03714)把语言模型调用组织成可组合、可优化的程序模块；[ADAS](?paper=2408.08435)用代码表达并搜索 agent 设计；[AFlow](?paper=2410.10762)搜索工作流的节点与连接。它们共同回应人工调系统的成本，但对结构预设的多少不同。[〔29〕](#review-ref-29)[〔30〕](#review-ref-30)[〔31〕](#review-ref-31)\n\n设计空间因此成为第一项研究选择：候选只能调整既有模块的参数和提示，还是能改变模块连接，乃至添加工具和执行逻辑？范围更开放，可能发现人工未预想的方案，也会扩大搜索和验证的难度。不能只依据“允许改全部代码”就推断方法更先进。\n\n### 从生成另一套系统，到修改自己实际使用的实现\n\n[SICA](?paper=2504.15228)让 coding agent 修改自身实现，再以更好的版本继续工作。[DGM](?paper=2505.22954)进一步保留历史 agent 档案，允许从不同版本继续产生新候选，避免只沿最近的最高分版本前进。DGM 的模型参数在这组实验中冻结；它把编程任务能力作为实证反馈，并依赖编程能力有助于后续自修改这一联系。这个联系在非编程领域是否成立，是后来讨论改进者能力时的重要问题。[〔32〕](#review-ref-32)[〔4〕](#review-ref-4)\n\n这里已经出现两种不同收益：新版本更会完成用户任务；新版本也可能更会编写下一轮修改代码。第一种能通过任务成绩观察，第二种还需要对改进过程作额外比较。把两者都称作“变强”会掩盖这一区别。\n\n### 同为改 harness，为什么 Self-Harness 的问题不同？\n\n[Self-Harness](?paper=2606.09498)强调各模型的失败模式不同，通用人工框架未必适配每个模型。它由同型号模型从执行记录中归纳弱点、提出小幅候选修改，再通过回归检查接受更新。实验涵盖 MiniMax M2.5、Qwen3.5-35B-A3B、GLM-5 与 Terminal-Bench 2.0、SWE-bench Verified、AppWorld。起点是 DeepAgent SDK 上的简化配置；论文的可修改配置面不等于任意改 SDK 全部源码。[〔33〕](#review-ref-33)\n\n这使对照问题变成：执行模型能否根据自己的弱点适配运行方式，是否仍需要更强的外部设计者？而 ADAS 的核心问题更偏向自动化系统设计空间，DGM 更突出自修改和多样版本探索。三者具有重叠实现，却不应以同一个泛化标签代替研究定位。\n\n### 另一种动机：强模型怎样通过 harness 帮助弱模型？\n\n[AI4AI at Test-Time](?paper=2608.12307)让更强的构建模型为冻结的弱模型设计执行代码、路由与检查流程。它研究的是无需训练弱模型参数，能否通过外部系统扩大其任务能力；不能把这种外部帮助和执行模型自行修改混为一谈。[〔34〕](#review-ref-34)\n\n同时，“改 harness”不都意味着改源码。[Recursive Harness Self-Improvement](?paper=2607.15524)比较相邻版本的任务产物，调整描述角色、通信、工作流与上下文管理的提示文本，执行代码和模型参数不变。它与代码自修改工作的区别，首先在于允许编辑的范围。[〔35〕](#review-ref-35)\n\n### 瓶颈一：失败记录很多，但怎样找到真正该改的机制？\n\n[Meta-Harness](?paper=2603.28052)允许修改者回看历次代码、分数和详细轨迹，避免只根据压缩摘要丢失的线索猜修改。[AHE](?paper=2604.25850)把改动关联到组件和执行证据，并检查组件之间的相互影响。[HarnessFix](?paper=2606.06324)进一步将运行步骤与对应 harness 实现对齐，归纳故障记录，再生成限定范围的修补。它们分别改善“能读到什么历史”“能否理解组件影响”“能否定位并约束修复”。[〔36〕](#review-ref-36)[〔37〕](#review-ref-37)[〔38〕](#review-ref-38)\n\n因此，日志长度本身不是诊断质量。真正有价值的是能否追溯：哪一步行为由哪个系统机制塑造，修改它预期修复什么，又可能影响什么。局部修复有效也不意味着定位完全正确；隔离组件、移植补丁、重复运行等对照可以帮助检验归因。\n\n[HarnessCompass](?paper=2608.01918)把诊断与泛化约束放在一起：限制框架写入题目专属信息，主动收集执行证据，并分开优化组件，减少记题式适配和同时改多处带来的干扰。这补充了“找到该改哪里”之后的两个问题：修改能否迁移，多个修改是否相互冲突。[〔39〕](#review-ref-39)\n\n### 瓶颈二：为什么更多候选仍可能搜不动？\n\n候选数量增加，却一直围绕相似起点作相似修改，搜索仍可能收缩。[HarnessBank](?paper=2607.13683)保存语义上不同、质量合格的版本，再进行重组和筛选。[RobustSGPO](?paper=2609.09646)控制改哪些 agent、采用什么修改操作，以及从哪些历史类别继续搜索。前者强调候选库的多样性与门控筛选，后者强调修改范围和操作的控制。[〔40〕](#review-ref-40)[〔41〕](#review-ref-41)\n\n### 瓶颈三：验证昂贵，而且总分可能掩盖退化\n\n[HarnessLens](?paper=2608.27311)针对每个候选都在固定题集上重复验证的浪费，选择与改动行为相关的任务，并检查可归因的变化。[SoL-Pi](?paper=2609.20519)则直接把运行效率纳入研究，搜索动作执行、上下文压缩、工具输出处理和委派阅读等机制。一个改善搜索阶段的验收效率，一个降低生成系统的运行成本；两种“省成本”不应混成同一个指标。[〔42〕](#review-ref-42)[〔43〕](#review-ref-43)\n\nSoL-Pi 的完整效率方案有成本下降，也有一定任务分数损失。评价它需要同时看性能、成本及候选选择协议，而不是把减少 token 自动解释为能力提升。对其他 harness 工作同样如此：应计入提出候选、运行验证和部署执行的不同成本。[〔43〕](#review-ref-43)\n\n| 已知瓶颈 | 代表工作怎样回应 | 下一步应控制的变量 |\n| --- | --- | --- |\n| 人工设计限制搜索范围 | ADAS／AFlow：自动搜索代码设计或工作流 | 预设模块与可编辑权限 |\n| 模型与通用框架不适配 | Self-Harness：从自身失败模式提出修改 | 固定模型、起点、外部帮助 |\n| 失败无法对应到系统原因 | Meta-Harness／AHE／HarnessFix：历史证据、组件归因、限定修复 | 日志信息量与诊断正确性 |\n| 候选越来越相似 | HarnessBank／RobustSGPO：多样版本与搜索操作控制 | 同预算下的有效候选比例 |\n| 验证或部署成本过高 | HarnessLens／SoL-Pi：选择性验收与运行机制优化 | 搜索成本和部署成本分别计算 |\n\n### 离线改好再部署，能否覆盖长期在线运行？\n\n[PILOT in the Loop](?paper=2608.26530)把执行与监督分开，运行中纠偏并留下技能；[Continual Harness](?paper=2605.09998)关心长程环境不中断、不重置时的持续适应；[Harness-of-Harness](?paper=2609.01481)面对多日软件开发的修复与功能推进。这些设置增加了离线候选比较中较弱的约束：当前任务还在运行，修改必须处理状态延续、恢复和新旧行为的兼容。[〔44〕](#review-ref-44)[〔45〕](#review-ref-45)[〔46〕](#review-ref-46)\n\n因此，harness 路线的研究重点可以从“又生成了一套代码”，收窄为**如何发现系统性故障、怎样搜索结构、怎样验收，以及如何把更新安全接入后续执行**。这里的“安全”需具体说明是控制旧能力退化、运行错误还是其他风险，不能仅作笼统标签。"
},
{
"id": "curriculum",
"title": "当目标和练习不再由人给定：系统怎样决定学什么？",
"thesis": "能优化明确的任务目标，与能自主建立有效学习目标，是两种能力。",
"groups": [
"curriculum",
"feedback"
],
"body": "### 固定题库为什么会成为限制？\n\n前面的大量设置默认已经有任务、数据和评分器，agent 主要寻找更好的执行方式。但如果任务本身仍由人不断提供，改进循环就依赖外部出题；如果训练题与真实目标失配，执行得更勤奋也可能沿错误方向前进。这里的研究对象从“怎么改模型或框架”，扩展为“用什么经历来驱动改进”。\n\n[AgentEvolver](?paper=2511.10395)针对新环境中人工任务构造和低效探索的成本，让系统组织交互任务并利用经验，提高轨迹利用率。[DiagEvo](?paper=2609.00768)从自身失败历史中归纳错误原因，再构造下一轮课程。前者偏向如何高效获取学习经历，后者偏向如何让练习针对暴露的能力缺口；二者都要进一步检验新数据是否真的改善了目标任务。[〔47〕](#review-ref-47)[〔48〕](#review-ref-48)\n\n### 从挑选已有任务，到创造仍然可验证的新挑战\n\n[BigBang](?paper=endlessfrontier.tech-assets-paper.pdf)回应的是另一个尺度的问题：当已有题目不再构成有效挑战时，系统能否持续合成更难且可验证的前沿任务。任务难度不能脱离正确性检查单独增长；若不能可靠判定结果，任务生成器与求解器可能共同迎合一个薄弱的评价机制。这里值得研究的是挑战性、可学习性与可验证性之间的配合。[〔49〕](#review-ref-49)\n\n数据研究本身也可以被单独评测。[RSIBench-Data](?paper=2607.25886)固定部分周边训练工程，考察 agent 能否诊断缺口、提出数据方案并保住后训练收益；[Curation-Bench](?paper=2606.04261)关注受控条件下的数据选择研究。这样才有机会区分：提升来自更好的数据决策，还是更大的模型、算力和训练工程改动。[〔50〕](#review-ref-50)[〔51〕](#review-ref-51)\n\n### 更强的要求：只有宽泛目标，能否建立正确的学习问题？\n\n[Aspire](?paper=2608.31111)把自主性进一步前移。系统只得到自然语言能力目标，需要选择数据、更新方式和自测方法。参数更新与 harness 更新是不同实验；后者正式评估使用题库中的 20 道写作题，不能把全部 520 道专家题都说成 harness 测试。部分参数设置允许查询目标题组的汇总分数，因此也不能把所有设置都描述为完全不可见的最终测试。[〔52〕](#review-ref-52)\n\n这个评测的意义在于把“改进循环跑通”与“朝正确能力方向进步”分开。作者观察到数据失配和狭窄自测等问题；这些现象提醒我们，自建课程不只是生成更多样本，还包含目标解释与诊断。结论应受论文模型、任务和预算约束，不能据此断言所有自主学习都无效。[〔52〕](#review-ref-52)\n\n| 自主程度增加在哪里 | 代表入口 | 需要额外验证的东西 |\n| --- | --- | --- |\n| 自己获取交互任务与训练轨迹 | AgentEvolver | 经历的覆盖、利用效率与真实学习价值 |\n| 从当前失败决定下一轮课程 | DiagEvo | 错误诊断是否正确，是否只反复强化局部题型 |\n| 持续创造新的挑战 | BigBang | 难度、可学习性、验证器可靠性 |\n| 自己把宽泛目标变成学习方案 | Aspire | 自测和真正目标能力是否对齐 |\n\n[S³Gym](?paper=2608.31100)提供另一个拆分视角：自主试验、判断经历与利用经历分别测，并比较保留历史、总结记忆和参数训练。即使能识别成功行为，也未必能把它转成可迁移策略。因此，选题、自评和学习不能用一个最终分数混为一谈。[〔53〕](#review-ref-53)\n\n### 当下一步不是练习题，而是开放研究决策\n\n[Argus](?paper=2608.05144)区分稳定的用户意图与可修订的操作目标、约束和验证条件。Manager、Planner、Engineer、Reviewer 围绕持续保存的项目状态推进任务；证据推翻原方案时，可修订目标与路线，并在指定节点交由操作者决策。它补充的是如何根据证据调整下一步研究，而非单纯生成更多训练题，也不意味着取消人的判断。[〔54〕](#review-ref-54)\n\n本站据此提出的研究问题是：**系统如何发现自己的学习信号已经偏离目标？** 这是从已有诊断结果引出的建议，并非宣称文献已经给出通用解决方案。"
},
{
"id": "weights",
"title": "从外部经验到参数学习：为什么需要 model–harness 协同？",
"thesis": "训练模型与修改运行系统解决不同瓶颈，联合路线的关键在于更新内容、顺序与预算。",
"groups": [
"weights"
],
"body": "### 为什么不能一直把经验留在 prompt、memory 或 skill 中？\n\n外部经验可以立即增改和检查，但每次使用都需要检索、读入和遵循。模型若没有执行相应策略的能力，增加说明未必有效。参数更新提供另一种保留经验的方式，却需要可学习的训练材料、奖励和算力，还可能带来遗忘。选择哪条路线，要根据实际瓶颈验证，不能把参数更新自动视为更深或更高级的进化。\n\n[SEAL](?paper=2506.10943)让模型根据新输入生成用于微调的数据及更新指令，执行参数更新，再用更新后模型的下游表现作为奖励，训练它生成更有效的学习材料。学习的不只是输入中的知识，还包括“怎样把输入改写成更容易学会的材料”。[SEED](?paper=2607.14777)从近期执行经历中提炼更具体的行动监督，补足长任务只给最终成败时的信息不足。前者强调如何为参数适应组织材料，后者强调交互训练中的逐步学习信号；两者都涉及数据到参数的转化，但训练对象和反馈链条不同。[〔55〕](#review-ref-55)[〔56〕](#review-ref-56)\n\n### 外部状态与模型行为可以一起成为学习对象\n\n[EvoHarness-RL](?paper=2608.05446)研究模型如何学习创建和读写外部状态，而非只依赖手写的固定读写时机。[SkillRise](?paper=2607.26784)让为未来任务整理技能的收益进入训练目标。它们提示了一种中间路线：经验仍以外部形式存在，但怎样产生和使用经验的策略可以被训练。把所有记忆工作归为“参数不变”会遗漏这种差别。[〔57〕](#review-ref-57)[〔58〕](#review-ref-58)\n\n### 从二选一，到研究两侧怎样互相限制\n\n[SIA](?paper=2605.27276)在一个改进流程中允许改运行框架或训练模型，关注怎样根据失败选择更新位置。[WHALE](?paper=2609.00196)则把交替更新的节奏作为具体问题：固定模型搜 harness，再固定 harness 训练模型，比较长阶段与小步交替等安排。新的模型可能适合不同框架，新框架又可能产生原来得不到的成功轨迹；因此两侧单独优化的结果未必能直接组合。[〔59〕](#review-ref-59)[〔60〕](#review-ref-60)\n\nWHALE 的任务反馈也不是单一 verifier：问答使用模型将回答与参考答案比对，数学用程序提取答案并匹配，棋题检查合法性及参考着法。三种环境的同一个“正确”标签来自不同实现。论文主要比较各运行的最佳测试点，这种结果与开发集选定版本后只测一次不同；评价交替机制时应保留这一条件。[〔60〕](#review-ref-60)\n\n### ScienceBuddy 为什么属于这一条路线？\n\n[ScienceBuddy](?paper=2609.17523)把科研请求、反馈和执行证据组织为持续学习材料：内层固定模型改 harness，外层在改进后的 harness 下训练模型。其单独 harness 实验主要改指令与技能文本，执行循环、工具及上下文处理固定，不能概括为重写完整框架。它的研究重点是交互、系统适配和模型学习如何衔接。[〔61〕](#review-ref-61)\n\n它也说明“有用户反馈”需要拆开写：harness 阶段由私有检查结果支持受限的模拟用户回复；参数学习按固定评分条目汇总可执行检查与 judge 分数。模拟用户的诊断、训练奖励和最终评价各有用途，不能相互替代。具体任务、数据划分与原文缺项在单篇实验表中保留。[〔61〕](#review-ref-61)\n\n[NeoHorse-1](?paper=2609.08183)提供另一种交互到训练的连接：routing harness 记录任务的能力需求、模型分配和执行轨迹，对轨迹检查、标注后组织训练课程，再根据能力评估调整下一轮数据配比。这里更新的是模型参数和训练数据安排；报告未展示 harness 代码自动进化，也把持续多轮更新留作后续工作。[〔62〕](#review-ref-62)\n\n| 论文切口 | 真正需要比较什么 |\n| --- | --- |\n| SEAL：自建有效适应材料 | 相同更新预算下，材料组织是否改善学习效果 |\n| EvoHarness-RL／SkillRise：学习外部经验的读写与整理策略 | 训练得到的策略是否改善后续任务，而非只帮助当前题 |\n| SIA：决定更新哪一侧 | 选择策略是否超过固定只改 harness 或只练模型 |\n| WHALE：决定何时切换 | 同总预算下，交替节奏是否带来额外收益 |\n| ScienceBuddy：将持续交互接入两层更新 | 交互材料、harness 改进和参数训练各自贡献多少 |\n\n[HELIX](?paper=2608.13951)强调可追踪、适合模型学习的执行轨迹，当前证据重点包括框架与数据接口；[Macaron-V1](?paper=2608.09819)的系统架构连接模型专长与运行配置，但直接框架搜索实验主要在冻结模型下进行。因此，“架构支持共同更新”“各侧分别有效”“多轮交替相互促进”应当分别记录。[〔63〕](#review-ref-63)[〔64〕](#review-ref-64)\n\n这一节的综合判断是：联合进化最有信息量的研究问题，通常不是“有没有同时改两样东西”，而是**什么失败该触发哪种更新、一次改多少、何时切换，以及改完后另一侧的最佳选择怎样变化**。"
},
{
"id": "improver",
"title": "从更强的 agent，到更有效的改进者：递归多了什么？",
"thesis": "改任务知识、改修改规则、训练编辑模型和修改优化程序，是不同层次的研究操作。",
"groups": [
"improver"
],
"body": "### 为什么做题更强，未必更会改系统？\n\n一个模型可能擅长完成任务，却不能准确解释失败原因；也可能能写出合理补丁，却不会验证其副作用。DGM 借助编程能力与代码自修改之间的联系形成改进循环。[Hyperagents](?paper=2603.19461)明确指出这种联系不一定推广到编程之外，因此将 task agent 与修改它的 meta-agent 同时放进可编辑程序，并考察修改机制的跨领域迁移。问题由“任务程序变好了”推进到“产生新任务程序的机制是否也变好了”。[〔65〕](#review-ref-65)\n\n### 路线一：改变指导修改的规则\n\n[Promptbreeder](?paper=2309.16797)让任务提示与指导提示变异的提示共同变化。[MetaSkill-Evolve](?paper=2607.05297)区分任务技能与指导技能修订的说明，以不同节奏更新。相对于固定一句“请改进”，这些方法把变异、反思或诊断的文字规则也作为学习对象。变化仍可能受到固定候选生成、预算与接受机制的约束，不能从“元提示可变”推断整个改进过程都不再有人类预设。[〔66〕](#review-ref-66)[〔67〕](#review-ref-67)\n\n### 路线二：训练一个更会提出有效修改的模型\n\n[Harness-R1](?paper=2608.02276)把独立编辑模型作为训练对象。它读取目标 agent 的失败记录，在预留的执行环节生成辅助代码；补丁装上后，冻结目标模型在任务上重新执行，实际奖励变化用于训练编辑模型。这与固定编辑者反复采样、选出最好补丁不同：学习信号改变了未来提出修改的策略。用于训练奖励的同批重跑，也不等于独立测试泛化。[〔68〕](#review-ref-68)\n\n[MetaEvolve](?paper=2607.21971)则训练利用反馈、反思和历史方案开展改进的通用能力，考察训练外优化问题。读这类工作时要明确参数属于谁：训练修改者，和训练执行任务的模型，是不同的系统变化。即便修改者训练后更有效，也还要另问它在部署时是否继续修改自己的学习机制。[〔69〕](#review-ref-69)\n\n[Frontis-MA1](?paper=2607.28568)把修改者学习放到机器学习工程中：训练模型执行起草、改进、调试和组合程序等操作，再让固定的搜索框架调用这些操作。它分别检查训练后模型与搜索经验的迁移，说明“修改者参数变了”与“外层搜索程序变了”是两种不同设置。[〔70〕](#review-ref-70)\n\n### 路线三：修改优化程序，而不只修改它输出的方案\n\n[STOP](?paper=2310.02304)研究支持代码改进的程序能否修改自身优化程序。Hyperagents 将这一问题扩展到任务程序与改进程序的共同可编辑结构，并通过迁移观察修改机制的作用。此时应检查每轮保留下来的究竟是更好的任务解，还是新的搜索、记忆、选择或诊断机制。[〔71〕](#review-ref-71)\n\n[Dream-RSI](?paper=2609.14858)提供了一个更具体的效率切口：利用累积发现树构建回放环境，在其中低成本试验探索策略，再放回线上继续搜索。底层 coding agent 不变，变化主要在探索编排层。它说明产生改进的能力可以通过更好的搜索策略改善，并不一定要训练一个更大的底层模型。回放对线上搜索的代表性与最终发现成本，因而成为重要评价条件。[〔72〕](#review-ref-72)\n\n### 另外两条分流：增加辅助层，或学习怎样组合更新\n\n[Metaⁿ](?paper=2608.24735)重复应用同一个固定的生成操作，逐层增加策略上下文和可调用辅助代码。它问的是增加辅助层何时有效；层数增加本身不表示生成规则也在进化，论文将主要收益归于层间传递的上下文。[〔73〕](#review-ref-73)\n\n[MetaRSI-v1](?paper=metarsi-v1)把数据生成、harness 修改和参数训练做成可组合操作，让调度策略调整操作顺序及候选生成指令。其重点是根据当前状态选择怎样更新；API 模型设置不能更新权重，已有验证也主要来自可执行代码与有标准答案的推理任务，不能外推为任意开放研究都能持续自改进。[〔74〕](#review-ref-74)\n\n| 想改变的东西 | 代表工作 | 应证明的新增能力 |\n| --- | --- | --- |\n| 指导修改的文字规则 | Promptbreeder、MetaSkill-Evolve | 新规则在相同预算下产生更有效的候选 |\n| 提出修改的模型策略 | Harness-R1、MetaEvolve | 学到的编辑／优化能力对新任务或新执行者仍有效 |\n| 修改与搜索程序 | STOP、Hyperagents、Dream-RSI | 新程序提高后续改进效率，而不只是保存了特定任务的好答案 |\n\n### 如何把“递归”变成可检验的主张？\n\n[Harness Updating Is Not Harness Benefit](?paper=2605.30621)通过执行者与修改者的独立配对，区分做题能力、提出更新的能力，以及从更新中受益的能力。其受限技能、提示和记忆编辑设置不能代表任意源码自修改，但这个角色分离的实验思想很关键：执行模型换强了，不能算作修改者更会改；补丁写得好，执行者不遵循也不一定受益。[〔75〕](#review-ref-75)\n\n本文据此建议一个更明确的检验：把旧、新修改者分别接到相同的新起点和未见任务上，匹配交互及验证预算，比较产生有效更新的速度、比例和累计收益。如果优势只存在于原任务，或依赖携带原任务答案，就应缩小“通用递归改进”的结论。此处是本站提出的实验设计，不是替相关论文追加它们尚未做的结果。"
},
{
"id": "evaluation",
"title": "从报告涨分，到解释为什么涨分：评测怎样推进了领域？",
"thesis": "方法的研究对象越开放，越需要把执行能力、改进能力、预算和数据接触分开。",
"groups": [
"evaluation"
],
"body": "### 第一种混淆：是不是只是多做了几次题？\n\nharness 搜索需要反复执行任务。如果对照只能答一次，而新方法可以读测试反馈、修订并重试，最终差距可能包含更多推理和更强反馈的收益。[Rethinking the Evaluation of Harness Evolution](?paper=2607.12227)在相近反馈和推理预算下比较直接重试等方法，并检查留出任务。作者在所测 Terminal-Bench 设置下发现，harness 进化并不稳定优于这些简单对照，泛化也有限。该结论质疑的是特定协议下的收益来源，不能泛化成所有系统修改无效。[〔76〕](#review-ref-76)\n\n### 第二种混淆：是不是把开发题调熟了？\n\n数据边界至少应拆成三部分：用于产生更新的数据、用于挑选版本的数据、用于最终检验的数据。候选如果根据某组分数被接受，那组数据已经承担了选版本作用；名称写着 held-out，也不能自动当作从未参与决策的最终测试。持续任务流还要明确顺序：做完当前任务才利用其反馈，和预先看到后续任务答案，属于不同条件。\n\n[Evo-Bench](?paper=2608.09096)控制执行者与预算，研究修改者能否产生可泛化的 harness 改进；[HarnessDev](?paper=2609.01437)区分从弱起点建设系统与继续改进既有系统；[One Recipe, Many Harnesses](?paper=2608.10178)进一步研究进化内容是通用执行策略、语言特有知识，还是对某个模型弱点的补偿。它们分别处理能力分工、起点和迁移对象，不能合并成一个笼统的“跨任务有效”。[〔77〕](#review-ref-77)[〔78〕](#review-ref-78)[〔79〕](#review-ref-79)\n\n### 第三种混淆：后期分数更高，就说明持续学习更好？\n\n任务顺序可能改变难度，也可能影响经验适用性。[PATH-Bench](?paper=2608.01149)关注历史路径、迁移与干扰；[AgentStream](?paper=2608.00155)将同域与跨域混合任务流放在连续条件下观察；[SEAGym](?paper=2606.17546)跟踪泛化、遗忘和成本。研究对象因此从一个版本的成绩，转向整段更新过程。[〔80〕](#review-ref-80)[〔81〕](#review-ref-81)[〔82〕](#review-ref-82)\n\n理论工作 [Safe Harness Self-Evolution](?paper=2609.08175)把有益候选是否存在、能否生成、有限数据能否可靠验收，以及后续是否还能进步分开分析。它提醒我们，停止更新未必只因修改模型太弱，也可能因为验证样本不足，无法确认小幅收益。理论保证依赖其任务分布与抽样条件，不能直接成为任意开放环境的承诺。[〔83〕](#review-ref-83)\n\n[AutoLab](?paper=2606.05080)从另一侧检查长程能力：每题给出可运行但次优的工程产物，要求 agent 在固定时间预算里持续测量、修改和探索。它观察的是迭代研发过程，执行 harness 保持固定；因此，持续把某个产物改好并不直接证明 agent 自身也在学习。[〔84〕](#review-ref-84)\n\n### 第四种混淆：会做 AI 研发，是否等于能自主改进自己？\n\n[PostTrainBench](?paper=2603.08640)考察给定模型和算力后能否完成有效后训练。[RSI-Exam](?paper=rsi-exam)从可运行弱方法出发，让研究 agent 做多轮实验，再评价交付物。[MLS-Bench](?paper=2605.08678)更强调提出的方法跨设置、扩大规模后是否仍有效。它们都与 AI 改进 AI 有关，但改进目标是否由人给定、提交的是模型还是方法，以及是否回头增强研究者本身，需要分别说明。[〔85〕](#review-ref-85)[〔86〕](#review-ref-86)[〔87〕](#review-ref-87)\n\n| 想声称的能力 | 最低限度需要怎样的对照 |\n| --- | --- |\n| 当前结果更好 | 相近反馈与总推理预算下的直接重试／搜索 |\n| 学到了可复用经验或系统机制 | 固定最终版本，测未参与选版本的新任务；消融所保留内容 |\n| 能持续改进 | 完整曲线、旧任务保持、累计成本，而不只挑最佳点 |\n| 修改者更会产生改进 | 固定执行起点，将新旧修改者迁移到相同未见任务 |\n| 能自主定义学习方向 | 目标任务独立于自建练习与自测；检查两者一致性 |\n\n这张表是本文的实验设计归纳。它不是要求每篇论文同时解决全部问题，而是要求**结论的范围与其对照真正支持的范围一致**。"
},
{
"id": "synthesis",
"title": "怎样把这幅脉络变成研究选题？",
"thesis": "把尚未分清的机制变成受控比较，比重复搭一套大而全的循环更容易形成清楚的问题。",
"groups": [
"harness",
"improver",
"evaluation"
],
"body": "### 对整个领域的一个综合判断\n\n从以上对照看，可以观察到三种研究对象的扩展。第一种把当前任务中的反馈转成后续任务可用的资源；第二种把人工固定的执行机制开放给搜索或学习；第三种把产生修改的规则、模型或程序也纳入改进。与此同时，选题和评价两端变得更重要：改得更多，不能补偿训练方向失配；搜索得更久，也不能替代独立验证。\n\n因此，领域的发展不宜写成“prompt → memory → harness → weights”的单向升级。它们往往在解决不同瓶颈，且可以共同作用。更加有用的脉络是：**一个环节被开放之后，哪个仍然固定的环节开始限制收益？新的论文是否识别并验证了这个限制？** 这是一种阅读和形成假设的方法，不是已经被证明适用于一切系统的统一规律。\n\n### 对 harness 进化，六个可以进一步收窄的问题\n\n| 研究问题 | 已有工作提供了什么 | 可以进一步做的受控比较 |\n| --- | --- | --- |\n| 错误究竟该在哪一层修？ | HarnessFix 提供故障定位；SIA 开放两侧更新；WHALE 研究交替 | 同一失败集、同预算比较补技能、修代码、练参数以及位置选择策略 |\n| 什么反馈最值得付出成本？ | GEPA 利用反思；DemoEvolve 引入示范；Harness-R1 用实际补丁收益训练 | 匹配轨迹和生成成本，比较总分、定位信息、示范带来的未见任务收益 |\n| 技能表示本身是否有效？ | ExpeL、ACE 提炼经验；ContinualSkillBench 提供历史上下文对照 | 同上下文与整理预算，比较原始轨迹、文字技能、步骤图，并记录调用情况 |\n| 自建练习怎样对准真正目标？ | DiagEvo 从失败生成课程；Aspire 暴露目标与自测的落差 | 固定数据池，比较课程策略；用不回传成绩的目标任务检验迁移 |\n| 改进者到底学到了什么？ | Hyperagents 开放修改程序；Harness-R1 训练编辑者 | 固定新起点，迁移新旧修改者；检查有效更新速度和跨域保持 |\n| 怎样决定接受、保留分支或停止？ | HarnessBank 保留多样候选；HarnessLens 优化验证；SEAGym 跟踪过程 | 改变验收样本量、分支策略和停止规则，计入误接受、误拒绝与累计成本 |\n\n以上方向是根据前文证据提出的研究建议，不声称尚无人研究。选择其中一个后，仍需针对其最新工作补查，进一步收窄任务、反馈条件和实验范围。\n\n### 一个更容易落地的起步顺序\n\n以 harness 为重点时，可以先选一类能明确执行与判分的任务，固定一个有代表性的基础 harness，保留逐步轨迹。第一步复现“原框架、同预算重试、固定编辑者改框架”三组对照，先确认系统修改确实有额外收益。第二步只改变一个机制，例如诊断信息或候选保留方式，观察收益是否还能在未见任务上保持。第三步再研究修改者训练或可修改优化程序，避免同时变更多个环节而无法归因。\n\n这一路线不是唯一选择。若问题在目标或课程，不应先投入完整 harness 搜索；若问题在执行模型根本不会使用技能，也不应只继续增加经验条目。树状图负责定位，本文负责解释各路线为什么存在，单篇研究表负责检查模型、反馈、harness 与数据设置是否支持相应比较。\n\n### 阅读入口与证据性质\n\n[Shiyu 的《什么在进化？》](?paper=shiyu-rsi-what-evolves)提供 Model、Harness、Artifact 的修改对象坐标；[《递归如何闭环？》](?paper=shiyu-rsi-loop-closes)讨论可修改范围与实验闭环。可以把这两篇与本文的问题脉络交叉阅读：一个定位改动发生在哪里，一个追问为什么改、怎样证明改对了。[〔88〕](#review-ref-88)[〔89〕](#review-ref-89)\n\n产业观察则回答另一类问题：现实研发中 AI 正承担哪些工作。[OpenAI 的 Research Acceleration](?paper=openai-research-acceleration)与[Anthropic 的 When AI builds itself](?paper=anthropic-when-ai-builds-itself)提供内部使用与评审视角；这些证据能说明研发参与方式，却不能直接替代“固定条件下可自主递归改进”的实验。[Reef](?paper=reef)这类仓库提供工程接口，也应与使用这些接口得到的学习结果分别记录。[〔90〕](#review-ref-90)[〔91〕](#review-ref-91)[〔92〕](#review-ref-92)"
}
],
"references": [
{
"number": 1,
"paper": "deepmind-alphaevolve-blog",
"title": "AlphaEvolve: A Gemini-powered coding agent for designing advanced algorithms",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/",
"sources": [
{
"label": "Designing better algorithms with large language models；导言与应用案例",
"url": "https://deepmind.google/blog/alphaevolve-a-gemini-powered-coding-agent-for-designing-advanced-algorithms/"
}
]
},
{
"number": 2,
"paper": "2509.19349",
"title": "ShinkaEvolve: Towards Open-Ended and Sample-Efficient Program Evolution",
"url": "https://arxiv.org/abs/2509.19349",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2509.19349#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2509.19349#S3.SS3"
}
]
},
{
"number": 3,
"paper": "cs-0309048",
"title": "Gödel Machines: Self-Referential Universal Problem Solvers Making Provably Optimal Self-Improvements",
"url": "https://arxiv.org/abs/cs/0309048",
"sources": [
{
"label": "§3 Formal Description",
"url": "https://arxiv.org/abs/cs/0309048"
}
]
},
{
"number": 4,
"paper": "2505.22954",
"title": "Darwin Gödel Machine (DGM): Open-Ended Evolution of Self-Improving Agents",
"url": "https://arxiv.org/abs/2505.22954",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2505.22954#S1"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2505.22954#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2505.22954#S4.SS1"
},
{
"label": "附录C.3",
"url": "https://arxiv.org/html/2505.22954#A3.SS3"
}
]
},
{
"number": 5,
"paper": "2303.17651",
"title": "Self-Refine: Iterative Refinement with Self-Feedback",
"url": "https://arxiv.org/abs/2303.17651",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.17651#S3"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2303.17651#S4.SS0.SSS0.Px1"
}
]
},
{
"number": 6,
"paper": "2310.01798",
"title": "Large Language Models Cannot Self-Correct Reasoning Yet",
"url": "https://arxiv.org/abs/2310.01798",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2310.01798"
}
]
},
{
"number": 7,
"paper": "2305.11738",
"title": "CRITIC: Large Language Models Can Self-Correct with Tool-Interactive Critiquing",
"url": "https://arxiv.org/abs/2305.11738",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2305.11738#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2305.11738#S4.SS1"
},
{
"label": "附录D.1",
"url": "https://arxiv.org/html/2305.11738#A4.SS1"
}
]
},
{
"number": 8,
"paper": "2305.20050",
"title": "Let’s Verify Step by Step",
"url": "https://arxiv.org/abs/2305.20050",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.20050#S2"
},
{
"label": "§2.4",
"url": "https://arxiv.org/html/2305.20050#S2.SS4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2305.20050#S4.SS2"
}
]
},
{
"number": 9,
"paper": "2406.07496",
"title": "TextGrad: Automatic “Differentiation” via Text",
"url": "https://arxiv.org/abs/2406.07496",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2406.07496#S3.SS1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2406.07496#S3.SS3"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2406.07496#S3.SS5.SSS0.Px1"
},
{
"label": "附录G.3",
"url": "https://arxiv.org/html/2406.07496#A7.SS3.SSS0.Px2"
}
]
},
{
"number": 10,
"paper": "2507.19457",
"title": "GEPA: Reflective Prompt Evolution Can Outperform Reinforcement Learning",
"url": "https://arxiv.org/abs/2507.19457",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2507.19457#S3"
}
]
},
{
"number": 11,
"paper": "2605.24539",
"title": "DemoEvolve: Overcoming Sparse Feedback in Agentic Harness Evolution with Demonstrations",
"url": "https://arxiv.org/abs/2605.24539",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2605.24539#S4.SS1.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2605.24539#A3.SS2.SSS0.Px3"
},
{
"label": "附录A",
"url": "https://arxiv.org/html/2605.24539#A1.SS0.SSS0.Px4"
}
]
},
{
"number": 12,
"paper": "2606.26294",
"title": "Red Queen Gödel Machine",
"url": "https://arxiv.org/abs/2606.26294",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2606.26294"
}
]
},
{
"number": 13,
"paper": "2303.11366",
"title": "Reflexion: Language Agents with Verbal Reinforcement Learning",
"url": "https://arxiv.org/abs/2303.11366",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2303.11366#S3.SS0.SSS0.Px3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2303.11366#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2303.11366#S4.SS3"
}
]
},
{
"number": 14,
"paper": "2308.10144",
"title": "ExpeL: LLM Agents Are Experiential Learners",
"url": "https://arxiv.org/abs/2308.10144",
"sources": [
{
"label": "实验数据与交叉验证",
"url": "https://arxiv.org/abs/2308.10144"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2308.10144#S4.SS4"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2308.10144#S5.SS1"
}
]
},
{
"number": 15,
"paper": "2305.16291",
"title": "Voyager: An Open-Ended Embodied Agent with Large Language Models",
"url": "https://arxiv.org/abs/2305.16291",
"sources": [
{
"label": "§2",
"url": "https://arxiv.org/html/2305.16291#S2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2305.16291#A1.SS1"
}
]
},
{
"number": 16,
"paper": "2409.07429",
"title": "Agent Workflow Memory (AWM)",
"url": "https://arxiv.org/abs/2409.07429",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2409.07429#S2.SS3.SSS0.Px2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2409.07429#S3.SS2.SSS2.Px1"
}
]
},
{
"number": 17,
"paper": "2603.18743",
"title": "Memento-Skills: Let Agents Design Agents",
"url": "https://arxiv.org/abs/2603.18743",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2603.18743#S2.SS1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px1"
},
{
"label": "§2.3",
"url": "https://arxiv.org/html/2603.18743#S2.SS3.SSS0.Px2"
}
]
},
{
"number": 18,
"paper": "2510.23601",
"title": "Alita-G: Self-Evolving Generative Agent for Agent Generation",
"url": "https://arxiv.org/abs/2510.23601",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2510.23601#S3.SS4"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.23601#S4.SS1.SSS0.Px1"
}
]
},
{
"number": 19,
"paper": "2510.04618",
"title": "Agentic Context Engineering (ACE)",
"url": "https://arxiv.org/abs/2510.04618",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2510.04618#S1"
},
{
"label": "§2.1",
"url": "https://arxiv.org/html/2510.04618#S2.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2510.04618#S4.SS1.SSS0.Px1"
}
]
},
{
"number": 20,
"paper": "2609.09153",
"title": "Procedural Graphs: Self-Evolving Execution Structures for LLM Agents",
"url": "https://arxiv.org/abs/2609.09153",
"sources": [
{
"label": "§3：图表示、使用与更新",
"url": "https://arxiv.org/html/2609.09153#S3"
},
{
"label": "附录D.2：五种图构造方案",
"url": "https://arxiv.org/html/2609.09153#A4.SS2"
},
{
"label": "§5.3：骨架构图与错误先验修复",
"url": "https://arxiv.org/html/2609.09153#S5.SS3"
}
]
},
{
"number": 21,
"paper": "2609.19656",
"title": "Self-Evolving Search Index",
"url": "https://arxiv.org/abs/2609.19656",
"sources": [
{
"label": "§3–4；附录 A.1–A.2、B.1–B.5",
"url": "https://arxiv.org/html/2609.19656v1"
}
]
},
{
"number": 22,
"paper": "2608.15071",
"title": "Evo-Harness: Context-to-Harness Skill Compilation for Self-Evolving Agents",
"url": "https://arxiv.org/abs/2608.15071",
"sources": [
{
"label": "§4.5",
"url": "https://arxiv.org/html/2608.15071#S4.SS5"
},
{
"label": "§4.6",
"url": "https://arxiv.org/html/2608.15071#S4.SS6"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2608.15071#A2"
},
{
"label": "§3.1–3.4：说明的选取、注入与修改",
"url": "https://arxiv.org/html/2608.15071#S3.SS1"
}
]
},
{
"number": 23,
"paper": "2605.23904",
"title": "SkillOpt: Executive Strategy for Self-Evolving Agent Skills",
"url": "https://arxiv.org/abs/2605.23904",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px2"
},
{
"label": "§4",
"url": "https://arxiv.org/html/2605.23904#S4.SS0.SSS0.Px3"
},
{
"label": "附录C.1",
"url": "https://arxiv.org/html/2605.23904#A3.SS1"
}
]
},
{
"number": 24,
"paper": "2608.24876",
"title": "Recuris",
"url": "https://arxiv.org/abs/2608.24876",
"sources": [
{
"label": "§2.1",
"url": "https://arxiv.org/html/2608.24876#S2.SS1.SSS2"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24876#S3.SS1"
},
{
"label": "附录E",
"url": "https://arxiv.org/html/2608.24876#A5"
},
{
"label": "Claude Code 官方说明",
"url": "https://code.claude.com/docs/en/overview"
}
]
},
{
"number": 25,
"paper": "2512.18746",
"title": "MemEvolve: Meta-Evolution of Agent Memory Systems",
"url": "https://arxiv.org/abs/2512.18746",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2512.18746#S1"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2512.18746#S3.SS3"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2512.18746#S5.SS2"
}
]
},
{
"number": 26,
"paper": "2602.07755",
"title": "ALMA: Learning to Continually Learn via Meta-learning Agentic Memory Designs",
"url": "https://arxiv.org/abs/2602.07755",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2602.07755#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2602.07755#S3.SS3"
}
]
},
{
"number": 27,
"paper": "2608.03874",
"title": "ContinualSkillBench: Can LLM Agents Truly Evolve Their Capabilities?",
"url": "https://arxiv.org/abs/2608.03874",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.03874"
}
]
},
{
"number": 28,
"paper": "2608.04003",
"title": "PAST-Bench",
"url": "https://arxiv.org/abs/2608.04003",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.04003"
}
]
},
{
"number": 29,
"paper": "2310.03714",
"title": "DSPy: Compiling Declarative Language Model Calls into Self-Improving Pipelines",
"url": "https://arxiv.org/abs/2310.03714",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2310.03714#S1"
}
]
},
{
"number": 30,
"paper": "2408.08435",
"title": "Automated Design of Agentic Systems (ADAS)",
"url": "https://arxiv.org/abs/2408.08435",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2408.08435#S3"
},
{
"label": "附录B",
"url": "https://arxiv.org/html/2408.08435#A2"
}
]
},
{
"number": 31,
"paper": "2410.10762",
"title": "AFlow: Automating Agentic Workflow Generation",
"url": "https://arxiv.org/abs/2410.10762",
"sources": [
{
"label": "§4",
"url": "https://arxiv.org/html/2410.10762#S4"
},
{
"label": "附录A.4",
"url": "https://arxiv.org/html/2410.10762#A1.SS4"
},
{
"label": "附录A.6",
"url": "https://arxiv.org/html/2410.10762#A1.SS6"
}
]
},
{
"number": 32,
"paper": "2504.15228",
"title": "A Self-Improving Coding Agent (SICA)",
"url": "https://arxiv.org/abs/2504.15228",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2504.15228#S3"
}
]
},
{
"number": 33,
"paper": "2606.09498",
"title": "Self-Harness: Harnesses That Improve Themselves",
"url": "https://arxiv.org/abs/2606.09498",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2606.09498#S1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2606.09498#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2606.09498#S4.SS1.SSS0.Px4"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2606.09498#S4.SS2"
},
{
"label": "实验任务子集与 promotion gate",
"url": "https://arxiv.org/abs/2606.09498"
}
]
},
{
"number": 34,
"paper": "2608.12307",
"title": "AI4AI at Test-Time: Strong-to-Weak Capability Transfer via Harnesses",
"url": "https://arxiv.org/abs/2608.12307",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.12307#S3"
}
]
},
{
"number": 35,
"paper": "2607.15524",
"title": "Recursive Harness Self-Improvement",
"url": "https://arxiv.org/abs/2607.15524",
"sources": [
{
"label": "§3.2",
"url": "https://arxiv.org/html/2607.15524#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2607.15524#S3.SS3"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.15524#S4.SS2"
}
]
},
{
"number": 36,
"paper": "2603.28052",
"title": "Meta-Harness: End-to-End Optimization of Model Harnesses",
"url": "https://arxiv.org/abs/2603.28052",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2603.28052#S3"
}
]
},
{
"number": 37,
"paper": "2604.25850",
"title": "Agentic Harness Engineering (AHE)",
"url": "https://arxiv.org/abs/2604.25850",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2604.25850#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2604.25850#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2604.25850#S3.SS2"
},
{
"label": "§3.3",
"url": "https://arxiv.org/html/2604.25850#S3.SS3"
}
]
},
{
"number": 38,
"paper": "2606.06324",
"title": "HarnessFix",
"url": "https://arxiv.org/abs/2606.06324",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2606.06324#S3"
},
{
"label": "§6",
"url": "https://arxiv.org/html/2606.06324#S6"
}
]
},
{
"number": 39,
"paper": "2608.01918",
"title": "HarnessCompass: Guiding Automatic Harness Evolution toward Generalizable and Effective Agent Harnesses",
"url": "https://arxiv.org/abs/2608.01918",
"sources": [
{
"label": "HarnessCompass",
"url": "https://arxiv.org/html/2608.01918#Sx3"
},
{
"label": "Constrained Evolution",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx2"
},
{
"label": "Aggregation and use.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx3.SSS0.Px3"
},
{
"label": "Running the tracks independently.",
"url": "https://arxiv.org/html/2608.01918#Sx3.SSx4.SSS0.Px1"
}
]
},
{
"number": 40,
"paper": "2607.13683",
"title": "HarnessBank / Self-Evolving Agent Harnesses via Gated Semantic Quality-Diversity",
"url": "https://arxiv.org/abs/2607.13683",
"sources": [
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.13683#S4.SS1"
},
{
"label": "§4.5",
"url": "https://arxiv.org/html/2607.13683#S4.SS5"
}
]
},
{
"number": 41,
"paper": "2609.09646",
"title": "RobustSGPO: Search-Space Control for Agent Harness Evolution",
"url": "https://arxiv.org/abs/2609.09646",
"sources": [
{
"label": "§5：指定改动、检查与保留版本",
"url": "https://arxiv.org/html/2609.09646#S5"
},
{
"label": "§4：三级修改权限",
"url": "https://arxiv.org/html/2609.09646#S4"
},
{
"label": "§6.4：任务切换实验",
"url": "https://arxiv.org/html/2609.09646#S6.SS4"
}
]
},
{
"number": 42,
"paper": "2608.27311",
"title": "HarnessLens: Verify Smarter, Evolve Further",
"url": "https://arxiv.org/abs/2608.27311",
"sources": [
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2608.27311#A1.SS1"
},
{
"label": "附录C.2",
"url": "https://arxiv.org/html/2608.27311#A3.SS2.SSS2"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2608.27311#S5.SS1.SSS0.Px2"
}
]
},
{
"number": 43,
"paper": "2609.20519",
"title": "SoL-Pi: Recursively Scaling Auto-Research Loops for Efficient Agent Harness",
"url": "https://arxiv.org/abs/2609.20519",
"sources": [
{
"label": "§2.1–2.5、§3.1–3.3、§5.1",
"url": "https://arxiv.org/html/2609.20519v1"
}
]
},
{
"number": 44,
"paper": "2608.26530",
"title": "PILOT in the Loop: Live Self-Improvement for Long-Horizon Agents",
"url": "https://arxiv.org/abs/2608.26530",
"sources": [
{
"label": "§2 双向监督与 Pi 实现",
"url": "https://arxiv.org/html/2608.26530#S2"
}
]
},
{
"number": 45,
"paper": "2605.09998",
"title": "Continual Harness: Online Adaptation for Self-Improving Foundation Agents",
"url": "https://arxiv.org/abs/2605.09998",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2605.09998"
}
]
},
{
"number": 46,
"paper": "2609.01481",
"title": "Harness-of-Harness: Multi-Day Autonomous Software Development with Continual Improvement",
"url": "https://arxiv.org/abs/2609.01481",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01481"
}
]
},
{
"number": 47,
"paper": "2511.10395",
"title": "AgentEvolver: Towards Efficient Self-Evolving Agent System",
"url": "https://arxiv.org/abs/2511.10395",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2511.10395#S1"
},
{
"label": "§3",
"url": "https://arxiv.org/html/2511.10395#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2511.10395#S3.SS1"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2511.10395#S4.SS1"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2511.10395#S5"
}
]
},
{
"number": 48,
"paper": "2609.00768",
"title": "DiagEvo: Diagnosis-Guided Self-Evolution via Hierarchical Error Memory",
"url": "https://arxiv.org/abs/2609.00768",
"sources": [
{
"label": "§3 自博弈、投票与诊断",
"url": "https://arxiv.org/html/2609.00768#S3"
}
]
},
{
"number": 49,
"paper": "endlessfrontier.tech-assets-paper.pdf",
"title": "BigBang: Pursuing Open-Ended Intelligence through Self-Evolving Synthesis of Verifiable Frontier Tasks",
"url": "https://endlessfrontier.tech/assets/paper.pdf",
"sources": [
{
"label": "§1 Introduction（第 1–3 页）",
"url": "https://endlessfrontier.tech/assets/paper.pdf#page=1"
},
{
"label": "摘要",
"url": "https://endlessfrontier.tech/assets/paper.pdf"
}
]
},
{
"number": 50,
"paper": "2607.25886",
"title": "RSIBench-Data: Benchmarking Data-Centric Research for Recursive Self-Improvement",
"url": "https://arxiv.org/abs/2607.25886",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.25886"
}
]
},
{
"number": 51,
"paper": "2606.04261",
"title": "Curation-Bench",
"url": "https://arxiv.org/abs/2606.04261",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.04261"
}
]
},
{
"number": 52,
"paper": "2608.31111",
"title": "Aspire: Can Models Self-Evolve from Vague Goals?",
"url": "https://arxiv.org/abs/2608.31111",
"sources": [
{
"label": "§1、§4.1：研究问题与实验安排",
"url": "https://arxiv.org/html/2608.31111#S4.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px2"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px2"
},
{
"label": "§4.2、附录 B：PostTrainBench 对照",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.31111#S3.SS2.SSS0.Px4"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px3"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.31111#S4.SS3.SSS0.Px5"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.31111#S4.SS4.SSS0.Px4"
},
{
"label": "附录 C.5：公开记录边界",
"url": "https://arxiv.org/html/2608.31111#A3.SS5"
},
{
"label": "§4.2：目标描述对照结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS2"
},
{
"label": "§4.3：参数更新结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS3"
},
{
"label": "§4.4：运行框架结果",
"url": "https://arxiv.org/html/2608.31111#S4.SS4"
}
]
},
{
"number": 53,
"paper": "2608.31100",
"title": "S³Gym: Can LLMs Turn Self-Testing and Self-Judging into Self-Improvement?",
"url": "https://arxiv.org/abs/2608.31100",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.31100"
}
]
},
{
"number": 54,
"paper": "2608.05144",
"title": "Argus: A General-Purpose Agentic Reasoning Runtime for Long-Horizon Tasks",
"url": "https://arxiv.org/abs/2608.05144",
"sources": [
{
"label": "摘要",
"url": "https://arxiv.org/abs/2608.05144"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.05144#S4.SS1"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2608.05144#S4.SS3"
},
{
"label": "§4.4",
"url": "https://arxiv.org/html/2608.05144#S4.SS4"
}
]
},
{
"number": 55,
"paper": "2506.10943",
"title": "SEAL / Self-Adapting Language Models",
"url": "https://arxiv.org/abs/2506.10943",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2506.10943#S3"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2506.10943#S3.SS1"
},
{
"label": "摘要",
"url": "https://arxiv.org/abs/2506.10943"
}
]
},
{
"number": 56,
"paper": "2607.14777",
"title": "SEED: Self-Evolving On-Policy Distillation for Agentic RL",
"url": "https://arxiv.org/abs/2607.14777",
"sources": [
{
"label": "附录B.3",
"url": "https://arxiv.org/html/2607.14777#A2.SS3"
}
]
},
{
"number": 57,
"paper": "2608.05446",
"title": "EvoHarness-RL: Learning Self-Evolving Runtime Harness for Long-Horizon LLM Agents",
"url": "https://arxiv.org/abs/2608.05446",
"sources": [
{
"label": "附录C",
"url": "https://arxiv.org/html/2608.05446#A3.SS0.SSS0.Px5"
},
{
"label": "附录D.2",
"url": "https://arxiv.org/html/2608.05446#A4.SS2"
}
]
},
{
"number": 58,
"paper": "2607.26784",
"title": "SkillRise: Agentic Reinforcement Learning for Cross-Task Skill Evolution",
"url": "https://arxiv.org/abs/2607.26784",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2607.26784#S3"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.26784#S4.SS1.SSS0.Px3"
}
]
},
{
"number": 59,
"paper": "2605.27276",
"title": "SIA: Self Improving AI with Harness & Weight Updates",
"url": "https://arxiv.org/abs/2605.27276",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.27276"
}
]
},
{
"number": 60,
"paper": "2609.00196",
"title": "WHALE: A Simple Recipe for Joint Harness-Weight Optimization",
"url": "https://arxiv.org/abs/2609.00196",
"sources": [
{
"label": "§6.2：交替节奏及自适应切换",
"url": "https://arxiv.org/html/2609.00196#S6.SS2"
},
{
"label": "§6.1：不同领域的瓶颈",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
},
{
"label": "附录B.3：三个固定判分器",
"url": "https://arxiv.org/html/2609.00196#A2.SS3"
},
{
"label": "§4：交替流程与返回规则",
"url": "https://arxiv.org/html/2609.00196#S4"
},
{
"label": "§5.1：三个领域的数据",
"url": "https://arxiv.org/html/2609.00196#S5.SS1"
},
{
"label": "§5.3、图2：最好测试成绩",
"url": "https://arxiv.org/html/2609.00196#S5.SS3"
},
{
"label": "§6.1：轨迹成本口径",
"url": "https://arxiv.org/html/2609.00196#S6.SS1"
}
]
},
{
"number": 61,
"paper": "2609.17523",
"title": "ScienceBuddy: Recursive-in-Recursive Self-Improvement for Interactive Scientific Agents",
"url": "https://arxiv.org/abs/2609.17523",
"sources": [
{
"label": "§2.2–2.4、§4.2–4.4、附录 §7.1–7.3 / 表1–2",
"url": "https://arxiv.org/html/2609.17523v1"
}
]
},
{
"number": 62,
"paper": "2609.08183",
"title": "NeoHorse-1: Towards Recursive Self-Improvement via Agentic Post-Training with Routing Harness",
"url": "https://arxiv.org/abs/2609.08183",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.08183#S1"
},
{
"label": "§3.5 按能力调整数据",
"url": "https://arxiv.org/html/2609.08183#S3.SS5"
},
{
"label": "§4 SFT 与 on-policy distillation",
"url": "https://arxiv.org/html/2609.08183#S4"
},
{
"label": "§3.1–3.4 轨迹、质量与路由",
"url": "https://arxiv.org/html/2609.08183#S3"
}
]
},
{
"number": 63,
"paper": "2608.13951",
"title": "HELIX: Model-Harness Co-evolution for Recursive Self-Improvement",
"url": "https://arxiv.org/abs/2608.13951",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.13951"
}
]
},
{
"number": 64,
"paper": "2608.09819",
"title": "Macaron-V1: Towards Open Continual Learning with Self-Improvement and Mixture-of-LoRA",
"url": "https://arxiv.org/abs/2608.09819",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09819"
}
]
},
{
"number": 65,
"paper": "2603.19461",
"title": "Hyperagents",
"url": "https://arxiv.org/abs/2603.19461",
"sources": [
{
"label": "§1 Introduction · 问题背景与研究动机",
"url": "https://arxiv.org/html/2603.19461#S1"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2603.19461#S5.SS1"
},
{
"label": "§5.3",
"url": "https://arxiv.org/html/2603.19461#S5.SS3"
}
]
},
{
"number": 66,
"paper": "2309.16797",
"title": "Promptbreeder: Self-Referential Self-Improvement Via Prompt Evolution",
"url": "https://arxiv.org/abs/2309.16797",
"sources": [
{
"label": "§3.1",
"url": "https://arxiv.org/html/2309.16797#S3.SS1"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2309.16797#A6"
}
]
},
{
"number": 67,
"paper": "2607.05297",
"title": "MetaSkill-Evolve: Recursive Self-Improvement of LLM Agents via Two-Timescale Meta-Skill Evolution",
"url": "https://arxiv.org/abs/2607.05297",
"sources": [
{
"label": "§3.4",
"url": "https://arxiv.org/html/2607.05297#S3.SS4"
},
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.05297#S3.SS5"
}
]
},
{
"number": 68,
"paper": "2608.02276",
"title": "Harness-R1: Learning to Edit Executable Runtime Harnesses from Agent Failure Trajectories",
"url": "https://arxiv.org/abs/2608.02276",
"sources": [
{
"label": "附录 D、Table D.1",
"url": "https://arxiv.org/abs/2608.02276"
},
{
"label": "§3.1：四处代码介入位置",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.02276#S3.SS1"
},
{
"label": "§3.2",
"url": "https://arxiv.org/html/2608.02276#S3.SS2"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2608.02276#S4.SS1.SSS0.Px4"
},
{
"label": "附录D",
"url": "https://arxiv.org/html/2608.02276#A4"
},
{
"label": "附录F",
"url": "https://arxiv.org/html/2608.02276#A6"
}
]
},
{
"number": 69,
"paper": "2607.21971",
"title": "Teaching LLMs to Self-Evolve: Cultivating Core Meta-Skills with Reinforcement Learning (MetaEvolve)",
"url": "https://arxiv.org/abs/2607.21971",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2607.21971"
}
]
},
{
"number": 70,
"paper": "2607.28568",
"title": "Frontis-MA1: Training an AI4AI Model towards Recursive Self-Improvement in Machine Learning Engineering",
"url": "https://arxiv.org/abs/2607.28568",
"sources": [
{
"label": "§3.5",
"url": "https://arxiv.org/html/2607.28568#S3.SS5"
},
{
"label": "§4.1",
"url": "https://arxiv.org/html/2607.28568#S4.SS1"
},
{
"label": "§4.2",
"url": "https://arxiv.org/html/2607.28568#S4.SS2"
},
{
"label": "§4.3",
"url": "https://arxiv.org/html/2607.28568#S4.SS3"
},
{
"label": "§5.1",
"url": "https://arxiv.org/html/2607.28568#S5.SS1"
},
{
"label": "§5.2",
"url": "https://arxiv.org/html/2607.28568#S5.SS2"
},
{
"label": "§5.4",
"url": "https://arxiv.org/html/2607.28568#S5.SS4"
}
]
},
{
"number": 71,
"paper": "2310.02304",
"title": "STOP: Self-Taught Optimizer / Recursively Self-Improving Code Generation",
"url": "https://arxiv.org/abs/2310.02304",
"sources": [
{
"label": "§5.1 Fixed Downstream Task",
"url": "https://arxiv.org/abs/2310.02304"
}
]
},
{
"number": 72,
"paper": "2609.14858",
"title": "Dream-RSI: Recursive Self-Improvement through Evolving Worlds",
"url": "https://arxiv.org/abs/2609.14858",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2609.14858#S1"
},
{
"label": "§3 探索策略、回放与选版本",
"url": "https://arxiv.org/html/2609.14858#S3"
}
]
},
{
"number": 73,
"paper": "2608.24735",
"title": "Meta^n: Recursive Self-Improvement through Emergent Depth",
"url": "https://arxiv.org/abs/2608.24735",
"sources": [
{
"label": "§3",
"url": "https://arxiv.org/html/2608.24735#S3.SS0.SSS0.Px1"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px4"
},
{
"label": "§3.1",
"url": "https://arxiv.org/html/2608.24735#S3.SS1.SSS0.Px6"
}
]
},
{
"number": 74,
"paper": "metarsi-v1",
"title": "MetaRSI-v1: A Meta-Recursive Self-Improving System for Recursive Self-Improving Systems Themselves",
"url": "https://www.cosmosmind.ai/research/metarsi-v1",
"sources": [
{
"label": "§4 统一循环与三类操作，PDF p9–20",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=9"
},
{
"label": "§5.1–5.2，PDF p21–22",
"url": "https://www.cosmosmind.ai/research/metarsi-v1.pdf#page=21"
}
]
},
{
"number": 75,
"paper": "2605.30621",
"title": "Harness Updating Is Not Harness Benefit: Disentangling Evolution Capabilities in Self-Evolving LLM Agents",
"url": "https://arxiv.org/abs/2605.30621",
"sources": [
{
"label": "§1：两种能力的分离",
"url": "https://arxiv.org/html/2605.30621#S1"
},
{
"label": "§3.3：固定一侧的能力指标",
"url": "https://arxiv.org/html/2605.30621#S3.SS3"
},
{
"label": "附录B.4：初始提示与可写范围",
"url": "https://arxiv.org/html/2605.30621#A2.SS4"
},
{
"label": "§4.1、附录B.2：模型配对",
"url": "https://arxiv.org/html/2605.30621#S4.SS1"
}
]
},
{
"number": 76,
"paper": "2607.12227",
"title": "Rethinking the Evaluation of Harness Evolution for Agents",
"url": "https://arxiv.org/abs/2607.12227",
"sources": [
{
"label": "§1 Introduction",
"url": "https://arxiv.org/html/2607.12227#S1"
},
{
"label": "§3 四种方法及反馈权限",
"url": "https://arxiv.org/html/2607.12227#S3"
},
{
"label": "§4.3 有测试反馈",
"url": "https://arxiv.org/html/2607.12227#S4.SS3"
},
{
"label": "§4.4 45/10/34 划分",
"url": "https://arxiv.org/html/2607.12227#S4.SS4"
},
{
"label": "§5.1–5.2 修改分析与限制",
"url": "https://arxiv.org/html/2607.12227#S5"
}
]
},
{
"number": 77,
"paper": "2608.09096",
"title": "Evo-Bench: Can Language Models Improve Agent Harness?",
"url": "https://arxiv.org/abs/2608.09096",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.09096"
}
]
},
{
"number": 78,
"paper": "2609.01437",
"title": "HarnessDev",
"url": "https://arxiv.org/abs/2609.01437",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2609.01437"
}
]
},
{
"number": 79,
"paper": "2608.10178",
"title": "One Recipe, Many Harnesses: What Self-Evolution Encodes Across Languages and Models",
"url": "https://arxiv.org/abs/2608.10178",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.10178"
}
]
},
{
"number": 80,
"paper": "2608.01149",
"title": "PATH-Bench",
"url": "https://arxiv.org/abs/2608.01149",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.01149"
}
]
},
{
"number": 81,
"paper": "2608.00155",
"title": "AgentStream",
"url": "https://arxiv.org/abs/2608.00155",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2608.00155"
}
]
},
{
"number": 82,
"paper": "2606.17546",
"title": "SEAGym: An Evaluation Environment for Self-Evolving LLM Agents",
"url": "https://arxiv.org/abs/2606.17546",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2606.17546"
}
]
},
{
"number": 83,
"paper": "2609.08175",
"title": "Safe Harness Self-Evolution: A Theoretical Analysis of Feasibility and Limits",
"url": "https://arxiv.org/abs/2609.08175",
"sources": [
{
"label": "§1–2，p1–4：研究对象与假设",
"url": "https://arxiv.org/pdf/2609.08175#page=1"
},
{
"label": "§3，p4–7：改进目标与生成概率",
"url": "https://arxiv.org/pdf/2609.08175#page=4"
},
{
"label": "§4，p7–10：有限数据验收",
"url": "https://arxiv.org/pdf/2609.08175#page=7"
}
]
},
{
"number": 84,
"paper": "2606.05080",
"title": "AutoLab",
"url": "https://arxiv.org/abs/2606.05080",
"sources": [
{
"label": "§2.3",
"url": "https://arxiv.org/html/2606.05080#S2.SS3"
},
{
"label": "§5",
"url": "https://arxiv.org/html/2606.05080#S5.SS0.SSS0.Px3"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px1"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px2"
},
{
"label": "附录A.1",
"url": "https://arxiv.org/html/2606.05080#A1.SS1.SSS0.Px3"
}
]
},
{
"number": 85,
"paper": "2603.08640",
"title": "PostTrainBench",
"url": "https://arxiv.org/abs/2603.08640",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2603.08640"
}
]
},
{
"number": 86,
"paper": "rsi-exam",
"title": "RSI-Exam: Benchmarking Recursive Self-Improvement through Executable Research",
"url": "https://rsi-exam.ai/blog.html",
"sources": [
{
"label": "官方报告 · 研究目标",
"url": "https://rsi-exam.ai/blog.html#introduction"
}
]
},
{
"number": 87,
"paper": "2605.08678",
"title": "MLS-Bench: A Holistic and Rigorous Assessment of AI Systems on Building Better AI",
"url": "https://arxiv.org/abs/2605.08678",
"sources": [
{
"label": "摘要：研究目标",
"url": "https://arxiv.org/abs/2605.08678"
}
]
},
{
"number": 88,
"paper": "shiyu-rsi-what-evolves",
"title": "什么在进化？——Model、Harness 与 Artifact 的三层地图",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/",
"sources": [
{
"label": "§1–2、§4、§6、§8；写在前面：来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-What-Evolves/"
}
]
},
{
"number": 89,
"paper": "shiyu-rsi-loop-closes",
"title": "递归如何闭环？——从可修改范围到 Agent as Service",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/",
"sources": [
{
"label": "定义；Everything as Service；Scoped → Joint → Recursive；来源与致谢",
"url": "https://shiyunee.github.io/blogs/RSI-Where-the-Loop-Closes/"
}
]
},
{
"number": 90,
"paper": "openai-research-acceleration",
"title": "Research Acceleration: The View Inside OpenAI",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/",
"sources": [
{
"label": "官方博客：导言与 §1–5",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
},
{
"label": "官方博客：Appendix / Our methods for this post",
"url": "https://openai.com/index/research-acceleration-view-inside-openai/"
}
]
},
{
"number": 91,
"paper": "anthropic-when-ai-builds-itself",
"title": "When AI builds itself",
"url": "https://www.anthropic.com/institute/recursive-self-improvement",
"sources": [
{
"label": "Evidence from within Anthropic；Update 9/18/2026；图注与脚注",
"url": "https://www.anthropic.com/institute/recursive-self-improvement"
}
]
},
{
"number": 92,
"paper": "reef",
"title": "Reef: Continual Learning Infrastructure for Self-Improving Agents",
"url": "https://github.com/Human-Agent-Society/reef",
"sources": [
{
"label": "README：How it works / When to use Reef",
"url": "https://github.com/Human-Agent-Society/reef"
}
]
}
],
"chineseCharacters": 11571,
"paperCount": 92
}
}