fix: 多项修复与优化

This commit is contained in:
Youzini-afk
2026-03-26 02:09:18 +08:00
parent 71011400d2
commit 0e590a6256
7 changed files with 2442 additions and 649 deletions

View File

@@ -1,23 +1,30 @@
// ST-BME: 层级压缩引擎 // ST-BME: 层级压缩引擎
// 超过阈值的节点被 LLM 总结为更高层级的压缩节点 // 超过阈值的节点被 LLM 总结为更高层级的压缩节点
import { createNode, addNode, createEdge, addEdge, getActiveNodes, getNode } from './graph.js'; import { embedText } from "./embedding.js";
import { callLLMForJSON } from './llm.js'; import {
import { embedText } from './embedding.js'; addEdge,
import { buildTaskPrompt } from './prompt-builder.js'; addNode,
import { applyTaskRegex } from './task-regex.js'; createEdge,
import { isDirectVectorConfig } from './vector-index.js'; createNode,
getActiveNodes,
getNode,
} from "./graph.js";
import { callLLMForJSON } from "./llm.js";
import { buildTaskPrompt } from "./prompt-builder.js";
import { applyTaskRegex } from "./task-regex.js";
import { isDirectVectorConfig } from "./vector-index.js";
function createAbortError(message = '操作已终止') { function createAbortError(message = "操作已终止") {
const error = new Error(message); const error = new Error(message);
error.name = 'AbortError'; error.name = "AbortError";
return error; return error;
} }
function throwIfAborted(signal) { function throwIfAborted(signal) {
if (signal?.aborted) { if (signal?.aborted) {
throw signal.reason instanceof Error ? signal.reason : createAbortError(); throw signal.reason instanceof Error ? signal.reason : createAbortError();
} }
} }
/** /**
@@ -30,193 +37,234 @@ function throwIfAborted(signal) {
* @param {boolean} [params.force=false] - 忽略阈值强制压缩 * @param {boolean} [params.force=false] - 忽略阈值强制压缩
* @returns {Promise<{created: number, archived: number}>} * @returns {Promise<{created: number, archived: number}>}
*/ */
export async function compressType({ graph, typeDef, embeddingConfig, force = false, customPrompt, signal, settings = {} }) { export async function compressType({
const compression = typeDef.compression; graph,
if (!compression || compression.mode !== 'hierarchical') { typeDef,
return { created: 0, archived: 0 }; embeddingConfig,
} force = false,
customPrompt,
signal,
settings = {},
}) {
const compression = typeDef.compression;
if (!compression || compression.mode !== "hierarchical") {
return { created: 0, archived: 0 };
}
let totalCreated = 0; let totalCreated = 0;
let totalArchived = 0; let totalArchived = 0;
// 从最低层级开始逐层压缩 // 从最低层级开始逐层压缩
for (let level = 0; level < compression.maxDepth; level++) { for (let level = 0; level < compression.maxDepth; level++) {
throwIfAborted(signal); throwIfAborted(signal);
const result = await compressLevel({ const result = await compressLevel({
graph, graph,
typeDef, typeDef,
level, level,
embeddingConfig, embeddingConfig,
force, force,
customPrompt, customPrompt,
signal, signal,
settings, settings,
}); });
totalCreated += result.created; totalCreated += result.created;
totalArchived += result.archived; totalArchived += result.archived;
// 如果这一层没有压缩发生,停止 // 如果这一层没有压缩发生,停止
if (result.created === 0) break; if (result.created === 0) break;
} }
return { created: totalCreated, archived: totalArchived }; return { created: totalCreated, archived: totalArchived };
} }
/** /**
* 压缩特定层级的节点 * 压缩特定层级的节点
*/ */
async function compressLevel({ graph, typeDef, level, embeddingConfig, force, customPrompt, signal, settings = {} }) { async function compressLevel({
const compression = typeDef.compression; graph,
throwIfAborted(signal); typeDef,
level,
embeddingConfig,
force,
customPrompt,
signal,
settings = {},
}) {
const compression = typeDef.compression;
throwIfAborted(signal);
// 获取该层级的活跃叶子节点 // 获取该层级的活跃叶子节点
const levelNodes = getActiveNodes(graph, typeDef.id) const levelNodes = getActiveNodes(graph, typeDef.id)
.filter(n => n.level === level) .filter((n) => n.level === level)
.sort((a, b) => a.seq - b.seq); .sort((a, b) => a.seq - b.seq);
const threshold = force ? Math.max(2, compression.fanIn) : compression.threshold; const threshold = force
const keepRecent = force ? 0 : compression.keepRecentLeaves; ? Math.max(2, compression.fanIn)
: compression.threshold;
const keepRecent = force ? 0 : compression.keepRecentLeaves;
// 不够阈值,无需压缩 // 不够阈值,无需压缩
if (levelNodes.length <= threshold) { if (levelNodes.length <= threshold) {
return { created: 0, archived: 0 }; return { created: 0, archived: 0 };
}
// 排除最近的节点
const compressible = levelNodes.slice(0, levelNodes.length - keepRecent);
if (compressible.length < compression.fanIn) {
return { created: 0, archived: 0 };
}
let created = 0;
let archived = 0;
// 按 fanIn 分组压缩
for (let i = 0; i < compressible.length; i += compression.fanIn) {
const batch = compressible.slice(i, i + compression.fanIn);
if (batch.length < 2) break; // 至少 2 个才压缩
// 调用 LLM 总结
const summaryResult = await summarizeBatch(
batch,
typeDef,
customPrompt,
signal,
settings,
);
if (!summaryResult) continue;
// 创建压缩节点
const compressedNode = createNode({
type: typeDef.id,
fields: summaryResult.fields,
seq: batch[batch.length - 1].seq,
seqRange: [
batch[0].seqRange?.[0] ?? batch[0].seq,
batch[batch.length - 1].seqRange?.[1] ?? batch[batch.length - 1].seq,
],
importance: Math.max(...batch.map((n) => n.importance)),
});
compressedNode.level = level + 1;
compressedNode.childIds = batch.map((n) => n.id);
// 生成 embedding
if (isDirectVectorConfig(embeddingConfig) && summaryResult.fields.summary) {
const vec = await embedText(
summaryResult.fields.summary,
embeddingConfig,
{ signal },
);
if (vec) compressedNode.embedding = Array.from(vec);
} }
// 排除最近的节点 addNode(graph, compressedNode);
const compressible = levelNodes.slice(0, levelNodes.length - keepRecent); migrateBatchEdges(graph, batch, compressedNode);
if (compressible.length < compression.fanIn) { created++;
return { created: 0, archived: 0 };
// 归档子节点
for (const child of batch) {
child.archived = true;
child.parentId = compressedNode.id;
archived++;
} }
}
let created = 0; return { created, archived };
let archived = 0;
// 按 fanIn 分组压缩
for (let i = 0; i < compressible.length; i += compression.fanIn) {
const batch = compressible.slice(i, i + compression.fanIn);
if (batch.length < 2) break; // 至少 2 个才压缩
// 调用 LLM 总结
const summaryResult = await summarizeBatch(batch, typeDef, customPrompt, signal, settings);
if (!summaryResult) continue;
// 创建压缩节点
const compressedNode = createNode({
type: typeDef.id,
fields: summaryResult.fields,
seq: batch[batch.length - 1].seq,
seqRange: [batch[0].seqRange?.[0] ?? batch[0].seq, batch[batch.length - 1].seqRange?.[1] ?? batch[batch.length - 1].seq],
importance: Math.max(...batch.map(n => n.importance)),
});
compressedNode.level = level + 1;
compressedNode.childIds = batch.map(n => n.id);
// 生成 embedding
if (isDirectVectorConfig(embeddingConfig) && summaryResult.fields.summary) {
const vec = await embedText(summaryResult.fields.summary, embeddingConfig, { signal });
if (vec) compressedNode.embedding = Array.from(vec);
}
addNode(graph, compressedNode);
migrateBatchEdges(graph, batch, compressedNode);
created++;
// 归档子节点
for (const child of batch) {
child.archived = true;
child.parentId = compressedNode.id;
archived++;
}
}
return { created, archived };
} }
function migrateBatchEdges(graph, batch, compressedNode) { function migrateBatchEdges(graph, batch, compressedNode) {
const batchIds = new Set(batch.map(node => node.id)); const batchIds = new Set(batch.map((node) => node.id));
const activeNodeIds = new Set(getActiveNodes(graph).map(node => node.id));
for (const edge of graph.edges) { for (const edge of graph.edges) {
if (edge.invalidAt || edge.expiredAt) continue; if (edge.invalidAt || edge.expiredAt) continue;
const fromInside = batchIds.has(edge.fromId); const fromInside = batchIds.has(edge.fromId);
const toInside = batchIds.has(edge.toId); const toInside = batchIds.has(edge.toId);
if (!fromInside && !toInside) continue; if (!fromInside && !toInside) continue;
if (fromInside && toInside) continue; if (fromInside && toInside) continue;
const newFromId = fromInside ? compressedNode.id : edge.fromId; const newFromId = fromInside ? compressedNode.id : edge.fromId;
const newToId = toInside ? compressedNode.id : edge.toId; const newToId = toInside ? compressedNode.id : edge.toId;
if (newFromId === newToId) continue; if (newFromId === newToId) continue;
if (!activeNodeIds.has(newFromId) || !activeNodeIds.has(newToId)) continue; if (!getNode(graph, newFromId) || !getNode(graph, newToId)) continue;
if (!getNode(graph, newFromId) || !getNode(graph, newToId)) continue;
const migratedEdge = createEdge({ const migratedEdge = createEdge({
fromId: newFromId, fromId: newFromId,
toId: newToId, toId: newToId,
relation: edge.relation, relation: edge.relation,
strength: edge.strength, strength: edge.strength,
edgeType: edge.edgeType, edgeType: edge.edgeType,
}); });
migratedEdge.validAt = edge.validAt ?? migratedEdge.validAt; migratedEdge.validAt = edge.validAt ?? migratedEdge.validAt;
migratedEdge.invalidAt = edge.invalidAt ?? migratedEdge.invalidAt; migratedEdge.invalidAt = edge.invalidAt ?? migratedEdge.invalidAt;
migratedEdge.expiredAt = edge.expiredAt ?? migratedEdge.expiredAt; migratedEdge.expiredAt = edge.expiredAt ?? migratedEdge.expiredAt;
addEdge(graph, migratedEdge); addEdge(graph, migratedEdge);
} }
} }
/** /**
* 调用 LLM 总结一批节点 * 调用 LLM 总结一批节点
*/ */
async function summarizeBatch(nodes, typeDef, customPrompt, signal, settings = {}) { async function summarizeBatch(
const nodeDescriptions = nodes.map((n, i) => { nodes,
const fieldsStr = Object.entries(n.fields) typeDef,
.filter(([_, v]) => v) customPrompt,
.map(([k, v]) => `${k}: ${v}`) signal,
.join('\n '); settings = {},
return `节点 ${i + 1} [楼层 ${n.seq}]:\n ${fieldsStr}`; ) {
}).join('\n\n'); const nodeDescriptions = nodes
.map((n, i) => {
const fieldsStr = Object.entries(n.fields)
.filter(([_, v]) => v)
.map(([k, v]) => `${k}: ${v}`)
.join("\n ");
return `节点 ${i + 1} [楼层 ${n.seq}]:\n ${fieldsStr}`;
})
.join("\n\n");
const instruction = typeDef.compression.instruction || '将以下节点压缩总结为一条精炼记录。'; const instruction =
typeDef.compression.instruction || "将以下节点压缩总结为一条精炼记录。";
const compressPromptBuild = buildTaskPrompt(settings, 'compress', { const compressPromptBuild = buildTaskPrompt(settings, "compress", {
taskName: 'compress', taskName: "compress",
nodeContent: nodeDescriptions, nodeContent: nodeDescriptions,
candidateNodes: nodeDescriptions, candidateNodes: nodeDescriptions,
currentRange: `${nodes[0]?.seq ?? '?'} ~ ${nodes[nodes.length - 1]?.seq ?? '?'}`, currentRange: `${nodes[0]?.seq ?? "?"} ~ ${nodes[nodes.length - 1]?.seq ?? "?"}`,
graphStats: `node_count=${nodes.length}, node_type=${typeDef.id}`, graphStats: `node_count=${nodes.length}, node_type=${typeDef.id}`,
}); });
const systemPrompt = applyTaskRegex( const systemPrompt = applyTaskRegex(
settings, settings,
'compress', "compress",
'finalPrompt', "finalPrompt",
compressPromptBuild.systemPrompt || customPrompt || [ compressPromptBuild.systemPrompt ||
'你是一个记忆压缩器。将多个同类型节点总结为一条更高层级的压缩节点。', customPrompt ||
[
"你是一个记忆压缩器。将多个同类型节点总结为一条更高层级的压缩节点。",
instruction, instruction,
'', "",
'输出格式为严格 JSON', "输出格式为严格 JSON",
`{"fields": {${typeDef.columns.map(c => `"${c.name}": "..."`).join(', ')}}}`, `{"fields": {${typeDef.columns.map((c) => `"${c.name}": "..."`).join(", ")}}}`,
'', "",
'规则:', "规则:",
'- 保留关键信息:因果关系、不可逆结果、未解决伏笔', "- 保留关键信息:因果关系、不可逆结果、未解决伏笔",
'- 去除重复和低信息密度内容', "- 去除重复和低信息密度内容",
'- 压缩后文本应精炼,目标 150 字左右', "- 压缩后文本应精炼,目标 150 字左右",
].join('\n'), ].join("\n"),
); );
const userPrompt = `请压缩以下 ${nodes.length} 个 "${typeDef.label}" 节点:\n\n${nodeDescriptions}`; const userPrompt = `请压缩以下 ${nodes.length} 个 "${typeDef.label}" 节点:\n\n${nodeDescriptions}`;
return await callLLMForJSON({ return await callLLMForJSON({
systemPrompt, systemPrompt,
userPrompt, userPrompt,
maxRetries: 1, maxRetries: 1,
signal, signal,
taskType: 'compress', taskType: "compress",
additionalMessages: compressPromptBuild.customMessages || [], additionalMessages: compressPromptBuild.customMessages || [],
}); });
} }
/** /**
@@ -228,20 +276,36 @@ async function summarizeBatch(nodes, typeDef, customPrompt, signal, settings = {
* @param {boolean} [force=false] * @param {boolean} [force=false]
* @returns {Promise<{created: number, archived: number}>} * @returns {Promise<{created: number, archived: number}>}
*/ */
export async function compressAll(graph, schema, embeddingConfig, force = false, customPrompt, signal, settings = {}) { export async function compressAll(
let totalCreated = 0; graph,
let totalArchived = 0; schema,
embeddingConfig,
force = false,
customPrompt,
signal,
settings = {},
) {
let totalCreated = 0;
let totalArchived = 0;
for (const typeDef of schema) { for (const typeDef of schema) {
throwIfAborted(signal); throwIfAborted(signal);
if (typeDef.compression?.mode === 'hierarchical') { if (typeDef.compression?.mode === "hierarchical") {
const result = await compressType({ graph, typeDef, embeddingConfig, force, customPrompt, signal, settings }); const result = await compressType({
totalCreated += result.created; graph,
totalArchived += result.archived; typeDef,
} embeddingConfig,
force,
customPrompt,
signal,
settings,
});
totalCreated += result.created;
totalArchived += result.archived;
} }
}
return { created: totalCreated, archived: totalArchived }; return { created: totalCreated, archived: totalArchived };
} }
// ==================== v2: 主动遗忘SleepGate 启发) ==================== // ==================== v2: 主动遗忘SleepGate 启发) ====================
@@ -249,40 +313,45 @@ export async function compressAll(graph, schema, embeddingConfig, force = false,
/** /**
* 睡眠清理周期 * 睡眠清理周期
* 评估每个节点的保留价值,低于阈值的归档(遗忘) * 评估每个节点的保留价值,低于阈值的归档(遗忘)
* *
* @param {object} graph - 图状态 * @param {object} graph - 图状态
* @param {object} settings - 包含 forgetThreshold 的设置 * @param {object} settings - 包含 forgetThreshold 的设置
* @returns {{forgotten: number}} 本次遗忘的节点数 * @returns {{forgotten: number}} 本次遗忘的节点数
*/ */
export function sleepCycle(graph, settings) { export function sleepCycle(graph, settings) {
const threshold = settings.forgetThreshold ?? 0.5; const threshold = settings.forgetThreshold ?? 0.5;
const nodes = getActiveNodes(graph); const nodes = getActiveNodes(graph);
const now = Date.now(); const now = Date.now();
let forgotten = 0; let forgotten = 0;
for (const node of nodes) { for (const node of nodes) {
// 跳过常驻类型synopsis, rule 等重要节点不应被遗忘) // 跳过常驻类型synopsis, rule 等重要节点不应被遗忘)
if (node.type === 'synopsis' || node.type === 'rule' || node.type === 'thread') continue; if (
// 跳过高重要性节点 node.type === "synopsis" ||
if (node.importance >= 8) continue; node.type === "rule" ||
// 跳过最近创建的节点(< 1 小时) node.type === "thread"
if (now - node.createdTime < 3600000) continue; )
continue;
// 跳过高重要性节点
if (node.importance >= 8) continue;
// 跳过最近创建的节点(< 1 小时)
if (now - node.createdTime < 3600000) continue;
// 计算保留价值 = importance × recency × (1 + accessFreq) // 计算保留价值 = importance × recency × (1 + accessFreq)
const ageHours = (now - node.createdTime) / 3600000; const ageHours = (now - node.createdTime) / 3600000;
const recency = 1 / (1 + Math.log10(1 + ageHours)); const recency = 1 / (1 + Math.log10(1 + ageHours));
const accessFreq = node.accessCount / Math.max(1, ageHours / 24); const accessFreq = node.accessCount / Math.max(1, ageHours / 24);
const retentionValue = (node.importance / 10) * recency * (1 + accessFreq); const retentionValue = (node.importance / 10) * recency * (1 + accessFreq);
if (retentionValue < threshold) { if (retentionValue < threshold) {
node.archived = true; node.archived = true;
forgotten++; forgotten++;
}
} }
}
if (forgotten > 0) { if (forgotten > 0) {
console.log(`[ST-BME] 主动遗忘: ${forgotten} 个低价值节点已归档`); console.log(`[ST-BME] 主动遗忘: ${forgotten} 个低价值节点已归档`);
} }
return { forgotten }; return { forgotten };
} }

View File

@@ -2,32 +2,32 @@
// 合并 Mem0 精确对照 + A-MEM 记忆进化为单一阶段 // 合并 Mem0 精确对照 + A-MEM 记忆进化为单一阶段
// 批量 embed + 批量查近邻 + 单次 LLM 调用 // 批量 embed + 批量查近邻 + 单次 LLM 调用
import { embedBatch, searchSimilar } from './embedding.js'; import { embedBatch, searchSimilar } from "./embedding.js";
import { addEdge, createEdge, getActiveNodes, getNode } from './graph.js'; import { addEdge, createEdge, getActiveNodes, getNode } from "./graph.js";
import { callLLMForJSON } from './llm.js'; import { callLLMForJSON } from "./llm.js";
import { buildTaskPrompt } from './prompt-builder.js'; import { buildTaskPrompt } from "./prompt-builder.js";
import { applyTaskRegex } from './task-regex.js'; import { applyTaskRegex } from "./task-regex.js";
import { import {
buildNodeVectorText, buildNodeVectorText,
findSimilarNodesByText, findSimilarNodesByText,
isDirectVectorConfig, isDirectVectorConfig,
validateVectorConfig, validateVectorConfig,
} from './vector-index.js'; } from "./vector-index.js";
function createAbortError(message = '操作已终止') { function createAbortError(message = "操作已终止") {
const error = new Error(message); const error = new Error(message);
error.name = 'AbortError'; error.name = "AbortError";
return error; return error;
} }
function isAbortError(error) { function isAbortError(error) {
return error?.name === 'AbortError'; return error?.name === "AbortError";
} }
function throwIfAborted(signal) { function throwIfAborted(signal) {
if (signal?.aborted) { if (signal?.aborted) {
throw signal.reason instanceof Error ? signal.reason : createAbortError(); throw signal.reason instanceof Error ? signal.reason : createAbortError();
} }
} }
/** /**
@@ -102,368 +102,408 @@ const CONSOLIDATION_SYSTEM_PROMPT = `你是一个记忆整合分析器。当新
* @returns {Promise<{merged: number, skipped: number, kept: number, evolved: number, connections: number, updates: number}>} * @returns {Promise<{merged: number, skipped: number, kept: number, evolved: number, connections: number, updates: number}>}
*/ */
export async function consolidateMemories({ export async function consolidateMemories({
graph, graph,
newNodeIds, newNodeIds,
embeddingConfig, embeddingConfig,
options = {}, options = {},
customPrompt, customPrompt,
signal, signal,
settings = {}, settings = {},
}) { }) {
const neighborCount = options.neighborCount ?? 5; const neighborCount = options.neighborCount ?? 5;
const conflictThreshold = options.conflictThreshold ?? 0.85; const conflictThreshold = options.conflictThreshold ?? 0.85;
const stats = { const stats = {
merged: 0, merged: 0,
skipped: 0, skipped: 0,
kept: 0, kept: 0,
evolved: 0, evolved: 0,
connections: 0, connections: 0,
updates: 0, updates: 0,
}; };
if (!newNodeIds || newNodeIds.length === 0) return stats; if (!newNodeIds || newNodeIds.length === 0) return stats;
if (!validateVectorConfig(embeddingConfig).valid) { if (!validateVectorConfig(embeddingConfig).valid) {
console.log('[ST-BME] 记忆整合跳过:向量配置不可用'); console.log("[ST-BME] 记忆整合跳过:向量配置不可用");
return stats; return stats;
}
// ══════════════════════════════════════════════
// Phase 0: 收集有效新节点
// ══════════════════════════════════════════════
const newEntries = [];
for (const id of newNodeIds) {
const node = getNode(graph, id);
if (!node || node.archived) continue;
const text = buildNodeVectorText(node);
if (!text) continue;
newEntries.push({ id, node, text });
}
if (newEntries.length === 0) return stats;
const activeNodes = getActiveNodes(graph).filter((n) => {
const text = buildNodeVectorText(n);
return typeof text === "string" && text.length > 0;
});
if (activeNodes.length < 2) {
// 图中节点不够,全部 keep
stats.kept = newEntries.length;
return stats;
}
throwIfAborted(signal);
console.log(`[ST-BME] 记忆整合开始: ${newEntries.length} 个新节点`);
// ══════════════════════════════════════════════
// Phase 1 + 2: 批量 Embed + 查近邻
// ══════════════════════════════════════════════
/** @type {Map<string, Array<{nodeId: string, score: number}>>} */
const neighborsMap = new Map();
if (isDirectVectorConfig(embeddingConfig)) {
// ── 直连模式: 1 次 embedBatch + N 次本地 cosine ──
const texts = newEntries.map((e) => e.text);
let queryVectors;
try {
queryVectors = await embedBatch(texts, embeddingConfig, { signal });
} catch (e) {
if (isAbortError(e)) throw e;
console.warn("[ST-BME] 批量 embed 失败,回退到逐条:", e.message);
queryVectors = null;
} }
// ══════════════════════════════════════════════ // 构建候选池(含 embedding 的活跃节点)
// Phase 0: 收集有效新节点 const candidatePool = activeNodes
// ══════════════════════════════════════════════ .filter((n) => Array.isArray(n.embedding) && n.embedding.length > 0)
const newEntries = []; .map((n) => ({ nodeId: n.id, embedding: n.embedding }));
for (const id of newNodeIds) {
const node = getNode(graph, id);
if (!node || node.archived) continue;
const text = buildNodeVectorText(node);
if (!text) continue;
newEntries.push({ id, node, text });
}
if (newEntries.length === 0) return stats;
const activeNodes = getActiveNodes(graph).filter(n => {
const text = buildNodeVectorText(n);
return typeof text === 'string' && text.length > 0;
});
if (activeNodes.length < 2) {
// 图中节点不够,全部 keep
stats.kept = newEntries.length;
return stats;
}
throwIfAborted(signal);
console.log(`[ST-BME] 记忆整合开始: ${newEntries.length} 个新节点`);
// ══════════════════════════════════════════════
// Phase 1 + 2: 批量 Embed + 查近邻
// ══════════════════════════════════════════════
/** @type {Map<string, Array<{nodeId: string, score: number}>>} */
const neighborsMap = new Map();
if (isDirectVectorConfig(embeddingConfig)) {
// ── 直连模式: 1 次 embedBatch + N 次本地 cosine ──
const texts = newEntries.map(e => e.text);
let queryVectors;
try {
queryVectors = await embedBatch(texts, embeddingConfig, { signal });
} catch (e) {
if (isAbortError(e)) throw e;
console.warn('[ST-BME] 批量 embed 失败,回退到逐条:', e.message);
queryVectors = null;
}
// 构建候选池(含 embedding 的活跃节点)
const candidatePool = activeNodes
.filter(n => Array.isArray(n.embedding) && n.embedding.length > 0)
.map(n => ({ nodeId: n.id, embedding: n.embedding }));
for (let i = 0; i < newEntries.length; i++) {
throwIfAborted(signal);
const entry = newEntries[i];
const candidates = candidatePool.filter(c => c.nodeId !== entry.id);
if (queryVectors?.[i] && candidates.length > 0) {
// 本地 cosine 搜索0 API 调用)
const neighbors = searchSimilar(queryVectors[i], candidates, neighborCount);
neighborsMap.set(entry.id, neighbors);
} else {
// fallback: 逐条 embed
try {
const neighbors = await findSimilarNodesByText(
graph, entry.text, embeddingConfig, neighborCount,
activeNodes.filter(n => n.id !== entry.id), signal,
);
neighborsMap.set(entry.id, neighbors);
} catch (e) {
if (isAbortError(e)) throw e;
console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
neighborsMap.set(entry.id, []);
}
}
}
} else {
// ── 后端模式: 逐条 /api/vector/query ──
for (let i = 0; i < newEntries.length; i++) {
throwIfAborted(signal);
const entry = newEntries[i];
try {
const neighbors = await findSimilarNodesByText(
graph, entry.text, embeddingConfig, neighborCount,
activeNodes.filter(n => n.id !== entry.id), signal,
);
neighborsMap.set(entry.id, neighbors);
} catch (e) {
if (isAbortError(e)) throw e;
console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
neighborsMap.set(entry.id, []);
}
}
}
// ══════════════════════════════════════════════
// Phase 3: 单次 LLM 批量判定
// ══════════════════════════════════════════════
throwIfAborted(signal);
const userPromptSections = [];
userPromptSections.push(`本轮共新增 ${newEntries.length} 条记忆,请逐条分析:\n`);
for (let i = 0; i < newEntries.length; i++) { for (let i = 0; i < newEntries.length; i++) {
const entry = newEntries[i]; throwIfAborted(signal);
const neighbors = neighborsMap.get(entry.id) || []; const entry = newEntries[i];
const candidates = candidatePool.filter((c) => c.nodeId !== entry.id);
const newNodeFieldsStr = Object.entries(entry.node.fields) if (queryVectors?.[i] && candidates.length > 0) {
.map(([k, v]) => `${k}: ${v}`) // 本地 cosine 搜索0 API 调用)
.join(', '); const neighbors = searchSimilar(
queryVectors[i],
// 构建近邻描述 candidates,
let neighborText; neighborCount,
if (neighbors.length === 0) { );
neighborText = ' (无近邻命中)'; neighborsMap.set(entry.id, neighbors);
} else { } else {
neighborText = neighbors.map(n => { // fallback: 逐条 embed
const node = getNode(graph, n.nodeId); try {
if (!node) return null; const neighbors = await findSimilarNodesByText(
const fieldsStr = Object.entries(node.fields) graph,
.map(([k, v]) => `${k}: ${v}`) entry.text,
.join(', '); embeddingConfig,
return ` - [${node.id}] 类型=${node.type}, ${fieldsStr} (相似度=${n.score.toFixed(3)})`; neighborCount,
}).filter(Boolean).join('\n'); activeNodes.filter((n) => n.id !== entry.id),
}
// 检查高相似度
const hasHighSimilarity = neighbors.length > 0 && neighbors[0].score > conflictThreshold;
const hint = hasHighSimilarity
? ` ⚠ 最高相似度 ${neighbors[0].score.toFixed(3)} 超过阈值 ${conflictThreshold}`
: '';
userPromptSections.push([
`### 新记忆 #${i + 1}`,
`[${entry.id}] 类型=${entry.node.type}, ${newNodeFieldsStr}`,
'近邻记忆:',
neighborText,
hint,
].filter(Boolean).join('\n'));
}
const userPrompt = userPromptSections.join('\n\n');
let decision;
const consolidationPromptBuild = buildTaskPrompt(settings, 'consolidation', {
taskName: 'consolidation',
candidateNodes: userPrompt,
candidateText: userPrompt,
graphStats: `new_entries=${newEntries.length}, threshold=${conflictThreshold}`,
});
try {
decision = await callLLMForJSON({
systemPrompt: applyTaskRegex(
settings,
'consolidation',
'finalPrompt',
consolidationPromptBuild.systemPrompt || customPrompt || CONSOLIDATION_SYSTEM_PROMPT,
),
userPrompt,
maxRetries: 1,
signal, signal,
taskType: 'consolidation', );
additionalMessages: consolidationPromptBuild.customMessages || [], neighborsMap.set(entry.id, neighbors);
}); } catch (e) {
} catch (e) { if (isAbortError(e)) throw e;
if (isAbortError(e)) throw e; console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
console.error('[ST-BME] 记忆整合 LLM 调用失败:', e); neighborsMap.set(entry.id, []);
stats.kept = newEntries.length;
return stats;
}
// ══════════════════════════════════════════════
// Phase 4: 逐个处理结果
// ══════════════════════════════════════════════
// 解析 LLM 返回——兼容单条和批量格式
let results;
if (Array.isArray(decision?.results)) {
results = decision.results;
} else if (decision?.action) {
// 单条返回格式LLM 可能忽略 results 包装)
results = [{ ...decision, node_id: newEntries[0]?.id }];
} else {
console.warn('[ST-BME] 记忆整合: LLM 返回格式异常,全部 keep');
stats.kept = newEntries.length;
return stats;
}
// 建立 node_id → result 的映射
const resultMap = new Map();
for (const r of results) {
if (r.node_id) resultMap.set(r.node_id, r);
}
// 处理每个新节点
for (const entry of newEntries) {
const result = resultMap.get(entry.id);
if (!result) {
// LLM 未返回此节点的结果fallback 为 keep
stats.kept++;
continue;
} }
}
}
} else {
// ── 后端模式: 逐条 /api/vector/query ──
for (let i = 0; i < newEntries.length; i++) {
throwIfAborted(signal);
const entry = newEntries[i];
try {
const neighbors = await findSimilarNodesByText(
graph,
entry.text,
embeddingConfig,
neighborCount,
activeNodes.filter((n) => n.id !== entry.id),
signal,
);
neighborsMap.set(entry.id, neighbors);
} catch (e) {
if (isAbortError(e)) throw e;
console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
neighborsMap.set(entry.id, []);
}
}
}
processOneResult(graph, entry, result, stats); // ══════════════════════════════════════════════
// Phase 3: 单次 LLM 批量判定
// ══════════════════════════════════════════════
throwIfAborted(signal);
const userPromptSections = [];
userPromptSections.push(
`本轮共新增 ${newEntries.length} 条记忆,请逐条分析:\n`,
);
for (let i = 0; i < newEntries.length; i++) {
const entry = newEntries[i];
const neighbors = neighborsMap.get(entry.id) || [];
const newNodeFieldsStr = Object.entries(entry.node.fields)
.map(([k, v]) => `${k}: ${v}`)
.join(", ");
// 构建近邻描述
let neighborText;
if (neighbors.length === 0) {
neighborText = " (无近邻命中)";
} else {
neighborText = neighbors
.map((n) => {
const node = getNode(graph, n.nodeId);
if (!node) return null;
const fieldsStr = Object.entries(node.fields)
.map(([k, v]) => `${k}: ${v}`)
.join(", ");
return ` - [${node.id}] 类型=${node.type}, ${fieldsStr} (相似度=${n.score.toFixed(3)})`;
})
.filter(Boolean)
.join("\n");
} }
// 日志 // 检查高相似度
const actionSummary = []; const hasHighSimilarity =
if (stats.merged > 0) actionSummary.push(`合并 ${stats.merged}`); neighbors.length > 0 && neighbors[0].score > conflictThreshold;
if (stats.skipped > 0) actionSummary.push(`跳过 ${stats.skipped}`); const hint = hasHighSimilarity
if (stats.kept > 0) actionSummary.push(`保留 ${stats.kept}`); ? ` ⚠ 最高相似度 ${neighbors[0].score.toFixed(3)} 超过阈值 ${conflictThreshold}`
if (stats.evolved > 0) actionSummary.push(`进化 ${stats.evolved}`); : "";
if (stats.connections > 0) actionSummary.push(`新链接 ${stats.connections}`);
if (stats.updates > 0) actionSummary.push(`回溯更新 ${stats.updates}`);
if (actionSummary.length > 0) { userPromptSections.push(
console.log(`[ST-BME] 记忆整合完成: ${actionSummary.join(', ')}`); [
} `### 新记忆 #${i + 1}`,
`[${entry.id}] 类型=${entry.node.type}, ${newNodeFieldsStr}`,
"近邻记忆:",
neighborText,
hint,
]
.filter(Boolean)
.join("\n"),
);
}
const userPrompt = userPromptSections.join("\n\n");
let decision;
const consolidationPromptBuild = buildTaskPrompt(settings, "consolidation", {
taskName: "consolidation",
candidateNodes: userPrompt,
candidateText: userPrompt,
graphStats: `new_entries=${newEntries.length}, threshold=${conflictThreshold}`,
});
try {
decision = await callLLMForJSON({
systemPrompt: applyTaskRegex(
settings,
"consolidation",
"finalPrompt",
consolidationPromptBuild.systemPrompt ||
customPrompt ||
CONSOLIDATION_SYSTEM_PROMPT,
),
userPrompt,
maxRetries: 1,
signal,
taskType: "consolidation",
additionalMessages: consolidationPromptBuild.customMessages || [],
});
} catch (e) {
if (isAbortError(e)) throw e;
console.error("[ST-BME] 记忆整合 LLM 调用失败:", e);
stats.kept = newEntries.length;
return stats; return stats;
}
// ══════════════════════════════════════════════
// Phase 4: 逐个处理结果
// ══════════════════════════════════════════════
// 解析 LLM 返回——兼容单条和批量格式
let results;
if (Array.isArray(decision?.results)) {
results = decision.results;
} else if (decision?.action) {
// 单条返回格式LLM 可能忽略 results 包装)
results = [{ ...decision, node_id: newEntries[0]?.id }];
} else {
console.warn("[ST-BME] 记忆整合: LLM 返回格式异常,全部 keep");
stats.kept = newEntries.length;
return stats;
}
// 建立 node_id → result 的映射
const resultMap = new Map();
for (const r of results) {
if (r.node_id) resultMap.set(r.node_id, r);
}
// 处理每个新节点
for (const entry of newEntries) {
const result = resultMap.get(entry.id);
if (!result) {
// LLM 未返回此节点的结果fallback 为 keep
stats.kept++;
continue;
}
processOneResult(graph, entry, result, stats);
}
// 日志
const actionSummary = [];
if (stats.merged > 0) actionSummary.push(`合并 ${stats.merged}`);
if (stats.skipped > 0) actionSummary.push(`跳过 ${stats.skipped}`);
if (stats.kept > 0) actionSummary.push(`保留 ${stats.kept}`);
if (stats.evolved > 0) actionSummary.push(`进化 ${stats.evolved}`);
if (stats.connections > 0) actionSummary.push(`新链接 ${stats.connections}`);
if (stats.updates > 0) actionSummary.push(`回溯更新 ${stats.updates}`);
if (actionSummary.length > 0) {
console.log(`[ST-BME] 记忆整合完成: ${actionSummary.join(", ")}`);
}
return stats;
} }
/** /**
* 处理单个节点的整合结果 * 处理单个节点的整合结果
*/ */
function processOneResult(graph, entry, result, stats) { function processOneResult(graph, entry, result, stats) {
const { id: newId, node: newNode } = entry; const { id: newId, node: newNode } = entry;
// ── 处理 action ── // ── 处理 action ──
switch (result.action) { switch (result.action) {
case 'skip': { case "skip": {
console.log(`[ST-BME] 记忆整合: skip (重复) — ${newId}`); console.log(`[ST-BME] 记忆整合: skip (重复) — ${newId}`);
newNode.archived = true; newNode.archived = true;
stats.skipped++; stats.skipped++;
break; break;
}
case 'merge': {
const targetId = result.merge_target_id;
const targetNode = targetId ? getNode(graph, targetId) : null;
if (targetNode && !targetNode.archived) {
console.log(`[ST-BME] 记忆整合: merge ${newId}${targetId}`);
if (result.merged_fields && typeof result.merged_fields === 'object') {
for (const [key, value] of Object.entries(result.merged_fields)) {
if (value != null && value !== '') {
targetNode.fields[key] = value;
}
}
} else {
for (const [key, value] of Object.entries(newNode.fields)) {
if (value != null && value !== '' && !targetNode.fields[key]) {
targetNode.fields[key] = value;
}
}
}
if (Number.isFinite(newNode.seq) && newNode.seq > (targetNode.seq || 0)) {
targetNode.seq = newNode.seq;
}
targetNode.embedding = null;
newNode.archived = true;
stats.merged++;
} else {
console.warn(`[ST-BME] 记忆整合: merge target ${targetId} 不存在,回退为 keep`);
stats.kept++;
}
break;
}
case 'keep':
default: {
stats.kept++;
break;
}
} }
// ── 处理 evolution ── case "merge": {
const evolution = result.evolution; const targetId = result.merge_target_id;
if (evolution?.should_evolve && !newNode.archived) { const targetNode = targetId ? getNode(graph, targetId) : null;
stats.evolved++;
console.log(`[ST-BME] 记忆整合/进化触发: ${result.reason || '(无理由)'}`);
if (Array.isArray(evolution.connections)) { if (targetNode && !targetNode.archived) {
for (const targetId of evolution.connections) { console.log(`[ST-BME] 记忆整合: merge ${newId}${targetId}`);
if (!getNode(graph, targetId)) continue;
const edge = createEdge({ if (result.merged_fields && typeof result.merged_fields === "object") {
fromId: newId, for (const [key, value] of Object.entries(result.merged_fields)) {
toId: targetId, if (value != null && value !== "") {
relation: 'related', targetNode.fields[key] = value;
strength: 0.7,
});
if (addEdge(graph, edge)) {
stats.connections++;
}
} }
}
} else {
for (const [key, value] of Object.entries(newNode.fields)) {
if (value != null && value !== "" && !targetNode.fields[key]) {
targetNode.fields[key] = value;
}
}
} }
if (Array.isArray(evolution.neighbor_updates)) { if (
for (const update of evolution.neighbor_updates) { Number.isFinite(newNode.seq) &&
if (!update.nodeId) continue; newNode.seq > (targetNode.seq || 0)
const oldNode = getNode(graph, update.nodeId); ) {
if (!oldNode || oldNode.archived) continue; targetNode.seq = newNode.seq;
let changed = false;
if (update.newContext && typeof update.newContext === 'string') {
if (oldNode.fields.state !== undefined) {
oldNode.fields.state = update.newContext;
changed = true;
} else if (oldNode.fields.summary !== undefined) {
oldNode.fields.summary = update.newContext;
changed = true;
} else if (oldNode.fields.core_note !== undefined) {
oldNode.fields.core_note = update.newContext;
changed = true;
}
}
if (update.newTags && Array.isArray(update.newTags)) {
oldNode.clusters = update.newTags;
changed = true;
}
if (changed) {
oldNode.embedding = null;
if (!oldNode._evolutionHistory) oldNode._evolutionHistory = [];
oldNode._evolutionHistory.push({
triggeredBy: newId,
timestamp: Date.now(),
reason: result.reason || '',
});
stats.updates++;
}
}
} }
const targetRange = Array.isArray(targetNode.seqRange)
? targetNode.seqRange
: [targetNode.seq || 0, targetNode.seq || 0];
const newRange = Array.isArray(newNode.seqRange)
? newNode.seqRange
: [newNode.seq || 0, newNode.seq || 0];
targetNode.seqRange = [
Math.min(targetRange[0], newRange[0]),
Math.max(targetRange[1], newRange[1]),
];
targetNode.embedding = null;
newNode.archived = true;
stats.merged++;
} else {
console.warn(
`[ST-BME] 记忆整合: merge target ${targetId} 不存在,回退为 keep`,
);
stats.kept++;
}
break;
} }
case "keep":
default: {
stats.kept++;
break;
}
}
// ── 处理 evolution ──
const evolution = result.evolution;
if (evolution?.should_evolve && !newNode.archived) {
stats.evolved++;
console.log(`[ST-BME] 记忆整合/进化触发: ${result.reason || "(无理由)"}`);
if (Array.isArray(evolution.connections)) {
for (const targetId of evolution.connections) {
if (!getNode(graph, targetId)) continue;
const edge = createEdge({
fromId: newId,
toId: targetId,
relation: "related",
strength: 0.7,
});
if (addEdge(graph, edge)) {
stats.connections++;
}
}
}
if (Array.isArray(evolution.neighbor_updates)) {
for (const update of evolution.neighbor_updates) {
if (!update.nodeId) continue;
const oldNode = getNode(graph, update.nodeId);
if (!oldNode || oldNode.archived) continue;
let changed = false;
if (update.newContext && typeof update.newContext === "string") {
if (oldNode.fields.state !== undefined) {
oldNode.fields.state = update.newContext;
changed = true;
} else if (oldNode.fields.summary !== undefined) {
oldNode.fields.summary = update.newContext;
changed = true;
} else if (oldNode.fields.core_note !== undefined) {
oldNode.fields.core_note = update.newContext;
changed = true;
}
}
if (update.newTags && Array.isArray(update.newTags)) {
oldNode.clusters = update.newTags;
changed = true;
}
if (changed) {
oldNode.embedding = null;
if (!oldNode._evolutionHistory) oldNode._evolutionHistory = [];
oldNode._evolutionHistory.push({
triggeredBy: newId,
timestamp: Date.now(),
reason: result.reason || "",
});
stats.updates++;
}
}
}
}
} }

View File

@@ -15,17 +15,11 @@ import {
updateNode, updateNode,
} from "./graph.js"; } from "./graph.js";
import { callLLMForJSON } from "./llm.js"; import { callLLMForJSON } from "./llm.js";
import { import { ensureEventTitle, getNodeDisplayName } from "./node-labels.js";
ensureEventTitle,
getNodeDisplayName,
} from "./node-labels.js";
import { buildTaskPrompt } from "./prompt-builder.js"; import { buildTaskPrompt } from "./prompt-builder.js";
import { applyTaskRegex } from "./task-regex.js";
import { RELATION_TYPES } from "./schema.js"; import { RELATION_TYPES } from "./schema.js";
import { import { applyTaskRegex } from "./task-regex.js";
buildNodeVectorText, import { buildNodeVectorText, isDirectVectorConfig } from "./vector-index.js";
isDirectVectorConfig,
} from "./vector-index.js";
function createAbortError(message = "操作已终止") { function createAbortError(message = "操作已终止") {
const error = new Error(message); const error = new Error(message);
@@ -39,9 +33,7 @@ function isAbortError(error) {
function throwIfAborted(signal) { function throwIfAborted(signal) {
if (signal?.aborted) { if (signal?.aborted) {
throw signal.reason instanceof Error throw signal.reason instanceof Error ? signal.reason : createAbortError();
? signal.reason
: createAbortError();
} }
} }
@@ -84,8 +76,6 @@ export async function extractMemories({
}; };
} }
const effectiveStartSeq = Number.isFinite(startSeq) const effectiveStartSeq = Number.isFinite(startSeq)
? startSeq ? startSeq
: (messages.find((m) => Number.isFinite(m.seq))?.seq ?? : (messages.find((m) => Number.isFinite(m.seq))?.seq ??
@@ -134,7 +124,9 @@ export async function extractMemories({
settings, settings,
"extract", "extract",
"finalPrompt", "finalPrompt",
promptBuild.systemPrompt || extractPrompt || buildDefaultExtractPrompt(schema), promptBuild.systemPrompt ||
extractPrompt ||
buildDefaultExtractPrompt(schema),
); );
// 用户提示词 // 用户提示词
@@ -175,12 +167,11 @@ export async function extractMemories({
}; };
} }
// 执行操作 // 执行操作
const stats = { newNodes: 0, updatedNodes: 0, newEdges: 0 }; const stats = { newNodes: 0, updatedNodes: 0, newEdges: 0 };
const newNodeIds = []; // v2: 收集新建节点 ID用于进化引擎 const newNodeIds = []; // v2: 收集新建节点 ID用于进化引擎
const refMap = new Map(); const refMap = new Map();
const operationErrors = [];
for (const op of result.operations) { for (const op of result.operations) {
try { try {
@@ -206,14 +197,29 @@ export async function extractMemories({
case "_skip": case "_skip":
// Mem0 对照判定为重复,跳过 // Mem0 对照判定为重复,跳过
break; break;
default: default: {
console.warn(`[ST-BME] 未知操作类型: ${op.action}`); const message = `[ST-BME] 未知操作类型: ${op?.action ?? "<missing>"}`;
console.warn(message, op);
operationErrors.push(message);
break;
}
} }
} catch (e) { } catch (e) {
console.error(`[ST-BME] 操作执行失败:`, op, e); console.error(`[ST-BME] 操作执行失败:`, op, e);
operationErrors.push(e?.message || String(e));
} }
} }
if (operationErrors.length > 0) {
return {
success: false,
error: operationErrors.join(" | "),
...stats,
newNodeIds,
processedRange: [effectiveStartSeq, effectiveEndSeq],
};
}
// 为新建节点生成 embedding。失败不应回滚整批图谱写入。 // 为新建节点生成 embedding。失败不应回滚整批图谱写入。
try { try {
await generateNodeEmbeddings(graph, embeddingConfig, signal); await generateNodeEmbeddings(graph, embeddingConfig, signal);
@@ -248,7 +254,7 @@ export async function extractMemories({
*/ */
function handleCreate(graph, op, seq, schema, refMap, stats) { function handleCreate(graph, op, seq, schema, refMap, stats) {
const normalizedFields = const normalizedFields =
op.type === "event" ? ensureEventTitle(op.fields || {}) : (op.fields || {}); op.type === "event" ? ensureEventTitle(op.fields || {}) : op.fields || {};
const typeDef = schema.find((s) => s.id === op.type); const typeDef = schema.find((s) => s.id === op.type);
if (!typeDef) { if (!typeDef) {
console.warn(`[ST-BME] 未知节点类型: ${op.type}`); console.warn(`[ST-BME] 未知节点类型: ${op.type}`);
@@ -480,7 +486,9 @@ async function generateNodeEmbeddings(graph, embeddingConfig, signal) {
if (needsEmbedding.length === 0) return; if (needsEmbedding.length === 0) return;
const texts = needsEmbedding.map((node) => buildNodeVectorText(node) || node.type); const texts = needsEmbedding.map(
(node) => buildNodeVectorText(node) || node.type,
);
console.log(`[ST-BME] 为 ${texts.length} 个节点生成 embedding`); console.log(`[ST-BME] 为 ${texts.length} 个节点生成 embedding`);
@@ -591,7 +599,14 @@ function buildDefaultExtractPrompt(schema) {
* @param {number} params.currentSeq * @param {number} params.currentSeq
* @returns {Promise<void>} * @returns {Promise<void>}
*/ */
export async function generateSynopsis({ graph, schema, currentSeq, customPrompt, signal, settings = {} }) { export async function generateSynopsis({
graph,
schema,
currentSeq,
customPrompt,
signal,
settings = {},
}) {
const eventNodes = getActiveNodes(graph, "event").sort( const eventNodes = getActiveNodes(graph, "event").sort(
(a, b) => a.seq - b.seq, (a, b) => a.seq - b.seq,
); );
@@ -623,11 +638,13 @@ export async function generateSynopsis({ graph, schema, currentSeq, customPrompt
settings, settings,
"synopsis", "synopsis",
"finalPrompt", "finalPrompt",
synopsisPromptBuild.systemPrompt || customPrompt || [ synopsisPromptBuild.systemPrompt ||
"你是故事概要生成器。根据事件线、角色和主线生成简洁的前情提要。", customPrompt ||
'输出 JSON{"summary": "前情提要文本200字以内"}', [
"要求:涵盖核心冲突、关键转折、主要角色当前状态。", "你是故事概要生成器。根据事件线、角色和主线生成简洁的前情提要。",
].join("\n"), '输出 JSON{"summary": "前情提要文本200字以内"}',
"要求:涵盖核心冲突、关键转折、主要角色当前状态。",
].join("\n"),
); );
const result = await callLLMForJSON({ const result = await callLLMForJSON({
@@ -677,7 +694,13 @@ export async function generateSynopsis({ graph, schema, currentSeq, customPrompt
} }
} }
export async function generateReflection({ graph, currentSeq, customPrompt, signal, settings = {} }) { export async function generateReflection({
graph,
currentSeq,
customPrompt,
signal,
settings = {},
}) {
const recentEvents = getActiveNodes(graph, "event") const recentEvents = getActiveNodes(graph, "event")
.sort((a, b) => b.seq - a.seq) .sort((a, b) => b.seq - a.seq)
.slice(0, 6) .slice(0, 6)
@@ -728,14 +751,16 @@ export async function generateReflection({ graph, currentSeq, customPrompt, sign
settings, settings,
"reflection", "reflection",
"finalPrompt", "finalPrompt",
reflectionPromptBuild.systemPrompt || customPrompt || [ reflectionPromptBuild.systemPrompt ||
"你是 RP 长期记忆系统的反思生成器。", customPrompt ||
'输出严格 JSON{"insight":"...","trigger":"...","suggestion":"...","importance":1-10}', [
"insight 应总结最近情节中最值得长期保留的变化、关系趋势或潜在线索。", "你是 RP 长期记忆系统的反思生成器。",
"trigger 说明触发这条反思的关键事件或矛盾。", '输出严格 JSON{"insight":"...","trigger":"...","suggestion":"...","importance":1-10}',
"suggestion 给出后续检索或叙事上值得关注的提示。", "insight 应总结最近情节中最值得长期保留的变化、关系趋势或潜在线索。",
"不要复述全部事件,要提炼高层结论。", "trigger 说明触发这条反思的关键事件或矛盾。",
].join("\n"), "suggestion 给出后续检索或叙事上值得关注的提示。",
"不要复述全部事件,要提炼高层结论。",
].join("\n"),
); );
const result = await callLLMForJSON({ const result = await callLLMForJSON({

470
index.js
View File

@@ -33,6 +33,10 @@ import { estimateTokens, formatInjection } from "./injector.js";
import { fetchMemoryLLMModels, testLLMConnection } from "./llm.js"; import { fetchMemoryLLMModels, testLLMConnection } from "./llm.js";
import { getNodeDisplayName } from "./node-labels.js"; import { getNodeDisplayName } from "./node-labels.js";
import { showManagedBmeNotice } from "./notice.js"; import { showManagedBmeNotice } from "./notice.js";
import {
createDefaultTaskProfiles,
migrateLegacyTaskProfiles,
} from "./prompt-profiles.js";
import { retrieve } from "./retriever.js"; import { retrieve } from "./retriever.js";
import { import {
appendBatchJournal, appendBatchJournal,
@@ -59,7 +63,6 @@ import {
testVectorConnection, testVectorConnection,
validateVectorConfig, validateVectorConfig,
} from "./vector-index.js"; } from "./vector-index.js";
import { createDefaultTaskProfiles, migrateLegacyTaskProfiles } from "./prompt-profiles.js";
// 操控面板模块(动态加载,防止加载失败崩溃整个扩展) // 操控面板模块(动态加载,防止加载失败崩溃整个扩展)
let _panelModule = null; let _panelModule = null;
@@ -198,9 +201,8 @@ let sendIntentHookRetryTimer = null;
let pendingHistoryRecoveryTimer = null; let pendingHistoryRecoveryTimer = null;
let pendingHistoryRecoveryTrigger = ""; let pendingHistoryRecoveryTrigger = "";
let pendingHistoryMutationCheckTimers = []; let pendingHistoryMutationCheckTimers = [];
let skipBeforeCombineRecallUntil = 0; const generationRecallTransactions = new Map();
let lastPreGenerationRecallKey = ""; const GENERATION_RECALL_TRANSACTION_TTL_MS = 15000;
let lastPreGenerationRecallAt = 0;
const stageNoticeHandles = { const stageNoticeHandles = {
extraction: null, extraction: null,
vector: null, vector: null,
@@ -956,6 +958,15 @@ function updateProcessedHistorySnapshot(chat, lastProcessedAssistantFloor) {
currentGraph.lastProcessedSeq = lastProcessedAssistantFloor; currentGraph.lastProcessedSeq = lastProcessedAssistantFloor;
} }
function shouldAdvanceProcessedHistory(batchStatus) {
if (!batchStatus || typeof batchStatus !== "object") return false;
return (
batchStatus.completed === true &&
batchStatus.outcome === "success" &&
batchStatus.consistency === "strong"
);
}
function computePostProcessArtifacts( function computePostProcessArtifacts(
beforeSnapshot, beforeSnapshot,
afterSnapshot, afterSnapshot,
@@ -1648,6 +1659,111 @@ function buildPreGenerationRecallKey(type, options = {}) {
].join(":"); ].join(":");
} }
function cleanupGenerationRecallTransactions(now = Date.now()) {
for (const [
transactionId,
transaction,
] of generationRecallTransactions.entries()) {
if (
!transaction ||
now - (transaction.updatedAt || 0) > GENERATION_RECALL_TRANSACTION_TTL_MS
) {
generationRecallTransactions.delete(transactionId);
}
}
}
function buildGenerationRecallTransactionId(chatId, generationType, recallKey) {
return [
String(chatId || ""),
String(generationType || "normal").trim() || "normal",
String(recallKey || ""),
].join(":");
}
function beginGenerationRecallTransaction({
chatId,
generationType = "normal",
recallKey = "",
} = {}) {
const normalizedChatId = String(chatId || "");
const normalizedGenerationType =
String(generationType || "normal").trim() || "normal";
const normalizedRecallKey = String(recallKey || "");
if (!normalizedChatId || !normalizedRecallKey) return null;
cleanupGenerationRecallTransactions();
const transactionId = buildGenerationRecallTransactionId(
normalizedChatId,
normalizedGenerationType,
normalizedRecallKey,
);
const now = Date.now();
const transaction = generationRecallTransactions.get(transactionId) || {
id: transactionId,
chatId: normalizedChatId,
generationType: normalizedGenerationType,
recallKey: normalizedRecallKey,
hookStates: {},
createdAt: now,
};
transaction.updatedAt = now;
generationRecallTransactions.set(transactionId, transaction);
return transaction;
}
function markGenerationRecallTransactionHookState(
transaction,
hookName,
state = "completed",
) {
if (!transaction?.id || !hookName) return transaction;
transaction.hookStates ||= {};
transaction.hookStates[hookName] = state;
transaction.updatedAt = Date.now();
generationRecallTransactions.set(transaction.id, transaction);
return transaction;
}
function shouldRunRecallForTransaction(transaction, hookName) {
if (!hookName) return true;
if (!transaction) return true;
const hookStates = transaction.hookStates || {};
if (hookStates[hookName] === "completed") {
return false;
}
if (
hookName === "GENERATE_BEFORE_COMBINE_PROMPTS" &&
hookStates.GENERATION_AFTER_COMMANDS === "completed"
) {
return false;
}
return true;
}
function createGenerationRecallContext({
hookName,
generationType = "normal",
recallOptions = {},
chatId = getCurrentChatId(),
} = {}) {
const recallKey =
recallOptions.recallKey ||
buildPreGenerationRecallKey(generationType, recallOptions);
const transaction = beginGenerationRecallTransaction({
chatId,
generationType,
recallKey,
});
return {
hookName,
generationType,
recallKey,
transaction,
shouldRun: shouldRunRecallForTransaction(transaction, hookName),
};
}
function getCurrentChatSeq(context = getContext()) { function getCurrentChatSeq(context = getContext()) {
const chat = context?.chat; const chat = context?.chat;
if (Array.isArray(chat) && chat.length > 0) { if (Array.isArray(chat) && chat.length > 0) {
@@ -1656,19 +1772,129 @@ function getCurrentChatSeq(context = getContext()) {
return currentGraph?.lastProcessedSeq ?? 0; return currentGraph?.lastProcessedSeq ?? 0;
} }
const BATCH_STAGE_ORDER = ["core", "structural", "semantic", "finalize"];
const BATCH_STAGE_SEVERITY = {
success: 0,
partial: 1,
failed: 2,
};
function createBatchStageStatus(stage, consistency = "strong") {
return {
stage,
outcome: "success",
consistency,
warnings: [],
errors: [],
artifacts: [],
};
}
function createBatchStatusSkeleton({ processedRange, extractionCountBefore }) {
return {
model: "layered-batch-v1",
processedRange: Array.isArray(processedRange)
? [...processedRange]
: [-1, -1],
extractionCountBefore: Number.isFinite(extractionCountBefore)
? extractionCountBefore
: extractionCount,
extractionCountAfter: Number.isFinite(extractionCount)
? extractionCount
: 0,
stages: {
core: createBatchStageStatus("core", "strong"),
structural: createBatchStageStatus("structural", "weak"),
semantic: createBatchStageStatus("semantic", "weak"),
finalize: createBatchStageStatus("finalize", "strong"),
},
outcome: "success",
consistency: "strong",
completed: false,
warnings: [],
errors: [],
};
}
function setBatchStageOutcome(status, stage, outcome, message = "") {
const stageStatus = status?.stages?.[stage];
if (!stageStatus) return;
const nextSeverity = BATCH_STAGE_SEVERITY[outcome] ?? 0;
const previousSeverity = BATCH_STAGE_SEVERITY[stageStatus.outcome] ?? 0;
if (nextSeverity >= previousSeverity) {
stageStatus.outcome = outcome;
}
if (!message) return;
if (outcome === "failed") {
stageStatus.errors.push(message);
} else if (outcome === "partial") {
stageStatus.warnings.push(message);
}
}
function pushBatchStageArtifact(status, stage, artifact) {
const stageStatus = status?.stages?.[stage];
if (!stageStatus || !artifact) return;
if (!stageStatus.artifacts.includes(artifact)) {
stageStatus.artifacts.push(artifact);
}
}
function finalizeBatchStatus(status) {
const stages = status?.stages || {};
const structuralOutcome = stages.structural?.outcome || "success";
const semanticOutcome = stages.semantic?.outcome || "success";
const finalizeOutcome = stages.finalize?.outcome || "failed";
const outcomeList = BATCH_STAGE_ORDER.map(
(stage) => stages[stage]?.outcome || "success",
);
if (finalizeOutcome !== "success") {
status.outcome = "failed";
} else if (outcomeList.includes("failed")) {
status.outcome = "failed";
} else if (structuralOutcome === "partial" || semanticOutcome === "partial") {
status.outcome = "partial";
} else {
status.outcome = "success";
}
status.consistency =
finalizeOutcome === "success" &&
stages.core?.outcome === "success" &&
stages.structural?.outcome === "success"
? "strong"
: "weak";
status.completed = finalizeOutcome === "success";
status.extractionCountAfter = Number.isFinite(extractionCount)
? extractionCount
: status.extractionCountAfter;
status.warnings = BATCH_STAGE_ORDER.flatMap(
(stage) => stages[stage]?.warnings || [],
);
status.errors = BATCH_STAGE_ORDER.flatMap(
(stage) => stages[stage]?.errors || [],
);
return status;
}
async function handleExtractionSuccess( async function handleExtractionSuccess(
result, result,
endIdx, endIdx,
settings, settings,
signal = undefined, signal = undefined,
status = createBatchStatusSkeleton({
processedRange: [endIdx, endIdx],
extractionCountBefore: extractionCount,
}),
) { ) {
const postProcessArtifacts = []; const postProcessArtifacts = [];
const warnings = [];
throwIfAborted(signal, "提取已终止"); throwIfAborted(signal, "提取已终止");
extractionCount++; extractionCount++;
ensureCurrentGraphRuntimeState(); ensureCurrentGraphRuntimeState();
currentGraph.historyState.extractionCount = extractionCount; currentGraph.historyState.extractionCount = extractionCount;
updateLastExtractedItems(result.newNodeIds || []); updateLastExtractedItems(result.newNodeIds || []);
setBatchStageOutcome(status, "core", "success");
if (settings.enableConsolidation && result.newNodeIds?.length > 0) { if (settings.enableConsolidation && result.newNodeIds?.length > 0) {
try { try {
@@ -1684,8 +1910,16 @@ async function handleExtractionSuccess(
signal, signal,
}); });
postProcessArtifacts.push("consolidation"); postProcessArtifacts.push("consolidation");
pushBatchStageArtifact(status, "structural", "consolidation");
} catch (e) { } catch (e) {
if (isAbortError(e)) throw e; if (isAbortError(e)) throw e;
const message = e?.message || String(e) || "记忆整合阶段失败";
setBatchStageOutcome(
status,
"structural",
"partial",
`记忆整合失败: ${message}`,
);
console.error("[ST-BME] 记忆整合失败:", e); console.error("[ST-BME] 记忆整合失败:", e);
} }
} }
@@ -1703,8 +1937,16 @@ async function handleExtractionSuccess(
signal, signal,
}); });
postProcessArtifacts.push("synopsis"); postProcessArtifacts.push("synopsis");
pushBatchStageArtifact(status, "semantic", "synopsis");
} catch (e) { } catch (e) {
if (isAbortError(e)) throw e; if (isAbortError(e)) throw e;
const message = e?.message || String(e) || "概要生成阶段失败";
setBatchStageOutcome(
status,
"semantic",
"failed",
`概要生成失败: ${message}`,
);
console.error("[ST-BME] 概要生成失败:", e); console.error("[ST-BME] 概要生成失败:", e);
} }
} }
@@ -1721,8 +1963,16 @@ async function handleExtractionSuccess(
signal, signal,
}); });
postProcessArtifacts.push("reflection"); postProcessArtifacts.push("reflection");
pushBatchStageArtifact(status, "semantic", "reflection");
} catch (e) { } catch (e) {
if (isAbortError(e)) throw e; if (isAbortError(e)) throw e;
const message = e?.message || String(e) || "反思生成阶段失败";
setBatchStageOutcome(
status,
"semantic",
"failed",
`反思生成失败: ${message}`,
);
console.error("[ST-BME] 反思生成失败:", e); console.error("[ST-BME] 反思生成失败:", e);
} }
} }
@@ -1734,7 +1984,15 @@ async function handleExtractionSuccess(
try { try {
sleepCycle(currentGraph, settings); sleepCycle(currentGraph, settings);
postProcessArtifacts.push("sleep"); postProcessArtifacts.push("sleep");
pushBatchStageArtifact(status, "semantic", "sleep");
} catch (e) { } catch (e) {
const message = e?.message || String(e) || "主动遗忘阶段失败";
setBatchStageOutcome(
status,
"semantic",
"failed",
`主动遗忘失败: ${message}`,
);
console.error("[ST-BME] 主动遗忘失败:", e); console.error("[ST-BME] 主动遗忘失败:", e);
} }
} }
@@ -1752,27 +2010,63 @@ async function handleExtractionSuccess(
); );
if (compressionResult.created > 0 || compressionResult.archived > 0) { if (compressionResult.created > 0 || compressionResult.archived > 0) {
postProcessArtifacts.push("compression"); postProcessArtifacts.push("compression");
pushBatchStageArtifact(status, "structural", "compression");
} }
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
const message = error?.message || String(error) || "压缩阶段失败"; const message = error?.message || String(error) || "压缩阶段失败";
warnings.push(`压缩阶段失败: ${message}`); setBatchStageOutcome(
status,
"structural",
"partial",
`压缩阶段失败: ${message}`,
);
console.error("[ST-BME] 记忆压缩失败:", error); console.error("[ST-BME] 记忆压缩失败:", error);
} }
const vectorSync = await syncVectorState({ signal }); let vectorSync = null;
try {
vectorSync = await syncVectorState({ signal });
} catch (error) {
if (isAbortError(error)) throw error;
const message = error?.message || String(error) || "向量同步阶段失败";
setBatchStageOutcome(
status,
"finalize",
"failed",
`向量同步失败: ${message}`,
);
return {
postProcessArtifacts,
vectorHashesInserted: [],
vectorStats: getVectorIndexStats(currentGraph),
vectorError: message,
warnings: status.warnings,
batchStatus: finalizeBatchStatus(status),
};
}
if (vectorSync?.aborted) { if (vectorSync?.aborted) {
throw createAbortError(vectorSync.error || "提取已终止"); throw createAbortError(vectorSync.error || "提取已终止");
} }
if (vectorSync?.error) { if (vectorSync?.error) {
warnings.push(`向量同步失败: ${vectorSync.error}`); setBatchStageOutcome(
status,
"finalize",
"failed",
`向量同步失败: ${vectorSync.error}`,
);
} else {
setBatchStageOutcome(status, "finalize", "success");
} }
return { return {
postProcessArtifacts, postProcessArtifacts,
vectorHashesInserted: vectorSync?.insertedHashes || [], vectorHashesInserted: vectorSync?.insertedHashes || [],
vectorStats: vectorSync?.stats || getVectorIndexStats(currentGraph), vectorStats: vectorSync?.stats || getVectorIndexStats(currentGraph),
vectorError: vectorSync?.error || "", vectorError: vectorSync?.error || "",
warnings, warnings: status.warnings,
batchStatus: finalizeBatchStatus(status),
}; };
} }
@@ -2160,6 +2454,15 @@ async function prepareVectorStateForReplay(
currentGraph.vectorIndexState.nodeToHash = {}; currentGraph.vectorIndexState.nodeToHash = {};
} }
currentGraph.vectorIndexState.dirty = true; currentGraph.vectorIndexState.dirty = true;
if (!currentGraph.vectorIndexState.dirtyReason) {
currentGraph.vectorIndexState.dirtyReason = skipBackendPurge
? "history-recovery-replay"
: "history-recovery-reset";
}
if (fullReset) {
currentGraph.vectorIndexState.replayRequiredNodeIds = [];
currentGraph.vectorIndexState.pendingRepairFromFloor = 0;
}
currentGraph.vectorIndexState.lastWarning = skipBackendPurge currentGraph.vectorIndexState.lastWarning = skipBackendPurge
? "历史恢复后需要修复受影响后缀的后端向量索引" ? "历史恢复后需要修复受影响后缀的后端向量索引"
: "历史恢复后需要重建后端向量索引"; : "历史恢复后需要重建后端向量索引";
@@ -2169,7 +2472,10 @@ async function prepareVectorStateForReplay(
if (fullReset) { if (fullReset) {
currentGraph.vectorIndexState.hashToNodeId = {}; currentGraph.vectorIndexState.hashToNodeId = {};
currentGraph.vectorIndexState.nodeToHash = {}; currentGraph.vectorIndexState.nodeToHash = {};
currentGraph.vectorIndexState.replayRequiredNodeIds = [];
currentGraph.vectorIndexState.dirty = true; currentGraph.vectorIndexState.dirty = true;
currentGraph.vectorIndexState.dirtyReason = "history-recovery-reset";
currentGraph.vectorIndexState.pendingRepairFromFloor = 0;
currentGraph.vectorIndexState.lastWarning = currentGraph.vectorIndexState.lastWarning =
"历史恢复后需要重嵌当前聊天向量"; "历史恢复后需要重嵌当前聊天向量";
} }
@@ -2189,6 +2495,10 @@ async function executeExtractionBatch({
const extractionCountBefore = extractionCount; const extractionCountBefore = extractionCount;
const beforeSnapshot = cloneGraphSnapshot(currentGraph); const beforeSnapshot = cloneGraphSnapshot(currentGraph);
const messages = buildExtractionMessages(chat, startIdx, endIdx, settings); const messages = buildExtractionMessages(chat, startIdx, endIdx, settings);
const batchStatus = createBatchStatusSkeleton({
processedRange: [startIdx, endIdx],
extractionCountBefore,
});
console.log( console.log(
`[ST-BME] 开始提取: 楼层 ${startIdx}-${endIdx}` + `[ST-BME] 开始提取: 楼层 ${startIdx}-${endIdx}` +
@@ -2211,21 +2521,41 @@ async function executeExtractionBatch({
}); });
if (!result.success) { if (!result.success) {
setBatchStageOutcome(
batchStatus,
"core",
"failed",
result?.error || "提取阶段未返回有效操作",
);
finalizeBatchStatus(batchStatus);
currentGraph.historyState.lastBatchStatus = batchStatus;
return { return {
success: false, success: false,
result, result,
effects: null, effects: null,
batchStatus,
error: result?.error || "提取阶段未返回有效操作", error: result?.error || "提取阶段未返回有效操作",
}; };
} }
setBatchStageOutcome(batchStatus, "core", "success");
const effects = await handleExtractionSuccess( const effects = await handleExtractionSuccess(
result, result,
endIdx, endIdx,
settings, settings,
signal, signal,
batchStatus,
); );
updateProcessedHistorySnapshot(chat, endIdx); const finalizedBatchStatus =
effects?.batchStatus || finalizeBatchStatus(batchStatus);
currentGraph.historyState.lastBatchStatus = {
...finalizedBatchStatus,
historyAdvanced: shouldAdvanceProcessedHistory(finalizedBatchStatus),
};
if (currentGraph.historyState.lastBatchStatus.historyAdvanced) {
updateProcessedHistorySnapshot(chat, endIdx);
}
const afterSnapshot = cloneGraphSnapshot(currentGraph); const afterSnapshot = cloneGraphSnapshot(currentGraph);
const postProcessArtifacts = computePostProcessArtifacts( const postProcessArtifacts = computePostProcessArtifacts(
@@ -2245,10 +2575,15 @@ async function executeExtractionBatch({
saveGraphToChat(); saveGraphToChat();
return { return {
success: true, success: finalizedBatchStatus.completed,
result, result,
effects, effects,
error: effects?.vectorError || "", batchStatus: finalizedBatchStatus,
error: finalizedBatchStatus.completed
? ""
: effects?.vectorError ||
finalizedBatchStatus.errors?.[0] ||
"批次未完成 finalize 闭环",
}; };
} }
@@ -2289,18 +2624,41 @@ async function replayExtractionFromHistory(chat, settings, signal = undefined) {
return replayedBatches; return replayedBatches;
} }
function collectAffectedInsertedHashes(affectedJournals = []) { function applyRecoveryPlanToVectorState(
const hashes = new Set(); recoveryPlan,
for (const journal of affectedJournals) { dirtyFallbackFloor = null,
const insertedHashes = ) {
journal?.vectorDelta?.insertedHashes || ensureCurrentGraphRuntimeState();
journal?.vectorHashesInserted || const vectorState = currentGraph.vectorIndexState;
[]; const replayRequiredNodeIds = new Set(
for (const hash of insertedHashes) { Array.isArray(vectorState.replayRequiredNodeIds)
if (hash) hashes.add(hash); ? vectorState.replayRequiredNodeIds.filter(Boolean)
} : [],
);
for (const nodeId of recoveryPlan?.replayRequiredNodeIds || []) {
if (nodeId) replayRequiredNodeIds.add(nodeId);
} }
return [...hashes];
vectorState.replayRequiredNodeIds = [...replayRequiredNodeIds];
vectorState.dirty = true;
vectorState.dirtyReason =
recoveryPlan?.dirtyReason ||
vectorState.dirtyReason ||
"history-recovery-replay";
const fallbackFloor = Number.isFinite(dirtyFallbackFloor)
? dirtyFallbackFloor
: currentGraph.historyState?.historyDirtyFrom;
vectorState.pendingRepairFromFloor = Number.isFinite(
recoveryPlan?.pendingRepairFromFloor,
)
? recoveryPlan.pendingRepairFromFloor
: Number.isFinite(fallbackFloor)
? fallbackFloor
: null;
vectorState.lastWarning = recoveryPlan?.legacyGapFallback
? "历史恢复检测到 legacy-gap向量索引需按受影响后缀修复"
: "历史恢复后需要修复受影响后缀的向量索引";
} }
function rollbackAffectedJournals(graph, affectedJournals = []) { function rollbackAffectedJournals(graph, affectedJournals = []) {
@@ -2370,18 +2728,23 @@ async function recoverHistoryIfNeeded(trigger = "history-recovery") {
recoveryPath = "reverse-journal"; recoveryPath = "reverse-journal";
affectedBatchCount = recoveryPoint.affectedBatchCount || 0; affectedBatchCount = recoveryPoint.affectedBatchCount || 0;
const config = getEmbeddingConfig(); const config = getEmbeddingConfig();
const insertedHashes = collectAffectedInsertedHashes( const recoveryPlan = buildReverseJournalRecoveryPlan(
recoveryPoint.affectedJournals, recoveryPoint.affectedJournals,
initialDirtyFrom,
); );
rollbackAffectedJournals(currentGraph, recoveryPoint.affectedJournals); rollbackAffectedJournals(currentGraph, recoveryPoint.affectedJournals);
currentGraph = normalizeGraphRuntimeState(currentGraph, chatId); currentGraph = normalizeGraphRuntimeState(currentGraph, chatId);
extractionCount = currentGraph.historyState.extractionCount || 0; extractionCount = currentGraph.historyState.extractionCount || 0;
applyRecoveryPlanToVectorState(recoveryPlan, initialDirtyFrom);
if (isBackendVectorConfig(config) && insertedHashes.length > 0) { if (
isBackendVectorConfig(config) &&
recoveryPlan.backendDeleteHashes.length > 0
) {
await deleteBackendVectorHashesForRecovery( await deleteBackendVectorHashesForRecovery(
currentGraph.vectorIndexState.collectionId, currentGraph.vectorIndexState.collectionId,
config, config,
insertedHashes, recoveryPlan.backendDeleteHashes,
historySignal, historySignal,
); );
} }
@@ -2926,33 +3289,58 @@ async function onGenerationAfterCommands(type, params = {}, dryRun = false) {
); );
if (!recallOptions?.overrideUserMessage) return; if (!recallOptions?.overrideUserMessage) return;
const recallKey = buildPreGenerationRecallKey(type, recallOptions); const recallContext = createGenerationRecallContext({
const recentlyHandled = hookName: "GENERATION_AFTER_COMMANDS",
lastPreGenerationRecallKey === recallKey && generationType: String(type || "normal").trim() || "normal",
Date.now() - lastPreGenerationRecallAt < 1500; recallOptions,
if (recentlyHandled) { });
if (!recallContext.shouldRun) {
return; return;
} }
markGenerationRecallTransactionHookState(
recallContext.transaction,
recallContext.hookName,
"running",
);
const didRecall = await runRecall({ const didRecall = await runRecall({
...recallOptions, ...recallOptions,
hookName: "GENERATION_AFTER_COMMANDS", recallKey: recallContext.recallKey,
hookName: recallContext.hookName,
signal: params?.signal, signal: params?.signal,
}); });
if (didRecall) { markGenerationRecallTransactionHookState(
lastPreGenerationRecallKey = recallKey; recallContext.transaction,
lastPreGenerationRecallAt = Date.now(); recallContext.hookName,
skipBeforeCombineRecallUntil = Date.now() + 1500; didRecall ? "completed" : "pending",
} );
} }
async function onBeforeCombinePrompts() { async function onBeforeCombinePrompts() {
if (skipBeforeCombineRecallUntil > Date.now()) { const recallContext = createGenerationRecallContext({
skipBeforeCombineRecallUntil = 0; hookName: "GENERATE_BEFORE_COMBINE_PROMPTS",
generationType: "normal",
recallOptions: {},
});
if (!recallContext.shouldRun) {
return; return;
} }
await runRecall({ hookName: "GENERATE_BEFORE_COMBINE_PROMPTS" });
markGenerationRecallTransactionHookState(
recallContext.transaction,
recallContext.hookName,
"running",
);
const didRecall = await runRecall({
recallKey: recallContext.recallKey,
hookName: recallContext.hookName,
});
markGenerationRecallTransactionHookState(
recallContext.transaction,
recallContext.hookName,
didRecall ? "completed" : "pending",
);
} }
function onMessageReceived() { function onMessageReceived() {
@@ -3502,6 +3890,8 @@ async function onReembedDirect() {
getLastExtractionStatus: () => lastExtractionStatus, getLastExtractionStatus: () => lastExtractionStatus,
getLastVectorStatus: () => lastVectorStatus, getLastVectorStatus: () => lastVectorStatus,
getLastRecallStatus: () => lastRecallStatus, getLastRecallStatus: () => lastRecallStatus,
getLastBatchStatus: () =>
currentGraph?.historyState?.lastBatchStatus || null,
getLastInjection: () => lastInjectionContent, getLastInjection: () => lastInjectionContent,
updateSettings: (patch) => { updateSettings: (patch) => {
const settings = updateModuleSettings(patch); const settings = updateModuleSettings(patch);

View File

@@ -17,6 +17,7 @@ export function createDefaultHistoryState(chatId = "") {
lastMutationSource: "", lastMutationSource: "",
extractionCount: 0, extractionCount: 0,
lastRecoveryResult: null, lastRecoveryResult: null,
lastBatchStatus: null,
}; };
} }
@@ -29,6 +30,9 @@ export function createDefaultVectorIndexState(chatId = "") {
hashToNodeId: {}, hashToNodeId: {},
nodeToHash: {}, nodeToHash: {},
dirty: false, dirty: false,
replayRequiredNodeIds: [],
dirtyReason: "",
pendingRepairFromFloor: null,
lastSyncAt: 0, lastSyncAt: 0,
lastStats: { lastStats: {
total: 0, total: 0,
@@ -60,7 +64,9 @@ export function normalizeGraphRuntimeState(graph, chatId = "") {
historyState.chatId = chatId || historyState.chatId || ""; historyState.chatId = chatId || historyState.chatId || "";
if (!Number.isFinite(historyState.lastProcessedAssistantFloor)) { if (!Number.isFinite(historyState.lastProcessedAssistantFloor)) {
historyState.lastProcessedAssistantFloor = Number.isFinite(graph.lastProcessedSeq) historyState.lastProcessedAssistantFloor = Number.isFinite(
graph.lastProcessedSeq,
)
? graph.lastProcessedSeq ? graph.lastProcessedSeq
: -1; : -1;
} }
@@ -70,6 +76,20 @@ export function normalizeGraphRuntimeState(graph, chatId = "") {
if (typeof historyState.lastMutationSource !== "string") { if (typeof historyState.lastMutationSource !== "string") {
historyState.lastMutationSource = ""; historyState.lastMutationSource = "";
} }
if (
!historyState.lastBatchStatus ||
typeof historyState.lastBatchStatus !== "object" ||
Array.isArray(historyState.lastBatchStatus)
) {
historyState.lastBatchStatus = null;
} else if (
typeof historyState.lastBatchStatus.historyAdvanced !== "boolean"
) {
historyState.lastBatchStatus = {
...historyState.lastBatchStatus,
historyAdvanced: false,
};
}
if ( if (
!historyState.processedMessageHashes || !historyState.processedMessageHashes ||
@@ -93,17 +113,42 @@ export function normalizeGraphRuntimeState(graph, chatId = "") {
) { ) {
vectorIndexState.nodeToHash = {}; vectorIndexState.nodeToHash = {};
} }
if (!vectorIndexState.lastStats || typeof vectorIndexState.lastStats !== "object") { if (
vectorIndexState.lastStats = createDefaultVectorIndexState(chatId).lastStats; !vectorIndexState.lastStats ||
typeof vectorIndexState.lastStats !== "object"
) {
vectorIndexState.lastStats =
createDefaultVectorIndexState(chatId).lastStats;
}
if (!Array.isArray(vectorIndexState.replayRequiredNodeIds)) {
vectorIndexState.replayRequiredNodeIds = [];
} else {
vectorIndexState.replayRequiredNodeIds = [
...new Set(vectorIndexState.replayRequiredNodeIds.filter(Boolean)),
];
}
if (typeof vectorIndexState.dirtyReason !== "string") {
vectorIndexState.dirtyReason = "";
}
if (!Number.isFinite(vectorIndexState.pendingRepairFromFloor)) {
vectorIndexState.pendingRepairFromFloor = null;
} }
const previousCollectionId = vectorIndexState.collectionId; const previousCollectionId = vectorIndexState.collectionId;
vectorIndexState.collectionId = buildVectorCollectionId(chatId || historyState.chatId); vectorIndexState.collectionId = buildVectorCollectionId(
chatId || historyState.chatId,
);
if (previousCollectionId && previousCollectionId !== vectorIndexState.collectionId) { if (
previousCollectionId &&
previousCollectionId !== vectorIndexState.collectionId
) {
vectorIndexState.hashToNodeId = {}; vectorIndexState.hashToNodeId = {};
vectorIndexState.nodeToHash = {}; vectorIndexState.nodeToHash = {};
vectorIndexState.replayRequiredNodeIds = [];
vectorIndexState.dirty = true; vectorIndexState.dirty = true;
vectorIndexState.dirtyReason = "chat-id-changed";
vectorIndexState.pendingRepairFromFloor = 0;
vectorIndexState.lastWarning = "聊天标识变化,向量索引已标记为待重建"; vectorIndexState.lastWarning = "聊天标识变化,向量索引已标记为待重建";
} }
@@ -155,7 +200,10 @@ export function buildMessageHash(message) {
return String(stableHashString(payload)); return String(stableHashString(payload));
} }
export function snapshotProcessedMessageHashes(chat, lastProcessedAssistantFloor) { export function snapshotProcessedMessageHashes(
chat,
lastProcessedAssistantFloor,
) {
const result = {}; const result = {};
if (!Array.isArray(chat) || lastProcessedAssistantFloor < 0) { if (!Array.isArray(chat) || lastProcessedAssistantFloor < 0) {
return result; return result;
@@ -268,6 +316,113 @@ function clonePlain(value) {
return JSON.parse(JSON.stringify(value)); return JSON.parse(JSON.stringify(value));
} }
function normalizeStringArray(values) {
if (!Array.isArray(values)) return [];
return [...new Set(values.filter(Boolean).map((value) => String(value)))];
}
function normalizeMappingArray(values) {
if (!Array.isArray(values)) return [];
const seen = new Set();
const mappings = [];
for (const entry of values) {
if (!entry || typeof entry !== "object") continue;
const nodeId = entry.nodeId ? String(entry.nodeId) : "";
const previousHash = entry.previousHash ? String(entry.previousHash) : "";
const nextHash = entry.nextHash ? String(entry.nextHash) : "";
if (!nodeId && !previousHash && !nextHash) continue;
const key = JSON.stringify([nodeId, previousHash, nextHash]);
if (seen.has(key)) continue;
seen.add(key);
mappings.push({ nodeId, previousHash, nextHash });
}
return mappings;
}
function buildVectorDelta(snapshotBefore, snapshotAfter, meta = {}) {
const beforeState = snapshotBefore?.vectorIndexState || {};
const afterState = snapshotAfter?.vectorIndexState || {};
const beforeNodeToHash = beforeState.nodeToHash || {};
const afterNodeToHash = afterState.nodeToHash || {};
const beforeHashSet = new Set(
Object.values(beforeState.hashToNodeId || {}).filter(Boolean),
);
const afterHashSet = new Set(
Object.values(afterState.hashToNodeId || {}).filter(Boolean),
);
const insertedHashes = new Set(
normalizeStringArray(meta.vectorHashesInserted),
);
const removedHashes = new Set(normalizeStringArray(meta.vectorHashesRemoved));
const touchedNodeIds = new Set(
normalizeStringArray(meta.vectorTouchedNodeIds),
);
const replayRequiredNodeIds = new Set(
normalizeStringArray(meta.vectorReplayRequiredNodeIds),
);
const backendDeleteHashes = new Set(
normalizeStringArray(meta.vectorBackendDeleteHashes),
);
const replacedMappings = normalizeMappingArray(meta.vectorReplacedMappings);
const nodeIds = new Set([
...Object.keys(beforeNodeToHash),
...Object.keys(afterNodeToHash),
]);
for (const hash of Object.keys(afterState.hashToNodeId || {})) {
if (!beforeHashSet.has(hash)) insertedHashes.add(hash);
}
for (const hash of Object.keys(beforeState.hashToNodeId || {})) {
if (!afterHashSet.has(hash)) removedHashes.add(hash);
}
for (const nodeId of nodeIds) {
const previousHash = beforeNodeToHash[nodeId]
? String(beforeNodeToHash[nodeId])
: "";
const nextHash = afterNodeToHash[nodeId]
? String(afterNodeToHash[nodeId])
: "";
if (previousHash === nextHash) continue;
touchedNodeIds.add(String(nodeId));
if (previousHash) {
removedHashes.add(previousHash);
backendDeleteHashes.add(previousHash);
}
if (nextHash) {
insertedHashes.add(nextHash);
}
if (previousHash || nextHash) {
const key = JSON.stringify([String(nodeId), previousHash, nextHash]);
const exists = replacedMappings.some(
(entry) =>
JSON.stringify([entry.nodeId, entry.previousHash, entry.nextHash]) ===
key,
);
if (!exists) {
replacedMappings.push({
nodeId: String(nodeId),
previousHash,
nextHash,
});
}
}
}
for (const nodeId of normalizeStringArray(afterState.replayRequiredNodeIds)) {
replayRequiredNodeIds.add(nodeId);
}
return {
insertedHashes: [...insertedHashes],
removedHashes: [...removedHashes],
replacedMappings,
touchedNodeIds: [...touchedNodeIds],
replayRequiredNodeIds: [...replayRequiredNodeIds],
backendDeleteHashes: [...backendDeleteHashes],
};
}
function buildJournalStateBefore(snapshotBefore, meta = {}) { function buildJournalStateBefore(snapshotBefore, meta = {}) {
return { return {
lastProcessedAssistantFloor: lastProcessedAssistantFloor:
@@ -277,7 +432,9 @@ function buildJournalStateBefore(snapshotBefore, meta = {}) {
processedMessageHashes: clonePlain( processedMessageHashes: clonePlain(
snapshotBefore?.historyState?.processedMessageHashes || {}, snapshotBefore?.historyState?.processedMessageHashes || {},
), ),
historyDirtyFrom: Number.isFinite(snapshotBefore?.historyState?.historyDirtyFrom) historyDirtyFrom: Number.isFinite(
snapshotBefore?.historyState?.historyDirtyFrom,
)
? snapshotBefore.historyState.historyDirtyFrom ? snapshotBefore.historyState.historyDirtyFrom
: null, : null,
vectorIndexState: clonePlain(snapshotBefore?.vectorIndexState || {}), vectorIndexState: clonePlain(snapshotBefore?.vectorIndexState || {}),
@@ -286,11 +443,15 @@ function buildJournalStateBefore(snapshotBefore, meta = {}) {
: null, : null,
extractionCount: Number.isFinite(meta.extractionCountBefore) extractionCount: Number.isFinite(meta.extractionCountBefore)
? meta.extractionCountBefore ? meta.extractionCountBefore
: snapshotBefore?.historyState?.extractionCount ?? 0, : (snapshotBefore?.historyState?.extractionCount ?? 0),
}; };
} }
export function createBatchJournalEntry(snapshotBefore, snapshotAfter, meta = {}) { export function createBatchJournalEntry(
snapshotBefore,
snapshotAfter,
meta = {},
) {
const beforeNodes = buildNodeMap(snapshotBefore?.nodes || []); const beforeNodes = buildNodeMap(snapshotBefore?.nodes || []);
const afterNodes = buildNodeMap(snapshotAfter?.nodes || []); const afterNodes = buildNodeMap(snapshotAfter?.nodes || []);
const beforeEdges = buildEdgeMap(snapshotBefore?.edges || []); const beforeEdges = buildEdgeMap(snapshotBefore?.edges || []);
@@ -333,11 +494,7 @@ export function createBatchJournalEntry(snapshotBefore, snapshotAfter, meta = {}
previousNodeSnapshots, previousNodeSnapshots,
previousEdgeSnapshots, previousEdgeSnapshots,
stateBefore: buildJournalStateBefore(snapshotBefore, meta), stateBefore: buildJournalStateBefore(snapshotBefore, meta),
vectorDelta: { vectorDelta: buildVectorDelta(snapshotBefore, snapshotAfter, meta),
insertedHashes: Array.isArray(meta.vectorHashesInserted)
? [...new Set(meta.vectorHashesInserted)]
: [],
},
postProcessArtifacts: Array.isArray(meta.postProcessArtifacts) postProcessArtifacts: Array.isArray(meta.postProcessArtifacts)
? meta.postProcessArtifacts ? meta.postProcessArtifacts
: [], : [],
@@ -427,9 +584,7 @@ export function rollbackBatch(graph, journal) {
journal.archivedNodeSnapshots || journal.archivedNodeSnapshots ||
[]; [];
const previousEdgeSnapshots = const previousEdgeSnapshots =
journal.previousEdgeSnapshots || journal.previousEdgeSnapshots || journal.invalidatedEdgeSnapshots || [];
journal.invalidatedEdgeSnapshots ||
[];
graph.edges = (graph.edges || []).filter( graph.edges = (graph.edges || []).filter(
(edge) => (edge) =>
@@ -437,7 +592,9 @@ export function rollbackBatch(graph, journal) {
!createdNodeIds.has(edge.fromId) && !createdNodeIds.has(edge.fromId) &&
!createdNodeIds.has(edge.toId), !createdNodeIds.has(edge.toId),
); );
graph.nodes = (graph.nodes || []).filter((node) => !createdNodeIds.has(node.id)); graph.nodes = (graph.nodes || []).filter(
(node) => !createdNodeIds.has(node.id),
);
for (const nodeSnapshot of previousNodeSnapshots) { for (const nodeSnapshot of previousNodeSnapshots) {
upsertById(graph.nodes, cloneGraphSnapshot(nodeSnapshot)); upsertById(graph.nodes, cloneGraphSnapshot(nodeSnapshot));
@@ -470,7 +627,9 @@ export function findJournalRecoveryPoint(graph, dirtyFromFloor) {
return { return {
path: "reverse-journal", path: "reverse-journal",
affectedIndex, affectedIndex,
affectedJournals: affectedJournals.map((journal) => cloneGraphSnapshot(journal)), affectedJournals: affectedJournals.map((journal) =>
cloneGraphSnapshot(journal),
),
affectedBatchCount: affectedJournals.length, affectedBatchCount: affectedJournals.length,
}; };
} }
@@ -489,6 +648,92 @@ export function findJournalRecoveryPoint(graph, dirtyFromFloor) {
return null; return null;
} }
export function buildReverseJournalRecoveryPlan(
affectedJournals = [],
dirtyFromFloor = null,
) {
const backendDeleteHashes = new Set();
const replayRequiredNodeIds = new Set();
const touchedNodeIds = new Set();
let hasLegacyGap = false;
let minProcessedFloor = Number.isFinite(dirtyFromFloor)
? dirtyFromFloor
: null;
for (const journal of affectedJournals) {
const vectorDelta = journal?.vectorDelta || {};
const insertedHashes = normalizeStringArray(
vectorDelta.insertedHashes || journal?.vectorHashesInserted || [],
);
const removedHashes = normalizeStringArray(vectorDelta.removedHashes);
const backendDeletes = normalizeStringArray(
vectorDelta.backendDeleteHashes,
);
const touchedNodes = normalizeStringArray(vectorDelta.touchedNodeIds);
const replayNodes = normalizeStringArray(vectorDelta.replayRequiredNodeIds);
const replacedMappings = normalizeMappingArray(
vectorDelta.replacedMappings,
);
const range = Array.isArray(journal?.processedRange)
? journal.processedRange
: [-1, -1];
if (Number.isFinite(range[0])) {
minProcessedFloor = Number.isFinite(minProcessedFloor)
? Math.min(minProcessedFloor, range[0])
: range[0];
}
for (const hash of insertedHashes) {
backendDeleteHashes.add(hash);
}
for (const hash of removedHashes) {
backendDeleteHashes.add(hash);
}
for (const hash of backendDeletes) {
backendDeleteHashes.add(hash);
}
for (const nodeId of touchedNodes) {
touchedNodeIds.add(nodeId);
replayRequiredNodeIds.add(nodeId);
}
for (const nodeId of replayNodes) {
replayRequiredNodeIds.add(nodeId);
}
for (const entry of replacedMappings) {
if (entry.nodeId) {
touchedNodeIds.add(entry.nodeId);
replayRequiredNodeIds.add(entry.nodeId);
}
if (entry.previousHash) backendDeleteHashes.add(entry.previousHash);
if (entry.nextHash) backendDeleteHashes.add(entry.nextHash);
}
if (
!Array.isArray(vectorDelta.removedHashes) ||
!Array.isArray(vectorDelta.replacedMappings) ||
!Array.isArray(vectorDelta.touchedNodeIds) ||
!Array.isArray(vectorDelta.replayRequiredNodeIds) ||
!Array.isArray(vectorDelta.backendDeleteHashes)
) {
hasLegacyGap = true;
}
}
const pendingRepairFromFloor = Number.isFinite(minProcessedFloor)
? minProcessedFloor
: null;
return {
backendDeleteHashes: [...backendDeleteHashes],
replayRequiredNodeIds: [...replayRequiredNodeIds],
touchedNodeIds: [...touchedNodeIds],
pendingRepairFromFloor,
legacyGapFallback: hasLegacyGap,
dirtyReason: hasLegacyGap ? "legacy-gap" : "history-recovery-replay",
};
}
export function buildRecoveryResult(status, extra = {}) { export function buildRecoveryResult(status, extra = {}) {
return { return {
status, status,

1015
tests/p0-regressions.mjs Normal file

File diff suppressed because it is too large Load Diff

View File

@@ -614,6 +614,7 @@ export async function syncGraphVectorIndex(
} }
} }
let directSyncHadFailures = false;
if (entriesToEmbed.length > 0) { if (entriesToEmbed.length > 0) {
throwIfAborted(signal); throwIfAborted(signal);
const embeddings = await embedBatch( const embeddings = await embedBatch(
@@ -634,6 +635,8 @@ export async function syncGraphVectorIndex(
state.hashToNodeId[entry.hash] = entry.nodeId; state.hashToNodeId[entry.hash] = entry.nodeId;
state.nodeToHash[entry.nodeId] = entry.hash; state.nodeToHash[entry.nodeId] = entry.hash;
insertedHashes.push(entry.hash); insertedHashes.push(entry.hash);
} else {
directSyncHadFailures = true;
} }
} }
} }
@@ -642,10 +645,16 @@ export async function syncGraphVectorIndex(
state.source = "direct"; state.source = "direct";
state.modelScope = getVectorModelScope(config); state.modelScope = getVectorModelScope(config);
state.collectionId = collectionId; state.collectionId = collectionId;
state.dirty = directSyncHadFailures;
state.lastWarning = directSyncHadFailures
? "部分节点 embedding 生成失败,向量索引仍待修复"
: "";
} }
state.dirty = false; if (state.mode !== "direct") {
state.lastWarning = ""; state.dirty = false;
state.lastWarning = "";
}
state.lastSyncAt = Date.now(); state.lastSyncAt = Date.now();
state.lastStats = computeVectorStats( state.lastStats = computeVectorStats(
graph, graph,