mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
Reorganize modules into layered directories
This commit is contained in:
Youzini-afk
229
maintenance/chat-history.js
Normal file
229
maintenance/chat-history.js
Normal file
@@ -0,0 +1,229 @@
|
||||
// ST-BME: 聊天历史纯函数
|
||||
// 此模块中的函数均不依赖 index.js 模块级可变状态，
|
||||
// 可被 index.js 及其他模块安全导入。
|
||||
|
||||
import { clampInt } from "../ui/ui-status.js";
|
||||
import { sanitizePlannerMessageText } from "../runtime/planner-tag-utils.js";
|
||||
import { rollbackBatch } from "../runtime/runtime-state.js";
|
||||
import { isInManagedHideRange } from "../ui/hide-engine.js";
|
||||
|
||||
export function isBmeManagedHiddenMessage(
|
||||
message,
|
||||
{ index = null, chat = null } = {},
|
||||
) {
|
||||
if (
|
||||
Number.isFinite(index) &&
|
||||
index > 0 &&
|
||||
isInManagedHideRange(index, chat)
|
||||
) {
|
||||
return true;
|
||||
}
|
||||
|
||||
return Boolean(
|
||||
message?.extra &&
|
||||
typeof message.extra === "object" &&
|
||||
message.extra.__st_bme_hide_managed === true,
|
||||
);
|
||||
}
|
||||
|
||||
export function isSystemMessageForExtraction(
|
||||
message,
|
||||
{ index = null, chat = null } = {},
|
||||
) {
|
||||
if (!message?.is_system) return false;
|
||||
if (Number.isFinite(index) && index === 0) return true;
|
||||
|
||||
return !isBmeManagedHiddenMessage(message, { index, chat });
|
||||
}
|
||||
|
||||
export function isAssistantChatMessage(
|
||||
message,
|
||||
{ index = null, chat = null } = {},
|
||||
) {
|
||||
return (
|
||||
Boolean(message) &&
|
||||
!message.is_user &&
|
||||
!isSystemMessageForExtraction(message, { index, chat })
|
||||
);
|
||||
}
|
||||
|
||||
export function getAssistantTurns(chat) {
|
||||
const assistantTurns = [];
|
||||
// 从 index 1 开始：index 0 是角色卡首条消息（greeting），不参与提取
|
||||
for (let index = 1; index < chat.length; index++) {
|
||||
if (!isAssistantChatMessage(chat[index], { index, chat })) continue;
|
||||
if (!String(chat[index]?.mes ?? "").trim()) continue;
|
||||
assistantTurns.push(index);
|
||||
}
|
||||
return assistantTurns;
|
||||
}
|
||||
|
||||
export function getMinExtractableAssistantFloor(chat) {
|
||||
const assistantTurns = getAssistantTurns(chat);
|
||||
return assistantTurns.length > 0 ? assistantTurns[0] : null;
|
||||
}
|
||||
|
||||
export function buildExtractionMessages(chat, startIdx, endIdx, settings) {
|
||||
const contextTurns = clampInt(settings.extractContextTurns, 2, 0, 20);
|
||||
const contextStart = Math.max(0, startIdx - contextTurns * 2);
|
||||
const messages = [];
|
||||
|
||||
for (
|
||||
let index = contextStart;
|
||||
index <= endIdx && index < chat.length;
|
||||
index++
|
||||
) {
|
||||
const msg = chat[index];
|
||||
if (isSystemMessageForExtraction(msg, { index, chat })) continue;
|
||||
const content = sanitizePlannerMessageText(msg);
|
||||
if (!String(content || "").trim()) continue;
|
||||
messages.push({
|
||||
seq: index,
|
||||
role: msg.is_user ? "user" : "assistant",
|
||||
content,
|
||||
});
|
||||
}
|
||||
|
||||
return messages;
|
||||
}
|
||||
|
||||
export function getChatIndexForPlayableSeq(chat, playableSeq) {
|
||||
if (!Array.isArray(chat) || !Number.isFinite(playableSeq)) return null;
|
||||
|
||||
let currentSeq = -1;
|
||||
for (let index = 0; index < chat.length; index++) {
|
||||
const message = chat[index];
|
||||
if (isSystemMessageForExtraction(message, { index, chat })) continue;
|
||||
currentSeq++;
|
||||
if (currentSeq >= playableSeq) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
return chat.length;
|
||||
}
|
||||
|
||||
export function getChatIndexForAssistantSeq(chat, assistantSeq) {
|
||||
if (!Array.isArray(chat) || !Number.isFinite(assistantSeq)) return null;
|
||||
|
||||
let currentSeq = -1;
|
||||
for (let index = 0; index < chat.length; index++) {
|
||||
if (!isAssistantChatMessage(chat[index], { index, chat })) continue;
|
||||
currentSeq++;
|
||||
if (currentSeq >= assistantSeq) {
|
||||
return index;
|
||||
}
|
||||
}
|
||||
|
||||
return chat.length;
|
||||
}
|
||||
|
||||
export function resolveDirtyFloorFromMutationMeta(trigger, primaryArg, meta, chat) {
|
||||
if (!meta || typeof meta !== "object") return null;
|
||||
|
||||
const candidates = [];
|
||||
const isDeleteTrigger = String(trigger || "").includes("message-deleted");
|
||||
const minExtractableFloor = getMinExtractableAssistantFloor(chat);
|
||||
|
||||
// 删除后 chat 已是收缩后的状态，删除事件携带的 seq 更接近"被删区间起点"，
|
||||
// 因此这里额外向前退一层，避免恢复仍停留在被删楼层对应的旧图谱边界。
|
||||
if (!isDeleteTrigger && Number.isFinite(meta.messageId)) {
|
||||
candidates.push({
|
||||
floor: meta.messageId,
|
||||
source: `${trigger}-meta`,
|
||||
});
|
||||
}
|
||||
if (Number.isFinite(meta.deletedPlayableSeqFrom)) {
|
||||
const floor = getChatIndexForPlayableSeq(chat, meta.deletedPlayableSeqFrom);
|
||||
if (Number.isFinite(floor)) {
|
||||
candidates.push({
|
||||
floor: Number.isFinite(minExtractableFloor)
|
||||
? Math.max(minExtractableFloor, floor - 1)
|
||||
: Math.max(0, floor - 1),
|
||||
source: `${trigger}-meta-delete-boundary`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (Number.isFinite(meta.deletedAssistantSeqFrom)) {
|
||||
const floor = getChatIndexForAssistantSeq(
|
||||
chat,
|
||||
meta.deletedAssistantSeqFrom,
|
||||
);
|
||||
if (Number.isFinite(floor)) {
|
||||
candidates.push({
|
||||
floor: Number.isFinite(minExtractableFloor)
|
||||
? Math.max(minExtractableFloor, floor - 1)
|
||||
: Math.max(0, floor - 1),
|
||||
source: `${trigger}-meta-delete-boundary`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!isDeleteTrigger && Number.isFinite(meta.playableSeq)) {
|
||||
const floor = getChatIndexForPlayableSeq(chat, meta.playableSeq);
|
||||
if (Number.isFinite(floor)) {
|
||||
candidates.push({
|
||||
floor,
|
||||
source: `${trigger}-meta`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!isDeleteTrigger && Number.isFinite(meta.assistantSeq)) {
|
||||
const floor = getChatIndexForAssistantSeq(chat, meta.assistantSeq);
|
||||
if (Number.isFinite(floor)) {
|
||||
candidates.push({
|
||||
floor,
|
||||
source: `${trigger}-meta`,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (!isDeleteTrigger && Number.isFinite(primaryArg)) {
|
||||
candidates.push({
|
||||
floor: primaryArg,
|
||||
source: `${trigger}-meta`,
|
||||
});
|
||||
}
|
||||
|
||||
if (candidates.length === 0) return null;
|
||||
const validCandidates = Number.isFinite(minExtractableFloor)
|
||||
? candidates.filter((c) => c.floor >= minExtractableFloor)
|
||||
: candidates;
|
||||
if (validCandidates.length === 0) return null;
|
||||
return validCandidates.reduce((earliest, current) =>
|
||||
current.floor < earliest.floor ? current : earliest,
|
||||
);
|
||||
}
|
||||
|
||||
export function clampRecoveryStartFloor(chat, floor) {
|
||||
if (!Number.isFinite(floor)) return floor;
|
||||
|
||||
const minExtractableFloor = getMinExtractableAssistantFloor(chat);
|
||||
if (!Number.isFinite(minExtractableFloor)) {
|
||||
return floor;
|
||||
}
|
||||
|
||||
return Math.max(floor, minExtractableFloor);
|
||||
}
|
||||
|
||||
export function rollbackAffectedJournals(graph, affectedJournals = []) {
|
||||
for (let index = affectedJournals.length - 1; index >= 0; index--) {
|
||||
rollbackBatch(graph, affectedJournals[index]);
|
||||
}
|
||||
graph.batchJournal = Array.isArray(graph.batchJournal)
|
||||
? graph.batchJournal.slice(
|
||||
0,
|
||||
Math.max(0, graph.batchJournal.length - affectedJournals.length),
|
||||
)
|
||||
: [];
|
||||
}
|
||||
|
||||
export function pruneProcessedMessageHashesFromFloor(graph, fromFloor) {
|
||||
if (!graph?.historyState?.processedMessageHashes) return;
|
||||
if (!Number.isFinite(fromFloor)) return;
|
||||
|
||||
const hashes = graph.historyState.processedMessageHashes;
|
||||
for (const key of Object.keys(hashes)) {
|
||||
if (Number(key) >= fromFloor) {
|
||||
delete hashes[key];
|
||||
}
|
||||
}
|
||||
}
|
||||
629
maintenance/compressor.js
Normal file
629
maintenance/compressor.js
Normal file
@@ -0,0 +1,629 @@
|
||||
// ST-BME: 层级压缩引擎
|
||||
// 超过阈值的节点被 LLM 总结为更高层级的压缩节点
|
||||
|
||||
import { debugLog } from "../runtime/debug-logging.js";
|
||||
import { embedText } from "../vector/embedding.js";
|
||||
import {
|
||||
addEdge,
|
||||
addNode,
|
||||
createEdge,
|
||||
createNode,
|
||||
getActiveNodes,
|
||||
getNode,
|
||||
} from "../graph/graph.js";
|
||||
import { callLLMForJSON } from "../llm/llm.js";
|
||||
import {
|
||||
getScopeOwnerKey,
|
||||
getScopeRegionKey,
|
||||
normalizeMemoryScope,
|
||||
} from "../graph/memory-scope.js";
|
||||
import { ensureEventTitle, getNodeDisplayName } from "../graph/node-labels.js";
|
||||
import {
|
||||
buildTaskExecutionDebugContext,
|
||||
buildTaskLlmPayload,
|
||||
buildTaskPrompt,
|
||||
} from "../prompting/prompt-builder.js";
|
||||
import { getSTContextForPrompt } from "../host/st-context.js";
|
||||
import { applyTaskRegex } from "../prompting/task-regex.js";
|
||||
import { isDirectVectorConfig } from "../vector/vector-index.js";
|
||||
|
||||
function createAbortError(message = "操作已终止") {
|
||||
const error = new Error(message);
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function createTaskLlmDebugContext(promptBuild, regexInput) {
|
||||
return typeof buildTaskExecutionDebugContext === "function"
|
||||
? buildTaskExecutionDebugContext(promptBuild, { regexInput })
|
||||
: null;
|
||||
}
|
||||
|
||||
function resolveTaskPromptPayload(promptBuild, fallbackUserPrompt = "") {
|
||||
if (typeof buildTaskLlmPayload === "function") {
|
||||
return buildTaskLlmPayload(promptBuild, fallbackUserPrompt);
|
||||
}
|
||||
|
||||
return {
|
||||
systemPrompt: String(promptBuild?.systemPrompt || ""),
|
||||
userPrompt: String(fallbackUserPrompt || ""),
|
||||
promptMessages: [],
|
||||
additionalMessages: Array.isArray(promptBuild?.privateTaskMessages)
|
||||
? promptBuild.privateTaskMessages
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function throwIfAborted(signal) {
|
||||
if (signal?.aborted) {
|
||||
throw signal.reason instanceof Error ? signal.reason : createAbortError();
|
||||
}
|
||||
}
|
||||
|
||||
function resolveCompressionWindow(compression = {}, force = false) {
|
||||
const fanIn = Number.isFinite(Number(compression?.fanIn))
|
||||
? Math.max(2, Number(compression.fanIn))
|
||||
: 2;
|
||||
const threshold = force
|
||||
? fanIn
|
||||
: Number.isFinite(Number(compression?.threshold))
|
||||
? Math.max(2, Number(compression.threshold))
|
||||
: fanIn;
|
||||
const keepRecent = force
|
||||
? 0
|
||||
: Number.isFinite(Number(compression?.keepRecentLeaves))
|
||||
? Math.max(0, Number(compression.keepRecentLeaves))
|
||||
: 0;
|
||||
|
||||
return {
|
||||
fanIn,
|
||||
threshold,
|
||||
keepRecent,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeCompressionFieldValue(value) {
|
||||
if (value == null) return "";
|
||||
if (typeof value === "string") return value.trim();
|
||||
if (typeof value === "number" || typeof value === "boolean") {
|
||||
return String(value);
|
||||
}
|
||||
if (Array.isArray(value)) {
|
||||
return value
|
||||
.map((item) => normalizeCompressionFieldValue(item))
|
||||
.filter(Boolean)
|
||||
.join("；");
|
||||
}
|
||||
if (typeof value === "object") {
|
||||
try {
|
||||
return JSON.stringify(value);
|
||||
} catch {
|
||||
return "";
|
||||
}
|
||||
}
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function buildCompressionFallbackSummary(batch = []) {
|
||||
return batch
|
||||
.map((node) =>
|
||||
normalizeCompressionFieldValue(
|
||||
node?.fields?.summary ||
|
||||
node?.fields?.title ||
|
||||
node?.fields?.name ||
|
||||
node?.fields?.insight ||
|
||||
getNodeDisplayName(node),
|
||||
),
|
||||
)
|
||||
.filter(Boolean)
|
||||
.slice(0, 6)
|
||||
.join("；");
|
||||
}
|
||||
|
||||
function normalizeCompressedFields(summaryResult, typeDef, batch = []) {
|
||||
const rawFields =
|
||||
summaryResult?.fields &&
|
||||
typeof summaryResult.fields === "object" &&
|
||||
!Array.isArray(summaryResult.fields)
|
||||
? summaryResult.fields
|
||||
: summaryResult && typeof summaryResult === "object" && !Array.isArray(summaryResult)
|
||||
? summaryResult
|
||||
: {};
|
||||
const columns = Array.isArray(typeDef?.columns) ? typeDef.columns : [];
|
||||
const normalized = {};
|
||||
|
||||
for (const column of columns) {
|
||||
const key = String(column?.name || "").trim();
|
||||
if (!key) continue;
|
||||
const normalizedValue = normalizeCompressionFieldValue(rawFields[key]);
|
||||
if (normalizedValue) {
|
||||
normalized[key] = normalizedValue;
|
||||
}
|
||||
}
|
||||
|
||||
const fallbackSummary = buildCompressionFallbackSummary(batch);
|
||||
if (!normalized.summary && columns.some((column) => column?.name === "summary")) {
|
||||
normalized.summary = fallbackSummary || "压缩批次摘要缺失";
|
||||
}
|
||||
if (!normalized.insight && columns.some((column) => column?.name === "insight")) {
|
||||
normalized.insight = fallbackSummary || "压缩批次洞察缺失";
|
||||
}
|
||||
if (!normalized.title && columns.some((column) => column?.name === "title")) {
|
||||
const titled = ensureEventTitle({ title: rawFields?.title, summary: normalized.summary });
|
||||
normalized.title =
|
||||
normalizeCompressionFieldValue(titled?.title) ||
|
||||
normalizeCompressionFieldValue(rawFields?.name) ||
|
||||
normalizeCompressionFieldValue(batch[batch.length - 1]?.fields?.title) ||
|
||||
normalizeCompressionFieldValue(batch[batch.length - 1]?.fields?.name) ||
|
||||
"压缩节点";
|
||||
}
|
||||
if (!normalized.name && columns.some((column) => column?.name === "name")) {
|
||||
normalized.name =
|
||||
normalizeCompressionFieldValue(rawFields?.title) ||
|
||||
normalizeCompressionFieldValue(rawFields?.name) ||
|
||||
normalizeCompressionFieldValue(batch[batch.length - 1]?.fields?.name) ||
|
||||
"压缩节点";
|
||||
}
|
||||
|
||||
return normalized;
|
||||
}
|
||||
|
||||
/**
|
||||
* 对指定类型执行层级压缩
|
||||
*
|
||||
* @param {object} params
|
||||
* @param {object} params.graph - 当前图状态
|
||||
* @param {object} params.typeDef - 要压缩的类型定义
|
||||
* @param {object} params.embeddingConfig - Embedding API 配置
|
||||
* @param {boolean} [params.force=false] - 忽略阈值强制压缩
|
||||
* @returns {Promise<{created: number, archived: number}>}
|
||||
*/
|
||||
export async function compressType({
|
||||
graph,
|
||||
typeDef,
|
||||
embeddingConfig,
|
||||
force = false,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings = {},
|
||||
}) {
|
||||
const compression = typeDef.compression;
|
||||
if (!compression || compression.mode !== "hierarchical") {
|
||||
return { created: 0, archived: 0 };
|
||||
}
|
||||
const maxDepth = Number.isFinite(Number(compression.maxDepth))
|
||||
? Math.max(1, Number(compression.maxDepth))
|
||||
: 1;
|
||||
|
||||
let totalCreated = 0;
|
||||
let totalArchived = 0;
|
||||
|
||||
// 从最低层级开始逐层压缩
|
||||
for (let level = 0; level < maxDepth; level++) {
|
||||
throwIfAborted(signal);
|
||||
const result = await compressLevel({
|
||||
graph,
|
||||
typeDef,
|
||||
level,
|
||||
embeddingConfig,
|
||||
force,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings,
|
||||
});
|
||||
|
||||
totalCreated += result.created;
|
||||
totalArchived += result.archived;
|
||||
|
||||
// 如果这一层没有压缩发生，停止
|
||||
if (result.created === 0) break;
|
||||
}
|
||||
|
||||
return { created: totalCreated, archived: totalArchived };
|
||||
}
|
||||
|
||||
/**
|
||||
* 压缩特定层级的节点
|
||||
*/
|
||||
async function compressLevel({
|
||||
graph,
|
||||
typeDef,
|
||||
level,
|
||||
embeddingConfig,
|
||||
force,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings = {},
|
||||
}) {
|
||||
const compression = typeDef.compression;
|
||||
const { fanIn, threshold, keepRecent } = resolveCompressionWindow(
|
||||
compression,
|
||||
force,
|
||||
);
|
||||
throwIfAborted(signal);
|
||||
|
||||
// 获取该层级的活跃叶子节点
|
||||
const levelNodes = getActiveNodes(graph, typeDef.id)
|
||||
.filter((n) => n.level === level)
|
||||
.sort((a, b) => a.seq - b.seq);
|
||||
let created = 0;
|
||||
let archived = 0;
|
||||
|
||||
for (const group of groupCompressionCandidates(levelNodes)) {
|
||||
if (force ? group.length < fanIn : group.length <= threshold) {
|
||||
continue;
|
||||
}
|
||||
|
||||
const compressible = group.slice(0, Math.max(0, group.length - keepRecent));
|
||||
if (compressible.length < fanIn) {
|
||||
continue;
|
||||
}
|
||||
|
||||
for (let i = 0; i < compressible.length; i += fanIn) {
|
||||
const batch = compressible.slice(i, i + fanIn);
|
||||
if (batch.length < 2) break;
|
||||
|
||||
const summaryResult = await summarizeBatch(
|
||||
batch,
|
||||
typeDef,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings,
|
||||
);
|
||||
if (!summaryResult) continue;
|
||||
const normalizedFields = normalizeCompressedFields(
|
||||
summaryResult,
|
||||
typeDef,
|
||||
batch,
|
||||
);
|
||||
if (Object.keys(normalizedFields).length === 0) {
|
||||
throw new Error(
|
||||
`压缩结果缺少可用 fields，无法创建 ${typeDef?.label || typeDef?.id || "压缩"} 节点`,
|
||||
);
|
||||
}
|
||||
|
||||
const compressedNode = createNode({
|
||||
type: typeDef.id,
|
||||
fields: normalizedFields,
|
||||
seq: batch[batch.length - 1].seq,
|
||||
seqRange: [
|
||||
batch[0].seqRange?.[0] ?? batch[0].seq,
|
||||
batch[batch.length - 1].seqRange?.[1] ?? batch[batch.length - 1].seq,
|
||||
],
|
||||
importance: Math.max(...batch.map((n) => n.importance)),
|
||||
scope: normalizeMemoryScope(batch[0]?.scope),
|
||||
});
|
||||
|
||||
compressedNode.level = level + 1;
|
||||
compressedNode.childIds = batch.map((n) => n.id);
|
||||
|
||||
const embeddingText =
|
||||
normalizeCompressionFieldValue(
|
||||
normalizedFields.summary ||
|
||||
normalizedFields.insight ||
|
||||
normalizedFields.title ||
|
||||
normalizedFields.name,
|
||||
) || "";
|
||||
if (isDirectVectorConfig(embeddingConfig) && embeddingText) {
|
||||
const vec = await embedText(
|
||||
embeddingText,
|
||||
embeddingConfig,
|
||||
{ signal },
|
||||
);
|
||||
if (vec) compressedNode.embedding = Array.from(vec);
|
||||
}
|
||||
|
||||
addNode(graph, compressedNode);
|
||||
migrateBatchEdges(graph, batch, compressedNode);
|
||||
created++;
|
||||
|
||||
for (const child of batch) {
|
||||
child.archived = true;
|
||||
child.parentId = compressedNode.id;
|
||||
archived++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return { created, archived };
|
||||
}
|
||||
|
||||
function groupCompressionCandidates(nodes = []) {
|
||||
const groups = new Map();
|
||||
for (const node of nodes) {
|
||||
const normalizedScope = normalizeMemoryScope(node?.scope);
|
||||
const key =
|
||||
normalizedScope.layer === "pov"
|
||||
? [
|
||||
"pov",
|
||||
getScopeOwnerKey(normalizedScope) || "owner:none",
|
||||
node.type || "",
|
||||
].join("::")
|
||||
: [
|
||||
"objective",
|
||||
getScopeRegionKey(normalizedScope) || "region:global",
|
||||
node.type || "",
|
||||
].join("::");
|
||||
if (!groups.has(key)) {
|
||||
groups.set(key, []);
|
||||
}
|
||||
groups.get(key).push(node);
|
||||
}
|
||||
return [...groups.values()].map((group) =>
|
||||
group.sort((a, b) => a.seq - b.seq),
|
||||
);
|
||||
}
|
||||
|
||||
function inspectCompressibleGroup(group = [], compression = {}, force = false) {
|
||||
const { fanIn, threshold, keepRecent } = resolveCompressionWindow(
|
||||
compression,
|
||||
force,
|
||||
);
|
||||
if (force ? group.length < fanIn : group.length <= threshold) {
|
||||
return null;
|
||||
}
|
||||
|
||||
const compressible = group.slice(0, Math.max(0, group.length - keepRecent));
|
||||
if (compressible.length < fanIn) {
|
||||
return null;
|
||||
}
|
||||
|
||||
return {
|
||||
candidateCount: compressible.length,
|
||||
fanIn,
|
||||
threshold,
|
||||
keepRecent,
|
||||
};
|
||||
}
|
||||
|
||||
export function inspectAutoCompressionCandidates(
|
||||
graph,
|
||||
schema = [],
|
||||
force = false,
|
||||
) {
|
||||
const safeSchema = Array.isArray(schema) ? schema : [];
|
||||
for (const typeDef of safeSchema) {
|
||||
if (typeDef?.compression?.mode !== "hierarchical") continue;
|
||||
const maxDepth = Number.isFinite(Number(typeDef?.compression?.maxDepth))
|
||||
? Math.max(1, Number(typeDef.compression.maxDepth))
|
||||
: 1;
|
||||
|
||||
for (let level = 0; level < maxDepth; level++) {
|
||||
const levelNodes = getActiveNodes(graph, typeDef.id)
|
||||
.filter((node) => Number(node?.level || 0) === level)
|
||||
.sort((a, b) => a.seq - b.seq);
|
||||
|
||||
for (const group of groupCompressionCandidates(levelNodes)) {
|
||||
const summary = inspectCompressibleGroup(
|
||||
group,
|
||||
typeDef.compression,
|
||||
force,
|
||||
);
|
||||
if (!summary) continue;
|
||||
return {
|
||||
hasCandidates: true,
|
||||
typeId: String(typeDef.id || ""),
|
||||
level,
|
||||
candidateCount: summary.candidateCount,
|
||||
threshold: summary.threshold,
|
||||
fanIn: summary.fanIn,
|
||||
keepRecent: summary.keepRecent,
|
||||
reason: "",
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
hasCandidates: false,
|
||||
typeId: "",
|
||||
level: null,
|
||||
candidateCount: 0,
|
||||
threshold: 0,
|
||||
fanIn: 0,
|
||||
keepRecent: 0,
|
||||
reason: "已到自动压缩周期，但当前没有达到内部压缩阈值的候选组",
|
||||
};
|
||||
}
|
||||
|
||||
function migrateBatchEdges(graph, batch, compressedNode) {
|
||||
const batchIds = new Set(batch.map((node) => node.id));
|
||||
|
||||
for (const edge of graph.edges) {
|
||||
if (edge.invalidAt || edge.expiredAt) continue;
|
||||
|
||||
const fromInside = batchIds.has(edge.fromId);
|
||||
const toInside = batchIds.has(edge.toId);
|
||||
if (!fromInside && !toInside) continue;
|
||||
if (fromInside && toInside) continue;
|
||||
|
||||
const newFromId = fromInside ? compressedNode.id : edge.fromId;
|
||||
const newToId = toInside ? compressedNode.id : edge.toId;
|
||||
|
||||
if (newFromId === newToId) continue;
|
||||
if (!getNode(graph, newFromId) || !getNode(graph, newToId)) continue;
|
||||
|
||||
const migratedEdge = createEdge({
|
||||
fromId: newFromId,
|
||||
toId: newToId,
|
||||
relation: edge.relation,
|
||||
strength: edge.strength,
|
||||
edgeType: edge.edgeType,
|
||||
scope: edge.scope,
|
||||
});
|
||||
migratedEdge.validAt = edge.validAt ?? migratedEdge.validAt;
|
||||
migratedEdge.invalidAt = edge.invalidAt ?? migratedEdge.invalidAt;
|
||||
migratedEdge.expiredAt = edge.expiredAt ?? migratedEdge.expiredAt;
|
||||
|
||||
addEdge(graph, migratedEdge);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 调用 LLM 总结一批节点
|
||||
*/
|
||||
async function summarizeBatch(
|
||||
nodes,
|
||||
typeDef,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings = {},
|
||||
) {
|
||||
const nodeDescriptions = nodes
|
||||
.map((n, i) => {
|
||||
const fieldsStr = Object.entries(n.fields)
|
||||
.filter(([_, v]) => v)
|
||||
.map(([k, v]) => `${k}: ${v}`)
|
||||
.join("\n ");
|
||||
return `节点 ${i + 1} [楼层 ${n.seq}]:\n ${fieldsStr}`;
|
||||
})
|
||||
.join("\n\n");
|
||||
|
||||
const instruction =
|
||||
typeDef.compression.instruction || "将以下节点压缩总结为一条精炼记录。";
|
||||
|
||||
const compressPromptBuild = await buildTaskPrompt(settings, "compress", {
|
||||
taskName: "compress",
|
||||
nodeContent: nodeDescriptions,
|
||||
candidateNodes: nodeDescriptions,
|
||||
currentRange: `${nodes[0]?.seq ?? "?"} ~ ${nodes[nodes.length - 1]?.seq ?? "?"}`,
|
||||
graphStats: `node_count=${nodes.length}, node_type=${typeDef.id}`,
|
||||
...getSTContextForPrompt(),
|
||||
});
|
||||
const compressRegexInput = { entries: [] };
|
||||
const systemPrompt = applyTaskRegex(
|
||||
settings,
|
||||
"compress",
|
||||
"finalPrompt",
|
||||
compressPromptBuild.systemPrompt ||
|
||||
customPrompt ||
|
||||
[
|
||||
"你是一个记忆压缩器。将多个同类型节点总结为一条更高层级的压缩节点。",
|
||||
instruction,
|
||||
"",
|
||||
"输出格式为严格 JSON：",
|
||||
`{"fields": {${typeDef.columns.map((c) => `"${c.name}": "..."`).join(", ")}}}`,
|
||||
"",
|
||||
"规则：",
|
||||
"- 保留关键信息：因果关系、不可逆结果、未解决伏笔",
|
||||
"- 去除重复和低信息密度内容",
|
||||
"- 压缩后文本应精炼，目标 150 字左右",
|
||||
].join("\n"),
|
||||
compressRegexInput,
|
||||
"system",
|
||||
);
|
||||
|
||||
const userPrompt = `请压缩以下 ${nodes.length} 个 "${typeDef.label}" 节点：\n\n${nodeDescriptions}`;
|
||||
const promptPayload = resolveTaskPromptPayload(
|
||||
compressPromptBuild,
|
||||
userPrompt,
|
||||
);
|
||||
const llmSystemPrompt =
|
||||
Array.isArray(promptPayload.promptMessages) &&
|
||||
promptPayload.promptMessages.length > 0
|
||||
? String(promptPayload.systemPrompt || "")
|
||||
: String(promptPayload.systemPrompt || systemPrompt || "");
|
||||
|
||||
return await callLLMForJSON({
|
||||
systemPrompt: llmSystemPrompt,
|
||||
userPrompt: promptPayload.userPrompt,
|
||||
maxRetries: 1,
|
||||
signal,
|
||||
taskType: "compress",
|
||||
debugContext: createTaskLlmDebugContext(
|
||||
compressPromptBuild,
|
||||
compressRegexInput,
|
||||
),
|
||||
promptMessages: promptPayload.promptMessages,
|
||||
additionalMessages: promptPayload.additionalMessages,
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* 对所有支持压缩的类型执行压缩
|
||||
*
|
||||
* @param {object} graph
|
||||
* @param {object[]} schema
|
||||
* @param {object} embeddingConfig
|
||||
* @param {boolean} [force=false]
|
||||
* @returns {Promise<{created: number, archived: number}>}
|
||||
*/
|
||||
export async function compressAll(
|
||||
graph,
|
||||
schema,
|
||||
embeddingConfig,
|
||||
force = false,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings = {},
|
||||
) {
|
||||
let totalCreated = 0;
|
||||
let totalArchived = 0;
|
||||
|
||||
for (const typeDef of schema) {
|
||||
throwIfAborted(signal);
|
||||
if (typeDef.compression?.mode === "hierarchical") {
|
||||
const result = await compressType({
|
||||
graph,
|
||||
typeDef,
|
||||
embeddingConfig,
|
||||
force,
|
||||
customPrompt,
|
||||
signal,
|
||||
settings,
|
||||
});
|
||||
totalCreated += result.created;
|
||||
totalArchived += result.archived;
|
||||
}
|
||||
}
|
||||
|
||||
return { created: totalCreated, archived: totalArchived };
|
||||
}
|
||||
|
||||
// ==================== v2: 主动遗忘（SleepGate 启发） ====================
|
||||
|
||||
/**
|
||||
* 睡眠清理周期
|
||||
* 评估每个节点的保留价值，低于阈值的归档（遗忘）
|
||||
*
|
||||
* @param {object} graph - 图状态
|
||||
* @param {object} settings - 包含 forgetThreshold 的设置
|
||||
* @returns {{forgotten: number}} 本次遗忘的节点数
|
||||
*/
|
||||
export function sleepCycle(graph, settings) {
|
||||
const threshold = settings.forgetThreshold ?? 0.5;
|
||||
const nodes = getActiveNodes(graph);
|
||||
const now = Date.now();
|
||||
let forgotten = 0;
|
||||
|
||||
for (const node of nodes) {
|
||||
// 跳过常驻类型（synopsis, rule 等重要节点不应被遗忘）
|
||||
if (
|
||||
node.type === "synopsis" ||
|
||||
node.type === "rule" ||
|
||||
node.type === "thread"
|
||||
)
|
||||
continue;
|
||||
// 跳过高重要性节点
|
||||
if (node.importance >= 8) continue;
|
||||
// 跳过最近创建的节点（< 1 小时）
|
||||
if (now - node.createdTime < 3600000) continue;
|
||||
|
||||
// 计算保留价值 = importance × recency × (1 + accessFreq)
|
||||
const ageHours = (now - node.createdTime) / 3600000;
|
||||
const recency = 1 / (1 + Math.log10(1 + ageHours));
|
||||
const accessFreq = node.accessCount / Math.max(1, ageHours / 24);
|
||||
const retentionValue = (node.importance / 10) * recency * (1 + accessFreq);
|
||||
|
||||
if (retentionValue < threshold) {
|
||||
node.archived = true;
|
||||
forgotten++;
|
||||
}
|
||||
}
|
||||
|
||||
if (forgotten > 0) {
|
||||
debugLog(`[ST-BME] 主动遗忘: ${forgotten} 个低价值节点已归档`);
|
||||
}
|
||||
|
||||
return { forgotten };
|
||||
}
|
||||
724
maintenance/consolidator.js
Normal file
724
maintenance/consolidator.js
Normal file
@@ -0,0 +1,724 @@
|
||||
// ST-BME: 统一记忆整合引擎（批量化版）
|
||||
// 合并 Mem0 精确对照 + A-MEM 记忆进化为单一阶段
|
||||
// 批量 embed + 批量查近邻 + 单次 LLM 调用
|
||||
|
||||
import { debugLog } from "../runtime/debug-logging.js";
|
||||
import { embedBatch, searchSimilar } from "../vector/embedding.js";
|
||||
import { addEdge, createEdge, getActiveNodes, getNode } from "../graph/graph.js";
|
||||
import { callLLMForJSON } from "../llm/llm.js";
|
||||
import {
|
||||
buildScopeBadgeText,
|
||||
canMergeScopedMemories,
|
||||
describeMemoryScope,
|
||||
} from "../graph/memory-scope.js";
|
||||
import {
|
||||
buildTaskExecutionDebugContext,
|
||||
buildTaskLlmPayload,
|
||||
buildTaskPrompt,
|
||||
} from "../prompting/prompt-builder.js";
|
||||
import { getSTContextForPrompt } from "../host/st-context.js";
|
||||
import { applyTaskRegex } from "../prompting/task-regex.js";
|
||||
import {
|
||||
buildNodeVectorText,
|
||||
findSimilarNodesByText,
|
||||
isDirectVectorConfig,
|
||||
validateVectorConfig,
|
||||
} from "../vector/vector-index.js";
|
||||
|
||||
function createAbortError(message = "操作已终止") {
|
||||
const error = new Error(message);
|
||||
error.name = "AbortError";
|
||||
return error;
|
||||
}
|
||||
|
||||
function createTaskLlmDebugContext(promptBuild, regexInput) {
|
||||
return typeof buildTaskExecutionDebugContext === "function"
|
||||
? buildTaskExecutionDebugContext(promptBuild, { regexInput })
|
||||
: null;
|
||||
}
|
||||
|
||||
function resolveTaskPromptPayload(promptBuild, fallbackUserPrompt = "") {
|
||||
if (typeof buildTaskLlmPayload === "function") {
|
||||
return buildTaskLlmPayload(promptBuild, fallbackUserPrompt);
|
||||
}
|
||||
|
||||
return {
|
||||
systemPrompt: String(promptBuild?.systemPrompt || ""),
|
||||
userPrompt: String(fallbackUserPrompt || ""),
|
||||
promptMessages: [],
|
||||
additionalMessages: Array.isArray(promptBuild?.privateTaskMessages)
|
||||
? promptBuild.privateTaskMessages
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
function isAbortError(error) {
|
||||
return error?.name === "AbortError";
|
||||
}
|
||||
|
||||
function throwIfAborted(signal) {
|
||||
if (signal?.aborted) {
|
||||
throw signal.reason instanceof Error ? signal.reason : createAbortError();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一记忆整合系统提示词（支持批量输出）
|
||||
*/
|
||||
const CONSOLIDATION_SYSTEM_PROMPT = `你是一个记忆整合分析器。当新记忆加入知识图谱时，你需要同时完成两项任务：
|
||||
|
||||
**任务一：冲突检测**
|
||||
判断新记忆与最近邻的已有记忆是否冲突或重复：
|
||||
- skip: 新记忆与已有记忆完全重复，应丢弃
|
||||
- merge: 新记忆是对旧记忆的修正或补充，应合并
|
||||
- keep: 新记忆是全新信息，应保留
|
||||
|
||||
**任务二：进化分析**（仅当 action=keep 时需要）
|
||||
分析新记忆是否揭示了关于旧记忆的新信息：
|
||||
- 建立有意义的关联连接
|
||||
- 反向更新旧记忆的描述或分类
|
||||
|
||||
输出严格 JSON：
|
||||
{
|
||||
"results": [
|
||||
{
|
||||
"node_id": "新记忆的节点 ID",
|
||||
"action": "keep" | "merge" | "skip",
|
||||
"merge_target_id": "仅 action=merge 时必填：要合并到的旧节点 ID",
|
||||
"merged_fields": { "仅 action=merge 时可选：合并后的字段更新" },
|
||||
"reason": "判定理由（简述）",
|
||||
"evolution": {
|
||||
"should_evolve": true/false,
|
||||
"connections": ["需要建立链接的旧记忆 ID 列表"],
|
||||
"neighbor_updates": [
|
||||
{
|
||||
"nodeId": "需更新的旧节点 ID",
|
||||
"newContext": "基于新信息修正后的描述（不需修改则为 null）",
|
||||
"newTags": ["更新后的分类标签，不需修改则为 null"]
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
整合规则：
|
||||
- 必须对每条新记忆都给出一个 result 条目
|
||||
- 当 action=skip 时，evolution 可省略或设 should_evolve=false
|
||||
- 当 action=merge 时，evolution 可省略或设 should_evolve=false
|
||||
- 仅当 action=keep 且新信息确实改变了对旧记忆的理解时，才设 should_evolve=true
|
||||
- 例如：揭露卧底身份 → 修正该角色之前事件中的动机描述
|
||||
- 例如：发现地点的隐藏特性 → 更新地点节点的描述
|
||||
- 不要对无关记忆强行建立联系
|
||||
- neighbor_updates 中每条必须有实际意义的修改`;
|
||||
|
||||
function normalizeLatestOnlyIdentityValue(value) {
|
||||
return String(value ?? "")
|
||||
.trim()
|
||||
.replace(/\s+/g, " ")
|
||||
.toLowerCase();
|
||||
}
|
||||
|
||||
export async function analyzeAutoConsolidationGate({
|
||||
graph,
|
||||
newNodeIds,
|
||||
embeddingConfig,
|
||||
schema = [],
|
||||
conflictThreshold = 0.85,
|
||||
signal,
|
||||
} = {}) {
|
||||
const normalizedThreshold = Number.isFinite(Number(conflictThreshold))
|
||||
? Math.max(0, Math.min(1, Number(conflictThreshold)))
|
||||
: 0.85;
|
||||
const safeNewNodeIds = Array.isArray(newNodeIds) ? newNodeIds : [];
|
||||
|
||||
if (!graph || safeNewNodeIds.length === 0) {
|
||||
return {
|
||||
triggered: false,
|
||||
reason: "本批新增少且无明显重复风险，跳过自动整合",
|
||||
matchedScore: null,
|
||||
matchedNodeId: "",
|
||||
detection: "none",
|
||||
};
|
||||
}
|
||||
|
||||
const schemaByType = new Map(
|
||||
(Array.isArray(schema) ? schema : [])
|
||||
.filter((typeDef) => typeDef?.id)
|
||||
.map((typeDef) => [String(typeDef.id), typeDef]),
|
||||
);
|
||||
const activeNodes = getActiveNodes(graph).filter((node) => !node?.archived);
|
||||
const vectorConfigValid = validateVectorConfig(embeddingConfig).valid;
|
||||
let bestVectorMatch = null;
|
||||
|
||||
for (const newNodeId of safeNewNodeIds) {
|
||||
throwIfAborted(signal);
|
||||
const node = getNode(graph, newNodeId);
|
||||
if (!node || node.archived) continue;
|
||||
|
||||
const typeDef = schemaByType.get(String(node.type || ""));
|
||||
const scopedCandidates = activeNodes.filter(
|
||||
(candidate) =>
|
||||
candidate?.id !== node.id && canMergeScopedMemories(node, candidate),
|
||||
);
|
||||
|
||||
if (typeDef?.latestOnly) {
|
||||
for (const field of ["name", "title"]) {
|
||||
const normalizedIdentity = normalizeLatestOnlyIdentityValue(
|
||||
node?.fields?.[field],
|
||||
);
|
||||
if (!normalizedIdentity) continue;
|
||||
const matchedNode = scopedCandidates.find(
|
||||
(candidate) =>
|
||||
candidate?.type === node.type &&
|
||||
normalizeLatestOnlyIdentityValue(candidate?.fields?.[field]) ===
|
||||
normalizedIdentity,
|
||||
);
|
||||
if (matchedNode) {
|
||||
return {
|
||||
triggered: true,
|
||||
reason: `本批仅新增 ${safeNewNodeIds.length} 个节点，但 latestOnly 的 ${field} 与旧记忆完全一致，已触发自动整合`,
|
||||
matchedScore: 1,
|
||||
matchedNodeId: matchedNode.id,
|
||||
detection: `latestOnly:${field}`,
|
||||
};
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!vectorConfigValid) continue;
|
||||
const text = buildNodeVectorText(node);
|
||||
if (!text) continue;
|
||||
|
||||
try {
|
||||
const neighbors = await findSimilarNodesByText(
|
||||
graph,
|
||||
text,
|
||||
embeddingConfig,
|
||||
1,
|
||||
scopedCandidates,
|
||||
signal,
|
||||
);
|
||||
const topNeighbor = Array.isArray(neighbors) ? neighbors[0] : null;
|
||||
if (!topNeighbor?.nodeId) continue;
|
||||
|
||||
if (
|
||||
!bestVectorMatch ||
|
||||
Number(topNeighbor.score || 0) > Number(bestVectorMatch.score || 0)
|
||||
) {
|
||||
bestVectorMatch = {
|
||||
score: Number(topNeighbor.score || 0),
|
||||
nodeId: topNeighbor.nodeId,
|
||||
};
|
||||
}
|
||||
|
||||
if (Number(topNeighbor.score || 0) >= normalizedThreshold) {
|
||||
return {
|
||||
triggered: true,
|
||||
reason: `本批仅新增 ${safeNewNodeIds.length} 个节点，但与旧记忆高度相似（${Number(topNeighbor.score || 0).toFixed(3)} >= ${normalizedThreshold.toFixed(2)}），已触发自动整合`,
|
||||
matchedScore: Number(topNeighbor.score || 0),
|
||||
matchedNodeId: topNeighbor.nodeId,
|
||||
detection: "vector",
|
||||
};
|
||||
}
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
console.warn(
|
||||
`[ST-BME] 自动整合门禁近邻查询失败 (${newNodeId}):`,
|
||||
error?.message || error,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
if (bestVectorMatch) {
|
||||
return {
|
||||
triggered: false,
|
||||
reason: `本批新增少且最高相似度 ${bestVectorMatch.score.toFixed(3)} 未达到阈值 ${normalizedThreshold.toFixed(2)}，跳过自动整合`,
|
||||
matchedScore: bestVectorMatch.score,
|
||||
matchedNodeId: bestVectorMatch.nodeId,
|
||||
detection: "vector",
|
||||
};
|
||||
}
|
||||
|
||||
if (!vectorConfigValid) {
|
||||
return {
|
||||
triggered: false,
|
||||
reason: "本批新增少且当前向量不可用，未检测到明确重复风险，跳过自动整合",
|
||||
matchedScore: null,
|
||||
matchedNodeId: "",
|
||||
detection: "vector-unavailable",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
triggered: false,
|
||||
reason: "本批新增少且无明显重复风险，跳过自动整合",
|
||||
matchedScore: null,
|
||||
matchedNodeId: "",
|
||||
detection: "none",
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* 统一记忆整合主函数（批量化版）
|
||||
*
|
||||
* 4 阶段架构：
|
||||
* Phase 0: 收集有效新节点
|
||||
* Phase 1: 批量 Embed（直连 1 次 embedBatch / 后端逐次）
|
||||
* Phase 2: 各节点查近邻（直连本地 cosine / 后端逐次 query）
|
||||
* Phase 3: 单次 LLM 批量判定
|
||||
* Phase 4: 逐个处理结果
|
||||
*
|
||||
* @param {object} params
|
||||
* @param {object} params.graph - 当前图状态
|
||||
* @param {string[]} params.newNodeIds - 本次新创建的节点 ID 列表
|
||||
* @param {object} params.embeddingConfig - Embedding API 配置
|
||||
* @param {object} [params.options]
|
||||
* @param {number} [params.options.neighborCount=5]
|
||||
* @param {number} [params.options.conflictThreshold=0.85]
|
||||
* @param {string} [params.customPrompt]
|
||||
* @param {AbortSignal} [params.signal]
|
||||
* @returns {Promise<{merged: number, skipped: number, kept: number, evolved: number, connections: number, updates: number}>}
|
||||
*/
|
||||
export async function consolidateMemories({
|
||||
graph,
|
||||
newNodeIds,
|
||||
embeddingConfig,
|
||||
options = {},
|
||||
customPrompt,
|
||||
signal,
|
||||
settings = {},
|
||||
}) {
|
||||
const neighborCount = options.neighborCount ?? 5;
|
||||
const conflictThreshold = options.conflictThreshold ?? 0.85;
|
||||
const stats = {
|
||||
merged: 0,
|
||||
skipped: 0,
|
||||
kept: 0,
|
||||
evolved: 0,
|
||||
connections: 0,
|
||||
updates: 0,
|
||||
};
|
||||
|
||||
if (!newNodeIds || newNodeIds.length === 0) return stats;
|
||||
if (!validateVectorConfig(embeddingConfig).valid) {
|
||||
debugLog("[ST-BME] 记忆整合跳过：向量配置不可用");
|
||||
return stats;
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════
|
||||
// Phase 0: 收集有效新节点
|
||||
// ══════════════════════════════════════════════
|
||||
const newEntries = [];
|
||||
for (const id of newNodeIds) {
|
||||
const node = getNode(graph, id);
|
||||
if (!node || node.archived) continue;
|
||||
const text = buildNodeVectorText(node);
|
||||
if (!text) continue;
|
||||
newEntries.push({ id, node, text });
|
||||
}
|
||||
|
||||
if (newEntries.length === 0) return stats;
|
||||
|
||||
const activeNodes = getActiveNodes(graph).filter((n) => {
|
||||
const text = buildNodeVectorText(n);
|
||||
return typeof text === "string" && text.length > 0;
|
||||
});
|
||||
|
||||
if (activeNodes.length < 2) {
|
||||
// 图中节点不够，全部 keep
|
||||
stats.kept = newEntries.length;
|
||||
return stats;
|
||||
}
|
||||
|
||||
throwIfAborted(signal);
|
||||
debugLog(`[ST-BME] 记忆整合开始: ${newEntries.length} 个新节点`);
|
||||
|
||||
// ══════════════════════════════════════════════
|
||||
// Phase 1 + 2: 批量 Embed + 查近邻
|
||||
// ══════════════════════════════════════════════
|
||||
/** @type {Map<string, Array<{nodeId: string, score: number}>>} */
|
||||
const neighborsMap = new Map();
|
||||
|
||||
if (isDirectVectorConfig(embeddingConfig)) {
|
||||
// ── 直连模式: 1 次 embedBatch + N 次本地 cosine ──
|
||||
const texts = newEntries.map((e) => e.text);
|
||||
let queryVectors;
|
||||
|
||||
try {
|
||||
queryVectors = await embedBatch(texts, embeddingConfig, { signal });
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) throw e;
|
||||
console.warn("[ST-BME] 批量 embed 失败，回退到逐条:", e.message);
|
||||
queryVectors = null;
|
||||
}
|
||||
|
||||
// 构建候选池（含 embedding 的活跃节点）
|
||||
const candidatePool = activeNodes
|
||||
.filter((n) => Array.isArray(n.embedding) && n.embedding.length > 0)
|
||||
.map((n) => ({ nodeId: n.id, embedding: n.embedding }));
|
||||
|
||||
for (let i = 0; i < newEntries.length; i++) {
|
||||
throwIfAborted(signal);
|
||||
const entry = newEntries[i];
|
||||
const candidates = candidatePool.filter((c) => {
|
||||
if (c.nodeId === entry.id) return false;
|
||||
const candidateNode = getNode(graph, c.nodeId);
|
||||
return canMergeScopedMemories(entry.node, candidateNode);
|
||||
});
|
||||
|
||||
if (queryVectors?.[i] && candidates.length > 0) {
|
||||
// 本地 cosine 搜索（0 API 调用）
|
||||
const neighbors = searchSimilar(
|
||||
queryVectors[i],
|
||||
candidates,
|
||||
neighborCount,
|
||||
);
|
||||
neighborsMap.set(entry.id, neighbors);
|
||||
} else {
|
||||
// fallback: 逐条 embed
|
||||
try {
|
||||
const neighbors = await findSimilarNodesByText(
|
||||
graph,
|
||||
entry.text,
|
||||
embeddingConfig,
|
||||
neighborCount,
|
||||
activeNodes.filter((n) => n.id !== entry.id),
|
||||
signal,
|
||||
);
|
||||
neighborsMap.set(entry.id, neighbors);
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) throw e;
|
||||
console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
|
||||
neighborsMap.set(entry.id, []);
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
// ── 后端模式: 逐条 /api/vector/query ──
|
||||
for (let i = 0; i < newEntries.length; i++) {
|
||||
throwIfAborted(signal);
|
||||
const entry = newEntries[i];
|
||||
try {
|
||||
const neighbors = await findSimilarNodesByText(
|
||||
graph,
|
||||
entry.text,
|
||||
embeddingConfig,
|
||||
neighborCount,
|
||||
activeNodes.filter(
|
||||
(n) => n.id !== entry.id && canMergeScopedMemories(entry.node, n),
|
||||
),
|
||||
signal,
|
||||
);
|
||||
neighborsMap.set(entry.id, neighbors);
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) throw e;
|
||||
console.warn(`[ST-BME] 近邻查询失败 (${entry.id}):`, e.message);
|
||||
neighborsMap.set(entry.id, []);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════
|
||||
// Phase 3: 单次 LLM 批量判定
|
||||
// ══════════════════════════════════════════════
|
||||
throwIfAborted(signal);
|
||||
|
||||
const userPromptSections = [];
|
||||
userPromptSections.push(
|
||||
`本轮共新增 ${newEntries.length} 条记忆，请逐条分析：\n`,
|
||||
);
|
||||
|
||||
for (let i = 0; i < newEntries.length; i++) {
|
||||
const entry = newEntries[i];
|
||||
const neighbors = neighborsMap.get(entry.id) || [];
|
||||
|
||||
const newNodeFieldsStr = Object.entries(entry.node.fields)
|
||||
.map(([k, v]) => `${k}: ${v}`)
|
||||
.join(", ");
|
||||
const newNodeScope = buildScopeBadgeText(entry.node.scope);
|
||||
|
||||
// 构建近邻描述
|
||||
let neighborText;
|
||||
if (neighbors.length === 0) {
|
||||
neighborText = " (无近邻命中)";
|
||||
} else {
|
||||
neighborText = neighbors
|
||||
.map((n) => {
|
||||
const node = getNode(graph, n.nodeId);
|
||||
if (!node) return null;
|
||||
const fieldsStr = Object.entries(node.fields)
|
||||
.map(([k, v]) => `${k}: ${v}`)
|
||||
.join(", ");
|
||||
return ` - [${node.id}] 类型=${node.type}, 作用域=${describeMemoryScope(node.scope)}, ${fieldsStr} (相似度=${n.score.toFixed(3)})`;
|
||||
})
|
||||
.filter(Boolean)
|
||||
.join("\n");
|
||||
}
|
||||
|
||||
// 检查高相似度
|
||||
const hasHighSimilarity =
|
||||
neighbors.length > 0 && neighbors[0].score > conflictThreshold;
|
||||
const hint = hasHighSimilarity
|
||||
? ` ⚠ 最高相似度 ${neighbors[0].score.toFixed(3)} 超过阈值 ${conflictThreshold}`
|
||||
: "";
|
||||
|
||||
userPromptSections.push(
|
||||
[
|
||||
`### 新记忆 #${i + 1}`,
|
||||
`[${entry.id}] 类型=${entry.node.type}, 作用域=${newNodeScope}, ${newNodeFieldsStr}`,
|
||||
"近邻记忆:",
|
||||
neighborText,
|
||||
hint,
|
||||
]
|
||||
.filter(Boolean)
|
||||
.join("\n"),
|
||||
);
|
||||
}
|
||||
|
||||
const userPrompt = userPromptSections.join("\n\n");
|
||||
|
||||
let decision;
|
||||
const consolidationPromptBuild = await buildTaskPrompt(settings, "consolidation", {
|
||||
taskName: "consolidation",
|
||||
candidateNodes: userPrompt,
|
||||
candidateText: userPrompt,
|
||||
graphStats: `new_entries=${newEntries.length}, threshold=${conflictThreshold}`,
|
||||
...getSTContextForPrompt(),
|
||||
});
|
||||
const consolidationRegexInput = { entries: [] };
|
||||
const consolidationSystemPrompt = applyTaskRegex(
|
||||
settings,
|
||||
"consolidation",
|
||||
"finalPrompt",
|
||||
consolidationPromptBuild.systemPrompt ||
|
||||
customPrompt ||
|
||||
CONSOLIDATION_SYSTEM_PROMPT,
|
||||
consolidationRegexInput,
|
||||
"system",
|
||||
);
|
||||
const promptPayload = resolveTaskPromptPayload(
|
||||
consolidationPromptBuild,
|
||||
userPrompt,
|
||||
);
|
||||
const llmSystemPrompt =
|
||||
Array.isArray(promptPayload.promptMessages) &&
|
||||
promptPayload.promptMessages.length > 0
|
||||
? String(promptPayload.systemPrompt || "")
|
||||
: String(promptPayload.systemPrompt || consolidationSystemPrompt || "");
|
||||
try {
|
||||
decision = await callLLMForJSON({
|
||||
systemPrompt: llmSystemPrompt,
|
||||
userPrompt: promptPayload.userPrompt,
|
||||
maxRetries: 1,
|
||||
signal,
|
||||
taskType: "consolidation",
|
||||
debugContext: createTaskLlmDebugContext(
|
||||
consolidationPromptBuild,
|
||||
consolidationRegexInput,
|
||||
),
|
||||
promptMessages: promptPayload.promptMessages,
|
||||
additionalMessages: promptPayload.additionalMessages,
|
||||
});
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) throw e;
|
||||
console.error("[ST-BME] 记忆整合 LLM 调用失败:", e);
|
||||
stats.kept = newEntries.length;
|
||||
return stats;
|
||||
}
|
||||
|
||||
// ══════════════════════════════════════════════
|
||||
// Phase 4: 逐个处理结果
|
||||
// ══════════════════════════════════════════════
|
||||
|
||||
// 解析 LLM 返回——兼容单条和批量格式
|
||||
let results;
|
||||
if (Array.isArray(decision?.results)) {
|
||||
results = decision.results;
|
||||
} else if (decision?.action) {
|
||||
// 单条返回格式（LLM 可能忽略 results 包装）
|
||||
results = [{ ...decision, node_id: newEntries[0]?.id }];
|
||||
} else {
|
||||
console.warn("[ST-BME] 记忆整合: LLM 返回格式异常，全部 keep");
|
||||
stats.kept = newEntries.length;
|
||||
return stats;
|
||||
}
|
||||
|
||||
// 建立 node_id → result 的映射
|
||||
const resultMap = new Map();
|
||||
for (const r of results) {
|
||||
if (r.node_id) resultMap.set(r.node_id, r);
|
||||
}
|
||||
|
||||
// 处理每个新节点
|
||||
for (const entry of newEntries) {
|
||||
const result = resultMap.get(entry.id);
|
||||
if (!result) {
|
||||
// LLM 未返回此节点的结果，fallback 为 keep
|
||||
stats.kept++;
|
||||
continue;
|
||||
}
|
||||
|
||||
processOneResult(graph, entry, result, stats);
|
||||
}
|
||||
|
||||
// 日志
|
||||
const actionSummary = [];
|
||||
if (stats.merged > 0) actionSummary.push(`合并 ${stats.merged}`);
|
||||
if (stats.skipped > 0) actionSummary.push(`跳过 ${stats.skipped}`);
|
||||
if (stats.kept > 0) actionSummary.push(`保留 ${stats.kept}`);
|
||||
if (stats.evolved > 0) actionSummary.push(`进化 ${stats.evolved}`);
|
||||
if (stats.connections > 0) actionSummary.push(`新链接 ${stats.connections}`);
|
||||
if (stats.updates > 0) actionSummary.push(`回溯更新 ${stats.updates}`);
|
||||
|
||||
if (actionSummary.length > 0) {
|
||||
debugLog(`[ST-BME] 记忆整合完成: ${actionSummary.join(", ")}`);
|
||||
}
|
||||
|
||||
return stats;
|
||||
}
|
||||
|
||||
/**
|
||||
* 处理单个节点的整合结果
|
||||
*/
|
||||
function processOneResult(graph, entry, result, stats) {
|
||||
const { id: newId, node: newNode } = entry;
|
||||
|
||||
// ── 处理 action ──
|
||||
switch (result.action) {
|
||||
case "skip": {
|
||||
debugLog(`[ST-BME] 记忆整合: skip (重复) — ${newId}`);
|
||||
newNode.archived = true;
|
||||
stats.skipped++;
|
||||
break;
|
||||
}
|
||||
|
||||
case "merge": {
|
||||
const targetId = result.merge_target_id;
|
||||
const targetNode = targetId ? getNode(graph, targetId) : null;
|
||||
|
||||
if (
|
||||
targetNode &&
|
||||
!targetNode.archived &&
|
||||
canMergeScopedMemories(newNode, targetNode)
|
||||
) {
|
||||
debugLog(`[ST-BME] 记忆整合: merge ${newId} → ${targetId}`);
|
||||
|
||||
if (result.merged_fields && typeof result.merged_fields === "object") {
|
||||
for (const [key, value] of Object.entries(result.merged_fields)) {
|
||||
if (value != null && value !== "") {
|
||||
targetNode.fields[key] = value;
|
||||
}
|
||||
}
|
||||
} else {
|
||||
for (const [key, value] of Object.entries(newNode.fields)) {
|
||||
if (value != null && value !== "" && !targetNode.fields[key]) {
|
||||
targetNode.fields[key] = value;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (
|
||||
Number.isFinite(newNode.seq) &&
|
||||
newNode.seq > (targetNode.seq || 0)
|
||||
) {
|
||||
targetNode.seq = newNode.seq;
|
||||
}
|
||||
|
||||
const targetRange = Array.isArray(targetNode.seqRange)
|
||||
? targetNode.seqRange
|
||||
: [targetNode.seq || 0, targetNode.seq || 0];
|
||||
const newRange = Array.isArray(newNode.seqRange)
|
||||
? newNode.seqRange
|
||||
: [newNode.seq || 0, newNode.seq || 0];
|
||||
targetNode.seqRange = [
|
||||
Math.min(targetRange[0], newRange[0]),
|
||||
Math.max(targetRange[1], newRange[1]),
|
||||
];
|
||||
|
||||
targetNode.embedding = null;
|
||||
newNode.archived = true;
|
||||
stats.merged++;
|
||||
} else {
|
||||
console.warn(
|
||||
`[ST-BME] 记忆整合: merge target ${targetId} 不存在，回退为 keep`,
|
||||
);
|
||||
stats.kept++;
|
||||
}
|
||||
break;
|
||||
}
|
||||
|
||||
case "keep":
|
||||
default: {
|
||||
stats.kept++;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
// ── 处理 evolution ──
|
||||
const evolution = result.evolution;
|
||||
if (evolution?.should_evolve && !newNode.archived) {
|
||||
stats.evolved++;
|
||||
debugLog(`[ST-BME] 记忆整合/进化触发: ${result.reason || "(无理由)"}`);
|
||||
|
||||
if (Array.isArray(evolution.connections)) {
|
||||
for (const targetId of evolution.connections) {
|
||||
if (!getNode(graph, targetId)) continue;
|
||||
const edge = createEdge({
|
||||
fromId: newId,
|
||||
toId: targetId,
|
||||
relation: "related",
|
||||
strength: 0.7,
|
||||
});
|
||||
if (addEdge(graph, edge)) {
|
||||
stats.connections++;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (Array.isArray(evolution.neighbor_updates)) {
|
||||
for (const update of evolution.neighbor_updates) {
|
||||
if (!update.nodeId) continue;
|
||||
const oldNode = getNode(graph, update.nodeId);
|
||||
if (
|
||||
!oldNode ||
|
||||
oldNode.archived ||
|
||||
!canMergeScopedMemories(newNode, oldNode)
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
let changed = false;
|
||||
|
||||
if (update.newContext && typeof update.newContext === "string") {
|
||||
if (oldNode.fields.state !== undefined) {
|
||||
oldNode.fields.state = update.newContext;
|
||||
changed = true;
|
||||
} else if (oldNode.fields.summary !== undefined) {
|
||||
oldNode.fields.summary = update.newContext;
|
||||
changed = true;
|
||||
} else if (oldNode.fields.core_note !== undefined) {
|
||||
oldNode.fields.core_note = update.newContext;
|
||||
changed = true;
|
||||
}
|
||||
}
|
||||
|
||||
if (update.newTags && Array.isArray(update.newTags)) {
|
||||
oldNode.clusters = update.newTags;
|
||||
changed = true;
|
||||
}
|
||||
|
||||
if (changed) {
|
||||
oldNode.embedding = null;
|
||||
if (!oldNode._evolutionHistory) oldNode._evolutionHistory = [];
|
||||
oldNode._evolutionHistory.push({
|
||||
triggeredBy: newId,
|
||||
timestamp: Date.now(),
|
||||
reason: result.reason || "",
|
||||
});
|
||||
stats.updates++;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
732
maintenance/extraction-controller.js
Normal file
732
maintenance/extraction-controller.js
Normal file
@@ -0,0 +1,732 @@
|
||||
// ST-BME: 提取编排控制器（纯函数）
|
||||
// 通过 runtime 依赖注入，避免直接访问 index.js 模块级状态。
|
||||
|
||||
import { debugLog } from "../runtime/debug-logging.js";
|
||||
|
||||
function toSafeFloor(value, fallback = null) {
|
||||
if (value == null || value === "") return fallback;
|
||||
const numeric = Number(value);
|
||||
return Number.isFinite(numeric) ? Math.floor(numeric) : fallback;
|
||||
}
|
||||
|
||||
function clampIntValue(value, fallback = 0, min = 0, max = 9999) {
|
||||
const numeric = Number(value);
|
||||
if (!Number.isFinite(numeric)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.trunc(numeric)));
|
||||
}
|
||||
|
||||
function isAssistantFloor(runtime, chat, index) {
|
||||
if (!Array.isArray(chat)) return false;
|
||||
const message = chat[index];
|
||||
if (!message) return false;
|
||||
if (typeof runtime?.isAssistantChatMessage === "function") {
|
||||
return Boolean(
|
||||
runtime.isAssistantChatMessage(message, {
|
||||
index,
|
||||
chat,
|
||||
}),
|
||||
);
|
||||
}
|
||||
return Boolean(message) && !message.is_user && !message.is_system;
|
||||
}
|
||||
|
||||
function getAssistantTurnsFallback(runtime, chat = []) {
|
||||
if (!Array.isArray(chat)) return [];
|
||||
const assistantTurns = [];
|
||||
for (let index = 0; index < chat.length; index++) {
|
||||
if (!isAssistantFloor(runtime, chat, index)) continue;
|
||||
if (!String(chat[index]?.mes ?? "").trim()) continue;
|
||||
assistantTurns.push(index);
|
||||
}
|
||||
return assistantTurns;
|
||||
}
|
||||
|
||||
function normalizeSmartTriggerDecision(decision = null) {
|
||||
if (!decision || typeof decision !== "object") {
|
||||
return { triggered: false, score: 0, reasons: [] };
|
||||
}
|
||||
return {
|
||||
triggered: decision.triggered === true,
|
||||
score: Number.isFinite(Number(decision.score)) ? Number(decision.score) : 0,
|
||||
reasons: Array.isArray(decision.reasons)
|
||||
? decision.reasons.map((item) => String(item || "")).filter(Boolean)
|
||||
: [],
|
||||
};
|
||||
}
|
||||
|
||||
export function resolveAutoExtractionPlanController(
|
||||
runtime,
|
||||
{
|
||||
chat = null,
|
||||
settings = null,
|
||||
lastProcessedAssistantFloor = null,
|
||||
lockedEndFloor = null,
|
||||
} = {},
|
||||
) {
|
||||
const resolvedChat = Array.isArray(chat)
|
||||
? chat
|
||||
: runtime?.getContext?.()?.chat || [];
|
||||
const resolvedSettings =
|
||||
settings && typeof settings === "object"
|
||||
? settings
|
||||
: runtime?.getSettings?.() || {};
|
||||
const safeLastProcessedAssistantFloor = toSafeFloor(
|
||||
lastProcessedAssistantFloor,
|
||||
toSafeFloor(runtime?.getLastProcessedAssistantFloor?.(), -1),
|
||||
);
|
||||
const safeLockedEndFloor = toSafeFloor(lockedEndFloor, null);
|
||||
const strategy =
|
||||
resolvedSettings.extractAutoDelayLatestAssistant === true
|
||||
? "lag-one-assistant"
|
||||
: "normal";
|
||||
const extractEvery = clampIntValue(
|
||||
resolvedSettings.extractEvery,
|
||||
1,
|
||||
1,
|
||||
50,
|
||||
);
|
||||
const assistantTurns =
|
||||
typeof runtime?.getAssistantTurns === "function"
|
||||
? runtime.getAssistantTurns(resolvedChat)
|
||||
: getAssistantTurnsFallback(runtime, resolvedChat);
|
||||
const pendingAssistantTurns = assistantTurns.filter(
|
||||
(floor) => floor > safeLastProcessedAssistantFloor,
|
||||
);
|
||||
const candidateAssistantTurns =
|
||||
safeLockedEndFloor == null
|
||||
? pendingAssistantTurns
|
||||
: pendingAssistantTurns.filter((floor) => floor <= safeLockedEndFloor);
|
||||
|
||||
let eligibleAssistantTurns = candidateAssistantTurns;
|
||||
let waitingForNextAssistant = false;
|
||||
if (safeLockedEndFloor == null && strategy === "lag-one-assistant") {
|
||||
if (candidateAssistantTurns.length <= 1) {
|
||||
eligibleAssistantTurns = [];
|
||||
waitingForNextAssistant = candidateAssistantTurns.length === 1;
|
||||
} else {
|
||||
eligibleAssistantTurns = candidateAssistantTurns.slice(0, -1);
|
||||
}
|
||||
}
|
||||
|
||||
const eligibleEndFloor =
|
||||
eligibleAssistantTurns.length > 0
|
||||
? eligibleAssistantTurns[eligibleAssistantTurns.length - 1]
|
||||
: null;
|
||||
const smartTriggerDecision =
|
||||
resolvedSettings.enableSmartTrigger && eligibleEndFloor != null
|
||||
? normalizeSmartTriggerDecision(
|
||||
runtime?.getSmartTriggerDecision?.(
|
||||
resolvedChat,
|
||||
safeLastProcessedAssistantFloor,
|
||||
resolvedSettings,
|
||||
eligibleEndFloor,
|
||||
),
|
||||
)
|
||||
: { triggered: false, score: 0, reasons: [] };
|
||||
const meetsExtractEvery = eligibleAssistantTurns.length >= extractEvery;
|
||||
const canRun =
|
||||
eligibleAssistantTurns.length > 0 &&
|
||||
(meetsExtractEvery || smartTriggerDecision.triggered);
|
||||
const batchAssistantTurns = canRun
|
||||
? smartTriggerDecision.triggered
|
||||
? eligibleAssistantTurns
|
||||
: eligibleAssistantTurns.slice(0, extractEvery)
|
||||
: [];
|
||||
const plannedBatchEndFloor =
|
||||
batchAssistantTurns.length > 0
|
||||
? batchAssistantTurns[batchAssistantTurns.length - 1]
|
||||
: null;
|
||||
|
||||
let reason = "";
|
||||
if (pendingAssistantTurns.length === 0) {
|
||||
reason = "no-unprocessed-assistant-turns";
|
||||
} else if (candidateAssistantTurns.length === 0) {
|
||||
reason =
|
||||
safeLockedEndFloor == null
|
||||
? "no-candidate-assistant-turns"
|
||||
: "locked-target-missing";
|
||||
} else if (waitingForNextAssistant) {
|
||||
reason = "waiting-next-assistant";
|
||||
} else if (!canRun && !smartTriggerDecision.triggered) {
|
||||
reason = "below-extract-every";
|
||||
}
|
||||
|
||||
return {
|
||||
strategy,
|
||||
chat: resolvedChat,
|
||||
settings: resolvedSettings,
|
||||
lastProcessedAssistantFloor: safeLastProcessedAssistantFloor,
|
||||
lockedEndFloor: safeLockedEndFloor,
|
||||
extractEvery,
|
||||
pendingAssistantTurns,
|
||||
candidateAssistantTurns,
|
||||
eligibleAssistantTurns,
|
||||
eligibleEndFloor,
|
||||
waitingForNextAssistant,
|
||||
smartTriggerDecision,
|
||||
meetsExtractEvery,
|
||||
canRun,
|
||||
batchAssistantTurns,
|
||||
plannedBatchEndFloor,
|
||||
startIdx: batchAssistantTurns[0] ?? null,
|
||||
endIdx: plannedBatchEndFloor,
|
||||
reason,
|
||||
};
|
||||
}
|
||||
|
||||
export async function executeExtractionBatchController(
|
||||
runtime,
|
||||
{
|
||||
chat,
|
||||
startIdx,
|
||||
endIdx,
|
||||
settings,
|
||||
smartTriggerDecision = null,
|
||||
signal = undefined,
|
||||
} = {},
|
||||
) {
|
||||
runtime.ensureCurrentGraphRuntimeState();
|
||||
runtime.throwIfAborted(signal, "提取已终止");
|
||||
|
||||
const currentGraph = runtime.getCurrentGraph();
|
||||
const lastProcessed = runtime.getLastProcessedAssistantFloor();
|
||||
const extractionCountBefore = runtime.getExtractionCount();
|
||||
const beforeSnapshot = runtime.cloneGraphSnapshot(currentGraph);
|
||||
const messages = runtime.buildExtractionMessages(chat, startIdx, endIdx, settings);
|
||||
const batchStatus = runtime.createBatchStatusSkeleton({
|
||||
processedRange: [startIdx, endIdx],
|
||||
extractionCountBefore,
|
||||
});
|
||||
|
||||
debugLog(
|
||||
`[ST-BME] 开始提取: 楼层 ${startIdx}-${endIdx}` +
|
||||
(smartTriggerDecision?.triggered
|
||||
? ` [智能触发 score=${smartTriggerDecision.score}; ${smartTriggerDecision.reasons.join(" / ")}]`
|
||||
: ""),
|
||||
);
|
||||
|
||||
const result = await runtime.extractMemories({
|
||||
graph: currentGraph,
|
||||
messages,
|
||||
startSeq: startIdx,
|
||||
endSeq: endIdx,
|
||||
lastProcessedSeq: lastProcessed,
|
||||
schema: runtime.getSchema(),
|
||||
embeddingConfig: runtime.getEmbeddingConfig(),
|
||||
extractPrompt: undefined,
|
||||
settings,
|
||||
signal,
|
||||
onStreamProgress: ({ previewText, receivedChars }) => {
|
||||
const preview =
|
||||
previewText?.length > 60 ? "…" + previewText.slice(-60) : previewText || "";
|
||||
runtime.setLastExtractionStatus(
|
||||
"AI 生成中",
|
||||
`${preview} [${receivedChars}字]`,
|
||||
"running",
|
||||
{ noticeMarquee: true },
|
||||
);
|
||||
},
|
||||
});
|
||||
|
||||
if (!result.success) {
|
||||
runtime.setBatchStageOutcome(
|
||||
batchStatus,
|
||||
"core",
|
||||
"failed",
|
||||
result?.error || "提取阶段未返回有效操作",
|
||||
);
|
||||
runtime.finalizeBatchStatus(batchStatus, runtime.getExtractionCount());
|
||||
runtime.getCurrentGraph().historyState.lastBatchStatus = batchStatus;
|
||||
return {
|
||||
success: false,
|
||||
result,
|
||||
effects: null,
|
||||
batchStatus,
|
||||
error: result?.error || "提取阶段未返回有效操作",
|
||||
};
|
||||
}
|
||||
|
||||
runtime.setBatchStageOutcome(batchStatus, "core", "success");
|
||||
const effects = await runtime.handleExtractionSuccess(
|
||||
result,
|
||||
endIdx,
|
||||
settings,
|
||||
signal,
|
||||
batchStatus,
|
||||
);
|
||||
const finalizedBatchStatus =
|
||||
effects?.batchStatus ||
|
||||
runtime.finalizeBatchStatus(batchStatus, runtime.getExtractionCount());
|
||||
|
||||
runtime.getCurrentGraph().historyState.lastBatchStatus = {
|
||||
...finalizedBatchStatus,
|
||||
historyAdvanced: runtime.shouldAdvanceProcessedHistory(finalizedBatchStatus),
|
||||
};
|
||||
|
||||
if (runtime.getCurrentGraph().historyState.lastBatchStatus.historyAdvanced) {
|
||||
runtime.updateProcessedHistorySnapshot(chat, endIdx);
|
||||
}
|
||||
|
||||
const afterSnapshot = runtime.cloneGraphSnapshot(runtime.getCurrentGraph());
|
||||
const postProcessArtifacts = runtime.computePostProcessArtifacts(
|
||||
beforeSnapshot,
|
||||
afterSnapshot,
|
||||
effects?.postProcessArtifacts || [],
|
||||
);
|
||||
runtime.appendBatchJournal(
|
||||
runtime.getCurrentGraph(),
|
||||
runtime.createBatchJournalEntry(beforeSnapshot, afterSnapshot, {
|
||||
processedRange: [startIdx, endIdx],
|
||||
postProcessArtifacts,
|
||||
vectorHashesInserted: effects?.vectorHashesInserted || [],
|
||||
extractionCountBefore,
|
||||
}),
|
||||
);
|
||||
runtime.saveGraphToChat({ reason: "extraction-batch-complete" });
|
||||
|
||||
return {
|
||||
success: finalizedBatchStatus.completed,
|
||||
result,
|
||||
effects,
|
||||
batchStatus: finalizedBatchStatus,
|
||||
error: finalizedBatchStatus.completed
|
||||
? ""
|
||||
: effects?.vectorError ||
|
||||
finalizedBatchStatus.errors?.[0] ||
|
||||
"批次未完成 finalize 闭环",
|
||||
};
|
||||
}
|
||||
|
||||
export async function runExtractionController(runtime, options = {}) {
|
||||
const lockedEndFloor = toSafeFloor(options?.lockedEndFloor, null);
|
||||
const triggerSource = String(options?.triggerSource || "auto").trim() || "auto";
|
||||
const settings = runtime.getSettings?.() || {};
|
||||
const context = runtime.getContext?.() || {};
|
||||
const chat = Array.isArray(context?.chat) ? context.chat : [];
|
||||
const plan = resolveAutoExtractionPlanController(runtime, {
|
||||
chat,
|
||||
settings,
|
||||
lockedEndFloor,
|
||||
});
|
||||
const deferredTargetEndFloor =
|
||||
plan.plannedBatchEndFloor ?? lockedEndFloor;
|
||||
|
||||
if (runtime.getIsExtracting()) {
|
||||
runtime.console?.debug?.("[ST-BME] auto extraction deferred: extraction already in progress");
|
||||
runtime.deferAutoExtraction?.("extracting", {
|
||||
targetEndFloor: deferredTargetEndFloor,
|
||||
strategy: plan.strategy,
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
if (!settings.enabled) return;
|
||||
if (!runtime.ensureGraphMutationReady("自动提取", { notify: false })) {
|
||||
runtime.console?.debug?.("[ST-BME] auto extraction blocked: graph-not-ready", {
|
||||
loadState: runtime.getGraphPersistenceState?.()?.loadState || "",
|
||||
});
|
||||
runtime.deferAutoExtraction?.("graph-not-ready", {
|
||||
targetEndFloor: deferredTargetEndFloor,
|
||||
strategy: plan.strategy,
|
||||
});
|
||||
runtime.setLastExtractionStatus(
|
||||
"等待图谱加载",
|
||||
runtime.getGraphMutationBlockReason("自动提取"),
|
||||
"warning",
|
||||
{ syncRuntime: true },
|
||||
);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!runtime.getCurrentGraph()) {
|
||||
runtime.ensureCurrentGraphRuntimeState?.();
|
||||
}
|
||||
|
||||
if (!(await runtime.recoverHistoryIfNeeded("auto-extract"))) {
|
||||
runtime.console?.debug?.("[ST-BME] auto extraction paused during history recovery", {
|
||||
recovering: runtime.getIsRecoveringHistory?.() === true,
|
||||
});
|
||||
if (runtime.getIsRecoveringHistory?.()) {
|
||||
runtime.deferAutoExtraction?.("history-recovering", {
|
||||
targetEndFloor: deferredTargetEndFloor,
|
||||
strategy: plan.strategy,
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (!chat || chat.length === 0) return;
|
||||
if (!plan.canRun || plan.startIdx == null || plan.endIdx == null) {
|
||||
return;
|
||||
}
|
||||
|
||||
const startIdx = plan.startIdx;
|
||||
const endIdx = plan.endIdx;
|
||||
const smartTriggerDecision = plan.smartTriggerDecision;
|
||||
runtime.setIsExtracting(true);
|
||||
const extractionController = runtime.beginStageAbortController("extraction");
|
||||
const extractionSignal = extractionController.signal;
|
||||
runtime.setLastExtractionStatus(
|
||||
"提取中",
|
||||
`楼层 ${startIdx}-${endIdx}${smartTriggerDecision.triggered ? " · 智能触发" : ""}${triggerSource !== "auto" ? ` · ${triggerSource}` : ""}`,
|
||||
"running",
|
||||
{ syncRuntime: true },
|
||||
);
|
||||
|
||||
try {
|
||||
const batchResult = await runtime.executeExtractionBatch({
|
||||
chat,
|
||||
startIdx,
|
||||
endIdx,
|
||||
settings,
|
||||
smartTriggerDecision,
|
||||
signal: extractionSignal,
|
||||
});
|
||||
|
||||
if (!batchResult.success) {
|
||||
const message =
|
||||
batchResult.error ||
|
||||
batchResult?.result?.error ||
|
||||
"提取批次未返回有效结果";
|
||||
runtime.console.warn("[ST-BME] 提取批次未返回有效结果:", message);
|
||||
runtime.notifyExtractionIssue(message);
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.setLastExtractionStatus(
|
||||
"提取完成",
|
||||
`楼层 ${startIdx}-${endIdx} · 新建 ${batchResult.result?.newNodes || 0} · 更新 ${batchResult.result?.updatedNodes || 0} · 新边 ${batchResult.result?.newEdges || 0}`,
|
||||
"success",
|
||||
{ syncRuntime: true },
|
||||
);
|
||||
} catch (e) {
|
||||
if (runtime.isAbortError(e)) {
|
||||
runtime.setLastExtractionStatus(
|
||||
"提取已终止",
|
||||
e?.message || "已手动终止当前提取",
|
||||
"warning",
|
||||
{
|
||||
syncRuntime: true,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
runtime.console.error("[ST-BME] 提取失败:", e);
|
||||
runtime.notifyExtractionIssue(e?.message || String(e) || "自动提取失败");
|
||||
} finally {
|
||||
runtime.finishStageAbortController("extraction", extractionController);
|
||||
runtime.setIsExtracting(false);
|
||||
}
|
||||
}
|
||||
|
||||
export async function onManualExtractController(runtime, options = {}) {
|
||||
if (runtime.getIsExtracting()) {
|
||||
runtime.toastr.info("记忆提取正在进行中，请稍候");
|
||||
return;
|
||||
}
|
||||
if (!runtime.ensureGraphMutationReady("手动提取")) return;
|
||||
if (!(await runtime.recoverHistoryIfNeeded("manual-extract"))) return;
|
||||
if (!runtime.getCurrentGraph()) {
|
||||
runtime.setCurrentGraph(
|
||||
runtime.normalizeGraphRuntimeState(
|
||||
runtime.createEmptyGraph(),
|
||||
runtime.getCurrentChatId(),
|
||||
),
|
||||
);
|
||||
}
|
||||
|
||||
const context = runtime.getContext();
|
||||
const chat = context.chat;
|
||||
if (!Array.isArray(chat) || chat.length === 0) {
|
||||
runtime.toastr.info("当前聊天为空，暂无可提取内容");
|
||||
return;
|
||||
}
|
||||
|
||||
const assistantTurns = runtime.getAssistantTurns(chat);
|
||||
const lastProcessed = runtime.getLastProcessedAssistantFloor();
|
||||
const pendingAssistantTurns = assistantTurns.filter((i) => i > lastProcessed);
|
||||
if (pendingAssistantTurns.length === 0) {
|
||||
runtime.toastr.info("没有待提取的新回复");
|
||||
return;
|
||||
}
|
||||
|
||||
const settings = runtime.getSettings();
|
||||
const extractEvery = runtime.clampInt(settings.extractEvery, 1, 1, 50);
|
||||
const totals = {
|
||||
newNodes: 0,
|
||||
updatedNodes: 0,
|
||||
newEdges: 0,
|
||||
batches: 0,
|
||||
};
|
||||
const warnings = [];
|
||||
|
||||
runtime.setIsExtracting(true);
|
||||
const extractionController = runtime.beginStageAbortController("extraction");
|
||||
const extractionSignal = extractionController.signal;
|
||||
runtime.setLastExtractionStatus(
|
||||
"手动提取中",
|
||||
`待处理 assistant 楼层 ${pendingAssistantTurns.length} 条`,
|
||||
"running",
|
||||
|
||||
{ syncRuntime: true, toastKind: "info", toastTitle: "ST-BME 手动提取" },
|
||||
);
|
||||
try {
|
||||
while (true) {
|
||||
const pendingTurns = runtime
|
||||
.getAssistantTurns(chat)
|
||||
.filter((i) => i > runtime.getLastProcessedAssistantFloor());
|
||||
if (pendingTurns.length === 0) break;
|
||||
|
||||
const batchAssistantTurns = pendingTurns.slice(0, extractEvery);
|
||||
const startIdx = batchAssistantTurns[0];
|
||||
const endIdx = batchAssistantTurns[batchAssistantTurns.length - 1];
|
||||
const batchResult = await runtime.executeExtractionBatch({
|
||||
chat,
|
||||
startIdx,
|
||||
endIdx,
|
||||
settings,
|
||||
signal: extractionSignal,
|
||||
});
|
||||
|
||||
if (!batchResult.success) {
|
||||
throw new Error(
|
||||
batchResult.error ||
|
||||
batchResult?.result?.error ||
|
||||
"手动提取未返回有效结果",
|
||||
);
|
||||
}
|
||||
|
||||
totals.newNodes += batchResult.result.newNodes || 0;
|
||||
totals.updatedNodes += batchResult.result.updatedNodes || 0;
|
||||
totals.newEdges += batchResult.result.newEdges || 0;
|
||||
totals.batches++;
|
||||
|
||||
if (Array.isArray(batchResult.effects?.warnings)) {
|
||||
warnings.push(...batchResult.effects.warnings);
|
||||
}
|
||||
|
||||
if (options?.drainAll === false) {
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (totals.batches === 0) {
|
||||
runtime.setLastExtractionStatus(
|
||||
"无待提取内容",
|
||||
"没有新的 assistant 回复需要处理",
|
||||
"info",
|
||||
{
|
||||
syncRuntime: true,
|
||||
},
|
||||
);
|
||||
runtime.toastr.info("没有待提取的新回复");
|
||||
return;
|
||||
}
|
||||
|
||||
runtime.toastr.success(
|
||||
`提取完成：${totals.batches} 批，新建 ${totals.newNodes}，更新 ${totals.updatedNodes}，新边 ${totals.newEdges}`,
|
||||
);
|
||||
runtime.setLastExtractionStatus(
|
||||
"手动提取完成",
|
||||
`${totals.batches} 批 · 新建 ${totals.newNodes} · 更新 ${totals.updatedNodes} · 新边 ${totals.newEdges}`,
|
||||
"success",
|
||||
{
|
||||
syncRuntime: true,
|
||||
toastKind: "success",
|
||||
toastTitle: "ST-BME 手动提取",
|
||||
},
|
||||
);
|
||||
if (warnings.length > 0) {
|
||||
runtime.toastr.warning(warnings.slice(0, 2).join("；"), "ST-BME 提取警告", {
|
||||
timeOut: 5000,
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
if (runtime.isAbortError(e)) {
|
||||
runtime.setLastExtractionStatus(
|
||||
"手动提取已终止",
|
||||
e?.message || "已手动终止当前提取",
|
||||
"warning",
|
||||
{
|
||||
syncRuntime: true,
|
||||
},
|
||||
);
|
||||
return;
|
||||
}
|
||||
runtime.console.error("[ST-BME] 手动提取失败:", e);
|
||||
runtime.setLastExtractionStatus("手动提取失败", e?.message || String(e), "error", {
|
||||
syncRuntime: true,
|
||||
toastKind: "",
|
||||
toastTitle: "ST-BME 手动提取",
|
||||
});
|
||||
runtime.toastr.error(`手动提取失败: ${e.message || e}`);
|
||||
} finally {
|
||||
runtime.finishStageAbortController("extraction", extractionController);
|
||||
runtime.setIsExtracting(false);
|
||||
runtime.refreshPanelLiveState();
|
||||
}
|
||||
}
|
||||
|
||||
export async function onRerollController(runtime, { fromFloor } = {}) {
|
||||
if (runtime.getIsExtracting?.()) {
|
||||
runtime.toastr?.info?.("记忆提取正在进行中，请稍候");
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: null,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: "busy",
|
||||
affectedBatchCount: 0,
|
||||
error: "记忆提取正在进行中",
|
||||
};
|
||||
}
|
||||
|
||||
if (
|
||||
typeof runtime.ensureGraphMutationReady === "function" &&
|
||||
!runtime.ensureGraphMutationReady("重新提取")
|
||||
) {
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: Number.isFinite(fromFloor) ? fromFloor : null,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: runtime.getGraphPersistenceState?.()?.loadState || "graph-not-ready",
|
||||
affectedBatchCount: 0,
|
||||
error:
|
||||
typeof runtime.getGraphMutationBlockReason === "function"
|
||||
? runtime.getGraphMutationBlockReason("重新提取")
|
||||
: "重新提取已暂停：图谱尚未就绪。",
|
||||
};
|
||||
}
|
||||
|
||||
if (!runtime.getCurrentGraph?.()) {
|
||||
runtime.toastr?.info?.("图谱为空，无需重 Roll");
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: null,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: "empty-graph",
|
||||
affectedBatchCount: 0,
|
||||
error: "图谱为空",
|
||||
};
|
||||
}
|
||||
|
||||
const context = runtime.getContext();
|
||||
const chat = context?.chat;
|
||||
if (!Array.isArray(chat) || chat.length === 0) {
|
||||
runtime.toastr?.info?.("当前聊天为空");
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: null,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: "empty-chat",
|
||||
affectedBatchCount: 0,
|
||||
error: "当前聊天为空",
|
||||
};
|
||||
}
|
||||
|
||||
let targetFloor = Number.isFinite(fromFloor) ? fromFloor : null;
|
||||
if (targetFloor === null) {
|
||||
const assistantTurns = runtime.getAssistantTurns(chat);
|
||||
if (assistantTurns.length === 0) {
|
||||
runtime.toastr?.info?.("聊天中没有 AI 回复");
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: null,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: "no-assistant-turn",
|
||||
affectedBatchCount: 0,
|
||||
error: "聊天中没有 AI 回复",
|
||||
};
|
||||
}
|
||||
targetFloor = assistantTurns[assistantTurns.length - 1];
|
||||
}
|
||||
|
||||
runtime.setRuntimeStatus(
|
||||
"重新提取中",
|
||||
Number.isFinite(targetFloor)
|
||||
? `准备从楼层 ${targetFloor} 开始回滚并重新提取`
|
||||
: "准备回滚最新 AI 楼并重新提取",
|
||||
"running",
|
||||
);
|
||||
|
||||
const lastProcessed = runtime.getLastProcessedAssistantFloor();
|
||||
const alreadyExtracted = targetFloor <= lastProcessed;
|
||||
|
||||
if (!alreadyExtracted) {
|
||||
runtime.toastr?.info?.("该楼层尚未提取，直接执行提取…", "ST-BME 重 Roll", {
|
||||
timeOut: 2000,
|
||||
});
|
||||
await runtime.onManualExtract();
|
||||
return {
|
||||
success: true,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: true,
|
||||
requestedFloor: targetFloor,
|
||||
effectiveFromFloor: lastProcessed + 1,
|
||||
recoveryPath: "direct-extract",
|
||||
affectedBatchCount: 0,
|
||||
extractionStatus: runtime.getLastExtractionStatusLevel?.() || "idle",
|
||||
error: "",
|
||||
};
|
||||
}
|
||||
|
||||
debugLog(`[ST-BME] 重 Roll 开始，目标楼层: ${targetFloor}`);
|
||||
let rollbackResult;
|
||||
try {
|
||||
rollbackResult = await runtime.rollbackGraphForReroll(targetFloor, context);
|
||||
} catch (e) {
|
||||
if (runtime.isAbortError(e)) {
|
||||
runtime.setRuntimeStatus(
|
||||
"重新提取已取消",
|
||||
e.message || "聊天已切换",
|
||||
"warning",
|
||||
);
|
||||
return {
|
||||
success: false,
|
||||
rollbackPerformed: false,
|
||||
extractionTriggered: false,
|
||||
requestedFloor: targetFloor,
|
||||
effectiveFromFloor: null,
|
||||
recoveryPath: "aborted",
|
||||
affectedBatchCount: 0,
|
||||
error: e.message || "聊天已切换，重新提取已取消",
|
||||
};
|
||||
}
|
||||
throw e;
|
||||
}
|
||||
|
||||
if (!rollbackResult?.success) {
|
||||
runtime.setRuntimeStatus(
|
||||
"重新提取失败",
|
||||
rollbackResult.error || "回滚失败",
|
||||
"error",
|
||||
);
|
||||
runtime.toastr?.error?.(rollbackResult.error, "ST-BME 重 Roll");
|
||||
return rollbackResult;
|
||||
}
|
||||
|
||||
const rerollDesc =
|
||||
rollbackResult.effectiveFromFloor !== targetFloor
|
||||
? `已按批次边界回滚到楼层 ${rollbackResult.effectiveFromFloor} 开始重新提取…`
|
||||
: `已回滚到楼层 ${targetFloor} 开始重新提取…`;
|
||||
runtime.toastr?.info?.(rerollDesc, "ST-BME 重 Roll", {
|
||||
timeOut: 2500,
|
||||
});
|
||||
|
||||
await runtime.onManualExtract({ drainAll: false });
|
||||
runtime.refreshPanelLiveState();
|
||||
return {
|
||||
...rollbackResult,
|
||||
extractionTriggered: true,
|
||||
extractionStatus: runtime.getLastExtractionStatusLevel?.() || "idle",
|
||||
};
|
||||
}
|
||||
1244
maintenance/extractor.js
Normal file
1244
maintenance/extractor.js
Normal file
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user