taozuqin/ST-Bionic-Memory-Ecology
1
0
Fork 0
mirror of https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git synced 2026-05-15 22:30:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

Add persistence and retrieval observability with native delta gating

Browse Source
This commit is contained in:
Youzini-afk
2026-04-21 20:32:03 +08:00
parent 5a8f563168
commit d2c3d1f5dd
8 changed files with 745 additions and 35 deletions
Download Patch File Download Diff File Expand all files Collapse all files

175
index.js
View File

@@ -1585,6 +1585,10 @@ function getGraphPersistenceLiveState() {
null,
),
persistDelta: cloneRuntimeDebugValue(graphPersistenceState.persistDelta, null),
loadDiagnostics: cloneRuntimeDebugValue(
graphPersistenceState.loadDiagnostics,
null,
),
};
return cloneRuntimeDebugValue(snapshot, snapshot);
@@ -1654,6 +1658,14 @@ function readPersistDeltaDiagnosticsNow() {
return Date.now();
}
function readLoadDiagnosticsNow() {
return readPersistDeltaDiagnosticsNow();
}
function normalizeLoadDiagnosticsMs(value = 0) {
return Math.round((Number(value) || 0) * 10) / 10;
}
function updatePersistDeltaDiagnostics(snapshot = null) {
const nextSnapshot =
snapshot && typeof snapshot === "object" && !Array.isArray(snapshot)
@@ -1671,6 +1683,23 @@ function updatePersistDeltaDiagnostics(snapshot = null) {
return nextSnapshot;
}
function updateLoadDiagnostics(snapshot = null) {
const nextSnapshot =
snapshot && typeof snapshot === "object" && !Array.isArray(snapshot)
? {
...(graphPersistenceState.loadDiagnostics &&
typeof graphPersistenceState.loadDiagnostics === "object" &&
!Array.isArray(graphPersistenceState.loadDiagnostics)
? cloneRuntimeDebugValue(graphPersistenceState.loadDiagnostics, {})
: {}),
...cloneRuntimeDebugValue(snapshot, {}),
updatedAt: new Date().toISOString(),
}
: null;
updateGraphPersistenceState({ loadDiagnostics: nextSnapshot });
return nextSnapshot;
}
function bumpGraphRevision(reason = "graph-mutation") {
const nextRevision =
Math.max(
@@ -9093,14 +9122,37 @@ function applyIndexedDbSnapshotToRuntime(
) {
const normalizedChatId = normalizeChatIdCandidate(chatId);
syncCommitMarkerToPersistenceState(getContext());
const loadStartedAt = readLoadDiagnosticsNow();
const recordLoadDiagnostics = (patch = {}) =>
updateLoadDiagnostics({
stage: "apply-indexeddb-snapshot",
source: String(source || reasonPrefix),
reasonPrefix: String(reasonPrefix || "indexeddb"),
statusLabel: String(statusLabel || "IndexedDB"),
chatId: normalizedChatId || "",
attemptIndex: Number.isFinite(Number(attemptIndex))
? Math.max(0, Math.floor(Number(attemptIndex)))
: 0,
storagePrimary: String(storagePrimary || "indexeddb"),
storageMode: String(storageMode || storagePrimary || "indexeddb"),
...cloneRuntimeDebugValue(patch, {}),
totalMs: normalizeLoadDiagnosticsMs(readLoadDiagnosticsNow() - loadStartedAt),
});
let hydrateMs = 0;
if (!normalizedChatId || !isIndexedDbSnapshotMeaningful(snapshot)) {
return {
const result = {
success: false,
loaded: false,
reason: `${reasonPrefix}-empty`,
chatId: normalizedChatId,
attemptIndex,
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: result.reason,
});
return result;
}
const revision = Math.max(
@@ -9145,7 +9197,7 @@ function applyIndexedDbSnapshotToRuntime(
revision,
staleDetail: staleDecision,
});
return {
const result = {
success: false,
loaded: false,
reason: `${reasonPrefix}-stale-runtime`,
@@ -9154,12 +9206,22 @@ function applyIndexedDbSnapshotToRuntime(
revision,
staleDetail: cloneRuntimeDebugValue(staleDecision, null),
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: result.reason,
revision,
staleDetail: cloneRuntimeDebugValue(staleDecision, null),
});
return result;
}
let graphFromSnapshot = null;
try {
const hydrateStartedAt = readLoadDiagnosticsNow();
graphFromSnapshot = buildGraphFromSnapshot(snapshot, {
chatId: normalizedChatId,
});
hydrateMs = readLoadDiagnosticsNow() - hydrateStartedAt;
} catch (error) {
const failureReason =
error?.code === "BME_SNAPSHOT_INTEGRITY_ERROR"
@@ -9194,7 +9256,7 @@ function applyIndexedDbSnapshotToRuntime(
detail: error?.message || String(error),
integrityReasons: Array.isArray(error?.reasons) ? error.reasons : [],
});
return {
const result = {
success: false,
loaded: false,
reason: failureReason,
@@ -9203,7 +9265,18 @@ function applyIndexedDbSnapshotToRuntime(
chatId: normalizedChatId,
attemptIndex,
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: failureReason,
revision,
hydrateMs: normalizeLoadDiagnosticsMs(hydrateMs),
error: error?.message || String(error),
integrityReasons: Array.isArray(error?.reasons) ? [...error.reasons] : [],
});
return result;
}
const applyRuntimeStartedAt = readLoadDiagnosticsNow();
currentGraph = graphFromSnapshot;
stampGraphPersistenceMeta(currentGraph, {
revision,
@@ -9298,7 +9371,7 @@ function applyIndexedDbSnapshotToRuntime(
...getGraphStats(currentGraph),
});
return {
const result = {
success: true,
loaded: true,
loadState: GRAPH_LOAD_STATES.LOADED,
@@ -9308,6 +9381,17 @@ function applyIndexedDbSnapshotToRuntime(
shadowSnapshotUsed: false,
revision,
};
recordLoadDiagnostics({
success: true,
loaded: true,
reason: result.reason,
revision,
hydrateMs: normalizeLoadDiagnosticsMs(hydrateMs),
applyRuntimeMs: normalizeLoadDiagnosticsMs(
readLoadDiagnosticsNow() - applyRuntimeStartedAt,
),
});
return result;
}
async function loadGraphFromIndexedDb(
@@ -9321,27 +9405,55 @@ async function loadGraphFromIndexedDb(
) {
const normalizedChatId = normalizeChatIdCandidate(chatId);
const commitMarker = syncCommitMarkerToPersistenceState(getContext());
const loadStartedAt = readLoadDiagnosticsNow();
const recordLoadDiagnostics = (patch = {}) =>
updateLoadDiagnostics({
stage: "load-indexeddb",
source: String(source || "indexeddb-probe"),
chatId: normalizedChatId || "",
attemptIndex: Number.isFinite(Number(attemptIndex))
? Math.max(0, Math.floor(Number(attemptIndex)))
: 0,
...cloneRuntimeDebugValue(patch, {}),
totalMs: normalizeLoadDiagnosticsMs(readLoadDiagnosticsNow() - loadStartedAt),
});
let exportSnapshotMs = 0;
let exportSnapshotSource = "";
if (!normalizedChatId) {
return {
const result = {
success: false,
loaded: false,
reason: "indexeddb-missing-chat-id",
chatId: "",
attemptIndex,
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: result.reason,
});
return result;
}
let localStore = getPreferredGraphLocalStorePresentationSync();
try {
const manager = ensureBmeChatManager();
if (!manager) {
return {
const result = {
success: false,
loaded: false,
reason: "indexeddb-manager-unavailable",
chatId: normalizedChatId,
attemptIndex,
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: result.reason,
storagePrimary: localStore.storagePrimary,
storageMode: localStore.storageMode,
});
return result;
}
const db = await manager.getCurrentDb(normalizedChatId);
localStore = resolveDbGraphStorePresentation(db);
@@ -9464,11 +9576,22 @@ async function loadGraphFromIndexedDb(
},
});
}
const snapshot =
identityRecoveryResult?.snapshot ||
localStoreMigrationResult?.snapshot ||
migrationResult?.snapshot ||
(await db.exportSnapshot({ includeTombstones: false }));
let snapshot = null;
if (identityRecoveryResult?.snapshot) {
snapshot = identityRecoveryResult.snapshot;
exportSnapshotSource = "identity-recovery";
} else if (localStoreMigrationResult?.snapshot) {
snapshot = localStoreMigrationResult.snapshot;
exportSnapshotSource = "local-store-migration";
} else if (migrationResult?.snapshot) {
snapshot = migrationResult.snapshot;
exportSnapshotSource = "legacy-migration";
} else {
const exportStartedAt = readLoadDiagnosticsNow();
snapshot = await db.exportSnapshot({ includeTombstones: false });
exportSnapshotMs = readLoadDiagnosticsNow() - exportStartedAt;
exportSnapshotSource = "indexeddb-export";
}
const shadowSnapshot = resolveCompatibleGraphShadowSnapshot(
resolveCurrentChatIdentity(getContext()),
);
@@ -9678,6 +9801,7 @@ async function loadGraphFromIndexedDb(
};
}
const applyInvokeStartedAt = readLoadDiagnosticsNow();
const loadResult = applyIndexedDbSnapshotToRuntime(normalizedChatId, snapshot, {
source,
attemptIndex,
@@ -9686,11 +9810,26 @@ async function loadGraphFromIndexedDb(
statusLabel: snapshotStore.statusLabel,
reasonPrefix: snapshotStore.reasonPrefix,
});
const applyInvokeMs = readLoadDiagnosticsNow() - applyInvokeStartedAt;
if (commitMarkerDiagnostic?.reason && loadResult?.loaded) {
updateGraphPersistenceState({
persistMismatchReason: commitMarkerDiagnostic.reason,
});
}
recordLoadDiagnostics({
success: loadResult?.success === true,
loaded: loadResult?.loaded === true,
reason: String(loadResult?.reason || ""),
revision: Number.isFinite(Number(loadResult?.revision))
? Number(loadResult.revision)
: snapshotRevision,
storagePrimary: snapshotStore.storagePrimary,
storageMode: snapshotStore.storageMode,
commitMarkerMismatched: commitMarkerMismatch.mismatched === true,
exportSnapshotSource: exportSnapshotSource || "snapshot-prepared",
exportSnapshotMs: normalizeLoadDiagnosticsMs(exportSnapshotMs),
applyInvokeMs: normalizeLoadDiagnosticsMs(applyInvokeMs),
});
return loadResult;
} catch (error) {
console.warn(`[ST-BME] ${localStore.statusLabel} 读取失败，回退 metadata:`, error);
@@ -9706,7 +9845,7 @@ async function loadGraphFromIndexedDb(
at: Date.now(),
},
});
return {
const result = {
success: false,
loaded: false,
reason: `${localStore.reasonPrefix}-read-failed`,
@@ -9714,6 +9853,17 @@ async function loadGraphFromIndexedDb(
attemptIndex,
error,
};
recordLoadDiagnostics({
success: false,
loaded: false,
reason: result.reason,
storagePrimary: localStore.storagePrimary,
storageMode: localStore.storageMode,
error: error?.message || String(error),
exportSnapshotSource: exportSnapshotSource || "unknown",
exportSnapshotMs: normalizeLoadDiagnosticsMs(exportSnapshotMs),
});
return result;
}
}
@@ -16030,6 +16180,7 @@ async function executeExtractionBatch({
getEmbeddingConfig,
getExtractionCount: () => extractionCount,
getLastProcessedAssistantFloor,
getSettings,
getSchema,
handleExtractionSuccess,
persistExtractionBatchResult,

87
maintenance/extraction-controller.js
View File

@@ -7,6 +7,8 @@ import {
normalizeDialogueFloorRange,
} from "./chat-history.js";
let nativePersistDeltaInstallPromise = null;
function toSafeFloor(value, fallback = null) {
if (value == null || value === "") return fallback;
const numeric = Number(value);
@@ -115,6 +117,31 @@ function cloneSerializable(value, fallback = null) {
}
}
function readNow() {
if (typeof performance === "object" && typeof performance.now === "function") {
return performance.now();
}
return Date.now();
}
async function ensureNativePersistDeltaHookInstalled() {
if (typeof globalThis.__stBmeNativeBuildPersistDelta === "function") {
return {
loaded: true,
source: "global-hook",
};
}
if (!nativePersistDeltaInstallPromise) {
nativePersistDeltaInstallPromise = import("../vendor/wasm/stbme_core.js")
.then((module) => module?.installNativePersistDeltaHook?.())
.catch((error) => {
nativePersistDeltaInstallPromise = null;
throw error;
});
}
return await nativePersistDeltaInstallPromise;
}
function setExtractionProgressStatus(
runtime,
text,
@@ -247,11 +274,12 @@ function buildRerunFallbackInfo(chat = [], targetDialogueRange = [-1, -1]) {
};
}
function buildCommittedBatchPersistSnapshot(
async function buildCommittedBatchPersistSnapshot(
runtime,
{
graph = null,
chat = [],
settings = null,
beforeSnapshot = null,
processedRange = [null, null],
postProcessArtifacts = [],
@@ -274,6 +302,10 @@ function buildCommittedBatchPersistSnapshot(
const range = Array.isArray(processedRange) ? processedRange : [null, null];
const rangeStart = Number.isFinite(Number(range[0])) ? Number(range[0]) : null;
const rangeEnd = Number.isFinite(Number(range[1])) ? Number(range[1]) : null;
const runtimeSettings =
settings && typeof settings === "object" && !Array.isArray(settings)
? settings
: runtime?.getSettings?.() || {};
const dialogueMap = buildDialogueFloorMap(chat);
const processedDialogueRange = [
Number.isFinite(Number(rangeStart))
@@ -290,7 +322,7 @@ function buildCommittedBatchPersistSnapshot(
Number(rangeStart) -
Math.max(
0,
Number(runtime?.getSettings?.()?.extractContextTurns) || 0,
Number(runtimeSettings?.extractContextTurns) || 0,
) *
2,
)
@@ -347,13 +379,45 @@ function buildCommittedBatchPersistSnapshot(
);
}
let persistDelta = null;
const shouldUseNativePersistDelta =
runtimeSettings?.persistUseNativeDelta === true &&
runtimeSettings?.graphNativeForceDisable !== true;
const nativeFailOpen = runtimeSettings?.nativeEngineFailOpen !== false;
if (typeof runtime.buildPersistDelta === "function") {
if (shouldUseNativePersistDelta) {
const preloadStartedAt = readNow();
try {
await ensureNativePersistDeltaHookInstalled();
} catch (error) {
if (!nativeFailOpen) {
throw error;
}
runtime?.console?.warn?.(
"[ST-BME] extraction native persist delta preload failed, fallback to JS delta:",
{
error: error?.message || String(error),
preloadMs: readNow() - preloadStartedAt,
},
);
}
}
persistDelta = runtime.buildPersistDelta(beforeSnapshot, committedGraphSnapshot, {
useNativeDelta: shouldUseNativePersistDelta,
nativeFailOpen,
persistNativeDeltaThresholdRecords:
runtimeSettings?.persistNativeDeltaThresholdRecords,
persistNativeDeltaThresholdStructuralDelta:
runtimeSettings?.persistNativeDeltaThresholdStructuralDelta,
persistNativeDeltaThresholdSerializedChars:
runtimeSettings?.persistNativeDeltaThresholdSerializedChars,
persistNativeDeltaBridgeMode: runtimeSettings?.persistNativeDeltaBridgeMode,
});
}
return {
persistDelta:
typeof runtime.buildPersistDelta === "function"
? runtime.buildPersistDelta(beforeSnapshot, committedGraphSnapshot, {
useNativeDelta: false,
})
: null,
persistDelta,
persistGraphSnapshot: committedGraphSnapshot,
committedBatchJournalEntry,
afterSnapshot,
@@ -367,7 +431,9 @@ function isPersistenceRevisionAccepted(runtime, persistence = null) {
if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) {
return false;
}
const lastAcceptedRevision = Number(graphPersistenceState?.lastAcceptedRevision || 0);
const lastAcceptedRevision = Number(
runtime?.getGraphPersistenceState?.()?.lastAcceptedRevision || 0,
);
return Number.isFinite(lastAcceptedRevision) && lastAcceptedRevision >= persistenceRevision;
}
@@ -633,9 +699,10 @@ export async function executeExtractionBatchController(
batchStatus,
);
const batchStatusRef = effects?.batchStatus || batchStatus;
const committedPersistState = buildCommittedBatchPersistSnapshot(runtime, {
const committedPersistState = await buildCommittedBatchPersistSnapshot(runtime, {
graph: runtime.getCurrentGraph(),
chat,
settings,
beforeSnapshot,
processedRange: [startIdx, endIdx],
postProcessArtifacts: runtime.computePostProcessArtifacts(

20
retrieval/retriever.js
View File

@@ -1379,6 +1379,24 @@ export async function retrieve({
)
? [...sharedRanking.diagnostics.lexicalTopHits]
: [];
retrievalMeta.timings.sharedQueryBlend = Number(
sharedRanking?.diagnostics?.timings?.queryBlend || 0,
);
retrievalMeta.timings.sharedLexical = Number(
sharedRanking?.diagnostics?.timings?.lexical || 0,
);
retrievalMeta.timings.sharedScoring = Number(
sharedRanking?.diagnostics?.timings?.scoring || 0,
);
retrievalMeta.timings.sharedTotal = Number(
sharedRanking?.diagnostics?.timings?.total || 0,
);
retrievalMeta.timings.sharedVector = Number(
sharedRanking?.diagnostics?.timings?.vector || 0,
);
retrievalMeta.timings.sharedDiffusion = Number(
sharedRanking?.diagnostics?.timings?.diffusion || 0,
);
retrievalMeta.timings.vector = Number(
sharedRanking?.diagnostics?.timings?.vector || 0,
);
@@ -1395,12 +1413,14 @@ export async function retrieve({
? [...sharedRanking.diffusionResults]
: [];
exactEntityAnchors.push(...(sharedRanking?.exactEntityAnchors || []));
const anchorCollectStartedAt = nowMs();
supplementalAnchorNodeIds = collectSupplementalAnchorNodeIds(
graph,
vectorResults,
exactEntityAnchors.map((item) => item.nodeId),
5,
);
retrievalMeta.timings.anchorCollect = roundMs(nowMs() - anchorCollectStartedAt);
let residualResult = {
triggered: false,

17
retrieval/shared-ranking.js
View File

@@ -536,6 +536,8 @@ export async function rankNodesForTaskContext({
? options.activeNodes.filter((node) => node && !node.archived)
: getActiveNodes(graph).filter((node) => node && !node.archived);
const vectorValidation = validateVectorConfig(embeddingConfig);
const rankingStartedAt = nowMs();
const queryBlendStartedAt = nowMs();
const contextQueryBlend = buildContextQueryBlend(userMessage, recentMessages, {
enabled: enableContextQueryBlend,
assistantWeight: contextAssistantWeight,
@@ -553,6 +555,7 @@ export async function rankNodesForTaskContext({
maxSegments: multiIntentMaxSegments,
},
);
const queryBlendMs = roundMs(nowMs() - queryBlendStartedAt);
const diagnostics = {
queryBlendActive: contextQueryBlend.active,
queryBlendParts: (contextQueryBlend.parts || []).map((part) => ({
@@ -577,12 +580,17 @@ export async function rankNodesForTaskContext({
lexicalTopHits: [],
skipReasons: [],
timings: {
queryBlend: queryBlendMs,
vector: 0,
diffusion: 0,
lexical: 0,
scoring: 0,
total: 0,
},
};
if (!graph || activeNodes.length === 0) {
diagnostics.timings.total = roundMs(nowMs() - rankingStartedAt);
return {
activeNodes,
contextQueryBlend,
@@ -688,13 +696,19 @@ export async function rankNodesForTaskContext({
}
}
const scoringStartedAt = nowMs();
let lexicalMs = 0;
const scoredNodes = [];
for (const [nodeId, scores] of scoreMap.entries()) {
const node = getNode(graph, nodeId);
if (!node || node.archived) continue;
const lexicalStartedAt = enableLexicalBoost ? nowMs() : 0;
const lexicalScore = enableLexicalBoost
? computeLexicalScore(node, lexicalQuery.sources)
: 0;
if (enableLexicalBoost) {
lexicalMs += nowMs() - lexicalStartedAt;
}
const finalScore = hybridScore(
{
graphScore: scores.graphScore,
@@ -719,6 +733,8 @@ export async function rankNodesForTaskContext({
weightedScore: finalScore,
});
}
diagnostics.timings.lexical = roundMs(lexicalMs);
diagnostics.timings.scoring = roundMs(nowMs() - scoringStartedAt);
scoredNodes.sort((left, right) => {
const weightedDelta =
@@ -737,6 +753,7 @@ export async function rankNodesForTaskContext({
(item) => (Number(item.lexicalScore) || 0) > 0,
).length;
diagnostics.lexicalTopHits = buildLexicalTopHits(scoredNodes);
diagnostics.timings.total = roundMs(nowMs() - rankingStartedAt);
return {
activeNodes,

101
tests/extraction-persistence-gating.mjs
View File

@@ -16,6 +16,7 @@ function createRuntime(persistResult) {
};
let processedHistoryUpdates = 0;
let persistedGraphSnapshot = null;
let lastPersistDeltaOptions = null;
return {
graph,
@@ -35,6 +36,22 @@ function createRuntime(persistResult) {
cloneGraphSnapshot(value) {
return JSON.parse(JSON.stringify(value));
},
buildPersistDelta(_beforeSnapshot, _afterSnapshot, options = {}) {
lastPersistDeltaOptions = { ...(options || {}) };
return {
upsertNodes: [],
upsertEdges: [],
deleteNodeIds: [],
deleteEdgeIds: [],
tombstones: [],
countDelta: {
nodes: 0,
edges: 0,
tombstones: 0,
},
runtimeMetaPatch: {},
};
},
buildExtractionMessages() {
return [{ seq: 5, role: "assistant", content: "测试消息" }];
},
@@ -101,6 +118,9 @@ function createRuntime(persistResult) {
get persistedGraphSnapshot() {
return persistedGraphSnapshot;
},
get lastPersistDeltaOptions() {
return lastPersistDeltaOptions;
},
};
}
@@ -212,4 +232,85 @@ function createRuntime(persistResult) {
assert.equal(runtime.graph.historyState.lastBatchStatus.persistence, null);
}
{
const originalNativeBuilder = globalThis.__stBmeNativeBuildPersistDelta;
globalThis.__stBmeNativeBuildPersistDelta = () => ({
upsertNodes: [],
upsertEdges: [],
deleteNodeIds: [],
deleteEdgeIds: [],
tombstones: [],
runtimeMetaPatch: {},
});
const runtime = createRuntime({
saved: true,
queued: false,
blocked: false,
accepted: true,
reason: "indexeddb",
revision: 9,
saveMode: "indexeddb",
storageTier: "indexeddb",
});
const result = await executeExtractionBatchController(runtime, {
chat: [{ is_user: false, mes: "测试" }],
startIdx: 5,
endIdx: 5,
settings: {
persistUseNativeDelta: true,
graphNativeForceDisable: false,
nativeEngineFailOpen: true,
persistNativeDeltaThresholdRecords: 123,
persistNativeDeltaThresholdStructuralDelta: 45,
persistNativeDeltaThresholdSerializedChars: 6789,
persistNativeDeltaBridgeMode: "hash",
},
});
assert.equal(result.success, true);
assert.equal(runtime.lastPersistDeltaOptions.useNativeDelta, true);
assert.equal(runtime.lastPersistDeltaOptions.nativeFailOpen, true);
assert.equal(runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdRecords, 123);
assert.equal(
runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdStructuralDelta,
45,
);
assert.equal(
runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdSerializedChars,
6789,
);
assert.equal(runtime.lastPersistDeltaOptions.persistNativeDeltaBridgeMode, "hash");
if (typeof originalNativeBuilder === "function") {
globalThis.__stBmeNativeBuildPersistDelta = originalNativeBuilder;
} else {
delete globalThis.__stBmeNativeBuildPersistDelta;
}
}
{
const runtime = createRuntime({
saved: true,
queued: false,
blocked: false,
accepted: true,
reason: "indexeddb",
revision: 10,
saveMode: "indexeddb",
storageTier: "indexeddb",
});
const result = await executeExtractionBatchController(runtime, {
chat: [{ is_user: false, mes: "测试" }],
startIdx: 5,
endIdx: 5,
settings: {
persistUseNativeDelta: true,
graphNativeForceDisable: true,
},
});
assert.equal(result.success, true);
assert.equal(runtime.lastPersistDeltaOptions.useNativeDelta, false);
}
console.log("extraction-persistence-gating tests passed");

177
ui/panel.js
View File

@@ -1461,6 +1461,159 @@ function _resolvePipelineStatus(statusObj) {
return { label: text || "IDLE", color, detail: meta };
}
function _readPersistenceDiagnosticObject(snapshot = null) {
if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) {
return null;
}
return snapshot;
}
function _formatLoadDiagnosticsStageLabel(stage = "") {
const normalized = String(stage || "").trim();
if (!normalized) return "—";
const labels = {
"load-indexeddb": "IndexedDB 加载",
"apply-indexeddb-snapshot": "快照应用",
};
return labels[normalized] || normalized;
}
function _formatPipelineLoadDiagnosticsMeta(loadDiagnostics = null) {
const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics);
if (!diagnostics) return "";
const totalText = _formatDurationMs(diagnostics.totalMs);
if (totalText !== "—") return `load ${totalText}`;
const stageLabel = _formatLoadDiagnosticsStageLabel(diagnostics.stage);
return stageLabel === "—" ? "" : stageLabel;
}
function _formatPipelinePersistDeltaMeta(persistDelta = null) {
const diagnostics = _readPersistenceDiagnosticObject(persistDelta);
if (!diagnostics) return "";
const parts = [];
const totalText = _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs);
if (totalText !== "—") {
parts.push(`delta ${totalText}`);
}
const gateText = String(_formatPersistDeltaGateText(diagnostics) || "").trim();
if (gateText) {
const compactGate = gateText.startsWith("已拦截") ? "已拦截" : gateText;
parts.push(`native ${compactGate}`);
}
return parts.join(" · ");
}
function _formatPersistenceLoadSummary(loadDiagnostics = null) {
const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics);
if (!diagnostics) return "暂无";
const statusText =
diagnostics.success === true
? "成功"
: diagnostics.success === false
? "失败"
: "未知";
const totalText = _formatDurationMs(diagnostics.totalMs);
const stageLabel = _formatLoadDiagnosticsStageLabel(diagnostics.stage);
const reasonText = String(diagnostics.reason || "").trim();
const parts = [statusText];
if (stageLabel !== "—") parts.push(stageLabel);
if (totalText !== "—") parts.push(`total ${totalText}`);
if (reasonText) parts.push(reasonText);
return parts.join(" · ");
}
function _formatPersistencePersistDeltaSummary(persistDelta = null) {
const diagnostics = _readPersistenceDiagnosticObject(persistDelta);
if (!diagnostics) return "暂无";
const pathText = String(diagnostics.path || "").trim() || "—";
const totalText = _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs);
const gateText = String(_formatPersistDeltaGateText(diagnostics) || "").trim();
const parts = [pathText];
if (totalText !== "—") parts.push(totalText);
if (gateText) parts.push(`native ${gateText}`);
return parts.join(" · ");
}
function _buildLoadDiagnosticRows(loadDiagnostics = null) {
const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics);
if (!diagnostics) {
return [["Load 诊断", "无"]];
}
const statusText =
diagnostics.success === true
? "成功"
: diagnostics.success === false
? "失败"
: "未知";
const updatedAtText = diagnostics.updatedAt
? _formatTaskProfileTime(diagnostics.updatedAt)
: "—";
return [
["Load 阶段", _formatLoadDiagnosticsStageLabel(diagnostics.stage)],
["Load 来源", String(diagnostics.source || diagnostics.statusLabel || "—")],
["Load 状态", statusText],
["Load 原因", String(diagnostics.reason || "—")],
["Load 总耗时", _formatDurationMs(diagnostics.totalMs)],
["导出快照", _formatDurationMs(diagnostics.exportSnapshotMs)],
["Hydrate", _formatDurationMs(diagnostics.hydrateMs)],
["Apply 调用", _formatDurationMs(diagnostics.applyInvokeMs)],
["Apply 运行", _formatDurationMs(diagnostics.applyRuntimeMs)],
["Load 更新时间", updatedAtText],
];
}
function _buildPersistDeltaDiagnosticRows(persistDelta = null) {
const diagnostics = _readPersistenceDiagnosticObject(persistDelta);
if (!diagnostics) {
return [["Persist Delta 诊断", "无"]];
}
const errorText = String(
diagnostics.moduleError || diagnostics.preloadError || diagnostics.nativeError || "",
).trim();
const bridgeText = `${String(diagnostics.requestedBridgeMode || "none")} → ${String(
diagnostics.preparedBridgeMode || "none",
)}`;
const deltaSizeText = `${Number(diagnostics.upsertNodeCount || 0)}N / ${Number(
diagnostics.upsertEdgeCount || 0,
)}E / ${Number(diagnostics.deleteNodeCount || 0)}DN / ${Number(
diagnostics.deleteEdgeCount || 0,
)}DE`;
const updatedAtText = diagnostics.updatedAt
? _formatTaskProfileTime(diagnostics.updatedAt)
: "—";
return [
["Persist 路径", String(diagnostics.path || "—")],
["Native Gate", _formatPersistDeltaGateText(diagnostics)],
["Bridge 模式", bridgeText],
["Persist 总耗时", _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs)],
["构建耗时", _formatDurationMs(diagnostics.buildMs)],
[
"Prepare / Native",
`${_formatDurationMs(diagnostics.prepareMs)} / ${_formatDurationMs(diagnostics.nativeAttemptMs)}`,
],
[
"Lookup / JS Diff",
`${_formatDurationMs(diagnostics.lookupMs)} / ${_formatDurationMs(diagnostics.jsDiffMs)}`,
],
["Hydrate", _formatDurationMs(diagnostics.hydrateMs)],
["Preload", String(diagnostics.preloadStatus || "—")],
["Native 来源", String(diagnostics.moduleSource || "—")],
["Fallback 原因", String(diagnostics.fallbackReason || "—")],
["Preload / Native 错误", errorText || "—"],
["增量规模", deltaSizeText],
["Persist 更新时间", updatedAtText],
];
}
function _refreshTaskPipelineOverview() {
const el = document.getElementById("bme-task-pipeline");
if (!el) return;
@@ -1473,9 +1626,22 @@ function _refreshTaskPipelineOverview() {
const vector = _resolvePipelineStatus(_getLastVectorStatus?.());
const recall = _resolvePipelineStatus(_getLastRecallStatus?.());
const persistLevel = loadInfo.loadState === "loaded" ? "info" : loadInfo.loadState === "loading" ? "info" : "warn";
const persistenceMetaParts = [`rev ${loadInfo.revision || 0}`];
const pipelineLoadMeta = _formatPipelineLoadDiagnosticsMeta(
loadInfo.loadDiagnostics,
);
if (pipelineLoadMeta) {
persistenceMetaParts.push(pipelineLoadMeta);
}
const pipelinePersistDeltaMeta = _formatPipelinePersistDeltaMeta(
loadInfo.persistDelta,
);
if (pipelinePersistDeltaMeta) {
persistenceMetaParts.push(pipelinePersistDeltaMeta);
}
const persistence = _resolvePipelineStatus({
text: loadInfo.loadState || "unknown",
meta: `rev ${loadInfo.revision || 0}`,
meta: persistenceMetaParts.join(" · "),
level: persistLevel,
});
@@ -2166,6 +2332,8 @@ function _refreshTaskPersistence() {
const graph = _getGraph?.() || {};
const ps = _getGraphPersistenceSnapshot();
const rs = graph.runtimeState || {};
const loadDiagnostics = _readPersistenceDiagnosticObject(ps.loadDiagnostics);
const persistDeltaDiagnostics = _readPersistenceDiagnosticObject(ps.persistDelta);
const LOAD_STATE_LABELS = {
"no-chat": "无聊天",
@@ -2307,6 +2475,8 @@ function _refreshTaskPersistence() {
const primaryRows = [
["当前状态", acceptedSummaryLabel],
["健康状态", healthLabel],
["Load 诊断", _formatPersistenceLoadSummary(loadDiagnostics)],
["Persist Delta", _formatPersistencePersistDeltaSummary(persistDeltaDiagnostics)],
["Chat Target", compactTargetLabel],
["主 durable", primaryTierLabel],
ps.hostProfile === "luker"
@@ -2358,6 +2528,10 @@ function _refreshTaskPersistence() {
["缓存落后", cacheLagLabel],
);
}
diagnosticRows.push(
..._buildLoadDiagnosticRows(loadDiagnostics),
..._buildPersistDeltaDiagnosticRows(persistDeltaDiagnostics),
);
el.innerHTML = `
<div class="bme-persist-grid">
@@ -11849,6 +12023,7 @@ function _getGraphPersistenceSnapshot() {
lastBackupFilename: "",
lastSyncError: "",
persistDelta: null,
loadDiagnostics: null,
};
}

1
ui/ui-status.js
View File

@@ -125,6 +125,7 @@ export function createGraphPersistenceState() {
lastSyncError: "",
dualWriteLastResult: null,
persistDelta: null,
loadDiagnostics: null,
updatedAt: new Date().toISOString(),
};
}

202
vector/vector-index.js
View File

@@ -63,6 +63,17 @@ function throwIfAborted(signal) {
}
}
function nowMs() {
if (typeof performance?.now === "function") {
return performance.now();
}
return Date.now();
}
function roundMs(value) {
return Math.round((Number(value) || 0) * 10) / 10;
}
export const BACKEND_DEFAULT_MODELS = {
openai: "text-embedding-3-small",
openrouter: "openai/text-embedding-3-small",
@@ -349,16 +360,38 @@ function getEligibleVectorNodes(graph, range = null) {
return nodes.filter((node) => buildNodeVectorText(node).length > 0);
}
function buildDesiredVectorEntries(graph, config, range = null) {
return getEligibleVectorNodes(graph, range).map((node) => {
const hash = buildNodeVectorHash(node, config);
function buildDesiredVectorEntries(graph, config, range = null, diagnostics = null) {
const modelScope = getVectorModelScope(config);
let textBuildMs = 0;
let hashBuildMs = 0;
const entries = getEligibleVectorNodes(graph, range).map((node) => {
const textStartedAt = diagnostics ? nowMs() : 0;
const text = buildNodeVectorText(node);
if (diagnostics) {
textBuildMs += nowMs() - textStartedAt;
}
const seqEnd = node?.seqRange?.[1] ?? node?.seq ?? 0;
const hashStartedAt = diagnostics ? nowMs() : 0;
const payload = [node?.id || "", text, String(seqEnd), modelScope].join("::");
const hash = stableHashString(payload);
if (diagnostics) {
hashBuildMs += nowMs() - hashStartedAt;
}
return {
nodeId: node.id,
hash,
text: buildNodeVectorText(node),
index: node?.seqRange?.[1] ?? node?.seq ?? 0,
text,
index: seqEnd,
};
});
if (diagnostics && typeof diagnostics === "object") {
diagnostics.textBuildMs = roundMs(textBuildMs);
diagnostics.hashBuildMs = roundMs(hashBuildMs);
diagnostics.entryCount = entries.length;
}
return entries;
}
function computeVectorStats(graph, desiredEntries) {
@@ -547,26 +580,54 @@ export async function syncGraphVectorIndex(
return {
insertedHashes: [],
stats: { total: 0, indexed: 0, stale: 0, pending: 0 },
timings: null,
};
}
throwIfAborted(signal);
const syncStartedAt = nowMs();
const syncMode = isBackendVectorConfig(config) ? "backend" : "direct";
const validation = validateVectorConfig(config);
if (!validation.valid) {
graph.vectorIndexState.lastWarning = validation.error;
graph.vectorIndexState.dirty = true;
return { insertedHashes: [], stats: graph.vectorIndexState.lastStats };
graph.vectorIndexState.lastTimings = {
mode: syncMode,
validationError: validation.error,
totalMs: roundMs(nowMs() - syncStartedAt),
updatedAt: Date.now(),
};
return {
insertedHashes: [],
stats: graph.vectorIndexState.lastStats,
timings: graph.vectorIndexState.lastTimings,
};
}
const state = graph.vectorIndexState;
const collectionId = buildVectorCollectionId(
chatId || graph?.historyState?.chatId,
);
const desiredEntries = buildDesiredVectorEntries(graph, config, range);
const desiredBuildDiagnostics = {};
const desiredBuildStartedAt = nowMs();
const desiredEntries = buildDesiredVectorEntries(
graph,
config,
range,
desiredBuildDiagnostics,
);
const desiredBuildMs = nowMs() - desiredBuildStartedAt;
const desiredByNodeId = new Map(
desiredEntries.map((entry) => [entry.nodeId, entry]),
);
const insertedHashes = [];
let backendPurgeMs = 0;
let backendDeleteMs = 0;
let backendInsertMs = 0;
let embedBatchMs = 0;
let deletedHashCount = 0;
let embeddingsRequested = 0;
const hasConcreteRange =
range && Number.isFinite(range.start) && Number.isFinite(range.end);
const rangedNodeIds = new Set(desiredEntries.map((entry) => entry.nodeId));
@@ -581,9 +642,13 @@ export async function syncGraphVectorIndex(
purge || state.dirty || scopeChanged || (force && !hasConcreteRange);
if (fullReset) {
const purgeStartedAt = nowMs();
await purgeVectorCollection(collectionId, signal);
backendPurgeMs += nowMs() - purgeStartedAt;
resetVectorMappings(graph, config, chatId);
const insertStartedAt = nowMs();
await insertVectorEntries(collectionId, config, desiredEntries, signal);
backendInsertMs += nowMs() - insertStartedAt;
for (const entry of desiredEntries) {
state.hashToNodeId[entry.hash] = entry.nodeId;
state.nodeToHash[entry.nodeId] = entry.hash;
@@ -623,8 +688,13 @@ export async function syncGraphVectorIndex(
entriesToInsert.push(entry);
}
deletedHashCount = hashesToDelete.length;
const deleteStartedAt = nowMs();
await deleteVectorHashes(collectionId, config, hashesToDelete, signal);
backendDeleteMs += nowMs() - deleteStartedAt;
const insertStartedAt = nowMs();
await insertVectorEntries(collectionId, config, entriesToInsert, signal);
backendInsertMs += nowMs() - insertStartedAt;
for (const entry of entriesToInsert) {
state.hashToNodeId[entry.hash] = entry.nodeId;
@@ -679,11 +749,14 @@ export async function syncGraphVectorIndex(
let directSyncHadFailures = false;
if (entriesToEmbed.length > 0) {
throwIfAborted(signal);
embeddingsRequested = entriesToEmbed.length;
const embedStartedAt = nowMs();
const embeddings = await embedBatch(
entriesToEmbed.map((entry) => entry.text),
config,
{ signal },
);
embedBatchMs += nowMs() - embedStartedAt;
for (let index = 0; index < entriesToEmbed.length; index++) {
const entry = entriesToEmbed[index];
@@ -718,14 +791,34 @@ export async function syncGraphVectorIndex(
state.lastWarning = "";
}
state.lastSyncAt = Date.now();
const statsBuildStartedAt = nowMs();
state.lastStats = computeVectorStats(
graph,
buildDesiredVectorEntries(graph, config),
);
const statsBuildMs = nowMs() - statsBuildStartedAt;
state.lastTimings = {
mode: syncMode,
desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length),
desiredBuildMs: roundMs(desiredBuildMs),
textBuildMs: Number(desiredBuildDiagnostics.textBuildMs || 0),
hashBuildMs: Number(desiredBuildDiagnostics.hashBuildMs || 0),
backendPurgeMs: roundMs(backendPurgeMs),
backendDeleteMs: roundMs(backendDeleteMs),
backendInsertMs: roundMs(backendInsertMs),
embedBatchMs: roundMs(embedBatchMs),
statsBuildMs: roundMs(statsBuildMs),
deletedHashes: Math.max(0, Math.floor(deletedHashCount)),
insertedEntries: insertedHashes.length,
embeddingsRequested: Math.max(0, Math.floor(embeddingsRequested)),
totalMs: roundMs(nowMs() - syncStartedAt),
updatedAt: Date.now(),
};
return {
insertedHashes,
stats: state.lastStats,
timings: state.lastTimings,
};
}
@@ -743,14 +836,52 @@ export async function findSimilarNodesByText(
const candidateNodes = Array.isArray(candidates)
? candidates
: getEligibleVectorNodes(graph);
const searchStartedAt = nowMs();
const mode = isDirectVectorConfig(config) ? "direct" : "backend";
const recordSearchTimings = (patch = {}) => {
const state = graph?.vectorIndexState;
if (!state || typeof state !== "object" || Array.isArray(state)) return;
state.lastSearchTimings = {
...(state.lastSearchTimings &&
typeof state.lastSearchTimings === "object" &&
!Array.isArray(state.lastSearchTimings)
? state.lastSearchTimings
: {}),
mode,
queryLength: String(text || "").length,
candidateCount: candidateNodes.length,
topK: Math.max(1, Math.floor(Number(topK) || 1)),
...patch,
totalMs: roundMs(nowMs() - searchStartedAt),
updatedAt: Date.now(),
};
};
if (candidateNodes.length === 0) return [];
if (candidateNodes.length === 0) {
recordSearchTimings({
success: true,
reason: "no-candidates",
resultCount: 0,
});
return [];
}
if (isDirectVectorConfig(config)) {
const queryEmbedStartedAt = nowMs();
const queryVec = await embedText(text, config, { signal });
if (!queryVec) return [];
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
if (!queryVec) {
recordSearchTimings({
success: false,
reason: "direct-query-embed-empty",
queryEmbedMs: roundMs(queryEmbedMs),
resultCount: 0,
});
return [];
}
return searchSimilar(
const localSearchStartedAt = nowMs();
const results = searchSimilar(
queryVec,
candidateNodes
.filter(
@@ -762,12 +893,29 @@ export async function findSimilarNodesByText(
})),
topK,
);
recordSearchTimings({
success: true,
reason: "ok",
queryEmbedMs: roundMs(queryEmbedMs),
searchMs: roundMs(nowMs() - localSearchStartedAt),
resultCount: results.length,
});
return results;
}
const validation = validateVectorConfig(config);
if (!validation.valid) return [];
if (!validation.valid) {
recordSearchTimings({
success: false,
reason: "vector-config-invalid",
error: validation.error,
resultCount: 0,
});
return [];
}
try {
const requestStartedAt = nowMs();
const response = await fetchWithTimeout(
"/api/vector/query",
{
@@ -784,6 +932,7 @@ export async function findSimilarNodesByText(
},
getConfiguredTimeoutMs(config),
);
const requestMs = nowMs() - requestStartedAt;
if (!response.ok) {
const errorText = await response.text().catch(() => response.statusText);
@@ -795,23 +944,47 @@ export async function findSimilarNodesByText(
"backend-query-failed",
`后端向量查询失败（${message}），已标记待重建`,
);
recordSearchTimings({
success: false,
reason: "backend-query-http-failed",
statusCode: Number(response.status || 0),
requestMs: roundMs(requestMs),
error: message,
resultCount: 0,
});
return [];
}
const parseStartedAt = nowMs();
const data = await response.json().catch(() => ({ hashes: [] }));
const parseMs = nowMs() - parseStartedAt;
const hashes = Array.isArray(data?.hashes) ? data.hashes : [];
const nodeIdByHash = graph.vectorIndexState?.hashToNodeId || {};
const allowedIds = new Set(candidateNodes.map((node) => node.id));
return hashes
const results = hashes
.map((hash, index) => ({
nodeId: nodeIdByHash[hash],
score: Math.max(0.01, 1 - index / Math.max(1, hashes.length)),
}))
.filter((entry) => entry.nodeId && allowedIds.has(entry.nodeId))
.slice(0, topK);
recordSearchTimings({
success: true,
reason: "ok",
requestMs: roundMs(requestMs),
parseMs: roundMs(parseMs),
resultCount: results.length,
hashCount: hashes.length,
});
return results;
} catch (error) {
if (isAbortError(error)) {
recordSearchTimings({
success: false,
reason: "aborted",
error: error?.message || String(error),
});
throw error;
}
const message = error?.message || String(error) || "后端向量查询失败";
@@ -821,6 +994,11 @@ export async function findSimilarNodesByText(
"backend-query-failed",
`后端向量查询失败（${message}），已标记待重建`,
);
recordSearchTimings({
success: false,
reason: "backend-query-exception",
error: message,
});
throw error;
}
}