From d2c3d1f5ddecde9d4618371922fca471b21dd02d Mon Sep 17 00:00:00 2001 From: Youzini-afk <13153778771cx@gmail.com> Date: Tue, 21 Apr 2026 20:32:03 +0800 Subject: [PATCH] Add persistence and retrieval observability with native delta gating --- index.js | 175 ++++++++++++++++++-- maintenance/extraction-controller.js | 87 ++++++++-- retrieval/retriever.js | 20 +++ retrieval/shared-ranking.js | 17 ++ tests/extraction-persistence-gating.mjs | 101 ++++++++++++ ui/panel.js | 177 ++++++++++++++++++++- ui/ui-status.js | 1 + vector/vector-index.js | 202 ++++++++++++++++++++++-- 8 files changed, 745 insertions(+), 35 deletions(-) diff --git a/index.js b/index.js index 1cbf5d4..7ab16e8 100644 --- a/index.js +++ b/index.js @@ -1585,6 +1585,10 @@ function getGraphPersistenceLiveState() { null, ), persistDelta: cloneRuntimeDebugValue(graphPersistenceState.persistDelta, null), + loadDiagnostics: cloneRuntimeDebugValue( + graphPersistenceState.loadDiagnostics, + null, + ), }; return cloneRuntimeDebugValue(snapshot, snapshot); @@ -1654,6 +1658,14 @@ function readPersistDeltaDiagnosticsNow() { return Date.now(); } +function readLoadDiagnosticsNow() { + return readPersistDeltaDiagnosticsNow(); +} + +function normalizeLoadDiagnosticsMs(value = 0) { + return Math.round((Number(value) || 0) * 10) / 10; +} + function updatePersistDeltaDiagnostics(snapshot = null) { const nextSnapshot = snapshot && typeof snapshot === "object" && !Array.isArray(snapshot) @@ -1671,6 +1683,23 @@ function updatePersistDeltaDiagnostics(snapshot = null) { return nextSnapshot; } +function updateLoadDiagnostics(snapshot = null) { + const nextSnapshot = + snapshot && typeof snapshot === "object" && !Array.isArray(snapshot) + ? { + ...(graphPersistenceState.loadDiagnostics && + typeof graphPersistenceState.loadDiagnostics === "object" && + !Array.isArray(graphPersistenceState.loadDiagnostics) + ? cloneRuntimeDebugValue(graphPersistenceState.loadDiagnostics, {}) + : {}), + ...cloneRuntimeDebugValue(snapshot, {}), + updatedAt: new Date().toISOString(), + } + : null; + updateGraphPersistenceState({ loadDiagnostics: nextSnapshot }); + return nextSnapshot; +} + function bumpGraphRevision(reason = "graph-mutation") { const nextRevision = Math.max( @@ -9093,14 +9122,37 @@ function applyIndexedDbSnapshotToRuntime( ) { const normalizedChatId = normalizeChatIdCandidate(chatId); syncCommitMarkerToPersistenceState(getContext()); + const loadStartedAt = readLoadDiagnosticsNow(); + const recordLoadDiagnostics = (patch = {}) => + updateLoadDiagnostics({ + stage: "apply-indexeddb-snapshot", + source: String(source || reasonPrefix), + reasonPrefix: String(reasonPrefix || "indexeddb"), + statusLabel: String(statusLabel || "IndexedDB"), + chatId: normalizedChatId || "", + attemptIndex: Number.isFinite(Number(attemptIndex)) + ? Math.max(0, Math.floor(Number(attemptIndex))) + : 0, + storagePrimary: String(storagePrimary || "indexeddb"), + storageMode: String(storageMode || storagePrimary || "indexeddb"), + ...cloneRuntimeDebugValue(patch, {}), + totalMs: normalizeLoadDiagnosticsMs(readLoadDiagnosticsNow() - loadStartedAt), + }); + let hydrateMs = 0; if (!normalizedChatId || !isIndexedDbSnapshotMeaningful(snapshot)) { - return { + const result = { success: false, loaded: false, reason: `${reasonPrefix}-empty`, chatId: normalizedChatId, attemptIndex, }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: result.reason, + }); + return result; } const revision = Math.max( @@ -9145,7 +9197,7 @@ function applyIndexedDbSnapshotToRuntime( revision, staleDetail: staleDecision, }); - return { + const result = { success: false, loaded: false, reason: `${reasonPrefix}-stale-runtime`, @@ -9154,12 +9206,22 @@ function applyIndexedDbSnapshotToRuntime( revision, staleDetail: cloneRuntimeDebugValue(staleDecision, null), }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: result.reason, + revision, + staleDetail: cloneRuntimeDebugValue(staleDecision, null), + }); + return result; } let graphFromSnapshot = null; try { + const hydrateStartedAt = readLoadDiagnosticsNow(); graphFromSnapshot = buildGraphFromSnapshot(snapshot, { chatId: normalizedChatId, }); + hydrateMs = readLoadDiagnosticsNow() - hydrateStartedAt; } catch (error) { const failureReason = error?.code === "BME_SNAPSHOT_INTEGRITY_ERROR" @@ -9194,7 +9256,7 @@ function applyIndexedDbSnapshotToRuntime( detail: error?.message || String(error), integrityReasons: Array.isArray(error?.reasons) ? error.reasons : [], }); - return { + const result = { success: false, loaded: false, reason: failureReason, @@ -9203,7 +9265,18 @@ function applyIndexedDbSnapshotToRuntime( chatId: normalizedChatId, attemptIndex, }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: failureReason, + revision, + hydrateMs: normalizeLoadDiagnosticsMs(hydrateMs), + error: error?.message || String(error), + integrityReasons: Array.isArray(error?.reasons) ? [...error.reasons] : [], + }); + return result; } + const applyRuntimeStartedAt = readLoadDiagnosticsNow(); currentGraph = graphFromSnapshot; stampGraphPersistenceMeta(currentGraph, { revision, @@ -9298,7 +9371,7 @@ function applyIndexedDbSnapshotToRuntime( ...getGraphStats(currentGraph), }); - return { + const result = { success: true, loaded: true, loadState: GRAPH_LOAD_STATES.LOADED, @@ -9308,6 +9381,17 @@ function applyIndexedDbSnapshotToRuntime( shadowSnapshotUsed: false, revision, }; + recordLoadDiagnostics({ + success: true, + loaded: true, + reason: result.reason, + revision, + hydrateMs: normalizeLoadDiagnosticsMs(hydrateMs), + applyRuntimeMs: normalizeLoadDiagnosticsMs( + readLoadDiagnosticsNow() - applyRuntimeStartedAt, + ), + }); + return result; } async function loadGraphFromIndexedDb( @@ -9321,27 +9405,55 @@ async function loadGraphFromIndexedDb( ) { const normalizedChatId = normalizeChatIdCandidate(chatId); const commitMarker = syncCommitMarkerToPersistenceState(getContext()); + const loadStartedAt = readLoadDiagnosticsNow(); + const recordLoadDiagnostics = (patch = {}) => + updateLoadDiagnostics({ + stage: "load-indexeddb", + source: String(source || "indexeddb-probe"), + chatId: normalizedChatId || "", + attemptIndex: Number.isFinite(Number(attemptIndex)) + ? Math.max(0, Math.floor(Number(attemptIndex))) + : 0, + ...cloneRuntimeDebugValue(patch, {}), + totalMs: normalizeLoadDiagnosticsMs(readLoadDiagnosticsNow() - loadStartedAt), + }); + let exportSnapshotMs = 0; + let exportSnapshotSource = ""; if (!normalizedChatId) { - return { + const result = { success: false, loaded: false, reason: "indexeddb-missing-chat-id", chatId: "", attemptIndex, }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: result.reason, + }); + return result; } let localStore = getPreferredGraphLocalStorePresentationSync(); try { const manager = ensureBmeChatManager(); if (!manager) { - return { + const result = { success: false, loaded: false, reason: "indexeddb-manager-unavailable", chatId: normalizedChatId, attemptIndex, }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: result.reason, + storagePrimary: localStore.storagePrimary, + storageMode: localStore.storageMode, + }); + return result; } const db = await manager.getCurrentDb(normalizedChatId); localStore = resolveDbGraphStorePresentation(db); @@ -9464,11 +9576,22 @@ async function loadGraphFromIndexedDb( }, }); } - const snapshot = - identityRecoveryResult?.snapshot || - localStoreMigrationResult?.snapshot || - migrationResult?.snapshot || - (await db.exportSnapshot({ includeTombstones: false })); + let snapshot = null; + if (identityRecoveryResult?.snapshot) { + snapshot = identityRecoveryResult.snapshot; + exportSnapshotSource = "identity-recovery"; + } else if (localStoreMigrationResult?.snapshot) { + snapshot = localStoreMigrationResult.snapshot; + exportSnapshotSource = "local-store-migration"; + } else if (migrationResult?.snapshot) { + snapshot = migrationResult.snapshot; + exportSnapshotSource = "legacy-migration"; + } else { + const exportStartedAt = readLoadDiagnosticsNow(); + snapshot = await db.exportSnapshot({ includeTombstones: false }); + exportSnapshotMs = readLoadDiagnosticsNow() - exportStartedAt; + exportSnapshotSource = "indexeddb-export"; + } const shadowSnapshot = resolveCompatibleGraphShadowSnapshot( resolveCurrentChatIdentity(getContext()), ); @@ -9678,6 +9801,7 @@ async function loadGraphFromIndexedDb( }; } + const applyInvokeStartedAt = readLoadDiagnosticsNow(); const loadResult = applyIndexedDbSnapshotToRuntime(normalizedChatId, snapshot, { source, attemptIndex, @@ -9686,11 +9810,26 @@ async function loadGraphFromIndexedDb( statusLabel: snapshotStore.statusLabel, reasonPrefix: snapshotStore.reasonPrefix, }); + const applyInvokeMs = readLoadDiagnosticsNow() - applyInvokeStartedAt; if (commitMarkerDiagnostic?.reason && loadResult?.loaded) { updateGraphPersistenceState({ persistMismatchReason: commitMarkerDiagnostic.reason, }); } + recordLoadDiagnostics({ + success: loadResult?.success === true, + loaded: loadResult?.loaded === true, + reason: String(loadResult?.reason || ""), + revision: Number.isFinite(Number(loadResult?.revision)) + ? Number(loadResult.revision) + : snapshotRevision, + storagePrimary: snapshotStore.storagePrimary, + storageMode: snapshotStore.storageMode, + commitMarkerMismatched: commitMarkerMismatch.mismatched === true, + exportSnapshotSource: exportSnapshotSource || "snapshot-prepared", + exportSnapshotMs: normalizeLoadDiagnosticsMs(exportSnapshotMs), + applyInvokeMs: normalizeLoadDiagnosticsMs(applyInvokeMs), + }); return loadResult; } catch (error) { console.warn(`[ST-BME] ${localStore.statusLabel} 读取失败，回退 metadata:`, error); @@ -9706,7 +9845,7 @@ async function loadGraphFromIndexedDb( at: Date.now(), }, }); - return { + const result = { success: false, loaded: false, reason: `${localStore.reasonPrefix}-read-failed`, @@ -9714,6 +9853,17 @@ async function loadGraphFromIndexedDb( attemptIndex, error, }; + recordLoadDiagnostics({ + success: false, + loaded: false, + reason: result.reason, + storagePrimary: localStore.storagePrimary, + storageMode: localStore.storageMode, + error: error?.message || String(error), + exportSnapshotSource: exportSnapshotSource || "unknown", + exportSnapshotMs: normalizeLoadDiagnosticsMs(exportSnapshotMs), + }); + return result; } } @@ -16030,6 +16180,7 @@ async function executeExtractionBatch({ getEmbeddingConfig, getExtractionCount: () => extractionCount, getLastProcessedAssistantFloor, + getSettings, getSchema, handleExtractionSuccess, persistExtractionBatchResult, diff --git a/maintenance/extraction-controller.js b/maintenance/extraction-controller.js index 38fa94e..0f1d78f 100644 --- a/maintenance/extraction-controller.js +++ b/maintenance/extraction-controller.js @@ -7,6 +7,8 @@ import { normalizeDialogueFloorRange, } from "./chat-history.js"; +let nativePersistDeltaInstallPromise = null; + function toSafeFloor(value, fallback = null) { if (value == null || value === "") return fallback; const numeric = Number(value); @@ -115,6 +117,31 @@ function cloneSerializable(value, fallback = null) { } } +function readNow() { + if (typeof performance === "object" && typeof performance.now === "function") { + return performance.now(); + } + return Date.now(); +} + +async function ensureNativePersistDeltaHookInstalled() { + if (typeof globalThis.__stBmeNativeBuildPersistDelta === "function") { + return { + loaded: true, + source: "global-hook", + }; + } + if (!nativePersistDeltaInstallPromise) { + nativePersistDeltaInstallPromise = import("../vendor/wasm/stbme_core.js") + .then((module) => module?.installNativePersistDeltaHook?.()) + .catch((error) => { + nativePersistDeltaInstallPromise = null; + throw error; + }); + } + return await nativePersistDeltaInstallPromise; +} + function setExtractionProgressStatus( runtime, text, @@ -247,11 +274,12 @@ function buildRerunFallbackInfo(chat = [], targetDialogueRange = [-1, -1]) { }; } -function buildCommittedBatchPersistSnapshot( +async function buildCommittedBatchPersistSnapshot( runtime, { graph = null, chat = [], + settings = null, beforeSnapshot = null, processedRange = [null, null], postProcessArtifacts = [], @@ -274,6 +302,10 @@ function buildCommittedBatchPersistSnapshot( const range = Array.isArray(processedRange) ? processedRange : [null, null]; const rangeStart = Number.isFinite(Number(range[0])) ? Number(range[0]) : null; const rangeEnd = Number.isFinite(Number(range[1])) ? Number(range[1]) : null; + const runtimeSettings = + settings && typeof settings === "object" && !Array.isArray(settings) + ? settings + : runtime?.getSettings?.() || {}; const dialogueMap = buildDialogueFloorMap(chat); const processedDialogueRange = [ Number.isFinite(Number(rangeStart)) @@ -290,7 +322,7 @@ function buildCommittedBatchPersistSnapshot( Number(rangeStart) - Math.max( 0, - Number(runtime?.getSettings?.()?.extractContextTurns) || 0, + Number(runtimeSettings?.extractContextTurns) || 0, ) * 2, ) @@ -347,13 +379,45 @@ function buildCommittedBatchPersistSnapshot( ); } + let persistDelta = null; + const shouldUseNativePersistDelta = + runtimeSettings?.persistUseNativeDelta === true && + runtimeSettings?.graphNativeForceDisable !== true; + const nativeFailOpen = runtimeSettings?.nativeEngineFailOpen !== false; + if (typeof runtime.buildPersistDelta === "function") { + if (shouldUseNativePersistDelta) { + const preloadStartedAt = readNow(); + try { + await ensureNativePersistDeltaHookInstalled(); + } catch (error) { + if (!nativeFailOpen) { + throw error; + } + runtime?.console?.warn?.( + "[ST-BME] extraction native persist delta preload failed, fallback to JS delta:", + { + error: error?.message || String(error), + preloadMs: readNow() - preloadStartedAt, + }, + ); + } + } + + persistDelta = runtime.buildPersistDelta(beforeSnapshot, committedGraphSnapshot, { + useNativeDelta: shouldUseNativePersistDelta, + nativeFailOpen, + persistNativeDeltaThresholdRecords: + runtimeSettings?.persistNativeDeltaThresholdRecords, + persistNativeDeltaThresholdStructuralDelta: + runtimeSettings?.persistNativeDeltaThresholdStructuralDelta, + persistNativeDeltaThresholdSerializedChars: + runtimeSettings?.persistNativeDeltaThresholdSerializedChars, + persistNativeDeltaBridgeMode: runtimeSettings?.persistNativeDeltaBridgeMode, + }); + } + return { - persistDelta: - typeof runtime.buildPersistDelta === "function" - ? runtime.buildPersistDelta(beforeSnapshot, committedGraphSnapshot, { - useNativeDelta: false, - }) - : null, + persistDelta, persistGraphSnapshot: committedGraphSnapshot, committedBatchJournalEntry, afterSnapshot, @@ -367,7 +431,9 @@ function isPersistenceRevisionAccepted(runtime, persistence = null) { if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) { return false; } - const lastAcceptedRevision = Number(graphPersistenceState?.lastAcceptedRevision || 0); + const lastAcceptedRevision = Number( + runtime?.getGraphPersistenceState?.()?.lastAcceptedRevision || 0, + ); return Number.isFinite(lastAcceptedRevision) && lastAcceptedRevision >= persistenceRevision; } @@ -633,9 +699,10 @@ export async function executeExtractionBatchController( batchStatus, ); const batchStatusRef = effects?.batchStatus || batchStatus; - const committedPersistState = buildCommittedBatchPersistSnapshot(runtime, { + const committedPersistState = await buildCommittedBatchPersistSnapshot(runtime, { graph: runtime.getCurrentGraph(), chat, + settings, beforeSnapshot, processedRange: [startIdx, endIdx], postProcessArtifacts: runtime.computePostProcessArtifacts( diff --git a/retrieval/retriever.js b/retrieval/retriever.js index ea97f90..504189c 100644 --- a/retrieval/retriever.js +++ b/retrieval/retriever.js @@ -1379,6 +1379,24 @@ export async function retrieve({ ) ? [...sharedRanking.diagnostics.lexicalTopHits] : []; + retrievalMeta.timings.sharedQueryBlend = Number( + sharedRanking?.diagnostics?.timings?.queryBlend || 0, + ); + retrievalMeta.timings.sharedLexical = Number( + sharedRanking?.diagnostics?.timings?.lexical || 0, + ); + retrievalMeta.timings.sharedScoring = Number( + sharedRanking?.diagnostics?.timings?.scoring || 0, + ); + retrievalMeta.timings.sharedTotal = Number( + sharedRanking?.diagnostics?.timings?.total || 0, + ); + retrievalMeta.timings.sharedVector = Number( + sharedRanking?.diagnostics?.timings?.vector || 0, + ); + retrievalMeta.timings.sharedDiffusion = Number( + sharedRanking?.diagnostics?.timings?.diffusion || 0, + ); retrievalMeta.timings.vector = Number( sharedRanking?.diagnostics?.timings?.vector || 0, ); @@ -1395,12 +1413,14 @@ export async function retrieve({ ? [...sharedRanking.diffusionResults] : []; exactEntityAnchors.push(...(sharedRanking?.exactEntityAnchors || [])); + const anchorCollectStartedAt = nowMs(); supplementalAnchorNodeIds = collectSupplementalAnchorNodeIds( graph, vectorResults, exactEntityAnchors.map((item) => item.nodeId), 5, ); + retrievalMeta.timings.anchorCollect = roundMs(nowMs() - anchorCollectStartedAt); let residualResult = { triggered: false, diff --git a/retrieval/shared-ranking.js b/retrieval/shared-ranking.js index 6e1fba3..05b782a 100644 --- a/retrieval/shared-ranking.js +++ b/retrieval/shared-ranking.js @@ -536,6 +536,8 @@ export async function rankNodesForTaskContext({ ? options.activeNodes.filter((node) => node && !node.archived) : getActiveNodes(graph).filter((node) => node && !node.archived); const vectorValidation = validateVectorConfig(embeddingConfig); + const rankingStartedAt = nowMs(); + const queryBlendStartedAt = nowMs(); const contextQueryBlend = buildContextQueryBlend(userMessage, recentMessages, { enabled: enableContextQueryBlend, assistantWeight: contextAssistantWeight, @@ -553,6 +555,7 @@ export async function rankNodesForTaskContext({ maxSegments: multiIntentMaxSegments, }, ); + const queryBlendMs = roundMs(nowMs() - queryBlendStartedAt); const diagnostics = { queryBlendActive: contextQueryBlend.active, queryBlendParts: (contextQueryBlend.parts || []).map((part) => ({ @@ -577,12 +580,17 @@ export async function rankNodesForTaskContext({ lexicalTopHits: [], skipReasons: [], timings: { + queryBlend: queryBlendMs, vector: 0, diffusion: 0, + lexical: 0, + scoring: 0, + total: 0, }, }; if (!graph || activeNodes.length === 0) { + diagnostics.timings.total = roundMs(nowMs() - rankingStartedAt); return { activeNodes, contextQueryBlend, @@ -688,13 +696,19 @@ export async function rankNodesForTaskContext({ } } + const scoringStartedAt = nowMs(); + let lexicalMs = 0; const scoredNodes = []; for (const [nodeId, scores] of scoreMap.entries()) { const node = getNode(graph, nodeId); if (!node || node.archived) continue; + const lexicalStartedAt = enableLexicalBoost ? nowMs() : 0; const lexicalScore = enableLexicalBoost ? computeLexicalScore(node, lexicalQuery.sources) : 0; + if (enableLexicalBoost) { + lexicalMs += nowMs() - lexicalStartedAt; + } const finalScore = hybridScore( { graphScore: scores.graphScore, @@ -719,6 +733,8 @@ export async function rankNodesForTaskContext({ weightedScore: finalScore, }); } + diagnostics.timings.lexical = roundMs(lexicalMs); + diagnostics.timings.scoring = roundMs(nowMs() - scoringStartedAt); scoredNodes.sort((left, right) => { const weightedDelta = @@ -737,6 +753,7 @@ export async function rankNodesForTaskContext({ (item) => (Number(item.lexicalScore) || 0) > 0, ).length; diagnostics.lexicalTopHits = buildLexicalTopHits(scoredNodes); + diagnostics.timings.total = roundMs(nowMs() - rankingStartedAt); return { activeNodes, diff --git a/tests/extraction-persistence-gating.mjs b/tests/extraction-persistence-gating.mjs index 897b09d..3741646 100644 --- a/tests/extraction-persistence-gating.mjs +++ b/tests/extraction-persistence-gating.mjs @@ -16,6 +16,7 @@ function createRuntime(persistResult) { }; let processedHistoryUpdates = 0; let persistedGraphSnapshot = null; + let lastPersistDeltaOptions = null; return { graph, @@ -35,6 +36,22 @@ function createRuntime(persistResult) { cloneGraphSnapshot(value) { return JSON.parse(JSON.stringify(value)); }, + buildPersistDelta(_beforeSnapshot, _afterSnapshot, options = {}) { + lastPersistDeltaOptions = { ...(options || {}) }; + return { + upsertNodes: [], + upsertEdges: [], + deleteNodeIds: [], + deleteEdgeIds: [], + tombstones: [], + countDelta: { + nodes: 0, + edges: 0, + tombstones: 0, + }, + runtimeMetaPatch: {}, + }; + }, buildExtractionMessages() { return [{ seq: 5, role: "assistant", content: "测试消息" }]; }, @@ -101,6 +118,9 @@ function createRuntime(persistResult) { get persistedGraphSnapshot() { return persistedGraphSnapshot; }, + get lastPersistDeltaOptions() { + return lastPersistDeltaOptions; + }, }; } @@ -212,4 +232,85 @@ function createRuntime(persistResult) { assert.equal(runtime.graph.historyState.lastBatchStatus.persistence, null); } +{ + const originalNativeBuilder = globalThis.__stBmeNativeBuildPersistDelta; + globalThis.__stBmeNativeBuildPersistDelta = () => ({ + upsertNodes: [], + upsertEdges: [], + deleteNodeIds: [], + deleteEdgeIds: [], + tombstones: [], + runtimeMetaPatch: {}, + }); + const runtime = createRuntime({ + saved: true, + queued: false, + blocked: false, + accepted: true, + reason: "indexeddb", + revision: 9, + saveMode: "indexeddb", + storageTier: "indexeddb", + }); + const result = await executeExtractionBatchController(runtime, { + chat: [{ is_user: false, mes: "测试" }], + startIdx: 5, + endIdx: 5, + settings: { + persistUseNativeDelta: true, + graphNativeForceDisable: false, + nativeEngineFailOpen: true, + persistNativeDeltaThresholdRecords: 123, + persistNativeDeltaThresholdStructuralDelta: 45, + persistNativeDeltaThresholdSerializedChars: 6789, + persistNativeDeltaBridgeMode: "hash", + }, + }); + + assert.equal(result.success, true); + assert.equal(runtime.lastPersistDeltaOptions.useNativeDelta, true); + assert.equal(runtime.lastPersistDeltaOptions.nativeFailOpen, true); + assert.equal(runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdRecords, 123); + assert.equal( + runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdStructuralDelta, + 45, + ); + assert.equal( + runtime.lastPersistDeltaOptions.persistNativeDeltaThresholdSerializedChars, + 6789, + ); + assert.equal(runtime.lastPersistDeltaOptions.persistNativeDeltaBridgeMode, "hash"); + + if (typeof originalNativeBuilder === "function") { + globalThis.__stBmeNativeBuildPersistDelta = originalNativeBuilder; + } else { + delete globalThis.__stBmeNativeBuildPersistDelta; + } +} + +{ + const runtime = createRuntime({ + saved: true, + queued: false, + blocked: false, + accepted: true, + reason: "indexeddb", + revision: 10, + saveMode: "indexeddb", + storageTier: "indexeddb", + }); + const result = await executeExtractionBatchController(runtime, { + chat: [{ is_user: false, mes: "测试" }], + startIdx: 5, + endIdx: 5, + settings: { + persistUseNativeDelta: true, + graphNativeForceDisable: true, + }, + }); + + assert.equal(result.success, true); + assert.equal(runtime.lastPersistDeltaOptions.useNativeDelta, false); +} + console.log("extraction-persistence-gating tests passed"); diff --git a/ui/panel.js b/ui/panel.js index cfbf1ae..67b4622 100644 --- a/ui/panel.js +++ b/ui/panel.js @@ -1461,6 +1461,159 @@ function _resolvePipelineStatus(statusObj) { return { label: text || "IDLE", color, detail: meta }; } +function _readPersistenceDiagnosticObject(snapshot = null) { + if (!snapshot || typeof snapshot !== "object" || Array.isArray(snapshot)) { + return null; + } + return snapshot; +} + +function _formatLoadDiagnosticsStageLabel(stage = "") { + const normalized = String(stage || "").trim(); + if (!normalized) return "—"; + const labels = { + "load-indexeddb": "IndexedDB 加载", + "apply-indexeddb-snapshot": "快照应用", + }; + return labels[normalized] || normalized; +} + +function _formatPipelineLoadDiagnosticsMeta(loadDiagnostics = null) { + const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics); + if (!diagnostics) return ""; + const totalText = _formatDurationMs(diagnostics.totalMs); + if (totalText !== "—") return `load ${totalText}`; + const stageLabel = _formatLoadDiagnosticsStageLabel(diagnostics.stage); + return stageLabel === "—" ? "" : stageLabel; +} + +function _formatPipelinePersistDeltaMeta(persistDelta = null) { + const diagnostics = _readPersistenceDiagnosticObject(persistDelta); + if (!diagnostics) return ""; + + const parts = []; + const totalText = _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs); + if (totalText !== "—") { + parts.push(`delta ${totalText}`); + } + + const gateText = String(_formatPersistDeltaGateText(diagnostics) || "").trim(); + if (gateText) { + const compactGate = gateText.startsWith("已拦截") ? "已拦截" : gateText; + parts.push(`native ${compactGate}`); + } + + return parts.join(" · "); +} + +function _formatPersistenceLoadSummary(loadDiagnostics = null) { + const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics); + if (!diagnostics) return "暂无"; + + const statusText = + diagnostics.success === true + ? "成功" + : diagnostics.success === false + ? "失败" + : "未知"; + const totalText = _formatDurationMs(diagnostics.totalMs); + const stageLabel = _formatLoadDiagnosticsStageLabel(diagnostics.stage); + const reasonText = String(diagnostics.reason || "").trim(); + const parts = [statusText]; + if (stageLabel !== "—") parts.push(stageLabel); + if (totalText !== "—") parts.push(`total ${totalText}`); + if (reasonText) parts.push(reasonText); + return parts.join(" · "); +} + +function _formatPersistencePersistDeltaSummary(persistDelta = null) { + const diagnostics = _readPersistenceDiagnosticObject(persistDelta); + if (!diagnostics) return "暂无"; + + const pathText = String(diagnostics.path || "").trim() || "—"; + const totalText = _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs); + const gateText = String(_formatPersistDeltaGateText(diagnostics) || "").trim(); + const parts = [pathText]; + if (totalText !== "—") parts.push(totalText); + if (gateText) parts.push(`native ${gateText}`); + return parts.join(" · "); +} + +function _buildLoadDiagnosticRows(loadDiagnostics = null) { + const diagnostics = _readPersistenceDiagnosticObject(loadDiagnostics); + if (!diagnostics) { + return [["Load 诊断", "无"]]; + } + + const statusText = + diagnostics.success === true + ? "成功" + : diagnostics.success === false + ? "失败" + : "未知"; + const updatedAtText = diagnostics.updatedAt + ? _formatTaskProfileTime(diagnostics.updatedAt) + : "—"; + + return [ + ["Load 阶段", _formatLoadDiagnosticsStageLabel(diagnostics.stage)], + ["Load 来源", String(diagnostics.source || diagnostics.statusLabel || "—")], + ["Load 状态", statusText], + ["Load 原因", String(diagnostics.reason || "—")], + ["Load 总耗时", _formatDurationMs(diagnostics.totalMs)], + ["导出快照", _formatDurationMs(diagnostics.exportSnapshotMs)], + ["Hydrate", _formatDurationMs(diagnostics.hydrateMs)], + ["Apply 调用", _formatDurationMs(diagnostics.applyInvokeMs)], + ["Apply 运行", _formatDurationMs(diagnostics.applyRuntimeMs)], + ["Load 更新时间", updatedAtText], + ]; +} + +function _buildPersistDeltaDiagnosticRows(persistDelta = null) { + const diagnostics = _readPersistenceDiagnosticObject(persistDelta); + if (!diagnostics) { + return [["Persist Delta 诊断", "无"]]; + } + + const errorText = String( + diagnostics.moduleError || diagnostics.preloadError || diagnostics.nativeError || "", + ).trim(); + const bridgeText = `${String(diagnostics.requestedBridgeMode || "none")} → ${String( + diagnostics.preparedBridgeMode || "none", + )}`; + const deltaSizeText = `${Number(diagnostics.upsertNodeCount || 0)}N / ${Number( + diagnostics.upsertEdgeCount || 0, + )}E / ${Number(diagnostics.deleteNodeCount || 0)}DN / ${Number( + diagnostics.deleteEdgeCount || 0, + )}DE`; + const updatedAtText = diagnostics.updatedAt + ? _formatTaskProfileTime(diagnostics.updatedAt) + : "—"; + + return [ + ["Persist 路径", String(diagnostics.path || "—")], + ["Native Gate", _formatPersistDeltaGateText(diagnostics)], + ["Bridge 模式", bridgeText], + ["Persist 总耗时", _formatDurationMs(diagnostics.totalMs || diagnostics.buildMs)], + ["构建耗时", _formatDurationMs(diagnostics.buildMs)], + [ + "Prepare / Native", + `${_formatDurationMs(diagnostics.prepareMs)} / ${_formatDurationMs(diagnostics.nativeAttemptMs)}`, + ], + [ + "Lookup / JS Diff", + `${_formatDurationMs(diagnostics.lookupMs)} / ${_formatDurationMs(diagnostics.jsDiffMs)}`, + ], + ["Hydrate", _formatDurationMs(diagnostics.hydrateMs)], + ["Preload", String(diagnostics.preloadStatus || "—")], + ["Native 来源", String(diagnostics.moduleSource || "—")], + ["Fallback 原因", String(diagnostics.fallbackReason || "—")], + ["Preload / Native 错误", errorText || "—"], + ["增量规模", deltaSizeText], + ["Persist 更新时间", updatedAtText], + ]; +} + function _refreshTaskPipelineOverview() { const el = document.getElementById("bme-task-pipeline"); if (!el) return; @@ -1473,9 +1626,22 @@ function _refreshTaskPipelineOverview() { const vector = _resolvePipelineStatus(_getLastVectorStatus?.()); const recall = _resolvePipelineStatus(_getLastRecallStatus?.()); const persistLevel = loadInfo.loadState === "loaded" ? "info" : loadInfo.loadState === "loading" ? "info" : "warn"; + const persistenceMetaParts = [`rev ${loadInfo.revision || 0}`]; + const pipelineLoadMeta = _formatPipelineLoadDiagnosticsMeta( + loadInfo.loadDiagnostics, + ); + if (pipelineLoadMeta) { + persistenceMetaParts.push(pipelineLoadMeta); + } + const pipelinePersistDeltaMeta = _formatPipelinePersistDeltaMeta( + loadInfo.persistDelta, + ); + if (pipelinePersistDeltaMeta) { + persistenceMetaParts.push(pipelinePersistDeltaMeta); + } const persistence = _resolvePipelineStatus({ text: loadInfo.loadState || "unknown", - meta: `rev ${loadInfo.revision || 0}`, + meta: persistenceMetaParts.join(" · "), level: persistLevel, }); @@ -2166,6 +2332,8 @@ function _refreshTaskPersistence() { const graph = _getGraph?.() || {}; const ps = _getGraphPersistenceSnapshot(); const rs = graph.runtimeState || {}; + const loadDiagnostics = _readPersistenceDiagnosticObject(ps.loadDiagnostics); + const persistDeltaDiagnostics = _readPersistenceDiagnosticObject(ps.persistDelta); const LOAD_STATE_LABELS = { "no-chat": "无聊天", @@ -2307,6 +2475,8 @@ function _refreshTaskPersistence() { const primaryRows = [ ["当前状态", acceptedSummaryLabel], ["健康状态", healthLabel], + ["Load 诊断", _formatPersistenceLoadSummary(loadDiagnostics)], + ["Persist Delta", _formatPersistencePersistDeltaSummary(persistDeltaDiagnostics)], ["Chat Target", compactTargetLabel], ["主 durable", primaryTierLabel], ps.hostProfile === "luker" @@ -2358,6 +2528,10 @@ function _refreshTaskPersistence() { ["缓存落后", cacheLagLabel], ); } + diagnosticRows.push( + ..._buildLoadDiagnosticRows(loadDiagnostics), + ..._buildPersistDeltaDiagnosticRows(persistDeltaDiagnostics), + ); el.innerHTML = `
@@ -11849,6 +12023,7 @@ function _getGraphPersistenceSnapshot() { lastBackupFilename: "", lastSyncError: "", persistDelta: null, + loadDiagnostics: null, }; } diff --git a/ui/ui-status.js b/ui/ui-status.js index a492a01..debedd3 100644 --- a/ui/ui-status.js +++ b/ui/ui-status.js @@ -125,6 +125,7 @@ export function createGraphPersistenceState() { lastSyncError: "", dualWriteLastResult: null, persistDelta: null, + loadDiagnostics: null, updatedAt: new Date().toISOString(), }; } diff --git a/vector/vector-index.js b/vector/vector-index.js index 7fd4e54..b7fde57 100644 --- a/vector/vector-index.js +++ b/vector/vector-index.js @@ -63,6 +63,17 @@ function throwIfAborted(signal) { } } +function nowMs() { + if (typeof performance?.now === "function") { + return performance.now(); + } + return Date.now(); +} + +function roundMs(value) { + return Math.round((Number(value) || 0) * 10) / 10; +} + export const BACKEND_DEFAULT_MODELS = { openai: "text-embedding-3-small", openrouter: "openai/text-embedding-3-small", @@ -349,16 +360,38 @@ function getEligibleVectorNodes(graph, range = null) { return nodes.filter((node) => buildNodeVectorText(node).length > 0); } -function buildDesiredVectorEntries(graph, config, range = null) { - return getEligibleVectorNodes(graph, range).map((node) => { - const hash = buildNodeVectorHash(node, config); +function buildDesiredVectorEntries(graph, config, range = null, diagnostics = null) { + const modelScope = getVectorModelScope(config); + let textBuildMs = 0; + let hashBuildMs = 0; + const entries = getEligibleVectorNodes(graph, range).map((node) => { + const textStartedAt = diagnostics ? nowMs() : 0; + const text = buildNodeVectorText(node); + if (diagnostics) { + textBuildMs += nowMs() - textStartedAt; + } + const seqEnd = node?.seqRange?.[1] ?? node?.seq ?? 0; + const hashStartedAt = diagnostics ? nowMs() : 0; + const payload = [node?.id || "", text, String(seqEnd), modelScope].join("::"); + const hash = stableHashString(payload); + if (diagnostics) { + hashBuildMs += nowMs() - hashStartedAt; + } return { nodeId: node.id, hash, - text: buildNodeVectorText(node), - index: node?.seqRange?.[1] ?? node?.seq ?? 0, + text, + index: seqEnd, }; }); + + if (diagnostics && typeof diagnostics === "object") { + diagnostics.textBuildMs = roundMs(textBuildMs); + diagnostics.hashBuildMs = roundMs(hashBuildMs); + diagnostics.entryCount = entries.length; + } + + return entries; } function computeVectorStats(graph, desiredEntries) { @@ -547,26 +580,54 @@ export async function syncGraphVectorIndex( return { insertedHashes: [], stats: { total: 0, indexed: 0, stale: 0, pending: 0 }, + timings: null, }; } throwIfAborted(signal); + const syncStartedAt = nowMs(); + const syncMode = isBackendVectorConfig(config) ? "backend" : "direct"; + const validation = validateVectorConfig(config); if (!validation.valid) { graph.vectorIndexState.lastWarning = validation.error; graph.vectorIndexState.dirty = true; - return { insertedHashes: [], stats: graph.vectorIndexState.lastStats }; + graph.vectorIndexState.lastTimings = { + mode: syncMode, + validationError: validation.error, + totalMs: roundMs(nowMs() - syncStartedAt), + updatedAt: Date.now(), + }; + return { + insertedHashes: [], + stats: graph.vectorIndexState.lastStats, + timings: graph.vectorIndexState.lastTimings, + }; } const state = graph.vectorIndexState; const collectionId = buildVectorCollectionId( chatId || graph?.historyState?.chatId, ); - const desiredEntries = buildDesiredVectorEntries(graph, config, range); + const desiredBuildDiagnostics = {}; + const desiredBuildStartedAt = nowMs(); + const desiredEntries = buildDesiredVectorEntries( + graph, + config, + range, + desiredBuildDiagnostics, + ); + const desiredBuildMs = nowMs() - desiredBuildStartedAt; const desiredByNodeId = new Map( desiredEntries.map((entry) => [entry.nodeId, entry]), ); const insertedHashes = []; + let backendPurgeMs = 0; + let backendDeleteMs = 0; + let backendInsertMs = 0; + let embedBatchMs = 0; + let deletedHashCount = 0; + let embeddingsRequested = 0; const hasConcreteRange = range && Number.isFinite(range.start) && Number.isFinite(range.end); const rangedNodeIds = new Set(desiredEntries.map((entry) => entry.nodeId)); @@ -581,9 +642,13 @@ export async function syncGraphVectorIndex( purge || state.dirty || scopeChanged || (force && !hasConcreteRange); if (fullReset) { + const purgeStartedAt = nowMs(); await purgeVectorCollection(collectionId, signal); + backendPurgeMs += nowMs() - purgeStartedAt; resetVectorMappings(graph, config, chatId); + const insertStartedAt = nowMs(); await insertVectorEntries(collectionId, config, desiredEntries, signal); + backendInsertMs += nowMs() - insertStartedAt; for (const entry of desiredEntries) { state.hashToNodeId[entry.hash] = entry.nodeId; state.nodeToHash[entry.nodeId] = entry.hash; @@ -623,8 +688,13 @@ export async function syncGraphVectorIndex( entriesToInsert.push(entry); } + deletedHashCount = hashesToDelete.length; + const deleteStartedAt = nowMs(); await deleteVectorHashes(collectionId, config, hashesToDelete, signal); + backendDeleteMs += nowMs() - deleteStartedAt; + const insertStartedAt = nowMs(); await insertVectorEntries(collectionId, config, entriesToInsert, signal); + backendInsertMs += nowMs() - insertStartedAt; for (const entry of entriesToInsert) { state.hashToNodeId[entry.hash] = entry.nodeId; @@ -679,11 +749,14 @@ export async function syncGraphVectorIndex( let directSyncHadFailures = false; if (entriesToEmbed.length > 0) { throwIfAborted(signal); + embeddingsRequested = entriesToEmbed.length; + const embedStartedAt = nowMs(); const embeddings = await embedBatch( entriesToEmbed.map((entry) => entry.text), config, { signal }, ); + embedBatchMs += nowMs() - embedStartedAt; for (let index = 0; index < entriesToEmbed.length; index++) { const entry = entriesToEmbed[index]; @@ -718,14 +791,34 @@ export async function syncGraphVectorIndex( state.lastWarning = ""; } state.lastSyncAt = Date.now(); + const statsBuildStartedAt = nowMs(); state.lastStats = computeVectorStats( graph, buildDesiredVectorEntries(graph, config), ); + const statsBuildMs = nowMs() - statsBuildStartedAt; + state.lastTimings = { + mode: syncMode, + desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length), + desiredBuildMs: roundMs(desiredBuildMs), + textBuildMs: Number(desiredBuildDiagnostics.textBuildMs || 0), + hashBuildMs: Number(desiredBuildDiagnostics.hashBuildMs || 0), + backendPurgeMs: roundMs(backendPurgeMs), + backendDeleteMs: roundMs(backendDeleteMs), + backendInsertMs: roundMs(backendInsertMs), + embedBatchMs: roundMs(embedBatchMs), + statsBuildMs: roundMs(statsBuildMs), + deletedHashes: Math.max(0, Math.floor(deletedHashCount)), + insertedEntries: insertedHashes.length, + embeddingsRequested: Math.max(0, Math.floor(embeddingsRequested)), + totalMs: roundMs(nowMs() - syncStartedAt), + updatedAt: Date.now(), + }; return { insertedHashes, stats: state.lastStats, + timings: state.lastTimings, }; } @@ -743,14 +836,52 @@ export async function findSimilarNodesByText( const candidateNodes = Array.isArray(candidates) ? candidates : getEligibleVectorNodes(graph); + const searchStartedAt = nowMs(); + const mode = isDirectVectorConfig(config) ? "direct" : "backend"; + const recordSearchTimings = (patch = {}) => { + const state = graph?.vectorIndexState; + if (!state || typeof state !== "object" || Array.isArray(state)) return; + state.lastSearchTimings = { + ...(state.lastSearchTimings && + typeof state.lastSearchTimings === "object" && + !Array.isArray(state.lastSearchTimings) + ? state.lastSearchTimings + : {}), + mode, + queryLength: String(text || "").length, + candidateCount: candidateNodes.length, + topK: Math.max(1, Math.floor(Number(topK) || 1)), + ...patch, + totalMs: roundMs(nowMs() - searchStartedAt), + updatedAt: Date.now(), + }; + }; - if (candidateNodes.length === 0) return []; + if (candidateNodes.length === 0) { + recordSearchTimings({ + success: true, + reason: "no-candidates", + resultCount: 0, + }); + return []; + } if (isDirectVectorConfig(config)) { + const queryEmbedStartedAt = nowMs(); const queryVec = await embedText(text, config, { signal }); - if (!queryVec) return []; + const queryEmbedMs = nowMs() - queryEmbedStartedAt; + if (!queryVec) { + recordSearchTimings({ + success: false, + reason: "direct-query-embed-empty", + queryEmbedMs: roundMs(queryEmbedMs), + resultCount: 0, + }); + return []; + } - return searchSimilar( + const localSearchStartedAt = nowMs(); + const results = searchSimilar( queryVec, candidateNodes .filter( @@ -762,12 +893,29 @@ export async function findSimilarNodesByText( })), topK, ); + recordSearchTimings({ + success: true, + reason: "ok", + queryEmbedMs: roundMs(queryEmbedMs), + searchMs: roundMs(nowMs() - localSearchStartedAt), + resultCount: results.length, + }); + return results; } const validation = validateVectorConfig(config); - if (!validation.valid) return []; + if (!validation.valid) { + recordSearchTimings({ + success: false, + reason: "vector-config-invalid", + error: validation.error, + resultCount: 0, + }); + return []; + } try { + const requestStartedAt = nowMs(); const response = await fetchWithTimeout( "/api/vector/query", { @@ -784,6 +932,7 @@ export async function findSimilarNodesByText( }, getConfiguredTimeoutMs(config), ); + const requestMs = nowMs() - requestStartedAt; if (!response.ok) { const errorText = await response.text().catch(() => response.statusText); @@ -795,23 +944,47 @@ export async function findSimilarNodesByText( "backend-query-failed", `后端向量查询失败（${message}），已标记待重建`, ); + recordSearchTimings({ + success: false, + reason: "backend-query-http-failed", + statusCode: Number(response.status || 0), + requestMs: roundMs(requestMs), + error: message, + resultCount: 0, + }); return []; } + const parseStartedAt = nowMs(); const data = await response.json().catch(() => ({ hashes: [] })); + const parseMs = nowMs() - parseStartedAt; const hashes = Array.isArray(data?.hashes) ? data.hashes : []; const nodeIdByHash = graph.vectorIndexState?.hashToNodeId || {}; const allowedIds = new Set(candidateNodes.map((node) => node.id)); - return hashes + const results = hashes .map((hash, index) => ({ nodeId: nodeIdByHash[hash], score: Math.max(0.01, 1 - index / Math.max(1, hashes.length)), })) .filter((entry) => entry.nodeId && allowedIds.has(entry.nodeId)) .slice(0, topK); + recordSearchTimings({ + success: true, + reason: "ok", + requestMs: roundMs(requestMs), + parseMs: roundMs(parseMs), + resultCount: results.length, + hashCount: hashes.length, + }); + return results; } catch (error) { if (isAbortError(error)) { + recordSearchTimings({ + success: false, + reason: "aborted", + error: error?.message || String(error), + }); throw error; } const message = error?.message || String(error) || "后端向量查询失败"; @@ -821,6 +994,11 @@ export async function findSimilarNodesByText( "backend-query-failed", `后端向量查询失败（${message}），已标记待重建`, ); + recordSearchTimings({ + success: false, + reason: "backend-query-exception", + error: message, + }); throw error; } }