diff --git a/index.js b/index.js index 64de6eb..2e7eade 100644 --- a/index.js +++ b/index.js @@ -1070,11 +1070,119 @@ function normalizeGraphSyncState(value = "idle") { return "idle"; } +function normalizePersistenceHostProfile(value = "generic-st") { + const normalized = String(value || "generic-st") + .trim() + .toLowerCase(); + return normalized === "luker" ? "luker" : "generic-st"; +} + +function normalizePersistenceStorageTier(value = "none") { + const normalized = String(value || "none") + .trim() + .toLowerCase(); + if ( + [ + "indexeddb", + "opfs", + "chat-state", + "luker-chat-state", + "shadow", + "metadata-full", + "none", + ].includes(normalized) + ) { + return normalized; + } + return "none"; +} + +function resolveLocalStoreTierFromPresentation( + presentation = getPreferredGraphLocalStorePresentationSync(), +) { + const normalizedPresentation = + presentation && typeof presentation === "object" + ? presentation + : getPreferredGraphLocalStorePresentationSync(); + return normalizedPresentation.storagePrimary === "opfs" ? "opfs" : "indexeddb"; +} + +function hasValidLukerChatStateTarget(context = getContext()) { + if (!context || typeof context !== "object") { + return false; + } + const chatId = normalizeChatIdCandidate( + context.chatId || + (typeof context.getCurrentChatId === "function" + ? context.getCurrentChatId() + : getCurrentChatId(context)), + ); + if (!chatId) { + return false; + } + if (context.groupId != null && String(context.groupId || "").trim()) { + return true; + } + if (context.characterId != null && String(context.characterId || "").trim()) { + return true; + } + return true; +} + +function resolvePersistenceHostProfile(context = getContext()) { + const activeContext = + context && typeof context === "object" ? context : getContext(); + const hasLukerApi = + !!globalThis.Luker && typeof globalThis.Luker?.getContext === "function"; + if ( + hasLukerApi && + canUseGraphChatState(activeContext) && + hasValidLukerChatStateTarget(activeContext) + ) { + return "luker"; + } + return "generic-st"; +} + +function buildPersistenceEnvironment( + context = getContext(), + presentation = getPreferredGraphLocalStorePresentationSync(), +) { + const hostProfile = resolvePersistenceHostProfile(context); + const localStoreTier = resolveLocalStoreTierFromPresentation(presentation); + return { + hostProfile, + localStoreTier, + primaryStorageTier: + hostProfile === "luker" ? "luker-chat-state" : localStoreTier, + cacheStorageTier: hostProfile === "luker" ? localStoreTier : "none", + }; +} + +function isLukerPrimaryPersistenceHost(context = getContext()) { + return resolvePersistenceHostProfile(context) === "luker"; +} + function getGraphPersistenceLiveState() { const liveCommitMarker = cloneRuntimeDebugValue(graphPersistenceState.commitMarker, null) || readGraphCommitMarker(getContext()); const restoreLock = normalizeRestoreLockState(graphPersistenceState.restoreLock); + const persistenceEnvironment = buildPersistenceEnvironment( + getContext(), + getPreferredGraphLocalStorePresentationSync(), + ); + const hostProfile = normalizePersistenceHostProfile( + graphPersistenceState.hostProfile || persistenceEnvironment.hostProfile, + ); + const primaryStorageTier = normalizePersistenceStorageTier( + graphPersistenceState.primaryStorageTier || + persistenceEnvironment.primaryStorageTier, + ); + const cacheStorageTier = normalizePersistenceStorageTier( + graphPersistenceState.cacheStorageTier || + persistenceEnvironment.cacheStorageTier, + ); const snapshot = { loadState: graphPersistenceState.loadState, chatId: graphPersistenceState.chatId, @@ -1095,6 +1203,14 @@ function getGraphPersistenceLiveState() { pendingPersist: graphPersistenceState.pendingPersist, lastAcceptedRevision: Number(graphPersistenceState.lastAcceptedRevision || 0), acceptedStorageTier: String(graphPersistenceState.acceptedStorageTier || "none"), + hostProfile, + primaryStorageTier, + cacheStorageTier, + cacheMirrorState: String(graphPersistenceState.cacheMirrorState || "idle"), + persistDiagnosticTier: String( + graphPersistenceState.persistDiagnosticTier || "none", + ), + acceptedBy: String(graphPersistenceState.acceptedBy || "none"), lastRecoverableStorageTier: String( graphPersistenceState.lastRecoverableStorageTier || "none", ), @@ -1111,6 +1227,10 @@ function getGraphPersistenceLiveState() { updatedAt: graphPersistenceState.updatedAt, storagePrimary: graphPersistenceState.storagePrimary || "indexeddb", storageMode: graphPersistenceState.storageMode || "indexeddb", + opfsWriteLockState: cloneRuntimeDebugValue( + graphPersistenceState.opfsWriteLockState, + null, + ), dbReady: graphPersistenceState.dbReady ?? isGraphLoadStateDbReady(graphPersistenceState.loadState), @@ -1373,6 +1493,19 @@ function applyGraphLoadState( storagePrimary = graphPersistenceState.storagePrimary || "indexeddb", storageMode = graphPersistenceState.storageMode || BME_GRAPH_LOCAL_STORAGE_MODE_INDEXEDDB, + hostProfile = graphPersistenceState.hostProfile || resolvePersistenceHostProfile(), + primaryStorageTier = + graphPersistenceState.primaryStorageTier || + buildPersistenceEnvironment(getContext(), { + storagePrimary, + storageMode, + }).primaryStorageTier, + cacheStorageTier = + graphPersistenceState.cacheStorageTier || + buildPersistenceEnvironment(getContext(), { + storagePrimary, + storageMode, + }).cacheStorageTier, } = {}, ) { updateGraphPersistenceState({ @@ -1392,6 +1525,9 @@ function applyGraphLoadState( dbReady, storagePrimary, storageMode, + hostProfile, + primaryStorageTier, + cacheStorageTier, }); if (dbReady && isGraphLoadStateDbReady(loadState)) { @@ -4758,6 +4894,10 @@ async function syncIndexedDbMetaToPersistenceState( } const storePresentation = resolveDbGraphStorePresentation(db); + const persistenceEnvironment = buildPersistenceEnvironment( + getContext(), + storePresentation, + ); const [ revision, syncDirty, @@ -4793,6 +4933,9 @@ async function syncIndexedDbMetaToPersistenceState( ]); const patch = { + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, storagePrimary: storePresentation.storagePrimary, storageMode: storePresentation.storageMode, indexedDbRevision: normalizeIndexedDbRevision(revision), @@ -4807,6 +4950,10 @@ async function syncIndexedDbMetaToPersistenceState( lastBackupRollbackAt: Number(lastBackupRollbackAt) || 0, lastBackupFilename: String(lastBackupFilename || ""), lastSyncError: String(lastSyncError || ""), + opfsWriteLockState: + typeof db.getWriteLockSnapshot === "function" + ? cloneRuntimeDebugValue(db.getWriteLockSnapshot(), null) + : graphPersistenceState.opfsWriteLockState, }; updateGraphPersistenceState(patch); @@ -4831,6 +4978,19 @@ async function runBmeAutoSyncForChat(source = "unknown", chatId = "") { }; } + if (isLukerPrimaryPersistenceHost(getContext())) { + updateGraphPersistenceState({ + syncState: "idle", + lastSyncError: "", + }); + return { + synced: false, + skipped: true, + chatId: normalizedChatId, + reason: "luker-host-sync-disabled", + }; + } + updateGraphPersistenceState({ syncState: "syncing", lastSyncError: "", @@ -5129,6 +5289,14 @@ async function persistGraphToHostChatState( } const resolvedIdentity = resolveCurrentChatIdentity(context); + const persistenceEnvironment = buildPersistenceEnvironment( + context, + getPreferredGraphLocalStorePresentationSync(), + ); + const effectiveStorageTier = + storageTier === "chat-state" && persistenceEnvironment.hostProfile === "luker" + ? "luker-chat-state" + : storageTier; const nextIntegrity = getChatMetadataIntegrity(context) || normalizeChatIdCandidate(resolvedIdentity?.integrity) || @@ -5147,7 +5315,7 @@ async function persistGraphToHostChatState( { namespace: GRAPH_CHAT_STATE_NAMESPACE, revision, - storageTier, + storageTier: effectiveStorageTier, accepted, reason, chatId, @@ -5176,7 +5344,7 @@ async function persistGraphToHostChatState( accepted: false, reason: writeResult?.reason || "chat-state-save-failed", revision, - storageTier, + storageTier: effectiveStorageTier, error: writeResult?.error || null, }; } @@ -5184,6 +5352,15 @@ async function persistGraphToHostChatState( cacheChatStateSnapshot(chatId, writeResult.snapshot); rememberResolvedGraphIdentityAlias(context, chatId); updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: + mode === "mirror" && persistenceEnvironment.hostProfile === "luker" + ? writeResult?.updated === false + ? "saved" + : "saved" + : graphPersistenceState.cacheMirrorState, metadataIntegrity: String(nextIntegrity || graphPersistenceState.metadataIntegrity || ""), lastPersistReason: String(reason || ""), lastPersistMode: @@ -5195,8 +5372,16 @@ async function persistGraphToHostChatState( Number(writeResult.snapshot.revision || revision || 0), ) : Number(graphPersistenceState.lastAcceptedRevision || 0), + acceptedStorageTier: + accepted === true + ? normalizePersistenceStorageTier(effectiveStorageTier) + : graphPersistenceState.acceptedStorageTier, + acceptedBy: + accepted === true + ? normalizePersistenceStorageTier(effectiveStorageTier) + : graphPersistenceState.acceptedBy, dualWriteLastResult: { - action: "save", + action: mode === "mirror" ? "cache-mirror" : "save", target: "chat-state", success: true, chatId, @@ -5217,7 +5402,7 @@ async function persistGraphToHostChatState( revision: Number(writeResult.snapshot.revision || revision || 0), reason: String(reason || "graph-chat-state"), saveMode: mode === "mirror" ? "chat-state-mirror" : "chat-state", - storageTier, + storageTier: effectiveStorageTier, snapshot: writeResult.snapshot, }; } @@ -5232,6 +5417,7 @@ async function loadGraphFromChatState( ) { const normalizedChatId = normalizeChatIdCandidate(chatId); const context = getContext(); + const shouldFallbackToLocalStore = isLukerPrimaryPersistenceHost(context); if (!normalizedChatId) { return { success: false, @@ -5256,6 +5442,14 @@ async function loadGraphFromChatState( namespace: GRAPH_CHAT_STATE_NAMESPACE, })) || readCachedChatStateSnapshot(normalizedChatId); if (!payload?.serializedGraph) { + if (shouldFallbackToLocalStore) { + scheduleIndexedDbGraphProbe(normalizedChatId, { + source: `${source}:luker-local-cache-fallback`, + attemptIndex, + allowOverride: true, + applyEmptyState: true, + }); + } return { success: false, loaded: false, @@ -5277,6 +5471,14 @@ async function loadGraphFromChatState( ); } catch (error) { console.warn("[ST-BME] 聊天侧车图谱反序列化失败:", error); + if (shouldFallbackToLocalStore) { + scheduleIndexedDbGraphProbe(normalizedChatId, { + source: `${source}:luker-local-cache-fallback`, + attemptIndex, + allowOverride: true, + applyEmptyState: true, + }); + } return { success: false, loaded: false, @@ -5288,6 +5490,14 @@ async function loadGraphFromChatState( } if (isGraphEffectivelyEmpty(chatStateGraph)) { + if (shouldFallbackToLocalStore) { + scheduleIndexedDbGraphProbe(normalizedChatId, { + source: `${source}:luker-local-cache-fallback`, + attemptIndex, + allowOverride: true, + applyEmptyState: true, + }); + } return { success: false, loaded: false, @@ -5403,9 +5613,12 @@ async function loadGraphFromChatState( source, attemptIndex, storagePrimary: "chat-state", - storageMode: "chat-state", - statusLabel: "聊天侧车", - reasonPrefix: "chat-state", + storageMode: + shouldFallbackToLocalStore === true ? "luker-chat-state" : "chat-state", + statusLabel: + shouldFallbackToLocalStore === true ? "Luker 侧车" : "聊天侧车", + reasonPrefix: + shouldFallbackToLocalStore === true ? "luker-chat-state" : "chat-state", }); if (commitMarkerDiagnostic?.reason && loadResult?.loaded) { updateGraphPersistenceState({ @@ -7533,6 +7746,11 @@ function buildGraphPersistResult({ accepted = false, recoverable = false, storageTier = "none", + acceptedBy = "none", + primaryTier = graphPersistenceState.primaryStorageTier, + cacheTier = graphPersistenceState.cacheStorageTier, + cacheMirrored = false, + diagnosticTier = graphPersistenceState.persistDiagnosticTier, reason = "", loadState = graphPersistenceState.loadState, revision = graphPersistenceState.revision, @@ -7545,6 +7763,11 @@ function buildGraphPersistResult({ accepted, recoverable, storageTier: String(storageTier || "none"), + acceptedBy: String(acceptedBy || "none"), + primaryTier: String(primaryTier || "none"), + cacheTier: String(cacheTier || "none"), + cacheMirrored: cacheMirrored === true, + diagnosticTier: String(diagnosticTier || "none"), reason: String(reason || ""), loadState, revision: Number.isFinite(revision) ? revision : 0, @@ -7642,7 +7865,12 @@ function buildBatchPersistenceRecordFromPersistResult(persistResult = null) { const recoverable = persistResult?.recoverable === true; let outcome = "failed"; - if (accepted && String(persistResult?.storageTier || "") === "indexeddb") { + if ( + accepted && + ["indexeddb", "opfs", "luker-chat-state"].includes( + String(persistResult?.storageTier || ""), + ) + ) { outcome = "saved"; } else if (accepted) { outcome = "fallback"; @@ -7670,6 +7898,198 @@ function buildBatchPersistenceRecordFromPersistResult(persistResult = null) { }; } +async function persistGraphToConfiguredDurableTier( + context, + graph, + { + chatId, + revision, + reason, + lastProcessedAssistantFloor = null, + } = {}, +) { + const preferredLocalStore = getPreferredGraphLocalStorePresentationSync(); + const persistenceEnvironment = buildPersistenceEnvironment( + context, + preferredLocalStore, + ); + const localStoreTier = resolveLocalStoreTierFromPresentation(preferredLocalStore); + + if ( + persistenceEnvironment.hostProfile === "luker" && + canUseHostGraphChatStatePersistence(context) + ) { + const chatStateResult = await persistGraphToHostChatState(context, { + graph, + revision, + reason, + storageTier: "luker-chat-state", + accepted: true, + lastProcessedAssistantFloor, + extractionCount, + mode: "primary", + }); + if (chatStateResult?.saved) { + const acceptedRevision = Number(chatStateResult.revision || revision); + persistGraphCommitMarker(context, { + reason, + revision: acceptedRevision, + storageTier: "luker-chat-state", + accepted: true, + lastProcessedAssistantFloor, + extractionCount, + immediate: true, + }); + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: + persistenceEnvironment.cacheStorageTier !== "none" ? "queued" : "idle", + revision: Math.max( + Number(graphPersistenceState.revision || 0), + acceptedRevision, + ), + pendingPersist: false, + persistMismatchReason: "", + lastAcceptedRevision: Math.max( + Number(graphPersistenceState.lastAcceptedRevision || 0), + acceptedRevision, + ), + acceptedStorageTier: "luker-chat-state", + acceptedBy: "luker-chat-state", + lastRecoverableStorageTier: "none", + lastPersistReason: reason, + lastPersistMode: String(chatStateResult.saveMode || "chat-state"), + queuedPersistRevision: 0, + queuedPersistChatId: "", + queuedPersistMode: "", + queuedPersistRotateIntegrity: false, + queuedPersistReason: "", + persistDiagnosticTier: "none", + }); + clearPendingGraphPersistRetry(); + if (persistenceEnvironment.cacheStorageTier !== "none") { + queueGraphPersistToIndexedDb(chatId, graph, { + revision: acceptedRevision, + reason: `${reason}:local-cache-mirror`, + persistRole: "cache-mirror", + scheduleCloudUpload: false, + }); + } + return buildGraphPersistResult({ + saved: true, + accepted: true, + reason, + revision: acceptedRevision, + saveMode: String(chatStateResult.saveMode || "chat-state"), + storageTier: "luker-chat-state", + acceptedBy: "luker-chat-state", + primaryTier: persistenceEnvironment.primaryStorageTier, + cacheTier: persistenceEnvironment.cacheStorageTier, + cacheMirrored: persistenceEnvironment.cacheStorageTier === "none", + }); + } + } + + const indexedDbResult = await saveGraphToIndexedDb(chatId, graph, { + revision, + reason, + }); + if (indexedDbResult?.saved) { + persistGraphCommitMarker(context, { + reason, + revision: indexedDbResult.revision || revision, + storageTier: indexedDbResult.storageTier || localStoreTier, + accepted: true, + lastProcessedAssistantFloor, + extractionCount, + immediate: true, + }); + clearPendingGraphPersistRetry(); + return buildGraphPersistResult({ + saved: true, + accepted: true, + reason, + revision: indexedDbResult.revision || revision, + saveMode: String(indexedDbResult.saveMode || "indexeddb-delta"), + storageTier: indexedDbResult.storageTier || localStoreTier, + acceptedBy: indexedDbResult.storageTier || localStoreTier, + primaryTier: persistenceEnvironment.primaryStorageTier, + cacheTier: persistenceEnvironment.cacheStorageTier, + }); + } + + if (canUseHostGraphChatStatePersistence(context)) { + const chatStateResult = await persistGraphToHostChatState(context, { + graph, + revision, + reason: `${reason}:chat-state-fallback`, + storageTier: "chat-state", + accepted: true, + lastProcessedAssistantFloor, + extractionCount, + mode: "primary", + }); + if (chatStateResult?.saved) { + const acceptedRevision = Number(chatStateResult.revision || revision); + persistGraphCommitMarker(context, { + reason: `${reason}:chat-state-fallback`, + revision: acceptedRevision, + storageTier: "chat-state", + accepted: true, + lastProcessedAssistantFloor, + extractionCount, + immediate: true, + }); + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + revision: Math.max( + Number(graphPersistenceState.revision || 0), + acceptedRevision, + ), + pendingPersist: false, + persistMismatchReason: "", + lastAcceptedRevision: Math.max( + Number(graphPersistenceState.lastAcceptedRevision || 0), + acceptedRevision, + ), + acceptedStorageTier: "chat-state", + acceptedBy: "chat-state", + lastRecoverableStorageTier: "none", + lastPersistReason: `${reason}:chat-state-fallback`, + lastPersistMode: String(chatStateResult.saveMode || "chat-state"), + queuedPersistRevision: 0, + queuedPersistChatId: "", + queuedPersistMode: "", + queuedPersistRotateIntegrity: false, + queuedPersistReason: "", + persistDiagnosticTier: "none", + }); + clearPendingGraphPersistRetry(); + queueGraphPersistToIndexedDb(chatId, graph, { + revision: acceptedRevision, + reason: `${reason}:chat-state-fallback:promote-indexeddb`, + }); + return buildGraphPersistResult({ + saved: true, + accepted: true, + reason: `${reason}:chat-state-fallback`, + revision: acceptedRevision, + saveMode: String(chatStateResult.saveMode || "chat-state"), + storageTier: "chat-state", + acceptedBy: "chat-state", + primaryTier: persistenceEnvironment.primaryStorageTier, + cacheTier: persistenceEnvironment.cacheStorageTier, + }); + } + } + + return null; +} + function resolvePendingPersistLastProcessedAssistantFloor() { const processedRange = Array.isArray( currentGraph?.historyState?.lastBatchStatus?.processedRange, @@ -8175,113 +8595,25 @@ async function retryPendingGraphPersist({ ); const lastProcessedAssistantFloor = resolvePendingPersistLastProcessedAssistantFloor(); - const indexedDbResult = await saveGraphToIndexedDb(activeChatId, pendingPersistGraph, { - revision: targetRevision, - reason, - }); - if (indexedDbResult?.saved) { - if (canUseHostGraphChatStatePersistence(context)) { - await persistGraphToHostChatState(context, { - graph: pendingPersistGraph, - revision: indexedDbResult.revision || targetRevision, - reason: `${reason}:chat-state-mirror`, - storageTier: "chat-state", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - mode: "mirror", - }); - } - clearPendingGraphPersistRetry(); - persistGraphCommitMarker(context, { + const acceptedPersistResult = await persistGraphToConfiguredDurableTier( + context, + pendingPersistGraph, + { + chatId: activeChatId, + revision: targetRevision, reason, - revision: indexedDbResult.revision || targetRevision, - storageTier: "indexeddb", - accepted: true, lastProcessedAssistantFloor, - extractionCount, - immediate: true, - }); - const persistResult = buildGraphPersistResult({ - saved: true, - accepted: true, - reason, - revision: indexedDbResult.revision || targetRevision, - saveMode: String(indexedDbResult.saveMode || "indexeddb-delta"), - storageTier: "indexeddb", - }); - applyAcceptedPendingPersistState(persistResult, { + }, + ); + if (acceptedPersistResult?.accepted) { + applyAcceptedPendingPersistState(acceptedPersistResult, { lastProcessedAssistantFloor, persistedGraph: pendingPersistGraph, }); - void maybeResumePendingAutoExtraction("pending-persist-resolved:indexeddb"); - return persistResult; - } - - if (canUseHostGraphChatStatePersistence(context)) { - const chatStateResult = await persistGraphToHostChatState(context, { - graph: pendingPersistGraph, - revision: targetRevision, - reason: `${reason}:chat-state-fallback`, - storageTier: "chat-state", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - mode: "primary", - }); - if (chatStateResult?.saved) { - clearPendingGraphPersistRetry(); - persistGraphCommitMarker(context, { - reason: `${reason}:chat-state-fallback`, - revision: chatStateResult.revision || targetRevision, - storageTier: "chat-state", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - immediate: true, - }); - updateGraphPersistenceState({ - revision: Math.max( - Number(graphPersistenceState.revision || 0), - Number(chatStateResult.revision || targetRevision), - ), - pendingPersist: false, - persistMismatchReason: "", - lastAcceptedRevision: Math.max( - Number(graphPersistenceState.lastAcceptedRevision || 0), - Number(chatStateResult.revision || targetRevision), - ), - acceptedStorageTier: "chat-state", - lastRecoverableStorageTier: "none", - lastPersistReason: `${reason}:chat-state-fallback`, - lastPersistMode: String(chatStateResult.saveMode || "chat-state"), - queuedPersistRevision: 0, - queuedPersistChatId: "", - queuedPersistMode: "", - queuedPersistRotateIntegrity: false, - queuedPersistReason: "", - storagePrimary: "chat-state", - storageMode: "chat-state", - }); - const persistResult = buildGraphPersistResult({ - saved: true, - accepted: true, - reason: `${reason}:chat-state-fallback`, - revision: Number(chatStateResult.revision || targetRevision), - saveMode: String(chatStateResult.saveMode || "chat-state"), - storageTier: "chat-state", - }); - applyAcceptedPendingPersistState(persistResult, { - lastProcessedAssistantFloor, - persistedGraph: pendingPersistGraph, - }); - queueGraphPersistToIndexedDb(activeChatId, pendingPersistGraph, { - revision: Number(chatStateResult.revision || targetRevision), - reason: `${reason}:chat-state-fallback:promote-indexeddb`, - }); - void maybeResumePendingAutoExtraction("pending-persist-resolved:chat-state"); - return persistResult; - } + void maybeResumePendingAutoExtraction( + `pending-persist-resolved:${acceptedPersistResult.acceptedBy || acceptedPersistResult.storageTier || "accepted"}`, + ); + return acceptedPersistResult; } let recoverableTier = "none"; @@ -8378,101 +8710,18 @@ async function persistExtractionBatchResult({ } const revision = allocateRequestedPersistRevision(0, persistGraph); - const indexedDbResult = await saveGraphToIndexedDb(chatId, persistGraph, { - revision, - reason, - }); - if (indexedDbResult?.saved) { - if (canUseHostGraphChatStatePersistence(context)) { - await persistGraphToHostChatState(context, { - graph: persistGraph, - revision: indexedDbResult.revision || revision, - reason: `${reason}:chat-state-mirror`, - storageTier: "chat-state", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - mode: "mirror", - }); - } - persistGraphCommitMarker(context, { - reason, - revision: indexedDbResult.revision || revision, - storageTier: "indexeddb", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - immediate: true, - }); - clearPendingGraphPersistRetry(); - return buildGraphPersistResult({ - saved: true, - accepted: true, - reason, - revision: indexedDbResult.revision || revision, - saveMode: String(indexedDbResult.saveMode || "indexeddb-delta"), - storageTier: "indexeddb", - }); - } - - if (canUseHostGraphChatStatePersistence(context)) { - const chatStateResult = await persistGraphToHostChatState(context, { - graph: persistGraph, + const acceptedPersistResult = await persistGraphToConfiguredDurableTier( + context, + persistGraph, + { + chatId, revision, - reason: `${reason}:chat-state-fallback`, - storageTier: "chat-state", - accepted: true, + reason, lastProcessedAssistantFloor, - extractionCount, - mode: "primary", - }); - if (chatStateResult?.saved) { - persistGraphCommitMarker(context, { - reason: `${reason}:chat-state-fallback`, - revision: chatStateResult.revision || revision, - storageTier: "chat-state", - accepted: true, - lastProcessedAssistantFloor, - extractionCount, - immediate: true, - }); - updateGraphPersistenceState({ - revision: Math.max( - Number(graphPersistenceState.revision || 0), - Number(chatStateResult.revision || revision), - ), - pendingPersist: false, - persistMismatchReason: "", - lastAcceptedRevision: Math.max( - Number(graphPersistenceState.lastAcceptedRevision || 0), - Number(chatStateResult.revision || revision), - ), - acceptedStorageTier: "chat-state", - lastRecoverableStorageTier: "none", - lastPersistReason: `${reason}:chat-state-fallback`, - lastPersistMode: String(chatStateResult.saveMode || "chat-state"), - queuedPersistRevision: 0, - queuedPersistChatId: "", - queuedPersistMode: "", - queuedPersistRotateIntegrity: false, - queuedPersistReason: "", - storagePrimary: "chat-state", - storageMode: "chat-state", - }); - clearPendingGraphPersistRetry(); - queueGraphPersistToIndexedDb(chatId, persistGraph, { - revision: Number(chatStateResult.revision || revision), - reason: `${reason}:chat-state-fallback:promote-indexeddb`, - }); - return buildGraphPersistResult({ - saved: true, - accepted: true, - reason: `${reason}:chat-state-fallback`, - revision: Number(chatStateResult.revision || revision), - saveMode: String(chatStateResult.saveMode || "chat-state"), - storageTier: "chat-state", - }); - } + }, + ); + if (acceptedPersistResult?.accepted) { + return acceptedPersistResult; } let recoverableTier = "none"; @@ -8712,6 +8961,49 @@ function syncGraphLoadFromLiveContext(options = {}) { }; } + const persistenceEnvironment = buildPersistenceEnvironment( + context, + getPreferredGraphLocalStorePresentationSync(), + ); + if ( + persistenceEnvironment.hostProfile === "luker" && + canUseHostGraphChatStatePersistence(context) + ) { + scheduleGraphChatStateProbe(chatId, { + source: `${source}:luker-chat-state-probe`, + attemptIndex, + allowOverride: true, + }); + applyGraphLoadState(GRAPH_LOAD_STATES.LOADING, { + chatId, + reason: `luker-chat-state-probe-pending:${String(source || "direct-load")}`, + attemptIndex, + dbReady: false, + writesBlocked: true, + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + }); + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + storagePrimary: getPreferredGraphLocalStorePresentationSync().storagePrimary, + storageMode: getPreferredGraphLocalStorePresentationSync().storageMode, + dbReady: false, + indexedDbLastError: "", + }); + refreshPanelLiveState(); + return { + success: false, + loaded: false, + loadState: GRAPH_LOAD_STATES.LOADING, + reason: "luker-chat-state-probe-pending", + chatId, + attemptIndex, + }; + } + if (canUseHostGraphChatStatePersistence(context)) { scheduleGraphChatStateProbe(chatId, { source: `${source}:chat-state-probe`, @@ -10146,7 +10438,12 @@ function loadGraphFromChat(options = {}) { async function saveGraphToIndexedDb( chatId, graph, - { revision = 0, reason = "graph-save" } = {}, + { + revision = 0, + reason = "graph-save", + persistRole = "primary", + scheduleCloudUpload: scheduleCloudUploadOption = undefined, + } = {}, ) { const normalizedChatId = normalizeChatIdCandidate(chatId); if (!normalizedChatId || !graph) { @@ -10158,6 +10455,9 @@ async function saveGraphToIndexedDb( }; } + const context = getContext(); + let db = null; + let localStore = getPreferredGraphLocalStorePresentationSync(); try { const manager = ensureBmeChatManager(); if (!manager) { @@ -10168,9 +10468,10 @@ async function saveGraphToIndexedDb( revision: normalizeIndexedDbRevision(revision), }; } - const db = await manager.getCurrentDb(normalizedChatId); + db = await manager.getCurrentDb(normalizedChatId); localStore = resolveDbGraphStorePresentation(db); - const currentIdentity = resolveCurrentChatIdentity(getContext()); + const persistenceEnvironment = buildPersistenceEnvironment(context, localStore); + const currentIdentity = resolveCurrentChatIdentity(context); const baseSnapshot = readCachedIndexedDbSnapshot(normalizedChatId, localStore) || (await db.exportSnapshot()); @@ -10190,6 +10491,12 @@ async function saveGraphToIndexedDb( }, }); const currentSettings = getSettings(); + const localStoreTier = resolveLocalStoreTierFromPresentation(localStore); + const shouldScheduleCloudUpload = + scheduleCloudUploadOption != null + ? scheduleCloudUploadOption === true + : persistenceEnvironment.hostProfile !== "luker" && + persistRole !== "cache-mirror"; const nativePersistBridgeMode = String( currentSettings.persistNativeDeltaBridgeMode || "json", ); @@ -10310,25 +10617,27 @@ async function saveGraphToIndexedDb( chatId: normalizedChatId, integrity: currentIdentity.integrity || - getChatMetadataIntegrity(getContext()) || + getChatMetadataIntegrity(context) || graphPersistenceState.metadataIntegrity, }); } - try { - scheduleUpload( - normalizedChatId, - buildBmeSyncRuntimeOptions({ - trigger: `graph-mutation:${String(reason || "graph-save")}`, - }), - ); - } catch (error) { - scheduleUploadWarning = - error?.message || String(error) || "schedule-upload-failed"; - console.warn( - `[ST-BME] ${localStore.statusLabel} 已写入，但同步上传调度失败:`, - error, - ); + if (shouldScheduleCloudUpload) { + try { + scheduleUpload( + normalizedChatId, + buildBmeSyncRuntimeOptions({ + trigger: `graph-mutation:${String(reason || "graph-save")}`, + }), + ); + } catch (error) { + scheduleUploadWarning = + error?.message || String(error) || "schedule-upload-failed"; + console.warn( + `[ST-BME] ${localStore.statusLabel} 已写入，但同步上传调度失败:`, + error, + ); + } } const persistDeltaDiagnostics = { @@ -10391,7 +10700,57 @@ async function saveGraphToIndexedDb( ) : ""; + const opfsWriteLockState = + typeof db?.getWriteLockSnapshot === "function" + ? cloneRuntimeDebugValue(db.getWriteLockSnapshot(), null) + : null; + + if (persistRole === "cache-mirror") { + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: "saved", + storagePrimary: localStore.storagePrimary, + storageMode: localStore.storageMode, + indexedDbRevision: snapshot.meta.revision, + indexedDbLastError: "", + lastSyncError: scheduleUploadWarning, + opfsWriteLockState, + dualWriteLastResult: { + action: "cache-mirror", + target: localStore.storagePrimary, + success: true, + chatId: normalizedChatId, + revision: snapshot.meta.revision, + reason: String(reason || "graph-save"), + warning: scheduleUploadWarning || "", + delta: cloneRuntimeDebugValue(commitResult?.delta, null), + at: Date.now(), + }, + persistDelta: persistDeltaDiagnostics, + }); + return { + saved: true, + accepted: false, + mirrored: true, + chatId: normalizedChatId, + revision: snapshot.meta.revision, + reason: String(reason || "graph-save"), + saveMode: `${localStore.reasonPrefix}-cache-mirror`, + storageTier: localStoreTier, + warning: scheduleUploadWarning || "", + delta: cloneRuntimeDebugValue(commitResult?.delta, null), + snapshot, + }; + } + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: + persistenceEnvironment.hostProfile === "luker" ? "idle" : "none", revision: snapshot.meta.revision, storagePrimary: localStore.storagePrimary, storageMode: localStore.storageMode, @@ -10405,7 +10764,7 @@ async function saveGraphToIndexedDb( queuedPersistReason: "", indexedDbRevision: snapshot.meta.revision, metadataIntegrity: - getChatMetadataIntegrity(getContext()) || + getChatMetadataIntegrity(context) || currentIdentity.integrity || graphPersistenceState.metadataIntegrity, indexedDbLastError: "", @@ -10418,8 +10777,11 @@ async function saveGraphToIndexedDb( Number(graphPersistenceState.lastAcceptedRevision || 0), snapshot.meta.revision, ), - acceptedStorageTier: localStore.storagePrimary, + acceptedStorageTier: localStoreTier, + acceptedBy: localStoreTier, lastRecoverableStorageTier: "none", + persistDiagnosticTier: "none", + opfsWriteLockState, dualWriteLastResult: { action: "save", target: localStore.storagePrimary, @@ -10469,10 +10831,12 @@ async function saveGraphToIndexedDb( return { saved: true, + accepted: true, chatId: normalizedChatId, revision: snapshot.meta.revision, reason: String(reason || "graph-save"), saveMode: `${localStore.reasonPrefix}-delta`, + storageTier: localStoreTier, warning: scheduleUploadWarning || "", delta: cloneRuntimeDebugValue(commitResult?.delta, null), snapshot, @@ -10487,12 +10851,25 @@ async function saveGraphToIndexedDb( error: error?.message || String(error), failedAt: Date.now(), }); + const persistenceEnvironment = buildPersistenceEnvironment(context, localStore); + const opfsWriteLockState = + typeof db?.getWriteLockSnapshot === "function" + ? cloneRuntimeDebugValue(db.getWriteLockSnapshot(), null) + : null; updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: + persistRole === "cache-mirror" + ? "error" + : graphPersistenceState.cacheMirrorState, storagePrimary: localStore.storagePrimary, storageMode: localStore.storageMode, indexedDbLastError: error?.message || String(error), + opfsWriteLockState, dualWriteLastResult: { - action: "save", + action: persistRole === "cache-mirror" ? "cache-mirror" : "save", target: localStore.storagePrimary, success: false, chatId: normalizedChatId, @@ -10506,7 +10883,10 @@ async function saveGraphToIndexedDb( saved: false, chatId: normalizedChatId, revision: normalizeIndexedDbRevision(revision), - reason: "indexeddb-write-failed", + reason: + persistRole === "cache-mirror" + ? "cache-mirror-write-failed" + : "indexeddb-write-failed", error, }; } @@ -10515,11 +10895,29 @@ async function saveGraphToIndexedDb( function queueGraphPersistToIndexedDb( chatId, graph, - { revision = 0, reason = "graph-save" } = {}, + { + revision = 0, + reason = "graph-save", + persistRole = "primary", + scheduleCloudUpload = undefined, + } = {}, ) { const normalizedChatId = normalizeChatIdCandidate(chatId); if (!normalizedChatId || !graph) return; + if (persistRole === "cache-mirror") { + const persistenceEnvironment = buildPersistenceEnvironment( + getContext(), + getPreferredGraphLocalStorePresentationSync(), + ); + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + cacheMirrorState: "queued", + }); + } + const normalizedRevision = normalizeIndexedDbRevision(revision); const latestQueuedRevision = normalizeIndexedDbRevision( bmeIndexedDbLatestQueuedRevisionByChatId.get(normalizedChatId), @@ -10553,6 +10951,8 @@ function queueGraphPersistToIndexedDb( return await saveGraphToIndexedDb(normalizedChatId, graphSnapshot, { revision: normalizedRevision, reason, + persistRole, + scheduleCloudUpload, }); }) .finally(() => { @@ -10598,6 +10998,10 @@ function saveGraphToChat(options = {}) { const revision = markMutation ? allocateRequestedPersistRevision(0, currentGraph) : resolvePersistRevisionFloor(0, currentGraph); + const persistenceEnvironment = buildPersistenceEnvironment( + context, + getPreferredGraphLocalStorePresentationSync(), + ); if (captureShadow) { maybeCaptureGraphShadowSnapshot(reason); @@ -10632,6 +11036,55 @@ function saveGraphToChat(options = {}) { } } + if (persistenceEnvironment.hostProfile === "luker") { + const persistGraph = cloneGraphForPersistence(currentGraph, chatId); + const lastProcessedAssistantFloor = Number.isFinite( + Number(persistGraph?.historyState?.lastProcessedAssistantFloor), + ) + ? Number(persistGraph.historyState.lastProcessedAssistantFloor) + : null; + scheduleBmeIndexedDbTask(async () => { + const persistResult = await persistGraphToConfiguredDurableTier( + context, + persistGraph, + { + chatId, + revision, + reason, + lastProcessedAssistantFloor, + }, + ); + if (!persistResult?.accepted) { + queueGraphPersist(reason, revision, { + immediate, + graph: persistGraph, + chatId, + captureShadow, + }); + } + refreshPanelLiveState(); + }); + updateGraphPersistenceState({ + hostProfile: persistenceEnvironment.hostProfile, + primaryStorageTier: persistenceEnvironment.primaryStorageTier, + cacheStorageTier: persistenceEnvironment.cacheStorageTier, + lastPersistReason: String(reason || "graph-save"), + lastPersistMode: "luker-chat-state-queued", + }); + return buildGraphPersistResult({ + saved: false, + queued: true, + blocked: false, + accepted: false, + reason: "luker-chat-state-queued", + revision, + saveMode: "luker-chat-state-queued", + storageTier: "luker-chat-state", + primaryTier: persistenceEnvironment.primaryStorageTier, + cacheTier: persistenceEnvironment.cacheStorageTier, + }); + } + if (!metadataFallbackEnabled) { const preferredLocalStore = getPreferredGraphLocalStorePresentationSync(); const saveMode = shouldQueueIndexedDbPersist diff --git a/sync/bme-opfs-store.js b/sync/bme-opfs-store.js index dbc66b6..78f969f 100644 --- a/sync/bme-opfs-store.js +++ b/sync/bme-opfs-store.js @@ -267,6 +267,30 @@ function buildSnapshotFilename(prefix, revision = 0, stampMs = Date.now()) { return `${String(prefix || "snapshot")}.${normalizeRevision(revision)}.${normalizeTimestamp(stampMs)}.json`; } +function escapeRegex(value = "") { + return String(value || "").replace(/[.*+?^${}()|[\]\\]/g, "\\$&"); +} + +function parseSnapshotFilenameCandidate(name = "", prefix = "") { + const normalizedName = String(name || "").trim(); + const normalizedPrefix = String(prefix || "").trim(); + if (!normalizedName || !normalizedPrefix) { + return null; + } + const matcher = new RegExp( + `^${escapeRegex(normalizedPrefix)}\\.(\\d+)\\.(\\d+)\\.json$`, + ); + const match = normalizedName.match(matcher); + if (!match) { + return null; + } + return { + filename: normalizedName, + revision: normalizeRevision(match[1]), + stampMs: normalizeTimestamp(match[2], 0), + }; +} + function isNotFoundError(error) { const name = String(error?.name || ""); const message = String(error?.message || ""); @@ -330,6 +354,37 @@ async function deleteFileIfExists(parentHandle, name) { } } +async function listDirectoryFileNames(parentHandle) { + if (!parentHandle) return []; + + if (parentHandle.files instanceof Map) { + return Array.from(parentHandle.files.keys()).map((name) => String(name || "")); + } + + const names = []; + if (typeof parentHandle.keys === "function") { + for await (const key of parentHandle.keys()) { + if (typeof key === "string" && key) { + names.push(key); + } + } + return names; + } + + if (typeof parentHandle.entries === "function") { + for await (const [name, handle] of parentHandle.entries()) { + if ( + typeof name === "string" && + name && + (!handle || handle.kind === "file" || typeof handle.getFile === "function") + ) { + names.push(name); + } + } + } + return names; +} + function normalizeSnapshotState(snapshot = {}) { const meta = snapshot?.meta && typeof snapshot.meta === "object" && !Array.isArray(snapshot.meta) @@ -396,19 +451,30 @@ function buildSnapshotFromStoredParts(manifest, corePayload = {}, auxPayload = { const nodes = sanitizeSnapshotRecordArray(corePayload?.nodes); const edges = sanitizeSnapshotRecordArray(corePayload?.edges); const tombstones = sanitizeSnapshotRecordArray(auxPayload?.tombstones); + const mergedMeta = { + ...baseMeta, + ...coreMeta, + ...auxMeta, + }; const state = normalizeSnapshotState({ - meta: { - ...baseMeta, - ...coreMeta, - ...auxMeta, + meta: mergedMeta, + state: { + ...(corePayload?.state && + typeof corePayload.state === "object" && + !Array.isArray(corePayload.state) + ? corePayload.state + : {}), + ...(Number.isFinite(Number(baseMeta?.lastProcessedFloor)) + ? { lastProcessedFloor: Number(baseMeta.lastProcessedFloor) } + : {}), + ...(Number.isFinite(Number(baseMeta?.extractionCount)) + ? { extractionCount: Number(baseMeta.extractionCount) } + : {}), }, - state: corePayload?.state, }); const meta = { ...createDefaultMetaValues(baseMeta.chatId || manifest?.chatId || ""), - ...toPlainData(baseMeta, {}), - ...toPlainData(coreMeta, {}), - ...toPlainData(auxMeta, {}), + ...toPlainData(mergedMeta, {}), chatId: normalizeChatId(baseMeta.chatId || manifest?.chatId || ""), schemaVersion: BME_DB_SCHEMA_VERSION, nodeCount: nodes.length, @@ -524,6 +590,14 @@ export class OpfsGraphStore { : getDefaultOpfsRootDirectory; this._chatDirectoryPromise = null; this._manifestCache = null; + this._writeChain = Promise.resolve(); + this._writeQueueDepth = 0; + this._writeLockState = { + active: false, + queueDepth: 0, + lastReason: "", + updatedAt: 0, + }; } async open() { @@ -534,11 +608,79 @@ export class OpfsGraphStore { async close() { this._chatDirectoryPromise = null; this._manifestCache = null; + this._writeChain = Promise.resolve(); + this._writeQueueDepth = 0; + this._writeLockState = { + active: false, + queueDepth: 0, + lastReason: "", + updatedAt: 0, + }; + } + + getWriteLockSnapshot() { + return toPlainData(this._writeLockState, this._writeLockState); + } + + async _awaitPendingWrites() { + try { + await this._writeChain; + } catch { + // swallow previous write failure for read barrier + } + } + + _setWriteLockState(patch = {}) { + this._writeLockState = { + ...this._writeLockState, + ...(patch || {}), + updatedAt: Date.now(), + }; + return this._writeLockState; + } + + async _runSerializedWrite(reason = "opfs-write", task = null) { + if (typeof task !== "function") { + throw new Error("OpfsGraphStore serialized write task is required"); + } + this._writeQueueDepth += 1; + this._setWriteLockState({ + active: true, + queueDepth: this._writeQueueDepth, + lastReason: String(reason || "opfs-write"), + }); + const runTask = async () => { + try { + return await task(); + } finally { + this._writeQueueDepth = Math.max(0, this._writeQueueDepth - 1); + this._setWriteLockState({ + active: this._writeQueueDepth > 0, + queueDepth: this._writeQueueDepth, + lastReason: String(reason || "opfs-write"), + }); + } + }; + const nextWrite = this._writeChain.catch(() => null).then(runTask); + this._writeChain = nextWrite.catch(() => null); + return await nextWrite; } async getMeta(key, fallbackValue = null) { const normalizedKey = normalizeRecordId(key); if (!normalizedKey) return fallbackValue; + if (OPFS_MANIFEST_META_KEYS.has(normalizedKey)) { + const manifest = await this._ensureManifest(); + const manifestMeta = + manifest?.meta && + typeof manifest.meta === "object" && + !Array.isArray(manifest.meta) + ? manifest.meta + : {}; + return Object.prototype.hasOwnProperty.call(manifestMeta, normalizedKey) + ? manifestMeta[normalizedKey] + : fallbackValue; + } const snapshot = await this._loadSnapshot(); return Object.prototype.hasOwnProperty.call(snapshot.meta, normalizedKey) ? snapshot.meta[normalizedKey] @@ -548,22 +690,12 @@ export class OpfsGraphStore { async setMeta(key, value) { const normalizedKey = normalizeRecordId(key); if (!normalizedKey) return null; - const snapshot = await this._loadSnapshot(); - snapshot.meta[normalizedKey] = toPlainData(value, value); - if (normalizedKey === "lastProcessedFloor") { - snapshot.state.lastProcessedFloor = Number.isFinite(Number(value)) - ? Number(value) - : META_DEFAULT_LAST_PROCESSED_FLOOR; - } - if (normalizedKey === "extractionCount") { - snapshot.state.extractionCount = Number.isFinite(Number(value)) - ? Number(value) - : META_DEFAULT_EXTRACTION_COUNT; - } - await this._writeResolvedSnapshot(snapshot); + await this.patchMeta({ + [normalizedKey]: value, + }); return { key: normalizedKey, - value: snapshot.meta[normalizedKey], + value: await this.getMeta(normalizedKey, null), updatedAt: Date.now(), }; } @@ -572,27 +704,68 @@ export class OpfsGraphStore { if (!record || typeof record !== "object" || Array.isArray(record)) { return {}; } - const snapshot = await this._loadSnapshot(); - const entries = []; - for (const [rawKey, value] of Object.entries(record)) { - const key = normalizeRecordId(rawKey); - if (!key) continue; - const normalizedValue = toPlainData(value, value); - snapshot.meta[key] = normalizedValue; - if (key === "lastProcessedFloor") { - snapshot.state.lastProcessedFloor = Number.isFinite(Number(normalizedValue)) - ? Number(normalizedValue) - : META_DEFAULT_LAST_PROCESSED_FLOOR; - } - if (key === "extractionCount") { - snapshot.state.extractionCount = Number.isFinite(Number(normalizedValue)) - ? Number(normalizedValue) - : META_DEFAULT_EXTRACTION_COUNT; - } - entries.push([key, normalizedValue]); + const entries = Object.entries(record) + .map(([rawKey, value]) => [normalizeRecordId(rawKey), toPlainData(value, value)]) + .filter(([key]) => Boolean(key)); + if (!entries.length) { + return {}; } - await this._writeResolvedSnapshot(snapshot); - return Object.fromEntries(entries); + + const allManifestOnly = entries.every(([key]) => + OPFS_MANIFEST_META_KEYS.has(key), + ); + if (allManifestOnly) { + return await this._runSerializedWrite("patchMeta:manifest", async () => { + const manifest = await this._ensureManifest({ awaitWrites: false }); + const nextMeta = { + ...createDefaultMetaValues(this.chatId), + ...(manifest?.meta && typeof manifest.meta === "object" && !Array.isArray(manifest.meta) + ? toPlainData(manifest.meta, {}) + : {}), + chatId: this.chatId, + storagePrimary: OPFS_STORE_KIND, + storageMode: this.storeMode, + }; + for (const [key, normalizedValue] of entries) { + nextMeta[key] = normalizedValue; + } + const nextManifest = { + ...(manifest || {}), + version: OPFS_MANIFEST_VERSION, + chatId: this.chatId, + storeKind: OPFS_STORE_KIND, + storeMode: this.storeMode, + activeCoreFilename: String(manifest?.activeCoreFilename || ""), + activeAuxFilename: String(manifest?.activeAuxFilename || ""), + meta: nextMeta, + }; + const chatDirectory = await this._getChatDirectory(); + await writeJsonFile(chatDirectory, OPFS_MANIFEST_FILENAME, nextManifest); + this._manifestCache = nextManifest; + return Object.fromEntries(entries); + }); + } + + return await this._runSerializedWrite("patchMeta:snapshot", async () => { + const snapshot = await this._loadSnapshot({ awaitWrites: false }); + const appliedEntries = []; + for (const [key, normalizedValue] of entries) { + snapshot.meta[key] = normalizedValue; + if (key === "lastProcessedFloor") { + snapshot.state.lastProcessedFloor = Number.isFinite(Number(normalizedValue)) + ? Number(normalizedValue) + : META_DEFAULT_LAST_PROCESSED_FLOOR; + } + if (key === "extractionCount") { + snapshot.state.extractionCount = Number.isFinite(Number(normalizedValue)) + ? Number(normalizedValue) + : META_DEFAULT_EXTRACTION_COUNT; + } + appliedEntries.push([key, normalizedValue]); + } + await this._writeResolvedSnapshot(snapshot); + return Object.fromEntries(appliedEntries); + }); } async getRevision() { @@ -608,13 +781,16 @@ export class OpfsGraphStore { } async commitDelta(delta = {}, options = {}) { - const nowMs = Date.now(); - const normalizedDelta = - delta && typeof delta === "object" && !Array.isArray(delta) ? delta : {}; - const currentSnapshot = await this._loadSnapshot(); - const nodeMap = new Map(); - const edgeMap = new Map(); - const tombstoneMap = new Map(); + return await this._runSerializedWrite( + String(options?.reason || "commitDelta"), + async () => { + const nowMs = Date.now(); + const normalizedDelta = + delta && typeof delta === "object" && !Array.isArray(delta) ? delta : {}; + const currentSnapshot = await this._loadSnapshot({ awaitWrites: false }); + const nodeMap = new Map(); + const edgeMap = new Map(); + const tombstoneMap = new Map(); for (const node of sanitizeSnapshotRecordArray(currentSnapshot.nodes)) { const id = normalizeRecordId(node.id); @@ -721,31 +897,33 @@ export class OpfsGraphStore { ? Number(runtimeMetaPatch.extractionCount) : currentSnapshot.state.extractionCount, }; - const nextSnapshot = { - meta: nextMeta, - state: nextState, - nodes: Array.from(nodeMap.values()), - edges: Array.from(edgeMap.values()), - tombstones: Array.from(tombstoneMap.values()), - }; - await this._writeResolvedSnapshot(nextSnapshot); + const nextSnapshot = { + meta: nextMeta, + state: nextState, + nodes: Array.from(nodeMap.values()), + edges: Array.from(edgeMap.values()), + tombstones: Array.from(tombstoneMap.values()), + }; + await this._writeResolvedSnapshot(nextSnapshot); - return { - revision: nextRevision, - lastModified: nowMs, - imported: { - nodes: nextSnapshot.nodes.length, - edges: nextSnapshot.edges.length, - tombstones: nextSnapshot.tombstones.length, + return { + revision: nextRevision, + lastModified: nowMs, + imported: { + nodes: nextSnapshot.nodes.length, + edges: nextSnapshot.edges.length, + tombstones: nextSnapshot.tombstones.length, + }, + delta: { + upsertNodes: upsertNodes.length, + upsertEdges: upsertEdges.length, + deleteNodeIds: deleteNodeIds.length, + deleteEdgeIds: deleteEdgeIds.length, + tombstones: tombstones.length, + }, + }; }, - delta: { - upsertNodes: upsertNodes.length, - upsertEdges: upsertEdges.length, - deleteNodeIds: deleteNodeIds.length, - deleteEdgeIds: deleteEdgeIds.length, - tombstones: tombstones.length, - }, - }; + ); } async bulkUpsertNodes(nodes = []) { @@ -990,140 +1168,216 @@ export class OpfsGraphStore { } async importSnapshot(snapshot, options = {}) { - const normalizedSnapshot = sanitizeSnapshot(snapshot); - const mode = normalizeMode(options.mode); - const shouldMarkSyncDirty = options.markSyncDirty !== false; - const nowMs = Date.now(); - const currentSnapshot = await this._loadSnapshot(); - const nextSnapshot = - mode === "replace" - ? normalizedSnapshot - : { - meta: { - ...currentSnapshot.meta, - ...normalizedSnapshot.meta, - }, - state: { - ...currentSnapshot.state, - ...normalizedSnapshot.state, - }, - nodes: mergeSnapshotRecords(currentSnapshot.nodes, normalizedSnapshot.nodes), - edges: mergeSnapshotRecords(currentSnapshot.edges, normalizedSnapshot.edges), - tombstones: mergeSnapshotRecords( - currentSnapshot.tombstones, - normalizedSnapshot.tombstones, - ), - }; - const currentRevision = normalizeRevision(currentSnapshot.meta?.revision); - const incomingRevision = normalizeRevision(normalizedSnapshot.meta?.revision); - const explicitRevision = normalizeRevision(options.revision); - const requestedRevision = Number.isFinite(Number(options.revision)) - ? explicitRevision - : options.preserveRevision - ? incomingRevision - : currentRevision + 1; - const nextRevision = Math.max(currentRevision + 1, requestedRevision); - nextSnapshot.meta = { - ...nextSnapshot.meta, - chatId: this.chatId, - revision: nextRevision, - lastModified: nowMs, - lastMutationReason: "importSnapshot", - syncDirty: shouldMarkSyncDirty, - syncDirtyReason: "importSnapshot", - storagePrimary: OPFS_STORE_KIND, - storageMode: this.storeMode, - }; - nextSnapshot.state = { - ...nextSnapshot.state, - lastProcessedFloor: Number.isFinite(Number(nextSnapshot?.state?.lastProcessedFloor)) - ? Number(nextSnapshot.state.lastProcessedFloor) - : Number.isFinite(Number(nextSnapshot?.meta?.lastProcessedFloor)) - ? Number(nextSnapshot.meta.lastProcessedFloor) - : META_DEFAULT_LAST_PROCESSED_FLOOR, - extractionCount: Number.isFinite(Number(nextSnapshot?.state?.extractionCount)) - ? Number(nextSnapshot.state.extractionCount) - : Number.isFinite(Number(nextSnapshot?.meta?.extractionCount)) - ? Number(nextSnapshot.meta.extractionCount) - : META_DEFAULT_EXTRACTION_COUNT, - }; - await this._writeResolvedSnapshot(nextSnapshot); + return await this._runSerializedWrite("importSnapshot", async () => { + const normalizedSnapshot = sanitizeSnapshot(snapshot); + const mode = normalizeMode(options.mode); + const shouldMarkSyncDirty = options.markSyncDirty !== false; + const nowMs = Date.now(); + const currentSnapshot = await this._loadSnapshot({ awaitWrites: false }); + const nextSnapshot = + mode === "replace" + ? normalizedSnapshot + : { + meta: { + ...currentSnapshot.meta, + ...normalizedSnapshot.meta, + }, + state: { + ...currentSnapshot.state, + ...normalizedSnapshot.state, + }, + nodes: mergeSnapshotRecords(currentSnapshot.nodes, normalizedSnapshot.nodes), + edges: mergeSnapshotRecords(currentSnapshot.edges, normalizedSnapshot.edges), + tombstones: mergeSnapshotRecords( + currentSnapshot.tombstones, + normalizedSnapshot.tombstones, + ), + }; + const currentRevision = normalizeRevision(currentSnapshot.meta?.revision); + const incomingRevision = normalizeRevision(normalizedSnapshot.meta?.revision); + const explicitRevision = normalizeRevision(options.revision); + const requestedRevision = Number.isFinite(Number(options.revision)) + ? explicitRevision + : options.preserveRevision + ? incomingRevision + : currentRevision + 1; + const nextRevision = Math.max(currentRevision + 1, requestedRevision); + nextSnapshot.meta = { + ...nextSnapshot.meta, + chatId: this.chatId, + revision: nextRevision, + lastModified: nowMs, + lastMutationReason: "importSnapshot", + syncDirty: shouldMarkSyncDirty, + syncDirtyReason: "importSnapshot", + storagePrimary: OPFS_STORE_KIND, + storageMode: this.storeMode, + }; + nextSnapshot.state = { + ...nextSnapshot.state, + lastProcessedFloor: Number.isFinite(Number(nextSnapshot?.state?.lastProcessedFloor)) + ? Number(nextSnapshot.state.lastProcessedFloor) + : Number.isFinite(Number(nextSnapshot?.meta?.lastProcessedFloor)) + ? Number(nextSnapshot.meta.lastProcessedFloor) + : META_DEFAULT_LAST_PROCESSED_FLOOR, + extractionCount: Number.isFinite(Number(nextSnapshot?.state?.extractionCount)) + ? Number(nextSnapshot.state.extractionCount) + : Number.isFinite(Number(nextSnapshot?.meta?.extractionCount)) + ? Number(nextSnapshot.meta.extractionCount) + : META_DEFAULT_EXTRACTION_COUNT, + }; + await this._writeResolvedSnapshot(nextSnapshot); - return { - mode, - revision: nextRevision, - imported: { - nodes: nextSnapshot.nodes.length, - edges: nextSnapshot.edges.length, - tombstones: nextSnapshot.tombstones.length, - }, - }; + return { + mode, + revision: nextRevision, + imported: { + nodes: nextSnapshot.nodes.length, + edges: nextSnapshot.edges.length, + tombstones: nextSnapshot.tombstones.length, + }, + }; + }); } async clearAll() { - const currentRevision = await this.getRevision(); - const nextRevision = currentRevision + 1; - await this._writeResolvedSnapshot({ - meta: { + return await this._runSerializedWrite("clearAll", async () => { + const currentRevision = normalizeRevision( + (await this._readManifest({ awaitWrites: false }))?.meta?.revision, + ); + const nextRevision = currentRevision + 1; + await this._writeResolvedSnapshot({ + meta: { + revision: nextRevision, + lastModified: Date.now(), + lastMutationReason: "clearAll", + syncDirty: true, + syncDirtyReason: "clearAll", + storagePrimary: OPFS_STORE_KIND, + storageMode: this.storeMode, + }, + state: { + lastProcessedFloor: META_DEFAULT_LAST_PROCESSED_FLOOR, + extractionCount: META_DEFAULT_EXTRACTION_COUNT, + }, + nodes: [], + edges: [], + tombstones: [], + }); + return { + cleared: true, revision: nextRevision, - lastModified: Date.now(), - lastMutationReason: "clearAll", - syncDirty: true, - syncDirtyReason: "clearAll", - storagePrimary: OPFS_STORE_KIND, - storageMode: this.storeMode, - }, - state: { - lastProcessedFloor: META_DEFAULT_LAST_PROCESSED_FLOOR, - extractionCount: META_DEFAULT_EXTRACTION_COUNT, - }, - nodes: [], - edges: [], - tombstones: [], + }; }); - return { - cleared: true, - revision: nextRevision, - }; } async pruneExpiredTombstones(nowMs = Date.now()) { - const normalizedNow = normalizeTimestamp(nowMs, Date.now()); - const cutoffMs = normalizedNow - BME_TOMBSTONE_RETENTION_MS; - const snapshot = await this._loadSnapshot(); - const nextTombstones = snapshot.tombstones.filter( - (item) => normalizeTimestamp(item?.deletedAt, 0) >= cutoffMs, + return await this._runSerializedWrite( + "pruneExpiredTombstones", + async () => { + const normalizedNow = normalizeTimestamp(nowMs, Date.now()); + const cutoffMs = normalizedNow - BME_TOMBSTONE_RETENTION_MS; + const snapshot = await this._loadSnapshot({ awaitWrites: false }); + const nextTombstones = snapshot.tombstones.filter( + (item) => normalizeTimestamp(item?.deletedAt, 0) >= cutoffMs, + ); + const removedCount = snapshot.tombstones.length - nextTombstones.length; + if (removedCount <= 0) { + return { + pruned: 0, + revision: normalizeRevision(snapshot.meta?.revision), + cutoffMs, + }; + } + const nextRevision = normalizeRevision(snapshot.meta?.revision) + 1; + await this._writeResolvedSnapshot({ + meta: { + ...snapshot.meta, + revision: nextRevision, + lastModified: normalizedNow, + lastMutationReason: "pruneExpiredTombstones", + syncDirty: true, + syncDirtyReason: "pruneExpiredTombstones", + storagePrimary: OPFS_STORE_KIND, + storageMode: this.storeMode, + }, + state: snapshot.state, + nodes: snapshot.nodes, + edges: snapshot.edges, + tombstones: nextTombstones, + }); + return { + pruned: removedCount, + revision: nextRevision, + cutoffMs, + }; + }, ); - const removedCount = snapshot.tombstones.length - nextTombstones.length; - if (removedCount <= 0) { - return { - pruned: 0, - revision: normalizeRevision(snapshot.meta?.revision), - cutoffMs, - }; + } + + async _recoverManifestFromDirectory(chatDirectory, manifest = null) { + const fileNames = await listDirectoryFileNames(chatDirectory); + const coreCandidates = fileNames + .map((name) => parseSnapshotFilenameCandidate(name, OPFS_CORE_FILENAME_PREFIX)) + .filter(Boolean); + const auxCandidates = fileNames + .map((name) => parseSnapshotFilenameCandidate(name, OPFS_AUX_FILENAME_PREFIX)) + .filter(Boolean); + if (!coreCandidates.length || !auxCandidates.length) { + return null; } - const nextRevision = normalizeRevision(snapshot.meta?.revision) + 1; - await this._writeResolvedSnapshot({ + + const coreByRevision = new Map(); + const auxByRevision = new Map(); + for (const candidate of coreCandidates) { + const current = coreByRevision.get(candidate.revision) || null; + if (!current || candidate.stampMs > current.stampMs) { + coreByRevision.set(candidate.revision, candidate); + } + } + for (const candidate of auxCandidates) { + const current = auxByRevision.get(candidate.revision) || null; + if (!current || candidate.stampMs > current.stampMs) { + auxByRevision.set(candidate.revision, candidate); + } + } + + const candidateRevisions = Array.from(coreByRevision.keys()) + .filter((revision) => auxByRevision.has(revision)) + .sort((left, right) => right - left); + if (!candidateRevisions.length) { + return null; + } + + const recoveredRevision = candidateRevisions[0]; + const recoveredCore = coreByRevision.get(recoveredRevision); + const recoveredAux = auxByRevision.get(recoveredRevision); + if (!recoveredCore || !recoveredAux) { + return null; + } + + const nextManifest = { + ...(manifest || {}), + version: OPFS_MANIFEST_VERSION, + chatId: this.chatId, + storeKind: OPFS_STORE_KIND, + storeMode: this.storeMode, + activeCoreFilename: recoveredCore.filename, + activeAuxFilename: recoveredAux.filename, meta: { - ...snapshot.meta, - revision: nextRevision, - lastModified: normalizedNow, - lastMutationReason: "pruneExpiredTombstones", - syncDirty: true, - syncDirtyReason: "pruneExpiredTombstones", + ...createDefaultMetaValues(this.chatId), + ...(manifest?.meta && typeof manifest.meta === "object" && !Array.isArray(manifest.meta) + ? toPlainData(manifest.meta, {}) + : {}), + revision: recoveredRevision, + chatId: this.chatId, storagePrimary: OPFS_STORE_KIND, storageMode: this.storeMode, }, - state: snapshot.state, - nodes: snapshot.nodes, - edges: snapshot.edges, - tombstones: nextTombstones, - }); - return { - pruned: removedCount, - revision: nextRevision, - cutoffMs, }; + await writeJsonFile(chatDirectory, OPFS_MANIFEST_FILENAME, nextManifest); + this._manifestCache = nextManifest; + return nextManifest; } async _getChatDirectory() { @@ -1150,8 +1404,8 @@ export class OpfsGraphStore { return await this._chatDirectoryPromise; } - async _ensureManifest() { - const existingManifest = await this._readManifest(); + async _ensureManifest(options = {}) { + const existingManifest = await this._readManifest(options); if (existingManifest) { return existingManifest; } @@ -1172,7 +1426,10 @@ export class OpfsGraphStore { return manifest; } - async _readManifest() { + async _readManifest({ awaitWrites = true } = {}) { + if (awaitWrites) { + await this._awaitPendingWrites(); + } if (this._manifestCache) { return this._manifestCache; } @@ -1209,21 +1466,71 @@ export class OpfsGraphStore { return manifest; } - async _loadSnapshot() { - const manifest = await this._ensureManifest(); + async _loadSnapshot({ awaitWrites = true } = {}) { + if (awaitWrites) { + await this._awaitPendingWrites(); + } + let manifest = await this._ensureManifest({ + awaitWrites: false, + }); const chatDirectory = await this._getChatDirectory(); - const corePayload = manifest.activeCoreFilename - ? await readJsonFile(chatDirectory, manifest.activeCoreFilename, {}) - : {}; - const auxPayload = manifest.activeAuxFilename - ? await readJsonFile(chatDirectory, manifest.activeAuxFilename, {}) - : {}; + const activeCoreRevision = parseSnapshotFilenameCandidate( + manifest?.activeCoreFilename, + OPFS_CORE_FILENAME_PREFIX, + )?.revision; + const activeAuxRevision = parseSnapshotFilenameCandidate( + manifest?.activeAuxFilename, + OPFS_AUX_FILENAME_PREFIX, + )?.revision; + let shouldRecoverManifest = + Boolean(manifest?.activeCoreFilename) && + Boolean(manifest?.activeAuxFilename) && + Number.isFinite(activeCoreRevision) && + Number.isFinite(activeAuxRevision) && + activeCoreRevision !== activeAuxRevision; + let corePayload = {}; + let auxPayload = {}; + try { + corePayload = manifest.activeCoreFilename + ? await readJsonFile(chatDirectory, manifest.activeCoreFilename, null) + : {}; + auxPayload = manifest.activeAuxFilename + ? await readJsonFile(chatDirectory, manifest.activeAuxFilename, null) + : {}; + if ( + (manifest.activeCoreFilename && !corePayload) || + (manifest.activeAuxFilename && !auxPayload) + ) { + shouldRecoverManifest = true; + } + } catch { + shouldRecoverManifest = true; + } + + if (shouldRecoverManifest) { + const recoveredManifest = await this._recoverManifestFromDirectory( + chatDirectory, + manifest, + ); + if (!recoveredManifest) { + throw new Error("opfs-manifest-snapshot-mismatch"); + } + manifest = recoveredManifest; + corePayload = manifest.activeCoreFilename + ? await readJsonFile(chatDirectory, manifest.activeCoreFilename, {}) + : {}; + auxPayload = manifest.activeAuxFilename + ? await readJsonFile(chatDirectory, manifest.activeAuxFilename, {}) + : {}; + } return buildSnapshotFromStoredParts(manifest, corePayload, auxPayload); } async _writeResolvedSnapshot(snapshot) { const chatDirectory = await this._getChatDirectory(); - const previousManifest = await this._ensureManifest(); + const previousManifest = await this._ensureManifest({ + awaitWrites: false, + }); const normalizedSnapshot = sanitizeSnapshot(snapshot); const state = normalizeSnapshotState(normalizedSnapshot); const writeStamp = Date.now(); diff --git a/tests/graph-persistence.mjs b/tests/graph-persistence.mjs index b8dd4d8..ce29ca5 100644 --- a/tests/graph-persistence.mjs +++ b/tests/graph-persistence.mjs @@ -1098,6 +1098,7 @@ result = { applyGraphLoadState, maybeFlushQueuedGraphPersist, retryPendingGraphPersist, + persistExtractionBatchResult, saveGraphToIndexedDb, cloneGraphForPersistence, assertRecoveryChatStillActive, @@ -3185,4 +3186,89 @@ result = { assert.equal(result?.commitMarker?.storageTier, "chat-state"); } +{ + const harness = await createGraphPersistenceHarness({ + chatId: "chat-generic-primary-no-mirror", + globalChatId: "chat-generic-primary-no-mirror", + characterId: "char-generic", + chatMetadata: { + integrity: "meta-generic-primary-no-mirror", + }, + }); + const graph = stampPersistedGraph( + createMeaningfulGraph("chat-generic-primary-no-mirror", "generic-primary"), + { + revision: 5, + integrity: "meta-generic-primary-no-mirror", + chatId: "chat-generic-primary-no-mirror", + reason: "generic-primary-seed", + }, + ); + harness.api.setCurrentGraph(graph); + + const result = await harness.api.persistExtractionBatchResult({ + reason: "generic-primary-persist", + lastProcessedAssistantFloor: 6, + }); + + assert.equal(result.accepted, true); + assert.equal(result.storageTier, "indexeddb"); + assert.equal( + harness.runtimeContext.__chatContext.__chatStateStore.size, + 0, + "generic ST 主写成功后不应再常驻 mirror 到 chat-state", + ); +} + +{ + const harness = await createGraphPersistenceHarness({ + chatId: "chat-luker-primary", + globalChatId: "chat-luker-primary", + characterId: "char-luker", + chatMetadata: { + integrity: "meta-luker-primary", + }, + }); + harness.runtimeContext.Luker = { + getContext() { + return harness.runtimeContext.__chatContext; + }, + }; + const graph = stampPersistedGraph( + createMeaningfulGraph("chat-luker-primary", "luker-primary"), + { + revision: 8, + integrity: "meta-luker-primary", + chatId: "chat-luker-primary", + reason: "luker-primary-seed", + }, + ); + harness.api.setCurrentGraph(graph); + + const result = await harness.api.persistExtractionBatchResult({ + reason: "luker-primary-persist", + lastProcessedAssistantFloor: 6, + }); + + assert.equal(result.accepted, true); + assert.equal(result.storageTier, "luker-chat-state"); + assert.equal(result.acceptedBy, "luker-chat-state"); + + const stored = await harness.runtimeContext.__chatContext.getChatState( + GRAPH_CHAT_STATE_NAMESPACE, + ); + assert.equal(stored?.revision, result.revision); + assert.equal(stored?.storageTier, "luker-chat-state"); + await new Promise((resolve) => setTimeout(resolve, 0)); + assert.equal( + Number(harness.api.getIndexedDbSnapshot()?.meta?.revision || 0) >= result.revision, + true, + "Luker 主存储成功后应异步补写本地缓存", + ); + assert.equal( + harness.api.getGraphPersistenceState().acceptedStorageTier, + "luker-chat-state", + ); +} + console.log("graph-persistence tests passed"); diff --git a/tests/helpers/memory-opfs.mjs b/tests/helpers/memory-opfs.mjs new file mode 100644 index 0000000..ea53cd9 --- /dev/null +++ b/tests/helpers/memory-opfs.mjs @@ -0,0 +1,107 @@ +export function createNotFoundError(message) { + const error = new Error(String(message || "Not found")); + error.name = "NotFoundError"; + return error; +} + +export class MemoryOpfsFileHandle { + constructor(parent, name) { + this.parent = parent; + this.name = String(name || ""); + } + + async getFile() { + const parent = this.parent; + const name = this.name; + return { + async text() { + return String(parent.files.get(name) ?? ""); + }, + }; + } + + async createWritable() { + const parent = this.parent; + const name = this.name; + let buffer = String(parent.files.get(name) ?? ""); + return { + async write(chunk) { + if (typeof chunk === "string") { + buffer = chunk; + return; + } + if (chunk == null) { + buffer = ""; + return; + } + buffer = String(chunk); + }, + async close() { + if (Number(parent.writeDelayMs) > 0) { + await new Promise((resolve) => + setTimeout(resolve, Number(parent.writeDelayMs)), + ); + } + parent.files.set(name, buffer); + }, + }; + } +} + +export class MemoryOpfsDirectoryHandle { + constructor(name = "", { writeDelayMs = 0 } = {}) { + this.name = String(name || ""); + this.directories = new Map(); + this.files = new Map(); + this.writeDelayMs = Number(writeDelayMs) || 0; + } + + async getDirectoryHandle(name, options = {}) { + const normalizedName = String(name || ""); + let directory = this.directories.get(normalizedName) || null; + if (!directory) { + if (!options.create) { + throw createNotFoundError(`Directory not found: ${normalizedName}`); + } + directory = new MemoryOpfsDirectoryHandle(normalizedName, { + writeDelayMs: this.writeDelayMs, + }); + this.directories.set(normalizedName, directory); + } + return directory; + } + + async getFileHandle(name, options = {}) { + const normalizedName = String(name || ""); + if (!this.files.has(normalizedName)) { + if (!options.create) { + throw createNotFoundError(`File not found: ${normalizedName}`); + } + this.files.set(normalizedName, ""); + } + return new MemoryOpfsFileHandle(this, normalizedName); + } + + async removeEntry(name, options = {}) { + const normalizedName = String(name || ""); + if (this.files.delete(normalizedName)) { + return; + } + const directory = this.directories.get(normalizedName) || null; + if (directory) { + const canDelete = + options.recursive === true || + (directory.files.size === 0 && directory.directories.size === 0); + if (!canDelete) { + throw new Error(`Directory not empty: ${normalizedName}`); + } + this.directories.delete(normalizedName); + return; + } + throw createNotFoundError(`Entry not found: ${normalizedName}`); + } +} + +export function createMemoryOpfsRoot(options = {}) { + return new MemoryOpfsDirectoryHandle("root", options); +} diff --git a/tests/index-esm-entry-smoke.mjs b/tests/index-esm-entry-smoke.mjs new file mode 100644 index 0000000..739ec37 --- /dev/null +++ b/tests/index-esm-entry-smoke.mjs @@ -0,0 +1,180 @@ +import assert from "node:assert/strict"; +import fs from "node:fs/promises"; +import path from "node:path"; +import { fileURLToPath, pathToFileURL } from "node:url"; + +const moduleDir = path.dirname(fileURLToPath(import.meta.url)); +const indexPath = path.resolve(moduleDir, "../index.js"); +const indexSource = await fs.readFile(indexPath, "utf8"); + +function extractSnippet(startMarker, endMarker) { + const start = indexSource.indexOf(startMarker); + const end = indexSource.indexOf(endMarker, start); + if (start < 0 || end < 0 || end <= start) { + throw new Error(`无法提取 index.js 片段: ${startMarker} -> ${endMarker}`); + } + return indexSource.slice(start, end).replace(/^export\s+/gm, ""); +} + +const saveGraphSnippet = extractSnippet( + "async function saveGraphToIndexedDb(", + "function queueGraphPersistToIndexedDb(", +); + +const tempModulePath = path.resolve( + moduleDir, + "../.tmp-index-esm-entry-smoke.mjs", +); + +await fs.writeFile( + tempModulePath, + ` +const GRAPH_LOAD_STATES = { SHADOW_RESTORED: "shadow-restored", LOADED: "loaded" }; +let currentGraph = null; +let graphPersistenceState = { + metadataIntegrity: "", + loadState: "loaded", + revision: 0, + lastPersistedRevision: 0, + lastAcceptedRevision: 0, + cacheMirrorState: "idle", + persistDiagnosticTier: "none", + hostProfile: "generic-st", + primaryStorageTier: "indexeddb", + cacheStorageTier: "none", + shadowSnapshotRevision: 0, + shadowSnapshotUpdatedAt: "", + shadowSnapshotReason: "", +}; +function normalizeChatIdCandidate(value = "") { return String(value ?? "").trim(); } +function normalizeIndexedDbRevision(value, fallbackValue = 0) { + const parsed = Number(value); + return Number.isFinite(parsed) && parsed >= 0 ? Math.floor(parsed) : Math.max(0, Number(fallbackValue) || 0); +} +function getContext() { return { chatId: "chat-esm", chatMetadata: {}, characterId: "char-esm" }; } +function getSettings() { + return { + persistNativeDeltaBridgeMode: "json", + persistUseNativeDelta: false, + graphNativeForceDisable: false, + nativeEngineFailOpen: true, + }; +} +function ensureBmeChatManager() { + return { + async getCurrentDb() { + return { + async exportSnapshot() { + return { meta: { revision: 0 }, nodes: [], edges: [], tombstones: [], state: { lastProcessedFloor: -1, extractionCount: 0 } }; + }, + async commitDelta(delta, options = {}) { + if (globalThis.__testCommitShouldThrow) { + throw new Error("commit-failed"); + } + return { + revision: Number(options.requestedRevision || 1), + lastModified: Date.now(), + delta, + }; + }, + }; + }, + }; +} +function getPreferredGraphLocalStorePresentationSync() { + return { storagePrimary: "indexeddb", storageMode: "indexeddb", statusLabel: "IndexedDB", reasonPrefix: "indexeddb" }; +} +function resolveDbGraphStorePresentation(db) { + return { storagePrimary: "indexeddb", storageMode: "indexeddb", statusLabel: "IndexedDB", reasonPrefix: "indexeddb" }; +} +function buildPersistenceEnvironment() { + return { hostProfile: "generic-st", primaryStorageTier: "indexeddb", cacheStorageTier: "none" }; +} +function resolveCurrentChatIdentity() { + return { integrity: "meta-esm", hostChatId: "host-esm" }; +} +function readCachedIndexedDbSnapshot() { return null; } +function resolvePersistRevisionFloor(revision = 0) { return Number(revision) || 1; } +function buildSnapshotFromGraph(graph, options = {}) { + return { + meta: { + revision: Number(options.revision || 1), + storagePrimary: "indexeddb", + storageMode: "indexeddb", + integrity: "meta-esm", + }, + nodes: [], + edges: [], + tombstones: [], + state: { lastProcessedFloor: -1, extractionCount: 0 }, + }; +} +function evaluatePersistNativeDeltaGate() { + return { + allowed: false, + reasons: [], + minSnapshotRecords: 0, + minStructuralDelta: 0, + minCombinedSerializedChars: 0, + beforeRecordCount: 0, + afterRecordCount: 0, + maxSnapshotRecords: 0, + structuralDelta: 0, + }; +} +function readPersistDeltaDiagnosticsNow() { return Date.now(); } +function updatePersistDeltaDiagnostics() {} +function buildPersistDelta() { + return { + upsertNodes: [], + upsertEdges: [], + deleteNodeIds: [], + deleteEdgeIds: [], + tombstones: [], + runtimeMetaPatch: {}, + }; +} +function cloneRuntimeDebugValue(value, fallback = null) { return value == null ? fallback : JSON.parse(JSON.stringify(value)); } +function buildBmeSyncRuntimeOptions() { return {}; } +function scheduleUpload() {} +function cacheIndexedDbSnapshot() {} +function stampGraphPersistenceMeta() {} +function getChatMetadataIntegrity() { return "meta-esm"; } +function clearPendingGraphPersistRetry() {} +function areChatIdsEquivalentForResolvedIdentity() { return false; } +function applyGraphLoadState() {} +function rememberResolvedGraphIdentityAlias() {} +function resolveLocalStoreTierFromPresentation() { return "indexeddb"; } +function updateGraphPersistenceState(patch = {}) { graphPersistenceState = { ...graphPersistenceState, ...(patch || {}) }; return graphPersistenceState; } +${saveGraphSnippet} +export { saveGraphToIndexedDb }; +`, + "utf8", +); + +try { + const smokeModule = await import( + `${pathToFileURL(tempModulePath).href}?t=${Date.now()}` + ); + const success = await smokeModule.saveGraphToIndexedDb( + "chat-esm", + { historyState: {} }, + { revision: 2, reason: "esm-success" }, + ); + assert.equal(success.saved, true); + assert.equal(success.accepted, true); + + globalThis.__testCommitShouldThrow = true; + const failed = await smokeModule.saveGraphToIndexedDb( + "chat-esm", + { historyState: {} }, + { revision: 3, reason: "esm-failure" }, + ); + assert.equal(failed.saved, false); + assert.equal(failed.reason, "indexeddb-write-failed"); +} finally { + delete globalThis.__testCommitShouldThrow; + await fs.unlink(tempModulePath).catch(() => {}); +} + +console.log("index-esm-entry-smoke tests passed"); diff --git a/tests/native-persist-delta-failopen.mjs b/tests/native-persist-delta-failopen.mjs new file mode 100644 index 0000000..e82ba83 --- /dev/null +++ b/tests/native-persist-delta-failopen.mjs @@ -0,0 +1,53 @@ +import assert from "node:assert/strict"; + +function moduleUrl(tag) { + return `../vendor/wasm/stbme_core.js?test=${Date.now()}-${tag}`; +} + +globalThis.__stBmeDisableWasmPackArtifacts = true; +delete globalThis.__stBmeLoadRustWasmLayout; + +const firstLoad = await import(moduleUrl("native-persist-first")); +let firstError = ""; +try { + await firstLoad.installNativePersistDeltaHook(); +} catch (error) { + firstError = error?.message || String(error); +} + +assert.match( + firstError, + /native module unavailable|native persist delta builder unavailable|global-loader|Rust\/WASM artifact is not initialized/i, +); + +globalThis.__stBmeLoadRustWasmLayout = async () => ({ + solve_layout() { + return { + ok: true, + positions: [], + diagnostics: { + solver: "mock-rust-wasm", + }, + }; + }, + build_persist_delta() { + return { + upsertNodes: [], + upsertEdges: [], + deleteNodeIds: [], + deleteEdgeIds: [], + tombstones: [], + runtimeMetaPatch: {}, + }; + }, +}); + +const retryStatus = await firstLoad.installNativePersistDeltaHook(); +assert.equal(retryStatus.loaded, true); +assert.equal(typeof globalThis.__stBmeNativeBuildPersistDelta, "function"); + +delete globalThis.__stBmeNativeBuildPersistDelta; +delete globalThis.__stBmeLoadRustWasmLayout; +delete globalThis.__stBmeDisableWasmPackArtifacts; + +console.log("native-persist-delta-failopen tests passed"); diff --git a/tests/opfs-meta-fast-path.mjs b/tests/opfs-meta-fast-path.mjs new file mode 100644 index 0000000..e1c74fc --- /dev/null +++ b/tests/opfs-meta-fast-path.mjs @@ -0,0 +1,72 @@ +import assert from "node:assert/strict"; + +import { + BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW, + OpfsGraphStore, +} from "../sync/bme-opfs-store.js"; +import { createMemoryOpfsRoot } from "./helpers/memory-opfs.mjs"; + +const rootDirectory = createMemoryOpfsRoot(); +const store = new OpfsGraphStore("chat-opfs-meta-fast-path", { + rootDirectoryFactory: async () => rootDirectory, + storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW, +}); + +await store.open(); +await store.importSnapshot( + { + meta: { + revision: 3, + lastBackupFilename: "before.json", + lastSyncUploadedAt: 10, + }, + state: { + lastProcessedFloor: 2, + extractionCount: 1, + }, + nodes: [ + { + id: "node-1", + type: "event", + fields: { title: "A" }, + archived: false, + updatedAt: 1, + }, + ], + edges: [], + tombstones: [], + }, + { + mode: "replace", + preserveRevision: true, + }, +); + +const originalLoadSnapshot = store._loadSnapshot.bind(store); +let loadSnapshotCalls = 0; +store._loadSnapshot = async (...args) => { + loadSnapshotCalls += 1; + return await originalLoadSnapshot(...args); +}; + +assert.equal(await store.getMeta("lastBackupFilename", ""), "before.json"); +assert.equal(await store.getRevision(), 3); +await store.patchMeta({ + lastBackupFilename: "after.json", + lastProcessedFloor: 9, + extractionCount: 4, +}); + +assert.equal( + loadSnapshotCalls, + 0, + "manifest-only meta fast path should not load full snapshot", +); + +const snapshot = await originalLoadSnapshot(); +assert.equal(snapshot.meta.lastBackupFilename, "after.json"); +assert.equal(snapshot.state.lastProcessedFloor, 9); +assert.equal(snapshot.state.extractionCount, 4); +assert.equal(snapshot.nodes.length, 1); + +console.log("opfs-meta-fast-path tests passed"); diff --git a/tests/opfs-write-serialization.mjs b/tests/opfs-write-serialization.mjs new file mode 100644 index 0000000..8d31647 --- /dev/null +++ b/tests/opfs-write-serialization.mjs @@ -0,0 +1,133 @@ +import assert from "node:assert/strict"; + +import { + BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY, + OpfsGraphStore, +} from "../sync/bme-opfs-store.js"; +import { createMemoryOpfsRoot } from "./helpers/memory-opfs.mjs"; + +async function testCommitDeltaAndPatchMetaSerialize() { + const rootDirectory = createMemoryOpfsRoot({ + writeDelayMs: 5, + }); + const store = new OpfsGraphStore("chat-opfs-serialize-meta", { + rootDirectoryFactory: async () => rootDirectory, + storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY, + }); + await store.open(); + + await store.importSnapshot( + { + meta: { + revision: 1, + lastBackupFilename: "", + }, + state: { + lastProcessedFloor: 0, + extractionCount: 0, + }, + nodes: [], + edges: [], + tombstones: [], + }, + { + mode: "replace", + preserveRevision: true, + }, + ); + + await Promise.all([ + store.commitDelta( + { + upsertNodes: [ + { + id: "node-1", + type: "event", + fields: { + title: "serialized", + }, + archived: false, + updatedAt: 100, + }, + ], + }, + { + reason: "serialized-node", + }, + ), + store.patchMeta({ + lastBackupFilename: "backup-a.json", + lastProcessedFloor: 7, + extractionCount: 3, + }), + ]); + + const snapshot = await store.exportSnapshot(); + assert.equal(snapshot.nodes.length, 1); + assert.equal(snapshot.nodes[0]?.id, "node-1"); + assert.equal(snapshot.meta.lastBackupFilename, "backup-a.json"); + assert.equal(snapshot.state.lastProcessedFloor, 7); + assert.equal(snapshot.state.extractionCount, 3); +} + +async function testImportSnapshotAndClearAllSerialize() { + const rootDirectory = createMemoryOpfsRoot({ + writeDelayMs: 5, + }); + const store = new OpfsGraphStore("chat-opfs-serialize-clear", { + rootDirectoryFactory: async () => rootDirectory, + storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY, + }); + await store.open(); + + await store.importSnapshot( + { + meta: { revision: 2 }, + state: { lastProcessedFloor: 5, extractionCount: 2 }, + nodes: [ + { + id: "seed-node", + type: "event", + fields: { title: "seed" }, + archived: false, + updatedAt: 1, + }, + ], + edges: [], + tombstones: [], + }, + { mode: "replace", preserveRevision: true }, + ); + + await Promise.all([ + store.clearAll(), + store.importSnapshot( + { + meta: { revision: 4 }, + state: { lastProcessedFloor: 9, extractionCount: 4 }, + nodes: [ + { + id: "after-clear-node", + type: "fact", + fields: { title: "after-clear" }, + archived: false, + updatedAt: 2, + }, + ], + edges: [], + tombstones: [], + }, + { mode: "replace", preserveRevision: true }, + ), + ]); + + const snapshot = await store.exportSnapshot(); + assert.equal(snapshot.nodes.length, 1); + assert.equal(snapshot.nodes[0]?.id, "after-clear-node"); + assert.equal(snapshot.state.lastProcessedFloor, 9); + assert.equal(snapshot.state.extractionCount, 4); +} + +await testCommitDeltaAndPatchMetaSerialize(); +await testImportSnapshotAndClearAllSerialize(); +console.log("opfs-write-serialization tests passed"); diff --git a/ui/panel.js b/ui/panel.js index c154267..d0ef2e3 100644 --- a/ui/panel.js +++ b/ui/panel.js @@ -2032,20 +2032,58 @@ function _refreshTaskPersistence() { const STORAGE_TIER_LABELS = { none: "无", metadata: "元数据", + "metadata-full": "完整 metadata", indexeddb: "IndexedDB", - chat: "聊天存档", + opfs: "OPFS", + "chat-state": "聊天侧车", + "luker-chat-state": "Luker 侧车主存储", + shadow: "影子快照", + }; + const HOST_PROFILE_LABELS = { + "generic-st": "通用 ST", + luker: "Luker", + }; + const CACHE_MIRROR_LABELS = { + idle: "空闲", + none: "无", + queued: "排队中", + saved: "已更新", + error: "失败", }; const loadStateLabel = LOAD_STATE_LABELS[ps.loadState] || ps.loadState || "未知"; - const storageTierLabel = STORAGE_TIER_LABELS[ps.acceptedStorageTier || ps.storageTier] || ps.acceptedStorageTier || ps.storageTier || "—"; + const acceptedTierLabel = + STORAGE_TIER_LABELS[ps.acceptedStorageTier || ps.storageTier] || + ps.acceptedStorageTier || + ps.storageTier || + "—"; + const primaryTierLabel = + STORAGE_TIER_LABELS[ps.primaryStorageTier] || ps.primaryStorageTier || "—"; + const cacheTierLabel = + STORAGE_TIER_LABELS[ps.cacheStorageTier] || ps.cacheStorageTier || "—"; + const hostProfileLabel = + HOST_PROFILE_LABELS[ps.hostProfile] || ps.hostProfile || "未知"; + const opfsLock = ps.opfsWriteLockState || null; + const opfsLockLabel = opfsLock + ? opfsLock.active + ? `活跃中 · queue ${Number(opfsLock.queueDepth || 0)}` + : `空闲 · queue ${Number(opfsLock.queueDepth || 0)}` + : "—"; const kvs = [ ["加载状态", loadStateLabel], - ["存储层级", storageTierLabel], + ["宿主档案", hostProfileLabel], + ["主 durable", primaryTierLabel], + ["当前 accepted", acceptedTierLabel], + ["accepted by", ps.acceptedBy || "—"], + ["本地缓存", cacheTierLabel], + ["缓存镜像", CACHE_MIRROR_LABELS[ps.cacheMirrorState] || ps.cacheMirrorState || "—"], ["版本号", ps.revision ?? "—"], - ["提交标记", ps.commitMarker ? "存在" : "无"], + ["提交标记", ps.commitMarker ? "存在（诊断锚点）" : "无"], + ["诊断层", STORAGE_TIER_LABELS[ps.persistDiagnosticTier] || ps.persistDiagnosticTier || "无"], ["阻塞原因", ps.blockedReason || ps.reason || "—"], ["影子快照", ps.shadowSnapshotUsed ? "已使用" : "未使用"], + ["OPFS 写锁", opfsLockLabel], ]; const kvHtml = kvs.map(([k, v]) => `
${_escHtml(k)}${_escHtml(String(v))}
`).join(""); @@ -2061,11 +2099,18 @@ function _refreshTaskPersistence() { const guidePairs = [ ["加载状态", "记忆图谱在当前聊天中的加载进度。\"已加载\" 表示正常运行。"], - ["存储层级", "当前持久化使用的最高存储介质。IndexedDB 最快，聊天存档最稳。"], + ["宿主档案", "当前运行环境。Luker 会把聊天侧车当主 durable 存储，其它宿主仍以本地存储为主。"], + ["主 durable", "当前宿主下真正负责 accepted 的主存储层。"], + ["当前 accepted", "最近一次已确认持久化最终落在哪一层。"], + ["accepted by", "本批最近一次 accepted 是由哪一层确认的。"], + ["本地缓存", "主存储之外的本地缓存层。Luker 下这里通常是 IndexedDB 或 OPFS。"], + ["缓存镜像", "本地缓存 mirror 的当前状态。失败不会自动等价为主持久化失败。"], ["版本号", "图谱修订号，每次写入操作自增。用于检测并发冲突。"], - ["提交标记", "聊天元数据中的标记，指示是否有更高版本存在于本地 IndexedDB。"], + ["提交标记", "聊天元数据中的诊断锚点，只用于对账与修复建议，不再单独代表 accepted。"], + ["诊断层", "最近一次仅作诊断/恢复用途的层级，例如影子快照或完整 metadata。"], ["阻塞原因", "如果加载被阻塞，这里显示具体原因。\"—\" 表示未阻塞。"], ["影子快照", "是否在启动时使用了上次会话留下的影子快照来加速加载。"], + ["OPFS 写锁", "OPFS 本地存储的串行写状态。活跃表示当前有写任务排队或执行中。"], ["图谱节点 / 边", "当前内存中图谱的节点和边数量。"], ["批次日志", "尚未合并到主快照的增量操作日志条目数。"], ["运行版本", "运行时图谱的内部版本号，和版本号联动。"], @@ -11227,10 +11272,17 @@ function _getGraphPersistenceSnapshot() { shadowSnapshotUsed: false, pendingPersist: false, lastAcceptedRevision: 0, + hostProfile: "generic-st", + primaryStorageTier: "indexeddb", + cacheStorageTier: "none", + cacheMirrorState: "idle", + acceptedBy: "none", + persistDiagnosticTier: "none", persistMismatchReason: "", commitMarker: null, chatId: "", storageMode: "indexeddb", + opfsWriteLockState: null, dbReady: false, syncState: "idle", syncDirty: false, @@ -11408,7 +11460,7 @@ function _getGraphLoadLabel(loadState = "") { case "empty-confirmed": return "当前聊天还没有图谱"; case "blocked": - return "当前聊天图谱未能完成 IndexedDB 确认，请稍后重试"; + return "当前聊天图谱未能完成正式持久化确认，请稍后重试"; case "loaded": return "聊天图谱已加载"; case "no-chat": diff --git a/ui/ui-status.js b/ui/ui-status.js index 8884f2f..fe0ef2f 100644 --- a/ui/ui-status.js +++ b/ui/ui-status.js @@ -51,6 +51,12 @@ export function createGraphPersistenceState() { pendingPersist: false, lastAcceptedRevision: 0, acceptedStorageTier: "none", + hostProfile: "generic-st", + primaryStorageTier: "indexeddb", + cacheStorageTier: "none", + cacheMirrorState: "idle", + persistDiagnosticTier: "none", + acceptedBy: "none", lastRecoverableStorageTier: "none", persistMismatchReason: "", commitMarker: null, @@ -63,6 +69,12 @@ export function createGraphPersistenceState() { }, storagePrimary: "indexeddb", storageMode: "indexeddb", + opfsWriteLockState: { + active: false, + queueDepth: 0, + lastReason: "", + updatedAt: 0, + }, dbReady: false, indexedDbRevision: 0, indexedDbLastError: "", diff --git a/vendor/wasm/stbme_core.js b/vendor/wasm/stbme_core.js index ada40be..a1cf6b3 100644 --- a/vendor/wasm/stbme_core.js +++ b/vendor/wasm/stbme_core.js @@ -3,6 +3,21 @@ let triedLoad = false; let loadError = null; let moduleSource = "none"; +function shouldRetryNativeLoad() { + return ( + !cachedNativeModule && + triedLoad && + typeof globalThis.__stBmeLoadRustWasmLayout === "function" + ); +} + +export function resetNativeModuleStatus() { + cachedNativeModule = null; + triedLoad = false; + loadError = null; + moduleSource = "none"; +} + async function resolveWasmModuleInput(wasmUrl) { if ( wasmUrl && @@ -69,12 +84,18 @@ async function loadFromWasmPackArtifacts() { }; } -async function loadNativeModule() { +async function loadNativeModule(options = {}) { if (cachedNativeModule) return cachedNativeModule; - if (triedLoad) { + if (triedLoad && !(options?.forceRetry === true || shouldRetryNativeLoad())) { throw loadError || new Error("stbme_core native module unavailable"); } + if (triedLoad && (options?.forceRetry === true || shouldRetryNativeLoad())) { + triedLoad = false; + loadError = null; + moduleSource = "none"; + } + triedLoad = true; let wasmPackError = null; @@ -148,7 +169,9 @@ export async function solveLayout(payload) { } export async function installNativePersistDeltaHook() { - const module = await loadNativeModule(); + const module = await loadNativeModule({ + forceRetry: shouldRetryNativeLoad(), + }); if ( !module || (typeof module.build_persist_delta_compact_hash !== "function" &&