perf: reduce graph load memory and clone overhead

This commit is contained in:
Youzini-afk
2026-04-21 16:56:17 +08:00
parent 076618bc25
commit 4fd4786983
6 changed files with 238 additions and 105 deletions

View File

@@ -5423,11 +5423,8 @@ function applyShadowSnapshotToRuntime(
let shadowGraph = null; let shadowGraph = null;
try { try {
shadowGraph = cloneGraphForPersistence( shadowGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState( deserializeGraph(shadowSnapshot.serializedGraph),
deserializeGraph(shadowSnapshot.serializedGraph),
normalizedChatId,
),
normalizedChatId, normalizedChatId,
); );
} catch (error) { } catch (error) {
@@ -5970,6 +5967,12 @@ function isIndexedDbSnapshotMeaningful(snapshot = null) {
if (Array.isArray(snapshot.nodes) && snapshot.nodes.length > 0) return true; if (Array.isArray(snapshot.nodes) && snapshot.nodes.length > 0) return true;
if (Array.isArray(snapshot.edges) && snapshot.edges.length > 0) return true; if (Array.isArray(snapshot.edges) && snapshot.edges.length > 0) return true;
if (
snapshot.__stBmeTombstonesOmitted === true &&
Number(snapshot?.meta?.tombstoneCount || 0) > 0
) {
return true;
}
if (Array.isArray(snapshot.tombstones) && snapshot.tombstones.length > 0) if (Array.isArray(snapshot.tombstones) && snapshot.tombstones.length > 0)
return true; return true;
@@ -6017,6 +6020,7 @@ function isIndexedDbSnapshotMeaningful(snapshot = null) {
function cacheIndexedDbSnapshot(chatId, snapshot = null) { function cacheIndexedDbSnapshot(chatId, snapshot = null) {
const normalizedChatId = normalizeChatIdCandidate(chatId); const normalizedChatId = normalizeChatIdCandidate(chatId);
if (!normalizedChatId || !snapshot || typeof snapshot !== "object") return; if (!normalizedChatId || !snapshot || typeof snapshot !== "object") return;
if (snapshot.__stBmeTombstonesOmitted === true) return;
const snapshotStore = resolveSnapshotGraphStorePresentation(snapshot); const snapshotStore = resolveSnapshotGraphStorePresentation(snapshot);
bmeIndexedDbSnapshotCacheByChatId.set(normalizedChatId, { bmeIndexedDbSnapshotCacheByChatId.set(normalizedChatId, {
chatId: normalizedChatId, chatId: normalizedChatId,
@@ -6311,10 +6315,7 @@ async function readLocalCacheSnapshotForChat(chatId, source = "luker-sidecar-loa
const manager = ensureBmeChatManager(); const manager = ensureBmeChatManager();
if (!manager) return null; if (!manager) return null;
const db = await manager.getCurrentDb(normalizedChatId); const db = await manager.getCurrentDb(normalizedChatId);
const snapshot = await db.exportSnapshot(); const snapshot = await db.exportSnapshot({ includeTombstones: false });
if (snapshot) {
cacheIndexedDbSnapshot(normalizedChatId, snapshot);
}
return snapshot; return snapshot;
} catch (error) { } catch (error) {
console.warn("[ST-BME] 读取 Luker 本地缓存快照失败:", source, error); console.warn("[ST-BME] 读取 Luker 本地缓存快照失败:", source, error);
@@ -6394,11 +6395,8 @@ function buildSnapshotFromLukerSidecarState(
let snapshot = null; let snapshot = null;
if (sidecar?.checkpoint?.serializedGraph) { if (sidecar?.checkpoint?.serializedGraph) {
try { try {
const checkpointGraph = cloneGraphForPersistence( const checkpointGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState( deserializeGraph(sidecar.checkpoint.serializedGraph),
deserializeGraph(sidecar.checkpoint.serializedGraph),
normalizedChatId,
),
normalizedChatId, normalizedChatId,
); );
snapshot = buildSnapshotFromGraph(checkpointGraph, { snapshot = buildSnapshotFromGraph(checkpointGraph, {
@@ -6437,8 +6435,8 @@ function buildSnapshotFromLukerSidecarState(
headRevision: Number(normalizedManifest.headRevision || 0), headRevision: Number(normalizedManifest.headRevision || 0),
}; };
} else { } else {
const emptyGraph = cloneGraphForPersistence( const emptyGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState(createEmptyGraph(), normalizedChatId), createEmptyGraph(),
normalizedChatId, normalizedChatId,
); );
snapshot = buildSnapshotFromGraph(emptyGraph, { snapshot = buildSnapshotFromGraph(emptyGraph, {
@@ -7659,11 +7657,8 @@ async function loadGraphFromChatState(
let chatStateGraph = null; let chatStateGraph = null;
try { try {
chatStateGraph = cloneGraphForPersistence( chatStateGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState( deserializeGraph(payload.serializedGraph),
deserializeGraph(payload.serializedGraph),
normalizedChatId,
),
normalizedChatId, normalizedChatId,
); );
} catch (error) { } catch (error) {
@@ -7953,13 +7948,9 @@ async function readPersistedGraphForChatStateTarget(
}); });
if (sidecarResult?.ok && sidecarResult?.snapshot) { if (sidecarResult?.ok && sidecarResult?.snapshot) {
try { try {
return cloneGraphForPersistence( return buildGraphFromSnapshot(sidecarResult.snapshot, {
normalizeGraphRuntimeState( chatId: targetChatId,
buildGraphFromSnapshot(sidecarResult.snapshot), });
targetChatId,
),
targetChatId,
);
} catch (error) { } catch (error) {
console.warn("[ST-BME] 读取 Luker branch source snapshot 失败:", error); console.warn("[ST-BME] 读取 Luker branch source snapshot 失败:", error);
} }
@@ -7971,11 +7962,8 @@ async function readPersistedGraphForChatStateTarget(
}); });
if (legacySnapshot?.serializedGraph) { if (legacySnapshot?.serializedGraph) {
try { try {
return cloneGraphForPersistence( return normalizeGraphRuntimeState(
normalizeGraphRuntimeState( deserializeGraph(legacySnapshot.serializedGraph),
deserializeGraph(legacySnapshot.serializedGraph),
targetChatId,
),
targetChatId, targetChatId,
); );
} catch (error) { } catch (error) {
@@ -8175,10 +8163,13 @@ function readLegacyGraphFromChatMetadata(chatId, context = getContext()) {
typeof legacyGraph === "string" typeof legacyGraph === "string"
? deserializeGraph(legacyGraph) ? deserializeGraph(legacyGraph)
: legacyGraph; : legacyGraph;
return cloneGraphForPersistence( const normalizedLegacyGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState(hydratedLegacyGraph, normalizedChatId), hydratedLegacyGraph,
normalizedChatId, normalizedChatId,
); );
return typeof legacyGraph === "string"
? normalizedLegacyGraph
: cloneGraphForPersistence(normalizedLegacyGraph, normalizedChatId);
} catch (error) { } catch (error) {
console.warn("[ST-BME] 读取 legacy chat_metadata 图谱失败:", error); console.warn("[ST-BME] 读取 legacy chat_metadata 图谱失败:", error);
return null; return null;
@@ -9218,10 +9209,7 @@ function applyIndexedDbSnapshotToRuntime(
attemptIndex, attemptIndex,
}; };
} }
currentGraph = cloneGraphForPersistence( currentGraph = graphFromSnapshot;
normalizeGraphRuntimeState(graphFromSnapshot, normalizedChatId),
normalizedChatId,
);
stampGraphPersistenceMeta(currentGraph, { stampGraphPersistenceMeta(currentGraph, {
revision, revision,
reason: `${reasonPrefix}:${String(source || reasonPrefix)}`, reason: `${reasonPrefix}:${String(source || reasonPrefix)}`,
@@ -9485,7 +9473,7 @@ async function loadGraphFromIndexedDb(
identityRecoveryResult?.snapshot || identityRecoveryResult?.snapshot ||
localStoreMigrationResult?.snapshot || localStoreMigrationResult?.snapshot ||
migrationResult?.snapshot || migrationResult?.snapshot ||
(await db.exportSnapshot()); (await db.exportSnapshot({ includeTombstones: false }));
const shadowSnapshot = resolveCompatibleGraphShadowSnapshot( const shadowSnapshot = resolveCompatibleGraphShadowSnapshot(
resolveCurrentChatIdentity(getContext()), resolveCurrentChatIdentity(getContext()),
); );
@@ -10802,11 +10790,8 @@ function resolvePendingPersistGraphSource(chatId = "") {
shadowSnapshot.serializedGraph shadowSnapshot.serializedGraph
) { ) {
try { try {
const shadowGraph = cloneGraphForPersistence( const shadowGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState( deserializeGraph(shadowSnapshot.serializedGraph),
deserializeGraph(shadowSnapshot.serializedGraph),
normalizedChatId,
),
normalizedChatId, normalizedChatId,
); );
return { return {
@@ -11376,7 +11361,9 @@ async function persistExtractionBatchResult({
const context = getContext(); const context = getContext();
const persistGraph = const persistGraph =
graphSnapshot && typeof graphSnapshot === "object" graphSnapshot && typeof graphSnapshot === "object"
? cloneGraphSnapshot(graphSnapshot) ? graphSnapshot === currentGraph
? cloneGraphSnapshot(graphSnapshot)
: graphSnapshot
: currentGraph; : currentGraph;
if (!context || !persistGraph) { if (!context || !persistGraph) {
return buildGraphPersistResult({ return buildGraphPersistResult({
@@ -12847,10 +12834,14 @@ function loadGraphFromChat(options = {}) {
: undefined; : undefined;
if (savedData != null && savedData !== "") { if (savedData != null && savedData !== "") {
try { try {
const officialGraph = cloneGraphForPersistence( const hydratedOfficialGraph = normalizeGraphRuntimeState(
normalizeGraphRuntimeState(deserializeGraph(savedData), chatId), deserializeGraph(savedData),
chatId, chatId,
); );
const officialGraph =
typeof savedData === "string"
? hydratedOfficialGraph
: cloneGraphForPersistence(hydratedOfficialGraph, chatId);
const shadowDecision = shouldPreferShadowSnapshotOverOfficial( const shadowDecision = shouldPreferShadowSnapshotOverOfficial(
officialGraph, officialGraph,
shadowSnapshot, shadowSnapshot,

View File

@@ -297,7 +297,7 @@ function buildCommittedBatchPersistSnapshot(
: null, : null,
rangeEnd, rangeEnd,
]; ];
const afterSnapshot = runtime.cloneGraphSnapshot(graph); const afterSnapshot = graph;
const effectiveArtifacts = Array.isArray(postProcessArtifacts) const effectiveArtifacts = Array.isArray(postProcessArtifacts)
? [...postProcessArtifacts] ? [...postProcessArtifacts]
: []; : [];
@@ -357,17 +357,12 @@ function buildCommittedBatchPersistSnapshot(
persistGraphSnapshot: committedGraphSnapshot, persistGraphSnapshot: committedGraphSnapshot,
committedBatchJournalEntry, committedBatchJournalEntry,
afterSnapshot, afterSnapshot,
committedAfterSnapshot: runtime.cloneGraphSnapshot(committedGraphSnapshot), committedAfterSnapshot: committedGraphSnapshot,
postProcessArtifacts: effectiveArtifacts, postProcessArtifacts: effectiveArtifacts,
}; };
} }
function isPersistenceRevisionAccepted(runtime, persistence = null) { function isPersistenceRevisionAccepted(runtime, persistence = null) {
if (!persistence || persistence.accepted === true) return true;
const graphPersistenceState = runtime?.getGraphPersistenceState?.() || {};
if (graphPersistenceState.pendingPersist === true) {
return false;
}
const persistenceRevision = Number(persistence?.revision || 0); const persistenceRevision = Number(persistence?.revision || 0);
if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) { if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) {
return false; return false;
@@ -645,7 +640,7 @@ export async function executeExtractionBatchController(
processedRange: [startIdx, endIdx], processedRange: [startIdx, endIdx],
postProcessArtifacts: runtime.computePostProcessArtifacts( postProcessArtifacts: runtime.computePostProcessArtifacts(
beforeSnapshot, beforeSnapshot,
runtime.cloneGraphSnapshot(runtime.getCurrentGraph()), runtime.getCurrentGraph(),
effects?.postProcessArtifacts || [], effects?.postProcessArtifacts || [],
), ),
vectorHashesInserted: effects?.vectorHashesInserted || [], vectorHashesInserted: effects?.vectorHashesInserted || [],

View File

@@ -2013,6 +2013,14 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
normalizeChatId(options.chatId) || normalizeChatId(options.chatId) ||
normalizeChatId(snapshotMeta?.chatId) || normalizeChatId(snapshotMeta?.chatId) ||
normalizeChatId(snapshotState?.chatId); normalizeChatId(snapshotState?.chatId);
const snapshotHistoryState = toPlainData(
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY],
{},
);
const snapshotVectorState = toPlainData(
snapshotMeta?.[BME_RUNTIME_VECTOR_META_KEY],
{},
);
const runtimeGraph = createEmptyGraph(); const runtimeGraph = createEmptyGraph();
runtimeGraph.version = Number.isFinite( runtimeGraph.version = Number.isFinite(
@@ -2020,21 +2028,17 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
) )
? Number(snapshotMeta[BME_RUNTIME_GRAPH_VERSION_META_KEY]) ? Number(snapshotMeta[BME_RUNTIME_GRAPH_VERSION_META_KEY])
: runtimeGraph.version; : runtimeGraph.version;
runtimeGraph.nodes = toArray(snapshotView.nodes).map((node) => ({ runtimeGraph.nodes = toArray(toPlainData(snapshotView.nodes, []));
...(node || {}), runtimeGraph.edges = toArray(toPlainData(snapshotView.edges, []));
}));
runtimeGraph.edges = toArray(snapshotView.edges).map((edge) => ({
...(edge || {}),
}));
runtimeGraph.batchJournal = toArray( runtimeGraph.batchJournal = toArray(
snapshotMeta?.[BME_RUNTIME_BATCH_JOURNAL_META_KEY], toPlainData(snapshotMeta?.[BME_RUNTIME_BATCH_JOURNAL_META_KEY], []),
); );
runtimeGraph.lastRecallResult = toPlainData( runtimeGraph.lastRecallResult = toPlainData(
snapshotMeta?.[BME_RUNTIME_LAST_RECALL_META_KEY], snapshotMeta?.[BME_RUNTIME_LAST_RECALL_META_KEY],
null, null,
); );
runtimeGraph.maintenanceJournal = toArray( runtimeGraph.maintenanceJournal = toArray(
snapshotMeta?.[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY], toPlainData(snapshotMeta?.[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY], []),
); );
runtimeGraph.knowledgeState = toPlainData( runtimeGraph.knowledgeState = toPlainData(
snapshotMeta?.[BME_RUNTIME_KNOWLEDGE_STATE_META_KEY], snapshotMeta?.[BME_RUNTIME_KNOWLEDGE_STATE_META_KEY],
@@ -2073,22 +2077,21 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
runtimeGraph.historyState = { runtimeGraph.historyState = {
...(runtimeGraph.historyState || {}), ...(runtimeGraph.historyState || {}),
...(snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY] || {}), ...snapshotHistoryState,
lastProcessedAssistantFloor: Number.isFinite( lastProcessedAssistantFloor: Number.isFinite(
Number(snapshotState?.lastProcessedFloor), Number(snapshotState?.lastProcessedFloor),
) )
? Number(snapshotState.lastProcessedFloor) ? Number(snapshotState.lastProcessedFloor)
: Number( : Number(
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY] snapshotHistoryState?.lastProcessedAssistantFloor ??
?.lastProcessedAssistantFloor ?? META_DEFAULT_LAST_PROCESSED_FLOOR, META_DEFAULT_LAST_PROCESSED_FLOOR,
), ),
extractionCount: Number.isFinite( extractionCount: Number.isFinite(
Number(snapshotState?.extractionCount), Number(snapshotState?.extractionCount),
) )
? Number(snapshotState.extractionCount) ? Number(snapshotState.extractionCount)
: Number( : Number(
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY] snapshotHistoryState?.extractionCount ?? META_DEFAULT_EXTRACTION_COUNT,
?.extractionCount ?? META_DEFAULT_EXTRACTION_COUNT,
), ),
}; };
if ( if (
@@ -2146,10 +2149,10 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
} }
runtimeGraph.vectorIndexState = { runtimeGraph.vectorIndexState = {
...(runtimeGraph.vectorIndexState || {}), ...(runtimeGraph.vectorIndexState || {}),
...(snapshotMeta?.[BME_RUNTIME_VECTOR_META_KEY] || {}), ...snapshotVectorState,
collectionId: buildVectorCollectionId( collectionId: buildVectorCollectionId(
chatId || chatId ||
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY]?.chatId || snapshotHistoryState?.chatId ||
runtimeGraph.historyState?.chatId || runtimeGraph.historyState?.chatId ||
"", "",
), ),
@@ -3032,23 +3035,47 @@ export class BmeDatabase {
}; };
} }
async exportSnapshot() { async exportSnapshot(options = {}) {
const db = await this.open(); const db = await this.open();
const [nodes, edges, tombstones, metaRows] = await db.transaction( const includeTombstones =
"r", options && typeof options === "object"
db.table("nodes"), ? options.includeTombstones !== false
db.table("edges"), : options !== false;
db.table("tombstones"), let nodes = [];
db.table("meta"), let edges = [];
async () => let tombstones = [];
await Promise.all([ let metaRows = [];
db.table("nodes").toArray(),
db.table("edges").toArray(), if (includeTombstones) {
db.table("tombstones").toArray(), [nodes, edges, tombstones, metaRows] = await db.transaction(
db.table("meta").toArray(), "r",
]), db.table("nodes"),
); db.table("edges"),
db.table("tombstones"),
db.table("meta"),
async () =>
await Promise.all([
db.table("nodes").toArray(),
db.table("edges").toArray(),
db.table("tombstones").toArray(),
db.table("meta").toArray(),
]),
);
} else {
[nodes, edges, metaRows] = await db.transaction(
"r",
db.table("nodes"),
db.table("edges"),
db.table("meta"),
async () =>
await Promise.all([
db.table("nodes").toArray(),
db.table("edges").toArray(),
db.table("meta").toArray(),
]),
);
}
const metaMap = toMetaMap(metaRows); const metaMap = toMetaMap(metaRows);
const meta = { const meta = {
@@ -3058,7 +3085,10 @@ export class BmeDatabase {
revision: normalizeRevision(metaMap?.revision), revision: normalizeRevision(metaMap?.revision),
nodeCount: nodes.length, nodeCount: nodes.length,
edgeCount: edges.length, edgeCount: edges.length,
tombstoneCount: tombstones.length, tombstoneCount: normalizeNonNegativeInteger(
metaMap?.tombstoneCount,
tombstones.length,
),
}; };
const state = { const state = {
@@ -3070,13 +3100,19 @@ export class BmeDatabase {
: META_DEFAULT_EXTRACTION_COUNT, : META_DEFAULT_EXTRACTION_COUNT,
}; };
return { const snapshot = {
meta, meta,
nodes, nodes,
edges, edges,
tombstones, tombstones: includeTombstones ? tombstones : [],
state, state,
}; };
if (!includeTombstones) {
snapshot.__stBmeTombstonesOmitted = true;
}
return snapshot;
} }
async importSnapshot(snapshot, options = {}) { async importSnapshot(snapshot, options = {}) {

View File

@@ -1345,15 +1345,23 @@ class LegacyOpfsGraphStore {
}; };
} }
async exportSnapshot() { async exportSnapshot(options = {}) {
const snapshot = await this._loadSnapshot(); const includeTombstones =
return { options && typeof options === "object"
? options.includeTombstones !== false
: options !== false;
const snapshot = await this._loadSnapshot({ includeTombstones });
const exported = {
meta: toPlainData(snapshot.meta, {}), meta: toPlainData(snapshot.meta, {}),
nodes: toPlainData(snapshot.nodes, []), nodes: toPlainData(snapshot.nodes, []),
edges: toPlainData(snapshot.edges, []), edges: toPlainData(snapshot.edges, []),
tombstones: toPlainData(snapshot.tombstones, []), tombstones: includeTombstones ? toPlainData(snapshot.tombstones, []) : [],
state: toPlainData(snapshot.state, {}), state: toPlainData(snapshot.state, {}),
}; };
if (!includeTombstones) {
exported.__stBmeTombstonesOmitted = true;
}
return exported;
} }
async importSnapshot(snapshot, options = {}) { async importSnapshot(snapshot, options = {}) {
@@ -2643,15 +2651,23 @@ export class OpfsGraphStore {
}; };
} }
async exportSnapshot() { async exportSnapshot(options = {}) {
const snapshot = await this._loadSnapshot(); const includeTombstones =
return { options && typeof options === "object"
? options.includeTombstones !== false
: options !== false;
const snapshot = await this._loadSnapshot({ includeTombstones });
const exported = {
meta: toPlainData(snapshot.meta, {}), meta: toPlainData(snapshot.meta, {}),
nodes: toPlainData(snapshot.nodes, []), nodes: toPlainData(snapshot.nodes, []),
edges: toPlainData(snapshot.edges, []), edges: toPlainData(snapshot.edges, []),
tombstones: toPlainData(snapshot.tombstones, []), tombstones: includeTombstones ? toPlainData(snapshot.tombstones, []) : [],
state: toPlainData(snapshot.state, {}), state: toPlainData(snapshot.state, {}),
}; };
if (!includeTombstones) {
exported.__stBmeTombstonesOmitted = true;
}
return exported;
} }
async importSnapshot(snapshot, options = {}) { async importSnapshot(snapshot, options = {}) {
@@ -3263,7 +3279,7 @@ export class OpfsGraphStore {
return records; return records;
} }
async _loadBaseSnapshotFromV2(manifest = null) { async _loadBaseSnapshotFromV2(manifest = null, { includeTombstones = true } = {}) {
const normalizedManifest = manifest || (await this._ensureV2Ready()); const normalizedManifest = manifest || (await this._ensureV2Ready());
const runtimeMeta = await this._readRuntimeMetaEntries(); const runtimeMeta = await this._readRuntimeMetaEntries();
const nodes = []; const nodes = [];
@@ -3275,8 +3291,10 @@ export class OpfsGraphStore {
for (let index = 0; index < OPFS_V2_EDGE_BUCKET_COUNT; index += 1) { for (let index = 0; index < OPFS_V2_EDGE_BUCKET_COUNT; index += 1) {
edges.push(...(await this._readShardRecords("edges", index))); edges.push(...(await this._readShardRecords("edges", index)));
} }
for (let index = 0; index < OPFS_V2_TOMBSTONE_BUCKET_COUNT; index += 1) { if (includeTombstones) {
tombstones.push(...(await this._readShardRecords("tombstones", index))); for (let index = 0; index < OPFS_V2_TOMBSTONE_BUCKET_COUNT; index += 1) {
tombstones.push(...(await this._readShardRecords("tombstones", index)));
}
} }
const meta = { const meta = {
...createDefaultMetaValues(this.chatId), ...createDefaultMetaValues(this.chatId),
@@ -3311,7 +3329,7 @@ export class OpfsGraphStore {
return snapshot; return snapshot;
} }
async _loadSnapshot({ awaitWrites = true } = {}) { async _loadSnapshot({ awaitWrites = true, includeTombstones = true } = {}) {
if (awaitWrites) { if (awaitWrites) {
await this._awaitPendingWrites(); await this._awaitPendingWrites();
} }
@@ -3319,10 +3337,24 @@ export class OpfsGraphStore {
const headRevision = normalizeRevision( const headRevision = normalizeRevision(
manifest?.headRevision || manifest?.meta?.revision, manifest?.headRevision || manifest?.meta?.revision,
); );
if (this._snapshotCache && normalizeRevision(this._snapshotCache.meta?.revision) === headRevision) { if (
return this._snapshotCache; this._snapshotCache &&
normalizeRevision(this._snapshotCache.meta?.revision) === headRevision
) {
if (includeTombstones) {
return this._snapshotCache;
}
return {
meta: this._snapshotCache.meta,
state: this._snapshotCache.state,
nodes: this._snapshotCache.nodes,
edges: this._snapshotCache.edges,
tombstones: [],
};
} }
const snapshot = await this._loadBaseSnapshotFromV2(manifest); const snapshot = await this._loadBaseSnapshotFromV2(manifest, {
includeTombstones,
});
const walRecords = await this._readWalRecords(manifest); const walRecords = await this._readWalRecords(manifest);
for (const walRecord of walRecords) { for (const walRecord of walRecords) {
const nextSnapshot = applyOpfsV2DeltaToSnapshot(snapshot, walRecord.delta, walRecord.committedAt); const nextSnapshot = applyOpfsV2DeltaToSnapshot(snapshot, walRecord.delta, walRecord.committedAt);
@@ -3355,7 +3387,11 @@ export class OpfsGraphStore {
snapshot.state = normalizeSnapshotState(snapshot); snapshot.state = normalizeSnapshotState(snapshot);
snapshot.meta.lastProcessedFloor = snapshot.state.lastProcessedFloor; snapshot.meta.lastProcessedFloor = snapshot.state.lastProcessedFloor;
snapshot.meta.extractionCount = snapshot.state.extractionCount; snapshot.meta.extractionCount = snapshot.state.extractionCount;
this._snapshotCache = snapshot; if (includeTombstones) {
this._snapshotCache = snapshot;
return snapshot;
}
snapshot.tombstones = [];
return snapshot; return snapshot;
} }

View File

@@ -2,6 +2,9 @@ import assert from "node:assert/strict";
import { import {
BME_DB_SCHEMA_VERSION, BME_DB_SCHEMA_VERSION,
BME_RUNTIME_BATCH_JOURNAL_META_KEY,
BME_RUNTIME_HISTORY_META_KEY,
BME_RUNTIME_VECTOR_META_KEY,
BME_TOMBSTONE_RETENTION_MS, BME_TOMBSTONE_RETENTION_MS,
BmeDatabase, BmeDatabase,
buildBmeDbName, buildBmeDbName,
@@ -20,6 +23,7 @@ const chatIdsForCleanup = new Set([
"chat-manager-a", "chat-manager-a",
"chat-manager-b", "chat-manager-b",
"chat-manager-selector", "chat-manager-selector",
"chat-export-without-tombstones",
"chat-replace-reset", "chat-replace-reset",
]); ]);
@@ -196,6 +200,41 @@ async function testSnapshotExportImport() {
await db.close(); await db.close();
} }
async function testSnapshotExportWithoutTombstones() {
const db = new BmeDatabase("chat-export-without-tombstones", {
dexieClass: globalThis.Dexie,
});
await db.open();
await db.bulkUpsertNodes([
{
id: "node-light-snapshot",
type: "event",
sourceFloor: 3,
archived: false,
updatedAt: Date.now(),
},
]);
await db.bulkUpsertTombstones([
{
id: "tomb-light-snapshot",
kind: "node",
targetId: "node-deleted-light-snapshot",
deletedAt: Date.now(),
sourceDeviceId: "device-light-snapshot",
},
]);
const exported = await db.exportSnapshot({ includeTombstones: false });
assert.equal(exported.__stBmeTombstonesOmitted, true);
assert.ok(Array.isArray(exported.nodes));
assert.ok(Array.isArray(exported.edges));
assert.deepEqual(exported.tombstones, []);
assert.equal(exported.meta.tombstoneCount, 1);
await db.close();
}
async function testReplaceImportResetsStaleMeta() { async function testReplaceImportResetsStaleMeta() {
const chatId = "chat-replace-reset"; const chatId = "chat-replace-reset";
const db = new BmeDatabase(chatId, { dexieClass: globalThis.Dexie }); const db = new BmeDatabase(chatId, { dexieClass: globalThis.Dexie });
@@ -532,6 +571,9 @@ async function testGraphSnapshotConverters() {
id: "node-converter", id: "node-converter",
type: "event", type: "event",
sourceFloor: 9, sourceFloor: 9,
fields: {
title: "Converter Node",
},
updatedAt: Date.now(), updatedAt: Date.now(),
}); });
@@ -583,6 +625,32 @@ async function testGraphSnapshotConverters() {
assert.equal(rebuilt.regionState.activeRegion, "camp"); assert.equal(rebuilt.regionState.activeRegion, "camp");
assert.equal(rebuilt.timelineState.activeSegmentId, "segment-1"); assert.equal(rebuilt.timelineState.activeSegmentId, "segment-1");
assert.equal(rebuilt.summaryState.entries[0].id, "summary-1"); assert.equal(rebuilt.summaryState.entries[0].id, "summary-1");
rebuilt.nodes[0].fields.title = "Mutated Converter Node";
rebuilt.historyState.processedMessageHashes[1] = "mutated-hash";
rebuilt.vectorIndexState.hashToNodeId["vec-hash"] = "node-mutated";
rebuilt.batchJournal[0].processedRange[0] = 99;
assert.equal(
snapshot.nodes[0].fields.title,
"Converter Node",
"buildGraphFromSnapshot 不应复用 snapshot 节点的嵌套字段引用",
);
assert.equal(
snapshot.meta[BME_RUNTIME_HISTORY_META_KEY].processedMessageHashes[1],
"hash-1",
"buildGraphFromSnapshot 不应复用 snapshot historyState 的嵌套对象引用",
);
assert.equal(
snapshot.meta[BME_RUNTIME_VECTOR_META_KEY].hashToNodeId["vec-hash"],
"node-converter",
"buildGraphFromSnapshot 不应复用 snapshot vectorState 的嵌套对象引用",
);
assert.equal(
snapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY][0].processedRange[0],
8,
"buildGraphFromSnapshot 不应复用 snapshot batchJournal 的嵌套数组引用",
);
} }
async function main() { async function main() {
@@ -593,6 +661,7 @@ async function main() {
await testCrudAndMeta(); await testCrudAndMeta();
await testTransactionRollback(); await testTransactionRollback();
await testSnapshotExportImport(); await testSnapshotExportImport();
await testSnapshotExportWithoutTombstones();
await testReplaceImportResetsStaleMeta(); await testReplaceImportResetsStaleMeta();
await testRevisionMonotonicity(); await testRevisionMonotonicity();
await testTombstonePrune(); await testTombstonePrune();

View File

@@ -311,6 +311,12 @@ async function testImportExportPersistenceAndFileRotation() {
assert.equal(firstExportedSnapshot.state.extractionCount, 2); assert.equal(firstExportedSnapshot.state.extractionCount, 2);
assert.equal(firstExportedSnapshot.meta.storagePrimary, "opfs"); assert.equal(firstExportedSnapshot.meta.storagePrimary, "opfs");
assert.equal(firstExportedSnapshot.meta.storageMode, "opfs-primary"); assert.equal(firstExportedSnapshot.meta.storageMode, "opfs-primary");
const lightweightSnapshot = await store.exportSnapshot({
includeTombstones: false,
});
assert.equal(lightweightSnapshot.__stBmeTombstonesOmitted, true);
assert.deepEqual(lightweightSnapshot.tombstones, []);
assert.equal(lightweightSnapshot.meta.tombstoneCount, 1);
assert.deepEqual(firstExportedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], { assert.deepEqual(firstExportedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], {
pending: ["job-1"], pending: ["job-1"],
}); });