mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
perf: reduce graph load memory and clone overhead
This commit is contained in:
87
index.js
87
index.js
@@ -5423,11 +5423,8 @@ function applyShadowSnapshotToRuntime(
|
|||||||
|
|
||||||
let shadowGraph = null;
|
let shadowGraph = null;
|
||||||
try {
|
try {
|
||||||
shadowGraph = cloneGraphForPersistence(
|
shadowGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(
|
deserializeGraph(shadowSnapshot.serializedGraph),
|
||||||
deserializeGraph(shadowSnapshot.serializedGraph),
|
|
||||||
normalizedChatId,
|
|
||||||
),
|
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -5970,6 +5967,12 @@ function isIndexedDbSnapshotMeaningful(snapshot = null) {
|
|||||||
|
|
||||||
if (Array.isArray(snapshot.nodes) && snapshot.nodes.length > 0) return true;
|
if (Array.isArray(snapshot.nodes) && snapshot.nodes.length > 0) return true;
|
||||||
if (Array.isArray(snapshot.edges) && snapshot.edges.length > 0) return true;
|
if (Array.isArray(snapshot.edges) && snapshot.edges.length > 0) return true;
|
||||||
|
if (
|
||||||
|
snapshot.__stBmeTombstonesOmitted === true &&
|
||||||
|
Number(snapshot?.meta?.tombstoneCount || 0) > 0
|
||||||
|
) {
|
||||||
|
return true;
|
||||||
|
}
|
||||||
if (Array.isArray(snapshot.tombstones) && snapshot.tombstones.length > 0)
|
if (Array.isArray(snapshot.tombstones) && snapshot.tombstones.length > 0)
|
||||||
return true;
|
return true;
|
||||||
|
|
||||||
@@ -6017,6 +6020,7 @@ function isIndexedDbSnapshotMeaningful(snapshot = null) {
|
|||||||
function cacheIndexedDbSnapshot(chatId, snapshot = null) {
|
function cacheIndexedDbSnapshot(chatId, snapshot = null) {
|
||||||
const normalizedChatId = normalizeChatIdCandidate(chatId);
|
const normalizedChatId = normalizeChatIdCandidate(chatId);
|
||||||
if (!normalizedChatId || !snapshot || typeof snapshot !== "object") return;
|
if (!normalizedChatId || !snapshot || typeof snapshot !== "object") return;
|
||||||
|
if (snapshot.__stBmeTombstonesOmitted === true) return;
|
||||||
const snapshotStore = resolveSnapshotGraphStorePresentation(snapshot);
|
const snapshotStore = resolveSnapshotGraphStorePresentation(snapshot);
|
||||||
bmeIndexedDbSnapshotCacheByChatId.set(normalizedChatId, {
|
bmeIndexedDbSnapshotCacheByChatId.set(normalizedChatId, {
|
||||||
chatId: normalizedChatId,
|
chatId: normalizedChatId,
|
||||||
@@ -6311,10 +6315,7 @@ async function readLocalCacheSnapshotForChat(chatId, source = "luker-sidecar-loa
|
|||||||
const manager = ensureBmeChatManager();
|
const manager = ensureBmeChatManager();
|
||||||
if (!manager) return null;
|
if (!manager) return null;
|
||||||
const db = await manager.getCurrentDb(normalizedChatId);
|
const db = await manager.getCurrentDb(normalizedChatId);
|
||||||
const snapshot = await db.exportSnapshot();
|
const snapshot = await db.exportSnapshot({ includeTombstones: false });
|
||||||
if (snapshot) {
|
|
||||||
cacheIndexedDbSnapshot(normalizedChatId, snapshot);
|
|
||||||
}
|
|
||||||
return snapshot;
|
return snapshot;
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("[ST-BME] 读取 Luker 本地缓存快照失败:", source, error);
|
console.warn("[ST-BME] 读取 Luker 本地缓存快照失败:", source, error);
|
||||||
@@ -6394,11 +6395,8 @@ function buildSnapshotFromLukerSidecarState(
|
|||||||
let snapshot = null;
|
let snapshot = null;
|
||||||
if (sidecar?.checkpoint?.serializedGraph) {
|
if (sidecar?.checkpoint?.serializedGraph) {
|
||||||
try {
|
try {
|
||||||
const checkpointGraph = cloneGraphForPersistence(
|
const checkpointGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(
|
deserializeGraph(sidecar.checkpoint.serializedGraph),
|
||||||
deserializeGraph(sidecar.checkpoint.serializedGraph),
|
|
||||||
normalizedChatId,
|
|
||||||
),
|
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
snapshot = buildSnapshotFromGraph(checkpointGraph, {
|
snapshot = buildSnapshotFromGraph(checkpointGraph, {
|
||||||
@@ -6437,8 +6435,8 @@ function buildSnapshotFromLukerSidecarState(
|
|||||||
headRevision: Number(normalizedManifest.headRevision || 0),
|
headRevision: Number(normalizedManifest.headRevision || 0),
|
||||||
};
|
};
|
||||||
} else {
|
} else {
|
||||||
const emptyGraph = cloneGraphForPersistence(
|
const emptyGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(createEmptyGraph(), normalizedChatId),
|
createEmptyGraph(),
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
snapshot = buildSnapshotFromGraph(emptyGraph, {
|
snapshot = buildSnapshotFromGraph(emptyGraph, {
|
||||||
@@ -7659,11 +7657,8 @@ async function loadGraphFromChatState(
|
|||||||
|
|
||||||
let chatStateGraph = null;
|
let chatStateGraph = null;
|
||||||
try {
|
try {
|
||||||
chatStateGraph = cloneGraphForPersistence(
|
chatStateGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(
|
deserializeGraph(payload.serializedGraph),
|
||||||
deserializeGraph(payload.serializedGraph),
|
|
||||||
normalizedChatId,
|
|
||||||
),
|
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -7953,13 +7948,9 @@ async function readPersistedGraphForChatStateTarget(
|
|||||||
});
|
});
|
||||||
if (sidecarResult?.ok && sidecarResult?.snapshot) {
|
if (sidecarResult?.ok && sidecarResult?.snapshot) {
|
||||||
try {
|
try {
|
||||||
return cloneGraphForPersistence(
|
return buildGraphFromSnapshot(sidecarResult.snapshot, {
|
||||||
normalizeGraphRuntimeState(
|
chatId: targetChatId,
|
||||||
buildGraphFromSnapshot(sidecarResult.snapshot),
|
});
|
||||||
targetChatId,
|
|
||||||
),
|
|
||||||
targetChatId,
|
|
||||||
);
|
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("[ST-BME] 读取 Luker branch source snapshot 失败:", error);
|
console.warn("[ST-BME] 读取 Luker branch source snapshot 失败:", error);
|
||||||
}
|
}
|
||||||
@@ -7971,11 +7962,8 @@ async function readPersistedGraphForChatStateTarget(
|
|||||||
});
|
});
|
||||||
if (legacySnapshot?.serializedGraph) {
|
if (legacySnapshot?.serializedGraph) {
|
||||||
try {
|
try {
|
||||||
return cloneGraphForPersistence(
|
return normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(
|
deserializeGraph(legacySnapshot.serializedGraph),
|
||||||
deserializeGraph(legacySnapshot.serializedGraph),
|
|
||||||
targetChatId,
|
|
||||||
),
|
|
||||||
targetChatId,
|
targetChatId,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
@@ -8175,10 +8163,13 @@ function readLegacyGraphFromChatMetadata(chatId, context = getContext()) {
|
|||||||
typeof legacyGraph === "string"
|
typeof legacyGraph === "string"
|
||||||
? deserializeGraph(legacyGraph)
|
? deserializeGraph(legacyGraph)
|
||||||
: legacyGraph;
|
: legacyGraph;
|
||||||
return cloneGraphForPersistence(
|
const normalizedLegacyGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(hydratedLegacyGraph, normalizedChatId),
|
hydratedLegacyGraph,
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
|
return typeof legacyGraph === "string"
|
||||||
|
? normalizedLegacyGraph
|
||||||
|
: cloneGraphForPersistence(normalizedLegacyGraph, normalizedChatId);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
console.warn("[ST-BME] 读取 legacy chat_metadata 图谱失败:", error);
|
console.warn("[ST-BME] 读取 legacy chat_metadata 图谱失败:", error);
|
||||||
return null;
|
return null;
|
||||||
@@ -9218,10 +9209,7 @@ function applyIndexedDbSnapshotToRuntime(
|
|||||||
attemptIndex,
|
attemptIndex,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
currentGraph = cloneGraphForPersistence(
|
currentGraph = graphFromSnapshot;
|
||||||
normalizeGraphRuntimeState(graphFromSnapshot, normalizedChatId),
|
|
||||||
normalizedChatId,
|
|
||||||
);
|
|
||||||
stampGraphPersistenceMeta(currentGraph, {
|
stampGraphPersistenceMeta(currentGraph, {
|
||||||
revision,
|
revision,
|
||||||
reason: `${reasonPrefix}:${String(source || reasonPrefix)}`,
|
reason: `${reasonPrefix}:${String(source || reasonPrefix)}`,
|
||||||
@@ -9485,7 +9473,7 @@ async function loadGraphFromIndexedDb(
|
|||||||
identityRecoveryResult?.snapshot ||
|
identityRecoveryResult?.snapshot ||
|
||||||
localStoreMigrationResult?.snapshot ||
|
localStoreMigrationResult?.snapshot ||
|
||||||
migrationResult?.snapshot ||
|
migrationResult?.snapshot ||
|
||||||
(await db.exportSnapshot());
|
(await db.exportSnapshot({ includeTombstones: false }));
|
||||||
const shadowSnapshot = resolveCompatibleGraphShadowSnapshot(
|
const shadowSnapshot = resolveCompatibleGraphShadowSnapshot(
|
||||||
resolveCurrentChatIdentity(getContext()),
|
resolveCurrentChatIdentity(getContext()),
|
||||||
);
|
);
|
||||||
@@ -10802,11 +10790,8 @@ function resolvePendingPersistGraphSource(chatId = "") {
|
|||||||
shadowSnapshot.serializedGraph
|
shadowSnapshot.serializedGraph
|
||||||
) {
|
) {
|
||||||
try {
|
try {
|
||||||
const shadowGraph = cloneGraphForPersistence(
|
const shadowGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(
|
deserializeGraph(shadowSnapshot.serializedGraph),
|
||||||
deserializeGraph(shadowSnapshot.serializedGraph),
|
|
||||||
normalizedChatId,
|
|
||||||
),
|
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
);
|
);
|
||||||
return {
|
return {
|
||||||
@@ -11376,7 +11361,9 @@ async function persistExtractionBatchResult({
|
|||||||
const context = getContext();
|
const context = getContext();
|
||||||
const persistGraph =
|
const persistGraph =
|
||||||
graphSnapshot && typeof graphSnapshot === "object"
|
graphSnapshot && typeof graphSnapshot === "object"
|
||||||
? cloneGraphSnapshot(graphSnapshot)
|
? graphSnapshot === currentGraph
|
||||||
|
? cloneGraphSnapshot(graphSnapshot)
|
||||||
|
: graphSnapshot
|
||||||
: currentGraph;
|
: currentGraph;
|
||||||
if (!context || !persistGraph) {
|
if (!context || !persistGraph) {
|
||||||
return buildGraphPersistResult({
|
return buildGraphPersistResult({
|
||||||
@@ -12847,10 +12834,14 @@ function loadGraphFromChat(options = {}) {
|
|||||||
: undefined;
|
: undefined;
|
||||||
if (savedData != null && savedData !== "") {
|
if (savedData != null && savedData !== "") {
|
||||||
try {
|
try {
|
||||||
const officialGraph = cloneGraphForPersistence(
|
const hydratedOfficialGraph = normalizeGraphRuntimeState(
|
||||||
normalizeGraphRuntimeState(deserializeGraph(savedData), chatId),
|
deserializeGraph(savedData),
|
||||||
chatId,
|
chatId,
|
||||||
);
|
);
|
||||||
|
const officialGraph =
|
||||||
|
typeof savedData === "string"
|
||||||
|
? hydratedOfficialGraph
|
||||||
|
: cloneGraphForPersistence(hydratedOfficialGraph, chatId);
|
||||||
const shadowDecision = shouldPreferShadowSnapshotOverOfficial(
|
const shadowDecision = shouldPreferShadowSnapshotOverOfficial(
|
||||||
officialGraph,
|
officialGraph,
|
||||||
shadowSnapshot,
|
shadowSnapshot,
|
||||||
|
|||||||
@@ -297,7 +297,7 @@ function buildCommittedBatchPersistSnapshot(
|
|||||||
: null,
|
: null,
|
||||||
rangeEnd,
|
rangeEnd,
|
||||||
];
|
];
|
||||||
const afterSnapshot = runtime.cloneGraphSnapshot(graph);
|
const afterSnapshot = graph;
|
||||||
const effectiveArtifacts = Array.isArray(postProcessArtifacts)
|
const effectiveArtifacts = Array.isArray(postProcessArtifacts)
|
||||||
? [...postProcessArtifacts]
|
? [...postProcessArtifacts]
|
||||||
: [];
|
: [];
|
||||||
@@ -357,17 +357,12 @@ function buildCommittedBatchPersistSnapshot(
|
|||||||
persistGraphSnapshot: committedGraphSnapshot,
|
persistGraphSnapshot: committedGraphSnapshot,
|
||||||
committedBatchJournalEntry,
|
committedBatchJournalEntry,
|
||||||
afterSnapshot,
|
afterSnapshot,
|
||||||
committedAfterSnapshot: runtime.cloneGraphSnapshot(committedGraphSnapshot),
|
committedAfterSnapshot: committedGraphSnapshot,
|
||||||
postProcessArtifacts: effectiveArtifacts,
|
postProcessArtifacts: effectiveArtifacts,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function isPersistenceRevisionAccepted(runtime, persistence = null) {
|
function isPersistenceRevisionAccepted(runtime, persistence = null) {
|
||||||
if (!persistence || persistence.accepted === true) return true;
|
|
||||||
const graphPersistenceState = runtime?.getGraphPersistenceState?.() || {};
|
|
||||||
if (graphPersistenceState.pendingPersist === true) {
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
const persistenceRevision = Number(persistence?.revision || 0);
|
const persistenceRevision = Number(persistence?.revision || 0);
|
||||||
if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) {
|
if (!Number.isFinite(persistenceRevision) || persistenceRevision <= 0) {
|
||||||
return false;
|
return false;
|
||||||
@@ -645,7 +640,7 @@ export async function executeExtractionBatchController(
|
|||||||
processedRange: [startIdx, endIdx],
|
processedRange: [startIdx, endIdx],
|
||||||
postProcessArtifacts: runtime.computePostProcessArtifacts(
|
postProcessArtifacts: runtime.computePostProcessArtifacts(
|
||||||
beforeSnapshot,
|
beforeSnapshot,
|
||||||
runtime.cloneGraphSnapshot(runtime.getCurrentGraph()),
|
runtime.getCurrentGraph(),
|
||||||
effects?.postProcessArtifacts || [],
|
effects?.postProcessArtifacts || [],
|
||||||
),
|
),
|
||||||
vectorHashesInserted: effects?.vectorHashesInserted || [],
|
vectorHashesInserted: effects?.vectorHashesInserted || [],
|
||||||
|
|||||||
102
sync/bme-db.js
102
sync/bme-db.js
@@ -2013,6 +2013,14 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
|||||||
normalizeChatId(options.chatId) ||
|
normalizeChatId(options.chatId) ||
|
||||||
normalizeChatId(snapshotMeta?.chatId) ||
|
normalizeChatId(snapshotMeta?.chatId) ||
|
||||||
normalizeChatId(snapshotState?.chatId);
|
normalizeChatId(snapshotState?.chatId);
|
||||||
|
const snapshotHistoryState = toPlainData(
|
||||||
|
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY],
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
const snapshotVectorState = toPlainData(
|
||||||
|
snapshotMeta?.[BME_RUNTIME_VECTOR_META_KEY],
|
||||||
|
{},
|
||||||
|
);
|
||||||
|
|
||||||
const runtimeGraph = createEmptyGraph();
|
const runtimeGraph = createEmptyGraph();
|
||||||
runtimeGraph.version = Number.isFinite(
|
runtimeGraph.version = Number.isFinite(
|
||||||
@@ -2020,21 +2028,17 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
|||||||
)
|
)
|
||||||
? Number(snapshotMeta[BME_RUNTIME_GRAPH_VERSION_META_KEY])
|
? Number(snapshotMeta[BME_RUNTIME_GRAPH_VERSION_META_KEY])
|
||||||
: runtimeGraph.version;
|
: runtimeGraph.version;
|
||||||
runtimeGraph.nodes = toArray(snapshotView.nodes).map((node) => ({
|
runtimeGraph.nodes = toArray(toPlainData(snapshotView.nodes, []));
|
||||||
...(node || {}),
|
runtimeGraph.edges = toArray(toPlainData(snapshotView.edges, []));
|
||||||
}));
|
|
||||||
runtimeGraph.edges = toArray(snapshotView.edges).map((edge) => ({
|
|
||||||
...(edge || {}),
|
|
||||||
}));
|
|
||||||
runtimeGraph.batchJournal = toArray(
|
runtimeGraph.batchJournal = toArray(
|
||||||
snapshotMeta?.[BME_RUNTIME_BATCH_JOURNAL_META_KEY],
|
toPlainData(snapshotMeta?.[BME_RUNTIME_BATCH_JOURNAL_META_KEY], []),
|
||||||
);
|
);
|
||||||
runtimeGraph.lastRecallResult = toPlainData(
|
runtimeGraph.lastRecallResult = toPlainData(
|
||||||
snapshotMeta?.[BME_RUNTIME_LAST_RECALL_META_KEY],
|
snapshotMeta?.[BME_RUNTIME_LAST_RECALL_META_KEY],
|
||||||
null,
|
null,
|
||||||
);
|
);
|
||||||
runtimeGraph.maintenanceJournal = toArray(
|
runtimeGraph.maintenanceJournal = toArray(
|
||||||
snapshotMeta?.[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY],
|
toPlainData(snapshotMeta?.[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY], []),
|
||||||
);
|
);
|
||||||
runtimeGraph.knowledgeState = toPlainData(
|
runtimeGraph.knowledgeState = toPlainData(
|
||||||
snapshotMeta?.[BME_RUNTIME_KNOWLEDGE_STATE_META_KEY],
|
snapshotMeta?.[BME_RUNTIME_KNOWLEDGE_STATE_META_KEY],
|
||||||
@@ -2073,22 +2077,21 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
|||||||
|
|
||||||
runtimeGraph.historyState = {
|
runtimeGraph.historyState = {
|
||||||
...(runtimeGraph.historyState || {}),
|
...(runtimeGraph.historyState || {}),
|
||||||
...(snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY] || {}),
|
...snapshotHistoryState,
|
||||||
lastProcessedAssistantFloor: Number.isFinite(
|
lastProcessedAssistantFloor: Number.isFinite(
|
||||||
Number(snapshotState?.lastProcessedFloor),
|
Number(snapshotState?.lastProcessedFloor),
|
||||||
)
|
)
|
||||||
? Number(snapshotState.lastProcessedFloor)
|
? Number(snapshotState.lastProcessedFloor)
|
||||||
: Number(
|
: Number(
|
||||||
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY]
|
snapshotHistoryState?.lastProcessedAssistantFloor ??
|
||||||
?.lastProcessedAssistantFloor ?? META_DEFAULT_LAST_PROCESSED_FLOOR,
|
META_DEFAULT_LAST_PROCESSED_FLOOR,
|
||||||
),
|
),
|
||||||
extractionCount: Number.isFinite(
|
extractionCount: Number.isFinite(
|
||||||
Number(snapshotState?.extractionCount),
|
Number(snapshotState?.extractionCount),
|
||||||
)
|
)
|
||||||
? Number(snapshotState.extractionCount)
|
? Number(snapshotState.extractionCount)
|
||||||
: Number(
|
: Number(
|
||||||
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY]
|
snapshotHistoryState?.extractionCount ?? META_DEFAULT_EXTRACTION_COUNT,
|
||||||
?.extractionCount ?? META_DEFAULT_EXTRACTION_COUNT,
|
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
if (
|
if (
|
||||||
@@ -2146,10 +2149,10 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
|||||||
}
|
}
|
||||||
runtimeGraph.vectorIndexState = {
|
runtimeGraph.vectorIndexState = {
|
||||||
...(runtimeGraph.vectorIndexState || {}),
|
...(runtimeGraph.vectorIndexState || {}),
|
||||||
...(snapshotMeta?.[BME_RUNTIME_VECTOR_META_KEY] || {}),
|
...snapshotVectorState,
|
||||||
collectionId: buildVectorCollectionId(
|
collectionId: buildVectorCollectionId(
|
||||||
chatId ||
|
chatId ||
|
||||||
snapshotMeta?.[BME_RUNTIME_HISTORY_META_KEY]?.chatId ||
|
snapshotHistoryState?.chatId ||
|
||||||
runtimeGraph.historyState?.chatId ||
|
runtimeGraph.historyState?.chatId ||
|
||||||
"",
|
"",
|
||||||
),
|
),
|
||||||
@@ -3032,23 +3035,47 @@ export class BmeDatabase {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async exportSnapshot() {
|
async exportSnapshot(options = {}) {
|
||||||
const db = await this.open();
|
const db = await this.open();
|
||||||
|
|
||||||
const [nodes, edges, tombstones, metaRows] = await db.transaction(
|
const includeTombstones =
|
||||||
"r",
|
options && typeof options === "object"
|
||||||
db.table("nodes"),
|
? options.includeTombstones !== false
|
||||||
db.table("edges"),
|
: options !== false;
|
||||||
db.table("tombstones"),
|
let nodes = [];
|
||||||
db.table("meta"),
|
let edges = [];
|
||||||
async () =>
|
let tombstones = [];
|
||||||
await Promise.all([
|
let metaRows = [];
|
||||||
db.table("nodes").toArray(),
|
|
||||||
db.table("edges").toArray(),
|
if (includeTombstones) {
|
||||||
db.table("tombstones").toArray(),
|
[nodes, edges, tombstones, metaRows] = await db.transaction(
|
||||||
db.table("meta").toArray(),
|
"r",
|
||||||
]),
|
db.table("nodes"),
|
||||||
);
|
db.table("edges"),
|
||||||
|
db.table("tombstones"),
|
||||||
|
db.table("meta"),
|
||||||
|
async () =>
|
||||||
|
await Promise.all([
|
||||||
|
db.table("nodes").toArray(),
|
||||||
|
db.table("edges").toArray(),
|
||||||
|
db.table("tombstones").toArray(),
|
||||||
|
db.table("meta").toArray(),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
} else {
|
||||||
|
[nodes, edges, metaRows] = await db.transaction(
|
||||||
|
"r",
|
||||||
|
db.table("nodes"),
|
||||||
|
db.table("edges"),
|
||||||
|
db.table("meta"),
|
||||||
|
async () =>
|
||||||
|
await Promise.all([
|
||||||
|
db.table("nodes").toArray(),
|
||||||
|
db.table("edges").toArray(),
|
||||||
|
db.table("meta").toArray(),
|
||||||
|
]),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
const metaMap = toMetaMap(metaRows);
|
const metaMap = toMetaMap(metaRows);
|
||||||
const meta = {
|
const meta = {
|
||||||
@@ -3058,7 +3085,10 @@ export class BmeDatabase {
|
|||||||
revision: normalizeRevision(metaMap?.revision),
|
revision: normalizeRevision(metaMap?.revision),
|
||||||
nodeCount: nodes.length,
|
nodeCount: nodes.length,
|
||||||
edgeCount: edges.length,
|
edgeCount: edges.length,
|
||||||
tombstoneCount: tombstones.length,
|
tombstoneCount: normalizeNonNegativeInteger(
|
||||||
|
metaMap?.tombstoneCount,
|
||||||
|
tombstones.length,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
|
|
||||||
const state = {
|
const state = {
|
||||||
@@ -3070,13 +3100,19 @@ export class BmeDatabase {
|
|||||||
: META_DEFAULT_EXTRACTION_COUNT,
|
: META_DEFAULT_EXTRACTION_COUNT,
|
||||||
};
|
};
|
||||||
|
|
||||||
return {
|
const snapshot = {
|
||||||
meta,
|
meta,
|
||||||
nodes,
|
nodes,
|
||||||
edges,
|
edges,
|
||||||
tombstones,
|
tombstones: includeTombstones ? tombstones : [],
|
||||||
state,
|
state,
|
||||||
};
|
};
|
||||||
|
|
||||||
|
if (!includeTombstones) {
|
||||||
|
snapshot.__stBmeTombstonesOmitted = true;
|
||||||
|
}
|
||||||
|
|
||||||
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
async importSnapshot(snapshot, options = {}) {
|
async importSnapshot(snapshot, options = {}) {
|
||||||
|
|||||||
@@ -1345,15 +1345,23 @@ class LegacyOpfsGraphStore {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async exportSnapshot() {
|
async exportSnapshot(options = {}) {
|
||||||
const snapshot = await this._loadSnapshot();
|
const includeTombstones =
|
||||||
return {
|
options && typeof options === "object"
|
||||||
|
? options.includeTombstones !== false
|
||||||
|
: options !== false;
|
||||||
|
const snapshot = await this._loadSnapshot({ includeTombstones });
|
||||||
|
const exported = {
|
||||||
meta: toPlainData(snapshot.meta, {}),
|
meta: toPlainData(snapshot.meta, {}),
|
||||||
nodes: toPlainData(snapshot.nodes, []),
|
nodes: toPlainData(snapshot.nodes, []),
|
||||||
edges: toPlainData(snapshot.edges, []),
|
edges: toPlainData(snapshot.edges, []),
|
||||||
tombstones: toPlainData(snapshot.tombstones, []),
|
tombstones: includeTombstones ? toPlainData(snapshot.tombstones, []) : [],
|
||||||
state: toPlainData(snapshot.state, {}),
|
state: toPlainData(snapshot.state, {}),
|
||||||
};
|
};
|
||||||
|
if (!includeTombstones) {
|
||||||
|
exported.__stBmeTombstonesOmitted = true;
|
||||||
|
}
|
||||||
|
return exported;
|
||||||
}
|
}
|
||||||
|
|
||||||
async importSnapshot(snapshot, options = {}) {
|
async importSnapshot(snapshot, options = {}) {
|
||||||
@@ -2643,15 +2651,23 @@ export class OpfsGraphStore {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async exportSnapshot() {
|
async exportSnapshot(options = {}) {
|
||||||
const snapshot = await this._loadSnapshot();
|
const includeTombstones =
|
||||||
return {
|
options && typeof options === "object"
|
||||||
|
? options.includeTombstones !== false
|
||||||
|
: options !== false;
|
||||||
|
const snapshot = await this._loadSnapshot({ includeTombstones });
|
||||||
|
const exported = {
|
||||||
meta: toPlainData(snapshot.meta, {}),
|
meta: toPlainData(snapshot.meta, {}),
|
||||||
nodes: toPlainData(snapshot.nodes, []),
|
nodes: toPlainData(snapshot.nodes, []),
|
||||||
edges: toPlainData(snapshot.edges, []),
|
edges: toPlainData(snapshot.edges, []),
|
||||||
tombstones: toPlainData(snapshot.tombstones, []),
|
tombstones: includeTombstones ? toPlainData(snapshot.tombstones, []) : [],
|
||||||
state: toPlainData(snapshot.state, {}),
|
state: toPlainData(snapshot.state, {}),
|
||||||
};
|
};
|
||||||
|
if (!includeTombstones) {
|
||||||
|
exported.__stBmeTombstonesOmitted = true;
|
||||||
|
}
|
||||||
|
return exported;
|
||||||
}
|
}
|
||||||
|
|
||||||
async importSnapshot(snapshot, options = {}) {
|
async importSnapshot(snapshot, options = {}) {
|
||||||
@@ -3263,7 +3279,7 @@ export class OpfsGraphStore {
|
|||||||
return records;
|
return records;
|
||||||
}
|
}
|
||||||
|
|
||||||
async _loadBaseSnapshotFromV2(manifest = null) {
|
async _loadBaseSnapshotFromV2(manifest = null, { includeTombstones = true } = {}) {
|
||||||
const normalizedManifest = manifest || (await this._ensureV2Ready());
|
const normalizedManifest = manifest || (await this._ensureV2Ready());
|
||||||
const runtimeMeta = await this._readRuntimeMetaEntries();
|
const runtimeMeta = await this._readRuntimeMetaEntries();
|
||||||
const nodes = [];
|
const nodes = [];
|
||||||
@@ -3275,8 +3291,10 @@ export class OpfsGraphStore {
|
|||||||
for (let index = 0; index < OPFS_V2_EDGE_BUCKET_COUNT; index += 1) {
|
for (let index = 0; index < OPFS_V2_EDGE_BUCKET_COUNT; index += 1) {
|
||||||
edges.push(...(await this._readShardRecords("edges", index)));
|
edges.push(...(await this._readShardRecords("edges", index)));
|
||||||
}
|
}
|
||||||
for (let index = 0; index < OPFS_V2_TOMBSTONE_BUCKET_COUNT; index += 1) {
|
if (includeTombstones) {
|
||||||
tombstones.push(...(await this._readShardRecords("tombstones", index)));
|
for (let index = 0; index < OPFS_V2_TOMBSTONE_BUCKET_COUNT; index += 1) {
|
||||||
|
tombstones.push(...(await this._readShardRecords("tombstones", index)));
|
||||||
|
}
|
||||||
}
|
}
|
||||||
const meta = {
|
const meta = {
|
||||||
...createDefaultMetaValues(this.chatId),
|
...createDefaultMetaValues(this.chatId),
|
||||||
@@ -3311,7 +3329,7 @@ export class OpfsGraphStore {
|
|||||||
return snapshot;
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
async _loadSnapshot({ awaitWrites = true } = {}) {
|
async _loadSnapshot({ awaitWrites = true, includeTombstones = true } = {}) {
|
||||||
if (awaitWrites) {
|
if (awaitWrites) {
|
||||||
await this._awaitPendingWrites();
|
await this._awaitPendingWrites();
|
||||||
}
|
}
|
||||||
@@ -3319,10 +3337,24 @@ export class OpfsGraphStore {
|
|||||||
const headRevision = normalizeRevision(
|
const headRevision = normalizeRevision(
|
||||||
manifest?.headRevision || manifest?.meta?.revision,
|
manifest?.headRevision || manifest?.meta?.revision,
|
||||||
);
|
);
|
||||||
if (this._snapshotCache && normalizeRevision(this._snapshotCache.meta?.revision) === headRevision) {
|
if (
|
||||||
return this._snapshotCache;
|
this._snapshotCache &&
|
||||||
|
normalizeRevision(this._snapshotCache.meta?.revision) === headRevision
|
||||||
|
) {
|
||||||
|
if (includeTombstones) {
|
||||||
|
return this._snapshotCache;
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
meta: this._snapshotCache.meta,
|
||||||
|
state: this._snapshotCache.state,
|
||||||
|
nodes: this._snapshotCache.nodes,
|
||||||
|
edges: this._snapshotCache.edges,
|
||||||
|
tombstones: [],
|
||||||
|
};
|
||||||
}
|
}
|
||||||
const snapshot = await this._loadBaseSnapshotFromV2(manifest);
|
const snapshot = await this._loadBaseSnapshotFromV2(manifest, {
|
||||||
|
includeTombstones,
|
||||||
|
});
|
||||||
const walRecords = await this._readWalRecords(manifest);
|
const walRecords = await this._readWalRecords(manifest);
|
||||||
for (const walRecord of walRecords) {
|
for (const walRecord of walRecords) {
|
||||||
const nextSnapshot = applyOpfsV2DeltaToSnapshot(snapshot, walRecord.delta, walRecord.committedAt);
|
const nextSnapshot = applyOpfsV2DeltaToSnapshot(snapshot, walRecord.delta, walRecord.committedAt);
|
||||||
@@ -3355,7 +3387,11 @@ export class OpfsGraphStore {
|
|||||||
snapshot.state = normalizeSnapshotState(snapshot);
|
snapshot.state = normalizeSnapshotState(snapshot);
|
||||||
snapshot.meta.lastProcessedFloor = snapshot.state.lastProcessedFloor;
|
snapshot.meta.lastProcessedFloor = snapshot.state.lastProcessedFloor;
|
||||||
snapshot.meta.extractionCount = snapshot.state.extractionCount;
|
snapshot.meta.extractionCount = snapshot.state.extractionCount;
|
||||||
this._snapshotCache = snapshot;
|
if (includeTombstones) {
|
||||||
|
this._snapshotCache = snapshot;
|
||||||
|
return snapshot;
|
||||||
|
}
|
||||||
|
snapshot.tombstones = [];
|
||||||
return snapshot;
|
return snapshot;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -2,6 +2,9 @@ import assert from "node:assert/strict";
|
|||||||
|
|
||||||
import {
|
import {
|
||||||
BME_DB_SCHEMA_VERSION,
|
BME_DB_SCHEMA_VERSION,
|
||||||
|
BME_RUNTIME_BATCH_JOURNAL_META_KEY,
|
||||||
|
BME_RUNTIME_HISTORY_META_KEY,
|
||||||
|
BME_RUNTIME_VECTOR_META_KEY,
|
||||||
BME_TOMBSTONE_RETENTION_MS,
|
BME_TOMBSTONE_RETENTION_MS,
|
||||||
BmeDatabase,
|
BmeDatabase,
|
||||||
buildBmeDbName,
|
buildBmeDbName,
|
||||||
@@ -20,6 +23,7 @@ const chatIdsForCleanup = new Set([
|
|||||||
"chat-manager-a",
|
"chat-manager-a",
|
||||||
"chat-manager-b",
|
"chat-manager-b",
|
||||||
"chat-manager-selector",
|
"chat-manager-selector",
|
||||||
|
"chat-export-without-tombstones",
|
||||||
"chat-replace-reset",
|
"chat-replace-reset",
|
||||||
]);
|
]);
|
||||||
|
|
||||||
@@ -196,6 +200,41 @@ async function testSnapshotExportImport() {
|
|||||||
await db.close();
|
await db.close();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function testSnapshotExportWithoutTombstones() {
|
||||||
|
const db = new BmeDatabase("chat-export-without-tombstones", {
|
||||||
|
dexieClass: globalThis.Dexie,
|
||||||
|
});
|
||||||
|
await db.open();
|
||||||
|
|
||||||
|
await db.bulkUpsertNodes([
|
||||||
|
{
|
||||||
|
id: "node-light-snapshot",
|
||||||
|
type: "event",
|
||||||
|
sourceFloor: 3,
|
||||||
|
archived: false,
|
||||||
|
updatedAt: Date.now(),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
await db.bulkUpsertTombstones([
|
||||||
|
{
|
||||||
|
id: "tomb-light-snapshot",
|
||||||
|
kind: "node",
|
||||||
|
targetId: "node-deleted-light-snapshot",
|
||||||
|
deletedAt: Date.now(),
|
||||||
|
sourceDeviceId: "device-light-snapshot",
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
const exported = await db.exportSnapshot({ includeTombstones: false });
|
||||||
|
assert.equal(exported.__stBmeTombstonesOmitted, true);
|
||||||
|
assert.ok(Array.isArray(exported.nodes));
|
||||||
|
assert.ok(Array.isArray(exported.edges));
|
||||||
|
assert.deepEqual(exported.tombstones, []);
|
||||||
|
assert.equal(exported.meta.tombstoneCount, 1);
|
||||||
|
|
||||||
|
await db.close();
|
||||||
|
}
|
||||||
|
|
||||||
async function testReplaceImportResetsStaleMeta() {
|
async function testReplaceImportResetsStaleMeta() {
|
||||||
const chatId = "chat-replace-reset";
|
const chatId = "chat-replace-reset";
|
||||||
const db = new BmeDatabase(chatId, { dexieClass: globalThis.Dexie });
|
const db = new BmeDatabase(chatId, { dexieClass: globalThis.Dexie });
|
||||||
@@ -532,6 +571,9 @@ async function testGraphSnapshotConverters() {
|
|||||||
id: "node-converter",
|
id: "node-converter",
|
||||||
type: "event",
|
type: "event",
|
||||||
sourceFloor: 9,
|
sourceFloor: 9,
|
||||||
|
fields: {
|
||||||
|
title: "Converter Node",
|
||||||
|
},
|
||||||
updatedAt: Date.now(),
|
updatedAt: Date.now(),
|
||||||
});
|
});
|
||||||
|
|
||||||
@@ -583,6 +625,32 @@ async function testGraphSnapshotConverters() {
|
|||||||
assert.equal(rebuilt.regionState.activeRegion, "camp");
|
assert.equal(rebuilt.regionState.activeRegion, "camp");
|
||||||
assert.equal(rebuilt.timelineState.activeSegmentId, "segment-1");
|
assert.equal(rebuilt.timelineState.activeSegmentId, "segment-1");
|
||||||
assert.equal(rebuilt.summaryState.entries[0].id, "summary-1");
|
assert.equal(rebuilt.summaryState.entries[0].id, "summary-1");
|
||||||
|
|
||||||
|
rebuilt.nodes[0].fields.title = "Mutated Converter Node";
|
||||||
|
rebuilt.historyState.processedMessageHashes[1] = "mutated-hash";
|
||||||
|
rebuilt.vectorIndexState.hashToNodeId["vec-hash"] = "node-mutated";
|
||||||
|
rebuilt.batchJournal[0].processedRange[0] = 99;
|
||||||
|
|
||||||
|
assert.equal(
|
||||||
|
snapshot.nodes[0].fields.title,
|
||||||
|
"Converter Node",
|
||||||
|
"buildGraphFromSnapshot 不应复用 snapshot 节点的嵌套字段引用",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
snapshot.meta[BME_RUNTIME_HISTORY_META_KEY].processedMessageHashes[1],
|
||||||
|
"hash-1",
|
||||||
|
"buildGraphFromSnapshot 不应复用 snapshot historyState 的嵌套对象引用",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
snapshot.meta[BME_RUNTIME_VECTOR_META_KEY].hashToNodeId["vec-hash"],
|
||||||
|
"node-converter",
|
||||||
|
"buildGraphFromSnapshot 不应复用 snapshot vectorState 的嵌套对象引用",
|
||||||
|
);
|
||||||
|
assert.equal(
|
||||||
|
snapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY][0].processedRange[0],
|
||||||
|
8,
|
||||||
|
"buildGraphFromSnapshot 不应复用 snapshot batchJournal 的嵌套数组引用",
|
||||||
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function main() {
|
async function main() {
|
||||||
@@ -593,6 +661,7 @@ async function main() {
|
|||||||
await testCrudAndMeta();
|
await testCrudAndMeta();
|
||||||
await testTransactionRollback();
|
await testTransactionRollback();
|
||||||
await testSnapshotExportImport();
|
await testSnapshotExportImport();
|
||||||
|
await testSnapshotExportWithoutTombstones();
|
||||||
await testReplaceImportResetsStaleMeta();
|
await testReplaceImportResetsStaleMeta();
|
||||||
await testRevisionMonotonicity();
|
await testRevisionMonotonicity();
|
||||||
await testTombstonePrune();
|
await testTombstonePrune();
|
||||||
|
|||||||
@@ -311,6 +311,12 @@ async function testImportExportPersistenceAndFileRotation() {
|
|||||||
assert.equal(firstExportedSnapshot.state.extractionCount, 2);
|
assert.equal(firstExportedSnapshot.state.extractionCount, 2);
|
||||||
assert.equal(firstExportedSnapshot.meta.storagePrimary, "opfs");
|
assert.equal(firstExportedSnapshot.meta.storagePrimary, "opfs");
|
||||||
assert.equal(firstExportedSnapshot.meta.storageMode, "opfs-primary");
|
assert.equal(firstExportedSnapshot.meta.storageMode, "opfs-primary");
|
||||||
|
const lightweightSnapshot = await store.exportSnapshot({
|
||||||
|
includeTombstones: false,
|
||||||
|
});
|
||||||
|
assert.equal(lightweightSnapshot.__stBmeTombstonesOmitted, true);
|
||||||
|
assert.deepEqual(lightweightSnapshot.tombstones, []);
|
||||||
|
assert.equal(lightweightSnapshot.meta.tombstoneCount, 1);
|
||||||
assert.deepEqual(firstExportedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], {
|
assert.deepEqual(firstExportedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], {
|
||||||
pending: ["job-1"],
|
pending: ["job-1"],
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user