mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
591 lines
18 KiB
JavaScript
591 lines
18 KiB
JavaScript
import assert from "node:assert/strict";
|
|
|
|
import {
|
|
BME_RUNTIME_BATCH_JOURNAL_META_KEY,
|
|
BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY,
|
|
BME_TOMBSTONE_RETENTION_MS,
|
|
} from "../sync/bme-db.js";
|
|
import {
|
|
BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY,
|
|
BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW,
|
|
deleteAllOpfsStorage,
|
|
deleteOpfsChatStorage,
|
|
OpfsGraphStore,
|
|
detectOpfsSupport,
|
|
} from "../sync/bme-opfs-store.js";
|
|
import { createEdge, createEmptyGraph, createNode } from "../graph/graph.js";
|
|
|
|
const PREFIX = "[ST-BME][opfs-persistence]";
|
|
|
|
function createNotFoundError(message) {
|
|
const error = new Error(String(message || "Not found"));
|
|
error.name = "NotFoundError";
|
|
return error;
|
|
}
|
|
|
|
class MemoryOpfsFileHandle {
|
|
constructor(parent, name) {
|
|
this.parent = parent;
|
|
this.name = String(name || "");
|
|
}
|
|
|
|
async getFile() {
|
|
const parent = this.parent;
|
|
const name = this.name;
|
|
return {
|
|
async text() {
|
|
return String(parent.files.get(name) ?? "");
|
|
},
|
|
};
|
|
}
|
|
|
|
async createWritable() {
|
|
const parent = this.parent;
|
|
const name = this.name;
|
|
let buffer = String(parent.files.get(name) ?? "");
|
|
return {
|
|
async write(chunk) {
|
|
if (typeof chunk === "string") {
|
|
buffer = chunk;
|
|
return;
|
|
}
|
|
if (chunk == null) {
|
|
buffer = "";
|
|
return;
|
|
}
|
|
buffer = String(chunk);
|
|
},
|
|
async close() {
|
|
parent.files.set(name, buffer);
|
|
},
|
|
};
|
|
}
|
|
}
|
|
|
|
class MemoryOpfsDirectoryHandle {
|
|
constructor(name = "") {
|
|
this.name = String(name || "");
|
|
this.directories = new Map();
|
|
this.files = new Map();
|
|
}
|
|
|
|
async getDirectoryHandle(name, options = {}) {
|
|
const normalizedName = String(name || "");
|
|
let directory = this.directories.get(normalizedName) || null;
|
|
if (!directory) {
|
|
if (!options.create) {
|
|
throw createNotFoundError(`Directory not found: ${normalizedName}`);
|
|
}
|
|
directory = new MemoryOpfsDirectoryHandle(normalizedName);
|
|
this.directories.set(normalizedName, directory);
|
|
}
|
|
return directory;
|
|
}
|
|
|
|
async getFileHandle(name, options = {}) {
|
|
const normalizedName = String(name || "");
|
|
if (!this.files.has(normalizedName)) {
|
|
if (!options.create) {
|
|
throw createNotFoundError(`File not found: ${normalizedName}`);
|
|
}
|
|
this.files.set(normalizedName, "");
|
|
}
|
|
return new MemoryOpfsFileHandle(this, normalizedName);
|
|
}
|
|
|
|
async removeEntry(name, options = {}) {
|
|
const normalizedName = String(name || "");
|
|
if (this.files.delete(normalizedName)) {
|
|
return;
|
|
}
|
|
const directory = this.directories.get(normalizedName) || null;
|
|
if (directory) {
|
|
const canDelete =
|
|
options.recursive === true
|
|
|| (directory.files.size === 0 && directory.directories.size === 0);
|
|
if (!canDelete) {
|
|
throw new Error(`Directory not empty: ${normalizedName}`);
|
|
}
|
|
this.directories.delete(normalizedName);
|
|
return;
|
|
}
|
|
throw createNotFoundError(`Entry not found: ${normalizedName}`);
|
|
}
|
|
}
|
|
|
|
function createMemoryOpfsRoot() {
|
|
return new MemoryOpfsDirectoryHandle("root");
|
|
}
|
|
|
|
function getChatDirectory(rootDirectory, chatId) {
|
|
const opfsRoot = rootDirectory.directories.get("st-bme") || null;
|
|
assert.ok(opfsRoot, "OPFS 根目录必须存在");
|
|
const chatsDirectory = opfsRoot.directories.get("chats") || null;
|
|
assert.ok(chatsDirectory, "OPFS chats 目录必须存在");
|
|
const chatDirectory = chatsDirectory.directories.get(encodeURIComponent(chatId)) || null;
|
|
assert.ok(chatDirectory, `OPFS chat 目录必须存在: ${chatId}`);
|
|
return chatDirectory;
|
|
}
|
|
|
|
function getNestedDirectory(directoryHandle, ...names) {
|
|
let current = directoryHandle;
|
|
for (const name of names) {
|
|
current = current?.directories?.get(String(name || "")) || null;
|
|
assert.ok(current, `目录必须存在: ${names.join("/")}`);
|
|
}
|
|
return current;
|
|
}
|
|
|
|
function readJsonFromDirectory(directoryHandle, filename) {
|
|
assert.ok(directoryHandle.files.has(filename), `文件必须存在: ${filename}`);
|
|
return JSON.parse(String(directoryHandle.files.get(filename) || ""));
|
|
}
|
|
|
|
function buildLegacyGraph(chatId) {
|
|
const graph = createEmptyGraph();
|
|
graph.historyState.chatId = chatId;
|
|
|
|
const node = createNode({
|
|
type: "event",
|
|
fields: {
|
|
title: "legacy-node",
|
|
},
|
|
seq: 5,
|
|
seqRange: [4, 5],
|
|
});
|
|
node.id = "legacy-node";
|
|
node.updatedAt = 1000;
|
|
|
|
const edge = createEdge({
|
|
fromId: node.id,
|
|
toId: node.id,
|
|
relation: "self",
|
|
});
|
|
edge.id = "legacy-edge";
|
|
edge.updatedAt = 1001;
|
|
|
|
graph.nodes.push(node);
|
|
graph.edges.push(edge);
|
|
return graph;
|
|
}
|
|
|
|
async function testDetectOpfsSupport() {
|
|
const rootDirectory = createMemoryOpfsRoot();
|
|
const supported = await detectOpfsSupport({
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
});
|
|
assert.equal(supported.available, true);
|
|
assert.equal(supported.reason, "ok");
|
|
|
|
const missingHandle = await detectOpfsSupport({
|
|
rootDirectoryFactory: async () => ({}),
|
|
});
|
|
assert.equal(missingHandle.available, false);
|
|
assert.equal(missingHandle.reason, "missing-directory-handle");
|
|
|
|
const failing = await detectOpfsSupport({
|
|
rootDirectoryFactory: async () => {
|
|
throw new Error("opfs-unavailable");
|
|
},
|
|
});
|
|
assert.equal(failing.available, false);
|
|
assert.equal(failing.reason, "opfs-unavailable");
|
|
}
|
|
|
|
async function testImportExportPersistenceAndFileRotation() {
|
|
const rootDirectory = createMemoryOpfsRoot();
|
|
const store = new OpfsGraphStore("chat-opfs-persist", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY,
|
|
});
|
|
|
|
await store.open();
|
|
|
|
const initialSnapshot = await store.exportSnapshot();
|
|
assert.equal(initialSnapshot.meta.chatId, "chat-opfs-persist");
|
|
assert.equal(initialSnapshot.meta.storagePrimary, "opfs");
|
|
assert.equal(initialSnapshot.meta.storageMode, "opfs-primary");
|
|
|
|
const chatDirectory = getChatDirectory(rootDirectory, "chat-opfs-persist");
|
|
assert.deepEqual(Array.from(chatDirectory.files.keys()).sort(), ["manifest.json"]);
|
|
|
|
await store.importSnapshot(
|
|
{
|
|
meta: {
|
|
revision: 4,
|
|
deviceId: "device-1",
|
|
[BME_RUNTIME_BATCH_JOURNAL_META_KEY]: {
|
|
pending: ["job-1"],
|
|
},
|
|
[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY]: {
|
|
completedAt: 123,
|
|
},
|
|
},
|
|
state: {
|
|
lastProcessedFloor: 8,
|
|
extractionCount: 2,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "node-1",
|
|
type: "event",
|
|
fields: {
|
|
title: "A",
|
|
},
|
|
archived: false,
|
|
updatedAt: 1000,
|
|
},
|
|
],
|
|
edges: [
|
|
{
|
|
id: "edge-1",
|
|
fromId: "node-1",
|
|
toId: "node-1",
|
|
relation: "self",
|
|
updatedAt: 1001,
|
|
},
|
|
],
|
|
tombstones: [
|
|
{
|
|
id: "ts-1",
|
|
kind: "node",
|
|
targetId: "node-old",
|
|
sourceDeviceId: "device-1",
|
|
deletedAt: 1002,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
mode: "replace",
|
|
preserveRevision: true,
|
|
markSyncDirty: false,
|
|
},
|
|
);
|
|
|
|
const manifestAfterFirstImport = readJsonFromDirectory(chatDirectory, "manifest.json");
|
|
assert.equal(manifestAfterFirstImport.formatVersion, 2);
|
|
assert.equal(manifestAfterFirstImport.baseRevision, 4);
|
|
assert.equal(manifestAfterFirstImport.headRevision, 4);
|
|
assert.equal(manifestAfterFirstImport.wal.count, 0);
|
|
const metaDirectory = getNestedDirectory(chatDirectory, "meta");
|
|
const shardDirectory = getNestedDirectory(chatDirectory, "shards");
|
|
const nodeShardDirectory = getNestedDirectory(shardDirectory, "nodes");
|
|
const edgeShardDirectory = getNestedDirectory(shardDirectory, "edges");
|
|
const tombstoneShardDirectory = getNestedDirectory(shardDirectory, "tombstones");
|
|
const walDirectory = getNestedDirectory(chatDirectory, "wal");
|
|
assert.ok(metaDirectory.files.size > 0);
|
|
assert.ok(nodeShardDirectory.files.size > 0);
|
|
assert.ok(edgeShardDirectory.files.size > 0);
|
|
assert.ok(tombstoneShardDirectory.files.size > 0);
|
|
assert.equal(walDirectory.files.size, 0);
|
|
|
|
const firstExportedSnapshot = await store.exportSnapshot();
|
|
assert.equal(firstExportedSnapshot.meta.revision, 4);
|
|
assert.equal(firstExportedSnapshot.state.lastProcessedFloor, 8);
|
|
assert.equal(firstExportedSnapshot.state.extractionCount, 2);
|
|
assert.equal(firstExportedSnapshot.meta.storagePrimary, "opfs");
|
|
assert.equal(firstExportedSnapshot.meta.storageMode, "opfs-primary");
|
|
assert.deepEqual(firstExportedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], {
|
|
pending: ["job-1"],
|
|
});
|
|
assert.deepEqual(
|
|
firstExportedSnapshot.meta[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY],
|
|
{
|
|
completedAt: 123,
|
|
},
|
|
);
|
|
|
|
await store.close();
|
|
|
|
const reopenedStore = new OpfsGraphStore("chat-opfs-persist", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY,
|
|
});
|
|
await reopenedStore.open();
|
|
|
|
const reopenedSnapshot = await reopenedStore.exportSnapshot();
|
|
assert.equal(reopenedSnapshot.meta.revision, 4);
|
|
assert.equal(reopenedSnapshot.nodes.length, 1);
|
|
assert.equal(reopenedSnapshot.edges.length, 1);
|
|
assert.equal(reopenedSnapshot.tombstones.length, 1);
|
|
assert.deepEqual(reopenedSnapshot.meta[BME_RUNTIME_BATCH_JOURNAL_META_KEY], {
|
|
pending: ["job-1"],
|
|
});
|
|
assert.deepEqual(reopenedSnapshot.meta[BME_RUNTIME_MAINTENANCE_JOURNAL_META_KEY], {
|
|
completedAt: 123,
|
|
});
|
|
|
|
await reopenedStore.importSnapshot(
|
|
{
|
|
meta: {
|
|
revision: 6,
|
|
},
|
|
state: {
|
|
lastProcessedFloor: 9,
|
|
extractionCount: 4,
|
|
},
|
|
nodes: [
|
|
{
|
|
id: "node-2",
|
|
type: "fact",
|
|
fields: {
|
|
title: "B",
|
|
},
|
|
archived: false,
|
|
updatedAt: 2000,
|
|
},
|
|
],
|
|
edges: [],
|
|
tombstones: [],
|
|
},
|
|
{
|
|
mode: "replace",
|
|
preserveRevision: true,
|
|
},
|
|
);
|
|
|
|
const manifestAfterSecondImport = readJsonFromDirectory(chatDirectory, "manifest.json");
|
|
assert.equal(manifestAfterSecondImport.formatVersion, 2);
|
|
assert.equal(manifestAfterSecondImport.baseRevision, 6);
|
|
assert.equal(manifestAfterSecondImport.headRevision, 6);
|
|
assert.equal(manifestAfterSecondImport.wal.count, 0);
|
|
const secondSnapshot = await reopenedStore.exportSnapshot();
|
|
assert.deepEqual(secondSnapshot.nodes.map((item) => item.id), ["node-2"]);
|
|
assert.equal(secondSnapshot.edges.length, 0);
|
|
assert.equal(secondSnapshot.tombstones.length, 0);
|
|
|
|
await reopenedStore.close();
|
|
}
|
|
|
|
async function testImportLegacyGraphMigrationAndSkipPaths() {
|
|
const migrationRoot = createMemoryOpfsRoot();
|
|
const migrationStore = new OpfsGraphStore("chat-opfs-legacy", {
|
|
rootDirectoryFactory: async () => migrationRoot,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW,
|
|
});
|
|
await migrationStore.open();
|
|
|
|
const nowMs = 1_700_000_000_000;
|
|
const migrated = await migrationStore.importLegacyGraph(
|
|
buildLegacyGraph("chat-opfs-legacy"),
|
|
{
|
|
nowMs,
|
|
source: "chat_metadata",
|
|
legacyRetentionMs: 5000,
|
|
revision: 7,
|
|
},
|
|
);
|
|
assert.equal(migrated.migrated, true);
|
|
assert.equal(migrated.skipped, false);
|
|
assert.equal(migrated.reason, "migrated");
|
|
assert.equal(migrated.revision, 7);
|
|
|
|
const migratedSnapshot = await migrationStore.exportSnapshot();
|
|
assert.equal(migratedSnapshot.meta.migrationCompletedAt, nowMs);
|
|
assert.equal(migratedSnapshot.meta.migrationSource, "chat_metadata");
|
|
assert.equal(migratedSnapshot.meta.legacyRetentionUntil, nowMs + 5000);
|
|
assert.equal(migratedSnapshot.meta.storagePrimary, "opfs");
|
|
assert.equal(migratedSnapshot.meta.storageMode, "opfs-primary");
|
|
assert.equal(migratedSnapshot.nodes.length, 1);
|
|
assert.equal(migratedSnapshot.edges.length, 1);
|
|
assert.equal(migratedSnapshot.nodes[0]?.sourceFloor, 5);
|
|
assert.equal(migratedSnapshot.edges[0]?.sourceFloor, 5);
|
|
|
|
const repeatedMigration = await migrationStore.importLegacyGraph(
|
|
buildLegacyGraph("chat-opfs-legacy"),
|
|
{
|
|
nowMs: nowMs + 1000,
|
|
source: "chat_metadata",
|
|
},
|
|
);
|
|
assert.equal(repeatedMigration.migrated, false);
|
|
assert.equal(repeatedMigration.skipped, true);
|
|
assert.equal(repeatedMigration.reason, "migration-already-completed");
|
|
|
|
await migrationStore.close();
|
|
|
|
const nonEmptyRoot = createMemoryOpfsRoot();
|
|
const nonEmptyStore = new OpfsGraphStore("chat-opfs-non-empty", {
|
|
rootDirectoryFactory: async () => nonEmptyRoot,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW,
|
|
});
|
|
await nonEmptyStore.open();
|
|
await nonEmptyStore.importSnapshot(
|
|
{
|
|
meta: {
|
|
revision: 2,
|
|
},
|
|
state: {},
|
|
nodes: [
|
|
{
|
|
id: "existing-node",
|
|
type: "event",
|
|
fields: {
|
|
title: "existing",
|
|
},
|
|
archived: false,
|
|
updatedAt: 1,
|
|
},
|
|
],
|
|
edges: [],
|
|
tombstones: [],
|
|
},
|
|
{
|
|
mode: "replace",
|
|
preserveRevision: true,
|
|
},
|
|
);
|
|
|
|
const skippedBecauseNonEmpty = await nonEmptyStore.importLegacyGraph(
|
|
buildLegacyGraph("chat-opfs-non-empty"),
|
|
{
|
|
nowMs,
|
|
source: "chat_metadata",
|
|
},
|
|
);
|
|
assert.equal(skippedBecauseNonEmpty.migrated, false);
|
|
assert.equal(skippedBecauseNonEmpty.skipped, true);
|
|
assert.equal(skippedBecauseNonEmpty.reason, "local-store-not-empty");
|
|
|
|
const nonEmptySnapshot = await nonEmptyStore.exportSnapshot();
|
|
assert.deepEqual(nonEmptySnapshot.nodes.map((item) => item.id), ["existing-node"]);
|
|
|
|
await nonEmptyStore.close();
|
|
}
|
|
|
|
async function testPruneExpiredTombstonesAndClearAll() {
|
|
const rootDirectory = createMemoryOpfsRoot();
|
|
const store = new OpfsGraphStore("chat-opfs-prune", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_SHADOW,
|
|
});
|
|
await store.open();
|
|
|
|
const nowMs = BME_TOMBSTONE_RETENTION_MS + 100_000;
|
|
await store.importSnapshot(
|
|
{
|
|
meta: {
|
|
revision: 3,
|
|
},
|
|
state: {},
|
|
nodes: [],
|
|
edges: [],
|
|
tombstones: [
|
|
{
|
|
id: "ts-old",
|
|
kind: "node",
|
|
targetId: "node-old",
|
|
sourceDeviceId: "device-1",
|
|
deletedAt: nowMs - BME_TOMBSTONE_RETENTION_MS - 1,
|
|
},
|
|
{
|
|
id: "ts-fresh",
|
|
kind: "edge",
|
|
targetId: "edge-fresh",
|
|
sourceDeviceId: "device-1",
|
|
deletedAt: nowMs - BME_TOMBSTONE_RETENTION_MS + 1,
|
|
},
|
|
],
|
|
},
|
|
{
|
|
mode: "replace",
|
|
preserveRevision: true,
|
|
},
|
|
);
|
|
|
|
const emptyIgnoringTombstones = await store.isEmpty();
|
|
assert.equal(emptyIgnoringTombstones.empty, true);
|
|
const emptyIncludingTombstones = await store.isEmpty({ includeTombstones: true });
|
|
assert.equal(emptyIncludingTombstones.empty, false);
|
|
|
|
const pruneResult = await store.pruneExpiredTombstones(nowMs);
|
|
assert.equal(pruneResult.pruned, 1);
|
|
assert.equal(pruneResult.revision, 4);
|
|
|
|
const afterPruneSnapshot = await store.exportSnapshot();
|
|
assert.deepEqual(afterPruneSnapshot.tombstones.map((item) => item.id), ["ts-fresh"]);
|
|
|
|
const clearResult = await store.clearAll();
|
|
assert.equal(clearResult.cleared, true);
|
|
assert.equal(clearResult.revision, 5);
|
|
|
|
const afterClearSnapshot = await store.exportSnapshot();
|
|
assert.equal(afterClearSnapshot.nodes.length, 0);
|
|
assert.equal(afterClearSnapshot.edges.length, 0);
|
|
assert.equal(afterClearSnapshot.tombstones.length, 0);
|
|
assert.equal(afterClearSnapshot.meta.storagePrimary, "opfs");
|
|
assert.equal(afterClearSnapshot.meta.storageMode, "opfs-primary");
|
|
|
|
const emptyAfterClear = await store.isEmpty({ includeTombstones: true });
|
|
assert.equal(emptyAfterClear.empty, true);
|
|
|
|
await store.close();
|
|
}
|
|
|
|
async function testDeleteCurrentAndAllOpfsStorage() {
|
|
const rootDirectory = createMemoryOpfsRoot();
|
|
const storeA = new OpfsGraphStore("chat-opfs-delete-a", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY,
|
|
});
|
|
const storeB = new OpfsGraphStore("chat-opfs-delete-b", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
storeMode: BME_GRAPH_LOCAL_STORAGE_MODE_OPFS_PRIMARY,
|
|
});
|
|
await storeA.open();
|
|
await storeB.open();
|
|
|
|
await storeA.importSnapshot(
|
|
{
|
|
meta: { revision: 1 },
|
|
state: { lastProcessedFloor: 1, extractionCount: 1 },
|
|
nodes: [{ id: "node-a", type: "event", fields: { title: "A" }, archived: false, updatedAt: 1 }],
|
|
edges: [],
|
|
tombstones: [],
|
|
},
|
|
{ mode: "replace", preserveRevision: true },
|
|
);
|
|
await storeB.importSnapshot(
|
|
{
|
|
meta: { revision: 1 },
|
|
state: { lastProcessedFloor: 1, extractionCount: 1 },
|
|
nodes: [{ id: "node-b", type: "event", fields: { title: "B" }, archived: false, updatedAt: 1 }],
|
|
edges: [],
|
|
tombstones: [],
|
|
},
|
|
{ mode: "replace", preserveRevision: true },
|
|
);
|
|
|
|
const deleteCurrentResult = await deleteOpfsChatStorage("chat-opfs-delete-a", {
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
});
|
|
assert.equal(deleteCurrentResult.deleted, true);
|
|
|
|
const chatsDirectory = getNestedDirectory(
|
|
getNestedDirectory(rootDirectory, "st-bme"),
|
|
"chats",
|
|
);
|
|
assert.equal(chatsDirectory.directories.has(encodeURIComponent("chat-opfs-delete-a")), false);
|
|
assert.equal(chatsDirectory.directories.has(encodeURIComponent("chat-opfs-delete-b")), true);
|
|
|
|
const deleteAllResult = await deleteAllOpfsStorage({
|
|
rootDirectoryFactory: async () => rootDirectory,
|
|
});
|
|
assert.equal(deleteAllResult.deleted, true);
|
|
assert.equal(rootDirectory.directories.has("st-bme"), false);
|
|
}
|
|
|
|
async function main() {
|
|
console.log(`${PREFIX} starting`);
|
|
|
|
await testDetectOpfsSupport();
|
|
await testImportExportPersistenceAndFileRotation();
|
|
await testImportLegacyGraphMigrationAndSkipPaths();
|
|
await testPruneExpiredTombstonesAndClearAll();
|
|
await testDeleteCurrentAndAllOpfsStorage();
|
|
|
|
console.log("opfs-persistence tests passed");
|
|
}
|
|
|
|
await main();
|