perf: add native hydrate wasm path

tests/default-settings.mjs

@@ -76,6 +76,8 @@ assert.equal(defaultSettings.persistNativeDeltaThresholdRecords, 20000);
 assert.equal(defaultSettings.persistNativeDeltaThresholdStructuralDelta, 600);
 assert.equal(defaultSettings.persistNativeDeltaThresholdSerializedChars, 4000000);
 assert.equal(defaultSettings.persistNativeDeltaBridgeMode, "json");
+assert.equal(defaultSettings.loadUseNativeHydrate, false);
+assert.equal(defaultSettings.loadNativeHydrateThresholdRecords, 12000);
 assert.equal(defaultSettings.nativeEngineFailOpen, true);
 assert.equal(defaultSettings.graphNativeForceDisable, false);
 assert.equal(defaultSettings.taskProfilesVersion, 3);

tests/graph-persistence.mjs

@@ -9,6 +9,7 @@ import {
   buildGraphFromSnapshot,
   buildPersistDelta,
   buildSnapshotFromGraph,
+  evaluateNativeHydrateGate,
   evaluatePersistNativeDeltaGate,
 } from "../sync/bme-db.js";
 import { onMessageReceivedController } from "../host/event-binding.js";
@@ -1032,6 +1033,7 @@ async function createGraphPersistenceHarness({
     buildGraphFromSnapshot,
     buildPersistDelta,
     buildSnapshotFromGraph,
+    evaluateNativeHydrateGate,
     evaluatePersistNativeDeltaGate,
     buildBmeDbName,
     BME_GRAPH_LOCAL_STORAGE_MODE_AUTO: "auto",

tests/native-hydrate-failopen.mjs

@@ -0,0 +1,57 @@
+import assert from "node:assert/strict";
+
+function moduleUrl(tag) {
+  return `../vendor/wasm/stbme_core.js?test=${Date.now()}-${tag}`;
+}
+
+globalThis.__stBmeDisableWasmPackArtifacts = true;
+delete globalThis.__stBmeLoadRustWasmLayout;
+
+const firstLoad = await import(moduleUrl("native-hydrate-first"));
+let firstError = "";
+try {
+  await firstLoad.installNativeHydrateHook();
+} catch (error) {
+  firstError = error?.message || String(error);
+}
+
+assert.match(
+  firstError,
+  /native module unavailable|native hydrate builder unavailable|global-loader|Rust\/WASM artifact is not initialized/i,
+);
+
+globalThis.__stBmeLoadRustWasmLayout = async () => ({
+  solve_layout() {
+    return {
+      ok: true,
+      positions: [],
+      diagnostics: {
+        solver: "mock-rust-wasm",
+      },
+    };
+  },
+  build_hydrate_records() {
+    return {
+      ok: true,
+      usedNative: true,
+      nodes: [],
+      edges: [],
+      diagnostics: {
+        solver: "mock-rust-wasm",
+        nodeCount: 0,
+        edgeCount: 0,
+        recordsNormalized: false,
+      },
+    };
+  },
+});
+
+const retryStatus = await firstLoad.installNativeHydrateHook();
+assert.equal(retryStatus.loaded, true);
+assert.equal(typeof globalThis.__stBmeNativeHydrateSnapshotRecords, "function");
+
+delete globalThis.__stBmeNativeHydrateSnapshotRecords;
+delete globalThis.__stBmeLoadRustWasmLayout;
+delete globalThis.__stBmeDisableWasmPackArtifacts;
+
+console.log("native-hydrate-failopen tests passed");

tests/native-hydrate-hook.mjs

@@ -0,0 +1,208 @@
+import assert from "node:assert/strict";
+
+import {
+  BME_RUNTIME_HISTORY_META_KEY,
+  BME_RUNTIME_RECORDS_NORMALIZED_META_KEY,
+  BME_RUNTIME_VECTOR_META_KEY,
+  buildGraphFromSnapshot,
+  evaluateNativeHydrateGate,
+  resolveNativeHydrateGateOptions,
+} from "../sync/bme-db.js";
+
+function cloneValue(value) {
+  if (typeof globalThis.structuredClone === "function") {
+    return globalThis.structuredClone(value);
+  }
+  return JSON.parse(JSON.stringify(value));
+}
+
+const snapshot = {
+  meta: {
+    chatId: "chat-native-hydrate",
+    revision: 3,
+    [BME_RUNTIME_RECORDS_NORMALIZED_META_KEY]: true,
+    [BME_RUNTIME_HISTORY_META_KEY]: {
+      chatId: "chat-native-hydrate",
+      lastProcessedAssistantFloor: 7,
+      extractionCount: 2,
+      processedMessageHashes: {},
+      processedMessageHashVersion: 1,
+      processedMessageHashesNeedRefresh: false,
+      recentRecallOwnerKeys: [],
+      activeRecallOwnerKey: "",
+      activeRegion: "",
+      activeRegionSource: "",
+      activeStorySegmentId: "",
+      activeStoryTimeLabel: "",
+      activeStoryTimeSource: "",
+      lastBatchStatus: null,
+      lastMutationSource: "test",
+      lastExtractedRegion: "",
+      lastExtractedStorySegmentId: "",
+      activeCharacterPovOwner: "",
+      activeUserPovOwner: "",
+    },
+    [BME_RUNTIME_VECTOR_META_KEY]: {
+      chatId: "chat-native-hydrate",
+      collectionId: "",
+      hashToNodeId: {},
+      nodeToHash: {},
+      replayRequiredNodeIds: [],
+      dirty: false,
+      dirtyReason: "",
+      pendingRepairFromFloor: null,
+      lastIntegrityIssue: null,
+      lastStats: {
+        nodesIndexed: 0,
+        updatedAt: 0,
+      },
+    },
+  },
+  state: {
+    lastProcessedFloor: 7,
+    extractionCount: 2,
+  },
+  nodes: [
+    {
+      id: "native-node-1",
+      type: "event",
+      updatedAt: 10,
+      fields: {
+        title: "Native Node",
+      },
+      embedding: [1, 2, 3],
+      scope: {
+        ownerType: "character",
+        ownerId: "owner-1",
+        layer: "objective",
+        regionPrimary: "camp",
+        regionPath: ["camp"],
+        regionSecondary: [],
+      },
+      storyTime: {
+        label: "Dawn",
+        tense: "unknown",
+      },
+      storyTimeSpan: {
+        startLabel: "Dawn",
+        endLabel: "Dawn",
+        mixed: false,
+      },
+    },
+  ],
+  edges: [
+    {
+      id: "native-edge-1",
+      fromId: "native-node-1",
+      toId: "native-node-2",
+      relation: "related",
+      scope: {
+        ownerType: "character",
+        ownerId: "owner-1",
+        layer: "objective",
+        regionPrimary: "camp",
+        regionPath: ["camp"],
+        regionSecondary: [],
+      },
+    },
+  ],
+  tombstones: [],
+};
+
+const defaultGate = resolveNativeHydrateGateOptions({});
+assert.equal(defaultGate.minSnapshotRecords, 12000);
+const gatedSmall = evaluateNativeHydrateGate(snapshot, {});
+assert.equal(gatedSmall.allowed, false);
+assert.deepEqual(gatedSmall.reasons, ["below-min-snapshot-records"]);
+const gatedLarge = evaluateNativeHydrateGate(
+  {
+    nodes: new Array(6000).fill({ id: "node-x" }),
+    edges: new Array(6000).fill({ id: "edge-x" }),
+  },
+  {},
+);
+assert.equal(gatedLarge.allowed, true);
+assert.deepEqual(gatedLarge.reasons, []);
+
+const originalNativeBuilder = globalThis.__stBmeNativeHydrateSnapshotRecords;
+
+globalThis.__stBmeNativeHydrateSnapshotRecords = (snapshotView = {}, options = {}) => {
+  assert.equal(options.recordsNormalized, true);
+  return {
+    ok: true,
+    usedNative: true,
+    nodes: cloneValue(snapshotView.nodes).map((node) => ({
+      ...node,
+      nativeHydrated: true,
+    })),
+    edges: cloneValue(snapshotView.edges).map((edge) => ({
+      ...edge,
+      nativeHydrated: true,
+    })),
+    diagnostics: {
+      solver: "test-native-hydrate",
+      nodeCount: Array.isArray(snapshotView.nodes) ? snapshotView.nodes.length : 0,
+      edgeCount: Array.isArray(snapshotView.edges) ? snapshotView.edges.length : 0,
+      recordsNormalized: options.recordsNormalized === true,
+    },
+  };
+};
+
+let nativeDiagnostics = null;
+const rebuilt = buildGraphFromSnapshot(snapshot, {
+  chatId: "chat-native-hydrate",
+  useNativeHydrate: true,
+  minSnapshotRecords: 0,
+  onDiagnostics(snapshotValue) {
+    nativeDiagnostics = snapshotValue;
+  },
+});
+assert.equal(rebuilt.nodes[0].nativeHydrated, true);
+assert.equal(rebuilt.edges[0].nativeHydrated, true);
+assert.equal(rebuilt.historyState.lastProcessedAssistantFloor, 7);
+assert.equal(nativeDiagnostics.nativeRequested, true);
+assert.equal(nativeDiagnostics.nativeUsed, true);
+assert.equal(nativeDiagnostics.nativeStatus, "ok");
+assert.equal(nativeDiagnostics.nativeGateAllowed, true);
+assert.equal(nativeDiagnostics.nativeModuleDiagnostics?.solver, "test-native-hydrate");
+assert.equal(Number.isFinite(nativeDiagnostics.nativeRecordsMs), true);
+rebuilt.nodes[0].fields.title = "Mutated Native Node";
+rebuilt.nodes[0].embedding[0] = 99;
+assert.equal(snapshot.nodes[0].fields.title, "Native Node");
+assert.equal(snapshot.nodes[0].embedding[0], 1);
+
+delete globalThis.__stBmeNativeHydrateSnapshotRecords;
+
+let fallbackDiagnostics = null;
+const fallbackGraph = buildGraphFromSnapshot(snapshot, {
+  chatId: "chat-native-hydrate",
+  useNativeHydrate: true,
+  minSnapshotRecords: 0,
+  onDiagnostics(snapshotValue) {
+    fallbackDiagnostics = snapshotValue;
+  },
+});
+assert.equal(fallbackGraph.nodes.length, 1);
+assert.equal(fallbackDiagnostics.nativeRequested, true);
+assert.equal(fallbackDiagnostics.nativeUsed, false);
+assert.equal(fallbackDiagnostics.nativeStatus, "builder-unavailable");
+
+let threwUnavailable = false;
+try {
+  buildGraphFromSnapshot(snapshot, {
+    chatId: "chat-native-hydrate",
+    useNativeHydrate: true,
+    minSnapshotRecords: 0,
+    nativeFailOpen: false,
+  });
+} catch (error) {
+  threwUnavailable =
+    String(error?.message || "") === "native-hydrate-builder-unavailable";
+}
+assert.equal(threwUnavailable, true);
+
+if (typeof originalNativeBuilder === "function") {
+  globalThis.__stBmeNativeHydrateSnapshotRecords = originalNativeBuilder;
+}
+
+console.log("native-hydrate-hook tests passed");

tests/native-layout-wrapper.mjs

@@ -22,6 +22,24 @@ try {
         },
       };
     },
+    build_hydrate_records(payload = {}) {
+      return {
+        ok: true,
+        usedNative: true,
+        nodes: Array.isArray(payload?.nodes)
+          ? payload.nodes.map((node) => ({ ...node, nativeHydrated: true }))
+          : [],
+        edges: Array.isArray(payload?.edges)
+          ? payload.edges.map((edge) => ({ ...edge, nativeHydrated: true }))
+          : [],
+        diagnostics: {
+          solver: "mock-loader",
+          nodeCount: Array.isArray(payload?.nodes) ? payload.nodes.length : 0,
+          edgeCount: Array.isArray(payload?.edges) ? payload.edges.length : 0,
+          recordsNormalized: payload?.recordsNormalized === true,
+        },
+      };
+    },
     build_persist_delta_compact(payload = {}) {
       return {
         upsertNodeIds: Array.isArray(payload?.afterNodes?.ids)
@@ -105,8 +123,29 @@ try {
   assert.deepEqual(deltaResult.upsertNodes, [{ id: "persist-native-node", marker: "after-chat" }]);
   assert.equal(deltaResult.runtimeMetaPatch.native, true);
 
+  const hydrateInstallStatus = await wrapper.installNativeHydrateHook();
+  assert.equal(hydrateInstallStatus.loaded, true);
+  assert.equal(
+    typeof globalThis.__stBmeNativeHydrateSnapshotRecords,
+    "function",
+  );
+  const hydrateResult = globalThis.__stBmeNativeHydrateSnapshotRecords(
+    {
+      nodes: [{ id: "hydrate-node", type: "event" }],
+      edges: [{ id: "hydrate-edge", fromId: "hydrate-node", toId: "hydrate-node-2" }],
+    },
+    {
+      recordsNormalized: true,
+    },
+  );
+  assert.equal(hydrateResult.ok, true);
+  assert.equal(hydrateResult.nodes[0].nativeHydrated, true);
+  assert.equal(hydrateResult.edges[0].nativeHydrated, true);
+  assert.equal(hydrateResult.diagnostics.recordsNormalized, true);
+
   delete globalThis.__stBmeLoadRustWasmLayout;
   delete globalThis.__stBmeNativeBuildPersistDelta;
+  delete globalThis.__stBmeNativeHydrateSnapshotRecords;
   delete globalThis.__stBmeDisableWasmPackArtifacts;
 
   const wrapperNoLoader = await importFreshWrapper("no-loader");
@@ -136,6 +175,7 @@ try {
   }
   delete globalThis.__stBmeDisableWasmPackArtifacts;
   delete globalThis.__stBmeNativeBuildPersistDelta;
+  delete globalThis.__stBmeNativeHydrateSnapshotRecords;
 }
 
 console.log("native-layout-wrapper tests passed");

tests/perf/persist-load-bench.mjs

@@ -13,12 +13,27 @@ import { createMemoryOpfsRoot } from "../helpers/memory-opfs.mjs";
 
 const RUNS = 4;
 const outputJson = process.argv.includes("--json");
+const useNativeHydrate = process.argv.includes("--native-hydrate");
+const nativeHydrateThresholdArg = process.argv.find((entry) =>
+  String(entry || "").startsWith("--native-hydrate-threshold="),
+);
+const nativeHydrateThresholdRecords = nativeHydrateThresholdArg
+  ? Math.max(
+      0,
+      Math.floor(
+        Number(String(nativeHydrateThresholdArg).split("=").slice(1).join("=") || 0) || 0,
+      ),
+    )
+  : undefined;
 const SIZE_PRESETS = [
   { label: "M", seed: 17, nodeCount: 1200, edgeCount: 3600, churn: 0.08 },
   { label: "L", seed: 29, nodeCount: 3600, edgeCount: 10800, churn: 0.1 },
   { label: "XL", seed: 43, nodeCount: 7200, edgeCount: 21600, churn: 0.12 },
 ];
 
+let nativeHydratePreloadStatus = useNativeHydrate ? "pending" : "not-requested";
+let nativeHydratePreloadError = "";
+
 function summarize(values = []) {
   if (!values.length) {
     return { avg: 0, p95: 0, min: 0, max: 0 };
@@ -218,6 +233,9 @@ function measureHydrate(snapshot, chatId) {
   const startedAt = performance.now();
   buildGraphFromSnapshot(snapshot, {
     chatId,
+    useNativeHydrate,
+    loadNativeHydrateThresholdRecords: nativeHydrateThresholdRecords,
+    nativeFailOpen: true,
     onDiagnostics(snapshotValue) {
       diagnostics = snapshotValue;
     },
@@ -266,8 +284,10 @@ async function runPreset(preset) {
   const hydrateStateSamples = [];
   const hydrateNormalizeSamples = [];
   const hydrateIntegritySamples = [];
+  const hydrateNativeRecordsSamples = [];
   const walFileWriteSamples = [];
   const manifestFileWriteSamples = [];
+  let hydrateNativeUsedRuns = 0;
 
   for (let run = 0; run < RUNS; run += 1) {
     const pair = buildBenchPair({
@@ -306,6 +326,12 @@ async function runPreset(preset) {
     hydrateStateSamples.push(Number(hydrateResult.diagnostics?.stateMs || 0));
     hydrateNormalizeSamples.push(Number(hydrateResult.diagnostics?.normalizeMs || 0));
     hydrateIntegritySamples.push(Number(hydrateResult.diagnostics?.integrityMs || 0));
+    hydrateNativeRecordsSamples.push(
+      Number(hydrateResult.diagnostics?.nativeRecordsMs || 0),
+    );
+    if (hydrateResult.diagnostics?.nativeUsed === true) {
+      hydrateNativeUsedRuns += 1;
+    }
     walFileWriteSamples.push(Number(opfsCommitResult.diagnostics?.walFileWriteMs || 0));
     manifestFileWriteSamples.push(
       Number(opfsCommitResult.diagnostics?.manifestFileWriteMs || 0),
@@ -321,6 +347,11 @@ async function runPreset(preset) {
     hydrateStateMs: summarize(hydrateStateSamples),
     hydrateNormalizeMs: summarize(hydrateNormalizeSamples),
     hydrateIntegrityMs: summarize(hydrateIntegritySamples),
+    hydrateNativeRecordsMs: summarize(hydrateNativeRecordsSamples),
+    hydrateNativeUsedRuns,
+    nativeHydrateRequested: useNativeHydrate,
+    nativeHydrateThresholdRecords:
+      nativeHydrateThresholdRecords == null ? null : nativeHydrateThresholdRecords,
     hydrateRuntimeMetaMs: summarize(hydrateRuntimeMetaSamples),
     opfsCommitMs: summarize(opfsCommitSamples),
     opfsWalFileWriteMs: summarize(walFileWriteSamples),
@@ -338,6 +369,8 @@ async function runPreset(preset) {
       `edgesP95=${result.hydrateEdgesMs.p95.toFixed(2)}ms`,
       `normalizeP95=${result.hydrateNormalizeMs.p95.toFixed(2)}ms`,
       `integrityP95=${result.hydrateIntegrityMs.p95.toFixed(2)}ms`,
+      `nativeRecordsP95=${result.hydrateNativeRecordsMs.p95.toFixed(2)}ms`,
+      `nativeUsed=${result.hydrateNativeUsedRuns}/${RUNS}`,
       `runtimeMetaP95=${result.hydrateRuntimeMetaMs.p95.toFixed(2)}ms`,
     );
     console.log(
@@ -350,15 +383,36 @@ async function runPreset(preset) {
 }
 
 async function main() {
-  const results = {};
-  for (const preset of SIZE_PRESETS) {
-    results[preset.label] = await runPreset(preset);
+  if (useNativeHydrate) {
+    try {
+      const nativeModule = await import("../../vendor/wasm/stbme_core.js");
+      const nativeStatus = await nativeModule?.installNativeHydrateHook?.();
+      nativeHydratePreloadStatus = nativeStatus?.loaded ? "loaded" : "not-loaded";
+      nativeHydratePreloadError = String(nativeStatus?.error || "");
+    } catch (error) {
+      nativeHydratePreloadStatus = "failed";
+      nativeHydratePreloadError = error?.message || String(error);
+      console.warn(
+        "[ST-BME][persist-load-bench] native hydrate preload failed, fallback to JS hydrate:",
+        error,
+      );
+    }
   }
+  const presets = {};
+  for (const preset of SIZE_PRESETS) {
+    presets[preset.label] = await runPreset(preset);
+  }
+  const payload = {
+    runs: RUNS,
+    nativeHydrateRequested: useNativeHydrate,
+    nativeHydratePreloadStatus,
+    nativeHydratePreloadError,
+    nativeHydrateThresholdRecords:
+      nativeHydrateThresholdRecords == null ? null : nativeHydrateThresholdRecords,
+    presets,
+  };
   if (outputJson) {
-    console.log(JSON.stringify({
-      runs: RUNS,
-      presets: results,
-    }));
+    console.log(JSON.stringify(payload));
   }
 }