mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 14:20:35 +08:00
perf: add native hydrate wasm path
This commit is contained in:
Youzini-afk
192
index.js
192
index.js
@@ -23,6 +23,7 @@ import {
|
||||
buildPersistDelta,
|
||||
buildGraphFromSnapshot,
|
||||
buildSnapshotFromGraph,
|
||||
evaluateNativeHydrateGate,
|
||||
evaluatePersistNativeDeltaGate,
|
||||
ensureDexieLoaded,
|
||||
} from "./sync/bme-db.js";
|
||||
@@ -1207,6 +1208,7 @@ let isRecalling = false;
|
||||
let activeRecallPromise = null;
|
||||
let recallRunSequence = 0;
|
||||
let nativePersistDeltaInstallPromise = null;
|
||||
let nativeHydrateInstallPromise = null;
|
||||
let lastInjectionContent = "";
|
||||
let lastExtractedItems = []; // 最近提取的节点（面板展示用）
|
||||
let lastRecalledItems = []; // 最近召回的节点（面板展示用）
|
||||
@@ -9122,6 +9124,14 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
storageMode = storagePrimary,
|
||||
statusLabel = "IndexedDB",
|
||||
reasonPrefix = "indexeddb",
|
||||
currentSettings = null,
|
||||
nativeHydrateRequested = null,
|
||||
nativeHydrateForceDisabled = null,
|
||||
nativeHydrateGate = null,
|
||||
nativeHydratePreloadStatus = "",
|
||||
nativeHydratePreloadMs = 0,
|
||||
nativeHydratePreloadError = "",
|
||||
nativeHydrateModuleStatus = null,
|
||||
} = {},
|
||||
) {
|
||||
const normalizedChatId = normalizeChatIdCandidate(chatId);
|
||||
@@ -9221,10 +9231,35 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
}
|
||||
let graphFromSnapshot = null;
|
||||
let hydrateDiagnostics = null;
|
||||
const effectiveSettings = currentSettings || getSettings();
|
||||
const resolvedNativeHydrateRequested =
|
||||
nativeHydrateRequested == null
|
||||
? effectiveSettings.loadUseNativeHydrate === true
|
||||
: nativeHydrateRequested === true;
|
||||
const resolvedNativeHydrateForceDisabled =
|
||||
nativeHydrateForceDisabled == null
|
||||
? effectiveSettings.graphNativeForceDisable === true
|
||||
: nativeHydrateForceDisabled === true;
|
||||
const resolvedNativeHydrateGate =
|
||||
nativeHydrateGate && typeof nativeHydrateGate === "object"
|
||||
? nativeHydrateGate
|
||||
: evaluateNativeHydrateGate(snapshot, effectiveSettings);
|
||||
const shouldUseNativeHydrate =
|
||||
resolvedNativeHydrateRequested &&
|
||||
resolvedNativeHydrateForceDisabled !== true &&
|
||||
resolvedNativeHydrateGate.allowed;
|
||||
const resolvedNativeHydratePreloadStatus = String(
|
||||
nativeHydratePreloadStatus ||
|
||||
(resolvedNativeHydrateRequested ? "not-preloaded" : "not-requested"),
|
||||
);
|
||||
try {
|
||||
const hydrateStartedAt = readLoadDiagnosticsNow();
|
||||
graphFromSnapshot = buildGraphFromSnapshot(snapshot, {
|
||||
chatId: normalizedChatId,
|
||||
useNativeHydrate: shouldUseNativeHydrate,
|
||||
nativeFailOpen: effectiveSettings.nativeEngineFailOpen !== false,
|
||||
loadNativeHydrateThresholdRecords:
|
||||
effectiveSettings.loadNativeHydrateThresholdRecords,
|
||||
onDiagnostics(snapshotValue) {
|
||||
hydrateDiagnostics =
|
||||
snapshotValue &&
|
||||
@@ -9277,6 +9312,17 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
integrityReasons: Array.isArray(error?.reasons) ? error.reasons : [],
|
||||
chatId: normalizedChatId,
|
||||
attemptIndex,
|
||||
hydrateDiagnostics: cloneRuntimeDebugValue(hydrateDiagnostics, null),
|
||||
nativeHydrateRequested: resolvedNativeHydrateRequested,
|
||||
nativeHydrateForceDisabled: resolvedNativeHydrateForceDisabled,
|
||||
nativeHydrateGate: cloneRuntimeDebugValue(resolvedNativeHydrateGate, null),
|
||||
nativeHydratePreloadStatus: resolvedNativeHydratePreloadStatus,
|
||||
nativeHydratePreloadMs: nativeHydratePreloadMs,
|
||||
nativeHydratePreloadError: nativeHydratePreloadError,
|
||||
nativeHydrateModuleStatus: cloneRuntimeDebugValue(
|
||||
nativeHydrateModuleStatus,
|
||||
null,
|
||||
),
|
||||
};
|
||||
recordLoadDiagnostics({
|
||||
success: false,
|
||||
@@ -9296,6 +9342,27 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
hydrateIntegrityMs: normalizeLoadDiagnosticsMs(
|
||||
hydrateDiagnostics?.integrityMs,
|
||||
),
|
||||
hydrateNativeRequested: resolvedNativeHydrateRequested,
|
||||
hydrateNativeForceDisabled: resolvedNativeHydrateForceDisabled,
|
||||
hydrateNativeGateAllowed: resolvedNativeHydrateGate.allowed === true,
|
||||
hydrateNativeGateReasons: cloneRuntimeDebugValue(
|
||||
resolvedNativeHydrateGate.reasons,
|
||||
[],
|
||||
),
|
||||
hydrateNativePreloadStatus: resolvedNativeHydratePreloadStatus,
|
||||
hydrateNativePreloadMs: normalizeLoadDiagnosticsMs(nativeHydratePreloadMs),
|
||||
hydrateNativePreloadError: String(nativeHydratePreloadError || ""),
|
||||
hydrateNativeModuleLoaded: Boolean(nativeHydrateModuleStatus?.loaded),
|
||||
hydrateNativeModuleSource: String(nativeHydrateModuleStatus?.source || ""),
|
||||
hydrateNativeModuleError: String(
|
||||
nativeHydrateModuleStatus?.error || nativeHydratePreloadError || "",
|
||||
),
|
||||
hydrateNativeUsed: hydrateDiagnostics?.nativeUsed === true,
|
||||
hydrateNativeStatus: String(hydrateDiagnostics?.nativeStatus || ""),
|
||||
hydrateNativeError: String(hydrateDiagnostics?.nativeError || ""),
|
||||
hydrateNativeRecordsMs: normalizeLoadDiagnosticsMs(
|
||||
hydrateDiagnostics?.nativeRecordsMs,
|
||||
),
|
||||
error: error?.message || String(error),
|
||||
integrityReasons: Array.isArray(error?.reasons) ? [...error.reasons] : [],
|
||||
});
|
||||
@@ -9405,6 +9472,17 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
attemptIndex,
|
||||
shadowSnapshotUsed: false,
|
||||
revision,
|
||||
hydrateDiagnostics: cloneRuntimeDebugValue(hydrateDiagnostics, null),
|
||||
nativeHydrateRequested: resolvedNativeHydrateRequested,
|
||||
nativeHydrateForceDisabled: resolvedNativeHydrateForceDisabled,
|
||||
nativeHydrateGate: cloneRuntimeDebugValue(resolvedNativeHydrateGate, null),
|
||||
nativeHydratePreloadStatus: resolvedNativeHydratePreloadStatus,
|
||||
nativeHydratePreloadMs: nativeHydratePreloadMs,
|
||||
nativeHydratePreloadError: nativeHydratePreloadError,
|
||||
nativeHydrateModuleStatus: cloneRuntimeDebugValue(
|
||||
nativeHydrateModuleStatus,
|
||||
null,
|
||||
),
|
||||
};
|
||||
recordLoadDiagnostics({
|
||||
success: true,
|
||||
@@ -9424,6 +9502,27 @@ function applyIndexedDbSnapshotToRuntime(
|
||||
hydrateIntegrityMs: normalizeLoadDiagnosticsMs(
|
||||
hydrateDiagnostics?.integrityMs,
|
||||
),
|
||||
hydrateNativeRequested: resolvedNativeHydrateRequested,
|
||||
hydrateNativeForceDisabled: resolvedNativeHydrateForceDisabled,
|
||||
hydrateNativeGateAllowed: resolvedNativeHydrateGate.allowed === true,
|
||||
hydrateNativeGateReasons: cloneRuntimeDebugValue(
|
||||
resolvedNativeHydrateGate.reasons,
|
||||
[],
|
||||
),
|
||||
hydrateNativePreloadStatus: resolvedNativeHydratePreloadStatus,
|
||||
hydrateNativePreloadMs: normalizeLoadDiagnosticsMs(nativeHydratePreloadMs),
|
||||
hydrateNativePreloadError: String(nativeHydratePreloadError || ""),
|
||||
hydrateNativeModuleLoaded: Boolean(nativeHydrateModuleStatus?.loaded),
|
||||
hydrateNativeModuleSource: String(nativeHydrateModuleStatus?.source || ""),
|
||||
hydrateNativeModuleError: String(
|
||||
nativeHydrateModuleStatus?.error || nativeHydratePreloadError || "",
|
||||
),
|
||||
hydrateNativeUsed: hydrateDiagnostics?.nativeUsed === true,
|
||||
hydrateNativeStatus: String(hydrateDiagnostics?.nativeStatus || ""),
|
||||
hydrateNativeError: String(hydrateDiagnostics?.nativeError || ""),
|
||||
hydrateNativeRecordsMs: normalizeLoadDiagnosticsMs(
|
||||
hydrateDiagnostics?.nativeRecordsMs,
|
||||
),
|
||||
applyRuntimeMs: normalizeLoadDiagnosticsMs(
|
||||
readLoadDiagnosticsNow() - applyRuntimeStartedAt,
|
||||
),
|
||||
@@ -9458,6 +9557,7 @@ async function loadGraphFromIndexedDb(
|
||||
let exportProbeMs = 0;
|
||||
let preApplyMs = 0;
|
||||
let exportSnapshotSource = "";
|
||||
const currentSettings = getSettings();
|
||||
if (!normalizedChatId) {
|
||||
const result = {
|
||||
success: false,
|
||||
@@ -9917,6 +10017,57 @@ async function loadGraphFromIndexedDb(
|
||||
}
|
||||
cacheIndexedDbSnapshot(normalizedChatId, snapshot);
|
||||
|
||||
const nativeHydrateRequested = currentSettings.loadUseNativeHydrate === true;
|
||||
const nativeHydrateForceDisabled =
|
||||
currentSettings.graphNativeForceDisable === true;
|
||||
const nativeHydrateGate = evaluateNativeHydrateGate(snapshot, currentSettings);
|
||||
const shouldUseNativeHydrate =
|
||||
nativeHydrateRequested &&
|
||||
nativeHydrateForceDisabled !== true &&
|
||||
nativeHydrateGate.allowed;
|
||||
let nativeHydrateModuleStatus = null;
|
||||
let nativeHydratePreloadStatus = nativeHydrateRequested
|
||||
? nativeHydrateForceDisabled
|
||||
? "force-disabled"
|
||||
: nativeHydrateGate.allowed
|
||||
? "pending"
|
||||
: "gated-out"
|
||||
: "not-requested";
|
||||
let nativeHydratePreloadError = "";
|
||||
let nativeHydratePreloadMs = 0;
|
||||
if (shouldUseNativeHydrate) {
|
||||
const preloadStartedAt = readLoadDiagnosticsNow();
|
||||
try {
|
||||
if (!nativeHydrateInstallPromise) {
|
||||
nativeHydrateInstallPromise = import("./vendor/wasm/stbme_core.js")
|
||||
.then((module) => module?.installNativeHydrateHook?.())
|
||||
.catch((error) => {
|
||||
nativeHydrateInstallPromise = null;
|
||||
throw error;
|
||||
});
|
||||
}
|
||||
nativeHydrateModuleStatus = await nativeHydrateInstallPromise;
|
||||
nativeHydratePreloadStatus = nativeHydrateModuleStatus?.loaded
|
||||
? "loaded"
|
||||
: "not-loaded";
|
||||
nativeHydratePreloadMs =
|
||||
readLoadDiagnosticsNow() - preloadStartedAt;
|
||||
} catch (error) {
|
||||
nativeHydratePreloadStatus = "failed";
|
||||
nativeHydratePreloadMs =
|
||||
readLoadDiagnosticsNow() - preloadStartedAt;
|
||||
nativeHydratePreloadError = error?.message || String(error);
|
||||
if (currentSettings.nativeEngineFailOpen !== false) {
|
||||
console.warn(
|
||||
"[ST-BME] native hydrate preload failed, fallback to JS hydrate:",
|
||||
error,
|
||||
);
|
||||
} else {
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
preApplyMs = readLoadDiagnosticsNow() - loadStartedAt;
|
||||
const applyInvokeStartedAt = readLoadDiagnosticsNow();
|
||||
const loadResult = applyIndexedDbSnapshotToRuntime(normalizedChatId, snapshot, {
|
||||
@@ -9926,6 +10077,14 @@ async function loadGraphFromIndexedDb(
|
||||
storageMode: snapshotStore.storageMode,
|
||||
statusLabel: snapshotStore.statusLabel,
|
||||
reasonPrefix: snapshotStore.reasonPrefix,
|
||||
currentSettings,
|
||||
nativeHydrateRequested,
|
||||
nativeHydrateForceDisabled,
|
||||
nativeHydrateGate,
|
||||
nativeHydratePreloadStatus,
|
||||
nativeHydratePreloadMs,
|
||||
nativeHydratePreloadError,
|
||||
nativeHydrateModuleStatus,
|
||||
});
|
||||
const applyInvokeMs = readLoadDiagnosticsNow() - applyInvokeStartedAt;
|
||||
const totalLoadMs = readLoadDiagnosticsNow() - loadStartedAt;
|
||||
@@ -9952,6 +10111,39 @@ async function loadGraphFromIndexedDb(
|
||||
preApplyOtherMs: normalizeLoadDiagnosticsMs(
|
||||
Math.max(0, preApplyMs - exportSnapshotMs - exportProbeMs),
|
||||
),
|
||||
hydrateNativeRequested: loadResult?.nativeHydrateRequested === true,
|
||||
hydrateNativeForceDisabled: loadResult?.nativeHydrateForceDisabled === true,
|
||||
hydrateNativeGateAllowed: loadResult?.nativeHydrateGate?.allowed === true,
|
||||
hydrateNativeGateReasons: cloneRuntimeDebugValue(
|
||||
loadResult?.nativeHydrateGate?.reasons,
|
||||
[],
|
||||
),
|
||||
hydrateNativePreloadStatus: String(
|
||||
loadResult?.nativeHydratePreloadStatus || nativeHydratePreloadStatus || "",
|
||||
),
|
||||
hydrateNativePreloadMs: normalizeLoadDiagnosticsMs(
|
||||
loadResult?.nativeHydratePreloadMs,
|
||||
),
|
||||
hydrateNativePreloadError: String(
|
||||
loadResult?.nativeHydratePreloadError || "",
|
||||
),
|
||||
hydrateNativeModuleLoaded: Boolean(
|
||||
loadResult?.nativeHydrateModuleStatus?.loaded,
|
||||
),
|
||||
hydrateNativeModuleSource: String(
|
||||
loadResult?.nativeHydrateModuleStatus?.source || "",
|
||||
),
|
||||
hydrateNativeModuleError: String(
|
||||
loadResult?.nativeHydrateModuleStatus?.error || "",
|
||||
),
|
||||
hydrateNativeUsed: loadResult?.hydrateDiagnostics?.nativeUsed === true,
|
||||
hydrateNativeStatus: String(
|
||||
loadResult?.hydrateDiagnostics?.nativeStatus || "",
|
||||
),
|
||||
hydrateNativeError: String(loadResult?.hydrateDiagnostics?.nativeError || ""),
|
||||
hydrateNativeRecordsMs: normalizeLoadDiagnosticsMs(
|
||||
loadResult?.hydrateDiagnostics?.nativeRecordsMs,
|
||||
),
|
||||
applyInvokeMs: normalizeLoadDiagnosticsMs(applyInvokeMs),
|
||||
untrackedMs: normalizeLoadDiagnosticsMs(
|
||||
Math.max(0, totalLoadMs - loadAccountedMs),
|
||||
|
||||
@@ -22,6 +22,25 @@ struct LayoutNode {
|
||||
region_rect: RegionRect,
|
||||
}
|
||||
|
||||
fn solve_hydrate_records_in_rust(payload: HydrateRecordsPayload) -> HydrateRecordsResult {
|
||||
let nodes = clone_hydrate_records(payload.nodes);
|
||||
let edges = clone_hydrate_records(payload.edges);
|
||||
let node_count = nodes.len();
|
||||
let edge_count = edges.len();
|
||||
HydrateRecordsResult {
|
||||
ok: true,
|
||||
used_native: true,
|
||||
nodes,
|
||||
edges,
|
||||
diagnostics: HydrateRecordsDiagnostics {
|
||||
solver: "rust-wasm".to_string(),
|
||||
node_count,
|
||||
edge_count,
|
||||
records_normalized: payload.records_normalized,
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct LayoutEdge {
|
||||
@@ -224,6 +243,36 @@ struct PersistDeltaIdResult {
|
||||
upsert_tombstone_ids: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct HydrateRecordsPayload {
|
||||
#[serde(default)]
|
||||
nodes: Vec<JsonValue>,
|
||||
#[serde(default)]
|
||||
edges: Vec<JsonValue>,
|
||||
#[serde(default)]
|
||||
records_normalized: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct HydrateRecordsDiagnostics {
|
||||
solver: String,
|
||||
node_count: usize,
|
||||
edge_count: usize,
|
||||
records_normalized: bool,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct HydrateRecordsResult {
|
||||
ok: bool,
|
||||
used_native: bool,
|
||||
nodes: Vec<JsonValue>,
|
||||
edges: Vec<JsonValue>,
|
||||
diagnostics: HydrateRecordsDiagnostics,
|
||||
}
|
||||
|
||||
fn default_iterations() -> u32 {
|
||||
80
|
||||
}
|
||||
@@ -299,6 +348,13 @@ fn sanitize_json_records(records: Vec<JsonValue>) -> Vec<JsonValue> {
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn clone_hydrate_records(records: Vec<JsonValue>) -> Vec<JsonValue> {
|
||||
records
|
||||
.into_iter()
|
||||
.filter(|record| record.is_object())
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn sanitize_persist_snapshot(snapshot: PersistSnapshot) -> PersistSnapshot {
|
||||
PersistSnapshot {
|
||||
meta: snapshot.meta,
|
||||
@@ -1018,3 +1074,12 @@ pub fn build_persist_delta_compact_hash(payload: JsValue) -> Result<JsValue, JsV
|
||||
JsValue::from_str(&format!("serialize hash compact persist result failed: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub fn build_hydrate_records(payload: JsValue) -> Result<JsValue, JsValue> {
|
||||
let parsed: HydrateRecordsPayload = serde_wasm_bindgen::from_value(payload)
|
||||
.map_err(|error| JsValue::from_str(&format!("invalid hydrate payload: {error}")))?;
|
||||
let solved = solve_hydrate_records_in_rust(parsed);
|
||||
serde_wasm_bindgen::to_value(&solved)
|
||||
.map_err(|error| JsValue::from_str(&format!("serialize hydrate result failed: {error}")))
|
||||
}
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
"bench:graph-layout": "node tests/perf/graph-layout-bench.mjs",
|
||||
"bench:persist-delta": "node tests/perf/persist-delta-bench.mjs",
|
||||
"bench:persist-load": "node tests/perf/persist-load-bench.mjs",
|
||||
"bench:persist-load:native-hydrate": "node tests/perf/persist-load-bench.mjs --native-hydrate",
|
||||
"bench:load-preapply": "node tests/perf/load-preapply-bench.mjs",
|
||||
"bench:p1-compare": "node scripts/compare-p1-bench.mjs",
|
||||
"bench:native": "npm run bench:graph-layout && npm run bench:persist-delta",
|
||||
|
||||
@@ -122,6 +122,8 @@ export const defaultSettings = {
|
||||
persistNativeDeltaThresholdStructuralDelta: 600,
|
||||
persistNativeDeltaThresholdSerializedChars: 4000000,
|
||||
persistNativeDeltaBridgeMode: "json",
|
||||
loadUseNativeHydrate: false,
|
||||
loadNativeHydrateThresholdRecords: 12000,
|
||||
nativeEngineFailOpen: true,
|
||||
graphNativeForceDisable: false,
|
||||
|
||||
|
||||
@@ -17,6 +17,8 @@ const args = new Map(
|
||||
const baselineRef = String(args.get("--baseline") || "origin/main");
|
||||
const currentRef = String(args.get("--current") || "HEAD");
|
||||
const outputJson = args.has("--json");
|
||||
const useNativeHydrate = args.has("--native-hydrate");
|
||||
const nativeHydrateThreshold = args.get("--native-hydrate-threshold");
|
||||
|
||||
async function runCommand(command, commandArgs, cwd) {
|
||||
const { stdout, stderr } = await execFileAsync(command, commandArgs, {
|
||||
@@ -78,9 +80,16 @@ function printRows(rows = [], title = "") {
|
||||
}
|
||||
|
||||
async function runBenchSuite(cwd) {
|
||||
const persistLoadArgs = ["tests/perf/persist-load-bench.mjs", "--json"];
|
||||
if (useNativeHydrate) {
|
||||
persistLoadArgs.push("--native-hydrate");
|
||||
}
|
||||
if (nativeHydrateThreshold !== undefined && nativeHydrateThreshold !== true) {
|
||||
persistLoadArgs.push(`--native-hydrate-threshold=${nativeHydrateThreshold}`);
|
||||
}
|
||||
const persistLoad = await runCommand(
|
||||
process.execPath,
|
||||
["tests/perf/persist-load-bench.mjs", "--json"],
|
||||
persistLoadArgs,
|
||||
cwd,
|
||||
);
|
||||
const loadPreapply = await runCommand(
|
||||
@@ -153,6 +162,11 @@ async function main() {
|
||||
baselineSha,
|
||||
currentRef,
|
||||
currentSha,
|
||||
nativeHydrateRequested: useNativeHydrate,
|
||||
nativeHydrateThreshold:
|
||||
nativeHydrateThreshold !== undefined && nativeHydrateThreshold !== true
|
||||
? String(nativeHydrateThreshold)
|
||||
: null,
|
||||
compare,
|
||||
}),
|
||||
);
|
||||
@@ -161,6 +175,15 @@ async function main() {
|
||||
|
||||
console.log(`[ST-BME][P1-compare] baseline=${baselineRef} (${baselineSha.slice(0, 7)})`);
|
||||
console.log(`[ST-BME][P1-compare] current=${currentRef} (${currentSha.slice(0, 7)})`);
|
||||
if (useNativeHydrate) {
|
||||
console.log(
|
||||
`[ST-BME][P1-compare] nativeHydrate=on threshold=${
|
||||
nativeHydrateThreshold !== undefined && nativeHydrateThreshold !== true
|
||||
? nativeHydrateThreshold
|
||||
: "default"
|
||||
}`,
|
||||
);
|
||||
}
|
||||
|
||||
printRows(
|
||||
collectMetricRows(compare, (entry) => entry.opfsCommitMs?.p95, "opfsCommitMs.p95"),
|
||||
|
||||
216
sync/bme-db.js
216
sync/bme-db.js
@@ -19,6 +19,7 @@ const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_RECORDS = 20000;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_STRUCTURAL_DELTA = 600;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_SERIALIZED_CHARS = 4000000;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE = "json";
|
||||
const DEFAULT_NATIVE_HYDRATE_THRESHOLD_RECORDS = 12000;
|
||||
const SUPPORTED_PERSIST_NATIVE_DELTA_BRIDGE_MODES = new Set(["json", "hash"]);
|
||||
const PERSIST_RECORD_SERIALIZATION_CACHE_LIMIT = 50000;
|
||||
|
||||
@@ -131,6 +132,52 @@ function estimatePersistPayloadBytes(value = null) {
|
||||
}
|
||||
}
|
||||
|
||||
function tryBuildNativeHydrateRecords(snapshotView, options = {}) {
|
||||
if (options?.useNativeHydrate !== true) {
|
||||
return {
|
||||
rawResult: null,
|
||||
status: "not-requested",
|
||||
error: "",
|
||||
};
|
||||
}
|
||||
const nativeBuilder = globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
if (typeof nativeBuilder !== "function") {
|
||||
if (options?.nativeFailOpen === false) {
|
||||
throw new Error("native-hydrate-builder-unavailable");
|
||||
}
|
||||
return {
|
||||
rawResult: null,
|
||||
status: "builder-unavailable",
|
||||
error: "native-hydrate-builder-unavailable",
|
||||
};
|
||||
}
|
||||
|
||||
try {
|
||||
return {
|
||||
rawResult: nativeBuilder(
|
||||
{
|
||||
nodes: toArray(snapshotView?.nodes),
|
||||
edges: toArray(snapshotView?.edges),
|
||||
},
|
||||
{
|
||||
recordsNormalized: options?.recordsNormalized === true,
|
||||
},
|
||||
),
|
||||
status: "ok",
|
||||
error: "",
|
||||
};
|
||||
} catch (error) {
|
||||
if (options?.nativeFailOpen === false) {
|
||||
throw error;
|
||||
}
|
||||
return {
|
||||
rawResult: null,
|
||||
status: "builder-error",
|
||||
error: error?.message || String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
function toPlainData(value, fallbackValue = null) {
|
||||
if (value == null) {
|
||||
return fallbackValue;
|
||||
@@ -303,6 +350,72 @@ function cloneHydrateSnapshotNodeRecords(records = []) {
|
||||
return output;
|
||||
}
|
||||
|
||||
function hasSharedHydrateRecordReferences(records = [], sourceRecords = []) {
|
||||
const normalizedSourceRecords = toArray(sourceRecords);
|
||||
if (!normalizedSourceRecords.length || !Array.isArray(records) || !records.length) {
|
||||
return false;
|
||||
}
|
||||
const sourceRecordSet = new WeakSet();
|
||||
for (let index = 0; index < normalizedSourceRecords.length; index += 1) {
|
||||
const record = normalizedSourceRecords[index];
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) continue;
|
||||
sourceRecordSet.add(record);
|
||||
}
|
||||
for (let index = 0; index < records.length; index += 1) {
|
||||
const record = records[index];
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) continue;
|
||||
if (sourceRecordSet.has(record)) {
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
function normalizeNativeHydrateRecordArray(records = []) {
|
||||
const sourceRecords = toArray(records);
|
||||
if (sourceRecords.length === 0) return [];
|
||||
const output = new Array(sourceRecords.length);
|
||||
let writeIndex = 0;
|
||||
for (let index = 0; index < sourceRecords.length; index += 1) {
|
||||
const record = sourceRecords[index];
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) continue;
|
||||
output[writeIndex] = record;
|
||||
writeIndex += 1;
|
||||
}
|
||||
output.length = writeIndex;
|
||||
return output;
|
||||
}
|
||||
|
||||
function normalizeNativeHydrateResult(rawResult = null, snapshotView = {}) {
|
||||
if (!rawResult || typeof rawResult !== "object" || Array.isArray(rawResult)) {
|
||||
return null;
|
||||
}
|
||||
if (
|
||||
rawResult.nodes === snapshotView?.nodes ||
|
||||
rawResult.edges === snapshotView?.edges
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
const nodes = normalizeNativeHydrateRecordArray(rawResult.nodes);
|
||||
const edges = normalizeNativeHydrateRecordArray(rawResult.edges);
|
||||
if (
|
||||
hasSharedHydrateRecordReferences(nodes, snapshotView?.nodes) ||
|
||||
hasSharedHydrateRecordReferences(edges, snapshotView?.edges)
|
||||
) {
|
||||
return null;
|
||||
}
|
||||
return {
|
||||
nodes,
|
||||
edges,
|
||||
diagnostics:
|
||||
rawResult.diagnostics &&
|
||||
typeof rawResult.diagnostics === "object" &&
|
||||
!Array.isArray(rawResult.diagnostics)
|
||||
? rawResult.diagnostics
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
function cloneHydrateSnapshotEdgeRecords(records = []) {
|
||||
const sourceRecords = toArray(records);
|
||||
if (sourceRecords.length === 0) return [];
|
||||
@@ -452,6 +565,40 @@ function countPersistSnapshotRecords(snapshot = {}) {
|
||||
);
|
||||
}
|
||||
|
||||
function countHydrateSnapshotRecords(snapshot = {}) {
|
||||
return toArray(snapshot?.nodes).length + toArray(snapshot?.edges).length;
|
||||
}
|
||||
|
||||
export function resolveNativeHydrateGateOptions(options = {}) {
|
||||
return {
|
||||
minSnapshotRecords: normalizePersistNativeDeltaThreshold(
|
||||
options?.loadNativeHydrateThresholdRecords ??
|
||||
options?.hydrateNativeThresholdRecords ??
|
||||
options?.minSnapshotRecords,
|
||||
DEFAULT_NATIVE_HYDRATE_THRESHOLD_RECORDS,
|
||||
),
|
||||
};
|
||||
}
|
||||
|
||||
export function evaluateNativeHydrateGate(snapshot, options = {}) {
|
||||
const normalizedSnapshot = normalizePersistSnapshotView(snapshot);
|
||||
const gateOptions = resolveNativeHydrateGateOptions(options);
|
||||
const recordCount = countHydrateSnapshotRecords(normalizedSnapshot);
|
||||
const reasons = [];
|
||||
if (
|
||||
gateOptions.minSnapshotRecords > 0 &&
|
||||
recordCount < gateOptions.minSnapshotRecords
|
||||
) {
|
||||
reasons.push("below-min-snapshot-records");
|
||||
}
|
||||
return {
|
||||
allowed: reasons.length === 0,
|
||||
reasons,
|
||||
minSnapshotRecords: gateOptions.minSnapshotRecords,
|
||||
recordCount,
|
||||
};
|
||||
}
|
||||
|
||||
function countPersistSnapshotStructuralDelta(beforeSnapshot = {}, afterSnapshot = {}) {
|
||||
return (
|
||||
Math.abs(toArray(afterSnapshot?.nodes).length - toArray(beforeSnapshot?.nodes).length) +
|
||||
@@ -2272,6 +2419,14 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
||||
normalizeMs: 0,
|
||||
integrityMs: 0,
|
||||
integrityReasonCount: 0,
|
||||
nativeRequested: false,
|
||||
nativeUsed: false,
|
||||
nativeStatus: "not-requested",
|
||||
nativeError: "",
|
||||
nativeRecordsMs: 0,
|
||||
nativeGateAllowed: false,
|
||||
nativeGateReasons: [],
|
||||
nativeModuleDiagnostics: null,
|
||||
}
|
||||
: null;
|
||||
const snapshotView = normalizePersistSnapshotView(snapshot);
|
||||
@@ -2301,6 +2456,58 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
||||
);
|
||||
const snapshotRecordsNormalized =
|
||||
snapshotMeta?.[BME_RUNTIME_RECORDS_NORMALIZED_META_KEY] === true;
|
||||
const nativeHydrateGate =
|
||||
options?.useNativeHydrate === true
|
||||
? evaluateNativeHydrateGate(snapshotView, options)
|
||||
: null;
|
||||
const nativeHydrateStartedAt = shouldCollectDiagnostics ? readPersistDeltaNow() : 0;
|
||||
let nativeHydrateAttempt =
|
||||
options?.useNativeHydrate !== true
|
||||
? {
|
||||
rawResult: null,
|
||||
status: "not-requested",
|
||||
error: "",
|
||||
}
|
||||
: nativeHydrateGate?.allowed === false
|
||||
? {
|
||||
rawResult: null,
|
||||
status: "gated-out",
|
||||
error: "",
|
||||
}
|
||||
: tryBuildNativeHydrateRecords(
|
||||
snapshotView,
|
||||
{
|
||||
...options,
|
||||
recordsNormalized: snapshotRecordsNormalized,
|
||||
},
|
||||
);
|
||||
let nativeHydrateResult = normalizeNativeHydrateResult(
|
||||
nativeHydrateAttempt.rawResult,
|
||||
snapshotView,
|
||||
);
|
||||
if (nativeHydrateAttempt.rawResult && !nativeHydrateResult) {
|
||||
if (options?.nativeFailOpen === false) {
|
||||
throw new Error("native-hydrate-invalid-result");
|
||||
}
|
||||
nativeHydrateAttempt = {
|
||||
rawResult: null,
|
||||
status: "invalid-result",
|
||||
error: "native-hydrate-invalid-result",
|
||||
};
|
||||
}
|
||||
if (hydrateDiagnostics) {
|
||||
hydrateDiagnostics.nativeRequested = options?.useNativeHydrate === true;
|
||||
hydrateDiagnostics.nativeStatus = nativeHydrateAttempt.status;
|
||||
hydrateDiagnostics.nativeError = nativeHydrateAttempt.error;
|
||||
hydrateDiagnostics.nativeGateAllowed = nativeHydrateGate?.allowed ?? false;
|
||||
hydrateDiagnostics.nativeGateReasons = nativeHydrateGate?.reasons || [];
|
||||
hydrateDiagnostics.nativeModuleDiagnostics =
|
||||
nativeHydrateResult?.diagnostics || null;
|
||||
if (nativeHydrateAttempt.rawResult) {
|
||||
hydrateDiagnostics.nativeRecordsMs =
|
||||
readPersistDeltaNow() - nativeHydrateStartedAt;
|
||||
}
|
||||
}
|
||||
|
||||
const runtimeGraph = createEmptyGraph();
|
||||
runtimeGraph.version = Number.isFinite(
|
||||
@@ -2310,17 +2517,22 @@ export function buildGraphFromSnapshot(snapshot, options = {}) {
|
||||
: runtimeGraph.version;
|
||||
|
||||
const hydrateNodesStartedAt = shouldCollectDiagnostics ? readPersistDeltaNow() : 0;
|
||||
runtimeGraph.nodes = cloneHydrateSnapshotNodeRecords(snapshotView.nodes);
|
||||
runtimeGraph.nodes = nativeHydrateResult
|
||||
? nativeHydrateResult.nodes
|
||||
: cloneHydrateSnapshotNodeRecords(snapshotView.nodes);
|
||||
if (hydrateDiagnostics) {
|
||||
hydrateDiagnostics.nodeCount = runtimeGraph.nodes.length;
|
||||
hydrateDiagnostics.nodesMs = readPersistDeltaNow() - hydrateNodesStartedAt;
|
||||
}
|
||||
|
||||
const hydrateEdgesStartedAt = shouldCollectDiagnostics ? readPersistDeltaNow() : 0;
|
||||
runtimeGraph.edges = cloneHydrateSnapshotEdgeRecords(snapshotView.edges);
|
||||
runtimeGraph.edges = nativeHydrateResult
|
||||
? nativeHydrateResult.edges
|
||||
: cloneHydrateSnapshotEdgeRecords(snapshotView.edges);
|
||||
if (hydrateDiagnostics) {
|
||||
hydrateDiagnostics.edgeCount = runtimeGraph.edges.length;
|
||||
hydrateDiagnostics.edgesMs = readPersistDeltaNow() - hydrateEdgesStartedAt;
|
||||
hydrateDiagnostics.nativeUsed = Boolean(nativeHydrateResult);
|
||||
}
|
||||
|
||||
const hydrateRuntimeMetaStartedAt = shouldCollectDiagnostics
|
||||
|
||||
@@ -76,6 +76,8 @@ assert.equal(defaultSettings.persistNativeDeltaThresholdRecords, 20000);
|
||||
assert.equal(defaultSettings.persistNativeDeltaThresholdStructuralDelta, 600);
|
||||
assert.equal(defaultSettings.persistNativeDeltaThresholdSerializedChars, 4000000);
|
||||
assert.equal(defaultSettings.persistNativeDeltaBridgeMode, "json");
|
||||
assert.equal(defaultSettings.loadUseNativeHydrate, false);
|
||||
assert.equal(defaultSettings.loadNativeHydrateThresholdRecords, 12000);
|
||||
assert.equal(defaultSettings.nativeEngineFailOpen, true);
|
||||
assert.equal(defaultSettings.graphNativeForceDisable, false);
|
||||
assert.equal(defaultSettings.taskProfilesVersion, 3);
|
||||
|
||||
@@ -9,6 +9,7 @@ import {
|
||||
buildGraphFromSnapshot,
|
||||
buildPersistDelta,
|
||||
buildSnapshotFromGraph,
|
||||
evaluateNativeHydrateGate,
|
||||
evaluatePersistNativeDeltaGate,
|
||||
} from "../sync/bme-db.js";
|
||||
import { onMessageReceivedController } from "../host/event-binding.js";
|
||||
@@ -1032,6 +1033,7 @@ async function createGraphPersistenceHarness({
|
||||
buildGraphFromSnapshot,
|
||||
buildPersistDelta,
|
||||
buildSnapshotFromGraph,
|
||||
evaluateNativeHydrateGate,
|
||||
evaluatePersistNativeDeltaGate,
|
||||
buildBmeDbName,
|
||||
BME_GRAPH_LOCAL_STORAGE_MODE_AUTO: "auto",
|
||||
|
||||
57
tests/native-hydrate-failopen.mjs
Normal file
57
tests/native-hydrate-failopen.mjs
Normal file
@@ -0,0 +1,57 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
function moduleUrl(tag) {
|
||||
return `../vendor/wasm/stbme_core.js?test=${Date.now()}-${tag}`;
|
||||
}
|
||||
|
||||
globalThis.__stBmeDisableWasmPackArtifacts = true;
|
||||
delete globalThis.__stBmeLoadRustWasmLayout;
|
||||
|
||||
const firstLoad = await import(moduleUrl("native-hydrate-first"));
|
||||
let firstError = "";
|
||||
try {
|
||||
await firstLoad.installNativeHydrateHook();
|
||||
} catch (error) {
|
||||
firstError = error?.message || String(error);
|
||||
}
|
||||
|
||||
assert.match(
|
||||
firstError,
|
||||
/native module unavailable|native hydrate builder unavailable|global-loader|Rust\/WASM artifact is not initialized/i,
|
||||
);
|
||||
|
||||
globalThis.__stBmeLoadRustWasmLayout = async () => ({
|
||||
solve_layout() {
|
||||
return {
|
||||
ok: true,
|
||||
positions: [],
|
||||
diagnostics: {
|
||||
solver: "mock-rust-wasm",
|
||||
},
|
||||
};
|
||||
},
|
||||
build_hydrate_records() {
|
||||
return {
|
||||
ok: true,
|
||||
usedNative: true,
|
||||
nodes: [],
|
||||
edges: [],
|
||||
diagnostics: {
|
||||
solver: "mock-rust-wasm",
|
||||
nodeCount: 0,
|
||||
edgeCount: 0,
|
||||
recordsNormalized: false,
|
||||
},
|
||||
};
|
||||
},
|
||||
});
|
||||
|
||||
const retryStatus = await firstLoad.installNativeHydrateHook();
|
||||
assert.equal(retryStatus.loaded, true);
|
||||
assert.equal(typeof globalThis.__stBmeNativeHydrateSnapshotRecords, "function");
|
||||
|
||||
delete globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
delete globalThis.__stBmeLoadRustWasmLayout;
|
||||
delete globalThis.__stBmeDisableWasmPackArtifacts;
|
||||
|
||||
console.log("native-hydrate-failopen tests passed");
|
||||
208
tests/native-hydrate-hook.mjs
Normal file
208
tests/native-hydrate-hook.mjs
Normal file
@@ -0,0 +1,208 @@
|
||||
import assert from "node:assert/strict";
|
||||
|
||||
import {
|
||||
BME_RUNTIME_HISTORY_META_KEY,
|
||||
BME_RUNTIME_RECORDS_NORMALIZED_META_KEY,
|
||||
BME_RUNTIME_VECTOR_META_KEY,
|
||||
buildGraphFromSnapshot,
|
||||
evaluateNativeHydrateGate,
|
||||
resolveNativeHydrateGateOptions,
|
||||
} from "../sync/bme-db.js";
|
||||
|
||||
function cloneValue(value) {
|
||||
if (typeof globalThis.structuredClone === "function") {
|
||||
return globalThis.structuredClone(value);
|
||||
}
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
}
|
||||
|
||||
const snapshot = {
|
||||
meta: {
|
||||
chatId: "chat-native-hydrate",
|
||||
revision: 3,
|
||||
[BME_RUNTIME_RECORDS_NORMALIZED_META_KEY]: true,
|
||||
[BME_RUNTIME_HISTORY_META_KEY]: {
|
||||
chatId: "chat-native-hydrate",
|
||||
lastProcessedAssistantFloor: 7,
|
||||
extractionCount: 2,
|
||||
processedMessageHashes: {},
|
||||
processedMessageHashVersion: 1,
|
||||
processedMessageHashesNeedRefresh: false,
|
||||
recentRecallOwnerKeys: [],
|
||||
activeRecallOwnerKey: "",
|
||||
activeRegion: "",
|
||||
activeRegionSource: "",
|
||||
activeStorySegmentId: "",
|
||||
activeStoryTimeLabel: "",
|
||||
activeStoryTimeSource: "",
|
||||
lastBatchStatus: null,
|
||||
lastMutationSource: "test",
|
||||
lastExtractedRegion: "",
|
||||
lastExtractedStorySegmentId: "",
|
||||
activeCharacterPovOwner: "",
|
||||
activeUserPovOwner: "",
|
||||
},
|
||||
[BME_RUNTIME_VECTOR_META_KEY]: {
|
||||
chatId: "chat-native-hydrate",
|
||||
collectionId: "",
|
||||
hashToNodeId: {},
|
||||
nodeToHash: {},
|
||||
replayRequiredNodeIds: [],
|
||||
dirty: false,
|
||||
dirtyReason: "",
|
||||
pendingRepairFromFloor: null,
|
||||
lastIntegrityIssue: null,
|
||||
lastStats: {
|
||||
nodesIndexed: 0,
|
||||
updatedAt: 0,
|
||||
},
|
||||
},
|
||||
},
|
||||
state: {
|
||||
lastProcessedFloor: 7,
|
||||
extractionCount: 2,
|
||||
},
|
||||
nodes: [
|
||||
{
|
||||
id: "native-node-1",
|
||||
type: "event",
|
||||
updatedAt: 10,
|
||||
fields: {
|
||||
title: "Native Node",
|
||||
},
|
||||
embedding: [1, 2, 3],
|
||||
scope: {
|
||||
ownerType: "character",
|
||||
ownerId: "owner-1",
|
||||
layer: "objective",
|
||||
regionPrimary: "camp",
|
||||
regionPath: ["camp"],
|
||||
regionSecondary: [],
|
||||
},
|
||||
storyTime: {
|
||||
label: "Dawn",
|
||||
tense: "unknown",
|
||||
},
|
||||
storyTimeSpan: {
|
||||
startLabel: "Dawn",
|
||||
endLabel: "Dawn",
|
||||
mixed: false,
|
||||
},
|
||||
},
|
||||
],
|
||||
edges: [
|
||||
{
|
||||
id: "native-edge-1",
|
||||
fromId: "native-node-1",
|
||||
toId: "native-node-2",
|
||||
relation: "related",
|
||||
scope: {
|
||||
ownerType: "character",
|
||||
ownerId: "owner-1",
|
||||
layer: "objective",
|
||||
regionPrimary: "camp",
|
||||
regionPath: ["camp"],
|
||||
regionSecondary: [],
|
||||
},
|
||||
},
|
||||
],
|
||||
tombstones: [],
|
||||
};
|
||||
|
||||
const defaultGate = resolveNativeHydrateGateOptions({});
|
||||
assert.equal(defaultGate.minSnapshotRecords, 12000);
|
||||
const gatedSmall = evaluateNativeHydrateGate(snapshot, {});
|
||||
assert.equal(gatedSmall.allowed, false);
|
||||
assert.deepEqual(gatedSmall.reasons, ["below-min-snapshot-records"]);
|
||||
const gatedLarge = evaluateNativeHydrateGate(
|
||||
{
|
||||
nodes: new Array(6000).fill({ id: "node-x" }),
|
||||
edges: new Array(6000).fill({ id: "edge-x" }),
|
||||
},
|
||||
{},
|
||||
);
|
||||
assert.equal(gatedLarge.allowed, true);
|
||||
assert.deepEqual(gatedLarge.reasons, []);
|
||||
|
||||
const originalNativeBuilder = globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
|
||||
globalThis.__stBmeNativeHydrateSnapshotRecords = (snapshotView = {}, options = {}) => {
|
||||
assert.equal(options.recordsNormalized, true);
|
||||
return {
|
||||
ok: true,
|
||||
usedNative: true,
|
||||
nodes: cloneValue(snapshotView.nodes).map((node) => ({
|
||||
...node,
|
||||
nativeHydrated: true,
|
||||
})),
|
||||
edges: cloneValue(snapshotView.edges).map((edge) => ({
|
||||
...edge,
|
||||
nativeHydrated: true,
|
||||
})),
|
||||
diagnostics: {
|
||||
solver: "test-native-hydrate",
|
||||
nodeCount: Array.isArray(snapshotView.nodes) ? snapshotView.nodes.length : 0,
|
||||
edgeCount: Array.isArray(snapshotView.edges) ? snapshotView.edges.length : 0,
|
||||
recordsNormalized: options.recordsNormalized === true,
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
let nativeDiagnostics = null;
|
||||
const rebuilt = buildGraphFromSnapshot(snapshot, {
|
||||
chatId: "chat-native-hydrate",
|
||||
useNativeHydrate: true,
|
||||
minSnapshotRecords: 0,
|
||||
onDiagnostics(snapshotValue) {
|
||||
nativeDiagnostics = snapshotValue;
|
||||
},
|
||||
});
|
||||
assert.equal(rebuilt.nodes[0].nativeHydrated, true);
|
||||
assert.equal(rebuilt.edges[0].nativeHydrated, true);
|
||||
assert.equal(rebuilt.historyState.lastProcessedAssistantFloor, 7);
|
||||
assert.equal(nativeDiagnostics.nativeRequested, true);
|
||||
assert.equal(nativeDiagnostics.nativeUsed, true);
|
||||
assert.equal(nativeDiagnostics.nativeStatus, "ok");
|
||||
assert.equal(nativeDiagnostics.nativeGateAllowed, true);
|
||||
assert.equal(nativeDiagnostics.nativeModuleDiagnostics?.solver, "test-native-hydrate");
|
||||
assert.equal(Number.isFinite(nativeDiagnostics.nativeRecordsMs), true);
|
||||
rebuilt.nodes[0].fields.title = "Mutated Native Node";
|
||||
rebuilt.nodes[0].embedding[0] = 99;
|
||||
assert.equal(snapshot.nodes[0].fields.title, "Native Node");
|
||||
assert.equal(snapshot.nodes[0].embedding[0], 1);
|
||||
|
||||
delete globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
|
||||
let fallbackDiagnostics = null;
|
||||
const fallbackGraph = buildGraphFromSnapshot(snapshot, {
|
||||
chatId: "chat-native-hydrate",
|
||||
useNativeHydrate: true,
|
||||
minSnapshotRecords: 0,
|
||||
onDiagnostics(snapshotValue) {
|
||||
fallbackDiagnostics = snapshotValue;
|
||||
},
|
||||
});
|
||||
assert.equal(fallbackGraph.nodes.length, 1);
|
||||
assert.equal(fallbackDiagnostics.nativeRequested, true);
|
||||
assert.equal(fallbackDiagnostics.nativeUsed, false);
|
||||
assert.equal(fallbackDiagnostics.nativeStatus, "builder-unavailable");
|
||||
|
||||
let threwUnavailable = false;
|
||||
try {
|
||||
buildGraphFromSnapshot(snapshot, {
|
||||
chatId: "chat-native-hydrate",
|
||||
useNativeHydrate: true,
|
||||
minSnapshotRecords: 0,
|
||||
nativeFailOpen: false,
|
||||
});
|
||||
} catch (error) {
|
||||
threwUnavailable =
|
||||
String(error?.message || "") === "native-hydrate-builder-unavailable";
|
||||
}
|
||||
assert.equal(threwUnavailable, true);
|
||||
|
||||
if (typeof originalNativeBuilder === "function") {
|
||||
globalThis.__stBmeNativeHydrateSnapshotRecords = originalNativeBuilder;
|
||||
}
|
||||
|
||||
console.log("native-hydrate-hook tests passed");
|
||||
@@ -22,6 +22,24 @@ try {
|
||||
},
|
||||
};
|
||||
},
|
||||
build_hydrate_records(payload = {}) {
|
||||
return {
|
||||
ok: true,
|
||||
usedNative: true,
|
||||
nodes: Array.isArray(payload?.nodes)
|
||||
? payload.nodes.map((node) => ({ ...node, nativeHydrated: true }))
|
||||
: [],
|
||||
edges: Array.isArray(payload?.edges)
|
||||
? payload.edges.map((edge) => ({ ...edge, nativeHydrated: true }))
|
||||
: [],
|
||||
diagnostics: {
|
||||
solver: "mock-loader",
|
||||
nodeCount: Array.isArray(payload?.nodes) ? payload.nodes.length : 0,
|
||||
edgeCount: Array.isArray(payload?.edges) ? payload.edges.length : 0,
|
||||
recordsNormalized: payload?.recordsNormalized === true,
|
||||
},
|
||||
};
|
||||
},
|
||||
build_persist_delta_compact(payload = {}) {
|
||||
return {
|
||||
upsertNodeIds: Array.isArray(payload?.afterNodes?.ids)
|
||||
@@ -105,8 +123,29 @@ try {
|
||||
assert.deepEqual(deltaResult.upsertNodes, [{ id: "persist-native-node", marker: "after-chat" }]);
|
||||
assert.equal(deltaResult.runtimeMetaPatch.native, true);
|
||||
|
||||
const hydrateInstallStatus = await wrapper.installNativeHydrateHook();
|
||||
assert.equal(hydrateInstallStatus.loaded, true);
|
||||
assert.equal(
|
||||
typeof globalThis.__stBmeNativeHydrateSnapshotRecords,
|
||||
"function",
|
||||
);
|
||||
const hydrateResult = globalThis.__stBmeNativeHydrateSnapshotRecords(
|
||||
{
|
||||
nodes: [{ id: "hydrate-node", type: "event" }],
|
||||
edges: [{ id: "hydrate-edge", fromId: "hydrate-node", toId: "hydrate-node-2" }],
|
||||
},
|
||||
{
|
||||
recordsNormalized: true,
|
||||
},
|
||||
);
|
||||
assert.equal(hydrateResult.ok, true);
|
||||
assert.equal(hydrateResult.nodes[0].nativeHydrated, true);
|
||||
assert.equal(hydrateResult.edges[0].nativeHydrated, true);
|
||||
assert.equal(hydrateResult.diagnostics.recordsNormalized, true);
|
||||
|
||||
delete globalThis.__stBmeLoadRustWasmLayout;
|
||||
delete globalThis.__stBmeNativeBuildPersistDelta;
|
||||
delete globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
delete globalThis.__stBmeDisableWasmPackArtifacts;
|
||||
|
||||
const wrapperNoLoader = await importFreshWrapper("no-loader");
|
||||
@@ -136,6 +175,7 @@ try {
|
||||
}
|
||||
delete globalThis.__stBmeDisableWasmPackArtifacts;
|
||||
delete globalThis.__stBmeNativeBuildPersistDelta;
|
||||
delete globalThis.__stBmeNativeHydrateSnapshotRecords;
|
||||
}
|
||||
|
||||
console.log("native-layout-wrapper tests passed");
|
||||
|
||||
@@ -13,12 +13,27 @@ import { createMemoryOpfsRoot } from "../helpers/memory-opfs.mjs";
|
||||
|
||||
const RUNS = 4;
|
||||
const outputJson = process.argv.includes("--json");
|
||||
const useNativeHydrate = process.argv.includes("--native-hydrate");
|
||||
const nativeHydrateThresholdArg = process.argv.find((entry) =>
|
||||
String(entry || "").startsWith("--native-hydrate-threshold="),
|
||||
);
|
||||
const nativeHydrateThresholdRecords = nativeHydrateThresholdArg
|
||||
? Math.max(
|
||||
0,
|
||||
Math.floor(
|
||||
Number(String(nativeHydrateThresholdArg).split("=").slice(1).join("=") || 0) || 0,
|
||||
),
|
||||
)
|
||||
: undefined;
|
||||
const SIZE_PRESETS = [
|
||||
{ label: "M", seed: 17, nodeCount: 1200, edgeCount: 3600, churn: 0.08 },
|
||||
{ label: "L", seed: 29, nodeCount: 3600, edgeCount: 10800, churn: 0.1 },
|
||||
{ label: "XL", seed: 43, nodeCount: 7200, edgeCount: 21600, churn: 0.12 },
|
||||
];
|
||||
|
||||
let nativeHydratePreloadStatus = useNativeHydrate ? "pending" : "not-requested";
|
||||
let nativeHydratePreloadError = "";
|
||||
|
||||
function summarize(values = []) {
|
||||
if (!values.length) {
|
||||
return { avg: 0, p95: 0, min: 0, max: 0 };
|
||||
@@ -218,6 +233,9 @@ function measureHydrate(snapshot, chatId) {
|
||||
const startedAt = performance.now();
|
||||
buildGraphFromSnapshot(snapshot, {
|
||||
chatId,
|
||||
useNativeHydrate,
|
||||
loadNativeHydrateThresholdRecords: nativeHydrateThresholdRecords,
|
||||
nativeFailOpen: true,
|
||||
onDiagnostics(snapshotValue) {
|
||||
diagnostics = snapshotValue;
|
||||
},
|
||||
@@ -266,8 +284,10 @@ async function runPreset(preset) {
|
||||
const hydrateStateSamples = [];
|
||||
const hydrateNormalizeSamples = [];
|
||||
const hydrateIntegritySamples = [];
|
||||
const hydrateNativeRecordsSamples = [];
|
||||
const walFileWriteSamples = [];
|
||||
const manifestFileWriteSamples = [];
|
||||
let hydrateNativeUsedRuns = 0;
|
||||
|
||||
for (let run = 0; run < RUNS; run += 1) {
|
||||
const pair = buildBenchPair({
|
||||
@@ -306,6 +326,12 @@ async function runPreset(preset) {
|
||||
hydrateStateSamples.push(Number(hydrateResult.diagnostics?.stateMs || 0));
|
||||
hydrateNormalizeSamples.push(Number(hydrateResult.diagnostics?.normalizeMs || 0));
|
||||
hydrateIntegritySamples.push(Number(hydrateResult.diagnostics?.integrityMs || 0));
|
||||
hydrateNativeRecordsSamples.push(
|
||||
Number(hydrateResult.diagnostics?.nativeRecordsMs || 0),
|
||||
);
|
||||
if (hydrateResult.diagnostics?.nativeUsed === true) {
|
||||
hydrateNativeUsedRuns += 1;
|
||||
}
|
||||
walFileWriteSamples.push(Number(opfsCommitResult.diagnostics?.walFileWriteMs || 0));
|
||||
manifestFileWriteSamples.push(
|
||||
Number(opfsCommitResult.diagnostics?.manifestFileWriteMs || 0),
|
||||
@@ -321,6 +347,11 @@ async function runPreset(preset) {
|
||||
hydrateStateMs: summarize(hydrateStateSamples),
|
||||
hydrateNormalizeMs: summarize(hydrateNormalizeSamples),
|
||||
hydrateIntegrityMs: summarize(hydrateIntegritySamples),
|
||||
hydrateNativeRecordsMs: summarize(hydrateNativeRecordsSamples),
|
||||
hydrateNativeUsedRuns,
|
||||
nativeHydrateRequested: useNativeHydrate,
|
||||
nativeHydrateThresholdRecords:
|
||||
nativeHydrateThresholdRecords == null ? null : nativeHydrateThresholdRecords,
|
||||
hydrateRuntimeMetaMs: summarize(hydrateRuntimeMetaSamples),
|
||||
opfsCommitMs: summarize(opfsCommitSamples),
|
||||
opfsWalFileWriteMs: summarize(walFileWriteSamples),
|
||||
@@ -338,6 +369,8 @@ async function runPreset(preset) {
|
||||
`edgesP95=${result.hydrateEdgesMs.p95.toFixed(2)}ms`,
|
||||
`normalizeP95=${result.hydrateNormalizeMs.p95.toFixed(2)}ms`,
|
||||
`integrityP95=${result.hydrateIntegrityMs.p95.toFixed(2)}ms`,
|
||||
`nativeRecordsP95=${result.hydrateNativeRecordsMs.p95.toFixed(2)}ms`,
|
||||
`nativeUsed=${result.hydrateNativeUsedRuns}/${RUNS}`,
|
||||
`runtimeMetaP95=${result.hydrateRuntimeMetaMs.p95.toFixed(2)}ms`,
|
||||
);
|
||||
console.log(
|
||||
@@ -350,15 +383,36 @@ async function runPreset(preset) {
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const results = {};
|
||||
for (const preset of SIZE_PRESETS) {
|
||||
results[preset.label] = await runPreset(preset);
|
||||
if (useNativeHydrate) {
|
||||
try {
|
||||
const nativeModule = await import("../../vendor/wasm/stbme_core.js");
|
||||
const nativeStatus = await nativeModule?.installNativeHydrateHook?.();
|
||||
nativeHydratePreloadStatus = nativeStatus?.loaded ? "loaded" : "not-loaded";
|
||||
nativeHydratePreloadError = String(nativeStatus?.error || "");
|
||||
} catch (error) {
|
||||
nativeHydratePreloadStatus = "failed";
|
||||
nativeHydratePreloadError = error?.message || String(error);
|
||||
console.warn(
|
||||
"[ST-BME][persist-load-bench] native hydrate preload failed, fallback to JS hydrate:",
|
||||
error,
|
||||
);
|
||||
}
|
||||
}
|
||||
const presets = {};
|
||||
for (const preset of SIZE_PRESETS) {
|
||||
presets[preset.label] = await runPreset(preset);
|
||||
}
|
||||
const payload = {
|
||||
runs: RUNS,
|
||||
nativeHydrateRequested: useNativeHydrate,
|
||||
nativeHydratePreloadStatus,
|
||||
nativeHydratePreloadError,
|
||||
nativeHydrateThresholdRecords:
|
||||
nativeHydrateThresholdRecords == null ? null : nativeHydrateThresholdRecords,
|
||||
presets,
|
||||
};
|
||||
if (outputJson) {
|
||||
console.log(JSON.stringify({
|
||||
runs: RUNS,
|
||||
presets: results,
|
||||
}));
|
||||
console.log(JSON.stringify(payload));
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
24
vendor/wasm/stbme_core.js
vendored
24
vendor/wasm/stbme_core.js
vendored
@@ -69,6 +69,10 @@ async function loadFromWasmPackArtifacts() {
|
||||
|
||||
return {
|
||||
solve_layout: module.solve_layout,
|
||||
build_hydrate_records:
|
||||
typeof module.build_hydrate_records === "function"
|
||||
? module.build_hydrate_records
|
||||
: null,
|
||||
build_persist_delta_compact_hash:
|
||||
typeof module.build_persist_delta_compact_hash === "function"
|
||||
? module.build_persist_delta_compact_hash
|
||||
@@ -222,6 +226,26 @@ export async function installNativePersistDeltaHook() {
|
||||
return getNativeModuleStatus();
|
||||
}
|
||||
|
||||
export async function installNativeHydrateHook() {
|
||||
const module = await loadNativeModule({
|
||||
forceRetry: shouldRetryNativeLoad(),
|
||||
});
|
||||
if (!module || typeof module.build_hydrate_records !== "function") {
|
||||
throw new Error("native hydrate builder unavailable");
|
||||
}
|
||||
|
||||
globalThis.__stBmeNativeHydrateSnapshotRecords = (snapshotView = {}, options = {}) => {
|
||||
const raw = module.build_hydrate_records({
|
||||
nodes: Array.isArray(snapshotView?.nodes) ? snapshotView.nodes : [],
|
||||
edges: Array.isArray(snapshotView?.edges) ? snapshotView.edges : [],
|
||||
recordsNormalized: options?.recordsNormalized === true,
|
||||
});
|
||||
return raw && typeof raw === "object" ? raw : null;
|
||||
};
|
||||
|
||||
return getNativeModuleStatus();
|
||||
}
|
||||
|
||||
export function getNativeModuleStatus() {
|
||||
return {
|
||||
loaded: Boolean(cachedNativeModule),
|
||||
|
||||
Reference in New Issue
Block a user