From 67cf5fe7fa2794961f530b23419d585619ecf52c Mon Sep 17 00:00:00 2001 From: Youzini-afk <13153778771cx@gmail.com> Date: Mon, 13 Apr 2026 16:37:19 +0800 Subject: [PATCH] perf: add hash compact persist-delta bridge mode --- index.js | 7 + native/stbme-core/src/lib.rs | 169 ++++++++++++++++++++ runtime/settings-defaults.js | 1 + sync/bme-db.js | 237 ++++++++++++++++++++++------ tests/default-settings.mjs | 1 + tests/native-layout-wrapper.mjs | 25 +++ tests/native-persist-delta-hook.mjs | 29 +++- tests/perf/persist-delta-bench.mjs | 49 ++++-- ui/panel.js | 6 + vendor/wasm/stbme_core.js | 35 +++- 10 files changed, 489 insertions(+), 70 deletions(-) diff --git a/index.js b/index.js index 6461056..2876d93 100644 --- a/index.js +++ b/index.js @@ -9684,6 +9684,9 @@ async function saveGraphToIndexedDb( }, }); const currentSettings = getSettings(); + const nativePersistBridgeMode = String( + currentSettings.persistNativeDeltaBridgeMode || "json", + ); const nativePersistRequested = currentSettings.persistUseNativeDelta === true; const nativePersistForceDisabled = currentSettings.graphNativeForceDisable === true; const nativePersistGate = evaluatePersistNativeDeltaGate( @@ -9712,6 +9715,7 @@ async function saveGraphToIndexedDb( saveReason: String(reason || "graph-save"), requestedRevision, requestedNative: nativePersistRequested, + requestedBridgeMode: nativePersistBridgeMode, nativeForceDisabled: nativePersistForceDisabled, nativeFailOpen: currentSettings.nativeEngineFailOpen !== false, gateAllowed: nativePersistGate.allowed, @@ -9771,6 +9775,7 @@ async function saveGraphToIndexedDb( currentSettings.persistNativeDeltaThresholdStructuralDelta, persistNativeDeltaThresholdSerializedChars: currentSettings.persistNativeDeltaThresholdSerializedChars, + persistNativeDeltaBridgeMode: nativePersistBridgeMode, onDiagnostics(snapshot) { persistDeltaBuildDiagnostics = snapshot; }, @@ -9822,6 +9827,8 @@ async function saveGraphToIndexedDb( saveReason: String(reason || "graph-save"), requestedRevision, requestedNative: nativePersistRequested, + requestedBridgeMode: + persistDeltaBuildDiagnostics?.requestedBridgeMode || nativePersistBridgeMode, buildRequestedNative: Boolean(persistDeltaBuildDiagnostics?.requestedNative), nativeForceDisabled: nativePersistForceDisabled, nativeFailOpen: currentSettings.nativeEngineFailOpen !== false, diff --git a/native/stbme-core/src/lib.rs b/native/stbme-core/src/lib.rs index d4385a0..74db058 100644 --- a/native/stbme-core/src/lib.rs +++ b/native/stbme-core/src/lib.rs @@ -149,6 +149,15 @@ struct PersistCompactRecordSet { serialized: Vec, } +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct PersistCompactHashRecordSet { + #[serde(default)] + ids: Vec, + #[serde(default)] + hashes: Vec, +} + #[derive(Debug, Clone, Deserialize, Default)] #[serde(rename_all = "camelCase")] struct PersistCompactTombstoneSet { @@ -160,6 +169,17 @@ struct PersistCompactTombstoneSet { target_keys: Vec, } +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct PersistCompactHashTombstoneSet { + #[serde(default)] + ids: Vec, + #[serde(default)] + hashes: Vec, + #[serde(default)] + target_keys: Vec, +} + #[derive(Debug, Clone, Deserialize, Default)] #[serde(rename_all = "camelCase")] struct PersistDeltaCompactPayload { @@ -177,6 +197,23 @@ struct PersistDeltaCompactPayload { after_tombstones: PersistCompactTombstoneSet, } +#[derive(Debug, Clone, Deserialize, Default)] +#[serde(rename_all = "camelCase")] +struct PersistDeltaCompactHashPayload { + #[serde(default)] + before_nodes: PersistCompactHashRecordSet, + #[serde(default)] + after_nodes: PersistCompactHashRecordSet, + #[serde(default)] + before_edges: PersistCompactHashRecordSet, + #[serde(default)] + after_edges: PersistCompactHashRecordSet, + #[serde(default)] + before_tombstones: PersistCompactHashRecordSet, + #[serde(default)] + after_tombstones: PersistCompactHashTombstoneSet, +} + #[derive(Debug, Clone, Serialize)] #[serde(rename_all = "camelCase")] struct PersistDeltaIdResult { @@ -285,6 +322,19 @@ fn build_json_serialized_index(records: &[JsonValue]) -> HashMap map } +fn build_compact_hash_lookup<'a>(ids: &'a [String], hashes: &'a [u32]) -> HashMap<&'a str, u32> { + let mut map = HashMap::new(); + let len = ids.len().min(hashes.len()); + for index in 0..len { + let id = ids[index].trim(); + if id.is_empty() { + continue; + } + map.insert(id, hashes[index]); + } + map +} + fn build_json_value_index(records: &[JsonValue]) -> HashMap { let mut map = HashMap::new(); for record in records { @@ -605,6 +655,115 @@ fn solve_persist_delta_compact_in_rust(payload: PersistDeltaCompactPayload) -> P } } +fn solve_persist_delta_compact_hash_in_rust( + payload: PersistDeltaCompactHashPayload, +) -> PersistDeltaIdResult { + let before_node_hash_by_id = + build_compact_hash_lookup(&payload.before_nodes.ids, &payload.before_nodes.hashes); + let after_node_hash_by_id = + build_compact_hash_lookup(&payload.after_nodes.ids, &payload.after_nodes.hashes); + let before_edge_hash_by_id = + build_compact_hash_lookup(&payload.before_edges.ids, &payload.before_edges.hashes); + let after_edge_hash_by_id = + build_compact_hash_lookup(&payload.after_edges.ids, &payload.after_edges.hashes); + let before_tombstone_hash_by_id = build_compact_hash_lookup( + &payload.before_tombstones.ids, + &payload.before_tombstones.hashes, + ); + let after_tombstone_target_key_by_id = build_compact_target_key_lookup( + &payload.after_tombstones.ids, + &payload.after_tombstones.target_keys, + ); + + let mut upsert_node_ids = Vec::new(); + let after_node_len = payload + .after_nodes + .ids + .len() + .min(payload.after_nodes.hashes.len()); + for index in 0..after_node_len { + let id = payload.after_nodes.ids[index].trim(); + if id.is_empty() { + continue; + } + let hash = payload.after_nodes.hashes[index]; + if before_node_hash_by_id.get(id) != Some(&hash) { + upsert_node_ids.push(id.to_string()); + } + } + + let mut upsert_edge_ids = Vec::new(); + let after_edge_len = payload + .after_edges + .ids + .len() + .min(payload.after_edges.hashes.len()); + for index in 0..after_edge_len { + let id = payload.after_edges.ids[index].trim(); + if id.is_empty() { + continue; + } + let hash = payload.after_edges.hashes[index]; + if before_edge_hash_by_id.get(id) != Some(&hash) { + upsert_edge_ids.push(id.to_string()); + } + } + + let mut delete_node_ids = Vec::new(); + for id in &payload.before_nodes.ids { + let normalized_id = id.trim(); + if normalized_id.is_empty() { + continue; + } + if !after_node_hash_by_id.contains_key(normalized_id) { + delete_node_ids.push(normalized_id.to_string()); + } + } + + let mut delete_edge_ids = Vec::new(); + for id in &payload.before_edges.ids { + let normalized_id = id.trim(); + if normalized_id.is_empty() { + continue; + } + if !after_edge_hash_by_id.contains_key(normalized_id) { + delete_edge_ids.push(normalized_id.to_string()); + } + } + + let mut upsert_tombstone_ids = Vec::new(); + let after_tombstone_len = payload + .after_tombstones + .ids + .len() + .min(payload.after_tombstones.hashes.len()); + for index in 0..after_tombstone_len { + let id = payload.after_tombstones.ids[index].trim(); + if id.is_empty() { + continue; + } + let target_key = after_tombstone_target_key_by_id + .get(id) + .copied() + .unwrap_or_default(); + if target_key.is_empty() { + continue; + } + let hash = payload.after_tombstones.hashes[index]; + if before_tombstone_hash_by_id.get(id) != Some(&hash) { + upsert_tombstone_ids.push(id.to_string()); + } + } + + PersistDeltaIdResult { + upsert_node_ids, + upsert_edge_ids, + delete_node_ids, + delete_edge_ids, + upsert_tombstone_ids, + } +} + fn build_region_buckets(nodes: &[LayoutNode]) -> HashMap> { let mut region_buckets = HashMap::new(); for (index, node) in nodes.iter().enumerate() { @@ -849,3 +1008,13 @@ pub fn build_persist_delta_compact(payload: JsValue) -> Result JsValue::from_str(&format!("serialize compact persist result failed: {error}")) }) } + +#[wasm_bindgen] +pub fn build_persist_delta_compact_hash(payload: JsValue) -> Result { + let parsed: PersistDeltaCompactHashPayload = serde_wasm_bindgen::from_value(payload) + .map_err(|error| JsValue::from_str(&format!("invalid hash compact persist payload: {error}")))?; + let solved = solve_persist_delta_compact_hash_in_rust(parsed); + serde_wasm_bindgen::to_value(&solved).map_err(|error| { + JsValue::from_str(&format!("serialize hash compact persist result failed: {error}")) + }) +} diff --git a/runtime/settings-defaults.js b/runtime/settings-defaults.js index 06ffdac..af27db5 100644 --- a/runtime/settings-defaults.js +++ b/runtime/settings-defaults.js @@ -121,6 +121,7 @@ export const defaultSettings = { persistNativeDeltaThresholdRecords: 20000, persistNativeDeltaThresholdStructuralDelta: 600, persistNativeDeltaThresholdSerializedChars: 4000000, + persistNativeDeltaBridgeMode: "json", nativeEngineFailOpen: true, graphNativeForceDisable: false, diff --git a/sync/bme-db.js b/sync/bme-db.js index 3a63663..2269610 100644 --- a/sync/bme-db.js +++ b/sync/bme-db.js @@ -18,6 +18,8 @@ export const BME_LEGACY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000; const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_RECORDS = 20000; const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_STRUCTURAL_DELTA = 600; const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_SERIALIZED_CHARS = 4000000; +const DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE = "json"; +const SUPPORTED_PERSIST_NATIVE_DELTA_BRIDGE_MODES = new Set(["json", "hash"]); export const BME_RUNTIME_HISTORY_META_KEY = "runtimeHistoryState"; export const BME_RUNTIME_VECTOR_META_KEY = "runtimeVectorIndexState"; @@ -251,6 +253,20 @@ export function resolvePersistNativeDeltaGateOptions(options = {}) { }; } +export function resolvePersistNativeDeltaBridgeMode(options = {}) { + const rawMode = String( + options?.persistNativeDeltaBridgeMode ?? + options?.nativeDeltaBridgeMode ?? + DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE, + ) + .trim() + .toLowerCase(); + if (!rawMode) return DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE; + return SUPPORTED_PERSIST_NATIVE_DELTA_BRIDGE_MODES.has(rawMode) + ? rawMode + : DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE; +} + export function evaluatePersistNativeDeltaGate( beforeSnapshot, afterSnapshot, @@ -610,30 +626,44 @@ function normalizeSnapshotMetaState(snapshot = {}) { }; } +function hashPersistSerializedRecord32(value = "") { + let hash = 2166136261; + for (let index = 0; index < value.length; index++) { + hash ^= value.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return hash >>> 0; +} + function buildPreparedRecordSet( records = [], { retainRecords = false, includeTargetKeys = false, includeSerializedList = false, + includeHashList = false, + includeSerializedLookup = true, includeSerializedCharCount = false, } = {}, ) { + const sourceRecords = toArray(records); const ids = []; const serialized = includeSerializedList ? [] : null; - const serializedById = new Map(); + const hashes = includeHashList ? [] : null; + const serializedById = includeSerializedLookup ? new Map() : null; const recordById = retainRecords ? new Map() : null; const targetKeyById = includeTargetKeys ? new Map() : null; let serializedCharCount = 0; - for (const record of toArray(records)) { + for (const record of sourceRecords) { if (!record || typeof record !== "object" || Array.isArray(record)) continue; const id = normalizeRecordId(record.id); if (!id) continue; const json = JSON.stringify(record); ids.push(id); if (serialized) serialized.push(json); - serializedById.set(id, json); + if (hashes) hashes.push(hashPersistSerializedRecord32(json)); + if (serializedById) serializedById.set(id, json); if (includeSerializedCharCount) { serializedCharCount += json.length; } @@ -648,47 +678,91 @@ function buildPreparedRecordSet( return { ids, serialized, + hashes, serializedById, + sourceRecords, recordById, targetKeyById, serializedCharCount, }; } +function ensurePreparedSerializedLookup(recordSet = null) { + if (!recordSet || typeof recordSet !== "object") { + return new Map(); + } + if (recordSet.serializedById instanceof Map) { + return recordSet.serializedById; + } + + const map = new Map(); + for (const record of toArray(recordSet.sourceRecords)) { + if (!record || typeof record !== "object" || Array.isArray(record)) continue; + const id = normalizeRecordId(record.id); + if (!id) continue; + map.set(id, JSON.stringify(record)); + } + recordSet.serializedById = map; + return map; +} + function buildPreparedPersistDeltaContext( beforeSnapshot, afterSnapshot, nowMs, options = {}, ) { - const includeCompactPayload = options.includeCompactPayload === true; + const compactPayloadModeRaw = String(options.compactPayloadMode || "none") + .trim() + .toLowerCase(); + const compactPayloadMode = + compactPayloadModeRaw === "hash" + ? "hash" + : compactPayloadModeRaw === "json" + ? "json" + : "none"; + const includeCompactSerializedList = compactPayloadMode === "json"; + const includeCompactHashList = compactPayloadMode === "hash"; + const includeSerializedLookup = options.includeSerializedLookup !== false; const includeSerializedCharCount = options.includeSerializedCharCount === true; const beforeNodes = buildPreparedRecordSet(beforeSnapshot.nodes, { - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const afterNodes = buildPreparedRecordSet(afterSnapshot.nodes, { retainRecords: true, - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const beforeEdges = buildPreparedRecordSet(beforeSnapshot.edges, { - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const afterEdges = buildPreparedRecordSet(afterSnapshot.edges, { retainRecords: true, - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const beforeTombstones = buildPreparedRecordSet(beforeSnapshot.tombstones, { - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const afterTombstones = buildPreparedRecordSet(afterSnapshot.tombstones, { retainRecords: true, includeTargetKeys: true, - includeSerializedList: includeCompactPayload, + includeSerializedList: includeCompactSerializedList, + includeHashList: includeCompactHashList, + includeSerializedLookup, includeSerializedCharCount, }); const sourceDeviceId = normalizeRecordId( @@ -729,38 +803,72 @@ function buildPreparedPersistDeltaContext( Math.abs(afterTombstones.ids.length - beforeTombstones.ids.length), beforeSerializedChars, afterSerializedChars, - compactPayload: includeCompactPayload - ? { - nowMs, - beforeNodes: { - ids: beforeNodes.ids, - serialized: beforeNodes.serialized, - }, - afterNodes: { - ids: afterNodes.ids, - serialized: afterNodes.serialized, - }, - beforeEdges: { - ids: beforeEdges.ids, - serialized: beforeEdges.serialized, - }, - afterEdges: { - ids: afterEdges.ids, - serialized: afterEdges.serialized, - }, - beforeTombstones: { - ids: beforeTombstones.ids, - serialized: beforeTombstones.serialized, - }, - afterTombstones: { - ids: afterTombstones.ids, - serialized: afterTombstones.serialized, - targetKeys: afterTombstones.ids.map( - (id) => afterTombstones.targetKeyById?.get(id) || "", - ), - }, - } - : null, + compactPayload: + compactPayloadMode === "json" + ? { + bridgeMode: "json", + nowMs, + beforeNodes: { + ids: beforeNodes.ids, + serialized: beforeNodes.serialized, + }, + afterNodes: { + ids: afterNodes.ids, + serialized: afterNodes.serialized, + }, + beforeEdges: { + ids: beforeEdges.ids, + serialized: beforeEdges.serialized, + }, + afterEdges: { + ids: afterEdges.ids, + serialized: afterEdges.serialized, + }, + beforeTombstones: { + ids: beforeTombstones.ids, + serialized: beforeTombstones.serialized, + }, + afterTombstones: { + ids: afterTombstones.ids, + serialized: afterTombstones.serialized, + targetKeys: afterTombstones.ids.map( + (id) => afterTombstones.targetKeyById?.get(id) || "", + ), + }, + } + : compactPayloadMode === "hash" + ? { + bridgeMode: "hash", + nowMs, + beforeNodes: { + ids: beforeNodes.ids, + hashes: beforeNodes.hashes, + }, + afterNodes: { + ids: afterNodes.ids, + hashes: afterNodes.hashes, + }, + beforeEdges: { + ids: beforeEdges.ids, + hashes: beforeEdges.hashes, + }, + afterEdges: { + ids: afterEdges.ids, + hashes: afterEdges.hashes, + }, + beforeTombstones: { + ids: beforeTombstones.ids, + hashes: beforeTombstones.hashes, + }, + afterTombstones: { + ids: afterTombstones.ids, + hashes: afterTombstones.hashes, + targetKeys: afterTombstones.ids.map( + (id) => afterTombstones.targetKeyById?.get(id) || "", + ), + }, + } + : null, }; } @@ -998,6 +1106,7 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { const normalizedBefore = normalizePersistSnapshotView(beforeSnapshot); const normalizedAfter = normalizePersistSnapshotView(afterSnapshot); const nowMs = normalizeTimestamp(options.nowMs, Date.now()); + const nativeBridgeMode = resolvePersistNativeDeltaBridgeMode(options); const nativeGateOptions = options?.useNativeDelta === true ? resolvePersistNativeDeltaGateOptions(options) @@ -1011,7 +1120,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { normalizedAfter, nowMs, { - includeCompactPayload: options?.useNativeDelta === true, + compactPayloadMode: options?.useNativeDelta === true ? nativeBridgeMode : "none", + includeSerializedLookup: options?.useNativeDelta !== true, includeSerializedCharCount: shouldMeasureSerializedChars, }, ); @@ -1074,8 +1184,12 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { if (shouldCollectDiagnostics) { emitPersistDeltaDiagnostics(options, { requestedNative: options?.useNativeDelta === true, + requestedBridgeMode: options?.useNativeDelta === true ? nativeBridgeMode : "none", + preparedBridgeMode: preparedContext.compactPayload?.bridgeMode || "none", usedNative: true, - path: nativeIdDelta ? "native-compact" : "native-full", + path: nativeIdDelta + ? `native-compact-${preparedContext.compactPayload?.bridgeMode || "json"}` + : "native-full", gateAllowed: preparedNativeGate?.allowed ?? false, gateReasons: preparedNativeGate?.reasons || [], nativeAttemptStatus: nativeAttempt.status, @@ -1100,11 +1214,29 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { return result; } + const beforeNodeSerializedById = ensurePreparedSerializedLookup( + preparedContext.beforeNodes, + ); + const afterNodeSerializedById = ensurePreparedSerializedLookup( + preparedContext.afterNodes, + ); + const beforeEdgeSerializedById = ensurePreparedSerializedLookup( + preparedContext.beforeEdges, + ); + const afterEdgeSerializedById = ensurePreparedSerializedLookup( + preparedContext.afterEdges, + ); + const beforeTombstoneSerializedById = ensurePreparedSerializedLookup( + preparedContext.beforeTombstones, + ); + const afterTombstoneSerializedById = ensurePreparedSerializedLookup( + preparedContext.afterTombstones, + ); + const upsertNodes = []; for (const id of preparedContext.afterNodes.ids) { if ( - preparedContext.beforeNodes.serializedById.get(id) !== - preparedContext.afterNodes.serializedById.get(id) + beforeNodeSerializedById.get(id) !== afterNodeSerializedById.get(id) ) { const record = preparedContext.afterNodes.recordById?.get(id); if (record) upsertNodes.push(record); @@ -1114,8 +1246,7 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { const upsertEdges = []; for (const id of preparedContext.afterEdges.ids) { if ( - preparedContext.beforeEdges.serializedById.get(id) !== - preparedContext.afterEdges.serializedById.get(id) + beforeEdgeSerializedById.get(id) !== afterEdgeSerializedById.get(id) ) { const record = preparedContext.afterEdges.recordById?.get(id); if (record) upsertEdges.push(record); @@ -1124,14 +1255,14 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { const deleteNodeIds = []; for (const id of preparedContext.beforeNodes.ids) { - if (!preparedContext.afterNodes.serializedById.has(id)) { + if (!afterNodeSerializedById.has(id)) { deleteNodeIds.push(id); } } const deleteEdgeIds = []; for (const id of preparedContext.beforeEdges.ids) { - if (!preparedContext.afterEdges.serializedById.has(id)) { + if (!afterEdgeSerializedById.has(id)) { deleteEdgeIds.push(id); } } @@ -1139,8 +1270,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { const tombstoneMap = new Map(); for (const id of preparedContext.afterTombstones.ids) { if ( - preparedContext.beforeTombstones.serializedById.get(id) !== - preparedContext.afterTombstones.serializedById.get(id) + beforeTombstoneSerializedById.get(id) !== + afterTombstoneSerializedById.get(id) ) { const record = preparedContext.afterTombstones.recordById?.get(id); const targetKey = preparedContext.afterTombstones.targetKeyById?.get(id) || ""; @@ -1186,6 +1317,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) { if (shouldCollectDiagnostics) { emitPersistDeltaDiagnostics(options, { requestedNative: options?.useNativeDelta === true, + requestedBridgeMode: options?.useNativeDelta === true ? nativeBridgeMode : "none", + preparedBridgeMode: preparedContext.compactPayload?.bridgeMode || "none", usedNative: false, path: "js", gateAllowed: preparedNativeGate?.allowed ?? false, diff --git a/tests/default-settings.mjs b/tests/default-settings.mjs index 39eaf70..0bdaebd 100644 --- a/tests/default-settings.mjs +++ b/tests/default-settings.mjs @@ -75,6 +75,7 @@ assert.equal(defaultSettings.persistUseNativeDelta, false); assert.equal(defaultSettings.persistNativeDeltaThresholdRecords, 20000); assert.equal(defaultSettings.persistNativeDeltaThresholdStructuralDelta, 600); assert.equal(defaultSettings.persistNativeDeltaThresholdSerializedChars, 4000000); +assert.equal(defaultSettings.persistNativeDeltaBridgeMode, "json"); assert.equal(defaultSettings.nativeEngineFailOpen, true); assert.equal(defaultSettings.graphNativeForceDisable, false); assert.equal(defaultSettings.taskProfilesVersion, 3); diff --git a/tests/native-layout-wrapper.mjs b/tests/native-layout-wrapper.mjs index 52512dd..6e543ce 100644 --- a/tests/native-layout-wrapper.mjs +++ b/tests/native-layout-wrapper.mjs @@ -33,6 +33,17 @@ try { upsertTombstoneIds: [], }; }, + build_persist_delta_compact_hash(payload = {}) { + return { + upsertNodeIds: Array.isArray(payload?.afterNodes?.ids) + ? payload.afterNodes.ids.slice(0, 1) + : [], + upsertEdgeIds: [], + deleteNodeIds: [], + deleteEdgeIds: [], + upsertTombstoneIds: [], + }; + }, build_persist_delta(payload = {}) { return { upsertNodes: [{ id: "persist-native-node", marker: payload?.afterSnapshot?.meta?.chatId || "" }], @@ -66,12 +77,26 @@ try { { nowMs: 123, preparedDeltaInput: { + bridgeMode: "json", afterNodes: { ids: ["persist-native-node"], serialized: ["{}"] }, }, }, ); assert.deepEqual(compactDeltaResult.upsertNodeIds, ["persist-native-node"]); + const hashCompactDeltaResult = globalThis.__stBmeNativeBuildPersistDelta( + { meta: { chatId: "before-chat" }, nodes: [], edges: [], tombstones: [], state: {} }, + { meta: { chatId: "after-chat" }, nodes: [], edges: [], tombstones: [], state: {} }, + { + nowMs: 123, + preparedDeltaInput: { + bridgeMode: "hash", + afterNodes: { ids: ["persist-native-node"], hashes: [1] }, + }, + }, + ); + assert.deepEqual(hashCompactDeltaResult.upsertNodeIds, ["persist-native-node"]); + const deltaResult = globalThis.__stBmeNativeBuildPersistDelta( { meta: { chatId: "before-chat" }, nodes: [], edges: [], tombstones: [], state: {} }, { meta: { chatId: "after-chat" }, nodes: [], edges: [], tombstones: [], state: {} }, diff --git a/tests/native-persist-delta-hook.mjs b/tests/native-persist-delta-hook.mjs index b5ffcd0..8da80ec 100644 --- a/tests/native-persist-delta-hook.mjs +++ b/tests/native-persist-delta-hook.mjs @@ -3,6 +3,7 @@ import assert from "node:assert/strict"; import { buildPersistDelta, evaluatePersistNativeDeltaGate, + resolvePersistNativeDeltaBridgeMode, resolvePersistNativeDeltaGateOptions, shouldUseNativePersistDeltaForSnapshots, } from "../sync/bme-db.js"; @@ -40,6 +41,9 @@ const defaultGate = resolvePersistNativeDeltaGateOptions({}); assert.equal(defaultGate.minSnapshotRecords, 20000); assert.equal(defaultGate.minStructuralDelta, 600); assert.equal(defaultGate.minCombinedSerializedChars, 4000000); +assert.equal(resolvePersistNativeDeltaBridgeMode({}), "json"); +assert.equal(resolvePersistNativeDeltaBridgeMode({ persistNativeDeltaBridgeMode: "hash" }), "hash"); +assert.equal(resolvePersistNativeDeltaBridgeMode({ persistNativeDeltaBridgeMode: "unknown" }), "json"); assert.equal( shouldUseNativePersistDeltaForSnapshots(beforeSnapshot, afterSnapshot, defaultGate), false, @@ -160,9 +164,32 @@ assert.deepEqual(compactNativeDelta.upsertEdges, []); assert.deepEqual(compactNativeDelta.deleteNodeIds, []); assert.equal(compactNativeDelta.runtimeMetaPatch.compact, true); assert.equal(compactNativeDelta.runtimeMetaPatch.chatId, "chat-native"); -assert.equal(compactDiagnostics.path, "native-compact"); +assert.equal(compactDiagnostics.path, "native-compact-json"); +assert.equal(compactDiagnostics.preparedBridgeMode, "json"); +assert.equal(compactDiagnostics.requestedBridgeMode, "json"); assert.equal(compactDiagnostics.usedNative, true); +let hashDiagnostics = null; +const hashNativeDelta = buildPersistDelta(beforeSnapshot, afterSnapshot, { + useNativeDelta: true, + minSnapshotRecords: 0, + minStructuralDelta: 0, + minCombinedSerializedChars: 0, + persistNativeDeltaBridgeMode: "hash", + runtimeMetaPatch: { hashMode: true }, + onDiagnostics(snapshot) { + hashDiagnostics = snapshot; + }, +}); +assert.deepEqual(hashNativeDelta.upsertNodes, [ + { id: "n1", type: "event", fields: { text: "after" }, updatedAt: 2 }, +]); +assert.equal(hashNativeDelta.runtimeMetaPatch.hashMode, true); +assert.equal(hashDiagnostics.path, "native-compact-hash"); +assert.equal(hashDiagnostics.preparedBridgeMode, "hash"); +assert.equal(hashDiagnostics.requestedBridgeMode, "hash"); +assert.equal(hashDiagnostics.usedNative, true); + delete globalThis.__stBmeNativeBuildPersistDelta; let threwUnavailable = false; diff --git a/tests/perf/persist-delta-bench.mjs b/tests/perf/persist-delta-bench.mjs index 80764a1..fcbe137 100644 --- a/tests/perf/persist-delta-bench.mjs +++ b/tests/perf/persist-delta-bench.mjs @@ -104,7 +104,8 @@ async function main() { await installNativePersistDeltaHook(); const nativeStatus = getNativeModuleStatus(); const jsSamples = []; - const nativeSamples = []; + const nativeJsonSamples = []; + const nativeHashSamples = []; for (let run = 0; run < RUNS; run++) { const snapshots = buildSnapshots(17 + run, 5000, 12000, 0.12); const jsStartedAt = performance.now(); @@ -120,26 +121,50 @@ async function main() { deleteEdgeIds: jsDelta.deleteEdgeIds.length, }); - const nativeStartedAt = performance.now(); - const nativeDelta = buildPersistDelta(snapshots.before, snapshots.after, { + const nativeJsonStartedAt = performance.now(); + const nativeJsonDelta = buildPersistDelta(snapshots.before, snapshots.after, { useNativeDelta: true, minSnapshotRecords: 0, minStructuralDelta: 0, minCombinedSerializedChars: 0, + persistNativeDeltaBridgeMode: "json", nativeFailOpen: false, }); - const nativeElapsedMs = performance.now() - nativeStartedAt; - nativeSamples.push({ - elapsedMs: nativeElapsedMs, - upsertNodes: nativeDelta.upsertNodes.length, - upsertEdges: nativeDelta.upsertEdges.length, - deleteNodeIds: nativeDelta.deleteNodeIds.length, - deleteEdgeIds: nativeDelta.deleteEdgeIds.length, + const nativeJsonElapsedMs = performance.now() - nativeJsonStartedAt; + nativeJsonSamples.push({ + elapsedMs: nativeJsonElapsedMs, + upsertNodes: nativeJsonDelta.upsertNodes.length, + upsertEdges: nativeJsonDelta.upsertEdges.length, + deleteNodeIds: nativeJsonDelta.deleteNodeIds.length, + deleteEdgeIds: nativeJsonDelta.deleteEdgeIds.length, + }); + + const nativeHashStartedAt = performance.now(); + const nativeHashDelta = buildPersistDelta(snapshots.before, snapshots.after, { + useNativeDelta: true, + minSnapshotRecords: 0, + minStructuralDelta: 0, + minCombinedSerializedChars: 0, + persistNativeDeltaBridgeMode: "hash", + nativeFailOpen: false, + }); + const nativeHashElapsedMs = performance.now() - nativeHashStartedAt; + nativeHashSamples.push({ + elapsedMs: nativeHashElapsedMs, + upsertNodes: nativeHashDelta.upsertNodes.length, + upsertEdges: nativeHashDelta.upsertEdges.length, + deleteNodeIds: nativeHashDelta.deleteNodeIds.length, + deleteEdgeIds: nativeHashDelta.deleteEdgeIds.length, }); } const jsTimingSummary = summarize(jsSamples.map((sample) => sample.elapsedMs)); - const nativeTimingSummary = summarize(nativeSamples.map((sample) => sample.elapsedMs)); + const nativeJsonTimingSummary = summarize( + nativeJsonSamples.map((sample) => sample.elapsedMs), + ); + const nativeHashTimingSummary = summarize( + nativeHashSamples.map((sample) => sample.elapsedMs), + ); const avgUpserts = jsSamples.reduce((acc, sample) => acc + sample.upsertNodes + sample.upsertEdges, 0) / jsSamples.length; @@ -151,7 +176,7 @@ async function main() { `[ST-BME][bench] persist-delta native-source=${nativeStatus.source || "unknown"}`, ); console.log( - `[ST-BME][bench] persist-delta runs=${RUNS} | js avg=${jsTimingSummary.avg.toFixed(2)}ms p95=${jsTimingSummary.p95.toFixed(2)}ms min=${jsTimingSummary.min.toFixed(2)}ms max=${jsTimingSummary.max.toFixed(2)}ms | native avg=${nativeTimingSummary.avg.toFixed(2)}ms p95=${nativeTimingSummary.p95.toFixed(2)}ms min=${nativeTimingSummary.min.toFixed(2)}ms max=${nativeTimingSummary.max.toFixed(2)}ms | avgUpserts=${avgUpserts.toFixed(1)} avgDeletes=${avgDeletes.toFixed(1)}`, + `[ST-BME][bench] persist-delta runs=${RUNS} | js avg=${jsTimingSummary.avg.toFixed(2)}ms p95=${jsTimingSummary.p95.toFixed(2)}ms min=${jsTimingSummary.min.toFixed(2)}ms max=${jsTimingSummary.max.toFixed(2)}ms | native-json avg=${nativeJsonTimingSummary.avg.toFixed(2)}ms p95=${nativeJsonTimingSummary.p95.toFixed(2)}ms min=${nativeJsonTimingSummary.min.toFixed(2)}ms max=${nativeJsonTimingSummary.max.toFixed(2)}ms | native-hash avg=${nativeHashTimingSummary.avg.toFixed(2)}ms p95=${nativeHashTimingSummary.p95.toFixed(2)}ms min=${nativeHashTimingSummary.min.toFixed(2)}ms max=${nativeHashTimingSummary.max.toFixed(2)}ms | avgUpserts=${avgUpserts.toFixed(1)} avgDeletes=${avgDeletes.toFixed(1)}`, ); } diff --git a/ui/panel.js b/ui/panel.js index 259d3f3..1f86c6b 100644 --- a/ui/panel.js +++ b/ui/panel.js @@ -8060,6 +8060,12 @@ function _renderPersistDeltaTraceCard(state) { 执行路径 ${_escHtml(String(diagnostics.path || "—"))} +
+ Bridge 模式 + ${_escHtml( + `${String(diagnostics.requestedBridgeMode || "none")} → ${String(diagnostics.preparedBridgeMode || "none")}`, + )} +
Native Gate ${_escHtml(_formatPersistDeltaGateText(diagnostics))} diff --git a/vendor/wasm/stbme_core.js b/vendor/wasm/stbme_core.js index 86e1e49..ada40be 100644 --- a/vendor/wasm/stbme_core.js +++ b/vendor/wasm/stbme_core.js @@ -54,6 +54,10 @@ async function loadFromWasmPackArtifacts() { return { solve_layout: module.solve_layout, + build_persist_delta_compact_hash: + typeof module.build_persist_delta_compact_hash === "function" + ? module.build_persist_delta_compact_hash + : null, build_persist_delta_compact: typeof module.build_persist_delta_compact === "function" ? module.build_persist_delta_compact @@ -147,7 +151,8 @@ export async function installNativePersistDeltaHook() { const module = await loadNativeModule(); if ( !module || - (typeof module.build_persist_delta_compact !== "function" && + (typeof module.build_persist_delta_compact_hash !== "function" && + typeof module.build_persist_delta_compact !== "function" && typeof module.build_persist_delta !== "function") ) { throw new Error("native persist delta builder unavailable"); @@ -155,12 +160,32 @@ export async function installNativePersistDeltaHook() { globalThis.__stBmeNativeBuildPersistDelta = (beforeSnapshot, afterSnapshot, options = {}) => { let raw = null; + const preparedInput = + options?.preparedDeltaInput && typeof options.preparedDeltaInput === "object" + ? options.preparedDeltaInput + : null; + const preparedBridgeMode = String(preparedInput?.bridgeMode || "") + .trim() + .toLowerCase(); if ( - typeof module.build_persist_delta_compact === "function" && - options?.preparedDeltaInput && - typeof options.preparedDeltaInput === "object" + typeof module.build_persist_delta_compact_hash === "function" && + preparedInput && + preparedBridgeMode === "hash" ) { - raw = module.build_persist_delta_compact(options.preparedDeltaInput); + raw = module.build_persist_delta_compact_hash(preparedInput); + } else if ( + typeof module.build_persist_delta_compact === "function" && + preparedInput && + (preparedBridgeMode === "json" || preparedBridgeMode === "") + ) { + raw = module.build_persist_delta_compact(preparedInput); + } else if ( + typeof module.build_persist_delta_compact === "function" && + preparedInput && + preparedBridgeMode === "hash" && + Array.isArray(preparedInput?.afterNodes?.serialized) + ) { + raw = module.build_persist_delta_compact(preparedInput); } else if (typeof module.build_persist_delta === "function") { raw = module.build_persist_delta({ beforeSnapshot,