mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-06-13 18:31:16 +08:00
perf: add hash compact persist-delta bridge mode
This commit is contained in:
7
index.js
7
index.js
@@ -9684,6 +9684,9 @@ async function saveGraphToIndexedDb(
|
||||
},
|
||||
});
|
||||
const currentSettings = getSettings();
|
||||
const nativePersistBridgeMode = String(
|
||||
currentSettings.persistNativeDeltaBridgeMode || "json",
|
||||
);
|
||||
const nativePersistRequested = currentSettings.persistUseNativeDelta === true;
|
||||
const nativePersistForceDisabled = currentSettings.graphNativeForceDisable === true;
|
||||
const nativePersistGate = evaluatePersistNativeDeltaGate(
|
||||
@@ -9712,6 +9715,7 @@ async function saveGraphToIndexedDb(
|
||||
saveReason: String(reason || "graph-save"),
|
||||
requestedRevision,
|
||||
requestedNative: nativePersistRequested,
|
||||
requestedBridgeMode: nativePersistBridgeMode,
|
||||
nativeForceDisabled: nativePersistForceDisabled,
|
||||
nativeFailOpen: currentSettings.nativeEngineFailOpen !== false,
|
||||
gateAllowed: nativePersistGate.allowed,
|
||||
@@ -9771,6 +9775,7 @@ async function saveGraphToIndexedDb(
|
||||
currentSettings.persistNativeDeltaThresholdStructuralDelta,
|
||||
persistNativeDeltaThresholdSerializedChars:
|
||||
currentSettings.persistNativeDeltaThresholdSerializedChars,
|
||||
persistNativeDeltaBridgeMode: nativePersistBridgeMode,
|
||||
onDiagnostics(snapshot) {
|
||||
persistDeltaBuildDiagnostics = snapshot;
|
||||
},
|
||||
@@ -9822,6 +9827,8 @@ async function saveGraphToIndexedDb(
|
||||
saveReason: String(reason || "graph-save"),
|
||||
requestedRevision,
|
||||
requestedNative: nativePersistRequested,
|
||||
requestedBridgeMode:
|
||||
persistDeltaBuildDiagnostics?.requestedBridgeMode || nativePersistBridgeMode,
|
||||
buildRequestedNative: Boolean(persistDeltaBuildDiagnostics?.requestedNative),
|
||||
nativeForceDisabled: nativePersistForceDisabled,
|
||||
nativeFailOpen: currentSettings.nativeEngineFailOpen !== false,
|
||||
|
||||
@@ -149,6 +149,15 @@ struct PersistCompactRecordSet {
|
||||
serialized: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistCompactHashRecordSet {
|
||||
#[serde(default)]
|
||||
ids: Vec<String>,
|
||||
#[serde(default)]
|
||||
hashes: Vec<u32>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistCompactTombstoneSet {
|
||||
@@ -160,6 +169,17 @@ struct PersistCompactTombstoneSet {
|
||||
target_keys: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistCompactHashTombstoneSet {
|
||||
#[serde(default)]
|
||||
ids: Vec<String>,
|
||||
#[serde(default)]
|
||||
hashes: Vec<u32>,
|
||||
#[serde(default)]
|
||||
target_keys: Vec<String>,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistDeltaCompactPayload {
|
||||
@@ -177,6 +197,23 @@ struct PersistDeltaCompactPayload {
|
||||
after_tombstones: PersistCompactTombstoneSet,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Deserialize, Default)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistDeltaCompactHashPayload {
|
||||
#[serde(default)]
|
||||
before_nodes: PersistCompactHashRecordSet,
|
||||
#[serde(default)]
|
||||
after_nodes: PersistCompactHashRecordSet,
|
||||
#[serde(default)]
|
||||
before_edges: PersistCompactHashRecordSet,
|
||||
#[serde(default)]
|
||||
after_edges: PersistCompactHashRecordSet,
|
||||
#[serde(default)]
|
||||
before_tombstones: PersistCompactHashRecordSet,
|
||||
#[serde(default)]
|
||||
after_tombstones: PersistCompactHashTombstoneSet,
|
||||
}
|
||||
|
||||
#[derive(Debug, Clone, Serialize)]
|
||||
#[serde(rename_all = "camelCase")]
|
||||
struct PersistDeltaIdResult {
|
||||
@@ -285,6 +322,19 @@ fn build_json_serialized_index(records: &[JsonValue]) -> HashMap<String, String>
|
||||
map
|
||||
}
|
||||
|
||||
fn build_compact_hash_lookup<'a>(ids: &'a [String], hashes: &'a [u32]) -> HashMap<&'a str, u32> {
|
||||
let mut map = HashMap::new();
|
||||
let len = ids.len().min(hashes.len());
|
||||
for index in 0..len {
|
||||
let id = ids[index].trim();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
map.insert(id, hashes[index]);
|
||||
}
|
||||
map
|
||||
}
|
||||
|
||||
fn build_json_value_index(records: &[JsonValue]) -> HashMap<String, JsonValue> {
|
||||
let mut map = HashMap::new();
|
||||
for record in records {
|
||||
@@ -605,6 +655,115 @@ fn solve_persist_delta_compact_in_rust(payload: PersistDeltaCompactPayload) -> P
|
||||
}
|
||||
}
|
||||
|
||||
fn solve_persist_delta_compact_hash_in_rust(
|
||||
payload: PersistDeltaCompactHashPayload,
|
||||
) -> PersistDeltaIdResult {
|
||||
let before_node_hash_by_id =
|
||||
build_compact_hash_lookup(&payload.before_nodes.ids, &payload.before_nodes.hashes);
|
||||
let after_node_hash_by_id =
|
||||
build_compact_hash_lookup(&payload.after_nodes.ids, &payload.after_nodes.hashes);
|
||||
let before_edge_hash_by_id =
|
||||
build_compact_hash_lookup(&payload.before_edges.ids, &payload.before_edges.hashes);
|
||||
let after_edge_hash_by_id =
|
||||
build_compact_hash_lookup(&payload.after_edges.ids, &payload.after_edges.hashes);
|
||||
let before_tombstone_hash_by_id = build_compact_hash_lookup(
|
||||
&payload.before_tombstones.ids,
|
||||
&payload.before_tombstones.hashes,
|
||||
);
|
||||
let after_tombstone_target_key_by_id = build_compact_target_key_lookup(
|
||||
&payload.after_tombstones.ids,
|
||||
&payload.after_tombstones.target_keys,
|
||||
);
|
||||
|
||||
let mut upsert_node_ids = Vec::new();
|
||||
let after_node_len = payload
|
||||
.after_nodes
|
||||
.ids
|
||||
.len()
|
||||
.min(payload.after_nodes.hashes.len());
|
||||
for index in 0..after_node_len {
|
||||
let id = payload.after_nodes.ids[index].trim();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let hash = payload.after_nodes.hashes[index];
|
||||
if before_node_hash_by_id.get(id) != Some(&hash) {
|
||||
upsert_node_ids.push(id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut upsert_edge_ids = Vec::new();
|
||||
let after_edge_len = payload
|
||||
.after_edges
|
||||
.ids
|
||||
.len()
|
||||
.min(payload.after_edges.hashes.len());
|
||||
for index in 0..after_edge_len {
|
||||
let id = payload.after_edges.ids[index].trim();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let hash = payload.after_edges.hashes[index];
|
||||
if before_edge_hash_by_id.get(id) != Some(&hash) {
|
||||
upsert_edge_ids.push(id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut delete_node_ids = Vec::new();
|
||||
for id in &payload.before_nodes.ids {
|
||||
let normalized_id = id.trim();
|
||||
if normalized_id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !after_node_hash_by_id.contains_key(normalized_id) {
|
||||
delete_node_ids.push(normalized_id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut delete_edge_ids = Vec::new();
|
||||
for id in &payload.before_edges.ids {
|
||||
let normalized_id = id.trim();
|
||||
if normalized_id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
if !after_edge_hash_by_id.contains_key(normalized_id) {
|
||||
delete_edge_ids.push(normalized_id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
let mut upsert_tombstone_ids = Vec::new();
|
||||
let after_tombstone_len = payload
|
||||
.after_tombstones
|
||||
.ids
|
||||
.len()
|
||||
.min(payload.after_tombstones.hashes.len());
|
||||
for index in 0..after_tombstone_len {
|
||||
let id = payload.after_tombstones.ids[index].trim();
|
||||
if id.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let target_key = after_tombstone_target_key_by_id
|
||||
.get(id)
|
||||
.copied()
|
||||
.unwrap_or_default();
|
||||
if target_key.is_empty() {
|
||||
continue;
|
||||
}
|
||||
let hash = payload.after_tombstones.hashes[index];
|
||||
if before_tombstone_hash_by_id.get(id) != Some(&hash) {
|
||||
upsert_tombstone_ids.push(id.to_string());
|
||||
}
|
||||
}
|
||||
|
||||
PersistDeltaIdResult {
|
||||
upsert_node_ids,
|
||||
upsert_edge_ids,
|
||||
delete_node_ids,
|
||||
delete_edge_ids,
|
||||
upsert_tombstone_ids,
|
||||
}
|
||||
}
|
||||
|
||||
fn build_region_buckets(nodes: &[LayoutNode]) -> HashMap<String, Vec<usize>> {
|
||||
let mut region_buckets = HashMap::new();
|
||||
for (index, node) in nodes.iter().enumerate() {
|
||||
@@ -849,3 +1008,13 @@ pub fn build_persist_delta_compact(payload: JsValue) -> Result<JsValue, JsValue>
|
||||
JsValue::from_str(&format!("serialize compact persist result failed: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
#[wasm_bindgen]
|
||||
pub fn build_persist_delta_compact_hash(payload: JsValue) -> Result<JsValue, JsValue> {
|
||||
let parsed: PersistDeltaCompactHashPayload = serde_wasm_bindgen::from_value(payload)
|
||||
.map_err(|error| JsValue::from_str(&format!("invalid hash compact persist payload: {error}")))?;
|
||||
let solved = solve_persist_delta_compact_hash_in_rust(parsed);
|
||||
serde_wasm_bindgen::to_value(&solved).map_err(|error| {
|
||||
JsValue::from_str(&format!("serialize hash compact persist result failed: {error}"))
|
||||
})
|
||||
}
|
||||
|
||||
@@ -121,6 +121,7 @@ export const defaultSettings = {
|
||||
persistNativeDeltaThresholdRecords: 20000,
|
||||
persistNativeDeltaThresholdStructuralDelta: 600,
|
||||
persistNativeDeltaThresholdSerializedChars: 4000000,
|
||||
persistNativeDeltaBridgeMode: "json",
|
||||
nativeEngineFailOpen: true,
|
||||
graphNativeForceDisable: false,
|
||||
|
||||
|
||||
237
sync/bme-db.js
237
sync/bme-db.js
@@ -18,6 +18,8 @@ export const BME_LEGACY_RETENTION_MS = 30 * 24 * 60 * 60 * 1000;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_RECORDS = 20000;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_STRUCTURAL_DELTA = 600;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_THRESHOLD_SERIALIZED_CHARS = 4000000;
|
||||
const DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE = "json";
|
||||
const SUPPORTED_PERSIST_NATIVE_DELTA_BRIDGE_MODES = new Set(["json", "hash"]);
|
||||
|
||||
export const BME_RUNTIME_HISTORY_META_KEY = "runtimeHistoryState";
|
||||
export const BME_RUNTIME_VECTOR_META_KEY = "runtimeVectorIndexState";
|
||||
@@ -251,6 +253,20 @@ export function resolvePersistNativeDeltaGateOptions(options = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
export function resolvePersistNativeDeltaBridgeMode(options = {}) {
|
||||
const rawMode = String(
|
||||
options?.persistNativeDeltaBridgeMode ??
|
||||
options?.nativeDeltaBridgeMode ??
|
||||
DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE,
|
||||
)
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (!rawMode) return DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE;
|
||||
return SUPPORTED_PERSIST_NATIVE_DELTA_BRIDGE_MODES.has(rawMode)
|
||||
? rawMode
|
||||
: DEFAULT_PERSIST_NATIVE_DELTA_BRIDGE_MODE;
|
||||
}
|
||||
|
||||
export function evaluatePersistNativeDeltaGate(
|
||||
beforeSnapshot,
|
||||
afterSnapshot,
|
||||
@@ -610,30 +626,44 @@ function normalizeSnapshotMetaState(snapshot = {}) {
|
||||
};
|
||||
}
|
||||
|
||||
function hashPersistSerializedRecord32(value = "") {
|
||||
let hash = 2166136261;
|
||||
for (let index = 0; index < value.length; index++) {
|
||||
hash ^= value.charCodeAt(index);
|
||||
hash = Math.imul(hash, 16777619);
|
||||
}
|
||||
return hash >>> 0;
|
||||
}
|
||||
|
||||
function buildPreparedRecordSet(
|
||||
records = [],
|
||||
{
|
||||
retainRecords = false,
|
||||
includeTargetKeys = false,
|
||||
includeSerializedList = false,
|
||||
includeHashList = false,
|
||||
includeSerializedLookup = true,
|
||||
includeSerializedCharCount = false,
|
||||
} = {},
|
||||
) {
|
||||
const sourceRecords = toArray(records);
|
||||
const ids = [];
|
||||
const serialized = includeSerializedList ? [] : null;
|
||||
const serializedById = new Map();
|
||||
const hashes = includeHashList ? [] : null;
|
||||
const serializedById = includeSerializedLookup ? new Map() : null;
|
||||
const recordById = retainRecords ? new Map() : null;
|
||||
const targetKeyById = includeTargetKeys ? new Map() : null;
|
||||
let serializedCharCount = 0;
|
||||
|
||||
for (const record of toArray(records)) {
|
||||
for (const record of sourceRecords) {
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) continue;
|
||||
const id = normalizeRecordId(record.id);
|
||||
if (!id) continue;
|
||||
const json = JSON.stringify(record);
|
||||
ids.push(id);
|
||||
if (serialized) serialized.push(json);
|
||||
serializedById.set(id, json);
|
||||
if (hashes) hashes.push(hashPersistSerializedRecord32(json));
|
||||
if (serializedById) serializedById.set(id, json);
|
||||
if (includeSerializedCharCount) {
|
||||
serializedCharCount += json.length;
|
||||
}
|
||||
@@ -648,47 +678,91 @@ function buildPreparedRecordSet(
|
||||
return {
|
||||
ids,
|
||||
serialized,
|
||||
hashes,
|
||||
serializedById,
|
||||
sourceRecords,
|
||||
recordById,
|
||||
targetKeyById,
|
||||
serializedCharCount,
|
||||
};
|
||||
}
|
||||
|
||||
function ensurePreparedSerializedLookup(recordSet = null) {
|
||||
if (!recordSet || typeof recordSet !== "object") {
|
||||
return new Map();
|
||||
}
|
||||
if (recordSet.serializedById instanceof Map) {
|
||||
return recordSet.serializedById;
|
||||
}
|
||||
|
||||
const map = new Map();
|
||||
for (const record of toArray(recordSet.sourceRecords)) {
|
||||
if (!record || typeof record !== "object" || Array.isArray(record)) continue;
|
||||
const id = normalizeRecordId(record.id);
|
||||
if (!id) continue;
|
||||
map.set(id, JSON.stringify(record));
|
||||
}
|
||||
recordSet.serializedById = map;
|
||||
return map;
|
||||
}
|
||||
|
||||
function buildPreparedPersistDeltaContext(
|
||||
beforeSnapshot,
|
||||
afterSnapshot,
|
||||
nowMs,
|
||||
options = {},
|
||||
) {
|
||||
const includeCompactPayload = options.includeCompactPayload === true;
|
||||
const compactPayloadModeRaw = String(options.compactPayloadMode || "none")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
const compactPayloadMode =
|
||||
compactPayloadModeRaw === "hash"
|
||||
? "hash"
|
||||
: compactPayloadModeRaw === "json"
|
||||
? "json"
|
||||
: "none";
|
||||
const includeCompactSerializedList = compactPayloadMode === "json";
|
||||
const includeCompactHashList = compactPayloadMode === "hash";
|
||||
const includeSerializedLookup = options.includeSerializedLookup !== false;
|
||||
const includeSerializedCharCount = options.includeSerializedCharCount === true;
|
||||
const beforeNodes = buildPreparedRecordSet(beforeSnapshot.nodes, {
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const afterNodes = buildPreparedRecordSet(afterSnapshot.nodes, {
|
||||
retainRecords: true,
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const beforeEdges = buildPreparedRecordSet(beforeSnapshot.edges, {
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const afterEdges = buildPreparedRecordSet(afterSnapshot.edges, {
|
||||
retainRecords: true,
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const beforeTombstones = buildPreparedRecordSet(beforeSnapshot.tombstones, {
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const afterTombstones = buildPreparedRecordSet(afterSnapshot.tombstones, {
|
||||
retainRecords: true,
|
||||
includeTargetKeys: true,
|
||||
includeSerializedList: includeCompactPayload,
|
||||
includeSerializedList: includeCompactSerializedList,
|
||||
includeHashList: includeCompactHashList,
|
||||
includeSerializedLookup,
|
||||
includeSerializedCharCount,
|
||||
});
|
||||
const sourceDeviceId = normalizeRecordId(
|
||||
@@ -729,38 +803,72 @@ function buildPreparedPersistDeltaContext(
|
||||
Math.abs(afterTombstones.ids.length - beforeTombstones.ids.length),
|
||||
beforeSerializedChars,
|
||||
afterSerializedChars,
|
||||
compactPayload: includeCompactPayload
|
||||
? {
|
||||
nowMs,
|
||||
beforeNodes: {
|
||||
ids: beforeNodes.ids,
|
||||
serialized: beforeNodes.serialized,
|
||||
},
|
||||
afterNodes: {
|
||||
ids: afterNodes.ids,
|
||||
serialized: afterNodes.serialized,
|
||||
},
|
||||
beforeEdges: {
|
||||
ids: beforeEdges.ids,
|
||||
serialized: beforeEdges.serialized,
|
||||
},
|
||||
afterEdges: {
|
||||
ids: afterEdges.ids,
|
||||
serialized: afterEdges.serialized,
|
||||
},
|
||||
beforeTombstones: {
|
||||
ids: beforeTombstones.ids,
|
||||
serialized: beforeTombstones.serialized,
|
||||
},
|
||||
afterTombstones: {
|
||||
ids: afterTombstones.ids,
|
||||
serialized: afterTombstones.serialized,
|
||||
targetKeys: afterTombstones.ids.map(
|
||||
(id) => afterTombstones.targetKeyById?.get(id) || "",
|
||||
),
|
||||
},
|
||||
}
|
||||
: null,
|
||||
compactPayload:
|
||||
compactPayloadMode === "json"
|
||||
? {
|
||||
bridgeMode: "json",
|
||||
nowMs,
|
||||
beforeNodes: {
|
||||
ids: beforeNodes.ids,
|
||||
serialized: beforeNodes.serialized,
|
||||
},
|
||||
afterNodes: {
|
||||
ids: afterNodes.ids,
|
||||
serialized: afterNodes.serialized,
|
||||
},
|
||||
beforeEdges: {
|
||||
ids: beforeEdges.ids,
|
||||
serialized: beforeEdges.serialized,
|
||||
},
|
||||
afterEdges: {
|
||||
ids: afterEdges.ids,
|
||||
serialized: afterEdges.serialized,
|
||||
},
|
||||
beforeTombstones: {
|
||||
ids: beforeTombstones.ids,
|
||||
serialized: beforeTombstones.serialized,
|
||||
},
|
||||
afterTombstones: {
|
||||
ids: afterTombstones.ids,
|
||||
serialized: afterTombstones.serialized,
|
||||
targetKeys: afterTombstones.ids.map(
|
||||
(id) => afterTombstones.targetKeyById?.get(id) || "",
|
||||
),
|
||||
},
|
||||
}
|
||||
: compactPayloadMode === "hash"
|
||||
? {
|
||||
bridgeMode: "hash",
|
||||
nowMs,
|
||||
beforeNodes: {
|
||||
ids: beforeNodes.ids,
|
||||
hashes: beforeNodes.hashes,
|
||||
},
|
||||
afterNodes: {
|
||||
ids: afterNodes.ids,
|
||||
hashes: afterNodes.hashes,
|
||||
},
|
||||
beforeEdges: {
|
||||
ids: beforeEdges.ids,
|
||||
hashes: beforeEdges.hashes,
|
||||
},
|
||||
afterEdges: {
|
||||
ids: afterEdges.ids,
|
||||
hashes: afterEdges.hashes,
|
||||
},
|
||||
beforeTombstones: {
|
||||
ids: beforeTombstones.ids,
|
||||
hashes: beforeTombstones.hashes,
|
||||
},
|
||||
afterTombstones: {
|
||||
ids: afterTombstones.ids,
|
||||
hashes: afterTombstones.hashes,
|
||||
targetKeys: afterTombstones.ids.map(
|
||||
(id) => afterTombstones.targetKeyById?.get(id) || "",
|
||||
),
|
||||
},
|
||||
}
|
||||
: null,
|
||||
};
|
||||
}
|
||||
|
||||
@@ -998,6 +1106,7 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
const normalizedBefore = normalizePersistSnapshotView(beforeSnapshot);
|
||||
const normalizedAfter = normalizePersistSnapshotView(afterSnapshot);
|
||||
const nowMs = normalizeTimestamp(options.nowMs, Date.now());
|
||||
const nativeBridgeMode = resolvePersistNativeDeltaBridgeMode(options);
|
||||
const nativeGateOptions =
|
||||
options?.useNativeDelta === true
|
||||
? resolvePersistNativeDeltaGateOptions(options)
|
||||
@@ -1011,7 +1120,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
normalizedAfter,
|
||||
nowMs,
|
||||
{
|
||||
includeCompactPayload: options?.useNativeDelta === true,
|
||||
compactPayloadMode: options?.useNativeDelta === true ? nativeBridgeMode : "none",
|
||||
includeSerializedLookup: options?.useNativeDelta !== true,
|
||||
includeSerializedCharCount: shouldMeasureSerializedChars,
|
||||
},
|
||||
);
|
||||
@@ -1074,8 +1184,12 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
if (shouldCollectDiagnostics) {
|
||||
emitPersistDeltaDiagnostics(options, {
|
||||
requestedNative: options?.useNativeDelta === true,
|
||||
requestedBridgeMode: options?.useNativeDelta === true ? nativeBridgeMode : "none",
|
||||
preparedBridgeMode: preparedContext.compactPayload?.bridgeMode || "none",
|
||||
usedNative: true,
|
||||
path: nativeIdDelta ? "native-compact" : "native-full",
|
||||
path: nativeIdDelta
|
||||
? `native-compact-${preparedContext.compactPayload?.bridgeMode || "json"}`
|
||||
: "native-full",
|
||||
gateAllowed: preparedNativeGate?.allowed ?? false,
|
||||
gateReasons: preparedNativeGate?.reasons || [],
|
||||
nativeAttemptStatus: nativeAttempt.status,
|
||||
@@ -1100,11 +1214,29 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
return result;
|
||||
}
|
||||
|
||||
const beforeNodeSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.beforeNodes,
|
||||
);
|
||||
const afterNodeSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.afterNodes,
|
||||
);
|
||||
const beforeEdgeSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.beforeEdges,
|
||||
);
|
||||
const afterEdgeSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.afterEdges,
|
||||
);
|
||||
const beforeTombstoneSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.beforeTombstones,
|
||||
);
|
||||
const afterTombstoneSerializedById = ensurePreparedSerializedLookup(
|
||||
preparedContext.afterTombstones,
|
||||
);
|
||||
|
||||
const upsertNodes = [];
|
||||
for (const id of preparedContext.afterNodes.ids) {
|
||||
if (
|
||||
preparedContext.beforeNodes.serializedById.get(id) !==
|
||||
preparedContext.afterNodes.serializedById.get(id)
|
||||
beforeNodeSerializedById.get(id) !== afterNodeSerializedById.get(id)
|
||||
) {
|
||||
const record = preparedContext.afterNodes.recordById?.get(id);
|
||||
if (record) upsertNodes.push(record);
|
||||
@@ -1114,8 +1246,7 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
const upsertEdges = [];
|
||||
for (const id of preparedContext.afterEdges.ids) {
|
||||
if (
|
||||
preparedContext.beforeEdges.serializedById.get(id) !==
|
||||
preparedContext.afterEdges.serializedById.get(id)
|
||||
beforeEdgeSerializedById.get(id) !== afterEdgeSerializedById.get(id)
|
||||
) {
|
||||
const record = preparedContext.afterEdges.recordById?.get(id);
|
||||
if (record) upsertEdges.push(record);
|
||||
@@ -1124,14 +1255,14 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
|
||||
const deleteNodeIds = [];
|
||||
for (const id of preparedContext.beforeNodes.ids) {
|
||||
if (!preparedContext.afterNodes.serializedById.has(id)) {
|
||||
if (!afterNodeSerializedById.has(id)) {
|
||||
deleteNodeIds.push(id);
|
||||
}
|
||||
}
|
||||
|
||||
const deleteEdgeIds = [];
|
||||
for (const id of preparedContext.beforeEdges.ids) {
|
||||
if (!preparedContext.afterEdges.serializedById.has(id)) {
|
||||
if (!afterEdgeSerializedById.has(id)) {
|
||||
deleteEdgeIds.push(id);
|
||||
}
|
||||
}
|
||||
@@ -1139,8 +1270,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
const tombstoneMap = new Map();
|
||||
for (const id of preparedContext.afterTombstones.ids) {
|
||||
if (
|
||||
preparedContext.beforeTombstones.serializedById.get(id) !==
|
||||
preparedContext.afterTombstones.serializedById.get(id)
|
||||
beforeTombstoneSerializedById.get(id) !==
|
||||
afterTombstoneSerializedById.get(id)
|
||||
) {
|
||||
const record = preparedContext.afterTombstones.recordById?.get(id);
|
||||
const targetKey = preparedContext.afterTombstones.targetKeyById?.get(id) || "";
|
||||
@@ -1186,6 +1317,8 @@ export function buildPersistDelta(beforeSnapshot, afterSnapshot, options = {}) {
|
||||
if (shouldCollectDiagnostics) {
|
||||
emitPersistDeltaDiagnostics(options, {
|
||||
requestedNative: options?.useNativeDelta === true,
|
||||
requestedBridgeMode: options?.useNativeDelta === true ? nativeBridgeMode : "none",
|
||||
preparedBridgeMode: preparedContext.compactPayload?.bridgeMode || "none",
|
||||
usedNative: false,
|
||||
path: "js",
|
||||
gateAllowed: preparedNativeGate?.allowed ?? false,
|
||||
|
||||
@@ -75,6 +75,7 @@ assert.equal(defaultSettings.persistUseNativeDelta, false);
|
||||
assert.equal(defaultSettings.persistNativeDeltaThresholdRecords, 20000);
|
||||
assert.equal(defaultSettings.persistNativeDeltaThresholdStructuralDelta, 600);
|
||||
assert.equal(defaultSettings.persistNativeDeltaThresholdSerializedChars, 4000000);
|
||||
assert.equal(defaultSettings.persistNativeDeltaBridgeMode, "json");
|
||||
assert.equal(defaultSettings.nativeEngineFailOpen, true);
|
||||
assert.equal(defaultSettings.graphNativeForceDisable, false);
|
||||
assert.equal(defaultSettings.taskProfilesVersion, 3);
|
||||
|
||||
@@ -33,6 +33,17 @@ try {
|
||||
upsertTombstoneIds: [],
|
||||
};
|
||||
},
|
||||
build_persist_delta_compact_hash(payload = {}) {
|
||||
return {
|
||||
upsertNodeIds: Array.isArray(payload?.afterNodes?.ids)
|
||||
? payload.afterNodes.ids.slice(0, 1)
|
||||
: [],
|
||||
upsertEdgeIds: [],
|
||||
deleteNodeIds: [],
|
||||
deleteEdgeIds: [],
|
||||
upsertTombstoneIds: [],
|
||||
};
|
||||
},
|
||||
build_persist_delta(payload = {}) {
|
||||
return {
|
||||
upsertNodes: [{ id: "persist-native-node", marker: payload?.afterSnapshot?.meta?.chatId || "" }],
|
||||
@@ -66,12 +77,26 @@ try {
|
||||
{
|
||||
nowMs: 123,
|
||||
preparedDeltaInput: {
|
||||
bridgeMode: "json",
|
||||
afterNodes: { ids: ["persist-native-node"], serialized: ["{}"] },
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.deepEqual(compactDeltaResult.upsertNodeIds, ["persist-native-node"]);
|
||||
|
||||
const hashCompactDeltaResult = globalThis.__stBmeNativeBuildPersistDelta(
|
||||
{ meta: { chatId: "before-chat" }, nodes: [], edges: [], tombstones: [], state: {} },
|
||||
{ meta: { chatId: "after-chat" }, nodes: [], edges: [], tombstones: [], state: {} },
|
||||
{
|
||||
nowMs: 123,
|
||||
preparedDeltaInput: {
|
||||
bridgeMode: "hash",
|
||||
afterNodes: { ids: ["persist-native-node"], hashes: [1] },
|
||||
},
|
||||
},
|
||||
);
|
||||
assert.deepEqual(hashCompactDeltaResult.upsertNodeIds, ["persist-native-node"]);
|
||||
|
||||
const deltaResult = globalThis.__stBmeNativeBuildPersistDelta(
|
||||
{ meta: { chatId: "before-chat" }, nodes: [], edges: [], tombstones: [], state: {} },
|
||||
{ meta: { chatId: "after-chat" }, nodes: [], edges: [], tombstones: [], state: {} },
|
||||
|
||||
@@ -3,6 +3,7 @@ import assert from "node:assert/strict";
|
||||
import {
|
||||
buildPersistDelta,
|
||||
evaluatePersistNativeDeltaGate,
|
||||
resolvePersistNativeDeltaBridgeMode,
|
||||
resolvePersistNativeDeltaGateOptions,
|
||||
shouldUseNativePersistDeltaForSnapshots,
|
||||
} from "../sync/bme-db.js";
|
||||
@@ -40,6 +41,9 @@ const defaultGate = resolvePersistNativeDeltaGateOptions({});
|
||||
assert.equal(defaultGate.minSnapshotRecords, 20000);
|
||||
assert.equal(defaultGate.minStructuralDelta, 600);
|
||||
assert.equal(defaultGate.minCombinedSerializedChars, 4000000);
|
||||
assert.equal(resolvePersistNativeDeltaBridgeMode({}), "json");
|
||||
assert.equal(resolvePersistNativeDeltaBridgeMode({ persistNativeDeltaBridgeMode: "hash" }), "hash");
|
||||
assert.equal(resolvePersistNativeDeltaBridgeMode({ persistNativeDeltaBridgeMode: "unknown" }), "json");
|
||||
assert.equal(
|
||||
shouldUseNativePersistDeltaForSnapshots(beforeSnapshot, afterSnapshot, defaultGate),
|
||||
false,
|
||||
@@ -160,9 +164,32 @@ assert.deepEqual(compactNativeDelta.upsertEdges, []);
|
||||
assert.deepEqual(compactNativeDelta.deleteNodeIds, []);
|
||||
assert.equal(compactNativeDelta.runtimeMetaPatch.compact, true);
|
||||
assert.equal(compactNativeDelta.runtimeMetaPatch.chatId, "chat-native");
|
||||
assert.equal(compactDiagnostics.path, "native-compact");
|
||||
assert.equal(compactDiagnostics.path, "native-compact-json");
|
||||
assert.equal(compactDiagnostics.preparedBridgeMode, "json");
|
||||
assert.equal(compactDiagnostics.requestedBridgeMode, "json");
|
||||
assert.equal(compactDiagnostics.usedNative, true);
|
||||
|
||||
let hashDiagnostics = null;
|
||||
const hashNativeDelta = buildPersistDelta(beforeSnapshot, afterSnapshot, {
|
||||
useNativeDelta: true,
|
||||
minSnapshotRecords: 0,
|
||||
minStructuralDelta: 0,
|
||||
minCombinedSerializedChars: 0,
|
||||
persistNativeDeltaBridgeMode: "hash",
|
||||
runtimeMetaPatch: { hashMode: true },
|
||||
onDiagnostics(snapshot) {
|
||||
hashDiagnostics = snapshot;
|
||||
},
|
||||
});
|
||||
assert.deepEqual(hashNativeDelta.upsertNodes, [
|
||||
{ id: "n1", type: "event", fields: { text: "after" }, updatedAt: 2 },
|
||||
]);
|
||||
assert.equal(hashNativeDelta.runtimeMetaPatch.hashMode, true);
|
||||
assert.equal(hashDiagnostics.path, "native-compact-hash");
|
||||
assert.equal(hashDiagnostics.preparedBridgeMode, "hash");
|
||||
assert.equal(hashDiagnostics.requestedBridgeMode, "hash");
|
||||
assert.equal(hashDiagnostics.usedNative, true);
|
||||
|
||||
delete globalThis.__stBmeNativeBuildPersistDelta;
|
||||
|
||||
let threwUnavailable = false;
|
||||
|
||||
@@ -104,7 +104,8 @@ async function main() {
|
||||
await installNativePersistDeltaHook();
|
||||
const nativeStatus = getNativeModuleStatus();
|
||||
const jsSamples = [];
|
||||
const nativeSamples = [];
|
||||
const nativeJsonSamples = [];
|
||||
const nativeHashSamples = [];
|
||||
for (let run = 0; run < RUNS; run++) {
|
||||
const snapshots = buildSnapshots(17 + run, 5000, 12000, 0.12);
|
||||
const jsStartedAt = performance.now();
|
||||
@@ -120,26 +121,50 @@ async function main() {
|
||||
deleteEdgeIds: jsDelta.deleteEdgeIds.length,
|
||||
});
|
||||
|
||||
const nativeStartedAt = performance.now();
|
||||
const nativeDelta = buildPersistDelta(snapshots.before, snapshots.after, {
|
||||
const nativeJsonStartedAt = performance.now();
|
||||
const nativeJsonDelta = buildPersistDelta(snapshots.before, snapshots.after, {
|
||||
useNativeDelta: true,
|
||||
minSnapshotRecords: 0,
|
||||
minStructuralDelta: 0,
|
||||
minCombinedSerializedChars: 0,
|
||||
persistNativeDeltaBridgeMode: "json",
|
||||
nativeFailOpen: false,
|
||||
});
|
||||
const nativeElapsedMs = performance.now() - nativeStartedAt;
|
||||
nativeSamples.push({
|
||||
elapsedMs: nativeElapsedMs,
|
||||
upsertNodes: nativeDelta.upsertNodes.length,
|
||||
upsertEdges: nativeDelta.upsertEdges.length,
|
||||
deleteNodeIds: nativeDelta.deleteNodeIds.length,
|
||||
deleteEdgeIds: nativeDelta.deleteEdgeIds.length,
|
||||
const nativeJsonElapsedMs = performance.now() - nativeJsonStartedAt;
|
||||
nativeJsonSamples.push({
|
||||
elapsedMs: nativeJsonElapsedMs,
|
||||
upsertNodes: nativeJsonDelta.upsertNodes.length,
|
||||
upsertEdges: nativeJsonDelta.upsertEdges.length,
|
||||
deleteNodeIds: nativeJsonDelta.deleteNodeIds.length,
|
||||
deleteEdgeIds: nativeJsonDelta.deleteEdgeIds.length,
|
||||
});
|
||||
|
||||
const nativeHashStartedAt = performance.now();
|
||||
const nativeHashDelta = buildPersistDelta(snapshots.before, snapshots.after, {
|
||||
useNativeDelta: true,
|
||||
minSnapshotRecords: 0,
|
||||
minStructuralDelta: 0,
|
||||
minCombinedSerializedChars: 0,
|
||||
persistNativeDeltaBridgeMode: "hash",
|
||||
nativeFailOpen: false,
|
||||
});
|
||||
const nativeHashElapsedMs = performance.now() - nativeHashStartedAt;
|
||||
nativeHashSamples.push({
|
||||
elapsedMs: nativeHashElapsedMs,
|
||||
upsertNodes: nativeHashDelta.upsertNodes.length,
|
||||
upsertEdges: nativeHashDelta.upsertEdges.length,
|
||||
deleteNodeIds: nativeHashDelta.deleteNodeIds.length,
|
||||
deleteEdgeIds: nativeHashDelta.deleteEdgeIds.length,
|
||||
});
|
||||
}
|
||||
|
||||
const jsTimingSummary = summarize(jsSamples.map((sample) => sample.elapsedMs));
|
||||
const nativeTimingSummary = summarize(nativeSamples.map((sample) => sample.elapsedMs));
|
||||
const nativeJsonTimingSummary = summarize(
|
||||
nativeJsonSamples.map((sample) => sample.elapsedMs),
|
||||
);
|
||||
const nativeHashTimingSummary = summarize(
|
||||
nativeHashSamples.map((sample) => sample.elapsedMs),
|
||||
);
|
||||
const avgUpserts =
|
||||
jsSamples.reduce((acc, sample) => acc + sample.upsertNodes + sample.upsertEdges, 0) /
|
||||
jsSamples.length;
|
||||
@@ -151,7 +176,7 @@ async function main() {
|
||||
`[ST-BME][bench] persist-delta native-source=${nativeStatus.source || "unknown"}`,
|
||||
);
|
||||
console.log(
|
||||
`[ST-BME][bench] persist-delta runs=${RUNS} | js avg=${jsTimingSummary.avg.toFixed(2)}ms p95=${jsTimingSummary.p95.toFixed(2)}ms min=${jsTimingSummary.min.toFixed(2)}ms max=${jsTimingSummary.max.toFixed(2)}ms | native avg=${nativeTimingSummary.avg.toFixed(2)}ms p95=${nativeTimingSummary.p95.toFixed(2)}ms min=${nativeTimingSummary.min.toFixed(2)}ms max=${nativeTimingSummary.max.toFixed(2)}ms | avgUpserts=${avgUpserts.toFixed(1)} avgDeletes=${avgDeletes.toFixed(1)}`,
|
||||
`[ST-BME][bench] persist-delta runs=${RUNS} | js avg=${jsTimingSummary.avg.toFixed(2)}ms p95=${jsTimingSummary.p95.toFixed(2)}ms min=${jsTimingSummary.min.toFixed(2)}ms max=${jsTimingSummary.max.toFixed(2)}ms | native-json avg=${nativeJsonTimingSummary.avg.toFixed(2)}ms p95=${nativeJsonTimingSummary.p95.toFixed(2)}ms min=${nativeJsonTimingSummary.min.toFixed(2)}ms max=${nativeJsonTimingSummary.max.toFixed(2)}ms | native-hash avg=${nativeHashTimingSummary.avg.toFixed(2)}ms p95=${nativeHashTimingSummary.p95.toFixed(2)}ms min=${nativeHashTimingSummary.min.toFixed(2)}ms max=${nativeHashTimingSummary.max.toFixed(2)}ms | avgUpserts=${avgUpserts.toFixed(1)} avgDeletes=${avgDeletes.toFixed(1)}`,
|
||||
);
|
||||
}
|
||||
|
||||
|
||||
@@ -8060,6 +8060,12 @@ function _renderPersistDeltaTraceCard(state) {
|
||||
<span>执行路径</span>
|
||||
<strong>${_escHtml(String(diagnostics.path || "—"))}</strong>
|
||||
</div>
|
||||
<div class="bme-ai-monitor-kv__row">
|
||||
<span>Bridge 模式</span>
|
||||
<strong>${_escHtml(
|
||||
`${String(diagnostics.requestedBridgeMode || "none")} → ${String(diagnostics.preparedBridgeMode || "none")}`,
|
||||
)}</strong>
|
||||
</div>
|
||||
<div class="bme-ai-monitor-kv__row">
|
||||
<span>Native Gate</span>
|
||||
<strong>${_escHtml(_formatPersistDeltaGateText(diagnostics))}</strong>
|
||||
|
||||
35
vendor/wasm/stbme_core.js
vendored
35
vendor/wasm/stbme_core.js
vendored
@@ -54,6 +54,10 @@ async function loadFromWasmPackArtifacts() {
|
||||
|
||||
return {
|
||||
solve_layout: module.solve_layout,
|
||||
build_persist_delta_compact_hash:
|
||||
typeof module.build_persist_delta_compact_hash === "function"
|
||||
? module.build_persist_delta_compact_hash
|
||||
: null,
|
||||
build_persist_delta_compact:
|
||||
typeof module.build_persist_delta_compact === "function"
|
||||
? module.build_persist_delta_compact
|
||||
@@ -147,7 +151,8 @@ export async function installNativePersistDeltaHook() {
|
||||
const module = await loadNativeModule();
|
||||
if (
|
||||
!module ||
|
||||
(typeof module.build_persist_delta_compact !== "function" &&
|
||||
(typeof module.build_persist_delta_compact_hash !== "function" &&
|
||||
typeof module.build_persist_delta_compact !== "function" &&
|
||||
typeof module.build_persist_delta !== "function")
|
||||
) {
|
||||
throw new Error("native persist delta builder unavailable");
|
||||
@@ -155,12 +160,32 @@ export async function installNativePersistDeltaHook() {
|
||||
|
||||
globalThis.__stBmeNativeBuildPersistDelta = (beforeSnapshot, afterSnapshot, options = {}) => {
|
||||
let raw = null;
|
||||
const preparedInput =
|
||||
options?.preparedDeltaInput && typeof options.preparedDeltaInput === "object"
|
||||
? options.preparedDeltaInput
|
||||
: null;
|
||||
const preparedBridgeMode = String(preparedInput?.bridgeMode || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (
|
||||
typeof module.build_persist_delta_compact === "function" &&
|
||||
options?.preparedDeltaInput &&
|
||||
typeof options.preparedDeltaInput === "object"
|
||||
typeof module.build_persist_delta_compact_hash === "function" &&
|
||||
preparedInput &&
|
||||
preparedBridgeMode === "hash"
|
||||
) {
|
||||
raw = module.build_persist_delta_compact(options.preparedDeltaInput);
|
||||
raw = module.build_persist_delta_compact_hash(preparedInput);
|
||||
} else if (
|
||||
typeof module.build_persist_delta_compact === "function" &&
|
||||
preparedInput &&
|
||||
(preparedBridgeMode === "json" || preparedBridgeMode === "")
|
||||
) {
|
||||
raw = module.build_persist_delta_compact(preparedInput);
|
||||
} else if (
|
||||
typeof module.build_persist_delta_compact === "function" &&
|
||||
preparedInput &&
|
||||
preparedBridgeMode === "hash" &&
|
||||
Array.isArray(preparedInput?.afterNodes?.serialized)
|
||||
) {
|
||||
raw = module.build_persist_delta_compact(preparedInput);
|
||||
} else if (typeof module.build_persist_delta === "function") {
|
||||
raw = module.build_persist_delta({
|
||||
beforeSnapshot,
|
||||
|
||||
Reference in New Issue
Block a user