mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
Integrate Authority Blob storage
This commit is contained in:
235
index.js
235
index.js
@@ -373,6 +373,10 @@ import {
|
|||||||
createAuthorityJobAdapter,
|
createAuthorityJobAdapter,
|
||||||
normalizeAuthorityJobConfig,
|
normalizeAuthorityJobConfig,
|
||||||
} from "./maintenance/authority-job-adapter.js";
|
} from "./maintenance/authority-job-adapter.js";
|
||||||
|
import {
|
||||||
|
createAuthorityBlobAdapter,
|
||||||
|
normalizeAuthorityBlobConfig,
|
||||||
|
} from "./maintenance/authority-blob-adapter.js";
|
||||||
|
|
||||||
export { DEFAULT_TRIGGER_KEYWORDS, getSmartTriggerDecision };
|
export { DEFAULT_TRIGGER_KEYWORDS, getSmartTriggerDecision };
|
||||||
|
|
||||||
@@ -1505,6 +1509,7 @@ function buildAuthorityPersistenceStatePatch(settings = getSettings()) {
|
|||||||
authorityStoragePrimaryReady: Boolean(capability.storagePrimaryReady),
|
authorityStoragePrimaryReady: Boolean(capability.storagePrimaryReady),
|
||||||
authorityTriviumPrimaryReady: Boolean(capability.triviumPrimaryReady),
|
authorityTriviumPrimaryReady: Boolean(capability.triviumPrimaryReady),
|
||||||
authorityJobsReady: Boolean(capability.jobsReady),
|
authorityJobsReady: Boolean(capability.jobsReady),
|
||||||
|
authorityBlobReady: Boolean(capability.blobReady),
|
||||||
authorityBrowserCacheMode: String(browserState.mode || "minimal"),
|
authorityBrowserCacheMode: String(browserState.mode || "minimal"),
|
||||||
authorityOfflineQueueBytes: Number(browserState.offlineQueueBytes || 0),
|
authorityOfflineQueueBytes: Number(browserState.offlineQueueBytes || 0),
|
||||||
authorityOfflineQueueItems: Number(browserState.offlineQueueItems || 0),
|
authorityOfflineQueueItems: Number(browserState.offlineQueueItems || 0),
|
||||||
@@ -1965,6 +1970,206 @@ function recordAuthorityJobSnapshot(job = null, options = {}) {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function recordAuthorityBlobSnapshot(event = {}) {
|
||||||
|
const normalizedEvent =
|
||||||
|
event && typeof event === "object" && !Array.isArray(event) ? event : {};
|
||||||
|
updateGraphPersistenceState({
|
||||||
|
authorityBlobState: normalizedEvent.ok === false ? "error" : "active",
|
||||||
|
authorityLastBlobEvent: cloneRuntimeDebugValue(normalizedEvent, null),
|
||||||
|
authorityLastBlobAction: String(normalizedEvent.action || ""),
|
||||||
|
authorityLastBlobBackend: String(normalizedEvent.backend || ""),
|
||||||
|
authorityLastBlobPath: String(normalizedEvent.path || ""),
|
||||||
|
authorityLastBlobReason: String(normalizedEvent.reason || ""),
|
||||||
|
authorityLastBlobError: String(normalizedEvent.error || ""),
|
||||||
|
authorityLastBlobUpdatedAt: String(
|
||||||
|
normalizedEvent.updatedAt || new Date().toISOString(),
|
||||||
|
),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthorityBlobFileHash(input = "") {
|
||||||
|
let hash = 2166136261;
|
||||||
|
const text = String(input ?? "");
|
||||||
|
for (let index = 0; index < text.length; index += 1) {
|
||||||
|
hash ^= text.charCodeAt(index);
|
||||||
|
hash = Math.imul(hash, 16777619);
|
||||||
|
}
|
||||||
|
return (hash >>> 0).toString(36);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthorityBlobSafeSlug(input = "", fallback = "unknown") {
|
||||||
|
const normalized = String(input || fallback)
|
||||||
|
.trim()
|
||||||
|
.replace(/[^A-Za-z0-9._-]+/g, "_")
|
||||||
|
.replace(/_+/g, "_")
|
||||||
|
.replace(/^[_.-]+|[_.-]+$/g, "")
|
||||||
|
.slice(0, 96);
|
||||||
|
return normalized || fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldUseAuthorityBlobCheckpoint() {
|
||||||
|
const settings = getSettings();
|
||||||
|
const authoritySettings = normalizeAuthoritySettings(settings);
|
||||||
|
const { capability } = getAuthorityRuntimeSnapshot(settings);
|
||||||
|
return Boolean(
|
||||||
|
authoritySettings.enabled &&
|
||||||
|
authoritySettings.blobCheckpointEnabled &&
|
||||||
|
capability.blobReady,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAuthorityBlobAdapter(options = {}) {
|
||||||
|
const settings = getSettings();
|
||||||
|
const config = normalizeAuthorityBlobConfig(settings);
|
||||||
|
return createAuthorityBlobAdapter(config, {
|
||||||
|
fetchImpl: globalThis.fetch?.bind(globalThis),
|
||||||
|
headerProvider:
|
||||||
|
typeof getRequestHeaders === "function" ? () => getRequestHeaders() : null,
|
||||||
|
...options,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeAuthorityLukerCheckpointBlob(
|
||||||
|
checkpoint = null,
|
||||||
|
{ chatId = "", reason = "luker-checkpoint", signal = undefined } = {},
|
||||||
|
) {
|
||||||
|
if (!checkpoint || !shouldUseAuthorityBlobCheckpoint()) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
reason: "authority-blob-unavailable",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const normalizedChatId = normalizeChatIdCandidate(chatId || checkpoint.chatId);
|
||||||
|
const safeChatId = buildAuthorityBlobSafeSlug(normalizedChatId);
|
||||||
|
const hash = buildAuthorityBlobFileHash(normalizedChatId || safeChatId);
|
||||||
|
const path = `user/files/ST-BME_luker_checkpoint_${safeChatId}-${hash}.json`;
|
||||||
|
try {
|
||||||
|
const adapter = getAuthorityBlobAdapter();
|
||||||
|
const result = await adapter.writeJson(path, checkpoint, {
|
||||||
|
signal,
|
||||||
|
metadata: {
|
||||||
|
chatId: normalizedChatId,
|
||||||
|
revision: Number(checkpoint?.revision || 0),
|
||||||
|
reason: String(reason || ""),
|
||||||
|
kind: "luker-checkpoint",
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const event = {
|
||||||
|
action: "checkpoint-write",
|
||||||
|
ok: result?.ok !== false,
|
||||||
|
backend: "authority-blob",
|
||||||
|
path: result?.path || path,
|
||||||
|
reason: String(reason || ""),
|
||||||
|
revision: Number(checkpoint?.revision || 0),
|
||||||
|
};
|
||||||
|
recordAuthorityBlobSnapshot(event);
|
||||||
|
updateGraphPersistenceState({
|
||||||
|
authorityBlobCheckpointPath: event.path,
|
||||||
|
authorityBlobCheckpointRevision: event.revision,
|
||||||
|
authorityBlobCheckpointUpdatedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
ok: event.ok,
|
||||||
|
path: event.path,
|
||||||
|
result,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const message = error?.message || String(error) || "Authority Blob checkpoint failed";
|
||||||
|
recordAuthorityBlobSnapshot({
|
||||||
|
action: "checkpoint-write",
|
||||||
|
ok: false,
|
||||||
|
backend: "authority-blob",
|
||||||
|
path,
|
||||||
|
reason: String(reason || ""),
|
||||||
|
error: message,
|
||||||
|
revision: Number(checkpoint?.revision || 0),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
path,
|
||||||
|
reason: "authority-blob-checkpoint-error",
|
||||||
|
error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readAuthorityLukerCheckpointBlob(chatId = "", options = {}) {
|
||||||
|
if (!shouldUseAuthorityBlobCheckpoint()) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
exists: false,
|
||||||
|
reason: "authority-blob-unavailable",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const normalizedChatId = normalizeChatIdCandidate(chatId);
|
||||||
|
if (!normalizedChatId) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
exists: false,
|
||||||
|
reason: "missing-chat-id",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const safeChatId = buildAuthorityBlobSafeSlug(normalizedChatId);
|
||||||
|
const hash = buildAuthorityBlobFileHash(normalizedChatId || safeChatId);
|
||||||
|
const path = `user/files/ST-BME_luker_checkpoint_${safeChatId}-${hash}.json`;
|
||||||
|
try {
|
||||||
|
const adapter = getAuthorityBlobAdapter();
|
||||||
|
const result = await adapter.readJson(path, options);
|
||||||
|
const exists = Boolean(result?.exists && result?.payload);
|
||||||
|
recordAuthorityBlobSnapshot({
|
||||||
|
action: "checkpoint-read",
|
||||||
|
ok: result?.ok !== false,
|
||||||
|
backend: "authority-blob",
|
||||||
|
path: result?.path || path,
|
||||||
|
reason: exists ? "checkpoint-found" : "checkpoint-missing",
|
||||||
|
revision: Number(result?.payload?.revision || 0),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
ok: result?.ok !== false,
|
||||||
|
exists,
|
||||||
|
path: result?.path || path,
|
||||||
|
checkpoint: exists ? result.payload : null,
|
||||||
|
result,
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const message = error?.message || String(error) || "Authority Blob checkpoint read failed";
|
||||||
|
recordAuthorityBlobSnapshot({
|
||||||
|
action: "checkpoint-read",
|
||||||
|
ok: false,
|
||||||
|
backend: "authority-blob",
|
||||||
|
path,
|
||||||
|
reason: "authority-blob-checkpoint-read-error",
|
||||||
|
error: message,
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
exists: false,
|
||||||
|
path,
|
||||||
|
reason: "authority-blob-checkpoint-read-error",
|
||||||
|
error,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readLukerGraphSidecarV2WithAuthorityBlob(context = null, options = {}) {
|
||||||
|
const sidecar = await readLukerGraphSidecarV2(context, options);
|
||||||
|
if (sidecar?.checkpoint) return sidecar;
|
||||||
|
const chatId =
|
||||||
|
normalizeChatIdCandidate(options.chatId) ||
|
||||||
|
normalizeChatIdCandidate(sidecar?.manifest?.chatId) ||
|
||||||
|
normalizeChatIdCandidate(getCurrentChatId(context));
|
||||||
|
const blobResult = await readAuthorityLukerCheckpointBlob(chatId);
|
||||||
|
if (!blobResult?.exists || !blobResult?.checkpoint) return sidecar;
|
||||||
|
return {
|
||||||
|
...(sidecar || {}),
|
||||||
|
checkpoint: blobResult.checkpoint,
|
||||||
|
authorityBlobCheckpoint: {
|
||||||
|
path: blobResult.path,
|
||||||
|
backend: "authority-blob",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function submitAuthorityVectorRebuildJob({
|
async function submitAuthorityVectorRebuildJob({
|
||||||
config = null,
|
config = null,
|
||||||
range = null,
|
range = null,
|
||||||
@@ -6448,6 +6653,9 @@ async function refreshRuntimeGraphAfterSyncApplied(syncPayload = {}) {
|
|||||||
function buildBmeSyncRuntimeOptions(extra = {}) {
|
function buildBmeSyncRuntimeOptions(extra = {}) {
|
||||||
const normalizedExtra =
|
const normalizedExtra =
|
||||||
extra && typeof extra === "object" && !Array.isArray(extra) ? extra : {};
|
extra && typeof extra === "object" && !Array.isArray(extra) ? extra : {};
|
||||||
|
const settings = getSettings();
|
||||||
|
const authoritySettings = normalizeAuthoritySettings(settings);
|
||||||
|
const { capability } = getAuthorityRuntimeSnapshot(settings);
|
||||||
const defaultOptions = {
|
const defaultOptions = {
|
||||||
getDb: async (chatId) => {
|
getDb: async (chatId) => {
|
||||||
const manager = ensureBmeChatManager();
|
const manager = ensureBmeChatManager();
|
||||||
@@ -6468,6 +6676,16 @@ function buildBmeSyncRuntimeOptions(extra = {}) {
|
|||||||
getCurrentChatId: () => getCurrentChatId(),
|
getCurrentChatId: () => getCurrentChatId(),
|
||||||
getCloudStorageMode: () => getSettings().cloudStorageMode || "automatic",
|
getCloudStorageMode: () => getSettings().cloudStorageMode || "automatic",
|
||||||
getRequestHeaders,
|
getRequestHeaders,
|
||||||
|
authorityBlobEnabled: Boolean(
|
||||||
|
authoritySettings.enabled &&
|
||||||
|
authoritySettings.blobCheckpointEnabled &&
|
||||||
|
capability.blobReady,
|
||||||
|
),
|
||||||
|
authorityBlobFailOpen: authoritySettings.failOpen,
|
||||||
|
authorityBlobConfig: {
|
||||||
|
...authoritySettings,
|
||||||
|
},
|
||||||
|
onAuthorityBlobEvent: recordAuthorityBlobSnapshot,
|
||||||
onSyncApplied: async (payload = {}) => {
|
onSyncApplied: async (payload = {}) => {
|
||||||
await refreshRuntimeGraphAfterSyncApplied(payload);
|
await refreshRuntimeGraphAfterSyncApplied(payload);
|
||||||
},
|
},
|
||||||
@@ -7436,6 +7654,10 @@ async function compactLukerGraphSidecarV2(
|
|||||||
error: checkpointResult?.error || null,
|
error: checkpointResult?.error || null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
await writeAuthorityLukerCheckpointBlob(checkpointResult.checkpoint, {
|
||||||
|
chatId: normalizedChatId,
|
||||||
|
reason,
|
||||||
|
});
|
||||||
|
|
||||||
const emptyJournal = buildLukerGraphJournalV2([], {
|
const emptyJournal = buildLukerGraphJournalV2([], {
|
||||||
chatId: normalizedChatId,
|
chatId: normalizedChatId,
|
||||||
@@ -7651,11 +7873,12 @@ async function persistGraphToLukerSidecarV2(
|
|||||||
? cloneRuntimeDebugValue(persistDelta, persistDelta)
|
? cloneRuntimeDebugValue(persistDelta, persistDelta)
|
||||||
: null;
|
: null;
|
||||||
|
|
||||||
const existingSidecar = await readLukerGraphSidecarV2(context, {
|
const existingSidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, {
|
||||||
manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE,
|
manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE,
|
||||||
journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE,
|
journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE,
|
||||||
checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE,
|
checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE,
|
||||||
chatStateTarget: normalizedTarget,
|
chatStateTarget: normalizedTarget,
|
||||||
|
chatId,
|
||||||
});
|
});
|
||||||
if (existingSidecar?.manifest) {
|
if (existingSidecar?.manifest) {
|
||||||
cacheChatStateManifest(chatId, existingSidecar.manifest);
|
cacheChatStateManifest(chatId, existingSidecar.manifest);
|
||||||
@@ -7742,6 +7965,10 @@ async function persistGraphToLukerSidecarV2(
|
|||||||
error: checkpointResult?.error || null,
|
error: checkpointResult?.error || null,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
await writeAuthorityLukerCheckpointBlob(checkpointResult.checkpoint, {
|
||||||
|
chatId,
|
||||||
|
reason: `${reason}:bootstrap`,
|
||||||
|
});
|
||||||
const emptyJournal = buildLukerGraphJournalV2([], {
|
const emptyJournal = buildLukerGraphJournalV2([], {
|
||||||
chatId,
|
chatId,
|
||||||
integrity: nextIntegrity,
|
integrity: nextIntegrity,
|
||||||
@@ -8071,11 +8298,12 @@ async function loadGraphFromLukerSidecarV2(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const sidecar = await readLukerGraphSidecarV2(context, {
|
const sidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, {
|
||||||
manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE,
|
manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE,
|
||||||
journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE,
|
journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE,
|
||||||
checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE,
|
checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE,
|
||||||
chatStateTarget: normalizedTarget,
|
chatStateTarget: normalizedTarget,
|
||||||
|
chatId: normalizedChatId,
|
||||||
});
|
});
|
||||||
const manifest = sidecar?.manifest || null;
|
const manifest = sidecar?.manifest || null;
|
||||||
if (!manifest) {
|
if (!manifest) {
|
||||||
@@ -8784,8 +9012,9 @@ async function readPersistedGraphForChatStateTarget(
|
|||||||
return null;
|
return null;
|
||||||
}
|
}
|
||||||
|
|
||||||
const sidecar = await readLukerGraphSidecarV2(context, {
|
const sidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, {
|
||||||
chatStateTarget: normalizedTarget,
|
chatStateTarget: normalizedTarget,
|
||||||
|
chatId: targetChatId,
|
||||||
});
|
});
|
||||||
const sidecarResult = buildSnapshotFromLukerSidecarState(sidecar, {
|
const sidecarResult = buildSnapshotFromLukerSidecarState(sidecar, {
|
||||||
chatId: targetChatId,
|
chatId: targetChatId,
|
||||||
|
|||||||
329
maintenance/authority-blob-adapter.js
Normal file
329
maintenance/authority-blob-adapter.js
Normal file
@@ -0,0 +1,329 @@
|
|||||||
|
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||||
|
|
||||||
|
export const AUTHORITY_BLOB_ENDPOINT = "/v1/blob";
|
||||||
|
|
||||||
|
function toPlainData(value, fallbackValue = null) {
|
||||||
|
if (value == null) return fallbackValue;
|
||||||
|
if (typeof globalThis.structuredClone === "function") {
|
||||||
|
try {
|
||||||
|
return globalThis.structuredClone(value);
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.parse(JSON.stringify(value));
|
||||||
|
} catch {
|
||||||
|
return fallbackValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRecordId(value) {
|
||||||
|
return String(value ?? "").trim();
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeInteger(value, fallback = 0, min = 0, max = Number.MAX_SAFE_INTEGER) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
if (!Number.isFinite(parsed)) return fallback;
|
||||||
|
return Math.min(max, Math.max(min, Math.trunc(parsed)));
|
||||||
|
}
|
||||||
|
|
||||||
|
function decodeBase64Utf8(base64Text = "") {
|
||||||
|
const normalizedBase64 = String(base64Text ?? "");
|
||||||
|
if (!normalizedBase64) return "";
|
||||||
|
if (typeof globalThis.atob === "function" && typeof globalThis.TextDecoder === "function") {
|
||||||
|
const binary = globalThis.atob(normalizedBase64);
|
||||||
|
const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0));
|
||||||
|
return new TextDecoder().decode(bytes);
|
||||||
|
}
|
||||||
|
if (typeof Buffer !== "undefined") {
|
||||||
|
return Buffer.from(normalizedBase64, "base64").toString("utf8");
|
||||||
|
}
|
||||||
|
return normalizedBase64;
|
||||||
|
}
|
||||||
|
|
||||||
|
function tryParseJsonText(text, fallbackValue = null) {
|
||||||
|
if (typeof text !== "string") return fallbackValue;
|
||||||
|
try {
|
||||||
|
return JSON.parse(text);
|
||||||
|
} catch {
|
||||||
|
return fallbackValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAuthorityBlobPath(path = "") {
|
||||||
|
const normalized = String(path ?? "")
|
||||||
|
.trim()
|
||||||
|
.replace(/\\/g, "/")
|
||||||
|
.replace(/^authority:\/\/private\//i, "")
|
||||||
|
.replace(/^\/+/, "")
|
||||||
|
.replace(/\/+/g, "/");
|
||||||
|
return normalized.replace(/\/+$/g, "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeBlobPayload(result = null) {
|
||||||
|
if (!result || typeof result !== "object" || Array.isArray(result)) return result;
|
||||||
|
const source = result.file || result.blob || result.result || result;
|
||||||
|
if (source.payload !== undefined) return source.payload;
|
||||||
|
if (source.json !== undefined) return source.json;
|
||||||
|
if (source.value !== undefined) return source.value;
|
||||||
|
if (source.data !== undefined) {
|
||||||
|
if (source.encoding === "base64" || source.base64 === true) {
|
||||||
|
return tryParseJsonText(decodeBase64Utf8(source.data), source.data);
|
||||||
|
}
|
||||||
|
if (typeof source.data === "string") {
|
||||||
|
return tryParseJsonText(source.data, source.data);
|
||||||
|
}
|
||||||
|
return source.data;
|
||||||
|
}
|
||||||
|
if (source.content !== undefined) {
|
||||||
|
if (typeof source.content === "string") {
|
||||||
|
return tryParseJsonText(source.content, source.content);
|
||||||
|
}
|
||||||
|
return source.content;
|
||||||
|
}
|
||||||
|
if (source.body !== undefined) {
|
||||||
|
if (typeof source.body === "string") {
|
||||||
|
return tryParseJsonText(source.body, source.body);
|
||||||
|
}
|
||||||
|
return source.body;
|
||||||
|
}
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeBlobRecordSource(input = null) {
|
||||||
|
if (!input || typeof input !== "object" || Array.isArray(input)) return {};
|
||||||
|
return input.file || input.blob || input.result || input;
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") {
|
||||||
|
const source = normalizeBlobRecordSource(input);
|
||||||
|
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath);
|
||||||
|
const missing =
|
||||||
|
source.exists === false ||
|
||||||
|
source.found === false ||
|
||||||
|
source.missing === true ||
|
||||||
|
Number(source.status || source.statusCode || 0) === 404;
|
||||||
|
if (missing) {
|
||||||
|
return {
|
||||||
|
exists: false,
|
||||||
|
path,
|
||||||
|
payload: null,
|
||||||
|
contentType: String(source.contentType || source.type || ""),
|
||||||
|
raw: toPlainData(input, input),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
exists: input != null && source.ok !== false,
|
||||||
|
path,
|
||||||
|
payload: normalizeBlobPayload(input),
|
||||||
|
contentType: String(source.contentType || source.type || "application/json"),
|
||||||
|
etag: String(source.etag || source.hash || ""),
|
||||||
|
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "",
|
||||||
|
raw: toPlainData(input, input),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") {
|
||||||
|
const source = normalizeBlobRecordSource(input);
|
||||||
|
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath);
|
||||||
|
return {
|
||||||
|
ok: input == null ? true : source.ok !== false && source.error == null,
|
||||||
|
path,
|
||||||
|
url: String(source.url || source.href || ""),
|
||||||
|
size: normalizeInteger(source.size || source.bytes, 0, 0),
|
||||||
|
etag: String(source.etag || source.hash || ""),
|
||||||
|
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "",
|
||||||
|
raw: toPlainData(input, input),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAuthorityBlobDeleteResult(input = null, fallbackPath = "") {
|
||||||
|
const source = normalizeBlobRecordSource(input);
|
||||||
|
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath);
|
||||||
|
const missing =
|
||||||
|
source.exists === false ||
|
||||||
|
source.found === false ||
|
||||||
|
source.missing === true ||
|
||||||
|
Number(source.status || source.statusCode || 0) === 404;
|
||||||
|
return {
|
||||||
|
ok: input == null ? true : source.ok !== false,
|
||||||
|
deleted: missing ? false : source.deleted !== false && source.ok !== false,
|
||||||
|
missing,
|
||||||
|
path,
|
||||||
|
raw: toPlainData(input, input),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export function normalizeAuthorityBlobConfig(settings = {}, overrides = {}) {
|
||||||
|
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
|
||||||
|
return {
|
||||||
|
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
|
||||||
|
enabled:
|
||||||
|
source.authorityBlobCheckpointEnabled !== false &&
|
||||||
|
source.blobCheckpointEnabled !== false &&
|
||||||
|
source.authorityBlobEnabled !== false,
|
||||||
|
failOpen: source.authorityFailOpen !== false && source.failOpen !== false,
|
||||||
|
namespace: normalizeRecordId(source.authorityBlobNamespace || source.blobNamespace || "st-bme"),
|
||||||
|
...overrides,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthorityBlobHttpClient {
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
||||||
|
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
||||||
|
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async request(action, payload = {}) {
|
||||||
|
if (typeof this.fetchImpl !== "function") {
|
||||||
|
throw new Error("Authority Blob fetch unavailable");
|
||||||
|
}
|
||||||
|
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_BLOB_ENDPOINT}`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
||||||
|
},
|
||||||
|
body: JSON.stringify({ action, ...payload }),
|
||||||
|
});
|
||||||
|
if (!response?.ok) {
|
||||||
|
const text = await response?.text?.().catch(() => "");
|
||||||
|
throw new Error(text || `Authority Blob HTTP ${response?.status || "unknown"}`);
|
||||||
|
}
|
||||||
|
return await response.json().catch(() => ({}));
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeJson(payload = {}) {
|
||||||
|
return await this.request("writeJson", payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeText(payload = {}) {
|
||||||
|
return await this.request("writeText", payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
async readJson(payload = {}) {
|
||||||
|
return await this.request("readJson", payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(payload = {}) {
|
||||||
|
return await this.request("delete", payload);
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(payload = {}) {
|
||||||
|
return await this.request("stat", payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAuthorityBlobClient(config = {}, options = {}) {
|
||||||
|
const injected = options.blobClient || config.blobClient || globalThis.__stBmeAuthorityBlobClient;
|
||||||
|
if (injected) return injected;
|
||||||
|
return new AuthorityBlobHttpClient({
|
||||||
|
baseUrl: config.baseUrl,
|
||||||
|
fetchImpl: options.fetchImpl || config.fetchImpl,
|
||||||
|
headerProvider: options.headerProvider || config.headerProvider,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
async function callClient(client, methodNames = [], action = "request", payload = {}) {
|
||||||
|
for (const methodName of methodNames) {
|
||||||
|
if (typeof client?.[methodName] === "function") {
|
||||||
|
return await client[methodName](payload);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof client?.request === "function") {
|
||||||
|
return await client.request(action, payload);
|
||||||
|
}
|
||||||
|
if (typeof client === "function") {
|
||||||
|
return await client({ action, ...payload });
|
||||||
|
}
|
||||||
|
throw new Error(`Authority Blob ${action} unavailable`);
|
||||||
|
}
|
||||||
|
|
||||||
|
function throwIfAborted(signal) {
|
||||||
|
if (signal?.aborted) {
|
||||||
|
throw signal.reason instanceof Error
|
||||||
|
? signal.reason
|
||||||
|
: Object.assign(new Error("操作已终止"), { name: "AbortError" });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthorityBlobAdapter {
|
||||||
|
constructor(config = {}, options = {}) {
|
||||||
|
this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {});
|
||||||
|
this.client = createAuthorityBlobClient(this.config, options);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeJson(path, payload = null, options = {}) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
|
if (!normalizedPath) throw new Error("Authority Blob path is required");
|
||||||
|
const result = await callClient(this.client, ["writeJson", "putJson", "writeFile", "put"], "writeJson", {
|
||||||
|
namespace: options.namespace || this.config.namespace,
|
||||||
|
path: normalizedPath,
|
||||||
|
name: normalizedPath,
|
||||||
|
contentType: options.contentType || "application/json",
|
||||||
|
payload: toPlainData(payload, payload),
|
||||||
|
data: toPlainData(payload, payload),
|
||||||
|
metadata: toPlainData(options.metadata, {}),
|
||||||
|
});
|
||||||
|
return normalizeAuthorityBlobWriteResult(result, normalizedPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async writeText(path, text = "", options = {}) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
|
if (!normalizedPath) throw new Error("Authority Blob path is required");
|
||||||
|
const result = await callClient(this.client, ["writeText", "writeFile", "putText", "put"], "writeText", {
|
||||||
|
namespace: options.namespace || this.config.namespace,
|
||||||
|
path: normalizedPath,
|
||||||
|
name: normalizedPath,
|
||||||
|
contentType: options.contentType || "text/plain; charset=utf-8",
|
||||||
|
text: String(text ?? ""),
|
||||||
|
data: String(text ?? ""),
|
||||||
|
metadata: toPlainData(options.metadata, {}),
|
||||||
|
});
|
||||||
|
return normalizeAuthorityBlobWriteResult(result, normalizedPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async readJson(path, options = {}) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
|
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
||||||
|
const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", {
|
||||||
|
namespace: options.namespace || this.config.namespace,
|
||||||
|
path: normalizedPath,
|
||||||
|
name: normalizedPath,
|
||||||
|
});
|
||||||
|
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async delete(path, options = {}) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
|
if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, "");
|
||||||
|
const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", {
|
||||||
|
namespace: options.namespace || this.config.namespace,
|
||||||
|
path: normalizedPath,
|
||||||
|
name: normalizedPath,
|
||||||
|
});
|
||||||
|
return normalizeAuthorityBlobDeleteResult(result, normalizedPath);
|
||||||
|
}
|
||||||
|
|
||||||
|
async stat(path, options = {}) {
|
||||||
|
throwIfAborted(options.signal);
|
||||||
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
|
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
||||||
|
const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", {
|
||||||
|
namespace: options.namespace || this.config.namespace,
|
||||||
|
path: normalizedPath,
|
||||||
|
name: normalizedPath,
|
||||||
|
});
|
||||||
|
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAuthorityBlobAdapter(config = {}, options = {}) {
|
||||||
|
return new AuthorityBlobAdapter(config, options);
|
||||||
|
}
|
||||||
713
sync/bme-sync.js
713
sync/bme-sync.js
@@ -3,6 +3,7 @@ import {
|
|||||||
MANUAL_BACKUP_BATCH_JOURNAL_COVERAGE_KEY,
|
MANUAL_BACKUP_BATCH_JOURNAL_COVERAGE_KEY,
|
||||||
PROCESSED_MESSAGE_HASH_VERSION,
|
PROCESSED_MESSAGE_HASH_VERSION,
|
||||||
} from "../runtime/runtime-state.js";
|
} from "../runtime/runtime-state.js";
|
||||||
|
import { createAuthorityBlobAdapter } from "../maintenance/authority-blob-adapter.js";
|
||||||
|
|
||||||
const BME_SYNC_FILE_PREFIX = "ST-BME_sync_";
|
const BME_SYNC_FILE_PREFIX = "ST-BME_sync_";
|
||||||
const BME_SYNC_FILE_SUFFIX = ".json";
|
const BME_SYNC_FILE_SUFFIX = ".json";
|
||||||
@@ -336,6 +337,454 @@ function getFetch(options = {}) {
|
|||||||
return fetchImpl;
|
return fetchImpl;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeRemoteFileName(fileName = "") {
|
||||||
|
const normalized = String(fileName ?? "")
|
||||||
|
.trim()
|
||||||
|
.replace(/^\/+/, "");
|
||||||
|
if (!normalized || /[\\/]/.test(normalized)) return "";
|
||||||
|
return normalized;
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeRemoteServerPath(pathOrFilename = "", fallbackFilename = "") {
|
||||||
|
const raw = String(pathOrFilename || fallbackFilename || "")
|
||||||
|
.trim()
|
||||||
|
.replace(/\\/g, "/")
|
||||||
|
.replace(/^authority:\/\/private\//i, "")
|
||||||
|
.replace(/^\/+/, "")
|
||||||
|
.split("?")[0];
|
||||||
|
if (!raw) return "";
|
||||||
|
if (raw.startsWith("user/files/")) {
|
||||||
|
const fileName = normalizeRemoteFileName(raw.slice("user/files/".length));
|
||||||
|
return fileName ? `user/files/${fileName}` : "";
|
||||||
|
}
|
||||||
|
const fileName = normalizeRemoteFileName(raw) || normalizeRemoteFileName(fallbackFilename);
|
||||||
|
return fileName ? `user/files/${fileName}` : "";
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRemoteFileUrl(pathOrFilename = "", fallbackFilename = "") {
|
||||||
|
const serverPath = normalizeRemoteServerPath(pathOrFilename, fallbackFilename);
|
||||||
|
if (!serverPath) return "";
|
||||||
|
const fileName = normalizeRemoteFileName(serverPath.slice("user/files/".length));
|
||||||
|
return fileName ? `/user/files/${encodeURIComponent(fileName)}` : `/${serverPath}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
function resolveRemoteFileName(pathOrFilename = "", fallbackFilename = "") {
|
||||||
|
const serverPath = normalizeRemoteServerPath(pathOrFilename, fallbackFilename);
|
||||||
|
return normalizeRemoteFileName(serverPath.slice("user/files/".length));
|
||||||
|
}
|
||||||
|
|
||||||
|
function parseSerializedJsonPayload(payload = "") {
|
||||||
|
try {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
value: JSON.parse(String(payload ?? "")),
|
||||||
|
};
|
||||||
|
} catch {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
value: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function shouldUseAuthorityBlobFiles(options = {}) {
|
||||||
|
if (options.authorityBlobEnabled === false) return false;
|
||||||
|
if (options.authorityBlobAdapter || options.authorityBlobClient || options.blobClient) return true;
|
||||||
|
return options.authorityBlobEnabled === true;
|
||||||
|
}
|
||||||
|
|
||||||
|
function getAuthorityBlobAdapter(options = {}) {
|
||||||
|
if (!shouldUseAuthorityBlobFiles(options)) return null;
|
||||||
|
if (options.authorityBlobAdapter) return options.authorityBlobAdapter;
|
||||||
|
return createAuthorityBlobAdapter(options.authorityBlobConfig || options.authoritySettings || {}, {
|
||||||
|
blobClient: options.authorityBlobClient || options.blobClient,
|
||||||
|
fetchImpl: options.fetch,
|
||||||
|
headerProvider: () => getRequestHeadersSafe(options),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
function readAuthorityBlobFailOpen(options = {}) {
|
||||||
|
return options.authorityBlobFailOpen !== false;
|
||||||
|
}
|
||||||
|
|
||||||
|
function recordAuthorityBlobFileEvent(options = {}, event = {}) {
|
||||||
|
if (typeof options.onAuthorityBlobEvent !== "function") return;
|
||||||
|
try {
|
||||||
|
options.onAuthorityBlobEvent({
|
||||||
|
...event,
|
||||||
|
updatedAt: new Date().toISOString(),
|
||||||
|
});
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readAuthorityBlobJsonFile(pathOrFilename = "", options = {}) {
|
||||||
|
const adapter = getAuthorityBlobAdapter(options);
|
||||||
|
const path = normalizeRemoteServerPath(pathOrFilename);
|
||||||
|
if (!adapter || !path) {
|
||||||
|
return {
|
||||||
|
available: false,
|
||||||
|
exists: false,
|
||||||
|
path,
|
||||||
|
payload: null,
|
||||||
|
reason: "authority-blob-disabled",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
const startedAt = readSyncTimingNow();
|
||||||
|
try {
|
||||||
|
const result = await adapter.readJson(path, {
|
||||||
|
signal: options.signal,
|
||||||
|
namespace: options.authorityBlobNamespace,
|
||||||
|
});
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
const exists = result?.exists === true;
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "read",
|
||||||
|
ok: result?.ok !== false,
|
||||||
|
path,
|
||||||
|
backend: "authority-blob",
|
||||||
|
reason: exists ? "ok" : "not-found",
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
available: true,
|
||||||
|
exists,
|
||||||
|
path: result?.path || path,
|
||||||
|
payload: exists ? result.payload : null,
|
||||||
|
reason: exists ? "ok" : "not-found",
|
||||||
|
timings: {
|
||||||
|
authorityBlobMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "read",
|
||||||
|
ok: false,
|
||||||
|
path,
|
||||||
|
backend: "authority-blob",
|
||||||
|
reason: "authority-blob-error",
|
||||||
|
error: error instanceof Error ? error.message : String(error || ""),
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
if (!readAuthorityBlobFailOpen(options)) throw error;
|
||||||
|
return {
|
||||||
|
available: true,
|
||||||
|
exists: false,
|
||||||
|
path,
|
||||||
|
payload: null,
|
||||||
|
reason: "authority-blob-error",
|
||||||
|
error,
|
||||||
|
timings: {
|
||||||
|
authorityBlobMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function writeRemoteJsonFile(pathOrFilename = "", serializedPayload = "", options = {}) {
|
||||||
|
const serverPath = normalizeRemoteServerPath(pathOrFilename);
|
||||||
|
const fileName = resolveRemoteFileName(serverPath);
|
||||||
|
if (!serverPath || !fileName) throw new Error("remote filename is required");
|
||||||
|
const adapter = getAuthorityBlobAdapter(options);
|
||||||
|
if (adapter) {
|
||||||
|
const startedAt = readSyncTimingNow();
|
||||||
|
try {
|
||||||
|
const parsedPayload = parseSerializedJsonPayload(serializedPayload);
|
||||||
|
const result = parsedPayload.ok
|
||||||
|
? await adapter.writeJson(serverPath, parsedPayload.value, {
|
||||||
|
signal: options.signal,
|
||||||
|
namespace: options.authorityBlobNamespace,
|
||||||
|
metadata: {
|
||||||
|
filename: fileName,
|
||||||
|
},
|
||||||
|
})
|
||||||
|
: await adapter.writeText(serverPath, String(serializedPayload ?? ""), {
|
||||||
|
signal: options.signal,
|
||||||
|
namespace: options.authorityBlobNamespace,
|
||||||
|
contentType: "application/json; charset=utf-8",
|
||||||
|
metadata: {
|
||||||
|
filename: fileName,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
if (result?.ok !== false) {
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "write",
|
||||||
|
ok: true,
|
||||||
|
path: serverPath,
|
||||||
|
backend: "authority-blob",
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
path: `/${serverPath}`,
|
||||||
|
authorityPath: result?.path || serverPath,
|
||||||
|
backend: "authority-blob",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "write",
|
||||||
|
ok: false,
|
||||||
|
path: serverPath,
|
||||||
|
backend: "authority-blob",
|
||||||
|
reason: "authority-blob-error",
|
||||||
|
error: error instanceof Error ? error.message : String(error || ""),
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
if (!readAuthorityBlobFailOpen(options)) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fetchImpl = getFetch(options);
|
||||||
|
const response = await fetchImpl("/api/files/upload", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
...getRequestHeadersSafe(options),
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
name: fileName,
|
||||||
|
data: encodeBase64Utf8(serializedPayload),
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text().catch(() => response.statusText);
|
||||||
|
throw new Error(errorText || `HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const uploadResult = await response.json().catch(() => ({}));
|
||||||
|
return {
|
||||||
|
path: String(uploadResult?.path || `/${serverPath}`),
|
||||||
|
backend: "user-files",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readRemoteJsonFileResult(pathOrFilename = "", options = {}) {
|
||||||
|
const serverPath = normalizeRemoteServerPath(pathOrFilename);
|
||||||
|
const fileName = resolveRemoteFileName(serverPath);
|
||||||
|
if (!serverPath || !fileName) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
reason: "remote-file-name-invalid",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const authorityResult = await readAuthorityBlobJsonFile(serverPath, options);
|
||||||
|
if (authorityResult.exists) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
reason: "ok",
|
||||||
|
path: authorityResult.path || serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: authorityResult.payload,
|
||||||
|
backend: "authority-blob",
|
||||||
|
timings: authorityResult.timings || {},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const fetchImpl = getFetch(options);
|
||||||
|
let response;
|
||||||
|
let networkMs = 0;
|
||||||
|
let parseMs = 0;
|
||||||
|
try {
|
||||||
|
const networkStartedAt = readSyncTimingNow();
|
||||||
|
response = await fetchImpl(`${buildRemoteFileUrl(serverPath)}?t=${Date.now()}`, {
|
||||||
|
method: "GET",
|
||||||
|
cache: "no-store",
|
||||||
|
});
|
||||||
|
networkMs = readSyncTimingNow() - networkStartedAt;
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 0,
|
||||||
|
reason: "network-error",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: null,
|
||||||
|
error,
|
||||||
|
timings: {
|
||||||
|
networkMs: normalizeSyncTimingMs(networkMs),
|
||||||
|
parseMs: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (response.status === 404) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
reason: "not-found",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: null,
|
||||||
|
timings: {
|
||||||
|
networkMs: normalizeSyncTimingMs(networkMs),
|
||||||
|
parseMs: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text().catch(() => response.statusText);
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: response.status,
|
||||||
|
reason: "http-error",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: null,
|
||||||
|
error: new Error(errorText || `HTTP ${response.status}`),
|
||||||
|
timings: {
|
||||||
|
networkMs: normalizeSyncTimingMs(networkMs),
|
||||||
|
parseMs: 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
try {
|
||||||
|
const parseStartedAt = readSyncTimingNow();
|
||||||
|
const payload = await response.json();
|
||||||
|
parseMs = readSyncTimingNow() - parseStartedAt;
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
reason: "ok",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload,
|
||||||
|
backend: "user-files",
|
||||||
|
timings: {
|
||||||
|
networkMs: normalizeSyncTimingMs(networkMs),
|
||||||
|
parseMs: normalizeSyncTimingMs(parseMs),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
} catch (error) {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 0,
|
||||||
|
reason: "invalid-json",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
payload: null,
|
||||||
|
error,
|
||||||
|
timings: {
|
||||||
|
networkMs: normalizeSyncTimingMs(networkMs),
|
||||||
|
parseMs: normalizeSyncTimingMs(parseMs),
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async function deleteRemoteJsonFile(pathOrFilename = "", options = {}) {
|
||||||
|
const serverPath = normalizeRemoteServerPath(pathOrFilename);
|
||||||
|
const fileName = resolveRemoteFileName(serverPath);
|
||||||
|
if (!serverPath || !fileName) {
|
||||||
|
return {
|
||||||
|
deleted: false,
|
||||||
|
reason: "remote-file-name-invalid",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const adapter = getAuthorityBlobAdapter(options);
|
||||||
|
let authorityDeleted = false;
|
||||||
|
if (adapter) {
|
||||||
|
const startedAt = readSyncTimingNow();
|
||||||
|
try {
|
||||||
|
const result = await adapter.delete(serverPath, {
|
||||||
|
signal: options.signal,
|
||||||
|
namespace: options.authorityBlobNamespace,
|
||||||
|
});
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
authorityDeleted = result?.deleted === true;
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "delete",
|
||||||
|
ok: result?.ok !== false,
|
||||||
|
path: serverPath,
|
||||||
|
backend: "authority-blob",
|
||||||
|
reason: authorityDeleted ? "ok" : "not-found",
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
if (authorityDeleted) {
|
||||||
|
try {
|
||||||
|
const fetchImpl = getFetch(options);
|
||||||
|
await fetchImpl("/api/files/delete", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
...getRequestHeadersSafe(options),
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
path: `/${serverPath}`,
|
||||||
|
}),
|
||||||
|
}).catch(() => null);
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
deleted: true,
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
backend: "authority-blob",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
} catch (error) {
|
||||||
|
const elapsedMs = readSyncTimingNow() - startedAt;
|
||||||
|
recordAuthorityBlobFileEvent(options, {
|
||||||
|
action: "delete",
|
||||||
|
ok: false,
|
||||||
|
path: serverPath,
|
||||||
|
backend: "authority-blob",
|
||||||
|
reason: "authority-blob-error",
|
||||||
|
error: error instanceof Error ? error.message : String(error || ""),
|
||||||
|
elapsedMs: normalizeSyncTimingMs(elapsedMs),
|
||||||
|
});
|
||||||
|
if (!readAuthorityBlobFailOpen(options)) throw error;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
const fetchImpl = getFetch(options);
|
||||||
|
const response = await fetchImpl("/api/files/delete", {
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
...getRequestHeadersSafe(options),
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
body: JSON.stringify({
|
||||||
|
path: `/${serverPath}`,
|
||||||
|
}),
|
||||||
|
});
|
||||||
|
|
||||||
|
if (response.status === 404) {
|
||||||
|
return {
|
||||||
|
deleted: false,
|
||||||
|
reason: "not-found",
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text().catch(() => response.statusText);
|
||||||
|
throw new Error(errorText || `HTTP ${response.status}`);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
deleted: true,
|
||||||
|
path: serverPath,
|
||||||
|
filename: fileName,
|
||||||
|
backend: "user-files",
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
async function getSafetyDb(chatId, options = {}) {
|
async function getSafetyDb(chatId, options = {}) {
|
||||||
if (typeof options.getSafetyDb === "function") {
|
if (typeof options.getSafetyDb === "function") {
|
||||||
return await options.getSafetyDb(chatId);
|
return await options.getSafetyDb(chatId);
|
||||||
@@ -347,22 +796,14 @@ async function getSafetyDb(chatId, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function fetchBackupManifest(options = {}) {
|
async function fetchBackupManifest(options = {}) {
|
||||||
const fetchImpl = getFetch(options);
|
const result = await readRemoteJsonFileResult(BME_BACKUP_MANIFEST_FILENAME, options);
|
||||||
const response = await fetchImpl(
|
if (result.status === 404) {
|
||||||
`/user/files/${BME_BACKUP_MANIFEST_FILENAME}?t=${Date.now()}`,
|
|
||||||
{
|
|
||||||
method: "GET",
|
|
||||||
cache: "no-store",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
if (response.status === 404) {
|
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
if (!response.ok) {
|
if (!result.ok) {
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
throw result.error || new Error(result.reason || "manifest read failed");
|
||||||
throw new Error(errorText || `manifest read failed: HTTP ${response.status}`);
|
|
||||||
}
|
}
|
||||||
const rawPayload = await response.json();
|
const rawPayload = result.payload;
|
||||||
if (!Array.isArray(rawPayload)) {
|
if (!Array.isArray(rawPayload)) {
|
||||||
throw new Error("backup manifest payload is not an array");
|
throw new Error("backup manifest payload is not an array");
|
||||||
}
|
}
|
||||||
@@ -370,24 +811,8 @@ async function fetchBackupManifest(options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
async function writeBackupManifest(entries = [], options = {}) {
|
async function writeBackupManifest(entries = [], options = {}) {
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const payload = JSON.stringify(entries);
|
const payload = JSON.stringify(entries);
|
||||||
const response = await fetchImpl("/api/files/upload", {
|
await writeRemoteJsonFile(BME_BACKUP_MANIFEST_FILENAME, payload, options);
|
||||||
method: "POST",
|
|
||||||
headers: {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
name: BME_BACKUP_MANIFEST_FILENAME,
|
|
||||||
data: encodeBase64Utf8(payload),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async function upsertBackupManifestEntry(entry, options = {}) {
|
async function upsertBackupManifestEntry(entry, options = {}) {
|
||||||
@@ -523,50 +948,50 @@ async function readBackupEnvelope(chatId, options = {}) {
|
|||||||
const lookupStartedAt = readSyncTimingNow();
|
const lookupStartedAt = readSyncTimingNow();
|
||||||
const lookup = await resolveBackupLookupContext(normalizedChatId, options);
|
const lookup = await resolveBackupLookupContext(normalizedChatId, options);
|
||||||
const lookupMs = readSyncTimingNow() - lookupStartedAt;
|
const lookupMs = readSyncTimingNow() - lookupStartedAt;
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const fallbackFilename = buildBackupFilename(normalizedChatId);
|
const fallbackFilename = buildBackupFilename(normalizedChatId);
|
||||||
let lastMissingFilename = lookup.candidates[0]?.filename || fallbackFilename;
|
let lastMissingFilename = lookup.candidates[0]?.filename || fallbackFilename;
|
||||||
let networkMs = 0;
|
let networkMs = 0;
|
||||||
let parseMs = 0;
|
let parseMs = 0;
|
||||||
|
let authorityBlobMs = 0;
|
||||||
|
|
||||||
for (const candidate of lookup.candidates) {
|
for (const candidate of lookup.candidates) {
|
||||||
try {
|
try {
|
||||||
const networkStartedAt = readSyncTimingNow();
|
const result = await readRemoteJsonFileResult(
|
||||||
const response = await fetchImpl(
|
candidate.serverPath || candidate.filename,
|
||||||
`${candidate.serverPath || `/user/files/${encodeURIComponent(candidate.filename)}`}?t=${Date.now()}`,
|
options,
|
||||||
{
|
|
||||||
method: "GET",
|
|
||||||
cache: "no-store",
|
|
||||||
},
|
|
||||||
);
|
);
|
||||||
networkMs += readSyncTimingNow() - networkStartedAt;
|
networkMs += Number(result.timings?.networkMs || 0);
|
||||||
if (response.status === 404) {
|
parseMs += Number(result.timings?.parseMs || 0);
|
||||||
|
authorityBlobMs += Number(result.timings?.authorityBlobMs || 0);
|
||||||
|
if (result.status === 404) {
|
||||||
lastMissingFilename = candidate.filename;
|
lastMissingFilename = candidate.filename;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
if (!response.ok) {
|
if (!result.ok) {
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
return {
|
return {
|
||||||
exists: false,
|
exists: false,
|
||||||
filename: candidate.filename,
|
filename: candidate.filename,
|
||||||
envelope: null,
|
envelope: null,
|
||||||
reason: "backup-read-error",
|
reason: "backup-read-error",
|
||||||
error: new Error(errorText || `HTTP ${response.status}`),
|
error: result.error || new Error(result.reason || "backup-read-error"),
|
||||||
timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt),
|
timings: finalizeSyncTimings(
|
||||||
|
{ lookupMs, networkMs, parseMs, authorityBlobMs },
|
||||||
|
readStartedAt,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const parseStartedAt = readSyncTimingNow();
|
const envelope = normalizeBackupEnvelope(result.payload, normalizedChatId);
|
||||||
const payload = await response.json();
|
|
||||||
parseMs += readSyncTimingNow() - parseStartedAt;
|
|
||||||
const envelope = normalizeBackupEnvelope(payload, normalizedChatId);
|
|
||||||
if (!envelope) {
|
if (!envelope) {
|
||||||
return {
|
return {
|
||||||
exists: false,
|
exists: false,
|
||||||
filename: candidate.filename,
|
filename: candidate.filename,
|
||||||
envelope: null,
|
envelope: null,
|
||||||
reason: "invalid-backup",
|
reason: "invalid-backup",
|
||||||
timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt),
|
timings: finalizeSyncTimings(
|
||||||
|
{ lookupMs, networkMs, parseMs, authorityBlobMs },
|
||||||
|
readStartedAt,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
return {
|
return {
|
||||||
@@ -574,7 +999,10 @@ async function readBackupEnvelope(chatId, options = {}) {
|
|||||||
filename: candidate.filename,
|
filename: candidate.filename,
|
||||||
envelope,
|
envelope,
|
||||||
reason: "ok",
|
reason: "ok",
|
||||||
timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt),
|
timings: finalizeSyncTimings(
|
||||||
|
{ lookupMs, networkMs, parseMs, authorityBlobMs },
|
||||||
|
readStartedAt,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
return {
|
return {
|
||||||
@@ -583,7 +1011,10 @@ async function readBackupEnvelope(chatId, options = {}) {
|
|||||||
envelope: null,
|
envelope: null,
|
||||||
reason: "backup-read-error",
|
reason: "backup-read-error",
|
||||||
error,
|
error,
|
||||||
timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt),
|
timings: finalizeSyncTimings(
|
||||||
|
{ lookupMs, networkMs, parseMs, authorityBlobMs },
|
||||||
|
readStartedAt,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -594,7 +1025,10 @@ async function readBackupEnvelope(chatId, options = {}) {
|
|||||||
envelope: null,
|
envelope: null,
|
||||||
reason: "not-found",
|
reason: "not-found",
|
||||||
manifestError: lookup.manifestError,
|
manifestError: lookup.manifestError,
|
||||||
timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt),
|
timings: finalizeSyncTimings(
|
||||||
|
{ lookupMs, networkMs, parseMs, authorityBlobMs },
|
||||||
|
readStartedAt,
|
||||||
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -622,40 +1056,22 @@ async function writeBackupEnvelope(envelope, chatId, options = {}) {
|
|||||||
const writeStartedAt = readSyncTimingNow();
|
const writeStartedAt = readSyncTimingNow();
|
||||||
const normalizedChatId = normalizeChatId(chatId);
|
const normalizedChatId = normalizeChatId(chatId);
|
||||||
const filename = buildBackupFilename(normalizedChatId);
|
const filename = buildBackupFilename(normalizedChatId);
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const serializeStartedAt = readSyncTimingNow();
|
const serializeStartedAt = readSyncTimingNow();
|
||||||
const payload = JSON.stringify(envelope);
|
const payload = JSON.stringify(envelope);
|
||||||
const serializeMs = readSyncTimingNow() - serializeStartedAt;
|
const serializeMs = readSyncTimingNow() - serializeStartedAt;
|
||||||
const uploadStartedAt = readSyncTimingNow();
|
const uploadStartedAt = readSyncTimingNow();
|
||||||
const response = await fetchImpl("/api/files/upload", {
|
const uploadResult = await writeRemoteJsonFile(filename, payload, options);
|
||||||
method: "POST",
|
|
||||||
headers: {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
name: filename,
|
|
||||||
data: encodeBase64Utf8(payload),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
const uploadMs = readSyncTimingNow() - uploadStartedAt;
|
const uploadMs = readSyncTimingNow() - uploadStartedAt;
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const responseParseStartedAt = readSyncTimingNow();
|
|
||||||
const uploadResult = await response.json().catch(() => ({}));
|
|
||||||
const responseParseMs = readSyncTimingNow() - responseParseStartedAt;
|
|
||||||
return {
|
return {
|
||||||
filename,
|
filename,
|
||||||
path: String(uploadResult?.path || `/user/files/${filename}`),
|
path: String(uploadResult?.path || `/user/files/${filename}`),
|
||||||
|
backend: String(uploadResult?.backend || ""),
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{
|
{
|
||||||
serializeMs,
|
serializeMs,
|
||||||
uploadMs,
|
uploadMs,
|
||||||
responseParseMs,
|
responseParseMs: 0,
|
||||||
},
|
},
|
||||||
writeStartedAt,
|
writeStartedAt,
|
||||||
),
|
),
|
||||||
@@ -1890,7 +2306,6 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const resolveStartedAt = readSyncTimingNow();
|
const resolveStartedAt = readSyncTimingNow();
|
||||||
const candidateFilenames = await resolveSyncFilenameCandidates(
|
const candidateFilenames = await resolveSyncFilenameCandidates(
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
@@ -1902,20 +2317,15 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
let parseMs = 0;
|
let parseMs = 0;
|
||||||
let chunkReadMs = 0;
|
let chunkReadMs = 0;
|
||||||
let normalizeMs = 0;
|
let normalizeMs = 0;
|
||||||
|
let authorityBlobMs = 0;
|
||||||
|
|
||||||
for (const filename of candidateFilenames) {
|
for (const filename of candidateFilenames) {
|
||||||
const cacheBust = `t=${Date.now()}`;
|
const result = await readRemoteJsonFileResult(filename, options);
|
||||||
const url = `/user/files/${encodeURIComponent(filename)}?${cacheBust}`;
|
networkMs += Number(result.timings?.networkMs || 0);
|
||||||
|
parseMs += Number(result.timings?.parseMs || 0);
|
||||||
let response;
|
authorityBlobMs += Number(result.timings?.authorityBlobMs || 0);
|
||||||
try {
|
if (result.reason === "network-error") {
|
||||||
const networkStartedAt = readSyncTimingNow();
|
const error = result.error || new Error("network-error");
|
||||||
response = await fetchImpl(url, {
|
|
||||||
method: "GET",
|
|
||||||
cache: "no-store",
|
|
||||||
});
|
|
||||||
networkMs += readSyncTimingNow() - networkStartedAt;
|
|
||||||
} catch (error) {
|
|
||||||
console.warn("[ST-BME] 读取远端同步文件失败:", error);
|
console.warn("[ST-BME] 读取远端同步文件失败:", error);
|
||||||
return {
|
return {
|
||||||
exists: false,
|
exists: false,
|
||||||
@@ -1924,39 +2334,36 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
snapshot: null,
|
snapshot: null,
|
||||||
error,
|
error,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs },
|
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs },
|
||||||
readStartedAt,
|
readStartedAt,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (response.status === 404) {
|
if (result.status === 404) {
|
||||||
lastNotFoundFilename = filename;
|
lastNotFoundFilename = filename;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
if (!result.ok) {
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
const error = result.error || new Error(result.reason || "remote-read-error");
|
||||||
const error = new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
console.warn("[ST-BME] 读取远端同步文件失败:", error);
|
console.warn("[ST-BME] 读取远端同步文件失败:", error);
|
||||||
return {
|
return {
|
||||||
exists: false,
|
exists: false,
|
||||||
status: "http-error",
|
status: result.reason || "http-error",
|
||||||
filename,
|
filename,
|
||||||
snapshot: null,
|
snapshot: null,
|
||||||
error,
|
error,
|
||||||
statusCode: response.status,
|
statusCode: result.status,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs },
|
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs },
|
||||||
readStartedAt,
|
readStartedAt,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const parseStartedAt = readSyncTimingNow();
|
const remotePayload = result.payload;
|
||||||
const remotePayload = await response.json();
|
|
||||||
parseMs += readSyncTimingNow() - parseStartedAt;
|
|
||||||
let snapshot = null;
|
let snapshot = null;
|
||||||
if (Number(remotePayload?.formatVersion || 0) === BME_REMOTE_SYNC_FORMAT_VERSION_V2) {
|
if (Number(remotePayload?.formatVersion || 0) === BME_REMOTE_SYNC_FORMAT_VERSION_V2) {
|
||||||
const manifestResult = await readRemoteSnapshotV2Manifest(
|
const manifestResult = await readRemoteSnapshotV2Manifest(
|
||||||
@@ -1982,7 +2389,7 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
filename,
|
filename,
|
||||||
snapshot,
|
snapshot,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs },
|
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs },
|
||||||
readStartedAt,
|
readStartedAt,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
@@ -1995,7 +2402,7 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
snapshot: null,
|
snapshot: null,
|
||||||
error,
|
error,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs },
|
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs },
|
||||||
readStartedAt,
|
readStartedAt,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
@@ -2008,29 +2415,21 @@ async function readRemoteSnapshot(chatId, options = {}) {
|
|||||||
filename: lastNotFoundFilename,
|
filename: lastNotFoundFilename,
|
||||||
snapshot: null,
|
snapshot: null,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs },
|
{ resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs },
|
||||||
readStartedAt,
|
readStartedAt,
|
||||||
),
|
),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readRemoteJsonFile(filename, options = {}) {
|
async function readRemoteJsonFile(filename, options = {}) {
|
||||||
const fetchImpl = getFetch(options);
|
const result = await readRemoteJsonFileResult(filename, options);
|
||||||
const response = await fetchImpl(
|
if (result.status === 404) {
|
||||||
`/user/files/${encodeURIComponent(filename)}?t=${Date.now()}`,
|
|
||||||
{
|
|
||||||
method: "GET",
|
|
||||||
cache: "no-store",
|
|
||||||
},
|
|
||||||
);
|
|
||||||
if (response.status === 404) {
|
|
||||||
throw new Error("remote-chunk-not-found");
|
throw new Error("remote-chunk-not-found");
|
||||||
}
|
}
|
||||||
if (!response.ok) {
|
if (!result.ok) {
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
throw result.error || new Error(result.reason || `HTTP ${result.status || "unknown"}`);
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
}
|
||||||
return await response.json();
|
return result.payload;
|
||||||
}
|
}
|
||||||
|
|
||||||
async function readRemoteSnapshotV2Manifest(manifest = {}, chatId = "", options = {}) {
|
async function readRemoteSnapshotV2Manifest(manifest = {}, chatId = "", options = {}) {
|
||||||
@@ -2107,7 +2506,6 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) {
|
|||||||
const normalizedChatId = normalizeChatId(chatId);
|
const normalizedChatId = normalizeChatId(chatId);
|
||||||
const normalizedSnapshot = normalizeSyncSnapshot(snapshot, normalizedChatId);
|
const normalizedSnapshot = normalizeSyncSnapshot(snapshot, normalizedChatId);
|
||||||
const filename = await resolveSyncFilename(normalizedChatId, options);
|
const filename = await resolveSyncFilename(normalizedChatId, options);
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const envelopeBuildStartedAt = readSyncTimingNow();
|
const envelopeBuildStartedAt = readSyncTimingNow();
|
||||||
const syncEnvelope = buildRemoteSyncEnvelopeV2(
|
const syncEnvelope = buildRemoteSyncEnvelopeV2(
|
||||||
normalizedSnapshot,
|
normalizedSnapshot,
|
||||||
@@ -2115,10 +2513,6 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) {
|
|||||||
filename,
|
filename,
|
||||||
);
|
);
|
||||||
const envelopeBuildMs = readSyncTimingNow() - envelopeBuildStartedAt;
|
const envelopeBuildMs = readSyncTimingNow() - envelopeBuildStartedAt;
|
||||||
const requestHeaders = {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
};
|
|
||||||
let chunkSerializeMs = 0;
|
let chunkSerializeMs = 0;
|
||||||
let chunkUploadMs = 0;
|
let chunkUploadMs = 0;
|
||||||
for (const chunk of syncEnvelope.chunks) {
|
for (const chunk of syncEnvelope.chunks) {
|
||||||
@@ -2126,45 +2520,20 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) {
|
|||||||
const chunkPayload = JSON.stringify(chunk.payload, null, 2);
|
const chunkPayload = JSON.stringify(chunk.payload, null, 2);
|
||||||
chunkSerializeMs += readSyncTimingNow() - serializeStartedAt;
|
chunkSerializeMs += readSyncTimingNow() - serializeStartedAt;
|
||||||
const uploadStartedAt = readSyncTimingNow();
|
const uploadStartedAt = readSyncTimingNow();
|
||||||
const chunkResponse = await fetchImpl("/api/files/upload", {
|
await writeRemoteJsonFile(chunk.filename, chunkPayload, options);
|
||||||
method: "POST",
|
|
||||||
headers: requestHeaders,
|
|
||||||
body: JSON.stringify({
|
|
||||||
name: chunk.filename,
|
|
||||||
data: encodeBase64Utf8(chunkPayload),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
chunkUploadMs += readSyncTimingNow() - uploadStartedAt;
|
chunkUploadMs += readSyncTimingNow() - uploadStartedAt;
|
||||||
if (!chunkResponse.ok) {
|
|
||||||
const errorText = await chunkResponse.text().catch(() => chunkResponse.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${chunkResponse.status}`);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
const manifestSerializeStartedAt = readSyncTimingNow();
|
const manifestSerializeStartedAt = readSyncTimingNow();
|
||||||
const manifestPayload = JSON.stringify(syncEnvelope.manifest, null, 2);
|
const manifestPayload = JSON.stringify(syncEnvelope.manifest, null, 2);
|
||||||
const manifestSerializeMs = readSyncTimingNow() - manifestSerializeStartedAt;
|
const manifestSerializeMs = readSyncTimingNow() - manifestSerializeStartedAt;
|
||||||
const manifestUploadStartedAt = readSyncTimingNow();
|
const manifestUploadStartedAt = readSyncTimingNow();
|
||||||
const response = await fetchImpl("/api/files/upload", {
|
const uploadResult = await writeRemoteJsonFile(filename, manifestPayload, options);
|
||||||
method: "POST",
|
|
||||||
headers: requestHeaders,
|
|
||||||
body: JSON.stringify({
|
|
||||||
name: filename,
|
|
||||||
data: encodeBase64Utf8(manifestPayload),
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
const manifestUploadMs = readSyncTimingNow() - manifestUploadStartedAt;
|
const manifestUploadMs = readSyncTimingNow() - manifestUploadStartedAt;
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
const responseParseStartedAt = readSyncTimingNow();
|
|
||||||
const uploadResult = await response.json().catch(() => ({}));
|
|
||||||
const responseParseMs = readSyncTimingNow() - responseParseStartedAt;
|
|
||||||
return {
|
return {
|
||||||
filename,
|
filename,
|
||||||
path: String(uploadResult?.path || ""),
|
path: String(uploadResult?.path || ""),
|
||||||
|
backend: String(uploadResult?.backend || ""),
|
||||||
payload: syncEnvelope.manifest,
|
payload: syncEnvelope.manifest,
|
||||||
timings: finalizeSyncTimings(
|
timings: finalizeSyncTimings(
|
||||||
{
|
{
|
||||||
@@ -2173,7 +2542,7 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) {
|
|||||||
chunkUploadMs,
|
chunkUploadMs,
|
||||||
manifestSerializeMs,
|
manifestSerializeMs,
|
||||||
manifestUploadMs,
|
manifestUploadMs,
|
||||||
responseParseMs,
|
responseParseMs: 0,
|
||||||
},
|
},
|
||||||
writeStartedAt,
|
writeStartedAt,
|
||||||
),
|
),
|
||||||
@@ -2610,24 +2979,9 @@ export async function deleteServerBackup(chatId, options = {}) {
|
|||||||
const serverPath =
|
const serverPath =
|
||||||
targetCandidate.serverPath ||
|
targetCandidate.serverPath ||
|
||||||
normalizeSelectedBackupServerPath("", filename);
|
normalizeSelectedBackupServerPath("", filename);
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const response = await fetchImpl("/api/files/delete", {
|
const deleteResult = await deleteRemoteJsonFile(serverPath || filename, options);
|
||||||
method: "POST",
|
|
||||||
headers: {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
path: serverPath,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (!response.ok && response.status !== 404) {
|
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const existingEntries =
|
const existingEntries =
|
||||||
@@ -2655,6 +3009,7 @@ export async function deleteServerBackup(chatId, options = {}) {
|
|||||||
deleted: true,
|
deleted: true,
|
||||||
chatId: normalizedChatId,
|
chatId: normalizedChatId,
|
||||||
filename,
|
filename,
|
||||||
|
remoteDeleted: deleteResult.deleted === true,
|
||||||
localMetaUpdated,
|
localMetaUpdated,
|
||||||
};
|
};
|
||||||
} catch (manifestError) {
|
} catch (manifestError) {
|
||||||
@@ -3315,7 +3670,6 @@ export async function deleteRemoteSyncFile(chatId, options = {}) {
|
|||||||
}
|
}
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const fetchImpl = getFetch(options);
|
|
||||||
const filenames = await resolveSyncFilenameCandidates(
|
const filenames = await resolveSyncFilenameCandidates(
|
||||||
normalizedChatId,
|
normalizedChatId,
|
||||||
options,
|
options,
|
||||||
@@ -3329,47 +3683,24 @@ export async function deleteRemoteSyncFile(chatId, options = {}) {
|
|||||||
for (const chunk of Array.isArray(manifestPayload?.chunks) ? manifestPayload.chunks : []) {
|
for (const chunk of Array.isArray(manifestPayload?.chunks) ? manifestPayload.chunks : []) {
|
||||||
const chunkFilename = String(chunk?.filename || "").trim();
|
const chunkFilename = String(chunk?.filename || "").trim();
|
||||||
if (!chunkFilename) continue;
|
if (!chunkFilename) continue;
|
||||||
await fetchImpl("/api/files/delete", {
|
await deleteRemoteJsonFile(chunkFilename, options).catch(() => null);
|
||||||
method: "POST",
|
|
||||||
headers: {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
path: `/user/files/${chunkFilename}`,
|
|
||||||
}),
|
|
||||||
}).catch(() => null);
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
} catch {
|
} catch {
|
||||||
// best-effort chunk cleanup
|
// best-effort chunk cleanup
|
||||||
}
|
}
|
||||||
const response = await fetchImpl("/api/files/delete", {
|
const deleteResult = await deleteRemoteJsonFile(filename, options);
|
||||||
method: "POST",
|
if (!deleteResult.deleted) {
|
||||||
headers: {
|
|
||||||
...getRequestHeadersSafe(options),
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
},
|
|
||||||
body: JSON.stringify({
|
|
||||||
path: `/user/files/${filename}`,
|
|
||||||
}),
|
|
||||||
});
|
|
||||||
|
|
||||||
if (response.status === 404) {
|
|
||||||
lastNotFoundFilename = filename;
|
lastNotFoundFilename = filename;
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!response.ok) {
|
|
||||||
const errorText = await response.text().catch(() => response.statusText);
|
|
||||||
throw new Error(errorText || `HTTP ${response.status}`);
|
|
||||||
}
|
|
||||||
|
|
||||||
sanitizedFilenameByChatId.delete(normalizedChatId);
|
sanitizedFilenameByChatId.delete(normalizedChatId);
|
||||||
return {
|
return {
|
||||||
deleted: true,
|
deleted: true,
|
||||||
chatId: normalizedChatId,
|
chatId: normalizedChatId,
|
||||||
filename,
|
filename,
|
||||||
|
backend: String(deleteResult.backend || ""),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
439
tests/authority-blob.mjs
Normal file
439
tests/authority-blob.mjs
Normal file
@@ -0,0 +1,439 @@
|
|||||||
|
import assert from "node:assert/strict";
|
||||||
|
|
||||||
|
import {
|
||||||
|
createAuthorityBlobAdapter,
|
||||||
|
normalizeAuthorityBlobPath,
|
||||||
|
normalizeAuthorityBlobReadResult,
|
||||||
|
} from "../maintenance/authority-blob-adapter.js";
|
||||||
|
import {
|
||||||
|
backupToServer,
|
||||||
|
download,
|
||||||
|
listServerBackups,
|
||||||
|
restoreFromServer,
|
||||||
|
upload,
|
||||||
|
} from "../sync/bme-sync.js";
|
||||||
|
|
||||||
|
class MemoryStorage {
|
||||||
|
constructor() {
|
||||||
|
this.map = new Map();
|
||||||
|
}
|
||||||
|
|
||||||
|
getItem(key) {
|
||||||
|
return this.map.has(key) ? this.map.get(key) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
setItem(key, value) {
|
||||||
|
this.map.set(String(key), String(value));
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
class FakeDb {
|
||||||
|
constructor(chatId, snapshot = null) {
|
||||||
|
this.chatId = chatId;
|
||||||
|
this.snapshot = snapshot || {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId,
|
||||||
|
deviceId: "",
|
||||||
|
revision: 1,
|
||||||
|
lastModified: 10,
|
||||||
|
nodeCount: 1,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [{ id: `${chatId}-node`, updatedAt: 10 }],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: {
|
||||||
|
lastProcessedFloor: 1,
|
||||||
|
extractionCount: 1,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
this.meta = new Map([
|
||||||
|
["syncDirty", false],
|
||||||
|
["syncDirtyReason", ""],
|
||||||
|
["lastSyncedRevision", 0],
|
||||||
|
]);
|
||||||
|
this.lastImportPayload = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async exportSnapshot() {
|
||||||
|
return JSON.parse(JSON.stringify(this.snapshot));
|
||||||
|
}
|
||||||
|
|
||||||
|
async importSnapshot(snapshot) {
|
||||||
|
this.lastImportPayload = JSON.parse(JSON.stringify(snapshot));
|
||||||
|
this.snapshot = JSON.parse(JSON.stringify(snapshot));
|
||||||
|
}
|
||||||
|
|
||||||
|
async getMeta(key, fallback = null) {
|
||||||
|
return this.meta.has(key) ? this.meta.get(key) : fallback;
|
||||||
|
}
|
||||||
|
|
||||||
|
async patchMeta(record = {}) {
|
||||||
|
for (const [key, value] of Object.entries(record)) {
|
||||||
|
this.meta.set(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
async setMeta(key, value) {
|
||||||
|
this.meta.set(key, value);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockAuthorityBlobClient() {
|
||||||
|
const files = new Map();
|
||||||
|
const calls = [];
|
||||||
|
return {
|
||||||
|
files,
|
||||||
|
calls,
|
||||||
|
async writeJson(payload = {}) {
|
||||||
|
calls.push(["writeJson", { ...payload }]);
|
||||||
|
files.set(String(payload.path || ""), JSON.parse(JSON.stringify(payload.payload)));
|
||||||
|
return { ok: true, path: payload.path, size: JSON.stringify(payload.payload).length };
|
||||||
|
},
|
||||||
|
async writeText(payload = {}) {
|
||||||
|
calls.push(["writeText", { ...payload }]);
|
||||||
|
files.set(String(payload.path || ""), String(payload.text ?? payload.data ?? ""));
|
||||||
|
return { ok: true, path: payload.path, size: String(payload.text ?? "").length };
|
||||||
|
},
|
||||||
|
async readJson(payload = {}) {
|
||||||
|
calls.push(["readJson", { ...payload }]);
|
||||||
|
const path = String(payload.path || "");
|
||||||
|
if (!files.has(path)) return { exists: false, path };
|
||||||
|
return { exists: true, path, payload: JSON.parse(JSON.stringify(files.get(path))) };
|
||||||
|
},
|
||||||
|
async delete(payload = {}) {
|
||||||
|
calls.push(["delete", { ...payload }]);
|
||||||
|
const path = String(payload.path || "");
|
||||||
|
const existed = files.delete(path);
|
||||||
|
return { ok: true, deleted: existed, exists: existed, path };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createMockFetch() {
|
||||||
|
const logs = {
|
||||||
|
getCalls: 0,
|
||||||
|
uploadCalls: 0,
|
||||||
|
deleteCalls: 0,
|
||||||
|
sanitizeCalls: 0,
|
||||||
|
};
|
||||||
|
const response = (status, body) => ({
|
||||||
|
ok: status >= 200 && status < 300,
|
||||||
|
status,
|
||||||
|
statusText: String(status),
|
||||||
|
async json() {
|
||||||
|
return JSON.parse(JSON.stringify(body));
|
||||||
|
},
|
||||||
|
async text() {
|
||||||
|
return typeof body === "string" ? body : JSON.stringify(body);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const fetch = async (url, options = {}) => {
|
||||||
|
const method = String(options.method || "GET").toUpperCase();
|
||||||
|
if (url === "/api/files/sanitize-filename" && method === "POST") {
|
||||||
|
logs.sanitizeCalls += 1;
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
return response(200, {
|
||||||
|
fileName: String(body.fileName || "").replace(/[^A-Za-z0-9._~-]+/g, "_"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (url === "/api/files/upload" && method === "POST") {
|
||||||
|
logs.uploadCalls += 1;
|
||||||
|
return response(500, "legacy upload should not be used");
|
||||||
|
}
|
||||||
|
if (url === "/api/files/delete" && method === "POST") {
|
||||||
|
logs.deleteCalls += 1;
|
||||||
|
return response(404, "not found");
|
||||||
|
}
|
||||||
|
if (String(url).startsWith("/user/files/") && method === "GET") {
|
||||||
|
logs.getCalls += 1;
|
||||||
|
return response(404, "not found");
|
||||||
|
}
|
||||||
|
return response(404, "unsupported route");
|
||||||
|
};
|
||||||
|
return { fetch, logs };
|
||||||
|
}
|
||||||
|
|
||||||
|
function createLegacyFileFetch() {
|
||||||
|
const files = new Map();
|
||||||
|
const logs = {
|
||||||
|
getCalls: 0,
|
||||||
|
uploadCalls: 0,
|
||||||
|
deleteCalls: 0,
|
||||||
|
sanitizeCalls: 0,
|
||||||
|
};
|
||||||
|
const response = (status, body) => ({
|
||||||
|
ok: status >= 200 && status < 300,
|
||||||
|
status,
|
||||||
|
statusText: String(status),
|
||||||
|
async json() {
|
||||||
|
return typeof body === "string" ? JSON.parse(body) : JSON.parse(JSON.stringify(body));
|
||||||
|
},
|
||||||
|
async text() {
|
||||||
|
return typeof body === "string" ? body : JSON.stringify(body);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
const decodeUploadData = (value = "") =>
|
||||||
|
Buffer.from(String(value || ""), "base64").toString("utf8");
|
||||||
|
const fetch = async (url, options = {}) => {
|
||||||
|
const method = String(options.method || "GET").toUpperCase();
|
||||||
|
if (url === "/api/files/sanitize-filename" && method === "POST") {
|
||||||
|
logs.sanitizeCalls += 1;
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
return response(200, {
|
||||||
|
fileName: String(body.fileName || "").replace(/[^A-Za-z0-9._~-]+/g, "_"),
|
||||||
|
});
|
||||||
|
}
|
||||||
|
if (url === "/api/files/upload" && method === "POST") {
|
||||||
|
logs.uploadCalls += 1;
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
const name = String(body.name || "");
|
||||||
|
files.set(name, decodeUploadData(body.data));
|
||||||
|
return response(200, { path: `/user/files/${name}` });
|
||||||
|
}
|
||||||
|
if (url === "/api/files/delete" && method === "POST") {
|
||||||
|
logs.deleteCalls += 1;
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
const name = decodeURIComponent(String(body.path || "").split("/").pop() || "");
|
||||||
|
const deleted = files.delete(name);
|
||||||
|
return response(deleted ? 200 : 404, deleted ? { deleted: true } : "not found");
|
||||||
|
}
|
||||||
|
if (String(url).startsWith("/user/files/") && method === "GET") {
|
||||||
|
logs.getCalls += 1;
|
||||||
|
const path = String(url).split("?")[0];
|
||||||
|
const name = decodeURIComponent(path.slice("/user/files/".length));
|
||||||
|
if (!files.has(name)) return response(404, "not found");
|
||||||
|
return response(200, JSON.parse(files.get(name)));
|
||||||
|
}
|
||||||
|
return response(404, "unsupported route");
|
||||||
|
};
|
||||||
|
return { fetch, files, logs };
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildRuntimeOptions({ dbByChatId, fetch, blobClient, onAuthorityBlobEvent = null }) {
|
||||||
|
return {
|
||||||
|
fetch,
|
||||||
|
blobClient,
|
||||||
|
authorityBlobEnabled: true,
|
||||||
|
authorityBlobFailOpen: true,
|
||||||
|
getDb: async (chatId) => {
|
||||||
|
const db = dbByChatId.get(chatId);
|
||||||
|
if (!db) throw new Error(`missing db: ${chatId}`);
|
||||||
|
return db;
|
||||||
|
},
|
||||||
|
getSafetyDb: async (chatId) => new FakeDb(`__restore_safety__${chatId}`),
|
||||||
|
getRequestHeaders: () => ({ "X-Test": "1" }),
|
||||||
|
onAuthorityBlobEvent,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function createFailingAuthorityBlobClient() {
|
||||||
|
const calls = [];
|
||||||
|
const fail = async (method, payload = {}) => {
|
||||||
|
calls.push([method, { ...payload }]);
|
||||||
|
throw new Error("blob unavailable");
|
||||||
|
};
|
||||||
|
return {
|
||||||
|
calls,
|
||||||
|
writeJson: (payload) => fail("writeJson", payload),
|
||||||
|
writeText: (payload) => fail("writeText", payload),
|
||||||
|
readJson: (payload) => fail("readJson", payload),
|
||||||
|
delete: (payload) => fail("delete", payload),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testAdapterBasics() {
|
||||||
|
const client = createMockAuthorityBlobClient();
|
||||||
|
const adapter = createAuthorityBlobAdapter({}, { blobClient: client });
|
||||||
|
assert.equal(normalizeAuthorityBlobPath("/user/files/demo.json"), "user/files/demo.json");
|
||||||
|
assert.equal(
|
||||||
|
normalizeAuthorityBlobReadResult({ data: JSON.stringify({ ok: true }) }, "a.json").payload.ok,
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
|
||||||
|
const writeResult = await adapter.writeJson("/user/files/demo.json", { hello: "world" });
|
||||||
|
assert.equal(writeResult.ok, true);
|
||||||
|
const readResult = await adapter.readJson("user/files/demo.json");
|
||||||
|
assert.equal(readResult.exists, true);
|
||||||
|
assert.deepEqual(readResult.payload, { hello: "world" });
|
||||||
|
const deleteResult = await adapter.delete("user/files/demo.json");
|
||||||
|
assert.equal(deleteResult.deleted, true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testAuthorityBlobFailOpenFallsBackToUserFiles() {
|
||||||
|
globalThis.localStorage = new MemoryStorage();
|
||||||
|
const blobClient = createFailingAuthorityBlobClient();
|
||||||
|
const { fetch, logs } = createLegacyFileFetch();
|
||||||
|
const dbByChatId = new Map();
|
||||||
|
const db = new FakeDb("blob-fallback", {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-fallback",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 9,
|
||||||
|
lastModified: 90,
|
||||||
|
nodeCount: 1,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [{ id: "fallback-node", updatedAt: 90 }],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: 6, extractionCount: 3 },
|
||||||
|
});
|
||||||
|
dbByChatId.set("blob-fallback", db);
|
||||||
|
const events = [];
|
||||||
|
const runtime = buildRuntimeOptions({
|
||||||
|
dbByChatId,
|
||||||
|
fetch,
|
||||||
|
blobClient,
|
||||||
|
onAuthorityBlobEvent: (event) => events.push(event),
|
||||||
|
});
|
||||||
|
|
||||||
|
const backupResult = await backupToServer("blob-fallback", runtime);
|
||||||
|
assert.equal(backupResult.backedUp, true);
|
||||||
|
assert.ok(logs.uploadCalls > 0);
|
||||||
|
assert.ok(blobClient.calls.some(([method]) => method === "writeJson"));
|
||||||
|
assert.ok(events.some((event) => event.reason === "authority-blob-error"));
|
||||||
|
|
||||||
|
db.snapshot = {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-fallback",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 1,
|
||||||
|
lastModified: 10,
|
||||||
|
nodeCount: 0,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: -1, extractionCount: 0 },
|
||||||
|
};
|
||||||
|
const restoreResult = await restoreFromServer("blob-fallback", runtime);
|
||||||
|
assert.equal(restoreResult.restored, true);
|
||||||
|
assert.equal(db.snapshot.nodes[0].id, "fallback-node");
|
||||||
|
assert.ok(logs.getCalls > 0);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testBackupRestoreUsesAuthorityBlob() {
|
||||||
|
globalThis.localStorage = new MemoryStorage();
|
||||||
|
const blobClient = createMockAuthorityBlobClient();
|
||||||
|
const { fetch, logs } = createMockFetch();
|
||||||
|
const dbByChatId = new Map();
|
||||||
|
const db = new FakeDb("blob-backup", {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-backup",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 7,
|
||||||
|
lastModified: 70,
|
||||||
|
nodeCount: 1,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [{ id: "blob-node", updatedAt: 70 }],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: 3, extractionCount: 2 },
|
||||||
|
});
|
||||||
|
dbByChatId.set("blob-backup", db);
|
||||||
|
const events = [];
|
||||||
|
const runtime = buildRuntimeOptions({
|
||||||
|
dbByChatId,
|
||||||
|
fetch,
|
||||||
|
blobClient,
|
||||||
|
onAuthorityBlobEvent: (event) => events.push(event),
|
||||||
|
});
|
||||||
|
|
||||||
|
const backupResult = await backupToServer("blob-backup", runtime);
|
||||||
|
assert.equal(backupResult.backedUp, true);
|
||||||
|
assert.equal(logs.uploadCalls, 0);
|
||||||
|
assert.equal(blobClient.files.has("user/files/ST-BME_BackupManifest.json"), true);
|
||||||
|
assert.equal(blobClient.files.has(`user/files/${backupResult.filename}`), true);
|
||||||
|
|
||||||
|
const manifest = await listServerBackups(runtime);
|
||||||
|
assert.equal(manifest.entries.length, 1);
|
||||||
|
assert.equal(manifest.entries[0].chatId, "blob-backup");
|
||||||
|
|
||||||
|
db.snapshot = {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-backup",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 1,
|
||||||
|
lastModified: 10,
|
||||||
|
nodeCount: 0,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: -1, extractionCount: 0 },
|
||||||
|
};
|
||||||
|
const restoreResult = await restoreFromServer("blob-backup", runtime);
|
||||||
|
assert.equal(restoreResult.restored, true);
|
||||||
|
assert.equal(db.snapshot.nodes[0].id, "blob-node");
|
||||||
|
assert.equal(events.some((event) => event.backend === "authority-blob"), true);
|
||||||
|
}
|
||||||
|
|
||||||
|
async function testSyncUploadDownloadUsesAuthorityBlob() {
|
||||||
|
globalThis.localStorage = new MemoryStorage();
|
||||||
|
const blobClient = createMockAuthorityBlobClient();
|
||||||
|
const { fetch, logs } = createMockFetch();
|
||||||
|
const dbByChatId = new Map();
|
||||||
|
const db = new FakeDb("blob-sync", {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-sync",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 5,
|
||||||
|
lastModified: 50,
|
||||||
|
nodeCount: 1,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [{ id: "sync-blob-node", updatedAt: 50 }],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: 4, extractionCount: 1 },
|
||||||
|
});
|
||||||
|
dbByChatId.set("blob-sync", db);
|
||||||
|
const runtime = buildRuntimeOptions({ dbByChatId, fetch, blobClient });
|
||||||
|
|
||||||
|
const uploadResult = await upload("blob-sync", runtime);
|
||||||
|
assert.equal(uploadResult.uploaded, true);
|
||||||
|
assert.equal(logs.uploadCalls, 0);
|
||||||
|
assert.equal(blobClient.files.has("user/files/ST-BME_sync_blob-sync.json"), true);
|
||||||
|
|
||||||
|
db.snapshot = {
|
||||||
|
meta: {
|
||||||
|
schemaVersion: 1,
|
||||||
|
chatId: "blob-sync",
|
||||||
|
deviceId: "",
|
||||||
|
revision: 1,
|
||||||
|
lastModified: 10,
|
||||||
|
nodeCount: 0,
|
||||||
|
edgeCount: 0,
|
||||||
|
tombstoneCount: 0,
|
||||||
|
},
|
||||||
|
nodes: [],
|
||||||
|
edges: [],
|
||||||
|
tombstones: [],
|
||||||
|
state: { lastProcessedFloor: -1, extractionCount: 0 },
|
||||||
|
};
|
||||||
|
const downloadResult = await download("blob-sync", runtime);
|
||||||
|
assert.equal(downloadResult.downloaded, true);
|
||||||
|
assert.equal(db.snapshot.nodes[0].id, "sync-blob-node");
|
||||||
|
}
|
||||||
|
|
||||||
|
await testAdapterBasics();
|
||||||
|
await testAuthorityBlobFailOpenFallsBackToUserFiles();
|
||||||
|
await testBackupRestoreUsesAuthorityBlob();
|
||||||
|
await testSyncUploadDownloadUsesAuthorityBlob();
|
||||||
|
console.log("authority-blob tests passed");
|
||||||
@@ -137,6 +137,7 @@ export function createGraphPersistenceState() {
|
|||||||
authorityServerPrimaryReady: false,
|
authorityServerPrimaryReady: false,
|
||||||
authorityStoragePrimaryReady: false,
|
authorityStoragePrimaryReady: false,
|
||||||
authorityTriviumPrimaryReady: false,
|
authorityTriviumPrimaryReady: false,
|
||||||
|
authorityBlobReady: false,
|
||||||
authorityBrowserCacheMode: "minimal",
|
authorityBrowserCacheMode: "minimal",
|
||||||
authorityOfflineQueueBytes: 0,
|
authorityOfflineQueueBytes: 0,
|
||||||
authorityOfflineQueueItems: 0,
|
authorityOfflineQueueItems: 0,
|
||||||
@@ -155,6 +156,14 @@ export function createGraphPersistenceState() {
|
|||||||
authorityLastJobProgress: 0,
|
authorityLastJobProgress: 0,
|
||||||
authorityLastJobError: "",
|
authorityLastJobError: "",
|
||||||
authorityLastJobUpdatedAt: "",
|
authorityLastJobUpdatedAt: "",
|
||||||
|
authorityBlobState: "idle",
|
||||||
|
authorityLastBlobEvent: null,
|
||||||
|
authorityLastBlobAction: "",
|
||||||
|
authorityLastBlobBackend: "",
|
||||||
|
authorityLastBlobPath: "",
|
||||||
|
authorityLastBlobReason: "",
|
||||||
|
authorityLastBlobError: "",
|
||||||
|
authorityLastBlobUpdatedAt: "",
|
||||||
localStoreFormatVersion: 1,
|
localStoreFormatVersion: 1,
|
||||||
localStoreMigrationState: "idle",
|
localStoreMigrationState: "idle",
|
||||||
opfsWriteLockState: {
|
opfsWriteLockState: {
|
||||||
|
|||||||
Reference in New Issue
Block a user