From d7cbbb20c1dcc9e806ac7df43ecd461d0cdbe959 Mon Sep 17 00:00:00 2001 From: Youzini-afk <13153778771cx@gmail.com> Date: Tue, 28 Apr 2026 12:52:52 +0800 Subject: [PATCH] Integrate Authority Blob storage --- index.js | 235 ++++++++- maintenance/authority-blob-adapter.js | 329 ++++++++++++ sync/bme-sync.js | 713 +++++++++++++++++++------- tests/authority-blob.mjs | 439 ++++++++++++++++ ui/ui-status.js | 9 + 5 files changed, 1531 insertions(+), 194 deletions(-) create mode 100644 maintenance/authority-blob-adapter.js create mode 100644 tests/authority-blob.mjs diff --git a/index.js b/index.js index ce44e8e..4f278e2 100644 --- a/index.js +++ b/index.js @@ -373,6 +373,10 @@ import { createAuthorityJobAdapter, normalizeAuthorityJobConfig, } from "./maintenance/authority-job-adapter.js"; +import { + createAuthorityBlobAdapter, + normalizeAuthorityBlobConfig, +} from "./maintenance/authority-blob-adapter.js"; export { DEFAULT_TRIGGER_KEYWORDS, getSmartTriggerDecision }; @@ -1505,6 +1509,7 @@ function buildAuthorityPersistenceStatePatch(settings = getSettings()) { authorityStoragePrimaryReady: Boolean(capability.storagePrimaryReady), authorityTriviumPrimaryReady: Boolean(capability.triviumPrimaryReady), authorityJobsReady: Boolean(capability.jobsReady), + authorityBlobReady: Boolean(capability.blobReady), authorityBrowserCacheMode: String(browserState.mode || "minimal"), authorityOfflineQueueBytes: Number(browserState.offlineQueueBytes || 0), authorityOfflineQueueItems: Number(browserState.offlineQueueItems || 0), @@ -1965,6 +1970,206 @@ function recordAuthorityJobSnapshot(job = null, options = {}) { }); } +function recordAuthorityBlobSnapshot(event = {}) { + const normalizedEvent = + event && typeof event === "object" && !Array.isArray(event) ? event : {}; + updateGraphPersistenceState({ + authorityBlobState: normalizedEvent.ok === false ? "error" : "active", + authorityLastBlobEvent: cloneRuntimeDebugValue(normalizedEvent, null), + authorityLastBlobAction: String(normalizedEvent.action || ""), + authorityLastBlobBackend: String(normalizedEvent.backend || ""), + authorityLastBlobPath: String(normalizedEvent.path || ""), + authorityLastBlobReason: String(normalizedEvent.reason || ""), + authorityLastBlobError: String(normalizedEvent.error || ""), + authorityLastBlobUpdatedAt: String( + normalizedEvent.updatedAt || new Date().toISOString(), + ), + }); +} + +function buildAuthorityBlobFileHash(input = "") { + let hash = 2166136261; + const text = String(input ?? ""); + for (let index = 0; index < text.length; index += 1) { + hash ^= text.charCodeAt(index); + hash = Math.imul(hash, 16777619); + } + return (hash >>> 0).toString(36); +} + +function buildAuthorityBlobSafeSlug(input = "", fallback = "unknown") { + const normalized = String(input || fallback) + .trim() + .replace(/[^A-Za-z0-9._-]+/g, "_") + .replace(/_+/g, "_") + .replace(/^[_.-]+|[_.-]+$/g, "") + .slice(0, 96); + return normalized || fallback; +} + +function shouldUseAuthorityBlobCheckpoint() { + const settings = getSettings(); + const authoritySettings = normalizeAuthoritySettings(settings); + const { capability } = getAuthorityRuntimeSnapshot(settings); + return Boolean( + authoritySettings.enabled && + authoritySettings.blobCheckpointEnabled && + capability.blobReady, + ); +} + +function getAuthorityBlobAdapter(options = {}) { + const settings = getSettings(); + const config = normalizeAuthorityBlobConfig(settings); + return createAuthorityBlobAdapter(config, { + fetchImpl: globalThis.fetch?.bind(globalThis), + headerProvider: + typeof getRequestHeaders === "function" ? () => getRequestHeaders() : null, + ...options, + }); +} + +async function writeAuthorityLukerCheckpointBlob( + checkpoint = null, + { chatId = "", reason = "luker-checkpoint", signal = undefined } = {}, +) { + if (!checkpoint || !shouldUseAuthorityBlobCheckpoint()) { + return { + ok: false, + reason: "authority-blob-unavailable", + }; + } + const normalizedChatId = normalizeChatIdCandidate(chatId || checkpoint.chatId); + const safeChatId = buildAuthorityBlobSafeSlug(normalizedChatId); + const hash = buildAuthorityBlobFileHash(normalizedChatId || safeChatId); + const path = `user/files/ST-BME_luker_checkpoint_${safeChatId}-${hash}.json`; + try { + const adapter = getAuthorityBlobAdapter(); + const result = await adapter.writeJson(path, checkpoint, { + signal, + metadata: { + chatId: normalizedChatId, + revision: Number(checkpoint?.revision || 0), + reason: String(reason || ""), + kind: "luker-checkpoint", + }, + }); + const event = { + action: "checkpoint-write", + ok: result?.ok !== false, + backend: "authority-blob", + path: result?.path || path, + reason: String(reason || ""), + revision: Number(checkpoint?.revision || 0), + }; + recordAuthorityBlobSnapshot(event); + updateGraphPersistenceState({ + authorityBlobCheckpointPath: event.path, + authorityBlobCheckpointRevision: event.revision, + authorityBlobCheckpointUpdatedAt: new Date().toISOString(), + }); + return { + ok: event.ok, + path: event.path, + result, + }; + } catch (error) { + const message = error?.message || String(error) || "Authority Blob checkpoint failed"; + recordAuthorityBlobSnapshot({ + action: "checkpoint-write", + ok: false, + backend: "authority-blob", + path, + reason: String(reason || ""), + error: message, + revision: Number(checkpoint?.revision || 0), + }); + return { + ok: false, + path, + reason: "authority-blob-checkpoint-error", + error, + }; + } +} + +async function readAuthorityLukerCheckpointBlob(chatId = "", options = {}) { + if (!shouldUseAuthorityBlobCheckpoint()) { + return { + ok: false, + exists: false, + reason: "authority-blob-unavailable", + }; + } + const normalizedChatId = normalizeChatIdCandidate(chatId); + if (!normalizedChatId) { + return { + ok: false, + exists: false, + reason: "missing-chat-id", + }; + } + const safeChatId = buildAuthorityBlobSafeSlug(normalizedChatId); + const hash = buildAuthorityBlobFileHash(normalizedChatId || safeChatId); + const path = `user/files/ST-BME_luker_checkpoint_${safeChatId}-${hash}.json`; + try { + const adapter = getAuthorityBlobAdapter(); + const result = await adapter.readJson(path, options); + const exists = Boolean(result?.exists && result?.payload); + recordAuthorityBlobSnapshot({ + action: "checkpoint-read", + ok: result?.ok !== false, + backend: "authority-blob", + path: result?.path || path, + reason: exists ? "checkpoint-found" : "checkpoint-missing", + revision: Number(result?.payload?.revision || 0), + }); + return { + ok: result?.ok !== false, + exists, + path: result?.path || path, + checkpoint: exists ? result.payload : null, + result, + }; + } catch (error) { + const message = error?.message || String(error) || "Authority Blob checkpoint read failed"; + recordAuthorityBlobSnapshot({ + action: "checkpoint-read", + ok: false, + backend: "authority-blob", + path, + reason: "authority-blob-checkpoint-read-error", + error: message, + }); + return { + ok: false, + exists: false, + path, + reason: "authority-blob-checkpoint-read-error", + error, + }; + } +} + +async function readLukerGraphSidecarV2WithAuthorityBlob(context = null, options = {}) { + const sidecar = await readLukerGraphSidecarV2(context, options); + if (sidecar?.checkpoint) return sidecar; + const chatId = + normalizeChatIdCandidate(options.chatId) || + normalizeChatIdCandidate(sidecar?.manifest?.chatId) || + normalizeChatIdCandidate(getCurrentChatId(context)); + const blobResult = await readAuthorityLukerCheckpointBlob(chatId); + if (!blobResult?.exists || !blobResult?.checkpoint) return sidecar; + return { + ...(sidecar || {}), + checkpoint: blobResult.checkpoint, + authorityBlobCheckpoint: { + path: blobResult.path, + backend: "authority-blob", + }, + }; +} + async function submitAuthorityVectorRebuildJob({ config = null, range = null, @@ -6448,6 +6653,9 @@ async function refreshRuntimeGraphAfterSyncApplied(syncPayload = {}) { function buildBmeSyncRuntimeOptions(extra = {}) { const normalizedExtra = extra && typeof extra === "object" && !Array.isArray(extra) ? extra : {}; + const settings = getSettings(); + const authoritySettings = normalizeAuthoritySettings(settings); + const { capability } = getAuthorityRuntimeSnapshot(settings); const defaultOptions = { getDb: async (chatId) => { const manager = ensureBmeChatManager(); @@ -6468,6 +6676,16 @@ function buildBmeSyncRuntimeOptions(extra = {}) { getCurrentChatId: () => getCurrentChatId(), getCloudStorageMode: () => getSettings().cloudStorageMode || "automatic", getRequestHeaders, + authorityBlobEnabled: Boolean( + authoritySettings.enabled && + authoritySettings.blobCheckpointEnabled && + capability.blobReady, + ), + authorityBlobFailOpen: authoritySettings.failOpen, + authorityBlobConfig: { + ...authoritySettings, + }, + onAuthorityBlobEvent: recordAuthorityBlobSnapshot, onSyncApplied: async (payload = {}) => { await refreshRuntimeGraphAfterSyncApplied(payload); }, @@ -7436,6 +7654,10 @@ async function compactLukerGraphSidecarV2( error: checkpointResult?.error || null, }; } + await writeAuthorityLukerCheckpointBlob(checkpointResult.checkpoint, { + chatId: normalizedChatId, + reason, + }); const emptyJournal = buildLukerGraphJournalV2([], { chatId: normalizedChatId, @@ -7651,11 +7873,12 @@ async function persistGraphToLukerSidecarV2( ? cloneRuntimeDebugValue(persistDelta, persistDelta) : null; - const existingSidecar = await readLukerGraphSidecarV2(context, { + const existingSidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, { manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE, journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE, checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE, chatStateTarget: normalizedTarget, + chatId, }); if (existingSidecar?.manifest) { cacheChatStateManifest(chatId, existingSidecar.manifest); @@ -7742,6 +7965,10 @@ async function persistGraphToLukerSidecarV2( error: checkpointResult?.error || null, }; } + await writeAuthorityLukerCheckpointBlob(checkpointResult.checkpoint, { + chatId, + reason: `${reason}:bootstrap`, + }); const emptyJournal = buildLukerGraphJournalV2([], { chatId, integrity: nextIntegrity, @@ -8071,11 +8298,12 @@ async function loadGraphFromLukerSidecarV2( }; } - const sidecar = await readLukerGraphSidecarV2(context, { + const sidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, { manifestNamespace: LUKER_GRAPH_MANIFEST_NAMESPACE, journalNamespace: LUKER_GRAPH_JOURNAL_NAMESPACE, checkpointNamespace: LUKER_GRAPH_CHECKPOINT_NAMESPACE, chatStateTarget: normalizedTarget, + chatId: normalizedChatId, }); const manifest = sidecar?.manifest || null; if (!manifest) { @@ -8784,8 +9012,9 @@ async function readPersistedGraphForChatStateTarget( return null; } - const sidecar = await readLukerGraphSidecarV2(context, { + const sidecar = await readLukerGraphSidecarV2WithAuthorityBlob(context, { chatStateTarget: normalizedTarget, + chatId: targetChatId, }); const sidecarResult = buildSnapshotFromLukerSidecarState(sidecar, { chatId: targetChatId, diff --git a/maintenance/authority-blob-adapter.js b/maintenance/authority-blob-adapter.js new file mode 100644 index 0000000..1986dd8 --- /dev/null +++ b/maintenance/authority-blob-adapter.js @@ -0,0 +1,329 @@ +import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; + +export const AUTHORITY_BLOB_ENDPOINT = "/v1/blob"; + +function toPlainData(value, fallbackValue = null) { + if (value == null) return fallbackValue; + if (typeof globalThis.structuredClone === "function") { + try { + return globalThis.structuredClone(value); + } catch { + } + } + try { + return JSON.parse(JSON.stringify(value)); + } catch { + return fallbackValue; + } +} + +function normalizeRecordId(value) { + return String(value ?? "").trim(); +} + +function normalizeInteger(value, fallback = 0, min = 0, max = Number.MAX_SAFE_INTEGER) { + const parsed = Number(value); + if (!Number.isFinite(parsed)) return fallback; + return Math.min(max, Math.max(min, Math.trunc(parsed))); +} + +function decodeBase64Utf8(base64Text = "") { + const normalizedBase64 = String(base64Text ?? ""); + if (!normalizedBase64) return ""; + if (typeof globalThis.atob === "function" && typeof globalThis.TextDecoder === "function") { + const binary = globalThis.atob(normalizedBase64); + const bytes = Uint8Array.from(binary, (char) => char.charCodeAt(0)); + return new TextDecoder().decode(bytes); + } + if (typeof Buffer !== "undefined") { + return Buffer.from(normalizedBase64, "base64").toString("utf8"); + } + return normalizedBase64; +} + +function tryParseJsonText(text, fallbackValue = null) { + if (typeof text !== "string") return fallbackValue; + try { + return JSON.parse(text); + } catch { + return fallbackValue; + } +} + +export function normalizeAuthorityBlobPath(path = "") { + const normalized = String(path ?? "") + .trim() + .replace(/\\/g, "/") + .replace(/^authority:\/\/private\//i, "") + .replace(/^\/+/, "") + .replace(/\/+/g, "/"); + return normalized.replace(/\/+$/g, ""); +} + +function normalizeBlobPayload(result = null) { + if (!result || typeof result !== "object" || Array.isArray(result)) return result; + const source = result.file || result.blob || result.result || result; + if (source.payload !== undefined) return source.payload; + if (source.json !== undefined) return source.json; + if (source.value !== undefined) return source.value; + if (source.data !== undefined) { + if (source.encoding === "base64" || source.base64 === true) { + return tryParseJsonText(decodeBase64Utf8(source.data), source.data); + } + if (typeof source.data === "string") { + return tryParseJsonText(source.data, source.data); + } + return source.data; + } + if (source.content !== undefined) { + if (typeof source.content === "string") { + return tryParseJsonText(source.content, source.content); + } + return source.content; + } + if (source.body !== undefined) { + if (typeof source.body === "string") { + return tryParseJsonText(source.body, source.body); + } + return source.body; + } + return null; +} + +function normalizeBlobRecordSource(input = null) { + if (!input || typeof input !== "object" || Array.isArray(input)) return {}; + return input.file || input.blob || input.result || input; +} + +export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") { + const source = normalizeBlobRecordSource(input); + const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); + const missing = + source.exists === false || + source.found === false || + source.missing === true || + Number(source.status || source.statusCode || 0) === 404; + if (missing) { + return { + exists: false, + path, + payload: null, + contentType: String(source.contentType || source.type || ""), + raw: toPlainData(input, input), + }; + } + return { + exists: input != null && source.ok !== false, + path, + payload: normalizeBlobPayload(input), + contentType: String(source.contentType || source.type || "application/json"), + etag: String(source.etag || source.hash || ""), + updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", + raw: toPlainData(input, input), + }; +} + +export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") { + const source = normalizeBlobRecordSource(input); + const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); + return { + ok: input == null ? true : source.ok !== false && source.error == null, + path, + url: String(source.url || source.href || ""), + size: normalizeInteger(source.size || source.bytes, 0, 0), + etag: String(source.etag || source.hash || ""), + updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", + raw: toPlainData(input, input), + }; +} + +export function normalizeAuthorityBlobDeleteResult(input = null, fallbackPath = "") { + const source = normalizeBlobRecordSource(input); + const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); + const missing = + source.exists === false || + source.found === false || + source.missing === true || + Number(source.status || source.statusCode || 0) === 404; + return { + ok: input == null ? true : source.ok !== false, + deleted: missing ? false : source.deleted !== false && source.ok !== false, + missing, + path, + raw: toPlainData(input, input), + }; +} + +export function normalizeAuthorityBlobConfig(settings = {}, overrides = {}) { + const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {}; + return { + baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl), + enabled: + source.authorityBlobCheckpointEnabled !== false && + source.blobCheckpointEnabled !== false && + source.authorityBlobEnabled !== false, + failOpen: source.authorityFailOpen !== false && source.failOpen !== false, + namespace: normalizeRecordId(source.authorityBlobNamespace || source.blobNamespace || "st-bme"), + ...overrides, + }; +} + +export class AuthorityBlobHttpClient { + constructor(options = {}) { + this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); + this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); + this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + } + + async request(action, payload = {}) { + if (typeof this.fetchImpl !== "function") { + throw new Error("Authority Blob fetch unavailable"); + } + const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_BLOB_ENDPOINT}`, { + method: "POST", + headers: { + Accept: "application/json", + "Content-Type": "application/json", + ...(this.headerProvider ? this.headerProvider() || {} : {}), + }, + body: JSON.stringify({ action, ...payload }), + }); + if (!response?.ok) { + const text = await response?.text?.().catch(() => ""); + throw new Error(text || `Authority Blob HTTP ${response?.status || "unknown"}`); + } + return await response.json().catch(() => ({})); + } + + async writeJson(payload = {}) { + return await this.request("writeJson", payload); + } + + async writeText(payload = {}) { + return await this.request("writeText", payload); + } + + async readJson(payload = {}) { + return await this.request("readJson", payload); + } + + async delete(payload = {}) { + return await this.request("delete", payload); + } + + async stat(payload = {}) { + return await this.request("stat", payload); + } +} + +export function createAuthorityBlobClient(config = {}, options = {}) { + const injected = options.blobClient || config.blobClient || globalThis.__stBmeAuthorityBlobClient; + if (injected) return injected; + return new AuthorityBlobHttpClient({ + baseUrl: config.baseUrl, + fetchImpl: options.fetchImpl || config.fetchImpl, + headerProvider: options.headerProvider || config.headerProvider, + }); +} + +async function callClient(client, methodNames = [], action = "request", payload = {}) { + for (const methodName of methodNames) { + if (typeof client?.[methodName] === "function") { + return await client[methodName](payload); + } + } + if (typeof client?.request === "function") { + return await client.request(action, payload); + } + if (typeof client === "function") { + return await client({ action, ...payload }); + } + throw new Error(`Authority Blob ${action} unavailable`); +} + +function throwIfAborted(signal) { + if (signal?.aborted) { + throw signal.reason instanceof Error + ? signal.reason + : Object.assign(new Error("操作已终止"), { name: "AbortError" }); + } +} + +export class AuthorityBlobAdapter { + constructor(config = {}, options = {}) { + this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {}); + this.client = createAuthorityBlobClient(this.config, options); + } + + async writeJson(path, payload = null, options = {}) { + throwIfAborted(options.signal); + const normalizedPath = normalizeAuthorityBlobPath(path); + if (!normalizedPath) throw new Error("Authority Blob path is required"); + const result = await callClient(this.client, ["writeJson", "putJson", "writeFile", "put"], "writeJson", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + contentType: options.contentType || "application/json", + payload: toPlainData(payload, payload), + data: toPlainData(payload, payload), + metadata: toPlainData(options.metadata, {}), + }); + return normalizeAuthorityBlobWriteResult(result, normalizedPath); + } + + async writeText(path, text = "", options = {}) { + throwIfAborted(options.signal); + const normalizedPath = normalizeAuthorityBlobPath(path); + if (!normalizedPath) throw new Error("Authority Blob path is required"); + const result = await callClient(this.client, ["writeText", "writeFile", "putText", "put"], "writeText", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + contentType: options.contentType || "text/plain; charset=utf-8", + text: String(text ?? ""), + data: String(text ?? ""), + metadata: toPlainData(options.metadata, {}), + }); + return normalizeAuthorityBlobWriteResult(result, normalizedPath); + } + + async readJson(path, options = {}) { + throwIfAborted(options.signal); + const normalizedPath = normalizeAuthorityBlobPath(path); + if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); + const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobReadResult(result, normalizedPath); + } + + async delete(path, options = {}) { + throwIfAborted(options.signal); + const normalizedPath = normalizeAuthorityBlobPath(path); + if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, ""); + const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobDeleteResult(result, normalizedPath); + } + + async stat(path, options = {}) { + throwIfAborted(options.signal); + const normalizedPath = normalizeAuthorityBlobPath(path); + if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); + const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobReadResult(result, normalizedPath); + } +} + +export function createAuthorityBlobAdapter(config = {}, options = {}) { + return new AuthorityBlobAdapter(config, options); +} diff --git a/sync/bme-sync.js b/sync/bme-sync.js index 7fd4ac7..82b869a 100644 --- a/sync/bme-sync.js +++ b/sync/bme-sync.js @@ -3,6 +3,7 @@ import { MANUAL_BACKUP_BATCH_JOURNAL_COVERAGE_KEY, PROCESSED_MESSAGE_HASH_VERSION, } from "../runtime/runtime-state.js"; +import { createAuthorityBlobAdapter } from "../maintenance/authority-blob-adapter.js"; const BME_SYNC_FILE_PREFIX = "ST-BME_sync_"; const BME_SYNC_FILE_SUFFIX = ".json"; @@ -336,6 +337,454 @@ function getFetch(options = {}) { return fetchImpl; } +function normalizeRemoteFileName(fileName = "") { + const normalized = String(fileName ?? "") + .trim() + .replace(/^\/+/, ""); + if (!normalized || /[\\/]/.test(normalized)) return ""; + return normalized; +} + +function normalizeRemoteServerPath(pathOrFilename = "", fallbackFilename = "") { + const raw = String(pathOrFilename || fallbackFilename || "") + .trim() + .replace(/\\/g, "/") + .replace(/^authority:\/\/private\//i, "") + .replace(/^\/+/, "") + .split("?")[0]; + if (!raw) return ""; + if (raw.startsWith("user/files/")) { + const fileName = normalizeRemoteFileName(raw.slice("user/files/".length)); + return fileName ? `user/files/${fileName}` : ""; + } + const fileName = normalizeRemoteFileName(raw) || normalizeRemoteFileName(fallbackFilename); + return fileName ? `user/files/${fileName}` : ""; +} + +function buildRemoteFileUrl(pathOrFilename = "", fallbackFilename = "") { + const serverPath = normalizeRemoteServerPath(pathOrFilename, fallbackFilename); + if (!serverPath) return ""; + const fileName = normalizeRemoteFileName(serverPath.slice("user/files/".length)); + return fileName ? `/user/files/${encodeURIComponent(fileName)}` : `/${serverPath}`; +} + +function resolveRemoteFileName(pathOrFilename = "", fallbackFilename = "") { + const serverPath = normalizeRemoteServerPath(pathOrFilename, fallbackFilename); + return normalizeRemoteFileName(serverPath.slice("user/files/".length)); +} + +function parseSerializedJsonPayload(payload = "") { + try { + return { + ok: true, + value: JSON.parse(String(payload ?? "")), + }; + } catch { + return { + ok: false, + value: null, + }; + } +} + +function shouldUseAuthorityBlobFiles(options = {}) { + if (options.authorityBlobEnabled === false) return false; + if (options.authorityBlobAdapter || options.authorityBlobClient || options.blobClient) return true; + return options.authorityBlobEnabled === true; +} + +function getAuthorityBlobAdapter(options = {}) { + if (!shouldUseAuthorityBlobFiles(options)) return null; + if (options.authorityBlobAdapter) return options.authorityBlobAdapter; + return createAuthorityBlobAdapter(options.authorityBlobConfig || options.authoritySettings || {}, { + blobClient: options.authorityBlobClient || options.blobClient, + fetchImpl: options.fetch, + headerProvider: () => getRequestHeadersSafe(options), + }); +} + +function readAuthorityBlobFailOpen(options = {}) { + return options.authorityBlobFailOpen !== false; +} + +function recordAuthorityBlobFileEvent(options = {}, event = {}) { + if (typeof options.onAuthorityBlobEvent !== "function") return; + try { + options.onAuthorityBlobEvent({ + ...event, + updatedAt: new Date().toISOString(), + }); + } catch { + } +} + +async function readAuthorityBlobJsonFile(pathOrFilename = "", options = {}) { + const adapter = getAuthorityBlobAdapter(options); + const path = normalizeRemoteServerPath(pathOrFilename); + if (!adapter || !path) { + return { + available: false, + exists: false, + path, + payload: null, + reason: "authority-blob-disabled", + }; + } + const startedAt = readSyncTimingNow(); + try { + const result = await adapter.readJson(path, { + signal: options.signal, + namespace: options.authorityBlobNamespace, + }); + const elapsedMs = readSyncTimingNow() - startedAt; + const exists = result?.exists === true; + recordAuthorityBlobFileEvent(options, { + action: "read", + ok: result?.ok !== false, + path, + backend: "authority-blob", + reason: exists ? "ok" : "not-found", + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + return { + available: true, + exists, + path: result?.path || path, + payload: exists ? result.payload : null, + reason: exists ? "ok" : "not-found", + timings: { + authorityBlobMs: normalizeSyncTimingMs(elapsedMs), + }, + }; + } catch (error) { + const elapsedMs = readSyncTimingNow() - startedAt; + recordAuthorityBlobFileEvent(options, { + action: "read", + ok: false, + path, + backend: "authority-blob", + reason: "authority-blob-error", + error: error instanceof Error ? error.message : String(error || ""), + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + if (!readAuthorityBlobFailOpen(options)) throw error; + return { + available: true, + exists: false, + path, + payload: null, + reason: "authority-blob-error", + error, + timings: { + authorityBlobMs: normalizeSyncTimingMs(elapsedMs), + }, + }; + } +} + +async function writeRemoteJsonFile(pathOrFilename = "", serializedPayload = "", options = {}) { + const serverPath = normalizeRemoteServerPath(pathOrFilename); + const fileName = resolveRemoteFileName(serverPath); + if (!serverPath || !fileName) throw new Error("remote filename is required"); + const adapter = getAuthorityBlobAdapter(options); + if (adapter) { + const startedAt = readSyncTimingNow(); + try { + const parsedPayload = parseSerializedJsonPayload(serializedPayload); + const result = parsedPayload.ok + ? await adapter.writeJson(serverPath, parsedPayload.value, { + signal: options.signal, + namespace: options.authorityBlobNamespace, + metadata: { + filename: fileName, + }, + }) + : await adapter.writeText(serverPath, String(serializedPayload ?? ""), { + signal: options.signal, + namespace: options.authorityBlobNamespace, + contentType: "application/json; charset=utf-8", + metadata: { + filename: fileName, + }, + }); + const elapsedMs = readSyncTimingNow() - startedAt; + if (result?.ok !== false) { + recordAuthorityBlobFileEvent(options, { + action: "write", + ok: true, + path: serverPath, + backend: "authority-blob", + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + return { + path: `/${serverPath}`, + authorityPath: result?.path || serverPath, + backend: "authority-blob", + }; + } + } catch (error) { + const elapsedMs = readSyncTimingNow() - startedAt; + recordAuthorityBlobFileEvent(options, { + action: "write", + ok: false, + path: serverPath, + backend: "authority-blob", + reason: "authority-blob-error", + error: error instanceof Error ? error.message : String(error || ""), + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + if (!readAuthorityBlobFailOpen(options)) throw error; + } + } + + const fetchImpl = getFetch(options); + const response = await fetchImpl("/api/files/upload", { + method: "POST", + headers: { + ...getRequestHeadersSafe(options), + "Content-Type": "application/json", + }, + body: JSON.stringify({ + name: fileName, + data: encodeBase64Utf8(serializedPayload), + }), + }); + + if (!response.ok) { + const errorText = await response.text().catch(() => response.statusText); + throw new Error(errorText || `HTTP ${response.status}`); + } + + const uploadResult = await response.json().catch(() => ({})); + return { + path: String(uploadResult?.path || `/${serverPath}`), + backend: "user-files", + }; +} + +async function readRemoteJsonFileResult(pathOrFilename = "", options = {}) { + const serverPath = normalizeRemoteServerPath(pathOrFilename); + const fileName = resolveRemoteFileName(serverPath); + if (!serverPath || !fileName) { + return { + ok: false, + status: 404, + reason: "remote-file-name-invalid", + path: serverPath, + filename: fileName, + payload: null, + }; + } + + const authorityResult = await readAuthorityBlobJsonFile(serverPath, options); + if (authorityResult.exists) { + return { + ok: true, + status: 200, + reason: "ok", + path: authorityResult.path || serverPath, + filename: fileName, + payload: authorityResult.payload, + backend: "authority-blob", + timings: authorityResult.timings || {}, + }; + } + + const fetchImpl = getFetch(options); + let response; + let networkMs = 0; + let parseMs = 0; + try { + const networkStartedAt = readSyncTimingNow(); + response = await fetchImpl(`${buildRemoteFileUrl(serverPath)}?t=${Date.now()}`, { + method: "GET", + cache: "no-store", + }); + networkMs = readSyncTimingNow() - networkStartedAt; + } catch (error) { + return { + ok: false, + status: 0, + reason: "network-error", + path: serverPath, + filename: fileName, + payload: null, + error, + timings: { + networkMs: normalizeSyncTimingMs(networkMs), + parseMs: 0, + }, + }; + } + + if (response.status === 404) { + return { + ok: false, + status: 404, + reason: "not-found", + path: serverPath, + filename: fileName, + payload: null, + timings: { + networkMs: normalizeSyncTimingMs(networkMs), + parseMs: 0, + }, + }; + } + + if (!response.ok) { + const errorText = await response.text().catch(() => response.statusText); + return { + ok: false, + status: response.status, + reason: "http-error", + path: serverPath, + filename: fileName, + payload: null, + error: new Error(errorText || `HTTP ${response.status}`), + timings: { + networkMs: normalizeSyncTimingMs(networkMs), + parseMs: 0, + }, + }; + } + + try { + const parseStartedAt = readSyncTimingNow(); + const payload = await response.json(); + parseMs = readSyncTimingNow() - parseStartedAt; + return { + ok: true, + status: 200, + reason: "ok", + path: serverPath, + filename: fileName, + payload, + backend: "user-files", + timings: { + networkMs: normalizeSyncTimingMs(networkMs), + parseMs: normalizeSyncTimingMs(parseMs), + }, + }; + } catch (error) { + return { + ok: false, + status: 0, + reason: "invalid-json", + path: serverPath, + filename: fileName, + payload: null, + error, + timings: { + networkMs: normalizeSyncTimingMs(networkMs), + parseMs: normalizeSyncTimingMs(parseMs), + }, + }; + } +} + +async function deleteRemoteJsonFile(pathOrFilename = "", options = {}) { + const serverPath = normalizeRemoteServerPath(pathOrFilename); + const fileName = resolveRemoteFileName(serverPath); + if (!serverPath || !fileName) { + return { + deleted: false, + reason: "remote-file-name-invalid", + path: serverPath, + filename: fileName, + }; + } + + const adapter = getAuthorityBlobAdapter(options); + let authorityDeleted = false; + if (adapter) { + const startedAt = readSyncTimingNow(); + try { + const result = await adapter.delete(serverPath, { + signal: options.signal, + namespace: options.authorityBlobNamespace, + }); + const elapsedMs = readSyncTimingNow() - startedAt; + authorityDeleted = result?.deleted === true; + recordAuthorityBlobFileEvent(options, { + action: "delete", + ok: result?.ok !== false, + path: serverPath, + backend: "authority-blob", + reason: authorityDeleted ? "ok" : "not-found", + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + if (authorityDeleted) { + try { + const fetchImpl = getFetch(options); + await fetchImpl("/api/files/delete", { + method: "POST", + headers: { + ...getRequestHeadersSafe(options), + "Content-Type": "application/json", + }, + body: JSON.stringify({ + path: `/${serverPath}`, + }), + }).catch(() => null); + } catch { + } + return { + deleted: true, + path: serverPath, + filename: fileName, + backend: "authority-blob", + }; + } + } catch (error) { + const elapsedMs = readSyncTimingNow() - startedAt; + recordAuthorityBlobFileEvent(options, { + action: "delete", + ok: false, + path: serverPath, + backend: "authority-blob", + reason: "authority-blob-error", + error: error instanceof Error ? error.message : String(error || ""), + elapsedMs: normalizeSyncTimingMs(elapsedMs), + }); + if (!readAuthorityBlobFailOpen(options)) throw error; + } + } + + const fetchImpl = getFetch(options); + const response = await fetchImpl("/api/files/delete", { + method: "POST", + headers: { + ...getRequestHeadersSafe(options), + "Content-Type": "application/json", + }, + body: JSON.stringify({ + path: `/${serverPath}`, + }), + }); + + if (response.status === 404) { + return { + deleted: false, + reason: "not-found", + path: serverPath, + filename: fileName, + }; + } + + if (!response.ok) { + const errorText = await response.text().catch(() => response.statusText); + throw new Error(errorText || `HTTP ${response.status}`); + } + + return { + deleted: true, + path: serverPath, + filename: fileName, + backend: "user-files", + }; +} + async function getSafetyDb(chatId, options = {}) { if (typeof options.getSafetyDb === "function") { return await options.getSafetyDb(chatId); @@ -347,22 +796,14 @@ async function getSafetyDb(chatId, options = {}) { } async function fetchBackupManifest(options = {}) { - const fetchImpl = getFetch(options); - const response = await fetchImpl( - `/user/files/${BME_BACKUP_MANIFEST_FILENAME}?t=${Date.now()}`, - { - method: "GET", - cache: "no-store", - }, - ); - if (response.status === 404) { + const result = await readRemoteJsonFileResult(BME_BACKUP_MANIFEST_FILENAME, options); + if (result.status === 404) { return []; } - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `manifest read failed: HTTP ${response.status}`); + if (!result.ok) { + throw result.error || new Error(result.reason || "manifest read failed"); } - const rawPayload = await response.json(); + const rawPayload = result.payload; if (!Array.isArray(rawPayload)) { throw new Error("backup manifest payload is not an array"); } @@ -370,24 +811,8 @@ async function fetchBackupManifest(options = {}) { } async function writeBackupManifest(entries = [], options = {}) { - const fetchImpl = getFetch(options); const payload = JSON.stringify(entries); - const response = await fetchImpl("/api/files/upload", { - method: "POST", - headers: { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }, - body: JSON.stringify({ - name: BME_BACKUP_MANIFEST_FILENAME, - data: encodeBase64Utf8(payload), - }), - }); - - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); - } + await writeRemoteJsonFile(BME_BACKUP_MANIFEST_FILENAME, payload, options); } async function upsertBackupManifestEntry(entry, options = {}) { @@ -523,50 +948,50 @@ async function readBackupEnvelope(chatId, options = {}) { const lookupStartedAt = readSyncTimingNow(); const lookup = await resolveBackupLookupContext(normalizedChatId, options); const lookupMs = readSyncTimingNow() - lookupStartedAt; - const fetchImpl = getFetch(options); const fallbackFilename = buildBackupFilename(normalizedChatId); let lastMissingFilename = lookup.candidates[0]?.filename || fallbackFilename; let networkMs = 0; let parseMs = 0; + let authorityBlobMs = 0; for (const candidate of lookup.candidates) { try { - const networkStartedAt = readSyncTimingNow(); - const response = await fetchImpl( - `${candidate.serverPath || `/user/files/${encodeURIComponent(candidate.filename)}`}?t=${Date.now()}`, - { - method: "GET", - cache: "no-store", - }, + const result = await readRemoteJsonFileResult( + candidate.serverPath || candidate.filename, + options, ); - networkMs += readSyncTimingNow() - networkStartedAt; - if (response.status === 404) { + networkMs += Number(result.timings?.networkMs || 0); + parseMs += Number(result.timings?.parseMs || 0); + authorityBlobMs += Number(result.timings?.authorityBlobMs || 0); + if (result.status === 404) { lastMissingFilename = candidate.filename; continue; } - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); + if (!result.ok) { return { exists: false, filename: candidate.filename, envelope: null, reason: "backup-read-error", - error: new Error(errorText || `HTTP ${response.status}`), - timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt), + error: result.error || new Error(result.reason || "backup-read-error"), + timings: finalizeSyncTimings( + { lookupMs, networkMs, parseMs, authorityBlobMs }, + readStartedAt, + ), }; } - const parseStartedAt = readSyncTimingNow(); - const payload = await response.json(); - parseMs += readSyncTimingNow() - parseStartedAt; - const envelope = normalizeBackupEnvelope(payload, normalizedChatId); + const envelope = normalizeBackupEnvelope(result.payload, normalizedChatId); if (!envelope) { return { exists: false, filename: candidate.filename, envelope: null, reason: "invalid-backup", - timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt), + timings: finalizeSyncTimings( + { lookupMs, networkMs, parseMs, authorityBlobMs }, + readStartedAt, + ), }; } return { @@ -574,7 +999,10 @@ async function readBackupEnvelope(chatId, options = {}) { filename: candidate.filename, envelope, reason: "ok", - timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt), + timings: finalizeSyncTimings( + { lookupMs, networkMs, parseMs, authorityBlobMs }, + readStartedAt, + ), }; } catch (error) { return { @@ -583,7 +1011,10 @@ async function readBackupEnvelope(chatId, options = {}) { envelope: null, reason: "backup-read-error", error, - timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt), + timings: finalizeSyncTimings( + { lookupMs, networkMs, parseMs, authorityBlobMs }, + readStartedAt, + ), }; } } @@ -594,7 +1025,10 @@ async function readBackupEnvelope(chatId, options = {}) { envelope: null, reason: "not-found", manifestError: lookup.manifestError, - timings: finalizeSyncTimings({ lookupMs, networkMs, parseMs }, readStartedAt), + timings: finalizeSyncTimings( + { lookupMs, networkMs, parseMs, authorityBlobMs }, + readStartedAt, + ), }; } @@ -622,40 +1056,22 @@ async function writeBackupEnvelope(envelope, chatId, options = {}) { const writeStartedAt = readSyncTimingNow(); const normalizedChatId = normalizeChatId(chatId); const filename = buildBackupFilename(normalizedChatId); - const fetchImpl = getFetch(options); const serializeStartedAt = readSyncTimingNow(); const payload = JSON.stringify(envelope); const serializeMs = readSyncTimingNow() - serializeStartedAt; const uploadStartedAt = readSyncTimingNow(); - const response = await fetchImpl("/api/files/upload", { - method: "POST", - headers: { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }, - body: JSON.stringify({ - name: filename, - data: encodeBase64Utf8(payload), - }), - }); + const uploadResult = await writeRemoteJsonFile(filename, payload, options); const uploadMs = readSyncTimingNow() - uploadStartedAt; - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); - } - - const responseParseStartedAt = readSyncTimingNow(); - const uploadResult = await response.json().catch(() => ({})); - const responseParseMs = readSyncTimingNow() - responseParseStartedAt; return { filename, path: String(uploadResult?.path || `/user/files/${filename}`), + backend: String(uploadResult?.backend || ""), timings: finalizeSyncTimings( { serializeMs, uploadMs, - responseParseMs, + responseParseMs: 0, }, writeStartedAt, ), @@ -1890,7 +2306,6 @@ async function readRemoteSnapshot(chatId, options = {}) { }; } - const fetchImpl = getFetch(options); const resolveStartedAt = readSyncTimingNow(); const candidateFilenames = await resolveSyncFilenameCandidates( normalizedChatId, @@ -1902,20 +2317,15 @@ async function readRemoteSnapshot(chatId, options = {}) { let parseMs = 0; let chunkReadMs = 0; let normalizeMs = 0; + let authorityBlobMs = 0; for (const filename of candidateFilenames) { - const cacheBust = `t=${Date.now()}`; - const url = `/user/files/${encodeURIComponent(filename)}?${cacheBust}`; - - let response; - try { - const networkStartedAt = readSyncTimingNow(); - response = await fetchImpl(url, { - method: "GET", - cache: "no-store", - }); - networkMs += readSyncTimingNow() - networkStartedAt; - } catch (error) { + const result = await readRemoteJsonFileResult(filename, options); + networkMs += Number(result.timings?.networkMs || 0); + parseMs += Number(result.timings?.parseMs || 0); + authorityBlobMs += Number(result.timings?.authorityBlobMs || 0); + if (result.reason === "network-error") { + const error = result.error || new Error("network-error"); console.warn("[ST-BME] 读取远端同步文件失败:", error); return { exists: false, @@ -1924,39 +2334,36 @@ async function readRemoteSnapshot(chatId, options = {}) { snapshot: null, error, timings: finalizeSyncTimings( - { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs }, + { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs }, readStartedAt, ), }; } - if (response.status === 404) { + if (result.status === 404) { lastNotFoundFilename = filename; continue; } - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - const error = new Error(errorText || `HTTP ${response.status}`); + if (!result.ok) { + const error = result.error || new Error(result.reason || "remote-read-error"); console.warn("[ST-BME] 读取远端同步文件失败:", error); return { exists: false, - status: "http-error", + status: result.reason || "http-error", filename, snapshot: null, error, - statusCode: response.status, + statusCode: result.status, timings: finalizeSyncTimings( - { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs }, + { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs }, readStartedAt, ), }; } try { - const parseStartedAt = readSyncTimingNow(); - const remotePayload = await response.json(); - parseMs += readSyncTimingNow() - parseStartedAt; + const remotePayload = result.payload; let snapshot = null; if (Number(remotePayload?.formatVersion || 0) === BME_REMOTE_SYNC_FORMAT_VERSION_V2) { const manifestResult = await readRemoteSnapshotV2Manifest( @@ -1982,7 +2389,7 @@ async function readRemoteSnapshot(chatId, options = {}) { filename, snapshot, timings: finalizeSyncTimings( - { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs }, + { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs }, readStartedAt, ), }; @@ -1995,7 +2402,7 @@ async function readRemoteSnapshot(chatId, options = {}) { snapshot: null, error, timings: finalizeSyncTimings( - { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs }, + { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs }, readStartedAt, ), }; @@ -2008,29 +2415,21 @@ async function readRemoteSnapshot(chatId, options = {}) { filename: lastNotFoundFilename, snapshot: null, timings: finalizeSyncTimings( - { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs }, + { resolveCandidatesMs, networkMs, parseMs, chunkReadMs, normalizeMs, authorityBlobMs }, readStartedAt, ), }; } async function readRemoteJsonFile(filename, options = {}) { - const fetchImpl = getFetch(options); - const response = await fetchImpl( - `/user/files/${encodeURIComponent(filename)}?t=${Date.now()}`, - { - method: "GET", - cache: "no-store", - }, - ); - if (response.status === 404) { + const result = await readRemoteJsonFileResult(filename, options); + if (result.status === 404) { throw new Error("remote-chunk-not-found"); } - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); + if (!result.ok) { + throw result.error || new Error(result.reason || `HTTP ${result.status || "unknown"}`); } - return await response.json(); + return result.payload; } async function readRemoteSnapshotV2Manifest(manifest = {}, chatId = "", options = {}) { @@ -2107,7 +2506,6 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) { const normalizedChatId = normalizeChatId(chatId); const normalizedSnapshot = normalizeSyncSnapshot(snapshot, normalizedChatId); const filename = await resolveSyncFilename(normalizedChatId, options); - const fetchImpl = getFetch(options); const envelopeBuildStartedAt = readSyncTimingNow(); const syncEnvelope = buildRemoteSyncEnvelopeV2( normalizedSnapshot, @@ -2115,10 +2513,6 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) { filename, ); const envelopeBuildMs = readSyncTimingNow() - envelopeBuildStartedAt; - const requestHeaders = { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }; let chunkSerializeMs = 0; let chunkUploadMs = 0; for (const chunk of syncEnvelope.chunks) { @@ -2126,45 +2520,20 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) { const chunkPayload = JSON.stringify(chunk.payload, null, 2); chunkSerializeMs += readSyncTimingNow() - serializeStartedAt; const uploadStartedAt = readSyncTimingNow(); - const chunkResponse = await fetchImpl("/api/files/upload", { - method: "POST", - headers: requestHeaders, - body: JSON.stringify({ - name: chunk.filename, - data: encodeBase64Utf8(chunkPayload), - }), - }); + await writeRemoteJsonFile(chunk.filename, chunkPayload, options); chunkUploadMs += readSyncTimingNow() - uploadStartedAt; - if (!chunkResponse.ok) { - const errorText = await chunkResponse.text().catch(() => chunkResponse.statusText); - throw new Error(errorText || `HTTP ${chunkResponse.status}`); - } } const manifestSerializeStartedAt = readSyncTimingNow(); const manifestPayload = JSON.stringify(syncEnvelope.manifest, null, 2); const manifestSerializeMs = readSyncTimingNow() - manifestSerializeStartedAt; const manifestUploadStartedAt = readSyncTimingNow(); - const response = await fetchImpl("/api/files/upload", { - method: "POST", - headers: requestHeaders, - body: JSON.stringify({ - name: filename, - data: encodeBase64Utf8(manifestPayload), - }), - }); + const uploadResult = await writeRemoteJsonFile(filename, manifestPayload, options); const manifestUploadMs = readSyncTimingNow() - manifestUploadStartedAt; - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); - } - - const responseParseStartedAt = readSyncTimingNow(); - const uploadResult = await response.json().catch(() => ({})); - const responseParseMs = readSyncTimingNow() - responseParseStartedAt; return { filename, path: String(uploadResult?.path || ""), + backend: String(uploadResult?.backend || ""), payload: syncEnvelope.manifest, timings: finalizeSyncTimings( { @@ -2173,7 +2542,7 @@ async function writeSnapshotToRemote(snapshot, chatId, options = {}) { chunkUploadMs, manifestSerializeMs, manifestUploadMs, - responseParseMs, + responseParseMs: 0, }, writeStartedAt, ), @@ -2610,24 +2979,9 @@ export async function deleteServerBackup(chatId, options = {}) { const serverPath = targetCandidate.serverPath || normalizeSelectedBackupServerPath("", filename); - const fetchImpl = getFetch(options); try { - const response = await fetchImpl("/api/files/delete", { - method: "POST", - headers: { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }, - body: JSON.stringify({ - path: serverPath, - }), - }); - - if (!response.ok && response.status !== 404) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); - } + const deleteResult = await deleteRemoteJsonFile(serverPath || filename, options); try { const existingEntries = @@ -2655,6 +3009,7 @@ export async function deleteServerBackup(chatId, options = {}) { deleted: true, chatId: normalizedChatId, filename, + remoteDeleted: deleteResult.deleted === true, localMetaUpdated, }; } catch (manifestError) { @@ -3315,7 +3670,6 @@ export async function deleteRemoteSyncFile(chatId, options = {}) { } try { - const fetchImpl = getFetch(options); const filenames = await resolveSyncFilenameCandidates( normalizedChatId, options, @@ -3329,47 +3683,24 @@ export async function deleteRemoteSyncFile(chatId, options = {}) { for (const chunk of Array.isArray(manifestPayload?.chunks) ? manifestPayload.chunks : []) { const chunkFilename = String(chunk?.filename || "").trim(); if (!chunkFilename) continue; - await fetchImpl("/api/files/delete", { - method: "POST", - headers: { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }, - body: JSON.stringify({ - path: `/user/files/${chunkFilename}`, - }), - }).catch(() => null); + await deleteRemoteJsonFile(chunkFilename, options).catch(() => null); } } } catch { // best-effort chunk cleanup } - const response = await fetchImpl("/api/files/delete", { - method: "POST", - headers: { - ...getRequestHeadersSafe(options), - "Content-Type": "application/json", - }, - body: JSON.stringify({ - path: `/user/files/${filename}`, - }), - }); - - if (response.status === 404) { + const deleteResult = await deleteRemoteJsonFile(filename, options); + if (!deleteResult.deleted) { lastNotFoundFilename = filename; continue; } - if (!response.ok) { - const errorText = await response.text().catch(() => response.statusText); - throw new Error(errorText || `HTTP ${response.status}`); - } - sanitizedFilenameByChatId.delete(normalizedChatId); return { deleted: true, chatId: normalizedChatId, filename, + backend: String(deleteResult.backend || ""), }; } diff --git a/tests/authority-blob.mjs b/tests/authority-blob.mjs new file mode 100644 index 0000000..4fc88d5 --- /dev/null +++ b/tests/authority-blob.mjs @@ -0,0 +1,439 @@ +import assert from "node:assert/strict"; + +import { + createAuthorityBlobAdapter, + normalizeAuthorityBlobPath, + normalizeAuthorityBlobReadResult, +} from "../maintenance/authority-blob-adapter.js"; +import { + backupToServer, + download, + listServerBackups, + restoreFromServer, + upload, +} from "../sync/bme-sync.js"; + +class MemoryStorage { + constructor() { + this.map = new Map(); + } + + getItem(key) { + return this.map.has(key) ? this.map.get(key) : null; + } + + setItem(key, value) { + this.map.set(String(key), String(value)); + } +} + +class FakeDb { + constructor(chatId, snapshot = null) { + this.chatId = chatId; + this.snapshot = snapshot || { + meta: { + schemaVersion: 1, + chatId, + deviceId: "", + revision: 1, + lastModified: 10, + nodeCount: 1, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [{ id: `${chatId}-node`, updatedAt: 10 }], + edges: [], + tombstones: [], + state: { + lastProcessedFloor: 1, + extractionCount: 1, + }, + }; + this.meta = new Map([ + ["syncDirty", false], + ["syncDirtyReason", ""], + ["lastSyncedRevision", 0], + ]); + this.lastImportPayload = null; + } + + async exportSnapshot() { + return JSON.parse(JSON.stringify(this.snapshot)); + } + + async importSnapshot(snapshot) { + this.lastImportPayload = JSON.parse(JSON.stringify(snapshot)); + this.snapshot = JSON.parse(JSON.stringify(snapshot)); + } + + async getMeta(key, fallback = null) { + return this.meta.has(key) ? this.meta.get(key) : fallback; + } + + async patchMeta(record = {}) { + for (const [key, value] of Object.entries(record)) { + this.meta.set(key, value); + } + } + + async setMeta(key, value) { + this.meta.set(key, value); + } +} + +function createMockAuthorityBlobClient() { + const files = new Map(); + const calls = []; + return { + files, + calls, + async writeJson(payload = {}) { + calls.push(["writeJson", { ...payload }]); + files.set(String(payload.path || ""), JSON.parse(JSON.stringify(payload.payload))); + return { ok: true, path: payload.path, size: JSON.stringify(payload.payload).length }; + }, + async writeText(payload = {}) { + calls.push(["writeText", { ...payload }]); + files.set(String(payload.path || ""), String(payload.text ?? payload.data ?? "")); + return { ok: true, path: payload.path, size: String(payload.text ?? "").length }; + }, + async readJson(payload = {}) { + calls.push(["readJson", { ...payload }]); + const path = String(payload.path || ""); + if (!files.has(path)) return { exists: false, path }; + return { exists: true, path, payload: JSON.parse(JSON.stringify(files.get(path))) }; + }, + async delete(payload = {}) { + calls.push(["delete", { ...payload }]); + const path = String(payload.path || ""); + const existed = files.delete(path); + return { ok: true, deleted: existed, exists: existed, path }; + }, + }; +} + +function createMockFetch() { + const logs = { + getCalls: 0, + uploadCalls: 0, + deleteCalls: 0, + sanitizeCalls: 0, + }; + const response = (status, body) => ({ + ok: status >= 200 && status < 300, + status, + statusText: String(status), + async json() { + return JSON.parse(JSON.stringify(body)); + }, + async text() { + return typeof body === "string" ? body : JSON.stringify(body); + }, + }); + const fetch = async (url, options = {}) => { + const method = String(options.method || "GET").toUpperCase(); + if (url === "/api/files/sanitize-filename" && method === "POST") { + logs.sanitizeCalls += 1; + const body = JSON.parse(String(options.body || "{}")); + return response(200, { + fileName: String(body.fileName || "").replace(/[^A-Za-z0-9._~-]+/g, "_"), + }); + } + if (url === "/api/files/upload" && method === "POST") { + logs.uploadCalls += 1; + return response(500, "legacy upload should not be used"); + } + if (url === "/api/files/delete" && method === "POST") { + logs.deleteCalls += 1; + return response(404, "not found"); + } + if (String(url).startsWith("/user/files/") && method === "GET") { + logs.getCalls += 1; + return response(404, "not found"); + } + return response(404, "unsupported route"); + }; + return { fetch, logs }; +} + +function createLegacyFileFetch() { + const files = new Map(); + const logs = { + getCalls: 0, + uploadCalls: 0, + deleteCalls: 0, + sanitizeCalls: 0, + }; + const response = (status, body) => ({ + ok: status >= 200 && status < 300, + status, + statusText: String(status), + async json() { + return typeof body === "string" ? JSON.parse(body) : JSON.parse(JSON.stringify(body)); + }, + async text() { + return typeof body === "string" ? body : JSON.stringify(body); + }, + }); + const decodeUploadData = (value = "") => + Buffer.from(String(value || ""), "base64").toString("utf8"); + const fetch = async (url, options = {}) => { + const method = String(options.method || "GET").toUpperCase(); + if (url === "/api/files/sanitize-filename" && method === "POST") { + logs.sanitizeCalls += 1; + const body = JSON.parse(String(options.body || "{}")); + return response(200, { + fileName: String(body.fileName || "").replace(/[^A-Za-z0-9._~-]+/g, "_"), + }); + } + if (url === "/api/files/upload" && method === "POST") { + logs.uploadCalls += 1; + const body = JSON.parse(String(options.body || "{}")); + const name = String(body.name || ""); + files.set(name, decodeUploadData(body.data)); + return response(200, { path: `/user/files/${name}` }); + } + if (url === "/api/files/delete" && method === "POST") { + logs.deleteCalls += 1; + const body = JSON.parse(String(options.body || "{}")); + const name = decodeURIComponent(String(body.path || "").split("/").pop() || ""); + const deleted = files.delete(name); + return response(deleted ? 200 : 404, deleted ? { deleted: true } : "not found"); + } + if (String(url).startsWith("/user/files/") && method === "GET") { + logs.getCalls += 1; + const path = String(url).split("?")[0]; + const name = decodeURIComponent(path.slice("/user/files/".length)); + if (!files.has(name)) return response(404, "not found"); + return response(200, JSON.parse(files.get(name))); + } + return response(404, "unsupported route"); + }; + return { fetch, files, logs }; +} + +function buildRuntimeOptions({ dbByChatId, fetch, blobClient, onAuthorityBlobEvent = null }) { + return { + fetch, + blobClient, + authorityBlobEnabled: true, + authorityBlobFailOpen: true, + getDb: async (chatId) => { + const db = dbByChatId.get(chatId); + if (!db) throw new Error(`missing db: ${chatId}`); + return db; + }, + getSafetyDb: async (chatId) => new FakeDb(`__restore_safety__${chatId}`), + getRequestHeaders: () => ({ "X-Test": "1" }), + onAuthorityBlobEvent, + }; +} + +function createFailingAuthorityBlobClient() { + const calls = []; + const fail = async (method, payload = {}) => { + calls.push([method, { ...payload }]); + throw new Error("blob unavailable"); + }; + return { + calls, + writeJson: (payload) => fail("writeJson", payload), + writeText: (payload) => fail("writeText", payload), + readJson: (payload) => fail("readJson", payload), + delete: (payload) => fail("delete", payload), + }; +} + +async function testAdapterBasics() { + const client = createMockAuthorityBlobClient(); + const adapter = createAuthorityBlobAdapter({}, { blobClient: client }); + assert.equal(normalizeAuthorityBlobPath("/user/files/demo.json"), "user/files/demo.json"); + assert.equal( + normalizeAuthorityBlobReadResult({ data: JSON.stringify({ ok: true }) }, "a.json").payload.ok, + true, + ); + + const writeResult = await adapter.writeJson("/user/files/demo.json", { hello: "world" }); + assert.equal(writeResult.ok, true); + const readResult = await adapter.readJson("user/files/demo.json"); + assert.equal(readResult.exists, true); + assert.deepEqual(readResult.payload, { hello: "world" }); + const deleteResult = await adapter.delete("user/files/demo.json"); + assert.equal(deleteResult.deleted, true); +} + +async function testAuthorityBlobFailOpenFallsBackToUserFiles() { + globalThis.localStorage = new MemoryStorage(); + const blobClient = createFailingAuthorityBlobClient(); + const { fetch, logs } = createLegacyFileFetch(); + const dbByChatId = new Map(); + const db = new FakeDb("blob-fallback", { + meta: { + schemaVersion: 1, + chatId: "blob-fallback", + deviceId: "", + revision: 9, + lastModified: 90, + nodeCount: 1, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [{ id: "fallback-node", updatedAt: 90 }], + edges: [], + tombstones: [], + state: { lastProcessedFloor: 6, extractionCount: 3 }, + }); + dbByChatId.set("blob-fallback", db); + const events = []; + const runtime = buildRuntimeOptions({ + dbByChatId, + fetch, + blobClient, + onAuthorityBlobEvent: (event) => events.push(event), + }); + + const backupResult = await backupToServer("blob-fallback", runtime); + assert.equal(backupResult.backedUp, true); + assert.ok(logs.uploadCalls > 0); + assert.ok(blobClient.calls.some(([method]) => method === "writeJson")); + assert.ok(events.some((event) => event.reason === "authority-blob-error")); + + db.snapshot = { + meta: { + schemaVersion: 1, + chatId: "blob-fallback", + deviceId: "", + revision: 1, + lastModified: 10, + nodeCount: 0, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [], + edges: [], + tombstones: [], + state: { lastProcessedFloor: -1, extractionCount: 0 }, + }; + const restoreResult = await restoreFromServer("blob-fallback", runtime); + assert.equal(restoreResult.restored, true); + assert.equal(db.snapshot.nodes[0].id, "fallback-node"); + assert.ok(logs.getCalls > 0); +} + +async function testBackupRestoreUsesAuthorityBlob() { + globalThis.localStorage = new MemoryStorage(); + const blobClient = createMockAuthorityBlobClient(); + const { fetch, logs } = createMockFetch(); + const dbByChatId = new Map(); + const db = new FakeDb("blob-backup", { + meta: { + schemaVersion: 1, + chatId: "blob-backup", + deviceId: "", + revision: 7, + lastModified: 70, + nodeCount: 1, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [{ id: "blob-node", updatedAt: 70 }], + edges: [], + tombstones: [], + state: { lastProcessedFloor: 3, extractionCount: 2 }, + }); + dbByChatId.set("blob-backup", db); + const events = []; + const runtime = buildRuntimeOptions({ + dbByChatId, + fetch, + blobClient, + onAuthorityBlobEvent: (event) => events.push(event), + }); + + const backupResult = await backupToServer("blob-backup", runtime); + assert.equal(backupResult.backedUp, true); + assert.equal(logs.uploadCalls, 0); + assert.equal(blobClient.files.has("user/files/ST-BME_BackupManifest.json"), true); + assert.equal(blobClient.files.has(`user/files/${backupResult.filename}`), true); + + const manifest = await listServerBackups(runtime); + assert.equal(manifest.entries.length, 1); + assert.equal(manifest.entries[0].chatId, "blob-backup"); + + db.snapshot = { + meta: { + schemaVersion: 1, + chatId: "blob-backup", + deviceId: "", + revision: 1, + lastModified: 10, + nodeCount: 0, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [], + edges: [], + tombstones: [], + state: { lastProcessedFloor: -1, extractionCount: 0 }, + }; + const restoreResult = await restoreFromServer("blob-backup", runtime); + assert.equal(restoreResult.restored, true); + assert.equal(db.snapshot.nodes[0].id, "blob-node"); + assert.equal(events.some((event) => event.backend === "authority-blob"), true); +} + +async function testSyncUploadDownloadUsesAuthorityBlob() { + globalThis.localStorage = new MemoryStorage(); + const blobClient = createMockAuthorityBlobClient(); + const { fetch, logs } = createMockFetch(); + const dbByChatId = new Map(); + const db = new FakeDb("blob-sync", { + meta: { + schemaVersion: 1, + chatId: "blob-sync", + deviceId: "", + revision: 5, + lastModified: 50, + nodeCount: 1, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [{ id: "sync-blob-node", updatedAt: 50 }], + edges: [], + tombstones: [], + state: { lastProcessedFloor: 4, extractionCount: 1 }, + }); + dbByChatId.set("blob-sync", db); + const runtime = buildRuntimeOptions({ dbByChatId, fetch, blobClient }); + + const uploadResult = await upload("blob-sync", runtime); + assert.equal(uploadResult.uploaded, true); + assert.equal(logs.uploadCalls, 0); + assert.equal(blobClient.files.has("user/files/ST-BME_sync_blob-sync.json"), true); + + db.snapshot = { + meta: { + schemaVersion: 1, + chatId: "blob-sync", + deviceId: "", + revision: 1, + lastModified: 10, + nodeCount: 0, + edgeCount: 0, + tombstoneCount: 0, + }, + nodes: [], + edges: [], + tombstones: [], + state: { lastProcessedFloor: -1, extractionCount: 0 }, + }; + const downloadResult = await download("blob-sync", runtime); + assert.equal(downloadResult.downloaded, true); + assert.equal(db.snapshot.nodes[0].id, "sync-blob-node"); +} + +await testAdapterBasics(); +await testAuthorityBlobFailOpenFallsBackToUserFiles(); +await testBackupRestoreUsesAuthorityBlob(); +await testSyncUploadDownloadUsesAuthorityBlob(); +console.log("authority-blob tests passed"); diff --git a/ui/ui-status.js b/ui/ui-status.js index 6f94c43..4b24c7c 100644 --- a/ui/ui-status.js +++ b/ui/ui-status.js @@ -137,6 +137,7 @@ export function createGraphPersistenceState() { authorityServerPrimaryReady: false, authorityStoragePrimaryReady: false, authorityTriviumPrimaryReady: false, + authorityBlobReady: false, authorityBrowserCacheMode: "minimal", authorityOfflineQueueBytes: 0, authorityOfflineQueueItems: 0, @@ -155,6 +156,14 @@ export function createGraphPersistenceState() { authorityLastJobProgress: 0, authorityLastJobError: "", authorityLastJobUpdatedAt: "", + authorityBlobState: "idle", + authorityLastBlobEvent: null, + authorityLastBlobAction: "", + authorityLastBlobBackend: "", + authorityLastBlobPath: "", + authorityLastBlobReason: "", + authorityLastBlobError: "", + authorityLastBlobUpdatedAt: "", localStoreFormatVersion: 1, localStoreMigrationState: "idle", opfsWriteLockState: {