diff --git a/maintenance/authority-blob-adapter.js b/maintenance/authority-blob-adapter.js index 1986dd8..2a34f90 100644 --- a/maintenance/authority-blob-adapter.js +++ b/maintenance/authority-blob-adapter.js @@ -1,6 +1,7 @@ import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; +import { AuthorityHttpClient } from "../runtime/authority-http-client.js"; -export const AUTHORITY_BLOB_ENDPOINT = "/v1/blob"; +export const AUTHORITY_BLOB_ENDPOINT = "/fs/private"; function toPlainData(value, fallbackValue = null) { if (value == null) return fallbackValue; @@ -97,7 +98,8 @@ function normalizeBlobRecordSource(input = null) { export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") { const source = normalizeBlobRecordSource(input); - const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); + const entry = source.entry && typeof source.entry === "object" ? source.entry : null; + const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath); const missing = source.exists === false || source.found === false || @@ -118,21 +120,22 @@ export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "" payload: normalizeBlobPayload(input), contentType: String(source.contentType || source.type || "application/json"), etag: String(source.etag || source.hash || ""), - updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", + updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "", raw: toPlainData(input, input), }; } export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") { const source = normalizeBlobRecordSource(input); - const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); + const entry = source.entry && typeof source.entry === "object" ? source.entry : null; + const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath); return { ok: input == null ? true : source.ok !== false && source.error == null, path, url: String(source.url || source.href || ""), - size: normalizeInteger(source.size || source.bytes, 0, 0), + size: normalizeInteger(source.size || source.bytes || entry?.sizeBytes, 0, 0), etag: String(source.etag || source.hash || ""), - updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", + updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "", raw: toPlainData(input, input), }; } @@ -170,49 +173,56 @@ export function normalizeAuthorityBlobConfig(settings = {}, overrides = {}) { export class AuthorityBlobHttpClient { constructor(options = {}) { - this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); - this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); - this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + this.http = new AuthorityHttpClient({ + ...options, + baseUrl: normalizeAuthorityBaseUrl(options.baseUrl), + }); } - async request(action, payload = {}) { - if (typeof this.fetchImpl !== "function") { - throw new Error("Authority Blob fetch unavailable"); - } - const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_BLOB_ENDPOINT}`, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - ...(this.headerProvider ? this.headerProvider() || {} : {}), - }, - body: JSON.stringify({ action, ...payload }), + async request(path, payload = {}, options = {}) { + return await this.http.requestJson(path, { + method: options.method || "POST", + body: payload, + session: true, + signal: options.signal, }); - if (!response?.ok) { - const text = await response?.text?.().catch(() => ""); - throw new Error(text || `Authority Blob HTTP ${response?.status || "unknown"}`); - } - return await response.json().catch(() => ({})); } async writeJson(payload = {}) { - return await this.request("writeJson", payload); + return await this.writeText({ + ...payload, + contentType: payload.contentType || "application/json", + text: JSON.stringify(toPlainData(payload.payload ?? payload.data, payload.payload ?? payload.data)), + }); } async writeText(payload = {}) { - return await this.request("writeText", payload); + return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/write-file`, { + path: normalizeAuthorityBlobPath(payload.path || payload.name), + content: String(payload.text ?? payload.data ?? payload.content ?? ""), + encoding: "utf8", + createParents: true, + }, { signal: payload.signal }); } async readJson(payload = {}) { - return await this.request("readJson", payload); + return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/read-file`, { + path: normalizeAuthorityBlobPath(payload.path || payload.name), + encoding: "utf8", + }, { signal: payload.signal }); } async delete(payload = {}) { - return await this.request("delete", payload); + return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/delete`, { + path: normalizeAuthorityBlobPath(payload.path || payload.name), + recursive: false, + }, { signal: payload.signal }); } async stat(payload = {}) { - return await this.request("stat", payload); + return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/stat`, { + path: normalizeAuthorityBlobPath(payload.path || payload.name), + }, { signal: payload.signal }); } } @@ -249,6 +259,10 @@ function throwIfAborted(signal) { } } +function isMissingAuthorityBlobError(error) { + return Number(error?.status || 0) === 404; +} + export class AuthorityBlobAdapter { constructor(config = {}, options = {}) { this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {}); @@ -291,36 +305,57 @@ export class AuthorityBlobAdapter { throwIfAborted(options.signal); const normalizedPath = normalizeAuthorityBlobPath(path); if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); - const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", { - namespace: options.namespace || this.config.namespace, - path: normalizedPath, - name: normalizedPath, - }); - return normalizeAuthorityBlobReadResult(result, normalizedPath); + try { + const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobReadResult(result, normalizedPath); + } catch (error) { + if (isMissingAuthorityBlobError(error)) { + return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath); + } + throw error; + } } async delete(path, options = {}) { throwIfAborted(options.signal); const normalizedPath = normalizeAuthorityBlobPath(path); if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, ""); - const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", { - namespace: options.namespace || this.config.namespace, - path: normalizedPath, - name: normalizedPath, - }); - return normalizeAuthorityBlobDeleteResult(result, normalizedPath); + try { + const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobDeleteResult(result, normalizedPath); + } catch (error) { + if (isMissingAuthorityBlobError(error)) { + return normalizeAuthorityBlobDeleteResult({ exists: false, status: 404 }, normalizedPath); + } + throw error; + } } async stat(path, options = {}) { throwIfAborted(options.signal); const normalizedPath = normalizeAuthorityBlobPath(path); if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); - const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", { - namespace: options.namespace || this.config.namespace, - path: normalizedPath, - name: normalizedPath, - }); - return normalizeAuthorityBlobReadResult(result, normalizedPath); + try { + const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", { + namespace: options.namespace || this.config.namespace, + path: normalizedPath, + name: normalizedPath, + }); + return normalizeAuthorityBlobReadResult(result, normalizedPath); + } catch (error) { + if (isMissingAuthorityBlobError(error)) { + return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath); + } + throw error; + } } } diff --git a/maintenance/authority-job-adapter.js b/maintenance/authority-job-adapter.js index 76a572e..3823bb2 100644 --- a/maintenance/authority-job-adapter.js +++ b/maintenance/authority-job-adapter.js @@ -1,6 +1,7 @@ import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; +import { AuthorityHttpClient } from "../runtime/authority-http-client.js"; -export const AUTHORITY_JOB_ENDPOINT = "/v1/jobs"; +export const AUTHORITY_JOB_ENDPOINT = "/jobs"; export const AUTHORITY_JOB_STATUS_TERMINAL = new Set([ "completed", "succeeded", @@ -158,8 +159,8 @@ export function normalizeAuthorityJobList(payload = null) { const jobs = readJobRows(payload).map(normalizeAuthorityJobRecord).filter((job) => job.id); return { jobs, - nextCursor: String(source.nextCursor || source.next_cursor || source.cursor?.next || ""), - hasMore: Boolean(source.hasMore || source.has_more || source.cursor?.hasMore), + nextCursor: String(source.nextCursor || source.next_cursor || source.page?.nextCursor || source.cursor?.next || ""), + hasMore: Boolean(source.hasMore || source.has_more || source.page?.hasMore || source.cursor?.hasMore), raw: toPlainData(payload, payload), }; } @@ -251,49 +252,68 @@ export function normalizeAuthorityJobConfig(settings = {}, overrides = {}) { export class AuthorityJobHttpClient { constructor(options = {}) { - this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); - this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); - this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + this.http = new AuthorityHttpClient({ + ...options, + baseUrl: normalizeAuthorityBaseUrl(options.baseUrl), + }); } - async request(action, payload = {}) { - if (typeof this.fetchImpl !== "function") { - throw new Error("Authority Jobs fetch unavailable"); - } - const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_JOB_ENDPOINT}`, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - ...(this.headerProvider ? this.headerProvider() || {} : {}), - }, - body: JSON.stringify({ action, ...payload }), + async request(path, payload = {}, options = {}) { + return await this.http.requestJson(path, { + method: options.method || "POST", + body: payload, + session: true, + signal: options.signal, }); - if (!response?.ok) { - const text = await response?.text?.().catch(() => ""); - throw new Error(text || `Authority Jobs HTTP ${response?.status || "unknown"}`); - } - return await response.json().catch(() => ({})); } async submit(payload = {}) { - return await this.request("submit", payload); + return await this.request(`${AUTHORITY_JOB_ENDPOINT}/create`, { + type: String(payload.type || payload.kind || "").trim(), + payload: toPlainData(payload.payload, payload.payload), + ...(payload.timeoutMs != null ? { timeoutMs: normalizeInteger(payload.timeoutMs, 0, 0, 3600000) } : {}), + ...(payload.idempotencyKey ? { idempotencyKey: String(payload.idempotencyKey) } : {}), + ...(payload.maxAttempts != null ? { maxAttempts: normalizeInteger(payload.maxAttempts, 1, 1, 1000) } : {}), + }); } async listPage(payload = {}) { - return await this.request("listPage", payload); + return await this.request(`${AUTHORITY_JOB_ENDPOINT}/list`, { + page: { + ...(payload.cursor ? { cursor: String(payload.cursor) } : {}), + limit: normalizeInteger(payload.limit, 20, 1, 100), + }, + ...(payload.filter && typeof payload.filter === "object" && !Array.isArray(payload.filter) && Object.keys(payload.filter).length > 0 + ? { filter: toPlainData(payload.filter, {}) } + : {}), + }); } - async waitForCompletion(payload = {}) { - return await this.request("waitForCompletion", payload); + async get(payload = {}) { + const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id)); + if (!id) { + throw new Error("Authority Jobs get requires job id"); + } + return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}`, undefined, { + method: "GET", + signal: payload.signal, + }); } async requeue(payload = {}) { - return await this.request("requeue", payload); + const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id)); + if (!id) { + throw new Error("Authority Jobs requeue requires job id"); + } + return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/requeue`, {}, { signal: payload.signal }); } async cancel(payload = {}) { - return await this.request("cancel", payload); + const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id)); + if (!id) { + throw new Error("Authority Jobs cancel requires job id"); + } + return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/cancel`, {}, { signal: payload.signal }); } } diff --git a/retrieval/authority-candidate-provider.js b/retrieval/authority-candidate-provider.js index 1823e71..05a4133 100644 --- a/retrieval/authority-candidate-provider.js +++ b/retrieval/authority-candidate-provider.js @@ -9,6 +9,7 @@ import { queryAuthorityTriviumNeighbors, searchAuthorityTriviumNodes, } from "../vector/authority-vector-primary-adapter.js"; +import { embedText } from "../vector/embedding.js"; function nowMs() { if (typeof performance?.now === "function") { @@ -247,6 +248,11 @@ export async function resolveAuthorityRecallCandidates({ const searchStartedAt = nowMs(); for (const queryEntry of queryPlan.queries) { try { + const queryVec = await embedText(queryEntry.text, embeddingConfig, { signal, isQuery: true }); + if (!queryVec) { + diagnostics.fallbackReason = diagnostics.fallbackReason || "authority-candidate-query-embed-empty"; + continue; + } const searchResults = await searchAuthorityTriviumNodes( graph, queryEntry.text, @@ -257,6 +263,7 @@ export async function resolveAuthorityRecallCandidates({ chatId, topK: limit, candidateIds: filteredIds.length > 0 ? filteredIds : undefined, + queryVector: Array.from(queryVec), signal, }, ); diff --git a/retrieval/retrieval-enhancer.js b/retrieval/retrieval-enhancer.js index 4e34356..7cdbe8b 100644 --- a/retrieval/retrieval-enhancer.js +++ b/retrieval/retrieval-enhancer.js @@ -484,7 +484,7 @@ export async function runResidualRecall({ }; } - const queryVec = await embedText(queryText, embeddingConfig, { signal }); + const queryVec = await embedText(queryText, embeddingConfig, { signal, isQuery: true }); if (!queryVec || queryVec.length === 0) { return { triggered: false, diff --git a/runtime/authority-capabilities.js b/runtime/authority-capabilities.js index bdeee38..bd4e23b 100644 --- a/runtime/authority-capabilities.js +++ b/runtime/authority-capabilities.js @@ -1,11 +1,11 @@ const DEFAULT_AUTHORITY_BASE_URL = "/api/plugins/authority"; const DEFAULT_AUTHORITY_PROBE_INTERVAL_MS = 60000; -const SQL_FEATURES = ["sql", "sql.query", "sql.page", "sql.pageall", "querysql"]; -const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.write", "sql.transaction"]; -const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert"]; -const JOB_FEATURES = ["jobs", "jobs.list", "jobs.wait", "events", "sse"]; -const BLOB_FEATURES = ["blob", "blob.write", "privatefiles", "private.files", "files.private"]; +const SQL_FEATURES = ["sql", "sql.query", "sql.querypage", "sql.page", "sql.pageall", "querysql"]; +const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.exec", "sql.write", "sql.transaction"]; +const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert", "trivium.upsert", "trivium.bulkmutations"]; +const JOB_FEATURES = ["jobs", "jobs.background", "jobs.list", "jobs.wait", "diagnostics.jobspage", "events", "sse"]; +const BLOB_FEATURES = ["blob", "blob.write", "storage.blob", "transfers.blob", "transfers.fs", "fs.private", "privatefiles", "private.files", "files.private"]; function toBoolean(value, fallback = false) { if (typeof value === "boolean") return value; @@ -89,6 +89,189 @@ function readNowMs() { return Date.now(); } +function clonePlain(value, fallbackValue = null) { + if (value == null) return fallbackValue; + if (typeof globalThis.structuredClone === "function") { + try { + return globalThis.structuredClone(value); + } catch { + } + } + try { + return JSON.parse(JSON.stringify(value)); + } catch { + return fallbackValue; + } +} + +function normalizeHeaderName(name = "") { + return String(name || "").trim().toLowerCase(); +} + +function buildDefaultSessionInitConfig(source = {}) { + const config = source && typeof source === "object" && !Array.isArray(source) ? source : {}; + return { + extensionId: String(config.extensionId || "third-party/st-bme"), + displayName: String(config.displayName || "ST-BME"), + version: String(config.version || "0.0.0"), + installType: String(config.installType || "local"), + declaredPermissions: clonePlain(config.declaredPermissions, null) || { + storage: { kv: true, blob: true }, + fs: { private: true }, + sql: { private: true }, + trivium: { private: true }, + jobs: { background: true }, + events: { channels: true }, + }, + ...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}), + }; +} + +function withJsonHeaders(headers = {}) { + return { + Accept: "application/json", + "Content-Type": "application/json", + ...(headers || {}), + }; +} + +async function readResponsePayload(response = null) { + if (!response) return {}; + if (typeof response.json === "function") { + try { + return await response.json(); + } catch { + } + } + if (typeof response.text === "function") { + try { + return { error: await response.text() }; + } catch { + return {}; + } + } + return {}; +} + +function readPayloadMessage(payload = {}, fallback = "") { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback; + return String(payload.error || payload.message || payload.reason || fallback || ""); +} + +function buildAuthorityPermissionEvaluateRequests(settings = {}, readiness = {}, options = {}) { + const requests = []; + const sqlTarget = String(options.sqlTarget || settings.sqlTarget || "default"); + const triviumTarget = String(options.triviumTarget || settings.triviumTarget || "st_bme_vectors"); + const jobTarget = String(options.jobTarget || settings.jobTarget || "delay"); + if (readiness.sql || readiness.sqlMutation) { + requests.push({ resource: "sql.private", target: sqlTarget, reason: `Probe SQL capability for ${sqlTarget}` }); + } + if (readiness.trivium) { + requests.push({ resource: "trivium.private", target: triviumTarget, reason: `Probe Trivium capability for ${triviumTarget}` }); + } + if (readiness.blob) { + requests.push({ resource: "fs.private", reason: "Probe private file capability for Authority Blob adapter" }); + } + if (readiness.jobs) { + requests.push({ resource: "jobs.background", target: jobTarget, reason: `Probe Jobs capability for ${jobTarget}` }); + } + return requests; +} + +async function verifyAuthorityDataPlane(baseUrl, fetchImpl, headers, settings = {}, readiness = {}, options = {}) { + const initHeaders = withJsonHeaders(headers); + const initResponse = await fetchImpl(`${baseUrl}/session/init`, { + method: "POST", + headers: initHeaders, + body: JSON.stringify(buildDefaultSessionInitConfig(options.sessionInitConfig || settings)), + }); + const initStatus = Number(initResponse?.status || 0); + const initPayload = await readResponsePayload(initResponse); + if (!initResponse?.ok) { + return { + sessionReady: false, + permissionReady: false, + reason: initStatus === 401 || initStatus === 403 ? "session-init-denied" : "session-init-failed", + lastError: readPayloadMessage(initPayload, `HTTP ${initStatus || "unknown"}`), + status: initStatus, + }; + } + + const sessionToken = String(initPayload?.sessionToken || initPayload?.token || ""); + if (!sessionToken) { + return { + sessionReady: false, + permissionReady: false, + reason: "session-token-missing", + lastError: "session token missing", + status: initStatus, + }; + } + + const sessionHeaders = { + ...withJsonHeaders(headers), + ...(Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === "x-authority-session-token") + ? {} + : { "x-authority-session-token": sessionToken }), + }; + const currentResponse = await fetchImpl(`${baseUrl}/session/current`, { + method: "GET", + headers: sessionHeaders, + }); + const currentStatus = Number(currentResponse?.status || 0); + const currentPayload = await readResponsePayload(currentResponse); + if (!currentResponse?.ok) { + return { + sessionReady: false, + permissionReady: false, + reason: currentStatus === 401 || currentStatus === 403 ? "session-invalid" : "session-current-failed", + lastError: readPayloadMessage(currentPayload, `HTTP ${currentStatus || "unknown"}`), + status: currentStatus, + }; + } + + const requests = buildAuthorityPermissionEvaluateRequests(settings, readiness, options); + if (!requests.length) { + return { + sessionReady: true, + permissionReady: true, + reason: "ok", + lastError: "", + status: currentStatus, + }; + } + + const permissionResponse = await fetchImpl(`${baseUrl}/permissions/evaluate-batch`, { + method: "POST", + headers: sessionHeaders, + body: JSON.stringify({ requests }), + }); + const permissionStatus = Number(permissionResponse?.status || 0); + const permissionPayload = await readResponsePayload(permissionResponse); + if (!permissionResponse?.ok) { + return { + sessionReady: true, + permissionReady: false, + reason: permissionStatus === 401 || permissionStatus === 403 ? "permission-denied" : "permission-evaluate-failed", + lastError: readPayloadMessage(permissionPayload, `HTTP ${permissionStatus || "unknown"}`), + status: permissionStatus, + }; + } + + const results = Array.isArray(permissionPayload?.results) ? permissionPayload.results : []; + const permissionReady = results.length === requests.length && results.every((result) => { + const decision = String(result?.decision || result?.grant?.status || "").trim().toLowerCase(); + return decision === "granted"; + }); + return { + sessionReady: true, + permissionReady, + reason: permissionReady ? "ok" : "permission-not-ready", + lastError: permissionReady ? "" : "required Authority permissions are not granted", + status: permissionStatus || currentStatus, + }; +} + export function normalizeAuthorityBaseUrl(baseUrl = DEFAULT_AUTHORITY_BASE_URL) { const normalized = String(baseUrl || DEFAULT_AUTHORITY_BASE_URL).trim() || DEFAULT_AUTHORITY_BASE_URL; return normalized.replace(/\/+$/, ""); @@ -116,12 +299,7 @@ export function normalizeAuthoritySettings(settings = {}) { export function buildAuthorityProbeUrls(baseUrl = DEFAULT_AUTHORITY_BASE_URL) { const normalizedBaseUrl = normalizeAuthorityBaseUrl(baseUrl); - return [ - `${normalizedBaseUrl}/v1/diagnostics/probe`, - `${normalizedBaseUrl}/v1/probe`, - `${normalizedBaseUrl}/probe`, - normalizedBaseUrl, - ]; + return [`${normalizedBaseUrl}/probe`]; } export function collectAuthorityFeatures(payload = {}) { @@ -295,7 +473,7 @@ export async function probeAuthorityCapabilities(options = {}) { for (const endpoint of buildAuthorityProbeUrls(settings.baseUrl)) { const startedAt = readNowMs(); try { - const response = await fetchImpl(endpoint, { method: "GET", headers }); + const response = await fetchImpl(endpoint, { method: "POST", headers }); const finishedAt = readNowMs(); const status = Number(response?.status || 0); lastStatus = status; @@ -340,13 +518,43 @@ export async function probeAuthorityCapabilities(options = {}) { } catch { payload = {}; } - return normalizeAuthorityProbeResponse(payload, { + const features = collectAuthorityFeatures(payload); + const readiness = createFeatureReadiness(features); + const missingFeatures = collectMissingFeatures(readiness); + const healthy = payload?.healthy ?? payload?.ok ?? true; + let sessionReady = payload?.sessionReady ?? payload?.session?.ready ?? payload?.session?.active; + let permissionReady = payload?.permissionReady ?? payload?.permissions?.ready ?? payload?.authorized; + let reason = missingFeatures.length ? "missing-required-features" : "ok"; + let dataPlaneLastError = ""; + let dataPlaneStatus = status; + if (healthy) { + const verified = await verifyAuthorityDataPlane(settings.baseUrl, fetchImpl, headers, settings, readiness, options); + sessionReady = verified.sessionReady; + permissionReady = verified.permissionReady; + dataPlaneStatus = Number(verified.status || status || 0); + dataPlaneLastError = String(verified.lastError || ""); + if (verified.reason && verified.reason !== "ok") { + reason = verified.reason; + } + } + return normalizeAuthorityCapabilityState( + { + installed: true, + healthy: Boolean(healthy), + sessionReady: Boolean(sessionReady), + permissionReady: Boolean(permissionReady), + features: Array.from(features), + missingFeatures, + reason, + lastError: dataPlaneLastError, + endpoint, + status: dataPlaneStatus, + latencyMs: normalizeLatencyMs(startedAt, finishedAt), + lastProbeAt: nowMs, + updatedAt: new Date(nowMs).toISOString(), + }, settings, - endpoint, - status, - latencyMs: normalizeLatencyMs(startedAt, finishedAt), - nowMs, - }); + ); } catch (error) { lastError = error?.message || String(error); } diff --git a/runtime/authority-http-client.js b/runtime/authority-http-client.js new file mode 100644 index 0000000..4ac5aff --- /dev/null +++ b/runtime/authority-http-client.js @@ -0,0 +1,194 @@ +import { normalizeAuthorityBaseUrl } from "./authority-capabilities.js"; + +export const AUTHORITY_PROTOCOL_AUTO = "auto"; +export const AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06 = "server-plugin-v06"; +export const AUTHORITY_SESSION_HEADER = "x-authority-session-token"; + +function normalizeProtocol(value = AUTHORITY_PROTOCOL_AUTO) { + const normalized = String(value || AUTHORITY_PROTOCOL_AUTO).trim().toLowerCase(); + if (["v06", "v0.6", "server-plugin", AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06].includes(normalized)) { + return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06; + } + return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06; +} + +function clonePlain(value, fallbackValue = null) { + if (value == null) return fallbackValue; + if (typeof globalThis.structuredClone === "function") { + try { + return globalThis.structuredClone(value); + } catch { + } + } + try { + return JSON.parse(JSON.stringify(value)); + } catch { + return fallbackValue; + } +} + +function normalizeHeaderName(name = "") { + return String(name || "").trim().toLowerCase(); +} + +function hasSessionHeader(headers = {}) { + return Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === AUTHORITY_SESSION_HEADER); +} + +function buildDefaultSessionInitConfig(source = {}) { + const config = source && typeof source === "object" && !Array.isArray(source) ? source : {}; + return { + extensionId: String(config.extensionId || "third-party/st-bme"), + displayName: String(config.displayName || "ST-BME"), + version: String(config.version || "0.0.0"), + installType: String(config.installType || "local"), + declaredPermissions: clonePlain(config.declaredPermissions, null) || { + storage: { kv: true, blob: true }, + fs: { private: true }, + sql: { private: true }, + trivium: { private: true }, + jobs: { background: true }, + events: { channels: true }, + }, + ...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}), + }; +} + +function readPayloadErrorMessage(payload = null, fallback = "") { + if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback; + return String(payload.error || payload.message || payload.reason || fallback || ""); +} + +async function readResponsePayload(response = null) { + if (!response) return {}; + const contentType = String(response.headers?.get?.("content-type") || "").toLowerCase(); + if (contentType.includes("application/json") && typeof response.json === "function") { + try { + return await response.json(); + } catch { + return {}; + } + } + if (typeof response.json === "function") { + try { + return await response.json(); + } catch { + } + } + if (typeof response.text === "function") { + try { + return { error: await response.text() }; + } catch { + return {}; + } + } + return {}; +} + +export class AuthorityHttpError extends Error { + constructor(message, options = {}) { + super(message); + this.name = "AuthorityHttpError"; + this.status = Number(options.status || 0); + this.code = String(options.code || ""); + this.category = String(options.category || ""); + this.payload = clonePlain(options.payload, null); + this.path = String(options.path || ""); + this.protocol = String(options.protocol || ""); + } +} + +export class AuthorityHttpClient { + constructor(options = {}) { + this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); + this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); + this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + this.protocol = normalizeProtocol(options.protocol || options.authorityProtocol); + this.sessionToken = String(options.sessionToken || options.authoritySessionToken || ""); + this.sessionInitConfig = buildDefaultSessionInitConfig(options.sessionInitConfig || options.initConfig || options); + this.sessionPromise = null; + } + + async buildHeaders({ session = false } = {}) { + let provided = {}; + if (this.headerProvider) { + provided = await this.headerProvider() || {}; + } + const headers = { + Accept: "application/json", + "Content-Type": "application/json", + ...provided, + }; + if (session && this.sessionToken && !hasSessionHeader(headers)) { + headers[AUTHORITY_SESSION_HEADER] = this.sessionToken; + } + return headers; + } + + async ensureSession() { + if (this.sessionToken) return this.sessionToken; + if (!this.sessionPromise) { + this.sessionPromise = this.requestJson("/session/init", { + method: "POST", + body: this.sessionInitConfig, + session: false, + protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, + }).then((payload) => { + const token = String(payload?.sessionToken || payload?.token || ""); + if (!token) { + throw new AuthorityHttpError("Authority session init did not return a session token", { + status: 0, + path: "/session/init", + protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, + payload, + }); + } + this.sessionToken = token; + return token; + }).catch((error) => { + this.sessionPromise = null; + throw error; + }); + } + return await this.sessionPromise; + } + + async requestJson(path, options = {}) { + if (typeof this.fetchImpl !== "function") { + throw new AuthorityHttpError("Authority fetch unavailable", { + path, + protocol: options.protocol || this.protocol, + }); + } + const method = String(options.method || "POST").toUpperCase(); + const session = Boolean(options.session); + if (session && !this.sessionToken) { + await this.ensureSession(); + } + const headers = await this.buildHeaders({ session }); + const response = await this.fetchImpl(`${this.baseUrl}${path}`, { + method, + headers, + ...(method === "GET" || options.body === undefined ? {} : { body: JSON.stringify(options.body) }), + ...(options.signal ? { signal: options.signal } : {}), + }); + const status = Number(response?.status || 0); + const payload = await readResponsePayload(response); + if (!response?.ok) { + const message = readPayloadErrorMessage(payload, `Authority HTTP ${status || "unknown"}`); + throw new AuthorityHttpError(message || `Authority HTTP ${status || "unknown"}`, { + status, + code: payload?.code, + category: payload?.category, + payload, + path, + protocol: options.protocol || this.protocol, + }); + } + return payload; + } +} + +export function createAuthorityHttpClient(options = {}) { + return new AuthorityHttpClient(options); +} diff --git a/scripts/check-syntax.mjs b/scripts/check-syntax.mjs index 7d781ee..6057774 100644 --- a/scripts/check-syntax.mjs +++ b/scripts/check-syntax.mjs @@ -20,6 +20,86 @@ const SOURCE_ROOTS = [ "native", ]; +function toDataModuleUrl(source = "") { + return `data:text/javascript,${encodeURIComponent(String(source || ""))}`; +} + +const CHECK_SYNTAX_HOOK_BOOTSTRAP_URL = toDataModuleUrl(` +import * as nodeModule from "node:module"; + +const register = typeof nodeModule.register === "function" ? nodeModule.register : undefined; +const registerHooks = typeof nodeModule.registerHooks === "function" + ? nodeModule.registerHooks + : undefined; + +const scriptShimUrl = ${JSON.stringify(toDataModuleUrl([ + "export function getRequestHeaders() { return {}; }", + "export function saveMetadata() {}", + "export function saveSettingsDebounced() {}", + "export function substituteParamsExtended(value) { return String(value ?? ''); }", +].join("\n")))}; +const extensionsShimUrl = ${JSON.stringify(toDataModuleUrl([ + "export const extension_settings = { st_bme: {} };", + "export function getContext() { return {}; }", + "export function saveMetadataDebounced() {}", +].join("\n")))}; +const openaiShimUrl = ${JSON.stringify(toDataModuleUrl([ + "export const chat_completion_sources = {};", + "export async function sendOpenAIRequest() { return {}; }", +].join("\n")))}; + +function resolveShim(specifier) { + const normalized = String(specifier || ""); + if (normalized.endsWith("/script.js")) return scriptShimUrl; + if (normalized.endsWith("/extensions.js")) return extensionsShimUrl; + if (normalized.endsWith("/openai.js")) return openaiShimUrl; + return ""; +} + +if (typeof registerHooks === "function") { + registerHooks({ + resolve(specifier, context, nextResolve) { + const shimUrl = resolveShim(specifier); + if (shimUrl) { + return { + shortCircuit: true, + url: shimUrl, + }; + } + return nextResolve(specifier, context); + }, + }); +} else if (typeof register === "function") { + register(${JSON.stringify(toDataModuleUrl(` +export async function resolve(specifier, context, nextResolve) { + const normalized = String(specifier || ""); + if (normalized.endsWith("/script.js")) { + return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([ + "export function getRequestHeaders() { return {}; }", + "export function saveMetadata() {}", + "export function saveSettingsDebounced() {}", + "export function substituteParamsExtended(value) { return String(value ?? ''); }", + ].join("\n")))} }; + } + if (normalized.endsWith("/extensions.js")) { + return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([ + "export const extension_settings = { st_bme: {} };", + "export function getContext() { return {}; }", + "export function saveMetadataDebounced() {}", + ].join("\n")))} }; + } + if (normalized.endsWith("/openai.js")) { + return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([ + "export const chat_completion_sources = {};", + "export async function sendOpenAIRequest() { return {}; }", + ].join("\n")))} }; + } + return nextResolve(specifier, context); +} +`))}, import.meta.url); +} +`); + async function collectFiles(targetPath) { const absolutePath = path.resolve(process.cwd(), targetPath); const fileStat = await stat(absolutePath); @@ -48,7 +128,7 @@ function toPosixPath(filePath) { async function runNodeCheck(filePath) { return await new Promise((resolve, reject) => { - const child = spawn(process.execPath, ["--check", filePath], { + const child = spawn(process.execPath, ["--import", CHECK_SYNTAX_HOOK_BOOTSTRAP_URL, "--check", filePath], { cwd: process.cwd(), stdio: "inherit", windowsHide: true, diff --git a/sync/authority-graph-store.js b/sync/authority-graph-store.js index 0db5d20..2e89e15 100644 --- a/sync/authority-graph-store.js +++ b/sync/authority-graph-store.js @@ -7,13 +7,17 @@ import { buildSnapshotFromGraph, } from "./bme-db.js"; import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; +import { AuthorityHttpClient } from "../runtime/authority-http-client.js"; export const AUTHORITY_GRAPH_STORE_KIND = "authority"; export const AUTHORITY_GRAPH_STORE_MODE = "authority-sql-primary"; const META_DEFAULT_LAST_PROCESSED_FLOOR = -1; const META_DEFAULT_EXTRACTION_COUNT = 0; -const AUTHORITY_SQL_ENDPOINT = "/v1/sql"; +const DEFAULT_AUTHORITY_SQL_DATABASE = "default"; +const AUTHORITY_SQL_QUERY_ENDPOINT = "/sql/query"; +const AUTHORITY_SQL_EXEC_ENDPOINT = "/sql/exec"; +const AUTHORITY_SQL_TRANSACTION_ENDPOINT = "/sql/transaction"; const AUTHORITY_TABLES = Object.freeze({ meta: "st_bme_graph_meta", @@ -340,41 +344,47 @@ function normalizeUpsertCountDelta(delta = {}) { export class AuthoritySqlHttpClient { constructor(options = {}) { - this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); - this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); - this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + this.http = new AuthorityHttpClient({ + ...options, + baseUrl: normalizeAuthorityBaseUrl(options.baseUrl), + }); + this.database = normalizeRecordId(options.database) || DEFAULT_AUTHORITY_SQL_DATABASE; } async query(sql, params = {}) { - return await this._request({ action: "query", sql, params }); + return await this._request(AUTHORITY_SQL_QUERY_ENDPOINT, { + database: this.database, + statement: String(sql || ""), + params, + }); } async execute(sql, params = {}) { - return await this._request({ action: "execute", sql, params }); + return await this._request(AUTHORITY_SQL_EXEC_ENDPOINT, { + database: this.database, + statement: String(sql || ""), + params, + }); } async transaction(statements = []) { - return await this._request({ action: "transaction", statements }); + return await this._request(AUTHORITY_SQL_TRANSACTION_ENDPOINT, { + database: this.database, + statements: toArray(statements) + .filter((statement) => statement?.sql) + .map((statement) => ({ + statement: String(statement.sql || ""), + params: statement.params || {}, + })), + }); } - async _request(body = {}) { - if (typeof this.fetchImpl !== "function") { - throw new Error("Authority SQL fetch unavailable"); - } - const headers = { - Accept: "application/json", - "Content-Type": "application/json", - ...(this.headerProvider ? this.headerProvider() || {} : {}), - }; - const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_SQL_ENDPOINT}`, { + async _request(path, body = {}) { + return await this.http.requestJson(path, { method: "POST", - headers, - body: JSON.stringify(body), + body, + session: true, }); - if (!response?.ok) { - throw new Error(`Authority SQL HTTP ${response?.status || "unknown"}`); - } - return await response.json().catch(() => ({})); } } diff --git a/tests/authority-blob.mjs b/tests/authority-blob.mjs index 4fc88d5..0f736fe 100644 --- a/tests/authority-blob.mjs +++ b/tests/authority-blob.mjs @@ -432,8 +432,150 @@ async function testSyncUploadDownloadUsesAuthorityBlob() { assert.equal(db.snapshot.nodes[0].id, "sync-blob-node"); } +async function testAuthorityBlobHttpBoundary() { + const requests = []; + const adapter = createAuthorityBlobAdapter( + { authorityBaseUrl: "https://authority.example.test/root" }, + { + headerProvider: () => ({ "X-Test": "1" }), + fetchImpl: async (url, options = {}) => { + requests.push({ url, options }); + if (url.endsWith("/session/init")) { + return { + ok: true, + status: 200, + async json() { + return { sessionToken: "blob-session-token" }; + }, + }; + } + if (url.endsWith("/fs/private/write-file")) { + return { + ok: true, + status: 200, + async json() { + return { + entry: { + path: "user/files/demo.json", + sizeBytes: 17, + updatedAt: "2026-04-28T12:00:00.000Z", + }, + }; + }, + }; + } + if (url.endsWith("/fs/private/read-file")) { + const body = JSON.parse(String(options.body || "{}")); + if (body.path === "user/files/missing.json") { + return { + ok: false, + status: 404, + async json() { + return { error: "not found" }; + }, + async text() { + return "not found"; + }, + headers: { get: () => "application/json" }, + }; + } + return { + ok: true, + status: 200, + async json() { + return { + entry: { + path: "user/files/demo.json", + sizeBytes: 17, + updatedAt: "2026-04-28T12:00:00.000Z", + }, + content: JSON.stringify({ hello: "world" }), + encoding: "utf8", + }; + }, + }; + } + if (url.endsWith("/fs/private/stat")) { + return { + ok: true, + status: 200, + async json() { + return { + entry: { + path: "user/files/demo.json", + sizeBytes: 17, + updatedAt: "2026-04-28T12:00:00.000Z", + }, + }; + }, + }; + } + if (url.endsWith("/fs/private/delete")) { + return { + ok: true, + status: 200, + async json() { + return { ok: true }; + }, + }; + } + return { + ok: false, + status: 404, + async json() { + return {}; + }, + async text() { + return "not found"; + }, + headers: { get: () => "application/json" }, + }; + }, + }, + ); + + const writeResult = await adapter.writeJson("user/files/demo.json", { hello: "world" }); + assert.equal(writeResult.ok, true); + assert.equal(writeResult.path, "user/files/demo.json"); + + const readResult = await adapter.readJson("user/files/demo.json"); + assert.equal(readResult.exists, true); + assert.deepEqual(readResult.payload, { hello: "world" }); + + const statResult = await adapter.stat("user/files/demo.json"); + assert.equal(statResult.exists, true); + assert.equal(statResult.path, "user/files/demo.json"); + + const missingResult = await adapter.readJson("user/files/missing.json"); + assert.equal(missingResult.exists, false); + + const deleteResult = await adapter.delete("user/files/demo.json"); + assert.equal(deleteResult.ok, true); + + assert.deepEqual( + requests.map((request) => request.url), + [ + "https://authority.example.test/root/session/init", + "https://authority.example.test/root/fs/private/write-file", + "https://authority.example.test/root/fs/private/read-file", + "https://authority.example.test/root/fs/private/stat", + "https://authority.example.test/root/fs/private/read-file", + "https://authority.example.test/root/fs/private/delete", + ], + ); + assert.equal(requests[1].options.headers["x-authority-session-token"], "blob-session-token"); + assert.equal(requests[1].options.headers["X-Test"], "1"); + assert.deepEqual(JSON.parse(String(requests[1].options.body || "{}")), { + path: "user/files/demo.json", + content: JSON.stringify({ hello: "world" }), + encoding: "utf8", + createParents: true, + }); +} + await testAdapterBasics(); await testAuthorityBlobFailOpenFallsBackToUserFiles(); await testBackupRestoreUsesAuthorityBlob(); await testSyncUploadDownloadUsesAuthorityBlob(); +await testAuthorityBlobHttpBoundary(); console.log("authority-blob tests passed"); diff --git a/tests/authority-capabilities.mjs b/tests/authority-capabilities.mjs index 658a691..5c22e52 100644 --- a/tests/authority-capabilities.mjs +++ b/tests/authority-capabilities.mjs @@ -18,31 +18,29 @@ assert.equal(normalizedSettings.vectorMode, "auto-primary"); assert.equal(normalizedSettings.primaryWhenAvailable, true); assert.deepEqual(buildAuthorityProbeUrls("/api/plugins/authority/"), [ - "/api/plugins/authority/v1/diagnostics/probe", - "/api/plugins/authority/v1/probe", "/api/plugins/authority/probe", - "/api/plugins/authority", ]); const collected = collectAuthorityFeatures({ - features: ["sql.query", "trivium.search"], - services: { - sql: true, - jobs: true, - blob: true, + features: { + sql: { queryPage: true }, + trivium: { upsert: true }, + jobs: { background: true }, + transfers: { fs: true }, }, }); -assert.equal(collected.has("sql.query"), true); -assert.equal(collected.has("trivium.search"), true); assert.equal(collected.has("sql"), true); -assert.equal(collected.has("jobs"), true); -assert.equal(collected.has("blob"), true); +assert.equal(collected.has("sql.querypage"), true); +assert.equal(collected.has("sql"), true); +assert.equal(collected.has("trivium"), true); +assert.equal(collected.has("jobs.background"), true); +assert.equal(collected.has("transfers.fs"), true); const readyState = normalizeAuthorityCapabilityState( { installed: true, healthy: true, - features: ["sql", "trivium", "jobs", "blob"], + features: ["sql", "trivium", "jobs", "transfers.fs"], }, defaultSettings, ); @@ -77,28 +75,91 @@ assert.equal(disabledState.reason, "disabled"); assert.equal(disabledState.serverPrimaryReady, false); assert.equal(disabledState.lastProbeAt, 1000); -let requestedUrl = ""; +const requestedUrls = []; const probedState = await probeAuthorityCapabilities({ settings: defaultSettings, allowRelativeUrl: true, nowMs: 2000, - fetchImpl: async (url) => { - requestedUrl = url; + fetchImpl: async (url, options = {}) => { + requestedUrls.push([url, options.method || "GET", options.headers || {}]); + if (url.endsWith("/probe")) { + assert.equal(options.method, "POST"); + return { + ok: true, + status: 200, + async json() { + return { + healthy: true, + features: { + sql: { queryPage: true }, + trivium: { upsert: true }, + jobs: { background: true }, + transfers: { fs: true }, + }, + }; + }, + }; + } + if (url.endsWith("/session/init")) { + assert.equal(options.method, "POST"); + return { + ok: true, + status: 200, + async json() { + return { sessionToken: "session-probe-token" }; + }, + }; + } + if (url.endsWith("/session/current")) { + assert.equal(options.method, "GET"); + assert.equal(options.headers["x-authority-session-token"], "session-probe-token"); + return { + ok: true, + status: 200, + async json() { + return { ok: true }; + }, + }; + } + if (url.endsWith("/permissions/evaluate-batch")) { + assert.equal(options.method, "POST"); + assert.equal(options.headers["x-authority-session-token"], "session-probe-token"); + const body = JSON.parse(String(options.body || "{}")); + assert.equal(Array.isArray(body.requests), true); + assert.equal(body.requests.some((request) => request.resource === "fs.private"), true); + return { + ok: true, + status: 200, + async json() { + return { + results: body.requests.map((request) => ({ + decision: "granted", + resource: request.resource, + target: request.target || "", + })), + }; + }, + }; + } return { - ok: true, - status: 200, + ok: false, + status: 404, async json() { - return { - healthy: true, - sessionReady: true, - permissionReady: true, - features: ["sql", "trivium", "jobs", "blob"], - }; + return {}; }, }; }, }); -assert.equal(requestedUrl, "/api/plugins/authority/v1/diagnostics/probe"); +assert.equal(requestedUrls[0]?.[0], "/api/plugins/authority/probe"); +assert.deepEqual( + requestedUrls.map(([url]) => url), + [ + "/api/plugins/authority/probe", + "/api/plugins/authority/session/init", + "/api/plugins/authority/session/current", + "/api/plugins/authority/permissions/evaluate-batch", + ], +); assert.equal(probedState.installed, true); assert.equal(probedState.healthy, true); assert.equal(probedState.serverPrimaryReady, true); diff --git a/tests/authority-graph-store.mjs b/tests/authority-graph-store.mjs index b1fda72..9ef59ed 100644 --- a/tests/authority-graph-store.mjs +++ b/tests/authority-graph-store.mjs @@ -301,6 +301,15 @@ async function testHttpSqlClientBoundary() { headerProvider: () => ({ "X-Test": "1" }), fetchImpl: async (url, init) => { requests.push({ url, init }); + if (url.endsWith("/session/init")) { + return { + ok: true, + status: 200, + async json() { + return { sessionToken: "sql-session-token" }; + }, + }; + } return { ok: true, status: 200, @@ -313,12 +322,19 @@ async function testHttpSqlClientBoundary() { const result = await client.query("SELECT 1", { chatId: "chat" }); assert.deepEqual(result, { rows: [{ value: 1 }] }); - assert.equal(requests[0].url, "https://authority.example.test/root/v1/sql"); - assert.equal(requests[0].init.method, "POST"); - assert.equal(requests[0].init.headers["X-Test"], "1"); - assert.deepEqual(JSON.parse(requests[0].init.body), { - action: "query", - sql: "SELECT 1", + assert.deepEqual( + requests.map((request) => request.url), + [ + "https://authority.example.test/root/session/init", + "https://authority.example.test/root/sql/query", + ], + ); + assert.equal(requests[1].init.method, "POST"); + assert.equal(requests[1].init.headers["X-Test"], "1"); + assert.equal(requests[1].init.headers["x-authority-session-token"], "sql-session-token"); + assert.deepEqual(JSON.parse(requests[1].init.body), { + database: "default", + statement: "SELECT 1", params: { chatId: "chat" }, }); } diff --git a/tests/authority-jobs.mjs b/tests/authority-jobs.mjs index 04d172b..b0d1372 100644 --- a/tests/authority-jobs.mjs +++ b/tests/authority-jobs.mjs @@ -455,4 +455,150 @@ const rangeSync = rangeRuntime.calls.find(([name]) => name === "syncVectorState" assert.equal(rangeSync?.[1]?.purge, false); assert.equal(rangeSync?.[1]?.range, range); +const httpRequests = []; +const httpAdapter = createAuthorityJobAdapter( + { authorityBaseUrl: "https://authority.example.test/root" }, + { + headerProvider: () => ({ "X-Test": "1" }), + fetchImpl: async (url, options = {}) => { + httpRequests.push({ url, options }); + if (url.endsWith("/session/init")) { + return { + ok: true, + status: 200, + async json() { + return { sessionToken: "job-session-token" }; + }, + }; + } + if (url.endsWith("/jobs/create")) { + return { + ok: true, + status: 200, + async json() { + return { + id: "job-http-1", + type: "authority.vector.rebuild", + status: "queued", + progress: 0, + idempotencyKey: "idem-http-1", + }; + }, + }; + } + if (url.endsWith("/jobs/job-http-1")) { + return { + ok: true, + status: 200, + async json() { + return { + id: "job-http-1", + type: "authority.vector.rebuild", + status: "completed", + progress: 1, + }; + }, + }; + } + if (url.endsWith("/jobs/list")) { + return { + ok: true, + status: 200, + async json() { + return { + jobs: [ + { + id: "job-http-1", + type: "authority.vector.rebuild", + status: "completed", + progress: 1, + }, + ], + page: { + nextCursor: "next-http-1", + hasMore: true, + }, + }; + }, + }; + } + if (url.endsWith("/jobs/job-http-1/requeue")) { + return { + ok: true, + status: 200, + async json() { + return { + id: "job-http-2", + type: "authority.vector.rebuild", + status: "queued", + progress: 0, + }; + }, + }; + } + if (url.endsWith("/jobs/job-http-1/cancel")) { + return { + ok: true, + status: 200, + async json() { + return { + id: "job-http-1", + type: "authority.vector.rebuild", + status: "cancelled", + progress: 1, + }; + }, + }; + } + return { + ok: false, + status: 404, + async json() { + return {}; + }, + }; + }, + }, +); + +const httpSubmitted = await httpAdapter.submit( + "authority.vector.rebuild", + { chatId: "chat-http" }, + { idempotencyKey: "idem-http-1" }, +); +assert.equal(httpSubmitted.id, "job-http-1"); +const httpLoaded = await httpAdapter.get("job-http-1"); +assert.equal(httpLoaded.status, "completed"); +const httpPage = await httpAdapter.listPage({ cursor: "cursor-http", limit: 10 }); +assert.equal(httpPage.nextCursor, "next-http-1"); +assert.equal(httpPage.hasMore, true); +const httpRequeued = await httpAdapter.requeue("job-http-1"); +assert.equal(httpRequeued.id, "job-http-2"); +const httpCancelled = await httpAdapter.cancel("job-http-1"); +assert.equal(httpCancelled.status, "cancelled"); +assert.deepEqual( + httpRequests.map((request) => request.url), + [ + "https://authority.example.test/root/session/init", + "https://authority.example.test/root/jobs/create", + "https://authority.example.test/root/jobs/job-http-1", + "https://authority.example.test/root/jobs/list", + "https://authority.example.test/root/jobs/job-http-1/requeue", + "https://authority.example.test/root/jobs/job-http-1/cancel", + ], +); +assert.equal(httpRequests[1].options.headers["x-authority-session-token"], "job-session-token"); +assert.equal(httpRequests[1].options.headers["X-Test"], "1"); +assert.deepEqual(JSON.parse(String(httpRequests[1].options.body || "{}")), { + type: "authority.vector.rebuild", + payload: { chatId: "chat-http" }, + idempotencyKey: "idem-http-1", +}); +assert.deepEqual(JSON.parse(String(httpRequests[3].options.body || "{}")), { + page: { + cursor: "cursor-http", + limit: 10, + }, +}); + console.log("authority-jobs tests passed"); diff --git a/tests/authority-recall-candidates.mjs b/tests/authority-recall-candidates.mjs index 9e41474..5cd0b7e 100644 --- a/tests/authority-recall-candidates.mjs +++ b/tests/authority-recall-candidates.mjs @@ -16,6 +16,15 @@ installResolveHooks([ }, ]); +globalThis.__stBmeTestOverrides = { + embedding: { + async embedText(text) { + const seed = String(text || "").length || 1; + return [seed / 100, 0.2, 0.3]; + }, + }, +}; + const { normalizeAuthorityVectorConfig } = await import( "../vector/authority-vector-primary-adapter.js" ); @@ -153,6 +162,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN const config = normalizeAuthorityVectorConfig( { authorityBaseUrl: "/api/plugins/authority", + authorityEmbeddingApiUrl: "https://example.com/v1", + authorityEmbeddingModel: "test-embedding", authorityVectorFailOpen: true, }, { triviumClient }, @@ -216,6 +227,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN const config = normalizeAuthorityVectorConfig( { authorityBaseUrl: "/api/plugins/authority", + authorityEmbeddingApiUrl: "https://example.com/v1", + authorityEmbeddingModel: "test-embedding", authorityVectorFailOpen: true, }, { triviumClient }, diff --git a/tests/authority-vector-primary.mjs b/tests/authority-vector-primary.mjs index 169b139..c5dbd42 100644 --- a/tests/authority-vector-primary.mjs +++ b/tests/authority-vector-primary.mjs @@ -16,6 +16,17 @@ installResolveHooks([ }, ]); +globalThis.__stBmeTestOverrides = { + embedding: { + async embedBatch(texts = []) { + return texts.map((text, index) => [1, index / 10, String(text || "").length / 100]); + }, + async embedText(text = "") { + return [1, 0.5, String(text || "").length / 100]; + }, + }, +}; + const { filterAuthorityTriviumNodes, isAuthorityVectorConfig, @@ -112,8 +123,20 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) { }; } +async function withMockFetch(handler, fn) { + const previousFetch = globalThis.fetch; + globalThis.fetch = handler; + try { + return await fn(); + } finally { + globalThis.fetch = previousFetch; + } +} + const config = normalizeAuthorityVectorConfig({ authorityBaseUrl: "/api/plugins/authority", + authorityEmbeddingApiUrl: "https://example.com/v1", + authorityEmbeddingModel: "test-embedding", authorityVectorSyncChunkSize: 1, authorityVectorFailOpen: true, }); @@ -144,6 +167,10 @@ assert.equal(isAuthorityVectorConfig(config), true); upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(), ["node-a", "node-b"], ); + assert.equal( + upserts.every(([, payload]) => payload.items.every((item) => Array.isArray(item.vector) && item.vector.length > 0)), + true, + ); const linkCall = triviumClient.calls.find(([name]) => name === "linkMany"); assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a"); assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b"); @@ -170,6 +197,8 @@ assert.equal(isAuthorityVectorConfig(config), true); assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]); const searchCall = triviumClient.calls.find(([name]) => name === "search"); assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]); + assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true); + assert.ok(searchCall?.[1]?.queryVector.length > 0); assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority"); assert.equal(graph.vectorIndexState.lastSearchTimings.success, true); } @@ -221,4 +250,72 @@ assert.equal(isAuthorityVectorConfig(config), true); assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]); } +{ + const previousOverrides = globalThis.__stBmeTestOverrides; + globalThis.__stBmeTestOverrides = {}; + const fetchCalls = []; + try { + await withMockFetch(async (url, options = {}) => { + fetchCalls.push([url, JSON.parse(String(options.body || "{}"))]); + return { + ok: true, + status: 200, + async json() { + const body = JSON.parse(String(options.body || "{}")); + if (Array.isArray(body.texts)) { + return { + vectors: body.texts.map((text, index) => [1, index + 1, String(text || "").length / 100]), + }; + } + return { + vector: [1, 9, String(body.text || "").length / 100], + }; + }, + async text() { + return ""; + }, + }; + }, async () => { + const backendConfig = normalizeAuthorityVectorConfig({ + authorityBaseUrl: "/api/plugins/authority", + embeddingTransportMode: "backend", + embeddingBackendSource: "openai", + embeddingBackendModel: "text-embedding-3-small", + authorityVectorSyncChunkSize: 2, + }); + const { graph, first, second } = createAuthorityVectorGraph(); + first.embedding = null; + second.embedding = null; + const triviumClient = createMockTriviumClient(); + await syncGraphVectorIndexFromIndex(graph, backendConfig, { + chatId: "chat-authority-vector", + purge: true, + triviumClient, + }); + const results = await findSimilarNodesByTextFromIndex( + graph, + "archive door", + { ...backendConfig, triviumClient }, + 5, + [first, second], + ); + assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]); + const upsertCall = triviumClient.calls.find(([name]) => name === "bulkUpsert"); + assert.equal( + upsertCall?.[1]?.items?.every((item) => Array.isArray(item.vector) && item.vector.length > 0), + true, + ); + const searchCall = triviumClient.calls.find(([name]) => name === "search"); + assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true); + assert.equal(fetchCalls.every(([url]) => url === "/api/vector/embed"), true); + assert.equal(fetchCalls[0]?.[1]?.source, "openai"); + assert.equal(fetchCalls[0]?.[1]?.model, "text-embedding-3-small"); + assert.equal(Array.isArray(fetchCalls[0]?.[1]?.texts), true); + assert.equal(fetchCalls[fetchCalls.length - 1]?.[1]?.isQuery, true); + }); + } finally { + globalThis.__stBmeTestOverrides = previousOverrides; + } +} + console.log("authority-vector-primary tests passed"); diff --git a/tests/e2e/authority-checkpoint-restore.mjs b/tests/e2e/authority-checkpoint-restore.mjs index babb6c7..53bfce1 100644 --- a/tests/e2e/authority-checkpoint-restore.mjs +++ b/tests/e2e/authority-checkpoint-restore.mjs @@ -1,19 +1,36 @@ import assert from "node:assert/strict"; -import { buildLukerGraphCheckpointV2 } from "../../graph/graph-persistence.js"; import { - applyAuthorityCheckpointToStore, - buildAuthorityConsistencyAudit, - buildAuthorityCheckpointImportSnapshot, -} from "../../maintenance/authority-consistency.js"; -import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; -import { AuthorityGraphStore } from "../../sync/authority-graph-store.js"; + installResolveHooks, + toDataModuleUrl, +} from "../helpers/register-hooks-compat.mjs"; + +installResolveHooks([ + { + specifiers: ["../../../../../script.js"], + url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"), + }, + { + specifiers: ["../../../../extensions.js"], + url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"), + }, +]); + import { createAuthorityE2eContext, createAuthorityE2eContractGraph, runAuthorityE2eStep, } from "../helpers/authority-e2e-context.mjs"; +const { buildLukerGraphCheckpointV2 } = await import("../../graph/graph-persistence.js"); +const { + applyAuthorityCheckpointToStore, + buildAuthorityConsistencyAudit, + buildAuthorityCheckpointImportSnapshot, +} = await import("../../maintenance/authority-consistency.js"); +const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js"); +const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js"); + const context = createAuthorityE2eContext({ skipMessage: "authority checkpoint restore E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server", diff --git a/tests/e2e/authority-diagnostics-roundtrip.mjs b/tests/e2e/authority-diagnostics-roundtrip.mjs index f408a70..49756be 100644 --- a/tests/e2e/authority-diagnostics-roundtrip.mjs +++ b/tests/e2e/authority-diagnostics-roundtrip.mjs @@ -1,17 +1,34 @@ import assert from "node:assert/strict"; import { - buildAuthorityDiagnosticsBundle, - buildAuthorityPerformanceBaseline, - writeAuthorityDiagnosticsBundle, -} from "../../maintenance/authority-diagnostics-bundle.js"; -import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; + installResolveHooks, + toDataModuleUrl, +} from "../helpers/register-hooks-compat.mjs"; + +installResolveHooks([ + { + specifiers: ["../../../../../script.js"], + url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"), + }, + { + specifiers: ["../../../../extensions.js"], + url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"), + }, +]); + import { createAuthorityE2eContext, createAuthorityE2eContractGraph, runAuthorityE2eStep, } from "../helpers/authority-e2e-context.mjs"; +const { + buildAuthorityDiagnosticsBundle, + buildAuthorityPerformanceBaseline, + writeAuthorityDiagnosticsBundle, +} = await import("../../maintenance/authority-diagnostics-bundle.js"); +const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js"); + const context = createAuthorityE2eContext({ skipMessage: "authority diagnostics E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server", diff --git a/tests/e2e/authority-server-primary.mjs b/tests/e2e/authority-server-primary.mjs index 9c5b98c..f0d8faa 100644 --- a/tests/e2e/authority-server-primary.mjs +++ b/tests/e2e/authority-server-primary.mjs @@ -1,8 +1,32 @@ import assert from "node:assert/strict"; -import { probeAuthorityCapabilities } from "../../runtime/authority-capabilities.js"; -import { AuthorityGraphStore } from "../../sync/authority-graph-store.js"; import { + installResolveHooks, + toDataModuleUrl, +} from "../helpers/register-hooks-compat.mjs"; + +installResolveHooks([ + { + specifiers: ["../../../../../script.js"], + url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"), + }, + { + specifiers: ["../../../../extensions.js"], + url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"), + }, +]); + +import { + buildAuthorityE2eVectorEntries, + createAuthorityE2eContext, + createAuthorityE2eContractGraph, + createAuthorityE2eContractNode, + runAuthorityE2eStep, +} from "../helpers/authority-e2e-context.mjs"; + +const { probeAuthorityCapabilities } = await import("../../runtime/authority-capabilities.js"); +const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js"); +const { deleteAuthorityTriviumNodes, filterAuthorityTriviumNodes, normalizeAuthorityVectorConfig, @@ -11,19 +35,12 @@ import { searchAuthorityTriviumNodes, syncAuthorityTriviumLinks, upsertAuthorityTriviumEntries, -} from "../../vector/authority-vector-primary-adapter.js"; -import { +} = await import("../../vector/authority-vector-primary-adapter.js"); +const { buildAuthorityJobIdempotencyKey, createAuthorityJobAdapter, -} from "../../maintenance/authority-job-adapter.js"; -import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; -import { - buildAuthorityE2eVectorEntries, - createAuthorityE2eContext, - createAuthorityE2eContractGraph, - createAuthorityE2eContractNode, - runAuthorityE2eStep, -} from "../helpers/authority-e2e-context.mjs"; +} = await import("../../maintenance/authority-job-adapter.js"); +const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js"); const context = createAuthorityE2eContext({ skipMessage: @@ -35,7 +52,7 @@ if (context.skip) { process.exit(0); } const resolvedBaseUrl = context.baseUrl; -const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId } = context; +const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId, embeddingApiUrl, embeddingApiKey, embeddingModel } = context; const graph = createAuthorityE2eContractGraph(chatId, runId); const runContext = { @@ -105,7 +122,12 @@ await runAuthorityE2eStep("sql", async () => { }); await runAuthorityE2eStep("trivium", async () => { - const config = normalizeAuthorityVectorConfig({ authorityBaseUrl: resolvedBaseUrl }); + const config = normalizeAuthorityVectorConfig({ + authorityBaseUrl: resolvedBaseUrl, + authorityEmbeddingApiUrl: embeddingApiUrl, + authorityEmbeddingApiKey: embeddingApiKey, + authorityEmbeddingModel: embeddingModel, + }); const entries = buildAuthorityE2eVectorEntries(graph); await purgeAuthorityTriviumNamespace(config, { namespace, @@ -141,6 +163,7 @@ await runAuthorityE2eStep("trivium", async () => { namespace, collectionId, chatId, + queryVector: [1, 1, 0.18], topK: 5, fetchImpl, headerProvider, diff --git a/tests/helpers/authority-e2e-context.mjs b/tests/helpers/authority-e2e-context.mjs index f008732..80952d7 100644 --- a/tests/helpers/authority-e2e-context.mjs +++ b/tests/helpers/authority-e2e-context.mjs @@ -97,6 +97,9 @@ export function createAuthorityE2eContext(options = {}) { const namespace = String(env.AUTHORITY_E2E_NAMESPACE || `st-bme-e2e-${runId}`); const collectionId = String(env.AUTHORITY_E2E_COLLECTION_ID || `${namespace}::${chatId}`); const blobPath = String(env.AUTHORITY_E2E_BLOB_PATH || `st-bme/e2e/${runId}/contract.json`); + const embeddingApiUrl = String(env.AUTHORITY_E2E_EMBEDDING_API_URL || "").trim(); + const embeddingApiKey = String(env.AUTHORITY_E2E_EMBEDDING_API_KEY || "").trim(); + const embeddingModel = String(env.AUTHORITY_E2E_EMBEDDING_MODEL || "").trim(); return { skip: false, env, @@ -110,6 +113,9 @@ export function createAuthorityE2eContext(options = {}) { namespace, collectionId, blobPath, + embeddingApiUrl, + embeddingApiKey, + embeddingModel, }; } @@ -181,6 +187,7 @@ export function buildAuthorityE2eVectorEntries(graph = null) { index, hash: `${node.id}:hash`, text: `${node.fields?.title || node.id}. ${node.fields?.summary || ""}`, + vector: [1, index + 1, String(node.fields?.title || node.id || "").length / 100], })); } diff --git a/tests/perf/authority-recall-bench.mjs b/tests/perf/authority-recall-bench.mjs index 9da700a..729df95 100644 --- a/tests/perf/authority-recall-bench.mjs +++ b/tests/perf/authority-recall-bench.mjs @@ -16,6 +16,14 @@ installResolveHooks([ }, ]); +globalThis.__stBmeTestOverrides = { + embedding: { + async embedText(text = "") { + return [1, 0.25, String(text || "").length / 100]; + }, + }, +}; + const outputJson = process.argv.includes("--json"); const RUNS = 5; const SIZE_PRESETS = [ @@ -233,6 +241,8 @@ async function runPreset(preset) { const config = normalizeAuthorityVectorConfig( { authorityBaseUrl: "/api/plugins/authority", + authorityEmbeddingApiUrl: "https://example.com/v1", + authorityEmbeddingModel: "test-embedding", authorityVectorFailOpen: true, }, { triviumClient }, diff --git a/tests/vector-config.mjs b/tests/vector-config.mjs index f3e077e..3a2b66e 100644 --- a/tests/vector-config.mjs +++ b/tests/vector-config.mjs @@ -70,6 +70,54 @@ const defaultModeConfig = getVectorConfigFromSettings({ assert.equal(defaultModeConfig.mode, "direct"); assert.equal(validateVectorConfig(defaultModeConfig).valid, true); +const validAuthorityConfig = { + mode: "authority", + source: "authority-trivium", + baseUrl: "/api/plugins/authority", + apiUrl: "https://example.com/v1", + model: "text-embedding-3-small", +}; +assert.equal(validateVectorConfig(validAuthorityConfig).valid, true); + +const invalidAuthorityConfig = { + mode: "authority", + source: "authority-trivium", + baseUrl: "/api/plugins/authority", + apiUrl: "", + model: "", +}; +assert.equal(validateVectorConfig(invalidAuthorityConfig).valid, false); + +const validAuthorityBackendConfig = { + mode: "authority", + source: "authority-trivium", + baseUrl: "/api/plugins/authority", + embeddingMode: "backend", + embeddingSource: "openai", + apiUrl: "", + model: "text-embedding-3-small", +}; +assert.equal(validateVectorConfig(validAuthorityBackendConfig).valid, true); + +const invalidAuthorityBackendConfig = { + mode: "authority", + source: "authority-trivium", + baseUrl: "/api/plugins/authority", + embeddingMode: "backend", + embeddingSource: "vllm", + apiUrl: "", + model: "BAAI/bge-m3", +}; +assert.equal(validateVectorConfig(invalidAuthorityBackendConfig).valid, false); + +const authorityLikeConfig = getVectorConfigFromSettings({ + embeddingApiUrl: "https://example.com/v1/embeddings", + embeddingApiKey: "sk-test", + embeddingModel: "text-embedding-3-small", +}); +assert.equal(authorityLikeConfig.apiUrl, "https://example.com/v1"); +assert.equal(authorityLikeConfig.model, "text-embedding-3-small"); + const invalidBackendConfig = getVectorConfigFromSettings({ embeddingTransportMode: "backend", embeddingBackendSource: "vllm", diff --git a/vector/authority-vector-primary-adapter.js b/vector/authority-vector-primary-adapter.js index 06ff428..f8ded38 100644 --- a/vector/authority-vector-primary-adapter.js +++ b/vector/authority-vector-primary-adapter.js @@ -1,11 +1,17 @@ import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; +import { + AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, + AuthorityHttpClient, +} from "../runtime/authority-http-client.js"; +import { embedText } from "./embedding.js"; export const AUTHORITY_VECTOR_MODE = "authority"; export const AUTHORITY_VECTOR_SOURCE = "authority-trivium"; -const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium"; +const DEFAULT_AUTHORITY_TRIVIUM_DATABASE = "st_bme_vectors"; const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000; const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000; +const DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE = "openai"; function clampInteger(value, fallback, min, max) { const parsed = Number(value); @@ -36,6 +42,27 @@ function normalizeRecordId(value) { return String(value ?? "").trim(); } +function normalizeVector(value = null) { + const source = ArrayBuffer.isView(value) ? Array.from(value) : value; + if (!Array.isArray(source)) return []; + return source + .map((item) => Number(item)) + .filter((item) => Number.isFinite(item)); +} + +function normalizePositiveInteger(value, fallback = 0) { + const parsed = Number(value); + if (!Number.isFinite(parsed) || parsed <= 0) return fallback; + return Math.floor(parsed); +} + +function normalizeOpenAICompatibleBaseUrl(value) { + return String(value || "") + .trim() + .replace(/\/+(chat\/completions|embeddings)$/i, "") + .replace(/\/+$/, ""); +} + function readNestedValue(source = null, path = []) { let current = source; for (const key of path) { @@ -68,12 +95,14 @@ function readResultRows(payload = null) { if (Array.isArray(payload.data)) return payload.data; if (Array.isArray(payload.neighbors)) return payload.neighbors; if (Array.isArray(payload.links)) return payload.links; + if (Array.isArray(payload.nodes)) return payload.nodes; if (Array.isArray(payload.result?.results)) return payload.result.results; if (Array.isArray(payload.result?.items)) return payload.result.items; if (Array.isArray(payload.result?.rows)) return payload.result.rows; if (Array.isArray(payload.result?.data)) return payload.result.data; if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors; if (Array.isArray(payload.result?.links)) return payload.result.links; + if (Array.isArray(payload.result?.nodes)) return payload.result.nodes; return []; } @@ -145,6 +174,32 @@ function normalizeSearchResults(payload = null) { .filter(Boolean); } +function buildOpenOptions(config = {}, payload = {}) { + const database = normalizeRecordId(payload.database || config.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE; + return { + database, + ...(normalizePositiveInteger(payload.dim ?? config.dim, 0) > 0 ? { dim: normalizePositiveInteger(payload.dim ?? config.dim, 0) } : {}), + ...(payload.dtype || config.dtype ? { dtype: String(payload.dtype || config.dtype) } : {}), + ...(payload.syncMode || config.syncMode ? { syncMode: String(payload.syncMode || config.syncMode) } : {}), + ...(payload.storageMode || config.storageMode ? { storageMode: String(payload.storageMode || config.storageMode) } : {}), + }; +} + +function getNamespace(payload = {}) { + return normalizeRecordId(payload.namespace || payload.collectionId || payload.chatId); +} + +function buildNodeReference(id, namespace = "") { + return { + externalId: normalizeRecordId(id), + ...(namespace ? { namespace } : {}), + }; +} + +function buildV06PayloadSource(payload = {}) { + return payload && typeof payload === "object" && !Array.isArray(payload) ? payload : {}; +} + function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) { const scope = node?.scope && typeof node.scope === "object" ? node.scope : {}; const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0]; @@ -166,6 +221,7 @@ function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelSc regionKey: String(scope.regionKey || node?.regionKey || ""), storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""), storyTimeLabel: String(node?.storyTime?.label || ""), + text: String(entry?.text || ""), title: getNodeFieldText(node, ["title"]), name: getNodeFieldText(node, ["name"]), summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]), @@ -190,6 +246,7 @@ function buildAuthorityVectorItems(graph, entries = [], options = {}) { text: String(entry?.text || ""), index: Number(entry?.index || 0) || 0, hash: String(entry?.hash || ""), + vector: normalizeVector(entry?.vector || entry?.embedding || node?.embedding), payload, }; }) @@ -229,11 +286,43 @@ export function isAuthorityVectorConfig(config = null) { export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) { const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {}; + const hasAuthorityEmbeddingOverride = [ + source.authorityEmbeddingApiUrl, + source.authorityEmbeddingApiKey, + source.authorityEmbeddingModel, + ].some((value) => String(value ?? "").trim()); + const embeddingMode = hasAuthorityEmbeddingOverride + ? "direct" + : String(source.embeddingTransportMode || "direct").trim().toLowerCase() === "backend" + ? "backend" + : "direct"; + const embeddingSource = embeddingMode === "backend" + ? String(source.embeddingBackendSource || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE).trim().toLowerCase() || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE + : "direct"; return { mode: AUTHORITY_VECTOR_MODE, source: AUTHORITY_VECTOR_SOURCE, baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl), - model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(), + protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, + database: normalizeRecordId(source.authorityTriviumDatabase ?? source.triviumDatabase) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE, + dim: normalizePositiveInteger(source.authorityTriviumDim ?? source.triviumDim, 0), + dtype: String(source.authorityTriviumDtype ?? source.triviumDtype ?? "").trim(), + syncMode: String(source.authorityTriviumSyncMode ?? source.triviumSyncMode ?? "").trim(), + storageMode: String(source.authorityTriviumStorageMode ?? source.triviumStorageMode ?? "").trim(), + embeddingMode, + embeddingSource, + apiUrl: normalizeOpenAICompatibleBaseUrl( + embeddingMode === "backend" + ? source.embeddingBackendApiUrl + : source.authorityEmbeddingApiUrl ?? source.embeddingApiUrl ?? source.embeddingBackendApiUrl, + ), + apiKey: embeddingMode === "backend" + ? "" + : String(source.authorityEmbeddingApiKey ?? source.embeddingApiKey ?? "").trim(), + model: embeddingMode === "backend" + ? String(source.embeddingBackendModel ?? source.embeddingModel ?? "").trim() + : String(source.authorityEmbeddingModel ?? source.embeddingModel ?? source.embeddingBackendModel ?? "").trim(), + autoSuffix: source.embeddingAutoSuffix !== false, chunkSize: clampInteger( source.authorityVectorSyncChunkSize ?? source.chunkSize, DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE, @@ -251,61 +340,225 @@ export class AuthorityTriviumHttpClient { this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; + this.protocol = AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06; + this.config = { + database: normalizeRecordId(options.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE, + dim: normalizePositiveInteger(options.dim, 0), + dtype: String(options.dtype || "").trim(), + syncMode: String(options.syncMode || "").trim(), + storageMode: String(options.storageMode || "").trim(), + }; + this.http = new AuthorityHttpClient({ + ...options, + baseUrl: this.baseUrl, + fetchImpl: this.fetchImpl, + headerProvider: this.headerProvider, + protocol: this.protocol, + }); } async request(action, payload = {}) { - if (typeof this.fetchImpl !== "function") { - throw new Error("Authority Trivium fetch unavailable"); - } - const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, { - method: "POST", - headers: { - Accept: "application/json", - "Content-Type": "application/json", - ...(this.headerProvider ? this.headerProvider() || {} : {}), - }, - body: JSON.stringify({ action, ...payload }), + if (action === "purge") return await this.purge(payload); + if (action === "bulkUpsert") return await this.bulkUpsert(payload); + if (action === "deleteMany") return await this.deleteMany(payload); + if (action === "linkMany") return await this.linkMany(payload); + if (action === "search") return await this.search(payload); + if (action === "filterWhere") return await this.filterWhere(payload); + if (action === "queryPage") return await this.queryPage(payload); + if (action === "neighbors") return await this.neighbors(payload); + if (action === "stat") return await this.stat(payload); + throw new Error(`Authority Trivium v0.6 action unavailable: ${action}`); + } + + async requestV06(path, payload = {}, method = "POST") { + return await this.http.requestJson(path, { + method, + body: payload, + session: true, + protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, }); - if (!response?.ok) { - throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`); - } - return await response.json().catch(() => ({})); + } + + buildOpenOptions(payload = {}) { + return buildOpenOptions(this.config, payload); } async purge(payload = {}) { - return await this.request("purge", payload); + const namespace = getNamespace(payload); + const openOptions = this.buildOpenOptions(payload); + let cursor = ""; + let deleted = 0; + let scanned = 0; + for (let pageIndex = 0; pageIndex < 100; pageIndex++) { + const page = await this.requestV06("/trivium/list-mappings", { + ...openOptions, + namespace, + page: { cursor, limit: 200 }, + }); + const mappings = toArray(page?.mappings); + if (!mappings.length && !page?.page?.hasMore) break; + scanned += mappings.length; + const items = mappings + .map((item) => buildNodeReference(item?.externalId, item?.namespace || namespace)) + .filter((item) => item.externalId); + if (items.length) { + const result = await this.requestV06("/trivium/bulk-delete", { + ...openOptions, + items, + }); + deleted += Number(result?.successCount ?? items.length) || 0; + } + if (!page?.page?.hasMore) break; + cursor = String(page?.page?.nextCursor || ""); + if (!cursor) break; + } + return { ok: true, scanned, deleted }; } async bulkUpsert(payload = {}) { - return await this.request("bulkUpsert", payload); + const namespace = getNamespace(payload); + const items = toArray(payload.items); + const missingVector = items.find((item) => !normalizeVector(item?.vector || item?.embedding).length); + if (missingVector) { + throw new Error("Authority Trivium v0.6 bulkUpsert requires vector for every item"); + } + const mappedItems = items.map((item) => { + const nodeId = normalizeRecordId(item?.externalId || item?.nodeId || item?.id); + const payloadSource = buildV06PayloadSource(item?.payload); + return { + externalId: nodeId, + namespace, + vector: normalizeVector(item?.vector || item?.embedding), + payload: { + ...payloadSource, + nodeId: payloadSource.nodeId || nodeId, + externalId: payloadSource.externalId || nodeId, + collectionId: payload.collectionId || payloadSource.collectionId || "", + text: payloadSource.text || item?.text || "", + contentHash: payloadSource.contentHash || item?.hash || "", + index: Number(item?.index || payloadSource.index || 0) || 0, + }, + }; + }); + return await this.requestV06("/trivium/bulk-upsert", { + ...this.buildOpenOptions(payload), + items: mappedItems, + }); } async deleteMany(payload = {}) { - return await this.request("deleteMany", payload); + const namespace = getNamespace(payload); + const ids = [ + ...toArray(payload.ids), + ...toArray(payload.externalIds), + ...toArray(payload.items).map((item) => item?.externalId || item?.nodeId || item?.id), + ].map(normalizeRecordId).filter(Boolean); + return await this.requestV06("/trivium/bulk-delete", { + ...this.buildOpenOptions(payload), + items: ids.map((id) => buildNodeReference(id, namespace)), + }); } async linkMany(payload = {}) { - return await this.request("linkMany", payload); + const namespace = getNamespace(payload); + const sourceLinks = toArray(payload.links || payload.items); + return await this.requestV06("/trivium/bulk-link", { + ...this.buildOpenOptions(payload), + items: sourceLinks + .map((link) => { + const src = normalizeRecordId(link?.fromId || link?.src || link?.sourceId); + const dst = normalizeRecordId(link?.toId || link?.dst || link?.targetId); + if (!src || !dst) return null; + return { + src: buildNodeReference(src, namespace), + dst: buildNodeReference(dst, namespace), + label: String(link?.relation || link?.label || "related"), + weight: Number(link?.weight ?? link?.strength ?? 1) || 1, + }; + }) + .filter(Boolean), + }); } async search(payload = {}) { - return await this.request("search", payload); + const vector = normalizeVector(payload.vector || payload.embedding || payload.queryVector); + if (!vector.length) { + throw new Error("Authority Trivium v0.6 search requires vector"); + } + const queryText = String(payload.queryText || payload.text || payload.searchText || payload.query || ""); + const body = { + ...this.buildOpenOptions(payload), + vector, + topK: Number(payload.topK || payload.limit || 0) || undefined, + expandDepth: Number(payload.expandDepth || payload.depth || 0) || undefined, + minScore: Number.isFinite(Number(payload.minScore)) ? Number(payload.minScore) : undefined, + ...(payload.payloadFilter || payload.filter ? { payloadFilter: payload.payloadFilter || payload.filter } : {}), + }; + if (queryText) { + return await this.requestV06("/trivium/search-hybrid", { + ...body, + queryText, + hybridAlpha: Number.isFinite(Number(payload.hybridAlpha)) ? Number(payload.hybridAlpha) : undefined, + }); + } + return await this.requestV06("/trivium/search", body); } async filterWhere(payload = {}) { - return await this.request("filterWhere", payload); + const namespace = getNamespace(payload); + const result = await this.requestV06("/trivium/list-mappings", { + ...this.buildOpenOptions(payload), + namespace, + page: { + limit: Number(payload.limit || payload.topK || payload.pageSize || 100) || 100, + }, + }); + return { items: toArray(result?.mappings) }; } async queryPage(payload = {}) { - return await this.request("queryPage", payload); + return await this.filterWhere(payload); } async neighbors(payload = {}) { - return await this.request("neighbors", payload); + const namespace = getNamespace(payload); + const seedIds = [ + ...toArray(payload.ids), + ...toArray(payload.nodeIds), + ...toArray(payload.seedIds), + payload.id, + ].map(normalizeRecordId).filter(Boolean); + const openOptions = this.buildOpenOptions(payload); + const resolved = await this.requestV06("/trivium/resolve-many", { + ...openOptions, + items: seedIds.map((id) => buildNodeReference(id, namespace)), + }); + const neighbors = []; + for (const item of toArray(resolved?.items)) { + const internalId = Number(item?.id); + if (!Number.isFinite(internalId) || internalId <= 0) continue; + const result = await this.requestV06("/trivium/neighbors", { + ...openOptions, + id: internalId, + depth: Number(payload.depth || payload.expandDepth || 1) || 1, + }); + for (const node of toArray(result?.nodes)) { + neighbors.push({ + externalId: node?.externalId, + nodeId: node?.externalId, + id: node?.id, + namespace: node?.namespace, + }); + } + } + return { neighbors }; } async stat(payload = {}) { - return await this.request("stat", payload); + return await this.requestV06("/trivium/stat", { + ...this.buildOpenOptions(payload), + ...(payload.includeMappingIntegrity ? { includeMappingIntegrity: true } : {}), + }); } } @@ -313,9 +566,13 @@ export function createAuthorityTriviumClient(config = {}, options = {}) { const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient; if (injected) return injected; return new AuthorityTriviumHttpClient({ + ...config, baseUrl: config.baseUrl, fetchImpl: options.fetchImpl || config.fetchImpl, headerProvider: options.headerProvider || config.headerProvider, + protocol: config.protocol, + sessionToken: options.sessionToken || config.sessionToken, + sessionInitConfig: options.sessionInitConfig || config.sessionInitConfig, }); } @@ -451,6 +708,8 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti chatId: options.chatId, text: String(text || ""), searchText: String(text || ""), + vector: normalizeVector(options.vector || options.queryVector || options.embedding), + queryVector: normalizeVector(options.queryVector || options.vector || options.embedding), topK: Math.max(1, Math.floor(Number(options.topK) || 1)), candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean), }); @@ -458,11 +717,15 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti } export async function testAuthorityTriviumConnection(config = {}, options = {}) { + const probeVector = await embedText("test connection", config, { isQuery: true }); + if (!probeVector || probeVector.length === 0) { + return { success: false, dimensions: 0, error: "Embedding API 返回空结果" }; + } const client = createAuthorityTriviumClient(config, options); await callClient(client, ["stat"], "stat", { namespace: options.namespace, collectionId: options.collectionId, chatId: options.chatId, }); - return { success: true, dimensions: 0, error: "" }; + return { success: true, dimensions: probeVector.length, error: "" }; } diff --git a/vector/embedding.js b/vector/embedding.js index 11f07a0..d90f2d7 100644 --- a/vector/embedding.js +++ b/vector/embedding.js @@ -6,11 +6,17 @@ * 调用外部 API 获取文本向量，并提供暴力搜索 cosine 相似度 */ +import { getRequestHeaders } from "../../../../../script.js"; import { extension_settings } from "../../../../extensions.js"; import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js"; const MODULE_NAME = "st_bme"; const EMBEDDING_REQUEST_TIMEOUT_MS = 300000; +const BACKEND_SOURCES_REQUIRING_API_URL = new Set([ + "ollama", + "llamacpp", + "vllm", +]); function getEmbeddingTestOverride(name) { const override = globalThis.__stBmeTestOverrides?.embedding?.[name]; @@ -41,6 +47,69 @@ function normalizeOpenAICompatibleBaseUrl(value) { .replace(/\/+$/, ""); } +function normalizeVector(value) { + if (!Array.isArray(value)) return null; + const vector = value.map((item) => Number(item)).filter((item) => Number.isFinite(item)); + return vector.length ? new Float64Array(vector) : null; +} + +function readEmbeddingMode(config = {}) { + return String(config?.embeddingMode || config?.mode || "direct").trim().toLowerCase(); +} + +function readEmbeddingSource(config = {}) { + return String(config?.embeddingSource || config?.source || "openai").trim().toLowerCase() || "openai"; +} + +function buildBackendEmbeddingRequestBody(config = {}, payload = {}) { + const source = readEmbeddingSource(config); + const body = { + source, + model: String(config?.model || "").trim(), + isQuery: Boolean(payload.isQuery), + }; + if (payload.text !== undefined) { + body.text = String(payload.text ?? ""); + } + if (Array.isArray(payload.texts)) { + body.texts = payload.texts.map((item) => String(item ?? "")); + } + if (BACKEND_SOURCES_REQUIRING_API_URL.has(source)) { + body.apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl); + } + if (source === "ollama") { + body.keep = false; + } + return body; +} + +async function requestBackendEmbeddings(config = {}, payload = {}, { signal } = {}) { + const response = await fetchWithTimeout( + "/api/vector/embed", + { + method: "POST", + headers: { + ...getRequestHeaders(), + "Content-Type": "application/json", + }, + signal, + body: JSON.stringify(buildBackendEmbeddingRequestBody(config, payload)), + }, + getConfiguredTimeoutMs(config), + ); + + if (!response.ok) { + const errorText = await response.text().catch(() => response.statusText); + console.error( + `[ST-BME] Backend Embedding API 错误 (${response.status}):`, + errorText, + ); + return null; + } + + return await response.json().catch(() => ({})); +} + function createCombinedAbortSignal(...signals) { const validSignals = signals.filter(Boolean); if (validSignals.length <= 1) { @@ -107,10 +176,31 @@ async function fetchWithTimeout( * @param {string} config.model - 模型名（如 text-embedding-3-small） * @returns {Promise} 向量或 null */ -export async function embedText(text, config, { signal } = {}) { +export async function embedText(text, config, { signal, isQuery = false } = {}) { const override = getEmbeddingTestOverride("embedText"); if (override) { - return await override(text, config, { signal }); + return await override(text, config, { signal, isQuery }); + } + + if (readEmbeddingMode(config) === "backend") { + if (!text || !config?.model) { + console.warn("[ST-BME] Embedding 配置不完整，跳过"); + return null; + } + try { + const payload = await requestBackendEmbeddings( + config, + { text, isQuery }, + { signal }, + ); + return normalizeVector(payload?.vector); + } catch (e) { + if (isAbortError(e)) { + throw e; + } + console.error("[ST-BME] Backend Embedding 调用失败:", e); + return null; + } } const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl); @@ -173,10 +263,31 @@ export async function embedText(text, config, { signal } = {}) { * @param {object} config * @returns {Promise<(Float64Array|null)[]>} */ -export async function embedBatch(texts, config, { signal } = {}) { +export async function embedBatch(texts, config, { signal, isQuery = false } = {}) { const override = getEmbeddingTestOverride("embedBatch"); if (override) { - return await override(texts, config, { signal }); + return await override(texts, config, { signal, isQuery }); + } + + if (readEmbeddingMode(config) === "backend") { + if (!texts.length || !config?.model) { + return texts.map(() => null); + } + try { + const payload = await requestBackendEmbeddings( + config, + { texts, isQuery }, + { signal }, + ); + const vectors = Array.isArray(payload?.vectors) ? payload.vectors : []; + return texts.map((_, index) => normalizeVector(vectors[index])); + } catch (e) { + if (isAbortError(e)) { + throw e; + } + console.error("[ST-BME] Backend Embedding 批量调用失败:", e); + return texts.map(() => null); + } } const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl); diff --git a/vector/vector-index.js b/vector/vector-index.js index 9da3d54..fe1dc10 100644 --- a/vector/vector-index.js +++ b/vector/vector-index.js @@ -236,6 +236,9 @@ export function getVectorModelScope(config) { return [ "authority", config.source || "authority-trivium", + config.embeddingMode || "direct", + config.embeddingSource || "direct", + normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix), normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""), config.model || "", ].join("|"); @@ -266,6 +269,20 @@ export function validateVectorConfig(config) { if (!config.baseUrl) { return { valid: false, error: "Authority Trivium 地址不可用" }; } + if (!config.model) { + return { valid: false, error: "请先填写 Embedding 模型（Authority 默认复用当前用户设置）" }; + } + const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase(); + const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase(); + if (authorityEmbeddingMode === "backend") { + if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) { + return { valid: false, error: "当前后端 Embedding 源需要 API 地址（Authority 默认复用当前用户设置）" }; + } + return { valid: true, error: "" }; + } + if (!config.apiUrl) { + return { valid: false, error: "请先填写 Embedding API 地址（Authority 默认复用当前用户设置）" }; + } return { valid: true, error: "" }; } @@ -606,7 +623,7 @@ function markBackendVectorStateDirty( function markAuthorityVectorStateDirty( graph, - config, + config = {}, reason = "authority-trivium-failed", warning = "Authority Trivium 索引失败，已标记待重建", ) { @@ -640,6 +657,42 @@ function markAuthorityVectorStateDirty( state.lastWarning = String(warning || "Authority Trivium 索引失败，已标记待重建"); } +async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) { + const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node])); + const entriesToEmbed = []; + for (const entry of entries || []) { + const node = nodesById.get(String(entry?.nodeId || "")); + const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0; + if (node && !hasEmbedding) { + entriesToEmbed.push({ entry, node }); + } + } + if (!entriesToEmbed.length) { + return { requested: 0, failures: 0, elapsedMs: 0 }; + } + throwIfAborted(signal); + const startedAt = nowMs(); + const embeddings = await embedBatch( + entriesToEmbed.map(({ entry }) => entry.text), + config, + { signal }, + ); + let failures = 0; + for (let index = 0; index < entriesToEmbed.length; index++) { + const embedding = embeddings[index]; + if (embedding) { + entriesToEmbed[index].node.embedding = Array.from(embedding); + } else { + failures += 1; + } + } + return { + requested: entriesToEmbed.length, + failures, + elapsedMs: nowMs() - startedAt, + }; +} + export async function syncGraphVectorIndex( graph, config, @@ -741,6 +794,12 @@ export async function syncGraphVectorIndex( try { if (fullReset) { + const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal); + embeddingsRequested += embeddingResult.requested; + embedBatchMs += embeddingResult.elapsedMs; + if (embeddingResult.failures > 0) { + throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`); + } const purgeStartedAt = nowMs(); await purgeAuthorityTriviumNamespace(config, authorityOptions); authorityPurgeMs += nowMs() - purgeStartedAt; @@ -794,6 +853,12 @@ export async function syncGraphVectorIndex( queuedNodeIds.add(entry.nodeId); } + const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal); + embeddingsRequested += embeddingResult.requested; + embedBatchMs += embeddingResult.elapsedMs; + if (embeddingResult.failures > 0) { + throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`); + } deletedNodeCount = nodeIdsToDelete.length; const deleteStartedAt = nowMs(); await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions); @@ -1108,7 +1173,7 @@ export async function findSimilarNodesByText( if (isDirectVectorConfig(config)) { const queryEmbedStartedAt = nowMs(); - const queryVec = await embedText(text, config, { signal }); + const queryVec = await embedText(text, config, { signal, isQuery: true }); const queryEmbedMs = nowMs() - queryEmbedStartedAt; if (!queryVec) { recordSearchTimings({ @@ -1157,6 +1222,18 @@ export async function findSimilarNodesByText( if (isAuthorityVectorConfig(config)) { const requestStartedAt = nowMs(); try { + const queryEmbedStartedAt = nowMs(); + const queryVec = await embedText(text, config, { signal, isQuery: true }); + const queryEmbedMs = nowMs() - queryEmbedStartedAt; + if (!queryVec) { + recordSearchTimings({ + success: false, + reason: "authority-query-embed-empty", + queryEmbedMs: roundMs(queryEmbedMs), + resultCount: 0, + }); + return []; + } const allowedIds = new Set(candidateNodes.map((node) => node.id)); const results = ( await searchAuthorityTriviumNodes(graph, text, config, { @@ -1166,6 +1243,7 @@ export async function findSimilarNodesByText( modelScope: getVectorModelScope(config), topK, candidateIds: candidateNodes.map((node) => node.id), + queryVector: Array.from(queryVec), signal, }) ) @@ -1174,6 +1252,7 @@ export async function findSimilarNodesByText( recordSearchTimings({ success: true, reason: "ok", + queryEmbedMs: roundMs(queryEmbedMs), requestMs: roundMs(nowMs() - requestStartedAt), resultCount: results.length, });