taozuqin/ST-Bionic-Memory-Ecology
1
0
Fork 0
mirror of https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git synced 2026-05-15 14:20:35 +08:00
Code Issues Packages Projects Releases Wiki Activity

refactor(authority): complete v0.6-only sql/blob/jobs rollout

Browse Source
This commit is contained in:
Youzini-afk
2026-04-28 21:45:21 +08:00
parent a5b822682a
commit a7e2edac88
23 changed files with 1816 additions and 212 deletions
Download Patch File Download Diff File Expand all files Collapse all files

133
maintenance/authority-blob-adapter.js
View File

@@ -1,6 +1,7 @@
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
export const AUTHORITY_BLOB_ENDPOINT = "/v1/blob"; export const AUTHORITY_BLOB_ENDPOINT = "/fs/private";
function toPlainData(value, fallbackValue = null) { function toPlainData(value, fallbackValue = null) {
if (value == null) return fallbackValue; if (value == null) return fallbackValue;
@@ -97,7 +98,8 @@ function normalizeBlobRecordSource(input = null) {
export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") { export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") {
const source = normalizeBlobRecordSource(input); const source = normalizeBlobRecordSource(input);
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); const entry = source.entry && typeof source.entry === "object" ? source.entry : null;
const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath);
const missing = const missing =
source.exists === false || source.exists === false ||
source.found === false || source.found === false ||
@@ -118,21 +120,22 @@ export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = ""
payload: normalizeBlobPayload(input), payload: normalizeBlobPayload(input),
contentType: String(source.contentType || source.type || "application/json"), contentType: String(source.contentType || source.type || "application/json"),
etag: String(source.etag || source.hash || ""), etag: String(source.etag || source.hash || ""),
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "",
raw: toPlainData(input, input), raw: toPlainData(input, input),
}; };
} }
export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") { export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") {
const source = normalizeBlobRecordSource(input); const source = normalizeBlobRecordSource(input);
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath); const entry = source.entry && typeof source.entry === "object" ? source.entry : null;
const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath);
return { return {
ok: input == null ? true : source.ok !== false && source.error == null, ok: input == null ? true : source.ok !== false && source.error == null,
path, path,
url: String(source.url || source.href || ""), url: String(source.url || source.href || ""),
size: normalizeInteger(source.size || source.bytes, 0, 0), size: normalizeInteger(source.size || source.bytes || entry?.sizeBytes, 0, 0),
etag: String(source.etag || source.hash || ""), etag: String(source.etag || source.hash || ""),
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "", updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "",
raw: toPlainData(input, input), raw: toPlainData(input, input),
}; };
} }
@@ -170,49 +173,56 @@ export function normalizeAuthorityBlobConfig(settings = {}, overrides = {}) {
export class AuthorityBlobHttpClient { export class AuthorityBlobHttpClient {
constructor(options = {}) { constructor(options = {}) {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); this.http = new AuthorityHttpClient({
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); ...options,
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
});
} }
async request(action, payload = {}) { async request(path, payload = {}, options = {}) {
if (typeof this.fetchImpl !== "function") { return await this.http.requestJson(path, {
throw new Error("Authority Blob fetch unavailable"); method: options.method || "POST",
} body: payload,
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_BLOB_ENDPOINT}`, { session: true,
method: "POST", signal: options.signal,
headers: {
Accept: "application/json",
"Content-Type": "application/json",
...(this.headerProvider ? this.headerProvider() || {} : {}),
},
body: JSON.stringify({ action, ...payload }),
}); });
if (!response?.ok) {
const text = await response?.text?.().catch(() => "");
throw new Error(text || `Authority Blob HTTP ${response?.status || "unknown"}`);
}
return await response.json().catch(() => ({}));
} }
async writeJson(payload = {}) { async writeJson(payload = {}) {
return await this.request("writeJson", payload); return await this.writeText({
...payload,
contentType: payload.contentType || "application/json",
text: JSON.stringify(toPlainData(payload.payload ?? payload.data, payload.payload ?? payload.data)),
});
} }
async writeText(payload = {}) { async writeText(payload = {}) {
return await this.request("writeText", payload); return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/write-file`, {
path: normalizeAuthorityBlobPath(payload.path || payload.name),
content: String(payload.text ?? payload.data ?? payload.content ?? ""),
encoding: "utf8",
createParents: true,
}, { signal: payload.signal });
} }
async readJson(payload = {}) { async readJson(payload = {}) {
return await this.request("readJson", payload); return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/read-file`, {
path: normalizeAuthorityBlobPath(payload.path || payload.name),
encoding: "utf8",
}, { signal: payload.signal });
} }
async delete(payload = {}) { async delete(payload = {}) {
return await this.request("delete", payload); return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/delete`, {
path: normalizeAuthorityBlobPath(payload.path || payload.name),
recursive: false,
}, { signal: payload.signal });
} }
async stat(payload = {}) { async stat(payload = {}) {
return await this.request("stat", payload); return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/stat`, {
path: normalizeAuthorityBlobPath(payload.path || payload.name),
}, { signal: payload.signal });
} }
} }
@@ -249,6 +259,10 @@ function throwIfAborted(signal) {
} }
} }
function isMissingAuthorityBlobError(error) {
return Number(error?.status || 0) === 404;
}
export class AuthorityBlobAdapter { export class AuthorityBlobAdapter {
constructor(config = {}, options = {}) { constructor(config = {}, options = {}) {
this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {}); this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {});
@@ -291,36 +305,57 @@ export class AuthorityBlobAdapter {
throwIfAborted(options.signal); throwIfAborted(options.signal);
const normalizedPath = normalizeAuthorityBlobPath(path); const normalizedPath = normalizeAuthorityBlobPath(path);
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", { try {
namespace: options.namespace || this.config.namespace, const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", {
path: normalizedPath, namespace: options.namespace || this.config.namespace,
name: normalizedPath, path: normalizedPath,
}); name: normalizedPath,
return normalizeAuthorityBlobReadResult(result, normalizedPath); });
return normalizeAuthorityBlobReadResult(result, normalizedPath);
} catch (error) {
if (isMissingAuthorityBlobError(error)) {
return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath);
}
throw error;
}
} }
async delete(path, options = {}) { async delete(path, options = {}) {
throwIfAborted(options.signal); throwIfAborted(options.signal);
const normalizedPath = normalizeAuthorityBlobPath(path); const normalizedPath = normalizeAuthorityBlobPath(path);
if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, ""); if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, "");
const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", { try {
namespace: options.namespace || this.config.namespace, const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", {
path: normalizedPath, namespace: options.namespace || this.config.namespace,
name: normalizedPath, path: normalizedPath,
}); name: normalizedPath,
return normalizeAuthorityBlobDeleteResult(result, normalizedPath); });
return normalizeAuthorityBlobDeleteResult(result, normalizedPath);
} catch (error) {
if (isMissingAuthorityBlobError(error)) {
return normalizeAuthorityBlobDeleteResult({ exists: false, status: 404 }, normalizedPath);
}
throw error;
}
} }
async stat(path, options = {}) { async stat(path, options = {}) {
throwIfAborted(options.signal); throwIfAborted(options.signal);
const normalizedPath = normalizeAuthorityBlobPath(path); const normalizedPath = normalizeAuthorityBlobPath(path);
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, ""); if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", { try {
namespace: options.namespace || this.config.namespace, const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", {
path: normalizedPath, namespace: options.namespace || this.config.namespace,
name: normalizedPath, path: normalizedPath,
}); name: normalizedPath,
return normalizeAuthorityBlobReadResult(result, normalizedPath); });
return normalizeAuthorityBlobReadResult(result, normalizedPath);
} catch (error) {
if (isMissingAuthorityBlobError(error)) {
return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath);
}
throw error;
}
} }
} }

78
maintenance/authority-job-adapter.js
View File

@@ -1,6 +1,7 @@
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
export const AUTHORITY_JOB_ENDPOINT = "/v1/jobs"; export const AUTHORITY_JOB_ENDPOINT = "/jobs";
export const AUTHORITY_JOB_STATUS_TERMINAL = new Set([ export const AUTHORITY_JOB_STATUS_TERMINAL = new Set([
"completed", "completed",
"succeeded", "succeeded",
@@ -158,8 +159,8 @@ export function normalizeAuthorityJobList(payload = null) {
const jobs = readJobRows(payload).map(normalizeAuthorityJobRecord).filter((job) => job.id); const jobs = readJobRows(payload).map(normalizeAuthorityJobRecord).filter((job) => job.id);
return { return {
jobs, jobs,
nextCursor: String(source.nextCursor || source.next_cursor || source.cursor?.next || ""), nextCursor: String(source.nextCursor || source.next_cursor || source.page?.nextCursor || source.cursor?.next || ""),
hasMore: Boolean(source.hasMore || source.has_more || source.cursor?.hasMore), hasMore: Boolean(source.hasMore || source.has_more || source.page?.hasMore || source.cursor?.hasMore),
raw: toPlainData(payload, payload), raw: toPlainData(payload, payload),
}; };
} }
@@ -251,49 +252,68 @@ export function normalizeAuthorityJobConfig(settings = {}, overrides = {}) {
export class AuthorityJobHttpClient { export class AuthorityJobHttpClient {
constructor(options = {}) { constructor(options = {}) {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); this.http = new AuthorityHttpClient({
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); ...options,
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
});
} }
async request(action, payload = {}) { async request(path, payload = {}, options = {}) {
if (typeof this.fetchImpl !== "function") { return await this.http.requestJson(path, {
throw new Error("Authority Jobs fetch unavailable"); method: options.method || "POST",
} body: payload,
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_JOB_ENDPOINT}`, { session: true,
method: "POST", signal: options.signal,
headers: {
Accept: "application/json",
"Content-Type": "application/json",
...(this.headerProvider ? this.headerProvider() || {} : {}),
},
body: JSON.stringify({ action, ...payload }),
}); });
if (!response?.ok) {
const text = await response?.text?.().catch(() => "");
throw new Error(text || `Authority Jobs HTTP ${response?.status || "unknown"}`);
}
return await response.json().catch(() => ({}));
} }
async submit(payload = {}) { async submit(payload = {}) {
return await this.request("submit", payload); return await this.request(`${AUTHORITY_JOB_ENDPOINT}/create`, {
type: String(payload.type || payload.kind || "").trim(),
payload: toPlainData(payload.payload, payload.payload),
...(payload.timeoutMs != null ? { timeoutMs: normalizeInteger(payload.timeoutMs, 0, 0, 3600000) } : {}),
...(payload.idempotencyKey ? { idempotencyKey: String(payload.idempotencyKey) } : {}),
...(payload.maxAttempts != null ? { maxAttempts: normalizeInteger(payload.maxAttempts, 1, 1, 1000) } : {}),
});
} }
async listPage(payload = {}) { async listPage(payload = {}) {
return await this.request("listPage", payload); return await this.request(`${AUTHORITY_JOB_ENDPOINT}/list`, {
page: {
...(payload.cursor ? { cursor: String(payload.cursor) } : {}),
limit: normalizeInteger(payload.limit, 20, 1, 100),
},
...(payload.filter && typeof payload.filter === "object" && !Array.isArray(payload.filter) && Object.keys(payload.filter).length > 0
? { filter: toPlainData(payload.filter, {}) }
: {}),
});
} }
async waitForCompletion(payload = {}) { async get(payload = {}) {
return await this.request("waitForCompletion", payload); const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
if (!id) {
throw new Error("Authority Jobs get requires job id");
}
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}`, undefined, {
method: "GET",
signal: payload.signal,
});
} }
async requeue(payload = {}) { async requeue(payload = {}) {
return await this.request("requeue", payload); const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
if (!id) {
throw new Error("Authority Jobs requeue requires job id");
}
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/requeue`, {}, { signal: payload.signal });
} }
async cancel(payload = {}) { async cancel(payload = {}) {
return await this.request("cancel", payload); const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
if (!id) {
throw new Error("Authority Jobs cancel requires job id");
}
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/cancel`, {}, { signal: payload.signal });
} }
} }

7
retrieval/authority-candidate-provider.js
View File

@@ -9,6 +9,7 @@ import {
queryAuthorityTriviumNeighbors, queryAuthorityTriviumNeighbors,
searchAuthorityTriviumNodes, searchAuthorityTriviumNodes,
} from "../vector/authority-vector-primary-adapter.js"; } from "../vector/authority-vector-primary-adapter.js";
import { embedText } from "../vector/embedding.js";
function nowMs() { function nowMs() {
if (typeof performance?.now === "function") { if (typeof performance?.now === "function") {
@@ -247,6 +248,11 @@ export async function resolveAuthorityRecallCandidates({
const searchStartedAt = nowMs(); const searchStartedAt = nowMs();
for (const queryEntry of queryPlan.queries) { for (const queryEntry of queryPlan.queries) {
try { try {
const queryVec = await embedText(queryEntry.text, embeddingConfig, { signal, isQuery: true });
if (!queryVec) {
diagnostics.fallbackReason = diagnostics.fallbackReason || "authority-candidate-query-embed-empty";
continue;
}
const searchResults = await searchAuthorityTriviumNodes( const searchResults = await searchAuthorityTriviumNodes(
graph, graph,
queryEntry.text, queryEntry.text,
@@ -257,6 +263,7 @@ export async function resolveAuthorityRecallCandidates({
chatId, chatId,
topK: limit, topK: limit,
candidateIds: filteredIds.length > 0 ? filteredIds : undefined, candidateIds: filteredIds.length > 0 ? filteredIds : undefined,
queryVector: Array.from(queryVec),
signal, signal,
}, },
); );

2
retrieval/retrieval-enhancer.js
View File

@@ -484,7 +484,7 @@ export async function runResidualRecall({
}; };
} }
const queryVec = await embedText(queryText, embeddingConfig, { signal }); const queryVec = await embedText(queryText, embeddingConfig, { signal, isQuery: true });
if (!queryVec || queryVec.length === 0) { if (!queryVec || queryVec.length === 0) {
return { return {
triggered: false, triggered: false,

244
runtime/authority-capabilities.js
View File

@@ -1,11 +1,11 @@
const DEFAULT_AUTHORITY_BASE_URL = "/api/plugins/authority"; const DEFAULT_AUTHORITY_BASE_URL = "/api/plugins/authority";
const DEFAULT_AUTHORITY_PROBE_INTERVAL_MS = 60000; const DEFAULT_AUTHORITY_PROBE_INTERVAL_MS = 60000;
const SQL_FEATURES = ["sql", "sql.query", "sql.page", "sql.pageall", "querysql"]; const SQL_FEATURES = ["sql", "sql.query", "sql.querypage", "sql.page", "sql.pageall", "querysql"];
const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.write", "sql.transaction"]; const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.exec", "sql.write", "sql.transaction"];
const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert"]; const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert", "trivium.upsert", "trivium.bulkmutations"];
const JOB_FEATURES = ["jobs", "jobs.list", "jobs.wait", "events", "sse"]; const JOB_FEATURES = ["jobs", "jobs.background", "jobs.list", "jobs.wait", "diagnostics.jobspage", "events", "sse"];
const BLOB_FEATURES = ["blob", "blob.write", "privatefiles", "private.files", "files.private"]; const BLOB_FEATURES = ["blob", "blob.write", "storage.blob", "transfers.blob", "transfers.fs", "fs.private", "privatefiles", "private.files", "files.private"];
function toBoolean(value, fallback = false) { function toBoolean(value, fallback = false) {
if (typeof value === "boolean") return value; if (typeof value === "boolean") return value;
@@ -89,6 +89,189 @@ function readNowMs() {
return Date.now(); return Date.now();
} }
function clonePlain(value, fallbackValue = null) {
if (value == null) return fallbackValue;
if (typeof globalThis.structuredClone === "function") {
try {
return globalThis.structuredClone(value);
} catch {
}
}
try {
return JSON.parse(JSON.stringify(value));
} catch {
return fallbackValue;
}
}
function normalizeHeaderName(name = "") {
return String(name || "").trim().toLowerCase();
}
function buildDefaultSessionInitConfig(source = {}) {
const config = source && typeof source === "object" && !Array.isArray(source) ? source : {};
return {
extensionId: String(config.extensionId || "third-party/st-bme"),
displayName: String(config.displayName || "ST-BME"),
version: String(config.version || "0.0.0"),
installType: String(config.installType || "local"),
declaredPermissions: clonePlain(config.declaredPermissions, null) || {
storage: { kv: true, blob: true },
fs: { private: true },
sql: { private: true },
trivium: { private: true },
jobs: { background: true },
events: { channels: true },
},
...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}),
};
}
function withJsonHeaders(headers = {}) {
return {
Accept: "application/json",
"Content-Type": "application/json",
...(headers || {}),
};
}
async function readResponsePayload(response = null) {
if (!response) return {};
if (typeof response.json === "function") {
try {
return await response.json();
} catch {
}
}
if (typeof response.text === "function") {
try {
return { error: await response.text() };
} catch {
return {};
}
}
return {};
}
function readPayloadMessage(payload = {}, fallback = "") {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback;
return String(payload.error || payload.message || payload.reason || fallback || "");
}
function buildAuthorityPermissionEvaluateRequests(settings = {}, readiness = {}, options = {}) {
const requests = [];
const sqlTarget = String(options.sqlTarget || settings.sqlTarget || "default");
const triviumTarget = String(options.triviumTarget || settings.triviumTarget || "st_bme_vectors");
const jobTarget = String(options.jobTarget || settings.jobTarget || "delay");
if (readiness.sql || readiness.sqlMutation) {
requests.push({ resource: "sql.private", target: sqlTarget, reason: `Probe SQL capability for ${sqlTarget}` });
}
if (readiness.trivium) {
requests.push({ resource: "trivium.private", target: triviumTarget, reason: `Probe Trivium capability for ${triviumTarget}` });
}
if (readiness.blob) {
requests.push({ resource: "fs.private", reason: "Probe private file capability for Authority Blob adapter" });
}
if (readiness.jobs) {
requests.push({ resource: "jobs.background", target: jobTarget, reason: `Probe Jobs capability for ${jobTarget}` });
}
return requests;
}
async function verifyAuthorityDataPlane(baseUrl, fetchImpl, headers, settings = {}, readiness = {}, options = {}) {
const initHeaders = withJsonHeaders(headers);
const initResponse = await fetchImpl(`${baseUrl}/session/init`, {
method: "POST",
headers: initHeaders,
body: JSON.stringify(buildDefaultSessionInitConfig(options.sessionInitConfig || settings)),
});
const initStatus = Number(initResponse?.status || 0);
const initPayload = await readResponsePayload(initResponse);
if (!initResponse?.ok) {
return {
sessionReady: false,
permissionReady: false,
reason: initStatus === 401 || initStatus === 403 ? "session-init-denied" : "session-init-failed",
lastError: readPayloadMessage(initPayload, `HTTP ${initStatus || "unknown"}`),
status: initStatus,
};
}
const sessionToken = String(initPayload?.sessionToken || initPayload?.token || "");
if (!sessionToken) {
return {
sessionReady: false,
permissionReady: false,
reason: "session-token-missing",
lastError: "session token missing",
status: initStatus,
};
}
const sessionHeaders = {
...withJsonHeaders(headers),
...(Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === "x-authority-session-token")
? {}
: { "x-authority-session-token": sessionToken }),
};
const currentResponse = await fetchImpl(`${baseUrl}/session/current`, {
method: "GET",
headers: sessionHeaders,
});
const currentStatus = Number(currentResponse?.status || 0);
const currentPayload = await readResponsePayload(currentResponse);
if (!currentResponse?.ok) {
return {
sessionReady: false,
permissionReady: false,
reason: currentStatus === 401 || currentStatus === 403 ? "session-invalid" : "session-current-failed",
lastError: readPayloadMessage(currentPayload, `HTTP ${currentStatus || "unknown"}`),
status: currentStatus,
};
}
const requests = buildAuthorityPermissionEvaluateRequests(settings, readiness, options);
if (!requests.length) {
return {
sessionReady: true,
permissionReady: true,
reason: "ok",
lastError: "",
status: currentStatus,
};
}
const permissionResponse = await fetchImpl(`${baseUrl}/permissions/evaluate-batch`, {
method: "POST",
headers: sessionHeaders,
body: JSON.stringify({ requests }),
});
const permissionStatus = Number(permissionResponse?.status || 0);
const permissionPayload = await readResponsePayload(permissionResponse);
if (!permissionResponse?.ok) {
return {
sessionReady: true,
permissionReady: false,
reason: permissionStatus === 401 || permissionStatus === 403 ? "permission-denied" : "permission-evaluate-failed",
lastError: readPayloadMessage(permissionPayload, `HTTP ${permissionStatus || "unknown"}`),
status: permissionStatus,
};
}
const results = Array.isArray(permissionPayload?.results) ? permissionPayload.results : [];
const permissionReady = results.length === requests.length && results.every((result) => {
const decision = String(result?.decision || result?.grant?.status || "").trim().toLowerCase();
return decision === "granted";
});
return {
sessionReady: true,
permissionReady,
reason: permissionReady ? "ok" : "permission-not-ready",
lastError: permissionReady ? "" : "required Authority permissions are not granted",
status: permissionStatus || currentStatus,
};
}
export function normalizeAuthorityBaseUrl(baseUrl = DEFAULT_AUTHORITY_BASE_URL) { export function normalizeAuthorityBaseUrl(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
const normalized = String(baseUrl || DEFAULT_AUTHORITY_BASE_URL).trim() || DEFAULT_AUTHORITY_BASE_URL; const normalized = String(baseUrl || DEFAULT_AUTHORITY_BASE_URL).trim() || DEFAULT_AUTHORITY_BASE_URL;
return normalized.replace(/\/+$/, ""); return normalized.replace(/\/+$/, "");
@@ -116,12 +299,7 @@ export function normalizeAuthoritySettings(settings = {}) {
export function buildAuthorityProbeUrls(baseUrl = DEFAULT_AUTHORITY_BASE_URL) { export function buildAuthorityProbeUrls(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
const normalizedBaseUrl = normalizeAuthorityBaseUrl(baseUrl); const normalizedBaseUrl = normalizeAuthorityBaseUrl(baseUrl);
return [ return [`${normalizedBaseUrl}/probe`];
`${normalizedBaseUrl}/v1/diagnostics/probe`,
`${normalizedBaseUrl}/v1/probe`,
`${normalizedBaseUrl}/probe`,
normalizedBaseUrl,
];
} }
export function collectAuthorityFeatures(payload = {}) { export function collectAuthorityFeatures(payload = {}) {
@@ -295,7 +473,7 @@ export async function probeAuthorityCapabilities(options = {}) {
for (const endpoint of buildAuthorityProbeUrls(settings.baseUrl)) { for (const endpoint of buildAuthorityProbeUrls(settings.baseUrl)) {
const startedAt = readNowMs(); const startedAt = readNowMs();
try { try {
const response = await fetchImpl(endpoint, { method: "GET", headers }); const response = await fetchImpl(endpoint, { method: "POST", headers });
const finishedAt = readNowMs(); const finishedAt = readNowMs();
const status = Number(response?.status || 0); const status = Number(response?.status || 0);
lastStatus = status; lastStatus = status;
@@ -340,13 +518,43 @@ export async function probeAuthorityCapabilities(options = {}) {
} catch { } catch {
payload = {}; payload = {};
} }
return normalizeAuthorityProbeResponse(payload, { const features = collectAuthorityFeatures(payload);
const readiness = createFeatureReadiness(features);
const missingFeatures = collectMissingFeatures(readiness);
const healthy = payload?.healthy ?? payload?.ok ?? true;
let sessionReady = payload?.sessionReady ?? payload?.session?.ready ?? payload?.session?.active;
let permissionReady = payload?.permissionReady ?? payload?.permissions?.ready ?? payload?.authorized;
let reason = missingFeatures.length ? "missing-required-features" : "ok";
let dataPlaneLastError = "";
let dataPlaneStatus = status;
if (healthy) {
const verified = await verifyAuthorityDataPlane(settings.baseUrl, fetchImpl, headers, settings, readiness, options);
sessionReady = verified.sessionReady;
permissionReady = verified.permissionReady;
dataPlaneStatus = Number(verified.status || status || 0);
dataPlaneLastError = String(verified.lastError || "");
if (verified.reason && verified.reason !== "ok") {
reason = verified.reason;
}
}
return normalizeAuthorityCapabilityState(
{
installed: true,
healthy: Boolean(healthy),
sessionReady: Boolean(sessionReady),
permissionReady: Boolean(permissionReady),
features: Array.from(features),
missingFeatures,
reason,
lastError: dataPlaneLastError,
endpoint,
status: dataPlaneStatus,
latencyMs: normalizeLatencyMs(startedAt, finishedAt),
lastProbeAt: nowMs,
updatedAt: new Date(nowMs).toISOString(),
},
settings, settings,
endpoint, );
status,
latencyMs: normalizeLatencyMs(startedAt, finishedAt),
nowMs,
});
} catch (error) { } catch (error) {
lastError = error?.message || String(error); lastError = error?.message || String(error);
} }

194
runtime/authority-http-client.js Normal file
View File

@@ -0,0 +1,194 @@
import { normalizeAuthorityBaseUrl } from "./authority-capabilities.js";
export const AUTHORITY_PROTOCOL_AUTO = "auto";
export const AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06 = "server-plugin-v06";
export const AUTHORITY_SESSION_HEADER = "x-authority-session-token";
function normalizeProtocol(value = AUTHORITY_PROTOCOL_AUTO) {
const normalized = String(value || AUTHORITY_PROTOCOL_AUTO).trim().toLowerCase();
if (["v06", "v0.6", "server-plugin", AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06].includes(normalized)) {
return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
}
return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
}
function clonePlain(value, fallbackValue = null) {
if (value == null) return fallbackValue;
if (typeof globalThis.structuredClone === "function") {
try {
return globalThis.structuredClone(value);
} catch {
}
}
try {
return JSON.parse(JSON.stringify(value));
} catch {
return fallbackValue;
}
}
function normalizeHeaderName(name = "") {
return String(name || "").trim().toLowerCase();
}
function hasSessionHeader(headers = {}) {
return Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === AUTHORITY_SESSION_HEADER);
}
function buildDefaultSessionInitConfig(source = {}) {
const config = source && typeof source === "object" && !Array.isArray(source) ? source : {};
return {
extensionId: String(config.extensionId || "third-party/st-bme"),
displayName: String(config.displayName || "ST-BME"),
version: String(config.version || "0.0.0"),
installType: String(config.installType || "local"),
declaredPermissions: clonePlain(config.declaredPermissions, null) || {
storage: { kv: true, blob: true },
fs: { private: true },
sql: { private: true },
trivium: { private: true },
jobs: { background: true },
events: { channels: true },
},
...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}),
};
}
function readPayloadErrorMessage(payload = null, fallback = "") {
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback;
return String(payload.error || payload.message || payload.reason || fallback || "");
}
async function readResponsePayload(response = null) {
if (!response) return {};
const contentType = String(response.headers?.get?.("content-type") || "").toLowerCase();
if (contentType.includes("application/json") && typeof response.json === "function") {
try {
return await response.json();
} catch {
return {};
}
}
if (typeof response.json === "function") {
try {
return await response.json();
} catch {
}
}
if (typeof response.text === "function") {
try {
return { error: await response.text() };
} catch {
return {};
}
}
return {};
}
export class AuthorityHttpError extends Error {
constructor(message, options = {}) {
super(message);
this.name = "AuthorityHttpError";
this.status = Number(options.status || 0);
this.code = String(options.code || "");
this.category = String(options.category || "");
this.payload = clonePlain(options.payload, null);
this.path = String(options.path || "");
this.protocol = String(options.protocol || "");
}
}
export class AuthorityHttpClient {
constructor(options = {}) {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
this.protocol = normalizeProtocol(options.protocol || options.authorityProtocol);
this.sessionToken = String(options.sessionToken || options.authoritySessionToken || "");
this.sessionInitConfig = buildDefaultSessionInitConfig(options.sessionInitConfig || options.initConfig || options);
this.sessionPromise = null;
}
async buildHeaders({ session = false } = {}) {
let provided = {};
if (this.headerProvider) {
provided = await this.headerProvider() || {};
}
const headers = {
Accept: "application/json",
"Content-Type": "application/json",
...provided,
};
if (session && this.sessionToken && !hasSessionHeader(headers)) {
headers[AUTHORITY_SESSION_HEADER] = this.sessionToken;
}
return headers;
}
async ensureSession() {
if (this.sessionToken) return this.sessionToken;
if (!this.sessionPromise) {
this.sessionPromise = this.requestJson("/session/init", {
method: "POST",
body: this.sessionInitConfig,
session: false,
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
}).then((payload) => {
const token = String(payload?.sessionToken || payload?.token || "");
if (!token) {
throw new AuthorityHttpError("Authority session init did not return a session token", {
status: 0,
path: "/session/init",
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
payload,
});
}
this.sessionToken = token;
return token;
}).catch((error) => {
this.sessionPromise = null;
throw error;
});
}
return await this.sessionPromise;
}
async requestJson(path, options = {}) {
if (typeof this.fetchImpl !== "function") {
throw new AuthorityHttpError("Authority fetch unavailable", {
path,
protocol: options.protocol || this.protocol,
});
}
const method = String(options.method || "POST").toUpperCase();
const session = Boolean(options.session);
if (session && !this.sessionToken) {
await this.ensureSession();
}
const headers = await this.buildHeaders({ session });
const response = await this.fetchImpl(`${this.baseUrl}${path}`, {
method,
headers,
...(method === "GET" || options.body === undefined ? {} : { body: JSON.stringify(options.body) }),
...(options.signal ? { signal: options.signal } : {}),
});
const status = Number(response?.status || 0);
const payload = await readResponsePayload(response);
if (!response?.ok) {
const message = readPayloadErrorMessage(payload, `Authority HTTP ${status || "unknown"}`);
throw new AuthorityHttpError(message || `Authority HTTP ${status || "unknown"}`, {
status,
code: payload?.code,
category: payload?.category,
payload,
path,
protocol: options.protocol || this.protocol,
});
}
return payload;
}
}
export function createAuthorityHttpClient(options = {}) {
return new AuthorityHttpClient(options);
}

82
scripts/check-syntax.mjs
View File

@@ -20,6 +20,86 @@ const SOURCE_ROOTS = [
"native", "native",
]; ];
function toDataModuleUrl(source = "") {
return `data:text/javascript,${encodeURIComponent(String(source || ""))}`;
}
const CHECK_SYNTAX_HOOK_BOOTSTRAP_URL = toDataModuleUrl(`
import * as nodeModule from "node:module";
const register = typeof nodeModule.register === "function" ? nodeModule.register : undefined;
const registerHooks = typeof nodeModule.registerHooks === "function"
? nodeModule.registerHooks
: undefined;
const scriptShimUrl = ${JSON.stringify(toDataModuleUrl([
"export function getRequestHeaders() { return {}; }",
"export function saveMetadata() {}",
"export function saveSettingsDebounced() {}",
"export function substituteParamsExtended(value) { return String(value ?? ''); }",
].join("\n")))};
const extensionsShimUrl = ${JSON.stringify(toDataModuleUrl([
"export const extension_settings = { st_bme: {} };",
"export function getContext() { return {}; }",
"export function saveMetadataDebounced() {}",
].join("\n")))};
const openaiShimUrl = ${JSON.stringify(toDataModuleUrl([
"export const chat_completion_sources = {};",
"export async function sendOpenAIRequest() { return {}; }",
].join("\n")))};
function resolveShim(specifier) {
const normalized = String(specifier || "");
if (normalized.endsWith("/script.js")) return scriptShimUrl;
if (normalized.endsWith("/extensions.js")) return extensionsShimUrl;
if (normalized.endsWith("/openai.js")) return openaiShimUrl;
return "";
}
if (typeof registerHooks === "function") {
registerHooks({
resolve(specifier, context, nextResolve) {
const shimUrl = resolveShim(specifier);
if (shimUrl) {
return {
shortCircuit: true,
url: shimUrl,
};
}
return nextResolve(specifier, context);
},
});
} else if (typeof register === "function") {
register(${JSON.stringify(toDataModuleUrl(`
export async function resolve(specifier, context, nextResolve) {
const normalized = String(specifier || "");
if (normalized.endsWith("/script.js")) {
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
"export function getRequestHeaders() { return {}; }",
"export function saveMetadata() {}",
"export function saveSettingsDebounced() {}",
"export function substituteParamsExtended(value) { return String(value ?? ''); }",
].join("\n")))} };
}
if (normalized.endsWith("/extensions.js")) {
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
"export const extension_settings = { st_bme: {} };",
"export function getContext() { return {}; }",
"export function saveMetadataDebounced() {}",
].join("\n")))} };
}
if (normalized.endsWith("/openai.js")) {
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
"export const chat_completion_sources = {};",
"export async function sendOpenAIRequest() { return {}; }",
].join("\n")))} };
}
return nextResolve(specifier, context);
}
`))}, import.meta.url);
}
`);
async function collectFiles(targetPath) { async function collectFiles(targetPath) {
const absolutePath = path.resolve(process.cwd(), targetPath); const absolutePath = path.resolve(process.cwd(), targetPath);
const fileStat = await stat(absolutePath); const fileStat = await stat(absolutePath);
@@ -48,7 +128,7 @@ function toPosixPath(filePath) {
async function runNodeCheck(filePath) { async function runNodeCheck(filePath) {
return await new Promise((resolve, reject) => { return await new Promise((resolve, reject) => {
const child = spawn(process.execPath, ["--check", filePath], { const child = spawn(process.execPath, ["--import", CHECK_SYNTAX_HOOK_BOOTSTRAP_URL, "--check", filePath], {
cwd: process.cwd(), cwd: process.cwd(),
stdio: "inherit", stdio: "inherit",
windowsHide: true, windowsHide: true,

56
sync/authority-graph-store.js
View File

@@ -7,13 +7,17 @@ import {
buildSnapshotFromGraph, buildSnapshotFromGraph,
} from "./bme-db.js"; } from "./bme-db.js";
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
export const AUTHORITY_GRAPH_STORE_KIND = "authority"; export const AUTHORITY_GRAPH_STORE_KIND = "authority";
export const AUTHORITY_GRAPH_STORE_MODE = "authority-sql-primary"; export const AUTHORITY_GRAPH_STORE_MODE = "authority-sql-primary";
const META_DEFAULT_LAST_PROCESSED_FLOOR = -1; const META_DEFAULT_LAST_PROCESSED_FLOOR = -1;
const META_DEFAULT_EXTRACTION_COUNT = 0; const META_DEFAULT_EXTRACTION_COUNT = 0;
const AUTHORITY_SQL_ENDPOINT = "/v1/sql"; const DEFAULT_AUTHORITY_SQL_DATABASE = "default";
const AUTHORITY_SQL_QUERY_ENDPOINT = "/sql/query";
const AUTHORITY_SQL_EXEC_ENDPOINT = "/sql/exec";
const AUTHORITY_SQL_TRANSACTION_ENDPOINT = "/sql/transaction";
const AUTHORITY_TABLES = Object.freeze({ const AUTHORITY_TABLES = Object.freeze({
meta: "st_bme_graph_meta", meta: "st_bme_graph_meta",
@@ -340,41 +344,47 @@ function normalizeUpsertCountDelta(delta = {}) {
export class AuthoritySqlHttpClient { export class AuthoritySqlHttpClient {
constructor(options = {}) { constructor(options = {}) {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); this.http = new AuthorityHttpClient({
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); ...options,
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
});
this.database = normalizeRecordId(options.database) || DEFAULT_AUTHORITY_SQL_DATABASE;
} }
async query(sql, params = {}) { async query(sql, params = {}) {
return await this._request({ action: "query", sql, params }); return await this._request(AUTHORITY_SQL_QUERY_ENDPOINT, {
database: this.database,
statement: String(sql || ""),
params,
});
} }
async execute(sql, params = {}) { async execute(sql, params = {}) {
return await this._request({ action: "execute", sql, params }); return await this._request(AUTHORITY_SQL_EXEC_ENDPOINT, {
database: this.database,
statement: String(sql || ""),
params,
});
} }
async transaction(statements = []) { async transaction(statements = []) {
return await this._request({ action: "transaction", statements }); return await this._request(AUTHORITY_SQL_TRANSACTION_ENDPOINT, {
database: this.database,
statements: toArray(statements)
.filter((statement) => statement?.sql)
.map((statement) => ({
statement: String(statement.sql || ""),
params: statement.params || {},
})),
});
} }
async _request(body = {}) { async _request(path, body = {}) {
if (typeof this.fetchImpl !== "function") { return await this.http.requestJson(path, {
throw new Error("Authority SQL fetch unavailable");
}
const headers = {
Accept: "application/json",
"Content-Type": "application/json",
...(this.headerProvider ? this.headerProvider() || {} : {}),
};
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_SQL_ENDPOINT}`, {
method: "POST", method: "POST",
headers, body,
body: JSON.stringify(body), session: true,
}); });
if (!response?.ok) {
throw new Error(`Authority SQL HTTP ${response?.status || "unknown"}`);
}
return await response.json().catch(() => ({}));
} }
} }

142
tests/authority-blob.mjs
View File

@@ -432,8 +432,150 @@ async function testSyncUploadDownloadUsesAuthorityBlob() {
assert.equal(db.snapshot.nodes[0].id, "sync-blob-node"); assert.equal(db.snapshot.nodes[0].id, "sync-blob-node");
} }
async function testAuthorityBlobHttpBoundary() {
const requests = [];
const adapter = createAuthorityBlobAdapter(
{ authorityBaseUrl: "https://authority.example.test/root" },
{
headerProvider: () => ({ "X-Test": "1" }),
fetchImpl: async (url, options = {}) => {
requests.push({ url, options });
if (url.endsWith("/session/init")) {
return {
ok: true,
status: 200,
async json() {
return { sessionToken: "blob-session-token" };
},
};
}
if (url.endsWith("/fs/private/write-file")) {
return {
ok: true,
status: 200,
async json() {
return {
entry: {
path: "user/files/demo.json",
sizeBytes: 17,
updatedAt: "2026-04-28T12:00:00.000Z",
},
};
},
};
}
if (url.endsWith("/fs/private/read-file")) {
const body = JSON.parse(String(options.body || "{}"));
if (body.path === "user/files/missing.json") {
return {
ok: false,
status: 404,
async json() {
return { error: "not found" };
},
async text() {
return "not found";
},
headers: { get: () => "application/json" },
};
}
return {
ok: true,
status: 200,
async json() {
return {
entry: {
path: "user/files/demo.json",
sizeBytes: 17,
updatedAt: "2026-04-28T12:00:00.000Z",
},
content: JSON.stringify({ hello: "world" }),
encoding: "utf8",
};
},
};
}
if (url.endsWith("/fs/private/stat")) {
return {
ok: true,
status: 200,
async json() {
return {
entry: {
path: "user/files/demo.json",
sizeBytes: 17,
updatedAt: "2026-04-28T12:00:00.000Z",
},
};
},
};
}
if (url.endsWith("/fs/private/delete")) {
return {
ok: true,
status: 200,
async json() {
return { ok: true };
},
};
}
return {
ok: false,
status: 404,
async json() {
return {};
},
async text() {
return "not found";
},
headers: { get: () => "application/json" },
};
},
},
);
const writeResult = await adapter.writeJson("user/files/demo.json", { hello: "world" });
assert.equal(writeResult.ok, true);
assert.equal(writeResult.path, "user/files/demo.json");
const readResult = await adapter.readJson("user/files/demo.json");
assert.equal(readResult.exists, true);
assert.deepEqual(readResult.payload, { hello: "world" });
const statResult = await adapter.stat("user/files/demo.json");
assert.equal(statResult.exists, true);
assert.equal(statResult.path, "user/files/demo.json");
const missingResult = await adapter.readJson("user/files/missing.json");
assert.equal(missingResult.exists, false);
const deleteResult = await adapter.delete("user/files/demo.json");
assert.equal(deleteResult.ok, true);
assert.deepEqual(
requests.map((request) => request.url),
[
"https://authority.example.test/root/session/init",
"https://authority.example.test/root/fs/private/write-file",
"https://authority.example.test/root/fs/private/read-file",
"https://authority.example.test/root/fs/private/stat",
"https://authority.example.test/root/fs/private/read-file",
"https://authority.example.test/root/fs/private/delete",
],
);
assert.equal(requests[1].options.headers["x-authority-session-token"], "blob-session-token");
assert.equal(requests[1].options.headers["X-Test"], "1");
assert.deepEqual(JSON.parse(String(requests[1].options.body || "{}")), {
path: "user/files/demo.json",
content: JSON.stringify({ hello: "world" }),
encoding: "utf8",
createParents: true,
});
}
await testAdapterBasics(); await testAdapterBasics();
await testAuthorityBlobFailOpenFallsBackToUserFiles(); await testAuthorityBlobFailOpenFallsBackToUserFiles();
await testBackupRestoreUsesAuthorityBlob(); await testBackupRestoreUsesAuthorityBlob();
await testSyncUploadDownloadUsesAuthorityBlob(); await testSyncUploadDownloadUsesAuthorityBlob();
await testAuthorityBlobHttpBoundary();
console.log("authority-blob tests passed"); console.log("authority-blob tests passed");

111
tests/authority-capabilities.mjs
View File

@@ -18,31 +18,29 @@ assert.equal(normalizedSettings.vectorMode, "auto-primary");
assert.equal(normalizedSettings.primaryWhenAvailable, true); assert.equal(normalizedSettings.primaryWhenAvailable, true);
assert.deepEqual(buildAuthorityProbeUrls("/api/plugins/authority/"), [ assert.deepEqual(buildAuthorityProbeUrls("/api/plugins/authority/"), [
"/api/plugins/authority/v1/diagnostics/probe",
"/api/plugins/authority/v1/probe",
"/api/plugins/authority/probe", "/api/plugins/authority/probe",
"/api/plugins/authority",
]); ]);
const collected = collectAuthorityFeatures({ const collected = collectAuthorityFeatures({
features: ["sql.query", "trivium.search"], features: {
services: { sql: { queryPage: true },
sql: true, trivium: { upsert: true },
jobs: true, jobs: { background: true },
blob: true, transfers: { fs: true },
}, },
}); });
assert.equal(collected.has("sql.query"), true);
assert.equal(collected.has("trivium.search"), true);
assert.equal(collected.has("sql"), true); assert.equal(collected.has("sql"), true);
assert.equal(collected.has("jobs"), true); assert.equal(collected.has("sql.querypage"), true);
assert.equal(collected.has("blob"), true); assert.equal(collected.has("sql"), true);
assert.equal(collected.has("trivium"), true);
assert.equal(collected.has("jobs.background"), true);
assert.equal(collected.has("transfers.fs"), true);
const readyState = normalizeAuthorityCapabilityState( const readyState = normalizeAuthorityCapabilityState(
{ {
installed: true, installed: true,
healthy: true, healthy: true,
features: ["sql", "trivium", "jobs", "blob"], features: ["sql", "trivium", "jobs", "transfers.fs"],
}, },
defaultSettings, defaultSettings,
); );
@@ -77,28 +75,91 @@ assert.equal(disabledState.reason, "disabled");
assert.equal(disabledState.serverPrimaryReady, false); assert.equal(disabledState.serverPrimaryReady, false);
assert.equal(disabledState.lastProbeAt, 1000); assert.equal(disabledState.lastProbeAt, 1000);
let requestedUrl = ""; const requestedUrls = [];
const probedState = await probeAuthorityCapabilities({ const probedState = await probeAuthorityCapabilities({
settings: defaultSettings, settings: defaultSettings,
allowRelativeUrl: true, allowRelativeUrl: true,
nowMs: 2000, nowMs: 2000,
fetchImpl: async (url) => { fetchImpl: async (url, options = {}) => {
requestedUrl = url; requestedUrls.push([url, options.method || "GET", options.headers || {}]);
if (url.endsWith("/probe")) {
assert.equal(options.method, "POST");
return {
ok: true,
status: 200,
async json() {
return {
healthy: true,
features: {
sql: { queryPage: true },
trivium: { upsert: true },
jobs: { background: true },
transfers: { fs: true },
},
};
},
};
}
if (url.endsWith("/session/init")) {
assert.equal(options.method, "POST");
return {
ok: true,
status: 200,
async json() {
return { sessionToken: "session-probe-token" };
},
};
}
if (url.endsWith("/session/current")) {
assert.equal(options.method, "GET");
assert.equal(options.headers["x-authority-session-token"], "session-probe-token");
return {
ok: true,
status: 200,
async json() {
return { ok: true };
},
};
}
if (url.endsWith("/permissions/evaluate-batch")) {
assert.equal(options.method, "POST");
assert.equal(options.headers["x-authority-session-token"], "session-probe-token");
const body = JSON.parse(String(options.body || "{}"));
assert.equal(Array.isArray(body.requests), true);
assert.equal(body.requests.some((request) => request.resource === "fs.private"), true);
return {
ok: true,
status: 200,
async json() {
return {
results: body.requests.map((request) => ({
decision: "granted",
resource: request.resource,
target: request.target || "",
})),
};
},
};
}
return { return {
ok: true, ok: false,
status: 200, status: 404,
async json() { async json() {
return { return {};
healthy: true,
sessionReady: true,
permissionReady: true,
features: ["sql", "trivium", "jobs", "blob"],
};
}, },
}; };
}, },
}); });
assert.equal(requestedUrl, "/api/plugins/authority/v1/diagnostics/probe"); assert.equal(requestedUrls[0]?.[0], "/api/plugins/authority/probe");
assert.deepEqual(
requestedUrls.map(([url]) => url),
[
"/api/plugins/authority/probe",
"/api/plugins/authority/session/init",
"/api/plugins/authority/session/current",
"/api/plugins/authority/permissions/evaluate-batch",
],
);
assert.equal(probedState.installed, true); assert.equal(probedState.installed, true);
assert.equal(probedState.healthy, true); assert.equal(probedState.healthy, true);
assert.equal(probedState.serverPrimaryReady, true); assert.equal(probedState.serverPrimaryReady, true);

28
tests/authority-graph-store.mjs
View File

@@ -301,6 +301,15 @@ async function testHttpSqlClientBoundary() {
headerProvider: () => ({ "X-Test": "1" }), headerProvider: () => ({ "X-Test": "1" }),
fetchImpl: async (url, init) => { fetchImpl: async (url, init) => {
requests.push({ url, init }); requests.push({ url, init });
if (url.endsWith("/session/init")) {
return {
ok: true,
status: 200,
async json() {
return { sessionToken: "sql-session-token" };
},
};
}
return { return {
ok: true, ok: true,
status: 200, status: 200,
@@ -313,12 +322,19 @@ async function testHttpSqlClientBoundary() {
const result = await client.query("SELECT 1", { chatId: "chat" }); const result = await client.query("SELECT 1", { chatId: "chat" });
assert.deepEqual(result, { rows: [{ value: 1 }] }); assert.deepEqual(result, { rows: [{ value: 1 }] });
assert.equal(requests[0].url, "https://authority.example.test/root/v1/sql"); assert.deepEqual(
assert.equal(requests[0].init.method, "POST"); requests.map((request) => request.url),
assert.equal(requests[0].init.headers["X-Test"], "1"); [
assert.deepEqual(JSON.parse(requests[0].init.body), { "https://authority.example.test/root/session/init",
action: "query", "https://authority.example.test/root/sql/query",
sql: "SELECT 1", ],
);
assert.equal(requests[1].init.method, "POST");
assert.equal(requests[1].init.headers["X-Test"], "1");
assert.equal(requests[1].init.headers["x-authority-session-token"], "sql-session-token");
assert.deepEqual(JSON.parse(requests[1].init.body), {
database: "default",
statement: "SELECT 1",
params: { chatId: "chat" }, params: { chatId: "chat" },
}); });
} }

146
tests/authority-jobs.mjs
View File

@@ -455,4 +455,150 @@ const rangeSync = rangeRuntime.calls.find(([name]) => name === "syncVectorState"
assert.equal(rangeSync?.[1]?.purge, false); assert.equal(rangeSync?.[1]?.purge, false);
assert.equal(rangeSync?.[1]?.range, range); assert.equal(rangeSync?.[1]?.range, range);
const httpRequests = [];
const httpAdapter = createAuthorityJobAdapter(
{ authorityBaseUrl: "https://authority.example.test/root" },
{
headerProvider: () => ({ "X-Test": "1" }),
fetchImpl: async (url, options = {}) => {
httpRequests.push({ url, options });
if (url.endsWith("/session/init")) {
return {
ok: true,
status: 200,
async json() {
return { sessionToken: "job-session-token" };
},
};
}
if (url.endsWith("/jobs/create")) {
return {
ok: true,
status: 200,
async json() {
return {
id: "job-http-1",
type: "authority.vector.rebuild",
status: "queued",
progress: 0,
idempotencyKey: "idem-http-1",
};
},
};
}
if (url.endsWith("/jobs/job-http-1")) {
return {
ok: true,
status: 200,
async json() {
return {
id: "job-http-1",
type: "authority.vector.rebuild",
status: "completed",
progress: 1,
};
},
};
}
if (url.endsWith("/jobs/list")) {
return {
ok: true,
status: 200,
async json() {
return {
jobs: [
{
id: "job-http-1",
type: "authority.vector.rebuild",
status: "completed",
progress: 1,
},
],
page: {
nextCursor: "next-http-1",
hasMore: true,
},
};
},
};
}
if (url.endsWith("/jobs/job-http-1/requeue")) {
return {
ok: true,
status: 200,
async json() {
return {
id: "job-http-2",
type: "authority.vector.rebuild",
status: "queued",
progress: 0,
};
},
};
}
if (url.endsWith("/jobs/job-http-1/cancel")) {
return {
ok: true,
status: 200,
async json() {
return {
id: "job-http-1",
type: "authority.vector.rebuild",
status: "cancelled",
progress: 1,
};
},
};
}
return {
ok: false,
status: 404,
async json() {
return {};
},
};
},
},
);
const httpSubmitted = await httpAdapter.submit(
"authority.vector.rebuild",
{ chatId: "chat-http" },
{ idempotencyKey: "idem-http-1" },
);
assert.equal(httpSubmitted.id, "job-http-1");
const httpLoaded = await httpAdapter.get("job-http-1");
assert.equal(httpLoaded.status, "completed");
const httpPage = await httpAdapter.listPage({ cursor: "cursor-http", limit: 10 });
assert.equal(httpPage.nextCursor, "next-http-1");
assert.equal(httpPage.hasMore, true);
const httpRequeued = await httpAdapter.requeue("job-http-1");
assert.equal(httpRequeued.id, "job-http-2");
const httpCancelled = await httpAdapter.cancel("job-http-1");
assert.equal(httpCancelled.status, "cancelled");
assert.deepEqual(
httpRequests.map((request) => request.url),
[
"https://authority.example.test/root/session/init",
"https://authority.example.test/root/jobs/create",
"https://authority.example.test/root/jobs/job-http-1",
"https://authority.example.test/root/jobs/list",
"https://authority.example.test/root/jobs/job-http-1/requeue",
"https://authority.example.test/root/jobs/job-http-1/cancel",
],
);
assert.equal(httpRequests[1].options.headers["x-authority-session-token"], "job-session-token");
assert.equal(httpRequests[1].options.headers["X-Test"], "1");
assert.deepEqual(JSON.parse(String(httpRequests[1].options.body || "{}")), {
type: "authority.vector.rebuild",
payload: { chatId: "chat-http" },
idempotencyKey: "idem-http-1",
});
assert.deepEqual(JSON.parse(String(httpRequests[3].options.body || "{}")), {
page: {
cursor: "cursor-http",
limit: 10,
},
});
console.log("authority-jobs tests passed"); console.log("authority-jobs tests passed");

13
tests/authority-recall-candidates.mjs
View File

@@ -16,6 +16,15 @@ installResolveHooks([
}, },
]); ]);
globalThis.__stBmeTestOverrides = {
embedding: {
async embedText(text) {
const seed = String(text || "").length || 1;
return [seed / 100, 0.2, 0.3];
},
},
};
const { normalizeAuthorityVectorConfig } = await import( const { normalizeAuthorityVectorConfig } = await import(
"../vector/authority-vector-primary-adapter.js" "../vector/authority-vector-primary-adapter.js"
); );
@@ -153,6 +162,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN
const config = normalizeAuthorityVectorConfig( const config = normalizeAuthorityVectorConfig(
{ {
authorityBaseUrl: "/api/plugins/authority", authorityBaseUrl: "/api/plugins/authority",
authorityEmbeddingApiUrl: "https://example.com/v1",
authorityEmbeddingModel: "test-embedding",
authorityVectorFailOpen: true, authorityVectorFailOpen: true,
}, },
{ triviumClient }, { triviumClient },
@@ -216,6 +227,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN
const config = normalizeAuthorityVectorConfig( const config = normalizeAuthorityVectorConfig(
{ {
authorityBaseUrl: "/api/plugins/authority", authorityBaseUrl: "/api/plugins/authority",
authorityEmbeddingApiUrl: "https://example.com/v1",
authorityEmbeddingModel: "test-embedding",
authorityVectorFailOpen: true, authorityVectorFailOpen: true,
}, },
{ triviumClient }, { triviumClient },

97
tests/authority-vector-primary.mjs
View File

@@ -16,6 +16,17 @@ installResolveHooks([
}, },
]); ]);
globalThis.__stBmeTestOverrides = {
embedding: {
async embedBatch(texts = []) {
return texts.map((text, index) => [1, index / 10, String(text || "").length / 100]);
},
async embedText(text = "") {
return [1, 0.5, String(text || "").length / 100];
},
},
};
const { const {
filterAuthorityTriviumNodes, filterAuthorityTriviumNodes,
isAuthorityVectorConfig, isAuthorityVectorConfig,
@@ -112,8 +123,20 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) {
}; };
} }
async function withMockFetch(handler, fn) {
const previousFetch = globalThis.fetch;
globalThis.fetch = handler;
try {
return await fn();
} finally {
globalThis.fetch = previousFetch;
}
}
const config = normalizeAuthorityVectorConfig({ const config = normalizeAuthorityVectorConfig({
authorityBaseUrl: "/api/plugins/authority", authorityBaseUrl: "/api/plugins/authority",
authorityEmbeddingApiUrl: "https://example.com/v1",
authorityEmbeddingModel: "test-embedding",
authorityVectorSyncChunkSize: 1, authorityVectorSyncChunkSize: 1,
authorityVectorFailOpen: true, authorityVectorFailOpen: true,
}); });
@@ -144,6 +167,10 @@ assert.equal(isAuthorityVectorConfig(config), true);
upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(), upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(),
["node-a", "node-b"], ["node-a", "node-b"],
); );
assert.equal(
upserts.every(([, payload]) => payload.items.every((item) => Array.isArray(item.vector) && item.vector.length > 0)),
true,
);
const linkCall = triviumClient.calls.find(([name]) => name === "linkMany"); const linkCall = triviumClient.calls.find(([name]) => name === "linkMany");
assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a"); assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a");
assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b"); assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b");
@@ -170,6 +197,8 @@ assert.equal(isAuthorityVectorConfig(config), true);
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]); assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
const searchCall = triviumClient.calls.find(([name]) => name === "search"); const searchCall = triviumClient.calls.find(([name]) => name === "search");
assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]); assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]);
assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true);
assert.ok(searchCall?.[1]?.queryVector.length > 0);
assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority"); assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority");
assert.equal(graph.vectorIndexState.lastSearchTimings.success, true); assert.equal(graph.vectorIndexState.lastSearchTimings.success, true);
} }
@@ -221,4 +250,72 @@ assert.equal(isAuthorityVectorConfig(config), true);
assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]); assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]);
} }
{
const previousOverrides = globalThis.__stBmeTestOverrides;
globalThis.__stBmeTestOverrides = {};
const fetchCalls = [];
try {
await withMockFetch(async (url, options = {}) => {
fetchCalls.push([url, JSON.parse(String(options.body || "{}"))]);
return {
ok: true,
status: 200,
async json() {
const body = JSON.parse(String(options.body || "{}"));
if (Array.isArray(body.texts)) {
return {
vectors: body.texts.map((text, index) => [1, index + 1, String(text || "").length / 100]),
};
}
return {
vector: [1, 9, String(body.text || "").length / 100],
};
},
async text() {
return "";
},
};
}, async () => {
const backendConfig = normalizeAuthorityVectorConfig({
authorityBaseUrl: "/api/plugins/authority",
embeddingTransportMode: "backend",
embeddingBackendSource: "openai",
embeddingBackendModel: "text-embedding-3-small",
authorityVectorSyncChunkSize: 2,
});
const { graph, first, second } = createAuthorityVectorGraph();
first.embedding = null;
second.embedding = null;
const triviumClient = createMockTriviumClient();
await syncGraphVectorIndexFromIndex(graph, backendConfig, {
chatId: "chat-authority-vector",
purge: true,
triviumClient,
});
const results = await findSimilarNodesByTextFromIndex(
graph,
"archive door",
{ ...backendConfig, triviumClient },
5,
[first, second],
);
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
const upsertCall = triviumClient.calls.find(([name]) => name === "bulkUpsert");
assert.equal(
upsertCall?.[1]?.items?.every((item) => Array.isArray(item.vector) && item.vector.length > 0),
true,
);
const searchCall = triviumClient.calls.find(([name]) => name === "search");
assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true);
assert.equal(fetchCalls.every(([url]) => url === "/api/vector/embed"), true);
assert.equal(fetchCalls[0]?.[1]?.source, "openai");
assert.equal(fetchCalls[0]?.[1]?.model, "text-embedding-3-small");
assert.equal(Array.isArray(fetchCalls[0]?.[1]?.texts), true);
assert.equal(fetchCalls[fetchCalls.length - 1]?.[1]?.isQuery, true);
});
} finally {
globalThis.__stBmeTestOverrides = previousOverrides;
}
}
console.log("authority-vector-primary tests passed"); console.log("authority-vector-primary tests passed");

31
tests/e2e/authority-checkpoint-restore.mjs
View File

@@ -1,19 +1,36 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { buildLukerGraphCheckpointV2 } from "../../graph/graph-persistence.js";
import { import {
applyAuthorityCheckpointToStore, installResolveHooks,
buildAuthorityConsistencyAudit, toDataModuleUrl,
buildAuthorityCheckpointImportSnapshot, } from "../helpers/register-hooks-compat.mjs";
} from "../../maintenance/authority-consistency.js";
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; installResolveHooks([
import { AuthorityGraphStore } from "../../sync/authority-graph-store.js"; {
specifiers: ["../../../../../script.js"],
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
},
{
specifiers: ["../../../../extensions.js"],
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
},
]);
import { import {
createAuthorityE2eContext, createAuthorityE2eContext,
createAuthorityE2eContractGraph, createAuthorityE2eContractGraph,
runAuthorityE2eStep, runAuthorityE2eStep,
} from "../helpers/authority-e2e-context.mjs"; } from "../helpers/authority-e2e-context.mjs";
const { buildLukerGraphCheckpointV2 } = await import("../../graph/graph-persistence.js");
const {
applyAuthorityCheckpointToStore,
buildAuthorityConsistencyAudit,
buildAuthorityCheckpointImportSnapshot,
} = await import("../../maintenance/authority-consistency.js");
const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js");
const context = createAuthorityE2eContext({ const context = createAuthorityE2eContext({
skipMessage: skipMessage:
"authority checkpoint restore E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server", "authority checkpoint restore E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",

27
tests/e2e/authority-diagnostics-roundtrip.mjs
View File

@@ -1,17 +1,34 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { import {
buildAuthorityDiagnosticsBundle, installResolveHooks,
buildAuthorityPerformanceBaseline, toDataModuleUrl,
writeAuthorityDiagnosticsBundle, } from "../helpers/register-hooks-compat.mjs";
} from "../../maintenance/authority-diagnostics-bundle.js";
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; installResolveHooks([
{
specifiers: ["../../../../../script.js"],
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
},
{
specifiers: ["../../../../extensions.js"],
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
},
]);
import { import {
createAuthorityE2eContext, createAuthorityE2eContext,
createAuthorityE2eContractGraph, createAuthorityE2eContractGraph,
runAuthorityE2eStep, runAuthorityE2eStep,
} from "../helpers/authority-e2e-context.mjs"; } from "../helpers/authority-e2e-context.mjs";
const {
buildAuthorityDiagnosticsBundle,
buildAuthorityPerformanceBaseline,
writeAuthorityDiagnosticsBundle,
} = await import("../../maintenance/authority-diagnostics-bundle.js");
const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
const context = createAuthorityE2eContext({ const context = createAuthorityE2eContext({
skipMessage: skipMessage:
"authority diagnostics E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server", "authority diagnostics E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",

53
tests/e2e/authority-server-primary.mjs
View File

@@ -1,8 +1,32 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { probeAuthorityCapabilities } from "../../runtime/authority-capabilities.js";
import { AuthorityGraphStore } from "../../sync/authority-graph-store.js";
import { import {
installResolveHooks,
toDataModuleUrl,
} from "../helpers/register-hooks-compat.mjs";
installResolveHooks([
{
specifiers: ["../../../../../script.js"],
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
},
{
specifiers: ["../../../../extensions.js"],
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
},
]);
import {
buildAuthorityE2eVectorEntries,
createAuthorityE2eContext,
createAuthorityE2eContractGraph,
createAuthorityE2eContractNode,
runAuthorityE2eStep,
} from "../helpers/authority-e2e-context.mjs";
const { probeAuthorityCapabilities } = await import("../../runtime/authority-capabilities.js");
const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js");
const {
deleteAuthorityTriviumNodes, deleteAuthorityTriviumNodes,
filterAuthorityTriviumNodes, filterAuthorityTriviumNodes,
normalizeAuthorityVectorConfig, normalizeAuthorityVectorConfig,
@@ -11,19 +35,12 @@ import {
searchAuthorityTriviumNodes, searchAuthorityTriviumNodes,
syncAuthorityTriviumLinks, syncAuthorityTriviumLinks,
upsertAuthorityTriviumEntries, upsertAuthorityTriviumEntries,
} from "../../vector/authority-vector-primary-adapter.js"; } = await import("../../vector/authority-vector-primary-adapter.js");
import { const {
buildAuthorityJobIdempotencyKey, buildAuthorityJobIdempotencyKey,
createAuthorityJobAdapter, createAuthorityJobAdapter,
} from "../../maintenance/authority-job-adapter.js"; } = await import("../../maintenance/authority-job-adapter.js");
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js"; const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
import {
buildAuthorityE2eVectorEntries,
createAuthorityE2eContext,
createAuthorityE2eContractGraph,
createAuthorityE2eContractNode,
runAuthorityE2eStep,
} from "../helpers/authority-e2e-context.mjs";
const context = createAuthorityE2eContext({ const context = createAuthorityE2eContext({
skipMessage: skipMessage:
@@ -35,7 +52,7 @@ if (context.skip) {
process.exit(0); process.exit(0);
} }
const resolvedBaseUrl = context.baseUrl; const resolvedBaseUrl = context.baseUrl;
const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId } = context; const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId, embeddingApiUrl, embeddingApiKey, embeddingModel } = context;
const graph = createAuthorityE2eContractGraph(chatId, runId); const graph = createAuthorityE2eContractGraph(chatId, runId);
const runContext = { const runContext = {
@@ -105,7 +122,12 @@ await runAuthorityE2eStep("sql", async () => {
}); });
await runAuthorityE2eStep("trivium", async () => { await runAuthorityE2eStep("trivium", async () => {
const config = normalizeAuthorityVectorConfig({ authorityBaseUrl: resolvedBaseUrl }); const config = normalizeAuthorityVectorConfig({
authorityBaseUrl: resolvedBaseUrl,
authorityEmbeddingApiUrl: embeddingApiUrl,
authorityEmbeddingApiKey: embeddingApiKey,
authorityEmbeddingModel: embeddingModel,
});
const entries = buildAuthorityE2eVectorEntries(graph); const entries = buildAuthorityE2eVectorEntries(graph);
await purgeAuthorityTriviumNamespace(config, { await purgeAuthorityTriviumNamespace(config, {
namespace, namespace,
@@ -141,6 +163,7 @@ await runAuthorityE2eStep("trivium", async () => {
namespace, namespace,
collectionId, collectionId,
chatId, chatId,
queryVector: [1, 1, 0.18],
topK: 5, topK: 5,
fetchImpl, fetchImpl,
headerProvider, headerProvider,

7
tests/helpers/authority-e2e-context.mjs
View File

@@ -97,6 +97,9 @@ export function createAuthorityE2eContext(options = {}) {
const namespace = String(env.AUTHORITY_E2E_NAMESPACE || `st-bme-e2e-${runId}`); const namespace = String(env.AUTHORITY_E2E_NAMESPACE || `st-bme-e2e-${runId}`);
const collectionId = String(env.AUTHORITY_E2E_COLLECTION_ID || `${namespace}::${chatId}`); const collectionId = String(env.AUTHORITY_E2E_COLLECTION_ID || `${namespace}::${chatId}`);
const blobPath = String(env.AUTHORITY_E2E_BLOB_PATH || `st-bme/e2e/${runId}/contract.json`); const blobPath = String(env.AUTHORITY_E2E_BLOB_PATH || `st-bme/e2e/${runId}/contract.json`);
const embeddingApiUrl = String(env.AUTHORITY_E2E_EMBEDDING_API_URL || "").trim();
const embeddingApiKey = String(env.AUTHORITY_E2E_EMBEDDING_API_KEY || "").trim();
const embeddingModel = String(env.AUTHORITY_E2E_EMBEDDING_MODEL || "").trim();
return { return {
skip: false, skip: false,
env, env,
@@ -110,6 +113,9 @@ export function createAuthorityE2eContext(options = {}) {
namespace, namespace,
collectionId, collectionId,
blobPath, blobPath,
embeddingApiUrl,
embeddingApiKey,
embeddingModel,
}; };
} }
@@ -181,6 +187,7 @@ export function buildAuthorityE2eVectorEntries(graph = null) {
index, index,
hash: `${node.id}:hash`, hash: `${node.id}:hash`,
text: `${node.fields?.title || node.id}. ${node.fields?.summary || ""}`, text: `${node.fields?.title || node.id}. ${node.fields?.summary || ""}`,
vector: [1, index + 1, String(node.fields?.title || node.id || "").length / 100],
})); }));
} }

10
tests/perf/authority-recall-bench.mjs
View File

@@ -16,6 +16,14 @@ installResolveHooks([
}, },
]); ]);
globalThis.__stBmeTestOverrides = {
embedding: {
async embedText(text = "") {
return [1, 0.25, String(text || "").length / 100];
},
},
};
const outputJson = process.argv.includes("--json"); const outputJson = process.argv.includes("--json");
const RUNS = 5; const RUNS = 5;
const SIZE_PRESETS = [ const SIZE_PRESETS = [
@@ -233,6 +241,8 @@ async function runPreset(preset) {
const config = normalizeAuthorityVectorConfig( const config = normalizeAuthorityVectorConfig(
{ {
authorityBaseUrl: "/api/plugins/authority", authorityBaseUrl: "/api/plugins/authority",
authorityEmbeddingApiUrl: "https://example.com/v1",
authorityEmbeddingModel: "test-embedding",
authorityVectorFailOpen: true, authorityVectorFailOpen: true,
}, },
{ triviumClient }, { triviumClient },

48
tests/vector-config.mjs
View File

@@ -70,6 +70,54 @@ const defaultModeConfig = getVectorConfigFromSettings({
assert.equal(defaultModeConfig.mode, "direct"); assert.equal(defaultModeConfig.mode, "direct");
assert.equal(validateVectorConfig(defaultModeConfig).valid, true); assert.equal(validateVectorConfig(defaultModeConfig).valid, true);
const validAuthorityConfig = {
mode: "authority",
source: "authority-trivium",
baseUrl: "/api/plugins/authority",
apiUrl: "https://example.com/v1",
model: "text-embedding-3-small",
};
assert.equal(validateVectorConfig(validAuthorityConfig).valid, true);
const invalidAuthorityConfig = {
mode: "authority",
source: "authority-trivium",
baseUrl: "/api/plugins/authority",
apiUrl: "",
model: "",
};
assert.equal(validateVectorConfig(invalidAuthorityConfig).valid, false);
const validAuthorityBackendConfig = {
mode: "authority",
source: "authority-trivium",
baseUrl: "/api/plugins/authority",
embeddingMode: "backend",
embeddingSource: "openai",
apiUrl: "",
model: "text-embedding-3-small",
};
assert.equal(validateVectorConfig(validAuthorityBackendConfig).valid, true);
const invalidAuthorityBackendConfig = {
mode: "authority",
source: "authority-trivium",
baseUrl: "/api/plugins/authority",
embeddingMode: "backend",
embeddingSource: "vllm",
apiUrl: "",
model: "BAAI/bge-m3",
};
assert.equal(validateVectorConfig(invalidAuthorityBackendConfig).valid, false);
const authorityLikeConfig = getVectorConfigFromSettings({
embeddingApiUrl: "https://example.com/v1/embeddings",
embeddingApiKey: "sk-test",
embeddingModel: "text-embedding-3-small",
});
assert.equal(authorityLikeConfig.apiUrl, "https://example.com/v1");
assert.equal(authorityLikeConfig.model, "text-embedding-3-small");
const invalidBackendConfig = getVectorConfigFromSettings({ const invalidBackendConfig = getVectorConfigFromSettings({
embeddingTransportMode: "backend", embeddingTransportMode: "backend",
embeddingBackendSource: "vllm", embeddingBackendSource: "vllm",

317
vector/authority-vector-primary-adapter.js
View File

@@ -1,11 +1,17 @@
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"; import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
import {
AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
AuthorityHttpClient,
} from "../runtime/authority-http-client.js";
import { embedText } from "./embedding.js";
export const AUTHORITY_VECTOR_MODE = "authority"; export const AUTHORITY_VECTOR_MODE = "authority";
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium"; export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium"; const DEFAULT_AUTHORITY_TRIVIUM_DATABASE = "st_bme_vectors";
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000; const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000; const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
const DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE = "openai";
function clampInteger(value, fallback, min, max) { function clampInteger(value, fallback, min, max) {
const parsed = Number(value); const parsed = Number(value);
@@ -36,6 +42,27 @@ function normalizeRecordId(value) {
return String(value ?? "").trim(); return String(value ?? "").trim();
} }
function normalizeVector(value = null) {
const source = ArrayBuffer.isView(value) ? Array.from(value) : value;
if (!Array.isArray(source)) return [];
return source
.map((item) => Number(item))
.filter((item) => Number.isFinite(item));
}
function normalizePositiveInteger(value, fallback = 0) {
const parsed = Number(value);
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
return Math.floor(parsed);
}
function normalizeOpenAICompatibleBaseUrl(value) {
return String(value || "")
.trim()
.replace(/\/+(chat\/completions|embeddings)$/i, "")
.replace(/\/+$/, "");
}
function readNestedValue(source = null, path = []) { function readNestedValue(source = null, path = []) {
let current = source; let current = source;
for (const key of path) { for (const key of path) {
@@ -68,12 +95,14 @@ function readResultRows(payload = null) {
if (Array.isArray(payload.data)) return payload.data; if (Array.isArray(payload.data)) return payload.data;
if (Array.isArray(payload.neighbors)) return payload.neighbors; if (Array.isArray(payload.neighbors)) return payload.neighbors;
if (Array.isArray(payload.links)) return payload.links; if (Array.isArray(payload.links)) return payload.links;
if (Array.isArray(payload.nodes)) return payload.nodes;
if (Array.isArray(payload.result?.results)) return payload.result.results; if (Array.isArray(payload.result?.results)) return payload.result.results;
if (Array.isArray(payload.result?.items)) return payload.result.items; if (Array.isArray(payload.result?.items)) return payload.result.items;
if (Array.isArray(payload.result?.rows)) return payload.result.rows; if (Array.isArray(payload.result?.rows)) return payload.result.rows;
if (Array.isArray(payload.result?.data)) return payload.result.data; if (Array.isArray(payload.result?.data)) return payload.result.data;
if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors; if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors;
if (Array.isArray(payload.result?.links)) return payload.result.links; if (Array.isArray(payload.result?.links)) return payload.result.links;
if (Array.isArray(payload.result?.nodes)) return payload.result.nodes;
return []; return [];
} }
@@ -145,6 +174,32 @@ function normalizeSearchResults(payload = null) {
.filter(Boolean); .filter(Boolean);
} }
function buildOpenOptions(config = {}, payload = {}) {
const database = normalizeRecordId(payload.database || config.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE;
return {
database,
...(normalizePositiveInteger(payload.dim ?? config.dim, 0) > 0 ? { dim: normalizePositiveInteger(payload.dim ?? config.dim, 0) } : {}),
...(payload.dtype || config.dtype ? { dtype: String(payload.dtype || config.dtype) } : {}),
...(payload.syncMode || config.syncMode ? { syncMode: String(payload.syncMode || config.syncMode) } : {}),
...(payload.storageMode || config.storageMode ? { storageMode: String(payload.storageMode || config.storageMode) } : {}),
};
}
function getNamespace(payload = {}) {
return normalizeRecordId(payload.namespace || payload.collectionId || payload.chatId);
}
function buildNodeReference(id, namespace = "") {
return {
externalId: normalizeRecordId(id),
...(namespace ? { namespace } : {}),
};
}
function buildV06PayloadSource(payload = {}) {
return payload && typeof payload === "object" && !Array.isArray(payload) ? payload : {};
}
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) { function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {}; const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0]; const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
@@ -166,6 +221,7 @@ function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelSc
regionKey: String(scope.regionKey || node?.regionKey || ""), regionKey: String(scope.regionKey || node?.regionKey || ""),
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""), storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
storyTimeLabel: String(node?.storyTime?.label || ""), storyTimeLabel: String(node?.storyTime?.label || ""),
text: String(entry?.text || ""),
title: getNodeFieldText(node, ["title"]), title: getNodeFieldText(node, ["title"]),
name: getNodeFieldText(node, ["name"]), name: getNodeFieldText(node, ["name"]),
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]), summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
@@ -190,6 +246,7 @@ function buildAuthorityVectorItems(graph, entries = [], options = {}) {
text: String(entry?.text || ""), text: String(entry?.text || ""),
index: Number(entry?.index || 0) || 0, index: Number(entry?.index || 0) || 0,
hash: String(entry?.hash || ""), hash: String(entry?.hash || ""),
vector: normalizeVector(entry?.vector || entry?.embedding || node?.embedding),
payload, payload,
}; };
}) })
@@ -229,11 +286,43 @@ export function isAuthorityVectorConfig(config = null) {
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) { export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {}; const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
const hasAuthorityEmbeddingOverride = [
source.authorityEmbeddingApiUrl,
source.authorityEmbeddingApiKey,
source.authorityEmbeddingModel,
].some((value) => String(value ?? "").trim());
const embeddingMode = hasAuthorityEmbeddingOverride
? "direct"
: String(source.embeddingTransportMode || "direct").trim().toLowerCase() === "backend"
? "backend"
: "direct";
const embeddingSource = embeddingMode === "backend"
? String(source.embeddingBackendSource || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE).trim().toLowerCase() || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE
: "direct";
return { return {
mode: AUTHORITY_VECTOR_MODE, mode: AUTHORITY_VECTOR_MODE,
source: AUTHORITY_VECTOR_SOURCE, source: AUTHORITY_VECTOR_SOURCE,
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl), baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(), protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
database: normalizeRecordId(source.authorityTriviumDatabase ?? source.triviumDatabase) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
dim: normalizePositiveInteger(source.authorityTriviumDim ?? source.triviumDim, 0),
dtype: String(source.authorityTriviumDtype ?? source.triviumDtype ?? "").trim(),
syncMode: String(source.authorityTriviumSyncMode ?? source.triviumSyncMode ?? "").trim(),
storageMode: String(source.authorityTriviumStorageMode ?? source.triviumStorageMode ?? "").trim(),
embeddingMode,
embeddingSource,
apiUrl: normalizeOpenAICompatibleBaseUrl(
embeddingMode === "backend"
? source.embeddingBackendApiUrl
: source.authorityEmbeddingApiUrl ?? source.embeddingApiUrl ?? source.embeddingBackendApiUrl,
),
apiKey: embeddingMode === "backend"
? ""
: String(source.authorityEmbeddingApiKey ?? source.embeddingApiKey ?? "").trim(),
model: embeddingMode === "backend"
? String(source.embeddingBackendModel ?? source.embeddingModel ?? "").trim()
: String(source.authorityEmbeddingModel ?? source.embeddingModel ?? source.embeddingBackendModel ?? "").trim(),
autoSuffix: source.embeddingAutoSuffix !== false,
chunkSize: clampInteger( chunkSize: clampInteger(
source.authorityVectorSyncChunkSize ?? source.chunkSize, source.authorityVectorSyncChunkSize ?? source.chunkSize,
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE, DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
@@ -251,61 +340,225 @@ export class AuthorityTriviumHttpClient {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl); this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null); this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null; this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
this.protocol = AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
this.config = {
database: normalizeRecordId(options.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
dim: normalizePositiveInteger(options.dim, 0),
dtype: String(options.dtype || "").trim(),
syncMode: String(options.syncMode || "").trim(),
storageMode: String(options.storageMode || "").trim(),
};
this.http = new AuthorityHttpClient({
...options,
baseUrl: this.baseUrl,
fetchImpl: this.fetchImpl,
headerProvider: this.headerProvider,
protocol: this.protocol,
});
} }
async request(action, payload = {}) { async request(action, payload = {}) {
if (typeof this.fetchImpl !== "function") { if (action === "purge") return await this.purge(payload);
throw new Error("Authority Trivium fetch unavailable"); if (action === "bulkUpsert") return await this.bulkUpsert(payload);
} if (action === "deleteMany") return await this.deleteMany(payload);
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, { if (action === "linkMany") return await this.linkMany(payload);
method: "POST", if (action === "search") return await this.search(payload);
headers: { if (action === "filterWhere") return await this.filterWhere(payload);
Accept: "application/json", if (action === "queryPage") return await this.queryPage(payload);
"Content-Type": "application/json", if (action === "neighbors") return await this.neighbors(payload);
...(this.headerProvider ? this.headerProvider() || {} : {}), if (action === "stat") return await this.stat(payload);
}, throw new Error(`Authority Trivium v0.6 action unavailable: ${action}`);
body: JSON.stringify({ action, ...payload }), }
async requestV06(path, payload = {}, method = "POST") {
return await this.http.requestJson(path, {
method,
body: payload,
session: true,
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
}); });
if (!response?.ok) { }
throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`);
} buildOpenOptions(payload = {}) {
return await response.json().catch(() => ({})); return buildOpenOptions(this.config, payload);
} }
async purge(payload = {}) { async purge(payload = {}) {
return await this.request("purge", payload); const namespace = getNamespace(payload);
const openOptions = this.buildOpenOptions(payload);
let cursor = "";
let deleted = 0;
let scanned = 0;
for (let pageIndex = 0; pageIndex < 100; pageIndex++) {
const page = await this.requestV06("/trivium/list-mappings", {
...openOptions,
namespace,
page: { cursor, limit: 200 },
});
const mappings = toArray(page?.mappings);
if (!mappings.length && !page?.page?.hasMore) break;
scanned += mappings.length;
const items = mappings
.map((item) => buildNodeReference(item?.externalId, item?.namespace || namespace))
.filter((item) => item.externalId);
if (items.length) {
const result = await this.requestV06("/trivium/bulk-delete", {
...openOptions,
items,
});
deleted += Number(result?.successCount ?? items.length) || 0;
}
if (!page?.page?.hasMore) break;
cursor = String(page?.page?.nextCursor || "");
if (!cursor) break;
}
return { ok: true, scanned, deleted };
} }
async bulkUpsert(payload = {}) { async bulkUpsert(payload = {}) {
return await this.request("bulkUpsert", payload); const namespace = getNamespace(payload);
const items = toArray(payload.items);
const missingVector = items.find((item) => !normalizeVector(item?.vector || item?.embedding).length);
if (missingVector) {
throw new Error("Authority Trivium v0.6 bulkUpsert requires vector for every item");
}
const mappedItems = items.map((item) => {
const nodeId = normalizeRecordId(item?.externalId || item?.nodeId || item?.id);
const payloadSource = buildV06PayloadSource(item?.payload);
return {
externalId: nodeId,
namespace,
vector: normalizeVector(item?.vector || item?.embedding),
payload: {
...payloadSource,
nodeId: payloadSource.nodeId || nodeId,
externalId: payloadSource.externalId || nodeId,
collectionId: payload.collectionId || payloadSource.collectionId || "",
text: payloadSource.text || item?.text || "",
contentHash: payloadSource.contentHash || item?.hash || "",
index: Number(item?.index || payloadSource.index || 0) || 0,
},
};
});
return await this.requestV06("/trivium/bulk-upsert", {
...this.buildOpenOptions(payload),
items: mappedItems,
});
} }
async deleteMany(payload = {}) { async deleteMany(payload = {}) {
return await this.request("deleteMany", payload); const namespace = getNamespace(payload);
const ids = [
...toArray(payload.ids),
...toArray(payload.externalIds),
...toArray(payload.items).map((item) => item?.externalId || item?.nodeId || item?.id),
].map(normalizeRecordId).filter(Boolean);
return await this.requestV06("/trivium/bulk-delete", {
...this.buildOpenOptions(payload),
items: ids.map((id) => buildNodeReference(id, namespace)),
});
} }
async linkMany(payload = {}) { async linkMany(payload = {}) {
return await this.request("linkMany", payload); const namespace = getNamespace(payload);
const sourceLinks = toArray(payload.links || payload.items);
return await this.requestV06("/trivium/bulk-link", {
...this.buildOpenOptions(payload),
items: sourceLinks
.map((link) => {
const src = normalizeRecordId(link?.fromId || link?.src || link?.sourceId);
const dst = normalizeRecordId(link?.toId || link?.dst || link?.targetId);
if (!src || !dst) return null;
return {
src: buildNodeReference(src, namespace),
dst: buildNodeReference(dst, namespace),
label: String(link?.relation || link?.label || "related"),
weight: Number(link?.weight ?? link?.strength ?? 1) || 1,
};
})
.filter(Boolean),
});
} }
async search(payload = {}) { async search(payload = {}) {
return await this.request("search", payload); const vector = normalizeVector(payload.vector || payload.embedding || payload.queryVector);
if (!vector.length) {
throw new Error("Authority Trivium v0.6 search requires vector");
}
const queryText = String(payload.queryText || payload.text || payload.searchText || payload.query || "");
const body = {
...this.buildOpenOptions(payload),
vector,
topK: Number(payload.topK || payload.limit || 0) || undefined,
expandDepth: Number(payload.expandDepth || payload.depth || 0) || undefined,
minScore: Number.isFinite(Number(payload.minScore)) ? Number(payload.minScore) : undefined,
...(payload.payloadFilter || payload.filter ? { payloadFilter: payload.payloadFilter || payload.filter } : {}),
};
if (queryText) {
return await this.requestV06("/trivium/search-hybrid", {
...body,
queryText,
hybridAlpha: Number.isFinite(Number(payload.hybridAlpha)) ? Number(payload.hybridAlpha) : undefined,
});
}
return await this.requestV06("/trivium/search", body);
} }
async filterWhere(payload = {}) { async filterWhere(payload = {}) {
return await this.request("filterWhere", payload); const namespace = getNamespace(payload);
const result = await this.requestV06("/trivium/list-mappings", {
...this.buildOpenOptions(payload),
namespace,
page: {
limit: Number(payload.limit || payload.topK || payload.pageSize || 100) || 100,
},
});
return { items: toArray(result?.mappings) };
} }
async queryPage(payload = {}) { async queryPage(payload = {}) {
return await this.request("queryPage", payload); return await this.filterWhere(payload);
} }
async neighbors(payload = {}) { async neighbors(payload = {}) {
return await this.request("neighbors", payload); const namespace = getNamespace(payload);
const seedIds = [
...toArray(payload.ids),
...toArray(payload.nodeIds),
...toArray(payload.seedIds),
payload.id,
].map(normalizeRecordId).filter(Boolean);
const openOptions = this.buildOpenOptions(payload);
const resolved = await this.requestV06("/trivium/resolve-many", {
...openOptions,
items: seedIds.map((id) => buildNodeReference(id, namespace)),
});
const neighbors = [];
for (const item of toArray(resolved?.items)) {
const internalId = Number(item?.id);
if (!Number.isFinite(internalId) || internalId <= 0) continue;
const result = await this.requestV06("/trivium/neighbors", {
...openOptions,
id: internalId,
depth: Number(payload.depth || payload.expandDepth || 1) || 1,
});
for (const node of toArray(result?.nodes)) {
neighbors.push({
externalId: node?.externalId,
nodeId: node?.externalId,
id: node?.id,
namespace: node?.namespace,
});
}
}
return { neighbors };
} }
async stat(payload = {}) { async stat(payload = {}) {
return await this.request("stat", payload); return await this.requestV06("/trivium/stat", {
...this.buildOpenOptions(payload),
...(payload.includeMappingIntegrity ? { includeMappingIntegrity: true } : {}),
});
} }
} }
@@ -313,9 +566,13 @@ export function createAuthorityTriviumClient(config = {}, options = {}) {
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient; const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
if (injected) return injected; if (injected) return injected;
return new AuthorityTriviumHttpClient({ return new AuthorityTriviumHttpClient({
...config,
baseUrl: config.baseUrl, baseUrl: config.baseUrl,
fetchImpl: options.fetchImpl || config.fetchImpl, fetchImpl: options.fetchImpl || config.fetchImpl,
headerProvider: options.headerProvider || config.headerProvider, headerProvider: options.headerProvider || config.headerProvider,
protocol: config.protocol,
sessionToken: options.sessionToken || config.sessionToken,
sessionInitConfig: options.sessionInitConfig || config.sessionInitConfig,
}); });
} }
@@ -451,6 +708,8 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
chatId: options.chatId, chatId: options.chatId,
text: String(text || ""), text: String(text || ""),
searchText: String(text || ""), searchText: String(text || ""),
vector: normalizeVector(options.vector || options.queryVector || options.embedding),
queryVector: normalizeVector(options.queryVector || options.vector || options.embedding),
topK: Math.max(1, Math.floor(Number(options.topK) || 1)), topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean), candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
}); });
@@ -458,11 +717,15 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
} }
export async function testAuthorityTriviumConnection(config = {}, options = {}) { export async function testAuthorityTriviumConnection(config = {}, options = {}) {
const probeVector = await embedText("test connection", config, { isQuery: true });
if (!probeVector || probeVector.length === 0) {
return { success: false, dimensions: 0, error: "Embedding API 返回空结果" };
}
const client = createAuthorityTriviumClient(config, options); const client = createAuthorityTriviumClient(config, options);
await callClient(client, ["stat"], "stat", { await callClient(client, ["stat"], "stat", {
namespace: options.namespace, namespace: options.namespace,
collectionId: options.collectionId, collectionId: options.collectionId,
chatId: options.chatId, chatId: options.chatId,
}); });
return { success: true, dimensions: 0, error: "" }; return { success: true, dimensions: probeVector.length, error: "" };
} }

119
vector/embedding.js
View File

@@ -6,11 +6,17 @@
* 调用外部 API 获取文本向量，并提供暴力搜索 cosine 相似度 * 调用外部 API 获取文本向量，并提供暴力搜索 cosine 相似度
*/ */
import { getRequestHeaders } from "../../../../../script.js";
import { extension_settings } from "../../../../extensions.js"; import { extension_settings } from "../../../../extensions.js";
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js"; import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
const MODULE_NAME = "st_bme"; const MODULE_NAME = "st_bme";
const EMBEDDING_REQUEST_TIMEOUT_MS = 300000; const EMBEDDING_REQUEST_TIMEOUT_MS = 300000;
const BACKEND_SOURCES_REQUIRING_API_URL = new Set([
"ollama",
"llamacpp",
"vllm",
]);
function getEmbeddingTestOverride(name) { function getEmbeddingTestOverride(name) {
const override = globalThis.__stBmeTestOverrides?.embedding?.[name]; const override = globalThis.__stBmeTestOverrides?.embedding?.[name];
@@ -41,6 +47,69 @@ function normalizeOpenAICompatibleBaseUrl(value) {
.replace(/\/+$/, ""); .replace(/\/+$/, "");
} }
function normalizeVector(value) {
if (!Array.isArray(value)) return null;
const vector = value.map((item) => Number(item)).filter((item) => Number.isFinite(item));
return vector.length ? new Float64Array(vector) : null;
}
function readEmbeddingMode(config = {}) {
return String(config?.embeddingMode || config?.mode || "direct").trim().toLowerCase();
}
function readEmbeddingSource(config = {}) {
return String(config?.embeddingSource || config?.source || "openai").trim().toLowerCase() || "openai";
}
function buildBackendEmbeddingRequestBody(config = {}, payload = {}) {
const source = readEmbeddingSource(config);
const body = {
source,
model: String(config?.model || "").trim(),
isQuery: Boolean(payload.isQuery),
};
if (payload.text !== undefined) {
body.text = String(payload.text ?? "");
}
if (Array.isArray(payload.texts)) {
body.texts = payload.texts.map((item) => String(item ?? ""));
}
if (BACKEND_SOURCES_REQUIRING_API_URL.has(source)) {
body.apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
}
if (source === "ollama") {
body.keep = false;
}
return body;
}
async function requestBackendEmbeddings(config = {}, payload = {}, { signal } = {}) {
const response = await fetchWithTimeout(
"/api/vector/embed",
{
method: "POST",
headers: {
...getRequestHeaders(),
"Content-Type": "application/json",
},
signal,
body: JSON.stringify(buildBackendEmbeddingRequestBody(config, payload)),
},
getConfiguredTimeoutMs(config),
);
if (!response.ok) {
const errorText = await response.text().catch(() => response.statusText);
console.error(
`[ST-BME] Backend Embedding API 错误 (${response.status}):`,
errorText,
);
return null;
}
return await response.json().catch(() => ({}));
}
function createCombinedAbortSignal(...signals) { function createCombinedAbortSignal(...signals) {
const validSignals = signals.filter(Boolean); const validSignals = signals.filter(Boolean);
if (validSignals.length <= 1) { if (validSignals.length <= 1) {
@@ -107,10 +176,31 @@ async function fetchWithTimeout(
* @param {string} config.model - 模型名（如 text-embedding-3-small） * @param {string} config.model - 模型名（如 text-embedding-3-small）
* @returns {Promise<Float64Array|null>} 向量或 null * @returns {Promise<Float64Array|null>} 向量或 null
*/ */
export async function embedText(text, config, { signal } = {}) { export async function embedText(text, config, { signal, isQuery = false } = {}) {
const override = getEmbeddingTestOverride("embedText"); const override = getEmbeddingTestOverride("embedText");
if (override) { if (override) {
return await override(text, config, { signal }); return await override(text, config, { signal, isQuery });
}
if (readEmbeddingMode(config) === "backend") {
if (!text || !config?.model) {
console.warn("[ST-BME] Embedding 配置不完整，跳过");
return null;
}
try {
const payload = await requestBackendEmbeddings(
config,
{ text, isQuery },
{ signal },
);
return normalizeVector(payload?.vector);
} catch (e) {
if (isAbortError(e)) {
throw e;
}
console.error("[ST-BME] Backend Embedding 调用失败:", e);
return null;
}
} }
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl); const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
@@ -173,10 +263,31 @@ export async function embedText(text, config, { signal } = {}) {
* @param {object} config * @param {object} config
* @returns {Promise<(Float64Array|null)[]>} * @returns {Promise<(Float64Array|null)[]>}
*/ */
export async function embedBatch(texts, config, { signal } = {}) { export async function embedBatch(texts, config, { signal, isQuery = false } = {}) {
const override = getEmbeddingTestOverride("embedBatch"); const override = getEmbeddingTestOverride("embedBatch");
if (override) { if (override) {
return await override(texts, config, { signal }); return await override(texts, config, { signal, isQuery });
}
if (readEmbeddingMode(config) === "backend") {
if (!texts.length || !config?.model) {
return texts.map(() => null);
}
try {
const payload = await requestBackendEmbeddings(
config,
{ texts, isQuery },
{ signal },
);
const vectors = Array.isArray(payload?.vectors) ? payload.vectors : [];
return texts.map((_, index) => normalizeVector(vectors[index]));
} catch (e) {
if (isAbortError(e)) {
throw e;
}
console.error("[ST-BME] Backend Embedding 批量调用失败:", e);
return texts.map(() => null);
}
} }
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl); const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);

83
vector/vector-index.js
View File

@@ -236,6 +236,9 @@ export function getVectorModelScope(config) {
return [ return [
"authority", "authority",
config.source || "authority-trivium", config.source || "authority-trivium",
config.embeddingMode || "direct",
config.embeddingSource || "direct",
normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix),
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""), normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
config.model || "", config.model || "",
].join("|"); ].join("|");
@@ -266,6 +269,20 @@ export function validateVectorConfig(config) {
if (!config.baseUrl) { if (!config.baseUrl) {
return { valid: false, error: "Authority Trivium 地址不可用" }; return { valid: false, error: "Authority Trivium 地址不可用" };
} }
if (!config.model) {
return { valid: false, error: "请先填写 Embedding 模型（Authority 默认复用当前用户设置）" };
}
const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase();
const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase();
if (authorityEmbeddingMode === "backend") {
if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) {
return { valid: false, error: "当前后端 Embedding 源需要 API 地址（Authority 默认复用当前用户设置）" };
}
return { valid: true, error: "" };
}
if (!config.apiUrl) {
return { valid: false, error: "请先填写 Embedding API 地址（Authority 默认复用当前用户设置）" };
}
return { valid: true, error: "" }; return { valid: true, error: "" };
} }
@@ -606,7 +623,7 @@ function markBackendVectorStateDirty(
function markAuthorityVectorStateDirty( function markAuthorityVectorStateDirty(
graph, graph,
config, config = {},
reason = "authority-trivium-failed", reason = "authority-trivium-failed",
warning = "Authority Trivium 索引失败，已标记待重建", warning = "Authority Trivium 索引失败，已标记待重建",
) { ) {
@@ -640,6 +657,42 @@ function markAuthorityVectorStateDirty(
state.lastWarning = String(warning || "Authority Trivium 索引失败，已标记待重建"); state.lastWarning = String(warning || "Authority Trivium 索引失败，已标记待重建");
} }
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node]));
const entriesToEmbed = [];
for (const entry of entries || []) {
const node = nodesById.get(String(entry?.nodeId || ""));
const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0;
if (node && !hasEmbedding) {
entriesToEmbed.push({ entry, node });
}
}
if (!entriesToEmbed.length) {
return { requested: 0, failures: 0, elapsedMs: 0 };
}
throwIfAborted(signal);
const startedAt = nowMs();
const embeddings = await embedBatch(
entriesToEmbed.map(({ entry }) => entry.text),
config,
{ signal },
);
let failures = 0;
for (let index = 0; index < entriesToEmbed.length; index++) {
const embedding = embeddings[index];
if (embedding) {
entriesToEmbed[index].node.embedding = Array.from(embedding);
} else {
failures += 1;
}
}
return {
requested: entriesToEmbed.length,
failures,
elapsedMs: nowMs() - startedAt,
};
}
export async function syncGraphVectorIndex( export async function syncGraphVectorIndex(
graph, graph,
config, config,
@@ -741,6 +794,12 @@ export async function syncGraphVectorIndex(
try { try {
if (fullReset) { if (fullReset) {
const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal);
embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
}
const purgeStartedAt = nowMs(); const purgeStartedAt = nowMs();
await purgeAuthorityTriviumNamespace(config, authorityOptions); await purgeAuthorityTriviumNamespace(config, authorityOptions);
authorityPurgeMs += nowMs() - purgeStartedAt; authorityPurgeMs += nowMs() - purgeStartedAt;
@@ -794,6 +853,12 @@ export async function syncGraphVectorIndex(
queuedNodeIds.add(entry.nodeId); queuedNodeIds.add(entry.nodeId);
} }
const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal);
embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
}
deletedNodeCount = nodeIdsToDelete.length; deletedNodeCount = nodeIdsToDelete.length;
const deleteStartedAt = nowMs(); const deleteStartedAt = nowMs();
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions); await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
@@ -1108,7 +1173,7 @@ export async function findSimilarNodesByText(
if (isDirectVectorConfig(config)) { if (isDirectVectorConfig(config)) {
const queryEmbedStartedAt = nowMs(); const queryEmbedStartedAt = nowMs();
const queryVec = await embedText(text, config, { signal }); const queryVec = await embedText(text, config, { signal, isQuery: true });
const queryEmbedMs = nowMs() - queryEmbedStartedAt; const queryEmbedMs = nowMs() - queryEmbedStartedAt;
if (!queryVec) { if (!queryVec) {
recordSearchTimings({ recordSearchTimings({
@@ -1157,6 +1222,18 @@ export async function findSimilarNodesByText(
if (isAuthorityVectorConfig(config)) { if (isAuthorityVectorConfig(config)) {
const requestStartedAt = nowMs(); const requestStartedAt = nowMs();
try { try {
const queryEmbedStartedAt = nowMs();
const queryVec = await embedText(text, config, { signal, isQuery: true });
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
if (!queryVec) {
recordSearchTimings({
success: false,
reason: "authority-query-embed-empty",
queryEmbedMs: roundMs(queryEmbedMs),
resultCount: 0,
});
return [];
}
const allowedIds = new Set(candidateNodes.map((node) => node.id)); const allowedIds = new Set(candidateNodes.map((node) => node.id));
const results = ( const results = (
await searchAuthorityTriviumNodes(graph, text, config, { await searchAuthorityTriviumNodes(graph, text, config, {
@@ -1166,6 +1243,7 @@ export async function findSimilarNodesByText(
modelScope: getVectorModelScope(config), modelScope: getVectorModelScope(config),
topK, topK,
candidateIds: candidateNodes.map((node) => node.id), candidateIds: candidateNodes.map((node) => node.id),
queryVector: Array.from(queryVec),
signal, signal,
}) })
) )
@@ -1174,6 +1252,7 @@ export async function findSimilarNodesByText(
recordSearchTimings({ recordSearchTimings({
success: true, success: true,
reason: "ok", reason: "ok",
queryEmbedMs: roundMs(queryEmbedMs),
requestMs: roundMs(nowMs() - requestStartedAt), requestMs: roundMs(nowMs() - requestStartedAt),
resultCount: results.length, resultCount: results.length,
}); });