mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 14:20:35 +08:00
refactor(authority): complete v0.6-only sql/blob/jobs rollout
This commit is contained in:
@@ -1,6 +1,7 @@
|
|||||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||||
|
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
|
||||||
|
|
||||||
export const AUTHORITY_BLOB_ENDPOINT = "/v1/blob";
|
export const AUTHORITY_BLOB_ENDPOINT = "/fs/private";
|
||||||
|
|
||||||
function toPlainData(value, fallbackValue = null) {
|
function toPlainData(value, fallbackValue = null) {
|
||||||
if (value == null) return fallbackValue;
|
if (value == null) return fallbackValue;
|
||||||
@@ -97,7 +98,8 @@ function normalizeBlobRecordSource(input = null) {
|
|||||||
|
|
||||||
export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") {
|
export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = "") {
|
||||||
const source = normalizeBlobRecordSource(input);
|
const source = normalizeBlobRecordSource(input);
|
||||||
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath);
|
const entry = source.entry && typeof source.entry === "object" ? source.entry : null;
|
||||||
|
const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath);
|
||||||
const missing =
|
const missing =
|
||||||
source.exists === false ||
|
source.exists === false ||
|
||||||
source.found === false ||
|
source.found === false ||
|
||||||
@@ -118,21 +120,22 @@ export function normalizeAuthorityBlobReadResult(input = null, fallbackPath = ""
|
|||||||
payload: normalizeBlobPayload(input),
|
payload: normalizeBlobPayload(input),
|
||||||
contentType: String(source.contentType || source.type || "application/json"),
|
contentType: String(source.contentType || source.type || "application/json"),
|
||||||
etag: String(source.etag || source.hash || ""),
|
etag: String(source.etag || source.hash || ""),
|
||||||
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "",
|
updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "",
|
||||||
raw: toPlainData(input, input),
|
raw: toPlainData(input, input),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") {
|
export function normalizeAuthorityBlobWriteResult(input = null, fallbackPath = "") {
|
||||||
const source = normalizeBlobRecordSource(input);
|
const source = normalizeBlobRecordSource(input);
|
||||||
const path = normalizeAuthorityBlobPath(source.path || source.name || fallbackPath);
|
const entry = source.entry && typeof source.entry === "object" ? source.entry : null;
|
||||||
|
const path = normalizeAuthorityBlobPath(entry?.path || source.path || source.name || fallbackPath);
|
||||||
return {
|
return {
|
||||||
ok: input == null ? true : source.ok !== false && source.error == null,
|
ok: input == null ? true : source.ok !== false && source.error == null,
|
||||||
path,
|
path,
|
||||||
url: String(source.url || source.href || ""),
|
url: String(source.url || source.href || ""),
|
||||||
size: normalizeInteger(source.size || source.bytes, 0, 0),
|
size: normalizeInteger(source.size || source.bytes || entry?.sizeBytes, 0, 0),
|
||||||
etag: String(source.etag || source.hash || ""),
|
etag: String(source.etag || source.hash || ""),
|
||||||
updatedAt: source.updatedAt || source.updated_at || source.lastModified || "",
|
updatedAt: source.updatedAt || source.updated_at || source.lastModified || entry?.updatedAt || "",
|
||||||
raw: toPlainData(input, input),
|
raw: toPlainData(input, input),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -170,49 +173,56 @@ export function normalizeAuthorityBlobConfig(settings = {}, overrides = {}) {
|
|||||||
|
|
||||||
export class AuthorityBlobHttpClient {
|
export class AuthorityBlobHttpClient {
|
||||||
constructor(options = {}) {
|
constructor(options = {}) {
|
||||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
this.http = new AuthorityHttpClient({
|
||||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
...options,
|
||||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async request(action, payload = {}) {
|
async request(path, payload = {}, options = {}) {
|
||||||
if (typeof this.fetchImpl !== "function") {
|
return await this.http.requestJson(path, {
|
||||||
throw new Error("Authority Blob fetch unavailable");
|
method: options.method || "POST",
|
||||||
}
|
body: payload,
|
||||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_BLOB_ENDPOINT}`, {
|
session: true,
|
||||||
method: "POST",
|
signal: options.signal,
|
||||||
headers: {
|
|
||||||
Accept: "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ action, ...payload }),
|
|
||||||
});
|
});
|
||||||
if (!response?.ok) {
|
|
||||||
const text = await response?.text?.().catch(() => "");
|
|
||||||
throw new Error(text || `Authority Blob HTTP ${response?.status || "unknown"}`);
|
|
||||||
}
|
|
||||||
return await response.json().catch(() => ({}));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async writeJson(payload = {}) {
|
async writeJson(payload = {}) {
|
||||||
return await this.request("writeJson", payload);
|
return await this.writeText({
|
||||||
|
...payload,
|
||||||
|
contentType: payload.contentType || "application/json",
|
||||||
|
text: JSON.stringify(toPlainData(payload.payload ?? payload.data, payload.payload ?? payload.data)),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async writeText(payload = {}) {
|
async writeText(payload = {}) {
|
||||||
return await this.request("writeText", payload);
|
return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/write-file`, {
|
||||||
|
path: normalizeAuthorityBlobPath(payload.path || payload.name),
|
||||||
|
content: String(payload.text ?? payload.data ?? payload.content ?? ""),
|
||||||
|
encoding: "utf8",
|
||||||
|
createParents: true,
|
||||||
|
}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
|
|
||||||
async readJson(payload = {}) {
|
async readJson(payload = {}) {
|
||||||
return await this.request("readJson", payload);
|
return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/read-file`, {
|
||||||
|
path: normalizeAuthorityBlobPath(payload.path || payload.name),
|
||||||
|
encoding: "utf8",
|
||||||
|
}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
|
|
||||||
async delete(payload = {}) {
|
async delete(payload = {}) {
|
||||||
return await this.request("delete", payload);
|
return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/delete`, {
|
||||||
|
path: normalizeAuthorityBlobPath(payload.path || payload.name),
|
||||||
|
recursive: false,
|
||||||
|
}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
|
|
||||||
async stat(payload = {}) {
|
async stat(payload = {}) {
|
||||||
return await this.request("stat", payload);
|
return await this.request(`${AUTHORITY_BLOB_ENDPOINT}/stat`, {
|
||||||
|
path: normalizeAuthorityBlobPath(payload.path || payload.name),
|
||||||
|
}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -249,6 +259,10 @@ function throwIfAborted(signal) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function isMissingAuthorityBlobError(error) {
|
||||||
|
return Number(error?.status || 0) === 404;
|
||||||
|
}
|
||||||
|
|
||||||
export class AuthorityBlobAdapter {
|
export class AuthorityBlobAdapter {
|
||||||
constructor(config = {}, options = {}) {
|
constructor(config = {}, options = {}) {
|
||||||
this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {});
|
this.config = normalizeAuthorityBlobConfig(config, options.configOverrides || {});
|
||||||
@@ -291,36 +305,57 @@ export class AuthorityBlobAdapter {
|
|||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const normalizedPath = normalizeAuthorityBlobPath(path);
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
||||||
|
try {
|
||||||
const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", {
|
const result = await callClient(this.client, ["readJson", "getJson", "readFile", "get"], "readJson", {
|
||||||
namespace: options.namespace || this.config.namespace,
|
namespace: options.namespace || this.config.namespace,
|
||||||
path: normalizedPath,
|
path: normalizedPath,
|
||||||
name: normalizedPath,
|
name: normalizedPath,
|
||||||
});
|
});
|
||||||
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingAuthorityBlobError(error)) {
|
||||||
|
return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async delete(path, options = {}) {
|
async delete(path, options = {}) {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const normalizedPath = normalizeAuthorityBlobPath(path);
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, "");
|
if (!normalizedPath) return normalizeAuthorityBlobDeleteResult({ exists: false }, "");
|
||||||
|
try {
|
||||||
const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", {
|
const result = await callClient(this.client, ["delete", "deleteFile", "remove", "unlink"], "delete", {
|
||||||
namespace: options.namespace || this.config.namespace,
|
namespace: options.namespace || this.config.namespace,
|
||||||
path: normalizedPath,
|
path: normalizedPath,
|
||||||
name: normalizedPath,
|
name: normalizedPath,
|
||||||
});
|
});
|
||||||
return normalizeAuthorityBlobDeleteResult(result, normalizedPath);
|
return normalizeAuthorityBlobDeleteResult(result, normalizedPath);
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingAuthorityBlobError(error)) {
|
||||||
|
return normalizeAuthorityBlobDeleteResult({ exists: false, status: 404 }, normalizedPath);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
async stat(path, options = {}) {
|
async stat(path, options = {}) {
|
||||||
throwIfAborted(options.signal);
|
throwIfAborted(options.signal);
|
||||||
const normalizedPath = normalizeAuthorityBlobPath(path);
|
const normalizedPath = normalizeAuthorityBlobPath(path);
|
||||||
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
if (!normalizedPath) return normalizeAuthorityBlobReadResult({ exists: false }, "");
|
||||||
|
try {
|
||||||
const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", {
|
const result = await callClient(this.client, ["stat", "head", "metadata"], "stat", {
|
||||||
namespace: options.namespace || this.config.namespace,
|
namespace: options.namespace || this.config.namespace,
|
||||||
path: normalizedPath,
|
path: normalizedPath,
|
||||||
name: normalizedPath,
|
name: normalizedPath,
|
||||||
});
|
});
|
||||||
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
return normalizeAuthorityBlobReadResult(result, normalizedPath);
|
||||||
|
} catch (error) {
|
||||||
|
if (isMissingAuthorityBlobError(error)) {
|
||||||
|
return normalizeAuthorityBlobReadResult({ exists: false, status: 404 }, normalizedPath);
|
||||||
|
}
|
||||||
|
throw error;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -1,6 +1,7 @@
|
|||||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||||
|
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
|
||||||
|
|
||||||
export const AUTHORITY_JOB_ENDPOINT = "/v1/jobs";
|
export const AUTHORITY_JOB_ENDPOINT = "/jobs";
|
||||||
export const AUTHORITY_JOB_STATUS_TERMINAL = new Set([
|
export const AUTHORITY_JOB_STATUS_TERMINAL = new Set([
|
||||||
"completed",
|
"completed",
|
||||||
"succeeded",
|
"succeeded",
|
||||||
@@ -158,8 +159,8 @@ export function normalizeAuthorityJobList(payload = null) {
|
|||||||
const jobs = readJobRows(payload).map(normalizeAuthorityJobRecord).filter((job) => job.id);
|
const jobs = readJobRows(payload).map(normalizeAuthorityJobRecord).filter((job) => job.id);
|
||||||
return {
|
return {
|
||||||
jobs,
|
jobs,
|
||||||
nextCursor: String(source.nextCursor || source.next_cursor || source.cursor?.next || ""),
|
nextCursor: String(source.nextCursor || source.next_cursor || source.page?.nextCursor || source.cursor?.next || ""),
|
||||||
hasMore: Boolean(source.hasMore || source.has_more || source.cursor?.hasMore),
|
hasMore: Boolean(source.hasMore || source.has_more || source.page?.hasMore || source.cursor?.hasMore),
|
||||||
raw: toPlainData(payload, payload),
|
raw: toPlainData(payload, payload),
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
@@ -251,49 +252,68 @@ export function normalizeAuthorityJobConfig(settings = {}, overrides = {}) {
|
|||||||
|
|
||||||
export class AuthorityJobHttpClient {
|
export class AuthorityJobHttpClient {
|
||||||
constructor(options = {}) {
|
constructor(options = {}) {
|
||||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
this.http = new AuthorityHttpClient({
|
||||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
...options,
|
||||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async request(action, payload = {}) {
|
async request(path, payload = {}, options = {}) {
|
||||||
if (typeof this.fetchImpl !== "function") {
|
return await this.http.requestJson(path, {
|
||||||
throw new Error("Authority Jobs fetch unavailable");
|
method: options.method || "POST",
|
||||||
}
|
body: payload,
|
||||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_JOB_ENDPOINT}`, {
|
session: true,
|
||||||
method: "POST",
|
signal: options.signal,
|
||||||
headers: {
|
|
||||||
Accept: "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
|
||||||
},
|
|
||||||
body: JSON.stringify({ action, ...payload }),
|
|
||||||
});
|
});
|
||||||
if (!response?.ok) {
|
|
||||||
const text = await response?.text?.().catch(() => "");
|
|
||||||
throw new Error(text || `Authority Jobs HTTP ${response?.status || "unknown"}`);
|
|
||||||
}
|
|
||||||
return await response.json().catch(() => ({}));
|
|
||||||
}
|
}
|
||||||
|
|
||||||
async submit(payload = {}) {
|
async submit(payload = {}) {
|
||||||
return await this.request("submit", payload);
|
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/create`, {
|
||||||
|
type: String(payload.type || payload.kind || "").trim(),
|
||||||
|
payload: toPlainData(payload.payload, payload.payload),
|
||||||
|
...(payload.timeoutMs != null ? { timeoutMs: normalizeInteger(payload.timeoutMs, 0, 0, 3600000) } : {}),
|
||||||
|
...(payload.idempotencyKey ? { idempotencyKey: String(payload.idempotencyKey) } : {}),
|
||||||
|
...(payload.maxAttempts != null ? { maxAttempts: normalizeInteger(payload.maxAttempts, 1, 1, 1000) } : {}),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async listPage(payload = {}) {
|
async listPage(payload = {}) {
|
||||||
return await this.request("listPage", payload);
|
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/list`, {
|
||||||
|
page: {
|
||||||
|
...(payload.cursor ? { cursor: String(payload.cursor) } : {}),
|
||||||
|
limit: normalizeInteger(payload.limit, 20, 1, 100),
|
||||||
|
},
|
||||||
|
...(payload.filter && typeof payload.filter === "object" && !Array.isArray(payload.filter) && Object.keys(payload.filter).length > 0
|
||||||
|
? { filter: toPlainData(payload.filter, {}) }
|
||||||
|
: {}),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async waitForCompletion(payload = {}) {
|
async get(payload = {}) {
|
||||||
return await this.request("waitForCompletion", payload);
|
const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
|
||||||
|
if (!id) {
|
||||||
|
throw new Error("Authority Jobs get requires job id");
|
||||||
|
}
|
||||||
|
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}`, undefined, {
|
||||||
|
method: "GET",
|
||||||
|
signal: payload.signal,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async requeue(payload = {}) {
|
async requeue(payload = {}) {
|
||||||
return await this.request("requeue", payload);
|
const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
|
||||||
|
if (!id) {
|
||||||
|
throw new Error("Authority Jobs requeue requires job id");
|
||||||
|
}
|
||||||
|
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/requeue`, {}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
|
|
||||||
async cancel(payload = {}) {
|
async cancel(payload = {}) {
|
||||||
return await this.request("cancel", payload);
|
const id = encodeURIComponent(normalizeRecordId(payload.jobId || payload.id));
|
||||||
|
if (!id) {
|
||||||
|
throw new Error("Authority Jobs cancel requires job id");
|
||||||
|
}
|
||||||
|
return await this.request(`${AUTHORITY_JOB_ENDPOINT}/${id}/cancel`, {}, { signal: payload.signal });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -9,6 +9,7 @@ import {
|
|||||||
queryAuthorityTriviumNeighbors,
|
queryAuthorityTriviumNeighbors,
|
||||||
searchAuthorityTriviumNodes,
|
searchAuthorityTriviumNodes,
|
||||||
} from "../vector/authority-vector-primary-adapter.js";
|
} from "../vector/authority-vector-primary-adapter.js";
|
||||||
|
import { embedText } from "../vector/embedding.js";
|
||||||
|
|
||||||
function nowMs() {
|
function nowMs() {
|
||||||
if (typeof performance?.now === "function") {
|
if (typeof performance?.now === "function") {
|
||||||
@@ -247,6 +248,11 @@ export async function resolveAuthorityRecallCandidates({
|
|||||||
const searchStartedAt = nowMs();
|
const searchStartedAt = nowMs();
|
||||||
for (const queryEntry of queryPlan.queries) {
|
for (const queryEntry of queryPlan.queries) {
|
||||||
try {
|
try {
|
||||||
|
const queryVec = await embedText(queryEntry.text, embeddingConfig, { signal, isQuery: true });
|
||||||
|
if (!queryVec) {
|
||||||
|
diagnostics.fallbackReason = diagnostics.fallbackReason || "authority-candidate-query-embed-empty";
|
||||||
|
continue;
|
||||||
|
}
|
||||||
const searchResults = await searchAuthorityTriviumNodes(
|
const searchResults = await searchAuthorityTriviumNodes(
|
||||||
graph,
|
graph,
|
||||||
queryEntry.text,
|
queryEntry.text,
|
||||||
@@ -257,6 +263,7 @@ export async function resolveAuthorityRecallCandidates({
|
|||||||
chatId,
|
chatId,
|
||||||
topK: limit,
|
topK: limit,
|
||||||
candidateIds: filteredIds.length > 0 ? filteredIds : undefined,
|
candidateIds: filteredIds.length > 0 ? filteredIds : undefined,
|
||||||
|
queryVector: Array.from(queryVec),
|
||||||
signal,
|
signal,
|
||||||
},
|
},
|
||||||
);
|
);
|
||||||
|
|||||||
@@ -484,7 +484,7 @@ export async function runResidualRecall({
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
const queryVec = await embedText(queryText, embeddingConfig, { signal });
|
const queryVec = await embedText(queryText, embeddingConfig, { signal, isQuery: true });
|
||||||
if (!queryVec || queryVec.length === 0) {
|
if (!queryVec || queryVec.length === 0) {
|
||||||
return {
|
return {
|
||||||
triggered: false,
|
triggered: false,
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
const DEFAULT_AUTHORITY_BASE_URL = "/api/plugins/authority";
|
const DEFAULT_AUTHORITY_BASE_URL = "/api/plugins/authority";
|
||||||
const DEFAULT_AUTHORITY_PROBE_INTERVAL_MS = 60000;
|
const DEFAULT_AUTHORITY_PROBE_INTERVAL_MS = 60000;
|
||||||
|
|
||||||
const SQL_FEATURES = ["sql", "sql.query", "sql.page", "sql.pageall", "querysql"];
|
const SQL_FEATURES = ["sql", "sql.query", "sql.querypage", "sql.page", "sql.pageall", "querysql"];
|
||||||
const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.write", "sql.transaction"];
|
const SQL_MUTATION_FEATURES = ["sql", "sql.mutation", "sql.execute", "sql.exec", "sql.write", "sql.transaction"];
|
||||||
const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert"];
|
const TRIVIUM_FEATURES = ["trivium", "trivium.search", "trivium.query", "trivium.filterwhere", "trivium.bulkupsert", "trivium.upsert", "trivium.bulkmutations"];
|
||||||
const JOB_FEATURES = ["jobs", "jobs.list", "jobs.wait", "events", "sse"];
|
const JOB_FEATURES = ["jobs", "jobs.background", "jobs.list", "jobs.wait", "diagnostics.jobspage", "events", "sse"];
|
||||||
const BLOB_FEATURES = ["blob", "blob.write", "privatefiles", "private.files", "files.private"];
|
const BLOB_FEATURES = ["blob", "blob.write", "storage.blob", "transfers.blob", "transfers.fs", "fs.private", "privatefiles", "private.files", "files.private"];
|
||||||
|
|
||||||
function toBoolean(value, fallback = false) {
|
function toBoolean(value, fallback = false) {
|
||||||
if (typeof value === "boolean") return value;
|
if (typeof value === "boolean") return value;
|
||||||
@@ -89,6 +89,189 @@ function readNowMs() {
|
|||||||
return Date.now();
|
return Date.now();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function clonePlain(value, fallbackValue = null) {
|
||||||
|
if (value == null) return fallbackValue;
|
||||||
|
if (typeof globalThis.structuredClone === "function") {
|
||||||
|
try {
|
||||||
|
return globalThis.structuredClone(value);
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.parse(JSON.stringify(value));
|
||||||
|
} catch {
|
||||||
|
return fallbackValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeHeaderName(name = "") {
|
||||||
|
return String(name || "").trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildDefaultSessionInitConfig(source = {}) {
|
||||||
|
const config = source && typeof source === "object" && !Array.isArray(source) ? source : {};
|
||||||
|
return {
|
||||||
|
extensionId: String(config.extensionId || "third-party/st-bme"),
|
||||||
|
displayName: String(config.displayName || "ST-BME"),
|
||||||
|
version: String(config.version || "0.0.0"),
|
||||||
|
installType: String(config.installType || "local"),
|
||||||
|
declaredPermissions: clonePlain(config.declaredPermissions, null) || {
|
||||||
|
storage: { kv: true, blob: true },
|
||||||
|
fs: { private: true },
|
||||||
|
sql: { private: true },
|
||||||
|
trivium: { private: true },
|
||||||
|
jobs: { background: true },
|
||||||
|
events: { channels: true },
|
||||||
|
},
|
||||||
|
...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function withJsonHeaders(headers = {}) {
|
||||||
|
return {
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...(headers || {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readResponsePayload(response = null) {
|
||||||
|
if (!response) return {};
|
||||||
|
if (typeof response.json === "function") {
|
||||||
|
try {
|
||||||
|
return await response.json();
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof response.text === "function") {
|
||||||
|
try {
|
||||||
|
return { error: await response.text() };
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPayloadMessage(payload = {}, fallback = "") {
|
||||||
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback;
|
||||||
|
return String(payload.error || payload.message || payload.reason || fallback || "");
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildAuthorityPermissionEvaluateRequests(settings = {}, readiness = {}, options = {}) {
|
||||||
|
const requests = [];
|
||||||
|
const sqlTarget = String(options.sqlTarget || settings.sqlTarget || "default");
|
||||||
|
const triviumTarget = String(options.triviumTarget || settings.triviumTarget || "st_bme_vectors");
|
||||||
|
const jobTarget = String(options.jobTarget || settings.jobTarget || "delay");
|
||||||
|
if (readiness.sql || readiness.sqlMutation) {
|
||||||
|
requests.push({ resource: "sql.private", target: sqlTarget, reason: `Probe SQL capability for ${sqlTarget}` });
|
||||||
|
}
|
||||||
|
if (readiness.trivium) {
|
||||||
|
requests.push({ resource: "trivium.private", target: triviumTarget, reason: `Probe Trivium capability for ${triviumTarget}` });
|
||||||
|
}
|
||||||
|
if (readiness.blob) {
|
||||||
|
requests.push({ resource: "fs.private", reason: "Probe private file capability for Authority Blob adapter" });
|
||||||
|
}
|
||||||
|
if (readiness.jobs) {
|
||||||
|
requests.push({ resource: "jobs.background", target: jobTarget, reason: `Probe Jobs capability for ${jobTarget}` });
|
||||||
|
}
|
||||||
|
return requests;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function verifyAuthorityDataPlane(baseUrl, fetchImpl, headers, settings = {}, readiness = {}, options = {}) {
|
||||||
|
const initHeaders = withJsonHeaders(headers);
|
||||||
|
const initResponse = await fetchImpl(`${baseUrl}/session/init`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: initHeaders,
|
||||||
|
body: JSON.stringify(buildDefaultSessionInitConfig(options.sessionInitConfig || settings)),
|
||||||
|
});
|
||||||
|
const initStatus = Number(initResponse?.status || 0);
|
||||||
|
const initPayload = await readResponsePayload(initResponse);
|
||||||
|
if (!initResponse?.ok) {
|
||||||
|
return {
|
||||||
|
sessionReady: false,
|
||||||
|
permissionReady: false,
|
||||||
|
reason: initStatus === 401 || initStatus === 403 ? "session-init-denied" : "session-init-failed",
|
||||||
|
lastError: readPayloadMessage(initPayload, `HTTP ${initStatus || "unknown"}`),
|
||||||
|
status: initStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionToken = String(initPayload?.sessionToken || initPayload?.token || "");
|
||||||
|
if (!sessionToken) {
|
||||||
|
return {
|
||||||
|
sessionReady: false,
|
||||||
|
permissionReady: false,
|
||||||
|
reason: "session-token-missing",
|
||||||
|
lastError: "session token missing",
|
||||||
|
status: initStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const sessionHeaders = {
|
||||||
|
...withJsonHeaders(headers),
|
||||||
|
...(Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === "x-authority-session-token")
|
||||||
|
? {}
|
||||||
|
: { "x-authority-session-token": sessionToken }),
|
||||||
|
};
|
||||||
|
const currentResponse = await fetchImpl(`${baseUrl}/session/current`, {
|
||||||
|
method: "GET",
|
||||||
|
headers: sessionHeaders,
|
||||||
|
});
|
||||||
|
const currentStatus = Number(currentResponse?.status || 0);
|
||||||
|
const currentPayload = await readResponsePayload(currentResponse);
|
||||||
|
if (!currentResponse?.ok) {
|
||||||
|
return {
|
||||||
|
sessionReady: false,
|
||||||
|
permissionReady: false,
|
||||||
|
reason: currentStatus === 401 || currentStatus === 403 ? "session-invalid" : "session-current-failed",
|
||||||
|
lastError: readPayloadMessage(currentPayload, `HTTP ${currentStatus || "unknown"}`),
|
||||||
|
status: currentStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const requests = buildAuthorityPermissionEvaluateRequests(settings, readiness, options);
|
||||||
|
if (!requests.length) {
|
||||||
|
return {
|
||||||
|
sessionReady: true,
|
||||||
|
permissionReady: true,
|
||||||
|
reason: "ok",
|
||||||
|
lastError: "",
|
||||||
|
status: currentStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const permissionResponse = await fetchImpl(`${baseUrl}/permissions/evaluate-batch`, {
|
||||||
|
method: "POST",
|
||||||
|
headers: sessionHeaders,
|
||||||
|
body: JSON.stringify({ requests }),
|
||||||
|
});
|
||||||
|
const permissionStatus = Number(permissionResponse?.status || 0);
|
||||||
|
const permissionPayload = await readResponsePayload(permissionResponse);
|
||||||
|
if (!permissionResponse?.ok) {
|
||||||
|
return {
|
||||||
|
sessionReady: true,
|
||||||
|
permissionReady: false,
|
||||||
|
reason: permissionStatus === 401 || permissionStatus === 403 ? "permission-denied" : "permission-evaluate-failed",
|
||||||
|
lastError: readPayloadMessage(permissionPayload, `HTTP ${permissionStatus || "unknown"}`),
|
||||||
|
status: permissionStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
const results = Array.isArray(permissionPayload?.results) ? permissionPayload.results : [];
|
||||||
|
const permissionReady = results.length === requests.length && results.every((result) => {
|
||||||
|
const decision = String(result?.decision || result?.grant?.status || "").trim().toLowerCase();
|
||||||
|
return decision === "granted";
|
||||||
|
});
|
||||||
|
return {
|
||||||
|
sessionReady: true,
|
||||||
|
permissionReady,
|
||||||
|
reason: permissionReady ? "ok" : "permission-not-ready",
|
||||||
|
lastError: permissionReady ? "" : "required Authority permissions are not granted",
|
||||||
|
status: permissionStatus || currentStatus,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export function normalizeAuthorityBaseUrl(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
|
export function normalizeAuthorityBaseUrl(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
|
||||||
const normalized = String(baseUrl || DEFAULT_AUTHORITY_BASE_URL).trim() || DEFAULT_AUTHORITY_BASE_URL;
|
const normalized = String(baseUrl || DEFAULT_AUTHORITY_BASE_URL).trim() || DEFAULT_AUTHORITY_BASE_URL;
|
||||||
return normalized.replace(/\/+$/, "");
|
return normalized.replace(/\/+$/, "");
|
||||||
@@ -116,12 +299,7 @@ export function normalizeAuthoritySettings(settings = {}) {
|
|||||||
|
|
||||||
export function buildAuthorityProbeUrls(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
|
export function buildAuthorityProbeUrls(baseUrl = DEFAULT_AUTHORITY_BASE_URL) {
|
||||||
const normalizedBaseUrl = normalizeAuthorityBaseUrl(baseUrl);
|
const normalizedBaseUrl = normalizeAuthorityBaseUrl(baseUrl);
|
||||||
return [
|
return [`${normalizedBaseUrl}/probe`];
|
||||||
`${normalizedBaseUrl}/v1/diagnostics/probe`,
|
|
||||||
`${normalizedBaseUrl}/v1/probe`,
|
|
||||||
`${normalizedBaseUrl}/probe`,
|
|
||||||
normalizedBaseUrl,
|
|
||||||
];
|
|
||||||
}
|
}
|
||||||
|
|
||||||
export function collectAuthorityFeatures(payload = {}) {
|
export function collectAuthorityFeatures(payload = {}) {
|
||||||
@@ -295,7 +473,7 @@ export async function probeAuthorityCapabilities(options = {}) {
|
|||||||
for (const endpoint of buildAuthorityProbeUrls(settings.baseUrl)) {
|
for (const endpoint of buildAuthorityProbeUrls(settings.baseUrl)) {
|
||||||
const startedAt = readNowMs();
|
const startedAt = readNowMs();
|
||||||
try {
|
try {
|
||||||
const response = await fetchImpl(endpoint, { method: "GET", headers });
|
const response = await fetchImpl(endpoint, { method: "POST", headers });
|
||||||
const finishedAt = readNowMs();
|
const finishedAt = readNowMs();
|
||||||
const status = Number(response?.status || 0);
|
const status = Number(response?.status || 0);
|
||||||
lastStatus = status;
|
lastStatus = status;
|
||||||
@@ -340,13 +518,43 @@ export async function probeAuthorityCapabilities(options = {}) {
|
|||||||
} catch {
|
} catch {
|
||||||
payload = {};
|
payload = {};
|
||||||
}
|
}
|
||||||
return normalizeAuthorityProbeResponse(payload, {
|
const features = collectAuthorityFeatures(payload);
|
||||||
settings,
|
const readiness = createFeatureReadiness(features);
|
||||||
|
const missingFeatures = collectMissingFeatures(readiness);
|
||||||
|
const healthy = payload?.healthy ?? payload?.ok ?? true;
|
||||||
|
let sessionReady = payload?.sessionReady ?? payload?.session?.ready ?? payload?.session?.active;
|
||||||
|
let permissionReady = payload?.permissionReady ?? payload?.permissions?.ready ?? payload?.authorized;
|
||||||
|
let reason = missingFeatures.length ? "missing-required-features" : "ok";
|
||||||
|
let dataPlaneLastError = "";
|
||||||
|
let dataPlaneStatus = status;
|
||||||
|
if (healthy) {
|
||||||
|
const verified = await verifyAuthorityDataPlane(settings.baseUrl, fetchImpl, headers, settings, readiness, options);
|
||||||
|
sessionReady = verified.sessionReady;
|
||||||
|
permissionReady = verified.permissionReady;
|
||||||
|
dataPlaneStatus = Number(verified.status || status || 0);
|
||||||
|
dataPlaneLastError = String(verified.lastError || "");
|
||||||
|
if (verified.reason && verified.reason !== "ok") {
|
||||||
|
reason = verified.reason;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return normalizeAuthorityCapabilityState(
|
||||||
|
{
|
||||||
|
installed: true,
|
||||||
|
healthy: Boolean(healthy),
|
||||||
|
sessionReady: Boolean(sessionReady),
|
||||||
|
permissionReady: Boolean(permissionReady),
|
||||||
|
features: Array.from(features),
|
||||||
|
missingFeatures,
|
||||||
|
reason,
|
||||||
|
lastError: dataPlaneLastError,
|
||||||
endpoint,
|
endpoint,
|
||||||
status,
|
status: dataPlaneStatus,
|
||||||
latencyMs: normalizeLatencyMs(startedAt, finishedAt),
|
latencyMs: normalizeLatencyMs(startedAt, finishedAt),
|
||||||
nowMs,
|
lastProbeAt: nowMs,
|
||||||
});
|
updatedAt: new Date(nowMs).toISOString(),
|
||||||
|
},
|
||||||
|
settings,
|
||||||
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
lastError = error?.message || String(error);
|
lastError = error?.message || String(error);
|
||||||
}
|
}
|
||||||
|
|||||||
194
runtime/authority-http-client.js
Normal file
194
runtime/authority-http-client.js
Normal file
@@ -0,0 +1,194 @@
|
|||||||
|
import { normalizeAuthorityBaseUrl } from "./authority-capabilities.js";
|
||||||
|
|
||||||
|
export const AUTHORITY_PROTOCOL_AUTO = "auto";
|
||||||
|
export const AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06 = "server-plugin-v06";
|
||||||
|
export const AUTHORITY_SESSION_HEADER = "x-authority-session-token";
|
||||||
|
|
||||||
|
function normalizeProtocol(value = AUTHORITY_PROTOCOL_AUTO) {
|
||||||
|
const normalized = String(value || AUTHORITY_PROTOCOL_AUTO).trim().toLowerCase();
|
||||||
|
if (["v06", "v0.6", "server-plugin", AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06].includes(normalized)) {
|
||||||
|
return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
|
||||||
|
}
|
||||||
|
return AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
|
||||||
|
}
|
||||||
|
|
||||||
|
function clonePlain(value, fallbackValue = null) {
|
||||||
|
if (value == null) return fallbackValue;
|
||||||
|
if (typeof globalThis.structuredClone === "function") {
|
||||||
|
try {
|
||||||
|
return globalThis.structuredClone(value);
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
return JSON.parse(JSON.stringify(value));
|
||||||
|
} catch {
|
||||||
|
return fallbackValue;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeHeaderName(name = "") {
|
||||||
|
return String(name || "").trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function hasSessionHeader(headers = {}) {
|
||||||
|
return Object.keys(headers || {}).some((name) => normalizeHeaderName(name) === AUTHORITY_SESSION_HEADER);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildDefaultSessionInitConfig(source = {}) {
|
||||||
|
const config = source && typeof source === "object" && !Array.isArray(source) ? source : {};
|
||||||
|
return {
|
||||||
|
extensionId: String(config.extensionId || "third-party/st-bme"),
|
||||||
|
displayName: String(config.displayName || "ST-BME"),
|
||||||
|
version: String(config.version || "0.0.0"),
|
||||||
|
installType: String(config.installType || "local"),
|
||||||
|
declaredPermissions: clonePlain(config.declaredPermissions, null) || {
|
||||||
|
storage: { kv: true, blob: true },
|
||||||
|
fs: { private: true },
|
||||||
|
sql: { private: true },
|
||||||
|
trivium: { private: true },
|
||||||
|
jobs: { background: true },
|
||||||
|
events: { channels: true },
|
||||||
|
},
|
||||||
|
...(config.uiLabel ? { uiLabel: String(config.uiLabel) } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function readPayloadErrorMessage(payload = null, fallback = "") {
|
||||||
|
if (!payload || typeof payload !== "object" || Array.isArray(payload)) return fallback;
|
||||||
|
return String(payload.error || payload.message || payload.reason || fallback || "");
|
||||||
|
}
|
||||||
|
|
||||||
|
async function readResponsePayload(response = null) {
|
||||||
|
if (!response) return {};
|
||||||
|
const contentType = String(response.headers?.get?.("content-type") || "").toLowerCase();
|
||||||
|
if (contentType.includes("application/json") && typeof response.json === "function") {
|
||||||
|
try {
|
||||||
|
return await response.json();
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof response.json === "function") {
|
||||||
|
try {
|
||||||
|
return await response.json();
|
||||||
|
} catch {
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (typeof response.text === "function") {
|
||||||
|
try {
|
||||||
|
return { error: await response.text() };
|
||||||
|
} catch {
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {};
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthorityHttpError extends Error {
|
||||||
|
constructor(message, options = {}) {
|
||||||
|
super(message);
|
||||||
|
this.name = "AuthorityHttpError";
|
||||||
|
this.status = Number(options.status || 0);
|
||||||
|
this.code = String(options.code || "");
|
||||||
|
this.category = String(options.category || "");
|
||||||
|
this.payload = clonePlain(options.payload, null);
|
||||||
|
this.path = String(options.path || "");
|
||||||
|
this.protocol = String(options.protocol || "");
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export class AuthorityHttpClient {
|
||||||
|
constructor(options = {}) {
|
||||||
|
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
||||||
|
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
||||||
|
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
||||||
|
this.protocol = normalizeProtocol(options.protocol || options.authorityProtocol);
|
||||||
|
this.sessionToken = String(options.sessionToken || options.authoritySessionToken || "");
|
||||||
|
this.sessionInitConfig = buildDefaultSessionInitConfig(options.sessionInitConfig || options.initConfig || options);
|
||||||
|
this.sessionPromise = null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async buildHeaders({ session = false } = {}) {
|
||||||
|
let provided = {};
|
||||||
|
if (this.headerProvider) {
|
||||||
|
provided = await this.headerProvider() || {};
|
||||||
|
}
|
||||||
|
const headers = {
|
||||||
|
Accept: "application/json",
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
...provided,
|
||||||
|
};
|
||||||
|
if (session && this.sessionToken && !hasSessionHeader(headers)) {
|
||||||
|
headers[AUTHORITY_SESSION_HEADER] = this.sessionToken;
|
||||||
|
}
|
||||||
|
return headers;
|
||||||
|
}
|
||||||
|
|
||||||
|
async ensureSession() {
|
||||||
|
if (this.sessionToken) return this.sessionToken;
|
||||||
|
if (!this.sessionPromise) {
|
||||||
|
this.sessionPromise = this.requestJson("/session/init", {
|
||||||
|
method: "POST",
|
||||||
|
body: this.sessionInitConfig,
|
||||||
|
session: false,
|
||||||
|
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||||
|
}).then((payload) => {
|
||||||
|
const token = String(payload?.sessionToken || payload?.token || "");
|
||||||
|
if (!token) {
|
||||||
|
throw new AuthorityHttpError("Authority session init did not return a session token", {
|
||||||
|
status: 0,
|
||||||
|
path: "/session/init",
|
||||||
|
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||||
|
payload,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
this.sessionToken = token;
|
||||||
|
return token;
|
||||||
|
}).catch((error) => {
|
||||||
|
this.sessionPromise = null;
|
||||||
|
throw error;
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return await this.sessionPromise;
|
||||||
|
}
|
||||||
|
|
||||||
|
async requestJson(path, options = {}) {
|
||||||
|
if (typeof this.fetchImpl !== "function") {
|
||||||
|
throw new AuthorityHttpError("Authority fetch unavailable", {
|
||||||
|
path,
|
||||||
|
protocol: options.protocol || this.protocol,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
const method = String(options.method || "POST").toUpperCase();
|
||||||
|
const session = Boolean(options.session);
|
||||||
|
if (session && !this.sessionToken) {
|
||||||
|
await this.ensureSession();
|
||||||
|
}
|
||||||
|
const headers = await this.buildHeaders({ session });
|
||||||
|
const response = await this.fetchImpl(`${this.baseUrl}${path}`, {
|
||||||
|
method,
|
||||||
|
headers,
|
||||||
|
...(method === "GET" || options.body === undefined ? {} : { body: JSON.stringify(options.body) }),
|
||||||
|
...(options.signal ? { signal: options.signal } : {}),
|
||||||
|
});
|
||||||
|
const status = Number(response?.status || 0);
|
||||||
|
const payload = await readResponsePayload(response);
|
||||||
|
if (!response?.ok) {
|
||||||
|
const message = readPayloadErrorMessage(payload, `Authority HTTP ${status || "unknown"}`);
|
||||||
|
throw new AuthorityHttpError(message || `Authority HTTP ${status || "unknown"}`, {
|
||||||
|
status,
|
||||||
|
code: payload?.code,
|
||||||
|
category: payload?.category,
|
||||||
|
payload,
|
||||||
|
path,
|
||||||
|
protocol: options.protocol || this.protocol,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return payload;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
export function createAuthorityHttpClient(options = {}) {
|
||||||
|
return new AuthorityHttpClient(options);
|
||||||
|
}
|
||||||
@@ -20,6 +20,86 @@ const SOURCE_ROOTS = [
|
|||||||
"native",
|
"native",
|
||||||
];
|
];
|
||||||
|
|
||||||
|
function toDataModuleUrl(source = "") {
|
||||||
|
return `data:text/javascript,${encodeURIComponent(String(source || ""))}`;
|
||||||
|
}
|
||||||
|
|
||||||
|
const CHECK_SYNTAX_HOOK_BOOTSTRAP_URL = toDataModuleUrl(`
|
||||||
|
import * as nodeModule from "node:module";
|
||||||
|
|
||||||
|
const register = typeof nodeModule.register === "function" ? nodeModule.register : undefined;
|
||||||
|
const registerHooks = typeof nodeModule.registerHooks === "function"
|
||||||
|
? nodeModule.registerHooks
|
||||||
|
: undefined;
|
||||||
|
|
||||||
|
const scriptShimUrl = ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export function getRequestHeaders() { return {}; }",
|
||||||
|
"export function saveMetadata() {}",
|
||||||
|
"export function saveSettingsDebounced() {}",
|
||||||
|
"export function substituteParamsExtended(value) { return String(value ?? ''); }",
|
||||||
|
].join("\n")))};
|
||||||
|
const extensionsShimUrl = ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export const extension_settings = { st_bme: {} };",
|
||||||
|
"export function getContext() { return {}; }",
|
||||||
|
"export function saveMetadataDebounced() {}",
|
||||||
|
].join("\n")))};
|
||||||
|
const openaiShimUrl = ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export const chat_completion_sources = {};",
|
||||||
|
"export async function sendOpenAIRequest() { return {}; }",
|
||||||
|
].join("\n")))};
|
||||||
|
|
||||||
|
function resolveShim(specifier) {
|
||||||
|
const normalized = String(specifier || "");
|
||||||
|
if (normalized.endsWith("/script.js")) return scriptShimUrl;
|
||||||
|
if (normalized.endsWith("/extensions.js")) return extensionsShimUrl;
|
||||||
|
if (normalized.endsWith("/openai.js")) return openaiShimUrl;
|
||||||
|
return "";
|
||||||
|
}
|
||||||
|
|
||||||
|
if (typeof registerHooks === "function") {
|
||||||
|
registerHooks({
|
||||||
|
resolve(specifier, context, nextResolve) {
|
||||||
|
const shimUrl = resolveShim(specifier);
|
||||||
|
if (shimUrl) {
|
||||||
|
return {
|
||||||
|
shortCircuit: true,
|
||||||
|
url: shimUrl,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return nextResolve(specifier, context);
|
||||||
|
},
|
||||||
|
});
|
||||||
|
} else if (typeof register === "function") {
|
||||||
|
register(${JSON.stringify(toDataModuleUrl(`
|
||||||
|
export async function resolve(specifier, context, nextResolve) {
|
||||||
|
const normalized = String(specifier || "");
|
||||||
|
if (normalized.endsWith("/script.js")) {
|
||||||
|
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export function getRequestHeaders() { return {}; }",
|
||||||
|
"export function saveMetadata() {}",
|
||||||
|
"export function saveSettingsDebounced() {}",
|
||||||
|
"export function substituteParamsExtended(value) { return String(value ?? ''); }",
|
||||||
|
].join("\n")))} };
|
||||||
|
}
|
||||||
|
if (normalized.endsWith("/extensions.js")) {
|
||||||
|
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export const extension_settings = { st_bme: {} };",
|
||||||
|
"export function getContext() { return {}; }",
|
||||||
|
"export function saveMetadataDebounced() {}",
|
||||||
|
].join("\n")))} };
|
||||||
|
}
|
||||||
|
if (normalized.endsWith("/openai.js")) {
|
||||||
|
return { shortCircuit: true, url: ${JSON.stringify(toDataModuleUrl([
|
||||||
|
"export const chat_completion_sources = {};",
|
||||||
|
"export async function sendOpenAIRequest() { return {}; }",
|
||||||
|
].join("\n")))} };
|
||||||
|
}
|
||||||
|
return nextResolve(specifier, context);
|
||||||
|
}
|
||||||
|
`))}, import.meta.url);
|
||||||
|
}
|
||||||
|
`);
|
||||||
|
|
||||||
async function collectFiles(targetPath) {
|
async function collectFiles(targetPath) {
|
||||||
const absolutePath = path.resolve(process.cwd(), targetPath);
|
const absolutePath = path.resolve(process.cwd(), targetPath);
|
||||||
const fileStat = await stat(absolutePath);
|
const fileStat = await stat(absolutePath);
|
||||||
@@ -48,7 +128,7 @@ function toPosixPath(filePath) {
|
|||||||
|
|
||||||
async function runNodeCheck(filePath) {
|
async function runNodeCheck(filePath) {
|
||||||
return await new Promise((resolve, reject) => {
|
return await new Promise((resolve, reject) => {
|
||||||
const child = spawn(process.execPath, ["--check", filePath], {
|
const child = spawn(process.execPath, ["--import", CHECK_SYNTAX_HOOK_BOOTSTRAP_URL, "--check", filePath], {
|
||||||
cwd: process.cwd(),
|
cwd: process.cwd(),
|
||||||
stdio: "inherit",
|
stdio: "inherit",
|
||||||
windowsHide: true,
|
windowsHide: true,
|
||||||
|
|||||||
@@ -7,13 +7,17 @@ import {
|
|||||||
buildSnapshotFromGraph,
|
buildSnapshotFromGraph,
|
||||||
} from "./bme-db.js";
|
} from "./bme-db.js";
|
||||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||||
|
import { AuthorityHttpClient } from "../runtime/authority-http-client.js";
|
||||||
|
|
||||||
export const AUTHORITY_GRAPH_STORE_KIND = "authority";
|
export const AUTHORITY_GRAPH_STORE_KIND = "authority";
|
||||||
export const AUTHORITY_GRAPH_STORE_MODE = "authority-sql-primary";
|
export const AUTHORITY_GRAPH_STORE_MODE = "authority-sql-primary";
|
||||||
|
|
||||||
const META_DEFAULT_LAST_PROCESSED_FLOOR = -1;
|
const META_DEFAULT_LAST_PROCESSED_FLOOR = -1;
|
||||||
const META_DEFAULT_EXTRACTION_COUNT = 0;
|
const META_DEFAULT_EXTRACTION_COUNT = 0;
|
||||||
const AUTHORITY_SQL_ENDPOINT = "/v1/sql";
|
const DEFAULT_AUTHORITY_SQL_DATABASE = "default";
|
||||||
|
const AUTHORITY_SQL_QUERY_ENDPOINT = "/sql/query";
|
||||||
|
const AUTHORITY_SQL_EXEC_ENDPOINT = "/sql/exec";
|
||||||
|
const AUTHORITY_SQL_TRANSACTION_ENDPOINT = "/sql/transaction";
|
||||||
|
|
||||||
const AUTHORITY_TABLES = Object.freeze({
|
const AUTHORITY_TABLES = Object.freeze({
|
||||||
meta: "st_bme_graph_meta",
|
meta: "st_bme_graph_meta",
|
||||||
@@ -340,41 +344,47 @@ function normalizeUpsertCountDelta(delta = {}) {
|
|||||||
|
|
||||||
export class AuthoritySqlHttpClient {
|
export class AuthoritySqlHttpClient {
|
||||||
constructor(options = {}) {
|
constructor(options = {}) {
|
||||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
this.http = new AuthorityHttpClient({
|
||||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
...options,
|
||||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
baseUrl: normalizeAuthorityBaseUrl(options.baseUrl),
|
||||||
|
});
|
||||||
|
this.database = normalizeRecordId(options.database) || DEFAULT_AUTHORITY_SQL_DATABASE;
|
||||||
}
|
}
|
||||||
|
|
||||||
async query(sql, params = {}) {
|
async query(sql, params = {}) {
|
||||||
return await this._request({ action: "query", sql, params });
|
return await this._request(AUTHORITY_SQL_QUERY_ENDPOINT, {
|
||||||
|
database: this.database,
|
||||||
|
statement: String(sql || ""),
|
||||||
|
params,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async execute(sql, params = {}) {
|
async execute(sql, params = {}) {
|
||||||
return await this._request({ action: "execute", sql, params });
|
return await this._request(AUTHORITY_SQL_EXEC_ENDPOINT, {
|
||||||
|
database: this.database,
|
||||||
|
statement: String(sql || ""),
|
||||||
|
params,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async transaction(statements = []) {
|
async transaction(statements = []) {
|
||||||
return await this._request({ action: "transaction", statements });
|
return await this._request(AUTHORITY_SQL_TRANSACTION_ENDPOINT, {
|
||||||
|
database: this.database,
|
||||||
|
statements: toArray(statements)
|
||||||
|
.filter((statement) => statement?.sql)
|
||||||
|
.map((statement) => ({
|
||||||
|
statement: String(statement.sql || ""),
|
||||||
|
params: statement.params || {},
|
||||||
|
})),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async _request(body = {}) {
|
async _request(path, body = {}) {
|
||||||
if (typeof this.fetchImpl !== "function") {
|
return await this.http.requestJson(path, {
|
||||||
throw new Error("Authority SQL fetch unavailable");
|
|
||||||
}
|
|
||||||
const headers = {
|
|
||||||
Accept: "application/json",
|
|
||||||
"Content-Type": "application/json",
|
|
||||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
|
||||||
};
|
|
||||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_SQL_ENDPOINT}`, {
|
|
||||||
method: "POST",
|
method: "POST",
|
||||||
headers,
|
body,
|
||||||
body: JSON.stringify(body),
|
session: true,
|
||||||
});
|
});
|
||||||
if (!response?.ok) {
|
|
||||||
throw new Error(`Authority SQL HTTP ${response?.status || "unknown"}`);
|
|
||||||
}
|
|
||||||
return await response.json().catch(() => ({}));
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -432,8 +432,150 @@ async function testSyncUploadDownloadUsesAuthorityBlob() {
|
|||||||
assert.equal(db.snapshot.nodes[0].id, "sync-blob-node");
|
assert.equal(db.snapshot.nodes[0].id, "sync-blob-node");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function testAuthorityBlobHttpBoundary() {
|
||||||
|
const requests = [];
|
||||||
|
const adapter = createAuthorityBlobAdapter(
|
||||||
|
{ authorityBaseUrl: "https://authority.example.test/root" },
|
||||||
|
{
|
||||||
|
headerProvider: () => ({ "X-Test": "1" }),
|
||||||
|
fetchImpl: async (url, options = {}) => {
|
||||||
|
requests.push({ url, options });
|
||||||
|
if (url.endsWith("/session/init")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { sessionToken: "blob-session-token" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/fs/private/write-file")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
entry: {
|
||||||
|
path: "user/files/demo.json",
|
||||||
|
sizeBytes: 17,
|
||||||
|
updatedAt: "2026-04-28T12:00:00.000Z",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/fs/private/read-file")) {
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
if (body.path === "user/files/missing.json") {
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
async json() {
|
||||||
|
return { error: "not found" };
|
||||||
|
},
|
||||||
|
async text() {
|
||||||
|
return "not found";
|
||||||
|
},
|
||||||
|
headers: { get: () => "application/json" },
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
entry: {
|
||||||
|
path: "user/files/demo.json",
|
||||||
|
sizeBytes: 17,
|
||||||
|
updatedAt: "2026-04-28T12:00:00.000Z",
|
||||||
|
},
|
||||||
|
content: JSON.stringify({ hello: "world" }),
|
||||||
|
encoding: "utf8",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/fs/private/stat")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
entry: {
|
||||||
|
path: "user/files/demo.json",
|
||||||
|
sizeBytes: 17,
|
||||||
|
updatedAt: "2026-04-28T12:00:00.000Z",
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/fs/private/delete")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
async json() {
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
async text() {
|
||||||
|
return "not found";
|
||||||
|
},
|
||||||
|
headers: { get: () => "application/json" },
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const writeResult = await adapter.writeJson("user/files/demo.json", { hello: "world" });
|
||||||
|
assert.equal(writeResult.ok, true);
|
||||||
|
assert.equal(writeResult.path, "user/files/demo.json");
|
||||||
|
|
||||||
|
const readResult = await adapter.readJson("user/files/demo.json");
|
||||||
|
assert.equal(readResult.exists, true);
|
||||||
|
assert.deepEqual(readResult.payload, { hello: "world" });
|
||||||
|
|
||||||
|
const statResult = await adapter.stat("user/files/demo.json");
|
||||||
|
assert.equal(statResult.exists, true);
|
||||||
|
assert.equal(statResult.path, "user/files/demo.json");
|
||||||
|
|
||||||
|
const missingResult = await adapter.readJson("user/files/missing.json");
|
||||||
|
assert.equal(missingResult.exists, false);
|
||||||
|
|
||||||
|
const deleteResult = await adapter.delete("user/files/demo.json");
|
||||||
|
assert.equal(deleteResult.ok, true);
|
||||||
|
|
||||||
|
assert.deepEqual(
|
||||||
|
requests.map((request) => request.url),
|
||||||
|
[
|
||||||
|
"https://authority.example.test/root/session/init",
|
||||||
|
"https://authority.example.test/root/fs/private/write-file",
|
||||||
|
"https://authority.example.test/root/fs/private/read-file",
|
||||||
|
"https://authority.example.test/root/fs/private/stat",
|
||||||
|
"https://authority.example.test/root/fs/private/read-file",
|
||||||
|
"https://authority.example.test/root/fs/private/delete",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.equal(requests[1].options.headers["x-authority-session-token"], "blob-session-token");
|
||||||
|
assert.equal(requests[1].options.headers["X-Test"], "1");
|
||||||
|
assert.deepEqual(JSON.parse(String(requests[1].options.body || "{}")), {
|
||||||
|
path: "user/files/demo.json",
|
||||||
|
content: JSON.stringify({ hello: "world" }),
|
||||||
|
encoding: "utf8",
|
||||||
|
createParents: true,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
await testAdapterBasics();
|
await testAdapterBasics();
|
||||||
await testAuthorityBlobFailOpenFallsBackToUserFiles();
|
await testAuthorityBlobFailOpenFallsBackToUserFiles();
|
||||||
await testBackupRestoreUsesAuthorityBlob();
|
await testBackupRestoreUsesAuthorityBlob();
|
||||||
await testSyncUploadDownloadUsesAuthorityBlob();
|
await testSyncUploadDownloadUsesAuthorityBlob();
|
||||||
|
await testAuthorityBlobHttpBoundary();
|
||||||
console.log("authority-blob tests passed");
|
console.log("authority-blob tests passed");
|
||||||
|
|||||||
@@ -18,31 +18,29 @@ assert.equal(normalizedSettings.vectorMode, "auto-primary");
|
|||||||
assert.equal(normalizedSettings.primaryWhenAvailable, true);
|
assert.equal(normalizedSettings.primaryWhenAvailable, true);
|
||||||
|
|
||||||
assert.deepEqual(buildAuthorityProbeUrls("/api/plugins/authority/"), [
|
assert.deepEqual(buildAuthorityProbeUrls("/api/plugins/authority/"), [
|
||||||
"/api/plugins/authority/v1/diagnostics/probe",
|
|
||||||
"/api/plugins/authority/v1/probe",
|
|
||||||
"/api/plugins/authority/probe",
|
"/api/plugins/authority/probe",
|
||||||
"/api/plugins/authority",
|
|
||||||
]);
|
]);
|
||||||
|
|
||||||
const collected = collectAuthorityFeatures({
|
const collected = collectAuthorityFeatures({
|
||||||
features: ["sql.query", "trivium.search"],
|
features: {
|
||||||
services: {
|
sql: { queryPage: true },
|
||||||
sql: true,
|
trivium: { upsert: true },
|
||||||
jobs: true,
|
jobs: { background: true },
|
||||||
blob: true,
|
transfers: { fs: true },
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
assert.equal(collected.has("sql.query"), true);
|
|
||||||
assert.equal(collected.has("trivium.search"), true);
|
|
||||||
assert.equal(collected.has("sql"), true);
|
assert.equal(collected.has("sql"), true);
|
||||||
assert.equal(collected.has("jobs"), true);
|
assert.equal(collected.has("sql.querypage"), true);
|
||||||
assert.equal(collected.has("blob"), true);
|
assert.equal(collected.has("sql"), true);
|
||||||
|
assert.equal(collected.has("trivium"), true);
|
||||||
|
assert.equal(collected.has("jobs.background"), true);
|
||||||
|
assert.equal(collected.has("transfers.fs"), true);
|
||||||
|
|
||||||
const readyState = normalizeAuthorityCapabilityState(
|
const readyState = normalizeAuthorityCapabilityState(
|
||||||
{
|
{
|
||||||
installed: true,
|
installed: true,
|
||||||
healthy: true,
|
healthy: true,
|
||||||
features: ["sql", "trivium", "jobs", "blob"],
|
features: ["sql", "trivium", "jobs", "transfers.fs"],
|
||||||
},
|
},
|
||||||
defaultSettings,
|
defaultSettings,
|
||||||
);
|
);
|
||||||
@@ -77,28 +75,91 @@ assert.equal(disabledState.reason, "disabled");
|
|||||||
assert.equal(disabledState.serverPrimaryReady, false);
|
assert.equal(disabledState.serverPrimaryReady, false);
|
||||||
assert.equal(disabledState.lastProbeAt, 1000);
|
assert.equal(disabledState.lastProbeAt, 1000);
|
||||||
|
|
||||||
let requestedUrl = "";
|
const requestedUrls = [];
|
||||||
const probedState = await probeAuthorityCapabilities({
|
const probedState = await probeAuthorityCapabilities({
|
||||||
settings: defaultSettings,
|
settings: defaultSettings,
|
||||||
allowRelativeUrl: true,
|
allowRelativeUrl: true,
|
||||||
nowMs: 2000,
|
nowMs: 2000,
|
||||||
fetchImpl: async (url) => {
|
fetchImpl: async (url, options = {}) => {
|
||||||
requestedUrl = url;
|
requestedUrls.push([url, options.method || "GET", options.headers || {}]);
|
||||||
|
if (url.endsWith("/probe")) {
|
||||||
|
assert.equal(options.method, "POST");
|
||||||
return {
|
return {
|
||||||
ok: true,
|
ok: true,
|
||||||
status: 200,
|
status: 200,
|
||||||
async json() {
|
async json() {
|
||||||
return {
|
return {
|
||||||
healthy: true,
|
healthy: true,
|
||||||
sessionReady: true,
|
features: {
|
||||||
permissionReady: true,
|
sql: { queryPage: true },
|
||||||
features: ["sql", "trivium", "jobs", "blob"],
|
trivium: { upsert: true },
|
||||||
|
jobs: { background: true },
|
||||||
|
transfers: { fs: true },
|
||||||
|
},
|
||||||
};
|
};
|
||||||
},
|
},
|
||||||
};
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/session/init")) {
|
||||||
|
assert.equal(options.method, "POST");
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { sessionToken: "session-probe-token" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/session/current")) {
|
||||||
|
assert.equal(options.method, "GET");
|
||||||
|
assert.equal(options.headers["x-authority-session-token"], "session-probe-token");
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { ok: true };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/permissions/evaluate-batch")) {
|
||||||
|
assert.equal(options.method, "POST");
|
||||||
|
assert.equal(options.headers["x-authority-session-token"], "session-probe-token");
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
assert.equal(Array.isArray(body.requests), true);
|
||||||
|
assert.equal(body.requests.some((request) => request.resource === "fs.private"), true);
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
results: body.requests.map((request) => ({
|
||||||
|
decision: "granted",
|
||||||
|
resource: request.resource,
|
||||||
|
target: request.target || "",
|
||||||
|
})),
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
async json() {
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
};
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
assert.equal(requestedUrl, "/api/plugins/authority/v1/diagnostics/probe");
|
assert.equal(requestedUrls[0]?.[0], "/api/plugins/authority/probe");
|
||||||
|
assert.deepEqual(
|
||||||
|
requestedUrls.map(([url]) => url),
|
||||||
|
[
|
||||||
|
"/api/plugins/authority/probe",
|
||||||
|
"/api/plugins/authority/session/init",
|
||||||
|
"/api/plugins/authority/session/current",
|
||||||
|
"/api/plugins/authority/permissions/evaluate-batch",
|
||||||
|
],
|
||||||
|
);
|
||||||
assert.equal(probedState.installed, true);
|
assert.equal(probedState.installed, true);
|
||||||
assert.equal(probedState.healthy, true);
|
assert.equal(probedState.healthy, true);
|
||||||
assert.equal(probedState.serverPrimaryReady, true);
|
assert.equal(probedState.serverPrimaryReady, true);
|
||||||
|
|||||||
@@ -301,6 +301,15 @@ async function testHttpSqlClientBoundary() {
|
|||||||
headerProvider: () => ({ "X-Test": "1" }),
|
headerProvider: () => ({ "X-Test": "1" }),
|
||||||
fetchImpl: async (url, init) => {
|
fetchImpl: async (url, init) => {
|
||||||
requests.push({ url, init });
|
requests.push({ url, init });
|
||||||
|
if (url.endsWith("/session/init")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { sessionToken: "sql-session-token" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
return {
|
return {
|
||||||
ok: true,
|
ok: true,
|
||||||
status: 200,
|
status: 200,
|
||||||
@@ -313,12 +322,19 @@ async function testHttpSqlClientBoundary() {
|
|||||||
|
|
||||||
const result = await client.query("SELECT 1", { chatId: "chat" });
|
const result = await client.query("SELECT 1", { chatId: "chat" });
|
||||||
assert.deepEqual(result, { rows: [{ value: 1 }] });
|
assert.deepEqual(result, { rows: [{ value: 1 }] });
|
||||||
assert.equal(requests[0].url, "https://authority.example.test/root/v1/sql");
|
assert.deepEqual(
|
||||||
assert.equal(requests[0].init.method, "POST");
|
requests.map((request) => request.url),
|
||||||
assert.equal(requests[0].init.headers["X-Test"], "1");
|
[
|
||||||
assert.deepEqual(JSON.parse(requests[0].init.body), {
|
"https://authority.example.test/root/session/init",
|
||||||
action: "query",
|
"https://authority.example.test/root/sql/query",
|
||||||
sql: "SELECT 1",
|
],
|
||||||
|
);
|
||||||
|
assert.equal(requests[1].init.method, "POST");
|
||||||
|
assert.equal(requests[1].init.headers["X-Test"], "1");
|
||||||
|
assert.equal(requests[1].init.headers["x-authority-session-token"], "sql-session-token");
|
||||||
|
assert.deepEqual(JSON.parse(requests[1].init.body), {
|
||||||
|
database: "default",
|
||||||
|
statement: "SELECT 1",
|
||||||
params: { chatId: "chat" },
|
params: { chatId: "chat" },
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -455,4 +455,150 @@ const rangeSync = rangeRuntime.calls.find(([name]) => name === "syncVectorState"
|
|||||||
assert.equal(rangeSync?.[1]?.purge, false);
|
assert.equal(rangeSync?.[1]?.purge, false);
|
||||||
assert.equal(rangeSync?.[1]?.range, range);
|
assert.equal(rangeSync?.[1]?.range, range);
|
||||||
|
|
||||||
|
const httpRequests = [];
|
||||||
|
const httpAdapter = createAuthorityJobAdapter(
|
||||||
|
{ authorityBaseUrl: "https://authority.example.test/root" },
|
||||||
|
{
|
||||||
|
headerProvider: () => ({ "X-Test": "1" }),
|
||||||
|
fetchImpl: async (url, options = {}) => {
|
||||||
|
httpRequests.push({ url, options });
|
||||||
|
if (url.endsWith("/session/init")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return { sessionToken: "job-session-token" };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/jobs/create")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
id: "job-http-1",
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
status: "queued",
|
||||||
|
progress: 0,
|
||||||
|
idempotencyKey: "idem-http-1",
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/jobs/job-http-1")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
id: "job-http-1",
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
status: "completed",
|
||||||
|
progress: 1,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/jobs/list")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
jobs: [
|
||||||
|
{
|
||||||
|
id: "job-http-1",
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
status: "completed",
|
||||||
|
progress: 1,
|
||||||
|
},
|
||||||
|
],
|
||||||
|
page: {
|
||||||
|
nextCursor: "next-http-1",
|
||||||
|
hasMore: true,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/jobs/job-http-1/requeue")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
id: "job-http-2",
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
status: "queued",
|
||||||
|
progress: 0,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
if (url.endsWith("/jobs/job-http-1/cancel")) {
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
return {
|
||||||
|
id: "job-http-1",
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
status: "cancelled",
|
||||||
|
progress: 1,
|
||||||
|
};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
ok: false,
|
||||||
|
status: 404,
|
||||||
|
async json() {
|
||||||
|
return {};
|
||||||
|
},
|
||||||
|
};
|
||||||
|
},
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
const httpSubmitted = await httpAdapter.submit(
|
||||||
|
"authority.vector.rebuild",
|
||||||
|
{ chatId: "chat-http" },
|
||||||
|
{ idempotencyKey: "idem-http-1" },
|
||||||
|
);
|
||||||
|
assert.equal(httpSubmitted.id, "job-http-1");
|
||||||
|
const httpLoaded = await httpAdapter.get("job-http-1");
|
||||||
|
assert.equal(httpLoaded.status, "completed");
|
||||||
|
const httpPage = await httpAdapter.listPage({ cursor: "cursor-http", limit: 10 });
|
||||||
|
assert.equal(httpPage.nextCursor, "next-http-1");
|
||||||
|
assert.equal(httpPage.hasMore, true);
|
||||||
|
const httpRequeued = await httpAdapter.requeue("job-http-1");
|
||||||
|
assert.equal(httpRequeued.id, "job-http-2");
|
||||||
|
const httpCancelled = await httpAdapter.cancel("job-http-1");
|
||||||
|
assert.equal(httpCancelled.status, "cancelled");
|
||||||
|
assert.deepEqual(
|
||||||
|
httpRequests.map((request) => request.url),
|
||||||
|
[
|
||||||
|
"https://authority.example.test/root/session/init",
|
||||||
|
"https://authority.example.test/root/jobs/create",
|
||||||
|
"https://authority.example.test/root/jobs/job-http-1",
|
||||||
|
"https://authority.example.test/root/jobs/list",
|
||||||
|
"https://authority.example.test/root/jobs/job-http-1/requeue",
|
||||||
|
"https://authority.example.test/root/jobs/job-http-1/cancel",
|
||||||
|
],
|
||||||
|
);
|
||||||
|
assert.equal(httpRequests[1].options.headers["x-authority-session-token"], "job-session-token");
|
||||||
|
assert.equal(httpRequests[1].options.headers["X-Test"], "1");
|
||||||
|
assert.deepEqual(JSON.parse(String(httpRequests[1].options.body || "{}")), {
|
||||||
|
type: "authority.vector.rebuild",
|
||||||
|
payload: { chatId: "chat-http" },
|
||||||
|
idempotencyKey: "idem-http-1",
|
||||||
|
});
|
||||||
|
assert.deepEqual(JSON.parse(String(httpRequests[3].options.body || "{}")), {
|
||||||
|
page: {
|
||||||
|
cursor: "cursor-http",
|
||||||
|
limit: 10,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
|
||||||
console.log("authority-jobs tests passed");
|
console.log("authority-jobs tests passed");
|
||||||
|
|||||||
@@ -16,6 +16,15 @@ installResolveHooks([
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
globalThis.__stBmeTestOverrides = {
|
||||||
|
embedding: {
|
||||||
|
async embedText(text) {
|
||||||
|
const seed = String(text || "").length || 1;
|
||||||
|
return [seed / 100, 0.2, 0.3];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
const { normalizeAuthorityVectorConfig } = await import(
|
const { normalizeAuthorityVectorConfig } = await import(
|
||||||
"../vector/authority-vector-primary-adapter.js"
|
"../vector/authority-vector-primary-adapter.js"
|
||||||
);
|
);
|
||||||
@@ -153,6 +162,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN
|
|||||||
const config = normalizeAuthorityVectorConfig(
|
const config = normalizeAuthorityVectorConfig(
|
||||||
{
|
{
|
||||||
authorityBaseUrl: "/api/plugins/authority",
|
authorityBaseUrl: "/api/plugins/authority",
|
||||||
|
authorityEmbeddingApiUrl: "https://example.com/v1",
|
||||||
|
authorityEmbeddingModel: "test-embedding",
|
||||||
authorityVectorFailOpen: true,
|
authorityVectorFailOpen: true,
|
||||||
},
|
},
|
||||||
{ triviumClient },
|
{ triviumClient },
|
||||||
@@ -216,6 +227,8 @@ function createMockTriviumClient({ failFilter = false, failSearch = false, failN
|
|||||||
const config = normalizeAuthorityVectorConfig(
|
const config = normalizeAuthorityVectorConfig(
|
||||||
{
|
{
|
||||||
authorityBaseUrl: "/api/plugins/authority",
|
authorityBaseUrl: "/api/plugins/authority",
|
||||||
|
authorityEmbeddingApiUrl: "https://example.com/v1",
|
||||||
|
authorityEmbeddingModel: "test-embedding",
|
||||||
authorityVectorFailOpen: true,
|
authorityVectorFailOpen: true,
|
||||||
},
|
},
|
||||||
{ triviumClient },
|
{ triviumClient },
|
||||||
|
|||||||
@@ -16,6 +16,17 @@ installResolveHooks([
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
globalThis.__stBmeTestOverrides = {
|
||||||
|
embedding: {
|
||||||
|
async embedBatch(texts = []) {
|
||||||
|
return texts.map((text, index) => [1, index / 10, String(text || "").length / 100]);
|
||||||
|
},
|
||||||
|
async embedText(text = "") {
|
||||||
|
return [1, 0.5, String(text || "").length / 100];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
const {
|
const {
|
||||||
filterAuthorityTriviumNodes,
|
filterAuthorityTriviumNodes,
|
||||||
isAuthorityVectorConfig,
|
isAuthorityVectorConfig,
|
||||||
@@ -112,8 +123,20 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function withMockFetch(handler, fn) {
|
||||||
|
const previousFetch = globalThis.fetch;
|
||||||
|
globalThis.fetch = handler;
|
||||||
|
try {
|
||||||
|
return await fn();
|
||||||
|
} finally {
|
||||||
|
globalThis.fetch = previousFetch;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const config = normalizeAuthorityVectorConfig({
|
const config = normalizeAuthorityVectorConfig({
|
||||||
authorityBaseUrl: "/api/plugins/authority",
|
authorityBaseUrl: "/api/plugins/authority",
|
||||||
|
authorityEmbeddingApiUrl: "https://example.com/v1",
|
||||||
|
authorityEmbeddingModel: "test-embedding",
|
||||||
authorityVectorSyncChunkSize: 1,
|
authorityVectorSyncChunkSize: 1,
|
||||||
authorityVectorFailOpen: true,
|
authorityVectorFailOpen: true,
|
||||||
});
|
});
|
||||||
@@ -144,6 +167,10 @@ assert.equal(isAuthorityVectorConfig(config), true);
|
|||||||
upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(),
|
upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(),
|
||||||
["node-a", "node-b"],
|
["node-a", "node-b"],
|
||||||
);
|
);
|
||||||
|
assert.equal(
|
||||||
|
upserts.every(([, payload]) => payload.items.every((item) => Array.isArray(item.vector) && item.vector.length > 0)),
|
||||||
|
true,
|
||||||
|
);
|
||||||
const linkCall = triviumClient.calls.find(([name]) => name === "linkMany");
|
const linkCall = triviumClient.calls.find(([name]) => name === "linkMany");
|
||||||
assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a");
|
assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a");
|
||||||
assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b");
|
assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b");
|
||||||
@@ -170,6 +197,8 @@ assert.equal(isAuthorityVectorConfig(config), true);
|
|||||||
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
|
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
|
||||||
const searchCall = triviumClient.calls.find(([name]) => name === "search");
|
const searchCall = triviumClient.calls.find(([name]) => name === "search");
|
||||||
assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]);
|
assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]);
|
||||||
|
assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true);
|
||||||
|
assert.ok(searchCall?.[1]?.queryVector.length > 0);
|
||||||
assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority");
|
assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority");
|
||||||
assert.equal(graph.vectorIndexState.lastSearchTimings.success, true);
|
assert.equal(graph.vectorIndexState.lastSearchTimings.success, true);
|
||||||
}
|
}
|
||||||
@@ -221,4 +250,72 @@ assert.equal(isAuthorityVectorConfig(config), true);
|
|||||||
assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]);
|
assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
{
|
||||||
|
const previousOverrides = globalThis.__stBmeTestOverrides;
|
||||||
|
globalThis.__stBmeTestOverrides = {};
|
||||||
|
const fetchCalls = [];
|
||||||
|
try {
|
||||||
|
await withMockFetch(async (url, options = {}) => {
|
||||||
|
fetchCalls.push([url, JSON.parse(String(options.body || "{}"))]);
|
||||||
|
return {
|
||||||
|
ok: true,
|
||||||
|
status: 200,
|
||||||
|
async json() {
|
||||||
|
const body = JSON.parse(String(options.body || "{}"));
|
||||||
|
if (Array.isArray(body.texts)) {
|
||||||
|
return {
|
||||||
|
vectors: body.texts.map((text, index) => [1, index + 1, String(text || "").length / 100]),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
vector: [1, 9, String(body.text || "").length / 100],
|
||||||
|
};
|
||||||
|
},
|
||||||
|
async text() {
|
||||||
|
return "";
|
||||||
|
},
|
||||||
|
};
|
||||||
|
}, async () => {
|
||||||
|
const backendConfig = normalizeAuthorityVectorConfig({
|
||||||
|
authorityBaseUrl: "/api/plugins/authority",
|
||||||
|
embeddingTransportMode: "backend",
|
||||||
|
embeddingBackendSource: "openai",
|
||||||
|
embeddingBackendModel: "text-embedding-3-small",
|
||||||
|
authorityVectorSyncChunkSize: 2,
|
||||||
|
});
|
||||||
|
const { graph, first, second } = createAuthorityVectorGraph();
|
||||||
|
first.embedding = null;
|
||||||
|
second.embedding = null;
|
||||||
|
const triviumClient = createMockTriviumClient();
|
||||||
|
await syncGraphVectorIndexFromIndex(graph, backendConfig, {
|
||||||
|
chatId: "chat-authority-vector",
|
||||||
|
purge: true,
|
||||||
|
triviumClient,
|
||||||
|
});
|
||||||
|
const results = await findSimilarNodesByTextFromIndex(
|
||||||
|
graph,
|
||||||
|
"archive door",
|
||||||
|
{ ...backendConfig, triviumClient },
|
||||||
|
5,
|
||||||
|
[first, second],
|
||||||
|
);
|
||||||
|
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
|
||||||
|
const upsertCall = triviumClient.calls.find(([name]) => name === "bulkUpsert");
|
||||||
|
assert.equal(
|
||||||
|
upsertCall?.[1]?.items?.every((item) => Array.isArray(item.vector) && item.vector.length > 0),
|
||||||
|
true,
|
||||||
|
);
|
||||||
|
const searchCall = triviumClient.calls.find(([name]) => name === "search");
|
||||||
|
assert.equal(Array.isArray(searchCall?.[1]?.queryVector), true);
|
||||||
|
assert.equal(fetchCalls.every(([url]) => url === "/api/vector/embed"), true);
|
||||||
|
assert.equal(fetchCalls[0]?.[1]?.source, "openai");
|
||||||
|
assert.equal(fetchCalls[0]?.[1]?.model, "text-embedding-3-small");
|
||||||
|
assert.equal(Array.isArray(fetchCalls[0]?.[1]?.texts), true);
|
||||||
|
assert.equal(fetchCalls[fetchCalls.length - 1]?.[1]?.isQuery, true);
|
||||||
|
});
|
||||||
|
} finally {
|
||||||
|
globalThis.__stBmeTestOverrides = previousOverrides;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
console.log("authority-vector-primary tests passed");
|
console.log("authority-vector-primary tests passed");
|
||||||
|
|||||||
@@ -1,19 +1,36 @@
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
|
|
||||||
import { buildLukerGraphCheckpointV2 } from "../../graph/graph-persistence.js";
|
|
||||||
import {
|
import {
|
||||||
applyAuthorityCheckpointToStore,
|
installResolveHooks,
|
||||||
buildAuthorityConsistencyAudit,
|
toDataModuleUrl,
|
||||||
buildAuthorityCheckpointImportSnapshot,
|
} from "../helpers/register-hooks-compat.mjs";
|
||||||
} from "../../maintenance/authority-consistency.js";
|
|
||||||
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js";
|
installResolveHooks([
|
||||||
import { AuthorityGraphStore } from "../../sync/authority-graph-store.js";
|
{
|
||||||
|
specifiers: ["../../../../../script.js"],
|
||||||
|
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
specifiers: ["../../../../extensions.js"],
|
||||||
|
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
import {
|
import {
|
||||||
createAuthorityE2eContext,
|
createAuthorityE2eContext,
|
||||||
createAuthorityE2eContractGraph,
|
createAuthorityE2eContractGraph,
|
||||||
runAuthorityE2eStep,
|
runAuthorityE2eStep,
|
||||||
} from "../helpers/authority-e2e-context.mjs";
|
} from "../helpers/authority-e2e-context.mjs";
|
||||||
|
|
||||||
|
const { buildLukerGraphCheckpointV2 } = await import("../../graph/graph-persistence.js");
|
||||||
|
const {
|
||||||
|
applyAuthorityCheckpointToStore,
|
||||||
|
buildAuthorityConsistencyAudit,
|
||||||
|
buildAuthorityCheckpointImportSnapshot,
|
||||||
|
} = await import("../../maintenance/authority-consistency.js");
|
||||||
|
const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
|
||||||
|
const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js");
|
||||||
|
|
||||||
const context = createAuthorityE2eContext({
|
const context = createAuthorityE2eContext({
|
||||||
skipMessage:
|
skipMessage:
|
||||||
"authority checkpoint restore E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",
|
"authority checkpoint restore E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",
|
||||||
|
|||||||
@@ -1,17 +1,34 @@
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
buildAuthorityDiagnosticsBundle,
|
installResolveHooks,
|
||||||
buildAuthorityPerformanceBaseline,
|
toDataModuleUrl,
|
||||||
writeAuthorityDiagnosticsBundle,
|
} from "../helpers/register-hooks-compat.mjs";
|
||||||
} from "../../maintenance/authority-diagnostics-bundle.js";
|
|
||||||
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js";
|
installResolveHooks([
|
||||||
|
{
|
||||||
|
specifiers: ["../../../../../script.js"],
|
||||||
|
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
specifiers: ["../../../../extensions.js"],
|
||||||
|
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
import {
|
import {
|
||||||
createAuthorityE2eContext,
|
createAuthorityE2eContext,
|
||||||
createAuthorityE2eContractGraph,
|
createAuthorityE2eContractGraph,
|
||||||
runAuthorityE2eStep,
|
runAuthorityE2eStep,
|
||||||
} from "../helpers/authority-e2e-context.mjs";
|
} from "../helpers/authority-e2e-context.mjs";
|
||||||
|
|
||||||
|
const {
|
||||||
|
buildAuthorityDiagnosticsBundle,
|
||||||
|
buildAuthorityPerformanceBaseline,
|
||||||
|
writeAuthorityDiagnosticsBundle,
|
||||||
|
} = await import("../../maintenance/authority-diagnostics-bundle.js");
|
||||||
|
const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
|
||||||
|
|
||||||
const context = createAuthorityE2eContext({
|
const context = createAuthorityE2eContext({
|
||||||
skipMessage:
|
skipMessage:
|
||||||
"authority diagnostics E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",
|
"authority diagnostics E2E skipped: set AUTHORITY_E2E_BASE_URL to run against a real Authority server",
|
||||||
|
|||||||
@@ -1,8 +1,32 @@
|
|||||||
import assert from "node:assert/strict";
|
import assert from "node:assert/strict";
|
||||||
|
|
||||||
import { probeAuthorityCapabilities } from "../../runtime/authority-capabilities.js";
|
|
||||||
import { AuthorityGraphStore } from "../../sync/authority-graph-store.js";
|
|
||||||
import {
|
import {
|
||||||
|
installResolveHooks,
|
||||||
|
toDataModuleUrl,
|
||||||
|
} from "../helpers/register-hooks-compat.mjs";
|
||||||
|
|
||||||
|
installResolveHooks([
|
||||||
|
{
|
||||||
|
specifiers: ["../../../../../script.js"],
|
||||||
|
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
|
||||||
|
},
|
||||||
|
{
|
||||||
|
specifiers: ["../../../../extensions.js"],
|
||||||
|
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
|
||||||
|
},
|
||||||
|
]);
|
||||||
|
|
||||||
|
import {
|
||||||
|
buildAuthorityE2eVectorEntries,
|
||||||
|
createAuthorityE2eContext,
|
||||||
|
createAuthorityE2eContractGraph,
|
||||||
|
createAuthorityE2eContractNode,
|
||||||
|
runAuthorityE2eStep,
|
||||||
|
} from "../helpers/authority-e2e-context.mjs";
|
||||||
|
|
||||||
|
const { probeAuthorityCapabilities } = await import("../../runtime/authority-capabilities.js");
|
||||||
|
const { AuthorityGraphStore } = await import("../../sync/authority-graph-store.js");
|
||||||
|
const {
|
||||||
deleteAuthorityTriviumNodes,
|
deleteAuthorityTriviumNodes,
|
||||||
filterAuthorityTriviumNodes,
|
filterAuthorityTriviumNodes,
|
||||||
normalizeAuthorityVectorConfig,
|
normalizeAuthorityVectorConfig,
|
||||||
@@ -11,19 +35,12 @@ import {
|
|||||||
searchAuthorityTriviumNodes,
|
searchAuthorityTriviumNodes,
|
||||||
syncAuthorityTriviumLinks,
|
syncAuthorityTriviumLinks,
|
||||||
upsertAuthorityTriviumEntries,
|
upsertAuthorityTriviumEntries,
|
||||||
} from "../../vector/authority-vector-primary-adapter.js";
|
} = await import("../../vector/authority-vector-primary-adapter.js");
|
||||||
import {
|
const {
|
||||||
buildAuthorityJobIdempotencyKey,
|
buildAuthorityJobIdempotencyKey,
|
||||||
createAuthorityJobAdapter,
|
createAuthorityJobAdapter,
|
||||||
} from "../../maintenance/authority-job-adapter.js";
|
} = await import("../../maintenance/authority-job-adapter.js");
|
||||||
import { createAuthorityBlobAdapter } from "../../maintenance/authority-blob-adapter.js";
|
const { createAuthorityBlobAdapter } = await import("../../maintenance/authority-blob-adapter.js");
|
||||||
import {
|
|
||||||
buildAuthorityE2eVectorEntries,
|
|
||||||
createAuthorityE2eContext,
|
|
||||||
createAuthorityE2eContractGraph,
|
|
||||||
createAuthorityE2eContractNode,
|
|
||||||
runAuthorityE2eStep,
|
|
||||||
} from "../helpers/authority-e2e-context.mjs";
|
|
||||||
|
|
||||||
const context = createAuthorityE2eContext({
|
const context = createAuthorityE2eContext({
|
||||||
skipMessage:
|
skipMessage:
|
||||||
@@ -35,7 +52,7 @@ if (context.skip) {
|
|||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
const resolvedBaseUrl = context.baseUrl;
|
const resolvedBaseUrl = context.baseUrl;
|
||||||
const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId } = context;
|
const { chatId, namespace, collectionId, blobPath, fetchImpl, headerProvider, runId, embeddingApiUrl, embeddingApiKey, embeddingModel } = context;
|
||||||
const graph = createAuthorityE2eContractGraph(chatId, runId);
|
const graph = createAuthorityE2eContractGraph(chatId, runId);
|
||||||
|
|
||||||
const runContext = {
|
const runContext = {
|
||||||
@@ -105,7 +122,12 @@ await runAuthorityE2eStep("sql", async () => {
|
|||||||
});
|
});
|
||||||
|
|
||||||
await runAuthorityE2eStep("trivium", async () => {
|
await runAuthorityE2eStep("trivium", async () => {
|
||||||
const config = normalizeAuthorityVectorConfig({ authorityBaseUrl: resolvedBaseUrl });
|
const config = normalizeAuthorityVectorConfig({
|
||||||
|
authorityBaseUrl: resolvedBaseUrl,
|
||||||
|
authorityEmbeddingApiUrl: embeddingApiUrl,
|
||||||
|
authorityEmbeddingApiKey: embeddingApiKey,
|
||||||
|
authorityEmbeddingModel: embeddingModel,
|
||||||
|
});
|
||||||
const entries = buildAuthorityE2eVectorEntries(graph);
|
const entries = buildAuthorityE2eVectorEntries(graph);
|
||||||
await purgeAuthorityTriviumNamespace(config, {
|
await purgeAuthorityTriviumNamespace(config, {
|
||||||
namespace,
|
namespace,
|
||||||
@@ -141,6 +163,7 @@ await runAuthorityE2eStep("trivium", async () => {
|
|||||||
namespace,
|
namespace,
|
||||||
collectionId,
|
collectionId,
|
||||||
chatId,
|
chatId,
|
||||||
|
queryVector: [1, 1, 0.18],
|
||||||
topK: 5,
|
topK: 5,
|
||||||
fetchImpl,
|
fetchImpl,
|
||||||
headerProvider,
|
headerProvider,
|
||||||
|
|||||||
@@ -97,6 +97,9 @@ export function createAuthorityE2eContext(options = {}) {
|
|||||||
const namespace = String(env.AUTHORITY_E2E_NAMESPACE || `st-bme-e2e-${runId}`);
|
const namespace = String(env.AUTHORITY_E2E_NAMESPACE || `st-bme-e2e-${runId}`);
|
||||||
const collectionId = String(env.AUTHORITY_E2E_COLLECTION_ID || `${namespace}::${chatId}`);
|
const collectionId = String(env.AUTHORITY_E2E_COLLECTION_ID || `${namespace}::${chatId}`);
|
||||||
const blobPath = String(env.AUTHORITY_E2E_BLOB_PATH || `st-bme/e2e/${runId}/contract.json`);
|
const blobPath = String(env.AUTHORITY_E2E_BLOB_PATH || `st-bme/e2e/${runId}/contract.json`);
|
||||||
|
const embeddingApiUrl = String(env.AUTHORITY_E2E_EMBEDDING_API_URL || "").trim();
|
||||||
|
const embeddingApiKey = String(env.AUTHORITY_E2E_EMBEDDING_API_KEY || "").trim();
|
||||||
|
const embeddingModel = String(env.AUTHORITY_E2E_EMBEDDING_MODEL || "").trim();
|
||||||
return {
|
return {
|
||||||
skip: false,
|
skip: false,
|
||||||
env,
|
env,
|
||||||
@@ -110,6 +113,9 @@ export function createAuthorityE2eContext(options = {}) {
|
|||||||
namespace,
|
namespace,
|
||||||
collectionId,
|
collectionId,
|
||||||
blobPath,
|
blobPath,
|
||||||
|
embeddingApiUrl,
|
||||||
|
embeddingApiKey,
|
||||||
|
embeddingModel,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -181,6 +187,7 @@ export function buildAuthorityE2eVectorEntries(graph = null) {
|
|||||||
index,
|
index,
|
||||||
hash: `${node.id}:hash`,
|
hash: `${node.id}:hash`,
|
||||||
text: `${node.fields?.title || node.id}. ${node.fields?.summary || ""}`,
|
text: `${node.fields?.title || node.id}. ${node.fields?.summary || ""}`,
|
||||||
|
vector: [1, index + 1, String(node.fields?.title || node.id || "").length / 100],
|
||||||
}));
|
}));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -16,6 +16,14 @@ installResolveHooks([
|
|||||||
},
|
},
|
||||||
]);
|
]);
|
||||||
|
|
||||||
|
globalThis.__stBmeTestOverrides = {
|
||||||
|
embedding: {
|
||||||
|
async embedText(text = "") {
|
||||||
|
return [1, 0.25, String(text || "").length / 100];
|
||||||
|
},
|
||||||
|
},
|
||||||
|
};
|
||||||
|
|
||||||
const outputJson = process.argv.includes("--json");
|
const outputJson = process.argv.includes("--json");
|
||||||
const RUNS = 5;
|
const RUNS = 5;
|
||||||
const SIZE_PRESETS = [
|
const SIZE_PRESETS = [
|
||||||
@@ -233,6 +241,8 @@ async function runPreset(preset) {
|
|||||||
const config = normalizeAuthorityVectorConfig(
|
const config = normalizeAuthorityVectorConfig(
|
||||||
{
|
{
|
||||||
authorityBaseUrl: "/api/plugins/authority",
|
authorityBaseUrl: "/api/plugins/authority",
|
||||||
|
authorityEmbeddingApiUrl: "https://example.com/v1",
|
||||||
|
authorityEmbeddingModel: "test-embedding",
|
||||||
authorityVectorFailOpen: true,
|
authorityVectorFailOpen: true,
|
||||||
},
|
},
|
||||||
{ triviumClient },
|
{ triviumClient },
|
||||||
|
|||||||
@@ -70,6 +70,54 @@ const defaultModeConfig = getVectorConfigFromSettings({
|
|||||||
assert.equal(defaultModeConfig.mode, "direct");
|
assert.equal(defaultModeConfig.mode, "direct");
|
||||||
assert.equal(validateVectorConfig(defaultModeConfig).valid, true);
|
assert.equal(validateVectorConfig(defaultModeConfig).valid, true);
|
||||||
|
|
||||||
|
const validAuthorityConfig = {
|
||||||
|
mode: "authority",
|
||||||
|
source: "authority-trivium",
|
||||||
|
baseUrl: "/api/plugins/authority",
|
||||||
|
apiUrl: "https://example.com/v1",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
};
|
||||||
|
assert.equal(validateVectorConfig(validAuthorityConfig).valid, true);
|
||||||
|
|
||||||
|
const invalidAuthorityConfig = {
|
||||||
|
mode: "authority",
|
||||||
|
source: "authority-trivium",
|
||||||
|
baseUrl: "/api/plugins/authority",
|
||||||
|
apiUrl: "",
|
||||||
|
model: "",
|
||||||
|
};
|
||||||
|
assert.equal(validateVectorConfig(invalidAuthorityConfig).valid, false);
|
||||||
|
|
||||||
|
const validAuthorityBackendConfig = {
|
||||||
|
mode: "authority",
|
||||||
|
source: "authority-trivium",
|
||||||
|
baseUrl: "/api/plugins/authority",
|
||||||
|
embeddingMode: "backend",
|
||||||
|
embeddingSource: "openai",
|
||||||
|
apiUrl: "",
|
||||||
|
model: "text-embedding-3-small",
|
||||||
|
};
|
||||||
|
assert.equal(validateVectorConfig(validAuthorityBackendConfig).valid, true);
|
||||||
|
|
||||||
|
const invalidAuthorityBackendConfig = {
|
||||||
|
mode: "authority",
|
||||||
|
source: "authority-trivium",
|
||||||
|
baseUrl: "/api/plugins/authority",
|
||||||
|
embeddingMode: "backend",
|
||||||
|
embeddingSource: "vllm",
|
||||||
|
apiUrl: "",
|
||||||
|
model: "BAAI/bge-m3",
|
||||||
|
};
|
||||||
|
assert.equal(validateVectorConfig(invalidAuthorityBackendConfig).valid, false);
|
||||||
|
|
||||||
|
const authorityLikeConfig = getVectorConfigFromSettings({
|
||||||
|
embeddingApiUrl: "https://example.com/v1/embeddings",
|
||||||
|
embeddingApiKey: "sk-test",
|
||||||
|
embeddingModel: "text-embedding-3-small",
|
||||||
|
});
|
||||||
|
assert.equal(authorityLikeConfig.apiUrl, "https://example.com/v1");
|
||||||
|
assert.equal(authorityLikeConfig.model, "text-embedding-3-small");
|
||||||
|
|
||||||
const invalidBackendConfig = getVectorConfigFromSettings({
|
const invalidBackendConfig = getVectorConfigFromSettings({
|
||||||
embeddingTransportMode: "backend",
|
embeddingTransportMode: "backend",
|
||||||
embeddingBackendSource: "vllm",
|
embeddingBackendSource: "vllm",
|
||||||
|
|||||||
@@ -1,11 +1,17 @@
|
|||||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||||
|
import {
|
||||||
|
AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||||
|
AuthorityHttpClient,
|
||||||
|
} from "../runtime/authority-http-client.js";
|
||||||
|
import { embedText } from "./embedding.js";
|
||||||
|
|
||||||
export const AUTHORITY_VECTOR_MODE = "authority";
|
export const AUTHORITY_VECTOR_MODE = "authority";
|
||||||
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
|
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
|
||||||
|
|
||||||
const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium";
|
const DEFAULT_AUTHORITY_TRIVIUM_DATABASE = "st_bme_vectors";
|
||||||
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
|
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
|
||||||
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
|
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
|
||||||
|
const DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE = "openai";
|
||||||
|
|
||||||
function clampInteger(value, fallback, min, max) {
|
function clampInteger(value, fallback, min, max) {
|
||||||
const parsed = Number(value);
|
const parsed = Number(value);
|
||||||
@@ -36,6 +42,27 @@ function normalizeRecordId(value) {
|
|||||||
return String(value ?? "").trim();
|
return String(value ?? "").trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeVector(value = null) {
|
||||||
|
const source = ArrayBuffer.isView(value) ? Array.from(value) : value;
|
||||||
|
if (!Array.isArray(source)) return [];
|
||||||
|
return source
|
||||||
|
.map((item) => Number(item))
|
||||||
|
.filter((item) => Number.isFinite(item));
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizePositiveInteger(value, fallback = 0) {
|
||||||
|
const parsed = Number(value);
|
||||||
|
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||||
|
return Math.floor(parsed);
|
||||||
|
}
|
||||||
|
|
||||||
|
function normalizeOpenAICompatibleBaseUrl(value) {
|
||||||
|
return String(value || "")
|
||||||
|
.trim()
|
||||||
|
.replace(/\/+(chat\/completions|embeddings)$/i, "")
|
||||||
|
.replace(/\/+$/, "");
|
||||||
|
}
|
||||||
|
|
||||||
function readNestedValue(source = null, path = []) {
|
function readNestedValue(source = null, path = []) {
|
||||||
let current = source;
|
let current = source;
|
||||||
for (const key of path) {
|
for (const key of path) {
|
||||||
@@ -68,12 +95,14 @@ function readResultRows(payload = null) {
|
|||||||
if (Array.isArray(payload.data)) return payload.data;
|
if (Array.isArray(payload.data)) return payload.data;
|
||||||
if (Array.isArray(payload.neighbors)) return payload.neighbors;
|
if (Array.isArray(payload.neighbors)) return payload.neighbors;
|
||||||
if (Array.isArray(payload.links)) return payload.links;
|
if (Array.isArray(payload.links)) return payload.links;
|
||||||
|
if (Array.isArray(payload.nodes)) return payload.nodes;
|
||||||
if (Array.isArray(payload.result?.results)) return payload.result.results;
|
if (Array.isArray(payload.result?.results)) return payload.result.results;
|
||||||
if (Array.isArray(payload.result?.items)) return payload.result.items;
|
if (Array.isArray(payload.result?.items)) return payload.result.items;
|
||||||
if (Array.isArray(payload.result?.rows)) return payload.result.rows;
|
if (Array.isArray(payload.result?.rows)) return payload.result.rows;
|
||||||
if (Array.isArray(payload.result?.data)) return payload.result.data;
|
if (Array.isArray(payload.result?.data)) return payload.result.data;
|
||||||
if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors;
|
if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors;
|
||||||
if (Array.isArray(payload.result?.links)) return payload.result.links;
|
if (Array.isArray(payload.result?.links)) return payload.result.links;
|
||||||
|
if (Array.isArray(payload.result?.nodes)) return payload.result.nodes;
|
||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -145,6 +174,32 @@ function normalizeSearchResults(payload = null) {
|
|||||||
.filter(Boolean);
|
.filter(Boolean);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function buildOpenOptions(config = {}, payload = {}) {
|
||||||
|
const database = normalizeRecordId(payload.database || config.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE;
|
||||||
|
return {
|
||||||
|
database,
|
||||||
|
...(normalizePositiveInteger(payload.dim ?? config.dim, 0) > 0 ? { dim: normalizePositiveInteger(payload.dim ?? config.dim, 0) } : {}),
|
||||||
|
...(payload.dtype || config.dtype ? { dtype: String(payload.dtype || config.dtype) } : {}),
|
||||||
|
...(payload.syncMode || config.syncMode ? { syncMode: String(payload.syncMode || config.syncMode) } : {}),
|
||||||
|
...(payload.storageMode || config.storageMode ? { storageMode: String(payload.storageMode || config.storageMode) } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function getNamespace(payload = {}) {
|
||||||
|
return normalizeRecordId(payload.namespace || payload.collectionId || payload.chatId);
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildNodeReference(id, namespace = "") {
|
||||||
|
return {
|
||||||
|
externalId: normalizeRecordId(id),
|
||||||
|
...(namespace ? { namespace } : {}),
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildV06PayloadSource(payload = {}) {
|
||||||
|
return payload && typeof payload === "object" && !Array.isArray(payload) ? payload : {};
|
||||||
|
}
|
||||||
|
|
||||||
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
|
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
|
||||||
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
|
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
|
||||||
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
|
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
|
||||||
@@ -166,6 +221,7 @@ function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelSc
|
|||||||
regionKey: String(scope.regionKey || node?.regionKey || ""),
|
regionKey: String(scope.regionKey || node?.regionKey || ""),
|
||||||
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
|
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
|
||||||
storyTimeLabel: String(node?.storyTime?.label || ""),
|
storyTimeLabel: String(node?.storyTime?.label || ""),
|
||||||
|
text: String(entry?.text || ""),
|
||||||
title: getNodeFieldText(node, ["title"]),
|
title: getNodeFieldText(node, ["title"]),
|
||||||
name: getNodeFieldText(node, ["name"]),
|
name: getNodeFieldText(node, ["name"]),
|
||||||
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
|
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
|
||||||
@@ -190,6 +246,7 @@ function buildAuthorityVectorItems(graph, entries = [], options = {}) {
|
|||||||
text: String(entry?.text || ""),
|
text: String(entry?.text || ""),
|
||||||
index: Number(entry?.index || 0) || 0,
|
index: Number(entry?.index || 0) || 0,
|
||||||
hash: String(entry?.hash || ""),
|
hash: String(entry?.hash || ""),
|
||||||
|
vector: normalizeVector(entry?.vector || entry?.embedding || node?.embedding),
|
||||||
payload,
|
payload,
|
||||||
};
|
};
|
||||||
})
|
})
|
||||||
@@ -229,11 +286,43 @@ export function isAuthorityVectorConfig(config = null) {
|
|||||||
|
|
||||||
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
|
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
|
||||||
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
|
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
|
||||||
|
const hasAuthorityEmbeddingOverride = [
|
||||||
|
source.authorityEmbeddingApiUrl,
|
||||||
|
source.authorityEmbeddingApiKey,
|
||||||
|
source.authorityEmbeddingModel,
|
||||||
|
].some((value) => String(value ?? "").trim());
|
||||||
|
const embeddingMode = hasAuthorityEmbeddingOverride
|
||||||
|
? "direct"
|
||||||
|
: String(source.embeddingTransportMode || "direct").trim().toLowerCase() === "backend"
|
||||||
|
? "backend"
|
||||||
|
: "direct";
|
||||||
|
const embeddingSource = embeddingMode === "backend"
|
||||||
|
? String(source.embeddingBackendSource || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE).trim().toLowerCase() || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE
|
||||||
|
: "direct";
|
||||||
return {
|
return {
|
||||||
mode: AUTHORITY_VECTOR_MODE,
|
mode: AUTHORITY_VECTOR_MODE,
|
||||||
source: AUTHORITY_VECTOR_SOURCE,
|
source: AUTHORITY_VECTOR_SOURCE,
|
||||||
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
|
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
|
||||||
model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(),
|
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||||
|
database: normalizeRecordId(source.authorityTriviumDatabase ?? source.triviumDatabase) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
|
||||||
|
dim: normalizePositiveInteger(source.authorityTriviumDim ?? source.triviumDim, 0),
|
||||||
|
dtype: String(source.authorityTriviumDtype ?? source.triviumDtype ?? "").trim(),
|
||||||
|
syncMode: String(source.authorityTriviumSyncMode ?? source.triviumSyncMode ?? "").trim(),
|
||||||
|
storageMode: String(source.authorityTriviumStorageMode ?? source.triviumStorageMode ?? "").trim(),
|
||||||
|
embeddingMode,
|
||||||
|
embeddingSource,
|
||||||
|
apiUrl: normalizeOpenAICompatibleBaseUrl(
|
||||||
|
embeddingMode === "backend"
|
||||||
|
? source.embeddingBackendApiUrl
|
||||||
|
: source.authorityEmbeddingApiUrl ?? source.embeddingApiUrl ?? source.embeddingBackendApiUrl,
|
||||||
|
),
|
||||||
|
apiKey: embeddingMode === "backend"
|
||||||
|
? ""
|
||||||
|
: String(source.authorityEmbeddingApiKey ?? source.embeddingApiKey ?? "").trim(),
|
||||||
|
model: embeddingMode === "backend"
|
||||||
|
? String(source.embeddingBackendModel ?? source.embeddingModel ?? "").trim()
|
||||||
|
: String(source.authorityEmbeddingModel ?? source.embeddingModel ?? source.embeddingBackendModel ?? "").trim(),
|
||||||
|
autoSuffix: source.embeddingAutoSuffix !== false,
|
||||||
chunkSize: clampInteger(
|
chunkSize: clampInteger(
|
||||||
source.authorityVectorSyncChunkSize ?? source.chunkSize,
|
source.authorityVectorSyncChunkSize ?? source.chunkSize,
|
||||||
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
|
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
|
||||||
@@ -251,61 +340,225 @@ export class AuthorityTriviumHttpClient {
|
|||||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
||||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
||||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
||||||
|
this.protocol = AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
|
||||||
|
this.config = {
|
||||||
|
database: normalizeRecordId(options.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
|
||||||
|
dim: normalizePositiveInteger(options.dim, 0),
|
||||||
|
dtype: String(options.dtype || "").trim(),
|
||||||
|
syncMode: String(options.syncMode || "").trim(),
|
||||||
|
storageMode: String(options.storageMode || "").trim(),
|
||||||
|
};
|
||||||
|
this.http = new AuthorityHttpClient({
|
||||||
|
...options,
|
||||||
|
baseUrl: this.baseUrl,
|
||||||
|
fetchImpl: this.fetchImpl,
|
||||||
|
headerProvider: this.headerProvider,
|
||||||
|
protocol: this.protocol,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async request(action, payload = {}) {
|
async request(action, payload = {}) {
|
||||||
if (typeof this.fetchImpl !== "function") {
|
if (action === "purge") return await this.purge(payload);
|
||||||
throw new Error("Authority Trivium fetch unavailable");
|
if (action === "bulkUpsert") return await this.bulkUpsert(payload);
|
||||||
|
if (action === "deleteMany") return await this.deleteMany(payload);
|
||||||
|
if (action === "linkMany") return await this.linkMany(payload);
|
||||||
|
if (action === "search") return await this.search(payload);
|
||||||
|
if (action === "filterWhere") return await this.filterWhere(payload);
|
||||||
|
if (action === "queryPage") return await this.queryPage(payload);
|
||||||
|
if (action === "neighbors") return await this.neighbors(payload);
|
||||||
|
if (action === "stat") return await this.stat(payload);
|
||||||
|
throw new Error(`Authority Trivium v0.6 action unavailable: ${action}`);
|
||||||
}
|
}
|
||||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, {
|
|
||||||
method: "POST",
|
async requestV06(path, payload = {}, method = "POST") {
|
||||||
headers: {
|
return await this.http.requestJson(path, {
|
||||||
Accept: "application/json",
|
method,
|
||||||
"Content-Type": "application/json",
|
body: payload,
|
||||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
session: true,
|
||||||
},
|
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||||
body: JSON.stringify({ action, ...payload }),
|
|
||||||
});
|
});
|
||||||
if (!response?.ok) {
|
|
||||||
throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`);
|
|
||||||
}
|
}
|
||||||
return await response.json().catch(() => ({}));
|
|
||||||
|
buildOpenOptions(payload = {}) {
|
||||||
|
return buildOpenOptions(this.config, payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
async purge(payload = {}) {
|
async purge(payload = {}) {
|
||||||
return await this.request("purge", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const openOptions = this.buildOpenOptions(payload);
|
||||||
|
let cursor = "";
|
||||||
|
let deleted = 0;
|
||||||
|
let scanned = 0;
|
||||||
|
for (let pageIndex = 0; pageIndex < 100; pageIndex++) {
|
||||||
|
const page = await this.requestV06("/trivium/list-mappings", {
|
||||||
|
...openOptions,
|
||||||
|
namespace,
|
||||||
|
page: { cursor, limit: 200 },
|
||||||
|
});
|
||||||
|
const mappings = toArray(page?.mappings);
|
||||||
|
if (!mappings.length && !page?.page?.hasMore) break;
|
||||||
|
scanned += mappings.length;
|
||||||
|
const items = mappings
|
||||||
|
.map((item) => buildNodeReference(item?.externalId, item?.namespace || namespace))
|
||||||
|
.filter((item) => item.externalId);
|
||||||
|
if (items.length) {
|
||||||
|
const result = await this.requestV06("/trivium/bulk-delete", {
|
||||||
|
...openOptions,
|
||||||
|
items,
|
||||||
|
});
|
||||||
|
deleted += Number(result?.successCount ?? items.length) || 0;
|
||||||
|
}
|
||||||
|
if (!page?.page?.hasMore) break;
|
||||||
|
cursor = String(page?.page?.nextCursor || "");
|
||||||
|
if (!cursor) break;
|
||||||
|
}
|
||||||
|
return { ok: true, scanned, deleted };
|
||||||
}
|
}
|
||||||
|
|
||||||
async bulkUpsert(payload = {}) {
|
async bulkUpsert(payload = {}) {
|
||||||
return await this.request("bulkUpsert", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const items = toArray(payload.items);
|
||||||
|
const missingVector = items.find((item) => !normalizeVector(item?.vector || item?.embedding).length);
|
||||||
|
if (missingVector) {
|
||||||
|
throw new Error("Authority Trivium v0.6 bulkUpsert requires vector for every item");
|
||||||
|
}
|
||||||
|
const mappedItems = items.map((item) => {
|
||||||
|
const nodeId = normalizeRecordId(item?.externalId || item?.nodeId || item?.id);
|
||||||
|
const payloadSource = buildV06PayloadSource(item?.payload);
|
||||||
|
return {
|
||||||
|
externalId: nodeId,
|
||||||
|
namespace,
|
||||||
|
vector: normalizeVector(item?.vector || item?.embedding),
|
||||||
|
payload: {
|
||||||
|
...payloadSource,
|
||||||
|
nodeId: payloadSource.nodeId || nodeId,
|
||||||
|
externalId: payloadSource.externalId || nodeId,
|
||||||
|
collectionId: payload.collectionId || payloadSource.collectionId || "",
|
||||||
|
text: payloadSource.text || item?.text || "",
|
||||||
|
contentHash: payloadSource.contentHash || item?.hash || "",
|
||||||
|
index: Number(item?.index || payloadSource.index || 0) || 0,
|
||||||
|
},
|
||||||
|
};
|
||||||
|
});
|
||||||
|
return await this.requestV06("/trivium/bulk-upsert", {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
items: mappedItems,
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async deleteMany(payload = {}) {
|
async deleteMany(payload = {}) {
|
||||||
return await this.request("deleteMany", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const ids = [
|
||||||
|
...toArray(payload.ids),
|
||||||
|
...toArray(payload.externalIds),
|
||||||
|
...toArray(payload.items).map((item) => item?.externalId || item?.nodeId || item?.id),
|
||||||
|
].map(normalizeRecordId).filter(Boolean);
|
||||||
|
return await this.requestV06("/trivium/bulk-delete", {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
items: ids.map((id) => buildNodeReference(id, namespace)),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async linkMany(payload = {}) {
|
async linkMany(payload = {}) {
|
||||||
return await this.request("linkMany", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const sourceLinks = toArray(payload.links || payload.items);
|
||||||
|
return await this.requestV06("/trivium/bulk-link", {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
items: sourceLinks
|
||||||
|
.map((link) => {
|
||||||
|
const src = normalizeRecordId(link?.fromId || link?.src || link?.sourceId);
|
||||||
|
const dst = normalizeRecordId(link?.toId || link?.dst || link?.targetId);
|
||||||
|
if (!src || !dst) return null;
|
||||||
|
return {
|
||||||
|
src: buildNodeReference(src, namespace),
|
||||||
|
dst: buildNodeReference(dst, namespace),
|
||||||
|
label: String(link?.relation || link?.label || "related"),
|
||||||
|
weight: Number(link?.weight ?? link?.strength ?? 1) || 1,
|
||||||
|
};
|
||||||
|
})
|
||||||
|
.filter(Boolean),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
async search(payload = {}) {
|
async search(payload = {}) {
|
||||||
return await this.request("search", payload);
|
const vector = normalizeVector(payload.vector || payload.embedding || payload.queryVector);
|
||||||
|
if (!vector.length) {
|
||||||
|
throw new Error("Authority Trivium v0.6 search requires vector");
|
||||||
|
}
|
||||||
|
const queryText = String(payload.queryText || payload.text || payload.searchText || payload.query || "");
|
||||||
|
const body = {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
vector,
|
||||||
|
topK: Number(payload.topK || payload.limit || 0) || undefined,
|
||||||
|
expandDepth: Number(payload.expandDepth || payload.depth || 0) || undefined,
|
||||||
|
minScore: Number.isFinite(Number(payload.minScore)) ? Number(payload.minScore) : undefined,
|
||||||
|
...(payload.payloadFilter || payload.filter ? { payloadFilter: payload.payloadFilter || payload.filter } : {}),
|
||||||
|
};
|
||||||
|
if (queryText) {
|
||||||
|
return await this.requestV06("/trivium/search-hybrid", {
|
||||||
|
...body,
|
||||||
|
queryText,
|
||||||
|
hybridAlpha: Number.isFinite(Number(payload.hybridAlpha)) ? Number(payload.hybridAlpha) : undefined,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
return await this.requestV06("/trivium/search", body);
|
||||||
}
|
}
|
||||||
|
|
||||||
async filterWhere(payload = {}) {
|
async filterWhere(payload = {}) {
|
||||||
return await this.request("filterWhere", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const result = await this.requestV06("/trivium/list-mappings", {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
namespace,
|
||||||
|
page: {
|
||||||
|
limit: Number(payload.limit || payload.topK || payload.pageSize || 100) || 100,
|
||||||
|
},
|
||||||
|
});
|
||||||
|
return { items: toArray(result?.mappings) };
|
||||||
}
|
}
|
||||||
|
|
||||||
async queryPage(payload = {}) {
|
async queryPage(payload = {}) {
|
||||||
return await this.request("queryPage", payload);
|
return await this.filterWhere(payload);
|
||||||
}
|
}
|
||||||
|
|
||||||
async neighbors(payload = {}) {
|
async neighbors(payload = {}) {
|
||||||
return await this.request("neighbors", payload);
|
const namespace = getNamespace(payload);
|
||||||
|
const seedIds = [
|
||||||
|
...toArray(payload.ids),
|
||||||
|
...toArray(payload.nodeIds),
|
||||||
|
...toArray(payload.seedIds),
|
||||||
|
payload.id,
|
||||||
|
].map(normalizeRecordId).filter(Boolean);
|
||||||
|
const openOptions = this.buildOpenOptions(payload);
|
||||||
|
const resolved = await this.requestV06("/trivium/resolve-many", {
|
||||||
|
...openOptions,
|
||||||
|
items: seedIds.map((id) => buildNodeReference(id, namespace)),
|
||||||
|
});
|
||||||
|
const neighbors = [];
|
||||||
|
for (const item of toArray(resolved?.items)) {
|
||||||
|
const internalId = Number(item?.id);
|
||||||
|
if (!Number.isFinite(internalId) || internalId <= 0) continue;
|
||||||
|
const result = await this.requestV06("/trivium/neighbors", {
|
||||||
|
...openOptions,
|
||||||
|
id: internalId,
|
||||||
|
depth: Number(payload.depth || payload.expandDepth || 1) || 1,
|
||||||
|
});
|
||||||
|
for (const node of toArray(result?.nodes)) {
|
||||||
|
neighbors.push({
|
||||||
|
externalId: node?.externalId,
|
||||||
|
nodeId: node?.externalId,
|
||||||
|
id: node?.id,
|
||||||
|
namespace: node?.namespace,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return { neighbors };
|
||||||
}
|
}
|
||||||
|
|
||||||
async stat(payload = {}) {
|
async stat(payload = {}) {
|
||||||
return await this.request("stat", payload);
|
return await this.requestV06("/trivium/stat", {
|
||||||
|
...this.buildOpenOptions(payload),
|
||||||
|
...(payload.includeMappingIntegrity ? { includeMappingIntegrity: true } : {}),
|
||||||
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -313,9 +566,13 @@ export function createAuthorityTriviumClient(config = {}, options = {}) {
|
|||||||
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
|
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
|
||||||
if (injected) return injected;
|
if (injected) return injected;
|
||||||
return new AuthorityTriviumHttpClient({
|
return new AuthorityTriviumHttpClient({
|
||||||
|
...config,
|
||||||
baseUrl: config.baseUrl,
|
baseUrl: config.baseUrl,
|
||||||
fetchImpl: options.fetchImpl || config.fetchImpl,
|
fetchImpl: options.fetchImpl || config.fetchImpl,
|
||||||
headerProvider: options.headerProvider || config.headerProvider,
|
headerProvider: options.headerProvider || config.headerProvider,
|
||||||
|
protocol: config.protocol,
|
||||||
|
sessionToken: options.sessionToken || config.sessionToken,
|
||||||
|
sessionInitConfig: options.sessionInitConfig || config.sessionInitConfig,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -451,6 +708,8 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
|
|||||||
chatId: options.chatId,
|
chatId: options.chatId,
|
||||||
text: String(text || ""),
|
text: String(text || ""),
|
||||||
searchText: String(text || ""),
|
searchText: String(text || ""),
|
||||||
|
vector: normalizeVector(options.vector || options.queryVector || options.embedding),
|
||||||
|
queryVector: normalizeVector(options.queryVector || options.vector || options.embedding),
|
||||||
topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
|
topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
|
||||||
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
|
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
|
||||||
});
|
});
|
||||||
@@ -458,11 +717,15 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
|
|||||||
}
|
}
|
||||||
|
|
||||||
export async function testAuthorityTriviumConnection(config = {}, options = {}) {
|
export async function testAuthorityTriviumConnection(config = {}, options = {}) {
|
||||||
|
const probeVector = await embedText("test connection", config, { isQuery: true });
|
||||||
|
if (!probeVector || probeVector.length === 0) {
|
||||||
|
return { success: false, dimensions: 0, error: "Embedding API 返回空结果" };
|
||||||
|
}
|
||||||
const client = createAuthorityTriviumClient(config, options);
|
const client = createAuthorityTriviumClient(config, options);
|
||||||
await callClient(client, ["stat"], "stat", {
|
await callClient(client, ["stat"], "stat", {
|
||||||
namespace: options.namespace,
|
namespace: options.namespace,
|
||||||
collectionId: options.collectionId,
|
collectionId: options.collectionId,
|
||||||
chatId: options.chatId,
|
chatId: options.chatId,
|
||||||
});
|
});
|
||||||
return { success: true, dimensions: 0, error: "" };
|
return { success: true, dimensions: probeVector.length, error: "" };
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -6,11 +6,17 @@
|
|||||||
* 调用外部 API 获取文本向量,并提供暴力搜索 cosine 相似度
|
* 调用外部 API 获取文本向量,并提供暴力搜索 cosine 相似度
|
||||||
*/
|
*/
|
||||||
|
|
||||||
|
import { getRequestHeaders } from "../../../../../script.js";
|
||||||
import { extension_settings } from "../../../../extensions.js";
|
import { extension_settings } from "../../../../extensions.js";
|
||||||
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
|
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
|
||||||
|
|
||||||
const MODULE_NAME = "st_bme";
|
const MODULE_NAME = "st_bme";
|
||||||
const EMBEDDING_REQUEST_TIMEOUT_MS = 300000;
|
const EMBEDDING_REQUEST_TIMEOUT_MS = 300000;
|
||||||
|
const BACKEND_SOURCES_REQUIRING_API_URL = new Set([
|
||||||
|
"ollama",
|
||||||
|
"llamacpp",
|
||||||
|
"vllm",
|
||||||
|
]);
|
||||||
|
|
||||||
function getEmbeddingTestOverride(name) {
|
function getEmbeddingTestOverride(name) {
|
||||||
const override = globalThis.__stBmeTestOverrides?.embedding?.[name];
|
const override = globalThis.__stBmeTestOverrides?.embedding?.[name];
|
||||||
@@ -41,6 +47,69 @@ function normalizeOpenAICompatibleBaseUrl(value) {
|
|||||||
.replace(/\/+$/, "");
|
.replace(/\/+$/, "");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function normalizeVector(value) {
|
||||||
|
if (!Array.isArray(value)) return null;
|
||||||
|
const vector = value.map((item) => Number(item)).filter((item) => Number.isFinite(item));
|
||||||
|
return vector.length ? new Float64Array(vector) : null;
|
||||||
|
}
|
||||||
|
|
||||||
|
function readEmbeddingMode(config = {}) {
|
||||||
|
return String(config?.embeddingMode || config?.mode || "direct").trim().toLowerCase();
|
||||||
|
}
|
||||||
|
|
||||||
|
function readEmbeddingSource(config = {}) {
|
||||||
|
return String(config?.embeddingSource || config?.source || "openai").trim().toLowerCase() || "openai";
|
||||||
|
}
|
||||||
|
|
||||||
|
function buildBackendEmbeddingRequestBody(config = {}, payload = {}) {
|
||||||
|
const source = readEmbeddingSource(config);
|
||||||
|
const body = {
|
||||||
|
source,
|
||||||
|
model: String(config?.model || "").trim(),
|
||||||
|
isQuery: Boolean(payload.isQuery),
|
||||||
|
};
|
||||||
|
if (payload.text !== undefined) {
|
||||||
|
body.text = String(payload.text ?? "");
|
||||||
|
}
|
||||||
|
if (Array.isArray(payload.texts)) {
|
||||||
|
body.texts = payload.texts.map((item) => String(item ?? ""));
|
||||||
|
}
|
||||||
|
if (BACKEND_SOURCES_REQUIRING_API_URL.has(source)) {
|
||||||
|
body.apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||||
|
}
|
||||||
|
if (source === "ollama") {
|
||||||
|
body.keep = false;
|
||||||
|
}
|
||||||
|
return body;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function requestBackendEmbeddings(config = {}, payload = {}, { signal } = {}) {
|
||||||
|
const response = await fetchWithTimeout(
|
||||||
|
"/api/vector/embed",
|
||||||
|
{
|
||||||
|
method: "POST",
|
||||||
|
headers: {
|
||||||
|
...getRequestHeaders(),
|
||||||
|
"Content-Type": "application/json",
|
||||||
|
},
|
||||||
|
signal,
|
||||||
|
body: JSON.stringify(buildBackendEmbeddingRequestBody(config, payload)),
|
||||||
|
},
|
||||||
|
getConfiguredTimeoutMs(config),
|
||||||
|
);
|
||||||
|
|
||||||
|
if (!response.ok) {
|
||||||
|
const errorText = await response.text().catch(() => response.statusText);
|
||||||
|
console.error(
|
||||||
|
`[ST-BME] Backend Embedding API 错误 (${response.status}):`,
|
||||||
|
errorText,
|
||||||
|
);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
|
||||||
|
return await response.json().catch(() => ({}));
|
||||||
|
}
|
||||||
|
|
||||||
function createCombinedAbortSignal(...signals) {
|
function createCombinedAbortSignal(...signals) {
|
||||||
const validSignals = signals.filter(Boolean);
|
const validSignals = signals.filter(Boolean);
|
||||||
if (validSignals.length <= 1) {
|
if (validSignals.length <= 1) {
|
||||||
@@ -107,10 +176,31 @@ async function fetchWithTimeout(
|
|||||||
* @param {string} config.model - 模型名(如 text-embedding-3-small)
|
* @param {string} config.model - 模型名(如 text-embedding-3-small)
|
||||||
* @returns {Promise<Float64Array|null>} 向量或 null
|
* @returns {Promise<Float64Array|null>} 向量或 null
|
||||||
*/
|
*/
|
||||||
export async function embedText(text, config, { signal } = {}) {
|
export async function embedText(text, config, { signal, isQuery = false } = {}) {
|
||||||
const override = getEmbeddingTestOverride("embedText");
|
const override = getEmbeddingTestOverride("embedText");
|
||||||
if (override) {
|
if (override) {
|
||||||
return await override(text, config, { signal });
|
return await override(text, config, { signal, isQuery });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (readEmbeddingMode(config) === "backend") {
|
||||||
|
if (!text || !config?.model) {
|
||||||
|
console.warn("[ST-BME] Embedding 配置不完整,跳过");
|
||||||
|
return null;
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const payload = await requestBackendEmbeddings(
|
||||||
|
config,
|
||||||
|
{ text, isQuery },
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
return normalizeVector(payload?.vector);
|
||||||
|
} catch (e) {
|
||||||
|
if (isAbortError(e)) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
console.error("[ST-BME] Backend Embedding 调用失败:", e);
|
||||||
|
return null;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||||
@@ -173,10 +263,31 @@ export async function embedText(text, config, { signal } = {}) {
|
|||||||
* @param {object} config
|
* @param {object} config
|
||||||
* @returns {Promise<(Float64Array|null)[]>}
|
* @returns {Promise<(Float64Array|null)[]>}
|
||||||
*/
|
*/
|
||||||
export async function embedBatch(texts, config, { signal } = {}) {
|
export async function embedBatch(texts, config, { signal, isQuery = false } = {}) {
|
||||||
const override = getEmbeddingTestOverride("embedBatch");
|
const override = getEmbeddingTestOverride("embedBatch");
|
||||||
if (override) {
|
if (override) {
|
||||||
return await override(texts, config, { signal });
|
return await override(texts, config, { signal, isQuery });
|
||||||
|
}
|
||||||
|
|
||||||
|
if (readEmbeddingMode(config) === "backend") {
|
||||||
|
if (!texts.length || !config?.model) {
|
||||||
|
return texts.map(() => null);
|
||||||
|
}
|
||||||
|
try {
|
||||||
|
const payload = await requestBackendEmbeddings(
|
||||||
|
config,
|
||||||
|
{ texts, isQuery },
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
const vectors = Array.isArray(payload?.vectors) ? payload.vectors : [];
|
||||||
|
return texts.map((_, index) => normalizeVector(vectors[index]));
|
||||||
|
} catch (e) {
|
||||||
|
if (isAbortError(e)) {
|
||||||
|
throw e;
|
||||||
|
}
|
||||||
|
console.error("[ST-BME] Backend Embedding 批量调用失败:", e);
|
||||||
|
return texts.map(() => null);
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||||
|
|||||||
@@ -236,6 +236,9 @@ export function getVectorModelScope(config) {
|
|||||||
return [
|
return [
|
||||||
"authority",
|
"authority",
|
||||||
config.source || "authority-trivium",
|
config.source || "authority-trivium",
|
||||||
|
config.embeddingMode || "direct",
|
||||||
|
config.embeddingSource || "direct",
|
||||||
|
normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix),
|
||||||
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
|
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
|
||||||
config.model || "",
|
config.model || "",
|
||||||
].join("|");
|
].join("|");
|
||||||
@@ -266,6 +269,20 @@ export function validateVectorConfig(config) {
|
|||||||
if (!config.baseUrl) {
|
if (!config.baseUrl) {
|
||||||
return { valid: false, error: "Authority Trivium 地址不可用" };
|
return { valid: false, error: "Authority Trivium 地址不可用" };
|
||||||
}
|
}
|
||||||
|
if (!config.model) {
|
||||||
|
return { valid: false, error: "请先填写 Embedding 模型(Authority 默认复用当前用户设置)" };
|
||||||
|
}
|
||||||
|
const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase();
|
||||||
|
const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase();
|
||||||
|
if (authorityEmbeddingMode === "backend") {
|
||||||
|
if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) {
|
||||||
|
return { valid: false, error: "当前后端 Embedding 源需要 API 地址(Authority 默认复用当前用户设置)" };
|
||||||
|
}
|
||||||
|
return { valid: true, error: "" };
|
||||||
|
}
|
||||||
|
if (!config.apiUrl) {
|
||||||
|
return { valid: false, error: "请先填写 Embedding API 地址(Authority 默认复用当前用户设置)" };
|
||||||
|
}
|
||||||
return { valid: true, error: "" };
|
return { valid: true, error: "" };
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -606,7 +623,7 @@ function markBackendVectorStateDirty(
|
|||||||
|
|
||||||
function markAuthorityVectorStateDirty(
|
function markAuthorityVectorStateDirty(
|
||||||
graph,
|
graph,
|
||||||
config,
|
config = {},
|
||||||
reason = "authority-trivium-failed",
|
reason = "authority-trivium-failed",
|
||||||
warning = "Authority Trivium 索引失败,已标记待重建",
|
warning = "Authority Trivium 索引失败,已标记待重建",
|
||||||
) {
|
) {
|
||||||
@@ -640,6 +657,42 @@ function markAuthorityVectorStateDirty(
|
|||||||
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
|
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
|
||||||
}
|
}
|
||||||
|
|
||||||
|
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
|
||||||
|
const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node]));
|
||||||
|
const entriesToEmbed = [];
|
||||||
|
for (const entry of entries || []) {
|
||||||
|
const node = nodesById.get(String(entry?.nodeId || ""));
|
||||||
|
const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0;
|
||||||
|
if (node && !hasEmbedding) {
|
||||||
|
entriesToEmbed.push({ entry, node });
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if (!entriesToEmbed.length) {
|
||||||
|
return { requested: 0, failures: 0, elapsedMs: 0 };
|
||||||
|
}
|
||||||
|
throwIfAborted(signal);
|
||||||
|
const startedAt = nowMs();
|
||||||
|
const embeddings = await embedBatch(
|
||||||
|
entriesToEmbed.map(({ entry }) => entry.text),
|
||||||
|
config,
|
||||||
|
{ signal },
|
||||||
|
);
|
||||||
|
let failures = 0;
|
||||||
|
for (let index = 0; index < entriesToEmbed.length; index++) {
|
||||||
|
const embedding = embeddings[index];
|
||||||
|
if (embedding) {
|
||||||
|
entriesToEmbed[index].node.embedding = Array.from(embedding);
|
||||||
|
} else {
|
||||||
|
failures += 1;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return {
|
||||||
|
requested: entriesToEmbed.length,
|
||||||
|
failures,
|
||||||
|
elapsedMs: nowMs() - startedAt,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
export async function syncGraphVectorIndex(
|
export async function syncGraphVectorIndex(
|
||||||
graph,
|
graph,
|
||||||
config,
|
config,
|
||||||
@@ -741,6 +794,12 @@ export async function syncGraphVectorIndex(
|
|||||||
|
|
||||||
try {
|
try {
|
||||||
if (fullReset) {
|
if (fullReset) {
|
||||||
|
const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal);
|
||||||
|
embeddingsRequested += embeddingResult.requested;
|
||||||
|
embedBatchMs += embeddingResult.elapsedMs;
|
||||||
|
if (embeddingResult.failures > 0) {
|
||||||
|
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
|
||||||
|
}
|
||||||
const purgeStartedAt = nowMs();
|
const purgeStartedAt = nowMs();
|
||||||
await purgeAuthorityTriviumNamespace(config, authorityOptions);
|
await purgeAuthorityTriviumNamespace(config, authorityOptions);
|
||||||
authorityPurgeMs += nowMs() - purgeStartedAt;
|
authorityPurgeMs += nowMs() - purgeStartedAt;
|
||||||
@@ -794,6 +853,12 @@ export async function syncGraphVectorIndex(
|
|||||||
queuedNodeIds.add(entry.nodeId);
|
queuedNodeIds.add(entry.nodeId);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal);
|
||||||
|
embeddingsRequested += embeddingResult.requested;
|
||||||
|
embedBatchMs += embeddingResult.elapsedMs;
|
||||||
|
if (embeddingResult.failures > 0) {
|
||||||
|
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
|
||||||
|
}
|
||||||
deletedNodeCount = nodeIdsToDelete.length;
|
deletedNodeCount = nodeIdsToDelete.length;
|
||||||
const deleteStartedAt = nowMs();
|
const deleteStartedAt = nowMs();
|
||||||
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
|
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
|
||||||
@@ -1108,7 +1173,7 @@ export async function findSimilarNodesByText(
|
|||||||
|
|
||||||
if (isDirectVectorConfig(config)) {
|
if (isDirectVectorConfig(config)) {
|
||||||
const queryEmbedStartedAt = nowMs();
|
const queryEmbedStartedAt = nowMs();
|
||||||
const queryVec = await embedText(text, config, { signal });
|
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||||||
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||||||
if (!queryVec) {
|
if (!queryVec) {
|
||||||
recordSearchTimings({
|
recordSearchTimings({
|
||||||
@@ -1157,6 +1222,18 @@ export async function findSimilarNodesByText(
|
|||||||
if (isAuthorityVectorConfig(config)) {
|
if (isAuthorityVectorConfig(config)) {
|
||||||
const requestStartedAt = nowMs();
|
const requestStartedAt = nowMs();
|
||||||
try {
|
try {
|
||||||
|
const queryEmbedStartedAt = nowMs();
|
||||||
|
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||||||
|
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||||||
|
if (!queryVec) {
|
||||||
|
recordSearchTimings({
|
||||||
|
success: false,
|
||||||
|
reason: "authority-query-embed-empty",
|
||||||
|
queryEmbedMs: roundMs(queryEmbedMs),
|
||||||
|
resultCount: 0,
|
||||||
|
});
|
||||||
|
return [];
|
||||||
|
}
|
||||||
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
||||||
const results = (
|
const results = (
|
||||||
await searchAuthorityTriviumNodes(graph, text, config, {
|
await searchAuthorityTriviumNodes(graph, text, config, {
|
||||||
@@ -1166,6 +1243,7 @@ export async function findSimilarNodesByText(
|
|||||||
modelScope: getVectorModelScope(config),
|
modelScope: getVectorModelScope(config),
|
||||||
topK,
|
topK,
|
||||||
candidateIds: candidateNodes.map((node) => node.id),
|
candidateIds: candidateNodes.map((node) => node.id),
|
||||||
|
queryVector: Array.from(queryVec),
|
||||||
signal,
|
signal,
|
||||||
})
|
})
|
||||||
)
|
)
|
||||||
@@ -1174,6 +1252,7 @@ export async function findSimilarNodesByText(
|
|||||||
recordSearchTimings({
|
recordSearchTimings({
|
||||||
success: true,
|
success: true,
|
||||||
reason: "ok",
|
reason: "ok",
|
||||||
|
queryEmbedMs: roundMs(queryEmbedMs),
|
||||||
requestMs: roundMs(nowMs() - requestStartedAt),
|
requestMs: roundMs(nowMs() - requestStartedAt),
|
||||||
resultCount: results.length,
|
resultCount: results.length,
|
||||||
});
|
});
|
||||||
|
|||||||
Reference in New Issue
Block a user