refactor(authority): complete v0.6-only sql/blob/jobs rollout

This commit is contained in:
Youzini-afk
2026-04-28 21:45:21 +08:00
parent a5b822682a
commit a7e2edac88
23 changed files with 1816 additions and 212 deletions

View File

@@ -1,11 +1,17 @@
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
import {
AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
AuthorityHttpClient,
} from "../runtime/authority-http-client.js";
import { embedText } from "./embedding.js";
export const AUTHORITY_VECTOR_MODE = "authority";
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium";
const DEFAULT_AUTHORITY_TRIVIUM_DATABASE = "st_bme_vectors";
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
const DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE = "openai";
function clampInteger(value, fallback, min, max) {
const parsed = Number(value);
@@ -36,6 +42,27 @@ function normalizeRecordId(value) {
return String(value ?? "").trim();
}
function normalizeVector(value = null) {
const source = ArrayBuffer.isView(value) ? Array.from(value) : value;
if (!Array.isArray(source)) return [];
return source
.map((item) => Number(item))
.filter((item) => Number.isFinite(item));
}
function normalizePositiveInteger(value, fallback = 0) {
const parsed = Number(value);
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
return Math.floor(parsed);
}
function normalizeOpenAICompatibleBaseUrl(value) {
return String(value || "")
.trim()
.replace(/\/+(chat\/completions|embeddings)$/i, "")
.replace(/\/+$/, "");
}
function readNestedValue(source = null, path = []) {
let current = source;
for (const key of path) {
@@ -68,12 +95,14 @@ function readResultRows(payload = null) {
if (Array.isArray(payload.data)) return payload.data;
if (Array.isArray(payload.neighbors)) return payload.neighbors;
if (Array.isArray(payload.links)) return payload.links;
if (Array.isArray(payload.nodes)) return payload.nodes;
if (Array.isArray(payload.result?.results)) return payload.result.results;
if (Array.isArray(payload.result?.items)) return payload.result.items;
if (Array.isArray(payload.result?.rows)) return payload.result.rows;
if (Array.isArray(payload.result?.data)) return payload.result.data;
if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors;
if (Array.isArray(payload.result?.links)) return payload.result.links;
if (Array.isArray(payload.result?.nodes)) return payload.result.nodes;
return [];
}
@@ -145,6 +174,32 @@ function normalizeSearchResults(payload = null) {
.filter(Boolean);
}
function buildOpenOptions(config = {}, payload = {}) {
const database = normalizeRecordId(payload.database || config.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE;
return {
database,
...(normalizePositiveInteger(payload.dim ?? config.dim, 0) > 0 ? { dim: normalizePositiveInteger(payload.dim ?? config.dim, 0) } : {}),
...(payload.dtype || config.dtype ? { dtype: String(payload.dtype || config.dtype) } : {}),
...(payload.syncMode || config.syncMode ? { syncMode: String(payload.syncMode || config.syncMode) } : {}),
...(payload.storageMode || config.storageMode ? { storageMode: String(payload.storageMode || config.storageMode) } : {}),
};
}
function getNamespace(payload = {}) {
return normalizeRecordId(payload.namespace || payload.collectionId || payload.chatId);
}
function buildNodeReference(id, namespace = "") {
return {
externalId: normalizeRecordId(id),
...(namespace ? { namespace } : {}),
};
}
function buildV06PayloadSource(payload = {}) {
return payload && typeof payload === "object" && !Array.isArray(payload) ? payload : {};
}
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
@@ -166,6 +221,7 @@ function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelSc
regionKey: String(scope.regionKey || node?.regionKey || ""),
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
storyTimeLabel: String(node?.storyTime?.label || ""),
text: String(entry?.text || ""),
title: getNodeFieldText(node, ["title"]),
name: getNodeFieldText(node, ["name"]),
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
@@ -190,6 +246,7 @@ function buildAuthorityVectorItems(graph, entries = [], options = {}) {
text: String(entry?.text || ""),
index: Number(entry?.index || 0) || 0,
hash: String(entry?.hash || ""),
vector: normalizeVector(entry?.vector || entry?.embedding || node?.embedding),
payload,
};
})
@@ -229,11 +286,43 @@ export function isAuthorityVectorConfig(config = null) {
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
const hasAuthorityEmbeddingOverride = [
source.authorityEmbeddingApiUrl,
source.authorityEmbeddingApiKey,
source.authorityEmbeddingModel,
].some((value) => String(value ?? "").trim());
const embeddingMode = hasAuthorityEmbeddingOverride
? "direct"
: String(source.embeddingTransportMode || "direct").trim().toLowerCase() === "backend"
? "backend"
: "direct";
const embeddingSource = embeddingMode === "backend"
? String(source.embeddingBackendSource || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE).trim().toLowerCase() || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE
: "direct";
return {
mode: AUTHORITY_VECTOR_MODE,
source: AUTHORITY_VECTOR_SOURCE,
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(),
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
database: normalizeRecordId(source.authorityTriviumDatabase ?? source.triviumDatabase) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
dim: normalizePositiveInteger(source.authorityTriviumDim ?? source.triviumDim, 0),
dtype: String(source.authorityTriviumDtype ?? source.triviumDtype ?? "").trim(),
syncMode: String(source.authorityTriviumSyncMode ?? source.triviumSyncMode ?? "").trim(),
storageMode: String(source.authorityTriviumStorageMode ?? source.triviumStorageMode ?? "").trim(),
embeddingMode,
embeddingSource,
apiUrl: normalizeOpenAICompatibleBaseUrl(
embeddingMode === "backend"
? source.embeddingBackendApiUrl
: source.authorityEmbeddingApiUrl ?? source.embeddingApiUrl ?? source.embeddingBackendApiUrl,
),
apiKey: embeddingMode === "backend"
? ""
: String(source.authorityEmbeddingApiKey ?? source.embeddingApiKey ?? "").trim(),
model: embeddingMode === "backend"
? String(source.embeddingBackendModel ?? source.embeddingModel ?? "").trim()
: String(source.authorityEmbeddingModel ?? source.embeddingModel ?? source.embeddingBackendModel ?? "").trim(),
autoSuffix: source.embeddingAutoSuffix !== false,
chunkSize: clampInteger(
source.authorityVectorSyncChunkSize ?? source.chunkSize,
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
@@ -251,61 +340,225 @@ export class AuthorityTriviumHttpClient {
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
this.protocol = AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
this.config = {
database: normalizeRecordId(options.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
dim: normalizePositiveInteger(options.dim, 0),
dtype: String(options.dtype || "").trim(),
syncMode: String(options.syncMode || "").trim(),
storageMode: String(options.storageMode || "").trim(),
};
this.http = new AuthorityHttpClient({
...options,
baseUrl: this.baseUrl,
fetchImpl: this.fetchImpl,
headerProvider: this.headerProvider,
protocol: this.protocol,
});
}
async request(action, payload = {}) {
if (typeof this.fetchImpl !== "function") {
throw new Error("Authority Trivium fetch unavailable");
}
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, {
method: "POST",
headers: {
Accept: "application/json",
"Content-Type": "application/json",
...(this.headerProvider ? this.headerProvider() || {} : {}),
},
body: JSON.stringify({ action, ...payload }),
if (action === "purge") return await this.purge(payload);
if (action === "bulkUpsert") return await this.bulkUpsert(payload);
if (action === "deleteMany") return await this.deleteMany(payload);
if (action === "linkMany") return await this.linkMany(payload);
if (action === "search") return await this.search(payload);
if (action === "filterWhere") return await this.filterWhere(payload);
if (action === "queryPage") return await this.queryPage(payload);
if (action === "neighbors") return await this.neighbors(payload);
if (action === "stat") return await this.stat(payload);
throw new Error(`Authority Trivium v0.6 action unavailable: ${action}`);
}
async requestV06(path, payload = {}, method = "POST") {
return await this.http.requestJson(path, {
method,
body: payload,
session: true,
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
});
if (!response?.ok) {
throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`);
}
return await response.json().catch(() => ({}));
}
buildOpenOptions(payload = {}) {
return buildOpenOptions(this.config, payload);
}
async purge(payload = {}) {
return await this.request("purge", payload);
const namespace = getNamespace(payload);
const openOptions = this.buildOpenOptions(payload);
let cursor = "";
let deleted = 0;
let scanned = 0;
for (let pageIndex = 0; pageIndex < 100; pageIndex++) {
const page = await this.requestV06("/trivium/list-mappings", {
...openOptions,
namespace,
page: { cursor, limit: 200 },
});
const mappings = toArray(page?.mappings);
if (!mappings.length && !page?.page?.hasMore) break;
scanned += mappings.length;
const items = mappings
.map((item) => buildNodeReference(item?.externalId, item?.namespace || namespace))
.filter((item) => item.externalId);
if (items.length) {
const result = await this.requestV06("/trivium/bulk-delete", {
...openOptions,
items,
});
deleted += Number(result?.successCount ?? items.length) || 0;
}
if (!page?.page?.hasMore) break;
cursor = String(page?.page?.nextCursor || "");
if (!cursor) break;
}
return { ok: true, scanned, deleted };
}
async bulkUpsert(payload = {}) {
return await this.request("bulkUpsert", payload);
const namespace = getNamespace(payload);
const items = toArray(payload.items);
const missingVector = items.find((item) => !normalizeVector(item?.vector || item?.embedding).length);
if (missingVector) {
throw new Error("Authority Trivium v0.6 bulkUpsert requires vector for every item");
}
const mappedItems = items.map((item) => {
const nodeId = normalizeRecordId(item?.externalId || item?.nodeId || item?.id);
const payloadSource = buildV06PayloadSource(item?.payload);
return {
externalId: nodeId,
namespace,
vector: normalizeVector(item?.vector || item?.embedding),
payload: {
...payloadSource,
nodeId: payloadSource.nodeId || nodeId,
externalId: payloadSource.externalId || nodeId,
collectionId: payload.collectionId || payloadSource.collectionId || "",
text: payloadSource.text || item?.text || "",
contentHash: payloadSource.contentHash || item?.hash || "",
index: Number(item?.index || payloadSource.index || 0) || 0,
},
};
});
return await this.requestV06("/trivium/bulk-upsert", {
...this.buildOpenOptions(payload),
items: mappedItems,
});
}
async deleteMany(payload = {}) {
return await this.request("deleteMany", payload);
const namespace = getNamespace(payload);
const ids = [
...toArray(payload.ids),
...toArray(payload.externalIds),
...toArray(payload.items).map((item) => item?.externalId || item?.nodeId || item?.id),
].map(normalizeRecordId).filter(Boolean);
return await this.requestV06("/trivium/bulk-delete", {
...this.buildOpenOptions(payload),
items: ids.map((id) => buildNodeReference(id, namespace)),
});
}
async linkMany(payload = {}) {
return await this.request("linkMany", payload);
const namespace = getNamespace(payload);
const sourceLinks = toArray(payload.links || payload.items);
return await this.requestV06("/trivium/bulk-link", {
...this.buildOpenOptions(payload),
items: sourceLinks
.map((link) => {
const src = normalizeRecordId(link?.fromId || link?.src || link?.sourceId);
const dst = normalizeRecordId(link?.toId || link?.dst || link?.targetId);
if (!src || !dst) return null;
return {
src: buildNodeReference(src, namespace),
dst: buildNodeReference(dst, namespace),
label: String(link?.relation || link?.label || "related"),
weight: Number(link?.weight ?? link?.strength ?? 1) || 1,
};
})
.filter(Boolean),
});
}
async search(payload = {}) {
return await this.request("search", payload);
const vector = normalizeVector(payload.vector || payload.embedding || payload.queryVector);
if (!vector.length) {
throw new Error("Authority Trivium v0.6 search requires vector");
}
const queryText = String(payload.queryText || payload.text || payload.searchText || payload.query || "");
const body = {
...this.buildOpenOptions(payload),
vector,
topK: Number(payload.topK || payload.limit || 0) || undefined,
expandDepth: Number(payload.expandDepth || payload.depth || 0) || undefined,
minScore: Number.isFinite(Number(payload.minScore)) ? Number(payload.minScore) : undefined,
...(payload.payloadFilter || payload.filter ? { payloadFilter: payload.payloadFilter || payload.filter } : {}),
};
if (queryText) {
return await this.requestV06("/trivium/search-hybrid", {
...body,
queryText,
hybridAlpha: Number.isFinite(Number(payload.hybridAlpha)) ? Number(payload.hybridAlpha) : undefined,
});
}
return await this.requestV06("/trivium/search", body);
}
async filterWhere(payload = {}) {
return await this.request("filterWhere", payload);
const namespace = getNamespace(payload);
const result = await this.requestV06("/trivium/list-mappings", {
...this.buildOpenOptions(payload),
namespace,
page: {
limit: Number(payload.limit || payload.topK || payload.pageSize || 100) || 100,
},
});
return { items: toArray(result?.mappings) };
}
async queryPage(payload = {}) {
return await this.request("queryPage", payload);
return await this.filterWhere(payload);
}
async neighbors(payload = {}) {
return await this.request("neighbors", payload);
const namespace = getNamespace(payload);
const seedIds = [
...toArray(payload.ids),
...toArray(payload.nodeIds),
...toArray(payload.seedIds),
payload.id,
].map(normalizeRecordId).filter(Boolean);
const openOptions = this.buildOpenOptions(payload);
const resolved = await this.requestV06("/trivium/resolve-many", {
...openOptions,
items: seedIds.map((id) => buildNodeReference(id, namespace)),
});
const neighbors = [];
for (const item of toArray(resolved?.items)) {
const internalId = Number(item?.id);
if (!Number.isFinite(internalId) || internalId <= 0) continue;
const result = await this.requestV06("/trivium/neighbors", {
...openOptions,
id: internalId,
depth: Number(payload.depth || payload.expandDepth || 1) || 1,
});
for (const node of toArray(result?.nodes)) {
neighbors.push({
externalId: node?.externalId,
nodeId: node?.externalId,
id: node?.id,
namespace: node?.namespace,
});
}
}
return { neighbors };
}
async stat(payload = {}) {
return await this.request("stat", payload);
return await this.requestV06("/trivium/stat", {
...this.buildOpenOptions(payload),
...(payload.includeMappingIntegrity ? { includeMappingIntegrity: true } : {}),
});
}
}
@@ -313,9 +566,13 @@ export function createAuthorityTriviumClient(config = {}, options = {}) {
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
if (injected) return injected;
return new AuthorityTriviumHttpClient({
...config,
baseUrl: config.baseUrl,
fetchImpl: options.fetchImpl || config.fetchImpl,
headerProvider: options.headerProvider || config.headerProvider,
protocol: config.protocol,
sessionToken: options.sessionToken || config.sessionToken,
sessionInitConfig: options.sessionInitConfig || config.sessionInitConfig,
});
}
@@ -451,6 +708,8 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
chatId: options.chatId,
text: String(text || ""),
searchText: String(text || ""),
vector: normalizeVector(options.vector || options.queryVector || options.embedding),
queryVector: normalizeVector(options.queryVector || options.vector || options.embedding),
topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
});
@@ -458,11 +717,15 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
}
export async function testAuthorityTriviumConnection(config = {}, options = {}) {
const probeVector = await embedText("test connection", config, { isQuery: true });
if (!probeVector || probeVector.length === 0) {
return { success: false, dimensions: 0, error: "Embedding API 返回空结果" };
}
const client = createAuthorityTriviumClient(config, options);
await callClient(client, ["stat"], "stat", {
namespace: options.namespace,
collectionId: options.collectionId,
chatId: options.chatId,
});
return { success: true, dimensions: 0, error: "" };
return { success: true, dimensions: probeVector.length, error: "" };
}

View File

@@ -6,11 +6,17 @@
* 调用外部 API 获取文本向量,并提供暴力搜索 cosine 相似度
*/
import { getRequestHeaders } from "../../../../../script.js";
import { extension_settings } from "../../../../extensions.js";
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
const MODULE_NAME = "st_bme";
const EMBEDDING_REQUEST_TIMEOUT_MS = 300000;
const BACKEND_SOURCES_REQUIRING_API_URL = new Set([
"ollama",
"llamacpp",
"vllm",
]);
function getEmbeddingTestOverride(name) {
const override = globalThis.__stBmeTestOverrides?.embedding?.[name];
@@ -41,6 +47,69 @@ function normalizeOpenAICompatibleBaseUrl(value) {
.replace(/\/+$/, "");
}
function normalizeVector(value) {
if (!Array.isArray(value)) return null;
const vector = value.map((item) => Number(item)).filter((item) => Number.isFinite(item));
return vector.length ? new Float64Array(vector) : null;
}
function readEmbeddingMode(config = {}) {
return String(config?.embeddingMode || config?.mode || "direct").trim().toLowerCase();
}
function readEmbeddingSource(config = {}) {
return String(config?.embeddingSource || config?.source || "openai").trim().toLowerCase() || "openai";
}
function buildBackendEmbeddingRequestBody(config = {}, payload = {}) {
const source = readEmbeddingSource(config);
const body = {
source,
model: String(config?.model || "").trim(),
isQuery: Boolean(payload.isQuery),
};
if (payload.text !== undefined) {
body.text = String(payload.text ?? "");
}
if (Array.isArray(payload.texts)) {
body.texts = payload.texts.map((item) => String(item ?? ""));
}
if (BACKEND_SOURCES_REQUIRING_API_URL.has(source)) {
body.apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
}
if (source === "ollama") {
body.keep = false;
}
return body;
}
async function requestBackendEmbeddings(config = {}, payload = {}, { signal } = {}) {
const response = await fetchWithTimeout(
"/api/vector/embed",
{
method: "POST",
headers: {
...getRequestHeaders(),
"Content-Type": "application/json",
},
signal,
body: JSON.stringify(buildBackendEmbeddingRequestBody(config, payload)),
},
getConfiguredTimeoutMs(config),
);
if (!response.ok) {
const errorText = await response.text().catch(() => response.statusText);
console.error(
`[ST-BME] Backend Embedding API 错误 (${response.status}):`,
errorText,
);
return null;
}
return await response.json().catch(() => ({}));
}
function createCombinedAbortSignal(...signals) {
const validSignals = signals.filter(Boolean);
if (validSignals.length <= 1) {
@@ -107,10 +176,31 @@ async function fetchWithTimeout(
* @param {string} config.model - 模型名(如 text-embedding-3-small
* @returns {Promise<Float64Array|null>} 向量或 null
*/
export async function embedText(text, config, { signal } = {}) {
export async function embedText(text, config, { signal, isQuery = false } = {}) {
const override = getEmbeddingTestOverride("embedText");
if (override) {
return await override(text, config, { signal });
return await override(text, config, { signal, isQuery });
}
if (readEmbeddingMode(config) === "backend") {
if (!text || !config?.model) {
console.warn("[ST-BME] Embedding 配置不完整,跳过");
return null;
}
try {
const payload = await requestBackendEmbeddings(
config,
{ text, isQuery },
{ signal },
);
return normalizeVector(payload?.vector);
} catch (e) {
if (isAbortError(e)) {
throw e;
}
console.error("[ST-BME] Backend Embedding 调用失败:", e);
return null;
}
}
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
@@ -173,10 +263,31 @@ export async function embedText(text, config, { signal } = {}) {
* @param {object} config
* @returns {Promise<(Float64Array|null)[]>}
*/
export async function embedBatch(texts, config, { signal } = {}) {
export async function embedBatch(texts, config, { signal, isQuery = false } = {}) {
const override = getEmbeddingTestOverride("embedBatch");
if (override) {
return await override(texts, config, { signal });
return await override(texts, config, { signal, isQuery });
}
if (readEmbeddingMode(config) === "backend") {
if (!texts.length || !config?.model) {
return texts.map(() => null);
}
try {
const payload = await requestBackendEmbeddings(
config,
{ texts, isQuery },
{ signal },
);
const vectors = Array.isArray(payload?.vectors) ? payload.vectors : [];
return texts.map((_, index) => normalizeVector(vectors[index]));
} catch (e) {
if (isAbortError(e)) {
throw e;
}
console.error("[ST-BME] Backend Embedding 批量调用失败:", e);
return texts.map(() => null);
}
}
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);

View File

@@ -236,6 +236,9 @@ export function getVectorModelScope(config) {
return [
"authority",
config.source || "authority-trivium",
config.embeddingMode || "direct",
config.embeddingSource || "direct",
normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix),
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
config.model || "",
].join("|");
@@ -266,6 +269,20 @@ export function validateVectorConfig(config) {
if (!config.baseUrl) {
return { valid: false, error: "Authority Trivium 地址不可用" };
}
if (!config.model) {
return { valid: false, error: "请先填写 Embedding 模型Authority 默认复用当前用户设置)" };
}
const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase();
const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase();
if (authorityEmbeddingMode === "backend") {
if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) {
return { valid: false, error: "当前后端 Embedding 源需要 API 地址Authority 默认复用当前用户设置)" };
}
return { valid: true, error: "" };
}
if (!config.apiUrl) {
return { valid: false, error: "请先填写 Embedding API 地址Authority 默认复用当前用户设置)" };
}
return { valid: true, error: "" };
}
@@ -606,7 +623,7 @@ function markBackendVectorStateDirty(
function markAuthorityVectorStateDirty(
graph,
config,
config = {},
reason = "authority-trivium-failed",
warning = "Authority Trivium 索引失败,已标记待重建",
) {
@@ -640,6 +657,42 @@ function markAuthorityVectorStateDirty(
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
}
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node]));
const entriesToEmbed = [];
for (const entry of entries || []) {
const node = nodesById.get(String(entry?.nodeId || ""));
const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0;
if (node && !hasEmbedding) {
entriesToEmbed.push({ entry, node });
}
}
if (!entriesToEmbed.length) {
return { requested: 0, failures: 0, elapsedMs: 0 };
}
throwIfAborted(signal);
const startedAt = nowMs();
const embeddings = await embedBatch(
entriesToEmbed.map(({ entry }) => entry.text),
config,
{ signal },
);
let failures = 0;
for (let index = 0; index < entriesToEmbed.length; index++) {
const embedding = embeddings[index];
if (embedding) {
entriesToEmbed[index].node.embedding = Array.from(embedding);
} else {
failures += 1;
}
}
return {
requested: entriesToEmbed.length,
failures,
elapsedMs: nowMs() - startedAt,
};
}
export async function syncGraphVectorIndex(
graph,
config,
@@ -741,6 +794,12 @@ export async function syncGraphVectorIndex(
try {
if (fullReset) {
const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal);
embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
}
const purgeStartedAt = nowMs();
await purgeAuthorityTriviumNamespace(config, authorityOptions);
authorityPurgeMs += nowMs() - purgeStartedAt;
@@ -794,6 +853,12 @@ export async function syncGraphVectorIndex(
queuedNodeIds.add(entry.nodeId);
}
const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal);
embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
}
deletedNodeCount = nodeIdsToDelete.length;
const deleteStartedAt = nowMs();
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
@@ -1108,7 +1173,7 @@ export async function findSimilarNodesByText(
if (isDirectVectorConfig(config)) {
const queryEmbedStartedAt = nowMs();
const queryVec = await embedText(text, config, { signal });
const queryVec = await embedText(text, config, { signal, isQuery: true });
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
if (!queryVec) {
recordSearchTimings({
@@ -1157,6 +1222,18 @@ export async function findSimilarNodesByText(
if (isAuthorityVectorConfig(config)) {
const requestStartedAt = nowMs();
try {
const queryEmbedStartedAt = nowMs();
const queryVec = await embedText(text, config, { signal, isQuery: true });
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
if (!queryVec) {
recordSearchTimings({
success: false,
reason: "authority-query-embed-empty",
queryEmbedMs: roundMs(queryEmbedMs),
resultCount: 0,
});
return [];
}
const allowedIds = new Set(candidateNodes.map((node) => node.id));
const results = (
await searchAuthorityTriviumNodes(graph, text, config, {
@@ -1166,6 +1243,7 @@ export async function findSimilarNodesByText(
modelScope: getVectorModelScope(config),
topK,
candidateIds: candidateNodes.map((node) => node.id),
queryVector: Array.from(queryVec),
signal,
})
)
@@ -1174,6 +1252,7 @@ export async function findSimilarNodesByText(
recordSearchTimings({
success: true,
reason: "ok",
queryEmbedMs: roundMs(queryEmbedMs),
requestMs: roundMs(nowMs() - requestStartedAt),
resultCount: results.length,
});