mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
refactor(authority): complete v0.6-only sql/blob/jobs rollout
This commit is contained in:
@@ -1,11 +1,17 @@
|
||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||
import {
|
||||
AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||
AuthorityHttpClient,
|
||||
} from "../runtime/authority-http-client.js";
|
||||
import { embedText } from "./embedding.js";
|
||||
|
||||
export const AUTHORITY_VECTOR_MODE = "authority";
|
||||
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
|
||||
|
||||
const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium";
|
||||
const DEFAULT_AUTHORITY_TRIVIUM_DATABASE = "st_bme_vectors";
|
||||
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
|
||||
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
|
||||
const DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE = "openai";
|
||||
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const parsed = Number(value);
|
||||
@@ -36,6 +42,27 @@ function normalizeRecordId(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function normalizeVector(value = null) {
|
||||
const source = ArrayBuffer.isView(value) ? Array.from(value) : value;
|
||||
if (!Array.isArray(source)) return [];
|
||||
return source
|
||||
.map((item) => Number(item))
|
||||
.filter((item) => Number.isFinite(item));
|
||||
}
|
||||
|
||||
function normalizePositiveInteger(value, fallback = 0) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed) || parsed <= 0) return fallback;
|
||||
return Math.floor(parsed);
|
||||
}
|
||||
|
||||
function normalizeOpenAICompatibleBaseUrl(value) {
|
||||
return String(value || "")
|
||||
.trim()
|
||||
.replace(/\/+(chat\/completions|embeddings)$/i, "")
|
||||
.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function readNestedValue(source = null, path = []) {
|
||||
let current = source;
|
||||
for (const key of path) {
|
||||
@@ -68,12 +95,14 @@ function readResultRows(payload = null) {
|
||||
if (Array.isArray(payload.data)) return payload.data;
|
||||
if (Array.isArray(payload.neighbors)) return payload.neighbors;
|
||||
if (Array.isArray(payload.links)) return payload.links;
|
||||
if (Array.isArray(payload.nodes)) return payload.nodes;
|
||||
if (Array.isArray(payload.result?.results)) return payload.result.results;
|
||||
if (Array.isArray(payload.result?.items)) return payload.result.items;
|
||||
if (Array.isArray(payload.result?.rows)) return payload.result.rows;
|
||||
if (Array.isArray(payload.result?.data)) return payload.result.data;
|
||||
if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors;
|
||||
if (Array.isArray(payload.result?.links)) return payload.result.links;
|
||||
if (Array.isArray(payload.result?.nodes)) return payload.result.nodes;
|
||||
return [];
|
||||
}
|
||||
|
||||
@@ -145,6 +174,32 @@ function normalizeSearchResults(payload = null) {
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function buildOpenOptions(config = {}, payload = {}) {
|
||||
const database = normalizeRecordId(payload.database || config.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE;
|
||||
return {
|
||||
database,
|
||||
...(normalizePositiveInteger(payload.dim ?? config.dim, 0) > 0 ? { dim: normalizePositiveInteger(payload.dim ?? config.dim, 0) } : {}),
|
||||
...(payload.dtype || config.dtype ? { dtype: String(payload.dtype || config.dtype) } : {}),
|
||||
...(payload.syncMode || config.syncMode ? { syncMode: String(payload.syncMode || config.syncMode) } : {}),
|
||||
...(payload.storageMode || config.storageMode ? { storageMode: String(payload.storageMode || config.storageMode) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function getNamespace(payload = {}) {
|
||||
return normalizeRecordId(payload.namespace || payload.collectionId || payload.chatId);
|
||||
}
|
||||
|
||||
function buildNodeReference(id, namespace = "") {
|
||||
return {
|
||||
externalId: normalizeRecordId(id),
|
||||
...(namespace ? { namespace } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
function buildV06PayloadSource(payload = {}) {
|
||||
return payload && typeof payload === "object" && !Array.isArray(payload) ? payload : {};
|
||||
}
|
||||
|
||||
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
|
||||
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
|
||||
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
|
||||
@@ -166,6 +221,7 @@ function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelSc
|
||||
regionKey: String(scope.regionKey || node?.regionKey || ""),
|
||||
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
|
||||
storyTimeLabel: String(node?.storyTime?.label || ""),
|
||||
text: String(entry?.text || ""),
|
||||
title: getNodeFieldText(node, ["title"]),
|
||||
name: getNodeFieldText(node, ["name"]),
|
||||
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
|
||||
@@ -190,6 +246,7 @@ function buildAuthorityVectorItems(graph, entries = [], options = {}) {
|
||||
text: String(entry?.text || ""),
|
||||
index: Number(entry?.index || 0) || 0,
|
||||
hash: String(entry?.hash || ""),
|
||||
vector: normalizeVector(entry?.vector || entry?.embedding || node?.embedding),
|
||||
payload,
|
||||
};
|
||||
})
|
||||
@@ -229,11 +286,43 @@ export function isAuthorityVectorConfig(config = null) {
|
||||
|
||||
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
|
||||
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
|
||||
const hasAuthorityEmbeddingOverride = [
|
||||
source.authorityEmbeddingApiUrl,
|
||||
source.authorityEmbeddingApiKey,
|
||||
source.authorityEmbeddingModel,
|
||||
].some((value) => String(value ?? "").trim());
|
||||
const embeddingMode = hasAuthorityEmbeddingOverride
|
||||
? "direct"
|
||||
: String(source.embeddingTransportMode || "direct").trim().toLowerCase() === "backend"
|
||||
? "backend"
|
||||
: "direct";
|
||||
const embeddingSource = embeddingMode === "backend"
|
||||
? String(source.embeddingBackendSource || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE).trim().toLowerCase() || DEFAULT_AUTHORITY_EMBEDDING_BACKEND_SOURCE
|
||||
: "direct";
|
||||
return {
|
||||
mode: AUTHORITY_VECTOR_MODE,
|
||||
source: AUTHORITY_VECTOR_SOURCE,
|
||||
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
|
||||
model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(),
|
||||
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||
database: normalizeRecordId(source.authorityTriviumDatabase ?? source.triviumDatabase) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
|
||||
dim: normalizePositiveInteger(source.authorityTriviumDim ?? source.triviumDim, 0),
|
||||
dtype: String(source.authorityTriviumDtype ?? source.triviumDtype ?? "").trim(),
|
||||
syncMode: String(source.authorityTriviumSyncMode ?? source.triviumSyncMode ?? "").trim(),
|
||||
storageMode: String(source.authorityTriviumStorageMode ?? source.triviumStorageMode ?? "").trim(),
|
||||
embeddingMode,
|
||||
embeddingSource,
|
||||
apiUrl: normalizeOpenAICompatibleBaseUrl(
|
||||
embeddingMode === "backend"
|
||||
? source.embeddingBackendApiUrl
|
||||
: source.authorityEmbeddingApiUrl ?? source.embeddingApiUrl ?? source.embeddingBackendApiUrl,
|
||||
),
|
||||
apiKey: embeddingMode === "backend"
|
||||
? ""
|
||||
: String(source.authorityEmbeddingApiKey ?? source.embeddingApiKey ?? "").trim(),
|
||||
model: embeddingMode === "backend"
|
||||
? String(source.embeddingBackendModel ?? source.embeddingModel ?? "").trim()
|
||||
: String(source.authorityEmbeddingModel ?? source.embeddingModel ?? source.embeddingBackendModel ?? "").trim(),
|
||||
autoSuffix: source.embeddingAutoSuffix !== false,
|
||||
chunkSize: clampInteger(
|
||||
source.authorityVectorSyncChunkSize ?? source.chunkSize,
|
||||
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
|
||||
@@ -251,61 +340,225 @@ export class AuthorityTriviumHttpClient {
|
||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
||||
this.protocol = AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06;
|
||||
this.config = {
|
||||
database: normalizeRecordId(options.database) || DEFAULT_AUTHORITY_TRIVIUM_DATABASE,
|
||||
dim: normalizePositiveInteger(options.dim, 0),
|
||||
dtype: String(options.dtype || "").trim(),
|
||||
syncMode: String(options.syncMode || "").trim(),
|
||||
storageMode: String(options.storageMode || "").trim(),
|
||||
};
|
||||
this.http = new AuthorityHttpClient({
|
||||
...options,
|
||||
baseUrl: this.baseUrl,
|
||||
fetchImpl: this.fetchImpl,
|
||||
headerProvider: this.headerProvider,
|
||||
protocol: this.protocol,
|
||||
});
|
||||
}
|
||||
|
||||
async request(action, payload = {}) {
|
||||
if (typeof this.fetchImpl !== "function") {
|
||||
throw new Error("Authority Trivium fetch unavailable");
|
||||
}
|
||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
||||
},
|
||||
body: JSON.stringify({ action, ...payload }),
|
||||
if (action === "purge") return await this.purge(payload);
|
||||
if (action === "bulkUpsert") return await this.bulkUpsert(payload);
|
||||
if (action === "deleteMany") return await this.deleteMany(payload);
|
||||
if (action === "linkMany") return await this.linkMany(payload);
|
||||
if (action === "search") return await this.search(payload);
|
||||
if (action === "filterWhere") return await this.filterWhere(payload);
|
||||
if (action === "queryPage") return await this.queryPage(payload);
|
||||
if (action === "neighbors") return await this.neighbors(payload);
|
||||
if (action === "stat") return await this.stat(payload);
|
||||
throw new Error(`Authority Trivium v0.6 action unavailable: ${action}`);
|
||||
}
|
||||
|
||||
async requestV06(path, payload = {}, method = "POST") {
|
||||
return await this.http.requestJson(path, {
|
||||
method,
|
||||
body: payload,
|
||||
session: true,
|
||||
protocol: AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
|
||||
});
|
||||
if (!response?.ok) {
|
||||
throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`);
|
||||
}
|
||||
return await response.json().catch(() => ({}));
|
||||
}
|
||||
|
||||
buildOpenOptions(payload = {}) {
|
||||
return buildOpenOptions(this.config, payload);
|
||||
}
|
||||
|
||||
async purge(payload = {}) {
|
||||
return await this.request("purge", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const openOptions = this.buildOpenOptions(payload);
|
||||
let cursor = "";
|
||||
let deleted = 0;
|
||||
let scanned = 0;
|
||||
for (let pageIndex = 0; pageIndex < 100; pageIndex++) {
|
||||
const page = await this.requestV06("/trivium/list-mappings", {
|
||||
...openOptions,
|
||||
namespace,
|
||||
page: { cursor, limit: 200 },
|
||||
});
|
||||
const mappings = toArray(page?.mappings);
|
||||
if (!mappings.length && !page?.page?.hasMore) break;
|
||||
scanned += mappings.length;
|
||||
const items = mappings
|
||||
.map((item) => buildNodeReference(item?.externalId, item?.namespace || namespace))
|
||||
.filter((item) => item.externalId);
|
||||
if (items.length) {
|
||||
const result = await this.requestV06("/trivium/bulk-delete", {
|
||||
...openOptions,
|
||||
items,
|
||||
});
|
||||
deleted += Number(result?.successCount ?? items.length) || 0;
|
||||
}
|
||||
if (!page?.page?.hasMore) break;
|
||||
cursor = String(page?.page?.nextCursor || "");
|
||||
if (!cursor) break;
|
||||
}
|
||||
return { ok: true, scanned, deleted };
|
||||
}
|
||||
|
||||
async bulkUpsert(payload = {}) {
|
||||
return await this.request("bulkUpsert", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const items = toArray(payload.items);
|
||||
const missingVector = items.find((item) => !normalizeVector(item?.vector || item?.embedding).length);
|
||||
if (missingVector) {
|
||||
throw new Error("Authority Trivium v0.6 bulkUpsert requires vector for every item");
|
||||
}
|
||||
const mappedItems = items.map((item) => {
|
||||
const nodeId = normalizeRecordId(item?.externalId || item?.nodeId || item?.id);
|
||||
const payloadSource = buildV06PayloadSource(item?.payload);
|
||||
return {
|
||||
externalId: nodeId,
|
||||
namespace,
|
||||
vector: normalizeVector(item?.vector || item?.embedding),
|
||||
payload: {
|
||||
...payloadSource,
|
||||
nodeId: payloadSource.nodeId || nodeId,
|
||||
externalId: payloadSource.externalId || nodeId,
|
||||
collectionId: payload.collectionId || payloadSource.collectionId || "",
|
||||
text: payloadSource.text || item?.text || "",
|
||||
contentHash: payloadSource.contentHash || item?.hash || "",
|
||||
index: Number(item?.index || payloadSource.index || 0) || 0,
|
||||
},
|
||||
};
|
||||
});
|
||||
return await this.requestV06("/trivium/bulk-upsert", {
|
||||
...this.buildOpenOptions(payload),
|
||||
items: mappedItems,
|
||||
});
|
||||
}
|
||||
|
||||
async deleteMany(payload = {}) {
|
||||
return await this.request("deleteMany", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const ids = [
|
||||
...toArray(payload.ids),
|
||||
...toArray(payload.externalIds),
|
||||
...toArray(payload.items).map((item) => item?.externalId || item?.nodeId || item?.id),
|
||||
].map(normalizeRecordId).filter(Boolean);
|
||||
return await this.requestV06("/trivium/bulk-delete", {
|
||||
...this.buildOpenOptions(payload),
|
||||
items: ids.map((id) => buildNodeReference(id, namespace)),
|
||||
});
|
||||
}
|
||||
|
||||
async linkMany(payload = {}) {
|
||||
return await this.request("linkMany", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const sourceLinks = toArray(payload.links || payload.items);
|
||||
return await this.requestV06("/trivium/bulk-link", {
|
||||
...this.buildOpenOptions(payload),
|
||||
items: sourceLinks
|
||||
.map((link) => {
|
||||
const src = normalizeRecordId(link?.fromId || link?.src || link?.sourceId);
|
||||
const dst = normalizeRecordId(link?.toId || link?.dst || link?.targetId);
|
||||
if (!src || !dst) return null;
|
||||
return {
|
||||
src: buildNodeReference(src, namespace),
|
||||
dst: buildNodeReference(dst, namespace),
|
||||
label: String(link?.relation || link?.label || "related"),
|
||||
weight: Number(link?.weight ?? link?.strength ?? 1) || 1,
|
||||
};
|
||||
})
|
||||
.filter(Boolean),
|
||||
});
|
||||
}
|
||||
|
||||
async search(payload = {}) {
|
||||
return await this.request("search", payload);
|
||||
const vector = normalizeVector(payload.vector || payload.embedding || payload.queryVector);
|
||||
if (!vector.length) {
|
||||
throw new Error("Authority Trivium v0.6 search requires vector");
|
||||
}
|
||||
const queryText = String(payload.queryText || payload.text || payload.searchText || payload.query || "");
|
||||
const body = {
|
||||
...this.buildOpenOptions(payload),
|
||||
vector,
|
||||
topK: Number(payload.topK || payload.limit || 0) || undefined,
|
||||
expandDepth: Number(payload.expandDepth || payload.depth || 0) || undefined,
|
||||
minScore: Number.isFinite(Number(payload.minScore)) ? Number(payload.minScore) : undefined,
|
||||
...(payload.payloadFilter || payload.filter ? { payloadFilter: payload.payloadFilter || payload.filter } : {}),
|
||||
};
|
||||
if (queryText) {
|
||||
return await this.requestV06("/trivium/search-hybrid", {
|
||||
...body,
|
||||
queryText,
|
||||
hybridAlpha: Number.isFinite(Number(payload.hybridAlpha)) ? Number(payload.hybridAlpha) : undefined,
|
||||
});
|
||||
}
|
||||
return await this.requestV06("/trivium/search", body);
|
||||
}
|
||||
|
||||
async filterWhere(payload = {}) {
|
||||
return await this.request("filterWhere", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const result = await this.requestV06("/trivium/list-mappings", {
|
||||
...this.buildOpenOptions(payload),
|
||||
namespace,
|
||||
page: {
|
||||
limit: Number(payload.limit || payload.topK || payload.pageSize || 100) || 100,
|
||||
},
|
||||
});
|
||||
return { items: toArray(result?.mappings) };
|
||||
}
|
||||
|
||||
async queryPage(payload = {}) {
|
||||
return await this.request("queryPage", payload);
|
||||
return await this.filterWhere(payload);
|
||||
}
|
||||
|
||||
async neighbors(payload = {}) {
|
||||
return await this.request("neighbors", payload);
|
||||
const namespace = getNamespace(payload);
|
||||
const seedIds = [
|
||||
...toArray(payload.ids),
|
||||
...toArray(payload.nodeIds),
|
||||
...toArray(payload.seedIds),
|
||||
payload.id,
|
||||
].map(normalizeRecordId).filter(Boolean);
|
||||
const openOptions = this.buildOpenOptions(payload);
|
||||
const resolved = await this.requestV06("/trivium/resolve-many", {
|
||||
...openOptions,
|
||||
items: seedIds.map((id) => buildNodeReference(id, namespace)),
|
||||
});
|
||||
const neighbors = [];
|
||||
for (const item of toArray(resolved?.items)) {
|
||||
const internalId = Number(item?.id);
|
||||
if (!Number.isFinite(internalId) || internalId <= 0) continue;
|
||||
const result = await this.requestV06("/trivium/neighbors", {
|
||||
...openOptions,
|
||||
id: internalId,
|
||||
depth: Number(payload.depth || payload.expandDepth || 1) || 1,
|
||||
});
|
||||
for (const node of toArray(result?.nodes)) {
|
||||
neighbors.push({
|
||||
externalId: node?.externalId,
|
||||
nodeId: node?.externalId,
|
||||
id: node?.id,
|
||||
namespace: node?.namespace,
|
||||
});
|
||||
}
|
||||
}
|
||||
return { neighbors };
|
||||
}
|
||||
|
||||
async stat(payload = {}) {
|
||||
return await this.request("stat", payload);
|
||||
return await this.requestV06("/trivium/stat", {
|
||||
...this.buildOpenOptions(payload),
|
||||
...(payload.includeMappingIntegrity ? { includeMappingIntegrity: true } : {}),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
@@ -313,9 +566,13 @@ export function createAuthorityTriviumClient(config = {}, options = {}) {
|
||||
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
|
||||
if (injected) return injected;
|
||||
return new AuthorityTriviumHttpClient({
|
||||
...config,
|
||||
baseUrl: config.baseUrl,
|
||||
fetchImpl: options.fetchImpl || config.fetchImpl,
|
||||
headerProvider: options.headerProvider || config.headerProvider,
|
||||
protocol: config.protocol,
|
||||
sessionToken: options.sessionToken || config.sessionToken,
|
||||
sessionInitConfig: options.sessionInitConfig || config.sessionInitConfig,
|
||||
});
|
||||
}
|
||||
|
||||
@@ -451,6 +708,8 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
|
||||
chatId: options.chatId,
|
||||
text: String(text || ""),
|
||||
searchText: String(text || ""),
|
||||
vector: normalizeVector(options.vector || options.queryVector || options.embedding),
|
||||
queryVector: normalizeVector(options.queryVector || options.vector || options.embedding),
|
||||
topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
|
||||
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
|
||||
});
|
||||
@@ -458,11 +717,15 @@ export async function searchAuthorityTriviumNodes(graph, text, config = {}, opti
|
||||
}
|
||||
|
||||
export async function testAuthorityTriviumConnection(config = {}, options = {}) {
|
||||
const probeVector = await embedText("test connection", config, { isQuery: true });
|
||||
if (!probeVector || probeVector.length === 0) {
|
||||
return { success: false, dimensions: 0, error: "Embedding API 返回空结果" };
|
||||
}
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
await callClient(client, ["stat"], "stat", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
});
|
||||
return { success: true, dimensions: 0, error: "" };
|
||||
return { success: true, dimensions: probeVector.length, error: "" };
|
||||
}
|
||||
|
||||
@@ -6,11 +6,17 @@
|
||||
* 调用外部 API 获取文本向量,并提供暴力搜索 cosine 相似度
|
||||
*/
|
||||
|
||||
import { getRequestHeaders } from "../../../../../script.js";
|
||||
import { extension_settings } from "../../../../extensions.js";
|
||||
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
|
||||
|
||||
const MODULE_NAME = "st_bme";
|
||||
const EMBEDDING_REQUEST_TIMEOUT_MS = 300000;
|
||||
const BACKEND_SOURCES_REQUIRING_API_URL = new Set([
|
||||
"ollama",
|
||||
"llamacpp",
|
||||
"vllm",
|
||||
]);
|
||||
|
||||
function getEmbeddingTestOverride(name) {
|
||||
const override = globalThis.__stBmeTestOverrides?.embedding?.[name];
|
||||
@@ -41,6 +47,69 @@ function normalizeOpenAICompatibleBaseUrl(value) {
|
||||
.replace(/\/+$/, "");
|
||||
}
|
||||
|
||||
function normalizeVector(value) {
|
||||
if (!Array.isArray(value)) return null;
|
||||
const vector = value.map((item) => Number(item)).filter((item) => Number.isFinite(item));
|
||||
return vector.length ? new Float64Array(vector) : null;
|
||||
}
|
||||
|
||||
function readEmbeddingMode(config = {}) {
|
||||
return String(config?.embeddingMode || config?.mode || "direct").trim().toLowerCase();
|
||||
}
|
||||
|
||||
function readEmbeddingSource(config = {}) {
|
||||
return String(config?.embeddingSource || config?.source || "openai").trim().toLowerCase() || "openai";
|
||||
}
|
||||
|
||||
function buildBackendEmbeddingRequestBody(config = {}, payload = {}) {
|
||||
const source = readEmbeddingSource(config);
|
||||
const body = {
|
||||
source,
|
||||
model: String(config?.model || "").trim(),
|
||||
isQuery: Boolean(payload.isQuery),
|
||||
};
|
||||
if (payload.text !== undefined) {
|
||||
body.text = String(payload.text ?? "");
|
||||
}
|
||||
if (Array.isArray(payload.texts)) {
|
||||
body.texts = payload.texts.map((item) => String(item ?? ""));
|
||||
}
|
||||
if (BACKEND_SOURCES_REQUIRING_API_URL.has(source)) {
|
||||
body.apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||
}
|
||||
if (source === "ollama") {
|
||||
body.keep = false;
|
||||
}
|
||||
return body;
|
||||
}
|
||||
|
||||
async function requestBackendEmbeddings(config = {}, payload = {}, { signal } = {}) {
|
||||
const response = await fetchWithTimeout(
|
||||
"/api/vector/embed",
|
||||
{
|
||||
method: "POST",
|
||||
headers: {
|
||||
...getRequestHeaders(),
|
||||
"Content-Type": "application/json",
|
||||
},
|
||||
signal,
|
||||
body: JSON.stringify(buildBackendEmbeddingRequestBody(config, payload)),
|
||||
},
|
||||
getConfiguredTimeoutMs(config),
|
||||
);
|
||||
|
||||
if (!response.ok) {
|
||||
const errorText = await response.text().catch(() => response.statusText);
|
||||
console.error(
|
||||
`[ST-BME] Backend Embedding API 错误 (${response.status}):`,
|
||||
errorText,
|
||||
);
|
||||
return null;
|
||||
}
|
||||
|
||||
return await response.json().catch(() => ({}));
|
||||
}
|
||||
|
||||
function createCombinedAbortSignal(...signals) {
|
||||
const validSignals = signals.filter(Boolean);
|
||||
if (validSignals.length <= 1) {
|
||||
@@ -107,10 +176,31 @@ async function fetchWithTimeout(
|
||||
* @param {string} config.model - 模型名(如 text-embedding-3-small)
|
||||
* @returns {Promise<Float64Array|null>} 向量或 null
|
||||
*/
|
||||
export async function embedText(text, config, { signal } = {}) {
|
||||
export async function embedText(text, config, { signal, isQuery = false } = {}) {
|
||||
const override = getEmbeddingTestOverride("embedText");
|
||||
if (override) {
|
||||
return await override(text, config, { signal });
|
||||
return await override(text, config, { signal, isQuery });
|
||||
}
|
||||
|
||||
if (readEmbeddingMode(config) === "backend") {
|
||||
if (!text || !config?.model) {
|
||||
console.warn("[ST-BME] Embedding 配置不完整,跳过");
|
||||
return null;
|
||||
}
|
||||
try {
|
||||
const payload = await requestBackendEmbeddings(
|
||||
config,
|
||||
{ text, isQuery },
|
||||
{ signal },
|
||||
);
|
||||
return normalizeVector(payload?.vector);
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) {
|
||||
throw e;
|
||||
}
|
||||
console.error("[ST-BME] Backend Embedding 调用失败:", e);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||
@@ -173,10 +263,31 @@ export async function embedText(text, config, { signal } = {}) {
|
||||
* @param {object} config
|
||||
* @returns {Promise<(Float64Array|null)[]>}
|
||||
*/
|
||||
export async function embedBatch(texts, config, { signal } = {}) {
|
||||
export async function embedBatch(texts, config, { signal, isQuery = false } = {}) {
|
||||
const override = getEmbeddingTestOverride("embedBatch");
|
||||
if (override) {
|
||||
return await override(texts, config, { signal });
|
||||
return await override(texts, config, { signal, isQuery });
|
||||
}
|
||||
|
||||
if (readEmbeddingMode(config) === "backend") {
|
||||
if (!texts.length || !config?.model) {
|
||||
return texts.map(() => null);
|
||||
}
|
||||
try {
|
||||
const payload = await requestBackendEmbeddings(
|
||||
config,
|
||||
{ texts, isQuery },
|
||||
{ signal },
|
||||
);
|
||||
const vectors = Array.isArray(payload?.vectors) ? payload.vectors : [];
|
||||
return texts.map((_, index) => normalizeVector(vectors[index]));
|
||||
} catch (e) {
|
||||
if (isAbortError(e)) {
|
||||
throw e;
|
||||
}
|
||||
console.error("[ST-BME] Backend Embedding 批量调用失败:", e);
|
||||
return texts.map(() => null);
|
||||
}
|
||||
}
|
||||
|
||||
const apiUrl = normalizeOpenAICompatibleBaseUrl(config?.apiUrl);
|
||||
|
||||
@@ -236,6 +236,9 @@ export function getVectorModelScope(config) {
|
||||
return [
|
||||
"authority",
|
||||
config.source || "authority-trivium",
|
||||
config.embeddingMode || "direct",
|
||||
config.embeddingSource || "direct",
|
||||
normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix),
|
||||
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
|
||||
config.model || "",
|
||||
].join("|");
|
||||
@@ -266,6 +269,20 @@ export function validateVectorConfig(config) {
|
||||
if (!config.baseUrl) {
|
||||
return { valid: false, error: "Authority Trivium 地址不可用" };
|
||||
}
|
||||
if (!config.model) {
|
||||
return { valid: false, error: "请先填写 Embedding 模型(Authority 默认复用当前用户设置)" };
|
||||
}
|
||||
const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase();
|
||||
const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase();
|
||||
if (authorityEmbeddingMode === "backend") {
|
||||
if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) {
|
||||
return { valid: false, error: "当前后端 Embedding 源需要 API 地址(Authority 默认复用当前用户设置)" };
|
||||
}
|
||||
return { valid: true, error: "" };
|
||||
}
|
||||
if (!config.apiUrl) {
|
||||
return { valid: false, error: "请先填写 Embedding API 地址(Authority 默认复用当前用户设置)" };
|
||||
}
|
||||
return { valid: true, error: "" };
|
||||
}
|
||||
|
||||
@@ -606,7 +623,7 @@ function markBackendVectorStateDirty(
|
||||
|
||||
function markAuthorityVectorStateDirty(
|
||||
graph,
|
||||
config,
|
||||
config = {},
|
||||
reason = "authority-trivium-failed",
|
||||
warning = "Authority Trivium 索引失败,已标记待重建",
|
||||
) {
|
||||
@@ -640,6 +657,42 @@ function markAuthorityVectorStateDirty(
|
||||
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
|
||||
}
|
||||
|
||||
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
|
||||
const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node]));
|
||||
const entriesToEmbed = [];
|
||||
for (const entry of entries || []) {
|
||||
const node = nodesById.get(String(entry?.nodeId || ""));
|
||||
const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0;
|
||||
if (node && !hasEmbedding) {
|
||||
entriesToEmbed.push({ entry, node });
|
||||
}
|
||||
}
|
||||
if (!entriesToEmbed.length) {
|
||||
return { requested: 0, failures: 0, elapsedMs: 0 };
|
||||
}
|
||||
throwIfAborted(signal);
|
||||
const startedAt = nowMs();
|
||||
const embeddings = await embedBatch(
|
||||
entriesToEmbed.map(({ entry }) => entry.text),
|
||||
config,
|
||||
{ signal },
|
||||
);
|
||||
let failures = 0;
|
||||
for (let index = 0; index < entriesToEmbed.length; index++) {
|
||||
const embedding = embeddings[index];
|
||||
if (embedding) {
|
||||
entriesToEmbed[index].node.embedding = Array.from(embedding);
|
||||
} else {
|
||||
failures += 1;
|
||||
}
|
||||
}
|
||||
return {
|
||||
requested: entriesToEmbed.length,
|
||||
failures,
|
||||
elapsedMs: nowMs() - startedAt,
|
||||
};
|
||||
}
|
||||
|
||||
export async function syncGraphVectorIndex(
|
||||
graph,
|
||||
config,
|
||||
@@ -741,6 +794,12 @@ export async function syncGraphVectorIndex(
|
||||
|
||||
try {
|
||||
if (fullReset) {
|
||||
const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal);
|
||||
embeddingsRequested += embeddingResult.requested;
|
||||
embedBatchMs += embeddingResult.elapsedMs;
|
||||
if (embeddingResult.failures > 0) {
|
||||
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
|
||||
}
|
||||
const purgeStartedAt = nowMs();
|
||||
await purgeAuthorityTriviumNamespace(config, authorityOptions);
|
||||
authorityPurgeMs += nowMs() - purgeStartedAt;
|
||||
@@ -794,6 +853,12 @@ export async function syncGraphVectorIndex(
|
||||
queuedNodeIds.add(entry.nodeId);
|
||||
}
|
||||
|
||||
const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal);
|
||||
embeddingsRequested += embeddingResult.requested;
|
||||
embedBatchMs += embeddingResult.elapsedMs;
|
||||
if (embeddingResult.failures > 0) {
|
||||
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`);
|
||||
}
|
||||
deletedNodeCount = nodeIdsToDelete.length;
|
||||
const deleteStartedAt = nowMs();
|
||||
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
|
||||
@@ -1108,7 +1173,7 @@ export async function findSimilarNodesByText(
|
||||
|
||||
if (isDirectVectorConfig(config)) {
|
||||
const queryEmbedStartedAt = nowMs();
|
||||
const queryVec = await embedText(text, config, { signal });
|
||||
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||||
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||||
if (!queryVec) {
|
||||
recordSearchTimings({
|
||||
@@ -1157,6 +1222,18 @@ export async function findSimilarNodesByText(
|
||||
if (isAuthorityVectorConfig(config)) {
|
||||
const requestStartedAt = nowMs();
|
||||
try {
|
||||
const queryEmbedStartedAt = nowMs();
|
||||
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||||
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||||
if (!queryVec) {
|
||||
recordSearchTimings({
|
||||
success: false,
|
||||
reason: "authority-query-embed-empty",
|
||||
queryEmbedMs: roundMs(queryEmbedMs),
|
||||
resultCount: 0,
|
||||
});
|
||||
return [];
|
||||
}
|
||||
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
||||
const results = (
|
||||
await searchAuthorityTriviumNodes(graph, text, config, {
|
||||
@@ -1166,6 +1243,7 @@ export async function findSimilarNodesByText(
|
||||
modelScope: getVectorModelScope(config),
|
||||
topK,
|
||||
candidateIds: candidateNodes.map((node) => node.id),
|
||||
queryVector: Array.from(queryVec),
|
||||
signal,
|
||||
})
|
||||
)
|
||||
@@ -1174,6 +1252,7 @@ export async function findSimilarNodesByText(
|
||||
recordSearchTimings({
|
||||
success: true,
|
||||
reason: "ok",
|
||||
queryEmbedMs: roundMs(queryEmbedMs),
|
||||
requestMs: roundMs(nowMs() - requestStartedAt),
|
||||
resultCount: results.length,
|
||||
});
|
||||
|
||||
Reference in New Issue
Block a user