mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
1849 lines
57 KiB
JavaScript
1849 lines
57 KiB
JavaScript
// ST-BME: 向量模式、后端索引与直连兜底
|
||
|
||
import { getRequestHeaders } from "../../../../../script.js";
|
||
import { embedBatch, embedText, searchSimilar } from "./embedding.js";
|
||
import { getActiveNodes } from "../graph/graph.js";
|
||
import { describeMemoryScope, normalizeMemoryScope } from "../graph/memory-scope.js";
|
||
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
|
||
import { buildVectorCollectionId, stableHashString } from "../runtime/runtime-state.js";
|
||
import {
|
||
AUTHORITY_VECTOR_MODE,
|
||
AUTHORITY_VECTOR_SOURCE,
|
||
applyAuthorityBmeVectorManifest,
|
||
deleteAuthorityTriviumNodes,
|
||
isAuthorityVectorConfig,
|
||
normalizeAuthorityVectorConfig,
|
||
purgeAuthorityTriviumNamespace,
|
||
searchAuthorityTriviumNodes,
|
||
syncAuthorityTriviumLinks,
|
||
testAuthorityTriviumConnection,
|
||
upsertAuthorityTriviumEntries,
|
||
} from "./authority-vector-primary-adapter.js";
|
||
|
||
export {
|
||
AUTHORITY_VECTOR_MODE,
|
||
AUTHORITY_VECTOR_SOURCE,
|
||
isAuthorityVectorConfig,
|
||
normalizeAuthorityVectorConfig,
|
||
};
|
||
|
||
export const BACKEND_VECTOR_SOURCES = [
|
||
"openai",
|
||
"openrouter",
|
||
"cohere",
|
||
"mistral",
|
||
"electronhub",
|
||
"chutes",
|
||
"nanogpt",
|
||
"ollama",
|
||
"llamacpp",
|
||
"vllm",
|
||
];
|
||
|
||
const BACKEND_SOURCES_REQUIRING_API_URL = new Set([
|
||
"ollama",
|
||
"llamacpp",
|
||
"vllm",
|
||
]);
|
||
|
||
const MODEL_LIST_ENDPOINTS = {
|
||
openrouter: "/api/openrouter/models/embedding",
|
||
chutes: "/api/openai/chutes/models/embedding",
|
||
nanogpt: "/api/openai/nanogpt/models/embedding",
|
||
electronhub: "/api/openai/electronhub/models",
|
||
};
|
||
const VECTOR_REQUEST_TIMEOUT_MS = 300000;
|
||
|
||
function getConfiguredTimeoutMs(config = {}) {
|
||
return typeof resolveConfiguredTimeoutMs === "function"
|
||
? resolveConfiguredTimeoutMs(config, VECTOR_REQUEST_TIMEOUT_MS)
|
||
: (() => {
|
||
const timeoutMs = Number(config?.timeoutMs);
|
||
return Number.isFinite(timeoutMs) && timeoutMs > 0
|
||
? timeoutMs
|
||
: VECTOR_REQUEST_TIMEOUT_MS;
|
||
})();
|
||
}
|
||
|
||
const VECTOR_CONNECTION_PROBE_TEXTS = ["test connection", "runtime batch probe"];
|
||
|
||
const BACKEND_STATUS_MODEL_SOURCES = {
|
||
openai: "openai",
|
||
cohere: "cohere",
|
||
mistral: "mistralai",
|
||
};
|
||
|
||
function isAbortError(error) {
|
||
return error?.name === "AbortError";
|
||
}
|
||
|
||
function throwIfAborted(signal) {
|
||
if (signal?.aborted) {
|
||
throw signal.reason instanceof Error
|
||
? signal.reason
|
||
: Object.assign(new Error("操作已终止"), { name: "AbortError" });
|
||
}
|
||
}
|
||
|
||
function nowMs() {
|
||
if (typeof performance?.now === "function") {
|
||
return performance.now();
|
||
}
|
||
return Date.now();
|
||
}
|
||
|
||
function roundMs(value) {
|
||
return Math.round((Number(value) || 0) * 10) / 10;
|
||
}
|
||
|
||
export const BACKEND_DEFAULT_MODELS = {
|
||
openai: "text-embedding-3-small",
|
||
openrouter: "openai/text-embedding-3-small",
|
||
cohere: "embed-multilingual-v3.0",
|
||
mistral: "mistral-embed",
|
||
electronhub: "text-embedding-3-small",
|
||
chutes: "chutes-qwen-qwen3-embedding-8b",
|
||
nanogpt: "text-embedding-3-small",
|
||
ollama: "nomic-embed-text",
|
||
llamacpp: "text-embedding-3-small",
|
||
vllm: "BAAI/bge-m3",
|
||
};
|
||
|
||
function createCombinedAbortSignal(...signals) {
|
||
const validSignals = signals.filter(Boolean);
|
||
if (validSignals.length <= 1) {
|
||
return validSignals[0] || undefined;
|
||
}
|
||
|
||
if (
|
||
typeof AbortSignal !== "undefined" &&
|
||
typeof AbortSignal.any === "function"
|
||
) {
|
||
return AbortSignal.any(validSignals);
|
||
}
|
||
|
||
const controller = new AbortController();
|
||
for (const signal of validSignals) {
|
||
if (signal.aborted) {
|
||
controller.abort(signal.reason);
|
||
return controller.signal;
|
||
}
|
||
signal.addEventListener("abort", () => controller.abort(signal.reason), {
|
||
once: true,
|
||
});
|
||
}
|
||
return controller.signal;
|
||
}
|
||
|
||
async function fetchWithTimeout(
|
||
url,
|
||
options = {},
|
||
timeoutMs = VECTOR_REQUEST_TIMEOUT_MS,
|
||
) {
|
||
const controller = new AbortController();
|
||
const timeout = setTimeout(
|
||
() =>
|
||
controller.abort(
|
||
new DOMException(
|
||
`向量请求超时 (${Math.round(timeoutMs / 1000)}s)`,
|
||
"AbortError",
|
||
),
|
||
),
|
||
timeoutMs,
|
||
);
|
||
const signal = options.signal
|
||
? createCombinedAbortSignal(options.signal, controller.signal)
|
||
: controller.signal;
|
||
|
||
try {
|
||
return await fetch(url, {
|
||
...options,
|
||
signal,
|
||
});
|
||
} finally {
|
||
clearTimeout(timeout);
|
||
}
|
||
}
|
||
|
||
export function normalizeOpenAICompatibleBaseUrl(value, autoSuffix = true) {
|
||
let normalized = String(value || "")
|
||
.trim()
|
||
.replace(/\/+(chat\/completions|embeddings)$/i, "")
|
||
.replace(/\/+$/, "");
|
||
|
||
if (autoSuffix && normalized && !/\/v\d+$/i.test(normalized)) {
|
||
normalized = normalized;
|
||
}
|
||
|
||
return normalized;
|
||
}
|
||
|
||
export function getVectorConfigFromSettings(settings = {}) {
|
||
const mode =
|
||
settings.embeddingTransportMode === "backend" ? "backend" : "direct";
|
||
const autoSuffix = settings.embeddingAutoSuffix !== false;
|
||
|
||
if (mode === "direct") {
|
||
return {
|
||
mode,
|
||
source: "direct",
|
||
apiUrl: normalizeOpenAICompatibleBaseUrl(
|
||
settings.embeddingApiUrl,
|
||
autoSuffix,
|
||
),
|
||
apiKey: String(settings.embeddingApiKey || "").trim(),
|
||
model: String(settings.embeddingModel || "").trim(),
|
||
autoSuffix,
|
||
timeoutMs: getConfiguredTimeoutMs(settings),
|
||
};
|
||
}
|
||
|
||
const source = BACKEND_VECTOR_SOURCES.includes(
|
||
settings.embeddingBackendSource,
|
||
)
|
||
? settings.embeddingBackendSource
|
||
: "openai";
|
||
|
||
return {
|
||
mode,
|
||
source,
|
||
apiUrl: normalizeOpenAICompatibleBaseUrl(
|
||
settings.embeddingBackendApiUrl,
|
||
autoSuffix,
|
||
),
|
||
apiKey: "",
|
||
model: String(
|
||
settings.embeddingBackendModel || BACKEND_DEFAULT_MODELS[source] || "",
|
||
).trim(),
|
||
autoSuffix,
|
||
timeoutMs: getConfiguredTimeoutMs(settings),
|
||
};
|
||
}
|
||
|
||
export function getSuggestedBackendModel(source) {
|
||
return BACKEND_DEFAULT_MODELS[source] || "text-embedding-3-small";
|
||
}
|
||
|
||
export function isBackendVectorConfig(config) {
|
||
return config?.mode === "backend";
|
||
}
|
||
|
||
export function isDirectVectorConfig(config) {
|
||
return config?.mode === "direct";
|
||
}
|
||
|
||
export function getVectorModelScope(config) {
|
||
if (!config) return "";
|
||
|
||
if (config?.mode === "authority" || config?.source === "authority-trivium") {
|
||
return [
|
||
"authority",
|
||
config.source || "authority-trivium",
|
||
config.embeddingMode || "direct",
|
||
config.embeddingSource || "direct",
|
||
normalizeOpenAICompatibleBaseUrl(config.apiUrl || "", config.autoSuffix),
|
||
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
|
||
config.model || "",
|
||
].join("|");
|
||
}
|
||
|
||
if (isDirectVectorConfig(config)) {
|
||
return [
|
||
"direct",
|
||
normalizeOpenAICompatibleBaseUrl(config.apiUrl, config.autoSuffix),
|
||
config.model || "",
|
||
].join("|");
|
||
}
|
||
|
||
return [
|
||
"backend",
|
||
config.source || "",
|
||
normalizeOpenAICompatibleBaseUrl(config.apiUrl, config.autoSuffix),
|
||
config.model || "",
|
||
].join("|");
|
||
}
|
||
|
||
export function validateVectorConfig(config) {
|
||
if (!config) {
|
||
return { valid: false, error: "未找到向量配置" };
|
||
}
|
||
|
||
if (config?.mode === "authority" || config?.source === "authority-trivium") {
|
||
if (!config.baseUrl) {
|
||
return { valid: false, error: "Authority Trivium 地址不可用" };
|
||
}
|
||
if (!config.model) {
|
||
return { valid: false, error: "请先填写 Embedding 模型（Authority 默认复用当前用户设置）" };
|
||
}
|
||
const authorityEmbeddingMode = String(config.embeddingMode || "direct").trim().toLowerCase();
|
||
const authorityEmbeddingSource = String(config.embeddingSource || "openai").trim().toLowerCase();
|
||
if (authorityEmbeddingMode === "backend") {
|
||
if (BACKEND_SOURCES_REQUIRING_API_URL.has(authorityEmbeddingSource) && !config.apiUrl) {
|
||
return { valid: false, error: "当前后端 Embedding 源需要 API 地址（Authority 默认复用当前用户设置）" };
|
||
}
|
||
return { valid: true, error: "" };
|
||
}
|
||
if (!config.apiUrl) {
|
||
return { valid: false, error: "请先填写 Embedding API 地址（Authority 默认复用当前用户设置）" };
|
||
}
|
||
return { valid: true, error: "" };
|
||
}
|
||
|
||
if (isDirectVectorConfig(config)) {
|
||
if (!config.apiUrl) {
|
||
return { valid: false, error: "请填写直连 Embedding API 地址" };
|
||
}
|
||
if (!config.model) {
|
||
return { valid: false, error: "请填写直连 Embedding 模型" };
|
||
}
|
||
return { valid: true, error: "" };
|
||
}
|
||
|
||
if (!config.model) {
|
||
return { valid: false, error: "请填写后端向量模型" };
|
||
}
|
||
|
||
if (BACKEND_SOURCES_REQUIRING_API_URL.has(config.source) && !config.apiUrl) {
|
||
return { valid: false, error: "当前后端向量源需要填写 API 地址" };
|
||
}
|
||
|
||
return { valid: true, error: "" };
|
||
}
|
||
|
||
export function buildNodeVectorText(node) {
|
||
const fields = node?.fields || {};
|
||
const preferredKeys = [
|
||
"summary",
|
||
"insight",
|
||
"title",
|
||
"name",
|
||
"state",
|
||
"traits",
|
||
"constraint",
|
||
"goal",
|
||
"participants",
|
||
"suggestion",
|
||
"status",
|
||
"scope",
|
||
];
|
||
|
||
const parts = [];
|
||
|
||
for (const key of preferredKeys) {
|
||
const value = fields[key];
|
||
if (value == null || value === "") continue;
|
||
if (Array.isArray(value)) {
|
||
if (value.length > 0) parts.push(value.join(", "));
|
||
} else if (typeof value === "object") {
|
||
parts.push(JSON.stringify(value));
|
||
} else {
|
||
parts.push(String(value));
|
||
}
|
||
}
|
||
|
||
for (const [key, value] of Object.entries(fields)) {
|
||
if (preferredKeys.includes(key) || value == null || value === "") continue;
|
||
if (key === "embedding") continue;
|
||
if (Array.isArray(value)) {
|
||
if (value.length > 0) parts.push(`${key}: ${value.join(", ")}`);
|
||
continue;
|
||
}
|
||
if (typeof value === "object") {
|
||
parts.push(`${key}: ${JSON.stringify(value)}`);
|
||
continue;
|
||
}
|
||
parts.push(`${key}: ${value}`);
|
||
}
|
||
|
||
const scope = normalizeMemoryScope(node?.scope);
|
||
const scopeText = describeMemoryScope(scope);
|
||
const regionPath = Array.isArray(scope?.regionPath) ? scope.regionPath : [];
|
||
const regionSecondary = Array.isArray(scope?.regionSecondary)
|
||
? scope.regionSecondary
|
||
: [];
|
||
if (scopeText) {
|
||
parts.push(`memory_scope: ${scopeText}`);
|
||
}
|
||
if (regionPath.length > 0) {
|
||
parts.push(`memory_region_path: ${regionPath.join(" / ")}`);
|
||
}
|
||
if (regionSecondary.length > 0) {
|
||
parts.push(`memory_region_secondary: ${regionSecondary.join(", ")}`);
|
||
}
|
||
|
||
return parts.join(" | ").trim();
|
||
}
|
||
|
||
export function buildNodeVectorHash(node, config) {
|
||
const text = buildNodeVectorText(node);
|
||
const seqEnd = node?.seqRange?.[1] ?? node?.seq ?? 0;
|
||
const payload = [
|
||
node?.id || "",
|
||
text,
|
||
String(seqEnd),
|
||
getVectorModelScope(config),
|
||
].join("::");
|
||
return stableHashString(payload);
|
||
}
|
||
|
||
function buildBackendSourceRequest(config) {
|
||
const body = {
|
||
source: config.source,
|
||
model: config.model,
|
||
};
|
||
|
||
if (BACKEND_SOURCES_REQUIRING_API_URL.has(config.source)) {
|
||
body.apiUrl = config.apiUrl;
|
||
}
|
||
|
||
if (config.source === "ollama") {
|
||
body.keep = false;
|
||
}
|
||
|
||
return body;
|
||
}
|
||
|
||
function getEligibleVectorNodes(graph, range = null) {
|
||
let nodes = getActiveNodes(graph).filter((node) => !node.archived);
|
||
|
||
if (range && Number.isFinite(range.start) && Number.isFinite(range.end)) {
|
||
const start = Math.min(range.start, range.end);
|
||
const end = Math.max(range.start, range.end);
|
||
nodes = nodes.filter((node) => {
|
||
const seqStart = node?.seqRange?.[0] ?? node?.seq ?? -1;
|
||
const seqEnd = node?.seqRange?.[1] ?? node?.seq ?? -1;
|
||
return seqEnd >= start && seqStart <= end;
|
||
});
|
||
}
|
||
|
||
return nodes.filter((node) => buildNodeVectorText(node).length > 0);
|
||
}
|
||
|
||
function buildDesiredVectorEntries(graph, config, range = null, diagnostics = null) {
|
||
const modelScope = getVectorModelScope(config);
|
||
let textBuildMs = 0;
|
||
let hashBuildMs = 0;
|
||
const entries = getEligibleVectorNodes(graph, range).map((node) => {
|
||
const textStartedAt = diagnostics ? nowMs() : 0;
|
||
const text = buildNodeVectorText(node);
|
||
if (diagnostics) {
|
||
textBuildMs += nowMs() - textStartedAt;
|
||
}
|
||
const seqEnd = node?.seqRange?.[1] ?? node?.seq ?? 0;
|
||
const hashStartedAt = diagnostics ? nowMs() : 0;
|
||
const payload = [node?.id || "", text, String(seqEnd), modelScope].join("::");
|
||
const hash = stableHashString(payload);
|
||
if (diagnostics) {
|
||
hashBuildMs += nowMs() - hashStartedAt;
|
||
}
|
||
return {
|
||
nodeId: node.id,
|
||
hash,
|
||
text,
|
||
index: seqEnd,
|
||
};
|
||
});
|
||
|
||
if (diagnostics && typeof diagnostics === "object") {
|
||
diagnostics.textBuildMs = roundMs(textBuildMs);
|
||
diagnostics.hashBuildMs = roundMs(hashBuildMs);
|
||
diagnostics.entryCount = entries.length;
|
||
}
|
||
|
||
return entries;
|
||
}
|
||
|
||
function computeVectorStats(graph, desiredEntries) {
|
||
const state = graph.vectorIndexState || {};
|
||
const desiredByNodeId = new Map(
|
||
desiredEntries.map((entry) => [entry.nodeId, entry]),
|
||
);
|
||
const nodeToHash = state.nodeToHash || {};
|
||
const hashToNodeId = state.hashToNodeId || {};
|
||
|
||
let indexed = 0;
|
||
let pending = 0;
|
||
|
||
for (const entry of desiredEntries) {
|
||
if (nodeToHash[entry.nodeId] === entry.hash) {
|
||
indexed++;
|
||
} else {
|
||
pending++;
|
||
}
|
||
}
|
||
|
||
let stale = 0;
|
||
for (const [nodeId, hash] of Object.entries(nodeToHash)) {
|
||
const desired = desiredByNodeId.get(nodeId);
|
||
if (!desired || desired.hash !== hash || hashToNodeId[hash] !== nodeId) {
|
||
stale++;
|
||
}
|
||
}
|
||
|
||
return {
|
||
total: desiredEntries.length,
|
||
indexed,
|
||
stale,
|
||
pending,
|
||
};
|
||
}
|
||
|
||
async function purgeVectorCollection(collectionId, signal) {
|
||
throwIfAborted(signal);
|
||
const response = await fetchWithTimeout(
|
||
"/api/vector/purge",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
signal,
|
||
body: JSON.stringify({ collectionId }),
|
||
},
|
||
getConfiguredTimeoutMs(),
|
||
);
|
||
|
||
if (!response.ok) {
|
||
const message = await response.text().catch(() => response.statusText);
|
||
throw new Error(message || `HTTP ${response.status}`);
|
||
}
|
||
}
|
||
|
||
async function deleteVectorHashes(collectionId, config, hashes, signal) {
|
||
if (!Array.isArray(hashes) || hashes.length === 0) return;
|
||
throwIfAborted(signal);
|
||
|
||
const response = await fetchWithTimeout(
|
||
"/api/vector/delete",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
signal,
|
||
body: JSON.stringify({
|
||
collectionId,
|
||
hashes,
|
||
...buildBackendSourceRequest(config),
|
||
}),
|
||
},
|
||
getConfiguredTimeoutMs(config),
|
||
);
|
||
|
||
if (!response.ok) {
|
||
const message = await response.text().catch(() => response.statusText);
|
||
throw new Error(message || `HTTP ${response.status}`);
|
||
}
|
||
}
|
||
|
||
export async function deleteBackendVectorHashesForRecovery(
|
||
collectionId,
|
||
config,
|
||
hashes,
|
||
signal = undefined,
|
||
) {
|
||
if (!collectionId || !isBackendVectorConfig(config)) return;
|
||
await deleteVectorHashes(collectionId, config, hashes, signal);
|
||
}
|
||
|
||
async function insertVectorEntries(collectionId, config, entries, signal) {
|
||
if (!Array.isArray(entries) || entries.length === 0) return;
|
||
throwIfAborted(signal);
|
||
|
||
const response = await fetchWithTimeout(
|
||
"/api/vector/insert",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
signal,
|
||
body: JSON.stringify({
|
||
collectionId,
|
||
items: entries.map((entry) => ({
|
||
hash: entry.hash,
|
||
text: entry.text,
|
||
index: entry.index,
|
||
})),
|
||
...buildBackendSourceRequest(config),
|
||
}),
|
||
},
|
||
getConfiguredTimeoutMs(config),
|
||
);
|
||
|
||
if (!response.ok) {
|
||
const message = await response.text().catch(() => response.statusText);
|
||
throw new Error(message || `HTTP ${response.status}`);
|
||
}
|
||
}
|
||
|
||
function resetVectorMappings(graph, config, chatId) {
|
||
graph.vectorIndexState.mode = config.mode;
|
||
graph.vectorIndexState.source = config.source || "";
|
||
graph.vectorIndexState.modelScope = getVectorModelScope(config);
|
||
graph.vectorIndexState.collectionId = buildVectorCollectionId(chatId);
|
||
graph.vectorIndexState.hashToNodeId = {};
|
||
graph.vectorIndexState.nodeToHash = {};
|
||
}
|
||
|
||
function markBackendVectorStateDirty(
|
||
graph,
|
||
config,
|
||
reason = "backend-query-failed",
|
||
warning = "后端向量查询失败，已标记待重建",
|
||
) {
|
||
if (!graph?.vectorIndexState || !isBackendVectorConfig(config)) {
|
||
return;
|
||
}
|
||
|
||
const state = graph.vectorIndexState;
|
||
const total = Math.max(
|
||
Number(state.lastStats?.total || 0),
|
||
Object.keys(state.nodeToHash || {}).length,
|
||
Object.keys(state.hashToNodeId || {}).length,
|
||
);
|
||
const previousIndexed = Number.isFinite(Number(state.lastStats?.indexed))
|
||
? Math.max(0, Math.floor(Number(state.lastStats.indexed)))
|
||
: 0;
|
||
const previousStale = Number.isFinite(Number(state.lastStats?.stale))
|
||
? Math.max(0, Math.floor(Number(state.lastStats.stale)))
|
||
: 0;
|
||
const previousPending = Number.isFinite(Number(state.lastStats?.pending))
|
||
? Math.max(0, Math.floor(Number(state.lastStats.pending)))
|
||
: 0;
|
||
|
||
state.mode = "backend";
|
||
state.source = config.source || state.source || "";
|
||
state.modelScope = getVectorModelScope(config) || state.modelScope || "";
|
||
state.collectionId = buildVectorCollectionId(graph?.historyState?.chatId);
|
||
state.dirty = true;
|
||
state.dirtyReason = String(reason || "backend-query-failed");
|
||
state.pendingRepairFromFloor = Number.isFinite(Number(state.pendingRepairFromFloor))
|
||
? Math.max(0, Math.floor(Number(state.pendingRepairFromFloor)))
|
||
: 0;
|
||
state.lastStats = {
|
||
total,
|
||
indexed: previousIndexed,
|
||
stale: Math.max(previousStale, total > 0 ? 1 : 0),
|
||
pending: total > 0 ? Math.max(1, previousPending) : previousPending,
|
||
};
|
||
state.lastWarning = String(warning || "后端向量查询失败，已标记待重建");
|
||
}
|
||
|
||
function markAuthorityVectorStateDirty(
|
||
graph,
|
||
config = {},
|
||
reason = "authority-trivium-failed",
|
||
warning = "Authority Trivium 索引失败，已标记待重建",
|
||
diagnostics = {},
|
||
) {
|
||
if (!graph?.vectorIndexState || !isAuthorityVectorConfig(config)) {
|
||
return;
|
||
}
|
||
const state = graph.vectorIndexState;
|
||
const total = Math.max(
|
||
Number(state.lastStats?.total || 0),
|
||
Object.keys(state.nodeToHash || {}).length,
|
||
Object.keys(state.hashToNodeId || {}).length,
|
||
);
|
||
const previousIndexed = Number.isFinite(Number(state.lastStats?.indexed))
|
||
? Math.max(0, Math.floor(Number(state.lastStats.indexed)))
|
||
: 0;
|
||
state.mode = "authority";
|
||
state.source = config.source || "authority-trivium";
|
||
state.modelScope = getVectorModelScope(config) || state.modelScope || "";
|
||
state.collectionId = state.collectionId || buildVectorCollectionId(graph?.historyState?.chatId);
|
||
state.dirty = true;
|
||
state.dirtyReason = String(reason || "authority-trivium-failed");
|
||
state.pendingRepairFromFloor = Number.isFinite(Number(state.pendingRepairFromFloor))
|
||
? Math.max(0, Math.floor(Number(state.pendingRepairFromFloor)))
|
||
: 0;
|
||
state.lastStats = {
|
||
total,
|
||
indexed: previousIndexed,
|
||
stale: total > 0 ? Math.max(1, Number(state.lastStats?.stale || 0)) : 0,
|
||
pending: total > 0 ? Math.max(1, Number(state.lastStats?.pending || 0)) : 0,
|
||
};
|
||
state.lastWarning = String(warning || "Authority Trivium 索引失败，已标记待重建");
|
||
const errorCategory = String(diagnostics.errorCategory || diagnostics.authorityErrorCategory || "").trim();
|
||
const errorDomain = String(diagnostics.errorDomain || diagnostics.authorityErrorDomain || "").trim();
|
||
if (errorCategory) state.lastErrorCategory = errorCategory;
|
||
if (errorDomain) state.lastErrorDomain = errorDomain;
|
||
}
|
||
|
||
function getErrorCategory(error = null) {
|
||
return String(error?.category || error?.errorCategory || "").trim();
|
||
}
|
||
|
||
function getErrorDomain(error = null, fallback = "") {
|
||
if (!error) return "";
|
||
if (error?.errorDomain) return String(error.errorDomain).trim();
|
||
if (getErrorCategory(error)) return fallback || "authority";
|
||
return fallback;
|
||
}
|
||
|
||
function getAuthorityDiagnosticsErrorPatch(error = null) {
|
||
const errorCategory = getErrorCategory(error);
|
||
const errorDomain = getErrorDomain(error, errorCategory ? "authority" : "");
|
||
return {
|
||
...(errorCategory ? { errorCategory, authorityErrorCategory: errorCategory } : {}),
|
||
...(errorDomain ? { errorDomain, authorityErrorDomain: errorDomain } : {}),
|
||
...(Number(error?.status || 0) > 0 ? { status: Number(error.status) } : {}),
|
||
};
|
||
}
|
||
|
||
function createEmbeddingProviderError(failures = 0) {
|
||
const count = Math.max(0, Math.floor(Number(failures) || 0));
|
||
const error = new Error(`Embedding provider failed for ${count} item(s)`);
|
||
error.errorCategory = "embedding-provider";
|
||
error.errorDomain = "embedding";
|
||
return error;
|
||
}
|
||
|
||
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
|
||
const nodesById = new Map((graph?.nodes || []).map((node) => [String(node?.id || ""), node]));
|
||
const entriesToEmbed = [];
|
||
for (const entry of entries || []) {
|
||
const node = nodesById.get(String(entry?.nodeId || ""));
|
||
const hasEmbedding = Array.isArray(node?.embedding) && node.embedding.length > 0;
|
||
if (node && !hasEmbedding) {
|
||
entriesToEmbed.push({ entry, node });
|
||
}
|
||
}
|
||
if (!entriesToEmbed.length) {
|
||
return { requested: 0, failures: 0, elapsedMs: 0 };
|
||
}
|
||
throwIfAborted(signal);
|
||
const startedAt = nowMs();
|
||
const embeddings = await embedBatch(
|
||
entriesToEmbed.map(({ entry }) => entry.text),
|
||
config,
|
||
{ signal },
|
||
);
|
||
let failures = 0;
|
||
for (let index = 0; index < entriesToEmbed.length; index++) {
|
||
const embedding = embeddings[index];
|
||
if (embedding) {
|
||
entriesToEmbed[index].node.embedding = Array.from(embedding);
|
||
} else {
|
||
failures += 1;
|
||
}
|
||
}
|
||
return {
|
||
requested: entriesToEmbed.length,
|
||
failures,
|
||
elapsedMs: nowMs() - startedAt,
|
||
};
|
||
}
|
||
|
||
export async function syncGraphVectorIndex(
|
||
graph,
|
||
config,
|
||
{
|
||
chatId = "",
|
||
purge = false,
|
||
force = false,
|
||
range = null,
|
||
signal = undefined,
|
||
triviumClient = undefined,
|
||
headerProvider = undefined,
|
||
fetchImpl = undefined,
|
||
} = {},
|
||
) {
|
||
if (!graph || !config) {
|
||
return {
|
||
insertedHashes: [],
|
||
stats: { total: 0, indexed: 0, stale: 0, pending: 0 },
|
||
timings: null,
|
||
};
|
||
}
|
||
throwIfAborted(signal);
|
||
|
||
const syncStartedAt = nowMs();
|
||
const syncMode = isAuthorityVectorConfig(config)
|
||
? "authority"
|
||
: isBackendVectorConfig(config)
|
||
? "backend"
|
||
: "direct";
|
||
|
||
const validation = validateVectorConfig(config);
|
||
if (!validation.valid) {
|
||
graph.vectorIndexState.lastWarning = validation.error;
|
||
graph.vectorIndexState.dirty = true;
|
||
graph.vectorIndexState.lastTimings = {
|
||
mode: syncMode,
|
||
validationError: validation.error,
|
||
totalMs: roundMs(nowMs() - syncStartedAt),
|
||
updatedAt: Date.now(),
|
||
};
|
||
return {
|
||
insertedHashes: [],
|
||
stats: graph.vectorIndexState.lastStats,
|
||
timings: graph.vectorIndexState.lastTimings,
|
||
};
|
||
}
|
||
|
||
const state = graph.vectorIndexState;
|
||
const collectionId = buildVectorCollectionId(
|
||
chatId || graph?.historyState?.chatId,
|
||
);
|
||
const desiredBuildDiagnostics = {};
|
||
const desiredBuildStartedAt = nowMs();
|
||
const desiredEntries = buildDesiredVectorEntries(
|
||
graph,
|
||
config,
|
||
range,
|
||
desiredBuildDiagnostics,
|
||
);
|
||
const desiredBuildMs = nowMs() - desiredBuildStartedAt;
|
||
const desiredByNodeId = new Map(
|
||
desiredEntries.map((entry) => [entry.nodeId, entry]),
|
||
);
|
||
const insertedHashes = [];
|
||
let backendPurgeMs = 0;
|
||
let backendDeleteMs = 0;
|
||
let backendInsertMs = 0;
|
||
let authorityPurgeMs = 0;
|
||
let authorityDeleteMs = 0;
|
||
let authorityUpsertMs = 0;
|
||
let authorityLinkMs = 0;
|
||
let authorityPurgeDiagnostics = null;
|
||
let authorityDeleteDiagnostics = null;
|
||
let authorityUpsertDiagnostics = null;
|
||
let authorityLinkDiagnostics = null;
|
||
let embedBatchMs = 0;
|
||
let deletedHashCount = 0;
|
||
let deletedNodeCount = 0;
|
||
let embeddingsRequested = 0;
|
||
const hasConcreteRange =
|
||
range && Number.isFinite(range.start) && Number.isFinite(range.end);
|
||
const rangedNodeIds = new Set(desiredEntries.map((entry) => entry.nodeId));
|
||
|
||
if (isAuthorityVectorConfig(config)) {
|
||
const effectiveChatId = chatId || graph?.historyState?.chatId || "";
|
||
const authorityOptions = {
|
||
namespace: collectionId,
|
||
collectionId,
|
||
chatId: effectiveChatId,
|
||
modelScope: getVectorModelScope(config),
|
||
revision: graph?.meta?.revision || graph?.revision || 0,
|
||
signal,
|
||
triviumClient,
|
||
headerProvider,
|
||
fetchImpl,
|
||
};
|
||
const scopeChanged =
|
||
state.mode !== "authority" ||
|
||
state.source !== (config.source || "authority-trivium") ||
|
||
state.modelScope !== getVectorModelScope(config) ||
|
||
state.collectionId !== collectionId;
|
||
const fullReset = purge || state.dirty || scopeChanged;
|
||
|
||
try {
|
||
if (fullReset) {
|
||
const embeddingResult = await ensureEntryEmbeddings(graph, desiredEntries, config, signal);
|
||
embeddingsRequested += embeddingResult.requested;
|
||
embedBatchMs += embeddingResult.elapsedMs;
|
||
if (embeddingResult.failures > 0) {
|
||
throw createEmbeddingProviderError(embeddingResult.failures);
|
||
}
|
||
let appliedViaBme = false;
|
||
if (config.bmeVectorApplyReady === true) {
|
||
try {
|
||
const applyStartedAt = nowMs();
|
||
const applyResult = await applyAuthorityBmeVectorManifest(
|
||
graph,
|
||
config,
|
||
desiredEntries,
|
||
authorityOptions,
|
||
);
|
||
authorityUpsertMs += nowMs() - applyStartedAt;
|
||
authorityUpsertDiagnostics = applyResult?.diagnostics || null;
|
||
authorityLinkDiagnostics = {
|
||
operation: "bmeVectorApply:links",
|
||
totalItems: Number(applyResult?.diagnostics?.linkItems || 0),
|
||
linked: Number(applyResult?.diagnostics?.linked || 0),
|
||
totalMs: 0,
|
||
};
|
||
resetVectorMappings(graph, config, effectiveChatId);
|
||
appliedViaBme = true;
|
||
} catch (applyError) {
|
||
if (isAbortError(applyError)) throw applyError;
|
||
console.warn("[ST-BME] BME 服务端向量 apply 失败，回退 Authority Trivium 旧路径:", applyError);
|
||
}
|
||
}
|
||
if (!appliedViaBme) {
|
||
const purgeStartedAt = nowMs();
|
||
const purgeResult = await purgeAuthorityTriviumNamespace(config, authorityOptions);
|
||
authorityPurgeMs += nowMs() - purgeStartedAt;
|
||
authorityPurgeDiagnostics = purgeResult?.diagnostics || null;
|
||
if (purgeResult?.truncated) {
|
||
throw new Error(`Authority Trivium purge truncated after ${purgeResult.pages || 0} page(s)`);
|
||
}
|
||
resetVectorMappings(graph, config, effectiveChatId);
|
||
const upsertStartedAt = nowMs();
|
||
const upsertResult = await upsertAuthorityTriviumEntries(
|
||
graph,
|
||
config,
|
||
desiredEntries,
|
||
authorityOptions,
|
||
);
|
||
authorityUpsertMs += nowMs() - upsertStartedAt;
|
||
authorityUpsertDiagnostics = upsertResult?.diagnostics || null;
|
||
}
|
||
for (const entry of desiredEntries) {
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
insertedHashes.push(entry.hash);
|
||
}
|
||
} else {
|
||
const nodeIdsToDelete = [];
|
||
const entriesToUpsert = [];
|
||
const queuedNodeIds = new Set();
|
||
|
||
if (force && hasConcreteRange) {
|
||
for (const entry of desiredEntries) {
|
||
entriesToUpsert.push(entry);
|
||
queuedNodeIds.add(entry.nodeId);
|
||
}
|
||
}
|
||
|
||
for (const [nodeId, hash] of Object.entries(state.nodeToHash || {})) {
|
||
if (hasConcreteRange && !rangedNodeIds.has(nodeId)) {
|
||
continue;
|
||
}
|
||
const desired = desiredByNodeId.get(nodeId);
|
||
if (!desired) {
|
||
nodeIdsToDelete.push(nodeId);
|
||
delete state.nodeToHash[nodeId];
|
||
delete state.hashToNodeId[hash];
|
||
} else if (desired.hash !== hash && !queuedNodeIds.has(nodeId)) {
|
||
entriesToUpsert.push(desired);
|
||
queuedNodeIds.add(nodeId);
|
||
delete state.hashToNodeId[hash];
|
||
}
|
||
}
|
||
|
||
for (const entry of desiredEntries) {
|
||
if (force && hasConcreteRange) continue;
|
||
if (state.nodeToHash[entry.nodeId] === entry.hash) continue;
|
||
if (queuedNodeIds.has(entry.nodeId)) continue;
|
||
entriesToUpsert.push(entry);
|
||
queuedNodeIds.add(entry.nodeId);
|
||
}
|
||
|
||
const embeddingResult = await ensureEntryEmbeddings(graph, entriesToUpsert, config, signal);
|
||
embeddingsRequested += embeddingResult.requested;
|
||
embedBatchMs += embeddingResult.elapsedMs;
|
||
if (embeddingResult.failures > 0) {
|
||
throw createEmbeddingProviderError(embeddingResult.failures);
|
||
}
|
||
deletedNodeCount = nodeIdsToDelete.length;
|
||
let appliedViaBme = false;
|
||
if (config.bmeVectorApplyReady === true && nodeIdsToDelete.length === 0) {
|
||
try {
|
||
const applyStartedAt = nowMs();
|
||
const applyResult = await applyAuthorityBmeVectorManifest(
|
||
graph,
|
||
config,
|
||
entriesToUpsert,
|
||
authorityOptions,
|
||
);
|
||
authorityUpsertMs += nowMs() - applyStartedAt;
|
||
authorityUpsertDiagnostics = applyResult?.diagnostics || null;
|
||
authorityLinkDiagnostics = {
|
||
operation: "bmeVectorApply:links",
|
||
totalItems: Number(applyResult?.diagnostics?.linkItems || 0),
|
||
linked: Number(applyResult?.diagnostics?.linked || 0),
|
||
totalMs: 0,
|
||
};
|
||
appliedViaBme = true;
|
||
} catch (applyError) {
|
||
if (isAbortError(applyError)) throw applyError;
|
||
console.warn("[ST-BME] BME 服务端向量 apply 失败，回退 Authority Trivium 旧路径:", applyError);
|
||
}
|
||
}
|
||
if (!appliedViaBme) {
|
||
const deleteStartedAt = nowMs();
|
||
const deleteResult = await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
|
||
authorityDeleteMs += nowMs() - deleteStartedAt;
|
||
authorityDeleteDiagnostics = deleteResult?.diagnostics || null;
|
||
const upsertStartedAt = nowMs();
|
||
const upsertResult = await upsertAuthorityTriviumEntries(
|
||
graph,
|
||
config,
|
||
entriesToUpsert,
|
||
authorityOptions,
|
||
);
|
||
authorityUpsertMs += nowMs() - upsertStartedAt;
|
||
authorityUpsertDiagnostics = upsertResult?.diagnostics || null;
|
||
}
|
||
|
||
for (const entry of entriesToUpsert) {
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
insertedHashes.push(entry.hash);
|
||
}
|
||
}
|
||
|
||
if (!authorityLinkDiagnostics || authorityLinkDiagnostics.operation !== "bmeVectorApply:links") {
|
||
const linkStartedAt = nowMs();
|
||
const linkResult = await syncAuthorityTriviumLinks(graph, config, authorityOptions);
|
||
authorityLinkMs += nowMs() - linkStartedAt;
|
||
authorityLinkDiagnostics = linkResult?.diagnostics || null;
|
||
}
|
||
|
||
for (const node of graph.nodes || []) {
|
||
if (Array.isArray(node.embedding) && node.embedding.length > 0) {
|
||
node.embedding = null;
|
||
}
|
||
}
|
||
state.mode = "authority";
|
||
state.source = config.source || "authority-trivium";
|
||
state.modelScope = getVectorModelScope(config);
|
||
state.collectionId = collectionId;
|
||
state.dirty = false;
|
||
state.lastWarning = "";
|
||
} catch (error) {
|
||
if (isAbortError(error)) throw error;
|
||
const message = error?.message || String(error) || "Authority Trivium 同步失败";
|
||
const errorCategory = getErrorCategory(error);
|
||
const errorDomain = getErrorDomain(error, errorCategory ? "authority" : "");
|
||
const dirtyReason = errorDomain === "embedding"
|
||
? "embedding-provider-sync-failed"
|
||
: "authority-trivium-sync-failed";
|
||
const warningPrefix = errorDomain === "embedding"
|
||
? "Embedding provider 同步失败"
|
||
: "Authority Trivium 同步失败";
|
||
markAuthorityVectorStateDirty(
|
||
graph,
|
||
config,
|
||
dirtyReason,
|
||
`${warningPrefix}（${message}），已标记待重建`,
|
||
{ errorCategory, errorDomain },
|
||
);
|
||
state.lastSyncAt = Date.now();
|
||
state.lastTimings = {
|
||
mode: syncMode,
|
||
success: false,
|
||
error: message,
|
||
...(errorCategory ? { errorCategory } : {}),
|
||
...(errorDomain ? { errorDomain } : {}),
|
||
...(errorCategory && errorDomain === "authority" ? { authorityErrorCategory: errorCategory, authorityErrorDomain: errorDomain } : {}),
|
||
desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length),
|
||
desiredBuildMs: roundMs(desiredBuildMs),
|
||
authorityPurgeMs: roundMs(authorityPurgeMs),
|
||
authorityDeleteMs: roundMs(authorityDeleteMs),
|
||
authorityUpsertMs: roundMs(authorityUpsertMs),
|
||
authorityLinkMs: roundMs(authorityLinkMs),
|
||
authorityDiagnostics: {
|
||
purge: authorityPurgeDiagnostics,
|
||
delete: authorityDeleteDiagnostics,
|
||
upsert: error?.authorityDiagnostics || authorityUpsertDiagnostics,
|
||
link: authorityLinkDiagnostics,
|
||
},
|
||
totalMs: roundMs(nowMs() - syncStartedAt),
|
||
updatedAt: Date.now(),
|
||
};
|
||
const result = {
|
||
insertedHashes,
|
||
stats: state.lastStats,
|
||
timings: state.lastTimings,
|
||
error: message,
|
||
...(errorCategory ? { errorCategory } : {}),
|
||
...(errorDomain ? { errorDomain } : {}),
|
||
};
|
||
if (config.failOpen === false) {
|
||
throw error;
|
||
}
|
||
return result;
|
||
}
|
||
} else if (isBackendVectorConfig(config)) {
|
||
const scopeChanged =
|
||
state.mode !== "backend" ||
|
||
state.source !== config.source ||
|
||
state.modelScope !== getVectorModelScope(config) ||
|
||
state.collectionId !== collectionId;
|
||
const fullReset =
|
||
purge || state.dirty || scopeChanged || (force && !hasConcreteRange);
|
||
|
||
if (fullReset) {
|
||
const purgeStartedAt = nowMs();
|
||
await purgeVectorCollection(collectionId, signal);
|
||
backendPurgeMs += nowMs() - purgeStartedAt;
|
||
resetVectorMappings(graph, config, chatId);
|
||
const insertStartedAt = nowMs();
|
||
await insertVectorEntries(collectionId, config, desiredEntries, signal);
|
||
backendInsertMs += nowMs() - insertStartedAt;
|
||
for (const entry of desiredEntries) {
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
insertedHashes.push(entry.hash);
|
||
}
|
||
} else {
|
||
const hashesToDelete = [];
|
||
const entriesToInsert = [];
|
||
|
||
if (force && hasConcreteRange) {
|
||
for (const entry of desiredEntries) {
|
||
const currentHash = state.nodeToHash[entry.nodeId];
|
||
if (currentHash) {
|
||
hashesToDelete.push(currentHash);
|
||
delete state.hashToNodeId[currentHash];
|
||
delete state.nodeToHash[entry.nodeId];
|
||
}
|
||
entriesToInsert.push(entry);
|
||
}
|
||
}
|
||
|
||
for (const [nodeId, hash] of Object.entries(state.nodeToHash)) {
|
||
if (hasConcreteRange && !rangedNodeIds.has(nodeId)) {
|
||
continue;
|
||
}
|
||
const desired = desiredByNodeId.get(nodeId);
|
||
if (!desired || desired.hash !== hash) {
|
||
hashesToDelete.push(hash);
|
||
delete state.nodeToHash[nodeId];
|
||
delete state.hashToNodeId[hash];
|
||
}
|
||
}
|
||
|
||
for (const entry of desiredEntries) {
|
||
if (force && hasConcreteRange) continue;
|
||
if (state.nodeToHash[entry.nodeId] === entry.hash) continue;
|
||
entriesToInsert.push(entry);
|
||
}
|
||
|
||
deletedHashCount = hashesToDelete.length;
|
||
const deleteStartedAt = nowMs();
|
||
await deleteVectorHashes(collectionId, config, hashesToDelete, signal);
|
||
backendDeleteMs += nowMs() - deleteStartedAt;
|
||
const insertStartedAt = nowMs();
|
||
await insertVectorEntries(collectionId, config, entriesToInsert, signal);
|
||
backendInsertMs += nowMs() - insertStartedAt;
|
||
|
||
for (const entry of entriesToInsert) {
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
insertedHashes.push(entry.hash);
|
||
}
|
||
}
|
||
|
||
for (const node of graph.nodes || []) {
|
||
if (Array.isArray(node.embedding) && node.embedding.length > 0) {
|
||
node.embedding = null;
|
||
}
|
||
}
|
||
} else {
|
||
const entriesToEmbed = [];
|
||
const hashByNodeId = {};
|
||
|
||
for (const entry of desiredEntries) {
|
||
hashByNodeId[entry.nodeId] = entry.hash;
|
||
const currentHash = state.nodeToHash?.[entry.nodeId];
|
||
const node = graph.nodes.find(
|
||
(candidate) => candidate.id === entry.nodeId,
|
||
);
|
||
const hasEmbedding =
|
||
Array.isArray(node?.embedding) && node.embedding.length > 0;
|
||
|
||
if (!force && !currentHash && hasEmbedding) {
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
continue;
|
||
}
|
||
|
||
if (force || purge || currentHash !== entry.hash || !hasEmbedding) {
|
||
entriesToEmbed.push(entry);
|
||
}
|
||
}
|
||
|
||
if (purge || state.mode !== "direct") {
|
||
resetVectorMappings(graph, config, chatId);
|
||
} else {
|
||
for (const [nodeId, hash] of Object.entries(state.nodeToHash || {})) {
|
||
if (hasConcreteRange && !rangedNodeIds.has(nodeId)) {
|
||
continue;
|
||
}
|
||
if (!hashByNodeId[nodeId]) {
|
||
delete state.nodeToHash[nodeId];
|
||
delete state.hashToNodeId[hash];
|
||
}
|
||
}
|
||
}
|
||
|
||
let directSyncHadFailures = false;
|
||
if (entriesToEmbed.length > 0) {
|
||
throwIfAborted(signal);
|
||
embeddingsRequested = entriesToEmbed.length;
|
||
const embedStartedAt = nowMs();
|
||
const embeddings = await embedBatch(
|
||
entriesToEmbed.map((entry) => entry.text),
|
||
config,
|
||
{ signal },
|
||
);
|
||
embedBatchMs += nowMs() - embedStartedAt;
|
||
|
||
for (let index = 0; index < entriesToEmbed.length; index++) {
|
||
const entry = entriesToEmbed[index];
|
||
const node = graph.nodes.find(
|
||
(candidate) => candidate.id === entry.nodeId,
|
||
);
|
||
if (!node) continue;
|
||
|
||
if (embeddings[index]) {
|
||
node.embedding = Array.from(embeddings[index]);
|
||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||
insertedHashes.push(entry.hash);
|
||
} else {
|
||
directSyncHadFailures = true;
|
||
}
|
||
}
|
||
}
|
||
|
||
state.mode = "direct";
|
||
state.source = "direct";
|
||
state.modelScope = getVectorModelScope(config);
|
||
state.collectionId = collectionId;
|
||
state.dirty = directSyncHadFailures;
|
||
state.lastWarning = directSyncHadFailures
|
||
? "部分节点 embedding 生成失败，向量索引仍待修复"
|
||
: "";
|
||
}
|
||
|
||
if (state.mode !== "direct") {
|
||
state.dirty = false;
|
||
state.lastWarning = "";
|
||
}
|
||
state.lastSyncAt = Date.now();
|
||
const statsBuildStartedAt = nowMs();
|
||
state.lastStats = computeVectorStats(
|
||
graph,
|
||
buildDesiredVectorEntries(graph, config),
|
||
);
|
||
const statsBuildMs = nowMs() - statsBuildStartedAt;
|
||
state.lastTimings = {
|
||
mode: syncMode,
|
||
desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length),
|
||
desiredBuildMs: roundMs(desiredBuildMs),
|
||
textBuildMs: Number(desiredBuildDiagnostics.textBuildMs || 0),
|
||
hashBuildMs: Number(desiredBuildDiagnostics.hashBuildMs || 0),
|
||
backendPurgeMs: roundMs(backendPurgeMs),
|
||
backendDeleteMs: roundMs(backendDeleteMs),
|
||
backendInsertMs: roundMs(backendInsertMs),
|
||
authorityPurgeMs: roundMs(authorityPurgeMs),
|
||
authorityDeleteMs: roundMs(authorityDeleteMs),
|
||
authorityUpsertMs: roundMs(authorityUpsertMs),
|
||
authorityLinkMs: roundMs(authorityLinkMs),
|
||
authorityDiagnostics: {
|
||
purge: authorityPurgeDiagnostics,
|
||
delete: authorityDeleteDiagnostics,
|
||
upsert: authorityUpsertDiagnostics,
|
||
link: authorityLinkDiagnostics,
|
||
},
|
||
embedBatchMs: roundMs(embedBatchMs),
|
||
statsBuildMs: roundMs(statsBuildMs),
|
||
deletedHashes: Math.max(0, Math.floor(deletedHashCount)),
|
||
deletedNodes: Math.max(0, Math.floor(deletedNodeCount)),
|
||
insertedEntries: insertedHashes.length,
|
||
embeddingsRequested: Math.max(0, Math.floor(embeddingsRequested)),
|
||
totalMs: roundMs(nowMs() - syncStartedAt),
|
||
updatedAt: Date.now(),
|
||
};
|
||
|
||
return {
|
||
insertedHashes,
|
||
stats: state.lastStats,
|
||
timings: state.lastTimings,
|
||
};
|
||
}
|
||
|
||
export async function findSimilarNodesByText(
|
||
graph,
|
||
text,
|
||
config,
|
||
topK = 10,
|
||
candidates = null,
|
||
signal = undefined,
|
||
) {
|
||
if (!text || !graph || !config) return [];
|
||
throwIfAborted(signal);
|
||
|
||
const candidateNodes = Array.isArray(candidates)
|
||
? candidates
|
||
: getEligibleVectorNodes(graph);
|
||
const searchStartedAt = nowMs();
|
||
const mode = isAuthorityVectorConfig(config)
|
||
? "authority"
|
||
: isDirectVectorConfig(config)
|
||
? "direct"
|
||
: "backend";
|
||
const recordSearchTimings = (patch = {}) => {
|
||
const state = graph?.vectorIndexState;
|
||
if (!state || typeof state !== "object" || Array.isArray(state)) return;
|
||
state.lastSearchTimings = {
|
||
...(state.lastSearchTimings &&
|
||
typeof state.lastSearchTimings === "object" &&
|
||
!Array.isArray(state.lastSearchTimings)
|
||
? state.lastSearchTimings
|
||
: {}),
|
||
mode,
|
||
queryLength: String(text || "").length,
|
||
candidateCount: candidateNodes.length,
|
||
topK: Math.max(1, Math.floor(Number(topK) || 1)),
|
||
...patch,
|
||
totalMs: roundMs(nowMs() - searchStartedAt),
|
||
updatedAt: Date.now(),
|
||
};
|
||
};
|
||
|
||
if (candidateNodes.length === 0) {
|
||
recordSearchTimings({
|
||
success: true,
|
||
reason: "no-candidates",
|
||
resultCount: 0,
|
||
});
|
||
return [];
|
||
}
|
||
|
||
if (isDirectVectorConfig(config)) {
|
||
const queryEmbedStartedAt = nowMs();
|
||
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||
if (!queryVec) {
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "direct-query-embed-empty",
|
||
queryEmbedMs: roundMs(queryEmbedMs),
|
||
resultCount: 0,
|
||
});
|
||
return [];
|
||
}
|
||
|
||
const localSearchStartedAt = nowMs();
|
||
const results = searchSimilar(
|
||
queryVec,
|
||
candidateNodes
|
||
.filter(
|
||
(node) => Array.isArray(node.embedding) && node.embedding.length > 0,
|
||
)
|
||
.map((node) => ({
|
||
nodeId: node.id,
|
||
embedding: node.embedding,
|
||
})),
|
||
topK,
|
||
);
|
||
recordSearchTimings({
|
||
success: true,
|
||
reason: "ok",
|
||
queryEmbedMs: roundMs(queryEmbedMs),
|
||
searchMs: roundMs(nowMs() - localSearchStartedAt),
|
||
resultCount: results.length,
|
||
});
|
||
return results;
|
||
}
|
||
|
||
const validation = validateVectorConfig(config);
|
||
if (!validation.valid) {
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "vector-config-invalid",
|
||
error: validation.error,
|
||
resultCount: 0,
|
||
});
|
||
return [];
|
||
}
|
||
|
||
if (isAuthorityVectorConfig(config)) {
|
||
const requestStartedAt = nowMs();
|
||
try {
|
||
const queryEmbedStartedAt = nowMs();
|
||
const queryVec = await embedText(text, config, { signal, isQuery: true });
|
||
const queryEmbedMs = nowMs() - queryEmbedStartedAt;
|
||
if (!queryVec) {
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "authority-query-embed-empty",
|
||
queryEmbedMs: roundMs(queryEmbedMs),
|
||
resultCount: 0,
|
||
});
|
||
return [];
|
||
}
|
||
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
||
const results = (
|
||
await searchAuthorityTriviumNodes(graph, text, config, {
|
||
namespace: graph.vectorIndexState?.collectionId,
|
||
collectionId: graph.vectorIndexState?.collectionId,
|
||
chatId: graph?.historyState?.chatId || "",
|
||
modelScope: getVectorModelScope(config),
|
||
topK,
|
||
candidateIds: candidateNodes.map((node) => node.id),
|
||
queryVector: Array.from(queryVec),
|
||
signal,
|
||
})
|
||
)
|
||
.filter((entry) => entry.nodeId && allowedIds.has(entry.nodeId))
|
||
.slice(0, topK);
|
||
recordSearchTimings({
|
||
success: true,
|
||
reason: "ok",
|
||
queryEmbedMs: roundMs(queryEmbedMs),
|
||
requestMs: roundMs(nowMs() - requestStartedAt),
|
||
resultCount: results.length,
|
||
});
|
||
return results;
|
||
} catch (error) {
|
||
if (isAbortError(error)) {
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "aborted",
|
||
error: error?.message || String(error),
|
||
});
|
||
throw error;
|
||
}
|
||
const message = error?.message || String(error) || "Authority Trivium 查询失败";
|
||
const errorPatch = getAuthorityDiagnosticsErrorPatch(error);
|
||
markAuthorityVectorStateDirty(
|
||
graph,
|
||
config,
|
||
"authority-trivium-query-failed",
|
||
`Authority Trivium 查询失败（${message}），已标记待重建`,
|
||
errorPatch,
|
||
);
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "authority-trivium-query-failed",
|
||
requestMs: roundMs(nowMs() - requestStartedAt),
|
||
error: message,
|
||
...errorPatch,
|
||
resultCount: 0,
|
||
});
|
||
if (config.failOpen === false) {
|
||
throw error;
|
||
}
|
||
return [];
|
||
}
|
||
}
|
||
|
||
try {
|
||
const requestStartedAt = nowMs();
|
||
const response = await fetchWithTimeout(
|
||
"/api/vector/query",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
signal,
|
||
body: JSON.stringify({
|
||
collectionId: graph.vectorIndexState.collectionId,
|
||
searchText: text,
|
||
topK,
|
||
threshold: 0,
|
||
...buildBackendSourceRequest(config),
|
||
}),
|
||
},
|
||
getConfiguredTimeoutMs(config),
|
||
);
|
||
const requestMs = nowMs() - requestStartedAt;
|
||
|
||
if (!response.ok) {
|
||
const errorText = await response.text().catch(() => response.statusText);
|
||
const message = errorText || response.statusText || `HTTP ${response.status}`;
|
||
console.warn("[ST-BME] 后端向量查询失败:", message);
|
||
markBackendVectorStateDirty(
|
||
graph,
|
||
config,
|
||
"backend-query-failed",
|
||
`后端向量查询失败（${message}），已标记待重建`,
|
||
);
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "backend-query-http-failed",
|
||
statusCode: Number(response.status || 0),
|
||
requestMs: roundMs(requestMs),
|
||
error: message,
|
||
resultCount: 0,
|
||
});
|
||
return [];
|
||
}
|
||
|
||
const parseStartedAt = nowMs();
|
||
const data = await response.json().catch(() => ({ hashes: [] }));
|
||
const parseMs = nowMs() - parseStartedAt;
|
||
const hashes = Array.isArray(data?.hashes) ? data.hashes : [];
|
||
const nodeIdByHash = graph.vectorIndexState?.hashToNodeId || {};
|
||
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
||
|
||
const results = hashes
|
||
.map((hash, index) => ({
|
||
nodeId: nodeIdByHash[hash],
|
||
score: Math.max(0.01, 1 - index / Math.max(1, hashes.length)),
|
||
}))
|
||
.filter((entry) => entry.nodeId && allowedIds.has(entry.nodeId))
|
||
.slice(0, topK);
|
||
recordSearchTimings({
|
||
success: true,
|
||
reason: "ok",
|
||
requestMs: roundMs(requestMs),
|
||
parseMs: roundMs(parseMs),
|
||
resultCount: results.length,
|
||
hashCount: hashes.length,
|
||
});
|
||
return results;
|
||
} catch (error) {
|
||
if (isAbortError(error)) {
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "aborted",
|
||
error: error?.message || String(error),
|
||
});
|
||
throw error;
|
||
}
|
||
const message = error?.message || String(error) || "后端向量查询失败";
|
||
markBackendVectorStateDirty(
|
||
graph,
|
||
config,
|
||
"backend-query-failed",
|
||
`后端向量查询失败（${message}），已标记待重建`,
|
||
);
|
||
recordSearchTimings({
|
||
success: false,
|
||
reason: "backend-query-exception",
|
||
error: message,
|
||
});
|
||
throw error;
|
||
}
|
||
}
|
||
|
||
export async function testVectorConnection(config, chatId = "connection-test") {
|
||
const validation = validateVectorConfig(config);
|
||
if (!validation.valid) {
|
||
return { success: false, dimensions: 0, error: validation.error };
|
||
}
|
||
|
||
if (isDirectVectorConfig(config) || isBackendVectorConfig(config)) {
|
||
try {
|
||
const vectors = await embedBatch(VECTOR_CONNECTION_PROBE_TEXTS, config, {
|
||
isQuery: true,
|
||
});
|
||
const firstVector = vectors.find((vector) => vector && vector.length > 0);
|
||
if (firstVector) {
|
||
if (isBackendVectorConfig(config)) {
|
||
const response = await fetchWithTimeout(
|
||
"/api/vector/query",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
body: JSON.stringify({
|
||
collectionId: buildVectorCollectionId(chatId),
|
||
searchText: VECTOR_CONNECTION_PROBE_TEXTS[0],
|
||
topK: 1,
|
||
threshold: 0,
|
||
...buildBackendSourceRequest(config),
|
||
}),
|
||
},
|
||
getConfiguredTimeoutMs(config),
|
||
);
|
||
const payload = await response.text().catch(() => "");
|
||
if (!response.ok) {
|
||
return {
|
||
success: false,
|
||
dimensions: firstVector.length,
|
||
error: payload || response.statusText,
|
||
batchCapable: true,
|
||
vectorStoreCapable: false,
|
||
mode: config.mode,
|
||
source: config.source || "backend",
|
||
};
|
||
}
|
||
}
|
||
return {
|
||
success: true,
|
||
dimensions: firstVector.length,
|
||
error: "",
|
||
batchCapable: true,
|
||
vectorStoreCapable: isBackendVectorConfig(config) ? true : undefined,
|
||
mode: config.mode,
|
||
source: config.source || "direct",
|
||
};
|
||
}
|
||
return {
|
||
success: false,
|
||
dimensions: 0,
|
||
error: "批量 Embedding API 返回空结果",
|
||
batchCapable: false,
|
||
mode: config.mode,
|
||
source: config.source || "direct",
|
||
};
|
||
} catch (error) {
|
||
return {
|
||
success: false,
|
||
dimensions: 0,
|
||
error: String(error),
|
||
batchCapable: false,
|
||
mode: config.mode,
|
||
source: config.source || "direct",
|
||
};
|
||
}
|
||
}
|
||
|
||
if (isAuthorityVectorConfig(config)) {
|
||
try {
|
||
return await testAuthorityTriviumConnection(config, {
|
||
collectionId: buildVectorCollectionId(chatId),
|
||
chatId,
|
||
});
|
||
} catch (error) {
|
||
return { success: false, dimensions: 0, error: String(error) };
|
||
}
|
||
}
|
||
|
||
return { success: false, dimensions: 0, error: "未知向量配置" };
|
||
}
|
||
export function getVectorIndexStats(graph) {
|
||
const state = graph?.vectorIndexState;
|
||
if (!state) {
|
||
return { total: 0, indexed: 0, stale: 0, pending: 0 };
|
||
}
|
||
return state.lastStats || { total: 0, indexed: 0, stale: 0, pending: 0 };
|
||
}
|
||
|
||
function normalizeModelOptions(items = [], { embeddingOnly = false } = {}) {
|
||
if (!Array.isArray(items)) return [];
|
||
|
||
const candidates = [];
|
||
for (const item of items) {
|
||
if (typeof item === "string") {
|
||
const id = item.trim();
|
||
if (id) candidates.push({ id, label: id, raw: item });
|
||
continue;
|
||
}
|
||
|
||
if (!item || typeof item !== "object") continue;
|
||
const id = String(
|
||
item.id || item.name || item.label || item.slug || item.value || "",
|
||
).trim();
|
||
const label = String(
|
||
item.label || item.name || item.id || item.slug || item.value || "",
|
||
).trim();
|
||
if (!id) continue;
|
||
|
||
if (
|
||
embeddingOnly &&
|
||
Array.isArray(item.endpoints) &&
|
||
!item.endpoints.includes("/v1/embeddings")
|
||
) {
|
||
continue;
|
||
}
|
||
|
||
candidates.push({ id, label: label || id, raw: item });
|
||
}
|
||
|
||
const embeddingRegex =
|
||
/(embed|embedding|bge|e5|gte|nomic|voyage|mxbai|jina|minilm)/i;
|
||
const embeddingTagged = candidates.filter(
|
||
(item) => embeddingRegex.test(item.id) || embeddingRegex.test(item.label),
|
||
);
|
||
const source = embeddingTagged.length > 0 ? embeddingTagged : candidates;
|
||
|
||
const seen = new Set();
|
||
return source
|
||
.filter((item) => {
|
||
if (seen.has(item.id)) return false;
|
||
seen.add(item.id);
|
||
return true;
|
||
})
|
||
.map(({ id, label }) => ({ id, label }));
|
||
}
|
||
|
||
async function fetchJsonEndpoint(url, { method = "POST" } = {}) {
|
||
const response = await fetchWithTimeout(url, {
|
||
method,
|
||
headers: getRequestHeaders({ omitContentType: true }),
|
||
});
|
||
|
||
const payload = await response.json().catch(() => []);
|
||
if (!response.ok) {
|
||
throw new Error(
|
||
(typeof payload === "object" && payload?.error) ||
|
||
response.statusText ||
|
||
`HTTP ${response.status}`,
|
||
);
|
||
}
|
||
|
||
return payload;
|
||
}
|
||
|
||
async function fetchBackendStatusModelList(source) {
|
||
const chatCompletionSource = BACKEND_STATUS_MODEL_SOURCES[source];
|
||
if (!chatCompletionSource) {
|
||
throw new Error("当前后端向量源暂不支持自动拉取模型，请手动填写");
|
||
}
|
||
|
||
const response = await fetchWithTimeout(
|
||
"/api/backends/chat-completions/status",
|
||
{
|
||
method: "POST",
|
||
headers: getRequestHeaders(),
|
||
body: JSON.stringify({
|
||
chat_completion_source: chatCompletionSource,
|
||
}),
|
||
},
|
||
);
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok || payload?.error) {
|
||
throw new Error(
|
||
payload?.message ||
|
||
payload?.error ||
|
||
response.statusText ||
|
||
`HTTP ${response.status}`,
|
||
);
|
||
}
|
||
|
||
return normalizeModelOptions(payload?.data || payload, {
|
||
embeddingOnly: false,
|
||
});
|
||
}
|
||
|
||
async function fetchOpenAICompatibleModelList(apiUrl, apiKey = "") {
|
||
const normalizedUrl = normalizeOpenAICompatibleBaseUrl(apiUrl);
|
||
if (!normalizedUrl) {
|
||
throw new Error("请先填写 API 地址");
|
||
}
|
||
|
||
const response = await fetchWithTimeout(`${normalizedUrl}/models`, {
|
||
method: "GET",
|
||
headers: {
|
||
...(apiKey ? { Authorization: `Bearer ${apiKey}` } : {}),
|
||
},
|
||
});
|
||
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok) {
|
||
throw new Error(
|
||
payload?.error?.message || payload?.message || response.statusText,
|
||
);
|
||
}
|
||
|
||
return normalizeModelOptions(payload?.data || payload, {
|
||
embeddingOnly: false,
|
||
});
|
||
}
|
||
|
||
async function fetchOllamaModelList(apiUrl) {
|
||
const normalizedUrl = normalizeOpenAICompatibleBaseUrl(apiUrl).replace(
|
||
/\/v1$/i,
|
||
"",
|
||
);
|
||
if (!normalizedUrl) {
|
||
throw new Error("请先填写 Ollama API 地址");
|
||
}
|
||
|
||
const response = await fetchWithTimeout(`${normalizedUrl}/api/tags`, {
|
||
method: "GET",
|
||
});
|
||
const payload = await response.json().catch(() => ({}));
|
||
if (!response.ok) {
|
||
throw new Error(payload?.error || payload?.message || response.statusText);
|
||
}
|
||
|
||
return normalizeModelOptions(
|
||
Array.isArray(payload?.models)
|
||
? payload.models.map((item) => ({
|
||
id: item?.model || item?.name,
|
||
name: item?.model || item?.name,
|
||
}))
|
||
: [],
|
||
{ embeddingOnly: false },
|
||
);
|
||
}
|
||
|
||
export async function fetchAvailableEmbeddingModels(config) {
|
||
const validation = validateVectorConfig(config);
|
||
if (!validation.valid) {
|
||
return { success: false, models: [], error: validation.error };
|
||
}
|
||
|
||
try {
|
||
if (isAuthorityVectorConfig(config)) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "Authority Trivium 使用服务端索引配置，无需拉取 Embedding 模型",
|
||
};
|
||
}
|
||
|
||
if (isDirectVectorConfig(config)) {
|
||
const models = normalizeModelOptions(
|
||
await fetchOpenAICompatibleModelList(config.apiUrl, config.apiKey),
|
||
);
|
||
if (models.length === 0) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "未拉取到可用 Embedding 模型",
|
||
};
|
||
}
|
||
return { success: true, models, error: "" };
|
||
}
|
||
|
||
if (config.source === "ollama") {
|
||
const models = await fetchOllamaModelList(config.apiUrl);
|
||
if (models.length === 0) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "未拉取到可用 Ollama 模型",
|
||
};
|
||
}
|
||
return { success: true, models, error: "" };
|
||
}
|
||
|
||
if (MODEL_LIST_ENDPOINTS[config.source]) {
|
||
const payload = await fetchJsonEndpoint(
|
||
MODEL_LIST_ENDPOINTS[config.source],
|
||
);
|
||
const models = normalizeModelOptions(payload, {
|
||
embeddingOnly: config.source === "electronhub",
|
||
});
|
||
if (models.length === 0) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "未拉取到可用 Embedding 模型",
|
||
};
|
||
}
|
||
return { success: true, models, error: "" };
|
||
}
|
||
|
||
if (BACKEND_STATUS_MODEL_SOURCES[config.source]) {
|
||
const models = await fetchBackendStatusModelList(config.source);
|
||
if (models.length === 0) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "未拉取到可用 Embedding 模型",
|
||
};
|
||
}
|
||
return { success: true, models, error: "" };
|
||
}
|
||
|
||
if (config.apiUrl) {
|
||
const models = normalizeModelOptions(
|
||
await fetchOpenAICompatibleModelList(config.apiUrl),
|
||
);
|
||
if (models.length === 0) {
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "未拉取到可用 Embedding 模型",
|
||
};
|
||
}
|
||
return { success: true, models, error: "" };
|
||
}
|
||
|
||
return {
|
||
success: false,
|
||
models: [],
|
||
error: "当前后端向量源暂不支持自动拉取模型，请手动填写",
|
||
};
|
||
} catch (error) {
|
||
return { success: false, models: [], error: String(error) };
|
||
}
|
||
}
|