mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-06-13 18:31:16 +08:00
feat(authority): add Trivium vector primary adapter
This commit is contained in:
23
index.js
23
index.js
@@ -358,8 +358,10 @@ import {
|
||||
fetchAvailableEmbeddingModels,
|
||||
getVectorConfigFromSettings,
|
||||
getVectorIndexStats,
|
||||
isAuthorityVectorConfig,
|
||||
isBackendVectorConfig,
|
||||
isDirectVectorConfig,
|
||||
normalizeAuthorityVectorConfig,
|
||||
syncGraphVectorIndex,
|
||||
testVectorConnection,
|
||||
validateVectorConfig,
|
||||
@@ -5744,6 +5746,18 @@ function getPlannerRecallTimeoutMs() {
|
||||
|
||||
function getEmbeddingConfig(mode = null) {
|
||||
const settings = getSettings();
|
||||
if (!mode) {
|
||||
const authorityRuntime = getAuthorityRuntimeSnapshot(settings);
|
||||
const vectorMode = String(settings.authorityVectorMode || "auto-primary");
|
||||
if (
|
||||
settings.authorityTriviumPrimary !== false &&
|
||||
vectorMode !== "off" &&
|
||||
vectorMode !== "local-fallback" &&
|
||||
authorityRuntime.capability.triviumPrimaryReady
|
||||
) {
|
||||
return normalizeAuthorityVectorConfig(settings, buildAuthorityGraphStoreOptions(settings));
|
||||
}
|
||||
}
|
||||
return getVectorConfigFromSettings(
|
||||
mode ? { ...settings, embeddingTransportMode: mode } : settings,
|
||||
);
|
||||
@@ -13676,7 +13690,15 @@ async function syncVectorState({
|
||||
purge,
|
||||
range,
|
||||
signal,
|
||||
headerProvider:
|
||||
typeof getRequestHeaders === "function" ? () => getRequestHeaders() : null,
|
||||
});
|
||||
if (result?.error) {
|
||||
setLastVectorStatus("向量待修复", result.error, "warning", {
|
||||
syncRuntime: true,
|
||||
});
|
||||
return result;
|
||||
}
|
||||
setLastVectorStatus(
|
||||
"向量完成",
|
||||
`${scopeLabel} · indexed ${result.stats?.indexed ?? 0} · pending ${result.stats?.pending ?? 0}`,
|
||||
@@ -20078,6 +20100,7 @@ async function onRebuildVectorIndex(range = null) {
|
||||
ensureGraphMutationReady,
|
||||
finishStageAbortController,
|
||||
getEmbeddingConfig,
|
||||
isAuthorityVectorConfig,
|
||||
isBackendVectorConfig,
|
||||
refreshPanelLiveState,
|
||||
saveGraphToChat,
|
||||
|
||||
174
tests/authority-vector-primary.mjs
Normal file
174
tests/authority-vector-primary.mjs
Normal file
@@ -0,0 +1,174 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { addEdge, addNode, createEdge, createEmptyGraph, createNode } from "../graph/graph.js";
|
||||
import {
|
||||
installResolveHooks,
|
||||
toDataModuleUrl,
|
||||
} from "./helpers/register-hooks-compat.mjs";
|
||||
|
||||
installResolveHooks([
|
||||
{
|
||||
specifiers: ["../../../../../script.js"],
|
||||
url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"),
|
||||
},
|
||||
{
|
||||
specifiers: ["../../../../extensions.js"],
|
||||
url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"),
|
||||
},
|
||||
]);
|
||||
|
||||
const {
|
||||
findSimilarNodesByText,
|
||||
isAuthorityVectorConfig,
|
||||
normalizeAuthorityVectorConfig,
|
||||
syncGraphVectorIndex,
|
||||
} = await import("../vector/vector-index.js");
|
||||
|
||||
function createAuthorityVectorGraph() {
|
||||
const graph = createEmptyGraph();
|
||||
graph.historyState.chatId = "chat-authority-vector";
|
||||
const first = createNode({
|
||||
type: "event",
|
||||
fields: { summary: "Alice finds the silver key" },
|
||||
seq: 1,
|
||||
});
|
||||
first.id = "node-a";
|
||||
first.embedding = [0.1, 0.2];
|
||||
const second = createNode({
|
||||
type: "event",
|
||||
fields: { summary: "Bob guards the archive door" },
|
||||
seq: 2,
|
||||
});
|
||||
second.id = "node-b";
|
||||
second.embedding = [0.2, 0.3];
|
||||
addNode(graph, first);
|
||||
addNode(graph, second);
|
||||
addEdge(
|
||||
graph,
|
||||
createEdge({
|
||||
fromId: first.id,
|
||||
toId: second.id,
|
||||
relation: "related",
|
||||
strength: 0.75,
|
||||
}),
|
||||
);
|
||||
return { graph, first, second };
|
||||
}
|
||||
|
||||
function createMockTriviumClient({ failBulkUpsert = false } = {}) {
|
||||
const calls = [];
|
||||
return {
|
||||
calls,
|
||||
async purge(payload) {
|
||||
calls.push(["purge", payload]);
|
||||
return { ok: true };
|
||||
},
|
||||
async bulkUpsert(payload) {
|
||||
calls.push(["bulkUpsert", payload]);
|
||||
if (failBulkUpsert) {
|
||||
throw new Error("trivium-down");
|
||||
}
|
||||
return { ok: true, upserted: payload.items?.length || 0 };
|
||||
},
|
||||
async deleteMany(payload) {
|
||||
calls.push(["deleteMany", payload]);
|
||||
return { ok: true };
|
||||
},
|
||||
async linkMany(payload) {
|
||||
calls.push(["linkMany", payload]);
|
||||
return { ok: true, linked: payload.links?.length || 0 };
|
||||
},
|
||||
async search(payload) {
|
||||
calls.push(["search", payload]);
|
||||
return {
|
||||
results: [
|
||||
{ nodeId: "node-b", score: 0.91 },
|
||||
{ nodeId: "node-outside", score: 0.88 },
|
||||
],
|
||||
};
|
||||
},
|
||||
async stat(payload) {
|
||||
calls.push(["stat", payload]);
|
||||
return { ok: true };
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
const config = normalizeAuthorityVectorConfig({
|
||||
authorityBaseUrl: "/api/plugins/authority",
|
||||
authorityVectorSyncChunkSize: 1,
|
||||
authorityVectorFailOpen: true,
|
||||
});
|
||||
assert.equal(isAuthorityVectorConfig(config), true);
|
||||
|
||||
{
|
||||
const { graph, first, second } = createAuthorityVectorGraph();
|
||||
const triviumClient = createMockTriviumClient();
|
||||
const result = await syncGraphVectorIndex(graph, config, {
|
||||
chatId: "chat-authority-vector",
|
||||
purge: true,
|
||||
triviumClient,
|
||||
});
|
||||
|
||||
assert.equal(graph.vectorIndexState.mode, "authority");
|
||||
assert.equal(graph.vectorIndexState.source, "authority-trivium");
|
||||
assert.equal(graph.vectorIndexState.dirty, false);
|
||||
assert.equal(graph.vectorIndexState.lastWarning, "");
|
||||
assert.equal(result.insertedHashes.length, 2);
|
||||
assert.equal(result.stats.indexed, 2);
|
||||
assert.equal(result.stats.pending, 0);
|
||||
assert.equal(first.embedding, null);
|
||||
assert.equal(second.embedding, null);
|
||||
assert.equal(triviumClient.calls.filter(([name]) => name === "purge").length, 1);
|
||||
const upserts = triviumClient.calls.filter(([name]) => name === "bulkUpsert");
|
||||
assert.equal(upserts.length, 2);
|
||||
assert.deepEqual(
|
||||
upserts.flatMap(([, payload]) => payload.items.map((item) => item.nodeId)).sort(),
|
||||
["node-a", "node-b"],
|
||||
);
|
||||
const linkCall = triviumClient.calls.find(([name]) => name === "linkMany");
|
||||
assert.equal(linkCall?.[1]?.links?.[0]?.fromId, "node-a");
|
||||
assert.equal(linkCall?.[1]?.links?.[0]?.toId, "node-b");
|
||||
}
|
||||
|
||||
{
|
||||
const { graph, first, second } = createAuthorityVectorGraph();
|
||||
const triviumClient = createMockTriviumClient();
|
||||
const queryConfig = { ...config, triviumClient };
|
||||
await syncGraphVectorIndex(graph, queryConfig, {
|
||||
chatId: "chat-authority-vector",
|
||||
purge: true,
|
||||
triviumClient,
|
||||
});
|
||||
|
||||
const results = await findSimilarNodesByText(
|
||||
graph,
|
||||
"archive door",
|
||||
queryConfig,
|
||||
5,
|
||||
[first, second],
|
||||
);
|
||||
|
||||
assert.deepEqual(results, [{ nodeId: "node-b", score: 0.91 }]);
|
||||
const searchCall = triviumClient.calls.find(([name]) => name === "search");
|
||||
assert.deepEqual(searchCall?.[1]?.candidateIds.sort(), ["node-a", "node-b"]);
|
||||
assert.equal(graph.vectorIndexState.lastSearchTimings.mode, "authority");
|
||||
assert.equal(graph.vectorIndexState.lastSearchTimings.success, true);
|
||||
}
|
||||
|
||||
{
|
||||
const { graph } = createAuthorityVectorGraph();
|
||||
const triviumClient = createMockTriviumClient({ failBulkUpsert: true });
|
||||
const result = await syncGraphVectorIndex(graph, config, {
|
||||
chatId: "chat-authority-vector",
|
||||
purge: true,
|
||||
triviumClient,
|
||||
});
|
||||
|
||||
assert.match(result.error, /trivium-down/);
|
||||
assert.equal(graph.vectorIndexState.mode, "authority");
|
||||
assert.equal(graph.vectorIndexState.dirty, true);
|
||||
assert.equal(graph.vectorIndexState.dirtyReason, "authority-trivium-sync-failed");
|
||||
assert.match(graph.vectorIndexState.lastWarning, /Authority Trivium 同步失败/);
|
||||
}
|
||||
|
||||
console.log("authority-vector-primary tests passed");
|
||||
@@ -589,7 +589,10 @@ export async function onRebuildVectorIndexController(runtime, range = null) {
|
||||
try {
|
||||
const result = await runtime.syncVectorState({
|
||||
force: true,
|
||||
purge: runtime.isBackendVectorConfig(config) && !range,
|
||||
purge:
|
||||
!range &&
|
||||
(runtime.isBackendVectorConfig(config) ||
|
||||
runtime.isAuthorityVectorConfig?.(config)),
|
||||
range,
|
||||
signal: vectorController.signal,
|
||||
});
|
||||
|
||||
345
vector/authority-vector-primary-adapter.js
Normal file
345
vector/authority-vector-primary-adapter.js
Normal file
@@ -0,0 +1,345 @@
|
||||
import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js";
|
||||
|
||||
export const AUTHORITY_VECTOR_MODE = "authority";
|
||||
export const AUTHORITY_VECTOR_SOURCE = "authority-trivium";
|
||||
|
||||
const AUTHORITY_TRIVIUM_ENDPOINT = "/v1/trivium";
|
||||
const DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE = 1000;
|
||||
const MAX_AUTHORITY_VECTOR_CHUNK_SIZE = 2000;
|
||||
|
||||
function clampInteger(value, fallback, min, max) {
|
||||
const parsed = Number(value);
|
||||
if (!Number.isFinite(parsed)) return fallback;
|
||||
return Math.min(max, Math.max(min, Math.trunc(parsed)));
|
||||
}
|
||||
|
||||
function toArray(value) {
|
||||
return Array.isArray(value) ? value : [];
|
||||
}
|
||||
|
||||
function clonePlain(value, fallbackValue = null) {
|
||||
if (value == null) return fallbackValue;
|
||||
if (typeof globalThis.structuredClone === "function") {
|
||||
try {
|
||||
return globalThis.structuredClone(value);
|
||||
} catch {
|
||||
}
|
||||
}
|
||||
try {
|
||||
return JSON.parse(JSON.stringify(value));
|
||||
} catch {
|
||||
return fallbackValue;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeRecordId(value) {
|
||||
return String(value ?? "").trim();
|
||||
}
|
||||
|
||||
function throwIfAborted(signal) {
|
||||
if (signal?.aborted) {
|
||||
throw signal.reason instanceof Error
|
||||
? signal.reason
|
||||
: Object.assign(new Error("操作已终止"), { name: "AbortError" });
|
||||
}
|
||||
}
|
||||
|
||||
function getNodeFieldText(node = {}, keys = []) {
|
||||
const fields = node?.fields && typeof node.fields === "object" ? node.fields : {};
|
||||
for (const key of keys) {
|
||||
const value = fields[key];
|
||||
if (typeof value === "string" && value.trim()) return value.trim();
|
||||
}
|
||||
return "";
|
||||
}
|
||||
|
||||
function normalizeSearchResults(payload = null) {
|
||||
const rows = Array.isArray(payload)
|
||||
? payload
|
||||
: Array.isArray(payload?.results)
|
||||
? payload.results
|
||||
: Array.isArray(payload?.hits)
|
||||
? payload.hits
|
||||
: Array.isArray(payload?.items)
|
||||
? payload.items
|
||||
: Array.isArray(payload?.data)
|
||||
? payload.data
|
||||
: [];
|
||||
return rows
|
||||
.map((item, index) => {
|
||||
const nodeId = normalizeRecordId(
|
||||
item?.nodeId || item?.externalId || item?.id || item?.payload?.nodeId,
|
||||
);
|
||||
if (!nodeId) return null;
|
||||
const rawScore = Number(item?.score ?? item?.similarity ?? item?.rankScore);
|
||||
const distance = Number(item?.distance);
|
||||
const score = Number.isFinite(rawScore)
|
||||
? rawScore
|
||||
: Number.isFinite(distance)
|
||||
? 1 / (1 + Math.max(0, distance))
|
||||
: Math.max(0.01, 1 - index / Math.max(1, rows.length));
|
||||
return { nodeId, score };
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function buildAuthorityNodePayload(node = {}, entry = {}, { chatId = "", modelScope = "", revision = 0 } = {}) {
|
||||
const scope = node?.scope && typeof node.scope === "object" ? node.scope : {};
|
||||
const seqRange = Array.isArray(node?.seqRange) ? node.seqRange : [node?.seq ?? 0, node?.seq ?? 0];
|
||||
return {
|
||||
chatId,
|
||||
nodeId: normalizeRecordId(node?.id || entry?.nodeId),
|
||||
type: String(node?.type || ""),
|
||||
archived: Boolean(node?.archived),
|
||||
seqStart: Number(seqRange[0] ?? node?.seq ?? 0) || 0,
|
||||
seqEnd: Number(seqRange[1] ?? node?.seq ?? 0) || 0,
|
||||
sourceFloor: Number(seqRange[1] ?? node?.seq ?? 0) || 0,
|
||||
importance: Number(node?.importance ?? 0) || 0,
|
||||
updatedAt: Number(node?.updatedAt || Date.now()) || Date.now(),
|
||||
scopeLayer: String(scope.layer || ""),
|
||||
scopeOwnerType: String(scope.ownerType || ""),
|
||||
scopeOwnerId: String(scope.ownerId || ""),
|
||||
scopeOwnerName: String(scope.ownerName || ""),
|
||||
scopeBucket: String(scope.bucket || ""),
|
||||
regionKey: String(scope.regionKey || node?.regionKey || ""),
|
||||
storySegmentId: String(node?.storySegmentId || node?.storyTime?.segmentId || ""),
|
||||
storyTimeLabel: String(node?.storyTime?.label || ""),
|
||||
title: getNodeFieldText(node, ["title"]),
|
||||
name: getNodeFieldText(node, ["name"]),
|
||||
summaryPreview: getNodeFieldText(node, ["summary", "insight", "state"]),
|
||||
contentHash: String(entry?.hash || ""),
|
||||
modelScope,
|
||||
revision: Math.max(0, Math.floor(Number(revision) || 0)),
|
||||
};
|
||||
}
|
||||
|
||||
function buildAuthorityVectorItems(graph, entries = [], options = {}) {
|
||||
const nodesById = new Map(toArray(graph?.nodes).map((node) => [String(node?.id || ""), node]));
|
||||
return toArray(entries)
|
||||
.map((entry) => {
|
||||
const nodeId = normalizeRecordId(entry?.nodeId);
|
||||
const node = nodesById.get(nodeId);
|
||||
if (!node) return null;
|
||||
const payload = buildAuthorityNodePayload(node, entry, options);
|
||||
return {
|
||||
id: nodeId,
|
||||
externalId: nodeId,
|
||||
nodeId,
|
||||
text: String(entry?.text || ""),
|
||||
index: Number(entry?.index || 0) || 0,
|
||||
hash: String(entry?.hash || ""),
|
||||
payload,
|
||||
};
|
||||
})
|
||||
.filter((item) => item?.nodeId && item.text);
|
||||
}
|
||||
|
||||
function buildAuthorityLinkItems(graph, { chatId = "", revision = 0 } = {}) {
|
||||
return toArray(graph?.edges)
|
||||
.filter((edge) => edge && !edge.invalidAt && !edge.expiredAt && !edge.deletedAt)
|
||||
.map((edge) => {
|
||||
const fromId = normalizeRecordId(edge.fromId || edge.sourceId || edge.from);
|
||||
const toId = normalizeRecordId(edge.toId || edge.targetId || edge.to);
|
||||
if (!fromId || !toId) return null;
|
||||
return {
|
||||
id: normalizeRecordId(edge.id) || `${fromId}->${toId}:${String(edge.relation || "related")}`,
|
||||
fromId,
|
||||
toId,
|
||||
relation: String(edge.relation || edge.type || "related"),
|
||||
weight: Number(edge.strength ?? edge.weight ?? 1) || 1,
|
||||
payload: {
|
||||
chatId,
|
||||
edgeId: normalizeRecordId(edge.id),
|
||||
relation: String(edge.relation || edge.type || "related"),
|
||||
strength: Number(edge.strength ?? edge.weight ?? 1) || 1,
|
||||
edgeType: String(edge.type || edge.edgeType || ""),
|
||||
revision: Math.max(0, Math.floor(Number(revision) || 0)),
|
||||
raw: clonePlain(edge, {}),
|
||||
},
|
||||
};
|
||||
})
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
export function isAuthorityVectorConfig(config = null) {
|
||||
return config?.mode === AUTHORITY_VECTOR_MODE || config?.source === AUTHORITY_VECTOR_SOURCE;
|
||||
}
|
||||
|
||||
export function normalizeAuthorityVectorConfig(settings = {}, overrides = {}) {
|
||||
const source = settings && typeof settings === "object" && !Array.isArray(settings) ? settings : {};
|
||||
return {
|
||||
mode: AUTHORITY_VECTOR_MODE,
|
||||
source: AUTHORITY_VECTOR_SOURCE,
|
||||
baseUrl: normalizeAuthorityBaseUrl(source.authorityBaseUrl ?? source.baseUrl),
|
||||
model: String(source.embeddingBackendModel || source.embeddingModel || "").trim(),
|
||||
chunkSize: clampInteger(
|
||||
source.authorityVectorSyncChunkSize ?? source.chunkSize,
|
||||
DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE,
|
||||
1,
|
||||
MAX_AUTHORITY_VECTOR_CHUNK_SIZE,
|
||||
),
|
||||
timeoutMs: Math.max(0, Number(source.timeoutMs || 0) || 0),
|
||||
failOpen: source.authorityVectorFailOpen !== false && source.failOpen !== false,
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
export class AuthorityTriviumHttpClient {
|
||||
constructor(options = {}) {
|
||||
this.baseUrl = normalizeAuthorityBaseUrl(options.baseUrl);
|
||||
this.fetchImpl = options.fetchImpl || (typeof fetch === "function" ? fetch.bind(globalThis) : null);
|
||||
this.headerProvider = typeof options.headerProvider === "function" ? options.headerProvider : null;
|
||||
}
|
||||
|
||||
async request(action, payload = {}) {
|
||||
if (typeof this.fetchImpl !== "function") {
|
||||
throw new Error("Authority Trivium fetch unavailable");
|
||||
}
|
||||
const response = await this.fetchImpl(`${this.baseUrl}${AUTHORITY_TRIVIUM_ENDPOINT}`, {
|
||||
method: "POST",
|
||||
headers: {
|
||||
Accept: "application/json",
|
||||
"Content-Type": "application/json",
|
||||
...(this.headerProvider ? this.headerProvider() || {} : {}),
|
||||
},
|
||||
body: JSON.stringify({ action, ...payload }),
|
||||
});
|
||||
if (!response?.ok) {
|
||||
throw new Error(`Authority Trivium HTTP ${response?.status || "unknown"}`);
|
||||
}
|
||||
return await response.json().catch(() => ({}));
|
||||
}
|
||||
|
||||
async purge(payload = {}) {
|
||||
return await this.request("purge", payload);
|
||||
}
|
||||
|
||||
async bulkUpsert(payload = {}) {
|
||||
return await this.request("bulkUpsert", payload);
|
||||
}
|
||||
|
||||
async deleteMany(payload = {}) {
|
||||
return await this.request("deleteMany", payload);
|
||||
}
|
||||
|
||||
async linkMany(payload = {}) {
|
||||
return await this.request("linkMany", payload);
|
||||
}
|
||||
|
||||
async search(payload = {}) {
|
||||
return await this.request("search", payload);
|
||||
}
|
||||
|
||||
async stat(payload = {}) {
|
||||
return await this.request("stat", payload);
|
||||
}
|
||||
}
|
||||
|
||||
export function createAuthorityTriviumClient(config = {}, options = {}) {
|
||||
const injected = options.triviumClient || config.triviumClient || globalThis.__stBmeAuthorityTriviumClient;
|
||||
if (injected) return injected;
|
||||
return new AuthorityTriviumHttpClient({
|
||||
baseUrl: config.baseUrl,
|
||||
fetchImpl: options.fetchImpl || config.fetchImpl,
|
||||
headerProvider: options.headerProvider || config.headerProvider,
|
||||
});
|
||||
}
|
||||
|
||||
async function callClient(client, methodNames = [], action = "request", payload = {}) {
|
||||
for (const methodName of methodNames) {
|
||||
if (typeof client?.[methodName] === "function") {
|
||||
return await client[methodName](payload);
|
||||
}
|
||||
}
|
||||
if (typeof client?.request === "function") {
|
||||
return await client.request(action, payload);
|
||||
}
|
||||
if (typeof client === "function") {
|
||||
return await client({ action, ...payload });
|
||||
}
|
||||
throw new Error(`Authority Trivium ${action} unavailable`);
|
||||
}
|
||||
|
||||
export async function purgeAuthorityTriviumNamespace(config = {}, options = {}) {
|
||||
throwIfAborted(options.signal);
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
return await callClient(client, ["purge"], "purge", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
});
|
||||
}
|
||||
|
||||
export async function deleteAuthorityTriviumNodes(config = {}, nodeIds = [], options = {}) {
|
||||
const ids = toArray(nodeIds).map(normalizeRecordId).filter(Boolean);
|
||||
if (!ids.length) return { deleted: 0 };
|
||||
throwIfAborted(options.signal);
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
return await callClient(client, ["deleteMany", "deleteNodes"], "deleteMany", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
ids,
|
||||
externalIds: ids,
|
||||
});
|
||||
}
|
||||
|
||||
export async function upsertAuthorityTriviumEntries(graph, config = {}, entries = [], options = {}) {
|
||||
const items = buildAuthorityVectorItems(graph, entries, options);
|
||||
if (!items.length) return { upserted: 0 };
|
||||
throwIfAborted(options.signal);
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
const chunkSize = clampInteger(config.chunkSize, DEFAULT_AUTHORITY_VECTOR_CHUNK_SIZE, 1, MAX_AUTHORITY_VECTOR_CHUNK_SIZE);
|
||||
let upserted = 0;
|
||||
for (let index = 0; index < items.length; index += chunkSize) {
|
||||
throwIfAborted(options.signal);
|
||||
const chunk = items.slice(index, index + chunkSize);
|
||||
await callClient(client, ["bulkUpsert", "upsertMany", "upsert"], "bulkUpsert", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
items: chunk,
|
||||
});
|
||||
upserted += chunk.length;
|
||||
}
|
||||
return { upserted };
|
||||
}
|
||||
|
||||
export async function syncAuthorityTriviumLinks(graph, config = {}, options = {}) {
|
||||
const links = buildAuthorityLinkItems(graph, options);
|
||||
if (!links.length) return { linked: 0 };
|
||||
throwIfAborted(options.signal);
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
await callClient(client, ["linkMany", "upsertLinks"], "linkMany", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
links,
|
||||
});
|
||||
return { linked: links.length };
|
||||
}
|
||||
|
||||
export async function searchAuthorityTriviumNodes(graph, text, config = {}, options = {}) {
|
||||
throwIfAborted(options.signal);
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
const payload = await callClient(client, ["search", "query"], "search", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
text: String(text || ""),
|
||||
searchText: String(text || ""),
|
||||
topK: Math.max(1, Math.floor(Number(options.topK) || 1)),
|
||||
candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean),
|
||||
});
|
||||
return normalizeSearchResults(payload);
|
||||
}
|
||||
|
||||
export async function testAuthorityTriviumConnection(config = {}, options = {}) {
|
||||
const client = createAuthorityTriviumClient(config, options);
|
||||
await callClient(client, ["stat"], "stat", {
|
||||
namespace: options.namespace,
|
||||
collectionId: options.collectionId,
|
||||
chatId: options.chatId,
|
||||
});
|
||||
return { success: true, dimensions: 0, error: "" };
|
||||
}
|
||||
@@ -6,6 +6,25 @@ import { getActiveNodes } from "../graph/graph.js";
|
||||
import { describeMemoryScope, normalizeMemoryScope } from "../graph/memory-scope.js";
|
||||
import { resolveConfiguredTimeoutMs } from "../runtime/request-timeout.js";
|
||||
import { buildVectorCollectionId, stableHashString } from "../runtime/runtime-state.js";
|
||||
import {
|
||||
AUTHORITY_VECTOR_MODE,
|
||||
AUTHORITY_VECTOR_SOURCE,
|
||||
deleteAuthorityTriviumNodes,
|
||||
isAuthorityVectorConfig,
|
||||
normalizeAuthorityVectorConfig,
|
||||
purgeAuthorityTriviumNamespace,
|
||||
searchAuthorityTriviumNodes,
|
||||
syncAuthorityTriviumLinks,
|
||||
testAuthorityTriviumConnection,
|
||||
upsertAuthorityTriviumEntries,
|
||||
} from "./authority-vector-primary-adapter.js";
|
||||
|
||||
export {
|
||||
AUTHORITY_VECTOR_MODE,
|
||||
AUTHORITY_VECTOR_SOURCE,
|
||||
isAuthorityVectorConfig,
|
||||
normalizeAuthorityVectorConfig,
|
||||
};
|
||||
|
||||
export const BACKEND_VECTOR_SOURCES = [
|
||||
"openai",
|
||||
@@ -213,6 +232,15 @@ export function isDirectVectorConfig(config) {
|
||||
export function getVectorModelScope(config) {
|
||||
if (!config) return "";
|
||||
|
||||
if (config?.mode === "authority" || config?.source === "authority-trivium") {
|
||||
return [
|
||||
"authority",
|
||||
config.source || "authority-trivium",
|
||||
normalizeOpenAICompatibleBaseUrl(config.baseUrl || ""),
|
||||
config.model || "",
|
||||
].join("|");
|
||||
}
|
||||
|
||||
if (isDirectVectorConfig(config)) {
|
||||
return [
|
||||
"direct",
|
||||
@@ -234,6 +262,13 @@ export function validateVectorConfig(config) {
|
||||
return { valid: false, error: "未找到向量配置" };
|
||||
}
|
||||
|
||||
if (config?.mode === "authority" || config?.source === "authority-trivium") {
|
||||
if (!config.baseUrl) {
|
||||
return { valid: false, error: "Authority Trivium 地址不可用" };
|
||||
}
|
||||
return { valid: true, error: "" };
|
||||
}
|
||||
|
||||
if (isDirectVectorConfig(config)) {
|
||||
if (!config.apiUrl) {
|
||||
return { valid: false, error: "请填写直连 Embedding API 地址" };
|
||||
@@ -569,6 +604,42 @@ function markBackendVectorStateDirty(
|
||||
state.lastWarning = String(warning || "后端向量查询失败,已标记待重建");
|
||||
}
|
||||
|
||||
function markAuthorityVectorStateDirty(
|
||||
graph,
|
||||
config,
|
||||
reason = "authority-trivium-failed",
|
||||
warning = "Authority Trivium 索引失败,已标记待重建",
|
||||
) {
|
||||
if (!graph?.vectorIndexState || !isAuthorityVectorConfig(config)) {
|
||||
return;
|
||||
}
|
||||
const state = graph.vectorIndexState;
|
||||
const total = Math.max(
|
||||
Number(state.lastStats?.total || 0),
|
||||
Object.keys(state.nodeToHash || {}).length,
|
||||
Object.keys(state.hashToNodeId || {}).length,
|
||||
);
|
||||
const previousIndexed = Number.isFinite(Number(state.lastStats?.indexed))
|
||||
? Math.max(0, Math.floor(Number(state.lastStats.indexed)))
|
||||
: 0;
|
||||
state.mode = "authority";
|
||||
state.source = config.source || "authority-trivium";
|
||||
state.modelScope = getVectorModelScope(config) || state.modelScope || "";
|
||||
state.collectionId = state.collectionId || buildVectorCollectionId(graph?.historyState?.chatId);
|
||||
state.dirty = true;
|
||||
state.dirtyReason = String(reason || "authority-trivium-failed");
|
||||
state.pendingRepairFromFloor = Number.isFinite(Number(state.pendingRepairFromFloor))
|
||||
? Math.max(0, Math.floor(Number(state.pendingRepairFromFloor)))
|
||||
: 0;
|
||||
state.lastStats = {
|
||||
total,
|
||||
indexed: previousIndexed,
|
||||
stale: total > 0 ? Math.max(1, Number(state.lastStats?.stale || 0)) : 0,
|
||||
pending: total > 0 ? Math.max(1, Number(state.lastStats?.pending || 0)) : 0,
|
||||
};
|
||||
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
|
||||
}
|
||||
|
||||
export async function syncGraphVectorIndex(
|
||||
graph,
|
||||
config,
|
||||
@@ -578,6 +649,9 @@ export async function syncGraphVectorIndex(
|
||||
force = false,
|
||||
range = null,
|
||||
signal = undefined,
|
||||
triviumClient = undefined,
|
||||
headerProvider = undefined,
|
||||
fetchImpl = undefined,
|
||||
} = {},
|
||||
) {
|
||||
if (!graph || !config) {
|
||||
@@ -590,7 +664,11 @@ export async function syncGraphVectorIndex(
|
||||
throwIfAborted(signal);
|
||||
|
||||
const syncStartedAt = nowMs();
|
||||
const syncMode = isBackendVectorConfig(config) ? "backend" : "direct";
|
||||
const syncMode = isAuthorityVectorConfig(config)
|
||||
? "authority"
|
||||
: isBackendVectorConfig(config)
|
||||
? "backend"
|
||||
: "direct";
|
||||
|
||||
const validation = validateVectorConfig(config);
|
||||
if (!validation.valid) {
|
||||
@@ -629,14 +707,163 @@ export async function syncGraphVectorIndex(
|
||||
let backendPurgeMs = 0;
|
||||
let backendDeleteMs = 0;
|
||||
let backendInsertMs = 0;
|
||||
let authorityPurgeMs = 0;
|
||||
let authorityDeleteMs = 0;
|
||||
let authorityUpsertMs = 0;
|
||||
let authorityLinkMs = 0;
|
||||
let embedBatchMs = 0;
|
||||
let deletedHashCount = 0;
|
||||
let deletedNodeCount = 0;
|
||||
let embeddingsRequested = 0;
|
||||
const hasConcreteRange =
|
||||
range && Number.isFinite(range.start) && Number.isFinite(range.end);
|
||||
const rangedNodeIds = new Set(desiredEntries.map((entry) => entry.nodeId));
|
||||
|
||||
if (isBackendVectorConfig(config)) {
|
||||
if (isAuthorityVectorConfig(config)) {
|
||||
const effectiveChatId = chatId || graph?.historyState?.chatId || "";
|
||||
const authorityOptions = {
|
||||
namespace: collectionId,
|
||||
collectionId,
|
||||
chatId: effectiveChatId,
|
||||
modelScope: getVectorModelScope(config),
|
||||
revision: graph?.meta?.revision || graph?.revision || 0,
|
||||
signal,
|
||||
triviumClient,
|
||||
headerProvider,
|
||||
fetchImpl,
|
||||
};
|
||||
const scopeChanged =
|
||||
state.mode !== "authority" ||
|
||||
state.source !== (config.source || "authority-trivium") ||
|
||||
state.modelScope !== getVectorModelScope(config) ||
|
||||
state.collectionId !== collectionId;
|
||||
const fullReset = purge || state.dirty || scopeChanged;
|
||||
|
||||
try {
|
||||
if (fullReset) {
|
||||
const purgeStartedAt = nowMs();
|
||||
await purgeAuthorityTriviumNamespace(config, authorityOptions);
|
||||
authorityPurgeMs += nowMs() - purgeStartedAt;
|
||||
resetVectorMappings(graph, config, effectiveChatId);
|
||||
const upsertStartedAt = nowMs();
|
||||
await upsertAuthorityTriviumEntries(
|
||||
graph,
|
||||
config,
|
||||
desiredEntries,
|
||||
authorityOptions,
|
||||
);
|
||||
authorityUpsertMs += nowMs() - upsertStartedAt;
|
||||
for (const entry of desiredEntries) {
|
||||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||||
insertedHashes.push(entry.hash);
|
||||
}
|
||||
} else {
|
||||
const nodeIdsToDelete = [];
|
||||
const entriesToUpsert = [];
|
||||
const queuedNodeIds = new Set();
|
||||
|
||||
if (force && hasConcreteRange) {
|
||||
for (const entry of desiredEntries) {
|
||||
entriesToUpsert.push(entry);
|
||||
queuedNodeIds.add(entry.nodeId);
|
||||
}
|
||||
}
|
||||
|
||||
for (const [nodeId, hash] of Object.entries(state.nodeToHash || {})) {
|
||||
if (hasConcreteRange && !rangedNodeIds.has(nodeId)) {
|
||||
continue;
|
||||
}
|
||||
const desired = desiredByNodeId.get(nodeId);
|
||||
if (!desired) {
|
||||
nodeIdsToDelete.push(nodeId);
|
||||
delete state.nodeToHash[nodeId];
|
||||
delete state.hashToNodeId[hash];
|
||||
} else if (desired.hash !== hash && !queuedNodeIds.has(nodeId)) {
|
||||
entriesToUpsert.push(desired);
|
||||
queuedNodeIds.add(nodeId);
|
||||
delete state.hashToNodeId[hash];
|
||||
}
|
||||
}
|
||||
|
||||
for (const entry of desiredEntries) {
|
||||
if (force && hasConcreteRange) continue;
|
||||
if (state.nodeToHash[entry.nodeId] === entry.hash) continue;
|
||||
if (queuedNodeIds.has(entry.nodeId)) continue;
|
||||
entriesToUpsert.push(entry);
|
||||
queuedNodeIds.add(entry.nodeId);
|
||||
}
|
||||
|
||||
deletedNodeCount = nodeIdsToDelete.length;
|
||||
const deleteStartedAt = nowMs();
|
||||
await deleteAuthorityTriviumNodes(config, nodeIdsToDelete, authorityOptions);
|
||||
authorityDeleteMs += nowMs() - deleteStartedAt;
|
||||
const upsertStartedAt = nowMs();
|
||||
await upsertAuthorityTriviumEntries(
|
||||
graph,
|
||||
config,
|
||||
entriesToUpsert,
|
||||
authorityOptions,
|
||||
);
|
||||
authorityUpsertMs += nowMs() - upsertStartedAt;
|
||||
|
||||
for (const entry of entriesToUpsert) {
|
||||
state.hashToNodeId[entry.hash] = entry.nodeId;
|
||||
state.nodeToHash[entry.nodeId] = entry.hash;
|
||||
insertedHashes.push(entry.hash);
|
||||
}
|
||||
}
|
||||
|
||||
const linkStartedAt = nowMs();
|
||||
await syncAuthorityTriviumLinks(graph, config, authorityOptions);
|
||||
authorityLinkMs += nowMs() - linkStartedAt;
|
||||
|
||||
for (const node of graph.nodes || []) {
|
||||
if (Array.isArray(node.embedding) && node.embedding.length > 0) {
|
||||
node.embedding = null;
|
||||
}
|
||||
}
|
||||
state.mode = "authority";
|
||||
state.source = config.source || "authority-trivium";
|
||||
state.modelScope = getVectorModelScope(config);
|
||||
state.collectionId = collectionId;
|
||||
state.dirty = false;
|
||||
state.lastWarning = "";
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) throw error;
|
||||
const message = error?.message || String(error) || "Authority Trivium 同步失败";
|
||||
markAuthorityVectorStateDirty(
|
||||
graph,
|
||||
config,
|
||||
"authority-trivium-sync-failed",
|
||||
`Authority Trivium 同步失败(${message}),已标记待重建`,
|
||||
);
|
||||
state.lastSyncAt = Date.now();
|
||||
state.lastTimings = {
|
||||
mode: syncMode,
|
||||
success: false,
|
||||
error: message,
|
||||
desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length),
|
||||
desiredBuildMs: roundMs(desiredBuildMs),
|
||||
authorityPurgeMs: roundMs(authorityPurgeMs),
|
||||
authorityDeleteMs: roundMs(authorityDeleteMs),
|
||||
authorityUpsertMs: roundMs(authorityUpsertMs),
|
||||
authorityLinkMs: roundMs(authorityLinkMs),
|
||||
totalMs: roundMs(nowMs() - syncStartedAt),
|
||||
updatedAt: Date.now(),
|
||||
};
|
||||
const result = {
|
||||
insertedHashes,
|
||||
stats: state.lastStats,
|
||||
timings: state.lastTimings,
|
||||
error: message,
|
||||
};
|
||||
if (config.failOpen === false) {
|
||||
throw error;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
} else if (isBackendVectorConfig(config)) {
|
||||
const scopeChanged =
|
||||
state.mode !== "backend" ||
|
||||
state.source !== config.source ||
|
||||
@@ -810,9 +1037,14 @@ export async function syncGraphVectorIndex(
|
||||
backendPurgeMs: roundMs(backendPurgeMs),
|
||||
backendDeleteMs: roundMs(backendDeleteMs),
|
||||
backendInsertMs: roundMs(backendInsertMs),
|
||||
authorityPurgeMs: roundMs(authorityPurgeMs),
|
||||
authorityDeleteMs: roundMs(authorityDeleteMs),
|
||||
authorityUpsertMs: roundMs(authorityUpsertMs),
|
||||
authorityLinkMs: roundMs(authorityLinkMs),
|
||||
embedBatchMs: roundMs(embedBatchMs),
|
||||
statsBuildMs: roundMs(statsBuildMs),
|
||||
deletedHashes: Math.max(0, Math.floor(deletedHashCount)),
|
||||
deletedNodes: Math.max(0, Math.floor(deletedNodeCount)),
|
||||
insertedEntries: insertedHashes.length,
|
||||
embeddingsRequested: Math.max(0, Math.floor(embeddingsRequested)),
|
||||
totalMs: roundMs(nowMs() - syncStartedAt),
|
||||
@@ -841,7 +1073,11 @@ export async function findSimilarNodesByText(
|
||||
? candidates
|
||||
: getEligibleVectorNodes(graph);
|
||||
const searchStartedAt = nowMs();
|
||||
const mode = isDirectVectorConfig(config) ? "direct" : "backend";
|
||||
const mode = isAuthorityVectorConfig(config)
|
||||
? "authority"
|
||||
: isDirectVectorConfig(config)
|
||||
? "direct"
|
||||
: "backend";
|
||||
const recordSearchTimings = (patch = {}) => {
|
||||
const state = graph?.vectorIndexState;
|
||||
if (!state || typeof state !== "object" || Array.isArray(state)) return;
|
||||
@@ -918,6 +1154,60 @@ export async function findSimilarNodesByText(
|
||||
return [];
|
||||
}
|
||||
|
||||
if (isAuthorityVectorConfig(config)) {
|
||||
const requestStartedAt = nowMs();
|
||||
try {
|
||||
const allowedIds = new Set(candidateNodes.map((node) => node.id));
|
||||
const results = (
|
||||
await searchAuthorityTriviumNodes(graph, text, config, {
|
||||
namespace: graph.vectorIndexState?.collectionId,
|
||||
collectionId: graph.vectorIndexState?.collectionId,
|
||||
chatId: graph?.historyState?.chatId || "",
|
||||
modelScope: getVectorModelScope(config),
|
||||
topK,
|
||||
candidateIds: candidateNodes.map((node) => node.id),
|
||||
signal,
|
||||
})
|
||||
)
|
||||
.filter((entry) => entry.nodeId && allowedIds.has(entry.nodeId))
|
||||
.slice(0, topK);
|
||||
recordSearchTimings({
|
||||
success: true,
|
||||
reason: "ok",
|
||||
requestMs: roundMs(nowMs() - requestStartedAt),
|
||||
resultCount: results.length,
|
||||
});
|
||||
return results;
|
||||
} catch (error) {
|
||||
if (isAbortError(error)) {
|
||||
recordSearchTimings({
|
||||
success: false,
|
||||
reason: "aborted",
|
||||
error: error?.message || String(error),
|
||||
});
|
||||
throw error;
|
||||
}
|
||||
const message = error?.message || String(error) || "Authority Trivium 查询失败";
|
||||
markAuthorityVectorStateDirty(
|
||||
graph,
|
||||
config,
|
||||
"authority-trivium-query-failed",
|
||||
`Authority Trivium 查询失败(${message}),已标记待重建`,
|
||||
);
|
||||
recordSearchTimings({
|
||||
success: false,
|
||||
reason: "authority-trivium-query-failed",
|
||||
requestMs: roundMs(nowMs() - requestStartedAt),
|
||||
error: message,
|
||||
resultCount: 0,
|
||||
});
|
||||
if (config.failOpen === false) {
|
||||
throw error;
|
||||
}
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const requestStartedAt = nowMs();
|
||||
const response = await fetchWithTimeout(
|
||||
@@ -1025,6 +1315,17 @@ export async function testVectorConnection(config, chatId = "connection-test") {
|
||||
}
|
||||
}
|
||||
|
||||
if (isAuthorityVectorConfig(config)) {
|
||||
try {
|
||||
return await testAuthorityTriviumConnection(config, {
|
||||
collectionId: buildVectorCollectionId(chatId),
|
||||
chatId,
|
||||
});
|
||||
} catch (error) {
|
||||
return { success: false, dimensions: 0, error: String(error) };
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const response = await fetchWithTimeout(
|
||||
"/api/vector/query",
|
||||
@@ -1223,6 +1524,14 @@ export async function fetchAvailableEmbeddingModels(config) {
|
||||
}
|
||||
|
||||
try {
|
||||
if (isAuthorityVectorConfig(config)) {
|
||||
return {
|
||||
success: false,
|
||||
models: [],
|
||||
error: "Authority Trivium 使用服务端索引配置,无需拉取 Embedding 模型",
|
||||
};
|
||||
}
|
||||
|
||||
if (isDirectVectorConfig(config)) {
|
||||
const models = normalizeModelOptions(
|
||||
await fetchOpenAICompatibleModelList(config.apiUrl, config.apiKey),
|
||||
|
||||
Reference in New Issue
Block a user