From 6c8c56df62c30fc0a9f7ac11a4b0921d50303cd6 Mon Sep 17 00:00:00 2001 From: Youzini-afk <13153778771cx@gmail.com> Date: Tue, 28 Apr 2026 15:01:14 +0800 Subject: [PATCH] feat(authority): add recall candidate provider --- retrieval/authority-candidate-provider.js | 340 +++++++++++++++++++++ retrieval/retriever.js | 93 +++++- tests/authority-recall-candidates.mjs | 256 ++++++++++++++++ tests/authority-vector-primary.mjs | 64 +++- tests/retrieval-config.mjs | 129 +++++++- vector/authority-vector-primary-adapter.js | 151 ++++++++- 6 files changed, 1005 insertions(+), 28 deletions(-) create mode 100644 retrieval/authority-candidate-provider.js create mode 100644 tests/authority-recall-candidates.mjs diff --git a/retrieval/authority-candidate-provider.js b/retrieval/authority-candidate-provider.js new file mode 100644 index 0000000..1823e71 --- /dev/null +++ b/retrieval/authority-candidate-provider.js @@ -0,0 +1,340 @@ +import { + buildContextQueryBlend, + buildVectorQueryPlan, + clampPositiveInt, +} from "./shared-ranking.js"; +import { + filterAuthorityTriviumNodes, + isAuthorityVectorConfig, + queryAuthorityTriviumNeighbors, + searchAuthorityTriviumNodes, +} from "../vector/authority-vector-primary-adapter.js"; + +function nowMs() { + if (typeof performance?.now === "function") { + return performance.now(); + } + return Date.now(); +} + +function roundMs(value) { + return Math.round((Number(value) || 0) * 10) / 10; +} + +function normalizeRecordId(value) { + return String(value ?? "").trim(); +} + +function toArray(value) { + return Array.isArray(value) ? value : []; +} + +function uniqueIds(values = []) { + const result = []; + const seen = new Set(); + for (const value of values) { + const normalized = normalizeRecordId(value); + if (!normalized || seen.has(normalized)) continue; + seen.add(normalized); + result.push(normalized); + } + return result; +} + +function isAbortError(error) { + return error?.name === "AbortError"; +} + +function buildAuthorityCandidateQueryPlan(userMessage, recentMessages = [], options = {}) { + const queryBlend = buildContextQueryBlend(userMessage, recentMessages, { + enabled: options.enableContextQueryBlend !== false, + assistantWeight: Number(options.contextAssistantWeight ?? 0.2), + previousUserWeight: Number(options.contextPreviousUserWeight ?? 0.1), + maxTextLength: Number(options.maxTextLength || 400), + }); + const vectorQueryPlan = buildVectorQueryPlan(queryBlend, { + enableMultiIntent: options.enableMultiIntent !== false, + maxSegments: clampPositiveInt(options.multiIntentMaxSegments, 4), + }); + const maxQueryTexts = clampPositiveInt(options.maxQueryTexts, 6); + const queries = []; + const seen = new Set(); + for (const part of vectorQueryPlan.plan || []) { + for (const queryText of part.queries || []) { + const normalizedText = String(queryText || "").trim(); + const key = normalizedText.toLowerCase(); + if (!normalizedText || seen.has(key)) continue; + seen.add(key); + queries.push({ + text: normalizedText, + weight: Math.max(0.05, Number(part.weight || 0) || 0.05), + }); + if (queries.length >= maxQueryTexts) { + return { + queryBlend, + vectorQueryPlan, + queries, + }; + } + } + } + return { + queryBlend, + vectorQueryPlan, + queries, + }; +} + +function resolveSceneOwnerNames(sceneOwnerCandidates = [], ownerKeys = []) { + const ownerKeySet = new Set(uniqueIds(ownerKeys)); + return uniqueIds( + toArray(sceneOwnerCandidates) + .filter((candidate) => ownerKeySet.has(normalizeRecordId(candidate?.ownerKey))) + .map((candidate) => candidate?.ownerName) + .filter(Boolean), + ); +} + +function buildAuthorityCandidateFilters({ + ownerKeys = [], + ownerNames = [], + regionKeys = [], + storySegmentIds = [], +} = {}) { + return { + archived: false, + ownerKeys: uniqueIds(ownerKeys), + ownerNames: uniqueIds(ownerNames), + regionKeys: uniqueIds(regionKeys), + storySegmentIds: uniqueIds(storySegmentIds), + }; +} + +function mapCandidateNodes(candidateIds = [], availableNodes = []) { + const nodeMap = new Map( + toArray(availableNodes) + .map((node) => [normalizeRecordId(node?.id), node]) + .filter(([nodeId]) => nodeId), + ); + return uniqueIds(candidateIds) + .map((nodeId) => nodeMap.get(nodeId)) + .filter(Boolean); +} + +export async function resolveAuthorityRecallCandidates({ + graph, + userMessage, + recentMessages = [], + embeddingConfig, + availableNodes = [], + activeRegion = "", + activeStoryContext = {}, + activeRecallOwnerKeys = [], + sceneOwnerCandidates = [], + signal = undefined, + options = {}, +} = {}) { + const startedAt = nowMs(); + const diagnostics = { + provider: "authority-trivium", + available: false, + used: false, + candidateCount: 0, + filteredCount: 0, + searchHits: 0, + neighborCount: 0, + queryTexts: [], + fallbackReason: "", + timings: { + total: 0, + filter: 0, + search: 0, + neighbors: 0, + }, + }; + const candidateNodes = toArray(availableNodes).filter((node) => node && !node.archived); + if (options.enabled === false) { + diagnostics.fallbackReason = "authority-graph-query-disabled"; + diagnostics.timings.total = roundMs(nowMs() - startedAt); + return { + available: false, + used: false, + candidateNodes: [], + diagnostics, + }; + } + if (!graph || candidateNodes.length === 0 || !isAuthorityVectorConfig(embeddingConfig)) { + diagnostics.fallbackReason = "authority-vector-unavailable"; + diagnostics.timings.total = roundMs(nowMs() - startedAt); + return { + available: false, + used: false, + candidateNodes: [], + diagnostics, + }; + } + + diagnostics.available = true; + const collectionId = normalizeRecordId( + options.collectionId || graph?.vectorIndexState?.collectionId, + ); + const chatId = normalizeRecordId(options.chatId || graph?.historyState?.chatId); + if (!collectionId) { + diagnostics.fallbackReason = "authority-collection-missing"; + diagnostics.timings.total = roundMs(nowMs() - startedAt); + return { + available: true, + used: false, + candidateNodes: [], + diagnostics, + }; + } + + const allowedIds = new Set(candidateNodes.map((node) => normalizeRecordId(node?.id))); + const limit = clampPositiveInt( + options.limit, + Math.min(candidateNodes.length, Math.max(Number(options.topK || 0) * 4, 24)), + ); + const neighborLimit = clampPositiveInt( + options.neighborLimit, + Math.min(limit, Math.max(4, Math.ceil(limit / 4))), + ); + const minimumUsedCandidateCount = clampPositiveInt( + options.minimumUsedCandidateCount, + Math.min(candidateNodes.length, Math.max(Number(options.maxRecallNodes || 0), 6)), + ); + const ownerKeys = uniqueIds(activeRecallOwnerKeys); + const ownerNames = resolveSceneOwnerNames(sceneOwnerCandidates, ownerKeys); + const regionKeys = uniqueIds([activeRegion]); + const storySegmentIds = uniqueIds([activeStoryContext?.activeSegmentId]); + const filterPayload = buildAuthorityCandidateFilters({ + ownerKeys, + ownerNames, + regionKeys, + storySegmentIds, + }); + const queryPlan = buildAuthorityCandidateQueryPlan(userMessage, recentMessages, options); + diagnostics.queryTexts = queryPlan.queries.map((entry) => entry.text); + + let filteredIds = []; + const filterStartedAt = nowMs(); + try { + filteredIds = (await filterAuthorityTriviumNodes(embeddingConfig, { + namespace: collectionId, + collectionId, + chatId, + limit, + topK: limit, + filters: filterPayload, + filter: filterPayload, + where: filterPayload, + searchText: queryPlan.queries[0]?.text || String(userMessage || "").trim(), + signal, + })) + .filter((nodeId) => allowedIds.has(normalizeRecordId(nodeId))) + .slice(0, limit); + diagnostics.filteredCount = filteredIds.length; + } catch (error) { + if (isAbortError(error)) throw error; + diagnostics.fallbackReason = "authority-candidate-filter-failed"; + if (embeddingConfig?.failOpen === false) { + throw error; + } + } + diagnostics.timings.filter = roundMs(nowMs() - filterStartedAt); + + const searchScores = new Map(); + const searchStartedAt = nowMs(); + for (const queryEntry of queryPlan.queries) { + try { + const searchResults = await searchAuthorityTriviumNodes( + graph, + queryEntry.text, + embeddingConfig, + { + namespace: collectionId, + collectionId, + chatId, + topK: limit, + candidateIds: filteredIds.length > 0 ? filteredIds : undefined, + signal, + }, + ); + for (const result of searchResults) { + const nodeId = normalizeRecordId(result?.nodeId); + if (!nodeId || !allowedIds.has(nodeId)) continue; + const weightedScore = Math.max(0.001, Number(result?.score || 0) || 0) * queryEntry.weight; + const previous = Number(searchScores.get(nodeId) || 0) || 0; + if (weightedScore > previous) { + searchScores.set(nodeId, weightedScore); + } + } + } catch (error) { + if (isAbortError(error)) throw error; + diagnostics.fallbackReason ||= "authority-candidate-search-failed"; + if (embeddingConfig?.failOpen === false) { + throw error; + } + } + } + diagnostics.timings.search = roundMs(nowMs() - searchStartedAt); + diagnostics.searchHits = searchScores.size; + + const seedIds = [...searchScores.entries()] + .sort((left, right) => right[1] - left[1]) + .map(([nodeId]) => nodeId) + .slice(0, Math.min(limit, Math.max(4, neighborLimit))); + + let neighborIds = []; + const neighborsStartedAt = nowMs(); + if (seedIds.length > 0) { + try { + neighborIds = (await queryAuthorityTriviumNeighbors(embeddingConfig, seedIds, { + namespace: collectionId, + collectionId, + chatId, + limit: neighborLimit, + topK: neighborLimit, + candidateIds: filteredIds.length > 0 ? filteredIds : undefined, + signal, + })) + .filter((nodeId) => allowedIds.has(normalizeRecordId(nodeId))) + .slice(0, neighborLimit); + diagnostics.neighborCount = neighborIds.length; + } catch (error) { + if (isAbortError(error)) throw error; + diagnostics.fallbackReason ||= "authority-candidate-neighbors-failed"; + if (embeddingConfig?.failOpen === false) { + throw error; + } + } + } + diagnostics.timings.neighbors = roundMs(nowMs() - neighborsStartedAt); + + const prioritizedNodeIds = uniqueIds([ + ...seedIds, + ...filteredIds, + ...neighborIds, + ]).slice(0, limit); + const resolvedCandidateNodes = mapCandidateNodes(prioritizedNodeIds, candidateNodes); + diagnostics.candidateCount = resolvedCandidateNodes.length; + diagnostics.used = + resolvedCandidateNodes.length >= minimumUsedCandidateCount && + resolvedCandidateNodes.length < candidateNodes.length; + if (!diagnostics.used && !diagnostics.fallbackReason) { + diagnostics.fallbackReason = + resolvedCandidateNodes.length === 0 + ? "authority-candidate-empty" + : resolvedCandidateNodes.length >= candidateNodes.length + ? "authority-candidate-not-reduced" + : "authority-candidate-too-small"; + } + diagnostics.timings.total = roundMs(nowMs() - startedAt); + + return { + available: true, + used: diagnostics.used, + candidateNodes: diagnostics.used ? resolvedCandidateNodes : [], + diagnostics, + }; +} diff --git a/retrieval/retriever.js b/retrieval/retriever.js index 504189c..a6b61c1 100644 --- a/retrieval/retriever.js +++ b/retrieval/retriever.js @@ -36,6 +36,7 @@ import { normalizeMemoryScope, resolveScopeBucketWeight, } from "../graph/memory-scope.js"; +import { resolveAuthorityRecallCandidates } from "./authority-candidate-provider.js"; import { rankNodesForTaskContext } from "./shared-ranking.js"; import { computeKnowledgeGateForNode, @@ -207,9 +208,19 @@ function createRetrievalMeta(enableLLMRecall) { activeRecallOwnerKey: "", activeRecallOwnerKeys: [], activeRecallOwnerScores: {}, + activeNodeCount: 0, + rankingNodeCount: 0, sceneOwnerResolutionMode: "unresolved", sceneOwnerCandidates: [], bucketWeights: {}, + authorityCandidateProvider: "", + authorityCandidateUsed: false, + authorityCandidateCount: 0, + authorityCandidateFilteredCount: 0, + authorityCandidateSearchHits: 0, + authorityCandidateNeighborCount: 0, + authorityCandidateQueryTexts: [], + authorityCandidateFallbackReason: "", selectedByBucket: {}, knowledgeGateMode: "disabled", knowledgeAnchoredNodes: [], @@ -1228,6 +1239,7 @@ export async function retrieve({ } const nodeCount = activeNodes.length; + let rankingActiveNodes = activeNodes; const normalizedTopK = Math.max(1, topK); const normalizedMaxRecallNodes = Math.max(1, maxRecallNodes); const normalizedDiffusionTopK = Math.max(1, diffusionTopK); @@ -1249,6 +1261,8 @@ export async function retrieve({ retrievalMeta.activeRecallOwnerKey = activeRecallOwnerKeys[0] || ""; retrievalMeta.activeRecallOwnerKeys = [...activeRecallOwnerKeys]; retrievalMeta.activeRecallOwnerScores = { ...activeRecallOwnerScores }; + retrievalMeta.activeNodeCount = nodeCount; + retrievalMeta.rankingNodeCount = nodeCount; retrievalMeta.sceneOwnerResolutionMode = sceneOwnerResolutionMode; retrievalMeta.sceneOwnerCandidates = preliminarySceneOwnerCandidates.map((candidate) => ({ ownerKey: candidate.ownerKey, @@ -1262,8 +1276,83 @@ export async function retrieve({ retrievalMeta.knowledgeGateMode = enableCognitiveMemory ? "anchored-soft-visibility" : "disabled"; + const authorityCandidateStartedAt = nowMs(); + const authorityCandidateResult = await resolveAuthorityRecallCandidates({ + graph, + userMessage, + recentMessages, + embeddingConfig, + availableNodes: activeNodes, + activeRegion, + activeStoryContext, + activeRecallOwnerKeys, + sceneOwnerCandidates: preliminarySceneOwnerCandidates, + signal, + options: { + enabled: settings.authorityGraphQueryEnabled !== false, + topK: normalizedTopK, + maxRecallNodes: normalizedMaxRecallNodes, + limit: options.authorityCandidateLimit, + neighborLimit: options.authorityCandidateNeighborLimit, + minimumUsedCandidateCount: options.authorityCandidateMinCount, + enableContextQueryBlend, + contextAssistantWeight, + contextPreviousUserWeight, + enableMultiIntent, + multiIntentMaxSegments, + }, + }); + retrievalMeta.authorityCandidateProvider = + String(authorityCandidateResult?.diagnostics?.provider || ""); + retrievalMeta.authorityCandidateUsed = authorityCandidateResult?.used === true; + retrievalMeta.authorityCandidateCount = Number( + authorityCandidateResult?.diagnostics?.candidateCount || 0, + ); + retrievalMeta.authorityCandidateFilteredCount = Number( + authorityCandidateResult?.diagnostics?.filteredCount || 0, + ); + retrievalMeta.authorityCandidateSearchHits = Number( + authorityCandidateResult?.diagnostics?.searchHits || 0, + ); + retrievalMeta.authorityCandidateNeighborCount = Number( + authorityCandidateResult?.diagnostics?.neighborCount || 0, + ); + retrievalMeta.authorityCandidateQueryTexts = Array.isArray( + authorityCandidateResult?.diagnostics?.queryTexts, + ) + ? [...authorityCandidateResult.diagnostics.queryTexts] + : []; + retrievalMeta.authorityCandidateFallbackReason = String( + authorityCandidateResult?.diagnostics?.fallbackReason || "", + ); + retrievalMeta.timings.authorityCandidates = Number( + authorityCandidateResult?.diagnostics?.timings?.total || 0, + ); + retrievalMeta.timings.authorityCandidateFilter = Number( + authorityCandidateResult?.diagnostics?.timings?.filter || 0, + ); + retrievalMeta.timings.authorityCandidateSearch = Number( + authorityCandidateResult?.diagnostics?.timings?.search || 0, + ); + retrievalMeta.timings.authorityCandidateNeighbors = Number( + authorityCandidateResult?.diagnostics?.timings?.neighbors || 0, + ); + if (authorityCandidateResult?.used && authorityCandidateResult?.candidateNodes?.length > 0) { + rankingActiveNodes = authorityCandidateResult.candidateNodes; + retrievalMeta.rankingNodeCount = rankingActiveNodes.length; + } else if (retrievalMeta.authorityCandidateFallbackReason) { + pushSkipReason( + retrievalMeta, + `authority-candidate:${retrievalMeta.authorityCandidateFallbackReason}`, + ); + } + if (!Number.isFinite(retrievalMeta.timings.authorityCandidates)) { + retrievalMeta.timings.authorityCandidates = roundMs( + nowMs() - authorityCandidateStartedAt, + ); + } debugLog( - `[ST-BME] 检索开始: ${nodeCount} 个活跃节点${enableVisibility ? " (认知边界已启用)" : ""}`, + `[ST-BME] 检索开始: ${nodeCount} 个活跃节点 -> ${rankingActiveNodes.length} 个候选${enableVisibility ? " (认知边界已启用)" : ""}`, ); let vectorResults = []; @@ -1338,7 +1427,7 @@ export async function retrieve({ enableLexicalBoost, lexicalWeight, weights, - activeNodes, + activeNodes: rankingActiveNodes, }, }); const contextQueryBlend = sharedRanking.contextQueryBlend; diff --git a/tests/authority-recall-candidates.mjs b/tests/authority-recall-candidates.mjs new file mode 100644 index 0000000..9e41474 --- /dev/null +++ b/tests/authority-recall-candidates.mjs @@ -0,0 +1,256 @@ +import assert from "node:assert/strict"; +import { addNode, createEmptyGraph, createNode } from "../graph/graph.js"; +import { + installResolveHooks, + toDataModuleUrl, +} from "./helpers/register-hooks-compat.mjs"; + +installResolveHooks([ + { + specifiers: ["../../../../../script.js"], + url: toDataModuleUrl("export function getRequestHeaders() { return {}; }"), + }, + { + specifiers: ["../../../../extensions.js"], + url: toDataModuleUrl("export const extension_settings = { st_bme: {} };"), + }, +]); + +const { normalizeAuthorityVectorConfig } = await import( + "../vector/authority-vector-primary-adapter.js" +); +const { resolveAuthorityRecallCandidates } = await import( + "../retrieval/authority-candidate-provider.js" +); + +function createRecallGraph() { + const graph = createEmptyGraph(); + graph.historyState.chatId = "chat-authority-candidates"; + graph.vectorIndexState.collectionId = "st-bme:chat-authority-candidates:nodes"; + + const first = createNode({ + type: "event", + seq: 10, + fields: { title: "Alice enters the archive", summary: "Alice reaches the archive gate" }, + importance: 6, + scope: { + layer: "objective", + ownerType: "", + ownerId: "", + ownerName: "", + bucket: "objectiveGlobal", + regionKey: "archive", + }, + }); + first.id = "node-archive"; + first.storySegmentId = "seg-archive"; + + const second = createNode({ + type: "event", + seq: 11, + fields: { title: "Bob opens the vault", summary: "Bob unlocks the hidden vault" }, + importance: 7, + scope: { + layer: "objective", + ownerType: "", + ownerId: "", + ownerName: "", + bucket: "objectiveGlobal", + regionKey: "archive", + }, + }); + second.id = "node-vault"; + second.storySegmentId = "seg-archive"; + + const third = createNode({ + type: "pov_memory", + seq: 12, + fields: { title: "Alice remembers the key", summary: "Alice knows where the silver key is" }, + importance: 9, + scope: { + layer: "pov", + ownerType: "character", + ownerId: "Alice", + ownerName: "Alice", + bucket: "characterPov", + regionKey: "archive", + }, + }); + third.id = "node-alice-memory"; + third.storySegmentId = "seg-archive"; + + const fourth = createNode({ + type: "event", + seq: 6, + fields: { title: "Market rumor", summary: "A rumor spreads in the market" }, + importance: 2, + scope: { + layer: "objective", + ownerType: "", + ownerId: "", + ownerName: "", + bucket: "objectiveGlobal", + regionKey: "market", + }, + }); + fourth.id = "node-market"; + fourth.storySegmentId = "seg-market"; + + addNode(graph, first); + addNode(graph, second); + addNode(graph, third); + addNode(graph, fourth); + return { graph, nodes: [first, second, third, fourth] }; +} + +function createMockTriviumClient({ failFilter = false, failSearch = false, failNeighbors = false } = {}) { + const calls = []; + return { + calls, + async filterWhere(payload = {}) { + calls.push(["filterWhere", payload]); + if (failFilter) { + throw new Error("filter-down"); + } + return { + items: [ + { externalId: "node-archive" }, + { payload: { nodeId: "node-alice-memory" } }, + ], + }; + }, + async search(payload = {}) { + calls.push(["search", payload]); + if (failSearch) { + throw new Error("search-down"); + } + return { + results: [ + { nodeId: "node-alice-memory", score: 0.96 }, + { nodeId: "node-vault", score: 0.88 }, + { nodeId: "node-outside", score: 0.77 }, + ], + }; + }, + async neighbors(payload = {}) { + calls.push(["neighbors", payload]); + if (failNeighbors) { + throw new Error("neighbors-down"); + } + return { + neighbors: [ + { fromId: "node-alice-memory", toId: "node-vault" }, + { fromId: "node-alice-memory", toId: "node-archive" }, + ], + }; + }, + }; +} + +{ + const { graph, nodes } = createRecallGraph(); + const triviumClient = createMockTriviumClient(); + const config = normalizeAuthorityVectorConfig( + { + authorityBaseUrl: "/api/plugins/authority", + authorityVectorFailOpen: true, + }, + { triviumClient }, + ); + const result = await resolveAuthorityRecallCandidates({ + graph, + userMessage: "Alice 现在在 archive 里找 silver key 吗？", + recentMessages: ["assistant: Alice just reached the archive gate."], + embeddingConfig: config, + availableNodes: nodes, + activeRegion: "archive", + activeStoryContext: { + activeSegmentId: "seg-archive", + }, + activeRecallOwnerKeys: ["character:Alice"], + sceneOwnerCandidates: [ + { + ownerKey: "character:Alice", + ownerName: "Alice", + }, + ], + options: { + enabled: true, + topK: 4, + maxRecallNodes: 2, + limit: 6, + neighborLimit: 2, + minimumUsedCandidateCount: 2, + enableMultiIntent: true, + }, + }); + + assert.equal(result.available, true); + assert.equal(result.used, true); + assert.deepEqual( + result.candidateNodes.map((node) => node.id), + ["node-alice-memory", "node-vault", "node-archive"], + ); + assert.equal(result.diagnostics.filteredCount, 2); + assert.equal(result.diagnostics.searchHits, 2); + assert.equal(result.diagnostics.neighborCount, 1); + const filterCall = triviumClient.calls.find(([name]) => name === "filterWhere"); + assert.equal(filterCall?.[1]?.filters?.archived, false); + assert.deepEqual(filterCall?.[1]?.filters?.regionKeys, ["archive"]); + assert.deepEqual(filterCall?.[1]?.filters?.ownerKeys, ["character:Alice"]); + assert.deepEqual(filterCall?.[1]?.filters?.storySegmentIds, ["seg-archive"]); + const searchCall = triviumClient.calls.find(([name]) => name === "search"); + assert.ok(Array.isArray(searchCall?.[1]?.candidateIds)); + assert.ok(searchCall?.[1]?.candidateIds.includes("node-alice-memory")); + const neighborCall = triviumClient.calls.find(([name]) => name === "neighbors"); + assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-alice-memory", "node-vault"]); +} + +{ + const { graph, nodes } = createRecallGraph(); + const triviumClient = createMockTriviumClient({ + failFilter: true, + failSearch: true, + failNeighbors: true, + }); + const config = normalizeAuthorityVectorConfig( + { + authorityBaseUrl: "/api/plugins/authority", + authorityVectorFailOpen: true, + }, + { triviumClient }, + ); + const result = await resolveAuthorityRecallCandidates({ + graph, + userMessage: "archive", + recentMessages: [], + embeddingConfig: config, + availableNodes: nodes, + activeRegion: "archive", + activeStoryContext: { + activeSegmentId: "seg-archive", + }, + activeRecallOwnerKeys: ["character:Alice"], + sceneOwnerCandidates: [ + { + ownerKey: "character:Alice", + ownerName: "Alice", + }, + ], + options: { + enabled: true, + topK: 4, + maxRecallNodes: 2, + limit: 6, + neighborLimit: 2, + minimumUsedCandidateCount: 2, + }, + }); + + assert.equal(result.available, true); + assert.equal(result.used, false); + assert.deepEqual(result.candidateNodes, []); + assert.match(result.diagnostics.fallbackReason, /authority-candidate-(filter|search|neighbors)-failed/); +} + +console.log("authority-recall-candidates tests passed"); diff --git a/tests/authority-vector-primary.mjs b/tests/authority-vector-primary.mjs index dd3b321..169b139 100644 --- a/tests/authority-vector-primary.mjs +++ b/tests/authority-vector-primary.mjs @@ -17,11 +17,12 @@ installResolveHooks([ ]); const { - findSimilarNodesByText, + filterAuthorityTriviumNodes, isAuthorityVectorConfig, normalizeAuthorityVectorConfig, - syncGraphVectorIndex, -} = await import("../vector/vector-index.js"); + queryAuthorityTriviumNeighbors, +} = await import("../vector/authority-vector-primary-adapter.js"); +const { findSimilarNodesByText: findSimilarNodesByTextFromIndex, syncGraphVectorIndex: syncGraphVectorIndexFromIndex } = await import("../vector/vector-index.js"); function createAuthorityVectorGraph() { const graph = createEmptyGraph(); @@ -86,6 +87,24 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) { ], }; }, + async filterWhere(payload) { + calls.push(["filterWhere", payload]); + return { + items: [ + { externalId: "node-a" }, + { payload: { nodeId: "node-b" } }, + ], + }; + }, + async neighbors(payload) { + calls.push(["neighbors", payload]); + return { + neighbors: [ + { fromId: "node-a", toId: "node-b" }, + { fromId: "node-a", toId: "node-c" }, + ], + }; + }, async stat(payload) { calls.push(["stat", payload]); return { ok: true }; @@ -103,7 +122,7 @@ assert.equal(isAuthorityVectorConfig(config), true); { const { graph, first, second } = createAuthorityVectorGraph(); const triviumClient = createMockTriviumClient(); - const result = await syncGraphVectorIndex(graph, config, { + const result = await syncGraphVectorIndexFromIndex(graph, config, { chatId: "chat-authority-vector", purge: true, triviumClient, @@ -134,13 +153,13 @@ assert.equal(isAuthorityVectorConfig(config), true); const { graph, first, second } = createAuthorityVectorGraph(); const triviumClient = createMockTriviumClient(); const queryConfig = { ...config, triviumClient }; - await syncGraphVectorIndex(graph, queryConfig, { + await syncGraphVectorIndexFromIndex(graph, queryConfig, { chatId: "chat-authority-vector", purge: true, triviumClient, }); - const results = await findSimilarNodesByText( + const results = await findSimilarNodesByTextFromIndex( graph, "archive door", queryConfig, @@ -158,7 +177,7 @@ assert.equal(isAuthorityVectorConfig(config), true); { const { graph } = createAuthorityVectorGraph(); const triviumClient = createMockTriviumClient({ failBulkUpsert: true }); - const result = await syncGraphVectorIndex(graph, config, { + const result = await syncGraphVectorIndexFromIndex(graph, config, { chatId: "chat-authority-vector", purge: true, triviumClient, @@ -171,4 +190,35 @@ assert.equal(isAuthorityVectorConfig(config), true); assert.match(graph.vectorIndexState.lastWarning, /Authority Trivium 同步失败/); } +{ + const triviumClient = createMockTriviumClient(); + const queryConfig = { ...config, triviumClient }; + const filteredIds = await filterAuthorityTriviumNodes(queryConfig, { + collectionId: "authority-filter", + chatId: "chat-authority-vector", + limit: 8, + filters: { + archived: false, + ownerKeys: ["character:Alice"], + }, + }); + assert.deepEqual(filteredIds, ["node-a", "node-b"]); + const filterCall = triviumClient.calls.find(([name]) => name === "filterWhere"); + assert.equal(filterCall?.[1]?.collectionId, "authority-filter"); + assert.equal(filterCall?.[1]?.filters?.ownerKeys?.[0], "character:Alice"); +} + +{ + const triviumClient = createMockTriviumClient(); + const queryConfig = { ...config, triviumClient }; + const neighborIds = await queryAuthorityTriviumNeighbors(queryConfig, ["node-a"], { + collectionId: "authority-filter", + chatId: "chat-authority-vector", + limit: 4, + }); + assert.deepEqual(neighborIds, ["node-b", "node-c"]); + const neighborCall = triviumClient.calls.find(([name]) => name === "neighbors"); + assert.deepEqual(neighborCall?.[1]?.nodeIds, ["node-a"]); +} + console.log("authority-vector-primary tests passed"); diff --git a/tests/retrieval-config.mjs b/tests/retrieval-config.mjs index 9ec80e5..4170da0 100644 --- a/tests/retrieval-config.mjs +++ b/tests/retrieval-config.mjs @@ -435,6 +435,7 @@ async function rankNodesForTaskContext({ skipReasons: [], timings: { vector: 0, diffusion: 0 }, }; + const activeNodeIds = new Set(activeNodes.map((node) => node.id)); let vectorResults = []; if (enableVectorPrefilter) { @@ -446,10 +447,12 @@ async function rankNodesForTaskContext({ { nodeId: "rule-1", score: 0.9 }, { nodeId: "rule-2", score: 0.8 }, { nodeId: "rule-3", score: 0.7 }, - ].map((item) => ({ - ...item, - score: item.score * Math.max(0, Number(part.weight) || 0), - })); + ] + .filter((item) => activeNodeIds.has(item.nodeId)) + .map((item) => ({ + ...item, + score: item.score * Math.max(0, Number(part.weight) || 0), + })); groups.push(results); } } @@ -487,7 +490,7 @@ async function rankNodesForTaskContext({ diffusionResults = [ { nodeId: "rule-2", energy: 1.2 }, { nodeId: "rule-3", energy: 0.9 }, - ]; + ].filter((item) => activeNodeIds.has(item.nodeId)); } } diagnostics.diffusionHits = diffusionResults.length; @@ -566,6 +569,10 @@ const state = { llmCandidateCount: 0, llmResponse: { selected_keys: ["R1", "R2"] }, llmOptions: [], + authorityCandidateCalls: [], + authorityCandidateEnabled: false, + authorityCandidateNodeIds: [], + authorityCandidateDiagnostics: null, }; const graph = createGraph(); @@ -575,6 +582,80 @@ const retrieve = await loadRetrieve({ createPromptNodeReferenceMap, getPromptNodeLabel, rankNodesForTaskContext, + async resolveAuthorityRecallCandidates({ + availableNodes = [], + activeRegion = "", + activeStoryContext = {}, + activeRecallOwnerKeys = [], + options = {}, + } = {}) { + state.authorityCandidateCalls.push({ + availableNodeIds: availableNodes.map((node) => node.id), + activeRegion, + activeStorySegmentId: String(activeStoryContext?.activeSegmentId || ""), + activeRecallOwnerKeys: [...(activeRecallOwnerKeys || [])], + minimumUsedCandidateCount: Number(options.minimumUsedCandidateCount || 0) || 0, + }); + if (!state.authorityCandidateEnabled) { + return { + available: false, + used: false, + candidateNodes: [], + diagnostics: { + provider: "authority-trivium", + candidateCount: 0, + filteredCount: 0, + searchHits: 0, + neighborCount: 0, + queryTexts: [], + fallbackReason: "authority-vector-unavailable", + timings: { + total: 0, + filter: 0, + search: 0, + neighbors: 0, + }, + }, + }; + } + const requestedIds = Array.isArray(state.authorityCandidateNodeIds) + ? state.authorityCandidateNodeIds + : []; + const candidateNodes = availableNodes.filter((node) => requestedIds.includes(node.id)); + const minimumUsedCandidateCount = Number(options.minimumUsedCandidateCount || 0) || 0; + const used = + candidateNodes.length > 0 && + candidateNodes.length < availableNodes.length && + candidateNodes.length >= minimumUsedCandidateCount; + const diagnostics = { + provider: "authority-trivium", + candidateCount: candidateNodes.length, + filteredCount: candidateNodes.length, + searchHits: candidateNodes.length, + neighborCount: 0, + queryTexts: ["authority-candidate-query"], + fallbackReason: used + ? "" + : candidateNodes.length === 0 + ? "authority-candidate-empty" + : candidateNodes.length >= availableNodes.length + ? "authority-candidate-not-reduced" + : "authority-candidate-too-small", + timings: { + total: 1, + filter: 0.2, + search: 0.4, + neighbors: 0, + }, + ...(state.authorityCandidateDiagnostics || {}), + }; + return { + available: true, + used, + candidateNodes: used ? candidateNodes : [], + diagnostics, + }; + }, STORY_TEMPORAL_BUCKETS: { CURRENT: "current", ADJACENT_PAST: "adjacentPast", @@ -902,6 +983,44 @@ assert.equal(state.diffusionCalls.length, 0); assert.equal(state.llmCalls.length, 0); assert.deepEqual(Array.from(noStageResult.selectedNodeIds), ["rule-2", "rule-1"]); +state.authorityCandidateCalls.length = 0; +state.authorityCandidateEnabled = true; +state.authorityCandidateNodeIds = ["rule-2"]; +state.authorityCandidateDiagnostics = null; +state.vectorCalls.length = 0; +state.diffusionCalls.length = 0; +const authorityCandidateResult = await retrieve({ + graph, + userMessage: "只看规则二", + recentMessages: ["assistant: 请聚焦最新规则。"], + embeddingConfig: { + mode: "authority", + source: "authority-trivium", + failOpen: true, + }, + schema, + options: { + topK: 2, + maxRecallNodes: 2, + enableVectorPrefilter: true, + enableGraphDiffusion: false, + enableLLMRecall: false, + authorityCandidateMinCount: 1, + }, + settings: { + authorityGraphQueryEnabled: true, + }, +}); +assert.equal(state.authorityCandidateCalls.length, 1); +assert.deepEqual(state.authorityCandidateCalls[0].availableNodeIds, ["rule-1", "rule-2", "rule-3"]); +assert.equal(authorityCandidateResult.meta.retrieval.authorityCandidateUsed, true); +assert.equal(authorityCandidateResult.meta.retrieval.authorityCandidateCount, 1); +assert.equal(authorityCandidateResult.meta.retrieval.rankingNodeCount, 1); +assert.deepEqual(Array.from(authorityCandidateResult.selectedNodeIds), ["rule-2"]); +state.authorityCandidateEnabled = false; +state.authorityCandidateNodeIds = []; +state.authorityCandidateDiagnostics = null; + state.vectorCalls.length = 0; await retrieve({ graph, diff --git a/vector/authority-vector-primary-adapter.js b/vector/authority-vector-primary-adapter.js index 460dfe3..06ff428 100644 --- a/vector/authority-vector-primary-adapter.js +++ b/vector/authority-vector-primary-adapter.js @@ -36,6 +36,80 @@ function normalizeRecordId(value) { return String(value ?? "").trim(); } +function readNestedValue(source = null, path = []) { + let current = source; + for (const key of path) { + if (!current || typeof current !== "object" || Array.isArray(current)) { + return undefined; + } + current = current[key]; + } + return current; +} + +function normalizeNodeResultId(item = null) { + return normalizeRecordId( + item?.nodeId || + item?.externalId || + item?.id || + readNestedValue(item, ["payload", "nodeId"]) || + readNestedValue(item, ["payload", "externalId"]) || + readNestedValue(item, ["payload", "id"]), + ); +} + +function readResultRows(payload = null) { + if (Array.isArray(payload)) return payload; + if (!payload || typeof payload !== "object") return []; + if (Array.isArray(payload.results)) return payload.results; + if (Array.isArray(payload.hits)) return payload.hits; + if (Array.isArray(payload.items)) return payload.items; + if (Array.isArray(payload.rows)) return payload.rows; + if (Array.isArray(payload.data)) return payload.data; + if (Array.isArray(payload.neighbors)) return payload.neighbors; + if (Array.isArray(payload.links)) return payload.links; + if (Array.isArray(payload.result?.results)) return payload.result.results; + if (Array.isArray(payload.result?.items)) return payload.result.items; + if (Array.isArray(payload.result?.rows)) return payload.result.rows; + if (Array.isArray(payload.result?.data)) return payload.result.data; + if (Array.isArray(payload.result?.neighbors)) return payload.result.neighbors; + if (Array.isArray(payload.result?.links)) return payload.result.links; + return []; +} + +function normalizeNodeIdRows(payload = null) { + const seen = new Set(); + const result = []; + for (const item of readResultRows(payload)) { + const nodeId = normalizeNodeResultId(item); + if (!nodeId || seen.has(nodeId)) continue; + seen.add(nodeId); + result.push(nodeId); + } + return result; +} + +function normalizeNeighborNodeIds(payload = null, seedIds = []) { + const seedSet = new Set((Array.isArray(seedIds) ? seedIds : []).map(normalizeRecordId)); + const seen = new Set(); + const result = []; + for (const item of readResultRows(payload)) { + const directId = normalizeNodeResultId(item); + const preferredId = + normalizeRecordId(item?.neighborId || item?.targetId || item?.toId) || directId; + const alternateId = normalizeRecordId(item?.sourceId || item?.fromId); + const nodeId = !seedSet.has(preferredId) + ? preferredId + : !seedSet.has(alternateId) + ? alternateId + : preferredId; + if (!nodeId || seedSet.has(nodeId) || seen.has(nodeId)) continue; + seen.add(nodeId); + result.push(nodeId); + } + return result; +} + function throwIfAborted(signal) { if (signal?.aborted) { throw signal.reason instanceof Error @@ -54,22 +128,10 @@ function getNodeFieldText(node = {}, keys = []) { } function normalizeSearchResults(payload = null) { - const rows = Array.isArray(payload) - ? payload - : Array.isArray(payload?.results) - ? payload.results - : Array.isArray(payload?.hits) - ? payload.hits - : Array.isArray(payload?.items) - ? payload.items - : Array.isArray(payload?.data) - ? payload.data - : []; + const rows = readResultRows(payload); return rows .map((item, index) => { - const nodeId = normalizeRecordId( - item?.nodeId || item?.externalId || item?.id || item?.payload?.nodeId, - ); + const nodeId = normalizeNodeResultId(item); if (!nodeId) return null; const rawScore = Number(item?.score ?? item?.similarity ?? item?.rankScore); const distance = Number(item?.distance); @@ -230,6 +292,18 @@ export class AuthorityTriviumHttpClient { return await this.request("search", payload); } + async filterWhere(payload = {}) { + return await this.request("filterWhere", payload); + } + + async queryPage(payload = {}) { + return await this.request("queryPage", payload); + } + + async neighbors(payload = {}) { + return await this.request("neighbors", payload); + } + async stat(payload = {}) { return await this.request("stat", payload); } @@ -284,6 +358,31 @@ export async function deleteAuthorityTriviumNodes(config = {}, nodeIds = [], opt }); } +export async function filterAuthorityTriviumNodes(config = {}, options = {}) { + throwIfAborted(options.signal); + const client = createAuthorityTriviumClient(config, options); + const payload = await callClient( + client, + ["filterWhere", "queryPage", "query"], + "filterWhere", + { + namespace: options.namespace, + collectionId: options.collectionId, + chatId: options.chatId, + limit: Number(options.limit || options.topK || 0) || undefined, + topK: Number(options.topK || options.limit || 0) || undefined, + pageSize: Number(options.limit || options.topK || 0) || undefined, + filters: options.filters, + filter: options.filter, + where: options.where, + query: String(options.query || options.searchText || ""), + searchText: String(options.searchText || options.query || ""), + candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean), + }, + ); + return normalizeNodeIdRows(payload); +} + export async function upsertAuthorityTriviumEntries(graph, config = {}, entries = [], options = {}) { const items = buildAuthorityVectorItems(graph, entries, options); if (!items.length) return { upserted: 0 }; @@ -319,6 +418,30 @@ export async function syncAuthorityTriviumLinks(graph, config = {}, options = {} return { linked: links.length }; } +export async function queryAuthorityTriviumNeighbors(config = {}, nodeIds = [], options = {}) { + const ids = toArray(nodeIds).map(normalizeRecordId).filter(Boolean); + if (!ids.length) return []; + throwIfAborted(options.signal); + const client = createAuthorityTriviumClient(config, options); + const payload = await callClient( + client, + ["neighbors", "queryLinks", "queryNeighbors"], + "neighbors", + { + namespace: options.namespace, + collectionId: options.collectionId, + chatId: options.chatId, + ids, + nodeIds: ids, + seedIds: ids, + limit: Number(options.limit || options.topK || 0) || undefined, + topK: Number(options.topK || options.limit || 0) || undefined, + candidateIds: toArray(options.candidateIds).map(normalizeRecordId).filter(Boolean), + }, + ); + return normalizeNeighborNodeIds(payload, ids); +} + export async function searchAuthorityTriviumNodes(graph, text, config = {}, options = {}) { throwIfAborted(options.signal); const client = createAuthorityTriviumClient(config, options);