fix(authority): distinguish embedding and Trivium failures

Ultraworked with [Sisyphus](https://github.com/code-yeongyu/oh-my-openagent)

Co-authored-by: Sisyphus <clio-agent@sisyphuslabs.ai>
This commit is contained in:
youzini
2026-05-09 09:06:19 +00:00
parent 880e2b9158
commit f443b388ee
3 changed files with 163 additions and 7 deletions

View File

@@ -1,5 +1,6 @@
import assert from "node:assert/strict"; import assert from "node:assert/strict";
import { addEdge, addNode, createEdge, createEmptyGraph, createNode } from "../graph/graph.js"; import { addEdge, addNode, createEdge, createEmptyGraph, createNode } from "../graph/graph.js";
import { AuthorityHttpError } from "../runtime/authority-http-client.js";
import { import {
installResolveHooks, installResolveHooks,
toDataModuleUrl, toDataModuleUrl,
@@ -33,7 +34,10 @@ const {
normalizeAuthorityVectorConfig, normalizeAuthorityVectorConfig,
queryAuthorityTriviumNeighbors, queryAuthorityTriviumNeighbors,
} = await import("../vector/authority-vector-primary-adapter.js"); } = await import("../vector/authority-vector-primary-adapter.js");
const { findSimilarNodesByText: findSimilarNodesByTextFromIndex, syncGraphVectorIndex: syncGraphVectorIndexFromIndex } = await import("../vector/vector-index.js"); const {
findSimilarNodesByText: findSimilarNodesByTextFromIndex,
syncGraphVectorIndex: syncGraphVectorIndexFromIndex,
} = await import("../vector/vector-index.js");
function createAuthorityVectorGraph() { function createAuthorityVectorGraph() {
const graph = createEmptyGraph(); const graph = createEmptyGraph();
@@ -66,7 +70,7 @@ function createAuthorityVectorGraph() {
return { graph, first, second }; return { graph, first, second };
} }
function createMockTriviumClient({ failBulkUpsert = false } = {}) { function createMockTriviumClient({ failBulkUpsert = false, failSearch = false } = {}) {
const calls = []; const calls = [];
return { return {
calls, calls,
@@ -88,7 +92,11 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) {
async bulkUpsert(payload) { async bulkUpsert(payload) {
calls.push(["bulkUpsert", payload]); calls.push(["bulkUpsert", payload]);
if (failBulkUpsert) { if (failBulkUpsert) {
throw new Error("trivium-down"); throw new AuthorityHttpError("trivium-down", {
status: 503,
category: "server",
path: "/trivium/bulk-upsert",
});
} }
return { ok: true, upserted: payload.items?.length || 0 }; return { ok: true, upserted: payload.items?.length || 0 };
}, },
@@ -102,6 +110,13 @@ function createMockTriviumClient({ failBulkUpsert = false } = {}) {
}, },
async search(payload) { async search(payload) {
calls.push(["search", payload]); calls.push(["search", payload]);
if (failSearch) {
throw new AuthorityHttpError("trivium search denied", {
status: 403,
category: "permission",
path: "/trivium/search",
});
}
return { return {
results: [ results: [
{ nodeId: "node-b", score: 0.91 }, { nodeId: "node-b", score: 0.91 },
@@ -234,9 +249,77 @@ assert.equal(isAuthorityVectorConfig(config), true);
assert.equal(graph.vectorIndexState.mode, "authority"); assert.equal(graph.vectorIndexState.mode, "authority");
assert.equal(graph.vectorIndexState.dirty, true); assert.equal(graph.vectorIndexState.dirty, true);
assert.equal(graph.vectorIndexState.dirtyReason, "authority-trivium-sync-failed"); assert.equal(graph.vectorIndexState.dirtyReason, "authority-trivium-sync-failed");
assert.equal(result.errorCategory, "server");
assert.equal(result.errorDomain, "authority");
assert.equal(result.timings.errorCategory, "server");
assert.equal(result.timings.authorityErrorCategory, "server");
assert.equal(graph.vectorIndexState.lastErrorCategory, "server");
assert.equal(graph.vectorIndexState.lastErrorDomain, "authority");
assert.equal(result.timings.authorityDiagnostics.upsert.errorCategory, "server");
assert.equal(result.timings.authorityDiagnostics.upsert.chunks[0].errorCategory, "server");
assert.match(graph.vectorIndexState.lastWarning, /Authority Trivium 同步失败/); assert.match(graph.vectorIndexState.lastWarning, /Authority Trivium 同步失败/);
} }
{
const previousOverrides = globalThis.__stBmeTestOverrides;
globalThis.__stBmeTestOverrides = {
embedding: {
async embedBatch(texts = []) {
return texts.map(() => null);
},
async embedText() {
return null;
},
},
};
try {
const { graph } = createAuthorityVectorGraph();
graph.nodes.forEach((node) => {
node.embedding = null;
});
const triviumClient = createMockTriviumClient();
const result = await syncGraphVectorIndexFromIndex(graph, config, {
chatId: "chat-authority-vector",
purge: true,
triviumClient,
});
assert.match(result.error, /Embedding provider failed/);
assert.doesNotMatch(result.error, /Authority Trivium embedding failed/);
assert.equal(result.errorCategory, "embedding-provider");
assert.equal(result.errorDomain, "embedding");
assert.equal(graph.vectorIndexState.dirtyReason, "embedding-provider-sync-failed");
assert.equal(graph.vectorIndexState.lastErrorCategory, "embedding-provider");
assert.equal(graph.vectorIndexState.lastErrorDomain, "embedding");
assert.match(graph.vectorIndexState.lastWarning, /Embedding provider 同步失败/);
assert.equal(triviumClient.calls.some(([name]) => name === "bulkUpsert"), false);
} finally {
globalThis.__stBmeTestOverrides = previousOverrides;
}
}
{
const { graph, first, second } = createAuthorityVectorGraph();
const triviumClient = createMockTriviumClient({ failSearch: true });
const queryConfig = { ...config, triviumClient };
await syncGraphVectorIndexFromIndex(graph, queryConfig, {
chatId: "chat-authority-vector",
purge: true,
triviumClient,
});
const results = await findSimilarNodesByTextFromIndex(
graph,
"archive door",
queryConfig,
5,
[first, second],
);
assert.deepEqual(results, []);
assert.equal(graph.vectorIndexState.lastSearchTimings.errorCategory, "permission");
assert.equal(graph.vectorIndexState.lastSearchTimings.authorityErrorCategory, "permission");
assert.equal(graph.vectorIndexState.lastErrorCategory, "permission");
assert.equal(graph.vectorIndexState.lastErrorDomain, "authority");
}
{ {
const triviumClient = createMockTriviumClient(); const triviumClient = createMockTriviumClient();
const queryConfig = { ...config, triviumClient }; const queryConfig = { ...config, triviumClient };

View File

@@ -2,6 +2,7 @@ import { normalizeAuthorityBaseUrl } from "../runtime/authority-capabilities.js"
import { import {
AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06, AUTHORITY_PROTOCOL_SERVER_PLUGIN_V06,
AuthorityHttpClient, AuthorityHttpClient,
AuthorityHttpError,
} from "../runtime/authority-http-client.js"; } from "../runtime/authority-http-client.js";
import { embedText } from "./embedding.js"; import { embedText } from "./embedding.js";
@@ -89,6 +90,25 @@ function hasPlainKeys(value = null) {
return isPlainObject(value) && Object.keys(value).length > 0; return isPlainObject(value) && Object.keys(value).length > 0;
} }
function getAuthorityErrorCategory(error = null) {
return String(error?.category || error?.errorCategory || "").trim();
}
function getAuthorityErrorDomain(error = null) {
if (!error) return "";
return error instanceof AuthorityHttpError || getAuthorityErrorCategory(error) ? "authority" : "";
}
function buildAuthorityErrorDiagnostics(error = null) {
const category = getAuthorityErrorCategory(error);
const domain = getAuthorityErrorDomain(error);
return {
...(category ? { errorCategory: category, authorityErrorCategory: category } : {}),
...(domain ? { errorDomain: domain, authorityErrorDomain: domain } : {}),
...(Number(error?.status || 0) > 0 ? { status: Number(error.status) } : {}),
};
}
function normalizeOpenAICompatibleBaseUrl(value) { function normalizeOpenAICompatibleBaseUrl(value) {
return String(value || "") return String(value || "")
.trim() .trim()
@@ -816,6 +836,7 @@ export async function upsertAuthorityTriviumEntries(graph, config = {}, entries
durationMs: roundMs(nowMs() - chunkStartedAt), durationMs: roundMs(nowMs() - chunkStartedAt),
ok: false, ok: false,
error: error?.message || String(error), error: error?.message || String(error),
...buildAuthorityErrorDiagnostics(error),
}); });
error.authorityDiagnostics = { error.authorityDiagnostics = {
operation: "bulkUpsert", operation: "bulkUpsert",
@@ -824,6 +845,7 @@ export async function upsertAuthorityTriviumEntries(graph, config = {}, entries
chunks, chunks,
totalBytes, totalBytes,
totalMs: roundMs(nowMs() - startedAt), totalMs: roundMs(nowMs() - startedAt),
...buildAuthorityErrorDiagnostics(error),
}; };
throw error; throw error;
} }

View File

@@ -626,6 +626,7 @@ function markAuthorityVectorStateDirty(
config = {}, config = {},
reason = "authority-trivium-failed", reason = "authority-trivium-failed",
warning = "Authority Trivium 索引失败,已标记待重建", warning = "Authority Trivium 索引失败,已标记待重建",
diagnostics = {},
) { ) {
if (!graph?.vectorIndexState || !isAuthorityVectorConfig(config)) { if (!graph?.vectorIndexState || !isAuthorityVectorConfig(config)) {
return; return;
@@ -655,6 +656,39 @@ function markAuthorityVectorStateDirty(
pending: total > 0 ? Math.max(1, Number(state.lastStats?.pending || 0)) : 0, pending: total > 0 ? Math.max(1, Number(state.lastStats?.pending || 0)) : 0,
}; };
state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建"); state.lastWarning = String(warning || "Authority Trivium 索引失败,已标记待重建");
const errorCategory = String(diagnostics.errorCategory || diagnostics.authorityErrorCategory || "").trim();
const errorDomain = String(diagnostics.errorDomain || diagnostics.authorityErrorDomain || "").trim();
if (errorCategory) state.lastErrorCategory = errorCategory;
if (errorDomain) state.lastErrorDomain = errorDomain;
}
function getErrorCategory(error = null) {
return String(error?.category || error?.errorCategory || "").trim();
}
function getErrorDomain(error = null, fallback = "") {
if (!error) return "";
if (error?.errorDomain) return String(error.errorDomain).trim();
if (getErrorCategory(error)) return fallback || "authority";
return fallback;
}
function getAuthorityDiagnosticsErrorPatch(error = null) {
const errorCategory = getErrorCategory(error);
const errorDomain = getErrorDomain(error, errorCategory ? "authority" : "");
return {
...(errorCategory ? { errorCategory, authorityErrorCategory: errorCategory } : {}),
...(errorDomain ? { errorDomain, authorityErrorDomain: errorDomain } : {}),
...(Number(error?.status || 0) > 0 ? { status: Number(error.status) } : {}),
};
}
function createEmbeddingProviderError(failures = 0) {
const count = Math.max(0, Math.floor(Number(failures) || 0));
const error = new Error(`Embedding provider failed for ${count} item(s)`);
error.errorCategory = "embedding-provider";
error.errorDomain = "embedding";
return error;
} }
async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) { async function ensureEntryEmbeddings(graph, entries = [], config = {}, signal = undefined) {
@@ -802,7 +836,7 @@ export async function syncGraphVectorIndex(
embeddingsRequested += embeddingResult.requested; embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs; embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) { if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`); throw createEmbeddingProviderError(embeddingResult.failures);
} }
const purgeStartedAt = nowMs(); const purgeStartedAt = nowMs();
const purgeResult = await purgeAuthorityTriviumNamespace(config, authorityOptions); const purgeResult = await purgeAuthorityTriviumNamespace(config, authorityOptions);
@@ -866,7 +900,7 @@ export async function syncGraphVectorIndex(
embeddingsRequested += embeddingResult.requested; embeddingsRequested += embeddingResult.requested;
embedBatchMs += embeddingResult.elapsedMs; embedBatchMs += embeddingResult.elapsedMs;
if (embeddingResult.failures > 0) { if (embeddingResult.failures > 0) {
throw new Error(`Authority Trivium embedding failed for ${embeddingResult.failures} item(s)`); throw createEmbeddingProviderError(embeddingResult.failures);
} }
deletedNodeCount = nodeIdsToDelete.length; deletedNodeCount = nodeIdsToDelete.length;
const deleteStartedAt = nowMs(); const deleteStartedAt = nowMs();
@@ -909,17 +943,29 @@ export async function syncGraphVectorIndex(
} catch (error) { } catch (error) {
if (isAbortError(error)) throw error; if (isAbortError(error)) throw error;
const message = error?.message || String(error) || "Authority Trivium 同步失败"; const message = error?.message || String(error) || "Authority Trivium 同步失败";
const errorCategory = getErrorCategory(error);
const errorDomain = getErrorDomain(error, errorCategory ? "authority" : "");
const dirtyReason = errorDomain === "embedding"
? "embedding-provider-sync-failed"
: "authority-trivium-sync-failed";
const warningPrefix = errorDomain === "embedding"
? "Embedding provider 同步失败"
: "Authority Trivium 同步失败";
markAuthorityVectorStateDirty( markAuthorityVectorStateDirty(
graph, graph,
config, config,
"authority-trivium-sync-failed", dirtyReason,
`Authority Trivium 同步失败${message}),已标记待重建`, `${warningPrefix}${message}),已标记待重建`,
{ errorCategory, errorDomain },
); );
state.lastSyncAt = Date.now(); state.lastSyncAt = Date.now();
state.lastTimings = { state.lastTimings = {
mode: syncMode, mode: syncMode,
success: false, success: false,
error: message, error: message,
...(errorCategory ? { errorCategory } : {}),
...(errorDomain ? { errorDomain } : {}),
...(errorCategory && errorDomain === "authority" ? { authorityErrorCategory: errorCategory, authorityErrorDomain: errorDomain } : {}),
desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length), desiredEntries: Number(desiredBuildDiagnostics.entryCount || desiredEntries.length),
desiredBuildMs: roundMs(desiredBuildMs), desiredBuildMs: roundMs(desiredBuildMs),
authorityPurgeMs: roundMs(authorityPurgeMs), authorityPurgeMs: roundMs(authorityPurgeMs),
@@ -940,6 +986,8 @@ export async function syncGraphVectorIndex(
stats: state.lastStats, stats: state.lastStats,
timings: state.lastTimings, timings: state.lastTimings,
error: message, error: message,
...(errorCategory ? { errorCategory } : {}),
...(errorDomain ? { errorDomain } : {}),
}; };
if (config.failOpen === false) { if (config.failOpen === false) {
throw error; throw error;
@@ -1291,17 +1339,20 @@ export async function findSimilarNodesByText(
throw error; throw error;
} }
const message = error?.message || String(error) || "Authority Trivium 查询失败"; const message = error?.message || String(error) || "Authority Trivium 查询失败";
const errorPatch = getAuthorityDiagnosticsErrorPatch(error);
markAuthorityVectorStateDirty( markAuthorityVectorStateDirty(
graph, graph,
config, config,
"authority-trivium-query-failed", "authority-trivium-query-failed",
`Authority Trivium 查询失败(${message}),已标记待重建`, `Authority Trivium 查询失败(${message}),已标记待重建`,
errorPatch,
); );
recordSearchTimings({ recordSearchTimings({
success: false, success: false,
reason: "authority-trivium-query-failed", reason: "authority-trivium-query-failed",
requestMs: roundMs(nowMs() - requestStartedAt), requestMs: roundMs(nowMs() - requestStartedAt),
error: message, error: message,
...errorPatch,
resultCount: 0, resultCount: 0,
}); });
if (config.failOpen === false) { if (config.failOpen === false) {