Narrow trivial input skip rules

This commit is contained in:
Hao19911125
2026-04-05 23:52:59 +08:00
parent e3c819b5cf
commit d77b64e05a
2 changed files with 3 additions and 50 deletions

View File

@@ -21,9 +21,9 @@ function testIsTrivialUserInputTable() {
["/echo hello", true, "slash-command"], ["/echo hello", true, "slash-command"],
["/", true, "slash-command"], ["/", true, "slash-command"],
[" /echo", true, "slash-command"], [" /echo", true, "slash-command"],
["a", true, "under-min-tokens"], ["a", false, ""],
["好", true, "under-min-tokens"], ["好", false, ""],
["ok", true, "under-min-tokens"], ["ok", false, ""],
["ok a", false, ""], ["ok a", false, ""],
["好的", false, ""], ["好的", false, ""],
["好的呀", false, ""], ["好的呀", false, ""],
@@ -68,27 +68,6 @@ async function testSlashCommandSkipsRecallAndExtraction() {
assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null); assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null);
} }
async function testUnderMinTokensSkipsRecallAndExtraction() {
const harness = await createGenerationRecallHarness();
harness.chat = [];
harness.__sendTextareaValue = "a";
const startResult = harness.result.onGenerationStarted("normal", {}, false);
assert.equal(startResult, null);
assert.equal(
harness.result.getCurrentGenerationTrivialSkip()?.reason,
"under-min-tokens",
);
await harness.result.onGenerationAfterCommands("normal", {}, false);
assert.equal(harness.runRecallCalls.length, 0);
harness.chat.push({ is_user: false, mes: "assistant reply" });
harness.invokeOnMessageReceived(0, "");
assert.equal(harness.runExtractionCalls.length, 0);
assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null);
}
async function testEmptyInputSkipsPriorHistoryFallback() { async function testEmptyInputSkipsPriorHistoryFallback() {
const harness = await createGenerationRecallHarness(); const harness = await createGenerationRecallHarness();
harness.chat = [{ is_user: true, mes: "older real user message" }]; harness.chat = [{ is_user: true, mes: "older real user message" }];
@@ -284,7 +263,6 @@ async function testSkipFlagTtlExpires() {
await Promise.resolve(); await Promise.resolve();
testIsTrivialUserInputTable(); testIsTrivialUserInputTable();
await testSlashCommandSkipsRecallAndExtraction(); await testSlashCommandSkipsRecallAndExtraction();
await testUnderMinTokensSkipsRecallAndExtraction();
await testEmptyInputSkipsPriorHistoryFallback(); await testEmptyInputSkipsPriorHistoryFallback();
await testNormalInputStillRecalls(); await testNormalInputStillRecalls();
await testSentinelBlocksHistoryFallback(); await testSentinelBlocksHistoryFallback();

View File

@@ -323,23 +323,6 @@ export function normalizeRecallInputText(value) {
.trim(); .trim();
} }
const TRIVIAL_INPUT_MIN_TOKENS = 2;
const TRIVIAL_INPUT_CJK_TOKEN_REGEX =
/\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul}/gu;
function estimateTrivialInputTokens(text = "") {
const normalized = normalizeRecallInputText(text);
if (!normalized) return 0;
const cjkMatches = normalized.match(TRIVIAL_INPUT_CJK_TOKEN_REGEX) || [];
const nonCjkText = normalized.replace(TRIVIAL_INPUT_CJK_TOKEN_REGEX, " ");
const wordTokens = nonCjkText
.split(/\s+/)
.filter(Boolean);
return cjkMatches.length + wordTokens.length;
}
export function isTrivialUserInput(text) { export function isTrivialUserInput(text) {
const normalizedText = normalizeRecallInputText(text); const normalizedText = normalizeRecallInputText(text);
if (!normalizedText) { if (!normalizedText) {
@@ -358,14 +341,6 @@ export function isTrivialUserInput(text) {
}; };
} }
if (estimateTrivialInputTokens(normalizedText) < TRIVIAL_INPUT_MIN_TOKENS) {
return {
trivial: true,
reason: "under-min-tokens",
normalizedText,
};
}
return { return {
trivial: false, trivial: false,
reason: "", reason: "",