mirror of
https://github.com/Youzini-afk/ST-Bionic-Memory-Ecology.git
synced 2026-05-15 22:30:38 +08:00
Narrow trivial input skip rules
This commit is contained in:
@@ -21,9 +21,9 @@ function testIsTrivialUserInputTable() {
|
|||||||
["/echo hello", true, "slash-command"],
|
["/echo hello", true, "slash-command"],
|
||||||
["/", true, "slash-command"],
|
["/", true, "slash-command"],
|
||||||
[" /echo", true, "slash-command"],
|
[" /echo", true, "slash-command"],
|
||||||
["a", true, "under-min-tokens"],
|
["a", false, ""],
|
||||||
["好", true, "under-min-tokens"],
|
["好", false, ""],
|
||||||
["ok", true, "under-min-tokens"],
|
["ok", false, ""],
|
||||||
["ok a", false, ""],
|
["ok a", false, ""],
|
||||||
["好的", false, ""],
|
["好的", false, ""],
|
||||||
["好的呀", false, ""],
|
["好的呀", false, ""],
|
||||||
@@ -68,27 +68,6 @@ async function testSlashCommandSkipsRecallAndExtraction() {
|
|||||||
assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null);
|
assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null);
|
||||||
}
|
}
|
||||||
|
|
||||||
async function testUnderMinTokensSkipsRecallAndExtraction() {
|
|
||||||
const harness = await createGenerationRecallHarness();
|
|
||||||
harness.chat = [];
|
|
||||||
harness.__sendTextareaValue = "a";
|
|
||||||
|
|
||||||
const startResult = harness.result.onGenerationStarted("normal", {}, false);
|
|
||||||
assert.equal(startResult, null);
|
|
||||||
assert.equal(
|
|
||||||
harness.result.getCurrentGenerationTrivialSkip()?.reason,
|
|
||||||
"under-min-tokens",
|
|
||||||
);
|
|
||||||
|
|
||||||
await harness.result.onGenerationAfterCommands("normal", {}, false);
|
|
||||||
assert.equal(harness.runRecallCalls.length, 0);
|
|
||||||
|
|
||||||
harness.chat.push({ is_user: false, mes: "assistant reply" });
|
|
||||||
harness.invokeOnMessageReceived(0, "");
|
|
||||||
assert.equal(harness.runExtractionCalls.length, 0);
|
|
||||||
assert.equal(harness.result.getCurrentGenerationTrivialSkip(), null);
|
|
||||||
}
|
|
||||||
|
|
||||||
async function testEmptyInputSkipsPriorHistoryFallback() {
|
async function testEmptyInputSkipsPriorHistoryFallback() {
|
||||||
const harness = await createGenerationRecallHarness();
|
const harness = await createGenerationRecallHarness();
|
||||||
harness.chat = [{ is_user: true, mes: "older real user message" }];
|
harness.chat = [{ is_user: true, mes: "older real user message" }];
|
||||||
@@ -284,7 +263,6 @@ async function testSkipFlagTtlExpires() {
|
|||||||
await Promise.resolve();
|
await Promise.resolve();
|
||||||
testIsTrivialUserInputTable();
|
testIsTrivialUserInputTable();
|
||||||
await testSlashCommandSkipsRecallAndExtraction();
|
await testSlashCommandSkipsRecallAndExtraction();
|
||||||
await testUnderMinTokensSkipsRecallAndExtraction();
|
|
||||||
await testEmptyInputSkipsPriorHistoryFallback();
|
await testEmptyInputSkipsPriorHistoryFallback();
|
||||||
await testNormalInputStillRecalls();
|
await testNormalInputStillRecalls();
|
||||||
await testSentinelBlocksHistoryFallback();
|
await testSentinelBlocksHistoryFallback();
|
||||||
|
|||||||
25
ui-status.js
25
ui-status.js
@@ -323,23 +323,6 @@ export function normalizeRecallInputText(value) {
|
|||||||
.trim();
|
.trim();
|
||||||
}
|
}
|
||||||
|
|
||||||
const TRIVIAL_INPUT_MIN_TOKENS = 2;
|
|
||||||
const TRIVIAL_INPUT_CJK_TOKEN_REGEX =
|
|
||||||
/\p{Script=Han}|\p{Script=Hiragana}|\p{Script=Katakana}|\p{Script=Hangul}/gu;
|
|
||||||
|
|
||||||
function estimateTrivialInputTokens(text = "") {
|
|
||||||
const normalized = normalizeRecallInputText(text);
|
|
||||||
if (!normalized) return 0;
|
|
||||||
|
|
||||||
const cjkMatches = normalized.match(TRIVIAL_INPUT_CJK_TOKEN_REGEX) || [];
|
|
||||||
const nonCjkText = normalized.replace(TRIVIAL_INPUT_CJK_TOKEN_REGEX, " ");
|
|
||||||
const wordTokens = nonCjkText
|
|
||||||
.split(/\s+/)
|
|
||||||
.filter(Boolean);
|
|
||||||
|
|
||||||
return cjkMatches.length + wordTokens.length;
|
|
||||||
}
|
|
||||||
|
|
||||||
export function isTrivialUserInput(text) {
|
export function isTrivialUserInput(text) {
|
||||||
const normalizedText = normalizeRecallInputText(text);
|
const normalizedText = normalizeRecallInputText(text);
|
||||||
if (!normalizedText) {
|
if (!normalizedText) {
|
||||||
@@ -358,14 +341,6 @@ export function isTrivialUserInput(text) {
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
if (estimateTrivialInputTokens(normalizedText) < TRIVIAL_INPUT_MIN_TOKENS) {
|
|
||||||
return {
|
|
||||||
trivial: true,
|
|
||||||
reason: "under-min-tokens",
|
|
||||||
normalizedText,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
return {
|
||||||
trivial: false,
|
trivial: false,
|
||||||
reason: "",
|
reason: "",
|
||||||
|
|||||||
Reference in New Issue
Block a user