Reorganize modules into layered directories

prompting/injection-sanitizer.js

@@ -0,0 +1,463 @@
+import { sanitizeMvuContent } from "./mvu-compat.js";
+import { applyHostRegexReuse } from "./task-regex.js";
+
+export const PROMPT_CONTENT_ORIGIN = Object.freeze({
+  TEMPLATE_OWNED: "template-owned",
+  HOST_INJECTED: "host-injected",
+  WORLD_INFO_RENDERED: "world-info-rendered",
+});
+
+function normalizeSanitizerMode(mode = "injection-safe") {
+  return String(mode || "").trim() === "final-injection-safe"
+    ? "final-safe"
+    : "aggressive";
+}
+
+function isSanitizationEligible(options = {}) {
+  if (options?.sanitizationEligible === false) {
+    return false;
+  }
+  return String(options?.contentOrigin || "") !== PROMPT_CONTENT_ORIGIN.TEMPLATE_OWNED;
+}
+
+function normalizeReasons(reasons = []) {
+  return Array.isArray(reasons)
+    ? reasons.map((item) => String(item || "").trim()).filter(Boolean)
+    : [];
+}
+
+function pushUnique(target = [], value = "") {
+  const normalized = String(value || "").trim();
+  if (!normalized || target.includes(normalized)) {
+    return;
+  }
+  target.push(normalized);
+}
+
+export function createEmptyInjectionSanitizerDebug() {
+  return {
+    sanitizedFieldCount: 0,
+    sanitizedFields: [],
+    finalMessageStripCount: 0,
+    worldInfoBlockedContentHits: 0,
+    sanitizerAppliedFields: [],
+    sanitizerHitKinds: [],
+    hostReuseAppliedFields: [],
+    hostReuseSkippedDisplayOnlyRules: 0,
+    regexExecutionMode: "host-unavailable",
+    hostFormatterAvailable: false,
+    hostFormatterSource: "",
+    fallbackReason: "",
+  };
+}
+
+function recordSanitizerDebug(debugState, path, result = {}, stage = "") {
+  if (!debugState || (!result.changed && !result.dropped)) {
+    return;
+  }
+
+  const reasons = normalizeReasons(result.reasons);
+  debugState.sanitizedFields.push({
+    name: String(path || ""),
+    stage: String(stage || ""),
+    changed: Boolean(result.changed),
+    dropped: Boolean(result.dropped),
+    reasons,
+    blockedHitCount: Number(result.blockedHitCount || 0),
+  });
+  debugState.sanitizedFieldCount = debugState.sanitizedFields.length;
+  pushUnique(debugState.sanitizerAppliedFields, path);
+  for (const reason of reasons) {
+    pushUnique(debugState.sanitizerHitKinds, reason);
+  }
+}
+
+function recordHostReuseDebug(debugState, path, result = {}) {
+  if (!debugState || !result || typeof result !== "object") {
+    return;
+  }
+  debugState.regexExecutionMode = String(
+    result.executionMode || debugState.regexExecutionMode || "host-unavailable",
+  );
+  debugState.hostFormatterAvailable = Boolean(result.formatterAvailable);
+  debugState.hostFormatterSource = String(result.formatterSource || "");
+  debugState.fallbackReason = String(result.fallbackReason || "");
+  debugState.hostReuseSkippedDisplayOnlyRules = Math.max(
+    Number(debugState.hostReuseSkippedDisplayOnlyRules || 0),
+    Number(result.skippedDisplayOnlyRuleCount || 0),
+  );
+  if (result.changed) {
+    pushUnique(debugState.hostReuseAppliedFields, path);
+  }
+}
+
+export function sanitizeInjectionText(
+  settings = {},
+  taskType,
+  text,
+  {
+    mode = "injection-safe",
+    blockedContents = [],
+    contentOrigin = PROMPT_CONTENT_ORIGIN.HOST_INJECTED,
+    sanitizationEligible = true,
+    regexSourceType = "",
+    role = "system",
+    formatterOptions = null,
+    debugState = null,
+    regexCollector = null,
+    applySanitizer = true,
+    applyHostRegex = true,
+    path = "",
+    stage = "",
+  } = {},
+) {
+  const originalText = typeof text === "string" ? text : "";
+  const eligible = sanitizationEligible && isSanitizationEligible({
+    sanitizationEligible,
+    contentOrigin,
+  });
+
+  const sanitizerResult = eligible && applySanitizer
+    ? sanitizeMvuContent(originalText, {
+        mode: normalizeSanitizerMode(mode),
+        blockedContents,
+      })
+    : {
+        text: originalText,
+        changed: false,
+        dropped: false,
+        reasons: [],
+        blockedHitCount: 0,
+        artifactRemovedCount: 0,
+      };
+
+  recordSanitizerDebug(debugState, path, sanitizerResult, stage);
+
+  const afterSanitizer = String(sanitizerResult.text || "");
+  const hostReuseResult = eligible && applyHostRegex && regexSourceType
+    ? applyHostRegexReuse(settings, taskType, afterSanitizer, {
+        sourceType: regexSourceType,
+        role,
+        debugCollector: regexCollector,
+        formatterOptions,
+      })
+    : {
+        text: afterSanitizer,
+        changed: false,
+        executionMode: "host-unavailable",
+        formatterAvailable: false,
+        formatterSource: "",
+        fallbackReason: "",
+        skippedDisplayOnlyRuleCount: 0,
+      };
+
+  recordHostReuseDebug(debugState, path, hostReuseResult);
+
+  const finalText = String(hostReuseResult.text || "");
+  return {
+    text: finalText,
+    changed: finalText !== originalText,
+    dropped: Boolean(sanitizerResult.dropped),
+    reasons: normalizeReasons(sanitizerResult.reasons),
+    blockedHitCount: Number(sanitizerResult.blockedHitCount || 0),
+    artifactRemovedCount: Number(sanitizerResult.artifactRemovedCount || 0),
+    hostReuseChanged: Boolean(hostReuseResult.changed),
+    executionMode: String(hostReuseResult.executionMode || "host-unavailable"),
+    formatterAvailable: Boolean(hostReuseResult.formatterAvailable),
+    formatterSource: String(hostReuseResult.formatterSource || ""),
+    fallbackReason: String(hostReuseResult.fallbackReason || ""),
+    skippedDisplayOnlyRuleCount: Number(
+      hostReuseResult.skippedDisplayOnlyRuleCount || 0,
+    ),
+  };
+}
+
+function looksLikeMvuStateContainer(value, seen = new WeakSet()) {
+  if (!value || typeof value !== "object") {
+    return false;
+  }
+  if (seen.has(value)) {
+    return false;
+  }
+  seen.add(value);
+
+  if (Array.isArray(value)) {
+    return value.some((item) => looksLikeMvuStateContainer(item, seen));
+  }
+
+  const keys = Object.keys(value).map((key) =>
+    String(key || "").trim().toLowerCase(),
+  );
+  if (
+    keys.some((key) =>
+      ["stat_data", "display_data", "delta_data", "$internal"].includes(key),
+    )
+  ) {
+    return true;
+  }
+
+  return Object.values(value).some((item) => looksLikeMvuStateContainer(item, seen));
+}
+
+function getMvuObjectKeyStripReason(key, value) {
+  const normalizedKey = String(key || "").trim().toLowerCase();
+  if (
+    ["stat_data", "display_data", "delta_data", "$internal"].includes(
+      normalizedKey,
+    )
+  ) {
+    return "mvu_state_key_removed";
+  }
+  if (
+    ["variables", "message_variables", "chat_variables"].includes(normalizedKey) &&
+    looksLikeMvuStateContainer(value)
+  ) {
+    return "mvu_variables_container_removed";
+  }
+  return "";
+}
+
+function joinStructuredPath(basePath = "", segment = "") {
+  const normalizedSegment = String(segment || "");
+  if (!normalizedSegment) {
+    return basePath;
+  }
+  if (!basePath) {
+    return normalizedSegment.startsWith("[")
+      ? normalizedSegment.slice(1, -1)
+      : normalizedSegment;
+  }
+  return normalizedSegment.startsWith("[")
+    ? `${basePath}${normalizedSegment}`
+    : `${basePath}.${normalizedSegment}`;
+}
+
+export function sanitizeInjectionStructuredValue(
+  settings = {},
+  taskType,
+  value,
+  {
+    fieldName = "",
+    path = fieldName,
+    mode = "injection-safe",
+    blockedContents = [],
+    contentOrigin = PROMPT_CONTENT_ORIGIN.HOST_INJECTED,
+    sanitizationEligible = true,
+    regexSourceType = "",
+    role = "system",
+    formatterOptions = null,
+    debugState = null,
+    regexCollector = null,
+    applySanitizer = true,
+    applyHostRegex = true,
+    stripMvuContainers = true,
+    seen = new WeakSet(),
+  } = {},
+) {
+  if (typeof value === "string") {
+    const sanitized = sanitizeInjectionText(settings, taskType, value, {
+      mode,
+      blockedContents,
+      contentOrigin,
+      sanitizationEligible,
+      regexSourceType,
+      role,
+      formatterOptions,
+      debugState,
+      regexCollector,
+      applySanitizer,
+      applyHostRegex,
+      path,
+      stage: mode,
+    });
+    return {
+      value: sanitized.text,
+      changed: Boolean(sanitized.changed || sanitized.dropped),
+      omit:
+        !String(sanitized.text || "").trim() &&
+        String(value || "").trim().length > 0,
+      details: sanitized,
+    };
+  }
+
+  if (Array.isArray(value)) {
+    const sanitizedArray = [];
+    let changed = false;
+    for (let index = 0; index < value.length; index += 1) {
+      const childResult = sanitizeInjectionStructuredValue(
+        settings,
+        taskType,
+        value[index],
+        {
+          fieldName,
+          path: joinStructuredPath(path, `[${index}]`),
+          mode,
+          blockedContents,
+          contentOrigin,
+          sanitizationEligible,
+          regexSourceType,
+          role,
+          formatterOptions,
+          debugState,
+          regexCollector,
+          applySanitizer,
+          applyHostRegex,
+          stripMvuContainers,
+          seen,
+        },
+      );
+      if (childResult.omit) {
+        changed = true;
+        continue;
+      }
+      sanitizedArray.push(childResult.value);
+      if (childResult.changed) {
+        changed = true;
+      }
+    }
+    return {
+      value: sanitizedArray,
+      changed: changed || sanitizedArray.length !== value.length,
+      omit: value.length > 0 && sanitizedArray.length === 0,
+      details: null,
+    };
+  }
+
+  if (value && typeof value === "object") {
+    if (seen.has(value)) {
+      return {
+        value,
+        changed: false,
+        omit: false,
+        details: null,
+      };
+    }
+    seen.add(value);
+
+    const originalLooksMvuContainer = looksLikeMvuStateContainer(value);
+    const sanitizedObject = {};
+    let changed = false;
+    let keptEntries = 0;
+
+    for (const [key, entryValue] of Object.entries(value)) {
+      const stripReason = stripMvuContainers
+        ? getMvuObjectKeyStripReason(key, entryValue)
+        : "";
+      if (stripReason) {
+        changed = true;
+        recordSanitizerDebug(
+          debugState,
+          joinStructuredPath(path, key),
+          {
+            changed: true,
+            dropped: true,
+            reasons: [stripReason],
+            blockedHitCount: 0,
+          },
+          mode,
+        );
+        continue;
+      }
+
+      const childResult = sanitizeInjectionStructuredValue(
+        settings,
+        taskType,
+        entryValue,
+        {
+          fieldName,
+          path: joinStructuredPath(path, key),
+          mode,
+          blockedContents,
+          contentOrigin,
+          sanitizationEligible,
+          regexSourceType,
+          role,
+          formatterOptions,
+          debugState,
+          regexCollector,
+          applySanitizer,
+          applyHostRegex,
+          stripMvuContainers,
+          seen,
+        },
+      );
+      if (childResult.omit) {
+        changed = true;
+        continue;
+      }
+      sanitizedObject[key] = childResult.value;
+      keptEntries += 1;
+      if (childResult.changed) {
+        changed = true;
+      }
+    }
+
+    return {
+      value: sanitizedObject,
+      changed,
+      omit: originalLooksMvuContainer && keptEntries === 0,
+      details: null,
+    };
+  }
+
+  return {
+    value,
+    changed: false,
+    omit: false,
+    details: null,
+  };
+}
+
+export function sanitizeInjectionMessages(
+  settings = {},
+  taskType,
+  messages = [],
+  {
+    blockedContents = [],
+    debugState = null,
+    regexCollector = null,
+  } = {},
+) {
+  return (Array.isArray(messages) ? messages : [])
+    .map((message, index) => {
+      const contentOrigin = String(message?.contentOrigin || "").trim() ||
+        PROMPT_CONTENT_ORIGIN.TEMPLATE_OWNED;
+      const sanitizationEligible =
+        message?.sanitizationEligible === true &&
+        contentOrigin !== PROMPT_CONTENT_ORIGIN.TEMPLATE_OWNED;
+      if (!sanitizationEligible) {
+        return message;
+      }
+
+      const sanitized = sanitizeInjectionText(
+        settings,
+        taskType,
+        String(message?.content || ""),
+        {
+          mode: "final-injection-safe",
+          blockedContents,
+          contentOrigin,
+          sanitizationEligible,
+          regexSourceType: String(message?.regexSourceType || ""),
+          role: message?.role || "system",
+          debugState,
+          regexCollector,
+          applySanitizer: true,
+          applyHostRegex: false,
+          path: `message[${index}]`,
+          stage: "final-injection-safe",
+        },
+      );
+      if (debugState && (sanitized.changed || sanitized.dropped)) {
+        debugState.finalMessageStripCount += 1;
+      }
+      if (!String(sanitized.text || "").trim()) {
+        return null;
+      }
+      return {
+        ...message,
+        content: sanitized.text,
+      };
+    })
+    .filter(Boolean);
+}

prompting/mvu-compat.js

@@ -0,0 +1,237 @@
+// ST-BME: MVU (MagVarUpdate) compatibility helpers for private task prompts.
+// These rules are intentionally narrow so we strip MVU artifacts without
+// disturbing normal prompt or world info content.
+
+export const MVU_ENTRY_COMMENT_REGEX = /\[(mvu_update|mvu_plot|initvar)\]/i;
+
+const MVU_UPDATE_BLOCK_REGEX =
+  /\n?<(update(?:variable)?|variableupdate)>(?:(?!<\1>).)*?<\/\1>/gis;
+const MVU_STATUS_PLACEHOLDER_REGEX = /\n?<StatusPlaceHolderImpl\/>/gi;
+const MVU_STATUS_CURRENT_VARIABLE_REPLACE_REGEX =
+  /\n?<status_current_variables?>[\s\S]*?<\/status_current_variables?>/gi;
+const MVU_STATUS_CURRENT_VARIABLE_DETECT_REGEX =
+  /<status_current_variables?>[\s\S]*?<\/status_current_variables?>/i;
+const MVU_MESSAGE_VARIABLE_MACRO_REGEX =
+  /\{\{\s*get_message_variable::(?:stat_data|display_data|delta_data)(?:\.[^}]+)?\s*}}/gi;
+const MVU_GETVAR_REFERENCE_REGEX =
+  /getvar\(\s*["'](?:stat_data|display_data|delta_data)["']\s*\)/gi;
+const MVU_STATEFUL_TEMPLATE_TAG_REGEX =
+  /<%[-=]?[\s\S]*?(?:SafeGetValue|getvar\(\s*["'](?:stat_data|display_data|delta_data)["']\s*\)|\b(?:stat_data|display_data|delta_data)\b)[\s\S]*?%>/gi;
+const EJS_TEMPLATE_TAG_REGEX = /<%[-=]?[\s\S]*?%>/gi;
+const MVU_VARIABLE_OUTPUT_ENTRY_REGEX = /变量输出格式:\s*[\s\S]*?<UpdateVariable>/i;
+const MVU_VARIABLE_RULES_ENTRY_REGEX =
+  /变量更新规则:\s*[\s\S]*?(?:type:\s*|check:\s*|当前时间:|近期事务:)/i;
+const MVU_FORMAT_EMPHASIS_ENTRY_REGEX =
+  /(?:变量输出格式强调|格式强调[：:]?-?变量更新规则|格式强调[：:]?-?剧情演绎|The following must be inserted to the end of (?:each )?reply,? and cannot be omitted)[\s\S]*?format:\s*\|-?/i;
+const MVU_STATE_OBJECT_FIELD_REGEX =
+  /["']?(?:stat_data|display_data|delta_data)["']?\s*:/i;
+const MVU_STATE_PATH_REFERENCE_REGEX =
+  /\b(?:stat_data|display_data|delta_data)(?:\.[\w$\u4e00-\u9fff\[\]"'-]+){1,}/i;
+const MVU_STATE_HELPER_REFERENCE_REGEX =
+  /\b(?:SafeGetValue\([^)]*(?:stat_data|display_data|delta_data)[^)]*\)|message_data\[\d+\]\.data\.(?:stat_data|display_data|delta_data))\b/i;
+
+function uniq(values = []) {
+  return [...new Set((Array.isArray(values) ? values : []).filter(Boolean))];
+}
+
+function normalizeText(value = "") {
+  return String(value || "").replace(/\r\n/g, "\n");
+}
+
+function collapseWhitespace(value = "") {
+  return String(value || "")
+    .replace(/[ \t]{2,}/g, " ")
+    .replace(/\n{3,}/g, "\n\n")
+    .trim();
+}
+
+function countRegexMatches(text = "", regex) {
+  if (!text || !(regex instanceof RegExp)) {
+    return 0;
+  }
+
+  const source = new RegExp(regex.source, regex.flags);
+  let count = 0;
+  while (source.exec(text)) {
+    count += 1;
+  }
+  return count;
+}
+
+function matchesRegex(text = "", regex) {
+  if (!text || !(regex instanceof RegExp)) {
+    return false;
+  }
+
+  return new RegExp(regex.source, regex.flags).test(text);
+}
+
+function stripMvuPromptArtifactsDetailed(content = "") {
+  const input = normalizeText(content);
+  if (!input) {
+    return {
+      text: "",
+      changed: false,
+      artifactRemovedCount: 0,
+    };
+  }
+
+  const statefulTemplateTagCount = countRegexMatches(
+    input,
+    MVU_STATEFUL_TEMPLATE_TAG_REGEX,
+  );
+  const artifactRemovedCount =
+    countRegexMatches(input, MVU_UPDATE_BLOCK_REGEX) +
+    countRegexMatches(input, MVU_STATUS_PLACEHOLDER_REGEX) +
+    countRegexMatches(input, MVU_STATUS_CURRENT_VARIABLE_REPLACE_REGEX) +
+    countRegexMatches(input, MVU_MESSAGE_VARIABLE_MACRO_REGEX) +
+    countRegexMatches(input, MVU_GETVAR_REFERENCE_REGEX) +
+    statefulTemplateTagCount;
+
+  let stripped = input
+    .replace(MVU_UPDATE_BLOCK_REGEX, "")
+    .replace(MVU_STATUS_PLACEHOLDER_REGEX, "")
+    .replace(MVU_STATUS_CURRENT_VARIABLE_REPLACE_REGEX, "")
+    .replace(MVU_MESSAGE_VARIABLE_MACRO_REGEX, "")
+    .replace(MVU_GETVAR_REFERENCE_REGEX, "")
+    .replace(MVU_STATEFUL_TEMPLATE_TAG_REGEX, "");
+  if (statefulTemplateTagCount > 0) {
+    stripped = stripped.replace(EJS_TEMPLATE_TAG_REGEX, "");
+  }
+
+  const normalized = collapseWhitespace(stripped);
+  return {
+    text: normalized,
+    changed: normalized !== collapseWhitespace(input),
+    artifactRemovedCount,
+  };
+}
+
+function stripBlockedPromptContentsDetailed(content = "", blockedContents = []) {
+  const input = normalizeText(content);
+  const normalizedBlocked = uniq(
+    (Array.isArray(blockedContents) ? blockedContents : [])
+      .map((item) => collapseWhitespace(item))
+      .filter(Boolean)
+      .sort((left, right) => right.length - left.length),
+  );
+
+  if (!input || normalizedBlocked.length === 0) {
+    return {
+      text: collapseWhitespace(input),
+      changed: false,
+      blockedHitCount: 0,
+    };
+  }
+
+  let output = input;
+  let blockedHitCount = 0;
+  for (const blocked of normalizedBlocked) {
+    let index = output.indexOf(blocked);
+    while (index >= 0) {
+      blockedHitCount += 1;
+      output = `${output.slice(0, index)}${output.slice(index + blocked.length)}`;
+      index = output.indexOf(blocked);
+    }
+  }
+
+  const normalized = collapseWhitespace(output);
+  return {
+    text: normalized,
+    changed: normalized !== collapseWhitespace(input),
+    blockedHitCount,
+  };
+}
+
+export function isMvuTaggedWorldInfoComment(comment = "") {
+  return MVU_ENTRY_COMMENT_REGEX.test(String(comment || ""));
+}
+
+export function isMvuTaggedWorldInfoNameOrComment(name = "", comment = "") {
+  return (
+    MVU_ENTRY_COMMENT_REGEX.test(String(name || "")) ||
+    MVU_ENTRY_COMMENT_REGEX.test(String(comment || ""))
+  );
+}
+
+export function isLikelyMvuWorldInfoContent(content = "") {
+  const normalized = collapseWhitespace(content);
+  if (!normalized) {
+    return false;
+  }
+  const stateKeyMentionCount =
+    normalized.match(/\b(?:stat_data|display_data|delta_data)\b/gi)?.length || 0;
+
+  const stateSignals = [
+    MVU_MESSAGE_VARIABLE_MACRO_REGEX,
+    MVU_GETVAR_REFERENCE_REGEX,
+    MVU_STATE_OBJECT_FIELD_REGEX,
+    MVU_STATE_PATH_REFERENCE_REGEX,
+    MVU_STATE_HELPER_REFERENCE_REGEX,
+  ].reduce(
+    (count, pattern) => count + (matchesRegex(normalized, pattern) ? 1 : 0),
+    0,
+  );
+
+  return (
+    matchesRegex(normalized, MVU_STATUS_CURRENT_VARIABLE_DETECT_REGEX) ||
+    matchesRegex(normalized, MVU_VARIABLE_OUTPUT_ENTRY_REGEX) ||
+    matchesRegex(normalized, MVU_VARIABLE_RULES_ENTRY_REGEX) ||
+    matchesRegex(normalized, MVU_FORMAT_EMPHASIS_ENTRY_REGEX) ||
+    stateSignals >= 2 ||
+    (stateSignals >= 1 && stateKeyMentionCount >= 2)
+  );
+}
+
+export function stripMvuPromptArtifacts(content = "") {
+  return stripMvuPromptArtifactsDetailed(content).text;
+}
+
+export function stripBlockedPromptContents(content = "", blockedContents = []) {
+  return stripBlockedPromptContentsDetailed(content, blockedContents).text;
+}
+
+export function sanitizeMvuContent(
+  content = "",
+  { mode = "aggressive", blockedContents = [] } = {},
+) {
+  const originalText = normalizeText(content);
+  const originalCollapsed = collapseWhitespace(originalText);
+  const sanitizedMode = String(mode || "aggressive").trim().toLowerCase();
+
+  const artifactResult = stripMvuPromptArtifactsDetailed(originalCollapsed);
+  const blockedResult = stripBlockedPromptContentsDetailed(
+    artifactResult.text,
+    blockedContents,
+  );
+
+  const reasons = [];
+  if (artifactResult.artifactRemovedCount > 0) {
+    reasons.push("artifact_stripped");
+  }
+  if (blockedResult.blockedHitCount > 0) {
+    reasons.push("blocked_content_removed");
+  }
+
+  let text = blockedResult.text;
+  let dropped = false;
+  if (sanitizedMode === "aggressive") {
+    if (
+      isLikelyMvuWorldInfoContent(originalCollapsed) ||
+      isLikelyMvuWorldInfoContent(text)
+    ) {
+      text = "";
+      dropped = true;
+      reasons.push("likely_mvu_content");
+    }
+  }
+
+  return {
+    text: collapseWhitespace(text),
+    changed: collapseWhitespace(text) !== originalCollapsed,
+    dropped,
+    reasons: uniq(reasons),
+    blockedHitCount: blockedResult.blockedHitCount,
+    artifactRemovedCount: artifactResult.artifactRemovedCount,
+  };
+}