fix(sync): harden manual server cleanup

This commit is contained in:
youzini
2026-06-06 10:59:38 +00:00
parent dba53cc21c
commit 4f1dad8f8d
3 changed files with 353 additions and 17 deletions

View File

@@ -3901,16 +3901,26 @@ export function autoSyncOnVisibility(options = {}) {
};
}
export async function deleteRemoteSyncFile(chatId, options = {}) {
function cancelPendingSyncUpload(chatId) {
const normalizedChatId = normalizeChatId(chatId);
if (!normalizedChatId) {
return {
deleted: false,
chatId: "",
reason: "missing-chat-id",
};
const pendingTimer = uploadDebounceTimerByChatId.get(normalizedChatId);
if (pendingTimer) {
clearTimeout(pendingTimer);
uploadDebounceTimerByChatId.delete(normalizedChatId);
return true;
}
return false;
}
async function waitForChatSyncIdle(chatId) {
const normalizedChatId = normalizeChatId(chatId);
const existingTask = syncInFlightByChatId.get(normalizedChatId);
if (!existingTask) return;
await existingTask.catch(() => null);
}
async function deleteRemoteSyncFileUnlocked(chatId, options = {}) {
const normalizedChatId = normalizeChatId(chatId);
try {
const filenames = await resolveSyncFilenameCandidates(
normalizedChatId,
@@ -3919,17 +3929,44 @@ export async function deleteRemoteSyncFile(chatId, options = {}) {
let lastNotFoundFilename = filenames[0] || "";
for (const filename of filenames) {
try {
const manifestPayload = await readRemoteJsonFile(filename, options);
const chunkFilenamesToDelete = new Set();
let cleanupAttempted = 0;
let cleanupDeleted = 0;
let cleanupSkipped = 0;
let cleanupFailed = 0;
let cleanupReason = "not-needed";
const manifestReadResult = await readRemoteJsonFileResult(filename, options);
if (manifestReadResult.status === 404) {
cleanupReason = "manifest-not-found";
} else if (!manifestReadResult.ok) {
return {
deleted: false,
chatId: normalizedChatId,
filename,
reason: "manifest-read-error",
status: manifestReadResult.status,
error: manifestReadResult.error || null,
cleanup: {
attempted: 0,
deleted: 0,
skipped: 0,
failed: 0,
reason: manifestReadResult.reason || "manifest-read-error",
},
};
} else {
const manifestPayload = manifestReadResult.payload;
if (Number(manifestPayload?.formatVersion || 0) === BME_REMOTE_SYNC_FORMAT_VERSION_V2) {
for (const chunk of Array.isArray(manifestPayload?.chunks) ? manifestPayload.chunks : []) {
const chunkFilename = String(chunk?.filename || "").trim();
if (!chunkFilename) continue;
await deleteRemoteJsonFile(chunkFilename, options).catch(() => null);
for (const chunkFilename of [
...collectRemoteSyncChunkFilenames(manifestPayload, filename),
...readRemoteSyncChunkGcPending(manifestPayload, filename).keys(),
]) {
chunkFilenamesToDelete.add(chunkFilename);
}
cleanupReason = chunkFilenamesToDelete.size ? "pending" : "no-chunks";
} else {
cleanupReason = "non-v2-manifest";
}
} catch {
// best-effort chunk cleanup
}
const deleteResult = await deleteRemoteJsonFile(filename, options);
if (!deleteResult.deleted) {
@@ -3937,12 +3974,41 @@ export async function deleteRemoteSyncFile(chatId, options = {}) {
continue;
}
const headAfterDelete = await readRemoteJsonFileResult(filename, options);
if (headAfterDelete.ok) {
cleanupReason = "remote-head-recreated";
} else if (headAfterDelete.status !== 404) {
cleanupReason = headAfterDelete.reason || "head-check-failed";
} else {
cleanupReason = chunkFilenamesToDelete.size ? "manifest-deleted" : cleanupReason;
}
if (cleanupReason === "manifest-deleted") {
for (const chunkFilename of chunkFilenamesToDelete) {
cleanupAttempted += 1;
try {
const chunkDeleteResult = await deleteRemoteJsonFile(chunkFilename, options);
if (chunkDeleteResult.deleted) cleanupDeleted += 1;
else cleanupSkipped += 1;
} catch {
cleanupFailed += 1;
}
}
}
sanitizedFilenameByChatId.delete(normalizedChatId);
return {
deleted: true,
chatId: normalizedChatId,
filename,
backend: String(deleteResult.backend || ""),
cleanup: {
attempted: cleanupAttempted,
deleted: cleanupDeleted,
skipped: cleanupSkipped,
failed: cleanupFailed,
reason: cleanupReason,
},
};
}
@@ -3963,6 +4029,29 @@ export async function deleteRemoteSyncFile(chatId, options = {}) {
}
}
export async function deleteRemoteSyncFile(chatId, options = {}) {
const normalizedChatId = normalizeChatId(chatId);
if (!normalizedChatId) {
return {
deleted: false,
chatId: "",
reason: "missing-chat-id",
};
}
cancelPendingSyncUpload(normalizedChatId);
await waitForChatSyncIdle(normalizedChatId);
const deleteTask = deleteRemoteSyncFileUnlocked(normalizedChatId, options);
syncInFlightByChatId.set(normalizedChatId, deleteTask);
try {
return await deleteTask;
} finally {
if (syncInFlightByChatId.get(normalizedChatId) === deleteTask) {
syncInFlightByChatId.delete(normalizedChatId);
}
}
}
export function __testOnlyDecodeBase64Utf8(base64Text) {
return decodeBase64Utf8(base64Text);
}

View File

@@ -1425,6 +1425,218 @@ async function testDeleteRemoteSyncFile() {
assert.equal(logs.deleteCalls > deleteCallsAfterFirstDelete, true);
}
async function testDeleteRemoteSyncFileV2CleansChunksAndGcPending() {
const { fetch, remoteFiles, logs } = createMockFetchEnvironment();
const dbByChatId = new Map();
const chatId = "chat-v2-delete-cleanup";
dbByChatId.set(chatId, new FakeDb(chatId));
// Manually set up a v2 manifest with chunks and chunkGc.pending entries in remote storage
const manifestFilename = "ST-BME_sync_chat-v2-delete-cleanup.json";
const chunkNodeFile = "ST-BME_sync_chat-v2-delete-cleanup.__nodes.000.abc123.json";
const chunkEdgeFile = "ST-BME_sync_chat-v2-delete-cleanup.__edges.000.def456.json";
const gcPendingFile = "ST-BME_sync_chat-v2-delete-cleanup.__runtime-meta.000.ghi789.json";
remoteFiles.set(chunkNodeFile, { kind: "nodes", index: 0, records: [{ id: "n1" }] });
remoteFiles.set(chunkEdgeFile, { kind: "edges", index: 0, records: [{ id: "e1" }] });
remoteFiles.set(gcPendingFile, { kind: "runtime-meta", index: 0, records: [] });
remoteFiles.set(manifestFilename, {
formatVersion: 2,
meta: { chatId, revision: 5, lastModified: 500, nodeCount: 1, edgeCount: 1, tombstoneCount: 0, schemaVersion: 1 },
state: { lastProcessedFloor: 3, extractionCount: 2 },
chunks: [
{ kind: "nodes", index: 0, count: 1, filename: chunkNodeFile },
{ kind: "edges", index: 0, count: 1, filename: chunkEdgeFile },
],
chunkGc: {
pending: [
{ filename: gcPendingFile, firstSeenAt: 400, eligibleAt: 900, sourceRevision: 4 },
],
},
});
const runtime = buildRuntimeOptions({ dbByChatId, fetch });
const deleteResult = await deleteRemoteSyncFile(chatId, runtime);
assert.equal(deleteResult.deleted, true);
assert.equal(deleteResult.chatId, chatId);
assert.equal(deleteResult.filename, manifestFilename);
// All chunk files and gc-pending files should be deleted
assert.equal(remoteFiles.has(chunkNodeFile), false, "manifest.chunks node file should be deleted");
assert.equal(remoteFiles.has(chunkEdgeFile), false, "manifest.chunks edge file should be deleted");
assert.equal(remoteFiles.has(gcPendingFile), false, "manifest.chunkGc.pending file should be deleted");
assert.equal(remoteFiles.has(manifestFilename), false, "manifest itself should be deleted");
assert.equal(deleteResult.cleanup.attempted, 3);
assert.equal(deleteResult.cleanup.deleted, 3);
assert.equal(deleteResult.cleanup.skipped, 0);
assert.equal(deleteResult.cleanup.failed, 0);
// Verify delete calls: 2 chunks + 1 gc-pending + 1 manifest = 4
assert.equal(logs.deleteCalls, 4, "should delete 2 chunks + 1 gc-pending + 1 manifest");
}
async function testDeleteRemoteSyncFileManifestDeleteFailureKeepsChunks() {
const { fetch, remoteFiles } = createMockFetchEnvironment();
const dbByChatId = new Map();
const chatId = "chat-delete-manifest-fails";
dbByChatId.set(chatId, new FakeDb(chatId));
const manifestFilename = "ST-BME_sync_chat-delete-manifest-fails.json";
const chunkNodeFile = "ST-BME_sync_chat-delete-manifest-fails.__nodes.000.abc123.json";
const gcPendingFile = "ST-BME_sync_chat-delete-manifest-fails.__runtime-meta.000.ghi789.json";
remoteFiles.set(chunkNodeFile, { kind: "nodes", index: 0, records: [{ id: "n1" }] });
remoteFiles.set(gcPendingFile, { kind: "runtime-meta", index: 0, records: [] });
remoteFiles.set(manifestFilename, {
formatVersion: 2,
meta: { chatId, revision: 5, lastModified: 500, nodeCount: 1, edgeCount: 0, tombstoneCount: 0, schemaVersion: 1 },
state: { lastProcessedFloor: 3, extractionCount: 2 },
chunks: [
{ kind: "nodes", index: 0, count: 1, filename: chunkNodeFile },
],
chunkGc: {
pending: [
{ filename: gcPendingFile, firstSeenAt: 400, eligibleAt: 900, sourceRevision: 4 },
],
},
});
const guardedFetch = async (url, options = {}) => {
if (url === "/api/files/delete" && String(options?.method || "").toUpperCase() === "POST") {
const body = JSON.parse(String(options.body || "{}"));
if (String(body.path || "") === `/user/files/${manifestFilename}`) {
return createJsonResponse(500, "manifest delete failed");
}
}
return await fetch(url, options);
};
const deleteResult = await deleteRemoteSyncFile(
chatId,
buildRuntimeOptions({ dbByChatId, fetch: guardedFetch }),
);
assert.equal(deleteResult.deleted, false);
assert.equal(deleteResult.reason, "delete-error");
assert.equal(remoteFiles.has(manifestFilename), true, "manifest remains after delete failure");
assert.equal(remoteFiles.has(chunkNodeFile), true, "chunk must remain when manifest delete fails");
assert.equal(remoteFiles.has(gcPendingFile), true, "pending chunk must remain when manifest delete fails");
}
async function testDeleteRemoteSyncFileManifestReadFailureAbortsDelete() {
const { fetch, remoteFiles } = createMockFetchEnvironment();
const dbByChatId = new Map();
const chatId = "chat-delete-manifest-read-fails";
dbByChatId.set(chatId, new FakeDb(chatId));
const manifestFilename = "ST-BME_sync_chat-delete-manifest-read-fails.json";
const chunkNodeFile = "ST-BME_sync_chat-delete-manifest-read-fails.__nodes.000.abc123.json";
remoteFiles.set(chunkNodeFile, { kind: "nodes", index: 0, records: [{ id: "n1" }] });
remoteFiles.set(manifestFilename, {
formatVersion: 2,
meta: { chatId, revision: 5, lastModified: 500, nodeCount: 1, edgeCount: 0, tombstoneCount: 0, schemaVersion: 1 },
state: { lastProcessedFloor: 3, extractionCount: 2 },
chunks: [
{ kind: "nodes", index: 0, count: 1, filename: chunkNodeFile },
],
});
const guardedFetch = async (url, options = {}) => {
if (
String(url).startsWith(`/user/files/${manifestFilename}`)
&& String(options?.method || "GET").toUpperCase() === "GET"
) {
return createJsonResponse(500, "manifest read failed");
}
return await fetch(url, options);
};
const deleteResult = await deleteRemoteSyncFile(
chatId,
buildRuntimeOptions({ dbByChatId, fetch: guardedFetch }),
);
assert.equal(deleteResult.deleted, false);
assert.equal(deleteResult.reason, "manifest-read-error");
assert.equal(deleteResult.cleanup.reason, "http-error");
assert.equal(remoteFiles.has(manifestFilename), true, "manifest must remain after read failure");
assert.equal(remoteFiles.has(chunkNodeFile), true, "chunk must remain after read failure");
}
async function testDeleteRemoteSyncFileRemoteHeadRecreatedSkipsChunkCleanup() {
const { fetch, remoteFiles } = createMockFetchEnvironment();
const dbByChatId = new Map();
const chatId = "chat-delete-head-recreated";
dbByChatId.set(chatId, new FakeDb(chatId));
const manifestFilename = "ST-BME_sync_chat-delete-head-recreated.json";
const chunkNodeFile = "ST-BME_sync_chat-delete-head-recreated.__nodes.000.abc123.json";
const manifestPayload = {
formatVersion: 2,
meta: { chatId, revision: 5, lastModified: 500, nodeCount: 1, edgeCount: 0, tombstoneCount: 0, schemaVersion: 1 },
state: { lastProcessedFloor: 3, extractionCount: 2 },
chunks: [
{ kind: "nodes", index: 0, count: 1, filename: chunkNodeFile },
],
};
remoteFiles.set(chunkNodeFile, { kind: "nodes", index: 0, records: [{ id: "n1" }] });
remoteFiles.set(manifestFilename, manifestPayload);
const guardedFetch = async (url, options = {}) => {
if (url === "/api/files/delete" && String(options?.method || "").toUpperCase() === "POST") {
const body = JSON.parse(String(options.body || "{}"));
if (String(body.path || "") === `/user/files/${manifestFilename}`) {
const response = await fetch(url, options);
remoteFiles.set(manifestFilename, {
...manifestPayload,
meta: { ...manifestPayload.meta, revision: 6, lastModified: 600 },
});
return response;
}
}
return await fetch(url, options);
};
const deleteResult = await deleteRemoteSyncFile(
chatId,
buildRuntimeOptions({ dbByChatId, fetch: guardedFetch }),
);
assert.equal(deleteResult.deleted, true);
assert.equal(deleteResult.cleanup.reason, "remote-head-recreated");
assert.equal(deleteResult.cleanup.attempted, 0);
assert.equal(remoteFiles.has(manifestFilename), true, "recreated manifest must remain");
assert.equal(remoteFiles.has(chunkNodeFile), true, "chunk must remain when head is recreated");
}
async function testDeleteRemoteSyncFileMissingManifestNoSpeculativeDelete() {
const { fetch, remoteFiles, logs } = createMockFetchEnvironment();
const dbByChatId = new Map();
const chatId = "chat-missing-manifest-no-delete";
dbByChatId.set(chatId, new FakeDb(chatId));
// Pre-populate orphan-looking chunk files that match the chatId naming pattern
const orphanChunk = "ST-BME_sync_chat-missing-manifest-no-delete.__nodes.000.orphan.json";
const orphanGcPending = "ST-BME_sync_chat-missing-manifest-no-delete.__edges.000.stale.json";
remoteFiles.set(orphanChunk, { kind: "nodes", index: 0, records: [] });
remoteFiles.set(orphanGcPending, { kind: "edges", index: 0, records: [] });
const deleteCallsBefore = logs.deleteCalls;
const runtime = buildRuntimeOptions({ dbByChatId, fetch });
const deleteResult = await deleteRemoteSyncFile(chatId, runtime);
assert.equal(deleteResult.deleted, false);
assert.equal(deleteResult.reason, "not-found");
// Orphan chunks must NOT be speculatively deleted — only manifest filename candidates
// may be attempted for deletion (which 404 because the manifest was never uploaded),
// but chunks and gc-pending files must remain untouched.
assert.equal(remoteFiles.has(orphanChunk), true, "orphan chunk must not be speculatively deleted");
assert.equal(remoteFiles.has(orphanGcPending), true, "orphan gc-pending must not be speculatively deleted");
assert.equal(remoteFiles.size, 2, "both orphan files should remain untouched after missing-manifest delete");
}
async function testDeleteRemoteSyncFileFallsBackToLegacyFilename() {
const { fetch, remoteFiles, logs } = createMockFetchEnvironment();
const dbByChatId = new Map();
@@ -1683,6 +1895,11 @@ async function main() {
await testDeleteUsesExplicitManifestFilenameAndClearsLocalBackupMeta();
await testSyncNowLockAndAutoSync();
await testDeleteRemoteSyncFile();
await testDeleteRemoteSyncFileV2CleansChunksAndGcPending();
await testDeleteRemoteSyncFileManifestDeleteFailureKeepsChunks();
await testDeleteRemoteSyncFileManifestReadFailureAbortsDelete();
await testDeleteRemoteSyncFileRemoteHeadRecreatedSkipsChunkCleanup();
await testDeleteRemoteSyncFileMissingManifestNoSpeculativeDelete();
await testDeleteRemoteSyncFileFallsBackToLegacyFilename();
await testAutoSyncOnVisibility();
await testSyncNowRemoteReadErrorPath();

View File

@@ -1523,7 +1523,7 @@ export async function onDeleteServerSyncFileController(runtime) {
}
const userInput = runtime.prompt(
"此操作会删除当前聊天在服务端的同步数据。\n\n如果该聊天已经升级到远端 v2同步 manifest chunk 文件都会一起删除。\n\n请输入 DELETE 确认:",
"此操作会删除当前聊天在服务端的同步 manifest。\n\n如果该聊天已经升级到远端 v2会在 manifest 删除成功后尝试清理当前 manifest 引用的 chunk,以及 manifest 记录的待清理 chunk。\n\n注意普通 SillyTavern 不提供 user/files 目录枚举,因此已经脱离 manifest 的历史孤儿 chunk 无法通过此按钮自动发现。\n\n请输入 DELETE 确认:",
);
if (userInput !== "DELETE") {
if (userInput != null) {
@@ -1535,11 +1535,41 @@ export async function onDeleteServerSyncFileController(runtime) {
try {
const result = await runtime.deleteRemoteSyncFile(chatId);
if (result?.deleted) {
runtime.toastr.success(`已删除服务端同步数据: ${result.filename}`);
const cleanup = result.cleanup || {};
const cleanupSummary = Number(cleanup.attempted || 0) > 0
? `chunk 清理 删除 ${Number(cleanup.deleted || 0)}/${Number(cleanup.attempted || 0)},跳过 ${Number(cleanup.skipped || 0)},失败 ${Number(cleanup.failed || 0)}`
: "";
const cleanupReason = String(cleanup.reason || "");
const cleanupReasonLabel = {
"remote-head-recreated": "远端 manifest 已被重新创建chunk 清理已跳过",
"head-check-failed": "删除后无法确认远端状态chunk 清理已跳过",
"manifest-read-error": "读取 manifest 失败chunk 列表不可用",
}[cleanupReason] || "";
const benignCleanupReasons = new Set([
"",
"manifest-deleted",
"no-chunks",
"non-v2-manifest",
"manifest-not-found",
"not-needed",
]);
const shouldWarnCleanup =
Number(cleanup.failed || 0) > 0
|| Number(cleanup.skipped || 0) > 0
|| Boolean(cleanupReasonLabel)
|| !benignCleanupReasons.has(cleanupReason);
const message = `已删除服务端同步 manifest: ${result.filename}${cleanupSummary}`;
if (shouldWarnCleanup) {
runtime.toastr.warning(`${message}${cleanupReasonLabel ? `${cleanupReasonLabel}` : ";部分 chunk 可能仍残留"}`);
} else {
runtime.toastr.success(message);
}
} else {
runtime.toastr.info(
result?.reason === "not-found"
? "服务端没有找到同步数据"
: result?.reason === "manifest-read-error"
? "读取服务端同步 manifest 失败,已取消删除以避免残留坏数据"
: `删除未成功: ${result?.reason || "未知原因"}`,
);
}