数据迁移并发提升性能
This commit is contained in:
mtvpls
@@ -377,6 +377,7 @@ dockge/komodo 等 docker compose UI 也有自动更新功能
|
||||
| TMDB_REVERSE_PROXY | TMDB 反向代理地址 | URL | (空) |
|
||||
| DANMAKU_API_BASE | 弹幕 API 地址 | URL | http://localhost:9321 |
|
||||
| DANMAKU_API_TOKEN | 弹幕 API Token | 任意字符串 | 87654321 |
|
||||
| DATA_MIGRATION_CHUNK_SIZE | 数据迁移批处理大小（控制导入导出时每批处理的用户数量和数据条数） | 正整数 | 10 |
|
||||
|
||||
NEXT_PUBLIC_DOUBAN_PROXY_TYPE 选项解释：
|
||||
|
||||
|
||||
@@ -8,6 +8,7 @@ import { getAuthInfoFromCookie } from '@/lib/auth';
|
||||
import { SimpleCrypto } from '@/lib/crypto';
|
||||
import { db } from '@/lib/db';
|
||||
import { CURRENT_VERSION } from '@/lib/version';
|
||||
import { updateProgress, clearProgress } from '../progress/route';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
@@ -77,64 +78,113 @@ export async function POST(req: NextRequest) {
|
||||
allUsers = Array.from(new Set(allUsers));
|
||||
console.log(`准备导出 ${allUsers.length} 个V2用户（包括站长）`);
|
||||
|
||||
// 为每个用户收集数据（只导出V2用户）
|
||||
// 为每个用户收集数据（只导出V2用户）- 使用并行处理
|
||||
console.log(`开始并行导出 ${allUsers.length} 个用户的数据...`);
|
||||
updateProgress(authInfo.username, 'export', 'collecting', 0, allUsers.length, '开始收集用户数据...');
|
||||
|
||||
// 分块处理用户，每批处理数量可通过环境变量配置
|
||||
const CHUNK_SIZE = parseInt(process.env.DATA_MIGRATION_CHUNK_SIZE || '10', 10);
|
||||
let exportedCount = 0;
|
||||
for (const username of allUsers) {
|
||||
// 站长特殊处理：使用环境变量密码
|
||||
let finalPasswordV2 = username === process.env.USERNAME ? process.env.PASSWORD : null;
|
||||
|
||||
// 如果不是站长，获取V2密码
|
||||
if (!finalPasswordV2) {
|
||||
finalPasswordV2 = await getUserPasswordV2(username);
|
||||
for (let i = 0; i < allUsers.length; i += CHUNK_SIZE) {
|
||||
const chunk = allUsers.slice(i, i + CHUNK_SIZE);
|
||||
console.log(`处理第 ${Math.floor(i / CHUNK_SIZE) + 1} 批用户 (${chunk.length} 个)`);
|
||||
|
||||
// 并行处理当前批次的用户
|
||||
const userDataPromises = chunk.map(async (username) => {
|
||||
try {
|
||||
// 站长特殊处理：使用环境变量密码
|
||||
let finalPasswordV2 = username === process.env.USERNAME ? process.env.PASSWORD : null;
|
||||
|
||||
// 如果不是站长，获取V2密码
|
||||
if (!finalPasswordV2) {
|
||||
finalPasswordV2 = await getUserPasswordV2(username);
|
||||
}
|
||||
|
||||
// 跳过没有V2密码的用户
|
||||
if (!finalPasswordV2) {
|
||||
console.log(`跳过用户 ${username}：没有V2密码`);
|
||||
return null;
|
||||
}
|
||||
|
||||
// 并行获取用户的所有数据
|
||||
const [
|
||||
playRecords,
|
||||
favorites,
|
||||
searchHistory,
|
||||
skipConfigs,
|
||||
musicPlayRecords,
|
||||
playlists
|
||||
] = await Promise.all([
|
||||
db.getAllPlayRecords(username),
|
||||
db.getAllFavorites(username),
|
||||
db.getSearchHistory(username),
|
||||
db.getAllSkipConfigs(username),
|
||||
db.getAllMusicPlayRecords(username),
|
||||
db.getUserMusicPlaylists(username)
|
||||
]);
|
||||
|
||||
// 并行获取所有歌单的歌曲
|
||||
const playlistsWithSongs = await Promise.all(
|
||||
playlists.map(async (playlist) => {
|
||||
const songs = await db.getPlaylistSongs(playlist.id);
|
||||
return { ...playlist, songs };
|
||||
})
|
||||
);
|
||||
|
||||
return {
|
||||
username,
|
||||
userData: {
|
||||
playRecords,
|
||||
favorites,
|
||||
searchHistory,
|
||||
skipConfigs,
|
||||
musicPlayRecords,
|
||||
musicPlaylists: playlistsWithSongs,
|
||||
passwordV2: finalPasswordV2
|
||||
}
|
||||
};
|
||||
} catch (error) {
|
||||
console.error(`导出用户 ${username} 数据失败:`, error);
|
||||
return null;
|
||||
}
|
||||
});
|
||||
|
||||
// 等待当前批次完成
|
||||
const results = await Promise.all(userDataPromises);
|
||||
|
||||
// 将结果添加到导出数据中，并实时更新进度
|
||||
for (const result of results) {
|
||||
if (result) {
|
||||
exportData.data.userData[result.username] = result.userData;
|
||||
exportedCount++;
|
||||
// 每处理完一个用户就更新进度
|
||||
updateProgress(
|
||||
authInfo.username,
|
||||
'export',
|
||||
'collecting',
|
||||
exportedCount,
|
||||
allUsers.length,
|
||||
`正在收集用户数据 (${exportedCount}/${allUsers.length})...`
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// 跳过没有V2密码的用户
|
||||
if (!finalPasswordV2) {
|
||||
console.log(`跳过用户 ${username}：没有V2密码`);
|
||||
continue;
|
||||
}
|
||||
|
||||
// 获取用户的所有歌单
|
||||
const playlists = await db.getUserMusicPlaylists(username);
|
||||
const playlistsWithSongs = [];
|
||||
for (const playlist of playlists) {
|
||||
const songs = await db.getPlaylistSongs(playlist.id);
|
||||
playlistsWithSongs.push({
|
||||
...playlist,
|
||||
songs
|
||||
});
|
||||
}
|
||||
|
||||
const userData = {
|
||||
// 播放记录
|
||||
playRecords: await db.getAllPlayRecords(username),
|
||||
// 收藏夹
|
||||
favorites: await db.getAllFavorites(username),
|
||||
// 搜索历史
|
||||
searchHistory: await db.getSearchHistory(username),
|
||||
// 跳过片头片尾配置
|
||||
skipConfigs: await db.getAllSkipConfigs(username),
|
||||
// 音乐播放记录
|
||||
musicPlayRecords: await db.getAllMusicPlayRecords(username),
|
||||
// 音乐歌单（包含歌曲）
|
||||
musicPlaylists: playlistsWithSongs,
|
||||
// V2用户的加密密码
|
||||
passwordV2: finalPasswordV2
|
||||
};
|
||||
|
||||
exportData.data.userData[username] = userData;
|
||||
exportedCount++;
|
||||
console.log(`已完成 ${exportedCount}/${allUsers.length} 个用户`);
|
||||
}
|
||||
|
||||
console.log(`成功导出 ${exportedCount} 个用户的数据`);
|
||||
|
||||
// 将数据转换为JSON字符串
|
||||
updateProgress(authInfo.username, 'export', 'serializing', exportedCount, exportedCount, '正在序列化数据...');
|
||||
const jsonData = JSON.stringify(exportData);
|
||||
|
||||
// 先压缩数据
|
||||
updateProgress(authInfo.username, 'export', 'compressing', exportedCount, exportedCount, '正在压缩数据...');
|
||||
const compressedData = await gzipAsync(jsonData);
|
||||
|
||||
// 使用提供的密码加密压缩后的数据
|
||||
updateProgress(authInfo.username, 'export', 'encrypting', exportedCount, exportedCount, '正在加密数据...');
|
||||
const encryptedData = SimpleCrypto.encrypt(compressedData.toString('base64'), password);
|
||||
|
||||
// 生成文件名
|
||||
@@ -142,6 +192,10 @@ export async function POST(req: NextRequest) {
|
||||
const timestamp = `${now.getFullYear()}${String(now.getMonth() + 1).padStart(2, '0')}${String(now.getDate()).padStart(2, '0')}-${String(now.getHours()).padStart(2, '0')}${String(now.getMinutes()).padStart(2, '0')}${String(now.getSeconds()).padStart(2, '0')}`;
|
||||
const filename = `moontv-backup-${timestamp}.dat`;
|
||||
|
||||
// 清除进度信息
|
||||
updateProgress(authInfo.username, 'export', 'completed', exportedCount, exportedCount, '导出完成！');
|
||||
setTimeout(() => clearProgress(authInfo.username, 'export'), 3000);
|
||||
|
||||
// 返回加密的数据作为文件下载
|
||||
return new NextResponse(encryptedData, {
|
||||
status: 200,
|
||||
@@ -154,6 +208,11 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
} catch (error) {
|
||||
console.error('数据导出失败:', error);
|
||||
// 清除进度信息
|
||||
const authInfo = getAuthInfoFromCookie(req);
|
||||
if (authInfo?.username) {
|
||||
clearProgress(authInfo.username, 'export');
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: error instanceof Error ? error.message : '导出失败' },
|
||||
{ status: 500 }
|
||||
|
||||
@@ -8,6 +8,7 @@ import { getAuthInfoFromCookie } from '@/lib/auth';
|
||||
import { configSelfCheck, setCachedConfig } from '@/lib/config';
|
||||
import { SimpleCrypto } from '@/lib/crypto';
|
||||
import { db } from '@/lib/db';
|
||||
import { updateProgress, clearProgress } from '../progress/route';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
@@ -78,6 +79,7 @@ export async function POST(req: NextRequest) {
|
||||
}
|
||||
|
||||
// 开始导入数据 - 先清空现有数据
|
||||
updateProgress(authInfo.username, 'import', 'clearing', 0, 1, '正在清空现有数据...');
|
||||
await db.clearAllData();
|
||||
|
||||
// 额外清除所有V2用户（clearAllData可能只清除旧版用户）
|
||||
@@ -109,185 +111,267 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
const userCount = Object.keys(userData).length;
|
||||
console.log(`准备导入 ${userCount} 个用户的数据`);
|
||||
updateProgress(authInfo.username, 'import', 'importing', 0, userCount, '开始导入用户数据...');
|
||||
|
||||
// 分块处理用户，每批处理数量可通过环境变量配置
|
||||
const CHUNK_SIZE = parseInt(process.env.DATA_MIGRATION_CHUNK_SIZE || '10', 10);
|
||||
const usernames = Object.keys(userData);
|
||||
let importedCount = 0;
|
||||
for (const username in userData) {
|
||||
const user = userData[username];
|
||||
|
||||
// 为所有有passwordV2的用户创建user:info
|
||||
if (user.passwordV2) {
|
||||
const userV2 = usersV2Map.get(username) as any;
|
||||
for (let i = 0; i < usernames.length; i += CHUNK_SIZE) {
|
||||
const chunk = usernames.slice(i, i + CHUNK_SIZE);
|
||||
console.log(`处理第 ${Math.floor(i / CHUNK_SIZE) + 1} 批用户 (${chunk.length} 个)`);
|
||||
updateProgress(
|
||||
authInfo.username,
|
||||
'import',
|
||||
'importing',
|
||||
importedCount,
|
||||
userCount,
|
||||
`正在导入用户数据 (${importedCount}/${userCount})...`
|
||||
);
|
||||
|
||||
// 确定角色：站长为owner，其他用户从usersV2获取或默认为user
|
||||
let role: 'owner' | 'admin' | 'user' = 'user';
|
||||
if (username === process.env.USERNAME) {
|
||||
role = 'owner';
|
||||
} else if (userV2) {
|
||||
role = userV2.role === 'owner' ? 'user' : userV2.role;
|
||||
}
|
||||
// 并行导入当前批次的用户
|
||||
const importPromises = chunk.map(async (username) => {
|
||||
try {
|
||||
const user = userData[username];
|
||||
// 数据批处理大小（用于播放记录、收藏夹等）
|
||||
const DATA_BATCH_SIZE = parseInt(process.env.DATA_MIGRATION_CHUNK_SIZE || '10', 10);
|
||||
|
||||
const createdAt = userV2?.created_at || Date.now();
|
||||
// 为所有有passwordV2的用户创建user:info
|
||||
if (user.passwordV2) {
|
||||
const userV2 = usersV2Map.get(username) as any;
|
||||
|
||||
// 根据存储类型使用不同的导入方法
|
||||
if (storageType === 'd1') {
|
||||
// D1 存储：使用 createUserWithHashedPassword 方法
|
||||
try {
|
||||
if (typeof storage.createUserWithHashedPassword === 'function') {
|
||||
await storage.createUserWithHashedPassword(
|
||||
username,
|
||||
user.passwordV2, // 已经是hash过的密码
|
||||
role,
|
||||
createdAt,
|
||||
userV2?.tags,
|
||||
userV2?.oidcSub,
|
||||
userV2?.enabledApis,
|
||||
userV2?.banned
|
||||
);
|
||||
importedCount++;
|
||||
console.log(`用户 ${username} 导入成功 (D1)`);
|
||||
// 确定角色：站长为owner，其他用户从usersV2获取或默认为user
|
||||
let role: 'owner' | 'admin' | 'user' = 'user';
|
||||
if (username === process.env.USERNAME) {
|
||||
role = 'owner';
|
||||
} else if (userV2) {
|
||||
role = userV2.role === 'owner' ? 'user' : userV2.role;
|
||||
}
|
||||
|
||||
const createdAt = userV2?.created_at || Date.now();
|
||||
|
||||
// 根据存储类型使用不同的导入方法
|
||||
if (storageType === 'd1') {
|
||||
// D1 存储：使用 createUserWithHashedPassword 方法
|
||||
if (typeof storage.createUserWithHashedPassword === 'function') {
|
||||
await storage.createUserWithHashedPassword(
|
||||
username,
|
||||
user.passwordV2,
|
||||
role,
|
||||
createdAt,
|
||||
userV2?.tags,
|
||||
userV2?.oidcSub,
|
||||
userV2?.enabledApis,
|
||||
userV2?.banned
|
||||
);
|
||||
console.log(`用户 ${username} 导入成功 (D1)`);
|
||||
} else {
|
||||
console.error(`D1 storage 缺少 createUserWithHashedPassword 方法`);
|
||||
return false;
|
||||
}
|
||||
} else if (storageType === 'postgres') {
|
||||
// Postgres 存储：使用 createUserWithHashedPassword 方法
|
||||
if (typeof storage.createUserWithHashedPassword === 'function') {
|
||||
await storage.createUserWithHashedPassword(
|
||||
username,
|
||||
user.passwordV2,
|
||||
role,
|
||||
createdAt,
|
||||
userV2?.tags,
|
||||
userV2?.oidcSub,
|
||||
userV2?.enabledApis,
|
||||
userV2?.banned
|
||||
);
|
||||
console.log(`用户 ${username} 导入成功 (Postgres)`);
|
||||
} else {
|
||||
console.error(`Postgres storage 缺少 createUserWithHashedPassword 方法`);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
console.error(`D1 storage 缺少 createUserWithHashedPassword 方法`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`导入用户 ${username} 失败:`, err);
|
||||
}
|
||||
} else if (storageType === 'postgres') {
|
||||
// Postgres 存储：使用 createUserWithHashedPassword 方法
|
||||
try {
|
||||
if (typeof storage.createUserWithHashedPassword === 'function') {
|
||||
await storage.createUserWithHashedPassword(
|
||||
username,
|
||||
user.passwordV2, // 已经是hash过的密码
|
||||
// Redis 存储：直接设置用户信息
|
||||
const userInfoKey = `user:${username}:info`;
|
||||
const userInfo: Record<string, string> = {
|
||||
role,
|
||||
createdAt,
|
||||
userV2?.tags,
|
||||
userV2?.oidcSub,
|
||||
userV2?.enabledApis,
|
||||
userV2?.banned
|
||||
);
|
||||
importedCount++;
|
||||
console.log(`用户 ${username} 导入成功 (Postgres)`);
|
||||
} else {
|
||||
console.error(`Postgres storage 缺少 createUserWithHashedPassword 方法`);
|
||||
banned: String(userV2?.banned || false),
|
||||
password: user.passwordV2,
|
||||
created_at: createdAt.toString(),
|
||||
};
|
||||
|
||||
if (userV2?.tags && userV2.tags.length > 0) {
|
||||
userInfo.tags = JSON.stringify(userV2.tags);
|
||||
}
|
||||
|
||||
if (userV2?.oidcSub) {
|
||||
userInfo.oidcSub = userV2.oidcSub;
|
||||
}
|
||||
|
||||
if (userV2?.enabledApis && userV2.enabledApis.length > 0) {
|
||||
userInfo.enabledApis = JSON.stringify(userV2.enabledApis);
|
||||
}
|
||||
|
||||
await storage.withRetry(() => storage.client.hSet(userInfoKey, userInfo));
|
||||
await storage.withRetry(() => storage.client.zAdd('user:list', {
|
||||
score: createdAt,
|
||||
value: username,
|
||||
}));
|
||||
|
||||
if (userV2?.oidcSub) {
|
||||
const oidcSubKey = `oidc:sub:${userV2.oidcSub}`;
|
||||
await storage.withRetry(() => storage.client.set(oidcSubKey, username));
|
||||
}
|
||||
|
||||
console.log(`用户 ${username} 导入成功 (Redis)`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(`导入用户 ${username} 失败:`, err);
|
||||
}
|
||||
} else {
|
||||
// Redis 存储：直接设置用户信息
|
||||
const userInfoKey = `user:${username}:info`;
|
||||
const userInfo: Record<string, string> = {
|
||||
role,
|
||||
banned: String(userV2?.banned || false),
|
||||
password: user.passwordV2, // 已经是hash过的密码，直接使用
|
||||
created_at: createdAt.toString(),
|
||||
};
|
||||
|
||||
if (userV2?.tags && userV2.tags.length > 0) {
|
||||
userInfo.tags = JSON.stringify(userV2.tags);
|
||||
} else {
|
||||
console.log(`跳过用户 ${username}：没有passwordV2`);
|
||||
return false;
|
||||
}
|
||||
|
||||
if (userV2?.oidcSub) {
|
||||
userInfo.oidcSub = userV2.oidcSub;
|
||||
}
|
||||
// 并行导入用户的各类数据
|
||||
await Promise.all([
|
||||
// 导入播放记录（批量）
|
||||
(async () => {
|
||||
if (user.playRecords) {
|
||||
const entries = Object.entries(user.playRecords);
|
||||
// 使用配置的批处理大小
|
||||
for (let j = 0; j < entries.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = entries.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(([key, record]) =>
|
||||
(db as any).storage.setPlayRecord(username, key, record)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
})(),
|
||||
|
||||
if (userV2?.enabledApis && userV2.enabledApis.length > 0) {
|
||||
userInfo.enabledApis = JSON.stringify(userV2.enabledApis);
|
||||
}
|
||||
// 导入收藏夹（批量）
|
||||
(async () => {
|
||||
if (user.favorites) {
|
||||
const entries = Object.entries(user.favorites);
|
||||
for (let j = 0; j < entries.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = entries.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(([key, favorite]) =>
|
||||
(db as any).storage.setFavorite(username, key, favorite)
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
})(),
|
||||
|
||||
// 使用storage.withRetry直接设置用户信息
|
||||
await storage.withRetry(() => storage.client.hSet(userInfoKey, userInfo));
|
||||
// 导入搜索历史（批量）
|
||||
(async () => {
|
||||
if (user.searchHistory && Array.isArray(user.searchHistory)) {
|
||||
const reversed = user.searchHistory.reverse();
|
||||
for (let j = 0; j < reversed.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = reversed.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(keyword => db.addSearchHistory(username, keyword))
|
||||
);
|
||||
}
|
||||
}
|
||||
})(),
|
||||
|
||||
// 添加到用户列表
|
||||
await storage.withRetry(() => storage.client.zAdd('user:list', {
|
||||
score: createdAt,
|
||||
value: username,
|
||||
}));
|
||||
// 导入跳过片头片尾配置（批量）
|
||||
(async () => {
|
||||
if (user.skipConfigs) {
|
||||
const entries = Object.entries(user.skipConfigs);
|
||||
for (let j = 0; j < entries.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = entries.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(([key, skipConfig]) => {
|
||||
const [source, id] = key.split('+');
|
||||
if (source && id) {
|
||||
return db.setSkipConfig(username, source, id, skipConfig as any);
|
||||
}
|
||||
return Promise.resolve();
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
})(),
|
||||
|
||||
// 如果有oidcSub，创建映射
|
||||
if (userV2?.oidcSub) {
|
||||
const oidcSubKey = `oidc:sub:${userV2.oidcSub}`;
|
||||
await storage.withRetry(() => storage.client.set(oidcSubKey, username));
|
||||
}
|
||||
// 导入音乐播放记录（批量）
|
||||
(async () => {
|
||||
if (user.musicPlayRecords) {
|
||||
const entries = Object.entries(user.musicPlayRecords);
|
||||
for (let j = 0; j < entries.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = entries.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(([key, record]) => {
|
||||
const [platform, id] = key.split('+');
|
||||
if (platform && id) {
|
||||
return db.saveMusicPlayRecord(username, platform, id, record as any);
|
||||
}
|
||||
return Promise.resolve();
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
})(),
|
||||
|
||||
importedCount++;
|
||||
console.log(`用户 ${username} 导入成功 (Redis)`);
|
||||
// 导入音乐歌单
|
||||
(async () => {
|
||||
if (user.musicPlaylists && Array.isArray(user.musicPlaylists)) {
|
||||
for (const playlist of user.musicPlaylists) {
|
||||
await db.createMusicPlaylist(username, {
|
||||
id: playlist.id,
|
||||
name: playlist.name,
|
||||
description: playlist.description,
|
||||
cover: playlist.cover,
|
||||
});
|
||||
|
||||
// 批量导入歌单中的歌曲
|
||||
if (playlist.songs && Array.isArray(playlist.songs)) {
|
||||
for (let j = 0; j < playlist.songs.length; j += DATA_BATCH_SIZE) {
|
||||
const batch = playlist.songs.slice(j, j + DATA_BATCH_SIZE);
|
||||
await Promise.all(
|
||||
batch.map(song =>
|
||||
db.addSongToPlaylist(playlist.id, {
|
||||
platform: song.platform,
|
||||
id: song.id,
|
||||
name: song.name,
|
||||
artist: song.artist,
|
||||
album: song.album,
|
||||
pic: song.pic,
|
||||
duration: song.duration || 0,
|
||||
})
|
||||
)
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
})()
|
||||
]);
|
||||
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.error(`导入用户 ${username} 失败:`, error);
|
||||
return false;
|
||||
}
|
||||
} else {
|
||||
console.log(`跳过用户 ${username}：没有passwordV2`);
|
||||
}
|
||||
});
|
||||
|
||||
// 导入播放记录
|
||||
if (user.playRecords) {
|
||||
for (const [key, record] of Object.entries(user.playRecords)) {
|
||||
await (db as any).storage.setPlayRecord(username, key, record);
|
||||
}
|
||||
}
|
||||
// 等待当前批次完成
|
||||
const results = await Promise.all(importPromises);
|
||||
importedCount += results.filter(r => r).length;
|
||||
|
||||
// 导入收藏夹
|
||||
if (user.favorites) {
|
||||
for (const [key, favorite] of Object.entries(user.favorites)) {
|
||||
await (db as any).storage.setFavorite(username, key, favorite);
|
||||
}
|
||||
}
|
||||
|
||||
// 导入搜索历史
|
||||
if (user.searchHistory && Array.isArray(user.searchHistory)) {
|
||||
for (const keyword of user.searchHistory.reverse()) { // 反转以保持顺序
|
||||
await db.addSearchHistory(username, keyword);
|
||||
}
|
||||
}
|
||||
|
||||
// 导入跳过片头片尾配置
|
||||
if (user.skipConfigs) {
|
||||
for (const [key, skipConfig] of Object.entries(user.skipConfigs)) {
|
||||
const [source, id] = key.split('+');
|
||||
if (source && id) {
|
||||
await db.setSkipConfig(username, source, id, skipConfig as any);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 导入音乐播放记录
|
||||
if (user.musicPlayRecords) {
|
||||
for (const [key, record] of Object.entries(user.musicPlayRecords)) {
|
||||
const [platform, id] = key.split('+');
|
||||
if (platform && id) {
|
||||
await db.saveMusicPlayRecord(username, platform, id, record as any);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// 导入音乐歌单
|
||||
if (user.musicPlaylists && Array.isArray(user.musicPlaylists)) {
|
||||
for (const playlist of user.musicPlaylists) {
|
||||
// 创建歌单
|
||||
await db.createMusicPlaylist(username, {
|
||||
id: playlist.id,
|
||||
name: playlist.name,
|
||||
description: playlist.description,
|
||||
cover: playlist.cover,
|
||||
});
|
||||
|
||||
// 导入歌单中的歌曲
|
||||
if (playlist.songs && Array.isArray(playlist.songs)) {
|
||||
for (const song of playlist.songs) {
|
||||
await db.addSongToPlaylist(playlist.id, {
|
||||
platform: song.platform,
|
||||
id: song.id,
|
||||
name: song.name,
|
||||
artist: song.artist,
|
||||
album: song.album,
|
||||
pic: song.pic,
|
||||
duration: song.duration || 0,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
console.log(`已完成 ${importedCount}/${userCount} 个用户`);
|
||||
updateProgress(
|
||||
authInfo.username,
|
||||
'import',
|
||||
'importing',
|
||||
importedCount,
|
||||
userCount,
|
||||
`已导入 ${importedCount}/${userCount} 个用户`
|
||||
);
|
||||
}
|
||||
|
||||
console.log(`成功导入 ${importedCount} 个用户的user:info`);
|
||||
updateProgress(authInfo.username, 'import', 'completed', importedCount, userCount, '导入完成！');
|
||||
setTimeout(() => clearProgress(authInfo.username, 'import'), 3000);
|
||||
|
||||
return NextResponse.json({
|
||||
message: '数据导入成功',
|
||||
@@ -299,6 +383,11 @@ export async function POST(req: NextRequest) {
|
||||
|
||||
} catch (error) {
|
||||
console.error('数据导入失败:', error);
|
||||
// 清除进度信息
|
||||
const authInfo = getAuthInfoFromCookie(req);
|
||||
if (authInfo?.username) {
|
||||
clearProgress(authInfo.username, 'import');
|
||||
}
|
||||
return NextResponse.json(
|
||||
{ error: error instanceof Error ? error.message : '导入失败' },
|
||||
{ status: 500 }
|
||||
|
||||
124
src/app/api/admin/data_migration/progress/route.ts
Normal file
124
src/app/api/admin/data_migration/progress/route.ts
Normal file
@@ -0,0 +1,124 @@
|
||||
/* eslint-disable @typescript-eslint/no-explicit-any */
|
||||
|
||||
import { NextRequest } from 'next/server';
|
||||
|
||||
import { getAuthInfoFromCookie } from '@/lib/auth';
|
||||
|
||||
export const runtime = 'nodejs';
|
||||
|
||||
// 存储进度信息的 Map
|
||||
const progressStore = new Map<string, {
|
||||
phase: string;
|
||||
current: number;
|
||||
total: number;
|
||||
message: string;
|
||||
timestamp: number;
|
||||
}>();
|
||||
|
||||
// 清理过期的进度信息（超过5分钟）
|
||||
setInterval(() => {
|
||||
const now = Date.now();
|
||||
for (const [key, value] of progressStore.entries()) {
|
||||
if (now - value.timestamp > 5 * 60 * 1000) {
|
||||
progressStore.delete(key);
|
||||
}
|
||||
}
|
||||
}, 60 * 1000);
|
||||
|
||||
export async function GET(req: NextRequest) {
|
||||
// 验证身份和权限
|
||||
const authInfo = getAuthInfoFromCookie(req);
|
||||
if (!authInfo || !authInfo.username) {
|
||||
return new Response('Unauthorized', { status: 401 });
|
||||
}
|
||||
|
||||
if (authInfo.username !== process.env.USERNAME) {
|
||||
return new Response('Forbidden', { status: 403 });
|
||||
}
|
||||
|
||||
const { searchParams } = new URL(req.url);
|
||||
const operation = searchParams.get('operation'); // 'export' or 'import'
|
||||
|
||||
if (!operation) {
|
||||
return new Response('Missing operation parameter', { status: 400 });
|
||||
}
|
||||
|
||||
const progressKey = `${authInfo.username}:${operation}`;
|
||||
|
||||
// 创建 SSE 响应
|
||||
const encoder = new TextEncoder();
|
||||
let interval: NodeJS.Timeout | null = null;
|
||||
let timeout: NodeJS.Timeout | null = null;
|
||||
|
||||
const stream = new ReadableStream({
|
||||
start(controller) {
|
||||
const sendProgress = () => {
|
||||
try {
|
||||
const progress = progressStore.get(progressKey);
|
||||
if (progress) {
|
||||
const data = JSON.stringify(progress);
|
||||
controller.enqueue(encoder.encode(`data: ${data}\n\n`));
|
||||
}
|
||||
} catch (error) {
|
||||
// 如果控制器已关闭，清理定时器
|
||||
if (interval) clearInterval(interval);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
}
|
||||
};
|
||||
|
||||
// 立即发送一次
|
||||
sendProgress();
|
||||
|
||||
// 每秒发送一次进度更新
|
||||
interval = setInterval(sendProgress, 1000);
|
||||
|
||||
// 30秒后自动关闭连接
|
||||
timeout = setTimeout(() => {
|
||||
if (interval) clearInterval(interval);
|
||||
try {
|
||||
controller.close();
|
||||
} catch (error) {
|
||||
// 控制器可能已经关闭
|
||||
}
|
||||
}, 30000);
|
||||
},
|
||||
cancel() {
|
||||
// 当客户端断开连接时清理
|
||||
if (interval) clearInterval(interval);
|
||||
if (timeout) clearTimeout(timeout);
|
||||
},
|
||||
});
|
||||
|
||||
return new Response(stream, {
|
||||
headers: {
|
||||
'Content-Type': 'text/event-stream',
|
||||
'Cache-Control': 'no-cache',
|
||||
'Connection': 'keep-alive',
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
// 辅助函数：更新进度
|
||||
export function updateProgress(
|
||||
username: string,
|
||||
operation: 'export' | 'import',
|
||||
phase: string,
|
||||
current: number,
|
||||
total: number,
|
||||
message: string
|
||||
) {
|
||||
const progressKey = `${username}:${operation}`;
|
||||
progressStore.set(progressKey, {
|
||||
phase,
|
||||
current,
|
||||
total,
|
||||
message,
|
||||
timestamp: Date.now(),
|
||||
});
|
||||
}
|
||||
|
||||
// 辅助函数：清除进度
|
||||
export function clearProgress(username: string, operation: 'export' | 'import') {
|
||||
const progressKey = `${username}:${operation}`;
|
||||
progressStore.delete(progressKey);
|
||||
}
|
||||
@@ -144,6 +144,18 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
const [selectedFile, setSelectedFile] = useState<File | null>(null);
|
||||
const [isExporting, setIsExporting] = useState(false);
|
||||
const [isImporting, setIsImporting] = useState(false);
|
||||
const [exportProgress, setExportProgress] = useState<{
|
||||
phase: string;
|
||||
current: number;
|
||||
total: number;
|
||||
message: string;
|
||||
} | null>(null);
|
||||
const [importProgress, setImportProgress] = useState<{
|
||||
phase: string;
|
||||
current: number;
|
||||
total: number;
|
||||
message: string;
|
||||
} | null>(null);
|
||||
const [alertModal, setAlertModal] = useState<{
|
||||
isOpen: boolean;
|
||||
type: 'success' | 'error' | 'warning';
|
||||
@@ -180,8 +192,22 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
return;
|
||||
}
|
||||
|
||||
let eventSource: EventSource | null = null;
|
||||
|
||||
try {
|
||||
setIsExporting(true);
|
||||
setExportProgress(null);
|
||||
|
||||
// 连接到进度 SSE 端点
|
||||
eventSource = new EventSource('/api/admin/data_migration/progress?operation=export');
|
||||
eventSource.onmessage = (event) => {
|
||||
try {
|
||||
const progress = JSON.parse(event.data);
|
||||
setExportProgress(progress);
|
||||
} catch (e) {
|
||||
console.error('Failed to parse progress:', e);
|
||||
}
|
||||
};
|
||||
|
||||
const response = await fetch('/api/admin/data_migration/export', {
|
||||
method: 'POST',
|
||||
@@ -234,6 +260,10 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
});
|
||||
} finally {
|
||||
setIsExporting(false);
|
||||
setExportProgress(null);
|
||||
if (eventSource) {
|
||||
eventSource.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -265,8 +295,22 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
return;
|
||||
}
|
||||
|
||||
let eventSource: EventSource | null = null;
|
||||
|
||||
try {
|
||||
setIsImporting(true);
|
||||
setImportProgress(null);
|
||||
|
||||
// 连接到进度 SSE 端点
|
||||
eventSource = new EventSource('/api/admin/data_migration/progress?operation=import');
|
||||
eventSource.onmessage = (event) => {
|
||||
try {
|
||||
const progress = JSON.parse(event.data);
|
||||
setImportProgress(progress);
|
||||
} catch (e) {
|
||||
console.error('Failed to parse progress:', e);
|
||||
}
|
||||
};
|
||||
|
||||
const formData = new FormData();
|
||||
formData.append('file', selectedFile);
|
||||
@@ -322,6 +366,10 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
});
|
||||
} finally {
|
||||
setIsImporting(false);
|
||||
setImportProgress(null);
|
||||
if (eventSource) {
|
||||
eventSource.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
@@ -393,9 +441,24 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
}`}
|
||||
>
|
||||
{isExporting ? (
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<div className="w-4 h-4 border-2 border-white border-t-transparent rounded-full animate-spin"></div>
|
||||
导出中...
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<div className="w-4 h-4 border-2 border-white border-t-transparent rounded-full animate-spin"></div>
|
||||
导出中...
|
||||
</div>
|
||||
{exportProgress && (
|
||||
<div className="bg-gray-100 dark:bg-gray-700 rounded-lg p-3 space-y-2">
|
||||
<div className="text-xs text-gray-900 dark:text-gray-100 font-medium">{exportProgress.message}</div>
|
||||
{exportProgress.total > 0 && (
|
||||
<div className="w-full bg-gray-300 dark:bg-gray-600 rounded-full h-3">
|
||||
<div
|
||||
className="bg-yellow-500 h-3 rounded-full transition-all duration-300"
|
||||
style={{ width: `${(exportProgress.current / exportProgress.total) * 100}%` }}
|
||||
></div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
@@ -469,9 +532,24 @@ const DataMigration = ({ onRefreshConfig }: DataMigrationProps) => {
|
||||
}`}
|
||||
>
|
||||
{isImporting ? (
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<div className="w-4 h-4 border-2 border-white border-t-transparent rounded-full animate-spin"></div>
|
||||
导入中...
|
||||
<div className="space-y-2">
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
<div className="w-4 h-4 border-2 border-white border-t-transparent rounded-full animate-spin"></div>
|
||||
导入中...
|
||||
</div>
|
||||
{importProgress && (
|
||||
<div className="bg-gray-100 dark:bg-gray-700 rounded-lg p-3 space-y-2">
|
||||
<div className="text-xs text-gray-900 dark:text-gray-100 font-medium">{importProgress.message}</div>
|
||||
{importProgress.total > 0 && (
|
||||
<div className="w-full bg-gray-300 dark:bg-gray-600 rounded-full h-3">
|
||||
<div
|
||||
className="bg-yellow-500 h-3 rounded-full transition-all duration-300"
|
||||
style={{ width: `${(importProgress.current / importProgress.total) * 100}%` }}
|
||||
></div>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
) : (
|
||||
<div className="flex items-center justify-center gap-2">
|
||||
|
||||
Reference in New Issue
Block a user