反爬增强

src/app/api/douban-comments/route.ts

@@ -1,5 +1,6 @@
 import { NextRequest, NextResponse } from 'next/server';
 import * as cheerio from 'cheerio';
+import { fetchDoubanWithVerification } from '@/lib/douban-anti-crawler';
 
 export const runtime = 'nodejs';
 
@@ -25,20 +26,10 @@ export async function GET(request: NextRequest) {
   }
 
   try {
-    // 请求豆瓣短评页面
+    // 请求豆瓣短评页面（使用反爬验证）
     const url = `https://movie.douban.com/subject/${doubanId}/comments?start=${start}&limit=${limit}&status=P&sort=new_score`;
 
-    const response = await fetch(url, {
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36',
-        Accept:
-          'text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8',
-        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
-        Referer: 'https://movie.douban.com/',
-		Cookie: 'bid=sadjkfhadskbfasdfdjh'
-      },
-    });
+    const response = await fetchDoubanWithVerification(url);
 
     if (!response.ok) {
       return NextResponse.json(

src/app/api/douban-recommendations/route.ts

@@ -1,6 +1,7 @@
 import { NextRequest, NextResponse } from 'next/server';
 import * as cheerio from 'cheerio';
 import { fetchDoubanData } from '@/lib/douban';
+import { fetchDoubanWithVerification } from '@/lib/douban-anti-crawler';
 
 export const runtime = 'nodejs';
 
@@ -26,26 +27,10 @@ export async function GET(request: NextRequest) {
   }
 
   try {
-    // 请求豆瓣电影页面，使用和其他豆瓣API相同的请求头
+    // 请求豆瓣电影页面（使用反爬验证）
     const url = `https://movie.douban.com/subject/${doubanId}/`;
 
-    const controller = new AbortController();
-    const timeoutId = setTimeout(() => controller.abort(), 10000);
-
-    const response = await fetch(url, {
-      signal: controller.signal,
-      headers: {
-        'User-Agent':
-          'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
-        Referer: 'https://movie.douban.com/',
-        Accept: 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-        'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
-        Origin: 'https://movie.douban.com',
-		Cookie: 'bid=sadjkfhadskbfasdfdjh'
-      },
-    });
-
-    clearTimeout(timeoutId);
+    const response = await fetchDoubanWithVerification(url);
 
     if (!response.ok) {
       return NextResponse.json(

src/lib/douban-anti-crawler.ts

@@ -0,0 +1,199 @@
+import * as cheerio from 'cheerio';
+import { createHash } from 'crypto';
+
+/**
+ * 计算 SHA-512 哈希值
+ */
+function sha512(data: string): string {
+  return createHash('sha512').update(data).digest('hex');
+}
+
+/**
+ * 工作量证明算法 - 寻找满足难度要求的 nonce
+ * @param data 要哈希的数据
+ * @param difficulty 难度（前导零的数量）
+ * @returns 满足条件的 nonce
+ */
+function proofOfWork(data: string, difficulty = 4): number {
+  let nonce = 0;
+  const targetSubStr = '0'.repeat(difficulty);
+
+  while (true) {
+    nonce += 1;
+    const hash = sha512(data + nonce);
+    if (hash.startsWith(targetSubStr)) {
+      return nonce;
+    }
+  }
+}
+
+/**
+ * 解析豆瓣验证页面，提取表单数据
+ */
+function parseVerificationPage(html: string): {
+  tok: string;
+  cha: string;
+  red: string;
+} | null {
+  const $ = cheerio.load(html);
+
+  const tok = $('#tok').val() as string;
+  const cha = $('#cha').val() as string;
+  const red = $('#red').val() as string;
+
+  if (!tok || !cha || !red) {
+    console.error('Failed to extract verification form data');
+    return null;
+  }
+
+  return { tok, cha, red };
+}
+
+/**
+ * 获取豆瓣访问 cookie（处理反爬验证）
+ * @param url 要访问的豆瓣 URL
+ * @returns cookie 字符串
+ */
+export async function getDoubanCookie(url: string): Promise<string> {
+  const headers = {
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+    'Referer': 'https://movie.douban.com/',
+  };
+
+  try {
+    // 第一步：访问目标 URL，可能会被重定向到验证页面
+    const firstResponse = await fetch(url, {
+      headers,
+      redirect: 'manual', // 手动处理重定向
+    });
+
+    // 如果没有重定向，直接返回 cookie
+    if (firstResponse.status === 200) {
+      const cookies = firstResponse.headers.get('set-cookie');
+      return cookies || '';
+    }
+
+    // 如果是 302 重定向到验证页面
+    if (firstResponse.status === 302) {
+      const location = firstResponse.headers.get('location');
+      if (!location || !location.includes('sec.douban.com')) {
+        throw new Error('Unexpected redirect location');
+      }
+
+      console.log('Detected anti-crawler verification, processing...');
+
+      // 第二步：访问验证页面
+      const verifyResponse = await fetch(location, {
+        headers,
+      });
+
+      if (!verifyResponse.ok) {
+        throw new Error(`Failed to fetch verification page: ${verifyResponse.status}`);
+      }
+
+      const verifyHtml = await verifyResponse.text();
+
+      // 第三步：解析验证页面，提取表单数据
+      const formData = parseVerificationPage(verifyHtml);
+      if (!formData) {
+        throw new Error('Failed to parse verification page');
+      }
+
+      console.log('Calculating proof of work...');
+
+      // 第四步：计算工作量证明
+      const sol = proofOfWork(formData.cha, 4);
+
+      console.log('Proof of work calculated:', sol);
+
+      // 第五步：提交验证表单
+      const formBody = new URLSearchParams({
+        tok: formData.tok,
+        cha: formData.cha,
+        sol: sol.toString(),
+        red: formData.red,
+      });
+
+      const submitResponse = await fetch('https://sec.douban.com/c', {
+        method: 'POST',
+        headers: {
+          ...headers,
+          'Content-Type': 'application/x-www-form-urlencoded',
+        },
+        body: formBody.toString(),
+        redirect: 'manual', // 手动处理重定向
+      });
+
+      // 第六步：从响应中提取 cookie
+      const setCookieHeader = submitResponse.headers.get('set-cookie');
+      if (!setCookieHeader) {
+        throw new Error('No cookie received after verification');
+      }
+
+      console.log('Successfully obtained douban cookie');
+      return setCookieHeader;
+    }
+
+    throw new Error(`Unexpected response status: ${firstResponse.status}`);
+  } catch (error) {
+    console.error('Failed to get douban cookie:', error);
+    throw error;
+  }
+}
+
+/**
+ * 带反爬验证的豆瓣请求
+ * @param url 要访问的豆瓣 URL
+ * @param options fetch 选项
+ * @returns Response 对象
+ */
+export async function fetchDoubanWithVerification(
+  url: string,
+  options: RequestInit = {}
+): Promise<Response> {
+  const headers = {
+    'User-Agent':
+      'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/121.0.0.0 Safari/537.36',
+    'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
+    'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
+    'Referer': 'https://movie.douban.com/',
+    ...options.headers,
+  };
+
+  try {
+    // 先尝试直接访问
+    let response = await fetch(url, {
+      ...options,
+      headers,
+      redirect: 'manual',
+    });
+
+    // 如果被重定向到验证页面，获取 cookie 后重试
+    if (response.status === 302) {
+      const location = response.headers.get('location');
+      if (location && location.includes('sec.douban.com')) {
+        console.log('Anti-crawler detected, obtaining cookie...');
+
+        // 获取验证 cookie
+        const cookie = await getDoubanCookie(url);
+
+        // 使用 cookie 重新请求
+        response = await fetch(url, {
+          ...options,
+          headers: {
+            ...headers,
+            Cookie: cookie,
+          },
+        });
+      }
+    }
+
+    return response;
+  } catch (error) {
+    console.error('Failed to fetch douban with verification:', error);
+    throw error;
+  }
+}