i18n: translate all Chinese comments, docstrings, and logger messages to English

Comprehensive translation of Chinese text to English across the entire codebase:

- api/: FastAPI server documentation and logger messages
- cache/: Cache abstraction layer comments and docstrings
- database/: Database models and MongoDB store documentation
- media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu)
- model/: Data model documentation
- proxy/: Proxy pool and provider documentation
- store/: Data storage layer comments
- tools/: Utility functions and browser automation
- test/: Test file documentation

Preserved: Chinese disclaimer header (lines 10-18) for legal compliance

🤖 Generated with [Claude Code](https://claude.com/claude-code)

Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
程序员阿江(Relakkes)
2025-12-26 23:27:19 +08:00
parent 1544d13dd5
commit 157ddfb21b
93 changed files with 1971 additions and 1955 deletions

View File

@@ -41,7 +41,7 @@ def run(
try:
await asyncio.wait_for(asyncio.shield(app_cleanup()), timeout=cleanup_timeout_seconds)
except asyncio.TimeoutError:
print(f"[Main] 清理超时({cleanup_timeout_seconds}s),跳过剩余清理。")
print(f"[Main] Cleanup timeout ({cleanup_timeout_seconds}s), skipping remaining cleanup.")
async def _cancel_remaining_tasks(timeout_seconds: float = 2.0) -> None:
current = asyncio.current_task()
@@ -70,11 +70,11 @@ def run(
nonlocal shutdown_requested
if shutdown_requested:
print("[Main] 再次收到中断信号,强制退出。")
print("[Main] Received interrupt signal again, force exit.")
os._exit(force_exit_code)
shutdown_requested = True
print(f"\n[Main] 收到中断信号 {signum},正在退出(清理最多{cleanup_timeout_seconds}s)...")
print(f"\n[Main] Received interrupt signal {signum}, exiting (cleanup max {cleanup_timeout_seconds}s)...")
if on_first_interrupt is not None:
try:
@@ -100,7 +100,7 @@ def run(
try:
await _cleanup_with_timeout()
except Exception as e:
print(f"[Main] 清理时出错: {e}")
print(f"[Main] Error during cleanup: {e}")
await _cancel_remaining_tasks()
if cancelled:

View File

@@ -33,8 +33,8 @@ from tools import utils
class BrowserLauncher:
"""
浏览器启动器,用于检测和启动用户的Chrome/Edge浏览器
支持Windows和macOS系统
Browser launcher for detecting and launching user's Chrome/Edge browser
Supports Windows and macOS systems
"""
def __init__(self):
@@ -44,19 +44,19 @@ class BrowserLauncher:
def detect_browser_paths(self) -> List[str]:
"""
检测系统中可用的浏览器路径
返回按优先级排序的浏览器路径列表
Detect available browser paths in system
Returns list of browser paths sorted by priority
"""
paths = []
if self.system == "Windows":
# Windows下的常见Chrome/Edge安装路径
# Common Chrome/Edge installation paths on Windows
possible_paths = [
# Chrome路径
# Chrome paths
os.path.expandvars(r"%PROGRAMFILES%\Google\Chrome\Application\chrome.exe"),
os.path.expandvars(r"%PROGRAMFILES(X86)%\Google\Chrome\Application\chrome.exe"),
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"),
# Edge路径
# Edge paths
os.path.expandvars(r"%PROGRAMFILES%\Microsoft\Edge\Application\msedge.exe"),
os.path.expandvars(r"%PROGRAMFILES(X86)%\Microsoft\Edge\Application\msedge.exe"),
# Chrome Beta/Dev/Canary
@@ -65,21 +65,21 @@ class BrowserLauncher:
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome SxS\Application\chrome.exe"),
]
elif self.system == "Darwin": # macOS
# macOS下的常见Chrome/Edge安装路径
# Common Chrome/Edge installation paths on macOS
possible_paths = [
# Chrome路径
# Chrome paths
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
"/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
"/Applications/Google Chrome Dev.app/Contents/MacOS/Google Chrome Dev",
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
# Edge路径
# Edge paths
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
"/Applications/Microsoft Edge Beta.app/Contents/MacOS/Microsoft Edge Beta",
"/Applications/Microsoft Edge Dev.app/Contents/MacOS/Microsoft Edge Dev",
"/Applications/Microsoft Edge Canary.app/Contents/MacOS/Microsoft Edge Canary",
]
else:
# Linux等其他系统
# Linux and other systems
possible_paths = [
"/usr/bin/google-chrome",
"/usr/bin/google-chrome-stable",
@@ -94,7 +94,7 @@ class BrowserLauncher:
"/usr/bin/microsoft-edge-dev",
]
# 检查路径是否存在且可执行
# Check if path exists and is executable
for path in possible_paths:
if os.path.isfile(path) and os.access(path, os.X_OK):
paths.append(path)
@@ -103,10 +103,10 @@ class BrowserLauncher:
def find_available_port(self, start_port: int = 9222) -> int:
"""
查找可用的端口
Find available port
"""
port = start_port
while port < start_port + 100: # 最多尝试100个端口
while port < start_port + 100: # Try up to 100 ports
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.bind(('localhost', port))
@@ -114,18 +114,18 @@ class BrowserLauncher:
except OSError:
port += 1
raise RuntimeError(f"无法找到可用的端口,已尝试 {start_port} {port-1}")
raise RuntimeError(f"Cannot find available port, tried {start_port} to {port-1}")
def launch_browser(self, browser_path: str, debug_port: int, headless: bool = False,
user_data_dir: Optional[str] = None) -> subprocess.Popen:
"""
启动浏览器进程
Launch browser process
"""
# 基本启动参数
# Basic launch arguments
args = [
browser_path,
f"--remote-debugging-port={debug_port}",
"--remote-debugging-address=0.0.0.0", # 允许远程访问
"--remote-debugging-address=0.0.0.0", # Allow remote access
"--no-first-run",
"--no-default-browser-check",
"--disable-background-timer-throttling",
@@ -136,36 +136,36 @@ class BrowserLauncher:
"--disable-hang-monitor",
"--disable-prompt-on-repost",
"--disable-sync",
"--disable-dev-shm-usage", # 避免共享内存问题
"--no-sandbox", # 在CDP模式下关闭沙箱
# 🔥 关键反检测参数
"--disable-blink-features=AutomationControlled", # 禁用自动化控制标记
"--exclude-switches=enable-automation", # 排除自动化开关
"--disable-infobars", # 禁用信息栏
"--disable-dev-shm-usage", # Avoid shared memory issues
"--no-sandbox", # Disable sandbox in CDP mode
# Key anti-detection arguments
"--disable-blink-features=AutomationControlled", # Disable automation control flag
"--exclude-switches=enable-automation", # Exclude automation switch
"--disable-infobars", # Disable info bars
]
# 无头模式
# Headless mode
if headless:
args.extend([
"--headless=new", # 使用新的headless模式
"--headless=new", # Use new headless mode
"--disable-gpu",
])
else:
# 非无头模式的额外参数
# Extra arguments for non-headless mode
args.extend([
"--start-maximized", # 最大化窗口,更像真实用户
"--start-maximized", # Maximize window, more like real user
])
# 用户数据目录
# User data directory
if user_data_dir:
args.append(f"--user-data-dir={user_data_dir}")
utils.logger.info(f"[BrowserLauncher] 启动浏览器: {browser_path}")
utils.logger.info(f"[BrowserLauncher] 调试端口: {debug_port}")
utils.logger.info(f"[BrowserLauncher] 无头模式: {headless}")
utils.logger.info(f"[BrowserLauncher] Launching browser: {browser_path}")
utils.logger.info(f"[BrowserLauncher] Debug port: {debug_port}")
utils.logger.info(f"[BrowserLauncher] Headless mode: {headless}")
try:
# Windows上,使用CREATE_NEW_PROCESS_GROUP避免Ctrl+C影响子进程
# On Windows, use CREATE_NEW_PROCESS_GROUP to prevent Ctrl+C from affecting subprocess
if self.system == "Windows":
process = subprocess.Popen(
args,
@@ -178,21 +178,21 @@ class BrowserLauncher:
args,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
preexec_fn=os.setsid # 创建新的进程组
preexec_fn=os.setsid # Create new process group
)
self.browser_process = process
return process
except Exception as e:
utils.logger.error(f"[BrowserLauncher] 启动浏览器失败: {e}")
utils.logger.error(f"[BrowserLauncher] Failed to launch browser: {e}")
raise
def wait_for_browser_ready(self, debug_port: int, timeout: int = 30) -> bool:
"""
等待浏览器准备就绪
Wait for browser to be ready
"""
utils.logger.info(f"[BrowserLauncher] 等待浏览器在端口 {debug_port} 上准备就绪...")
utils.logger.info(f"[BrowserLauncher] Waiting for browser to be ready on port {debug_port}...")
start_time = time.time()
while time.time() - start_time < timeout:
@@ -201,19 +201,19 @@ class BrowserLauncher:
s.settimeout(1)
result = s.connect_ex(('localhost', debug_port))
if result == 0:
utils.logger.info(f"[BrowserLauncher] 浏览器已在端口 {debug_port} 上准备就绪")
utils.logger.info(f"[BrowserLauncher] Browser is ready on port {debug_port}")
return True
except Exception:
pass
time.sleep(0.5)
utils.logger.error(f"[BrowserLauncher] 浏览器在 {timeout} 秒内未能准备就绪")
utils.logger.error(f"[BrowserLauncher] Browser failed to be ready within {timeout} seconds")
return False
def get_browser_info(self, browser_path: str) -> Tuple[str, str]:
"""
获取浏览器信息(名称和版本)
Get browser info (name and version)
"""
try:
if "chrome" in browser_path.lower():
@@ -225,7 +225,7 @@ class BrowserLauncher:
else:
name = "Unknown Browser"
# 尝试获取版本信息
# Try to get version info
try:
result = subprocess.run([browser_path, "--version"],
capture_output=True, text=True, timeout=5)
@@ -240,7 +240,7 @@ class BrowserLauncher:
def cleanup(self):
"""
清理资源,关闭浏览器进程
Cleanup resources, close browser process
"""
if not self.browser_process:
return
@@ -248,20 +248,20 @@ class BrowserLauncher:
process = self.browser_process
if process.poll() is not None:
utils.logger.info("[BrowserLauncher] 浏览器进程已退出,无需清理")
utils.logger.info("[BrowserLauncher] Browser process already exited, no cleanup needed")
self.browser_process = None
return
utils.logger.info("[BrowserLauncher] 正在关闭浏览器进程...")
utils.logger.info("[BrowserLauncher] Closing browser process...")
try:
if self.system == "Windows":
# 先尝试正常终止
# First try normal termination
process.terminate()
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
utils.logger.warning("[BrowserLauncher] 正常终止超时使用taskkill强制结束")
utils.logger.warning("[BrowserLauncher] Normal termination timeout, using taskkill to force kill")
subprocess.run(
["taskkill", "/F", "/T", "/PID", str(process.pid)],
capture_output=True,
@@ -273,17 +273,17 @@ class BrowserLauncher:
try:
os.killpg(pgid, signal.SIGTERM)
except ProcessLookupError:
utils.logger.info("[BrowserLauncher] 浏览器进程组不存在,可能已退出")
utils.logger.info("[BrowserLauncher] Browser process group does not exist, may have exited")
else:
try:
process.wait(timeout=5)
except subprocess.TimeoutExpired:
utils.logger.warning("[BrowserLauncher] 优雅关闭超时,发送SIGKILL")
utils.logger.warning("[BrowserLauncher] Graceful shutdown timeout, sending SIGKILL")
os.killpg(pgid, signal.SIGKILL)
process.wait(timeout=5)
utils.logger.info("[BrowserLauncher] 浏览器进程已关闭")
utils.logger.info("[BrowserLauncher] Browser process closed")
except Exception as e:
utils.logger.warning(f"[BrowserLauncher] 关闭浏览器进程时出错: {e}")
utils.logger.warning(f"[BrowserLauncher] Error closing browser process: {e}")
finally:
self.browser_process = None

View File

@@ -34,7 +34,7 @@ from tools import utils
class CDPBrowserManager:
"""
CDP浏览器管理器负责启动和管理通过CDP连接的浏览器
CDP browser manager, responsible for launching and managing browsers connected via CDP
"""
def __init__(self):
@@ -46,27 +46,27 @@ class CDPBrowserManager:
def _register_cleanup_handlers(self):
"""
注册清理处理器,确保程序退出时清理浏览器进程
Register cleanup handlers to ensure browser process cleanup on program exit
"""
if self._cleanup_registered:
return
def sync_cleanup():
"""同步清理函数,用于atexit"""
"""Synchronous cleanup function for atexit"""
if self.launcher and self.launcher.browser_process:
utils.logger.info("[CDPBrowserManager] atexit: 清理浏览器进程")
utils.logger.info("[CDPBrowserManager] atexit: Cleaning up browser process")
self.launcher.cleanup()
# 注册atexit清理
# Register atexit cleanup
atexit.register(sync_cleanup)
# 注册信号处理器(仅在没有自定义处理器时注册,避免覆盖主入口的信号处理逻辑)
# Register signal handlers (only when no custom handlers exist, to avoid overriding main entry signal handling logic)
prev_sigint = signal.getsignal(signal.SIGINT)
prev_sigterm = signal.getsignal(signal.SIGTERM)
def signal_handler(signum, frame):
"""信号处理器"""
utils.logger.info(f"[CDPBrowserManager] 收到信号 {signum},清理浏览器进程")
"""Signal handler"""
utils.logger.info(f"[CDPBrowserManager] Received signal {signum}, cleaning up browser process")
if self.launcher and self.launcher.browser_process:
self.launcher.cleanup()
@@ -80,19 +80,19 @@ class CDPBrowserManager:
install_sigint = prev_sigint in (signal.default_int_handler, signal.SIG_DFL)
install_sigterm = prev_sigterm == signal.SIG_DFL
# 注册SIGINT (Ctrl+C) SIGTERM
# Register SIGINT (Ctrl+C) and SIGTERM
if install_sigint:
signal.signal(signal.SIGINT, signal_handler)
else:
utils.logger.info("[CDPBrowserManager] 已存在SIGINT处理器,跳过注册以避免覆盖")
utils.logger.info("[CDPBrowserManager] SIGINT handler already exists, skipping registration to avoid override")
if install_sigterm:
signal.signal(signal.SIGTERM, signal_handler)
else:
utils.logger.info("[CDPBrowserManager] 已存在SIGTERM处理器,跳过注册以避免覆盖")
utils.logger.info("[CDPBrowserManager] SIGTERM handler already exists, skipping registration to avoid override")
self._cleanup_registered = True
utils.logger.info("[CDPBrowserManager] 清理处理器已注册")
utils.logger.info("[CDPBrowserManager] Cleanup handlers registered")
async def launch_and_connect(
self,
@@ -102,25 +102,25 @@ class CDPBrowserManager:
headless: bool = False,
) -> BrowserContext:
"""
启动浏览器并通过CDP连接
Launch browser and connect via CDP
"""
try:
# 1. 检测浏览器路径
# 1. Detect browser path
browser_path = await self._get_browser_path()
# 2. 获取可用端口
# 2. Get available port
self.debug_port = self.launcher.find_available_port(config.CDP_DEBUG_PORT)
# 3. 启动浏览器
# 3. Launch browser
await self._launch_browser(browser_path, headless)
# 4. 注册清理处理器(确保异常退出时也能清理)
# 4. Register cleanup handlers (ensure cleanup on abnormal exit)
self._register_cleanup_handlers()
# 5. 通过CDP连接
# 5. Connect via CDP
await self._connect_via_cdp(playwright)
# 5. 创建浏览器上下文
# 6. Create browser context
browser_context = await self._create_browser_context(
playwright_proxy, user_agent
)
@@ -129,68 +129,68 @@ class CDPBrowserManager:
return browser_context
except Exception as e:
utils.logger.error(f"[CDPBrowserManager] CDP浏览器启动失败: {e}")
utils.logger.error(f"[CDPBrowserManager] CDP browser launch failed: {e}")
await self.cleanup()
raise
async def _get_browser_path(self) -> str:
"""
获取浏览器路径
Get browser path
"""
# 优先使用用户自定义路径
# Prefer user-defined path
if config.CUSTOM_BROWSER_PATH and os.path.isfile(config.CUSTOM_BROWSER_PATH):
utils.logger.info(
f"[CDPBrowserManager] 使用自定义浏览器路径: {config.CUSTOM_BROWSER_PATH}"
f"[CDPBrowserManager] Using custom browser path: {config.CUSTOM_BROWSER_PATH}"
)
return config.CUSTOM_BROWSER_PATH
# 自动检测浏览器路径
# Auto-detect browser path
browser_paths = self.launcher.detect_browser_paths()
if not browser_paths:
raise RuntimeError(
"未找到可用的浏览器。请确保已安装Chrome或Edge浏览器"
"或在配置文件中设置CUSTOM_BROWSER_PATH指定浏览器路径。"
"No available browser found. Please ensure Chrome or Edge browser is installed, "
"or set CUSTOM_BROWSER_PATH in config file to specify browser path."
)
browser_path = browser_paths[0] # 使用第一个找到的浏览器
browser_path = browser_paths[0] # Use the first browser found
browser_name, browser_version = self.launcher.get_browser_info(browser_path)
utils.logger.info(
f"[CDPBrowserManager] 检测到浏览器: {browser_name} ({browser_version})"
f"[CDPBrowserManager] Detected browser: {browser_name} ({browser_version})"
)
utils.logger.info(f"[CDPBrowserManager] 浏览器路径: {browser_path}")
utils.logger.info(f"[CDPBrowserManager] Browser path: {browser_path}")
return browser_path
async def _test_cdp_connection(self, debug_port: int) -> bool:
"""
测试CDP连接是否可用
Test if CDP connection is available
"""
try:
# 简单的socket连接测试
# Simple socket connection test
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.settimeout(5)
result = s.connect_ex(("localhost", debug_port))
if result == 0:
utils.logger.info(
f"[CDPBrowserManager] CDP端口 {debug_port} 可访问"
f"[CDPBrowserManager] CDP port {debug_port} is accessible"
)
return True
else:
utils.logger.warning(
f"[CDPBrowserManager] CDP端口 {debug_port} 不可访问"
f"[CDPBrowserManager] CDP port {debug_port} is not accessible"
)
return False
except Exception as e:
utils.logger.warning(f"[CDPBrowserManager] CDP连接测试失败: {e}")
utils.logger.warning(f"[CDPBrowserManager] CDP connection test failed: {e}")
return False
async def _launch_browser(self, browser_path: str, headless: bool):
"""
启动浏览器进程
Launch browser process
"""
# 设置用户数据目录(如果启用了保存登录状态)
# Set user data directory (if save login state is enabled)
user_data_dir = None
if config.SAVE_LOGIN_STATE:
user_data_dir = os.path.join(
@@ -199,9 +199,9 @@ class CDPBrowserManager:
f"cdp_{config.USER_DATA_DIR % config.PLATFORM}",
)
os.makedirs(user_data_dir, exist_ok=True)
utils.logger.info(f"[CDPBrowserManager] 用户数据目录: {user_data_dir}")
utils.logger.info(f"[CDPBrowserManager] User data directory: {user_data_dir}")
# 启动浏览器
# Launch browser
self.launcher.browser_process = self.launcher.launch_browser(
browser_path=browser_path,
debug_port=self.debug_port,
@@ -209,24 +209,24 @@ class CDPBrowserManager:
user_data_dir=user_data_dir,
)
# 等待浏览器准备就绪
# Wait for browser to be ready
if not self.launcher.wait_for_browser_ready(
self.debug_port, config.BROWSER_LAUNCH_TIMEOUT
):
raise RuntimeError(f"浏览器在 {config.BROWSER_LAUNCH_TIMEOUT} 秒内未能启动")
raise RuntimeError(f"Browser failed to start within {config.BROWSER_LAUNCH_TIMEOUT} seconds")
# 额外等待一秒让CDP服务完全启动
# Extra wait for CDP service to fully start
await asyncio.sleep(1)
# 测试CDP连接
# Test CDP connection
if not await self._test_cdp_connection(self.debug_port):
utils.logger.warning(
"[CDPBrowserManager] CDP连接测试失败,但将继续尝试连接"
"[CDPBrowserManager] CDP connection test failed, but will continue to try connecting"
)
async def _get_browser_websocket_url(self, debug_port: int) -> str:
"""
获取浏览器的WebSocket连接URL
Get browser WebSocket connection URL
"""
try:
async with httpx.AsyncClient() as client:
@@ -238,196 +238,196 @@ class CDPBrowserManager:
ws_url = data.get("webSocketDebuggerUrl")
if ws_url:
utils.logger.info(
f"[CDPBrowserManager] 获取到浏览器WebSocket URL: {ws_url}"
f"[CDPBrowserManager] Got browser WebSocket URL: {ws_url}"
)
return ws_url
else:
raise RuntimeError("未找到webSocketDebuggerUrl")
raise RuntimeError("webSocketDebuggerUrl not found")
else:
raise RuntimeError(f"HTTP {response.status_code}: {response.text}")
except Exception as e:
utils.logger.error(f"[CDPBrowserManager] 获取WebSocket URL失败: {e}")
utils.logger.error(f"[CDPBrowserManager] Failed to get WebSocket URL: {e}")
raise
async def _connect_via_cdp(self, playwright: Playwright):
"""
通过CDP连接到浏览器
Connect to browser via CDP
"""
try:
# 获取正确的WebSocket URL
# Get correct WebSocket URL
ws_url = await self._get_browser_websocket_url(self.debug_port)
utils.logger.info(f"[CDPBrowserManager] 正在通过CDP连接到浏览器: {ws_url}")
utils.logger.info(f"[CDPBrowserManager] Connecting to browser via CDP: {ws_url}")
# 使用PlaywrightconnectOverCDP方法连接
# Use Playwright's connectOverCDP method to connect
self.browser = await playwright.chromium.connect_over_cdp(ws_url)
if self.browser.is_connected():
utils.logger.info("[CDPBrowserManager] 成功连接到浏览器")
utils.logger.info("[CDPBrowserManager] Successfully connected to browser")
utils.logger.info(
f"[CDPBrowserManager] 浏览器上下文数量: {len(self.browser.contexts)}"
f"[CDPBrowserManager] Browser contexts count: {len(self.browser.contexts)}"
)
else:
raise RuntimeError("CDP连接失败")
raise RuntimeError("CDP connection failed")
except Exception as e:
utils.logger.error(f"[CDPBrowserManager] CDP连接失败: {e}")
utils.logger.error(f"[CDPBrowserManager] CDP connection failed: {e}")
raise
async def _create_browser_context(
self, playwright_proxy: Optional[Dict] = None, user_agent: Optional[str] = None
) -> BrowserContext:
"""
创建或获取浏览器上下文
Create or get browser context
"""
if not self.browser:
raise RuntimeError("浏览器未连接")
raise RuntimeError("Browser not connected")
# 获取现有上下文或创建新的上下文
# Get existing context or create new context
contexts = self.browser.contexts
if contexts:
# 使用现有的第一个上下文
# Use existing first context
browser_context = contexts[0]
utils.logger.info("[CDPBrowserManager] 使用现有的浏览器上下文")
utils.logger.info("[CDPBrowserManager] Using existing browser context")
else:
# 创建新的上下文
# Create new context
context_options = {
"viewport": {"width": 1920, "height": 1080},
"accept_downloads": True,
}
# 设置用户代理
# Set user agent
if user_agent:
context_options["user_agent"] = user_agent
utils.logger.info(f"[CDPBrowserManager] 设置用户代理: {user_agent}")
utils.logger.info(f"[CDPBrowserManager] Setting user agent: {user_agent}")
# 注意CDP模式下代理设置可能不生效因为浏览器已经启动
# Note: Proxy settings may not work in CDP mode since browser is already launched
if playwright_proxy:
utils.logger.warning(
"[CDPBrowserManager] 警告: CDP模式下代理设置可能不生效"
"建议在浏览器启动前配置系统代理或浏览器代理扩展"
"[CDPBrowserManager] Warning: Proxy settings may not work in CDP mode, "
"recommend configuring system proxy or browser proxy extension before launching browser"
)
browser_context = await self.browser.new_context(**context_options)
utils.logger.info("[CDPBrowserManager] 创建新的浏览器上下文")
utils.logger.info("[CDPBrowserManager] Created new browser context")
return browser_context
async def add_stealth_script(self, script_path: str = "libs/stealth.min.js"):
"""
添加反检测脚本
Add anti-detection script
"""
if self.browser_context and os.path.exists(script_path):
try:
await self.browser_context.add_init_script(path=script_path)
utils.logger.info(
f"[CDPBrowserManager] 已添加反检测脚本: {script_path}"
f"[CDPBrowserManager] Added anti-detection script: {script_path}"
)
except Exception as e:
utils.logger.warning(f"[CDPBrowserManager] 添加反检测脚本失败: {e}")
utils.logger.warning(f"[CDPBrowserManager] Failed to add anti-detection script: {e}")
async def add_cookies(self, cookies: list):
"""
添加Cookie
Add cookies
"""
if self.browser_context:
try:
await self.browser_context.add_cookies(cookies)
utils.logger.info(f"[CDPBrowserManager] 已添加 {len(cookies)} 个Cookie")
utils.logger.info(f"[CDPBrowserManager] Added {len(cookies)} cookies")
except Exception as e:
utils.logger.warning(f"[CDPBrowserManager] 添加Cookie失败: {e}")
utils.logger.warning(f"[CDPBrowserManager] Failed to add cookies: {e}")
async def get_cookies(self) -> list:
"""
获取当前Cookie
Get current cookies
"""
if self.browser_context:
try:
cookies = await self.browser_context.cookies()
return cookies
except Exception as e:
utils.logger.warning(f"[CDPBrowserManager] 获取Cookie失败: {e}")
utils.logger.warning(f"[CDPBrowserManager] Failed to get cookies: {e}")
return []
return []
async def cleanup(self, force: bool = False):
"""
清理资源
Cleanup resources
Args:
force: 是否强制清理浏览器进程(忽略AUTO_CLOSE_BROWSER配置)
force: Whether to force cleanup browser process (ignoring AUTO_CLOSE_BROWSER config)
"""
try:
# 关闭浏览器上下文
# Close browser context
if self.browser_context:
try:
# 检查上下文是否已经关闭
# 尝试获取页面列表,如果失败说明已经关闭
# Check if context is already closed
# Try to get page list, if fails means already closed
try:
pages = self.browser_context.pages
if pages is not None:
await self.browser_context.close()
utils.logger.info("[CDPBrowserManager] 浏览器上下文已关闭")
utils.logger.info("[CDPBrowserManager] Browser context closed")
except:
utils.logger.debug("[CDPBrowserManager] 浏览器上下文已经被关闭")
utils.logger.debug("[CDPBrowserManager] Browser context already closed")
except Exception as context_error:
# 只在错误不是因为已关闭时才记录警告
# Only log warning if error is not due to already being closed
error_msg = str(context_error).lower()
if "closed" not in error_msg and "disconnected" not in error_msg:
utils.logger.warning(
f"[CDPBrowserManager] 关闭浏览器上下文失败: {context_error}"
f"[CDPBrowserManager] Failed to close browser context: {context_error}"
)
else:
utils.logger.debug(f"[CDPBrowserManager] 浏览器上下文已关闭: {context_error}")
utils.logger.debug(f"[CDPBrowserManager] Browser context already closed: {context_error}")
finally:
self.browser_context = None
# 断开浏览器连接
# Disconnect browser
if self.browser:
try:
# 检查浏览器是否仍然连接
# Check if browser is still connected
if self.browser.is_connected():
await self.browser.close()
utils.logger.info("[CDPBrowserManager] 浏览器连接已断开")
utils.logger.info("[CDPBrowserManager] Browser connection disconnected")
else:
utils.logger.debug("[CDPBrowserManager] 浏览器连接已经断开")
utils.logger.debug("[CDPBrowserManager] Browser connection already disconnected")
except Exception as browser_error:
# 只在错误不是因为已关闭时才记录警告
# Only log warning if error is not due to already being closed
error_msg = str(browser_error).lower()
if "closed" not in error_msg and "disconnected" not in error_msg:
utils.logger.warning(
f"[CDPBrowserManager] 关闭浏览器连接失败: {browser_error}"
f"[CDPBrowserManager] Failed to close browser connection: {browser_error}"
)
else:
utils.logger.debug(f"[CDPBrowserManager] 浏览器连接已关闭: {browser_error}")
utils.logger.debug(f"[CDPBrowserManager] Browser connection already closed: {browser_error}")
finally:
self.browser = None
# 关闭浏览器进程
# force=True 时强制关闭,忽略AUTO_CLOSE_BROWSER配置
# 这用于处理异常退出或手动清理的情况
# Close browser process
# force=True means force close, ignoring AUTO_CLOSE_BROWSER config
# Used for handling abnormal exit or manual cleanup
if force or config.AUTO_CLOSE_BROWSER:
if self.launcher and self.launcher.browser_process:
self.launcher.cleanup()
else:
utils.logger.debug("[CDPBrowserManager] 没有需要清理的浏览器进程")
utils.logger.debug("[CDPBrowserManager] No browser process to cleanup")
else:
utils.logger.info(
"[CDPBrowserManager] 浏览器进程保持运行(AUTO_CLOSE_BROWSER=False"
"[CDPBrowserManager] Browser process kept running (AUTO_CLOSE_BROWSER=False)"
)
except Exception as e:
utils.logger.error(f"[CDPBrowserManager] 清理资源时出错: {e}")
utils.logger.error(f"[CDPBrowserManager] Error during resource cleanup: {e}")
def is_connected(self) -> bool:
"""
检查是否已连接到浏览器
Check if connected to browser
"""
return self.browser is not None and self.browser.is_connected()
async def get_browser_info(self) -> Dict[str, Any]:
"""
获取浏览器信息
Get browser info
"""
if not self.browser:
return {}
@@ -443,5 +443,5 @@ class CDPBrowserManager:
"is_connected": self.is_connected(),
}
except Exception as e:
utils.logger.warning(f"[CDPBrowserManager] 获取浏览器信息失败: {e}")
utils.logger.warning(f"[CDPBrowserManager] Failed to get browser info: {e}")
return {}

View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 12:53
# @Desc : 爬虫相关的工具函数
# @Desc : Crawler utility functions
import base64
import json
@@ -73,13 +73,13 @@ async def find_qrcode_img_from_canvas(page: Page, canvas_selector: str) -> str:
"""
# 等待Canvas元素加载完成
# Wait for Canvas element to load
canvas = await page.wait_for_selector(canvas_selector)
# 截取Canvas元素的截图
# Take screenshot of Canvas element
screenshot = await canvas.screenshot()
# 将截图转换为base64格式
# Convert screenshot to base64 format
base64_image = base64.b64encode(screenshot).decode('utf-8')
return base64_image
@@ -185,7 +185,7 @@ def format_proxy_info(ip_proxy_info) -> Tuple[Optional[Dict], Optional[str]]:
"username": ip_proxy_info.user,
"password": ip_proxy_info.password,
}
# httpx 0.28.1 需要直接传入代理URL字符串而不是字典
# httpx 0.28.1 requires passing proxy URL string directly, not a dictionary
if ip_proxy_info.user and ip_proxy_info.password:
httpx_proxy = f"http://{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
else:

View File

@@ -17,13 +17,13 @@
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
"""
文件头版权声明管理工具
File header copyright declaration management tool
功能:
- 自动为Python文件添加标准化的版权声明和免责声明
- 智能检测现有文件头(编码声明、作者信息、免责声明等)
- 在合适位置插入版权信息,不破坏现有内容
- 支持批量处理和单文件检查模式
Features:
- Automatically add standardized copyright declaration and disclaimer to Python files
- Intelligently detect existing file headers (encoding declaration, author info, disclaimer, etc.)
- Insert copyright info at appropriate position without breaking existing content
- Support batch processing and single file check mode
"""
import os
@@ -31,14 +31,14 @@ import re
import sys
from typing import List, Tuple
# 项目配置
# Project configuration
REPO_URL = "https://github.com/NanmiCoder/MediaCrawler"
GITHUB_PROFILE = "https://github.com/NanmiCoder"
EMAIL = "relakkes@gmail.com"
COPYRIGHT_YEAR = "2025"
LICENSE_TYPE = "NON-COMMERCIAL LEARNING LICENSE 1.1"
# 免责声明标准文本
# Disclaimer standard text
DISCLAIMER = """# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
# 1. 不得用于任何商业用途。
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
@@ -52,27 +52,27 @@ DISCLAIMER = """# 声明:本代码仅供学习和研究目的使用。使用
def get_file_relative_path(file_path: str, project_root: str) -> str:
"""
获取文件相对于项目根目录的路径
Get file path relative to project root
Args:
file_path: 文件绝对路径
project_root: 项目根目录
file_path: File absolute path
project_root: Project root directory
Returns:
相对路径字符串
Relative path string
"""
return os.path.relpath(file_path, project_root)
def generate_copyright_header(relative_path: str) -> str:
"""
生成版权声明头部
Generate copyright declaration header
Args:
relative_path: 文件相对于项目根目录的路径
relative_path: File path relative to project root
Returns:
格式化的版权声明字符串
Formatted copyright declaration string
"""
file_url = f"{REPO_URL}/blob/main/{relative_path}"
@@ -89,53 +89,53 @@ def generate_copyright_header(relative_path: str) -> str:
def has_copyright_header(content: str) -> bool:
"""
检查文件是否已包含版权声明
Check if file already contains copyright declaration
Args:
content: 文件内容
content: File content
Returns:
True如果已包含版权声明
True if already contains copyright declaration
"""
# 检查是否包含Copyright关键字
# Check if contains Copyright keyword
return "Copyright (c)" in content and "MediaCrawler project" in content
def has_disclaimer(content: str) -> bool:
"""
检查文件是否已包含免责声明
Check if file already contains disclaimer
Args:
content: 文件内容
content: File content
Returns:
True如果已包含免责声明
True if already contains disclaimer
"""
return "声明:本代码仅供学习和研究目的使用" in content
def find_insert_position(lines: List[str]) -> Tuple[int, bool]:
"""
找到插入版权声明的位置
Find position to insert copyright declaration
Args:
lines: 文件内容行列表
lines: List of file content lines
Returns:
(插入行号, 是否需要在前面添加编码声明)
(insert line number, whether encoding declaration needs to be added)
"""
insert_pos = 0
has_encoding = False
# 检查第一行是否是shebang
# Check if first line is shebang
if lines and lines[0].startswith('#!'):
insert_pos = 1
# 检查编码声明通常在第1或2行
# Check encoding declaration (usually on line 1 or 2)
for i in range(insert_pos, min(insert_pos + 2, len(lines))):
if i < len(lines):
line = lines[i].strip()
# 匹配 # -*- coding: utf-8 -*- # coding: utf-8 等格式
# Match # -*- coding: utf-8 -*- or # coding: utf-8 etc.
if re.match(r'#.*coding[:=]\s*([-\w.]+)', line):
has_encoding = True
insert_pos = i + 1
@@ -146,59 +146,59 @@ def find_insert_position(lines: List[str]) -> Tuple[int, bool]:
def process_file(file_path: str, project_root: str, dry_run: bool = False) -> Tuple[bool, str]:
"""
处理单个Python文件
Process single Python file
Args:
file_path: 文件路径
project_root: 项目根目录
dry_run: 仅检查不修改
file_path: File path
project_root: Project root directory
dry_run: Check only without modification
Returns:
(是否需要修改, 状态消息)
(whether modification needed, status message)
"""
try:
with open(file_path, 'r', encoding='utf-8') as f:
content = f.read()
lines = content.splitlines(keepends=True)
# 如果已经有版权声明,跳过
# Skip if already has copyright header
if has_copyright_header(content):
return False, f"✓ Already has copyright header: {file_path}"
# 获取相对路径
# Get relative path
relative_path = get_file_relative_path(file_path, project_root)
# 生成版权声明
# Generate copyright header
copyright_header = generate_copyright_header(relative_path)
# 查找插入位置
# Find insert position
insert_pos, has_encoding = find_insert_position(lines)
# 构建新的文件内容
# Build new file content
new_lines = []
# 如果没有编码声明,添加一个
# Add encoding declaration if not present
if not has_encoding:
new_lines.append("# -*- coding: utf-8 -*-\n")
# 添加前面的部分shebang和编码声明
# Add front part (shebang and encoding declaration)
new_lines.extend(lines[:insert_pos])
# 添加版权声明
# Add copyright header
new_lines.append(copyright_header + "\n")
# 如果文件没有免责声明,添加免责声明
# Add disclaimer if file doesn't have one
if not has_disclaimer(content):
new_lines.append(DISCLAIMER + "\n")
# 添加一个空行(如果下一行不是空行)
# Add empty line (if next line is not empty)
if insert_pos < len(lines) and lines[insert_pos].strip():
new_lines.append("\n")
# 添加剩余的内容
# Add remaining content
new_lines.extend(lines[insert_pos:])
# 如果不是dry run,写入文件
# Write to file if not dry run
if not dry_run:
with open(file_path, 'w', encoding='utf-8') as f:
f.writelines(new_lines)
@@ -212,14 +212,14 @@ def process_file(file_path: str, project_root: str, dry_run: bool = False) -> Tu
def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List[str]:
"""
查找所有Python文件
Find all Python files
Args:
root_dir: 根目录
exclude_patterns: 排除的目录模式
root_dir: Root directory
exclude_patterns: Directory patterns to exclude
Returns:
Python文件路径列表
List of Python file paths
"""
if exclude_patterns is None:
exclude_patterns = ['venv', '.venv', 'node_modules', '__pycache__', '.git', 'build', 'dist', '.eggs']
@@ -227,7 +227,7 @@ def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List
python_files = []
for root, dirs, files in os.walk(root_dir):
# 排除特定目录
# Exclude specific directories
dirs[:] = [d for d in dirs if d not in exclude_patterns and not d.startswith('.')]
for file in files:
@@ -238,39 +238,39 @@ def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List
def main():
"""主函数"""
"""Main function"""
import argparse
parser = argparse.ArgumentParser(description='Python文件头版权声明管理工具')
parser.add_argument('files', nargs='*', help='要处理的文件路径(可选,默认处理所有.py文件')
parser.add_argument('--dry-run', action='store_true', help='仅检查不修改文件')
parser.add_argument('--project-root', default=None, help='项目根目录(默认为当前目录)')
parser.add_argument('--check', action='store_true', help='检查模式,如果有文件缺少版权声明则返回非零退出码')
parser = argparse.ArgumentParser(description='Python file header copyright declaration management tool')
parser.add_argument('files', nargs='*', help='File paths to process (optional, defaults to all .py files)')
parser.add_argument('--dry-run', action='store_true', help='Check only without modifying files')
parser.add_argument('--project-root', default=None, help='Project root directory (defaults to current directory)')
parser.add_argument('--check', action='store_true', help='Check mode, return non-zero exit code if files missing copyright declaration')
args = parser.parse_args()
# 确定项目根目录
# Determine project root directory
if args.project_root:
project_root = os.path.abspath(args.project_root)
else:
# 假设此脚本在 tools/ 目录下
# Assume this script is in tools/ directory
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
print(f"Project root: {project_root}")
print(f"Mode: {'DRY RUN' if args.dry_run else 'UPDATE'}")
print("-" * 60)
# 获取要处理的文件列表
# Get list of files to process
if args.files:
# 处理指定的文件
# Process specified files
files_to_process = [os.path.abspath(f) for f in args.files if f.endswith('.py')]
else:
# 处理所有Python文件
# Process all Python files
files_to_process = find_python_files(project_root)
print(f"Found {len(files_to_process)} Python files to process\n")
# 处理文件
# Process files
updated_count = 0
skipped_count = 0
error_count = 0
@@ -286,7 +286,7 @@ def main():
else:
skipped_count += 1
# 打印汇总
# Print summary
print("\n" + "=" * 60)
print(f"Summary:")
print(f" Total files: {len(files_to_process)}")
@@ -295,7 +295,7 @@ def main():
print(f" Errors: {error_count}")
print("=" * 60)
# 如果是check模式且有文件需要更新返回非零退出码
# Return non-zero exit code in check mode if files need update
if args.check and updated_count > 0:
sys.exit(1)
elif error_count > 0:

View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 12:55
# @Desc : 滑块相关的工具包
# @Desc : Slider verification utility package
import os
from typing import List
from urllib.parse import urlparse
@@ -38,8 +38,8 @@ class Slide:
"""
def __init__(self, gap, bg, gap_size=None, bg_size=None, out=None):
"""
:param gap: 缺口图片链接或者url
:param bg: 带缺口的图片链接或者url
:param gap: Gap image path or url
:param bg: Background image with gap path or url
"""
self.img_dir = os.path.join(os.getcwd(), 'temp_image')
if not os.path.exists(self.img_dir):
@@ -76,13 +76,13 @@ class Slide:
cv2.imwrite(img_path, image)
return img_path
else:
raise Exception(f"保存{img_type}图片失败")
raise Exception(f"Failed to save {img_type} image")
else:
return img
@staticmethod
def clear_white(img):
"""清除图片的空白区域,这里主要清除滑块的空白"""
"""Clear whitespace from image, mainly clearing slider whitespace"""
img = cv2.imread(img)
rows, cols, channel = img.shape
min_x = 255
@@ -108,16 +108,16 @@ class Slide:
def template_match(self, tpl, target):
th, tw = tpl.shape[:2]
result = cv2.matchTemplate(target, tpl, cv2.TM_CCOEFF_NORMED)
# 寻找矩阵(一维数组当作向量,用Mat定义) 中最小值和最大值的位置
# Find min and max value positions in matrix
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
tl = max_loc
br = (tl[0] + tw, tl[1] + th)
# 绘制矩形边框,将匹配区域标注出来
# target:目标图像
# tl:矩形定点
# br:矩形的宽高
# (0,0,255):矩形边框颜色
# 1:矩形边框大小
# Draw rectangle border to mark the matched area
# target: target image
# tl: rectangle top-left corner
# br: rectangle width and height
# (0,0,255): rectangle border color
# 1: rectangle border size
cv2.rectangle(target, tl, br, (0, 0, 255), 2)
cv2.imwrite(self.out, target)
return tl[0]
@@ -138,39 +138,39 @@ class Slide:
slide_pic = cv2.cvtColor(slide, cv2.COLOR_GRAY2RGB)
back_pic = cv2.cvtColor(back, cv2.COLOR_GRAY2RGB)
x = self.template_match(slide_pic, back_pic)
# 输出横坐标, 即 滑块在图片上的位置
# Output x-coordinate, i.e., slider position on image
return x
def get_track_simple(distance) -> List[int]:
# 有的检测移动速度的 如果匀速移动会被识别出来,来个简单点的 渐进
# distance为传入的总距离
# 移动轨迹
# Some detection checks movement speed - constant speed will be detected, so use gradual acceleration
# distance is the total distance to move
# Movement track
track: List[int] = []
# 当前位移
# Current displacement
current = 0
# 减速阈值
# Deceleration threshold
mid = distance * 4 / 5
# 计算间隔
# Time interval
t = 0.2
# 初速度
# Initial velocity
v = 1
while current < distance:
if current < mid:
# 加速度为2
# Acceleration = 4
a = 4
else:
# 加速度为-2
# Acceleration = -3
a = -3
v0 = v
# 当前速度
# Current velocity
v = v0 + a * t # type: ignore
# 移动距离
# Movement distance
move = v0 * t + 1 / 2 * a * t * t
# 当前位移
# Current displacement
current += move # type: ignore
# 加入轨迹
# Add to track
track.append(round(move))
return track

View File

@@ -21,7 +21,7 @@
# -*- coding: utf-8 -*-
# @Author : relakkes@gmail.com
# @Time : 2023/12/2 12:52
# @Desc : 时间相关的工具函数
# @Desc : Time utility functions
import time
from datetime import datetime, timedelta, timezone
@@ -29,7 +29,7 @@ from datetime import datetime, timedelta, timezone
def get_current_timestamp() -> int:
"""
获取当前的时间戳(13 位)1701493264496
Get current timestamp (13 digits): 1701493264496
:return:
"""
return int(time.time() * 1000)
@@ -37,21 +37,21 @@ def get_current_timestamp() -> int:
def get_current_time() -> str:
"""
获取当前的时间:'2023-12-02 13:01:23'
Get current time: '2023-12-02 13:01:23'
:return:
"""
return time.strftime('%Y-%m-%d %X', time.localtime())
def get_current_time_hour() -> str:
"""
获取当前的时间:'2023-12-02-13'
Get current time with hour: '2023-12-02-13'
:return:
"""
return time.strftime('%Y-%m-%d-%H', time.localtime())
def get_current_date() -> str:
"""
获取当前的日期:'2023-12-02'
Get current date: '2023-12-02'
:return:
"""
return time.strftime('%Y-%m-%d', time.localtime())
@@ -59,7 +59,7 @@ def get_current_date() -> str:
def get_time_str_from_unix_time(unixtime):
"""
unix 整数类型时间戳 ==> 字符串日期时间
Unix integer timestamp ==> datetime string
:param unixtime:
:return:
"""
@@ -70,7 +70,7 @@ def get_time_str_from_unix_time(unixtime):
def get_date_str_from_unix_time(unixtime):
"""
unix 整数类型时间戳 ==> 字符串日期
Unix integer timestamp ==> date string
:param unixtime:
:return:
"""
@@ -81,7 +81,7 @@ def get_date_str_from_unix_time(unixtime):
def get_unix_time_from_time_str(time_str):
"""
字符串时间 ==> unix 整数类型时间戳,精确到秒
Time string ==> Unix integer timestamp, precise to seconds
:param time_str:
:return:
"""
@@ -99,34 +99,34 @@ def get_unix_timestamp():
def rfc2822_to_china_datetime(rfc2822_time):
# 定义RFC 2822格式
# Define RFC 2822 format
rfc2822_format = "%a %b %d %H:%M:%S %z %Y"
# 将RFC 2822时间字符串转换为datetime对象
# Convert RFC 2822 time string to datetime object
dt_object = datetime.strptime(rfc2822_time, rfc2822_format)
# 将datetime对象的时区转换为中国时区
# Convert datetime object timezone to China timezone
dt_object_china = dt_object.astimezone(timezone(timedelta(hours=8)))
return dt_object_china
def rfc2822_to_timestamp(rfc2822_time):
# 定义RFC 2822格式
# Define RFC 2822 format
rfc2822_format = "%a %b %d %H:%M:%S %z %Y"
# 将RFC 2822时间字符串转换为datetime对象
# Convert RFC 2822 time string to datetime object
dt_object = datetime.strptime(rfc2822_time, rfc2822_format)
# 将datetime对象转换为UTC时间
# Convert datetime object to UTC time
dt_utc = dt_object.replace(tzinfo=timezone.utc)
# 计算UTC时间对应的Unix时间戳
# Calculate Unix timestamp from UTC time
timestamp = int(dt_utc.timestamp())
return timestamp
if __name__ == '__main__':
# 示例用法
# Example usage
_rfc2822_time = "Sat Dec 23 17:12:54 +0800 2023"
print(rfc2822_to_china_datetime(_rfc2822_time))

View File

@@ -36,7 +36,7 @@ def init_loging_config():
_logger = logging.getLogger("MediaCrawler")
_logger.setLevel(level)
# 关闭 httpx INFO 日志
# Disable httpx INFO level logs
logging.getLogger("httpx").setLevel(logging.WARNING)
return _logger