mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-15 14:17:54 +08:00
i18n: translate all Chinese comments, docstrings, and logger messages to English
Comprehensive translation of Chinese text to English across the entire codebase: - api/: FastAPI server documentation and logger messages - cache/: Cache abstraction layer comments and docstrings - database/: Database models and MongoDB store documentation - media_platform/: All platform crawlers (Bilibili, Douyin, Kuaishou, Tieba, Weibo, Xiaohongshu, Zhihu) - model/: Data model documentation - proxy/: Proxy pool and provider documentation - store/: Data storage layer comments - tools/: Utility functions and browser automation - test/: Test file documentation Preserved: Chinese disclaimer header (lines 10-18) for legal compliance 🤖 Generated with [Claude Code](https://claude.com/claude-code) Co-Authored-By: Claude Opus 4.5 <noreply@anthropic.com>
This commit is contained in:
@@ -41,7 +41,7 @@ def run(
|
||||
try:
|
||||
await asyncio.wait_for(asyncio.shield(app_cleanup()), timeout=cleanup_timeout_seconds)
|
||||
except asyncio.TimeoutError:
|
||||
print(f"[Main] 清理超时({cleanup_timeout_seconds}s),跳过剩余清理。")
|
||||
print(f"[Main] Cleanup timeout ({cleanup_timeout_seconds}s), skipping remaining cleanup.")
|
||||
|
||||
async def _cancel_remaining_tasks(timeout_seconds: float = 2.0) -> None:
|
||||
current = asyncio.current_task()
|
||||
@@ -70,11 +70,11 @@ def run(
|
||||
nonlocal shutdown_requested
|
||||
|
||||
if shutdown_requested:
|
||||
print("[Main] 再次收到中断信号,强制退出。")
|
||||
print("[Main] Received interrupt signal again, force exit.")
|
||||
os._exit(force_exit_code)
|
||||
|
||||
shutdown_requested = True
|
||||
print(f"\n[Main] 收到中断信号 {signum},正在退出(清理最多{cleanup_timeout_seconds}s)...")
|
||||
print(f"\n[Main] Received interrupt signal {signum}, exiting (cleanup max {cleanup_timeout_seconds}s)...")
|
||||
|
||||
if on_first_interrupt is not None:
|
||||
try:
|
||||
@@ -100,7 +100,7 @@ def run(
|
||||
try:
|
||||
await _cleanup_with_timeout()
|
||||
except Exception as e:
|
||||
print(f"[Main] 清理时出错: {e}")
|
||||
print(f"[Main] Error during cleanup: {e}")
|
||||
await _cancel_remaining_tasks()
|
||||
|
||||
if cancelled:
|
||||
|
||||
@@ -33,8 +33,8 @@ from tools import utils
|
||||
|
||||
class BrowserLauncher:
|
||||
"""
|
||||
浏览器启动器,用于检测和启动用户的Chrome/Edge浏览器
|
||||
支持Windows和macOS系统
|
||||
Browser launcher for detecting and launching user's Chrome/Edge browser
|
||||
Supports Windows and macOS systems
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
@@ -44,19 +44,19 @@ class BrowserLauncher:
|
||||
|
||||
def detect_browser_paths(self) -> List[str]:
|
||||
"""
|
||||
检测系统中可用的浏览器路径
|
||||
返回按优先级排序的浏览器路径列表
|
||||
Detect available browser paths in system
|
||||
Returns list of browser paths sorted by priority
|
||||
"""
|
||||
paths = []
|
||||
|
||||
if self.system == "Windows":
|
||||
# Windows下的常见Chrome/Edge安装路径
|
||||
# Common Chrome/Edge installation paths on Windows
|
||||
possible_paths = [
|
||||
# Chrome路径
|
||||
# Chrome paths
|
||||
os.path.expandvars(r"%PROGRAMFILES%\Google\Chrome\Application\chrome.exe"),
|
||||
os.path.expandvars(r"%PROGRAMFILES(X86)%\Google\Chrome\Application\chrome.exe"),
|
||||
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome\Application\chrome.exe"),
|
||||
# Edge路径
|
||||
# Edge paths
|
||||
os.path.expandvars(r"%PROGRAMFILES%\Microsoft\Edge\Application\msedge.exe"),
|
||||
os.path.expandvars(r"%PROGRAMFILES(X86)%\Microsoft\Edge\Application\msedge.exe"),
|
||||
# Chrome Beta/Dev/Canary
|
||||
@@ -65,21 +65,21 @@ class BrowserLauncher:
|
||||
os.path.expandvars(r"%LOCALAPPDATA%\Google\Chrome SxS\Application\chrome.exe"),
|
||||
]
|
||||
elif self.system == "Darwin": # macOS
|
||||
# macOS下的常见Chrome/Edge安装路径
|
||||
# Common Chrome/Edge installation paths on macOS
|
||||
possible_paths = [
|
||||
# Chrome路径
|
||||
# Chrome paths
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"/Applications/Google Chrome Beta.app/Contents/MacOS/Google Chrome Beta",
|
||||
"/Applications/Google Chrome Dev.app/Contents/MacOS/Google Chrome Dev",
|
||||
"/Applications/Google Chrome Canary.app/Contents/MacOS/Google Chrome Canary",
|
||||
# Edge路径
|
||||
# Edge paths
|
||||
"/Applications/Microsoft Edge.app/Contents/MacOS/Microsoft Edge",
|
||||
"/Applications/Microsoft Edge Beta.app/Contents/MacOS/Microsoft Edge Beta",
|
||||
"/Applications/Microsoft Edge Dev.app/Contents/MacOS/Microsoft Edge Dev",
|
||||
"/Applications/Microsoft Edge Canary.app/Contents/MacOS/Microsoft Edge Canary",
|
||||
]
|
||||
else:
|
||||
# Linux等其他系统
|
||||
# Linux and other systems
|
||||
possible_paths = [
|
||||
"/usr/bin/google-chrome",
|
||||
"/usr/bin/google-chrome-stable",
|
||||
@@ -94,7 +94,7 @@ class BrowserLauncher:
|
||||
"/usr/bin/microsoft-edge-dev",
|
||||
]
|
||||
|
||||
# 检查路径是否存在且可执行
|
||||
# Check if path exists and is executable
|
||||
for path in possible_paths:
|
||||
if os.path.isfile(path) and os.access(path, os.X_OK):
|
||||
paths.append(path)
|
||||
@@ -103,10 +103,10 @@ class BrowserLauncher:
|
||||
|
||||
def find_available_port(self, start_port: int = 9222) -> int:
|
||||
"""
|
||||
查找可用的端口
|
||||
Find available port
|
||||
"""
|
||||
port = start_port
|
||||
while port < start_port + 100: # 最多尝试100个端口
|
||||
while port < start_port + 100: # Try up to 100 ports
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(('localhost', port))
|
||||
@@ -114,18 +114,18 @@ class BrowserLauncher:
|
||||
except OSError:
|
||||
port += 1
|
||||
|
||||
raise RuntimeError(f"无法找到可用的端口,已尝试 {start_port} 到 {port-1}")
|
||||
raise RuntimeError(f"Cannot find available port, tried {start_port} to {port-1}")
|
||||
|
||||
def launch_browser(self, browser_path: str, debug_port: int, headless: bool = False,
|
||||
user_data_dir: Optional[str] = None) -> subprocess.Popen:
|
||||
"""
|
||||
启动浏览器进程
|
||||
Launch browser process
|
||||
"""
|
||||
# 基本启动参数
|
||||
# Basic launch arguments
|
||||
args = [
|
||||
browser_path,
|
||||
f"--remote-debugging-port={debug_port}",
|
||||
"--remote-debugging-address=0.0.0.0", # 允许远程访问
|
||||
"--remote-debugging-address=0.0.0.0", # Allow remote access
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check",
|
||||
"--disable-background-timer-throttling",
|
||||
@@ -136,36 +136,36 @@ class BrowserLauncher:
|
||||
"--disable-hang-monitor",
|
||||
"--disable-prompt-on-repost",
|
||||
"--disable-sync",
|
||||
"--disable-dev-shm-usage", # 避免共享内存问题
|
||||
"--no-sandbox", # 在CDP模式下关闭沙箱
|
||||
# 🔥 关键反检测参数
|
||||
"--disable-blink-features=AutomationControlled", # 禁用自动化控制标记
|
||||
"--exclude-switches=enable-automation", # 排除自动化开关
|
||||
"--disable-infobars", # 禁用信息栏
|
||||
"--disable-dev-shm-usage", # Avoid shared memory issues
|
||||
"--no-sandbox", # Disable sandbox in CDP mode
|
||||
# Key anti-detection arguments
|
||||
"--disable-blink-features=AutomationControlled", # Disable automation control flag
|
||||
"--exclude-switches=enable-automation", # Exclude automation switch
|
||||
"--disable-infobars", # Disable info bars
|
||||
]
|
||||
|
||||
# 无头模式
|
||||
# Headless mode
|
||||
if headless:
|
||||
args.extend([
|
||||
"--headless=new", # 使用新的headless模式
|
||||
"--headless=new", # Use new headless mode
|
||||
"--disable-gpu",
|
||||
])
|
||||
else:
|
||||
# 非无头模式的额外参数
|
||||
# Extra arguments for non-headless mode
|
||||
args.extend([
|
||||
"--start-maximized", # 最大化窗口,更像真实用户
|
||||
"--start-maximized", # Maximize window, more like real user
|
||||
])
|
||||
|
||||
# 用户数据目录
|
||||
# User data directory
|
||||
if user_data_dir:
|
||||
args.append(f"--user-data-dir={user_data_dir}")
|
||||
|
||||
utils.logger.info(f"[BrowserLauncher] 启动浏览器: {browser_path}")
|
||||
utils.logger.info(f"[BrowserLauncher] 调试端口: {debug_port}")
|
||||
utils.logger.info(f"[BrowserLauncher] 无头模式: {headless}")
|
||||
utils.logger.info(f"[BrowserLauncher] Launching browser: {browser_path}")
|
||||
utils.logger.info(f"[BrowserLauncher] Debug port: {debug_port}")
|
||||
utils.logger.info(f"[BrowserLauncher] Headless mode: {headless}")
|
||||
|
||||
try:
|
||||
# 在Windows上,使用CREATE_NEW_PROCESS_GROUP避免Ctrl+C影响子进程
|
||||
# On Windows, use CREATE_NEW_PROCESS_GROUP to prevent Ctrl+C from affecting subprocess
|
||||
if self.system == "Windows":
|
||||
process = subprocess.Popen(
|
||||
args,
|
||||
@@ -178,21 +178,21 @@ class BrowserLauncher:
|
||||
args,
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
preexec_fn=os.setsid # 创建新的进程组
|
||||
preexec_fn=os.setsid # Create new process group
|
||||
)
|
||||
|
||||
self.browser_process = process
|
||||
return process
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[BrowserLauncher] 启动浏览器失败: {e}")
|
||||
utils.logger.error(f"[BrowserLauncher] Failed to launch browser: {e}")
|
||||
raise
|
||||
|
||||
def wait_for_browser_ready(self, debug_port: int, timeout: int = 30) -> bool:
|
||||
"""
|
||||
等待浏览器准备就绪
|
||||
Wait for browser to be ready
|
||||
"""
|
||||
utils.logger.info(f"[BrowserLauncher] 等待浏览器在端口 {debug_port} 上准备就绪...")
|
||||
utils.logger.info(f"[BrowserLauncher] Waiting for browser to be ready on port {debug_port}...")
|
||||
|
||||
start_time = time.time()
|
||||
while time.time() - start_time < timeout:
|
||||
@@ -201,19 +201,19 @@ class BrowserLauncher:
|
||||
s.settimeout(1)
|
||||
result = s.connect_ex(('localhost', debug_port))
|
||||
if result == 0:
|
||||
utils.logger.info(f"[BrowserLauncher] 浏览器已在端口 {debug_port} 上准备就绪")
|
||||
utils.logger.info(f"[BrowserLauncher] Browser is ready on port {debug_port}")
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
time.sleep(0.5)
|
||||
|
||||
utils.logger.error(f"[BrowserLauncher] 浏览器在 {timeout} 秒内未能准备就绪")
|
||||
utils.logger.error(f"[BrowserLauncher] Browser failed to be ready within {timeout} seconds")
|
||||
return False
|
||||
|
||||
def get_browser_info(self, browser_path: str) -> Tuple[str, str]:
|
||||
"""
|
||||
获取浏览器信息(名称和版本)
|
||||
Get browser info (name and version)
|
||||
"""
|
||||
try:
|
||||
if "chrome" in browser_path.lower():
|
||||
@@ -225,7 +225,7 @@ class BrowserLauncher:
|
||||
else:
|
||||
name = "Unknown Browser"
|
||||
|
||||
# 尝试获取版本信息
|
||||
# Try to get version info
|
||||
try:
|
||||
result = subprocess.run([browser_path, "--version"],
|
||||
capture_output=True, text=True, timeout=5)
|
||||
@@ -240,7 +240,7 @@ class BrowserLauncher:
|
||||
|
||||
def cleanup(self):
|
||||
"""
|
||||
清理资源,关闭浏览器进程
|
||||
Cleanup resources, close browser process
|
||||
"""
|
||||
if not self.browser_process:
|
||||
return
|
||||
@@ -248,20 +248,20 @@ class BrowserLauncher:
|
||||
process = self.browser_process
|
||||
|
||||
if process.poll() is not None:
|
||||
utils.logger.info("[BrowserLauncher] 浏览器进程已退出,无需清理")
|
||||
utils.logger.info("[BrowserLauncher] Browser process already exited, no cleanup needed")
|
||||
self.browser_process = None
|
||||
return
|
||||
|
||||
utils.logger.info("[BrowserLauncher] 正在关闭浏览器进程...")
|
||||
utils.logger.info("[BrowserLauncher] Closing browser process...")
|
||||
|
||||
try:
|
||||
if self.system == "Windows":
|
||||
# 先尝试正常终止
|
||||
# First try normal termination
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
utils.logger.warning("[BrowserLauncher] 正常终止超时,使用taskkill强制结束")
|
||||
utils.logger.warning("[BrowserLauncher] Normal termination timeout, using taskkill to force kill")
|
||||
subprocess.run(
|
||||
["taskkill", "/F", "/T", "/PID", str(process.pid)],
|
||||
capture_output=True,
|
||||
@@ -273,17 +273,17 @@ class BrowserLauncher:
|
||||
try:
|
||||
os.killpg(pgid, signal.SIGTERM)
|
||||
except ProcessLookupError:
|
||||
utils.logger.info("[BrowserLauncher] 浏览器进程组不存在,可能已退出")
|
||||
utils.logger.info("[BrowserLauncher] Browser process group does not exist, may have exited")
|
||||
else:
|
||||
try:
|
||||
process.wait(timeout=5)
|
||||
except subprocess.TimeoutExpired:
|
||||
utils.logger.warning("[BrowserLauncher] 优雅关闭超时,发送SIGKILL")
|
||||
utils.logger.warning("[BrowserLauncher] Graceful shutdown timeout, sending SIGKILL")
|
||||
os.killpg(pgid, signal.SIGKILL)
|
||||
process.wait(timeout=5)
|
||||
|
||||
utils.logger.info("[BrowserLauncher] 浏览器进程已关闭")
|
||||
utils.logger.info("[BrowserLauncher] Browser process closed")
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[BrowserLauncher] 关闭浏览器进程时出错: {e}")
|
||||
utils.logger.warning(f"[BrowserLauncher] Error closing browser process: {e}")
|
||||
finally:
|
||||
self.browser_process = None
|
||||
|
||||
@@ -34,7 +34,7 @@ from tools import utils
|
||||
|
||||
class CDPBrowserManager:
|
||||
"""
|
||||
CDP浏览器管理器,负责启动和管理通过CDP连接的浏览器
|
||||
CDP browser manager, responsible for launching and managing browsers connected via CDP
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
@@ -46,27 +46,27 @@ class CDPBrowserManager:
|
||||
|
||||
def _register_cleanup_handlers(self):
|
||||
"""
|
||||
注册清理处理器,确保程序退出时清理浏览器进程
|
||||
Register cleanup handlers to ensure browser process cleanup on program exit
|
||||
"""
|
||||
if self._cleanup_registered:
|
||||
return
|
||||
|
||||
def sync_cleanup():
|
||||
"""同步清理函数,用于atexit"""
|
||||
"""Synchronous cleanup function for atexit"""
|
||||
if self.launcher and self.launcher.browser_process:
|
||||
utils.logger.info("[CDPBrowserManager] atexit: 清理浏览器进程")
|
||||
utils.logger.info("[CDPBrowserManager] atexit: Cleaning up browser process")
|
||||
self.launcher.cleanup()
|
||||
|
||||
# 注册atexit清理
|
||||
# Register atexit cleanup
|
||||
atexit.register(sync_cleanup)
|
||||
|
||||
# 注册信号处理器(仅在没有自定义处理器时注册,避免覆盖主入口的信号处理逻辑)
|
||||
# Register signal handlers (only when no custom handlers exist, to avoid overriding main entry signal handling logic)
|
||||
prev_sigint = signal.getsignal(signal.SIGINT)
|
||||
prev_sigterm = signal.getsignal(signal.SIGTERM)
|
||||
|
||||
def signal_handler(signum, frame):
|
||||
"""信号处理器"""
|
||||
utils.logger.info(f"[CDPBrowserManager] 收到信号 {signum},清理浏览器进程")
|
||||
"""Signal handler"""
|
||||
utils.logger.info(f"[CDPBrowserManager] Received signal {signum}, cleaning up browser process")
|
||||
if self.launcher and self.launcher.browser_process:
|
||||
self.launcher.cleanup()
|
||||
|
||||
@@ -80,19 +80,19 @@ class CDPBrowserManager:
|
||||
install_sigint = prev_sigint in (signal.default_int_handler, signal.SIG_DFL)
|
||||
install_sigterm = prev_sigterm == signal.SIG_DFL
|
||||
|
||||
# 注册SIGINT (Ctrl+C) 和 SIGTERM
|
||||
# Register SIGINT (Ctrl+C) and SIGTERM
|
||||
if install_sigint:
|
||||
signal.signal(signal.SIGINT, signal_handler)
|
||||
else:
|
||||
utils.logger.info("[CDPBrowserManager] 已存在SIGINT处理器,跳过注册以避免覆盖")
|
||||
utils.logger.info("[CDPBrowserManager] SIGINT handler already exists, skipping registration to avoid override")
|
||||
|
||||
if install_sigterm:
|
||||
signal.signal(signal.SIGTERM, signal_handler)
|
||||
else:
|
||||
utils.logger.info("[CDPBrowserManager] 已存在SIGTERM处理器,跳过注册以避免覆盖")
|
||||
utils.logger.info("[CDPBrowserManager] SIGTERM handler already exists, skipping registration to avoid override")
|
||||
|
||||
self._cleanup_registered = True
|
||||
utils.logger.info("[CDPBrowserManager] 清理处理器已注册")
|
||||
utils.logger.info("[CDPBrowserManager] Cleanup handlers registered")
|
||||
|
||||
async def launch_and_connect(
|
||||
self,
|
||||
@@ -102,25 +102,25 @@ class CDPBrowserManager:
|
||||
headless: bool = False,
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
启动浏览器并通过CDP连接
|
||||
Launch browser and connect via CDP
|
||||
"""
|
||||
try:
|
||||
# 1. 检测浏览器路径
|
||||
# 1. Detect browser path
|
||||
browser_path = await self._get_browser_path()
|
||||
|
||||
# 2. 获取可用端口
|
||||
# 2. Get available port
|
||||
self.debug_port = self.launcher.find_available_port(config.CDP_DEBUG_PORT)
|
||||
|
||||
# 3. 启动浏览器
|
||||
# 3. Launch browser
|
||||
await self._launch_browser(browser_path, headless)
|
||||
|
||||
# 4. 注册清理处理器(确保异常退出时也能清理)
|
||||
# 4. Register cleanup handlers (ensure cleanup on abnormal exit)
|
||||
self._register_cleanup_handlers()
|
||||
|
||||
# 5. 通过CDP连接
|
||||
# 5. Connect via CDP
|
||||
await self._connect_via_cdp(playwright)
|
||||
|
||||
# 5. 创建浏览器上下文
|
||||
# 6. Create browser context
|
||||
browser_context = await self._create_browser_context(
|
||||
playwright_proxy, user_agent
|
||||
)
|
||||
@@ -129,68 +129,68 @@ class CDPBrowserManager:
|
||||
return browser_context
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[CDPBrowserManager] CDP浏览器启动失败: {e}")
|
||||
utils.logger.error(f"[CDPBrowserManager] CDP browser launch failed: {e}")
|
||||
await self.cleanup()
|
||||
raise
|
||||
|
||||
async def _get_browser_path(self) -> str:
|
||||
"""
|
||||
获取浏览器路径
|
||||
Get browser path
|
||||
"""
|
||||
# 优先使用用户自定义路径
|
||||
# Prefer user-defined path
|
||||
if config.CUSTOM_BROWSER_PATH and os.path.isfile(config.CUSTOM_BROWSER_PATH):
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] 使用自定义浏览器路径: {config.CUSTOM_BROWSER_PATH}"
|
||||
f"[CDPBrowserManager] Using custom browser path: {config.CUSTOM_BROWSER_PATH}"
|
||||
)
|
||||
return config.CUSTOM_BROWSER_PATH
|
||||
|
||||
# 自动检测浏览器路径
|
||||
# Auto-detect browser path
|
||||
browser_paths = self.launcher.detect_browser_paths()
|
||||
|
||||
if not browser_paths:
|
||||
raise RuntimeError(
|
||||
"未找到可用的浏览器。请确保已安装Chrome或Edge浏览器,"
|
||||
"或在配置文件中设置CUSTOM_BROWSER_PATH指定浏览器路径。"
|
||||
"No available browser found. Please ensure Chrome or Edge browser is installed, "
|
||||
"or set CUSTOM_BROWSER_PATH in config file to specify browser path."
|
||||
)
|
||||
|
||||
browser_path = browser_paths[0] # 使用第一个找到的浏览器
|
||||
browser_path = browser_paths[0] # Use the first browser found
|
||||
browser_name, browser_version = self.launcher.get_browser_info(browser_path)
|
||||
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] 检测到浏览器: {browser_name} ({browser_version})"
|
||||
f"[CDPBrowserManager] Detected browser: {browser_name} ({browser_version})"
|
||||
)
|
||||
utils.logger.info(f"[CDPBrowserManager] 浏览器路径: {browser_path}")
|
||||
utils.logger.info(f"[CDPBrowserManager] Browser path: {browser_path}")
|
||||
|
||||
return browser_path
|
||||
|
||||
async def _test_cdp_connection(self, debug_port: int) -> bool:
|
||||
"""
|
||||
测试CDP连接是否可用
|
||||
Test if CDP connection is available
|
||||
"""
|
||||
try:
|
||||
# 简单的socket连接测试
|
||||
# Simple socket connection test
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.settimeout(5)
|
||||
result = s.connect_ex(("localhost", debug_port))
|
||||
if result == 0:
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] CDP端口 {debug_port} 可访问"
|
||||
f"[CDPBrowserManager] CDP port {debug_port} is accessible"
|
||||
)
|
||||
return True
|
||||
else:
|
||||
utils.logger.warning(
|
||||
f"[CDPBrowserManager] CDP端口 {debug_port} 不可访问"
|
||||
f"[CDPBrowserManager] CDP port {debug_port} is not accessible"
|
||||
)
|
||||
return False
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[CDPBrowserManager] CDP连接测试失败: {e}")
|
||||
utils.logger.warning(f"[CDPBrowserManager] CDP connection test failed: {e}")
|
||||
return False
|
||||
|
||||
async def _launch_browser(self, browser_path: str, headless: bool):
|
||||
"""
|
||||
启动浏览器进程
|
||||
Launch browser process
|
||||
"""
|
||||
# 设置用户数据目录(如果启用了保存登录状态)
|
||||
# Set user data directory (if save login state is enabled)
|
||||
user_data_dir = None
|
||||
if config.SAVE_LOGIN_STATE:
|
||||
user_data_dir = os.path.join(
|
||||
@@ -199,9 +199,9 @@ class CDPBrowserManager:
|
||||
f"cdp_{config.USER_DATA_DIR % config.PLATFORM}",
|
||||
)
|
||||
os.makedirs(user_data_dir, exist_ok=True)
|
||||
utils.logger.info(f"[CDPBrowserManager] 用户数据目录: {user_data_dir}")
|
||||
utils.logger.info(f"[CDPBrowserManager] User data directory: {user_data_dir}")
|
||||
|
||||
# 启动浏览器
|
||||
# Launch browser
|
||||
self.launcher.browser_process = self.launcher.launch_browser(
|
||||
browser_path=browser_path,
|
||||
debug_port=self.debug_port,
|
||||
@@ -209,24 +209,24 @@ class CDPBrowserManager:
|
||||
user_data_dir=user_data_dir,
|
||||
)
|
||||
|
||||
# 等待浏览器准备就绪
|
||||
# Wait for browser to be ready
|
||||
if not self.launcher.wait_for_browser_ready(
|
||||
self.debug_port, config.BROWSER_LAUNCH_TIMEOUT
|
||||
):
|
||||
raise RuntimeError(f"浏览器在 {config.BROWSER_LAUNCH_TIMEOUT} 秒内未能启动")
|
||||
raise RuntimeError(f"Browser failed to start within {config.BROWSER_LAUNCH_TIMEOUT} seconds")
|
||||
|
||||
# 额外等待一秒让CDP服务完全启动
|
||||
# Extra wait for CDP service to fully start
|
||||
await asyncio.sleep(1)
|
||||
|
||||
# 测试CDP连接
|
||||
# Test CDP connection
|
||||
if not await self._test_cdp_connection(self.debug_port):
|
||||
utils.logger.warning(
|
||||
"[CDPBrowserManager] CDP连接测试失败,但将继续尝试连接"
|
||||
"[CDPBrowserManager] CDP connection test failed, but will continue to try connecting"
|
||||
)
|
||||
|
||||
async def _get_browser_websocket_url(self, debug_port: int) -> str:
|
||||
"""
|
||||
获取浏览器的WebSocket连接URL
|
||||
Get browser WebSocket connection URL
|
||||
"""
|
||||
try:
|
||||
async with httpx.AsyncClient() as client:
|
||||
@@ -238,196 +238,196 @@ class CDPBrowserManager:
|
||||
ws_url = data.get("webSocketDebuggerUrl")
|
||||
if ws_url:
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] 获取到浏览器WebSocket URL: {ws_url}"
|
||||
f"[CDPBrowserManager] Got browser WebSocket URL: {ws_url}"
|
||||
)
|
||||
return ws_url
|
||||
else:
|
||||
raise RuntimeError("未找到webSocketDebuggerUrl")
|
||||
raise RuntimeError("webSocketDebuggerUrl not found")
|
||||
else:
|
||||
raise RuntimeError(f"HTTP {response.status_code}: {response.text}")
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[CDPBrowserManager] 获取WebSocket URL失败: {e}")
|
||||
utils.logger.error(f"[CDPBrowserManager] Failed to get WebSocket URL: {e}")
|
||||
raise
|
||||
|
||||
async def _connect_via_cdp(self, playwright: Playwright):
|
||||
"""
|
||||
通过CDP连接到浏览器
|
||||
Connect to browser via CDP
|
||||
"""
|
||||
try:
|
||||
# 获取正确的WebSocket URL
|
||||
# Get correct WebSocket URL
|
||||
ws_url = await self._get_browser_websocket_url(self.debug_port)
|
||||
utils.logger.info(f"[CDPBrowserManager] 正在通过CDP连接到浏览器: {ws_url}")
|
||||
utils.logger.info(f"[CDPBrowserManager] Connecting to browser via CDP: {ws_url}")
|
||||
|
||||
# 使用Playwright的connectOverCDP方法连接
|
||||
# Use Playwright's connectOverCDP method to connect
|
||||
self.browser = await playwright.chromium.connect_over_cdp(ws_url)
|
||||
|
||||
if self.browser.is_connected():
|
||||
utils.logger.info("[CDPBrowserManager] 成功连接到浏览器")
|
||||
utils.logger.info("[CDPBrowserManager] Successfully connected to browser")
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] 浏览器上下文数量: {len(self.browser.contexts)}"
|
||||
f"[CDPBrowserManager] Browser contexts count: {len(self.browser.contexts)}"
|
||||
)
|
||||
else:
|
||||
raise RuntimeError("CDP连接失败")
|
||||
raise RuntimeError("CDP connection failed")
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[CDPBrowserManager] CDP连接失败: {e}")
|
||||
utils.logger.error(f"[CDPBrowserManager] CDP connection failed: {e}")
|
||||
raise
|
||||
|
||||
async def _create_browser_context(
|
||||
self, playwright_proxy: Optional[Dict] = None, user_agent: Optional[str] = None
|
||||
) -> BrowserContext:
|
||||
"""
|
||||
创建或获取浏览器上下文
|
||||
Create or get browser context
|
||||
"""
|
||||
if not self.browser:
|
||||
raise RuntimeError("浏览器未连接")
|
||||
raise RuntimeError("Browser not connected")
|
||||
|
||||
# 获取现有上下文或创建新的上下文
|
||||
# Get existing context or create new context
|
||||
contexts = self.browser.contexts
|
||||
|
||||
if contexts:
|
||||
# 使用现有的第一个上下文
|
||||
# Use existing first context
|
||||
browser_context = contexts[0]
|
||||
utils.logger.info("[CDPBrowserManager] 使用现有的浏览器上下文")
|
||||
utils.logger.info("[CDPBrowserManager] Using existing browser context")
|
||||
else:
|
||||
# 创建新的上下文
|
||||
# Create new context
|
||||
context_options = {
|
||||
"viewport": {"width": 1920, "height": 1080},
|
||||
"accept_downloads": True,
|
||||
}
|
||||
|
||||
# 设置用户代理
|
||||
# Set user agent
|
||||
if user_agent:
|
||||
context_options["user_agent"] = user_agent
|
||||
utils.logger.info(f"[CDPBrowserManager] 设置用户代理: {user_agent}")
|
||||
utils.logger.info(f"[CDPBrowserManager] Setting user agent: {user_agent}")
|
||||
|
||||
# 注意:CDP模式下代理设置可能不生效,因为浏览器已经启动
|
||||
# Note: Proxy settings may not work in CDP mode since browser is already launched
|
||||
if playwright_proxy:
|
||||
utils.logger.warning(
|
||||
"[CDPBrowserManager] 警告: CDP模式下代理设置可能不生效,"
|
||||
"建议在浏览器启动前配置系统代理或浏览器代理扩展"
|
||||
"[CDPBrowserManager] Warning: Proxy settings may not work in CDP mode, "
|
||||
"recommend configuring system proxy or browser proxy extension before launching browser"
|
||||
)
|
||||
|
||||
browser_context = await self.browser.new_context(**context_options)
|
||||
utils.logger.info("[CDPBrowserManager] 创建新的浏览器上下文")
|
||||
utils.logger.info("[CDPBrowserManager] Created new browser context")
|
||||
|
||||
return browser_context
|
||||
|
||||
async def add_stealth_script(self, script_path: str = "libs/stealth.min.js"):
|
||||
"""
|
||||
添加反检测脚本
|
||||
Add anti-detection script
|
||||
"""
|
||||
if self.browser_context and os.path.exists(script_path):
|
||||
try:
|
||||
await self.browser_context.add_init_script(path=script_path)
|
||||
utils.logger.info(
|
||||
f"[CDPBrowserManager] 已添加反检测脚本: {script_path}"
|
||||
f"[CDPBrowserManager] Added anti-detection script: {script_path}"
|
||||
)
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[CDPBrowserManager] 添加反检测脚本失败: {e}")
|
||||
utils.logger.warning(f"[CDPBrowserManager] Failed to add anti-detection script: {e}")
|
||||
|
||||
async def add_cookies(self, cookies: list):
|
||||
"""
|
||||
添加Cookie
|
||||
Add cookies
|
||||
"""
|
||||
if self.browser_context:
|
||||
try:
|
||||
await self.browser_context.add_cookies(cookies)
|
||||
utils.logger.info(f"[CDPBrowserManager] 已添加 {len(cookies)} 个Cookie")
|
||||
utils.logger.info(f"[CDPBrowserManager] Added {len(cookies)} cookies")
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[CDPBrowserManager] 添加Cookie失败: {e}")
|
||||
utils.logger.warning(f"[CDPBrowserManager] Failed to add cookies: {e}")
|
||||
|
||||
async def get_cookies(self) -> list:
|
||||
"""
|
||||
获取当前Cookie
|
||||
Get current cookies
|
||||
"""
|
||||
if self.browser_context:
|
||||
try:
|
||||
cookies = await self.browser_context.cookies()
|
||||
return cookies
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[CDPBrowserManager] 获取Cookie失败: {e}")
|
||||
utils.logger.warning(f"[CDPBrowserManager] Failed to get cookies: {e}")
|
||||
return []
|
||||
return []
|
||||
|
||||
async def cleanup(self, force: bool = False):
|
||||
"""
|
||||
清理资源
|
||||
Cleanup resources
|
||||
|
||||
Args:
|
||||
force: 是否强制清理浏览器进程(忽略AUTO_CLOSE_BROWSER配置)
|
||||
force: Whether to force cleanup browser process (ignoring AUTO_CLOSE_BROWSER config)
|
||||
"""
|
||||
try:
|
||||
# 关闭浏览器上下文
|
||||
# Close browser context
|
||||
if self.browser_context:
|
||||
try:
|
||||
# 检查上下文是否已经关闭
|
||||
# 尝试获取页面列表,如果失败说明已经关闭
|
||||
# Check if context is already closed
|
||||
# Try to get page list, if fails means already closed
|
||||
try:
|
||||
pages = self.browser_context.pages
|
||||
if pages is not None:
|
||||
await self.browser_context.close()
|
||||
utils.logger.info("[CDPBrowserManager] 浏览器上下文已关闭")
|
||||
utils.logger.info("[CDPBrowserManager] Browser context closed")
|
||||
except:
|
||||
utils.logger.debug("[CDPBrowserManager] 浏览器上下文已经被关闭")
|
||||
utils.logger.debug("[CDPBrowserManager] Browser context already closed")
|
||||
except Exception as context_error:
|
||||
# 只在错误不是因为已关闭时才记录警告
|
||||
# Only log warning if error is not due to already being closed
|
||||
error_msg = str(context_error).lower()
|
||||
if "closed" not in error_msg and "disconnected" not in error_msg:
|
||||
utils.logger.warning(
|
||||
f"[CDPBrowserManager] 关闭浏览器上下文失败: {context_error}"
|
||||
f"[CDPBrowserManager] Failed to close browser context: {context_error}"
|
||||
)
|
||||
else:
|
||||
utils.logger.debug(f"[CDPBrowserManager] 浏览器上下文已关闭: {context_error}")
|
||||
utils.logger.debug(f"[CDPBrowserManager] Browser context already closed: {context_error}")
|
||||
finally:
|
||||
self.browser_context = None
|
||||
|
||||
# 断开浏览器连接
|
||||
# Disconnect browser
|
||||
if self.browser:
|
||||
try:
|
||||
# 检查浏览器是否仍然连接
|
||||
# Check if browser is still connected
|
||||
if self.browser.is_connected():
|
||||
await self.browser.close()
|
||||
utils.logger.info("[CDPBrowserManager] 浏览器连接已断开")
|
||||
utils.logger.info("[CDPBrowserManager] Browser connection disconnected")
|
||||
else:
|
||||
utils.logger.debug("[CDPBrowserManager] 浏览器连接已经断开")
|
||||
utils.logger.debug("[CDPBrowserManager] Browser connection already disconnected")
|
||||
except Exception as browser_error:
|
||||
# 只在错误不是因为已关闭时才记录警告
|
||||
# Only log warning if error is not due to already being closed
|
||||
error_msg = str(browser_error).lower()
|
||||
if "closed" not in error_msg and "disconnected" not in error_msg:
|
||||
utils.logger.warning(
|
||||
f"[CDPBrowserManager] 关闭浏览器连接失败: {browser_error}"
|
||||
f"[CDPBrowserManager] Failed to close browser connection: {browser_error}"
|
||||
)
|
||||
else:
|
||||
utils.logger.debug(f"[CDPBrowserManager] 浏览器连接已关闭: {browser_error}")
|
||||
utils.logger.debug(f"[CDPBrowserManager] Browser connection already closed: {browser_error}")
|
||||
finally:
|
||||
self.browser = None
|
||||
|
||||
# 关闭浏览器进程
|
||||
# force=True 时强制关闭,忽略AUTO_CLOSE_BROWSER配置
|
||||
# 这用于处理异常退出或手动清理的情况
|
||||
# Close browser process
|
||||
# force=True means force close, ignoring AUTO_CLOSE_BROWSER config
|
||||
# Used for handling abnormal exit or manual cleanup
|
||||
if force or config.AUTO_CLOSE_BROWSER:
|
||||
if self.launcher and self.launcher.browser_process:
|
||||
self.launcher.cleanup()
|
||||
else:
|
||||
utils.logger.debug("[CDPBrowserManager] 没有需要清理的浏览器进程")
|
||||
utils.logger.debug("[CDPBrowserManager] No browser process to cleanup")
|
||||
else:
|
||||
utils.logger.info(
|
||||
"[CDPBrowserManager] 浏览器进程保持运行(AUTO_CLOSE_BROWSER=False)"
|
||||
"[CDPBrowserManager] Browser process kept running (AUTO_CLOSE_BROWSER=False)"
|
||||
)
|
||||
|
||||
except Exception as e:
|
||||
utils.logger.error(f"[CDPBrowserManager] 清理资源时出错: {e}")
|
||||
utils.logger.error(f"[CDPBrowserManager] Error during resource cleanup: {e}")
|
||||
|
||||
def is_connected(self) -> bool:
|
||||
"""
|
||||
检查是否已连接到浏览器
|
||||
Check if connected to browser
|
||||
"""
|
||||
return self.browser is not None and self.browser.is_connected()
|
||||
|
||||
async def get_browser_info(self) -> Dict[str, Any]:
|
||||
"""
|
||||
获取浏览器信息
|
||||
Get browser info
|
||||
"""
|
||||
if not self.browser:
|
||||
return {}
|
||||
@@ -443,5 +443,5 @@ class CDPBrowserManager:
|
||||
"is_connected": self.is_connected(),
|
||||
}
|
||||
except Exception as e:
|
||||
utils.logger.warning(f"[CDPBrowserManager] 获取浏览器信息失败: {e}")
|
||||
utils.logger.warning(f"[CDPBrowserManager] Failed to get browser info: {e}")
|
||||
return {}
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/2 12:53
|
||||
# @Desc : 爬虫相关的工具函数
|
||||
# @Desc : Crawler utility functions
|
||||
|
||||
import base64
|
||||
import json
|
||||
@@ -73,13 +73,13 @@ async def find_qrcode_img_from_canvas(page: Page, canvas_selector: str) -> str:
|
||||
|
||||
"""
|
||||
|
||||
# 等待Canvas元素加载完成
|
||||
# Wait for Canvas element to load
|
||||
canvas = await page.wait_for_selector(canvas_selector)
|
||||
|
||||
# 截取Canvas元素的截图
|
||||
# Take screenshot of Canvas element
|
||||
screenshot = await canvas.screenshot()
|
||||
|
||||
# 将截图转换为base64格式
|
||||
# Convert screenshot to base64 format
|
||||
base64_image = base64.b64encode(screenshot).decode('utf-8')
|
||||
return base64_image
|
||||
|
||||
@@ -185,7 +185,7 @@ def format_proxy_info(ip_proxy_info) -> Tuple[Optional[Dict], Optional[str]]:
|
||||
"username": ip_proxy_info.user,
|
||||
"password": ip_proxy_info.password,
|
||||
}
|
||||
# httpx 0.28.1 需要直接传入代理URL字符串,而不是字典
|
||||
# httpx 0.28.1 requires passing proxy URL string directly, not a dictionary
|
||||
if ip_proxy_info.user and ip_proxy_info.password:
|
||||
httpx_proxy = f"http://{ip_proxy_info.user}:{ip_proxy_info.password}@{ip_proxy_info.ip}:{ip_proxy_info.port}"
|
||||
else:
|
||||
|
||||
@@ -17,13 +17,13 @@
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
"""
|
||||
文件头版权声明管理工具
|
||||
File header copyright declaration management tool
|
||||
|
||||
功能:
|
||||
- 自动为Python文件添加标准化的版权声明和免责声明
|
||||
- 智能检测现有文件头(编码声明、作者信息、免责声明等)
|
||||
- 在合适位置插入版权信息,不破坏现有内容
|
||||
- 支持批量处理和单文件检查模式
|
||||
Features:
|
||||
- Automatically add standardized copyright declaration and disclaimer to Python files
|
||||
- Intelligently detect existing file headers (encoding declaration, author info, disclaimer, etc.)
|
||||
- Insert copyright info at appropriate position without breaking existing content
|
||||
- Support batch processing and single file check mode
|
||||
"""
|
||||
|
||||
import os
|
||||
@@ -31,14 +31,14 @@ import re
|
||||
import sys
|
||||
from typing import List, Tuple
|
||||
|
||||
# 项目配置
|
||||
# Project configuration
|
||||
REPO_URL = "https://github.com/NanmiCoder/MediaCrawler"
|
||||
GITHUB_PROFILE = "https://github.com/NanmiCoder"
|
||||
EMAIL = "relakkes@gmail.com"
|
||||
COPYRIGHT_YEAR = "2025"
|
||||
LICENSE_TYPE = "NON-COMMERCIAL LEARNING LICENSE 1.1"
|
||||
|
||||
# 免责声明标准文本
|
||||
# Disclaimer standard text
|
||||
DISCLAIMER = """# 声明:本代码仅供学习和研究目的使用。使用者应遵守以下原则:
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
@@ -52,27 +52,27 @@ DISCLAIMER = """# 声明:本代码仅供学习和研究目的使用。使用
|
||||
|
||||
def get_file_relative_path(file_path: str, project_root: str) -> str:
|
||||
"""
|
||||
获取文件相对于项目根目录的路径
|
||||
Get file path relative to project root
|
||||
|
||||
Args:
|
||||
file_path: 文件绝对路径
|
||||
project_root: 项目根目录
|
||||
file_path: File absolute path
|
||||
project_root: Project root directory
|
||||
|
||||
Returns:
|
||||
相对路径字符串
|
||||
Relative path string
|
||||
"""
|
||||
return os.path.relpath(file_path, project_root)
|
||||
|
||||
|
||||
def generate_copyright_header(relative_path: str) -> str:
|
||||
"""
|
||||
生成版权声明头部
|
||||
Generate copyright declaration header
|
||||
|
||||
Args:
|
||||
relative_path: 文件相对于项目根目录的路径
|
||||
relative_path: File path relative to project root
|
||||
|
||||
Returns:
|
||||
格式化的版权声明字符串
|
||||
Formatted copyright declaration string
|
||||
"""
|
||||
file_url = f"{REPO_URL}/blob/main/{relative_path}"
|
||||
|
||||
@@ -89,53 +89,53 @@ def generate_copyright_header(relative_path: str) -> str:
|
||||
|
||||
def has_copyright_header(content: str) -> bool:
|
||||
"""
|
||||
检查文件是否已包含版权声明
|
||||
Check if file already contains copyright declaration
|
||||
|
||||
Args:
|
||||
content: 文件内容
|
||||
content: File content
|
||||
|
||||
Returns:
|
||||
True如果已包含版权声明
|
||||
True if already contains copyright declaration
|
||||
"""
|
||||
# 检查是否包含Copyright关键字
|
||||
# Check if contains Copyright keyword
|
||||
return "Copyright (c)" in content and "MediaCrawler project" in content
|
||||
|
||||
|
||||
def has_disclaimer(content: str) -> bool:
|
||||
"""
|
||||
检查文件是否已包含免责声明
|
||||
Check if file already contains disclaimer
|
||||
|
||||
Args:
|
||||
content: 文件内容
|
||||
content: File content
|
||||
|
||||
Returns:
|
||||
True如果已包含免责声明
|
||||
True if already contains disclaimer
|
||||
"""
|
||||
return "声明:本代码仅供学习和研究目的使用" in content
|
||||
|
||||
|
||||
def find_insert_position(lines: List[str]) -> Tuple[int, bool]:
|
||||
"""
|
||||
找到插入版权声明的位置
|
||||
Find position to insert copyright declaration
|
||||
|
||||
Args:
|
||||
lines: 文件内容行列表
|
||||
lines: List of file content lines
|
||||
|
||||
Returns:
|
||||
(插入行号, 是否需要在前面添加编码声明)
|
||||
(insert line number, whether encoding declaration needs to be added)
|
||||
"""
|
||||
insert_pos = 0
|
||||
has_encoding = False
|
||||
|
||||
# 检查第一行是否是shebang
|
||||
# Check if first line is shebang
|
||||
if lines and lines[0].startswith('#!'):
|
||||
insert_pos = 1
|
||||
|
||||
# 检查编码声明(通常在第1或2行)
|
||||
# Check encoding declaration (usually on line 1 or 2)
|
||||
for i in range(insert_pos, min(insert_pos + 2, len(lines))):
|
||||
if i < len(lines):
|
||||
line = lines[i].strip()
|
||||
# 匹配 # -*- coding: utf-8 -*- 或 # coding: utf-8 等格式
|
||||
# Match # -*- coding: utf-8 -*- or # coding: utf-8 etc.
|
||||
if re.match(r'#.*coding[:=]\s*([-\w.]+)', line):
|
||||
has_encoding = True
|
||||
insert_pos = i + 1
|
||||
@@ -146,59 +146,59 @@ def find_insert_position(lines: List[str]) -> Tuple[int, bool]:
|
||||
|
||||
def process_file(file_path: str, project_root: str, dry_run: bool = False) -> Tuple[bool, str]:
|
||||
"""
|
||||
处理单个Python文件
|
||||
Process single Python file
|
||||
|
||||
Args:
|
||||
file_path: 文件路径
|
||||
project_root: 项目根目录
|
||||
dry_run: 仅检查不修改
|
||||
file_path: File path
|
||||
project_root: Project root directory
|
||||
dry_run: Check only without modification
|
||||
|
||||
Returns:
|
||||
(是否需要修改, 状态消息)
|
||||
(whether modification needed, status message)
|
||||
"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
content = f.read()
|
||||
lines = content.splitlines(keepends=True)
|
||||
|
||||
# 如果已经有版权声明,跳过
|
||||
# Skip if already has copyright header
|
||||
if has_copyright_header(content):
|
||||
return False, f"✓ Already has copyright header: {file_path}"
|
||||
|
||||
# 获取相对路径
|
||||
# Get relative path
|
||||
relative_path = get_file_relative_path(file_path, project_root)
|
||||
|
||||
# 生成版权声明
|
||||
# Generate copyright header
|
||||
copyright_header = generate_copyright_header(relative_path)
|
||||
|
||||
# 查找插入位置
|
||||
# Find insert position
|
||||
insert_pos, has_encoding = find_insert_position(lines)
|
||||
|
||||
# 构建新的文件内容
|
||||
# Build new file content
|
||||
new_lines = []
|
||||
|
||||
# 如果没有编码声明,添加一个
|
||||
# Add encoding declaration if not present
|
||||
if not has_encoding:
|
||||
new_lines.append("# -*- coding: utf-8 -*-\n")
|
||||
|
||||
# 添加前面的部分(shebang和编码声明)
|
||||
# Add front part (shebang and encoding declaration)
|
||||
new_lines.extend(lines[:insert_pos])
|
||||
|
||||
# 添加版权声明
|
||||
# Add copyright header
|
||||
new_lines.append(copyright_header + "\n")
|
||||
|
||||
# 如果文件没有免责声明,添加免责声明
|
||||
# Add disclaimer if file doesn't have one
|
||||
if not has_disclaimer(content):
|
||||
new_lines.append(DISCLAIMER + "\n")
|
||||
|
||||
# 添加一个空行(如果下一行不是空行)
|
||||
# Add empty line (if next line is not empty)
|
||||
if insert_pos < len(lines) and lines[insert_pos].strip():
|
||||
new_lines.append("\n")
|
||||
|
||||
# 添加剩余的内容
|
||||
# Add remaining content
|
||||
new_lines.extend(lines[insert_pos:])
|
||||
|
||||
# 如果不是dry run,写入文件
|
||||
# Write to file if not dry run
|
||||
if not dry_run:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
f.writelines(new_lines)
|
||||
@@ -212,14 +212,14 @@ def process_file(file_path: str, project_root: str, dry_run: bool = False) -> Tu
|
||||
|
||||
def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List[str]:
|
||||
"""
|
||||
查找所有Python文件
|
||||
Find all Python files
|
||||
|
||||
Args:
|
||||
root_dir: 根目录
|
||||
exclude_patterns: 排除的目录模式
|
||||
root_dir: Root directory
|
||||
exclude_patterns: Directory patterns to exclude
|
||||
|
||||
Returns:
|
||||
Python文件路径列表
|
||||
List of Python file paths
|
||||
"""
|
||||
if exclude_patterns is None:
|
||||
exclude_patterns = ['venv', '.venv', 'node_modules', '__pycache__', '.git', 'build', 'dist', '.eggs']
|
||||
@@ -227,7 +227,7 @@ def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List
|
||||
python_files = []
|
||||
|
||||
for root, dirs, files in os.walk(root_dir):
|
||||
# 排除特定目录
|
||||
# Exclude specific directories
|
||||
dirs[:] = [d for d in dirs if d not in exclude_patterns and not d.startswith('.')]
|
||||
|
||||
for file in files:
|
||||
@@ -238,39 +238,39 @@ def find_python_files(root_dir: str, exclude_patterns: List[str] = None) -> List
|
||||
|
||||
|
||||
def main():
|
||||
"""主函数"""
|
||||
"""Main function"""
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(description='Python文件头版权声明管理工具')
|
||||
parser.add_argument('files', nargs='*', help='要处理的文件路径(可选,默认处理所有.py文件)')
|
||||
parser.add_argument('--dry-run', action='store_true', help='仅检查不修改文件')
|
||||
parser.add_argument('--project-root', default=None, help='项目根目录(默认为当前目录)')
|
||||
parser.add_argument('--check', action='store_true', help='检查模式,如果有文件缺少版权声明则返回非零退出码')
|
||||
parser = argparse.ArgumentParser(description='Python file header copyright declaration management tool')
|
||||
parser.add_argument('files', nargs='*', help='File paths to process (optional, defaults to all .py files)')
|
||||
parser.add_argument('--dry-run', action='store_true', help='Check only without modifying files')
|
||||
parser.add_argument('--project-root', default=None, help='Project root directory (defaults to current directory)')
|
||||
parser.add_argument('--check', action='store_true', help='Check mode, return non-zero exit code if files missing copyright declaration')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
# 确定项目根目录
|
||||
# Determine project root directory
|
||||
if args.project_root:
|
||||
project_root = os.path.abspath(args.project_root)
|
||||
else:
|
||||
# 假设此脚本在 tools/ 目录下
|
||||
# Assume this script is in tools/ directory
|
||||
project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
print(f"Project root: {project_root}")
|
||||
print(f"Mode: {'DRY RUN' if args.dry_run else 'UPDATE'}")
|
||||
print("-" * 60)
|
||||
|
||||
# 获取要处理的文件列表
|
||||
# Get list of files to process
|
||||
if args.files:
|
||||
# 处理指定的文件
|
||||
# Process specified files
|
||||
files_to_process = [os.path.abspath(f) for f in args.files if f.endswith('.py')]
|
||||
else:
|
||||
# 处理所有Python文件
|
||||
# Process all Python files
|
||||
files_to_process = find_python_files(project_root)
|
||||
|
||||
print(f"Found {len(files_to_process)} Python files to process\n")
|
||||
|
||||
# 处理文件
|
||||
# Process files
|
||||
updated_count = 0
|
||||
skipped_count = 0
|
||||
error_count = 0
|
||||
@@ -286,7 +286,7 @@ def main():
|
||||
else:
|
||||
skipped_count += 1
|
||||
|
||||
# 打印汇总
|
||||
# Print summary
|
||||
print("\n" + "=" * 60)
|
||||
print(f"Summary:")
|
||||
print(f" Total files: {len(files_to_process)}")
|
||||
@@ -295,7 +295,7 @@ def main():
|
||||
print(f" Errors: {error_count}")
|
||||
print("=" * 60)
|
||||
|
||||
# 如果是check模式且有文件需要更新,返回非零退出码
|
||||
# Return non-zero exit code in check mode if files need update
|
||||
if args.check and updated_count > 0:
|
||||
sys.exit(1)
|
||||
elif error_count > 0:
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/2 12:55
|
||||
# @Desc : 滑块相关的工具包
|
||||
# @Desc : Slider verification utility package
|
||||
import os
|
||||
from typing import List
|
||||
from urllib.parse import urlparse
|
||||
@@ -38,8 +38,8 @@ class Slide:
|
||||
"""
|
||||
def __init__(self, gap, bg, gap_size=None, bg_size=None, out=None):
|
||||
"""
|
||||
:param gap: 缺口图片链接或者url
|
||||
:param bg: 带缺口的图片链接或者url
|
||||
:param gap: Gap image path or url
|
||||
:param bg: Background image with gap path or url
|
||||
"""
|
||||
self.img_dir = os.path.join(os.getcwd(), 'temp_image')
|
||||
if not os.path.exists(self.img_dir):
|
||||
@@ -76,13 +76,13 @@ class Slide:
|
||||
cv2.imwrite(img_path, image)
|
||||
return img_path
|
||||
else:
|
||||
raise Exception(f"保存{img_type}图片失败")
|
||||
raise Exception(f"Failed to save {img_type} image")
|
||||
else:
|
||||
return img
|
||||
|
||||
@staticmethod
|
||||
def clear_white(img):
|
||||
"""清除图片的空白区域,这里主要清除滑块的空白"""
|
||||
"""Clear whitespace from image, mainly clearing slider whitespace"""
|
||||
img = cv2.imread(img)
|
||||
rows, cols, channel = img.shape
|
||||
min_x = 255
|
||||
@@ -108,16 +108,16 @@ class Slide:
|
||||
def template_match(self, tpl, target):
|
||||
th, tw = tpl.shape[:2]
|
||||
result = cv2.matchTemplate(target, tpl, cv2.TM_CCOEFF_NORMED)
|
||||
# 寻找矩阵(一维数组当作向量,用Mat定义) 中最小值和最大值的位置
|
||||
# Find min and max value positions in matrix
|
||||
min_val, max_val, min_loc, max_loc = cv2.minMaxLoc(result)
|
||||
tl = max_loc
|
||||
br = (tl[0] + tw, tl[1] + th)
|
||||
# 绘制矩形边框,将匹配区域标注出来
|
||||
# target:目标图像
|
||||
# tl:矩形定点
|
||||
# br:矩形的宽高
|
||||
# (0,0,255):矩形边框颜色
|
||||
# 1:矩形边框大小
|
||||
# Draw rectangle border to mark the matched area
|
||||
# target: target image
|
||||
# tl: rectangle top-left corner
|
||||
# br: rectangle width and height
|
||||
# (0,0,255): rectangle border color
|
||||
# 1: rectangle border size
|
||||
cv2.rectangle(target, tl, br, (0, 0, 255), 2)
|
||||
cv2.imwrite(self.out, target)
|
||||
return tl[0]
|
||||
@@ -138,39 +138,39 @@ class Slide:
|
||||
slide_pic = cv2.cvtColor(slide, cv2.COLOR_GRAY2RGB)
|
||||
back_pic = cv2.cvtColor(back, cv2.COLOR_GRAY2RGB)
|
||||
x = self.template_match(slide_pic, back_pic)
|
||||
# 输出横坐标, 即 滑块在图片上的位置
|
||||
# Output x-coordinate, i.e., slider position on image
|
||||
return x
|
||||
|
||||
|
||||
def get_track_simple(distance) -> List[int]:
|
||||
# 有的检测移动速度的 如果匀速移动会被识别出来,来个简单点的 渐进
|
||||
# distance为传入的总距离
|
||||
# 移动轨迹
|
||||
# Some detection checks movement speed - constant speed will be detected, so use gradual acceleration
|
||||
# distance is the total distance to move
|
||||
# Movement track
|
||||
track: List[int] = []
|
||||
# 当前位移
|
||||
# Current displacement
|
||||
current = 0
|
||||
# 减速阈值
|
||||
# Deceleration threshold
|
||||
mid = distance * 4 / 5
|
||||
# 计算间隔
|
||||
# Time interval
|
||||
t = 0.2
|
||||
# 初速度
|
||||
# Initial velocity
|
||||
v = 1
|
||||
|
||||
while current < distance:
|
||||
if current < mid:
|
||||
# 加速度为2
|
||||
# Acceleration = 4
|
||||
a = 4
|
||||
else:
|
||||
# 加速度为-2
|
||||
# Acceleration = -3
|
||||
a = -3
|
||||
v0 = v
|
||||
# 当前速度
|
||||
# Current velocity
|
||||
v = v0 + a * t # type: ignore
|
||||
# 移动距离
|
||||
# Movement distance
|
||||
move = v0 * t + 1 / 2 * a * t * t
|
||||
# 当前位移
|
||||
# Current displacement
|
||||
current += move # type: ignore
|
||||
# 加入轨迹
|
||||
# Add to track
|
||||
track.append(round(move))
|
||||
return track
|
||||
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Author : relakkes@gmail.com
|
||||
# @Time : 2023/12/2 12:52
|
||||
# @Desc : 时间相关的工具函数
|
||||
# @Desc : Time utility functions
|
||||
|
||||
import time
|
||||
from datetime import datetime, timedelta, timezone
|
||||
@@ -29,7 +29,7 @@ from datetime import datetime, timedelta, timezone
|
||||
|
||||
def get_current_timestamp() -> int:
|
||||
"""
|
||||
获取当前的时间戳(13 位):1701493264496
|
||||
Get current timestamp (13 digits): 1701493264496
|
||||
:return:
|
||||
"""
|
||||
return int(time.time() * 1000)
|
||||
@@ -37,21 +37,21 @@ def get_current_timestamp() -> int:
|
||||
|
||||
def get_current_time() -> str:
|
||||
"""
|
||||
获取当前的时间:'2023-12-02 13:01:23'
|
||||
Get current time: '2023-12-02 13:01:23'
|
||||
:return:
|
||||
"""
|
||||
return time.strftime('%Y-%m-%d %X', time.localtime())
|
||||
|
||||
def get_current_time_hour() -> str:
|
||||
"""
|
||||
获取当前的时间:'2023-12-02-13'
|
||||
Get current time with hour: '2023-12-02-13'
|
||||
:return:
|
||||
"""
|
||||
return time.strftime('%Y-%m-%d-%H', time.localtime())
|
||||
|
||||
def get_current_date() -> str:
|
||||
"""
|
||||
获取当前的日期:'2023-12-02'
|
||||
Get current date: '2023-12-02'
|
||||
:return:
|
||||
"""
|
||||
return time.strftime('%Y-%m-%d', time.localtime())
|
||||
@@ -59,7 +59,7 @@ def get_current_date() -> str:
|
||||
|
||||
def get_time_str_from_unix_time(unixtime):
|
||||
"""
|
||||
unix 整数类型时间戳 ==> 字符串日期时间
|
||||
Unix integer timestamp ==> datetime string
|
||||
:param unixtime:
|
||||
:return:
|
||||
"""
|
||||
@@ -70,7 +70,7 @@ def get_time_str_from_unix_time(unixtime):
|
||||
|
||||
def get_date_str_from_unix_time(unixtime):
|
||||
"""
|
||||
unix 整数类型时间戳 ==> 字符串日期
|
||||
Unix integer timestamp ==> date string
|
||||
:param unixtime:
|
||||
:return:
|
||||
"""
|
||||
@@ -81,7 +81,7 @@ def get_date_str_from_unix_time(unixtime):
|
||||
|
||||
def get_unix_time_from_time_str(time_str):
|
||||
"""
|
||||
字符串时间 ==> unix 整数类型时间戳,精确到秒
|
||||
Time string ==> Unix integer timestamp, precise to seconds
|
||||
:param time_str:
|
||||
:return:
|
||||
"""
|
||||
@@ -99,34 +99,34 @@ def get_unix_timestamp():
|
||||
|
||||
|
||||
def rfc2822_to_china_datetime(rfc2822_time):
|
||||
# 定义RFC 2822格式
|
||||
# Define RFC 2822 format
|
||||
rfc2822_format = "%a %b %d %H:%M:%S %z %Y"
|
||||
|
||||
# 将RFC 2822时间字符串转换为datetime对象
|
||||
# Convert RFC 2822 time string to datetime object
|
||||
dt_object = datetime.strptime(rfc2822_time, rfc2822_format)
|
||||
|
||||
# 将datetime对象的时区转换为中国时区
|
||||
# Convert datetime object timezone to China timezone
|
||||
dt_object_china = dt_object.astimezone(timezone(timedelta(hours=8)))
|
||||
return dt_object_china
|
||||
|
||||
|
||||
def rfc2822_to_timestamp(rfc2822_time):
|
||||
# 定义RFC 2822格式
|
||||
# Define RFC 2822 format
|
||||
rfc2822_format = "%a %b %d %H:%M:%S %z %Y"
|
||||
|
||||
# 将RFC 2822时间字符串转换为datetime对象
|
||||
# Convert RFC 2822 time string to datetime object
|
||||
dt_object = datetime.strptime(rfc2822_time, rfc2822_format)
|
||||
|
||||
# 将datetime对象转换为UTC时间
|
||||
# Convert datetime object to UTC time
|
||||
dt_utc = dt_object.replace(tzinfo=timezone.utc)
|
||||
|
||||
# 计算UTC时间对应的Unix时间戳
|
||||
# Calculate Unix timestamp from UTC time
|
||||
timestamp = int(dt_utc.timestamp())
|
||||
|
||||
return timestamp
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
# 示例用法
|
||||
# Example usage
|
||||
_rfc2822_time = "Sat Dec 23 17:12:54 +0800 2023"
|
||||
print(rfc2822_to_china_datetime(_rfc2822_time))
|
||||
|
||||
@@ -36,7 +36,7 @@ def init_loging_config():
|
||||
_logger = logging.getLogger("MediaCrawler")
|
||||
_logger.setLevel(level)
|
||||
|
||||
# 关闭 httpx 的 INFO 日志
|
||||
# Disable httpx INFO level logs
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
|
||||
return _logger
|
||||
|
||||
Reference in New Issue
Block a user