From 0ca7b29cf0753076ff7488e6f03cef4f2f82aa0f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Thu, 17 Sep 2026 22:54:22 +0800 Subject: [PATCH] =?UTF-8?q?feat(media):=20=E9=87=8D=E6=9E=84=E5=AA=92?= =?UTF-8?q?=E4=BD=93=E4=B8=8B=E8=BD=BD=EF=BC=8C=E6=94=AF=E6=8C=81=20xhs/dy?= =?UTF-8?q?/ks/bili/wb=20=E4=BA=94=E5=B9=B3=E5=8F=B0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验， 代码按平台复制粘贴了 4 份。本次用统一下载器替换： - 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、 路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流 - 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址， 与下载器解耦；快手首次接入下载能力 - 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI； 同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写 - 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/ - B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4， 避免低清文件阻塞后续的高清路径） - 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法 媒体下载失败只记录日志，不中断爬取主流程。 --- README.md | 34 + api/schemas/crawler.py | 1 + api/services/crawler_manager.py | 1 + base/base_crawler.py | 16 - cmd_arg/arg.py | 12 + config/base_config.py | 7 +- media_downloader/__init__.py | 46 + media_downloader/downloader.py | 581 ++++++++++++ media_downloader/ffmpeg.py | 115 +++ media_downloader/paths.py | 197 ++++ media_downloader/types.py | 81 ++ media_platform/bilibili/client.py | 23 +- media_platform/bilibili/core.py | 164 +++- media_platform/bilibili/media.py | 202 ++++ media_platform/douyin/client.py | 14 - media_platform/douyin/core.py | 103 +- media_platform/douyin/media.py | 144 +++ media_platform/kuaishou/core.py | 48 + media_platform/kuaishou/media.py | 137 +++ media_platform/weibo/client.py | 28 - media_platform/weibo/core.py | 78 +- media_platform/weibo/media.py | 181 ++++ media_platform/xhs/client.py | 23 - media_platform/xhs/core.py | 94 +- media_platform/xhs/media.py | 160 ++++ store/bilibili/__init__.py | 16 - store/bilibili/bilibilli_store_media.py | 82 -- store/douyin/__init__.py | 102 +- store/douyin/douyin_store_media.py | 129 --- store/weibo/__init__.py | 15 - store/weibo/weibo_store_media.py | 82 -- store/xhs/__init__.py | 68 +- store/xhs/xhs_store_media.py | 133 --- tests/media_server.py | 386 ++++++++ tests/test_api_limits.py | 29 + tests/test_cmd_arg_media.py | 73 ++ tests/test_media_core_integration.py | 513 ++++++++++ tests/test_media_downloader.py | 890 ++++++++++++++++++ tests/test_media_e2e.py | 172 ++++ tests/test_media_extractors.py | 741 +++++++++++++++ tests/test_media_ffmpeg.py | 147 +++ .../components/config/CrawlerConfigPanel.tsx | 19 +- webui/src/i18n/locales/en-US/config.json | 2 + webui/src/i18n/locales/zh-CN/config.json | 2 + webui/src/lib/api.ts | 1 + webui/src/store/crawlerStore.ts | 1 + webui/src/types/crawler.ts | 1 + 47 files changed, 5187 insertions(+), 907 deletions(-) create mode 100644 media_downloader/__init__.py create mode 100644 media_downloader/downloader.py create mode 100644 media_downloader/ffmpeg.py create mode 100644 media_downloader/paths.py create mode 100644 media_downloader/types.py create mode 100644 media_platform/bilibili/media.py create mode 100644 media_platform/douyin/media.py create mode 100644 media_platform/kuaishou/media.py create mode 100644 media_platform/weibo/media.py create mode 100644 media_platform/xhs/media.py delete mode 100644 store/bilibili/bilibilli_store_media.py delete mode 100644 store/douyin/douyin_store_media.py delete mode 100644 store/weibo/weibo_store_media.py delete mode 100644 store/xhs/xhs_store_media.py create mode 100644 tests/media_server.py create mode 100644 tests/test_cmd_arg_media.py create mode 100644 tests/test_media_core_integration.py create mode 100644 tests/test_media_downloader.py create mode 100644 tests/test_media_e2e.py create mode 100644 tests/test_media_extractors.py create mode 100644 tests/test_media_ffmpeg.py diff --git a/README.md b/README.md index 74afb5b..095f640 100644 --- a/README.md +++ b/README.md @@ -170,6 +170,40 @@ uv run main.py --platform xhs --lt qrcode --type detail uv run main.py --help ``` +
+📥 媒体下载（封面 / 视频 / 图文图片） + +默认关闭。开启后会在爬取的同时把媒体文件下载到本地，按帖子聚合存放： + +```shell +# 命令行开关 +uv run main.py --platform xhs --type detail --specified_id <帖子URL或ID> --get_media true + +# 或在 config/base_config.py 中设置 ENABLE_GET_MEDIA = True +``` + +**落盘结构**（`SAVE_DATA_PATH` 为空时落在 `data/` 下）： + +``` +data/xhs/media/{帖子ID}/ +├── cover.jpg # 封面 +├── video.mp4 # 视频（B站为 DASH 合流产物） +└── 001.jpg # 图文帖的图片 +``` + +**支持平台**：小红书、抖音、快手、B站、微博。贴吧与知乎的数据结构中没有媒体字段，暂不支持。 + +**关于 B 站**： +- 安装 [ffmpeg](https://ffmpeg.org/) 后可走 DASH 路径获取最高画质（音视频分轨下载后无损合流）；未安装时自动降级为 mp4 直链，产物名为 `video-durl.mp4` 以便区分 +- 清晰度由 `config/bilibili_config.py` 的 `BILI_QN` 控制（默认 80 = 1080p）；未登录或权限不足时若高清取流被拒，会自动逐档降级到实际可下载的清晰度 + +**说明**： +- 媒体下载失败只记录日志，不会中断爬取 +- 已存在的文件会被跳过；需要重新下载时删掉对应目录即可 +- 下载是串行跟随爬取流程的（未做并发下载），避免给平台 CDN 造成压力 + +
+
🖥️ WebUI 可视化操作界面 diff --git a/api/schemas/crawler.py b/api/schemas/crawler.py index 6eb3b1b..cfc995e 100644 --- a/api/schemas/crawler.py +++ b/api/schemas/crawler.py @@ -71,6 +71,7 @@ class CrawlerStartRequest(BaseModel): start_page: int = 1 enable_comments: bool = True enable_sub_comments: bool = False + enable_media: bool = False save_option: SaveDataOptionEnum = SaveDataOptionEnum.JSONL cookies: str = "" headless: bool = False diff --git a/api/services/crawler_manager.py b/api/services/crawler_manager.py index 9af954b..83119cc 100644 --- a/api/services/crawler_manager.py +++ b/api/services/crawler_manager.py @@ -224,6 +224,7 @@ class CrawlerManager: cmd.extend(["--get_comment", "true" if config.enable_comments else "false"]) cmd.extend(["--get_sub_comment", "true" if config.enable_sub_comments else "false"]) + cmd.extend(["--get_media", "true" if config.enable_media else "false"]) if config.max_notes_count is not None: cmd.extend(["--crawler_max_notes_count", str(config.max_notes_count)]) diff --git a/base/base_crawler.py b/base/base_crawler.py index 4fe9eaa..de6efc8 100644 --- a/base/base_crawler.py +++ b/base/base_crawler.py @@ -100,22 +100,6 @@ class AbstractStore(ABC): pass -class AbstractStoreImage(ABC): - # TODO: support all platform - # only weibo is supported - # @abstractmethod - async def store_image(self, image_content_item: Dict): - pass - - -class AbstractStoreVideo(ABC): - # TODO: support all platform - # only weibo is supported - # @abstractmethod - async def store_video(self, video_content_item: Dict): - pass - - class AbstractApiClient(ABC): @abstractmethod diff --git a/cmd_arg/arg.py b/cmd_arg/arg.py index 20d1d39..8ea133d 100644 --- a/cmd_arg/arg.py +++ b/cmd_arg/arg.py @@ -216,6 +216,15 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None): show_default=True, ), ] = str(config.ENABLE_GET_SUB_COMMENTS), + get_media: Annotated[ + str, + typer.Option( + "--get_media", + help="Whether to download media files (cover/video/images of the post), supports yes/true/t/y/1 or no/false/f/n/0 (xhs/dy/ks/bili/wb)", + rich_help_panel="Storage Configuration", + show_default=True, + ), + ] = str(config.ENABLE_GET_MEDIA), headless: Annotated[ str, typer.Option( @@ -337,6 +346,7 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None): enable_comment = _to_bool(get_comment) enable_sub_comment = _to_bool(get_sub_comment) + enable_media = _to_bool(get_media) enable_headless = _to_bool(headless) enable_ip_proxy_value = _to_bool(enable_ip_proxy) init_db_value = init_db.value if init_db else None @@ -353,6 +363,7 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None): config.KEYWORDS = keywords config.ENABLE_GET_COMMENTS = enable_comment config.ENABLE_GET_SUB_COMMENTS = enable_sub_comment + config.ENABLE_GET_MEDIA = enable_media config.HEADLESS = enable_headless config.CDP_HEADLESS = enable_headless config.SAVE_DATA_OPTION = save_data_option.value @@ -409,6 +420,7 @@ async def parse_cmd(argv: Optional[Sequence[str]] = None): keywords=config.KEYWORDS, get_comment=config.ENABLE_GET_COMMENTS, get_sub_comment=config.ENABLE_GET_SUB_COMMENTS, + get_media=config.ENABLE_GET_MEDIA, headless=config.HEADLESS, save_data_option=config.SAVE_DATA_OPTION, init_db=init_db_value, diff --git a/config/base_config.py b/config/base_config.py index 28a852e..cf6823c 100644 --- a/config/base_config.py +++ b/config/base_config.py @@ -104,8 +104,11 @@ CRAWLER_MAX_NOTES_COUNT = 15 # Controlling the number of concurrent crawlers MAX_CONCURRENCY_NUM = 1 -# Whether to enable crawling media mode (including image or video resources), crawling media is not enabled by default -ENABLE_GET_MEIDAS = False +# 是否启用媒体下载（封面、视频，以及图文帖的图片），默认关闭。 +# 开启后媒体文件按 {SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/ 目录聚合存放。 +# 支持的平台：xhs / dy / ks / bili / wb（tieba、zhihu 的数据结构中没有媒体字段，不支持）。 +# 命令行开关：--get_media +ENABLE_GET_MEDIA = False # Whether to enable comment crawling mode. Comment crawling is enabled by default. ENABLE_GET_COMMENTS = True diff --git a/media_downloader/__init__.py b/media_downloader/__init__.py new file mode 100644 index 0000000..4506739 --- /dev/null +++ b/media_downloader/__init__.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_downloader/__init__.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""统一媒体下载器：流式传输、断点续传、指数退避重试、大小校验、DASH 合流。 + +平台特有的"下载哪个 URL"由 ``media_platform//media.py`` 负责， +本包只处理"怎么把字节落到磁盘"，因此不认识任何平台。 +""" + +from .downloader import MediaDownloader +from .ffmpeg import ffmpeg_path, is_available as is_ffmpeg_available, merge_audio_video +from .types import ( + MediaDownloadError, + MediaFatalError, + MediaItem, + MediaRetryableError, + MediaType, +) + +__all__ = [ + "MediaDownloader", + "MediaDownloadError", + "MediaFatalError", + "MediaItem", + "MediaRetryableError", + "MediaType", + "ffmpeg_path", + "is_ffmpeg_available", + "merge_audio_video", +] diff --git a/media_downloader/downloader.py b/media_downloader/downloader.py new file mode 100644 index 0000000..b3eb244 --- /dev/null +++ b/media_downloader/downloader.py @@ -0,0 +1,581 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_downloader/downloader.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""统一媒体下载器。 + +设计要点（与常见实现的关键差异）： + +1. **零额外请求**：不做 HEAD 探测（大量 CDN 对 HEAD 返回 403/405 或与 GET 不一致）， + 大小、类型、是否支持续传全部从一次 ``stream GET`` 的响应头判定。 +2. **大小校验按 HTTP 语义**：206 响应的 ``Content-Length`` 是"剩余长度"， + 文件总大小必须取自 ``Content-Range`` 的 ``T``，否则续传场景必然误判失败。 +3. **显式 ``Accept-Encoding: identity``**：httpx 默认协商 gzip，若 CDN 压缩， + 解码后的字节数与 ``Content-Length`` 不等，大小校验会 100% 误判。 +4. **临时文件带 URL 指纹**：URL 变化时自动作废旧片段，避免把新内容拼到旧半成品上。 +5. **失败不打断爬虫**：所有异常在 ``download`` 内部收敛为返回值与日志。 +""" + +from __future__ import annotations + +import asyncio +import logging +import os +import random +import re +import shutil +import uuid +from pathlib import Path +from typing import Iterable, Mapping, Optional, Sequence +from urllib.parse import urlsplit + +import httpx + +from tools.httpx_util import make_async_client + +from .ffmpeg import merge_audio_video +from .paths import ( + ALLOWED_EXTENSIONS, + build_file_path, + build_media_dir, + build_media_path, + ensure_within, + guess_extension, + redact_url, + sanitize_component, + url_fingerprint, +) +from .types import MediaDownloadError, MediaFatalError, MediaItem, MediaRetryableError, MediaType + +logger = logging.getLogger("MediaCrawler.media_downloader") + +DEFAULT_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) +FALLBACK_EXTENSIONS = {MediaType.IMAGE: ".jpg", MediaType.VIDEO: ".mp4"} + +_CONTENT_RANGE = re.compile(r"bytes\s+(\d+)-(\d+)/(\d+|\*)", re.IGNORECASE) + +# 各媒体类型可接受的 Content-Type 前缀 +_EXPECTED_CONTENT_TYPES = { + MediaType.IMAGE: ("image/",), + # 音频单独成轨时素材类型仍是视频（DASH），因此一并接受 + MediaType.VIDEO: ("video/", "audio/"), +} +# 这些类型无法用于判定内容是否正常，一律放行 +_NEUTRAL_CONTENT_TYPES = ( + "application/octet-stream", + "binary/octet-stream", + "application/x-www-form-urlencoded", + # 少数 CDN 会用 application/* 返回真实媒体，不能当成错误页拒掉 + "application/mp4", + "application/x-m4a", +) +# 注意：HLS 播放列表（application/x-mpegurl、application/vnd.apple.mpegurl） +# **不能**放进中性名单——m3u8 是文本清单而不是媒体，放行会把它存成几十 KB 的假视频， +# 且会被"已存在即跳过"永久信任。下载器不支持 HLS，交给类型校验直接拒绝。 + + +def is_expected_content_type(content_type: Optional[str], media_type: MediaType) -> bool: + """判断响应的 Content-Type 是否与媒体类型相容。 + + CDN 的防盗链页、限流页常以 HTTP 200 + ``text/html`` 返回， + 若不拦截会被当作媒体文件落盘，并且因为文件非空而被永久跳过。 + """ + if not content_type: + return True + + main_type = content_type.split(";")[0].strip().lower() + if not main_type or main_type in _NEUTRAL_CONTENT_TYPES: + return True + + expected_prefixes = _EXPECTED_CONTENT_TYPES.get(media_type) + if not expected_prefixes: + return True + + return main_type.startswith(expected_prefixes) + + +class MediaDownloader: + """把 MediaItem 流式下载到本地磁盘。 + + 平台差异只通过构造参数（proxy/extra_headers）与 MediaItem 字段注入。 + ``platform`` 仅用于决定落盘目录，下载器不对平台名做任何分支判断 + （Referer 等反爬头由平台侧的 ``_media_headers()`` 提供）。 + """ + + # 这些状态码重试没有意义：403 多为签名过期，404/410 为资源不存在 + FATAL_STATUS_CODES = frozenset({400, 401, 403, 404, 405, 410, 451}) + + def __init__( + self, + platform: str, + *, + proxy: Optional[str] = None, + extra_headers: Optional[Mapping[str, str]] = None, + timeout: float = 60.0, + max_retries: int = 3, + retry_base_delay: float = 1.0, + retry_max_delay: float = 30.0, + max_candidates: int = 3, + base_dir: Optional[Path] = None, + overwrite: bool = False, + ) -> None: + self.platform = platform + self.proxy = proxy + self.extra_headers = dict(extra_headers or {}) + self.timeout = timeout + self.max_retries = max(0, max_retries) + self.retry_base_delay = retry_base_delay + self.retry_max_delay = retry_max_delay + # 主地址 + 备用地址的总数上限，防止个别平台给出超长 url_list 造成请求放大 + self.max_candidates = max(1, max_candidates) + self._base_dir = Path(base_dir) if base_dir is not None else None + self.overwrite = overwrite + + # ------------------------------------------------------------------ 对外 API + + @property + def base_dir(self) -> Path: + """媒体根目录，未显式指定时取 config.SAVE_DATA_PATH（为空则 data/）""" + if self._base_dir is None: + import config + + self._base_dir = Path(getattr(config, "SAVE_DATA_PATH", "") or "data") + return self._base_dir + + async def download_all(self, items: Sequence[MediaItem]) -> list[Path]: + """串行下载一个帖子的全部媒体，共享同一个 httpx 连接池""" + if not items: + return [] + downloaded: list[Path] = [] + async with make_async_client( + proxy=self.proxy, follow_redirects=True, timeout=self.timeout + ) as client: + for item in items: + path = await self.download(item, client=client) + if path is not None: + downloaded.append(path) + if len(downloaded) < len(items): + logger.warning( + "[download] 本组媒体部分失败: 成功 %d / 共 %d（失败项见上方日志）", + len(downloaded), + len(items), + ) + return downloaded + + async def download(self, item: MediaItem, client: Optional[httpx.AsyncClient] = None) -> Optional[Path]: + """下载单个媒体；成功返回最终路径，失败返回 None（不抛异常）。 + + 续传的作用范围是"本次调用内的重试"：``.part`` 片段带进程号， + 且媒体直链的签名通常跨运行就失效，因此不承诺跨进程续传。 + """ + try: + if not self.is_supported_url(item.url): + # 平台响应里偶尔会出现协议相对地址（//cdn/...）或空值，httpx 会直接报错 + logger.warning("[download] 非法媒体地址，跳过: %s", redact_url(item.url)) + return None + + existing = self._find_existing(item) + if existing is not None and not self.overwrite: + logger.info("[download] 已存在，跳过: %s", existing) + return existing + + if client is not None: + return await self._download_item(client, item) + async with make_async_client( + proxy=self.proxy, follow_redirects=True, timeout=self.timeout + ) as own_client: + return await self._download_item(own_client, item) + except Exception as exc: # 兜底：任何异常都不能打断爬虫 + logger.error("[download] 下载失败 %s: %s", redact_url(item.url), exc) + return None + + def update_credentials( + self, + proxy: Optional[str] = None, + extra_headers: Optional[Mapping[str, str]] = None, + ) -> None: + """同步平台 client 的最新代理与请求头。 + + 代理池刷新与 Cookie 续期都是就地改 client 的属性，而下载器是惰性创建、 + 长期复用的；不主动同步就会一直用创建时的旧代理/旧凭证， + 表现为 API 请求正常但媒体下载持续失败。 + """ + self.proxy = proxy + if extra_headers: + self.extra_headers.update(extra_headers) + + def build_path(self, item: MediaItem, extension: Optional[str] = None) -> Path: + """最终落盘路径（纯函数，便于测试与断言）""" + ext = extension or item.extension or guess_extension( + item.url, default=FALLBACK_EXTENSIONS.get(item.media_type, ".bin") + ) + return build_media_path(self.base_dir, self.platform, item.content_id, item.stem, ext) + + # ------------------------------------------------------------------ 内部实现 + + def _find_existing(self, item: MediaItem) -> Optional[Path]: + """查找已下载完成的同名文件（0 字节视为损坏，重新下载）""" + directory = build_media_dir(self.base_dir, self.platform, item.content_id) + if not directory.is_dir(): + return None + + if item.extension: + candidate = self.build_path(item, item.extension) + if candidate.is_file() and candidate.stat().st_size > 0: + return candidate + return None + + for candidate in sorted(directory.glob(f"{item.stem}.*")): + if candidate.suffix.lower() not in ALLOWED_EXTENSIONS: + continue + if candidate.is_file() and candidate.stat().st_size > 0: + return candidate + return None + + async def _download_item(self, client: httpx.AsyncClient, item: MediaItem) -> Optional[Path]: + directory = build_media_dir(self.base_dir, self.platform, item.content_id) + directory.mkdir(parents=True, exist_ok=True) + + if item.is_dash: + return await self._download_dash(client, item, directory) + + return await self._download_single( + client, item, directory, item.url, item.backup_urls, item.stem, item.extension + ) + + async def _download_single( + self, + client: httpx.AsyncClient, + item: MediaItem, + target_dir: Path, + url: str, + backup_urls: Iterable[str], + stem: str, + extension: Optional[str] = None, + ) -> Optional[Path]: + """按 主地址 -> 备用地址 的顺序下载到 ``target_dir``，每个地址内部做重试。 + + ``stem`` 与 ``extension`` 由调用方给出：DASH 的两路流会用不同的文件名 + 落在同一个临时目录里，不能共用 MediaItem 上的 stem。 + """ + # 临时文件名也要清洗：stem 来自 MediaItem 的公开契约，未清洗的 stem + # 会把 .part 写到 base_dir 之外（最终路径有 build_file_path 兜底，临时路径没有） + safe_stem = sanitize_component(stem, "media") + # 备用地址数量要有上限：抖音的 url_list 长度不可控，逐个试会放大成几十次请求 + candidates = [url, *[candidate for candidate in backup_urls if candidate]][: self.max_candidates] + + for candidate in candidates: + if not candidate: + continue + part_path = target_dir / f"{safe_stem}.{url_fingerprint(candidate)}.{os.getpid()}.part" + try: + content_type = await self._fetch_with_retry(client, item, candidate, part_path) + except MediaFatalError as exc: + logger.warning("[download] 放弃地址 %s: %s", redact_url(candidate), exc) + continue + except MediaDownloadError as exc: + logger.warning("[download] 地址重试耗尽 %s: %s", redact_url(candidate), exc) + continue + + resolved_extension = extension or guess_extension( + candidate, + content_type=content_type, + explicit=item.extension, + default=FALLBACK_EXTENSIONS.get(item.media_type, ".bin"), + ) + try: + final_path = ensure_within( + self.base_dir, build_file_path(target_dir, stem, resolved_extension) + ) + except ValueError as exc: + logger.error("[download] %s", exc) + part_path.unlink(missing_ok=True) + return None + + os.replace(part_path, final_path) + self._cleanup_parts(target_dir, safe_stem, keep=None) + logger.info( + "[download] 下载完成: %s (%d bytes)", + final_path, + final_path.stat().st_size, + ) + return final_path + + # 调用内重试会复用 .part 做续传；全部地址都失败后则不再保留， + # 否则每次失败都会在媒体目录里留下一个再也用不上的残片（媒体直链签名每次都会变） + self._cleanup_parts(target_dir, safe_stem) + logger.error("[download] 媒体下载失败，已尝试全部地址: %s", redact_url(url)) + return None + + @staticmethod + def _cleanup_parts(target_dir: Path, safe_stem: str, keep: Optional[Path] = None) -> None: + """清掉该 stem 的临时片段。 + + 成功下载后也要清理：某个候选地址中途失败、回退到下一个候选成功时， + 前一个候选的残片不会自己消失。 + """ + for stale_part in target_dir.glob(f"{safe_stem}.*.part"): + if keep is not None and stale_part == keep: + continue + stale_part.unlink(missing_ok=True) + + async def _download_dash( + self, client: httpx.AsyncClient, item: MediaItem, directory: Path + ) -> Optional[Path]: + """DASH 音视频分轨：两路流下载到临时目录，ffmpeg 合流后原子替换""" + tmp_dir = directory / f".tmp-{os.getpid()}-{uuid.uuid4().hex[:8]}" + tmp_dir.mkdir(parents=True, exist_ok=True) + try: + video_path = await self._download_single( + client, item, tmp_dir, item.url, item.backup_urls, "video", ".m4s" + ) + if video_path is None: + logger.error("[download] DASH 视频轨下载失败: %s", redact_url(item.url)) + return None + + audio_url = item.audio_url or "" + audio_path = await self._download_single( + client, item, tmp_dir, audio_url, item.audio_backup_urls, "audio", ".m4s" + ) + if audio_path is None: + logger.error("[download] DASH 音频轨下载失败: %s", redact_url(audio_url)) + return None + + merged_path = tmp_dir / "merged.mp4" + await merge_audio_video(video_path, audio_path, merged_path) + + final_path = ensure_within(self.base_dir, self.build_path(item, ".mp4")) + os.replace(merged_path, final_path) + logger.info( + "[download] DASH 合流完成: %s (%d bytes)", + final_path, + final_path.stat().st_size, + ) + return final_path + except MediaDownloadError as exc: + logger.error("[download] DASH 合流失败 %s: %s", redact_url(item.url), exc) + return None + finally: + shutil.rmtree(tmp_dir, ignore_errors=True) + + async def _fetch_with_retry( + self, + client: httpx.AsyncClient, + item: MediaItem, + url: str, + part_path: Path, + ) -> Optional[str]: + """带指数退避重试的单文件抓取，返回 Content-Type""" + last_error: Optional[Exception] = None + for attempt in range(self.max_retries + 1): + try: + return await self._fetch_once(client, item, url, part_path) + except MediaFatalError: + raise + # httpx.InvalidURL 不是 HTTPError 的子类（它是 Exception 的直接子类）， + # 漏掉它会让一个畸形地址直接终止整条候选回退链 + except (MediaRetryableError, httpx.HTTPError, httpx.InvalidURL, OSError) as exc: + last_error = exc + if attempt < self.max_retries: + delay = self._retry_delay(attempt) + logger.warning( + "[download] 第 %d/%d 次重试 %s（%.1fs 后）: %s", + attempt + 1, + self.max_retries, + redact_url(url), + delay, + exc, + ) + await asyncio.sleep(delay) + if last_error is None: + raise MediaRetryableError("未知错误") + # 部分网络异常的 str() 为空，带上类型名才有排查线索 + raise MediaRetryableError(f"{type(last_error).__name__}: {last_error}") + + def _retry_delay(self, attempt: int) -> float: + """指数退避 + full jitter，避免多个任务同步重试""" + ceiling = min(self.retry_base_delay * (2**attempt), self.retry_max_delay) + return random.uniform(0, ceiling) + + async def _fetch_once( + self, + client: httpx.AsyncClient, + item: MediaItem, + url: str, + part_path: Path, + ) -> Optional[str]: + """执行一次流式 GET，成功返回 Content-Type。 + + 续传语义： + - 本地存在 ``.part`` 片段 -> 带 ``Range`` 请求 + - 206 -> 追加写；200（服务端忽略 Range）-> 截断重写；416 -> 片段失效，重来 + """ + resume_from = part_path.stat().st_size if part_path.exists() else 0 + headers = self._build_headers(item) + if resume_from > 0: + headers["Range"] = f"bytes={resume_from}-" + + async with client.stream("GET", url, headers=headers) as response: + status = response.status_code + if status in self.FATAL_STATUS_CODES: + raise MediaFatalError(f"HTTP {status}") + if status == 416: + part_path.unlink(missing_ok=True) + raise MediaRetryableError("HTTP 416：本地残留片段已失效，重新下载") + if status not in (200, 206): + raise MediaRetryableError(f"HTTP {status}") + + content_type = response.headers.get("content-type") + if not is_expected_content_type(content_type, item.media_type): + # 典型的防盗链/限流错误页：HTTP 200 但返回 text/html。 + # 此时尚未读取响应体，本地 .part 仍是合法前缀，保留以便下次续传 + raise MediaRetryableError( + f"响应类型 {content_type} 与媒体类型 {item.media_type.value} 不符，疑似错误页" + ) + + if status == 200 and resume_from > 0: + logger.debug("[download] 服务端不支持 Range，从头下载: %s", redact_url(url)) + resume_from = 0 + + content_range = self._parse_content_range(response) + if status == 206: + # RFC 9110 要求 206 必须带可解析的 Content-Range。缺失时服务端的实际行为 + # 不可预期（有 CDN 会对带 Range 的请求回 206 + 全量体），此时若继续 append + # 会把整份内容接到本地片段后面，而"总长"又是按剩余长度反推的，大小校验 + # 反而会通过 —— 只能拒绝并丢弃片段重下。 + if content_range is None: + part_path.unlink(missing_ok=True) + raise MediaRetryableError("206 响应缺少 Content-Range，丢弃片段重新下载") + if content_range[0] != resume_from: + # 起点不符会得到内容错位或头部缺失的文件；未发 Range（resume_from=0） + # 却收到非 0 起点的 206 同样要拒绝 + part_path.unlink(missing_ok=True) + raise MediaRetryableError( + f"服务端返回的 Range 起点 {content_range[0]} 与请求的 {resume_from} 不符" + ) + if content_range[2] is None: + # total 为 * 表示服务端只返回了区间的一部分且不告知总长， + # 这种响应无法确认完整性（落盘的就是截断文件），丢弃片段后 + # 以无 Range 的方式重新请求完整内容 + part_path.unlink(missing_ok=True) + raise MediaRetryableError("206 的 Content-Range 未给出总长（*），重新完整下载") + + use_append = status == 206 and resume_from > 0 + total_size = self._resolve_total_size(content_range, response, status) + # 服务端若无视 identity 强制压缩，落盘的是解码后的字节，与 Content-Length 不等 + encoded = response.headers.get("content-encoding", "").strip().lower() not in ("", "identity") + + written = resume_from if use_append else 0 + mode = "ab" if use_append else "wb" + with open(part_path, mode) as file_obj: + # 必须使用不带 chunk_size 的 aiter_bytes()：传入 chunk_size 时 httpx 会 + # 缓冲满该长度才产出数据，响应中途断开会让已接收的字节全部丢失， + # 断点续传与"截断后重试"都会退化成从头再来。 + async for chunk in response.aiter_bytes(): + if not chunk: + continue + file_obj.write(chunk) + written += len(chunk) + + if written == 0: + part_path.unlink(missing_ok=True) + raise MediaRetryableError("响应体为空") + + if encoded: + logger.warning("[download] 服务端返回压缩内容，跳过大小校验: %s", redact_url(url)) + elif total_size is None: + logger.warning( + "[download] 响应无 Content-Length，跳过大小校验: %s", redact_url(url) + ) + elif written != total_size: + raise MediaRetryableError(f"大小不符：实际 {written} 字节，期望 {total_size} 字节") + + return content_type + + @staticmethod + def _parse_content_range(response: httpx.Response) -> Optional[tuple[int, int, Optional[int]]]: + """解析 ``Content-Range: bytes 100-999/1000``，返回 (start, end, total)""" + match = _CONTENT_RANGE.search(response.headers.get("content-range", "")) + if not match: + return None + total = None if match.group(3) == "*" else int(match.group(3)) + return int(match.group(1)), int(match.group(2)), total + + @staticmethod + def _resolve_total_size( + content_range: Optional[tuple[int, int, Optional[int]]], + response: httpx.Response, + status: int, + ) -> Optional[int]: + """解析文件总大小。 + + 206 响应的 Content-Length 只是"剩余长度"，文件总长只能取自 Content-Range 的 total。 + total 为 ``*``（未知）时不能拿"剩余长度 + 已下载量"反推：服务端有权只返回请求区间的 + 一部分，反推出来的"总长"会让截断的文件恰好通过校验。 + """ + if status == 206: + return content_range[2] if content_range else None + return MediaDownloader._parse_content_length(response) + + @staticmethod + def _parse_content_length(response: httpx.Response) -> Optional[int]: + # RFC 7230：存在 Transfer-Encoding 时必须忽略 Content-Length， + # 否则一个 chunked 的完整响应会因为服务端多写的 CL 被误判为"大小不符" + if response.headers.get("transfer-encoding"): + return None + raw = response.headers.get("content-length", "") + if raw.isdigit(): + return int(raw) + return None + + def _build_headers(self, item: MediaItem) -> dict: + """拼装请求头：通用 UA + 调用方覆盖 + 单项覆盖。 + + 平台特有的反爬头（Referer 等）由平台侧通过 ``extra_headers`` 注入， + 下载器不认识任何平台名。Cookie 同理不会出现在默认头里，避免所有平台的 + 媒体请求都带上账号凭证；注意 httpx 在重定向时一律剥离 Cookie（同源跳转也会剥）， + 所以注入的 Cookie 只对首跳生效。 + """ + headers = { + "User-Agent": DEFAULT_USER_AGENT, + "Accept": "*/*", + # 必须显式声明不压缩：否则解码后字节数与 Content-Length 不等，大小校验必然误判 + "Accept-Encoding": "identity", + } + headers.update(self.extra_headers) + if item.headers: + headers.update(item.headers) + return headers + + @staticmethod + def is_supported_url(url) -> bool: + """URL 是否为可下载的 http(s) 地址。 + + 平台响应里的字段可能是 int/list/dict，``urlsplit`` 对非字符串会抛 + TypeError/AttributeError，这里必须一并挡住，否则会击穿 "download 不抛异常" 的契约。 + """ + if not isinstance(url, str): + return False + try: + parts = urlsplit(url) + except ValueError: + return False + return parts.scheme in ("http", "https") and bool(parts.netloc) diff --git a/media_downloader/ffmpeg.py b/media_downloader/ffmpeg.py new file mode 100644 index 0000000..65af8e8 --- /dev/null +++ b/media_downloader/ffmpeg.py @@ -0,0 +1,115 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_downloader/ffmpeg.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""ffmpeg 检测与音视频合流。 + +DASH 资源（如 B 站）的视频轨与音频轨是分开的，需要 ffmpeg 无损封装成 mp4。 +本机没有 ffmpeg 时调用方应降级为单流直链，不做任何隐式安装。 +""" + +from __future__ import annotations + +import asyncio +import contextlib +import functools +import logging +import shutil +from pathlib import Path + +from .types import MediaDownloadError + +logger = logging.getLogger("MediaCrawler.media_downloader.ffmpeg") + +DEFAULT_MERGE_TIMEOUT = 300.0 + + +@functools.lru_cache(maxsize=1) +def ffmpeg_path() -> str | None: + """返回本机 ffmpeg 可执行文件路径，不存在则返回 None（进程内缓存一次）""" + return shutil.which("ffmpeg") + + +def is_available() -> bool: + """本机是否具备 ffmpeg""" + return ffmpeg_path() is not None + + +async def merge_audio_video( + video_path: Path, + audio_path: Path, + output_path: Path, + timeout: float = DEFAULT_MERGE_TIMEOUT, +) -> None: + """把独立的视频轨与音频轨无损封装成 mp4。 + + 必须使用 ``asyncio.create_subprocess_exec``：爬虫的采集与评论任务跑在同一个 + 事件循环上，同步的 ``subprocess.run`` 会阻塞整个循环数十秒。 + + :raises MediaDownloadError: ffmpeg 不存在、超时或返回非 0 + """ + executable = ffmpeg_path() + if not executable: + raise MediaDownloadError("未检测到 ffmpeg，无法合流 DASH 音视频") + + process = await asyncio.create_subprocess_exec( + executable, + "-hide_banner", + "-loglevel", + "error", + "-y", + "-i", + str(video_path), + "-i", + str(audio_path), + "-c", + "copy", + str(output_path), + stdout=asyncio.subprocess.DEVNULL, + stderr=asyncio.subprocess.PIPE, + ) + + try: + _, stderr = await asyncio.wait_for(process.communicate(), timeout=timeout) + except asyncio.TimeoutError as exc: + process.kill() + await process.wait() + raise MediaDownloadError(f"ffmpeg 合流超时（{timeout}s）") from exc + except BaseException: + # 外部取消（Ctrl-C / 任务被 abort）不会走上面的超时分支， + # 这里必须显式回收子进程，否则 ffmpeg 会变成孤儿继续跑满整个编解码任务 + if process.returncode is None: + process.kill() + # 等待子进程真正退出：否则 transport 会留到事件循环关闭后才被 GC 并报错。 + # 取消状态下这个 await 可能再次被取消，吞掉即可，原始异常照常抛出 + with contextlib.suppress(BaseException): + await process.wait() + raise + + if process.returncode != 0: + detail = stderr.decode("utf-8", errors="replace")[-500:] if stderr else "" + raise MediaDownloadError(f"ffmpeg 合流失败（退出码 {process.returncode}）: {detail}") + + if not output_path.exists() or output_path.stat().st_size == 0: + raise MediaDownloadError("ffmpeg 合流产物为空") + + logger.info( + "[merge_audio_video] 合流完成: %s (%d bytes)", + output_path.name, + output_path.stat().st_size, + ) diff --git a/media_downloader/paths.py b/media_downloader/paths.py new file mode 100644 index 0000000..352ba59 --- /dev/null +++ b/media_downloader/paths.py @@ -0,0 +1,197 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_downloader/paths.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""媒体落盘路径规则：目录布局、文件名清洗、扩展名推断。 + +本模块全部为纯函数，不发起任何 IO（``ensure_within`` 会做路径解析）。 +""" + +from __future__ import annotations + +import hashlib +import re +from pathlib import Path +from typing import Optional +from urllib.parse import urlsplit + +# 允许落盘的扩展名白名单，白名单之外的推断结果一律丢弃 +ALLOWED_EXTENSIONS = frozenset( + { + ".jpg", + ".jpeg", + ".png", + ".gif", + ".webp", + ".mp4", + ".m4s", + ".m4a", + ".mp3", + ".flv", + ".mov", + } +) + +# Content-Type -> 扩展名（只取主类型，忽略 charset 等参数） +CONTENT_TYPE_EXTENSIONS = { + "image/jpeg": ".jpg", + "image/jpg": ".jpg", + "image/png": ".png", + "image/gif": ".gif", + "image/webp": ".webp", + "video/mp4": ".mp4", + "video/quicktime": ".mov", + "video/x-flv": ".flv", + "video/x-m4v": ".mp4", + "audio/mp4": ".m4a", + "audio/x-m4a": ".m4a", + "audio/mpeg": ".mp3", + "audio/aac": ".m4a", +} + +_ILLEGAL_CHARS = re.compile(r"[^A-Za-z0-9._-]") +# 从 URL 路径里提取扩展名，兼容 "img.jpg!large" 这类 CDN 后缀写法 +_URL_EXTENSION = re.compile(r"\.([A-Za-z0-9]{2,5})(?:!.*)?$") +_WINDOWS_RESERVED = frozenset( + { + "CON", + "PRN", + "AUX", + "NUL", + *(f"COM{index}" for index in range(1, 10)), + *(f"LPT{index}" for index in range(1, 10)), + } +) +MAX_COMPONENT_LENGTH = 64 +MEDIA_DIR_NAME = "media" + + +def url_fingerprint(url: str) -> str: + """URL 的短指纹，用于命名下载中的临时文件，URL 变化时自动作废旧片段""" + return hashlib.sha1(url.encode("utf-8")).hexdigest()[:8] + + +def redact_url(url) -> str: + """日志安全的 URL 表示：去掉 query（媒体直链常带签名参数）。 + + 对非字符串输入返回占位符——它常被用在异常处理路径里，自身抛异常会掩盖原始错误。 + """ + if not isinstance(url, str): + return "" + try: + parts = urlsplit(url) + except ValueError: + return "" + if not parts.scheme or not parts.netloc: + return "" + return f"{parts.scheme}://{parts.netloc}{parts.path}" + + +def sanitize_component(value: str, fallback: str = "unknown") -> str: + """清洗单个路径片段，防止路径穿越与非法字符。 + + - 只保留 ``[A-Za-z0-9._-]``，其余替换为 ``_`` + - 去掉开头的 ``.``，避免隐藏文件与 ``..`` + - 命中 Windows 保留设备名时加前缀 + - 超长截断 + """ + cleaned = _ILLEGAL_CHARS.sub("_", (value or "").strip()) + cleaned = cleaned.lstrip(".") + cleaned = cleaned.replace("..", "_") + cleaned = cleaned[:MAX_COMPONENT_LENGTH] + if not cleaned: + return fallback + if cleaned.split(".")[0].upper() in _WINDOWS_RESERVED: + cleaned = f"_{cleaned}" + return cleaned + + +def guess_extension( + url: str = "", + content_type: Optional[str] = None, + explicit: Optional[str] = None, + default: str = ".bin", +) -> str: + """推断文件扩展名，优先级：显式指定 > URL 后缀 > Content-Type > 默认值。 + + 只返回白名单内的扩展名（default 除外，由调用方保证合理）。 + """ + if explicit: + normalized = explicit if explicit.startswith(".") else f".{explicit}" + normalized = normalized.lower() + if normalized in ALLOWED_EXTENSIONS: + return normalized + + if url: + path = urlsplit(url).path + match = _URL_EXTENSION.search(path) + if match: + candidate = f".{match.group(1).lower()}" + if candidate in ALLOWED_EXTENSIONS: + return candidate + + if content_type: + main_type = content_type.split(";")[0].strip().lower() + candidate = CONTENT_TYPE_EXTENSIONS.get(main_type) + if candidate: + return candidate + + return default + + +def build_media_dir(base_dir: Path, platform: str, content_id) -> Path: + """媒体目录：``{base}/{platform}/media/{content_id}`` + + content_id 来自外部响应，可能是 int 或 None，先归一成字符串再清洗。 + """ + safe_platform = sanitize_component(platform, "unknown") + raw_content_id = "" if content_id is None else str(content_id) + safe_content_id = sanitize_component( + raw_content_id, fallback=hashlib.sha1(raw_content_id.encode("utf-8")).hexdigest()[:16] + ) + return Path(base_dir) / safe_platform / MEDIA_DIR_NAME / safe_content_id + + +def build_file_path(directory: Path, stem: str, extension: str) -> Path: + """目录内的媒体文件路径：``{directory}/{stem}{ext}``""" + safe_stem = sanitize_component(stem, "media") + normalized_ext = extension if extension.startswith(".") else f".{extension}" + return Path(directory) / f"{safe_stem}{normalized_ext.lower()}" + + +def build_media_path( + base_dir: Path, + platform: str, + content_id: str, + stem: str, + extension: str, +) -> Path: + """媒体文件最终路径：``{base}/{platform}/media/{content_id}/{stem}{ext}``""" + return build_file_path(build_media_dir(base_dir, platform, content_id), stem, extension) + + +def ensure_within(base_dir: Path, target: Path) -> Path: + """校验目标路径位于 base_dir 之内，返回解析后的绝对路径。 + + 防止外部响应里的 content_id 构造出越界路径。 + """ + base_resolved = Path(base_dir).resolve() + target_resolved = Path(target).resolve() + if not target_resolved.is_relative_to(base_resolved): + raise ValueError(f"媒体落盘路径越界: {target_resolved} 不在 {base_resolved} 之内") + return target_resolved diff --git a/media_downloader/types.py b/media_downloader/types.py new file mode 100644 index 0000000..eb491d6 --- /dev/null +++ b/media_downloader/types.py @@ -0,0 +1,81 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_downloader/types.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""媒体下载的数据结构与异常定义。 + +本模块只描述"要下载什么"，不包含任何平台细节与 IO 逻辑。 +""" + +from __future__ import annotations + +from dataclasses import dataclass +from enum import Enum +from typing import Mapping, Optional, Tuple + + +class MediaType(str, Enum): + """媒体类型""" + + IMAGE = "image" + VIDEO = "video" + + +class MediaDownloadError(Exception): + """媒体下载失败（基类）""" + + +class MediaRetryableError(MediaDownloadError): + """可重试的下载失败：网络抖动、5xx、429、响应体不完整等""" + + +class MediaFatalError(MediaDownloadError): + """不可重试的下载失败：4xx（403 多为签名过期、404 为资源不存在）""" + + +@dataclass(slots=True) +class MediaItem: + """一个待下载的媒体资源。 + + 下载器只认识这个结构，平台差异全部通过字段注入： + + :param url: 主下载地址 + :param media_type: 媒体类型（图片/视频） + :param content_id: 所属帖子 ID，决定落盘目录 + :param stem: 文件名主干（如 001 / cover / video），扩展名由下载器推断 + :param extension: 显式扩展名（含点）。为空时依次从 URL 后缀、Content-Type 推断 + :param backup_urls: 主地址重试耗尽后依次尝试的备用地址（多 CDN 场景） + :param audio_url: 非空表示 DASH 音视频分轨资源，需要 ffmpeg 合流 + :param audio_backup_urls: 音轨的备用地址 + :param headers: 针对该资源的请求头覆盖项（如 B 站 Cookie） + """ + + url: str + media_type: MediaType + content_id: str + stem: str = "media" + extension: Optional[str] = None + backup_urls: Tuple[str, ...] = () + audio_url: Optional[str] = None + audio_backup_urls: Tuple[str, ...] = () + headers: Optional[Mapping[str, str]] = None + + @property + def is_dash(self) -> bool: + """是否为 DASH 音视频分轨资源（需要下载两路流后合流）""" + return bool(self.audio_url) diff --git a/media_platform/bilibili/client.py b/media_platform/bilibili/client.py index 6d46e42..4ff7948 100644 --- a/media_platform/bilibili/client.py +++ b/media_platform/bilibili/client.py @@ -42,6 +42,7 @@ if TYPE_CHECKING: from .exception import DataFetchError from .field import CommentOrderType, SearchOrderType from .help import BilibiliSign +from .media import DASH_FNVAL def _extract_pinned_comments(value: Any) -> List[Dict]: @@ -213,11 +214,13 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): params.update({"bvid": bvid}) return await self.get(uri, params, enable_params_sign=False) - async def get_video_play_url(self, aid: int, cid: int) -> Dict: + async def get_video_play_url(self, aid: int, cid: int, fnval: int = DASH_FNVAL) -> Dict: """ Bilibli web video play url api :param aid: Video aid :param cid: cid + :param fnval: 请求的媒体格式位掩码，默认请求 DASH（含 4K/8K/HDR/AV1）； + 传 1 则请求音视频合一的 mp4 直链 :return: """ if not aid or not cid or aid <= 0 or cid <= 0: @@ -229,28 +232,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin): "cid": cid, "qn": qn_value, "fourk": 1, - "fnval": 1, + "fnval": fnval, "platform": "pc", } return await self.get(uri, params, enable_params_sign=True) - async def get_video_media(self, url: str) -> Union[bytes, None]: - # Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206) - async with make_async_client(proxy=self.proxy, follow_redirects=True) as client: - try: - response = await client.request("GET", url, timeout=self.timeout, headers=self.headers) - response.raise_for_status() - if 200 <= response.status_code < 300: - return response.content - utils.logger.error( - f"[BilibiliClient.get_video_media] Unexpected status {response.status_code} for {url}" - ) - return None - except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx - utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep original exception type name for developer debugging - return None - async def get_video_comments( self, video_id: str, diff --git a/media_platform/bilibili/core.py b/media_platform/bilibili/core.py index 14b5099..a2c4068 100644 --- a/media_platform/bilibili/core.py +++ b/media_platform/bilibili/core.py @@ -41,12 +41,14 @@ from playwright._impl._errors import TargetClosedError import config from base.base_crawler import AbstractCrawler +from media_downloader import MediaDownloader, is_ffmpeg_available from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import bilibili as bilibili_store from tools import utils from tools.cdp_browser import CDPBrowserManager from var import crawler_type_var, source_keyword_var +from . import media as bili_media from .client import BilibiliClient from .exception import DataFetchError from .field import SearchOrderType @@ -66,6 +68,7 @@ class BilibiliCrawler(AbstractCrawler): self.user_agent = utils.get_user_agent() self.cdp_manager = None self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh + self._media_downloader: Optional[MediaDownloader] = None async def start(self): playwright_proxy_format, httpx_proxy_format = None, None @@ -228,7 +231,7 @@ class BilibiliCrawler(AbstractCrawler): video_id_list.append(video_item.get("View").get("aid")) await bilibili_store.update_bilibili_video(video_item) await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) + await self.download_media(video_item, semaphore) page += 1 # Sleep after page navigation @@ -308,7 +311,7 @@ class BilibiliCrawler(AbstractCrawler): video_id_list.append(video_item.get("View").get("aid")) await bilibili_store.update_bilibili_video(video_item) await bilibili_store.update_up_info(video_item) - await self.get_bilibili_video(video_item, semaphore) + await self.download_media(video_item, semaphore) page += 1 @@ -413,7 +416,7 @@ class BilibiliCrawler(AbstractCrawler): video_aids_list.append(video_aid) await bilibili_store.update_bilibili_video(video_detail) await bilibili_store.update_up_info(video_detail) - await self.get_bilibili_video(video_detail, semaphore) + await self.download_media(video_detail, semaphore) await self.batch_get_video_comments(video_aids_list) async def get_video_info_task(self, aid: int, bvid: str, semaphore: asyncio.Semaphore) -> Optional[Dict]: @@ -440,17 +443,24 @@ class BilibiliCrawler(AbstractCrawler): utils.logger.error(f"[BilibiliCrawler.get_video_info_task] have not fund note detail video_id:{bvid}, err: {ex}") return None - async def get_video_play_url_task(self, aid: int, cid: int, semaphore: asyncio.Semaphore) -> Union[Dict, None]: + async def get_video_play_url_task( + self, + aid: int, + cid: int, + semaphore: asyncio.Semaphore, + fnval: int = bili_media.DASH_FNVAL, + ) -> Union[Dict, None]: """ Get video play url :param aid: :param cid: :param semaphore: + :param fnval: 媒体格式位掩码，默认请求 DASH 分轨流 :return: """ async with semaphore: try: - result = await self.bili_client.get_video_play_url(aid=aid, cid=cid) + result = await self.bili_client.get_video_play_url(aid=aid, cid=cid, fnval=fnval) return result except DataFetchError as ex: utils.logger.error(f"[BilibiliCrawler.get_video_play_url_task] Get video play url error: {ex}") @@ -570,42 +580,128 @@ class BilibiliCrawler(AbstractCrawler): except Exception as e: utils.logger.error(f"[BilibiliCrawler.close] An error occurred during close: {e}") - async def get_bilibili_video(self, video_item: Dict, semaphore: asyncio.Semaphore): + async def download_media(self, video_item: Dict, semaphore: asyncio.Semaphore): + """下载 B 站视频与封面。 + + 优先走 DASH（音视频分轨 + ffmpeg 合流，画质最好）；本机没有 ffmpeg + 或 DASH 流不可用时，降级为 mp4 直链（音视频合一，清晰度受接口限制）。 + + :param video_item: 视频详情，需包含 View 字段（含 aid/cid/pic/bvid） + :param semaphore: 用于限制 playurl 接口请求并发 """ - download bilibili video - :param video_item: - :param semaphore: - :return: - """ - if not config.ENABLE_GET_MEIDAS: - utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Crawling image mode is not enabled") + if not config.ENABLE_GET_MEDIA: return - video_item_view: Dict = video_item.get("View") + try: + await self._download_media(video_item, semaphore) + except Exception as exc: + # 媒体下载是旁路能力：playurl 是额外的接口调用，网络抖动不能中断爬取主流程 + utils.logger.error(f"[BilibiliCrawler.download_media] 媒体下载异常: {exc}") + + async def _download_media(self, video_item: Dict, semaphore: asyncio.Semaphore) -> None: + video_item_view: Dict = video_item.get("View") or {} aid = video_item_view.get("aid") cid = video_item_view.get("cid") - result = await self.get_video_play_url_task(aid, cid, semaphore) - if result is None: - utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video play url failed") - return - durl_list = result.get("durl") - max_size = -1 - video_url = "" - for durl in durl_list: - size = durl.get("size") - if size > max_size: - max_size = size - video_url = durl.get("url") - if video_url == "": - utils.logger.info("[BilibiliCrawler.get_bilibili_video] get video url failed") + content_id = video_item_view.get("bvid") or (str(aid) if aid else "") + if not content_id: + utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/bvid，跳过媒体下载") return - content = await self.bili_client.get_video_media(video_url) - await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC) - utils.logger.info(f"[BilibiliCrawler.get_bilibili_video] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching video {aid}") - if content is None: + downloader = self._get_media_downloader() + + cover_item = bili_media.build_cover_item(video_item_view, content_id) + if cover_item is not None: + await downloader.download(cover_item) + + if not aid or not cid: + utils.logger.warning("[BilibiliCrawler.download_media] 缺少 aid/cid，无法获取播放地址") return - extension_file_name = f"video.mp4" - await bilibili_store.store_video(aid, content, extension_file_name) + + # ffmpeg 是否可用在请求之前就是确定的（进程内缓存），据此一次性决定 fnval： + # 没装 ffmpeg 却先按 DASH 请求一次，既多打一次风控最敏感的 playurl 接口， + # 也拿不到可用的 durl + dash_supported = is_ffmpeg_available() + play_fnval = bili_media.DASH_FNVAL if dash_supported else bili_media.MP4_FNVAL + play_info = await self.get_video_play_url_task(aid, cid, semaphore, fnval=play_fnval) + await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC) + if play_info is None: + utils.logger.error("[BilibiliCrawler.download_media] 获取播放地址失败") + return + + if dash_supported: + # 逐档降级：未登录或权限不足时 B 站会返回高清晰度 URL，但实际取流被 CDN 403， + # 此时退到更低的可用档位往往能成功，比直接放弃（或退回更低质的直链）更好 + quality: Optional[int] = getattr(config, "BILI_QN", 80) + while quality is not None: + dash_item = bili_media.build_dash_item(play_info, content_id, preferred_quality=quality) + if dash_item is None: + break + if await downloader.download(dash_item) is not None: + return + lower_quality = bili_media.next_lower_quality(play_info, quality) + if lower_quality is None: + break + utils.logger.warning( + f"[BilibiliCrawler.download_media] 清晰度 {quality} 取流失败，降级到 {lower_quality}" + ) + quality = lower_quality + + utils.logger.warning("[BilibiliCrawler.download_media] DASH 路径失败，降级为 mp4 直链") + else: + utils.logger.info("[BilibiliCrawler.download_media] 未检测到 ffmpeg，使用 mp4 直链下载（低清晰度）") + + durl_play_info = play_info + durl_item = bili_media.build_durl_item(durl_play_info, content_id) + if durl_item is None: + # DASH 请求不会返回 durl，需要再按 mp4 格式请求一次 + durl_play_info = await self.get_video_play_url_task( + aid, cid, semaphore, fnval=bili_media.MP4_FNVAL + ) or {} + await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC) + durl_item = bili_media.build_durl_item(durl_play_info, content_id) + + if durl_item is None: + utils.logger.error( + f"[BilibiliCrawler.download_media] 未能获取到视频直链 aid={aid} cid={cid}" + ) + return + + segment_count = bili_media.count_durl_segments(durl_play_info) + if segment_count > 1: + utils.logger.warning( + f"[BilibiliCrawler.download_media] 该视频直链被切成 {segment_count} 段，" + f"当前仅下载体积最大的一段，文件可能不完整；" + f"确认本机 ffmpeg 可用后走 DASH 路径可获取完整视频" + ) + + await downloader.download(durl_item) + + def _get_media_downloader(self) -> MediaDownloader: + """惰性创建媒体下载器，并同步最新的代理与 Cookie（两者都会就地刷新）""" + if self._media_downloader is None: + self._media_downloader = MediaDownloader( + platform="bili", + proxy=getattr(self.bili_client, "proxy", None), + extra_headers=self._media_headers(), + ) + else: + self._media_downloader.update_credentials( + proxy=getattr(self.bili_client, "proxy", None), + extra_headers=self._media_headers(), + ) + return self._media_downloader + + def _media_headers(self) -> Dict: + """媒体请求头：平台 Referer（防盗链）+ UA。 + + B 站清晰度由 playurl 接口依据 Cookie 决定，返回的直链里已固化流版本， + 且带 deadline/upsig 签名，下载本身不需要 Cookie（已实测无 Cookie 可直接取流）， + 因此不透传 Cookie，避免把账号凭证扩散到 CDN 域名。 + """ + headers = {"Referer": "https://www.bilibili.com/"} + client_headers = getattr(self.bili_client, "headers", {}) or {} + if client_headers.get("User-Agent"): + headers["User-Agent"] = client_headers["User-Agent"] + return headers async def get_all_creator_details(self, creator_url_list: List[str]): """ diff --git a/media_platform/bilibili/media.py b/media_platform/bilibili/media.py new file mode 100644 index 0000000..5ddc694 --- /dev/null +++ b/media_platform/bilibili/media.py @@ -0,0 +1,202 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/bilibili/media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""B 站媒体地址提取与流选择。 + +B 站的播放地址需要先调 playurl 接口获取： +- ``fnval=DASH_FNVAL`` 返回音视频分轨的 DASH 流，画质最好，但需要 ffmpeg 合流 +- ``fnval=MP4_FNVAL`` 返回音视频合一的 mp4 直链，无需外部依赖，但清晰度受限 + +本模块只做纯计算，接口调用与降级编排由 core 负责。 +""" + +from __future__ import annotations + +from typing import Dict, List, Optional + +import config +from media_downloader import MediaItem, MediaType + +# fnval 位掩码：16(DASH) | 64(HDR) | 128(4K) | 256(Dolby Audio) | 512(Dolby Vision) | 1024(8K) | 2048(AV1) +DASH_FNVAL = 4048 +# 请求音视频合一的 mp4 直链 +MP4_FNVAL = 1 + +# B 站 codecid：7=AVC(H.264) 12=HEVC(H.265) 13=AV1 14=AV2 +# 同一清晰度下优先 AVC：下载后的文件要进剪辑软件/播放器，H.264 的兼容性最好 +_CODEC_PRIORITY = {7: 0, 13: 1, 12: 2} +_UNKNOWN_CODEC_PRIORITY = 3 + + +def _as_dict(value) -> Dict: + """接口响应里同名字段可能是字符串或 None，统一收敛为 dict""" + return value if isinstance(value, dict) else {} + + +def _stream_url(stream) -> str: + if not isinstance(stream, dict): + return "" + return stream.get("base_url") or stream.get("baseUrl") or "" + + +def _to_int(value, default: int = 0) -> int: + """接口偶尔把 id/bandwidth 返回成字符串，排序与比较前统一转 int""" + try: + return int(value) + except (TypeError, ValueError): + return default + + +def _stream_sort_key(stream: Dict) -> tuple: + """同清晰度下把兼容性最好的编码排在后面（取 [-1] 即选中它）""" + codec_rank = _CODEC_PRIORITY.get(_to_int(stream.get("codecid"), -1), _UNKNOWN_CODEC_PRIORITY) + return (_to_int(stream.get("id")), -codec_rank, _to_int(stream.get("bandwidth"))) + + +def _stream_backup_urls(stream) -> tuple: + if not isinstance(stream, dict): + return () + backups = stream.get("backup_url") or stream.get("backupUrl") or [] + return tuple(url for url in backups if isinstance(url, str) and url) + + +def build_cover_item(view: Dict, content_id: str) -> Optional[MediaItem]: + """封面地址来自稿件详情里的 pic 字段""" + cover_url = _as_dict(view).get("pic") + if not isinstance(cover_url, str) or not cover_url: + return None + return MediaItem( + url=cover_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem="cover", + ) + + +def pick_video_stream(play_info: Dict, preferred_quality: Optional[int] = None) -> Optional[Dict]: + """挑选视频流：优先不超过用户清晰度设置的最高档，都超出时取最低档。 + + 同一清晰度可能同时提供 AVC/HEVC/AV1 多种编码，按 ``_CODEC_PRIORITY`` + 取兼容性最好的那一种（排序 key 让优先级高的排在末尾）。 + """ + if preferred_quality is None: + preferred_quality = getattr(config, "BILI_QN", 80) + + dash = _as_dict(play_info).get("dash") + if not isinstance(dash, dict): + return None + streams = [stream for stream in (dash.get("video") or []) if _stream_url(stream)] + if not streams: + return None + + streams.sort(key=_stream_sort_key) + within_quality = [stream for stream in streams if _to_int(stream.get("id")) <= preferred_quality] + if within_quality: + return within_quality[-1] + + # 所有档位都超过用户设置时取最低清晰度，但编码优先级仍要生效： + # 排序把每个 id 档内兼容性最好的编码放在该档最后，所以要在同 id 组里取最后一个 + lowest_id = _to_int(streams[0].get("id")) + same_quality = [stream for stream in streams if _to_int(stream.get("id")) == lowest_id] + return same_quality[-1] + + +def pick_audio_stream(play_info: Dict) -> Optional[Dict]: + """音频流取码率最高的一档（id 越大码率越高）""" + dash = _as_dict(play_info).get("dash") + if not isinstance(dash, dict): + return None + streams = [stream for stream in (dash.get("audio") or []) if _stream_url(stream)] + if not streams: + return None + return max(streams, key=lambda stream: (_to_int(stream.get("id")), _to_int(stream.get("bandwidth")))) + + +def build_dash_item(play_info: Dict, content_id: str, preferred_quality: Optional[int] = None) -> Optional[MediaItem]: + """构建 DASH 下载任务（视频轨 + 音频轨，交由下载器用 ffmpeg 合流）""" + video_stream = pick_video_stream(play_info, preferred_quality) + audio_stream = pick_audio_stream(play_info) + if video_stream is None or audio_stream is None: + return None + + return MediaItem( + url=_stream_url(video_stream), + backup_urls=_stream_backup_urls(video_stream), + audio_url=_stream_url(audio_stream), + audio_backup_urls=_stream_backup_urls(audio_stream), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video", + ) + + +def available_qualities(play_info: Dict) -> List[int]: + """响应中实际可用的视频清晰度档位（从高到低）""" + dash = _as_dict(play_info).get("dash") + if not isinstance(dash, dict): + return [] + qualities = { + _to_int(stream.get("id")) + for stream in (dash.get("video") or []) + if _stream_url(stream) + } + return sorted(qualities, reverse=True) + + +def next_lower_quality(play_info: Dict, current_quality: int) -> Optional[int]: + """比 ``current_quality`` 更低的下一个可用档位，没有则返回 None。 + + 用于"playurl 给了高清地址但 CDN 拒绝下载"时逐档降级 + （未登录或权限不足时 B 站会返回高清晰度 URL，但实际取流会被 403）。 + """ + lower = [quality for quality in available_qualities(play_info) if quality < current_quality] + return lower[0] if lower else None + + +def count_durl_segments(play_info: Dict) -> int: + """durl 的分段数量。大于 1 表示整片被切分，只下最大一段会得到不完整的文件。""" + return len( + [ + entry + for entry in _as_dict(play_info).get("durl") or [] + if isinstance(entry, dict) and entry.get("url") + ] + ) + + +def build_durl_item(play_info: Dict, content_id: str) -> Optional[MediaItem]: + """构建 mp4 直链下载任务（取体积最大的一段）。 + + stem 刻意与 DASH 产物（``video``）区分：直链画质受接口限制，是降级产物。 + 两者同名会让先下到的低清文件被"已存在即跳过"永久信任， + 之后即使装好 ffmpeg 也再拿不到高清。 + """ + durl_list: List[Dict] = _as_dict(play_info).get("durl") or [] + candidates = [entry for entry in durl_list if isinstance(entry, dict) and entry.get("url")] + if not candidates: + return None + + best = max(candidates, key=lambda entry: _to_int(entry.get("size"))) + return MediaItem( + url=best["url"], + backup_urls=tuple(best.get("backup_url") or ()), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video-durl", + ) diff --git a/media_platform/douyin/client.py b/media_platform/douyin/client.py index 3310cdb..759920e 100644 --- a/media_platform/douyin/client.py +++ b/media_platform/douyin/client.py @@ -344,20 +344,6 @@ class DouYinClient(AbstractApiClient, ProxyRefreshMixin): result.extend(aweme_list) return result - async def get_aweme_media(self, url: str) -> Union[bytes, None]: - async with make_async_client(proxy=self.proxy) as client: - try: - response = await client.request("GET", url, timeout=self.timeout, follow_redirects=True) - response.raise_for_status() - if not response.reason_phrase == "OK": - utils.logger.error(f"[DouYinClient.get_aweme_media] request {url} err, res:{response.text}") - return None - else: - return response.content - except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx - utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep the original exception type name for developers to debug - return None - async def resolve_short_url(self, short_url: str) -> str: """ 解析抖音短链接,获取重定向后的真实URL diff --git a/media_platform/douyin/core.py b/media_platform/douyin/core.py index c0e9372..10380a6 100644 --- a/media_platform/douyin/core.py +++ b/media_platform/douyin/core.py @@ -33,12 +33,14 @@ from playwright.async_api import ( import config from base.base_crawler import AbstractCrawler +from media_downloader import MediaDownloader from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import douyin as douyin_store from tools import utils from tools.cdp_browser import CDPBrowserManager from var import crawler_type_var, source_keyword_var +from . import media as douyin_media from .client import DouYinClient from .exception import DataFetchError from .field import PublishTimeType @@ -63,6 +65,7 @@ class DouYinCrawler(AbstractCrawler): ] self.cdp_manager = None self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh + self._media_downloader: Optional[MediaDownloader] = None async def start(self) -> None: playwright_proxy_format, httpx_proxy_format = None, None @@ -170,7 +173,7 @@ class DouYinCrawler(AbstractCrawler): aweme_list.append(aweme_info.get("aweme_id", "")) page_aweme_list.append(aweme_info.get("aweme_id", "")) await douyin_store.update_douyin_aweme(aweme_item=aweme_info) - await self.get_aweme_media(aweme_item=aweme_info) + await self.download_media(aweme_item=aweme_info) # Batch get note comments for the current page await self.batch_get_note_comments(page_aweme_list) @@ -212,7 +215,7 @@ class DouYinCrawler(AbstractCrawler): for aweme_detail in aweme_details: if aweme_detail is not None: await douyin_store.update_douyin_aweme(aweme_item=aweme_detail) - await self.get_aweme_media(aweme_item=aweme_detail) + await self.download_media(aweme_item=aweme_detail) await self.batch_get_note_comments(aweme_id_list) async def get_aweme_detail(self, aweme_id: str, semaphore: asyncio.Semaphore) -> Any: @@ -304,7 +307,7 @@ class DouYinCrawler(AbstractCrawler): for aweme_item in note_details: if aweme_item is not None: await douyin_store.update_douyin_aweme(aweme_item=aweme_item) - await self.get_aweme_media(aweme_item=aweme_item) + await self.download_media(aweme_item=aweme_item) async def create_douyin_client(self, httpx_proxy: Optional[str]) -> DouYinClient: """Create douyin client""" @@ -399,72 +402,44 @@ class DouYinCrawler(AbstractCrawler): await self.browser_context.close() utils.logger.info("[DouYinCrawler.close] Browser context closed ...") - async def get_aweme_media(self, aweme_item: Dict): - """ - 获取抖音媒体，自动判断媒体类型是短视频还是帖子图片并下载 + async def download_media(self, aweme_item: Dict) -> None: + """下载抖音作品的媒体资源（图集图片，或视频 + 封面） Args: aweme_item (Dict): 抖音作品详情 """ - if not config.ENABLE_GET_MEIDAS: - utils.logger.info(f"[DouYinCrawler.get_aweme_media] Crawling image mode is not enabled") + if not config.ENABLE_GET_MEDIA: return - # List of note urls. If it is a short video type, an empty list will be returned. - note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item) - # The video URL will always exist, but when it is a short video type, the file is actually an audio file. - video_download_url: str = douyin_store._extract_video_download_url(aweme_item) - # TODO: Douyin does not adopt the audio and video separation strategy, so the audio can be separated from the original video and will not be extracted for the time being. - if note_download_url: - await self.get_aweme_images(aweme_item) + try: + items = douyin_media.build_media_items(aweme_item) + if items: + await self._get_media_downloader().download_all(items) + except Exception as exc: + # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程 + utils.logger.error(f"[DouYinCrawler.download_media] 媒体下载异常: {exc}") + + def _get_media_downloader(self) -> MediaDownloader: + """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）""" + if self._media_downloader is None: + self._media_downloader = MediaDownloader( + platform="dy", + proxy=getattr(self.dy_client, "proxy", None), + extra_headers=self._media_headers(), + ) else: - await self.get_aweme_video(aweme_item) + self._media_downloader.update_credentials( + proxy=getattr(self.dy_client, "proxy", None), + extra_headers=self._media_headers(), + ) + return self._media_downloader - async def get_aweme_images(self, aweme_item: Dict): + def _media_headers(self) -> Dict: + """媒体请求头：平台 Referer（防盗链）+ UA。 + + client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。 """ - get aweme images. please use get_aweme_media - - Args: - aweme_item (Dict): 抖音作品详情 - """ - if not config.ENABLE_GET_MEIDAS: - return - aweme_id = aweme_item.get("aweme_id") - # List of note urls. If it is a short video type, an empty list will be returned. - note_download_url: List[str] = douyin_store._extract_note_image_list(aweme_item) - - if not note_download_url: - return - picNum = 0 - for url in note_download_url: - if not url: - continue - content = await self.dy_client.get_aweme_media(url) - await asyncio.sleep(random.random()) - if content is None: - continue - extension_file_name = f"{picNum:>03d}.jpeg" - picNum += 1 - await douyin_store.update_dy_aweme_image(aweme_id, content, extension_file_name) - - async def get_aweme_video(self, aweme_item: Dict): - """ - get aweme videos. please use get_aweme_media - - Args: - aweme_item (Dict): 抖音作品详情 - """ - if not config.ENABLE_GET_MEIDAS: - return - aweme_id = aweme_item.get("aweme_id") - - # The video URL will always exist, but when it is a short video type, the file is actually an audio file. - video_download_url: str = douyin_store._extract_video_download_url(aweme_item) - - if not video_download_url: - return - content = await self.dy_client.get_aweme_media(video_download_url) - await asyncio.sleep(random.random()) - if content is None: - return - extension_file_name = f"video.mp4" - await douyin_store.update_dy_aweme_video(aweme_id, content, extension_file_name) + headers = {"Referer": "https://www.douyin.com/"} + client_headers = getattr(self.dy_client, "headers", {}) or {} + if client_headers.get("User-Agent"): + headers["User-Agent"] = client_headers["User-Agent"] + return headers diff --git a/media_platform/douyin/media.py b/media_platform/douyin/media.py new file mode 100644 index 0000000..0038d67 --- /dev/null +++ b/media_platform/douyin/media.py @@ -0,0 +1,144 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/douyin/media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""抖音媒体地址提取。 + +输入是抖音作品详情（``/aweme/v1/web/aweme/detail/`` 的响应）， +输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。 +""" + +from __future__ import annotations + +from typing import Dict, List + +from media_downloader import MediaItem, MediaType + +# 视频清晰度字段，按优先级排列（h264 与 256 通常为无水印源） +_VIDEO_ADDR_KEYS = ("play_addr_h264", "play_addr_256", "play_addr") + +# 封面字段，按优先级排列 +_COVER_KEYS = ("raw_cover", "origin_cover", "cover", "dynamic_cover") + + +def _url_list_of(container) -> List[str]: + """接口响应里同名字段可能是字符串或 None，统一收敛为 url 列表""" + if not isinstance(container, dict): + return [] + return [url for url in (container.get("url_list") or []) if url and isinstance(url, str)] + + +def extract_image_urls(aweme_detail: Dict) -> List[str]: + """提取图集作品的图片地址（url_list 的最后一个通常是无水印原图）""" + urls: List[str] = [] + for image in aweme_detail.get("images") or []: + candidates = _url_list_of(image) + if candidates: + urls.append(candidates[-1]) + return urls + + +def extract_cover_url(aweme_detail: Dict) -> str: + """提取视频封面地址""" + video_item = aweme_detail.get("video") + if not isinstance(video_item, dict): + return "" + for key in _COVER_KEYS: + candidates = _url_list_of(video_item.get(key)) + if candidates: + return candidates[-1] + return "" + + +def extract_video_urls(aweme_detail: Dict) -> List[str]: + """提取视频地址候选列表。 + + 同一档清晰度会返回多个 CDN 地址，优先取 url_list 最后一个（通常无水印）， + 其余地址作为备用；主地址全部不可用时依次降级到更低清晰度档位。 + """ + video_item = aweme_detail.get("video") + if not isinstance(video_item, dict): + return [] + + for key in _VIDEO_ADDR_KEYS: + candidates = _url_list_of(video_item.get(key)) + if candidates: + return list(reversed(candidates)) + + # 兜底：bit_rate 列表里码率最高的那一档 + bit_rates = [entry for entry in (video_item.get("bit_rate") or []) if isinstance(entry, dict)] + for entry in sorted(bit_rates, key=lambda item: item.get("bit_rate", 0), reverse=True): + candidates = _url_list_of(entry.get("play_addr")) + if candidates: + return list(reversed(candidates)) + + return [] + + +def build_media_items(aweme_item: Dict) -> List[MediaItem]: + """把一个抖音作品转换成待下载的媒体任务列表。 + + - 图集作品：各张图片（第一张即封面，不再单独下载一份 cover） + - 视频作品：封面 + 视频 + """ + if not isinstance(aweme_item, dict): + return [] + + content_id = aweme_item.get("aweme_id") or "" + if not content_id: + return [] + + items: List[MediaItem] = [] + + image_urls = extract_image_urls(aweme_item) + if image_urls: + for index, image_url in enumerate(image_urls, start=1): + items.append( + MediaItem( + url=image_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem=f"{index:03d}", + ) + ) + return items + + cover_url = extract_cover_url(aweme_item) + if cover_url: + items.append( + MediaItem( + url=cover_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem="cover", + ) + ) + + video_urls = extract_video_urls(aweme_item) + if video_urls: + items.append( + MediaItem( + url=video_urls[0], + backup_urls=tuple(video_urls[1:]), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video", + ) + ) + + return items diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index e56a5d0..c904e18 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -35,6 +35,7 @@ from playwright.async_api import ( import config from base.base_crawler import AbstractCrawler +from media_downloader import MediaDownloader from model.m_kuaishou import VideoUrlInfo, CreatorUrlInfo from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import kuaishou as kuaishou_store @@ -42,6 +43,7 @@ from tools import utils from tools.cdp_browser import CDPBrowserManager from var import comment_tasks_var, crawler_type_var, source_keyword_var +from . import media as kuaishou_media from .client import KuaiShouClient from .exception import DataFetchError from .help import ( @@ -64,6 +66,7 @@ class KuaishouCrawler(AbstractCrawler): self.user_agent = utils.get_user_agent() self.cdp_manager = None self.ip_proxy_pool = None # Proxy IP pool, used for automatic proxy refresh + self._media_downloader: Optional[MediaDownloader] = None async def start(self): playwright_proxy_format, httpx_proxy_format = None, None @@ -177,6 +180,7 @@ class KuaishouCrawler(AbstractCrawler): for video_detail in videos_res.get("feeds", []): video_id_list.append(video_detail.get("photo", {}).get("id")) await kuaishou_store.update_kuaishou_video(video_item=video_detail) + await self.download_media(video_item=video_detail) utils.logger.info( f"[KuaishouCrawler.search] keyword: {keyword}, page: {page}, got {len(video_id_list)} videos" @@ -215,6 +219,7 @@ class KuaishouCrawler(AbstractCrawler): for video_detail in video_details: if video_detail is not None: await kuaishou_store.update_kuaishou_video(video_detail) + await self.download_media(video_item=video_detail) await self.batch_get_video_comments(video_ids) async def get_video_info_task( @@ -464,6 +469,49 @@ class KuaishouCrawler(AbstractCrawler): for video_detail in video_details: if video_detail is not None: await kuaishou_store.update_kuaishou_video(video_detail) + await self.download_media(video_item=video_detail) + + async def download_media(self, video_item: Dict): + """下载快手作品的媒体资源（封面 + 视频） + + Args: + video_item (Dict): 快手作品详情（结构为 {"photo": {...}, "author": {...}}） + """ + if not config.ENABLE_GET_MEDIA: + return + try: + items = kuaishou_media.build_media_items(video_item) + if items: + await self._get_media_downloader().download_all(items) + except Exception as exc: + # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程 + utils.logger.error(f"[KuaishouCrawler.download_media] 媒体下载异常: {exc}") + + def _get_media_downloader(self) -> MediaDownloader: + """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）""" + if self._media_downloader is None: + self._media_downloader = MediaDownloader( + platform="ks", + proxy=getattr(self.ks_client, "proxy", None), + extra_headers=self._media_headers(), + ) + else: + self._media_downloader.update_credentials( + proxy=getattr(self.ks_client, "proxy", None), + extra_headers=self._media_headers(), + ) + return self._media_downloader + + def _media_headers(self) -> Dict: + """媒体请求头：平台 Referer（防盗链）+ UA。 + + client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。 + """ + headers = {"Referer": "https://www.kuaishou.com/"} + client_headers = getattr(self.ks_client, "headers", {}) or {} + if client_headers.get("User-Agent"): + headers["User-Agent"] = client_headers["User-Agent"] + return headers async def close(self): """Close browser context""" diff --git a/media_platform/kuaishou/media.py b/media_platform/kuaishou/media.py new file mode 100644 index 0000000..76c2775 --- /dev/null +++ b/media_platform/kuaishou/media.py @@ -0,0 +1,137 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/kuaishou/media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""快手媒体地址提取。 + +输入是快手作品详情（``{"photo": {...}, "author": {...}}``）， +输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。 +""" + +from __future__ import annotations + +from typing import Dict, List + +from media_downloader import MediaItem, MediaType + + +def _first_url(urls) -> str: + """从 url 列表里取第一个非空字符串""" + if isinstance(urls, str): + return urls + for url in urls or []: + if isinstance(url, str) and url: + return url + return "" + + +def _photo_of(video_item: Dict) -> Dict: + """接口响应里 photo 字段可能是字符串或 None，统一收敛为 dict""" + photo = video_item.get("photo") + return photo if isinstance(photo, dict) else {} + + +def _extract_representation_url(codec_resource) -> str: + """从 videoResource.{hevc,h264} 的 adaptationSet 里取第一个播放地址""" + if not isinstance(codec_resource, dict): + return "" + for adaptation in codec_resource.get("adaptationSet") or []: + if not isinstance(adaptation, dict): + continue + for representation in adaptation.get("representation") or []: + if isinstance(representation, dict) and representation.get("url"): + return representation["url"] + return "" + + +def extract_video_urls(video_item: Dict) -> List[str]: + """提取视频地址候选列表（H265 优先，画质与体积更优）""" + photo = _photo_of(video_item) + if not photo: + return [] + + video_resource = photo.get("videoResource") if isinstance(photo.get("videoResource"), dict) else {} + candidates = [ + photo.get("photoH265Url"), + photo.get("photoUrl"), + _extract_representation_url(video_resource.get("hevc")), + _extract_representation_url(video_resource.get("h264")), + # 详情接口另有 manifest.adaptationSet[].representation[].url 一份等价描述， + # search 接口的 photo 对象字段更少时靠它兜底 + _extract_representation_url(photo.get("manifest")), + ] + + urls: List[str] = [] + for candidate in candidates: + if candidate and isinstance(candidate, str) and candidate not in urls: + urls.append(candidate) + return urls + + +def extract_cover_url(video_item: Dict) -> str: + """提取视频封面地址""" + photo = _photo_of(video_item) + if not photo: + return "" + + for candidate in ( + photo.get("coverUrl"), + _first_url([entry.get("url") for entry in photo.get("coverUrls") or [] if isinstance(entry, dict)]), + photo.get("animatedCoverUrl"), + ): + if isinstance(candidate, str) and candidate: + return candidate + return "" + + +def build_media_items(video_item: Dict) -> List[MediaItem]: + """把一个快手作品转换成待下载的媒体任务列表：封面 + 视频""" + if not isinstance(video_item, dict): + return [] + + content_id = _photo_of(video_item).get("id") + if not content_id: + return [] + + content_id = str(content_id) + items: List[MediaItem] = [] + + cover_url = extract_cover_url(video_item) + if cover_url: + items.append( + MediaItem( + url=cover_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem="cover", + ) + ) + + video_urls = extract_video_urls(video_item) + if video_urls: + items.append( + MediaItem( + url=video_urls[0], + backup_urls=tuple(video_urls[1:]), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video", + ) + ) + + return items diff --git a/media_platform/weibo/client.py b/media_platform/weibo/client.py index 21f8d1c..08c1742 100644 --- a/media_platform/weibo/client.py +++ b/media_platform/weibo/client.py @@ -65,7 +65,6 @@ class WeiboClient(ProxyRefreshMixin): self.cookie_urls = [self._host] self.playwright_page = playwright_page self.cookie_dict = cookie_dict - self._image_agent_host = "https://i1.wp.com/" # Initialize proxy pool (from ProxyRefreshMixin) self.init_proxy_pool(proxy_ip_pool) @@ -277,33 +276,6 @@ class WeiboClient(ProxyRefreshMixin): utils.logger.info(f"[WeiboClient.get_note_info_by_id] $render_data value not found") return dict() - async def get_note_image(self, image_url: str) -> bytes: - image_url = image_url[8:] # Remove https:// - sub_url = image_url.split("/") - image_url = "" - for i in range(len(sub_url)): - if i == 1: - image_url += "large/" # Get high-resolution images - elif i == len(sub_url) - 1: - image_url += sub_url[i] - else: - image_url += sub_url[i] + "/" - # Weibo image hosting has anti-hotlinking, so proxy access is needed - # Since Weibo images are accessed through i1.wp.com, we need to concatenate the URL - final_uri = (f"{self._image_agent_host}" - f"{image_url}") - async with make_async_client(proxy=self.proxy) as client: - try: - response = await client.request("GET", final_uri, timeout=self.timeout) - response.raise_for_status() - if not response.reason_phrase == "OK": - utils.logger.error(f"[WeiboClient.get_note_image] request {final_uri} err, res:{response.text}") - return None - else: - return response.content - except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx - utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep original exception type name for developer debugging - return None async def get_creator_container_info(self, creator_id: str) -> Dict: """ diff --git a/media_platform/weibo/core.py b/media_platform/weibo/core.py index 4f34da5..6a35056 100644 --- a/media_platform/weibo/core.py +++ b/media_platform/weibo/core.py @@ -38,12 +38,14 @@ from playwright.async_api import ( import config from base.base_crawler import AbstractCrawler +from media_downloader import MediaDownloader from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import weibo as weibo_store from tools import utils from tools.cdp_browser import CDPBrowserManager from var import crawler_type_var, source_keyword_var +from . import media as weibo_media from .client import WeiboClient from .exception import DataFetchError from .field import SearchType @@ -65,6 +67,7 @@ class WeiboCrawler(AbstractCrawler): self.mobile_user_agent = utils.get_mobile_user_agent() self.cdp_manager = None self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh + self._media_downloader: Optional[MediaDownloader] = None async def start(self): playwright_proxy_format, httpx_proxy_format = None, None @@ -179,7 +182,7 @@ class WeiboCrawler(AbstractCrawler): if mblog: note_id_list.append(mblog.get("id")) await weibo_store.update_weibo_note(note_item) - await self.get_note_images(mblog) + await self.download_media(mblog) page += 1 @@ -200,6 +203,9 @@ class WeiboCrawler(AbstractCrawler): for note_item in video_details: if note_item: await weibo_store.update_weibo_note(note_item) + mblog = note_item.get("mblog") + if mblog: + await self.download_media(mblog) await self.batch_get_notes_comments(config.WEIBO_SPECIFIED_ID_LIST) async def get_note_info_task(self, note_id: str, semaphore: asyncio.Semaphore) -> Optional[Dict]: @@ -269,36 +275,46 @@ class WeiboCrawler(AbstractCrawler): except Exception as e: utils.logger.error(f"[WeiboCrawler.get_note_comments] may be been blocked, err:{e}") - async def get_note_images(self, mblog: Dict): - """ - get note images - :param mblog: - :return: - """ - if not config.ENABLE_GET_MEIDAS: - utils.logger.info(f"[WeiboCrawler.get_note_images] Crawling image mode is not enabled") - return + async def download_media(self, mblog: Dict) -> None: + """下载微博的媒体资源（配图，或视频 + 封面） - pics: List = mblog.get("pics") - if not pics: + :param mblog: 微博正文数据 + """ + if not config.ENABLE_GET_MEDIA: return - for pic in pics: - if isinstance(pic, str): - url = pic - pid = url.split("/")[-1].split(".")[0] - elif isinstance(pic, dict): - url = pic.get("url") - pid = pic.get("pid", "") - else: - continue - if not url: - continue - content = await self.wb_client.get_note_image(url) - await asyncio.sleep(config.CRAWLER_MAX_SLEEP_SEC) - utils.logger.info(f"[WeiboCrawler.get_note_images] Sleeping for {config.CRAWLER_MAX_SLEEP_SEC} seconds after fetching image") - if content != None: - extension_file_name = url.split(".")[-1] - await weibo_store.update_weibo_note_image(pid, content, extension_file_name) + try: + items = weibo_media.build_media_items(mblog) + if items: + await self._get_media_downloader().download_all(items) + except Exception as exc: + # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程 + utils.logger.error(f"[WeiboCrawler.download_media] 媒体下载异常: {exc}") + + def _get_media_downloader(self) -> MediaDownloader: + """惰性创建媒体下载器，并同步最新的代理与 UA（代理池是就地刷新的）""" + if self._media_downloader is None: + self._media_downloader = MediaDownloader( + platform="wb", + proxy=getattr(self.wb_client, "proxy", None), + extra_headers=self._media_headers(), + ) + else: + self._media_downloader.update_credentials( + proxy=getattr(self.wb_client, "proxy", None), + extra_headers=self._media_headers(), + ) + return self._media_downloader + + def _media_headers(self) -> Dict: + """媒体请求头：平台 Referer（防盗链）+ UA。 + + client.headers 里含 Cookie，不能整体透传到 CDN 请求上，因此只取 UA。 + """ + headers = {"Referer": "https://weibo.com/"} + client_headers = getattr(self.wb_client, "headers", {}) or {} + if client_headers.get("User-Agent"): + headers["User-Agent"] = client_headers["User-Agent"] + return headers async def get_creators_and_notes(self) -> None: """ @@ -321,6 +337,10 @@ class WeiboCrawler(AbstractCrawler): # If full text fetching is enabled, batch get full text first updated_note_list = await self.batch_get_notes_full_text(note_list) await weibo_store.batch_update_weibo_notes(updated_note_list) + for note_item in updated_note_list: + mblog = (note_item or {}).get("mblog") + if mblog: + await self.download_media(mblog) # Get all note information of the creator all_notes_list = await self.wb_client.get_all_notes_by_creator_id( diff --git a/media_platform/weibo/media.py b/media_platform/weibo/media.py new file mode 100644 index 0000000..6c80099 --- /dev/null +++ b/media_platform/weibo/media.py @@ -0,0 +1,181 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/weibo/media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""微博媒体地址提取。 + +输入是微博 ``mblog``，输出是可下载的 ``MediaItem`` 列表。无 IO（少数平台会读取 config 决定策略）。 + +图片需要特殊处理：微博图床有防盗链，且缩略图路径里带尺寸段， +统一改写为 ``large`` 清晰度后经 i1.wp.com 代理访问。 +""" + +from __future__ import annotations + +from typing import Dict, List +from urllib.parse import urlsplit + +from media_downloader import MediaItem, MediaType + +# 微博图床存在防盗链，统一走该图片代理访问 +WEIBO_IMAGE_AGENT_HOST = "https://i1.wp.com/" + +# 视频地址字段，按清晰度从高到低尝试 +_VIDEO_KEYS = ( + "mp4_1080p_mp4", + "mp4_hd_mp4", + "mp4_720p_mp4", + "hevc_mp4_hd", + "mp4_ld_mp4", + "stream_url", +) + + +def rewrite_image_url(raw_url: str) -> str: + """把微博图床地址改写为 large 清晰度并加图片代理前缀。 + + ``https://wx1.sinaimg.cn/orj360/abc.jpg`` + -> ``https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg`` + """ + parts = urlsplit(raw_url) + if not parts.scheme or not parts.netloc: + return raw_url + + segments = [segment for segment in parts.path.split("/") if segment] + if segments: + segments[0] = "large" # 原始路径首段是尺寸标识（orj360 / thumbnail 等） + + path = "/" + "/".join(segments) if segments else parts.path + return f"{WEIBO_IMAGE_AGENT_HOST}{parts.netloc}{path}" + + +def extract_image_urls(mblog: Dict) -> List[str]: + """提取微博配图地址（已改写为可访问地址）""" + urls: List[str] = [] + for pic in mblog.get("pics") or []: + if isinstance(pic, str): + raw_url = pic + elif isinstance(pic, dict): + raw_url = pic.get("url") or "" + else: + continue + if isinstance(raw_url, str) and raw_url: + urls.append(rewrite_image_url(raw_url)) + return urls + + +def extract_video_urls(mblog: Dict) -> List[str]: + """提取微博视频地址候选列表（按清晰度从高到低）""" + page_info = mblog.get("page_info") or {} + if not isinstance(page_info, dict): + return [] + if page_info.get("type") != "video" and not page_info.get("media_info"): + return [] + + urls: List[str] = [] + for container_key in ("media_info", "urls"): + container = page_info.get(container_key) or {} + if not isinstance(container, dict): + continue + for key in _VIDEO_KEYS: + url = container.get(key) + if isinstance(url, str) and url and url not in urls: + urls.append(url) + return urls + + +def extract_cover_url(mblog: Dict) -> str: + """提取视频封面地址""" + page_info = mblog.get("page_info") or {} + if not isinstance(page_info, dict): + return "" + page_pic = page_info.get("page_pic") or {} + if isinstance(page_pic, dict): + url = page_pic.get("url") + return url if isinstance(url, str) else "" + return "" + + +def media_source_mblog(mblog: Dict) -> Dict: + """返回真正承载媒体内容的那一层微博。 + + 纯转发的 ``pics`` / ``page_info`` 位于 ``retweeted_status`` 里，顶层两者皆无； + 不处理的话转发帖会静默地一条媒体都下载不到。 + 若转发时自己带了图（顶层有媒体），则以顶层为准。 + """ + if mblog.get("pics") or mblog.get("page_info"): + return mblog + + retweeted = mblog.get("retweeted_status") + if isinstance(retweeted, dict): + return retweeted + return mblog + + +def build_media_items(mblog: Dict) -> List[MediaItem]: + """把一条微博转换成待下载的媒体任务列表。 + + - 视频微博：封面 + 视频（配图通常就是封面，不重复下载） + - 普通微博：各张配图 + - 转发微博：取原博的媒体，目录仍按转发帖自身的 id 组织 + """ + if not isinstance(mblog, dict): + return [] + + content_id = mblog.get("id") or mblog.get("mid") or "" + if not content_id: + return [] + content_id = str(content_id) + + source = media_source_mblog(mblog) + items: List[MediaItem] = [] + + video_urls = extract_video_urls(source) + if video_urls: + cover_url = extract_cover_url(source) + if cover_url: + items.append( + MediaItem( + url=cover_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem="cover", + ) + ) + items.append( + MediaItem( + url=video_urls[0], + backup_urls=tuple(video_urls[1:]), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video", + ) + ) + return items + + for index, image_url in enumerate(extract_image_urls(source), start=1): + items.append( + MediaItem( + url=image_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem=f"{index:03d}", + ) + ) + + return items diff --git a/media_platform/xhs/client.py b/media_platform/xhs/client.py index 4627cd2..f9d5193 100644 --- a/media_platform/xhs/client.py +++ b/media_platform/xhs/client.py @@ -246,29 +246,6 @@ class XiaoHongShuClient(AbstractApiClient, ProxyRefreshMixin): **kwargs, ) - async def get_note_media(self, url: str) -> Union[bytes, None]: - # Check if proxy is expired before request - await self._refresh_proxy_if_expired() - - async with make_async_client(proxy=self.proxy) as client: - try: - response = await client.request("GET", url, timeout=self.timeout) - response.raise_for_status() - if not response.reason_phrase == "OK": - utils.logger.error( - f"[XiaoHongShuClient.get_note_media] request {url} err, res:{response.text}" - ) - return None - else: - return response.content - except ( - httpx.HTTPError - ) as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx - utils.logger.error( - f"[XiaoHongShuClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}" - ) # Keep original exception type name for developer debugging - return None - async def query_self(self) -> Optional[Dict]: """ Query self user info to check login state diff --git a/media_platform/xhs/core.py b/media_platform/xhs/core.py index 2dc64e9..8110907 100644 --- a/media_platform/xhs/core.py +++ b/media_platform/xhs/core.py @@ -34,6 +34,7 @@ from tenacity import RetryError import config from base.base_crawler import AbstractCrawler +from media_downloader import MediaDownloader from model.m_xiaohongshu import NoteUrlInfo, CreatorUrlInfo from proxy.proxy_ip_pool import IpInfoModel, create_ip_pool from store import xhs as xhs_store @@ -41,6 +42,7 @@ from tools import utils from tools.cdp_browser import CDPBrowserManager from var import crawler_type_var, source_keyword_var +from . import media as xhs_media from .client import XiaoHongShuClient from .exception import ( DataFetchError, @@ -66,6 +68,7 @@ class XiaoHongShuCrawler(AbstractCrawler): self.user_agent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" self.cdp_manager = None self.ip_proxy_pool = None # Proxy IP pool for automatic proxy refresh + self._media_downloader: Optional[MediaDownloader] = None async def start(self) -> None: playwright_proxy_format, httpx_proxy_format = None, None @@ -176,7 +179,7 @@ class XiaoHongShuCrawler(AbstractCrawler): for note_detail in note_details: if note_detail: await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) + await self.download_media(note_detail) note_ids.append(note_detail.get("note_id")) xsec_tokens.append(note_detail.get("xsec_token")) page += 1 @@ -253,7 +256,7 @@ class XiaoHongShuCrawler(AbstractCrawler): for note_detail in note_details: if note_detail: await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) + await self.download_media(note_detail) async def get_specified_notes(self): """Get the information and comments of the specified post @@ -280,7 +283,7 @@ class XiaoHongShuCrawler(AbstractCrawler): need_get_comment_note_ids.append(note_detail.get("note_id", "")) xsec_tokens.append(note_detail.get("xsec_token", "")) await xhs_store.update_xhs_note(note_detail) - await self.get_notice_media(note_detail) + await self.download_media(note_detail) await self.batch_get_note_comments(need_get_comment_note_ids, xsec_tokens) async def get_note_detail_async_task( @@ -480,63 +483,40 @@ class XiaoHongShuCrawler(AbstractCrawler): await self.browser_context.close() utils.logger.info("[XiaoHongShuCrawler.close] Browser context closed ...") - async def get_notice_media(self, note_detail: Dict): - if not config.ENABLE_GET_MEIDAS: - utils.logger.info(f"[XiaoHongShuCrawler.get_notice_media] Crawling image mode is not enabled") - return - await self.get_note_images(note_detail) - await self.get_notice_video(note_detail) - - async def get_note_images(self, note_item: Dict): - """Get note images. Please use get_notice_media + async def download_media(self, note_detail: Dict) -> None: + """下载笔记的媒体资源（封面、视频或图文图片） Args: - note_item: Note item dictionary + note_detail: 笔记详情（原始 note_card 结构） """ - if not config.ENABLE_GET_MEIDAS: + if not config.ENABLE_GET_MEDIA: return - note_id = note_item.get("note_id") - image_list: List[Dict] = note_item.get("image_list", []) + try: + items = xhs_media.build_media_items(note_detail) + if items: + await self._get_media_downloader().download_all(items) + except Exception as exc: + # 媒体下载是旁路能力，解析异常/网络异常都不能中断爬取主流程 + utils.logger.error(f"[XiaoHongShuCrawler.download_media] 媒体下载异常: {exc}") - for img in image_list: - if img.get("url_default") != "": - img.update({"url": img.get("url_default")}) + def _get_media_downloader(self) -> MediaDownloader: + """惰性创建媒体下载器，并同步最新的代理设置（代理池是就地刷新的）""" + if self._media_downloader is None: + self._media_downloader = MediaDownloader( + platform="xhs", + proxy=getattr(self.xhs_client, "proxy", None), + extra_headers=self._media_headers(), + ) + else: + self._media_downloader.update_credentials( + proxy=getattr(self.xhs_client, "proxy", None), + extra_headers=self._media_headers(), + ) + return self._media_downloader - if not image_list: - return - picNum = 0 - for pic in image_list: - url = pic.get("url") - if not url: - continue - content = await self.xhs_client.get_note_media(url) - await asyncio.sleep(random.random()) - if content is None: - continue - extension_file_name = f"{picNum}.jpg" - picNum += 1 - await xhs_store.update_xhs_note_image(note_id, content, extension_file_name) - - async def get_notice_video(self, note_item: Dict): - """Get note videos. Please use get_notice_media - - Args: - note_item: Note item dictionary - """ - if not config.ENABLE_GET_MEIDAS: - return - note_id = note_item.get("note_id") - - videos = xhs_store.get_video_url_arr(note_item) - - if not videos: - return - videoNum = 0 - for url in videos: - content = await self.xhs_client.get_note_media(url) - await asyncio.sleep(random.random()) - if content is None: - continue - extension_file_name = f"{videoNum}.mp4" - videoNum += 1 - await xhs_store.update_xhs_note_video(note_id, content, extension_file_name) + def _media_headers(self) -> Dict: + """媒体请求头：平台 Referer（防盗链）+ UA""" + return { + "Referer": "https://www.xiaohongshu.com/", + "User-Agent": self.user_agent, + } diff --git a/media_platform/xhs/media.py b/media_platform/xhs/media.py new file mode 100644 index 0000000..d772ac3 --- /dev/null +++ b/media_platform/xhs/media.py @@ -0,0 +1,160 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/media_platform/xhs/media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： +# 1. 不得用于任何商业用途。 +# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 +# 3. 不得进行大规模爬取或对平台造成运营干扰。 +# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 +# 5. 不得用于任何非法或不当的用途。 +# +# 详细许可条款请参阅项目根目录下的LICENSE文件。 +# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 + +"""小红书媒体地址提取。 + +输入是 ``/api/sns/web/v1/feed`` 返回的原始 ``note_card``（snake_case 字段）， +输出是下载器可直接消费的 ``MediaItem`` 列表。无 IO（会读取 config 决定国际版策略）。 +""" + +from __future__ import annotations + +from typing import Dict, List + +import config +from media_downloader import MediaItem, MediaType + +# 小红书视频 CDN：origin_video_key 拼在此域名后可拿到无水印源片 +XHS_VIDEO_CDN_HOST = "https://sns-video-bd.xhscdn.com" + + +def _as_dict(value) -> Dict: + """接口响应里同名字段可能是字符串或 None，统一收敛为 dict""" + return value if isinstance(value, dict) else {} + + +def _extract_origin_video_key(video_dict: Dict) -> str: + """从 video.consumer 中取无水印源片 key（兼容 snake_case 与 camelCase）""" + consumer = _as_dict(video_dict.get("consumer")) + return consumer.get("origin_video_key") or consumer.get("originVideoKey") or "" + + +def extract_video_urls(note_item: Dict) -> List[str]: + """提取视频地址候选列表，按可用性排序。 + + 优先无水印源片（origin_video_key），其余为带水印的 h264 master_url 备用。 + 国际版（rednote）的 CDN 域名与国内不同，不能拼接 xhscdn 域名。 + """ + if note_item.get("type") != "video": + return [] + + video_dict = _as_dict(note_item.get("video")) + if not video_dict: + return [] + + urls: List[str] = [] + origin_video_key = _extract_origin_video_key(video_dict) + if origin_video_key and not getattr(config, "XHS_INTERNATIONAL", False): + urls.append(f"{XHS_VIDEO_CDN_HOST}/{origin_video_key}") + + stream = _as_dict(_as_dict(video_dict.get("media")).get("stream")) + for item in stream.get("h264") or []: + master_url = item.get("master_url") if isinstance(item, dict) else None + if master_url and master_url not in urls: + urls.append(master_url) + + return urls + + +def extract_image_urls(note_item: Dict) -> List[str]: + """提取图文笔记的图片地址（按原始顺序）""" + urls: List[str] = [] + for image_item in note_item.get("image_list") or []: + if not isinstance(image_item, dict): + continue + url = image_item.get("url_default") or image_item.get("url") or "" + if isinstance(url, str) and url: + urls.append(url) + return urls + + +def extract_cover_url(note_item: Dict) -> str: + """提取封面地址。 + + 小红书不同接口返回的封面字段位置不一致，这里按可靠性依次兜底； + 视频笔记的 image_list 通常就是封面图，作为最后兜底。 + """ + video_dict = _as_dict(note_item.get("video")) + cover_dict = _as_dict(note_item.get("cover")) + video_cover = _as_dict(video_dict.get("cover")) + + candidates = [ + video_cover.get("url_default"), + video_cover.get("url"), + cover_dict.get("url_default"), + cover_dict.get("url"), + ] + for candidate in candidates: + if candidate: + return candidate + + images = extract_image_urls(note_item) + return images[0] if images else "" + + +def build_media_items(note_item: Dict) -> List[MediaItem]: + """把一个笔记转换成待下载的媒体任务列表。 + + - 视频笔记：封面 + 视频。其 image_list 通常只有封面一张，不再按图集重复下载 + - 图文笔记：各张图片（第一张即封面，因此不再单独下载一份 cover） + """ + if not isinstance(note_item, dict): + return [] + + content_id = note_item.get("note_id") or note_item.get("id") or "" + if not content_id: + return [] + + items: List[MediaItem] = [] + + if note_item.get("type") == "video": + cover_url = extract_cover_url(note_item) + if cover_url: + items.append( + MediaItem( + url=cover_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem="cover", + ) + ) + + video_urls = extract_video_urls(note_item) + if video_urls: + items.append( + MediaItem( + url=video_urls[0], + backup_urls=tuple(video_urls[1:]), + media_type=MediaType.VIDEO, + content_id=content_id, + stem="video", + ) + ) + else: + for index, image_url in enumerate(extract_image_urls(note_item), start=1): + items.append( + MediaItem( + url=image_url, + media_type=MediaType.IMAGE, + content_id=content_id, + stem=f"{index:03d}", + ) + ) + + return items diff --git a/store/bilibili/__init__.py b/store/bilibili/__init__.py index b5eb921..3b8d85d 100644 --- a/store/bilibili/__init__.py +++ b/store/bilibili/__init__.py @@ -29,7 +29,6 @@ from var import source_keyword_var from tools.user_hash import anonymize_user_id, mask_nickname from ._store_impl import * -from .bilibilli_store_media import * class BiliStoreFactory: @@ -116,21 +115,6 @@ async def update_bilibili_video_comment(video_id: str, comment_item: Dict): await BiliStoreFactory.create_store().store_comment(comment_item=save_comment_item) -async def store_video(aid, video_content, extension_file_name): - """ - video video storage implementation - Args: - aid: - video_content: - extension_file_name: - """ - await BilibiliVideo().store_video({ - "aid": aid, - "video_content": video_content, - "extension_file_name": extension_file_name, - }) - - async def batch_update_bilibili_creator_fans(creator_info: Dict, fans_list: List[Dict]): # 教学版：不再采集/存储粉丝列表(其他用户的个人信息)，防骚扰。 return diff --git a/store/bilibili/bilibilli_store_media.py b/store/bilibili/bilibilli_store_media.py deleted file mode 100644 index b12048a..0000000 --- a/store/bilibili/bilibilli_store_media.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 relakkes@gmail.com -# -# This file is part of MediaCrawler project. -# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/bilibili/bilibilli_store_media.py -# GitHub: https://github.com/NanmiCoder -# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 -# - -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 -# 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - -# -*- coding: utf-8 -*- -# @Author : helloteemo -# @Time : 2024/7/12 20:01 -# @Desc : Bilibili media storage -import pathlib -from typing import Dict - -import aiofiles - -from base.base_crawler import AbstractStoreImage, AbstractStoreVideo -from tools import utils -import config - - -class BilibiliVideo(AbstractStoreVideo): - def __init__(self): - if config.SAVE_DATA_PATH: - self.video_store_path = f"{config.SAVE_DATA_PATH}/bili/videos" - else: - self.video_store_path = "data/bili/videos" - - async def store_video(self, video_content_item: Dict): - """ - store content - - Args: - video_content_item: - - Returns: - - """ - await self.save_video(video_content_item.get("aid"), video_content_item.get("video_content"), video_content_item.get("extension_file_name")) - - def make_save_file_name(self, aid: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - aid: aid - extension_file_name: video filename with extension - - Returns: - - """ - return f"{self.video_store_path}/{aid}/{extension_file_name}" - - async def save_video(self, aid: int, video_content: str, extension_file_name="mp4"): - """ - save video to local - - Args: - aid: aid - video_content: video content - extension_file_name: video filename with extension - - Returns: - - """ - pathlib.Path(self.video_store_path + "/" + str(aid)).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(str(aid), extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(video_content) - utils.logger.info(f"[BilibiliVideoImplement.save_video] save save_video {save_file_name} success ...") diff --git a/store/douyin/__init__.py b/store/douyin/__init__.py index a1e9207..1d5e2b9 100644 --- a/store/douyin/__init__.py +++ b/store/douyin/__init__.py @@ -24,11 +24,11 @@ from typing import List import config +from media_platform.douyin.media import extract_cover_url, extract_image_urls, extract_video_urls from var import source_keyword_var from tools.user_hash import anonymize_user_id, mask_nickname from ._store_impl import * -from .douyin_store_media import * class DouyinStoreFactory: @@ -51,30 +51,6 @@ class DouyinStoreFactory: return store_class() -def _extract_note_image_list(aweme_detail: Dict) -> List[str]: - """ - Extract note image list - - Args: - aweme_detail (Dict): Douyin content details - - Returns: - List[str]: Note image list - """ - images_res: List[str] = [] - images: List[Dict] = aweme_detail.get("images", []) - - if not images: - return [] - - for image in images: - image_url_list = image.get("url_list", []) # download_url_list has watermarked images, url_list has non-watermarked images - if image_url_list: - images_res.append(image_url_list[-1]) - - return images_res - - def _extract_comment_image_list(comment_item: Dict) -> List[str]: """ Extract comment image list @@ -99,46 +75,6 @@ def _extract_comment_image_list(comment_item: Dict) -> List[str]: return images_res -def _extract_content_cover_url(aweme_detail: Dict) -> str: - """ - Extract video cover URL - - Args: - aweme_detail (Dict): Douyin content details - - Returns: - str: Video cover URL - """ - res_cover_url = "" - - video_item = aweme_detail.get("video", {}) - raw_cover_url_list = (video_item.get("raw_cover", {}) or video_item.get("origin_cover", {})).get("url_list", []) - if raw_cover_url_list and len(raw_cover_url_list) > 1: - res_cover_url = raw_cover_url_list[1] - - return res_cover_url - - -def _extract_video_download_url(aweme_detail: Dict) -> str: - """ - Extract video download URL - - Args: - aweme_detail (Dict): Douyin video - - Returns: - str: Video download URL - """ - video_item = aweme_detail.get("video", {}) - url_h264_list = video_item.get("play_addr_h264", {}).get("url_list", []) - url_256_list = video_item.get("play_addr_256", {}).get("url_list", []) - url_list = video_item.get("play_addr", {}).get("url_list", []) - actual_url_list = url_h264_list or url_256_list or url_list - if not actual_url_list or len(actual_url_list) < 2: - return "" - return actual_url_list[-1] - - def _extract_music_download_url(aweme_detail: Dict) -> str: """ Extract music download URL @@ -173,10 +109,10 @@ async def update_douyin_aweme(aweme_item: Dict): "share_count": str(interact_info.get("share_count")), "last_modify_ts": utils.get_current_timestamp(), "aweme_url": f"https://www.douyin.com/video/{aweme_id}", - "cover_url": _extract_content_cover_url(aweme_item), - "video_download_url": _extract_video_download_url(aweme_item), + "cover_url": extract_cover_url(aweme_item), + "video_download_url": (extract_video_urls(aweme_item) or [""])[0], "music_download_url": _extract_music_download_url(aweme_item), - "note_download_url": ",".join(_extract_note_image_list(aweme_item)), + "note_download_url": ",".join(extract_image_urls(aweme_item)), "source_keyword": source_keyword_var.get(), } utils.logger.info(f"[store.douyin.update_douyin_aweme] douyin aweme id:{aweme_id}, title:{save_content_item.get('title')}") @@ -219,33 +155,3 @@ async def update_dy_aweme_comment(aweme_id: str, comment_item: Dict): async def save_creator(user_id: str, creator: Dict): # 教学版：创作者个人资料(昵称/性别/头像/签名/IP/粉丝数等)不再落库，防骚扰。 return - - -async def update_dy_aweme_image(aweme_id, pic_content, extension_file_name): - """ - Update Douyin note image - Args: - aweme_id: - pic_content: - extension_file_name: - - Returns: - - """ - - await DouYinImage().store_image({"aweme_id": aweme_id, "pic_content": pic_content, "extension_file_name": extension_file_name}) - - -async def update_dy_aweme_video(aweme_id, video_content, extension_file_name): - """ - Update Douyin short video - Args: - aweme_id: - video_content: - extension_file_name: - - Returns: - - """ - - await DouYinVideo().store_video({"aweme_id": aweme_id, "video_content": video_content, "extension_file_name": extension_file_name}) diff --git a/store/douyin/douyin_store_media.py b/store/douyin/douyin_store_media.py deleted file mode 100644 index baebf33..0000000 --- a/store/douyin/douyin_store_media.py +++ /dev/null @@ -1,129 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 relakkes@gmail.com -# -# This file is part of MediaCrawler project. -# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/douyin/douyin_store_media.py -# GitHub: https://github.com/NanmiCoder -# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 -# - -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 -# 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - -import pathlib -from typing import Dict - -import aiofiles - -from base.base_crawler import AbstractStoreImage, AbstractStoreVideo -from tools import utils -import config - - -class DouYinImage(AbstractStoreImage): - def __init__(self): - if config.SAVE_DATA_PATH: - self.image_store_path = f"{config.SAVE_DATA_PATH}/douyin/images" - else: - self.image_store_path = "data/douyin/images" - - async def store_image(self, image_content_item: Dict): - """ - store content - - Args: - image_content_item: - - Returns: - - """ - await self.save_image(image_content_item.get("aweme_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name")) - - def make_save_file_name(self, aweme_id: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - aweme_id: aweme id - extension_file_name: image filename with extension - - Returns: - - """ - return f"{self.image_store_path}/{aweme_id}/{extension_file_name}" - - async def save_image(self, aweme_id: str, pic_content: str, extension_file_name): - """ - save image to local - - Args: - aweme_id: aweme id - pic_content: image content - extension_file_name: image filename with extension - - Returns: - - """ - pathlib.Path(self.image_store_path + "/" + aweme_id).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(aweme_id, extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(pic_content) - utils.logger.info(f"[DouYinImageStoreImplement.save_image] save image {save_file_name} success ...") - - -class DouYinVideo(AbstractStoreVideo): - def __init__(self): - if config.SAVE_DATA_PATH: - self.video_store_path = f"{config.SAVE_DATA_PATH}/douyin/videos" - else: - self.video_store_path = "data/douyin/videos" - - async def store_video(self, video_content_item: Dict): - """ - store content - - Args: - video_content_item: - - Returns: - - """ - await self.save_video(video_content_item.get("aweme_id"), video_content_item.get("video_content"), video_content_item.get("extension_file_name")) - - def make_save_file_name(self, aweme_id: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - aweme_id: aweme id - extension_file_name: video filename with extension - - Returns: - - """ - return f"{self.video_store_path}/{aweme_id}/{extension_file_name}" - - async def save_video(self, aweme_id: str, video_content: str, extension_file_name): - """ - save video to local - - Args: - aweme_id: aweme id - video_content: video content - extension_file_name: video filename with extension - - Returns: - - """ - pathlib.Path(self.video_store_path + "/" + aweme_id).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(aweme_id, extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(video_content) - utils.logger.info(f"[DouYinVideoStoreImplement.save_video] save video {save_file_name} success ...") diff --git a/store/weibo/__init__.py b/store/weibo/__init__.py index bdf632b..5894011 100644 --- a/store/weibo/__init__.py +++ b/store/weibo/__init__.py @@ -28,7 +28,6 @@ from typing import List from tools.user_hash import anonymize_user_id, mask_nickname from var import source_keyword_var -from .weibo_store_media import * from ._store_impl import * @@ -160,20 +159,6 @@ async def update_weibo_note_comment(note_id: str, comment_item: Dict): await WeibostoreFactory.create_store().store_comment(comment_item=save_comment_item) -async def update_weibo_note_image(picid: str, pic_content, extension_file_name): - """ - Save weibo note image to local - Args: - picid: - pic_content: - extension_file_name: - - Returns: - - """ - await WeiboStoreImage().store_image({"pic_id": picid, "pic_content": pic_content, "extension_file_name": extension_file_name}) - - async def save_creator(user_id: str, user_info: Dict): """ Save creator information to local diff --git a/store/weibo/weibo_store_media.py b/store/weibo/weibo_store_media.py deleted file mode 100644 index c38dd5a..0000000 --- a/store/weibo/weibo_store_media.py +++ /dev/null @@ -1,82 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 relakkes@gmail.com -# -# This file is part of MediaCrawler project. -# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/weibo/weibo_store_media.py -# GitHub: https://github.com/NanmiCoder -# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 -# - -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 -# 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - -# -*- coding: utf-8 -*- -# @Author : Erm -# @Time : 2024/4/9 17:35 -# @Desc : Weibo media storage -import pathlib -from typing import Dict - -import aiofiles - -from base.base_crawler import AbstractStoreImage, AbstractStoreVideo -from tools import utils -import config - - -class WeiboStoreImage(AbstractStoreImage): - def __init__(self): - if config.SAVE_DATA_PATH: - self.image_store_path = f"{config.SAVE_DATA_PATH}/weibo/images" - else: - self.image_store_path = "data/weibo/images" - - async def store_image(self, image_content_item: Dict): - """ - store content - - Args: - image_content_item: - - Returns: - - """ - await self.save_image(image_content_item.get("pic_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name")) - - def make_save_file_name(self, picid: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - picid: image id - extension_file_name: video filename with extension - - Returns: - - """ - return f"{self.image_store_path}/{picid}.{extension_file_name}" - - async def save_image(self, picid: str, pic_content: str, extension_file_name="jpg"): - """ - save image to local - - Args: - picid: image id - pic_content: image content - extension_file_name: image filename with extension - - Returns: - - """ - pathlib.Path(self.image_store_path).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(picid, extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(pic_content) - utils.logger.info(f"[WeiboImageStoreImplement.save_image] save image {save_file_name} success ...") diff --git a/store/xhs/__init__.py b/store/xhs/__init__.py index 5ef8e3c..47b5c71 100644 --- a/store/xhs/__init__.py +++ b/store/xhs/__init__.py @@ -24,10 +24,10 @@ from typing import List import config +from media_platform.xhs.media import extract_video_urls from var import source_keyword_var from tools.user_hash import anonymize_user_id, mask_nickname -from .xhs_store_media import * from ._store_impl import * @@ -51,40 +51,6 @@ class XhsStoreFactory: return store_class() -def get_video_url_arr(note_item: Dict) -> List: - """ - Get video url array - Args: - note_item: - - Returns: - - """ - if note_item.get('type') != 'video': - return [] - - video_dict = note_item.get('video') - if not video_dict: - return [] - - videoArr = [] - consumer = video_dict.get('consumer', {}) - originVideoKey = consumer.get('origin_video_key', '') - if originVideoKey == '': - originVideoKey = consumer.get('originVideoKey', '') - # Fallback with watermark - if originVideoKey == '': - media = video_dict.get('media', {}) - stream = media.get('stream', {}) - videos = stream.get('h264') - if type(videos).__name__ == 'list': - videoArr = [v.get('master_url') for v in videos] - else: - videoArr = [f"http://sns-video-bd.xhscdn.com/{originVideoKey}"] - - return videoArr - - async def update_xhs_note(note_item: Dict): """ Update Xiaohongshu note @@ -104,7 +70,7 @@ async def update_xhs_note(note_item: Dict): if img.get('url_default') != '': img.update({'url': img.get('url_default')}) - video_url = ','.join(get_video_url_arr(note_item)) + video_url = ','.join(extract_video_urls(note_item)) local_db_item = { "note_id": note_item.get("note_id"), # Note ID @@ -190,33 +156,3 @@ async def save_creator(user_id: str, creator: Dict): """ # 教学版：创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库，防骚扰。 return - - -async def update_xhs_note_image(note_id, pic_content, extension_file_name): - """ - Update Xiaohongshu note image - Args: - note_id: - pic_content: - extension_file_name: - - Returns: - - """ - - await XiaoHongShuImage().store_image({"notice_id": note_id, "pic_content": pic_content, "extension_file_name": extension_file_name}) - - -async def update_xhs_note_video(note_id, video_content, extension_file_name): - """ - Update Xiaohongshu note video - Args: - note_id: - video_content: - extension_file_name: - - Returns: - - """ - - await XiaoHongShuVideo().store_video({"notice_id": note_id, "video_content": video_content, "extension_file_name": extension_file_name}) diff --git a/store/xhs/xhs_store_media.py b/store/xhs/xhs_store_media.py deleted file mode 100644 index c38a812..0000000 --- a/store/xhs/xhs_store_media.py +++ /dev/null @@ -1,133 +0,0 @@ -# -*- coding: utf-8 -*- -# Copyright (c) 2025 relakkes@gmail.com -# -# This file is part of MediaCrawler project. -# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/xhs/xhs_store_media.py -# GitHub: https://github.com/NanmiCoder -# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 -# - -# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则： -# 1. 不得用于任何商业用途。 -# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。 -# 3. 不得进行大规模爬取或对平台造成运营干扰。 -# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。 -# 5. 不得用于任何非法或不当的用途。 -# -# 详细许可条款请参阅项目根目录下的LICENSE文件。 -# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。 - -# -*- coding: utf-8 -*- -# @Author : helloteemo -# @Time : 2024/7/11 22:35 -# @Desc : Xiaohongshu media storage -import pathlib -from typing import Dict - -import aiofiles - -from base.base_crawler import AbstractStoreImage, AbstractStoreVideo -from tools import utils -import config - - -class XiaoHongShuImage(AbstractStoreImage): - def __init__(self): - if config.SAVE_DATA_PATH: - self.image_store_path = f"{config.SAVE_DATA_PATH}/xhs/images" - else: - self.image_store_path = "data/xhs/images" - - async def store_image(self, image_content_item: Dict): - """ - store content - - Args: - image_content_item: - - Returns: - - """ - await self.save_image(image_content_item.get("notice_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name")) - - def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - notice_id: notice id - extension_file_name: image filename with extension - - Returns: - - """ - return f"{self.image_store_path}/{notice_id}/{extension_file_name}" - - async def save_image(self, notice_id: str, pic_content: str, extension_file_name): - """ - save image to local - - Args: - notice_id: notice id - pic_content: image content - extension_file_name: image filename with extension - - Returns: - - """ - pathlib.Path(self.image_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(notice_id, extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(pic_content) - utils.logger.info(f"[XiaoHongShuImageStoreImplement.save_image] save image {save_file_name} success ...") - - -class XiaoHongShuVideo(AbstractStoreVideo): - def __init__(self): - if config.SAVE_DATA_PATH: - self.video_store_path = f"{config.SAVE_DATA_PATH}/xhs/videos" - else: - self.video_store_path = "data/xhs/videos" - - async def store_video(self, video_content_item: Dict): - """ - store content - - Args: - video_content_item: - - Returns: - - """ - await self.save_video(video_content_item.get("notice_id"), video_content_item.get("video_content"), video_content_item.get("extension_file_name")) - - def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str: - """ - make save file name by store type - - Args: - notice_id: notice id - extension_file_name: video filename with extension - - Returns: - - """ - return f"{self.video_store_path}/{notice_id}/{extension_file_name}" - - async def save_video(self, notice_id: str, video_content: str, extension_file_name): - """ - save video to local - - Args: - notice_id: notice id - video_content: video content - extension_file_name: video filename with extension - - Returns: - - """ - pathlib.Path(self.video_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True) - save_file_name = self.make_save_file_name(notice_id, extension_file_name) - async with aiofiles.open(save_file_name, 'wb') as f: - await f.write(video_content) - utils.logger.info(f"[XiaoHongShuVideoStoreImplement.save_video] save video {save_file_name} success ...") diff --git a/tests/media_server.py b/tests/media_server.py new file mode 100644 index 0000000..f2ae69e --- /dev/null +++ b/tests/media_server.py @@ -0,0 +1,386 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/media_server.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""媒体下载测试用的本地 HTTP 服务器。 + +基于标准库 ``ThreadingHTTPServer``，不引入额外依赖；覆盖下载器需要面对的各种 +CDN 行为：Range 续传、忽略 Range、chunked、响应截断、5xx 抖动、302 跳转、 +无扩展名但带 Content-Type、慢响应、空响应体等。 + +所有请求都会记录到 ``server.requests``，用于断言"跳过已下载时不发请求" +"403 只请求一次"这类行为。 +""" + +from __future__ import annotations + +import threading +import time +from dataclasses import dataclass, field +from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer +from typing import Dict, List, Optional, Tuple +from urllib.parse import parse_qs, urlsplit + +# 可识别的字节模式，便于断言续传后拼接结果正确 +DEFAULT_CONTENT = bytes(range(256)) * 16 # 4096 字节 +DEFAULT_CONTENT_TYPE = "image/jpeg" + + +@dataclass +class RequestRecord: + """一次进入服务器的请求""" + + method: str + path: str + headers: Dict[str, str] = field(default_factory=dict) + + +class _MediaRequestHandler(BaseHTTPRequestHandler): + protocol_version = "HTTP/1.1" + server_version = "MediaTestServer/1.0" + + # 静音默认的 stderr 访问日志 + def log_message(self, format: str, *args) -> None: # noqa: A002 + return + + # ------------------------------------------------------------------ 工具方法 + + @property + def media_server(self) -> "MediaTestServer": + return self.server.media_server # type: ignore[attr-defined] + + def _record(self) -> None: + self.media_server.requests.append( + RequestRecord( + method=self.command, + path=self.path, + headers={key.lower(): value for key, value in self.headers.items()}, + ) + ) + + def _content_for(self, name: str) -> bytes: + custom = self.media_server.contents.get(name) + return custom if custom is not None else DEFAULT_CONTENT + + def _send_body(self, status: int, body: bytes, content_type: str, extra_headers=None) -> None: + self.send_response(status) + self.send_header("Content-Type", content_type) + self.send_header("Content-Length", str(len(body))) + for key, value in (extra_headers or {}).items(): + self.send_header(key, value) + self.end_headers() + if body: + self.wfile.write(body) + + def _parse_range(self) -> Optional[int]: + """解析 ``Range: bytes=N-``，返回起始偏移；不合法或不存在返回 None""" + raw = self.headers.get("range", "") + if not raw.startswith("bytes="): + return None + spec = raw[len("bytes=") :].split(",")[0].strip() + if not spec.endswith("-"): + return None + start = spec[:-1].strip() + return int(start) if start.isdigit() else None + + # ------------------------------------------------------------------ 路由 + + def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler 约定 + self._record() + parsed = urlsplit(self.path) + path = parsed.path + query = parse_qs(parsed.query) + + try: + if path.startswith("/status/"): + code = int(path.rsplit("/", 1)[-1]) + self._send_body(code, b"", "text/plain") + elif path.startswith("/ok/") or path.startswith("/redirected/"): + self._handle_ok(path.rsplit("/", 1)[-1], query) + elif path.startswith("/no-range/"): + self._handle_no_range(path.rsplit("/", 1)[-1]) + elif path.startswith("/wrong-range/"): + self._handle_wrong_range(path.rsplit("/", 1)[-1], query) + elif path.startswith("/poison-206/"): + self._handle_poison_206(path.rsplit("/", 1)[-1], query) + elif path.startswith("/gzip/"): + self._handle_gzip(path.rsplit("/", 1)[-1], query) + elif path.startswith("/unknown-total/"): + self._handle_unknown_total(path.rsplit("/", 1)[-1]) + elif path.startswith("/no-content-range/"): + self._handle_no_content_range(path.rsplit("/", 1)[-1]) + elif path.startswith("/chunked-bogus-cl/"): + self._handle_chunked_with_bogus_content_length(path.rsplit("/", 1)[-1]) + elif path.startswith("/chunked/"): + self._handle_chunked(path.rsplit("/", 1)[-1]) + elif path.startswith("/truncated/"): + self._handle_truncated(path.rsplit("/", 1)[-1], query) + elif path.startswith("/flaky/"): + self._handle_flaky(path.rsplit("/", 1)[-1], query) + elif path.startswith("/redirect/"): + self._handle_redirect(path.rsplit("/", 1)[-1]) + elif path.startswith("/ctyped/"): + self._handle_ctyped(path.rsplit("/", 1)[-1]) + elif path.startswith("/slow/"): + self._handle_slow(path.rsplit("/", 1)[-1], query) + elif path.startswith("/empty/"): + self._send_body(200, b"", DEFAULT_CONTENT_TYPE) + else: + self._send_body(404, b"not found", "text/plain") + except (BrokenPipeError, ConnectionResetError): # 客户端主动断开 + self.close_connection = True + + # ------------------------------------------------------------------ 各路由实现 + + def _handle_ok(self, name: str, query) -> None: + """正常文件，支持 Range 续传""" + content = self._content_for(name) + total = len(content) + start = self._parse_range() + + if start is not None: + if start >= total: + self.send_response(416) + self.send_header("Content-Range", f"bytes */{total}") + self.send_header("Content-Length", "0") + self.end_headers() + return + body = content[start:] + content_type = query.get("ct", [DEFAULT_CONTENT_TYPE])[0] + self._send_body( + 206, + body, + content_type, + {"Content-Range": f"bytes {start}-{total - 1}/{total}"}, + ) + return + + self._send_body(200, content, query.get("ct", [DEFAULT_CONTENT_TYPE])[0]) + + def _handle_wrong_range(self, name: str, query) -> None: + """返回 206 但 Content-Range 起点与请求不一致（模拟按关键帧对齐的加速节点）。 + + ``?fail=N``：前 N 次返回错位响应，之后恢复正常， + 用于验证"下载器丢弃片段重下后能成功"。 + """ + fail_times = int(query.get("fail", ["1000"])[0]) + counter_key = f"/wrong-range/{name}" + with self.media_server.lock: + seen = self.media_server.counters.get(counter_key, 0) + self.media_server.counters[counter_key] = seen + 1 + + if seen >= fail_times: + self._handle_ok(name, query) + return + + content = self._content_for(name) + total = len(content) + requested = self._parse_range() or 0 + delta = int(query.get("delta", ["100"])[0]) + start = min(requested + delta, total - 1) + self._send_body( + 206, + content[start:], + DEFAULT_CONTENT_TYPE, + {"Content-Range": f"bytes {start}-{total - 1}/{total}"}, + ) + + def _handle_poison_206(self, name: str, query) -> None: + """206 声明非 0 起点，但 body 长度**恰好等于 total**（内容是错位的）。 + + 这是唯一能单独验证"起点守卫"的响应：长度自洽，大小校验拦不住， + 只有比对 Content-Range 起点才能发现异常。 + """ + content = self._content_for(name) + total = len(content) + start = int(query.get("start", ["100"])[0]) + body = content[start:] + b"\x00" * start + self._send_body( + 206, + body, + DEFAULT_CONTENT_TYPE, + {"Content-Range": f"bytes {start}-{total - 1}/{total}"}, + ) + + def _handle_gzip(self, name: str, query) -> None: + """无视 Accept-Encoding: identity，强制返回 gzip 压缩体""" + import gzip + + content = self._content_for(name) + compressed = gzip.compress(content) + self.send_response(200) + self.send_header("Content-Type", DEFAULT_CONTENT_TYPE) + self.send_header("Content-Encoding", "gzip") + self.send_header("Content-Length", str(len(compressed))) + self.end_headers() + self.wfile.write(compressed) + + def _handle_unknown_total(self, name: str) -> None: + """206 只返回区间的一部分，且 Content-Range 的 total 为 *（不告知总长）""" + content = self._content_for(name) + start = self._parse_range() or 0 + chunk = content[start : start + 1024] + self._send_body( + 206, + chunk, + DEFAULT_CONTENT_TYPE, + {"Content-Range": f"bytes {start}-{start + len(chunk) - 1}/*"}, + ) + + def _handle_no_content_range(self, name: str) -> None: + """206 但缺少 Content-Range（违反 RFC 9110），且回的是全量体""" + content = self._content_for(name) + self._send_body(206, content, DEFAULT_CONTENT_TYPE) + + def _handle_chunked_with_bogus_content_length(self, name: str) -> None: + """chunked 传输同时带一个错误的 Content-Length（RFC 7230 要求忽略后者）""" + content = self._content_for(name) + self.send_response(200) + self.send_header("Content-Type", DEFAULT_CONTENT_TYPE) + self.send_header("Transfer-Encoding", "chunked") + self.send_header("Content-Length", "999") + self.end_headers() + self.wfile.write(f"{len(content):X}\r\n".encode("ascii")) + self.wfile.write(content) + self.wfile.write(b"\r\n0\r\n\r\n") + self.wfile.flush() + + def _handle_no_range(self, name: str) -> None: + """声明支持 Range 但始终返回全量 200（模拟不支持续传的 CDN）""" + content = self._content_for(name) + self._send_body(200, content, DEFAULT_CONTENT_TYPE, {"Accept-Ranges": "bytes"}) + + def _handle_chunked(self, name: str) -> None: + """chunked 传输：无 Content-Length""" + content = self._content_for(name) + self.send_response(200) + self.send_header("Content-Type", DEFAULT_CONTENT_TYPE) + self.send_header("Transfer-Encoding", "chunked") + self.end_headers() + for offset in range(0, len(content), 1024): + chunk = content[offset : offset + 1024] + self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii")) + self.wfile.write(chunk) + self.wfile.write(b"\r\n") + self.wfile.write(b"0\r\n\r\n") + self.wfile.flush() + + def _handle_truncated(self, name: str, query) -> None: + """声明完整长度但只发送一半后断开连接。 + + ``?fail=N``：前 N 次截断，之后恢复正常（支持 Range）， + 用于验证"断流 -> 重试 -> 带 Range 续传 -> 拼接完整"。 + """ + fail_times = int(query.get("fail", ["1000"])[0]) + counter_key = f"/truncated/{name}" + with self.media_server.lock: + seen = self.media_server.counters.get(counter_key, 0) + self.media_server.counters[counter_key] = seen + 1 + + if seen >= fail_times: + self._handle_ok(name, query) + return + + content = self._content_for(name) + self.send_response(200) + self.send_header("Content-Type", DEFAULT_CONTENT_TYPE) + self.send_header("Content-Length", str(len(content))) + self.end_headers() + self.wfile.write(content[: len(content) // 2]) + self.wfile.flush() + self.close_connection = True + try: + self.connection.shutdown(2) # SHUT_RDWR + except OSError: + pass + + def _handle_flaky(self, name: str, query) -> None: + """前 N 次请求返回 500，之后正常（服务端计数）""" + fail_times = int(query.get("fail", ["1"])[0]) + counter_key = f"/flaky/{name}" + with self.media_server.lock: + seen = self.media_server.counters.get(counter_key, 0) + self.media_server.counters[counter_key] = seen + 1 + if seen < fail_times: + self._send_body(500, b"server error", "text/plain") + return + content = self._content_for(name) + self._send_body(200, content, DEFAULT_CONTENT_TYPE) + + def _handle_redirect(self, name: str) -> None: + self.send_response(302) + self.send_header("Location", f"/ok/{name}") + self.send_header("Content-Length", "0") + self.end_headers() + + def _handle_ctyped(self, name: str) -> None: + """URL 无扩展名，靠 Content-Type 推断""" + content_type = self.media_server.content_types.get(name, "image/webp") + self._send_body(200, self._content_for(name), content_type) + + def _handle_slow(self, name: str, query) -> None: + delay = float(query.get("s", ["1"])[0]) + time.sleep(delay) + self._send_body(200, self._content_for(name), DEFAULT_CONTENT_TYPE) + + +class MediaTestServer(ThreadingHTTPServer): + """带请求记录能力的本地测试服务器""" + + daemon_threads = True + allow_reuse_address = True + + def __init__(self) -> None: + super().__init__(("127.0.0.1", 0), _MediaRequestHandler) + # handler 通过 server.media_server 反向访问这些状态 + self.media_server: "MediaTestServer" = self + self.requests: List[RequestRecord] = [] + self.counters: Dict[str, int] = {} + self.contents: Dict[str, bytes] = {} + self.content_types: Dict[str, str] = {} + self.lock = threading.Lock() + + # ------------------------------------------------------------------ 测试辅助 + + @property + def base_url(self) -> str: + host, port = self.server_address[:2] + return f"http://{host}:{port}" + + def url(self, path: str) -> str: + return f"{self.base_url}{path}" + + def reset(self) -> None: + """清空请求记录与计数器（内容配置保留）""" + with self.lock: + self.requests.clear() + self.counters.clear() + + def set_content(self, name: str, content: bytes) -> None: + self.contents[name] = content + + def set_content_type(self, name: str, content_type: str) -> None: + self.content_types[name] = content_type + + @property + def request_count(self) -> int: + return len(self.requests) + + def paths(self) -> List[str]: + return [record.path for record in self.requests] + + def last_request(self) -> Optional[RequestRecord]: + return self.requests[-1] if self.requests else None + + +def start_server() -> Tuple[MediaTestServer, threading.Thread]: + """在后台线程启动服务器，返回 (server, thread)""" + server = MediaTestServer() + thread = threading.Thread(target=server.serve_forever, daemon=True) + thread.start() + return server, thread diff --git a/tests/test_api_limits.py b/tests/test_api_limits.py index 0cb65bc..aeee264 100644 --- a/tests/test_api_limits.py +++ b/tests/test_api_limits.py @@ -62,6 +62,35 @@ def test_crawler_manager_build_command(): idx_comments = cmd2.index("--max_comments_count_singlenotes") assert cmd2[idx_comments + 1] == "5" + +def test_crawler_manager_passes_media_switch(): + cm = CrawlerManager() + + req_off = CrawlerStartRequest( + platform=PlatformEnum.XHS, + login_type=LoginTypeEnum.QRCODE, + crawler_type=CrawlerTypeEnum.DETAIL, + specified_ids="note-1", + ) + cmd_off = cm._build_command(req_off) + idx_off = cmd_off.index("--get_media") + assert cmd_off[idx_off + 1] == "false" + + req_on = CrawlerStartRequest( + platform=PlatformEnum.XHS, + login_type=LoginTypeEnum.QRCODE, + crawler_type=CrawlerTypeEnum.DETAIL, + specified_ids="note-1", + enable_media=True, + ) + cmd_on = cm._build_command(req_on) + idx_on = cmd_on.index("--get_media") + assert cmd_on[idx_on + 1] == "true" + + +def test_api_schema_exposes_media_switch_default_off(): + assert CrawlerStartRequest(platform=PlatformEnum.XHS).enable_media is False + def test_api_start_crawler_with_limits(): client = TestClient(app) diff --git a/tests/test_cmd_arg_media.py b/tests/test_cmd_arg_media.py new file mode 100644 index 0000000..13ac3a1 --- /dev/null +++ b/tests/test_cmd_arg_media.py @@ -0,0 +1,73 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_cmd_arg_media.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""--get_media 命令行开关测试。 + +注意：parse_cmd 会就地覆盖全局 config，因此每个用例都用 monkeypatch 还原， +避免污染其他测试。 +""" + +from __future__ import annotations + +import config +import pytest +from cmd_arg import parse_cmd + + +@pytest.fixture(autouse=True) +def _isolate_config(monkeypatch): + monkeypatch.setattr(config, "ENABLE_GET_MEDIA", False) + monkeypatch.setattr(config, "PLATFORM", "xhs") + monkeypatch.setattr(config, "CRAWLER_TYPE", "search") + yield + + +BASE_ARGS = ["--platform", "xhs", "--type", "detail", "--specified_id", "note-1"] + + +@pytest.mark.asyncio +async def test_get_media_true_enables_switch(): + await parse_cmd([*BASE_ARGS, "--get_media", "true"]) + + assert config.ENABLE_GET_MEDIA is True + + +@pytest.mark.asyncio +async def test_get_media_false_keeps_switch_off(): + await parse_cmd([*BASE_ARGS, "--get_media", "false"]) + + assert config.ENABLE_GET_MEDIA is False + + +@pytest.mark.asyncio +async def test_get_media_defaults_to_config_value(monkeypatch): + monkeypatch.setattr(config, "ENABLE_GET_MEDIA", True) + + args = await parse_cmd(BASE_ARGS) + + assert config.ENABLE_GET_MEDIA is True + assert args.get_media is True + + +@pytest.mark.parametrize( + ("raw", "expected"), + [("yes", True), ("y", True), ("1", True), ("no", False), ("n", False), ("0", False)], +) +@pytest.mark.asyncio +async def test_get_media_accepts_boolean_aliases(raw, expected): + await parse_cmd([*BASE_ARGS, "--get_media", raw]) + + assert config.ENABLE_GET_MEDIA is expected + + +@pytest.mark.asyncio +async def test_get_media_returns_value_in_namespace(): + args = await parse_cmd([*BASE_ARGS, "--get_media", "true"]) + + assert args.get_media is True diff --git a/tests/test_media_core_integration.py b/tests/test_media_core_integration.py new file mode 100644 index 0000000..2296204 --- /dev/null +++ b/tests/test_media_core_integration.py @@ -0,0 +1,513 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_core_integration.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""平台 core 的 ``download_media`` 与统一下载器的集成测试。 + +用本地 HTTP 服务器冒充平台 CDN，验证「core 编排 -> 平台 URL 提取 -> 统一下载器 -> 落盘」 +的完整链路，不依赖外部网络、浏览器与登录态。 +""" + +from __future__ import annotations + +import asyncio +from pathlib import Path +from types import SimpleNamespace + +import config +import httpx +import pytest + +from media_platform.bilibili.core import BilibiliCrawler +from media_platform.douyin.core import DouYinCrawler +from media_platform.kuaishou.core import KuaishouCrawler +from media_platform.weibo.core import WeiboCrawler +from media_platform.xhs.core import XiaoHongShuCrawler +from tests.media_server import DEFAULT_CONTENT, MediaTestServer, start_server + + +@pytest.fixture(scope="module") +def server(): + srv, _thread = start_server() + yield srv + srv.shutdown() + srv.server_close() + + +@pytest.fixture(autouse=True) +def _reset_server(server: MediaTestServer): + server.reset() + + +@pytest.fixture(autouse=True) +def _media_config(monkeypatch, tmp_path: Path): + """打开媒体下载开关并把落盘根目录指到临时目录""" + monkeypatch.setattr(config, "ENABLE_GET_MEDIA", True) + monkeypatch.setattr(config, "SAVE_DATA_PATH", str(tmp_path)) + monkeypatch.setattr(config, "XHS_INTERNATIONAL", False) + # 平台在媒体下载之间会按爬虫限速 sleep，测试里不需要真的等 + monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 0) + + +def media_dir(tmp_path: Path, platform: str, content_id: str) -> Path: + return tmp_path / platform / "media" / content_id + + +def assert_downloaded(directory: Path, filename: str) -> Path: + path = directory / filename + assert path.is_file(), f"缺少文件 {path}，目录内容: {sorted(p.name for p in directory.iterdir())}" + assert path.stat().st_size == len(DEFAULT_CONTENT) + return path + + +# --------------------------------------------------------------------------- xhs + + +@pytest.mark.asyncio +async def test_xhs_core_downloads_cover_and_video(server, tmp_path): + note = { + "note_id": "xhs-e2e-video", + "type": "video", + "image_list": [], + "video": { + # 不带 origin_video_key，走 master_url 分支以便指向本地服务器 + "media": {"stream": {"h264": [{"master_url": server.url("/ok/xhs-video?ct=video/mp4")}]}}, + "cover": {"url_default": server.url("/ok/xhs-cover")}, + }, + } + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + await crawler.download_media(note) + + directory = media_dir(tmp_path, "xhs", "xhs-e2e-video") + assert_downloaded(directory, "cover.jpg") + assert_downloaded(directory, "video.mp4") + assert server.request_count == 2 + + +@pytest.mark.asyncio +async def test_xhs_core_downloads_numbered_images(server, tmp_path): + note = { + "note_id": "xhs-e2e-images", + "type": "normal", + "image_list": [ + {"url_default": server.url("/ok/xhs-img-1")}, + {"url_default": server.url("/ok/xhs-img-2")}, + ], + } + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + await crawler.download_media(note) + + directory = media_dir(tmp_path, "xhs", "xhs-e2e-images") + assert_downloaded(directory, "001.jpg") + assert_downloaded(directory, "002.jpg") + assert not (directory / "cover.jpg").exists(), "图文笔记不应再单独下载一份封面" + + +@pytest.mark.asyncio +async def test_xhs_core_sends_referer(server, tmp_path): + note = { + "note_id": "xhs-e2e-referer", + "type": "normal", + "image_list": [{"url_default": server.url("/ok/xhs-referer")}], + } + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + await crawler.download_media(note) + + assert server.requests[0].headers.get("referer") == "https://www.xiaohongshu.com/" + assert server.requests[0].headers.get("user-agent") == "integration-test-agent" + + +# --------------------------------------------------------------------------- douyin + + +@pytest.mark.asyncio +async def test_douyin_core_downloads_cover_and_video(server, tmp_path): + aweme = { + "aweme_id": "dy-e2e-video", + "video": { + "play_addr_h264": {"url_list": [server.url("/ok/dy-video?ct=video/mp4")]}, + "raw_cover": {"url_list": [server.url("/ok/dy-cover")]}, + }, + } + crawler = DouYinCrawler() + # client.headers 里带上 Cookie，用于验证它不会被透传到 CDN 请求上 + crawler.dy_client = SimpleNamespace( + proxy=None, headers={"User-Agent": "dy-agent", "Cookie": "session=secret"} + ) + + await crawler.download_media(aweme) + + directory = media_dir(tmp_path, "dy", "dy-e2e-video") + assert_downloaded(directory, "cover.jpg") + assert_downloaded(directory, "video.mp4") + sent = server.requests[0].headers + assert sent.get("referer") == "https://www.douyin.com/" + assert sent.get("user-agent") == "dy-agent" + assert sent.get("cookie") is None, "不应把平台 Cookie 带到 CDN 请求上" + + +@pytest.mark.asyncio +async def test_douyin_core_downloads_images(server, tmp_path): + aweme = { + "aweme_id": "dy-e2e-images", + "images": [ + {"url_list": [server.url("/ok/dy-img-1")]}, + {"url_list": [server.url("/ok/dy-img-2")]}, + ], + } + crawler = DouYinCrawler() + crawler.dy_client = SimpleNamespace(proxy=None, headers={}) + + await crawler.download_media(aweme) + + directory = media_dir(tmp_path, "dy", "dy-e2e-images") + assert_downloaded(directory, "001.jpg") + assert_downloaded(directory, "002.jpg") + + +# --------------------------------------------------------------------------- kuaishou + + +@pytest.mark.asyncio +async def test_kuaishou_core_downloads_cover_and_video(server, tmp_path): + video_item = { + "photo": { + "id": "ks-e2e-video", + "photoUrl": server.url("/ok/ks-video?ct=video/mp4"), + "coverUrl": server.url("/ok/ks-cover"), + } + } + crawler = KuaishouCrawler() + crawler.ks_client = SimpleNamespace(proxy=None, headers={}) + + await crawler.download_media(video_item) + + directory = media_dir(tmp_path, "ks", "ks-e2e-video") + assert_downloaded(directory, "cover.jpg") + assert_downloaded(directory, "video.mp4") + assert server.requests[0].headers.get("referer") == "https://www.kuaishou.com/" + + +# --------------------------------------------------------------------------- weibo + + +@pytest.mark.asyncio +async def test_weibo_core_downloads_cover_and_video(server, tmp_path): + mblog = { + "id": "wb-e2e-video", + "page_info": { + "type": "video", + "page_pic": {"url": server.url("/ok/wb-cover")}, + "media_info": {"mp4_hd_mp4": server.url("/ok/wb-video?ct=video/mp4")}, + }, + } + crawler = WeiboCrawler() + crawler.wb_client = SimpleNamespace(proxy=None, headers={}) + + await crawler.download_media(mblog) + + directory = media_dir(tmp_path, "wb", "wb-e2e-video") + assert_downloaded(directory, "cover.jpg") + assert_downloaded(directory, "video.mp4") + + +# --------------------------------------------------------------------------- bilibili + + +@pytest.mark.asyncio +async def test_bilibili_core_downloads_cover_even_without_play_info(server, tmp_path, monkeypatch): + """playurl 拿不到时，封面仍应正常落盘（两者是独立的下载任务）""" + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + async def fake_play_url(*args, **kwargs): + return None + + monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url) + + video_item = { + "View": { + "aid": 1, + "cid": 2, + "bvid": "BV1e2e", + "pic": server.url("/ok/bili-cover"), + } + } + + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + directory = media_dir(tmp_path, "bili", "BV1e2e") + assert_downloaded(directory, "cover.jpg") + assert not (directory / "video.mp4").exists() + + +@pytest.mark.asyncio +async def test_bilibili_core_skips_dash_when_ffmpeg_unavailable(server, tmp_path, monkeypatch): + """ffmpeg 不可用时应直接走直链：DASH 流的地址一次都不该被请求""" + import media_platform.bilibili.core as bili_core + from media_platform.bilibili.media import DASH_FNVAL, MP4_FNVAL + + monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: False) + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + requested_fnvals = [] + + async def fake_play_url(aid, cid, semaphore, fnval=None): + requested_fnvals.append(fnval if fnval is not None else DASH_FNVAL) + return { + "dash": { + "video": [{"id": 80, "base_url": server.url("/ok/dash-video-unused?ct=video/mp4")}], + "audio": [{"id": 30280, "base_url": server.url("/ok/dash-audio-unused?ct=audio/mp4")}], + }, + "durl": [{"url": server.url("/ok/bili-durl?ct=video/mp4"), "size": 100}], + } + + monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url) + + video_item = {"View": {"aid": 1, "cid": 2, "bvid": "BV1fallback", "pic": server.url("/ok/bili-cover2")}} + + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + directory = media_dir(tmp_path, "bili", "BV1fallback") + # 直链产物刻意与 DASH 产物（video.mp4）区分，避免低清文件阻塞后续的高清路径 + assert_downloaded(directory, "video-durl.mp4") + assert requested_fnvals == [MP4_FNVAL], "ffmpeg 不可用时应直接按 MP4 格式请求，不做无用的 DASH 请求" + + requested_paths = server.paths() + assert not any("unused" in path for path in requested_paths), ( + f"ffmpeg 不可用时不应下载 DASH 分轨，实际请求: {requested_paths}" + ) + + +@pytest.mark.asyncio +async def test_core_download_media_never_raises_on_malformed_payload(server, tmp_path): + """平台返回畸形结构时，download_media 只能记日志，不能把异常抛给爬取主流程""" + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + # video 字段是字符串，提取器若不做防御会 AttributeError + await crawler.download_media({"note_id": "x", "type": "video", "video": "oops"}) + + assert server.request_count == 0 + + +@pytest.mark.asyncio +async def test_core_survives_extractor_exception(server, tmp_path, monkeypatch): + """提取器抛异常时必须被 core 的壳层兜住（直接锁定 try/except，去掉它此用例会失败）""" + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + def boom(_raw): + raise RuntimeError("extractor exploded") + + monkeypatch.setattr("media_platform.xhs.core.xhs_media.build_media_items", boom) + + await crawler.download_media({"note_id": "boom"}) + + assert server.request_count == 0 + + +@pytest.mark.asyncio +async def test_bilibili_dash_success_skips_durl_fallback(server, tmp_path, monkeypatch): + """DASH 下载成功后必须立即结束，不能再下一份低清直链。 + + 这是 B 站的主路径（装好 ffmpeg 的机器），此前完全没有测试覆盖。 + """ + import media_downloader.downloader as downloader_module + import media_platform.bilibili.core as bili_core + + monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True) + + async def fake_merge(video_path, audio_path, output_path, timeout=300.0): + output_path.write_bytes(b"merged-mp4") + + monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge) + + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + async def fake_play_url(aid, cid, semaphore, fnval=None): + return { + "dash": { + "video": [{"id": 80, "codecid": 7, "base_url": server.url("/ok/dash-best?ct=video/mp4")}], + "audio": [{"id": 30280, "base_url": server.url("/ok/dash-audio?ct=audio/mp4")}], + }, + "durl": [{"url": server.url("/ok/low-quality-durl?ct=video/mp4"), "size": 1}], + } + + monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url) + + video_item = { + "View": {"aid": 1, "cid": 2, "bvid": "BV1dashok", "pic": server.url("/ok/bili-cover-5")} + } + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + directory = media_dir(tmp_path, "bili", "BV1dashok") + assert (directory / "video.mp4").is_file(), "DASH 合流产物应落在 video.mp4" + assert not (directory / "video-durl.mp4").exists(), "DASH 成功后不应再走直链" + assert not any("low-quality" in path for path in server.paths()), ( + f"DASH 成功时不应请求直链，实际请求: {server.paths()}" + ) + + +@pytest.mark.asyncio +async def test_bilibili_falls_back_to_lower_quality_when_cdn_rejects(server, tmp_path, monkeypatch): + """高清流取流被 CDN 拒绝时（未登录/权限不足的真实场景），应逐档降级而不是直接放弃""" + import media_downloader.downloader as downloader_module + import media_platform.bilibili.core as bili_core + + monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True) + monkeypatch.setattr(config, "BILI_QN", 80) + + async def fake_merge(video_path, audio_path, output_path, timeout=300.0): + output_path.write_bytes(b"merged-sd") + + monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge) + + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + async def fake_play_url(aid, cid, semaphore, fnval=None): + return { + "dash": { + "video": [ + {"id": 80, "codecid": 7, "base_url": server.url("/status/403")}, # 高清被 CDN 拒绝 + {"id": 32, "codecid": 7, "base_url": server.url("/ok/dash-sd?ct=video/mp4")}, + ], + "audio": [{"id": 30280, "base_url": server.url("/ok/dash-sd-audio?ct=audio/mp4")}], + } + } + + monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url) + + video_item = {"View": {"aid": 1, "cid": 2, "bvid": "BV1degrade", "pic": server.url("/ok/bili-cover-8")}} + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + directory = media_dir(tmp_path, "bili", "BV1degrade") + assert (directory / "video.mp4").read_bytes() == b"merged-sd" + assert any("dash-sd" in path for path in server.paths()), "应降级到低清晰度流" + + +@pytest.mark.asyncio +async def test_bilibili_durl_does_not_block_later_dash_download(server, tmp_path, monkeypatch): + """先前降级下载的低清直链产物，不能阻止之后（装好 ffmpeg）走 DASH 拿高清""" + import media_downloader.downloader as downloader_module + import media_platform.bilibili.core as bili_core + + directory = media_dir(tmp_path, "bili", "BV1upgrade") + directory.mkdir(parents=True) + (directory / "video-durl.mp4").write_bytes(b"low-resolution-durl-file") + + monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True) + + async def fake_merge(video_path, audio_path, output_path, timeout=300.0): + output_path.write_bytes(b"high-resolution-dash-file") + + monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge) + + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + async def fake_play_url(aid, cid, semaphore, fnval=None): + return { + "dash": { + "video": [{"id": 80, "codecid": 7, "base_url": server.url("/ok/dash-up?ct=video/mp4")}], + "audio": [{"id": 30280, "base_url": server.url("/ok/dash-up-a?ct=audio/mp4")}], + } + } + + monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url) + + video_item = { + "View": {"aid": 1, "cid": 2, "bvid": "BV1upgrade", "pic": server.url("/ok/bili-cover-6")} + } + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + assert (directory / "video.mp4").read_bytes() == b"high-resolution-dash-file" + assert server.request_count > 0, "已有的低清直链产物不应让 DASH 路径被跳过" + + +@pytest.mark.asyncio +async def test_bilibili_download_media_survives_play_url_failure(server, tmp_path, monkeypatch): + """playurl 抛出非 DataFetchError 的异常（如网络超时）时，壳层必须兜住。 + + 这条用例锁定的正是 core 里的 try/except：去掉它，异常会直接击穿爬取主流程。 + """ + crawler = BilibiliCrawler() + crawler.bili_client = SimpleNamespace(proxy=None, headers={}) + + async def boom(*args, **kwargs): + raise httpx.ReadTimeout("simulated timeout") + + monkeypatch.setattr(crawler, "get_video_play_url_task", boom) + + video_item = { + "View": {"aid": 1, "cid": 2, "bvid": "BV1boom", "pic": server.url("/ok/bili-cover-3")} + } + + await crawler.download_media(video_item, asyncio.Semaphore(1)) + + # 封面在 playurl 之前下载，失败不影响它 + directory = media_dir(tmp_path, "bili", "BV1boom") + assert_downloaded(directory, "cover.jpg") + assert not (directory / "video.mp4").exists() + + +# --------------------------------------------------------------------------- 开关 + + +@pytest.mark.asyncio +async def test_media_switch_off_downloads_nothing(server, tmp_path, monkeypatch): + monkeypatch.setattr(config, "ENABLE_GET_MEDIA", False) + note = { + "note_id": "xhs-switch-off", + "type": "normal", + "image_list": [{"url_default": server.url("/ok/should-not-download")}], + } + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + await crawler.download_media(note) + + assert server.request_count == 0 + assert not media_dir(tmp_path, "xhs", "xhs-switch-off").exists() + + +@pytest.mark.asyncio +async def test_download_all_keeps_going_after_one_failure(server, tmp_path): + """单个媒体失败不应影响同帖其它媒体""" + note = { + "note_id": "xhs-partial-failure", + "type": "normal", + "image_list": [ + {"url_default": server.url("/status/500")}, + {"url_default": server.url("/ok/xhs-good-image")}, + ], + } + crawler = XiaoHongShuCrawler() + crawler.xhs_client = SimpleNamespace(proxy=None) + crawler.user_agent = "integration-test-agent" + + await crawler.download_media(note) + + directory = media_dir(tmp_path, "xhs", "xhs-partial-failure") + assert not (directory / "001.jpg").exists() + assert_downloaded(directory, "002.jpg") diff --git a/tests/test_media_downloader.py b/tests/test_media_downloader.py new file mode 100644 index 0000000..bd99727 --- /dev/null +++ b/tests/test_media_downloader.py @@ -0,0 +1,890 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_downloader.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""MediaDownloader 单测。 + +全部基于 ``tests/media_server.py`` 提供的本地 HTTP 服务器，真实走 socket 与 httpx， +不使用任何 mock，也不访问外网。 +""" + +from __future__ import annotations + +import os +import shutil +import subprocess + +import httpx +from pathlib import Path + +import pytest + +from media_downloader import MediaDownloader, MediaItem, MediaType +from media_downloader.paths import ( + build_media_dir, + build_media_path, + ensure_within, + guess_extension, + redact_url, + sanitize_component, + url_fingerprint, +) +from tests.media_server import DEFAULT_CONTENT, MediaTestServer, start_server + + +@pytest.fixture(scope="module") +def server(): + srv, _thread = start_server() + yield srv + srv.shutdown() + srv.server_close() + + +@pytest.fixture(autouse=True) +def _reset_server(server: MediaTestServer): + server.reset() + yield + + +@pytest.fixture +def downloader(server: MediaTestServer, tmp_path: Path) -> MediaDownloader: + """重试间隔调到毫秒级，避免测试因退避等待变慢""" + return MediaDownloader( + platform="xhs", + base_dir=tmp_path, + max_retries=2, + retry_base_delay=0.001, + retry_max_delay=0.005, + ) + + +def make_item(server: MediaTestServer, path: str, **overrides) -> MediaItem: + params = { + "url": server.url(path), + "media_type": MediaType.IMAGE, + "content_id": "note-1", + "stem": "001", + } + params.update(overrides) + return MediaItem(**params) + + +def part_path_for(tmp_path: Path, item: MediaItem, url: str) -> Path: + directory = build_media_dir(tmp_path, "xhs", item.content_id) + directory.mkdir(parents=True, exist_ok=True) + return directory / f"{item.stem}.{url_fingerprint(url)}.{os.getpid()}.part" + + +# --------------------------------------------------------------------------- 基础 + + +@pytest.mark.asyncio +async def test_download_success_and_layout(server, downloader, tmp_path): + item = make_item(server, "/ok/photo") + result = await downloader.download(item) + + assert result == tmp_path / "xhs" / "media" / "note-1" / "001.jpg" + assert result.read_bytes() == DEFAULT_CONTENT + assert not list(result.parent.glob("*.part")), "下载完成后不应残留临时文件" + + +@pytest.mark.asyncio +async def test_download_all_shares_one_client(server, downloader, monkeypatch): + """同一帖子的多个媒体必须复用同一个 AsyncClient（而不是每个文件新建一个连接池）""" + import media_downloader.downloader as downloader_module + + created_clients = [] + original = downloader_module.make_async_client + + def counting_make_async_client(**kwargs): + client = original(**kwargs) + created_clients.append(client) + return client + + monkeypatch.setattr(downloader_module, "make_async_client", counting_make_async_client) + + items = [ + make_item(server, "/ok/a", stem="001"), + make_item(server, "/ok/b", stem="002"), + make_item(server, "/ok/c", stem="003"), + ] + paths = await downloader.download_all(items) + + assert len(paths) == 3 + assert [path.name for path in paths] == ["001.jpg", "002.jpg", "003.jpg"] + assert server.request_count == 3 + assert len(created_clients) == 1, f"3 个文件应共用一个 client，实际创建了 {len(created_clients)} 个" + + +@pytest.mark.asyncio +async def test_download_all_empty(downloader): + assert await downloader.download_all([]) == [] + + +# --------------------------------------------------------------------------- 跳过与覆盖 + + +@pytest.mark.asyncio +async def test_existing_file_is_skipped_without_request(server, downloader): + item = make_item(server, "/ok/photo") + first = await downloader.download(item) + assert first is not None + + server.reset() + second = await downloader.download(item) + + assert second == first + assert server.request_count == 0, "已下载完成的文件不应再次发起请求" + + +@pytest.mark.asyncio +async def test_overwrite_forces_redownload(server, tmp_path): + downloader = MediaDownloader( + platform="xhs", + base_dir=tmp_path, + max_retries=0, + overwrite=True, + ) + item = make_item(server, "/ok/photo") + await downloader.download(item) + + server.reset() + await downloader.download(item) + + assert server.request_count == 1 + + +@pytest.mark.asyncio +async def test_zero_byte_file_is_treated_as_broken(server, downloader): + item = make_item(server, "/ok/photo") + target = downloader.base_dir / "xhs" / "media" / "note-1" / "001.jpg" + target.parent.mkdir(parents=True, exist_ok=True) + target.write_bytes(b"") + + server.reset() + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 1 + + +# --------------------------------------------------------------------------- 续传 + + +@pytest.mark.asyncio +async def test_resume_from_partial_file(server, downloader, tmp_path): + """预置半个片段 -> 必须带 Range 请求，且大小校验以 Content-Range 的总长为准。 + + 这是关键回归：206 响应的 Content-Length 只是剩余长度（3072）， + 若被误当成文件总长，4096 != 3072 会误判为失败。 + """ + item = make_item(server, "/ok/photo") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:1024]) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 1 + assert server.requests[0].headers.get("range") == "bytes=1024-" + + +@pytest.mark.asyncio +async def test_server_ignoring_range_falls_back_to_full_download(server, downloader, tmp_path): + item = make_item(server, "/no-range/photo") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:1024]) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT, "服务端不支持 Range 时应截断重写，而不是拼接" + + +@pytest.mark.asyncio +async def test_truncated_response_retries_with_range(server, downloader, tmp_path): + """服务端中途断流 -> 重试时带 Range 续传 -> 最终拼接出完整文件""" + item = make_item(server, "/truncated/photo?fail=1") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 2 + assert server.requests[1].headers.get("range") == f"bytes={len(DEFAULT_CONTENT) // 2}-" + + +@pytest.mark.asyncio +async def test_stale_part_larger_than_remote_triggers_416_restart(server, downloader, tmp_path): + """本地残留片段比远端还大 -> 416 -> 清空片段重来""" + item = make_item(server, "/ok/photo") + oversized = len(DEFAULT_CONTENT) + 5000 + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(b"z" * oversized) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + # 第一次带越界的 Range 拿到 416，片段被丢弃后第二次必须不带 Range 重新下载 + assert server.request_count == 2 + assert server.requests[0].headers.get("range") == f"bytes={oversized}-" + assert server.requests[1].headers.get("range") is None + + +@pytest.mark.asyncio +async def test_mismatched_content_range_start_discards_partial_file(server, downloader, tmp_path): + """服务端返回的 Range 起点与请求不符时必须丢弃片段重下，而不是把错位内容拼进去""" + item = make_item(server, "/wrong-range/photo?delta=100&fail=1") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:1024]) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 2 + assert server.requests[0].headers.get("range") == "bytes=1024-" + assert server.requests[1].headers.get("range") is None, "片段被丢弃后应重新完整下载" + + +@pytest.mark.asyncio +async def test_mismatched_content_range_never_produces_corrupt_file(server, downloader, tmp_path): + """服务端持续返回错位 Range 时应放弃下载，绝不落盘内容错位的文件""" + item = make_item(server, "/wrong-range/broken?delta=100") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:1024]) + + result = await downloader.download(item) + + assert result is None + directory = build_media_dir(downloader.base_dir, "xhs", item.content_id) + assert not list(directory.glob("*.jpg")), "内容错位的响应不能落盘" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("bad_content_type", ["text/html", "text/plain", "application/json"]) +async def test_error_page_with_200_is_rejected(server, downloader, bad_content_type): + """CDN 防盗链页常以 200 + text/html 返回，不能当成媒体文件落盘""" + server.set_content_type("errpage", bad_content_type) + item = make_item(server, "/ctyped/errpage") + + result = await downloader.download(item) + + assert result is None + directory = build_media_dir(downloader.base_dir, "xhs", item.content_id) + assert not list(directory.glob("*.jpg")), "错误页不应被保存为媒体文件" + + +@pytest.mark.asyncio +async def test_content_type_mismatch_preserves_partial_progress(server, downloader, tmp_path): + """错误页在读 body 之前就被拦下，本地已有的片段必须保留以便续传""" + server.set_content_type("limited", "text/html") + item = make_item(server, "/ctyped/limited") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:3000]) + + result = await downloader.download(item) + + assert result is None + # 每一次尝试都要基于已有片段续传，而不是丢掉 3000 字节从头再来 + ranges = [record.headers.get("range") for record in server.requests] + assert ranges == ["bytes=3000-"] * server.request_count, ( + f"被拦截的响应没有消耗任何字节，已下载的进度不应被丢弃，实际: {ranges}" + ) + + +@pytest.mark.asyncio +async def test_unsolicited_partial_response_is_rejected(server, downloader): + """未发 Range 却收到起点非 0 的 206 时必须拒绝。 + + 这里用"长度自洽但内容错位"的毒响应，确保拦住它的是起点守卫本身， + 而不是被大小校验代偿（后者在长度恰好吻合时就失效了）。 + """ + item = make_item(server, "/poison-206/photo?start=100") + + result = await downloader.download(item) + + assert result is None + directory = build_media_dir(downloader.base_dir, "xhs", item.content_id) + assert not list(directory.glob("*.jpg")), "起点错位的内容不能落盘" + + +@pytest.mark.asyncio +async def test_partial_206_with_unknown_total_is_rejected(server, downloader): + """206 的 total 为 * 时无法确认完整性（服务端可能只给了区间的一部分），必须重下""" + item = make_item(server, "/unknown-total/photo") + + result = await downloader.download(item) + + assert result is None + directory = build_media_dir(downloader.base_dir, "xhs", item.content_id) + assert not list(directory.glob("*.jpg")), "无法确认完整性的内容不能落盘" + + +@pytest.mark.asyncio +async def test_partial_response_without_content_range_is_rejected(server, downloader, tmp_path): + """206 缺少 Content-Range 时服务端行为不可预期（可能回全量体），必须丢弃片段重下""" + item = make_item(server, "/no-content-range/photo") + part = part_path_for(tmp_path, item, item.url) + part.write_bytes(DEFAULT_CONTENT[:1024]) + + result = await downloader.download(item) + + assert result is None + assert not part.exists(), "无法确认区间语义的片段必须删除" + + +def test_total_size_semantics(): + """总长只能来自 Content-Range；total 未知时不反推，chunked 时忽略 Content-Length""" + # 206：以 Content-Range 的 total 为准 + response = httpx.Response(206, headers={"content-range": "bytes 100-999/1000"}) + content_range = MediaDownloader._parse_content_range(response) + assert content_range == (100, 999, 1000) + assert MediaDownloader._resolve_total_size(content_range, response, 206) == 1000 + + # total 为 *（服务端只返回部分区间）：不能拿 剩余长度+已下载量 反推出"总长" + response = httpx.Response( + 206, headers={"content-range": "bytes 100-999/*", "content-length": "900"} + ) + content_range = MediaDownloader._parse_content_range(response) + assert content_range == (100, 999, None) + assert MediaDownloader._resolve_total_size(content_range, response, 206) is None + + # 缺少 Content-Range 的 206 无法判定区间语义 + assert MediaDownloader._resolve_total_size(None, httpx.Response(206), 206) is None + + # chunked 响应即使带了 Content-Length 也必须忽略（RFC 7230） + response = httpx.Response(200, headers={"transfer-encoding": "chunked", "content-length": "999"}) + assert MediaDownloader._resolve_total_size(None, response, 200) is None + + # 普通 200 用 Content-Length + response = httpx.Response(200, headers={"content-length": "4096"}) + assert MediaDownloader._resolve_total_size(None, response, 200) == 4096 + + +@pytest.mark.asyncio +async def test_chunked_response_ignores_bogus_content_length(server, downloader): + """RFC 7230：有 Transfer-Encoding 时必须忽略 Content-Length，否则完整响应会被误判""" + item = make_item(server, "/chunked-bogus-cl/photo") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + + +@pytest.mark.asyncio +async def test_gzip_response_still_lands_decoded_bytes(server, downloader): + """服务端无视 identity 强制压缩时，落盘的应是解码后的原始字节""" + item = make_item(server, "/gzip/photo") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + + +@pytest.mark.asyncio +async def test_malformed_url_in_backup_chain_does_not_abort(server, downloader): + """畸形备用地址不能终止整条回退链（httpx.InvalidURL 不是 HTTPError 的子类）""" + item = make_item( + server, + "/status/500", + backup_urls=("http://h/a\nb.jpg", server.url("/ok/backup-after-bad")), + ) + + result = await downloader.download(item) + + assert result is not None, "畸形地址应被跳过，继续尝试后面的候选" + + +@pytest.mark.asyncio +async def test_backup_candidates_are_capped(server, tmp_path): + """备用地址数量必须有上限，否则超长 url_list 会放大成几十次请求""" + downloader = MediaDownloader("dy", base_dir=tmp_path, max_retries=0, max_candidates=2) + item = make_item( + server, + "/status/500", + backup_urls=tuple(server.url(f"/status/50{i}") for i in range(10)), + ) + + result = await downloader.download(item) + + assert result is None + assert server.request_count == 2, f"候选总数应被裁剪到 2，实际请求 {server.request_count} 次" + + +@pytest.mark.asyncio +async def test_non_string_url_never_escapes_contract(server, downloader): + """MediaItem 是公开契约，非字符串 url 必须收敛为 None 而不是抛异常""" + for bad_url in (123, ["http://x"], {"url": "http://x"}, None): + item = MediaItem(url=bad_url, media_type=MediaType.IMAGE, content_id="c") + assert await downloader.download(item) is None + + +@pytest.mark.asyncio +async def test_part_file_name_is_sanitized(server, downloader, tmp_path): + """stem 未清洗时 .part 会写到 base_dir 之外，临时文件名同样要清洗""" + item = make_item(server, "/ok/photo", stem="../../../../tmp/ESCAPED") + + result = await downloader.download(item) + + assert result is not None + assert result.is_relative_to(tmp_path) + assert not list(Path("/tmp").glob("ESCAPED*")), "临时文件不得落在 base_dir 之外" + + +@pytest.mark.asyncio +async def test_update_credentials_switches_proxy(server, tmp_path, monkeypatch): + """代理池就地刷新后，下载器必须跟着换代理，否则长跑时媒体下载会一直走失效代理""" + import media_downloader.downloader as downloader_module + + used_proxies = [] + original = downloader_module.make_async_client + + def spy(**kwargs): + used_proxies.append(kwargs.get("proxy")) + return original(**kwargs) + + monkeypatch.setattr(downloader_module, "make_async_client", spy) + downloader = MediaDownloader("xhs", base_dir=tmp_path, max_retries=0) + + await downloader.download(make_item(server, "/ok/cred-1")) + downloader.update_credentials(proxy="http://new-proxy:8080") + await downloader.download(make_item(server, "/ok/cred-2", stem="002")) + + assert used_proxies == [None, "http://new-proxy:8080"] + + +@pytest.mark.asyncio +async def test_binary_content_type_is_accepted(server, downloader): + """通用二进制类型无法判定内容，应当放行（CDN 常见）""" + server.set_content_type("blob", "application/octet-stream") + item = make_item(server, "/ctyped/blob") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + + +@pytest.mark.asyncio +async def test_video_content_type_accepted_for_audio_stream(server, downloader): + """DASH 音频轨的 Content-Type 是 audio/*，属于视频任务的一部分，必须放行""" + server.set_content_type("audio-track", "audio/mp4") + item = make_item(server, "/ctyped/audio-track", media_type=MediaType.VIDEO, stem="audio") + + result = await downloader.download(item) + + assert result is not None + assert result.suffix == ".m4a" + + +@pytest.mark.asyncio +async def test_stale_part_from_other_url_is_discarded(server, downloader, tmp_path): + """URL 变化时旧的临时片段必须作废（不能把新内容拼到旧半成品上），且下载成功后清理干净""" + item = make_item(server, "/ok/photo") + stale_part = part_path_for(tmp_path, item, server.url("/ok/other")) + stale_part.write_bytes(b"y" * 1024) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.requests[0].headers.get("range") is None, "不同 URL 的片段不应被当作续传基础" + assert not stale_part.exists(), "成功后应清理同 stem 的陈旧片段" + assert not list(result.parent.glob("*.part")) + + +# --------------------------------------------------------------------------- 错误处理 + + +@pytest.mark.asyncio +@pytest.mark.parametrize("status", [403, 404, 410]) +async def test_fatal_status_is_not_retried(server, downloader, status): + item = make_item(server, f"/status/{status}") + + result = await downloader.download(item) + + assert result is None + assert server.request_count == 1, f"HTTP {status} 不应重试" + + +@pytest.mark.asyncio +async def test_server_error_is_retried_until_success(server, tmp_path): + downloader = MediaDownloader( + platform="xhs", + base_dir=tmp_path, + max_retries=3, + retry_base_delay=0.001, + retry_max_delay=0.005, + ) + item = make_item(server, "/flaky/photo?fail=2") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 3 + + +@pytest.mark.asyncio +async def test_retry_exhausted_returns_none(server, downloader): + item = make_item(server, "/status/500") + + result = await downloader.download(item) + + assert result is None + assert server.request_count == 3 # 1 次 + 2 次重试 + + +@pytest.mark.asyncio +async def test_empty_body_is_rejected(server, downloader): + item = make_item(server, "/empty/photo") + + result = await downloader.download(item) + + assert result is None + + +@pytest.mark.asyncio +async def test_timeout_returns_none(server, tmp_path): + downloader = MediaDownloader( + platform="xhs", + base_dir=tmp_path, + timeout=0.2, + max_retries=0, + ) + item = make_item(server, "/slow/photo?s=2") + + result = await downloader.download(item) + + assert result is None + + +@pytest.mark.asyncio +@pytest.mark.parametrize( + "bad_url", + [ + "", + "//cdn.example.com/x.jpg", # 协议相对地址，httpx 无法处理 + "ftp://cdn.example.com/x.jpg", + "not-a-url", + ], +) +async def test_invalid_url_is_rejected_without_request(server, downloader, bad_url): + item = MediaItem(url=bad_url, media_type=MediaType.IMAGE, content_id="note-1") + + assert await downloader.download(item) is None + assert server.request_count == 0 + + +# --------------------------------------------------------------------------- 备用地址 + + +@pytest.mark.asyncio +async def test_backup_url_used_after_primary_fails(server, downloader): + item = make_item( + server, + "/status/403", + backup_urls=(server.url("/ok/backup"),), + ) + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + assert server.request_count == 2 + assert server.paths() == ["/status/403", "/ok/backup"] + + +# --------------------------------------------------------------------------- 响应头行为 + + +@pytest.mark.asyncio +async def test_chunked_response_without_content_length(server, downloader): + item = make_item(server, "/chunked/photo") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + + +@pytest.mark.asyncio +async def test_redirect_is_followed(server, downloader): + item = make_item(server, "/redirect/photo") + + result = await downloader.download(item) + + assert result is not None + assert result.read_bytes() == DEFAULT_CONTENT + + +@pytest.mark.asyncio +async def test_extension_inferred_from_content_type(server, downloader): + server.set_content_type("anon", "image/webp") + item = make_item(server, "/ctyped/anon") + + result = await downloader.download(item) + + assert result is not None + assert result.suffix == ".webp" + + +@pytest.mark.asyncio +async def test_explicit_extension_wins(server, downloader): + item = make_item(server, "/ctyped/anon", extension=".png") + + result = await downloader.download(item) + + assert result is not None + assert result.suffix == ".png" + + +@pytest.mark.asyncio +async def test_request_headers(server, downloader): + item = make_item(server, "/ok/photo") + + await downloader.download(item) + + headers = server.requests[0].headers + assert headers.get("accept-encoding") == "identity", ( + "必须显式声明不压缩，否则解码后字节数与 Content-Length 不等，大小校验会误判" + ) + assert "user-agent" in headers + # 下载器不认识平台，Referer 必须由平台侧注入（见 test_xhs_core_sends_referer） + assert headers.get("referer") is None + + +@pytest.mark.asyncio +async def test_item_headers_override_defaults(server, downloader): + item = make_item(server, "/ok/photo", headers={"Cookie": "session=abc"}) + + await downloader.download(item) + + assert server.requests[0].headers.get("cookie") == "session=abc" + + +@pytest.mark.asyncio +async def test_extra_headers_are_applied(server, tmp_path): + """平台侧注入的 Referer 等反爬头必须真正发出去""" + downloader = MediaDownloader( + platform="bili", + base_dir=tmp_path, + max_retries=0, + extra_headers={"Referer": "https://www.bilibili.com/"}, + ) + item = make_item(server, "/ok/photo") + + await downloader.download(item) + + headers = server.requests[0].headers + assert headers.get("referer") == "https://www.bilibili.com/" + + +@pytest.mark.asyncio +async def test_proxy_is_passed_to_client(server, tmp_path, monkeypatch): + import media_downloader.downloader as downloader_module + + captured: dict = {} + original = downloader_module.make_async_client + + def spy(**kwargs): + captured.update(kwargs) + return original(**kwargs) + + monkeypatch.setattr(downloader_module, "make_async_client", spy) + downloader = MediaDownloader( + platform="xhs", + base_dir=tmp_path, + proxy="http://127.0.0.1:9", + max_retries=0, + ) + item = make_item(server, "/ok/photo") + + # 代理不可用会失败，但我们要断言的是参数透传 + await downloader.download(item) + + assert captured.get("proxy") == "http://127.0.0.1:9" + assert captured.get("follow_redirects") is True + + +# --------------------------------------------------------------------------- 路径安全 + + +@pytest.mark.asyncio +async def test_malicious_content_id_stays_inside_base_dir(server, downloader, tmp_path): + item = make_item( + server, + "/ok/photo", + content_id="../../../etc", + stem="../../passwd", + ) + + result = await downloader.download(item) + + assert result is not None + assert result.resolve().is_relative_to(tmp_path.resolve()) + + +# --------------------------------------------------------------------------- DASH 合流 + + +def _make_dash_streams(tmp_path: Path) -> tuple[bytes, bytes]: + """用 ffmpeg 生成一对真实的 DASH 分轨素材（视频轨 / 音频轨）""" + ffmpeg = shutil.which("ffmpeg") + if not ffmpeg: + pytest.skip("本机未安装 ffmpeg") + video = tmp_path / "_src_video.m4s" + audio = tmp_path / "_src_audio.m4s" + subprocess.run( + [ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", + "-i", "color=c=blue:s=160x120:d=1", "-c:v", "mpeg4", "-f", "mp4", str(video)], + check=True, capture_output=True, + ) + subprocess.run( + [ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi", + "-i", "sine=f=440:d=1", "-c:a", "aac", "-f", "mp4", str(audio)], + check=True, capture_output=True, + ) + return video.read_bytes(), audio.read_bytes() + + +@pytest.mark.asyncio +async def test_dash_streams_are_downloaded_and_merged(server, tmp_path): + video_bytes, audio_bytes = _make_dash_streams(tmp_path) + server.set_content("dash-v", video_bytes) + server.set_content("dash-a", audio_bytes) + downloader = MediaDownloader("bili", base_dir=tmp_path / "out", max_retries=0) + + item = MediaItem( + url=server.url("/ok/dash-v?ct=video/mp4"), + audio_url=server.url("/ok/dash-a?ct=audio/mp4"), + media_type=MediaType.VIDEO, + content_id="BV1test", + stem="video", + ) + result = await downloader.download(item) + + assert result is not None + assert result.suffix == ".mp4" + assert result.stat().st_size > 0 + assert server.request_count == 2, "分轨应各下载一次" + assert not list(result.parent.glob(".tmp-*")), "合流完成后必须清理临时目录" + + probe = shutil.which("ffprobe") + if probe: + stream_types = subprocess.run( + [probe, "-v", "error", "-show_entries", "stream=codec_type", + "-of", "csv=p=0", str(result)], + check=True, capture_output=True, text=True, + ).stdout + assert "video" in stream_types and "audio" in stream_types + + +@pytest.mark.asyncio +async def test_dash_failure_cleans_up_temp_dir(server, tmp_path): + video_bytes, _ = _make_dash_streams(tmp_path) + server.set_content("dash-v2", video_bytes) + downloader = MediaDownloader("bili", base_dir=tmp_path / "out", max_retries=0) + + item = MediaItem( + url=server.url("/ok/dash-v2?ct=video/mp4"), + audio_url=server.url("/status/403"), + media_type=MediaType.VIDEO, + content_id="BV1fail", + stem="video", + ) + result = await downloader.download(item) + + assert result is None + media_dir = tmp_path / "out" / "bili" / "media" / "BV1fail" + assert media_dir.is_dir() + assert not list(media_dir.glob(".tmp-*")), "失败的 DASH 下载同样要清理临时目录" + + +# --------------------------------------------------------------------------- paths 纯函数 + + +@pytest.mark.parametrize( + ("raw", "expected"), + [ + ("note-123_ab", "note-123_ab"), + ("../../etc/passwd", "___etc_passwd"), + ("", "unknown"), + ("...", "unknown"), + ("a" * 100, "a" * 64), + ("CON", "_CON"), + ("con.txt", "_con.txt"), + ], +) +def test_sanitize_component(raw, expected): + assert sanitize_component(raw) == expected + + +@pytest.mark.parametrize("raw", ["..", ".", "", " "]) +def test_sanitize_component_fallback_used_for_empty(raw): + assert sanitize_component(raw, fallback="fb") == "fb" + + +@pytest.mark.parametrize( + ("url", "content_type", "explicit", "expected"), + [ + ("https://cdn.com/a/b.jpg", None, None, ".jpg"), + ("https://cdn.com/a/b.jpeg?x=1", None, None, ".jpeg"), + ("https://cdn.com/a/b.mp4!large", None, None, ".mp4"), + ("https://cdn.com/a/b", "image/webp; charset=utf-8", None, ".webp"), + ("https://cdn.com/a/b.unknownext", "video/mp4", None, ".mp4"), + ("https://cdn.com/a/b.jpg", "image/png", ".png", ".png"), + ("https://cdn.com/a/b.exe", None, None, ".jpg"), + ("", None, None, ".jpg"), + ], +) +def test_guess_extension(url, content_type, explicit, expected): + assert guess_extension(url, content_type=content_type, explicit=explicit, default=".jpg") == expected + + +def test_guess_extension_ignores_explicit_outside_whitelist(): + assert guess_extension("https://cdn.com/a.jpg", explicit=".exe", default=".jpg") == ".jpg" + + +@pytest.mark.parametrize( + ("platform", "content_id", "expected"), + [ + ("xhs", "note-1", "xhs/media/note-1"), + ("dy", "../../evil", "dy/media/___evil"), + ], +) +def test_build_media_dir(tmp_path, platform, content_id, expected): + assert build_media_dir(tmp_path, platform, content_id) == tmp_path / expected + + +def test_build_media_path_normalizes_extension(): + path = build_media_path("/base", "xhs", "note-1", "cover", "JPG") + assert path == Path("/base/xhs/media/note-1/cover.jpg") + + +def test_ensure_within_rejects_escape(tmp_path): + with pytest.raises(ValueError): + ensure_within(tmp_path, tmp_path / ".." / "outside.mp4") + + +def test_url_fingerprint_is_stable_and_short(): + fingerprint = url_fingerprint("https://cdn.com/a.jpg?sig=1") + assert fingerprint == url_fingerprint("https://cdn.com/a.jpg?sig=1") + assert fingerprint != url_fingerprint("https://cdn.com/a.jpg?sig=2") + assert len(fingerprint) == 8 + + +def test_redact_url_drops_query(): + assert redact_url("https://cdn.com/a.jpg?sign=secret&t=1") == "https://cdn.com/a.jpg" + assert redact_url("not-a-url") == "" diff --git a/tests/test_media_e2e.py b/tests/test_media_e2e.py new file mode 100644 index 0000000..e639889 --- /dev/null +++ b/tests/test_media_e2e.py @@ -0,0 +1,172 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_e2e.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""真实平台端到端测试（默认跳过）。 + +会真实访问 B 站接口与 CDN 并下载一个完整视频（约几十 MB），因此只在人工验证时运行： + + MEDIA_E2E=1 .venv/bin/python -m pytest tests/test_media_e2e.py -v + +B 站无需登录即可获取公开视频的基础清晰度，wbi 签名是纯算法， +所以这里不依赖浏览器与账号，只验证「提取 -> 下载 -> 合流」的真实链路。 +请勿把本文件接入 CI：它会对目标平台产生真实流量。 +""" + +from __future__ import annotations + +import os +import shutil +import subprocess +from pathlib import Path + +import config +import httpx +import pytest + +from media_downloader import MediaDownloader, is_ffmpeg_available +from media_platform.bilibili import media as bili_media +from media_platform.bilibili.help import BilibiliSign + +pytestmark = pytest.mark.skipif( + os.getenv("MEDIA_E2E") != "1", + reason="真实平台测试，设置 MEDIA_E2E=1 后运行", +) + +BV_ID = os.getenv("MEDIA_E2E_BVID", "BV1dwuKzmE26") +USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 " + "(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36" +) + + +async def _fetch_wbi_keys(client: httpx.AsyncClient) -> tuple[str, str]: + """从 nav 接口获取 wbi 签名所需的 img_key / sub_key（无需登录）""" + response = await client.get( + "https://api.bilibili.com/x/web-interface/nav", headers={"User-Agent": USER_AGENT} + ) + wbi_img = response.json()["data"]["wbi_img"] + img_key = wbi_img["img_url"].rsplit("/", 1)[1].split(".")[0] + sub_key = wbi_img["sub_url"].rsplit("/", 1)[1].split(".")[0] + return img_key, sub_key + + +@pytest.mark.asyncio +async def test_bilibili_downloads_cover_and_video(tmp_path: Path): + async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: + view_response = await client.get( + "https://api.bilibili.com/x/web-interface/view", + params={"bvid": BV_ID}, + headers={"User-Agent": USER_AGENT}, + ) + view_data = view_response.json()["data"] + aid, cid, pic = view_data["aid"], view_data["cid"], view_data["pic"] + + img_key, sub_key = await _fetch_wbi_keys(client) + params = { + "avid": aid, + "cid": cid, + "qn": 80, + "fourk": 1, + "fnval": bili_media.DASH_FNVAL, + "platform": "pc", + } + signed = BilibiliSign(img_key, sub_key).sign(params) + play_response = await client.get( + "https://api.bilibili.com/x/player/wbi/playurl", + params=signed, + headers={"User-Agent": USER_AGENT, "Referer": "https://www.bilibili.com"}, + ) + payload = play_response.json() + assert payload.get("code") == 0, f"playurl 接口返回异常: {payload}" + play_info = payload["data"] + + view = {"aid": aid, "cid": cid, "bvid": BV_ID, "pic": pic} + cover_item = bili_media.build_cover_item(view, BV_ID) + + # 生产路径由 core._media_headers() 注入 Referer；这里绕过 core 直接构造下载器， + # 必须自行带上，否则 B 站 CDN 一律 403 + downloader = MediaDownloader( + "bili", + base_dir=tmp_path, + max_retries=2, + timeout=120.0, + extra_headers={"Referer": "https://www.bilibili.com/", "User-Agent": USER_AGENT}, + ) + cover_path = await downloader.download(cover_item) + + # 与 core 的编排保持一致：未登录时 B 站会返回高清晰度 URL 但取流被 CDN 403， + # 需要逐档降级到实际可下载的档位 + video_item = None + video_path = None + if is_ffmpeg_available(): + quality = getattr(config, "BILI_QN", 80) + while quality is not None: + candidate = bili_media.build_dash_item(play_info, BV_ID, preferred_quality=quality) + if candidate is None: + break + video_item = candidate + video_path = await downloader.download(candidate) + if video_path is not None: + break + lower_quality = bili_media.next_lower_quality(play_info, quality) + if lower_quality is None: + break + print(f"[e2e] 清晰度 {quality} 取流失败，降级到 {lower_quality}") + quality = lower_quality + + if video_path is None: + video_item = bili_media.build_durl_item(play_info, BV_ID) + if video_item is None: + fallback = await _fetch_play_info_for_mp4(aid, cid) + video_item = bili_media.build_durl_item(fallback or {}, BV_ID) + if video_item is not None: + video_path = await downloader.download(video_item) + + assert video_item is not None, "未能从 playurl 响应中提取到视频流" + + assert cover_path is not None and cover_path.stat().st_size > 0 + assert cover_path.read_bytes()[:2] == b"\xff\xd8", "封面应为 JPEG" + + assert video_path is not None and video_path.stat().st_size > 0 + if video_item.is_dash: + assert video_path.suffix == ".mp4", "DASH 合流产物应为 mp4" + assert not list(video_path.parent.glob(".tmp-*")), "临时目录应被清理" + + probe = shutil.which("ffprobe") + if probe: + result = subprocess.run( + [probe, "-v", "error", "-show_entries", "stream=codec_type", + "-of", "csv=p=0", str(video_path)], + check=True, capture_output=True, text=True, + ) + stream_types = [line.strip() for line in result.stdout.splitlines() if line.strip()] + assert "video" in stream_types and "audio" in stream_types, ( + f"DASH 合流后应同时包含视频轨与音频轨，实际: {stream_types}" + ) + +async def _fetch_play_info_for_mp4(aid: int, cid: int) -> dict: + """按 mp4 格式再取一次播放地址（DASH 响应里不带 durl）""" + async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client: + img_key, sub_key = await _fetch_wbi_keys(client) + params = { + "avid": aid, + "cid": cid, + "qn": 80, + "fourk": 1, + "fnval": bili_media.MP4_FNVAL, + "platform": "pc", + } + signed = BilibiliSign(img_key, sub_key).sign(params) + response = await client.get( + "https://api.bilibili.com/x/player/wbi/playurl", + params=signed, + headers={"User-Agent": USER_AGENT, "Referer": "https://www.bilibili.com"}, + ) + payload = response.json() + return payload.get("data") or {} diff --git a/tests/test_media_extractors.py b/tests/test_media_extractors.py new file mode 100644 index 0000000..1bb4ded --- /dev/null +++ b/tests/test_media_extractors.py @@ -0,0 +1,741 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_extractors.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""各平台媒体地址提取器单测。 + +每个平台的 ``media.py`` 都是纯函数，这里用最小化的真实结构 fixture 直接断言 +产出的 ``MediaItem`` 列表，不发起任何网络请求。 +""" + +from __future__ import annotations + +import config +import pytest + +from media_downloader import MediaType +from media_platform.bilibili import media as bili_media +from media_platform.douyin import media as douyin_media +from media_platform.kuaishou import media as kuaishou_media +from media_platform.weibo import media as weibo_media +from media_platform.xhs import media as xhs_media + +# --------------------------------------------------------------------------- xhs fixture + +XHS_VIDEO_NOTE = { + "note_id": "video-note-1", + "type": "video", + "image_list": [{"url_default": "https://sns-webpic.xhscdn.com/img-cover"}], + "cover": {"url_default": "https://sns-webpic.xhscdn.com/top-cover"}, + "video": { + "consumer": {"origin_video_key": "spec/abc/def"}, + "media": {"stream": {"h264": [{"master_url": "https://sns-video-hw.xhscdn.com/master"}]}}, + "cover": {"url_default": "https://sns-webpic.xhscdn.com/video-cover"}, + }, +} + +XHS_IMAGE_NOTE = { + "note_id": "image-note-1", + "type": "normal", + "image_list": [ + {"url_default": "https://sns-webpic.xhscdn.com/1"}, + {"url_default": "https://sns-webpic.xhscdn.com/2"}, + {"url_default": "https://sns-webpic.xhscdn.com/3"}, + ], +} + + +# --------------------------------------------------------------------------- xhs 用例 + + +def test_xhs_video_note_yields_cover_and_video(): + items = xhs_media.build_media_items(XHS_VIDEO_NOTE) + + assert [item.stem for item in items] == ["cover", "video"] + assert [item.media_type for item in items] == [MediaType.IMAGE, MediaType.VIDEO] + + cover, video = items + assert cover.url == "https://sns-webpic.xhscdn.com/video-cover" + assert video.content_id == "video-note-1" + # 优先无水印源片，带水印的 master_url 作为备用 + assert video.url == "https://sns-video-bd.xhscdn.com/spec/abc/def" + assert video.backup_urls == ("https://sns-video-hw.xhscdn.com/master",) + + +def test_xhs_video_note_falls_back_to_top_level_cover(): + note = {**XHS_VIDEO_NOTE, "video": {**XHS_VIDEO_NOTE["video"], "cover": {}}} + note.pop("cover", None) + note["cover"] = {"url_default": "https://sns-webpic.xhscdn.com/top-cover"} + + items = xhs_media.build_media_items(note) + + assert items[0].url == "https://sns-webpic.xhscdn.com/top-cover" + + +def test_xhs_video_note_falls_back_to_first_image_when_no_cover(): + note = { + **XHS_VIDEO_NOTE, + "video": {**XHS_VIDEO_NOTE["video"], "cover": {}}, + "cover": {}, + } + + items = xhs_media.build_media_items(note) + + assert items[0].stem == "cover" + assert items[0].url == "https://sns-webpic.xhscdn.com/img-cover" + + +def test_xhs_video_note_without_any_cover_still_yields_video(): + note = { + **XHS_VIDEO_NOTE, + "video": {**XHS_VIDEO_NOTE["video"], "cover": {}}, + "cover": {}, + "image_list": [], + } + + items = xhs_media.build_media_items(note) + + assert [item.stem for item in items] == ["video"] + + +def test_xhs_video_note_without_origin_key_uses_master_url(): + note = { + **XHS_VIDEO_NOTE, + "video": {"media": {"stream": {"h264": [{"master_url": "https://sns-video-hw.xhscdn.com/master"}]}}}, + } + + items = xhs_media.build_media_items(note) + + video = next(item for item in items if item.media_type == MediaType.VIDEO) + assert video.url == "https://sns-video-hw.xhscdn.com/master" + assert video.backup_urls == () + + +def test_xhs_international_does_not_build_xhscdn_url(monkeypatch): + monkeypatch.setattr(config, "XHS_INTERNATIONAL", True) + + items = xhs_media.build_media_items(XHS_VIDEO_NOTE) + + video = next(item for item in items if item.media_type == MediaType.VIDEO) + assert video.url == "https://sns-video-hw.xhscdn.com/master" + assert "sns-video-bd" not in video.url + + +def test_xhs_image_note_yields_numbered_images_without_duplicate_cover(): + items = xhs_media.build_media_items(XHS_IMAGE_NOTE) + + assert [item.stem for item in items] == ["001", "002", "003"] + assert [item.url for item in items] == [ + "https://sns-webpic.xhscdn.com/1", + "https://sns-webpic.xhscdn.com/2", + "https://sns-webpic.xhscdn.com/3", + ] + assert all(item.media_type is MediaType.IMAGE for item in items) + assert not any(item.stem == "cover" for item in items) + + +def test_xhs_image_note_skips_entries_without_url(): + note = { + "note_id": "n1", + "type": "normal", + "image_list": [{"url_default": ""}, {"url": "https://cdn/x"}, {"no_url": True}], + } + + items = xhs_media.build_media_items(note) + + assert [item.url for item in items] == ["https://cdn/x"] + + +def test_xhs_missing_note_id_returns_empty(): + assert xhs_media.build_media_items({"type": "video"}) == [] + + +@pytest.mark.parametrize("payload", [None, "not-a-dict", [], 123]) +def test_xhs_non_dict_input_is_ignored(payload): + assert xhs_media.build_media_items(payload) == [] + + +# --------------------------------------------------------------------------- douyin fixture + +DY_VIDEO_ITEM = { + "aweme_id": "7300000000000000001", + "desc": "video", + "video": { + "play_addr_h264": {"url_list": ["https://cdn-a/h264", "https://cdn-b/h264"]}, + "play_addr": {"url_list": ["https://cdn-a/play"]}, + "raw_cover": {"url_list": ["https://cdn-a/cover", "https://cdn-b/cover"]}, + }, +} + +DY_IMAGE_ITEM = { + "aweme_id": "7300000000000000002", + "desc": "images", + "images": [ + {"url_list": ["https://cdn-a/img1-small", "https://cdn-b/img1"]}, + {"url_list": ["https://cdn-a/img2-small", "https://cdn-b/img2"]}, + ], + "video": {"raw_cover": {"url_list": ["https://cdn/cover"]}}, +} + + +# --------------------------------------------------------------------------- douyin 用例 + + +def test_douyin_video_item_yields_cover_and_video(): + items = douyin_media.build_media_items(DY_VIDEO_ITEM) + + assert [item.stem for item in items] == ["cover", "video"] + cover, video = items + assert cover.url == "https://cdn-b/cover" + # url_list 最后一个通常无水印，其余作为备用 + assert video.url == "https://cdn-b/h264" + assert video.backup_urls == ("https://cdn-a/h264",) + assert video.media_type is MediaType.VIDEO + + +def test_douyin_video_falls_back_to_lower_quality_addr(): + item = { + "aweme_id": "1", + "video": {"play_addr_256": {"url_list": ["https://cdn/only", "https://cdn/best"]}}, + } + + items = douyin_media.build_media_items(item) + + video = next(entry for entry in items if entry.media_type is MediaType.VIDEO) + assert video.url == "https://cdn/best" + assert video.backup_urls == ("https://cdn/only",) + + +def test_douyin_video_falls_back_to_bit_rate_list(): + item = { + "aweme_id": "1", + "video": { + "bit_rate": [ + {"bit_rate": 100, "play_addr": {"url_list": ["https://cdn/low"]}}, + {"bit_rate": 900, "play_addr": {"url_list": ["https://cdn/high"]}}, + ] + }, + } + + items = douyin_media.build_media_items(item) + + video = next(entry for entry in items if entry.media_type is MediaType.VIDEO) + assert video.url == "https://cdn/high" + + +def test_douyin_image_item_yields_numbered_images_without_cover(): + items = douyin_media.build_media_items(DY_IMAGE_ITEM) + + assert [item.stem for item in items] == ["001", "002"] + assert [item.url for item in items] == ["https://cdn-b/img1", "https://cdn-b/img2"] + assert not any(item.stem == "cover" for item in items) + + +def test_douyin_empty_video_yields_nothing(): + assert douyin_media.build_media_items({"aweme_id": "1", "video": {}}) == [] + + +def test_douyin_cover_key_priority_is_locked(): + """封面字段有明确优先级，不能只钉住第一个键""" + detail = { + "aweme_id": "1", + "video": { + "raw_cover": {"url_list": ["https://cdn/raw"]}, + "origin_cover": {"url_list": ["https://cdn/origin"]}, + "cover": {"url_list": ["https://cdn/cover"]}, + "dynamic_cover": {"url_list": ["https://cdn/dynamic"]}, + }, + } + + assert douyin_media.extract_cover_url(detail) == "https://cdn/raw" + del detail["video"]["raw_cover"] + assert douyin_media.extract_cover_url(detail) == "https://cdn/origin" + del detail["video"]["origin_cover"] + assert douyin_media.extract_cover_url(detail) == "https://cdn/cover" + del detail["video"]["cover"] + assert douyin_media.extract_cover_url(detail) == "https://cdn/dynamic" + + +def test_douyin_video_addr_key_priority_is_locked(): + """h264 > 256 > play_addr 的降级顺序必须被钉住""" + detail = { + "aweme_id": "1", + "video": { + "play_addr_h264": {"url_list": ["https://cdn/h264"]}, + "play_addr_256": {"url_list": ["https://cdn/256"]}, + "play_addr": {"url_list": ["https://cdn/plain"]}, + }, + } + + assert douyin_media.extract_video_urls(detail) == ["https://cdn/h264"] + del detail["video"]["play_addr_h264"] + assert douyin_media.extract_video_urls(detail) == ["https://cdn/256"] + del detail["video"]["play_addr_256"] + assert douyin_media.extract_video_urls(detail) == ["https://cdn/plain"] + + +def test_bilibili_falls_back_to_lowest_quality_keeping_codec_priority(monkeypatch): + """所有档位都超过用户设置时取最低清晰度，但编码优先级仍要生效（AVC 优先）""" + monkeypatch.setattr(config, "BILI_QN", 16) + play_info = { + "dash": { + "video": [ + {"id": 32, "codecid": 12, "bandwidth": 300000, "base_url": "https://upos/hevc.m4s"}, + {"id": 32, "codecid": 7, "bandwidth": 200000, "base_url": "https://upos/avc.m4s"}, + {"id": 80, "codecid": 7, "bandwidth": 900000, "base_url": "https://upos/80.m4s"}, + ] + } + } + + assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/avc.m4s" + + +def test_bilibili_tolerates_string_typed_ids(): + """接口偶发把 id/bandwidth 返回成字符串，不能因此整条媒体被跳过""" + play_info = { + "dash": { + "video": [ + {"id": "80", "codecid": "12", "bandwidth": "900000", "base_url": "https://upos/hevc.m4s"}, + {"id": "80", "codecid": "7", "bandwidth": "800000", "base_url": "https://upos/avc.m4s"}, + ], + "audio": [{"id": "30280", "bandwidth": "192000", "base_url": "https://upos/a.m4s"}], + } + } + + item = bili_media.build_dash_item(play_info, "BV1") + + assert item is not None + assert item.url == "https://upos/avc.m4s" + + +def test_bilibili_durl_stem_differs_from_dash_stem(): + """直链降级产物必须与 DASH 产物区分，否则低清文件会永久阻塞高清路径""" + dash_item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1") + durl_item = bili_media.build_durl_item(BILI_DURL_PLAY_INFO, "BV1") + + assert dash_item.stem == "video" + assert durl_item.stem == "video-durl" + + +# --------------------------------------------------------------------------- kuaishou fixture + +KS_VIDEO_ITEM = { + "photo": { + "id": "3xabcdefg", + "caption": "hello", + "photoH265Url": "https://ks/h265.mp4", + "photoUrl": "https://ks/h264.mp4", + "coverUrl": "https://ks/cover.jpg", + }, + "author": {"id": "u1"}, +} + + +# --------------------------------------------------------------------------- kuaishou 用例 + + +def test_kuaishou_video_item_yields_cover_and_video(): + items = kuaishou_media.build_media_items(KS_VIDEO_ITEM) + + assert [item.stem for item in items] == ["cover", "video"] + cover, video = items + assert cover.url == "https://ks/cover.jpg" + assert video.url == "https://ks/h265.mp4" + assert video.backup_urls == ("https://ks/h264.mp4",) + assert video.content_id == "3xabcdefg" + + +def test_kuaishou_falls_back_to_video_resource_representations(): + item = { + "photo": { + "id": "1", + "videoResource": { + "h264": { + "adaptationSet": [ + {"representation": [{"url": "https://ks/rep.mp4"}]} + ] + } + }, + } + } + + items = kuaishou_media.build_media_items(item) + + video = next(entry for entry in items if entry.media_type is MediaType.VIDEO) + assert video.url == "https://ks/rep.mp4" + + +def test_kuaishou_cover_falls_back_to_cover_urls_list(): + item = {"photo": {"id": "1", "coverUrls": [{"url": "https://ks/c1.jpg"}], "photoUrl": "https://ks/v.mp4"}} + + items = kuaishou_media.build_media_items(item) + + cover = next(entry for entry in items if entry.stem == "cover") + assert cover.url == "https://ks/c1.jpg" + + +def test_kuaishou_missing_photo_id_yields_nothing(): + assert kuaishou_media.build_media_items({"photo": {"photoUrl": "https://ks/v.mp4"}}) == [] + + +# --------------------------------------------------------------------------- weibo fixture + +WB_IMAGE_MBLOG = { + "id": "5000000000000000", + "pics": [ + {"url": "https://wx1.sinaimg.cn/orj360/abc.jpg", "pid": "abc"}, + "https://wx2.sinaimg.cn/thumbnail/def.jpg", + ], +} + +WB_VIDEO_MBLOG = { + "id": "5000000000000001", + "page_info": { + "type": "video", + "page_pic": {"url": "https://wx1.sinaimg.cn/orj360/cover.jpg"}, + "media_info": { + "mp4_hd_mp4": "https://f.video.weibocdn.com/hd.mp4", + "mp4_ld_mp4": "https://f.video.weibocdn.com/ld.mp4", + }, + }, +} + + +# --------------------------------------------------------------------------- weibo 用例 + + +def test_weibo_image_url_is_rewritten_to_large_via_agent_host(): + assert ( + weibo_media.rewrite_image_url("https://wx1.sinaimg.cn/orj360/abc.jpg") + == "https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg" + ) + + +def test_weibo_image_url_rewrite_ignores_query_string(): + assert ( + weibo_media.rewrite_image_url("https://wx1.sinaimg.cn/orj360/abc.jpg?v=1") + == "https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg" + ) + + +def test_weibo_image_note_yields_numbered_images(): + items = weibo_media.build_media_items(WB_IMAGE_MBLOG) + + assert [item.stem for item in items] == ["001", "002"] + assert [item.url for item in items] == [ + "https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg", + "https://i1.wp.com/wx2.sinaimg.cn/large/def.jpg", + ] + + +def test_weibo_video_note_yields_cover_and_video(): + items = weibo_media.build_media_items(WB_VIDEO_MBLOG) + + assert [item.stem for item in items] == ["cover", "video"] + cover, video = items + assert cover.url == "https://wx1.sinaimg.cn/orj360/cover.jpg" + assert video.url == "https://f.video.weibocdn.com/hd.mp4" + assert video.backup_urls == ("https://f.video.weibocdn.com/ld.mp4",) + + +def test_weibo_plain_note_yields_nothing(): + assert weibo_media.build_media_items({"id": "1", "text": "just text"}) == [] + + +def test_weibo_retweet_uses_original_media(): + """转发微博的媒体在 retweeted_status 里，顶层没有；不处理会一条都下不到""" + retweet = { + "id": "6000000000000000", + "text": "转发内容", + "retweeted_status": { + "id": "5000000000000009", + "pics": [{"url": "https://wx1.sinaimg.cn/orj360/original.jpg"}], + }, + } + + items = weibo_media.build_media_items(retweet) + + assert [item.stem for item in items] == ["001"] + assert items[0].url == "https://i1.wp.com/wx1.sinaimg.cn/large/original.jpg" + assert items[0].content_id == "6000000000000000", "目录应按转发帖自身 id 组织" + + +def test_weibo_retweet_with_video(): + retweet = { + "id": "6000000000000001", + "retweeted_status": { + "page_info": { + "type": "video", + "page_pic": {"url": "https://wx1.sinaimg.cn/orj360/cover.jpg"}, + "media_info": {"mp4_hd_mp4": "https://f.video.weibocdn.com/hd.mp4"}, + } + }, + } + + items = weibo_media.build_media_items(retweet) + + assert [item.stem for item in items] == ["cover", "video"] + assert items[1].url == "https://f.video.weibocdn.com/hd.mp4" + + +def test_weibo_own_media_wins_over_retweeted_status(): + """自己带媒体时不应去看被转发的内容""" + mblog = { + "id": "6000000000000002", + "pics": [{"url": "https://wx1.sinaimg.cn/orj360/mine.jpg"}], + "retweeted_status": {"pics": [{"url": "https://wx1.sinaimg.cn/orj360/theirs.jpg"}]}, + } + + items = weibo_media.build_media_items(mblog) + + assert items[0].url == "https://i1.wp.com/wx1.sinaimg.cn/large/mine.jpg" + + +def test_kuaishou_falls_back_to_manifest_representations(): + """search 接口的 photo 字段更少，manifest 是最后一道兜底""" + item = { + "photo": { + "id": "1", + "manifest": { + "adaptationSet": [{"representation": [{"url": "https://ks/manifest.mp4"}]}] + }, + } + } + + items = kuaishou_media.build_media_items(item) + + video = next(entry for entry in items if entry.media_type is MediaType.VIDEO) + assert video.url == "https://ks/manifest.mp4" + + +@pytest.mark.parametrize("scalar_video_resource", ["https://ks/raw.mp4", 123, None]) +def test_kuaishou_tolerates_scalar_video_resource(scalar_video_resource): + """GraphQL 里 videoResource 可能是 scalar 而非对象，不能因此抛异常""" + item = {"photo": {"id": "1", "videoResource": scalar_video_resource, "photoUrl": "https://ks/plain.mp4"}} + + video_items = [ + entry for entry in kuaishou_media.build_media_items(item) + if entry.media_type is MediaType.VIDEO + ] + + assert [entry.url for entry in video_items] == ["https://ks/plain.mp4"] + + +# --------------------------------------------------------------------------- bilibili fixture + +BILI_VIEW = { + "aid": 114514, + "cid": 1919810, + "bvid": "BV1dwuKzmE26", + "pic": "https://i0.hdslb.com/bfs/archive/cover.jpg", +} + +BILI_DASH_PLAY_INFO = { + "dash": { + "video": [ + {"id": 32, "bandwidth": 300000, "base_url": "https://upos/32.m4s", "backup_url": ["https://upos-b/32.m4s"]}, + {"id": 80, "bandwidth": 900000, "base_url": "https://upos/80.m4s"}, + {"id": 116, "bandwidth": 2000000, "baseUrl": "https://upos/116.m4s"}, + ], + "audio": [ + {"id": 30216, "bandwidth": 64000, "base_url": "https://upos/a64.m4s"}, + {"id": 30280, "bandwidth": 192000, "base_url": "https://upos/a192.m4s", "backup_url": ["https://upos-b/a192.m4s"]}, + ], + } +} + +BILI_DURL_PLAY_INFO = { + "durl": [ + {"url": "https://upos/seg1.mp4", "size": 1000, "backup_url": ["https://upos-b/seg1.mp4"]}, + {"url": "https://upos/seg2.mp4", "size": 5000}, + ] +} + + +# --------------------------------------------------------------------------- bilibili 用例 + + +def test_bilibili_cover_item_from_view_pic(): + item = bili_media.build_cover_item(BILI_VIEW, "BV1dwuKzmE26") + + assert item is not None + assert item.url == "https://i0.hdslb.com/bfs/archive/cover.jpg" + assert item.stem == "cover" + assert item.content_id == "BV1dwuKzmE26" + + +def test_bilibili_cover_item_missing_pic(): + assert bili_media.build_cover_item({"aid": 1}, "BV1") is None + assert bili_media.build_cover_item({}, "BV1") is None + + +def test_bilibili_pick_video_stream_respects_configured_quality(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 80) + + stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO) + + assert stream["id"] == 80 + + +def test_bilibili_pick_video_stream_falls_back_to_lowest_when_all_exceed(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 16) + + stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO) + + assert stream["id"] == 32 + + +def test_bilibili_pick_video_stream_picks_highest_when_all_below(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 127) + + stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO) + + assert stream["id"] == 116 + + +def test_bilibili_pick_audio_stream_takes_highest_bandwidth(): + stream = bili_media.pick_audio_stream(BILI_DASH_PLAY_INFO) + + assert stream["id"] == 30280 + + +def test_bilibili_prefers_avc_over_hevc_at_same_quality(monkeypatch): + """同一清晰度下应选兼容性最好的 AVC，而不是码率更高但兼容性差的 HEVC""" + monkeypatch.setattr(config, "BILI_QN", 80) + play_info = { + "dash": { + "video": [ + {"id": 80, "codecid": 12, "bandwidth": 2000000, "base_url": "https://upos/hevc.m4s"}, + {"id": 80, "codecid": 7, "bandwidth": 1500000, "base_url": "https://upos/avc.m4s"}, + {"id": 80, "codecid": 13, "bandwidth": 1200000, "base_url": "https://upos/av1.m4s"}, + ], + "audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}], + } + } + + assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/avc.m4s" + assert bili_media.build_dash_item(play_info, "BV1").url == "https://upos/avc.m4s" + + +def test_bilibili_falls_back_to_hevc_when_avc_unavailable(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 80) + play_info = { + "dash": { + "video": [ + {"id": 80, "codecid": 12, "bandwidth": 2000000, "base_url": "https://upos/hevc.m4s"}, + ], + "audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}], + } + } + + assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/hevc.m4s" + + +def test_bilibili_build_dash_item(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 80) + + item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1dwuKzmE26") + + assert item is not None + assert item.is_dash is True + assert item.url == "https://upos/80.m4s" + assert item.audio_url == "https://upos/a192.m4s" + assert item.audio_backup_urls == ("https://upos-b/a192.m4s",) + assert item.media_type is MediaType.VIDEO + + +def test_bilibili_build_dash_item_supports_camel_case_base_url(monkeypatch): + monkeypatch.setattr(config, "BILI_QN", 127) + + item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1") + + assert item.url == "https://upos/116.m4s" + + +def test_bilibili_build_dash_item_requires_audio(): + play_info = {"dash": {"video": [{"id": 80, "base_url": "https://upos/v.m4s"}]}} + + assert bili_media.build_dash_item(play_info, "BV1") is None + + +def test_bilibili_build_durl_item_takes_largest_segment(): + item = bili_media.build_durl_item(BILI_DURL_PLAY_INFO, "BV1") + + assert item is not None + assert item.url == "https://upos/seg2.mp4" + assert item.is_dash is False + + +def test_bilibili_build_durl_item_keeps_backup_urls(): + play_info = {"durl": [{"url": "https://upos/only.mp4", "size": 1, "backup_url": ["https://upos-b/only.mp4"]}]} + + item = bili_media.build_durl_item(play_info, "BV1") + + assert item.backup_urls == ("https://upos-b/only.mp4",) + + +@pytest.mark.parametrize("play_info", [{}, {"durl": []}, None]) +def test_bilibili_build_durl_item_empty(play_info): + assert bili_media.build_durl_item(play_info or {}, "BV1") is None + + +def test_bilibili_counts_durl_segments(): + assert bili_media.count_durl_segments(BILI_DURL_PLAY_INFO) == 2 + assert bili_media.count_durl_segments(BILI_DASH_PLAY_INFO) == 0 + assert bili_media.count_durl_segments(None) == 0 + assert bili_media.count_durl_segments({"durl": ["oops", {"no_url": 1}]}) == 0 + + +@pytest.mark.parametrize("payload", ["oops", 123, [], None]) +def test_douyin_extractors_tolerate_malformed_payload(payload): + """接口偶发返回非 dict 结构时不能抛异常，否则会中断整轮爬取""" + assert douyin_media.build_media_items(payload) == [] + + +@pytest.mark.parametrize("payload", ["oops", 123, [], None]) +def test_kuaishou_extractors_tolerate_malformed_payload(payload): + assert kuaishou_media.build_media_items(payload) == [] + + +@pytest.mark.parametrize("payload", ["oops", 123, [], None]) +def test_bilibili_extractors_tolerate_malformed_payload(payload): + assert bili_media.pick_video_stream(payload or {}) is None + assert bili_media.pick_audio_stream(payload or {}) is None + assert bili_media.build_dash_item(payload or {}, "BV1") is None + assert bili_media.build_durl_item(payload or {}, "BV1") is None + + +@pytest.mark.parametrize("payload", ["oops", 123, [], None]) +def test_weibo_extractors_tolerate_malformed_payload(payload): + assert weibo_media.build_media_items(payload) == [] + + +def test_xhs_extractors_tolerate_malformed_video_field(): + """实测：video 字段是字符串时旧逻辑会 AttributeError""" + assert xhs_media.build_media_items({"note_id": "n1", "type": "video", "video": "oops"}) == [] + assert xhs_media.build_media_items({"note_id": "n1", "type": "video", "video": None}) == [] + assert ( + xhs_media.build_media_items( + {"note_id": "n1", "type": "video", "video": {"media": "oops", "cover": "oops"}} + ) + == [] + ) + + +def test_bilibili_tolerates_non_dict_stream_entries(): + play_info = { + "dash": { + "video": ["oops", {"id": 80, "base_url": "https://upos/ok.m4s"}], + "audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}], + } + } + + item = bili_media.build_dash_item(play_info, "BV1") + + assert item is not None + assert item.url == "https://upos/ok.m4s" diff --git a/tests/test_media_ffmpeg.py b/tests/test_media_ffmpeg.py new file mode 100644 index 0000000..334d394 --- /dev/null +++ b/tests/test_media_ffmpeg.py @@ -0,0 +1,147 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_ffmpeg.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""ffmpeg 音视频合流测试。 + +本机没装 ffmpeg 时整体跳过（不影响其他测试）。素材用 ffmpeg 内置的 lavfi 源生成， +不依赖任何外部文件。 +""" + +from __future__ import annotations + +import asyncio +import shutil +import subprocess +from pathlib import Path + +import pytest + +from media_downloader import merge_audio_video +from media_downloader.ffmpeg import ffmpeg_path, is_available +from media_downloader.types import MediaDownloadError + +pytestmark = pytest.mark.skipif(not is_available(), reason="本机未安装 ffmpeg") + + +def _ffprobe_path() -> str: + probe = shutil.which("ffprobe") + if not probe: + pytest.skip("本机未安装 ffprobe") + return probe + + +def _make_video(path: Path, seconds: int = 1) -> None: + # 显式 -f mp4：.m4s 不是 ffmpeg 能自动识别的容器后缀，需模拟 DASH 分段的真实形态 + subprocess.run( + [ + ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", f"color=c=red:s=320x240:d={seconds}", + "-c:v", "mpeg4", "-pix_fmt", "yuv420p", "-f", "mp4", str(path), + ], + check=True, + capture_output=True, + ) + + +def _make_audio(path: Path, seconds: int = 1) -> None: + subprocess.run( + [ + ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y", + "-f", "lavfi", "-i", f"sine=f=440:d={seconds}", + "-c:a", "aac", "-f", "mp4", str(path), + ], + check=True, + capture_output=True, + ) + + +def _stream_types(path: Path) -> list[str]: + result = subprocess.run( + [ + _ffprobe_path(), "-v", "error", + "-show_entries", "stream=codec_type", + "-of", "csv=p=0", str(path), + ], + check=True, + capture_output=True, + text=True, + ) + return [line.strip() for line in result.stdout.splitlines() if line.strip()] + + +@pytest.mark.asyncio +async def test_merge_produces_playable_file_with_both_streams(tmp_path): + video = tmp_path / "video.m4s" + audio = tmp_path / "audio.m4s" + merged = tmp_path / "merged.mp4" + _make_video(video) + _make_audio(audio) + + await merge_audio_video(video, audio, merged) + + assert merged.exists() + assert merged.stat().st_size > 0 + assert _stream_types(merged) == ["video", "audio"] + + +@pytest.mark.asyncio +async def test_merge_rejects_corrupt_input(tmp_path): + video = tmp_path / "video.m4s" + bad_audio = tmp_path / "audio.m4s" + merged = tmp_path / "merged.mp4" + _make_video(video) + bad_audio.write_bytes(b"this is definitely not an audio stream") + + with pytest.raises(MediaDownloadError): + await merge_audio_video(video, bad_audio, merged) + + +@pytest.mark.asyncio +async def test_merge_kills_subprocess_when_cancelled(tmp_path, monkeypatch): + """任务被取消（Ctrl-C / abort）时不能留下孤儿 ffmpeg 进程""" + import media_downloader.ffmpeg as ffmpeg_module + + # 必须用 exec：否则 sh 会另起一个 sleep 子进程，它继承着 stderr 管道， + # 即便 kill 掉 sh，管道清理仍会阻塞到 sleep 自然结束 + fake_ffmpeg = tmp_path / "fake_ffmpeg.sh" + fake_ffmpeg.write_text("#!/bin/sh\nexec sleep 60\n") + fake_ffmpeg.chmod(0o755) + monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: str(fake_ffmpeg)) + + task = asyncio.create_task( + ffmpeg_module.merge_audio_video( + tmp_path / "v.m4s", tmp_path / "a.m4s", tmp_path / "o.mp4", timeout=60 + ) + ) + await asyncio.sleep(0.4) + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + await asyncio.sleep(0.4) + + probe = subprocess.run(["pgrep", "-f", str(fake_ffmpeg)], capture_output=True, text=True) + assert probe.returncode != 0, f"取消后子进程仍在运行: {probe.stdout.strip()}" + + +@pytest.mark.asyncio +async def test_merge_reports_missing_ffmpeg(tmp_path, monkeypatch): + import media_downloader.ffmpeg as ffmpeg_module + + monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: None) + video = tmp_path / "video.m4s" + audio = tmp_path / "audio.m4s" + video.write_bytes(b"x") + audio.write_bytes(b"y") + + with pytest.raises(MediaDownloadError, match="ffmpeg"): + await merge_audio_video(video, audio, tmp_path / "out.mp4") + + +def test_is_available_matches_executable(): + assert is_available() is bool(ffmpeg_path()) diff --git a/webui/src/components/config/CrawlerConfigPanel.tsx b/webui/src/components/config/CrawlerConfigPanel.tsx index 5226ea2..b09762a 100644 --- a/webui/src/components/config/CrawlerConfigPanel.tsx +++ b/webui/src/components/config/CrawlerConfigPanel.tsx @@ -1,7 +1,7 @@ import type { ComponentType, ReactNode, KeyboardEvent } from 'react' import { useState } from 'react' import { useTranslation } from 'react-i18next' -import { Database, Globe, KeyRound, MessageSquare, Play, Square, X } from 'lucide-react' +import { Database, Globe, Image as ImageIcon, KeyRound, MessageSquare, Play, Square, X } from 'lucide-react' import { Label } from '@/components/ui/label' import { Select, SelectContent, SelectItem, SelectTrigger, SelectValue } from '@/components/ui/select' import { Input } from '@/components/ui/input' @@ -371,6 +371,23 @@ export function CrawlerConfigPanel() {

{t('field.subComments')}

+
+ updateConfig({ enable_media: checked === true })} + disabled={isDisabled} + /> +
+
+ +

{t('field.downloadMedia')}

+
+

+ {t('field.downloadMediaHint')} +

+
+
+