fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-09-22 11:28:12 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台

Browse Source 
旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：

- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
  与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
  避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志，不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/media_server.py
+386
View File
@@ -0,0 +1,386 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/media_server.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""媒体下载测试用的本地 HTTP 服务器。
基于标准库 ``ThreadingHTTPServer``，不引入额外依赖；覆盖下载器需要面对的各种
CDN 行为：Range 续传、忽略 Range、chunked、响应截断、5xx 抖动、302 跳转、
无扩展名但带 Content-Type、慢响应、空响应体等。
所有请求都会记录到 ``server.requests``，用于断言"跳过已下载时不发请求"
"403 只请求一次"这类行为。
"""
from __future__ import annotations
import threading
import time
from dataclasses import dataclass, field
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from typing import Dict, List, Optional, Tuple
from urllib.parse import parse_qs, urlsplit
# 可识别的字节模式，便于断言续传后拼接结果正确
DEFAULT_CONTENT = bytes(range(256)) * 16 # 4096 字节
DEFAULT_CONTENT_TYPE = "image/jpeg"
@dataclass
class RequestRecord:
"""一次进入服务器的请求"""
method: str
path: str
headers: Dict[str, str] = field(default_factory=dict)
class _MediaRequestHandler(BaseHTTPRequestHandler):
protocol_version = "HTTP/1.1"
server_version = "MediaTestServer/1.0"
# 静音默认的 stderr 访问日志
def log_message(self, format: str, *args) -> None: # noqa: A002
return
# ------------------------------------------------------------------ 工具方法
@property
def media_server(self) -> "MediaTestServer":
return self.server.media_server # type: ignore[attr-defined]
def _record(self) -> None:
self.media_server.requests.append(
RequestRecord(
method=self.command,
path=self.path,
headers={key.lower(): value for key, value in self.headers.items()},
)
)
def _content_for(self, name: str) -> bytes:
custom = self.media_server.contents.get(name)
return custom if custom is not None else DEFAULT_CONTENT
def _send_body(self, status: int, body: bytes, content_type: str, extra_headers=None) -> None:
self.send_response(status)
self.send_header("Content-Type", content_type)
self.send_header("Content-Length", str(len(body)))
for key, value in (extra_headers or {}).items():
self.send_header(key, value)
self.end_headers()
if body:
self.wfile.write(body)
def _parse_range(self) -> Optional[int]:
"""解析 ``Range: bytes=N-``，返回起始偏移；不合法或不存在返回 None"""
raw = self.headers.get("range", "")
if not raw.startswith("bytes="):
return None
spec = raw[len("bytes=") :].split(",")[0].strip()
if not spec.endswith("-"):
return None
start = spec[:-1].strip()
return int(start) if start.isdigit() else None
# ------------------------------------------------------------------ 路由
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler 约定
self._record()
parsed = urlsplit(self.path)
path = parsed.path
query = parse_qs(parsed.query)
try:
if path.startswith("/status/"):
code = int(path.rsplit("/", 1)[-1])
self._send_body(code, b"", "text/plain")
elif path.startswith("/ok/") or path.startswith("/redirected/"):
self._handle_ok(path.rsplit("/", 1)[-1], query)
elif path.startswith("/no-range/"):
self._handle_no_range(path.rsplit("/", 1)[-1])
elif path.startswith("/wrong-range/"):
self._handle_wrong_range(path.rsplit("/", 1)[-1], query)
elif path.startswith("/poison-206/"):
self._handle_poison_206(path.rsplit("/", 1)[-1], query)
elif path.startswith("/gzip/"):
self._handle_gzip(path.rsplit("/", 1)[-1], query)
elif path.startswith("/unknown-total/"):
self._handle_unknown_total(path.rsplit("/", 1)[-1])
elif path.startswith("/no-content-range/"):
self._handle_no_content_range(path.rsplit("/", 1)[-1])
elif path.startswith("/chunked-bogus-cl/"):
self._handle_chunked_with_bogus_content_length(path.rsplit("/", 1)[-1])
elif path.startswith("/chunked/"):
self._handle_chunked(path.rsplit("/", 1)[-1])
elif path.startswith("/truncated/"):
self._handle_truncated(path.rsplit("/", 1)[-1], query)
elif path.startswith("/flaky/"):
self._handle_flaky(path.rsplit("/", 1)[-1], query)
elif path.startswith("/redirect/"):
self._handle_redirect(path.rsplit("/", 1)[-1])
elif path.startswith("/ctyped/"):
self._handle_ctyped(path.rsplit("/", 1)[-1])
elif path.startswith("/slow/"):
self._handle_slow(path.rsplit("/", 1)[-1], query)
elif path.startswith("/empty/"):
self._send_body(200, b"", DEFAULT_CONTENT_TYPE)
else:
self._send_body(404, b"not found", "text/plain")
except (BrokenPipeError, ConnectionResetError): # 客户端主动断开
self.close_connection = True
# ------------------------------------------------------------------ 各路由实现
def _handle_ok(self, name: str, query) -> None:
"""正常文件，支持 Range 续传"""
content = self._content_for(name)
total = len(content)
start = self._parse_range()
if start is not None:
if start >= total:
self.send_response(416)
self.send_header("Content-Range", f"bytes */{total}")
self.send_header("Content-Length", "0")
self.end_headers()
return
body = content[start:]
content_type = query.get("ct", [DEFAULT_CONTENT_TYPE])[0]
self._send_body(
206,
body,
content_type,
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
)
return
self._send_body(200, content, query.get("ct", [DEFAULT_CONTENT_TYPE])[0])
def _handle_wrong_range(self, name: str, query) -> None:
"""返回 206 但 Content-Range 起点与请求不一致（模拟按关键帧对齐的加速节点）。
``?fail=N``：前 N 次返回错位响应，之后恢复正常，
用于验证"下载器丢弃片段重下后能成功"。
"""
fail_times = int(query.get("fail", ["1000"])[0])
counter_key = f"/wrong-range/{name}"
with self.media_server.lock:
seen = self.media_server.counters.get(counter_key, 0)
self.media_server.counters[counter_key] = seen + 1
if seen >= fail_times:
self._handle_ok(name, query)
return
content = self._content_for(name)
total = len(content)
requested = self._parse_range() or 0
delta = int(query.get("delta", ["100"])[0])
start = min(requested + delta, total - 1)
self._send_body(
206,
content[start:],
DEFAULT_CONTENT_TYPE,
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
)
def _handle_poison_206(self, name: str, query) -> None:
"""206 声明非 0 起点，但 body 长度**恰好等于 total**（内容是错位的）。
这是唯一能单独验证"起点守卫"的响应：长度自洽，大小校验拦不住，
只有比对 Content-Range 起点才能发现异常。
"""
content = self._content_for(name)
total = len(content)
start = int(query.get("start", ["100"])[0])
body = content[start:] + b"\x00" * start
self._send_body(
206,
body,
DEFAULT_CONTENT_TYPE,
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
)
def _handle_gzip(self, name: str, query) -> None:
"""无视 Accept-Encoding: identity，强制返回 gzip 压缩体"""
import gzip
content = self._content_for(name)
compressed = gzip.compress(content)
self.send_response(200)
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
self.send_header("Content-Encoding", "gzip")
self.send_header("Content-Length", str(len(compressed)))
self.end_headers()
self.wfile.write(compressed)
def _handle_unknown_total(self, name: str) -> None:
"""206 只返回区间的一部分，且 Content-Range 的 total 为 *（不告知总长）"""
content = self._content_for(name)
start = self._parse_range() or 0
chunk = content[start : start + 1024]
self._send_body(
206,
chunk,
DEFAULT_CONTENT_TYPE,
{"Content-Range": f"bytes {start}-{start + len(chunk) - 1}/*"},
)
def _handle_no_content_range(self, name: str) -> None:
"""206 但缺少 Content-Range（违反 RFC 9110），且回的是全量体"""
content = self._content_for(name)
self._send_body(206, content, DEFAULT_CONTENT_TYPE)
def _handle_chunked_with_bogus_content_length(self, name: str) -> None:
"""chunked 传输同时带一个错误的 Content-Length（RFC 7230 要求忽略后者）"""
content = self._content_for(name)
self.send_response(200)
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
self.send_header("Transfer-Encoding", "chunked")
self.send_header("Content-Length", "999")
self.end_headers()
self.wfile.write(f"{len(content):X}\r\n".encode("ascii"))
self.wfile.write(content)
self.wfile.write(b"\r\n0\r\n\r\n")
self.wfile.flush()
def _handle_no_range(self, name: str) -> None:
"""声明支持 Range 但始终返回全量 200（模拟不支持续传的 CDN）"""
content = self._content_for(name)
self._send_body(200, content, DEFAULT_CONTENT_TYPE, {"Accept-Ranges": "bytes"})
def _handle_chunked(self, name: str) -> None:
"""chunked 传输：无 Content-Length"""
content = self._content_for(name)
self.send_response(200)
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
self.send_header("Transfer-Encoding", "chunked")
self.end_headers()
for offset in range(0, len(content), 1024):
chunk = content[offset : offset + 1024]
self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii"))
self.wfile.write(chunk)
self.wfile.write(b"\r\n")
self.wfile.write(b"0\r\n\r\n")
self.wfile.flush()
def _handle_truncated(self, name: str, query) -> None:
"""声明完整长度但只发送一半后断开连接。
``?fail=N``：前 N 次截断，之后恢复正常（支持 Range），
用于验证"断流 -> 重试 -> 带 Range 续传 -> 拼接完整"。
"""
fail_times = int(query.get("fail", ["1000"])[0])
counter_key = f"/truncated/{name}"
with self.media_server.lock:
seen = self.media_server.counters.get(counter_key, 0)
self.media_server.counters[counter_key] = seen + 1
if seen >= fail_times:
self._handle_ok(name, query)
return
content = self._content_for(name)
self.send_response(200)
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
self.send_header("Content-Length", str(len(content)))
self.end_headers()
self.wfile.write(content[: len(content) // 2])
self.wfile.flush()
self.close_connection = True
try:
self.connection.shutdown(2) # SHUT_RDWR
except OSError:
pass
def _handle_flaky(self, name: str, query) -> None:
"""前 N 次请求返回 500，之后正常（服务端计数）"""
fail_times = int(query.get("fail", ["1"])[0])
counter_key = f"/flaky/{name}"
with self.media_server.lock:
seen = self.media_server.counters.get(counter_key, 0)
self.media_server.counters[counter_key] = seen + 1
if seen < fail_times:
self._send_body(500, b"server error", "text/plain")
return
content = self._content_for(name)
self._send_body(200, content, DEFAULT_CONTENT_TYPE)
def _handle_redirect(self, name: str) -> None:
self.send_response(302)
self.send_header("Location", f"/ok/{name}")
self.send_header("Content-Length", "0")
self.end_headers()
def _handle_ctyped(self, name: str) -> None:
"""URL 无扩展名，靠 Content-Type 推断"""
content_type = self.media_server.content_types.get(name, "image/webp")
self._send_body(200, self._content_for(name), content_type)
def _handle_slow(self, name: str, query) -> None:
delay = float(query.get("s", ["1"])[0])
time.sleep(delay)
self._send_body(200, self._content_for(name), DEFAULT_CONTENT_TYPE)
class MediaTestServer(ThreadingHTTPServer):
"""带请求记录能力的本地测试服务器"""
daemon_threads = True
allow_reuse_address = True
def __init__(self) -> None:
super().__init__(("127.0.0.1", 0), _MediaRequestHandler)
# handler 通过 server.media_server 反向访问这些状态
self.media_server: "MediaTestServer" = self
self.requests: List[RequestRecord] = []
self.counters: Dict[str, int] = {}
self.contents: Dict[str, bytes] = {}
self.content_types: Dict[str, str] = {}
self.lock = threading.Lock()
# ------------------------------------------------------------------ 测试辅助
@property
def base_url(self) -> str:
host, port = self.server_address[:2]
return f"http://{host}:{port}"
def url(self, path: str) -> str:
return f"{self.base_url}{path}"
def reset(self) -> None:
"""清空请求记录与计数器（内容配置保留）"""
with self.lock:
self.requests.clear()
self.counters.clear()
def set_content(self, name: str, content: bytes) -> None:
self.contents[name] = content
def set_content_type(self, name: str, content_type: str) -> None:
self.content_types[name] = content_type
@property
def request_count(self) -> int:
return len(self.requests)
def paths(self) -> List[str]:
return [record.path for record in self.requests]
def last_request(self) -> Optional[RequestRecord]:
return self.requests[-1] if self.requests else None
def start_server() -> Tuple[MediaTestServer, threading.Thread]:
"""在后台线程启动服务器，返回 (server, thread)"""
server = MediaTestServer()
thread = threading.Thread(target=server.serve_forever, daemon=True)
thread.start()
return server, thread
tests/test_api_limits.py
+29
View File
@@ -62,6 +62,35 @@ def test_crawler_manager_build_command():
idx_comments = cmd2.index("--max_comments_count_singlenotes")
assert cmd2[idx_comments + 1] == "5"
def test_crawler_manager_passes_media_switch():
cm = CrawlerManager()
req_off = CrawlerStartRequest(
platform=PlatformEnum.XHS,
login_type=LoginTypeEnum.QRCODE,
crawler_type=CrawlerTypeEnum.DETAIL,
specified_ids="note-1",
)
cmd_off = cm._build_command(req_off)
idx_off = cmd_off.index("--get_media")
assert cmd_off[idx_off + 1] == "false"
req_on = CrawlerStartRequest(
platform=PlatformEnum.XHS,
login_type=LoginTypeEnum.QRCODE,
crawler_type=CrawlerTypeEnum.DETAIL,
specified_ids="note-1",
enable_media=True,
)
cmd_on = cm._build_command(req_on)
idx_on = cmd_on.index("--get_media")
assert cmd_on[idx_on + 1] == "true"
def test_api_schema_exposes_media_switch_default_off():
assert CrawlerStartRequest(platform=PlatformEnum.XHS).enable_media is False
def test_api_start_crawler_with_limits():
client = TestClient(app)
tests/test_cmd_arg_media.py
+73
View File
@@ -0,0 +1,73 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_cmd_arg_media.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""--get_media 命令行开关测试。
注意：parse_cmd 会就地覆盖全局 config，因此每个用例都用 monkeypatch 还原，
避免污染其他测试。
"""
from __future__ import annotations
import config
import pytest
from cmd_arg import parse_cmd
@pytest.fixture(autouse=True)
def _isolate_config(monkeypatch):
monkeypatch.setattr(config, "ENABLE_GET_MEDIA", False)
monkeypatch.setattr(config, "PLATFORM", "xhs")
monkeypatch.setattr(config, "CRAWLER_TYPE", "search")
yield
BASE_ARGS = ["--platform", "xhs", "--type", "detail", "--specified_id", "note-1"]
@pytest.mark.asyncio
async def test_get_media_true_enables_switch():
await parse_cmd([*BASE_ARGS, "--get_media", "true"])
assert config.ENABLE_GET_MEDIA is True
@pytest.mark.asyncio
async def test_get_media_false_keeps_switch_off():
await parse_cmd([*BASE_ARGS, "--get_media", "false"])
assert config.ENABLE_GET_MEDIA is False
@pytest.mark.asyncio
async def test_get_media_defaults_to_config_value(monkeypatch):
monkeypatch.setattr(config, "ENABLE_GET_MEDIA", True)
args = await parse_cmd(BASE_ARGS)
assert config.ENABLE_GET_MEDIA is True
assert args.get_media is True
@pytest.mark.parametrize(
("raw", "expected"),
[("yes", True), ("y", True), ("1", True), ("no", False), ("n", False), ("0", False)],
)
@pytest.mark.asyncio
async def test_get_media_accepts_boolean_aliases(raw, expected):
await parse_cmd([*BASE_ARGS, "--get_media", raw])
assert config.ENABLE_GET_MEDIA is expected
@pytest.mark.asyncio
async def test_get_media_returns_value_in_namespace():
args = await parse_cmd([*BASE_ARGS, "--get_media", "true"])
assert args.get_media is True
tests/test_media_core_integration.py
+513
View File
@@ -0,0 +1,513 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_core_integration.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""平台 core 的 ``download_media`` 与统一下载器的集成测试。
用本地 HTTP 服务器冒充平台 CDN，验证「core 编排 -> 平台 URL 提取 -> 统一下载器 -> 落盘」
的完整链路，不依赖外部网络、浏览器与登录态。
"""
from __future__ import annotations
import asyncio
from pathlib import Path
from types import SimpleNamespace
import config
import httpx
import pytest
from media_platform.bilibili.core import BilibiliCrawler
from media_platform.douyin.core import DouYinCrawler
from media_platform.kuaishou.core import KuaishouCrawler
from media_platform.weibo.core import WeiboCrawler
from media_platform.xhs.core import XiaoHongShuCrawler
from tests.media_server import DEFAULT_CONTENT, MediaTestServer, start_server
@pytest.fixture(scope="module")
def server():
srv, _thread = start_server()
yield srv
srv.shutdown()
srv.server_close()
@pytest.fixture(autouse=True)
def _reset_server(server: MediaTestServer):
server.reset()
@pytest.fixture(autouse=True)
def _media_config(monkeypatch, tmp_path: Path):
"""打开媒体下载开关并把落盘根目录指到临时目录"""
monkeypatch.setattr(config, "ENABLE_GET_MEDIA", True)
monkeypatch.setattr(config, "SAVE_DATA_PATH", str(tmp_path))
monkeypatch.setattr(config, "XHS_INTERNATIONAL", False)
# 平台在媒体下载之间会按爬虫限速 sleep，测试里不需要真的等
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 0)
def media_dir(tmp_path: Path, platform: str, content_id: str) -> Path:
return tmp_path / platform / "media" / content_id
def assert_downloaded(directory: Path, filename: str) -> Path:
path = directory / filename
assert path.is_file(), f"缺少文件 {path}，目录内容: {sorted(p.name for p in directory.iterdir())}"
assert path.stat().st_size == len(DEFAULT_CONTENT)
return path
# --------------------------------------------------------------------------- xhs
@pytest.mark.asyncio
async def test_xhs_core_downloads_cover_and_video(server, tmp_path):
note = {
"note_id": "xhs-e2e-video",
"type": "video",
"image_list": [],
"video": {
# 不带 origin_video_key，走 master_url 分支以便指向本地服务器
"media": {"stream": {"h264": [{"master_url": server.url("/ok/xhs-video?ct=video/mp4")}]}},
"cover": {"url_default": server.url("/ok/xhs-cover")},
},
}
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
await crawler.download_media(note)
directory = media_dir(tmp_path, "xhs", "xhs-e2e-video")
assert_downloaded(directory, "cover.jpg")
assert_downloaded(directory, "video.mp4")
assert server.request_count == 2
@pytest.mark.asyncio
async def test_xhs_core_downloads_numbered_images(server, tmp_path):
note = {
"note_id": "xhs-e2e-images",
"type": "normal",
"image_list": [
{"url_default": server.url("/ok/xhs-img-1")},
{"url_default": server.url("/ok/xhs-img-2")},
],
}
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
await crawler.download_media(note)
directory = media_dir(tmp_path, "xhs", "xhs-e2e-images")
assert_downloaded(directory, "001.jpg")
assert_downloaded(directory, "002.jpg")
assert not (directory / "cover.jpg").exists(), "图文笔记不应再单独下载一份封面"
@pytest.mark.asyncio
async def test_xhs_core_sends_referer(server, tmp_path):
note = {
"note_id": "xhs-e2e-referer",
"type": "normal",
"image_list": [{"url_default": server.url("/ok/xhs-referer")}],
}
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
await crawler.download_media(note)
assert server.requests[0].headers.get("referer") == "https://www.xiaohongshu.com/"
assert server.requests[0].headers.get("user-agent") == "integration-test-agent"
# --------------------------------------------------------------------------- douyin
@pytest.mark.asyncio
async def test_douyin_core_downloads_cover_and_video(server, tmp_path):
aweme = {
"aweme_id": "dy-e2e-video",
"video": {
"play_addr_h264": {"url_list": [server.url("/ok/dy-video?ct=video/mp4")]},
"raw_cover": {"url_list": [server.url("/ok/dy-cover")]},
},
}
crawler = DouYinCrawler()
# client.headers 里带上 Cookie，用于验证它不会被透传到 CDN 请求上
crawler.dy_client = SimpleNamespace(
proxy=None, headers={"User-Agent": "dy-agent", "Cookie": "session=secret"}
)
await crawler.download_media(aweme)
directory = media_dir(tmp_path, "dy", "dy-e2e-video")
assert_downloaded(directory, "cover.jpg")
assert_downloaded(directory, "video.mp4")
sent = server.requests[0].headers
assert sent.get("referer") == "https://www.douyin.com/"
assert sent.get("user-agent") == "dy-agent"
assert sent.get("cookie") is None, "不应把平台 Cookie 带到 CDN 请求上"
@pytest.mark.asyncio
async def test_douyin_core_downloads_images(server, tmp_path):
aweme = {
"aweme_id": "dy-e2e-images",
"images": [
{"url_list": [server.url("/ok/dy-img-1")]},
{"url_list": [server.url("/ok/dy-img-2")]},
],
}
crawler = DouYinCrawler()
crawler.dy_client = SimpleNamespace(proxy=None, headers={})
await crawler.download_media(aweme)
directory = media_dir(tmp_path, "dy", "dy-e2e-images")
assert_downloaded(directory, "001.jpg")
assert_downloaded(directory, "002.jpg")
# --------------------------------------------------------------------------- kuaishou
@pytest.mark.asyncio
async def test_kuaishou_core_downloads_cover_and_video(server, tmp_path):
video_item = {
"photo": {
"id": "ks-e2e-video",
"photoUrl": server.url("/ok/ks-video?ct=video/mp4"),
"coverUrl": server.url("/ok/ks-cover"),
}
}
crawler = KuaishouCrawler()
crawler.ks_client = SimpleNamespace(proxy=None, headers={})
await crawler.download_media(video_item)
directory = media_dir(tmp_path, "ks", "ks-e2e-video")
assert_downloaded(directory, "cover.jpg")
assert_downloaded(directory, "video.mp4")
assert server.requests[0].headers.get("referer") == "https://www.kuaishou.com/"
# --------------------------------------------------------------------------- weibo
@pytest.mark.asyncio
async def test_weibo_core_downloads_cover_and_video(server, tmp_path):
mblog = {
"id": "wb-e2e-video",
"page_info": {
"type": "video",
"page_pic": {"url": server.url("/ok/wb-cover")},
"media_info": {"mp4_hd_mp4": server.url("/ok/wb-video?ct=video/mp4")},
},
}
crawler = WeiboCrawler()
crawler.wb_client = SimpleNamespace(proxy=None, headers={})
await crawler.download_media(mblog)
directory = media_dir(tmp_path, "wb", "wb-e2e-video")
assert_downloaded(directory, "cover.jpg")
assert_downloaded(directory, "video.mp4")
# --------------------------------------------------------------------------- bilibili
@pytest.mark.asyncio
async def test_bilibili_core_downloads_cover_even_without_play_info(server, tmp_path, monkeypatch):
"""playurl 拿不到时，封面仍应正常落盘（两者是独立的下载任务）"""
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
async def fake_play_url(*args, **kwargs):
return None
monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url)
video_item = {
"View": {
"aid": 1,
"cid": 2,
"bvid": "BV1e2e",
"pic": server.url("/ok/bili-cover"),
}
}
await crawler.download_media(video_item, asyncio.Semaphore(1))
directory = media_dir(tmp_path, "bili", "BV1e2e")
assert_downloaded(directory, "cover.jpg")
assert not (directory / "video.mp4").exists()
@pytest.mark.asyncio
async def test_bilibili_core_skips_dash_when_ffmpeg_unavailable(server, tmp_path, monkeypatch):
"""ffmpeg 不可用时应直接走直链：DASH 流的地址一次都不该被请求"""
import media_platform.bilibili.core as bili_core
from media_platform.bilibili.media import DASH_FNVAL, MP4_FNVAL
monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: False)
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
requested_fnvals = []
async def fake_play_url(aid, cid, semaphore, fnval=None):
requested_fnvals.append(fnval if fnval is not None else DASH_FNVAL)
return {
"dash": {
"video": [{"id": 80, "base_url": server.url("/ok/dash-video-unused?ct=video/mp4")}],
"audio": [{"id": 30280, "base_url": server.url("/ok/dash-audio-unused?ct=audio/mp4")}],
},
"durl": [{"url": server.url("/ok/bili-durl?ct=video/mp4"), "size": 100}],
}
monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url)
video_item = {"View": {"aid": 1, "cid": 2, "bvid": "BV1fallback", "pic": server.url("/ok/bili-cover2")}}
await crawler.download_media(video_item, asyncio.Semaphore(1))
directory = media_dir(tmp_path, "bili", "BV1fallback")
# 直链产物刻意与 DASH 产物（video.mp4）区分，避免低清文件阻塞后续的高清路径
assert_downloaded(directory, "video-durl.mp4")
assert requested_fnvals == [MP4_FNVAL], "ffmpeg 不可用时应直接按 MP4 格式请求，不做无用的 DASH 请求"
requested_paths = server.paths()
assert not any("unused" in path for path in requested_paths), (
f"ffmpeg 不可用时不应下载 DASH 分轨，实际请求: {requested_paths}"
)
@pytest.mark.asyncio
async def test_core_download_media_never_raises_on_malformed_payload(server, tmp_path):
"""平台返回畸形结构时，download_media 只能记日志，不能把异常抛给爬取主流程"""
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
# video 字段是字符串，提取器若不做防御会 AttributeError
await crawler.download_media({"note_id": "x", "type": "video", "video": "oops"})
assert server.request_count == 0
@pytest.mark.asyncio
async def test_core_survives_extractor_exception(server, tmp_path, monkeypatch):
"""提取器抛异常时必须被 core 的壳层兜住（直接锁定 try/except，去掉它此用例会失败）"""
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
def boom(_raw):
raise RuntimeError("extractor exploded")
monkeypatch.setattr("media_platform.xhs.core.xhs_media.build_media_items", boom)
await crawler.download_media({"note_id": "boom"})
assert server.request_count == 0
@pytest.mark.asyncio
async def test_bilibili_dash_success_skips_durl_fallback(server, tmp_path, monkeypatch):
"""DASH 下载成功后必须立即结束，不能再下一份低清直链。
这是 B 站的主路径（装好 ffmpeg 的机器），此前完全没有测试覆盖。
"""
import media_downloader.downloader as downloader_module
import media_platform.bilibili.core as bili_core
monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True)
async def fake_merge(video_path, audio_path, output_path, timeout=300.0):
output_path.write_bytes(b"merged-mp4")
monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge)
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
async def fake_play_url(aid, cid, semaphore, fnval=None):
return {
"dash": {
"video": [{"id": 80, "codecid": 7, "base_url": server.url("/ok/dash-best?ct=video/mp4")}],
"audio": [{"id": 30280, "base_url": server.url("/ok/dash-audio?ct=audio/mp4")}],
},
"durl": [{"url": server.url("/ok/low-quality-durl?ct=video/mp4"), "size": 1}],
}
monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url)
video_item = {
"View": {"aid": 1, "cid": 2, "bvid": "BV1dashok", "pic": server.url("/ok/bili-cover-5")}
}
await crawler.download_media(video_item, asyncio.Semaphore(1))
directory = media_dir(tmp_path, "bili", "BV1dashok")
assert (directory / "video.mp4").is_file(), "DASH 合流产物应落在 video.mp4"
assert not (directory / "video-durl.mp4").exists(), "DASH 成功后不应再走直链"
assert not any("low-quality" in path for path in server.paths()), (
f"DASH 成功时不应请求直链，实际请求: {server.paths()}"
)
@pytest.mark.asyncio
async def test_bilibili_falls_back_to_lower_quality_when_cdn_rejects(server, tmp_path, monkeypatch):
"""高清流取流被 CDN 拒绝时（未登录/权限不足的真实场景），应逐档降级而不是直接放弃"""
import media_downloader.downloader as downloader_module
import media_platform.bilibili.core as bili_core
monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True)
monkeypatch.setattr(config, "BILI_QN", 80)
async def fake_merge(video_path, audio_path, output_path, timeout=300.0):
output_path.write_bytes(b"merged-sd")
monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge)
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
async def fake_play_url(aid, cid, semaphore, fnval=None):
return {
"dash": {
"video": [
{"id": 80, "codecid": 7, "base_url": server.url("/status/403")}, # 高清被 CDN 拒绝
{"id": 32, "codecid": 7, "base_url": server.url("/ok/dash-sd?ct=video/mp4")},
],
"audio": [{"id": 30280, "base_url": server.url("/ok/dash-sd-audio?ct=audio/mp4")}],
}
}
monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url)
video_item = {"View": {"aid": 1, "cid": 2, "bvid": "BV1degrade", "pic": server.url("/ok/bili-cover-8")}}
await crawler.download_media(video_item, asyncio.Semaphore(1))
directory = media_dir(tmp_path, "bili", "BV1degrade")
assert (directory / "video.mp4").read_bytes() == b"merged-sd"
assert any("dash-sd" in path for path in server.paths()), "应降级到低清晰度流"
@pytest.mark.asyncio
async def test_bilibili_durl_does_not_block_later_dash_download(server, tmp_path, monkeypatch):
"""先前降级下载的低清直链产物，不能阻止之后（装好 ffmpeg）走 DASH 拿高清"""
import media_downloader.downloader as downloader_module
import media_platform.bilibili.core as bili_core
directory = media_dir(tmp_path, "bili", "BV1upgrade")
directory.mkdir(parents=True)
(directory / "video-durl.mp4").write_bytes(b"low-resolution-durl-file")
monkeypatch.setattr(bili_core, "is_ffmpeg_available", lambda: True)
async def fake_merge(video_path, audio_path, output_path, timeout=300.0):
output_path.write_bytes(b"high-resolution-dash-file")
monkeypatch.setattr(downloader_module, "merge_audio_video", fake_merge)
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
async def fake_play_url(aid, cid, semaphore, fnval=None):
return {
"dash": {
"video": [{"id": 80, "codecid": 7, "base_url": server.url("/ok/dash-up?ct=video/mp4")}],
"audio": [{"id": 30280, "base_url": server.url("/ok/dash-up-a?ct=audio/mp4")}],
}
}
monkeypatch.setattr(crawler, "get_video_play_url_task", fake_play_url)
video_item = {
"View": {"aid": 1, "cid": 2, "bvid": "BV1upgrade", "pic": server.url("/ok/bili-cover-6")}
}
await crawler.download_media(video_item, asyncio.Semaphore(1))
assert (directory / "video.mp4").read_bytes() == b"high-resolution-dash-file"
assert server.request_count > 0, "已有的低清直链产物不应让 DASH 路径被跳过"
@pytest.mark.asyncio
async def test_bilibili_download_media_survives_play_url_failure(server, tmp_path, monkeypatch):
"""playurl 抛出非 DataFetchError 的异常（如网络超时）时，壳层必须兜住。
这条用例锁定的正是 core 里的 try/except：去掉它，异常会直接击穿爬取主流程。
"""
crawler = BilibiliCrawler()
crawler.bili_client = SimpleNamespace(proxy=None, headers={})
async def boom(*args, **kwargs):
raise httpx.ReadTimeout("simulated timeout")
monkeypatch.setattr(crawler, "get_video_play_url_task", boom)
video_item = {
"View": {"aid": 1, "cid": 2, "bvid": "BV1boom", "pic": server.url("/ok/bili-cover-3")}
}
await crawler.download_media(video_item, asyncio.Semaphore(1))
# 封面在 playurl 之前下载，失败不影响它
directory = media_dir(tmp_path, "bili", "BV1boom")
assert_downloaded(directory, "cover.jpg")
assert not (directory / "video.mp4").exists()
# --------------------------------------------------------------------------- 开关
@pytest.mark.asyncio
async def test_media_switch_off_downloads_nothing(server, tmp_path, monkeypatch):
monkeypatch.setattr(config, "ENABLE_GET_MEDIA", False)
note = {
"note_id": "xhs-switch-off",
"type": "normal",
"image_list": [{"url_default": server.url("/ok/should-not-download")}],
}
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
await crawler.download_media(note)
assert server.request_count == 0
assert not media_dir(tmp_path, "xhs", "xhs-switch-off").exists()
@pytest.mark.asyncio
async def test_download_all_keeps_going_after_one_failure(server, tmp_path):
"""单个媒体失败不应影响同帖其它媒体"""
note = {
"note_id": "xhs-partial-failure",
"type": "normal",
"image_list": [
{"url_default": server.url("/status/500")},
{"url_default": server.url("/ok/xhs-good-image")},
],
}
crawler = XiaoHongShuCrawler()
crawler.xhs_client = SimpleNamespace(proxy=None)
crawler.user_agent = "integration-test-agent"
await crawler.download_media(note)
directory = media_dir(tmp_path, "xhs", "xhs-partial-failure")
assert not (directory / "001.jpg").exists()
assert_downloaded(directory, "002.jpg")
tests/test_media_downloader.py
+890
View File
@@ -0,0 +1,890 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_downloader.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""MediaDownloader 单测。
全部基于 ``tests/media_server.py`` 提供的本地 HTTP 服务器，真实走 socket 与 httpx，
不使用任何 mock，也不访问外网。
"""
from __future__ import annotations
import os
import shutil
import subprocess
import httpx
from pathlib import Path
import pytest
from media_downloader import MediaDownloader, MediaItem, MediaType
from media_downloader.paths import (
build_media_dir,
build_media_path,
ensure_within,
guess_extension,
redact_url,
sanitize_component,
url_fingerprint,
)
from tests.media_server import DEFAULT_CONTENT, MediaTestServer, start_server
@pytest.fixture(scope="module")
def server():
srv, _thread = start_server()
yield srv
srv.shutdown()
srv.server_close()
@pytest.fixture(autouse=True)
def _reset_server(server: MediaTestServer):
server.reset()
yield
@pytest.fixture
def downloader(server: MediaTestServer, tmp_path: Path) -> MediaDownloader:
"""重试间隔调到毫秒级，避免测试因退避等待变慢"""
return MediaDownloader(
platform="xhs",
base_dir=tmp_path,
max_retries=2,
retry_base_delay=0.001,
retry_max_delay=0.005,
)
def make_item(server: MediaTestServer, path: str, **overrides) -> MediaItem:
params = {
"url": server.url(path),
"media_type": MediaType.IMAGE,
"content_id": "note-1",
"stem": "001",
}
params.update(overrides)
return MediaItem(**params)
def part_path_for(tmp_path: Path, item: MediaItem, url: str) -> Path:
directory = build_media_dir(tmp_path, "xhs", item.content_id)
directory.mkdir(parents=True, exist_ok=True)
return directory / f"{item.stem}.{url_fingerprint(url)}.{os.getpid()}.part"
# --------------------------------------------------------------------------- 基础
@pytest.mark.asyncio
async def test_download_success_and_layout(server, downloader, tmp_path):
item = make_item(server, "/ok/photo")
result = await downloader.download(item)
assert result == tmp_path / "xhs" / "media" / "note-1" / "001.jpg"
assert result.read_bytes() == DEFAULT_CONTENT
assert not list(result.parent.glob("*.part")), "下载完成后不应残留临时文件"
@pytest.mark.asyncio
async def test_download_all_shares_one_client(server, downloader, monkeypatch):
"""同一帖子的多个媒体必须复用同一个 AsyncClient（而不是每个文件新建一个连接池）"""
import media_downloader.downloader as downloader_module
created_clients = []
original = downloader_module.make_async_client
def counting_make_async_client(**kwargs):
client = original(**kwargs)
created_clients.append(client)
return client
monkeypatch.setattr(downloader_module, "make_async_client", counting_make_async_client)
items = [
make_item(server, "/ok/a", stem="001"),
make_item(server, "/ok/b", stem="002"),
make_item(server, "/ok/c", stem="003"),
]
paths = await downloader.download_all(items)
assert len(paths) == 3
assert [path.name for path in paths] == ["001.jpg", "002.jpg", "003.jpg"]
assert server.request_count == 3
assert len(created_clients) == 1, f"3 个文件应共用一个 client，实际创建了 {len(created_clients)} 个"
@pytest.mark.asyncio
async def test_download_all_empty(downloader):
assert await downloader.download_all([]) == []
# --------------------------------------------------------------------------- 跳过与覆盖
@pytest.mark.asyncio
async def test_existing_file_is_skipped_without_request(server, downloader):
item = make_item(server, "/ok/photo")
first = await downloader.download(item)
assert first is not None
server.reset()
second = await downloader.download(item)
assert second == first
assert server.request_count == 0, "已下载完成的文件不应再次发起请求"
@pytest.mark.asyncio
async def test_overwrite_forces_redownload(server, tmp_path):
downloader = MediaDownloader(
platform="xhs",
base_dir=tmp_path,
max_retries=0,
overwrite=True,
)
item = make_item(server, "/ok/photo")
await downloader.download(item)
server.reset()
await downloader.download(item)
assert server.request_count == 1
@pytest.mark.asyncio
async def test_zero_byte_file_is_treated_as_broken(server, downloader):
item = make_item(server, "/ok/photo")
target = downloader.base_dir / "xhs" / "media" / "note-1" / "001.jpg"
target.parent.mkdir(parents=True, exist_ok=True)
target.write_bytes(b"")
server.reset()
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 1
# --------------------------------------------------------------------------- 续传
@pytest.mark.asyncio
async def test_resume_from_partial_file(server, downloader, tmp_path):
"""预置半个片段 -> 必须带 Range 请求，且大小校验以 Content-Range 的总长为准。
这是关键回归：206 响应的 Content-Length 只是剩余长度（3072），
若被误当成文件总长，4096 != 3072 会误判为失败。
"""
item = make_item(server, "/ok/photo")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:1024])
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 1
assert server.requests[0].headers.get("range") == "bytes=1024-"
@pytest.mark.asyncio
async def test_server_ignoring_range_falls_back_to_full_download(server, downloader, tmp_path):
item = make_item(server, "/no-range/photo")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:1024])
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT, "服务端不支持 Range 时应截断重写，而不是拼接"
@pytest.mark.asyncio
async def test_truncated_response_retries_with_range(server, downloader, tmp_path):
"""服务端中途断流 -> 重试时带 Range 续传 -> 最终拼接出完整文件"""
item = make_item(server, "/truncated/photo?fail=1")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 2
assert server.requests[1].headers.get("range") == f"bytes={len(DEFAULT_CONTENT) // 2}-"
@pytest.mark.asyncio
async def test_stale_part_larger_than_remote_triggers_416_restart(server, downloader, tmp_path):
"""本地残留片段比远端还大 -> 416 -> 清空片段重来"""
item = make_item(server, "/ok/photo")
oversized = len(DEFAULT_CONTENT) + 5000
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(b"z" * oversized)
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
# 第一次带越界的 Range 拿到 416，片段被丢弃后第二次必须不带 Range 重新下载
assert server.request_count == 2
assert server.requests[0].headers.get("range") == f"bytes={oversized}-"
assert server.requests[1].headers.get("range") is None
@pytest.mark.asyncio
async def test_mismatched_content_range_start_discards_partial_file(server, downloader, tmp_path):
"""服务端返回的 Range 起点与请求不符时必须丢弃片段重下，而不是把错位内容拼进去"""
item = make_item(server, "/wrong-range/photo?delta=100&fail=1")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:1024])
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 2
assert server.requests[0].headers.get("range") == "bytes=1024-"
assert server.requests[1].headers.get("range") is None, "片段被丢弃后应重新完整下载"
@pytest.mark.asyncio
async def test_mismatched_content_range_never_produces_corrupt_file(server, downloader, tmp_path):
"""服务端持续返回错位 Range 时应放弃下载，绝不落盘内容错位的文件"""
item = make_item(server, "/wrong-range/broken?delta=100")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:1024])
result = await downloader.download(item)
assert result is None
directory = build_media_dir(downloader.base_dir, "xhs", item.content_id)
assert not list(directory.glob("*.jpg")), "内容错位的响应不能落盘"
@pytest.mark.asyncio
@pytest.mark.parametrize("bad_content_type", ["text/html", "text/plain", "application/json"])
async def test_error_page_with_200_is_rejected(server, downloader, bad_content_type):
"""CDN 防盗链页常以 200 + text/html 返回，不能当成媒体文件落盘"""
server.set_content_type("errpage", bad_content_type)
item = make_item(server, "/ctyped/errpage")
result = await downloader.download(item)
assert result is None
directory = build_media_dir(downloader.base_dir, "xhs", item.content_id)
assert not list(directory.glob("*.jpg")), "错误页不应被保存为媒体文件"
@pytest.mark.asyncio
async def test_content_type_mismatch_preserves_partial_progress(server, downloader, tmp_path):
"""错误页在读 body 之前就被拦下，本地已有的片段必须保留以便续传"""
server.set_content_type("limited", "text/html")
item = make_item(server, "/ctyped/limited")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:3000])
result = await downloader.download(item)
assert result is None
# 每一次尝试都要基于已有片段续传，而不是丢掉 3000 字节从头再来
ranges = [record.headers.get("range") for record in server.requests]
assert ranges == ["bytes=3000-"] * server.request_count, (
f"被拦截的响应没有消耗任何字节，已下载的进度不应被丢弃，实际: {ranges}"
)
@pytest.mark.asyncio
async def test_unsolicited_partial_response_is_rejected(server, downloader):
"""未发 Range 却收到起点非 0 的 206 时必须拒绝。
这里用"长度自洽但内容错位"的毒响应，确保拦住它的是起点守卫本身，
而不是被大小校验代偿（后者在长度恰好吻合时就失效了）。
"""
item = make_item(server, "/poison-206/photo?start=100")
result = await downloader.download(item)
assert result is None
directory = build_media_dir(downloader.base_dir, "xhs", item.content_id)
assert not list(directory.glob("*.jpg")), "起点错位的内容不能落盘"
@pytest.mark.asyncio
async def test_partial_206_with_unknown_total_is_rejected(server, downloader):
"""206 的 total 为 * 时无法确认完整性（服务端可能只给了区间的一部分），必须重下"""
item = make_item(server, "/unknown-total/photo")
result = await downloader.download(item)
assert result is None
directory = build_media_dir(downloader.base_dir, "xhs", item.content_id)
assert not list(directory.glob("*.jpg")), "无法确认完整性的内容不能落盘"
@pytest.mark.asyncio
async def test_partial_response_without_content_range_is_rejected(server, downloader, tmp_path):
"""206 缺少 Content-Range 时服务端行为不可预期（可能回全量体），必须丢弃片段重下"""
item = make_item(server, "/no-content-range/photo")
part = part_path_for(tmp_path, item, item.url)
part.write_bytes(DEFAULT_CONTENT[:1024])
result = await downloader.download(item)
assert result is None
assert not part.exists(), "无法确认区间语义的片段必须删除"
def test_total_size_semantics():
"""总长只能来自 Content-Range；total 未知时不反推，chunked 时忽略 Content-Length"""
# 206：以 Content-Range 的 total 为准
response = httpx.Response(206, headers={"content-range": "bytes 100-999/1000"})
content_range = MediaDownloader._parse_content_range(response)
assert content_range == (100, 999, 1000)
assert MediaDownloader._resolve_total_size(content_range, response, 206) == 1000
# total 为 *（服务端只返回部分区间）：不能拿 剩余长度+已下载量 反推出"总长"
response = httpx.Response(
206, headers={"content-range": "bytes 100-999/*", "content-length": "900"}
)
content_range = MediaDownloader._parse_content_range(response)
assert content_range == (100, 999, None)
assert MediaDownloader._resolve_total_size(content_range, response, 206) is None
# 缺少 Content-Range 的 206 无法判定区间语义
assert MediaDownloader._resolve_total_size(None, httpx.Response(206), 206) is None
# chunked 响应即使带了 Content-Length 也必须忽略（RFC 7230）
response = httpx.Response(200, headers={"transfer-encoding": "chunked", "content-length": "999"})
assert MediaDownloader._resolve_total_size(None, response, 200) is None
# 普通 200 用 Content-Length
response = httpx.Response(200, headers={"content-length": "4096"})
assert MediaDownloader._resolve_total_size(None, response, 200) == 4096
@pytest.mark.asyncio
async def test_chunked_response_ignores_bogus_content_length(server, downloader):
"""RFC 7230：有 Transfer-Encoding 时必须忽略 Content-Length，否则完整响应会被误判"""
item = make_item(server, "/chunked-bogus-cl/photo")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
@pytest.mark.asyncio
async def test_gzip_response_still_lands_decoded_bytes(server, downloader):
"""服务端无视 identity 强制压缩时，落盘的应是解码后的原始字节"""
item = make_item(server, "/gzip/photo")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
@pytest.mark.asyncio
async def test_malformed_url_in_backup_chain_does_not_abort(server, downloader):
"""畸形备用地址不能终止整条回退链（httpx.InvalidURL 不是 HTTPError 的子类）"""
item = make_item(
server,
"/status/500",
backup_urls=("http://h/a\nb.jpg", server.url("/ok/backup-after-bad")),
)
result = await downloader.download(item)
assert result is not None, "畸形地址应被跳过，继续尝试后面的候选"
@pytest.mark.asyncio
async def test_backup_candidates_are_capped(server, tmp_path):
"""备用地址数量必须有上限，否则超长 url_list 会放大成几十次请求"""
downloader = MediaDownloader("dy", base_dir=tmp_path, max_retries=0, max_candidates=2)
item = make_item(
server,
"/status/500",
backup_urls=tuple(server.url(f"/status/50{i}") for i in range(10)),
)
result = await downloader.download(item)
assert result is None
assert server.request_count == 2, f"候选总数应被裁剪到 2，实际请求 {server.request_count} 次"
@pytest.mark.asyncio
async def test_non_string_url_never_escapes_contract(server, downloader):
"""MediaItem 是公开契约，非字符串 url 必须收敛为 None 而不是抛异常"""
for bad_url in (123, ["http://x"], {"url": "http://x"}, None):
item = MediaItem(url=bad_url, media_type=MediaType.IMAGE, content_id="c")
assert await downloader.download(item) is None
@pytest.mark.asyncio
async def test_part_file_name_is_sanitized(server, downloader, tmp_path):
"""stem 未清洗时 .part 会写到 base_dir 之外，临时文件名同样要清洗"""
item = make_item(server, "/ok/photo", stem="../../../../tmp/ESCAPED")
result = await downloader.download(item)
assert result is not None
assert result.is_relative_to(tmp_path)
assert not list(Path("/tmp").glob("ESCAPED*")), "临时文件不得落在 base_dir 之外"
@pytest.mark.asyncio
async def test_update_credentials_switches_proxy(server, tmp_path, monkeypatch):
"""代理池就地刷新后，下载器必须跟着换代理，否则长跑时媒体下载会一直走失效代理"""
import media_downloader.downloader as downloader_module
used_proxies = []
original = downloader_module.make_async_client
def spy(**kwargs):
used_proxies.append(kwargs.get("proxy"))
return original(**kwargs)
monkeypatch.setattr(downloader_module, "make_async_client", spy)
downloader = MediaDownloader("xhs", base_dir=tmp_path, max_retries=0)
await downloader.download(make_item(server, "/ok/cred-1"))
downloader.update_credentials(proxy="http://new-proxy:8080")
await downloader.download(make_item(server, "/ok/cred-2", stem="002"))
assert used_proxies == [None, "http://new-proxy:8080"]
@pytest.mark.asyncio
async def test_binary_content_type_is_accepted(server, downloader):
"""通用二进制类型无法判定内容，应当放行（CDN 常见）"""
server.set_content_type("blob", "application/octet-stream")
item = make_item(server, "/ctyped/blob")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
@pytest.mark.asyncio
async def test_video_content_type_accepted_for_audio_stream(server, downloader):
"""DASH 音频轨的 Content-Type 是 audio/*，属于视频任务的一部分，必须放行"""
server.set_content_type("audio-track", "audio/mp4")
item = make_item(server, "/ctyped/audio-track", media_type=MediaType.VIDEO, stem="audio")
result = await downloader.download(item)
assert result is not None
assert result.suffix == ".m4a"
@pytest.mark.asyncio
async def test_stale_part_from_other_url_is_discarded(server, downloader, tmp_path):
"""URL 变化时旧的临时片段必须作废（不能把新内容拼到旧半成品上），且下载成功后清理干净"""
item = make_item(server, "/ok/photo")
stale_part = part_path_for(tmp_path, item, server.url("/ok/other"))
stale_part.write_bytes(b"y" * 1024)
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.requests[0].headers.get("range") is None, "不同 URL 的片段不应被当作续传基础"
assert not stale_part.exists(), "成功后应清理同 stem 的陈旧片段"
assert not list(result.parent.glob("*.part"))
# --------------------------------------------------------------------------- 错误处理
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [403, 404, 410])
async def test_fatal_status_is_not_retried(server, downloader, status):
item = make_item(server, f"/status/{status}")
result = await downloader.download(item)
assert result is None
assert server.request_count == 1, f"HTTP {status} 不应重试"
@pytest.mark.asyncio
async def test_server_error_is_retried_until_success(server, tmp_path):
downloader = MediaDownloader(
platform="xhs",
base_dir=tmp_path,
max_retries=3,
retry_base_delay=0.001,
retry_max_delay=0.005,
)
item = make_item(server, "/flaky/photo?fail=2")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 3
@pytest.mark.asyncio
async def test_retry_exhausted_returns_none(server, downloader):
item = make_item(server, "/status/500")
result = await downloader.download(item)
assert result is None
assert server.request_count == 3 # 1 次 + 2 次重试
@pytest.mark.asyncio
async def test_empty_body_is_rejected(server, downloader):
item = make_item(server, "/empty/photo")
result = await downloader.download(item)
assert result is None
@pytest.mark.asyncio
async def test_timeout_returns_none(server, tmp_path):
downloader = MediaDownloader(
platform="xhs",
base_dir=tmp_path,
timeout=0.2,
max_retries=0,
)
item = make_item(server, "/slow/photo?s=2")
result = await downloader.download(item)
assert result is None
@pytest.mark.asyncio
@pytest.mark.parametrize(
"bad_url",
[
"",
"//cdn.example.com/x.jpg", # 协议相对地址，httpx 无法处理
"ftp://cdn.example.com/x.jpg",
"not-a-url",
],
)
async def test_invalid_url_is_rejected_without_request(server, downloader, bad_url):
item = MediaItem(url=bad_url, media_type=MediaType.IMAGE, content_id="note-1")
assert await downloader.download(item) is None
assert server.request_count == 0
# --------------------------------------------------------------------------- 备用地址
@pytest.mark.asyncio
async def test_backup_url_used_after_primary_fails(server, downloader):
item = make_item(
server,
"/status/403",
backup_urls=(server.url("/ok/backup"),),
)
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
assert server.request_count == 2
assert server.paths() == ["/status/403", "/ok/backup"]
# --------------------------------------------------------------------------- 响应头行为
@pytest.mark.asyncio
async def test_chunked_response_without_content_length(server, downloader):
item = make_item(server, "/chunked/photo")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
@pytest.mark.asyncio
async def test_redirect_is_followed(server, downloader):
item = make_item(server, "/redirect/photo")
result = await downloader.download(item)
assert result is not None
assert result.read_bytes() == DEFAULT_CONTENT
@pytest.mark.asyncio
async def test_extension_inferred_from_content_type(server, downloader):
server.set_content_type("anon", "image/webp")
item = make_item(server, "/ctyped/anon")
result = await downloader.download(item)
assert result is not None
assert result.suffix == ".webp"
@pytest.mark.asyncio
async def test_explicit_extension_wins(server, downloader):
item = make_item(server, "/ctyped/anon", extension=".png")
result = await downloader.download(item)
assert result is not None
assert result.suffix == ".png"
@pytest.mark.asyncio
async def test_request_headers(server, downloader):
item = make_item(server, "/ok/photo")
await downloader.download(item)
headers = server.requests[0].headers
assert headers.get("accept-encoding") == "identity", (
"必须显式声明不压缩，否则解码后字节数与 Content-Length 不等，大小校验会误判"
)
assert "user-agent" in headers
# 下载器不认识平台，Referer 必须由平台侧注入（见 test_xhs_core_sends_referer）
assert headers.get("referer") is None
@pytest.mark.asyncio
async def test_item_headers_override_defaults(server, downloader):
item = make_item(server, "/ok/photo", headers={"Cookie": "session=abc"})
await downloader.download(item)
assert server.requests[0].headers.get("cookie") == "session=abc"
@pytest.mark.asyncio
async def test_extra_headers_are_applied(server, tmp_path):
"""平台侧注入的 Referer 等反爬头必须真正发出去"""
downloader = MediaDownloader(
platform="bili",
base_dir=tmp_path,
max_retries=0,
extra_headers={"Referer": "https://www.bilibili.com/"},
)
item = make_item(server, "/ok/photo")
await downloader.download(item)
headers = server.requests[0].headers
assert headers.get("referer") == "https://www.bilibili.com/"
@pytest.mark.asyncio
async def test_proxy_is_passed_to_client(server, tmp_path, monkeypatch):
import media_downloader.downloader as downloader_module
captured: dict = {}
original = downloader_module.make_async_client
def spy(**kwargs):
captured.update(kwargs)
return original(**kwargs)
monkeypatch.setattr(downloader_module, "make_async_client", spy)
downloader = MediaDownloader(
platform="xhs",
base_dir=tmp_path,
proxy="http://127.0.0.1:9",
max_retries=0,
)
item = make_item(server, "/ok/photo")
# 代理不可用会失败，但我们要断言的是参数透传
await downloader.download(item)
assert captured.get("proxy") == "http://127.0.0.1:9"
assert captured.get("follow_redirects") is True
# --------------------------------------------------------------------------- 路径安全
@pytest.mark.asyncio
async def test_malicious_content_id_stays_inside_base_dir(server, downloader, tmp_path):
item = make_item(
server,
"/ok/photo",
content_id="../../../etc",
stem="../../passwd",
)
result = await downloader.download(item)
assert result is not None
assert result.resolve().is_relative_to(tmp_path.resolve())
# --------------------------------------------------------------------------- DASH 合流
def _make_dash_streams(tmp_path: Path) -> tuple[bytes, bytes]:
"""用 ffmpeg 生成一对真实的 DASH 分轨素材（视频轨 / 音频轨）"""
ffmpeg = shutil.which("ffmpeg")
if not ffmpeg:
pytest.skip("本机未安装 ffmpeg")
video = tmp_path / "_src_video.m4s"
audio = tmp_path / "_src_audio.m4s"
subprocess.run(
[ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi",
"-i", "color=c=blue:s=160x120:d=1", "-c:v", "mpeg4", "-f", "mp4", str(video)],
check=True, capture_output=True,
)
subprocess.run(
[ffmpeg, "-hide_banner", "-loglevel", "error", "-y", "-f", "lavfi",
"-i", "sine=f=440:d=1", "-c:a", "aac", "-f", "mp4", str(audio)],
check=True, capture_output=True,
)
return video.read_bytes(), audio.read_bytes()
@pytest.mark.asyncio
async def test_dash_streams_are_downloaded_and_merged(server, tmp_path):
video_bytes, audio_bytes = _make_dash_streams(tmp_path)
server.set_content("dash-v", video_bytes)
server.set_content("dash-a", audio_bytes)
downloader = MediaDownloader("bili", base_dir=tmp_path / "out", max_retries=0)
item = MediaItem(
url=server.url("/ok/dash-v?ct=video/mp4"),
audio_url=server.url("/ok/dash-a?ct=audio/mp4"),
media_type=MediaType.VIDEO,
content_id="BV1test",
stem="video",
)
result = await downloader.download(item)
assert result is not None
assert result.suffix == ".mp4"
assert result.stat().st_size > 0
assert server.request_count == 2, "分轨应各下载一次"
assert not list(result.parent.glob(".tmp-*")), "合流完成后必须清理临时目录"
probe = shutil.which("ffprobe")
if probe:
stream_types = subprocess.run(
[probe, "-v", "error", "-show_entries", "stream=codec_type",
"-of", "csv=p=0", str(result)],
check=True, capture_output=True, text=True,
).stdout
assert "video" in stream_types and "audio" in stream_types
@pytest.mark.asyncio
async def test_dash_failure_cleans_up_temp_dir(server, tmp_path):
video_bytes, _ = _make_dash_streams(tmp_path)
server.set_content("dash-v2", video_bytes)
downloader = MediaDownloader("bili", base_dir=tmp_path / "out", max_retries=0)
item = MediaItem(
url=server.url("/ok/dash-v2?ct=video/mp4"),
audio_url=server.url("/status/403"),
media_type=MediaType.VIDEO,
content_id="BV1fail",
stem="video",
)
result = await downloader.download(item)
assert result is None
media_dir = tmp_path / "out" / "bili" / "media" / "BV1fail"
assert media_dir.is_dir()
assert not list(media_dir.glob(".tmp-*")), "失败的 DASH 下载同样要清理临时目录"
# --------------------------------------------------------------------------- paths 纯函数
@pytest.mark.parametrize(
("raw", "expected"),
[
("note-123_ab", "note-123_ab"),
("../../etc/passwd", "___etc_passwd"),
("", "unknown"),
("...", "unknown"),
("a" * 100, "a" * 64),
("CON", "_CON"),
("con.txt", "_con.txt"),
],
)
def test_sanitize_component(raw, expected):
assert sanitize_component(raw) == expected
@pytest.mark.parametrize("raw", ["..", ".", "", " "])
def test_sanitize_component_fallback_used_for_empty(raw):
assert sanitize_component(raw, fallback="fb") == "fb"
@pytest.mark.parametrize(
("url", "content_type", "explicit", "expected"),
[
("https://cdn.com/a/b.jpg", None, None, ".jpg"),
("https://cdn.com/a/b.jpeg?x=1", None, None, ".jpeg"),
("https://cdn.com/a/b.mp4!large", None, None, ".mp4"),
("https://cdn.com/a/b", "image/webp; charset=utf-8", None, ".webp"),
("https://cdn.com/a/b.unknownext", "video/mp4", None, ".mp4"),
("https://cdn.com/a/b.jpg", "image/png", ".png", ".png"),
("https://cdn.com/a/b.exe", None, None, ".jpg"),
("", None, None, ".jpg"),
],
)
def test_guess_extension(url, content_type, explicit, expected):
assert guess_extension(url, content_type=content_type, explicit=explicit, default=".jpg") == expected
def test_guess_extension_ignores_explicit_outside_whitelist():
assert guess_extension("https://cdn.com/a.jpg", explicit=".exe", default=".jpg") == ".jpg"
@pytest.mark.parametrize(
("platform", "content_id", "expected"),
[
("xhs", "note-1", "xhs/media/note-1"),
("dy", "../../evil", "dy/media/___evil"),
],
)
def test_build_media_dir(tmp_path, platform, content_id, expected):
assert build_media_dir(tmp_path, platform, content_id) == tmp_path / expected
def test_build_media_path_normalizes_extension():
path = build_media_path("/base", "xhs", "note-1", "cover", "JPG")
assert path == Path("/base/xhs/media/note-1/cover.jpg")
def test_ensure_within_rejects_escape(tmp_path):
with pytest.raises(ValueError):
ensure_within(tmp_path, tmp_path / ".." / "outside.mp4")
def test_url_fingerprint_is_stable_and_short():
fingerprint = url_fingerprint("https://cdn.com/a.jpg?sig=1")
assert fingerprint == url_fingerprint("https://cdn.com/a.jpg?sig=1")
assert fingerprint != url_fingerprint("https://cdn.com/a.jpg?sig=2")
assert len(fingerprint) == 8
def test_redact_url_drops_query():
assert redact_url("https://cdn.com/a.jpg?sign=secret&t=1") == "https://cdn.com/a.jpg"
assert redact_url("not-a-url") == "<invalid-url>"
tests/test_media_e2e.py
+172
View File
@@ -0,0 +1,172 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_e2e.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""真实平台端到端测试（默认跳过）。
会真实访问 B 站接口与 CDN 并下载一个完整视频（约几十 MB），因此只在人工验证时运行：
MEDIA_E2E=1 .venv/bin/python -m pytest tests/test_media_e2e.py -v
B 站无需登录即可获取公开视频的基础清晰度，wbi 签名是纯算法，
所以这里不依赖浏览器与账号，只验证「提取 -> 下载 -> 合流」的真实链路。
请勿把本文件接入 CI：它会对目标平台产生真实流量。
"""
from __future__ import annotations
import os
import shutil
import subprocess
from pathlib import Path
import config
import httpx
import pytest
from media_downloader import MediaDownloader, is_ffmpeg_available
from media_platform.bilibili import media as bili_media
from media_platform.bilibili.help import BilibiliSign
pytestmark = pytest.mark.skipif(
os.getenv("MEDIA_E2E") != "1",
reason="真实平台测试，设置 MEDIA_E2E=1 后运行",
)
BV_ID = os.getenv("MEDIA_E2E_BVID", "BV1dwuKzmE26")
USER_AGENT = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 "
"(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"
)
async def _fetch_wbi_keys(client: httpx.AsyncClient) -> tuple[str, str]:
"""从 nav 接口获取 wbi 签名所需的 img_key / sub_key（无需登录）"""
response = await client.get(
"https://api.bilibili.com/x/web-interface/nav", headers={"User-Agent": USER_AGENT}
)
wbi_img = response.json()["data"]["wbi_img"]
img_key = wbi_img["img_url"].rsplit("/", 1)[1].split(".")[0]
sub_key = wbi_img["sub_url"].rsplit("/", 1)[1].split(".")[0]
return img_key, sub_key
@pytest.mark.asyncio
async def test_bilibili_downloads_cover_and_video(tmp_path: Path):
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
view_response = await client.get(
"https://api.bilibili.com/x/web-interface/view",
params={"bvid": BV_ID},
headers={"User-Agent": USER_AGENT},
)
view_data = view_response.json()["data"]
aid, cid, pic = view_data["aid"], view_data["cid"], view_data["pic"]
img_key, sub_key = await _fetch_wbi_keys(client)
params = {
"avid": aid,
"cid": cid,
"qn": 80,
"fourk": 1,
"fnval": bili_media.DASH_FNVAL,
"platform": "pc",
}
signed = BilibiliSign(img_key, sub_key).sign(params)
play_response = await client.get(
"https://api.bilibili.com/x/player/wbi/playurl",
params=signed,
headers={"User-Agent": USER_AGENT, "Referer": "https://www.bilibili.com"},
)
payload = play_response.json()
assert payload.get("code") == 0, f"playurl 接口返回异常: {payload}"
play_info = payload["data"]
view = {"aid": aid, "cid": cid, "bvid": BV_ID, "pic": pic}
cover_item = bili_media.build_cover_item(view, BV_ID)
# 生产路径由 core._media_headers() 注入 Referer；这里绕过 core 直接构造下载器，
# 必须自行带上，否则 B 站 CDN 一律 403
downloader = MediaDownloader(
"bili",
base_dir=tmp_path,
max_retries=2,
timeout=120.0,
extra_headers={"Referer": "https://www.bilibili.com/", "User-Agent": USER_AGENT},
)
cover_path = await downloader.download(cover_item)
# 与 core 的编排保持一致：未登录时 B 站会返回高清晰度 URL 但取流被 CDN 403，
# 需要逐档降级到实际可下载的档位
video_item = None
video_path = None
if is_ffmpeg_available():
quality = getattr(config, "BILI_QN", 80)
while quality is not None:
candidate = bili_media.build_dash_item(play_info, BV_ID, preferred_quality=quality)
if candidate is None:
break
video_item = candidate
video_path = await downloader.download(candidate)
if video_path is not None:
break
lower_quality = bili_media.next_lower_quality(play_info, quality)
if lower_quality is None:
break
print(f"[e2e] 清晰度 {quality} 取流失败，降级到 {lower_quality}")
quality = lower_quality
if video_path is None:
video_item = bili_media.build_durl_item(play_info, BV_ID)
if video_item is None:
fallback = await _fetch_play_info_for_mp4(aid, cid)
video_item = bili_media.build_durl_item(fallback or {}, BV_ID)
if video_item is not None:
video_path = await downloader.download(video_item)
assert video_item is not None, "未能从 playurl 响应中提取到视频流"
assert cover_path is not None and cover_path.stat().st_size > 0
assert cover_path.read_bytes()[:2] == b"\xff\xd8", "封面应为 JPEG"
assert video_path is not None and video_path.stat().st_size > 0
if video_item.is_dash:
assert video_path.suffix == ".mp4", "DASH 合流产物应为 mp4"
assert not list(video_path.parent.glob(".tmp-*")), "临时目录应被清理"
probe = shutil.which("ffprobe")
if probe:
result = subprocess.run(
[probe, "-v", "error", "-show_entries", "stream=codec_type",
"-of", "csv=p=0", str(video_path)],
check=True, capture_output=True, text=True,
)
stream_types = [line.strip() for line in result.stdout.splitlines() if line.strip()]
assert "video" in stream_types and "audio" in stream_types, (
f"DASH 合流后应同时包含视频轨与音频轨，实际: {stream_types}"
)
async def _fetch_play_info_for_mp4(aid: int, cid: int) -> dict:
"""按 mp4 格式再取一次播放地址（DASH 响应里不带 durl）"""
async with httpx.AsyncClient(timeout=30, follow_redirects=True) as client:
img_key, sub_key = await _fetch_wbi_keys(client)
params = {
"avid": aid,
"cid": cid,
"qn": 80,
"fourk": 1,
"fnval": bili_media.MP4_FNVAL,
"platform": "pc",
}
signed = BilibiliSign(img_key, sub_key).sign(params)
response = await client.get(
"https://api.bilibili.com/x/player/wbi/playurl",
params=signed,
headers={"User-Agent": USER_AGENT, "Referer": "https://www.bilibili.com"},
)
payload = response.json()
return payload.get("data") or {}
tests/test_media_extractors.py
+741
View File
@@ -0,0 +1,741 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_extractors.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""各平台媒体地址提取器单测。
每个平台的 ``media.py`` 都是纯函数，这里用最小化的真实结构 fixture 直接断言
产出的 ``MediaItem`` 列表，不发起任何网络请求。
"""
from __future__ import annotations
import config
import pytest
from media_downloader import MediaType
from media_platform.bilibili import media as bili_media
from media_platform.douyin import media as douyin_media
from media_platform.kuaishou import media as kuaishou_media
from media_platform.weibo import media as weibo_media
from media_platform.xhs import media as xhs_media
# --------------------------------------------------------------------------- xhs fixture
XHS_VIDEO_NOTE = {
"note_id": "video-note-1",
"type": "video",
"image_list": [{"url_default": "https://sns-webpic.xhscdn.com/img-cover"}],
"cover": {"url_default": "https://sns-webpic.xhscdn.com/top-cover"},
"video": {
"consumer": {"origin_video_key": "spec/abc/def"},
"media": {"stream": {"h264": [{"master_url": "https://sns-video-hw.xhscdn.com/master"}]}},
"cover": {"url_default": "https://sns-webpic.xhscdn.com/video-cover"},
},
}
XHS_IMAGE_NOTE = {
"note_id": "image-note-1",
"type": "normal",
"image_list": [
{"url_default": "https://sns-webpic.xhscdn.com/1"},
{"url_default": "https://sns-webpic.xhscdn.com/2"},
{"url_default": "https://sns-webpic.xhscdn.com/3"},
],
}
# --------------------------------------------------------------------------- xhs 用例
def test_xhs_video_note_yields_cover_and_video():
items = xhs_media.build_media_items(XHS_VIDEO_NOTE)
assert [item.stem for item in items] == ["cover", "video"]
assert [item.media_type for item in items] == [MediaType.IMAGE, MediaType.VIDEO]
cover, video = items
assert cover.url == "https://sns-webpic.xhscdn.com/video-cover"
assert video.content_id == "video-note-1"
# 优先无水印源片，带水印的 master_url 作为备用
assert video.url == "https://sns-video-bd.xhscdn.com/spec/abc/def"
assert video.backup_urls == ("https://sns-video-hw.xhscdn.com/master",)
def test_xhs_video_note_falls_back_to_top_level_cover():
note = {**XHS_VIDEO_NOTE, "video": {**XHS_VIDEO_NOTE["video"], "cover": {}}}
note.pop("cover", None)
note["cover"] = {"url_default": "https://sns-webpic.xhscdn.com/top-cover"}
items = xhs_media.build_media_items(note)
assert items[0].url == "https://sns-webpic.xhscdn.com/top-cover"
def test_xhs_video_note_falls_back_to_first_image_when_no_cover():
note = {
**XHS_VIDEO_NOTE,
"video": {**XHS_VIDEO_NOTE["video"], "cover": {}},
"cover": {},
}
items = xhs_media.build_media_items(note)
assert items[0].stem == "cover"
assert items[0].url == "https://sns-webpic.xhscdn.com/img-cover"
def test_xhs_video_note_without_any_cover_still_yields_video():
note = {
**XHS_VIDEO_NOTE,
"video": {**XHS_VIDEO_NOTE["video"], "cover": {}},
"cover": {},
"image_list": [],
}
items = xhs_media.build_media_items(note)
assert [item.stem for item in items] == ["video"]
def test_xhs_video_note_without_origin_key_uses_master_url():
note = {
**XHS_VIDEO_NOTE,
"video": {"media": {"stream": {"h264": [{"master_url": "https://sns-video-hw.xhscdn.com/master"}]}}},
}
items = xhs_media.build_media_items(note)
video = next(item for item in items if item.media_type == MediaType.VIDEO)
assert video.url == "https://sns-video-hw.xhscdn.com/master"
assert video.backup_urls == ()
def test_xhs_international_does_not_build_xhscdn_url(monkeypatch):
monkeypatch.setattr(config, "XHS_INTERNATIONAL", True)
items = xhs_media.build_media_items(XHS_VIDEO_NOTE)
video = next(item for item in items if item.media_type == MediaType.VIDEO)
assert video.url == "https://sns-video-hw.xhscdn.com/master"
assert "sns-video-bd" not in video.url
def test_xhs_image_note_yields_numbered_images_without_duplicate_cover():
items = xhs_media.build_media_items(XHS_IMAGE_NOTE)
assert [item.stem for item in items] == ["001", "002", "003"]
assert [item.url for item in items] == [
"https://sns-webpic.xhscdn.com/1",
"https://sns-webpic.xhscdn.com/2",
"https://sns-webpic.xhscdn.com/3",
]
assert all(item.media_type is MediaType.IMAGE for item in items)
assert not any(item.stem == "cover" for item in items)
def test_xhs_image_note_skips_entries_without_url():
note = {
"note_id": "n1",
"type": "normal",
"image_list": [{"url_default": ""}, {"url": "https://cdn/x"}, {"no_url": True}],
}
items = xhs_media.build_media_items(note)
assert [item.url for item in items] == ["https://cdn/x"]
def test_xhs_missing_note_id_returns_empty():
assert xhs_media.build_media_items({"type": "video"}) == []
@pytest.mark.parametrize("payload", [None, "not-a-dict", [], 123])
def test_xhs_non_dict_input_is_ignored(payload):
assert xhs_media.build_media_items(payload) == []
# --------------------------------------------------------------------------- douyin fixture
DY_VIDEO_ITEM = {
"aweme_id": "7300000000000000001",
"desc": "video",
"video": {
"play_addr_h264": {"url_list": ["https://cdn-a/h264", "https://cdn-b/h264"]},
"play_addr": {"url_list": ["https://cdn-a/play"]},
"raw_cover": {"url_list": ["https://cdn-a/cover", "https://cdn-b/cover"]},
},
}
DY_IMAGE_ITEM = {
"aweme_id": "7300000000000000002",
"desc": "images",
"images": [
{"url_list": ["https://cdn-a/img1-small", "https://cdn-b/img1"]},
{"url_list": ["https://cdn-a/img2-small", "https://cdn-b/img2"]},
],
"video": {"raw_cover": {"url_list": ["https://cdn/cover"]}},
}
# --------------------------------------------------------------------------- douyin 用例
def test_douyin_video_item_yields_cover_and_video():
items = douyin_media.build_media_items(DY_VIDEO_ITEM)
assert [item.stem for item in items] == ["cover", "video"]
cover, video = items
assert cover.url == "https://cdn-b/cover"
# url_list 最后一个通常无水印，其余作为备用
assert video.url == "https://cdn-b/h264"
assert video.backup_urls == ("https://cdn-a/h264",)
assert video.media_type is MediaType.VIDEO
def test_douyin_video_falls_back_to_lower_quality_addr():
item = {
"aweme_id": "1",
"video": {"play_addr_256": {"url_list": ["https://cdn/only", "https://cdn/best"]}},
}
items = douyin_media.build_media_items(item)
video = next(entry for entry in items if entry.media_type is MediaType.VIDEO)
assert video.url == "https://cdn/best"
assert video.backup_urls == ("https://cdn/only",)
def test_douyin_video_falls_back_to_bit_rate_list():
item = {
"aweme_id": "1",
"video": {
"bit_rate": [
{"bit_rate": 100, "play_addr": {"url_list": ["https://cdn/low"]}},
{"bit_rate": 900, "play_addr": {"url_list": ["https://cdn/high"]}},
]
},
}
items = douyin_media.build_media_items(item)
video = next(entry for entry in items if entry.media_type is MediaType.VIDEO)
assert video.url == "https://cdn/high"
def test_douyin_image_item_yields_numbered_images_without_cover():
items = douyin_media.build_media_items(DY_IMAGE_ITEM)
assert [item.stem for item in items] == ["001", "002"]
assert [item.url for item in items] == ["https://cdn-b/img1", "https://cdn-b/img2"]
assert not any(item.stem == "cover" for item in items)
def test_douyin_empty_video_yields_nothing():
assert douyin_media.build_media_items({"aweme_id": "1", "video": {}}) == []
def test_douyin_cover_key_priority_is_locked():
"""封面字段有明确优先级，不能只钉住第一个键"""
detail = {
"aweme_id": "1",
"video": {
"raw_cover": {"url_list": ["https://cdn/raw"]},
"origin_cover": {"url_list": ["https://cdn/origin"]},
"cover": {"url_list": ["https://cdn/cover"]},
"dynamic_cover": {"url_list": ["https://cdn/dynamic"]},
},
}
assert douyin_media.extract_cover_url(detail) == "https://cdn/raw"
del detail["video"]["raw_cover"]
assert douyin_media.extract_cover_url(detail) == "https://cdn/origin"
del detail["video"]["origin_cover"]
assert douyin_media.extract_cover_url(detail) == "https://cdn/cover"
del detail["video"]["cover"]
assert douyin_media.extract_cover_url(detail) == "https://cdn/dynamic"
def test_douyin_video_addr_key_priority_is_locked():
"""h264 > 256 > play_addr 的降级顺序必须被钉住"""
detail = {
"aweme_id": "1",
"video": {
"play_addr_h264": {"url_list": ["https://cdn/h264"]},
"play_addr_256": {"url_list": ["https://cdn/256"]},
"play_addr": {"url_list": ["https://cdn/plain"]},
},
}
assert douyin_media.extract_video_urls(detail) == ["https://cdn/h264"]
del detail["video"]["play_addr_h264"]
assert douyin_media.extract_video_urls(detail) == ["https://cdn/256"]
del detail["video"]["play_addr_256"]
assert douyin_media.extract_video_urls(detail) == ["https://cdn/plain"]
def test_bilibili_falls_back_to_lowest_quality_keeping_codec_priority(monkeypatch):
"""所有档位都超过用户设置时取最低清晰度，但编码优先级仍要生效（AVC 优先）"""
monkeypatch.setattr(config, "BILI_QN", 16)
play_info = {
"dash": {
"video": [
{"id": 32, "codecid": 12, "bandwidth": 300000, "base_url": "https://upos/hevc.m4s"},
{"id": 32, "codecid": 7, "bandwidth": 200000, "base_url": "https://upos/avc.m4s"},
{"id": 80, "codecid": 7, "bandwidth": 900000, "base_url": "https://upos/80.m4s"},
]
}
}
assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/avc.m4s"
def test_bilibili_tolerates_string_typed_ids():
"""接口偶发把 id/bandwidth 返回成字符串，不能因此整条媒体被跳过"""
play_info = {
"dash": {
"video": [
{"id": "80", "codecid": "12", "bandwidth": "900000", "base_url": "https://upos/hevc.m4s"},
{"id": "80", "codecid": "7", "bandwidth": "800000", "base_url": "https://upos/avc.m4s"},
],
"audio": [{"id": "30280", "bandwidth": "192000", "base_url": "https://upos/a.m4s"}],
}
}
item = bili_media.build_dash_item(play_info, "BV1")
assert item is not None
assert item.url == "https://upos/avc.m4s"
def test_bilibili_durl_stem_differs_from_dash_stem():
"""直链降级产物必须与 DASH 产物区分，否则低清文件会永久阻塞高清路径"""
dash_item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1")
durl_item = bili_media.build_durl_item(BILI_DURL_PLAY_INFO, "BV1")
assert dash_item.stem == "video"
assert durl_item.stem == "video-durl"
# --------------------------------------------------------------------------- kuaishou fixture
KS_VIDEO_ITEM = {
"photo": {
"id": "3xabcdefg",
"caption": "hello",
"photoH265Url": "https://ks/h265.mp4",
"photoUrl": "https://ks/h264.mp4",
"coverUrl": "https://ks/cover.jpg",
},
"author": {"id": "u1"},
}
# --------------------------------------------------------------------------- kuaishou 用例
def test_kuaishou_video_item_yields_cover_and_video():
items = kuaishou_media.build_media_items(KS_VIDEO_ITEM)
assert [item.stem for item in items] == ["cover", "video"]
cover, video = items
assert cover.url == "https://ks/cover.jpg"
assert video.url == "https://ks/h265.mp4"
assert video.backup_urls == ("https://ks/h264.mp4",)
assert video.content_id == "3xabcdefg"
def test_kuaishou_falls_back_to_video_resource_representations():
item = {
"photo": {
"id": "1",
"videoResource": {
"h264": {
"adaptationSet": [
{"representation": [{"url": "https://ks/rep.mp4"}]}
]
}
},
}
}
items = kuaishou_media.build_media_items(item)
video = next(entry for entry in items if entry.media_type is MediaType.VIDEO)
assert video.url == "https://ks/rep.mp4"
def test_kuaishou_cover_falls_back_to_cover_urls_list():
item = {"photo": {"id": "1", "coverUrls": [{"url": "https://ks/c1.jpg"}], "photoUrl": "https://ks/v.mp4"}}
items = kuaishou_media.build_media_items(item)
cover = next(entry for entry in items if entry.stem == "cover")
assert cover.url == "https://ks/c1.jpg"
def test_kuaishou_missing_photo_id_yields_nothing():
assert kuaishou_media.build_media_items({"photo": {"photoUrl": "https://ks/v.mp4"}}) == []
# --------------------------------------------------------------------------- weibo fixture
WB_IMAGE_MBLOG = {
"id": "5000000000000000",
"pics": [
{"url": "https://wx1.sinaimg.cn/orj360/abc.jpg", "pid": "abc"},
"https://wx2.sinaimg.cn/thumbnail/def.jpg",
],
}
WB_VIDEO_MBLOG = {
"id": "5000000000000001",
"page_info": {
"type": "video",
"page_pic": {"url": "https://wx1.sinaimg.cn/orj360/cover.jpg"},
"media_info": {
"mp4_hd_mp4": "https://f.video.weibocdn.com/hd.mp4",
"mp4_ld_mp4": "https://f.video.weibocdn.com/ld.mp4",
},
},
}
# --------------------------------------------------------------------------- weibo 用例
def test_weibo_image_url_is_rewritten_to_large_via_agent_host():
assert (
weibo_media.rewrite_image_url("https://wx1.sinaimg.cn/orj360/abc.jpg")
== "https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg"
)
def test_weibo_image_url_rewrite_ignores_query_string():
assert (
weibo_media.rewrite_image_url("https://wx1.sinaimg.cn/orj360/abc.jpg?v=1")
== "https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg"
)
def test_weibo_image_note_yields_numbered_images():
items = weibo_media.build_media_items(WB_IMAGE_MBLOG)
assert [item.stem for item in items] == ["001", "002"]
assert [item.url for item in items] == [
"https://i1.wp.com/wx1.sinaimg.cn/large/abc.jpg",
"https://i1.wp.com/wx2.sinaimg.cn/large/def.jpg",
]
def test_weibo_video_note_yields_cover_and_video():
items = weibo_media.build_media_items(WB_VIDEO_MBLOG)
assert [item.stem for item in items] == ["cover", "video"]
cover, video = items
assert cover.url == "https://wx1.sinaimg.cn/orj360/cover.jpg"
assert video.url == "https://f.video.weibocdn.com/hd.mp4"
assert video.backup_urls == ("https://f.video.weibocdn.com/ld.mp4",)
def test_weibo_plain_note_yields_nothing():
assert weibo_media.build_media_items({"id": "1", "text": "just text"}) == []
def test_weibo_retweet_uses_original_media():
"""转发微博的媒体在 retweeted_status 里，顶层没有；不处理会一条都下不到"""
retweet = {
"id": "6000000000000000",
"text": "转发内容",
"retweeted_status": {
"id": "5000000000000009",
"pics": [{"url": "https://wx1.sinaimg.cn/orj360/original.jpg"}],
},
}
items = weibo_media.build_media_items(retweet)
assert [item.stem for item in items] == ["001"]
assert items[0].url == "https://i1.wp.com/wx1.sinaimg.cn/large/original.jpg"
assert items[0].content_id == "6000000000000000", "目录应按转发帖自身 id 组织"
def test_weibo_retweet_with_video():
retweet = {
"id": "6000000000000001",
"retweeted_status": {
"page_info": {
"type": "video",
"page_pic": {"url": "https://wx1.sinaimg.cn/orj360/cover.jpg"},
"media_info": {"mp4_hd_mp4": "https://f.video.weibocdn.com/hd.mp4"},
}
},
}
items = weibo_media.build_media_items(retweet)
assert [item.stem for item in items] == ["cover", "video"]
assert items[1].url == "https://f.video.weibocdn.com/hd.mp4"
def test_weibo_own_media_wins_over_retweeted_status():
"""自己带媒体时不应去看被转发的内容"""
mblog = {
"id": "6000000000000002",
"pics": [{"url": "https://wx1.sinaimg.cn/orj360/mine.jpg"}],
"retweeted_status": {"pics": [{"url": "https://wx1.sinaimg.cn/orj360/theirs.jpg"}]},
}
items = weibo_media.build_media_items(mblog)
assert items[0].url == "https://i1.wp.com/wx1.sinaimg.cn/large/mine.jpg"
def test_kuaishou_falls_back_to_manifest_representations():
"""search 接口的 photo 字段更少，manifest 是最后一道兜底"""
item = {
"photo": {
"id": "1",
"manifest": {
"adaptationSet": [{"representation": [{"url": "https://ks/manifest.mp4"}]}]
},
}
}
items = kuaishou_media.build_media_items(item)
video = next(entry for entry in items if entry.media_type is MediaType.VIDEO)
assert video.url == "https://ks/manifest.mp4"
@pytest.mark.parametrize("scalar_video_resource", ["https://ks/raw.mp4", 123, None])
def test_kuaishou_tolerates_scalar_video_resource(scalar_video_resource):
"""GraphQL 里 videoResource 可能是 scalar 而非对象，不能因此抛异常"""
item = {"photo": {"id": "1", "videoResource": scalar_video_resource, "photoUrl": "https://ks/plain.mp4"}}
video_items = [
entry for entry in kuaishou_media.build_media_items(item)
if entry.media_type is MediaType.VIDEO
]
assert [entry.url for entry in video_items] == ["https://ks/plain.mp4"]
# --------------------------------------------------------------------------- bilibili fixture
BILI_VIEW = {
"aid": 114514,
"cid": 1919810,
"bvid": "BV1dwuKzmE26",
"pic": "https://i0.hdslb.com/bfs/archive/cover.jpg",
}
BILI_DASH_PLAY_INFO = {
"dash": {
"video": [
{"id": 32, "bandwidth": 300000, "base_url": "https://upos/32.m4s", "backup_url": ["https://upos-b/32.m4s"]},
{"id": 80, "bandwidth": 900000, "base_url": "https://upos/80.m4s"},
{"id": 116, "bandwidth": 2000000, "baseUrl": "https://upos/116.m4s"},
],
"audio": [
{"id": 30216, "bandwidth": 64000, "base_url": "https://upos/a64.m4s"},
{"id": 30280, "bandwidth": 192000, "base_url": "https://upos/a192.m4s", "backup_url": ["https://upos-b/a192.m4s"]},
],
}
}
BILI_DURL_PLAY_INFO = {
"durl": [
{"url": "https://upos/seg1.mp4", "size": 1000, "backup_url": ["https://upos-b/seg1.mp4"]},
{"url": "https://upos/seg2.mp4", "size": 5000},
]
}
# --------------------------------------------------------------------------- bilibili 用例
def test_bilibili_cover_item_from_view_pic():
item = bili_media.build_cover_item(BILI_VIEW, "BV1dwuKzmE26")
assert item is not None
assert item.url == "https://i0.hdslb.com/bfs/archive/cover.jpg"
assert item.stem == "cover"
assert item.content_id == "BV1dwuKzmE26"
def test_bilibili_cover_item_missing_pic():
assert bili_media.build_cover_item({"aid": 1}, "BV1") is None
assert bili_media.build_cover_item({}, "BV1") is None
def test_bilibili_pick_video_stream_respects_configured_quality(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 80)
stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO)
assert stream["id"] == 80
def test_bilibili_pick_video_stream_falls_back_to_lowest_when_all_exceed(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 16)
stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO)
assert stream["id"] == 32
def test_bilibili_pick_video_stream_picks_highest_when_all_below(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 127)
stream = bili_media.pick_video_stream(BILI_DASH_PLAY_INFO)
assert stream["id"] == 116
def test_bilibili_pick_audio_stream_takes_highest_bandwidth():
stream = bili_media.pick_audio_stream(BILI_DASH_PLAY_INFO)
assert stream["id"] == 30280
def test_bilibili_prefers_avc_over_hevc_at_same_quality(monkeypatch):
"""同一清晰度下应选兼容性最好的 AVC，而不是码率更高但兼容性差的 HEVC"""
monkeypatch.setattr(config, "BILI_QN", 80)
play_info = {
"dash": {
"video": [
{"id": 80, "codecid": 12, "bandwidth": 2000000, "base_url": "https://upos/hevc.m4s"},
{"id": 80, "codecid": 7, "bandwidth": 1500000, "base_url": "https://upos/avc.m4s"},
{"id": 80, "codecid": 13, "bandwidth": 1200000, "base_url": "https://upos/av1.m4s"},
],
"audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}],
}
}
assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/avc.m4s"
assert bili_media.build_dash_item(play_info, "BV1").url == "https://upos/avc.m4s"
def test_bilibili_falls_back_to_hevc_when_avc_unavailable(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 80)
play_info = {
"dash": {
"video": [
{"id": 80, "codecid": 12, "bandwidth": 2000000, "base_url": "https://upos/hevc.m4s"},
],
"audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}],
}
}
assert bili_media.pick_video_stream(play_info)["base_url"] == "https://upos/hevc.m4s"
def test_bilibili_build_dash_item(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 80)
item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1dwuKzmE26")
assert item is not None
assert item.is_dash is True
assert item.url == "https://upos/80.m4s"
assert item.audio_url == "https://upos/a192.m4s"
assert item.audio_backup_urls == ("https://upos-b/a192.m4s",)
assert item.media_type is MediaType.VIDEO
def test_bilibili_build_dash_item_supports_camel_case_base_url(monkeypatch):
monkeypatch.setattr(config, "BILI_QN", 127)
item = bili_media.build_dash_item(BILI_DASH_PLAY_INFO, "BV1")
assert item.url == "https://upos/116.m4s"
def test_bilibili_build_dash_item_requires_audio():
play_info = {"dash": {"video": [{"id": 80, "base_url": "https://upos/v.m4s"}]}}
assert bili_media.build_dash_item(play_info, "BV1") is None
def test_bilibili_build_durl_item_takes_largest_segment():
item = bili_media.build_durl_item(BILI_DURL_PLAY_INFO, "BV1")
assert item is not None
assert item.url == "https://upos/seg2.mp4"
assert item.is_dash is False
def test_bilibili_build_durl_item_keeps_backup_urls():
play_info = {"durl": [{"url": "https://upos/only.mp4", "size": 1, "backup_url": ["https://upos-b/only.mp4"]}]}
item = bili_media.build_durl_item(play_info, "BV1")
assert item.backup_urls == ("https://upos-b/only.mp4",)
@pytest.mark.parametrize("play_info", [{}, {"durl": []}, None])
def test_bilibili_build_durl_item_empty(play_info):
assert bili_media.build_durl_item(play_info or {}, "BV1") is None
def test_bilibili_counts_durl_segments():
assert bili_media.count_durl_segments(BILI_DURL_PLAY_INFO) == 2
assert bili_media.count_durl_segments(BILI_DASH_PLAY_INFO) == 0
assert bili_media.count_durl_segments(None) == 0
assert bili_media.count_durl_segments({"durl": ["oops", {"no_url": 1}]}) == 0
@pytest.mark.parametrize("payload", ["oops", 123, [], None])
def test_douyin_extractors_tolerate_malformed_payload(payload):
"""接口偶发返回非 dict 结构时不能抛异常，否则会中断整轮爬取"""
assert douyin_media.build_media_items(payload) == []
@pytest.mark.parametrize("payload", ["oops", 123, [], None])
def test_kuaishou_extractors_tolerate_malformed_payload(payload):
assert kuaishou_media.build_media_items(payload) == []
@pytest.mark.parametrize("payload", ["oops", 123, [], None])
def test_bilibili_extractors_tolerate_malformed_payload(payload):
assert bili_media.pick_video_stream(payload or {}) is None
assert bili_media.pick_audio_stream(payload or {}) is None
assert bili_media.build_dash_item(payload or {}, "BV1") is None
assert bili_media.build_durl_item(payload or {}, "BV1") is None
@pytest.mark.parametrize("payload", ["oops", 123, [], None])
def test_weibo_extractors_tolerate_malformed_payload(payload):
assert weibo_media.build_media_items(payload) == []
def test_xhs_extractors_tolerate_malformed_video_field():
"""实测：video 字段是字符串时旧逻辑会 AttributeError"""
assert xhs_media.build_media_items({"note_id": "n1", "type": "video", "video": "oops"}) == []
assert xhs_media.build_media_items({"note_id": "n1", "type": "video", "video": None}) == []
assert (
xhs_media.build_media_items(
{"note_id": "n1", "type": "video", "video": {"media": "oops", "cover": "oops"}}
)
== []
)
def test_bilibili_tolerates_non_dict_stream_entries():
play_info = {
"dash": {
"video": ["oops", {"id": 80, "base_url": "https://upos/ok.m4s"}],
"audio": [{"id": 30280, "base_url": "https://upos/a.m4s"}],
}
}
item = bili_media.build_dash_item(play_info, "BV1")
assert item is not None
assert item.url == "https://upos/ok.m4s"
tests/test_media_ffmpeg.py
+147
View File
@@ -0,0 +1,147 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_ffmpeg.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""ffmpeg 音视频合流测试。
本机没装 ffmpeg 时整体跳过（不影响其他测试）。素材用 ffmpeg 内置的 lavfi 源生成，
不依赖任何外部文件。
"""
from __future__ import annotations
import asyncio
import shutil
import subprocess
from pathlib import Path
import pytest
from media_downloader import merge_audio_video
from media_downloader.ffmpeg import ffmpeg_path, is_available
from media_downloader.types import MediaDownloadError
pytestmark = pytest.mark.skipif(not is_available(), reason="本机未安装 ffmpeg")
def _ffprobe_path() -> str:
probe = shutil.which("ffprobe")
if not probe:
pytest.skip("本机未安装 ffprobe")
return probe
def _make_video(path: Path, seconds: int = 1) -> None:
# 显式 -f mp4：.m4s 不是 ffmpeg 能自动识别的容器后缀，需模拟 DASH 分段的真实形态
subprocess.run(
[
ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", f"color=c=red:s=320x240:d={seconds}",
"-c:v", "mpeg4", "-pix_fmt", "yuv420p", "-f", "mp4", str(path),
],
check=True,
capture_output=True,
)
def _make_audio(path: Path, seconds: int = 1) -> None:
subprocess.run(
[
ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", f"sine=f=440:d={seconds}",
"-c:a", "aac", "-f", "mp4", str(path),
],
check=True,
capture_output=True,
)
def _stream_types(path: Path) -> list[str]:
result = subprocess.run(
[
_ffprobe_path(), "-v", "error",
"-show_entries", "stream=codec_type",
"-of", "csv=p=0", str(path),
],
check=True,
capture_output=True,
text=True,
)
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
@pytest.mark.asyncio
async def test_merge_produces_playable_file_with_both_streams(tmp_path):
video = tmp_path / "video.m4s"
audio = tmp_path / "audio.m4s"
merged = tmp_path / "merged.mp4"
_make_video(video)
_make_audio(audio)
await merge_audio_video(video, audio, merged)
assert merged.exists()
assert merged.stat().st_size > 0
assert _stream_types(merged) == ["video", "audio"]
@pytest.mark.asyncio
async def test_merge_rejects_corrupt_input(tmp_path):
video = tmp_path / "video.m4s"
bad_audio = tmp_path / "audio.m4s"
merged = tmp_path / "merged.mp4"
_make_video(video)
bad_audio.write_bytes(b"this is definitely not an audio stream")
with pytest.raises(MediaDownloadError):
await merge_audio_video(video, bad_audio, merged)
@pytest.mark.asyncio
async def test_merge_kills_subprocess_when_cancelled(tmp_path, monkeypatch):
"""任务被取消（Ctrl-C / abort）时不能留下孤儿 ffmpeg 进程"""
import media_downloader.ffmpeg as ffmpeg_module
# 必须用 exec：否则 sh 会另起一个 sleep 子进程，它继承着 stderr 管道，
# 即便 kill 掉 sh，管道清理仍会阻塞到 sleep 自然结束
fake_ffmpeg = tmp_path / "fake_ffmpeg.sh"
fake_ffmpeg.write_text("#!/bin/sh\nexec sleep 60\n")
fake_ffmpeg.chmod(0o755)
monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: str(fake_ffmpeg))
task = asyncio.create_task(
ffmpeg_module.merge_audio_video(
tmp_path / "v.m4s", tmp_path / "a.m4s", tmp_path / "o.mp4", timeout=60
)
)
await asyncio.sleep(0.4)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
await asyncio.sleep(0.4)
probe = subprocess.run(["pgrep", "-f", str(fake_ffmpeg)], capture_output=True, text=True)
assert probe.returncode != 0, f"取消后子进程仍在运行: {probe.stdout.strip()}"
@pytest.mark.asyncio
async def test_merge_reports_missing_ffmpeg(tmp_path, monkeypatch):
import media_downloader.ffmpeg as ffmpeg_module
monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: None)
video = tmp_path / "video.m4s"
audio = tmp_path / "audio.m4s"
video.write_bytes(b"x")
audio.write_bytes(b"y")
with pytest.raises(MediaDownloadError, match="ffmpeg"):
await merge_audio_video(video, audio, tmp_path / "out.mp4")
def test_is_available_matches_executable():
assert is_available() is bool(ffmpeg_path())