mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-20 11:27:57 +08:00
feat(media): 重构媒体下载,支持 xhs/dy/ks/bili/wb 五平台
旧实现只覆盖 4 个平台,且把整个文件读进内存、无重试与完整性校验,
代码按平台复制粘贴了 4 份。本次用统一下载器替换:
- 新增 media_downloader/:流式写入、Range 续传、指数退避重试、大小校验、
路径穿越防护;B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py:从平台原始响应提取媒体地址,
与下载器解耦;快手首次接入下载能力
- 开关:config.ENABLE_GET_MEDIA 与 --get_media,并打通 API/WebUI;
同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合:{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质,否则降级 mp4 直链(产物 video-durl.mp4,
避免低清文件阻塞后续的高清路径)
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法
媒体下载失败只记录日志,不中断爬取主流程。
This commit is contained in:
@@ -62,6 +62,35 @@ def test_crawler_manager_build_command():
|
||||
idx_comments = cmd2.index("--max_comments_count_singlenotes")
|
||||
assert cmd2[idx_comments + 1] == "5"
|
||||
|
||||
|
||||
def test_crawler_manager_passes_media_switch():
|
||||
cm = CrawlerManager()
|
||||
|
||||
req_off = CrawlerStartRequest(
|
||||
platform=PlatformEnum.XHS,
|
||||
login_type=LoginTypeEnum.QRCODE,
|
||||
crawler_type=CrawlerTypeEnum.DETAIL,
|
||||
specified_ids="note-1",
|
||||
)
|
||||
cmd_off = cm._build_command(req_off)
|
||||
idx_off = cmd_off.index("--get_media")
|
||||
assert cmd_off[idx_off + 1] == "false"
|
||||
|
||||
req_on = CrawlerStartRequest(
|
||||
platform=PlatformEnum.XHS,
|
||||
login_type=LoginTypeEnum.QRCODE,
|
||||
crawler_type=CrawlerTypeEnum.DETAIL,
|
||||
specified_ids="note-1",
|
||||
enable_media=True,
|
||||
)
|
||||
cmd_on = cm._build_command(req_on)
|
||||
idx_on = cmd_on.index("--get_media")
|
||||
assert cmd_on[idx_on + 1] == "true"
|
||||
|
||||
|
||||
def test_api_schema_exposes_media_switch_default_off():
|
||||
assert CrawlerStartRequest(platform=PlatformEnum.XHS).enable_media is False
|
||||
|
||||
def test_api_start_crawler_with_limits():
|
||||
client = TestClient(app)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user