feat(media): 重构媒体下载,支持 xhs/dy/ks/bili/wb 五平台

旧实现只覆盖 4 个平台,且把整个文件读进内存、无重试与完整性校验,
代码按平台复制粘贴了 4 份。本次用统一下载器替换:

- 新增 media_downloader/:流式写入、Range 续传、指数退避重试、大小校验、
  路径穿越防护;B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py:从平台原始响应提取媒体地址,
  与下载器解耦;快手首次接入下载能力
- 开关:config.ENABLE_GET_MEDIA 与 --get_media,并打通 API/WebUI;
  同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合:{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质,否则降级 mp4 直链(产物 video-durl.mp4,
  避免低清文件阻塞后续的高清路径)
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法

媒体下载失败只记录日志,不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
2026-09-17 22:54:22 +08:00
parent 60e66f2a92
commit 0ca7b29cf0
47 changed files with 5187 additions and 907 deletions
+147
View File
@@ -0,0 +1,147 @@
# -*- coding: utf-8 -*-
# Copyright (c) 2025 relakkes@gmail.com
#
# This file is part of MediaCrawler project.
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_media_ffmpeg.py
# GitHub: https://github.com/NanmiCoder
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
#
"""ffmpeg 音视频合流测试。
本机没装 ffmpeg 时整体跳过(不影响其他测试)。素材用 ffmpeg 内置的 lavfi 源生成,
不依赖任何外部文件。
"""
from __future__ import annotations
import asyncio
import shutil
import subprocess
from pathlib import Path
import pytest
from media_downloader import merge_audio_video
from media_downloader.ffmpeg import ffmpeg_path, is_available
from media_downloader.types import MediaDownloadError
pytestmark = pytest.mark.skipif(not is_available(), reason="本机未安装 ffmpeg")
def _ffprobe_path() -> str:
probe = shutil.which("ffprobe")
if not probe:
pytest.skip("本机未安装 ffprobe")
return probe
def _make_video(path: Path, seconds: int = 1) -> None:
# 显式 -f mp4:.m4s 不是 ffmpeg 能自动识别的容器后缀,需模拟 DASH 分段的真实形态
subprocess.run(
[
ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", f"color=c=red:s=320x240:d={seconds}",
"-c:v", "mpeg4", "-pix_fmt", "yuv420p", "-f", "mp4", str(path),
],
check=True,
capture_output=True,
)
def _make_audio(path: Path, seconds: int = 1) -> None:
subprocess.run(
[
ffmpeg_path(), "-hide_banner", "-loglevel", "error", "-y",
"-f", "lavfi", "-i", f"sine=f=440:d={seconds}",
"-c:a", "aac", "-f", "mp4", str(path),
],
check=True,
capture_output=True,
)
def _stream_types(path: Path) -> list[str]:
result = subprocess.run(
[
_ffprobe_path(), "-v", "error",
"-show_entries", "stream=codec_type",
"-of", "csv=p=0", str(path),
],
check=True,
capture_output=True,
text=True,
)
return [line.strip() for line in result.stdout.splitlines() if line.strip()]
@pytest.mark.asyncio
async def test_merge_produces_playable_file_with_both_streams(tmp_path):
video = tmp_path / "video.m4s"
audio = tmp_path / "audio.m4s"
merged = tmp_path / "merged.mp4"
_make_video(video)
_make_audio(audio)
await merge_audio_video(video, audio, merged)
assert merged.exists()
assert merged.stat().st_size > 0
assert _stream_types(merged) == ["video", "audio"]
@pytest.mark.asyncio
async def test_merge_rejects_corrupt_input(tmp_path):
video = tmp_path / "video.m4s"
bad_audio = tmp_path / "audio.m4s"
merged = tmp_path / "merged.mp4"
_make_video(video)
bad_audio.write_bytes(b"this is definitely not an audio stream")
with pytest.raises(MediaDownloadError):
await merge_audio_video(video, bad_audio, merged)
@pytest.mark.asyncio
async def test_merge_kills_subprocess_when_cancelled(tmp_path, monkeypatch):
"""任务被取消(Ctrl-C / abort)时不能留下孤儿 ffmpeg 进程"""
import media_downloader.ffmpeg as ffmpeg_module
# 必须用 exec:否则 sh 会另起一个 sleep 子进程,它继承着 stderr 管道,
# 即便 kill 掉 sh,管道清理仍会阻塞到 sleep 自然结束
fake_ffmpeg = tmp_path / "fake_ffmpeg.sh"
fake_ffmpeg.write_text("#!/bin/sh\nexec sleep 60\n")
fake_ffmpeg.chmod(0o755)
monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: str(fake_ffmpeg))
task = asyncio.create_task(
ffmpeg_module.merge_audio_video(
tmp_path / "v.m4s", tmp_path / "a.m4s", tmp_path / "o.mp4", timeout=60
)
)
await asyncio.sleep(0.4)
task.cancel()
with pytest.raises(asyncio.CancelledError):
await task
await asyncio.sleep(0.4)
probe = subprocess.run(["pgrep", "-f", str(fake_ffmpeg)], capture_output=True, text=True)
assert probe.returncode != 0, f"取消后子进程仍在运行: {probe.stdout.strip()}"
@pytest.mark.asyncio
async def test_merge_reports_missing_ffmpeg(tmp_path, monkeypatch):
import media_downloader.ffmpeg as ffmpeg_module
monkeypatch.setattr(ffmpeg_module, "ffmpeg_path", lambda: None)
video = tmp_path / "video.m4s"
audio = tmp_path / "audio.m4s"
video.write_bytes(b"x")
audio.write_bytes(b"y")
with pytest.raises(MediaDownloadError, match="ffmpeg"):
await merge_audio_video(video, audio, tmp_path / "out.mp4")
def test_is_available_matches_executable():
assert is_available() is bool(ffmpeg_path())