mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-02 12:38:12 +08:00
feat(media): 重构媒体下载,支持 xhs/dy/ks/bili/wb 五平台
旧实现只覆盖 4 个平台,且把整个文件读进内存、无重试与完整性校验,
代码按平台复制粘贴了 4 份。本次用统一下载器替换:
- 新增 media_downloader/:流式写入、Range 续传、指数退避重试、大小校验、
路径穿越防护;B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py:从平台原始响应提取媒体地址,
与下载器解耦;快手首次接入下载能力
- 开关:config.ENABLE_GET_MEDIA 与 --get_media,并打通 API/WebUI;
同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合:{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质,否则降级 mp4 直链(产物 video-durl.mp4,
避免低清文件阻塞后续的高清路径)
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法
媒体下载失败只记录日志,不中断爬取主流程。
This commit is contained in:
@@ -42,6 +42,7 @@ if TYPE_CHECKING:
|
||||
from .exception import DataFetchError
|
||||
from .field import CommentOrderType, SearchOrderType
|
||||
from .help import BilibiliSign
|
||||
from .media import DASH_FNVAL
|
||||
|
||||
|
||||
def _extract_pinned_comments(value: Any) -> List[Dict]:
|
||||
@@ -213,11 +214,13 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
params.update({"bvid": bvid})
|
||||
return await self.get(uri, params, enable_params_sign=False)
|
||||
|
||||
async def get_video_play_url(self, aid: int, cid: int) -> Dict:
|
||||
async def get_video_play_url(self, aid: int, cid: int, fnval: int = DASH_FNVAL) -> Dict:
|
||||
"""
|
||||
Bilibli web video play url api
|
||||
:param aid: Video aid
|
||||
:param cid: cid
|
||||
:param fnval: 请求的媒体格式位掩码,默认请求 DASH(含 4K/8K/HDR/AV1);
|
||||
传 1 则请求音视频合一的 mp4 直链
|
||||
:return:
|
||||
"""
|
||||
if not aid or not cid or aid <= 0 or cid <= 0:
|
||||
@@ -229,28 +232,12 @@ class BilibiliClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
"cid": cid,
|
||||
"qn": qn_value,
|
||||
"fourk": 1,
|
||||
"fnval": 1,
|
||||
"fnval": fnval,
|
||||
"platform": "pc",
|
||||
}
|
||||
|
||||
return await self.get(uri, params, enable_params_sign=True)
|
||||
|
||||
async def get_video_media(self, url: str) -> Union[bytes, None]:
|
||||
# Follow CDN 302 redirects and treat any 2xx as success (some endpoints return 206)
|
||||
async with make_async_client(proxy=self.proxy, follow_redirects=True) as client:
|
||||
try:
|
||||
response = await client.request("GET", url, timeout=self.timeout, headers=self.headers)
|
||||
response.raise_for_status()
|
||||
if 200 <= response.status_code < 300:
|
||||
return response.content
|
||||
utils.logger.error(
|
||||
f"[BilibiliClient.get_video_media] Unexpected status {response.status_code} for {url}"
|
||||
)
|
||||
return None
|
||||
except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
|
||||
utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # Keep original exception type name for developer debugging
|
||||
return None
|
||||
|
||||
async def get_video_comments(
|
||||
self,
|
||||
video_id: str,
|
||||
|
||||
Reference in New Issue
Block a user