mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-01 01:08:11 +08:00
feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台
旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：
- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法
媒体下载失败只记录日志，不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
+2
-66
@@ -24,10 +24,10 @@
|
||||
from typing import List
|
||||
|
||||
import config
|
||||
from media_platform.xhs.media import extract_video_urls
|
||||
from var import source_keyword_var
|
||||
from tools.user_hash import anonymize_user_id, mask_nickname
|
||||
|
||||
from .xhs_store_media import *
|
||||
from ._store_impl import *
|
||||
|
||||
|
||||
@@ -51,40 +51,6 @@ class XhsStoreFactory:
|
||||
return store_class()
|
||||
|
||||
|
||||
def get_video_url_arr(note_item: Dict) -> List:
|
||||
"""
|
||||
Get video url array
|
||||
Args:
|
||||
note_item:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
if note_item.get('type') != 'video':
|
||||
return []
|
||||
|
||||
video_dict = note_item.get('video')
|
||||
if not video_dict:
|
||||
return []
|
||||
|
||||
videoArr = []
|
||||
consumer = video_dict.get('consumer', {})
|
||||
originVideoKey = consumer.get('origin_video_key', '')
|
||||
if originVideoKey == '':
|
||||
originVideoKey = consumer.get('originVideoKey', '')
|
||||
# Fallback with watermark
|
||||
if originVideoKey == '':
|
||||
media = video_dict.get('media', {})
|
||||
stream = media.get('stream', {})
|
||||
videos = stream.get('h264')
|
||||
if type(videos).__name__ == 'list':
|
||||
videoArr = [v.get('master_url') for v in videos]
|
||||
else:
|
||||
videoArr = [f"http://sns-video-bd.xhscdn.com/{originVideoKey}"]
|
||||
|
||||
return videoArr
|
||||
|
||||
|
||||
async def update_xhs_note(note_item: Dict):
|
||||
"""
|
||||
Update Xiaohongshu note
|
||||
@@ -104,7 +70,7 @@ async def update_xhs_note(note_item: Dict):
|
||||
if img.get('url_default') != '':
|
||||
img.update({'url': img.get('url_default')})
|
||||
|
||||
video_url = ','.join(get_video_url_arr(note_item))
|
||||
video_url = ','.join(extract_video_urls(note_item))
|
||||
|
||||
local_db_item = {
|
||||
"note_id": note_item.get("note_id"), # Note ID
|
||||
@@ -190,33 +156,3 @@ async def save_creator(user_id: str, creator: Dict):
|
||||
"""
|
||||
# 教学版：创作者个人资料(昵称/性别/头像/IP/粉丝数等)不再落库，防骚扰。
|
||||
return
|
||||
|
||||
|
||||
async def update_xhs_note_image(note_id, pic_content, extension_file_name):
|
||||
"""
|
||||
Update Xiaohongshu note image
|
||||
Args:
|
||||
note_id:
|
||||
pic_content:
|
||||
extension_file_name:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
|
||||
await XiaoHongShuImage().store_image({"notice_id": note_id, "pic_content": pic_content, "extension_file_name": extension_file_name})
|
||||
|
||||
|
||||
async def update_xhs_note_video(note_id, video_content, extension_file_name):
|
||||
"""
|
||||
Update Xiaohongshu note video
|
||||
Args:
|
||||
note_id:
|
||||
video_content:
|
||||
extension_file_name:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
|
||||
await XiaoHongShuVideo().store_video({"notice_id": note_id, "video_content": video_content, "extension_file_name": extension_file_name})
|
||||
|
||||
@@ -1,133 +0,0 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 relakkes@gmail.com
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/store/xhs/xhs_store_media.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
|
||||
# 声明：本代码仅供学习和研究目的使用。使用者应遵守以下原则：
|
||||
# 1. 不得用于任何商业用途。
|
||||
# 2. 使用时应遵守目标平台的使用条款和robots.txt规则。
|
||||
# 3. 不得进行大规模爬取或对平台造成运营干扰。
|
||||
# 4. 应合理控制请求频率，避免给目标平台带来不必要的负担。
|
||||
# 5. 不得用于任何非法或不当的用途。
|
||||
#
|
||||
# 详细许可条款请参阅项目根目录下的LICENSE文件。
|
||||
# 使用本代码即表示您同意遵守上述原则和LICENSE中的所有条款。
|
||||
|
||||
# -*- coding: utf-8 -*-
|
||||
# @Author : helloteemo
|
||||
# @Time : 2024/7/11 22:35
|
||||
# @Desc : Xiaohongshu media storage
|
||||
import pathlib
|
||||
from typing import Dict
|
||||
|
||||
import aiofiles
|
||||
|
||||
from base.base_crawler import AbstractStoreImage, AbstractStoreVideo
|
||||
from tools import utils
|
||||
import config
|
||||
|
||||
|
||||
class XiaoHongShuImage(AbstractStoreImage):
|
||||
def __init__(self):
|
||||
if config.SAVE_DATA_PATH:
|
||||
self.image_store_path = f"{config.SAVE_DATA_PATH}/xhs/images"
|
||||
else:
|
||||
self.image_store_path = "data/xhs/images"
|
||||
|
||||
async def store_image(self, image_content_item: Dict):
|
||||
"""
|
||||
store content
|
||||
|
||||
Args:
|
||||
image_content_item:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
await self.save_image(image_content_item.get("notice_id"), image_content_item.get("pic_content"), image_content_item.get("extension_file_name"))
|
||||
|
||||
def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str:
|
||||
"""
|
||||
make save file name by store type
|
||||
|
||||
Args:
|
||||
notice_id: notice id
|
||||
extension_file_name: image filename with extension
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
return f"{self.image_store_path}/{notice_id}/{extension_file_name}"
|
||||
|
||||
async def save_image(self, notice_id: str, pic_content: str, extension_file_name):
|
||||
"""
|
||||
save image to local
|
||||
|
||||
Args:
|
||||
notice_id: notice id
|
||||
pic_content: image content
|
||||
extension_file_name: image filename with extension
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
pathlib.Path(self.image_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True)
|
||||
save_file_name = self.make_save_file_name(notice_id, extension_file_name)
|
||||
async with aiofiles.open(save_file_name, 'wb') as f:
|
||||
await f.write(pic_content)
|
||||
utils.logger.info(f"[XiaoHongShuImageStoreImplement.save_image] save image {save_file_name} success ...")
|
||||
|
||||
|
||||
class XiaoHongShuVideo(AbstractStoreVideo):
|
||||
def __init__(self):
|
||||
if config.SAVE_DATA_PATH:
|
||||
self.video_store_path = f"{config.SAVE_DATA_PATH}/xhs/videos"
|
||||
else:
|
||||
self.video_store_path = "data/xhs/videos"
|
||||
|
||||
async def store_video(self, video_content_item: Dict):
|
||||
"""
|
||||
store content
|
||||
|
||||
Args:
|
||||
video_content_item:
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
await self.save_video(video_content_item.get("notice_id"), video_content_item.get("video_content"), video_content_item.get("extension_file_name"))
|
||||
|
||||
def make_save_file_name(self, notice_id: str, extension_file_name: str) -> str:
|
||||
"""
|
||||
make save file name by store type
|
||||
|
||||
Args:
|
||||
notice_id: notice id
|
||||
extension_file_name: video filename with extension
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
return f"{self.video_store_path}/{notice_id}/{extension_file_name}"
|
||||
|
||||
async def save_video(self, notice_id: str, video_content: str, extension_file_name):
|
||||
"""
|
||||
save video to local
|
||||
|
||||
Args:
|
||||
notice_id: notice id
|
||||
video_content: video content
|
||||
extension_file_name: video filename with extension
|
||||
|
||||
Returns:
|
||||
|
||||
"""
|
||||
pathlib.Path(self.video_store_path + "/" + notice_id).mkdir(parents=True, exist_ok=True)
|
||||
save_file_name = self.make_save_file_name(notice_id, extension_file_name)
|
||||
async with aiofiles.open(save_file_name, 'wb') as f:
|
||||
await f.write(video_content)
|
||||
utils.logger.info(f"[XiaoHongShuVideoStoreImplement.save_video] save video {save_file_name} success ...")
|
||||
Reference in New Issue
Block a user