mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-09-27 07:28:12 +08:00
feat(media): 重构媒体下载，支持 xhs/dy/ks/bili/wb 五平台
旧实现只覆盖 4 个平台，且把整个文件读进内存、无重试与完整性校验，
代码按平台复制粘贴了 4 份。本次用统一下载器替换：
- 新增 media_downloader/：流式写入、Range 续传、指数退避重试、大小校验、
路径穿越防护；B 站 DASH 音视频分轨下载后交由 ffmpeg 无损合流
- 新增 media_platform/<平台>/media.py：从平台原始响应提取媒体地址，
与下载器解耦；快手首次接入下载能力
- 开关：config.ENABLE_GET_MEDIA 与 --get_media，并打通 API/WebUI；
同时修正旧配置项 ENABLE_GET_MEIDAS 的拼写
- 落盘按帖子聚合：{SAVE_DATA_PATH 或 data}/{platform}/media/{内容ID}/
- B 站装好 ffmpeg 时走 DASH 最高画质，否则降级 mp4 直链（产物 video-durl.mp4，
避免低清文件阻塞后续的高清路径）
- 删除 4 个 *_store_media.py、AbstractStoreImage/Video 及各 client 的媒体 GET 方法
媒体下载失败只记录日志，不中断爬取主流程。
This commit is contained in:
程序员阿江(Relakkes)
@@ -0,0 +1,386 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
# Copyright (c) 2025 relakkes@gmail.com
|
||||
#
|
||||
# This file is part of MediaCrawler project.
|
||||
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/media_server.py
|
||||
# GitHub: https://github.com/NanmiCoder
|
||||
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||
#
|
||||
|
||||
"""媒体下载测试用的本地 HTTP 服务器。
|
||||
|
||||
基于标准库 ``ThreadingHTTPServer``，不引入额外依赖；覆盖下载器需要面对的各种
|
||||
CDN 行为：Range 续传、忽略 Range、chunked、响应截断、5xx 抖动、302 跳转、
|
||||
无扩展名但带 Content-Type、慢响应、空响应体等。
|
||||
|
||||
所有请求都会记录到 ``server.requests``，用于断言"跳过已下载时不发请求"
|
||||
"403 只请求一次"这类行为。
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.parse import parse_qs, urlsplit
|
||||
|
||||
# 可识别的字节模式，便于断言续传后拼接结果正确
|
||||
DEFAULT_CONTENT = bytes(range(256)) * 16 # 4096 字节
|
||||
DEFAULT_CONTENT_TYPE = "image/jpeg"
|
||||
|
||||
|
||||
@dataclass
|
||||
class RequestRecord:
|
||||
"""一次进入服务器的请求"""
|
||||
|
||||
method: str
|
||||
path: str
|
||||
headers: Dict[str, str] = field(default_factory=dict)
|
||||
|
||||
|
||||
class _MediaRequestHandler(BaseHTTPRequestHandler):
|
||||
protocol_version = "HTTP/1.1"
|
||||
server_version = "MediaTestServer/1.0"
|
||||
|
||||
# 静音默认的 stderr 访问日志
|
||||
def log_message(self, format: str, *args) -> None: # noqa: A002
|
||||
return
|
||||
|
||||
# ------------------------------------------------------------------ 工具方法
|
||||
|
||||
@property
|
||||
def media_server(self) -> "MediaTestServer":
|
||||
return self.server.media_server # type: ignore[attr-defined]
|
||||
|
||||
def _record(self) -> None:
|
||||
self.media_server.requests.append(
|
||||
RequestRecord(
|
||||
method=self.command,
|
||||
path=self.path,
|
||||
headers={key.lower(): value for key, value in self.headers.items()},
|
||||
)
|
||||
)
|
||||
|
||||
def _content_for(self, name: str) -> bytes:
|
||||
custom = self.media_server.contents.get(name)
|
||||
return custom if custom is not None else DEFAULT_CONTENT
|
||||
|
||||
def _send_body(self, status: int, body: bytes, content_type: str, extra_headers=None) -> None:
|
||||
self.send_response(status)
|
||||
self.send_header("Content-Type", content_type)
|
||||
self.send_header("Content-Length", str(len(body)))
|
||||
for key, value in (extra_headers or {}).items():
|
||||
self.send_header(key, value)
|
||||
self.end_headers()
|
||||
if body:
|
||||
self.wfile.write(body)
|
||||
|
||||
def _parse_range(self) -> Optional[int]:
|
||||
"""解析 ``Range: bytes=N-``，返回起始偏移；不合法或不存在返回 None"""
|
||||
raw = self.headers.get("range", "")
|
||||
if not raw.startswith("bytes="):
|
||||
return None
|
||||
spec = raw[len("bytes=") :].split(",")[0].strip()
|
||||
if not spec.endswith("-"):
|
||||
return None
|
||||
start = spec[:-1].strip()
|
||||
return int(start) if start.isdigit() else None
|
||||
|
||||
# ------------------------------------------------------------------ 路由
|
||||
|
||||
def do_GET(self) -> None: # noqa: N802 - BaseHTTPRequestHandler 约定
|
||||
self._record()
|
||||
parsed = urlsplit(self.path)
|
||||
path = parsed.path
|
||||
query = parse_qs(parsed.query)
|
||||
|
||||
try:
|
||||
if path.startswith("/status/"):
|
||||
code = int(path.rsplit("/", 1)[-1])
|
||||
self._send_body(code, b"", "text/plain")
|
||||
elif path.startswith("/ok/") or path.startswith("/redirected/"):
|
||||
self._handle_ok(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/no-range/"):
|
||||
self._handle_no_range(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/wrong-range/"):
|
||||
self._handle_wrong_range(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/poison-206/"):
|
||||
self._handle_poison_206(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/gzip/"):
|
||||
self._handle_gzip(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/unknown-total/"):
|
||||
self._handle_unknown_total(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/no-content-range/"):
|
||||
self._handle_no_content_range(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/chunked-bogus-cl/"):
|
||||
self._handle_chunked_with_bogus_content_length(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/chunked/"):
|
||||
self._handle_chunked(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/truncated/"):
|
||||
self._handle_truncated(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/flaky/"):
|
||||
self._handle_flaky(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/redirect/"):
|
||||
self._handle_redirect(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/ctyped/"):
|
||||
self._handle_ctyped(path.rsplit("/", 1)[-1])
|
||||
elif path.startswith("/slow/"):
|
||||
self._handle_slow(path.rsplit("/", 1)[-1], query)
|
||||
elif path.startswith("/empty/"):
|
||||
self._send_body(200, b"", DEFAULT_CONTENT_TYPE)
|
||||
else:
|
||||
self._send_body(404, b"not found", "text/plain")
|
||||
except (BrokenPipeError, ConnectionResetError): # 客户端主动断开
|
||||
self.close_connection = True
|
||||
|
||||
# ------------------------------------------------------------------ 各路由实现
|
||||
|
||||
def _handle_ok(self, name: str, query) -> None:
|
||||
"""正常文件，支持 Range 续传"""
|
||||
content = self._content_for(name)
|
||||
total = len(content)
|
||||
start = self._parse_range()
|
||||
|
||||
if start is not None:
|
||||
if start >= total:
|
||||
self.send_response(416)
|
||||
self.send_header("Content-Range", f"bytes */{total}")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
return
|
||||
body = content[start:]
|
||||
content_type = query.get("ct", [DEFAULT_CONTENT_TYPE])[0]
|
||||
self._send_body(
|
||||
206,
|
||||
body,
|
||||
content_type,
|
||||
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
|
||||
)
|
||||
return
|
||||
|
||||
self._send_body(200, content, query.get("ct", [DEFAULT_CONTENT_TYPE])[0])
|
||||
|
||||
def _handle_wrong_range(self, name: str, query) -> None:
|
||||
"""返回 206 但 Content-Range 起点与请求不一致（模拟按关键帧对齐的加速节点）。
|
||||
|
||||
``?fail=N``：前 N 次返回错位响应，之后恢复正常，
|
||||
用于验证"下载器丢弃片段重下后能成功"。
|
||||
"""
|
||||
fail_times = int(query.get("fail", ["1000"])[0])
|
||||
counter_key = f"/wrong-range/{name}"
|
||||
with self.media_server.lock:
|
||||
seen = self.media_server.counters.get(counter_key, 0)
|
||||
self.media_server.counters[counter_key] = seen + 1
|
||||
|
||||
if seen >= fail_times:
|
||||
self._handle_ok(name, query)
|
||||
return
|
||||
|
||||
content = self._content_for(name)
|
||||
total = len(content)
|
||||
requested = self._parse_range() or 0
|
||||
delta = int(query.get("delta", ["100"])[0])
|
||||
start = min(requested + delta, total - 1)
|
||||
self._send_body(
|
||||
206,
|
||||
content[start:],
|
||||
DEFAULT_CONTENT_TYPE,
|
||||
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
|
||||
)
|
||||
|
||||
def _handle_poison_206(self, name: str, query) -> None:
|
||||
"""206 声明非 0 起点，但 body 长度**恰好等于 total**（内容是错位的）。
|
||||
|
||||
这是唯一能单独验证"起点守卫"的响应：长度自洽，大小校验拦不住，
|
||||
只有比对 Content-Range 起点才能发现异常。
|
||||
"""
|
||||
content = self._content_for(name)
|
||||
total = len(content)
|
||||
start = int(query.get("start", ["100"])[0])
|
||||
body = content[start:] + b"\x00" * start
|
||||
self._send_body(
|
||||
206,
|
||||
body,
|
||||
DEFAULT_CONTENT_TYPE,
|
||||
{"Content-Range": f"bytes {start}-{total - 1}/{total}"},
|
||||
)
|
||||
|
||||
def _handle_gzip(self, name: str, query) -> None:
|
||||
"""无视 Accept-Encoding: identity，强制返回 gzip 压缩体"""
|
||||
import gzip
|
||||
|
||||
content = self._content_for(name)
|
||||
compressed = gzip.compress(content)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
|
||||
self.send_header("Content-Encoding", "gzip")
|
||||
self.send_header("Content-Length", str(len(compressed)))
|
||||
self.end_headers()
|
||||
self.wfile.write(compressed)
|
||||
|
||||
def _handle_unknown_total(self, name: str) -> None:
|
||||
"""206 只返回区间的一部分，且 Content-Range 的 total 为 *（不告知总长）"""
|
||||
content = self._content_for(name)
|
||||
start = self._parse_range() or 0
|
||||
chunk = content[start : start + 1024]
|
||||
self._send_body(
|
||||
206,
|
||||
chunk,
|
||||
DEFAULT_CONTENT_TYPE,
|
||||
{"Content-Range": f"bytes {start}-{start + len(chunk) - 1}/*"},
|
||||
)
|
||||
|
||||
def _handle_no_content_range(self, name: str) -> None:
|
||||
"""206 但缺少 Content-Range（违反 RFC 9110），且回的是全量体"""
|
||||
content = self._content_for(name)
|
||||
self._send_body(206, content, DEFAULT_CONTENT_TYPE)
|
||||
|
||||
def _handle_chunked_with_bogus_content_length(self, name: str) -> None:
|
||||
"""chunked 传输同时带一个错误的 Content-Length（RFC 7230 要求忽略后者）"""
|
||||
content = self._content_for(name)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self.send_header("Content-Length", "999")
|
||||
self.end_headers()
|
||||
self.wfile.write(f"{len(content):X}\r\n".encode("ascii"))
|
||||
self.wfile.write(content)
|
||||
self.wfile.write(b"\r\n0\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
|
||||
def _handle_no_range(self, name: str) -> None:
|
||||
"""声明支持 Range 但始终返回全量 200（模拟不支持续传的 CDN）"""
|
||||
content = self._content_for(name)
|
||||
self._send_body(200, content, DEFAULT_CONTENT_TYPE, {"Accept-Ranges": "bytes"})
|
||||
|
||||
def _handle_chunked(self, name: str) -> None:
|
||||
"""chunked 传输：无 Content-Length"""
|
||||
content = self._content_for(name)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
|
||||
self.send_header("Transfer-Encoding", "chunked")
|
||||
self.end_headers()
|
||||
for offset in range(0, len(content), 1024):
|
||||
chunk = content[offset : offset + 1024]
|
||||
self.wfile.write(f"{len(chunk):X}\r\n".encode("ascii"))
|
||||
self.wfile.write(chunk)
|
||||
self.wfile.write(b"\r\n")
|
||||
self.wfile.write(b"0\r\n\r\n")
|
||||
self.wfile.flush()
|
||||
|
||||
def _handle_truncated(self, name: str, query) -> None:
|
||||
"""声明完整长度但只发送一半后断开连接。
|
||||
|
||||
``?fail=N``：前 N 次截断，之后恢复正常（支持 Range），
|
||||
用于验证"断流 -> 重试 -> 带 Range 续传 -> 拼接完整"。
|
||||
"""
|
||||
fail_times = int(query.get("fail", ["1000"])[0])
|
||||
counter_key = f"/truncated/{name}"
|
||||
with self.media_server.lock:
|
||||
seen = self.media_server.counters.get(counter_key, 0)
|
||||
self.media_server.counters[counter_key] = seen + 1
|
||||
|
||||
if seen >= fail_times:
|
||||
self._handle_ok(name, query)
|
||||
return
|
||||
|
||||
content = self._content_for(name)
|
||||
self.send_response(200)
|
||||
self.send_header("Content-Type", DEFAULT_CONTENT_TYPE)
|
||||
self.send_header("Content-Length", str(len(content)))
|
||||
self.end_headers()
|
||||
self.wfile.write(content[: len(content) // 2])
|
||||
self.wfile.flush()
|
||||
self.close_connection = True
|
||||
try:
|
||||
self.connection.shutdown(2) # SHUT_RDWR
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
def _handle_flaky(self, name: str, query) -> None:
|
||||
"""前 N 次请求返回 500，之后正常（服务端计数）"""
|
||||
fail_times = int(query.get("fail", ["1"])[0])
|
||||
counter_key = f"/flaky/{name}"
|
||||
with self.media_server.lock:
|
||||
seen = self.media_server.counters.get(counter_key, 0)
|
||||
self.media_server.counters[counter_key] = seen + 1
|
||||
if seen < fail_times:
|
||||
self._send_body(500, b"server error", "text/plain")
|
||||
return
|
||||
content = self._content_for(name)
|
||||
self._send_body(200, content, DEFAULT_CONTENT_TYPE)
|
||||
|
||||
def _handle_redirect(self, name: str) -> None:
|
||||
self.send_response(302)
|
||||
self.send_header("Location", f"/ok/{name}")
|
||||
self.send_header("Content-Length", "0")
|
||||
self.end_headers()
|
||||
|
||||
def _handle_ctyped(self, name: str) -> None:
|
||||
"""URL 无扩展名，靠 Content-Type 推断"""
|
||||
content_type = self.media_server.content_types.get(name, "image/webp")
|
||||
self._send_body(200, self._content_for(name), content_type)
|
||||
|
||||
def _handle_slow(self, name: str, query) -> None:
|
||||
delay = float(query.get("s", ["1"])[0])
|
||||
time.sleep(delay)
|
||||
self._send_body(200, self._content_for(name), DEFAULT_CONTENT_TYPE)
|
||||
|
||||
|
||||
class MediaTestServer(ThreadingHTTPServer):
|
||||
"""带请求记录能力的本地测试服务器"""
|
||||
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
def __init__(self) -> None:
|
||||
super().__init__(("127.0.0.1", 0), _MediaRequestHandler)
|
||||
# handler 通过 server.media_server 反向访问这些状态
|
||||
self.media_server: "MediaTestServer" = self
|
||||
self.requests: List[RequestRecord] = []
|
||||
self.counters: Dict[str, int] = {}
|
||||
self.contents: Dict[str, bytes] = {}
|
||||
self.content_types: Dict[str, str] = {}
|
||||
self.lock = threading.Lock()
|
||||
|
||||
# ------------------------------------------------------------------ 测试辅助
|
||||
|
||||
@property
|
||||
def base_url(self) -> str:
|
||||
host, port = self.server_address[:2]
|
||||
return f"http://{host}:{port}"
|
||||
|
||||
def url(self, path: str) -> str:
|
||||
return f"{self.base_url}{path}"
|
||||
|
||||
def reset(self) -> None:
|
||||
"""清空请求记录与计数器（内容配置保留）"""
|
||||
with self.lock:
|
||||
self.requests.clear()
|
||||
self.counters.clear()
|
||||
|
||||
def set_content(self, name: str, content: bytes) -> None:
|
||||
self.contents[name] = content
|
||||
|
||||
def set_content_type(self, name: str, content_type: str) -> None:
|
||||
self.content_types[name] = content_type
|
||||
|
||||
@property
|
||||
def request_count(self) -> int:
|
||||
return len(self.requests)
|
||||
|
||||
def paths(self) -> List[str]:
|
||||
return [record.path for record in self.requests]
|
||||
|
||||
def last_request(self) -> Optional[RequestRecord]:
|
||||
return self.requests[-1] if self.requests else None
|
||||
|
||||
|
||||
def start_server() -> Tuple[MediaTestServer, threading.Thread]:
|
||||
"""在后台线程启动服务器，返回 (server, thread)"""
|
||||
server = MediaTestServer()
|
||||
thread = threading.Thread(target=server.serve_forever, daemon=True)
|
||||
thread.start()
|
||||
return server, thread
|
||||
Reference in New Issue
Block a user