mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-04 05:48:11 +08:00
fix(ks): 视频不可用时 photo 为 null 导致整轮爬取崩溃
快手对已删除/私密/不存在的视频,返回的 visionVideoDetail 里 photo/author 是
null——key 存在、值是 null。而 detail.get("photo", {}) 只在 key 缺失时给默认值,
key 存在且为 null 时拿到的仍是 None,紧接着的 photo.get(...) 抛 AttributeError。
该异常不在 get_video_info_task 的 except 列表里(只 catch DataFetchError /
KeyError),会穿过 asyncio.gather 直接把整轮爬取带崩。
- core.py: 用 (x or {}) 兜住 null;photo 为空时打 WARNING 并跳过该视频,
不再返回半残的 detail 让下游存空记录、再去下载媒体
- 新增 tests/test_kuaishou_unavailable_video.py(不发网络请求)
验证:photo=null / photo 缺失 / visionVideoDetail 缺失 均返回 None;
真实 cookie 端到端确认不可用视频被跳过、正常视频照常返回详情。
This commit is contained in:
@@ -237,15 +237,29 @@ class KuaishouCrawler(AbstractCrawler):
|
|||||||
utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {sleep_sec:.1f} seconds after fetching video details {video_id}")
|
utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {sleep_sec:.1f} seconds after fetching video details {video_id}")
|
||||||
|
|
||||||
detail = result.get("visionVideoDetail")
|
detail = result.get("visionVideoDetail")
|
||||||
if detail:
|
if not detail:
|
||||||
photo = detail.get("photo", {})
|
return None
|
||||||
author = detail.get("author", {})
|
|
||||||
utils.logger.info(
|
# 快手对不可用视频(已删除/私密/不存在)返回的是
|
||||||
f"[KuaishouCrawler.get_video_info_task] video detail: "
|
# visionVideoDetail: {photo: null, author: null}——key 在、值是 null。
|
||||||
f"id={photo.get('id', video_id)} author={author.get('name', '')} "
|
# 注意 .get("photo", {}) 只在 key **缺失** 时给默认值,key 存在且为 null
|
||||||
f"likes={photo.get('likeCount', '')} views={photo.get('viewCount', '')} "
|
# 时拿到的仍是 None,接着 .get() 就抛 AttributeError,而
|
||||||
f"caption={str(photo.get('caption', ''))[:50]}"
|
# asyncio.gather 不会拦住它,整轮爬取会直接带崩。
|
||||||
|
photo = detail.get("photo") or {}
|
||||||
|
if not photo:
|
||||||
|
utils.logger.warning(
|
||||||
|
f"[KuaishouCrawler.get_video_info_task] 视频不可用"
|
||||||
|
f"(photo 为空,可能已删除或私密),跳过 video_id={video_id}"
|
||||||
)
|
)
|
||||||
|
return None
|
||||||
|
|
||||||
|
author = detail.get("author") or {}
|
||||||
|
utils.logger.info(
|
||||||
|
f"[KuaishouCrawler.get_video_info_task] video detail: "
|
||||||
|
f"id={photo.get('id', video_id)} author={author.get('name', '')} "
|
||||||
|
f"likes={photo.get('likeCount', '')} views={photo.get('viewCount', '')} "
|
||||||
|
f"caption={str(photo.get('caption', ''))[:50]}"
|
||||||
|
)
|
||||||
return detail
|
return detail
|
||||||
except DataFetchError as ex:
|
except DataFetchError as ex:
|
||||||
utils.logger.error(
|
utils.logger.error(
|
||||||
|
|||||||
@@ -0,0 +1,128 @@
|
|||||||
|
# -*- coding: utf-8 -*-
|
||||||
|
# Copyright (c) 2025 relakkes@gmail.com
|
||||||
|
#
|
||||||
|
# This file is part of MediaCrawler project.
|
||||||
|
# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_kuaishou_unavailable_video.py
|
||||||
|
# GitHub: https://github.com/NanmiCoder
|
||||||
|
# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1
|
||||||
|
#
|
||||||
|
|
||||||
|
"""快手详情接口遇到「不可用视频」时的回归测试。
|
||||||
|
|
||||||
|
回归背景:快手对已删除/私密/不存在的视频返回的是
|
||||||
|
``visionVideoDetail: {photo: null, author: null}`` —— key 存在、值是 null。
|
||||||
|
而 ``detail.get("photo", {})`` 只在 key **缺失** 时给默认值,key 存在且为 null
|
||||||
|
时拿到的仍是 ``None``,紧接着的 ``photo.get(...)`` 抛 AttributeError;
|
||||||
|
该异常不在 ``get_video_info_task`` 的 except 列表里,又会穿过
|
||||||
|
``asyncio.gather``,把整轮爬取直接带崩。
|
||||||
|
|
||||||
|
这里不发起网络请求,只用 stub client 驱动真实的任务函数。
|
||||||
|
"""
|
||||||
|
|
||||||
|
from __future__ import annotations
|
||||||
|
|
||||||
|
import asyncio
|
||||||
|
import random
|
||||||
|
import sys
|
||||||
|
from pathlib import Path
|
||||||
|
from types import SimpleNamespace
|
||||||
|
|
||||||
|
import pytest
|
||||||
|
|
||||||
|
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
|
||||||
|
|
||||||
|
import config # noqa: E402
|
||||||
|
from media_platform.kuaishou.core import KuaishouCrawler # noqa: E402
|
||||||
|
|
||||||
|
PHOTO_ID = "3x3zxz4mjrsc8ke"
|
||||||
|
UNAVAILABLE_PHOTO_ID = "3xf8enb8dbj6uig"
|
||||||
|
|
||||||
|
# 真实响应里 photo/author 为 null 的那个视频
|
||||||
|
UNAVAILABLE_DETAIL = {
|
||||||
|
"visionVideoDetail": {
|
||||||
|
"status": 1,
|
||||||
|
"type": "video",
|
||||||
|
"author": None,
|
||||||
|
"photo": None,
|
||||||
|
"tags": [],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
NORMAL_DETAIL = {
|
||||||
|
"visionVideoDetail": {
|
||||||
|
"status": 1,
|
||||||
|
"type": "video",
|
||||||
|
"author": {"name": "余胜军说Java"},
|
||||||
|
"photo": {"id": PHOTO_ID, "caption": "我教你学Python", "likeCount": 167000},
|
||||||
|
"tags": [],
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
|
||||||
|
class _StubClient:
|
||||||
|
def __init__(self, payload):
|
||||||
|
self._payload = payload
|
||||||
|
|
||||||
|
async def get_video_info(self, photo_id): # noqa: ANN001
|
||||||
|
return self._payload
|
||||||
|
|
||||||
|
|
||||||
|
def _make_crawler(payload) -> SimpleNamespace:
|
||||||
|
"""只借 get_video_info_task 用到的 self.ks_client,不需要完整 crawler"""
|
||||||
|
return SimpleNamespace(ks_client=_StubClient(payload))
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.fixture(autouse=True)
|
||||||
|
def _no_sleep(monkeypatch):
|
||||||
|
"""去掉任务里的固定延时与随机抖动,测试不应真的等待"""
|
||||||
|
monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 0)
|
||||||
|
monkeypatch.setattr(random, "uniform", lambda a, b: 0)
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_unavailable_video_is_skipped_instead_of_crashing():
|
||||||
|
"""photo 为 null 时返回 None(跳过),而不是抛 AttributeError"""
|
||||||
|
crawler = _make_crawler(UNAVAILABLE_DETAIL)
|
||||||
|
|
||||||
|
result = await KuaishouCrawler.get_video_info_task(
|
||||||
|
crawler, UNAVAILABLE_PHOTO_ID, asyncio.Semaphore(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_missing_photo_key_is_also_skipped():
|
||||||
|
"""photo 字段整个缺失时同样跳过(.get 的默认值路径)"""
|
||||||
|
crawler = _make_crawler({"visionVideoDetail": {"status": 1, "author": None}})
|
||||||
|
|
||||||
|
result = await KuaishouCrawler.get_video_info_task(
|
||||||
|
crawler, UNAVAILABLE_PHOTO_ID, asyncio.Semaphore(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_normal_video_still_returns_detail():
|
||||||
|
"""正常视频不受影响"""
|
||||||
|
crawler = _make_crawler(NORMAL_DETAIL)
|
||||||
|
|
||||||
|
result = await KuaishouCrawler.get_video_info_task(
|
||||||
|
crawler, PHOTO_ID, asyncio.Semaphore(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is not None
|
||||||
|
assert result["photo"]["id"] == PHOTO_ID
|
||||||
|
|
||||||
|
|
||||||
|
@pytest.mark.asyncio
|
||||||
|
async def test_empty_vision_video_detail_returns_none():
|
||||||
|
"""visionVideoDetail 整体缺失时返回 None"""
|
||||||
|
crawler = _make_crawler({})
|
||||||
|
|
||||||
|
result = await KuaishouCrawler.get_video_info_task(
|
||||||
|
crawler, PHOTO_ID, asyncio.Semaphore(1)
|
||||||
|
)
|
||||||
|
|
||||||
|
assert result is None
|
||||||
Reference in New Issue
Block a user