From 1e1ae64cb4242f5afdb2dbc35b5da931ae6497a0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E7=A8=8B=E5=BA=8F=E5=91=98=E9=98=BF=E6=B1=9F=28Relakkes?= =?UTF-8?q?=29?= Date: Fri, 18 Sep 2026 17:18:01 +0800 Subject: [PATCH] =?UTF-8?q?fix(ks):=20=E8=A7=86=E9=A2=91=E4=B8=8D=E5=8F=AF?= =?UTF-8?q?=E7=94=A8=E6=97=B6=20photo=20=E4=B8=BA=20null=20=E5=AF=BC?= =?UTF-8?q?=E8=87=B4=E6=95=B4=E8=BD=AE=E7=88=AC=E5=8F=96=E5=B4=A9=E6=BA=83?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 快手对已删除/私密/不存在的视频，返回的 visionVideoDetail 里 photo/author 是 null——key 存在、值是 null。而 detail.get("photo", {}) 只在 key 缺失时给默认值， key 存在且为 null 时拿到的仍是 None，紧接着的 photo.get(...) 抛 AttributeError。 该异常不在 get_video_info_task 的 except 列表里（只 catch DataFetchError / KeyError），会穿过 asyncio.gather 直接把整轮爬取带崩。 - core.py: 用 (x or {}) 兜住 null；photo 为空时打 WARNING 并跳过该视频， 不再返回半残的 detail 让下游存空记录、再去下载媒体 - 新增 tests/test_kuaishou_unavailable_video.py（不发网络请求） 验证：photo=null / photo 缺失 / visionVideoDetail 缺失 均返回 None； 真实 cookie 端到端确认不可用视频被跳过、正常视频照常返回详情。 --- media_platform/kuaishou/core.py | 30 ++++-- tests/test_kuaishou_unavailable_video.py | 128 +++++++++++++++++++++++ 2 files changed, 150 insertions(+), 8 deletions(-) create mode 100644 tests/test_kuaishou_unavailable_video.py diff --git a/media_platform/kuaishou/core.py b/media_platform/kuaishou/core.py index c904e18..0c06217 100644 --- a/media_platform/kuaishou/core.py +++ b/media_platform/kuaishou/core.py @@ -237,15 +237,29 @@ class KuaishouCrawler(AbstractCrawler): utils.logger.info(f"[KuaishouCrawler.get_video_info_task] Sleeping for {sleep_sec:.1f} seconds after fetching video details {video_id}") detail = result.get("visionVideoDetail") - if detail: - photo = detail.get("photo", {}) - author = detail.get("author", {}) - utils.logger.info( - f"[KuaishouCrawler.get_video_info_task] video detail: " - f"id={photo.get('id', video_id)} author={author.get('name', '')} " - f"likes={photo.get('likeCount', '')} views={photo.get('viewCount', '')} " - f"caption={str(photo.get('caption', ''))[:50]}" + if not detail: + return None + + # 快手对不可用视频（已删除/私密/不存在）返回的是 + # visionVideoDetail: {photo: null, author: null}——key 在、值是 null。 + # 注意 .get("photo", {}) 只在 key **缺失** 时给默认值，key 存在且为 null + # 时拿到的仍是 None，接着 .get() 就抛 AttributeError，而 + # asyncio.gather 不会拦住它，整轮爬取会直接带崩。 + photo = detail.get("photo") or {} + if not photo: + utils.logger.warning( + f"[KuaishouCrawler.get_video_info_task] 视频不可用" + f"（photo 为空，可能已删除或私密），跳过 video_id={video_id}" ) + return None + + author = detail.get("author") or {} + utils.logger.info( + f"[KuaishouCrawler.get_video_info_task] video detail: " + f"id={photo.get('id', video_id)} author={author.get('name', '')} " + f"likes={photo.get('likeCount', '')} views={photo.get('viewCount', '')} " + f"caption={str(photo.get('caption', ''))[:50]}" + ) return detail except DataFetchError as ex: utils.logger.error( diff --git a/tests/test_kuaishou_unavailable_video.py b/tests/test_kuaishou_unavailable_video.py new file mode 100644 index 0000000..c09b433 --- /dev/null +++ b/tests/test_kuaishou_unavailable_video.py @@ -0,0 +1,128 @@ +# -*- coding: utf-8 -*- +# Copyright (c) 2025 relakkes@gmail.com +# +# This file is part of MediaCrawler project. +# Repository: https://github.com/NanmiCoder/MediaCrawler/blob/main/tests/test_kuaishou_unavailable_video.py +# GitHub: https://github.com/NanmiCoder +# Licensed under NON-COMMERCIAL LEARNING LICENSE 1.1 +# + +"""快手详情接口遇到「不可用视频」时的回归测试。 + +回归背景：快手对已删除/私密/不存在的视频返回的是 +``visionVideoDetail: {photo: null, author: null}`` —— key 存在、值是 null。 +而 ``detail.get("photo", {})`` 只在 key **缺失** 时给默认值，key 存在且为 null +时拿到的仍是 ``None``，紧接着的 ``photo.get(...)`` 抛 AttributeError； +该异常不在 ``get_video_info_task`` 的 except 列表里，又会穿过 +``asyncio.gather``，把整轮爬取直接带崩。 + +这里不发起网络请求，只用 stub client 驱动真实的任务函数。 +""" + +from __future__ import annotations + +import asyncio +import random +import sys +from pathlib import Path +from types import SimpleNamespace + +import pytest + +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +import config # noqa: E402 +from media_platform.kuaishou.core import KuaishouCrawler # noqa: E402 + +PHOTO_ID = "3x3zxz4mjrsc8ke" +UNAVAILABLE_PHOTO_ID = "3xf8enb8dbj6uig" + +# 真实响应里 photo/author 为 null 的那个视频 +UNAVAILABLE_DETAIL = { + "visionVideoDetail": { + "status": 1, + "type": "video", + "author": None, + "photo": None, + "tags": [], + } +} + +NORMAL_DETAIL = { + "visionVideoDetail": { + "status": 1, + "type": "video", + "author": {"name": "余胜军说Java"}, + "photo": {"id": PHOTO_ID, "caption": "我教你学Python", "likeCount": 167000}, + "tags": [], + } +} + + +class _StubClient: + def __init__(self, payload): + self._payload = payload + + async def get_video_info(self, photo_id): # noqa: ANN001 + return self._payload + + +def _make_crawler(payload) -> SimpleNamespace: + """只借 get_video_info_task 用到的 self.ks_client，不需要完整 crawler""" + return SimpleNamespace(ks_client=_StubClient(payload)) + + +@pytest.fixture(autouse=True) +def _no_sleep(monkeypatch): + """去掉任务里的固定延时与随机抖动，测试不应真的等待""" + monkeypatch.setattr(config, "CRAWLER_MAX_SLEEP_SEC", 0) + monkeypatch.setattr(random, "uniform", lambda a, b: 0) + + +@pytest.mark.asyncio +async def test_unavailable_video_is_skipped_instead_of_crashing(): + """photo 为 null 时返回 None（跳过），而不是抛 AttributeError""" + crawler = _make_crawler(UNAVAILABLE_DETAIL) + + result = await KuaishouCrawler.get_video_info_task( + crawler, UNAVAILABLE_PHOTO_ID, asyncio.Semaphore(1) + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_missing_photo_key_is_also_skipped(): + """photo 字段整个缺失时同样跳过（.get 的默认值路径）""" + crawler = _make_crawler({"visionVideoDetail": {"status": 1, "author": None}}) + + result = await KuaishouCrawler.get_video_info_task( + crawler, UNAVAILABLE_PHOTO_ID, asyncio.Semaphore(1) + ) + + assert result is None + + +@pytest.mark.asyncio +async def test_normal_video_still_returns_detail(): + """正常视频不受影响""" + crawler = _make_crawler(NORMAL_DETAIL) + + result = await KuaishouCrawler.get_video_info_task( + crawler, PHOTO_ID, asyncio.Semaphore(1) + ) + + assert result is not None + assert result["photo"]["id"] == PHOTO_ID + + +@pytest.mark.asyncio +async def test_empty_vision_video_detail_returns_none(): + """visionVideoDetail 整体缺失时返回 None""" + crawler = _make_crawler({}) + + result = await KuaishouCrawler.get_video_info_task( + crawler, PHOTO_ID, asyncio.Semaphore(1) + ) + + assert result is None