fork/MediaCrawler
1
0
Fork 0
mirror of https://github.com/NanmiCoder/MediaCrawler.git synced 2026-10-06 22:28:11 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #700 from 2513502304/main

Browse Source 
将捕获异常的类型由HTTPStatusError换成基类HTTPError，以便正确处理爬取媒体资源出现任何错误时，都不会导致爬取评论的中断，详情参见提交记录
This commit is contained in:
程序员阿江-Relakkes
2025-08-06 17:26:29 +08:00
committed by GitHub
parent e4e0f659e0 2c11e64dc9
commit 526c37822b
4 changed files with 8 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
media_platform/bilibili/client.py
+2 -2
View File
@@ -210,8 +210,8 @@ class BilibiliClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[BilibiliClient.get_video_media] {exc}") utils.logger.error(f"[BilibiliClient.get_video_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def get_video_comments( async def get_video_comments(
media_platform/douyin/client.py
+2 -2
View File
@@ -321,6 +321,6 @@ class DouYinClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
media_platform/weibo/client.py
+2 -2
View File
@@ -256,8 +256,8 @@ class WeiboClient:
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def get_creator_container_info(self, creator_id: str) -> Dict: async def get_creator_container_info(self, creator_id: str) -> Dict:
media_platform/xhs/client.py
+2 -2
View File
@@ -160,8 +160,8 @@ class XiaoHongShuClient(AbstractApiClient):
return None return None
else: else:
return response.content return response.content
except httpx.HTTPStatusError as exc: # some wrong when call httpx.request method, such as connection error, client error or server error except httpx.HTTPError as exc: # some wrong when call httpx.request method, such as connection error, client error, server error or response status code is not 2xx
utils.logger.error(f"[DouYinClient.get_aweme_media] {exc}") utils.logger.error(f"[DouYinClient.get_aweme_media] {exc.__class__.__name__} for {exc.request.url} - {exc}") # 保留原始异常类型名称，以便开发者调试
return None return None
async def pong(self) -> bool: async def pong(self) -> bool: