mirror of
https://github.com/NanmiCoder/MediaCrawler.git
synced 2026-10-10 13:10:55 +08:00
feat: ip proxy expired check
This commit is contained in:
@@ -21,7 +21,7 @@
|
||||
# -*- coding: utf-8 -*-
|
||||
import asyncio
|
||||
import json
|
||||
from typing import Any, Callable, Dict, List, Optional
|
||||
from typing import TYPE_CHECKING, Any, Callable, Dict, List, Optional
|
||||
from urllib.parse import urlencode
|
||||
|
||||
import httpx
|
||||
@@ -29,13 +29,17 @@ from playwright.async_api import BrowserContext, Page
|
||||
|
||||
import config
|
||||
from base.base_crawler import AbstractApiClient
|
||||
from proxy.proxy_mixin import ProxyRefreshMixin
|
||||
from tools import utils
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from proxy.proxy_ip_pool import ProxyIpPool
|
||||
|
||||
from .exception import DataFetchError
|
||||
from .graphql import KuaiShouGraphQL
|
||||
|
||||
|
||||
class KuaiShouClient(AbstractApiClient):
|
||||
class KuaiShouClient(AbstractApiClient, ProxyRefreshMixin):
|
||||
def __init__(
|
||||
self,
|
||||
timeout=10,
|
||||
@@ -44,6 +48,7 @@ class KuaiShouClient(AbstractApiClient):
|
||||
headers: Dict[str, str],
|
||||
playwright_page: Page,
|
||||
cookie_dict: Dict[str, str],
|
||||
proxy_ip_pool: Optional["ProxyIpPool"] = None,
|
||||
):
|
||||
self.proxy = proxy
|
||||
self.timeout = timeout
|
||||
@@ -52,8 +57,13 @@ class KuaiShouClient(AbstractApiClient):
|
||||
self.playwright_page = playwright_page
|
||||
self.cookie_dict = cookie_dict
|
||||
self.graphql = KuaiShouGraphQL()
|
||||
# 初始化代理池(来自 ProxyRefreshMixin)
|
||||
self.init_proxy_pool(proxy_ip_pool)
|
||||
|
||||
async def request(self, method, url, **kwargs) -> Any:
|
||||
# 每次请求前检测代理是否过期
|
||||
await self._refresh_proxy_if_expired()
|
||||
|
||||
async with httpx.AsyncClient(proxy=self.proxy) as client:
|
||||
response = await client.request(method, url, timeout=self.timeout, **kwargs)
|
||||
data: Dict = response.json()
|
||||
|
||||
Reference in New Issue
Block a user