feat: 抖音abogus参数更新

media_platform/douyin/client.py

@@ -2,11 +2,10 @@ import asyncio
 import copy
 import json
 import urllib.parse
-from typing import Any, Callable, Dict, List, Optional
+from typing import Any, Callable, Dict, Optional
 
-import execjs
-import httpx
-from playwright.async_api import BrowserContext, Page
+import requests
+from playwright.async_api import BrowserContext
 
 from base.base_crawler import AbstractApiClient
 from tools import utils
@@ -14,6 +13,7 @@ from var import request_keyword_var
 
 from .exception import *
 from .field import *
+from .help import *
 
 
 class DOUYINClient(AbstractApiClient):
@@ -33,51 +33,71 @@ class DOUYINClient(AbstractApiClient):
         self.playwright_page = playwright_page
         self.cookie_dict = cookie_dict
 
-    async def __process_req_params(self, params: Optional[Dict] = None, headers: Optional[Dict] = None):
+    async def __process_req_params(
+            self, params: Optional[Dict] = None, headers: Optional[Dict] = None,
+            request_method="GET"
+    ):
+
         if not params:
             return
         headers = headers or self.headers
         local_storage: Dict = await self.playwright_page.evaluate("() => window.localStorage")  # type: ignore
-        douyin_js_obj = execjs.compile(open('libs/douyin.js').read())
         common_params = {
             "device_platform": "webapp",
             "aid": "6383",
             "channel": "channel_pc_web",
+            "version_code": "190600",
+            "version_name": "19.6.0",
+            "update_version_code": "170400",
+            "pc_client_type": "1",
             "cookie_enabled": "true",
             "browser_language": "zh-CN",
-            "browser_platform": "Win32",
-            "browser_name": "Firefox",
-            "browser_version": "110.0",
+            "browser_platform": "MacIntel",
+            "browser_name": "Chrome",
+            "browser_version": "125.0.0.0",
             "browser_online": "true",
-            "engine_name": "Gecko",
-            "os_name": "Windows",
-            "os_version": "10",
+            "engine_name": "Blink",
+            "os_name": "Mac OS",
+            "os_version": "10.15.7",
+            "cpu_core_num": "8",
+            "device_memory": "8",
             "engine_version": "109.0",
             "platform": "PC",
-            "screen_width": "1920",
-            "screen_height": "1200",
-            # " webid": douyin_js_obj.call("get_web_id"),
-            # "msToken": local_storage.get("xmst"),
-            # "msToken": "abL8SeUTPa9-EToD8qfC7toScSADxpg6yLh2dbNcpWHzE0bT04txM_4UwquIcRvkRb9IU8sifwgM1Kwf1Lsld81o9Irt2_yNyUbbQPSUO8EfVlZJ_78FckDFnwVBVUVK",
+            "screen_width": "2560",
+            "screen_height": "1440",
+            'effective_type': '4g',
+            "round_trip_time": "50",
+            "webid": get_web_id(),
+            "msToken": local_storage.get("xmst"),
         }
         params.update(common_params)
-        query = '&'.join([f'{k}={v}' for k, v in params.items()])
-        x_bogus = douyin_js_obj.call('sign', query, headers["User-Agent"])
-        params["X-Bogus"] = x_bogus
-        # print(x_bogus, query)
+        query_string = urllib.parse.urlencode(params)
+
+        # 20240610 a-bogus更新（Playwright版本）
+        post_data = {}
+        if request_method == "POST":
+            post_data = params
+        a_bogus = await get_a_bogus(query_string, post_data, headers["User-Agent"], self.playwright_page)
+        params["a_bogus"] = a_bogus
 
     async def request(self, method, url, **kwargs):
-        async with httpx.AsyncClient(proxies=self.proxies) as client:
-            response = await client.request(
-                method, url, timeout=self.timeout,
-                **kwargs
-            )
-            try:
-                return response.json()
-            except Exception as e:
-                raise DataFetchError(f"{e}, {response.text}")
+        response = None
+        if method == "GET":
+            response = requests.request(method, url, **kwargs)
+        elif method == "POST":
+            response = requests.request(method, url, **kwargs)
+        try:
+            if response.text == "" or response.text == "blocked":
+                utils.logger.error(f"request params incrr, response.text: {response.text}")
+                raise Exception("account blocked")
+            return response.json()
+        except Exception as e:
+            raise DataFetchError(f"{e}, {response.text}")
 
     async def get(self, uri: str, params: Optional[Dict] = None, headers: Optional[Dict] = None):
+        """
+        GET请求
+        """
         await self.__process_req_params(params, headers)
         headers = headers or self.headers
         return await self.request(method="GET", url=f"{self._host}{uri}", params=params, headers=headers)
@@ -117,27 +137,30 @@ class DOUYINClient(AbstractApiClient):
         :param publish_time: ·
         :return:
         """
-        params = {
-            "keyword": urllib.parse.quote(keyword),
-            "search_channel": search_channel.value,
-            "search_source": "normal_search",
-            "query_correct_type": 1,
-            "is_filter_search": 0,
-            "offset": offset,
-            "count": 10  # must be set to 10
+        query_params = {
+            'search_channel': search_channel.value,
+            'enable_history': '1',
+            'keyword': urllib.parse.quote(keyword),
+            'search_source': 'tab_search',
+            'query_correct_type': '1',
+            'is_filter_search': '0',
+            'from_group_id': '7378810571505847586',
+            'offset': offset,
+            'count': '15',
+            'need_filter_settings': '1',
+            'list_type': 'multi',
         }
-        if sort_type != SearchSortType.GENERAL or publish_time != PublishTimeType.UNLIMITED:
-           params["filter_selected"] = urllib.parse.quote(json.dumps({
-               "sort_type": str(sort_type.value),
-               "publish_time": str(publish_time.value)
-           }))
-           params["is_filter_search"] = 1
-           params["search_source"] = "tab_search"
-        referer_url = "https://www.douyin.com/search/" + keyword
-        referer_url += f"?publish_time={publish_time.value}&sort_type={sort_type.value}&type=general"
+        if sort_type.value != SearchSortType.GENERAL.value or publish_time.value != PublishTimeType.UNLIMITED.value:
+            query_params["filter_selected"] = urllib.parse.quote(json.dumps({
+                "sort_type": str(sort_type.value),
+                "publish_time": str(publish_time.value)
+            }))
+            query_params["is_filter_search"] = 1
+            query_params["search_source"] = "tab_search"
+        referer_url = f"https://www.douyin.com/search/{keyword}?aid=f594bbd9-a0e2-4651-9319-ebe3cb6298c1&type=general"
         headers = copy.copy(self.headers)
         headers["Referer"] = urllib.parse.quote(referer_url, safe=':/')
-        return await self.get("/aweme/v1/web/general/search/single/", params, headers=headers)
+        return await self.get("/aweme/v1/web/general/search/single/", query_params, headers=headers)
 
     async def get_video_by_id(self, aweme_id: str) -> Any:
         """
@@ -149,7 +172,6 @@ class DOUYINClient(AbstractApiClient):
             "aweme_id": aweme_id
         }
         headers = copy.copy(self.headers)
-        # headers["Cookie"] = "s_v_web_id=verify_lol4a8dv_wpQ1QMyP_xemd_4wON_8Yzr_FJa8DN1vdY2m;"
         del headers["Origin"]
         res = await self.get("/aweme/v1/web/aweme/detail/", params, headers)
         return res.get("aweme_detail", {})
@@ -259,7 +281,9 @@ class DOUYINClient(AbstractApiClient):
             "count": 18,
             "max_cursor": max_cursor,
             "locate_query": "false",
-            "publish_video_strategy_type": 2
+            "publish_video_strategy_type": 2,
+            'verifyFp': 'verify_lx901cuk_K7kaK4dK_bn2E_4dgk_BxAA_E0XS1VtUi130',
+            'fp': 'verify_lx901cuk_K7kaK4dK_bn2E_4dgk_BxAA_E0XS1VtUi130'
         }
         return await self.get(uri, params)