Xefir/PyNyaaTa
Archived
1
0
Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Reuse cloudscraper
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Michel Roux 2022-12-19 19:02:37 +00:00
parent 3ff3fad357
commit ea5f514adb
2 changed files with 5 additions and 7 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
pynyaata/connectors/core.py
View File

@ -3,7 +3,7 @@ from enum import Enum
from functools import wraps from functools import wraps
from json import dumps, loads from json import dumps, loads
import requests from cloudscraper import create_scraper
from requests import RequestException from requests import RequestException
from redis.exceptions import RedisError from redis.exceptions import RedisError
@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
if REDIS_ENABLED: if REDIS_ENABLED:
from ..config import cache from ..config import cache
cloudproxy_session = None scraper = create_scraper()
class ConnectorReturn(Enum): class ConnectorReturn(Enum):
@ -80,19 +80,16 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
if ajax: if ajax:
headers['X-Requested-With'] = 'XMLHttpRequest' headers['X-Requested-With'] = 'XMLHttpRequest'
if cloudflare:
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
try: try:
if method == 'post': if method == 'post':
response = requests.post( response = scraper.post(
url, url,
params, params,
timeout=REQUESTS_TIMEOUT, timeout=REQUESTS_TIMEOUT,
headers=headers headers=headers
) )
else: else:
response = requests.get( response = scraper.get(
url, url,
timeout=REQUESTS_TIMEOUT, timeout=REQUESTS_TIMEOUT,
headers=headers headers=headers

1
requirements.txt
View File

@ -11,3 +11,4 @@ python-dotenv==0.21.0
dateparser==1.1.4 dateparser==1.1.4
redis==4.4.0 redis==4.4.0
transmission-rpc==3.4.0 transmission-rpc==3.4.0
cloudscraper==1.2.66