Revert "Reuse cloudscraper"
All checks were successful
continuous-integration/drone/push Build is passing

This reverts commit ea5f514adb.
This commit is contained in:
Michel Roux 2022-12-19 19:18:08 +00:00
parent ea5f514adb
commit 28b11257a4
2 changed files with 7 additions and 5 deletions

View File

@ -3,7 +3,7 @@ from enum import Enum
from functools import wraps from functools import wraps
from json import dumps, loads from json import dumps, loads
from cloudscraper import create_scraper import requests
from requests import RequestException from requests import RequestException
from redis.exceptions import RedisError from redis.exceptions import RedisError
@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
if REDIS_ENABLED: if REDIS_ENABLED:
from ..config import cache from ..config import cache
scraper = create_scraper() cloudproxy_session = None
class ConnectorReturn(Enum): class ConnectorReturn(Enum):
@ -80,16 +80,19 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
if ajax: if ajax:
headers['X-Requested-With'] = 'XMLHttpRequest' headers['X-Requested-With'] = 'XMLHttpRequest'
if cloudflare:
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
try: try:
if method == 'post': if method == 'post':
response = scraper.post( response = requests.post(
url, url,
params, params,
timeout=REQUESTS_TIMEOUT, timeout=REQUESTS_TIMEOUT,
headers=headers headers=headers
) )
else: else:
response = scraper.get( response = requests.get(
url, url,
timeout=REQUESTS_TIMEOUT, timeout=REQUESTS_TIMEOUT,
headers=headers headers=headers

View File

@ -11,4 +11,3 @@ python-dotenv==0.21.0
dateparser==1.1.4 dateparser==1.1.4
redis==4.4.0 redis==4.4.0
transmission-rpc==3.4.0 transmission-rpc==3.4.0
cloudscraper==1.2.66