Revert "Reuse cloudscraper"
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This reverts commit ea5f514adb
.
This commit is contained in:
parent
ea5f514adb
commit
28b11257a4
@ -3,7 +3,7 @@ from enum import Enum
|
||||
from functools import wraps
|
||||
from json import dumps, loads
|
||||
|
||||
from cloudscraper import create_scraper
|
||||
import requests
|
||||
from requests import RequestException
|
||||
from redis.exceptions import RedisError
|
||||
|
||||
@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
|
||||
if REDIS_ENABLED:
|
||||
from ..config import cache
|
||||
|
||||
scraper = create_scraper()
|
||||
cloudproxy_session = None
|
||||
|
||||
|
||||
class ConnectorReturn(Enum):
|
||||
@ -80,16 +80,19 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
|
||||
if ajax:
|
||||
headers['X-Requested-With'] = 'XMLHttpRequest'
|
||||
|
||||
if cloudflare:
|
||||
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
|
||||
|
||||
try:
|
||||
if method == 'post':
|
||||
response = scraper.post(
|
||||
response = requests.post(
|
||||
url,
|
||||
params,
|
||||
timeout=REQUESTS_TIMEOUT,
|
||||
headers=headers
|
||||
)
|
||||
else:
|
||||
response = scraper.get(
|
||||
response = requests.get(
|
||||
url,
|
||||
timeout=REQUESTS_TIMEOUT,
|
||||
headers=headers
|
||||
|
@ -11,4 +11,3 @@ python-dotenv==0.21.0
|
||||
dateparser==1.1.4
|
||||
redis==4.4.0
|
||||
transmission-rpc==3.4.0
|
||||
cloudscraper==1.2.66
|
||||
|
Reference in New Issue
Block a user