Revert "Reuse cloudscraper"
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
This reverts commit ea5f514adb
.
This commit is contained in:
parent
ea5f514adb
commit
28b11257a4
@ -3,7 +3,7 @@ from enum import Enum
|
|||||||
from functools import wraps
|
from functools import wraps
|
||||||
from json import dumps, loads
|
from json import dumps, loads
|
||||||
|
|
||||||
from cloudscraper import create_scraper
|
import requests
|
||||||
from requests import RequestException
|
from requests import RequestException
|
||||||
from redis.exceptions import RedisError
|
from redis.exceptions import RedisError
|
||||||
|
|
||||||
@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
|
|||||||
if REDIS_ENABLED:
|
if REDIS_ENABLED:
|
||||||
from ..config import cache
|
from ..config import cache
|
||||||
|
|
||||||
scraper = create_scraper()
|
cloudproxy_session = None
|
||||||
|
|
||||||
|
|
||||||
class ConnectorReturn(Enum):
|
class ConnectorReturn(Enum):
|
||||||
@ -80,16 +80,19 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
|
|||||||
if ajax:
|
if ajax:
|
||||||
headers['X-Requested-With'] = 'XMLHttpRequest'
|
headers['X-Requested-With'] = 'XMLHttpRequest'
|
||||||
|
|
||||||
|
if cloudflare:
|
||||||
|
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
|
||||||
|
|
||||||
try:
|
try:
|
||||||
if method == 'post':
|
if method == 'post':
|
||||||
response = scraper.post(
|
response = requests.post(
|
||||||
url,
|
url,
|
||||||
params,
|
params,
|
||||||
timeout=REQUESTS_TIMEOUT,
|
timeout=REQUESTS_TIMEOUT,
|
||||||
headers=headers
|
headers=headers
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
response = scraper.get(
|
response = requests.get(
|
||||||
url,
|
url,
|
||||||
timeout=REQUESTS_TIMEOUT,
|
timeout=REQUESTS_TIMEOUT,
|
||||||
headers=headers
|
headers=headers
|
||||||
|
@ -11,4 +11,3 @@ python-dotenv==0.21.0
|
|||||||
dateparser==1.1.4
|
dateparser==1.1.4
|
||||||
redis==4.4.0
|
redis==4.4.0
|
||||||
transmission-rpc==3.4.0
|
transmission-rpc==3.4.0
|
||||||
cloudscraper==1.2.66
|
|
||||||
|
Reference in New Issue
Block a user