diff --git a/pynyaata/connectors/core.py b/pynyaata/connectors/core.py index f81b3ab..4645e31 100644 --- a/pynyaata/connectors/core.py +++ b/pynyaata/connectors/core.py @@ -3,7 +3,7 @@ from enum import Enum from functools import wraps from json import dumps, loads -from cloudscraper import create_scraper +import requests from requests import RequestException from redis.exceptions import RedisError @@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED if REDIS_ENABLED: from ..config import cache -scraper = create_scraper() +cloudproxy_session = None class ConnectorReturn(Enum): @@ -80,16 +80,19 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False): if ajax: headers['X-Requested-With'] = 'XMLHttpRequest' + if cloudflare: + headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)' + try: if method == 'post': - response = scraper.post( + response = requests.post( url, params, timeout=REQUESTS_TIMEOUT, headers=headers ) else: - response = scraper.get( + response = requests.get( url, timeout=REQUESTS_TIMEOUT, headers=headers diff --git a/requirements.txt b/requirements.txt index 6d04bff..068a6af 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,3 @@ python-dotenv==0.21.0 dateparser==1.1.4 redis==4.4.0 transmission-rpc==3.4.0 -cloudscraper==1.2.66