Reuse cloudscraper

pynyaata/connectors/core.py

@@ -3,7 +3,7 @@ from enum import Enum
 from functools import wraps
 from json import dumps, loads
 
-import requests
+from cloudscraper import create_scraper
 from requests import RequestException
 from redis.exceptions import RedisError
 
@@ -12,7 +12,7 @@ from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
 if REDIS_ENABLED:
     from ..config import cache
 
-cloudproxy_session = None
+scraper = create_scraper()
 
 
 class ConnectorReturn(Enum):
@@ -80,19 +80,16 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
     if ajax:
         headers['X-Requested-With'] = 'XMLHttpRequest'
 
-    if cloudflare:
-        headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
-
     try:
         if method == 'post':
-            response = requests.post(
+            response = scraper.post(
                 url,
                 params,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers
             )
         else:
-            response = requests.get(
+            response = scraper.get(
                 url,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers

requirements.txt

@@ -11,3 +11,4 @@ python-dotenv==0.21.0
 dateparser==1.1.4
 redis==4.4.0
 transmission-rpc==3.4.0
+cloudscraper==1.2.66