This repository has been archived on 2023-10-01. You can view files and clone it, but cannot push or open issues or pull requests.
PyNyaaTa/pynyaata/connectors/core.py

187 lines
4.0 KiB
Python
Raw Normal View History

2020-04-24 19:01:44 +00:00
from abc import ABC, abstractmethod
from enum import Enum
from functools import wraps
2020-10-21 20:13:57 +00:00
from json import dumps, loads
2020-04-24 19:01:44 +00:00
import requests
2020-04-24 19:01:44 +00:00
from requests import RequestException
2021-07-10 21:15:06 +00:00
from redis.exceptions import RedisError
2020-04-24 19:01:44 +00:00
2021-12-07 10:17:01 +00:00
from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
2021-07-10 07:25:35 +00:00
if REDIS_ENABLED:
from ..config import cache
2020-04-24 19:01:44 +00:00
cloudproxy_session = None
2020-04-24 19:01:44 +00:00
class ConnectorReturn(Enum):
SEARCH = 1
HISTORY = 2
class ConnectorLang(Enum):
FR = '🇫🇷'
JP = '🇯🇵'
class Cache:
def cache_data(self, f):
@wraps(f)
def wrapper(*args, **kwds):
connector = args[0]
key = 'pynyaata.%s.%s.%s.%s' % (
connector.__class__.__name__,
f.__name__,
connector.query,
connector.page
)
2021-07-10 07:25:35 +00:00
if REDIS_ENABLED:
2021-07-10 21:15:06 +00:00
json = None
try:
json = cache.get(key)
except RedisError:
pass
2021-07-10 07:25:35 +00:00
if json:
data = loads(json)
connector.data = data['data']
connector.is_more = data['is_more']
connector.on_error = False
return
2020-04-24 19:01:44 +00:00
ret = f(*args, **kwds)
2021-07-10 07:25:35 +00:00
if not connector.on_error and REDIS_ENABLED:
2021-07-10 21:15:06 +00:00
try:
cache.set(key, dumps({
'data': connector.data,
'is_more': connector.is_more
}), CACHE_TIMEOUT)
except RedisError:
pass
2021-07-10 07:25:35 +00:00
2020-04-24 19:01:44 +00:00
return ret
return wrapper
ConnectorCache = Cache()
2021-12-16 22:32:51 +00:00
def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
2020-10-21 20:13:57 +00:00
output = ''
http_code = 500
method = 'post' if (params is not None) else 'get'
2021-12-16 22:32:51 +00:00
headers = {}
2020-10-21 20:13:57 +00:00
2020-04-24 19:01:44 +00:00
if ajax:
2021-12-16 22:32:51 +00:00
headers['X-Requested-With'] = 'XMLHttpRequest'
if cloudflare:
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
2020-04-24 19:01:44 +00:00
try:
2021-12-07 10:09:45 +00:00
if method == 'post':
response = requests.post(
2021-12-07 10:09:45 +00:00
url,
params,
timeout=REQUESTS_TIMEOUT,
headers=headers
)
else:
response = requests.get(
2021-12-07 10:09:45 +00:00
url,
timeout=REQUESTS_TIMEOUT,
headers=headers
)
output = response.text
http_code = response.status_code
2021-01-07 20:19:50 +00:00
except RequestException as e:
2020-07-25 14:27:21 +00:00
if debug:
2021-01-29 20:32:28 +00:00
logger.exception(e)
2020-04-24 19:01:44 +00:00
return {'http_code': http_code, 'output': output}
class ConnectorCore(ABC):
@property
@abstractmethod
def color(self):
pass
@property
@abstractmethod
def title(self):
pass
@property
@abstractmethod
def favicon(self):
pass
@property
@abstractmethod
def base_url(self):
pass
@property
@abstractmethod
def is_light(self):
pass
def __init__(self, query, page=1, return_type=ConnectorReturn.SEARCH):
self.query = query
self.data = []
self.is_more = False
self.on_error = True
self.page = page
self.return_type = return_type
@abstractmethod
def get_full_search_url(self):
pass
@abstractmethod
def search(self):
pass
@abstractmethod
def get_history(self):
pass
2021-01-30 18:40:36 +00:00
@abstractmethod
def is_vf(self, url):
pass
2020-04-24 19:01:44 +00:00
2021-01-03 19:41:58 +00:00
async def run(self):
2020-04-24 19:01:44 +00:00
if self.on_error:
if self.return_type is ConnectorReturn.SEARCH:
self.search()
elif self.return_type is ConnectorReturn.HISTORY:
self.get_history()
return self
class Other(ConnectorCore):
color = 'is-danger'
title = 'Other'
favicon = 'blank.png'
base_url = ''
is_light = True
def get_full_search_url(self):
pass
def search(self):
pass
def get_history(self):
pass
2021-01-30 21:11:52 +00:00
def is_vf(self, url):
return False