Cleaning mess
This commit is contained in:
parent
0adca89e76
commit
fc019007fa
3
app.py
3
app.py
@ -1,4 +1,3 @@
|
||||
from logging import getLogger
|
||||
from operator import attrgetter, itemgetter
|
||||
from time import sleep
|
||||
|
||||
@ -134,7 +133,7 @@ def clean_model(obj):
|
||||
if not attr.startswith('_') and getattr(obj, attr) is None:
|
||||
try:
|
||||
setattr(obj, attr, '')
|
||||
except Exception:
|
||||
except AttributeError:
|
||||
pass
|
||||
return obj
|
||||
|
||||
|
@ -9,6 +9,7 @@ from urllib.parse import quote
|
||||
|
||||
from bs4 import BeautifulSoup
|
||||
from cloudscraper import create_scraper
|
||||
from requests import RequestException
|
||||
|
||||
from config import IS_DEBUG, CACHE_TIMEOUT, BLACKLIST_WORDS
|
||||
from models import AnimeLink
|
||||
@ -78,6 +79,29 @@ class Cache:
|
||||
ConnectorCache = Cache()
|
||||
|
||||
|
||||
def curl_content(url, params=None, ajax=False):
|
||||
if ajax:
|
||||
headers = {'X-Requested-With': 'XMLHttpRequest'}
|
||||
else:
|
||||
headers = {}
|
||||
|
||||
try:
|
||||
if params is not None:
|
||||
response = scraper.post(url, params, timeout=5, headers=headers)
|
||||
else:
|
||||
response = scraper.get(url, timeout=5, headers=headers)
|
||||
|
||||
output = response.text
|
||||
http_code = response.status_code
|
||||
except RequestException as e:
|
||||
output = ''
|
||||
http_code = 500
|
||||
if IS_DEBUG:
|
||||
getLogger().exception(e)
|
||||
|
||||
return {'http_code': http_code, 'output': output}
|
||||
|
||||
|
||||
class Connector(ABC):
|
||||
@property
|
||||
@abstractmethod
|
||||
@ -132,28 +156,6 @@ class Connector(ABC):
|
||||
self.get_history()
|
||||
return self
|
||||
|
||||
def curl_content(self, url, params=None, ajax=False):
|
||||
if ajax:
|
||||
headers = {'X-Requested-With': 'XMLHttpRequest'}
|
||||
else:
|
||||
headers = {}
|
||||
|
||||
try:
|
||||
if params is not None:
|
||||
response = scraper.post(url, params, timeout=5, headers=headers)
|
||||
else:
|
||||
response = scraper.get(url, timeout=5, headers=headers)
|
||||
|
||||
output = response.text
|
||||
http_code = response.status_code
|
||||
except Exception as e:
|
||||
output = ''
|
||||
http_code = 500
|
||||
if IS_DEBUG:
|
||||
getLogger().exception(e)
|
||||
|
||||
return {'http_code': http_code, 'output': output}
|
||||
|
||||
@staticmethod
|
||||
def get_instance(url, query):
|
||||
if 'nyaa.si' in url:
|
||||
@ -202,7 +204,7 @@ class Nyaa(Connector):
|
||||
|
||||
@ConnectorCache.cache_data
|
||||
def search(self):
|
||||
response = self.curl_content(self.get_full_search_url())
|
||||
response = curl_content(self.get_full_search_url())
|
||||
|
||||
if response['http_code'] is 200:
|
||||
html = BeautifulSoup(response['output'], 'html.parser')
|
||||
@ -274,7 +276,7 @@ class Pantsu(Connector):
|
||||
|
||||
@ConnectorCache.cache_data
|
||||
def search(self):
|
||||
response = self.curl_content(self.get_full_search_url())
|
||||
response = curl_content(self.get_full_search_url())
|
||||
|
||||
if response['http_code'] is 200:
|
||||
html = BeautifulSoup(response['output'], 'html.parser')
|
||||
@ -320,9 +322,8 @@ class Pantsu(Connector):
|
||||
'href': href,
|
||||
'name': url_safe,
|
||||
'comment': '',
|
||||
'link': tds[2].decode_contents()
|
||||
.replace('icon-magnet', 'fa fa-fw fa-magnet')
|
||||
.replace('icon-floppy', 'fa fa-fw fa-download'),
|
||||
'link': tds[2].decode_contents().replace('icon-magnet', 'fa fa-fw fa-magnet').replace(
|
||||
'icon-floppy', 'fa fa-fw fa-download'),
|
||||
'size': tds[3].string,
|
||||
'date': formatted_date,
|
||||
'seeds': check_seeds,
|
||||
@ -360,7 +361,7 @@ class YggTorrent(Connector):
|
||||
@ConnectorCache.cache_data
|
||||
def search(self):
|
||||
if self.category:
|
||||
response = self.curl_content(self.get_full_search_url())
|
||||
response = curl_content(self.get_full_search_url())
|
||||
|
||||
if response['http_code'] is 200:
|
||||
html = BeautifulSoup(response['output'], 'html.parser')
|
||||
@ -434,7 +435,7 @@ class AnimeUltime(Connector):
|
||||
|
||||
@ConnectorCache.cache_data
|
||||
def search(self):
|
||||
response = self.curl_content(self.get_full_search_url(), {'search': self.query})
|
||||
response = curl_content(self.get_full_search_url(), {'search': self.query})
|
||||
|
||||
if response['http_code'] is 200:
|
||||
html = BeautifulSoup(response['output'], 'html.parser')
|
||||
@ -482,7 +483,7 @@ class AnimeUltime(Connector):
|
||||
|
||||
@ConnectorCache.cache_data
|
||||
def get_history(self):
|
||||
response = self.curl_content(self.get_full_search_url())
|
||||
response = curl_content(self.get_full_search_url())
|
||||
|
||||
if response['http_code'] is 200:
|
||||
html = BeautifulSoup(response['output'], 'html.parser')
|
||||
|
@ -1,13 +1,12 @@
|
||||
from config import app
|
||||
from connectors import Connector
|
||||
from connectors import curl_content
|
||||
from models import AnimeLink
|
||||
|
||||
app.config['SQLALCHEMY_ECHO'] = False
|
||||
links = AnimeLink.query.all()
|
||||
|
||||
for link in links:
|
||||
connect = Connector.get_instance(link.link, link.title.keyword)
|
||||
html = connect.curl_content(link.link)
|
||||
html = curl_content(link.link)
|
||||
|
||||
if html['http_code'] is not 200:
|
||||
print('(%d) %s %s : %s' % (html['http_code'], link.title.name, link.season, link.link))
|
||||
|
@ -2,7 +2,8 @@ Flask==1.0.2
|
||||
Flask-SQLAlchemy==2.1
|
||||
SQLAlchemy==1.2.18
|
||||
Flask-HTTPAuth==3.2.4
|
||||
Flask-WTF===0.14.2
|
||||
Flask-WTF==0.14.2
|
||||
WTForms==2.2.1
|
||||
PyMySQL==0.9.3
|
||||
requests==2.21.0
|
||||
beautifulsoup4==4.7.1
|
||||
|
Reference in New Issue
Block a user