diff --git a/README.md b/README.md index 21f720e..84f5581 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it ## Features -* Search on [Nyaa.si](https://nyaa.si/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1) +* Search on [Nyaa.si](https://nyaa.si/) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1) * Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search * Color official and bad links * Add seeded links to a database @@ -40,15 +40,6 @@ All is managed by environment variables. Please look into the `.env.dist` file to list all possible environment variables. You have to have a running database server to be able to access the admin panel. -### Bypassing CloudFlare for YggTorrent - -YggTorrent use CloudFlare to protect them to DDoS attacks. -This app will make abusive requests to their servers, and CloudFlare will try to detect if PyNyaaTa is a real human or not. *I think you have the answer to the question ...* -Over time, CloudFlare will ask you systematically to prove yourself. -To be able to see YggTorrent results, you have to have a FlareSolverr instance running. -Please refer to their [documentation](https://github.com/FlareSolverr/FlareSolverr#installation). -After that, change the `CLOUDPROXY_ENDPOINT` environment variable to refer to your CloudProxy instance. - ## Links - Project homepage: https://nyaa.crystalyx.net/ diff --git a/get404.py b/get404.py index f183b9d..7eda786 100644 --- a/get404.py +++ b/get404.py @@ -4,7 +4,7 @@ from pynyaata.models import AnimeLink links = AnimeLink.query.all() for link in links: - html = curl_content(link.link, debug=False, cloudflare=True) + html = curl_content(link.link, debug=False) if html['http_code'] != 200 and html['http_code'] != 500: print('(%d) %s %s : %s' % ( diff --git a/pynyaata/__init__.py b/pynyaata/__init__.py index f60b1f3..9d6004d 100644 --- a/pynyaata/__init__.py +++ b/pynyaata/__init__.py @@ -109,7 +109,7 @@ def latest(page=1): @app.route('/list') @app.route('/list/') @db_required -def list_animes(url_filters='nyaa,yggtorrent'): +def list_animes(url_filters='nyaa'): filters = None for i, to_filter in enumerate(url_filters.split(',')): if not i: diff --git a/pynyaata/config.py b/pynyaata/config.py index 6d87431..62d0338 100644 --- a/pynyaata/config.py +++ b/pynyaata/config.py @@ -17,7 +17,6 @@ APP_PORT = int(environ.get('FLASK_PORT', 5000)) CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 60 * 60)) REQUESTS_TIMEOUT = int(environ.get('REQUESTS_TIMEOUT', 5)) BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',') if environ.get('BLACKLIST_WORDS', '') else [] -CLOUDPROXY_ENDPOINT = environ.get('CLOUDPROXY_ENDPOINT') DB_ENABLED = False REDIS_ENABLED = False TRANSMISSION_ENABLED = False diff --git a/pynyaata/connectors/__init__.py b/pynyaata/connectors/__init__.py index c86f9db..115853f 100644 --- a/pynyaata/connectors/__init__.py +++ b/pynyaata/connectors/__init__.py @@ -3,14 +3,11 @@ from asyncio import gather from .animeultime import AnimeUltime from .core import Other from .nyaa import Nyaa -from .yggtorrent import YggAnimation, YggTorrent async def run_all(*args, **kwargs): coroutines = [Nyaa(*args, **kwargs).run(), - AnimeUltime(*args, **kwargs).run(), - YggTorrent(*args, **kwargs).run(), - YggAnimation(*args, **kwargs).run()] + AnimeUltime(*args, **kwargs).run()] return list(await gather(*coroutines)) @@ -20,7 +17,5 @@ def get_instance(url, query=''): return Nyaa(query) elif 'anime-ultime' in url: return AnimeUltime(query) - elif 'ygg' in url: - return YggTorrent(query) else: return Other(query) diff --git a/pynyaata/connectors/core.py b/pynyaata/connectors/core.py index 4bb2625..d27c96d 100644 --- a/pynyaata/connectors/core.py +++ b/pynyaata/connectors/core.py @@ -7,13 +7,10 @@ from redis.exceptions import RedisError from requests import RequestException, Session from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger -from ..flarerequests import FlareRequests if REDIS_ENABLED: from ..config import cache -cloudproxy_session = None - class ConnectorReturn(Enum): SEARCH = 1 @@ -71,11 +68,11 @@ class Cache: ConnectorCache = Cache() -def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False): +def curl_content(url, params=None, ajax=False, debug=True): output = '' http_code = 500 method = 'post' if (params is not None) else 'get' - request = FlareRequests() if cloudflare else Session() + request = Session() headers = {} if ajax: diff --git a/pynyaata/connectors/yggtorrent.py b/pynyaata/connectors/yggtorrent.py deleted file mode 100644 index a6e1069..0000000 --- a/pynyaata/connectors/yggtorrent.py +++ /dev/null @@ -1,98 +0,0 @@ -import re -from datetime import datetime -from urllib.parse import quote - -from bs4 import BeautifulSoup - -from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content -from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db, parse_date - - -class YggTorrent(ConnectorCore): - color = 'is-success' - title = 'YggTorrent' - favicon = 'yggtorrent.png' - base_url = 'https://www3.yggtorrent.wtf' - is_light = False - category = 2179 - - def get_full_search_url(self): - sort_type = 'size' - if self.return_type is ConnectorReturn.HISTORY: - sort_type = 'publish_date' - sort_page = '&page=%s' % ( - (self.page - 1) * 50 - ) if self.page > 1 else '' - - return '%s/engine/search?name=%s&category=2145&sub_category=%s&do=search&order=desc&sort=%s%s' % ( - self.base_url, self.query, self.category, sort_type, sort_page - ) - - def get_history(self): - self.search() - - @ConnectorCache.cache_data - def search(self): - if self.category: - response = curl_content( - self.get_full_search_url(), cloudflare=True - ) - - if response['http_code'] == 200: - html = BeautifulSoup(response['output'], 'html.parser') - trs = html.select('table.table tr') - valid_trs = 0 - - for i, tr in enumerate(trs): - if not i: - continue - - tds = tr.findAll('td') - check_downloads = int(tds[6].get_text()) - check_seeds = int(tds[7].get_text()) - - if check_downloads or check_seeds: - url = tds[1].a - url_safe = url.get_text() - - if check_blacklist_words(url_safe): - continue - - valid_trs = valid_trs + 1 - - self.data.append({ - 'vf': check_if_vf(url_safe), - 'href': url['href'], - 'name': url_safe, - 'comment': '%s' % - (url['href'], tds[3].decode_contents()), - 'link': '' - '' - '' % (self.base_url, - re.search(r'/(\d+)', url['href']).group(1)), - 'size': tds[5].get_text(), - 'date': parse_date(datetime.fromtimestamp(int(tds[4].div.get_text()))), - 'seeds': check_seeds, - 'leechs': tds[8].get_text(), - 'downloads': check_downloads, - 'class': self.color if link_exist_in_db(quote(url['href'], '/+:')) else '' - }) - - self.on_error = False - self.is_more = valid_trs and valid_trs != len(trs) - 1 - - @ConnectorCache.cache_data - def is_vf(self, url): - response = curl_content(url) - - if response['http_code'] == 200: - html = BeautifulSoup(response['output'], 'html.parser') - title = html.select('#title h1') - return check_if_vf(title[0].get_text()) - - return False - - -class YggAnimation(YggTorrent): - title = 'YggAnimation' - category = 2178 diff --git a/pynyaata/flarerequests.py b/pynyaata/flarerequests.py deleted file mode 100644 index 5cf2f11..0000000 --- a/pynyaata/flarerequests.py +++ /dev/null @@ -1,70 +0,0 @@ -from io import BytesIO -from urllib import parse -from charset_normalizer import detect -from requests import RequestException, Response, Session, post -from .config import CLOUDPROXY_ENDPOINT - - -class FlareRequests(Session): - def request(self, method, url, params=None, data=None, **kwargs): - if not CLOUDPROXY_ENDPOINT: - return super().request(method, url, params, data, **kwargs) - - sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json() - - if "sessions" in sessions and len(sessions["sessions"]) > 0: - FLARE_SESSION = sessions["sessions"][0] - else: - response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"}) - session = response.json() - - if "session" in session: - FLARE_SESSION = session["session"] - else: - raise RequestException(response) - - if params: - url += "&" if len(url.split("?")) > 1 else "?" - url = f"{url}{parse.urlencode(params)}" - - post_data = { - "cmd": f"request.{method.lower()}", - "session": FLARE_SESSION, - "url": url, - } - - if data: - post_data["postData"] = parse.urlencode(data) - - try: - response = post( - CLOUDPROXY_ENDPOINT, - json=post_data, - ) - - content = response.json() - - if "solution" in content: - solution = content["solution"] - raw = solution["response"].encode() - encoding = detect(raw) - - resolved = Response() - resolved.status_code = solution["status"] - resolved.headers = solution["headers"] - resolved.raw = BytesIO(raw) - resolved.url = url - resolved.encoding = encoding["encoding"] - resolved.reason = content["status"] - resolved.cookies = solution["cookies"] - - return resolved - - raise RequestException(response) - except RequestException: - session = post( - CLOUDPROXY_ENDPOINT, - json={"cmd": "sessions.destroy", "session": FLARE_SESSION}, - ) - - raise RequestException(solution)