diff --git a/.drone.yml b/.drone.yml index b455a4e..d5467ad 100644 --- a/.drone.yml +++ b/.drone.yml @@ -29,4 +29,3 @@ steps: - master event: - push - diff --git a/README.md b/README.md index e04d58b..21f720e 100644 --- a/README.md +++ b/README.md @@ -27,7 +27,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it ## Features -* Search on [Nyaa.si](https://nyaa.si/), [Nyaa.net (codename Pantsu)](https://nyaa.net/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1) +* Search on [Nyaa.si](https://nyaa.si/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1) * Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search * Color official and bad links * Add seeded links to a database @@ -40,6 +40,15 @@ All is managed by environment variables. Please look into the `.env.dist` file to list all possible environment variables. You have to have a running database server to be able to access the admin panel. +### Bypassing CloudFlare for YggTorrent + +YggTorrent use CloudFlare to protect them to DDoS attacks. +This app will make abusive requests to their servers, and CloudFlare will try to detect if PyNyaaTa is a real human or not. *I think you have the answer to the question ...* +Over time, CloudFlare will ask you systematically to prove yourself. +To be able to see YggTorrent results, you have to have a FlareSolverr instance running. +Please refer to their [documentation](https://github.com/FlareSolverr/FlareSolverr#installation). +After that, change the `CLOUDPROXY_ENDPOINT` environment variable to refer to your CloudProxy instance. + ## Links - Project homepage: https://nyaa.crystalyx.net/ diff --git a/pynyaata/__init__.py b/pynyaata/__init__.py index e825ad9..f60b1f3 100644 --- a/pynyaata/__init__.py +++ b/pynyaata/__init__.py @@ -1,14 +1,14 @@ -from asyncio import get_event_loop, set_event_loop, SelectorEventLoop +from asyncio import SelectorEventLoop, get_event_loop, set_event_loop from functools import wraps from operator import attrgetter, itemgetter -from flask import redirect, render_template, request, url_for, abort +from flask import abort, redirect, render_template, request, url_for from . import utils -from .config import app, auth, ADMIN_USERNAME, ADMIN_PASSWORD, DB_ENABLED, APP_PORT, IS_DEBUG, TRANSMISSION_ENABLED -from .connectors import get_instance, run_all, Nyaa +from .config import ADMIN_PASSWORD, ADMIN_USERNAME, APP_PORT, DB_ENABLED, IS_DEBUG, TRANSMISSION_ENABLED, app, auth +from .connectors import Nyaa, get_instance, run_all from .connectors.core import ConnectorLang, ConnectorReturn -from .forms import SearchForm, DeleteForm, EditForm, FolderDeleteForm, FolderEditForm +from .forms import DeleteForm, EditForm, FolderDeleteForm, FolderEditForm, SearchForm if DB_ENABLED: from .config import db diff --git a/pynyaata/config.py b/pynyaata/config.py index 62d0338..6d87431 100644 --- a/pynyaata/config.py +++ b/pynyaata/config.py @@ -17,6 +17,7 @@ APP_PORT = int(environ.get('FLASK_PORT', 5000)) CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 60 * 60)) REQUESTS_TIMEOUT = int(environ.get('REQUESTS_TIMEOUT', 5)) BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',') if environ.get('BLACKLIST_WORDS', '') else [] +CLOUDPROXY_ENDPOINT = environ.get('CLOUDPROXY_ENDPOINT') DB_ENABLED = False REDIS_ENABLED = False TRANSMISSION_ENABLED = False diff --git a/pynyaata/connectors/__init__.py b/pynyaata/connectors/__init__.py index ce94578..c86f9db 100644 --- a/pynyaata/connectors/__init__.py +++ b/pynyaata/connectors/__init__.py @@ -3,7 +3,7 @@ from asyncio import gather from .animeultime import AnimeUltime from .core import Other from .nyaa import Nyaa -from .yggtorrent import YggTorrent, YggAnimation +from .yggtorrent import YggAnimation, YggTorrent async def run_all(*args, **kwargs): diff --git a/pynyaata/connectors/animeultime.py b/pynyaata/connectors/animeultime.py index 49a3913..54996a7 100644 --- a/pynyaata/connectors/animeultime.py +++ b/pynyaata/connectors/animeultime.py @@ -2,8 +2,8 @@ from datetime import datetime, timedelta from bs4 import BeautifulSoup -from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import parse_date, link_exist_in_db +from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content +from ..utils import link_exist_in_db, parse_date class AnimeUltime(ConnectorCore): diff --git a/pynyaata/connectors/core.py b/pynyaata/connectors/core.py index 4645e31..4bb2625 100644 --- a/pynyaata/connectors/core.py +++ b/pynyaata/connectors/core.py @@ -3,11 +3,11 @@ from enum import Enum from functools import wraps from json import dumps, loads -import requests -from requests import RequestException from redis.exceptions import RedisError +from requests import RequestException, Session -from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED +from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger +from ..flarerequests import FlareRequests if REDIS_ENABLED: from ..config import cache @@ -75,24 +75,22 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False): output = '' http_code = 500 method = 'post' if (params is not None) else 'get' + request = FlareRequests() if cloudflare else Session() headers = {} if ajax: headers['X-Requested-With'] = 'XMLHttpRequest' - if cloudflare: - headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)' - try: if method == 'post': - response = requests.post( + response = request.post( url, params, timeout=REQUESTS_TIMEOUT, headers=headers ) else: - response = requests.get( + response = request.get( url, timeout=REQUESTS_TIMEOUT, headers=headers diff --git a/pynyaata/connectors/nyaa.py b/pynyaata/connectors/nyaa.py index b118b98..67f4b84 100644 --- a/pynyaata/connectors/nyaa.py +++ b/pynyaata/connectors/nyaa.py @@ -1,7 +1,7 @@ from bs4 import BeautifulSoup -from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import link_exist_in_db, check_blacklist_words, check_if_vf +from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content +from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db class Nyaa(ConnectorCore): diff --git a/pynyaata/connectors/yggtorrent.py b/pynyaata/connectors/yggtorrent.py index 4b8c621..0430db5 100644 --- a/pynyaata/connectors/yggtorrent.py +++ b/pynyaata/connectors/yggtorrent.py @@ -4,8 +4,8 @@ from urllib.parse import quote from bs4 import BeautifulSoup -from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf +from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content +from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db, parse_date class YggTorrent(ConnectorCore): diff --git a/pynyaata/flarerequests.py b/pynyaata/flarerequests.py new file mode 100644 index 0000000..71ca052 --- /dev/null +++ b/pynyaata/flarerequests.py @@ -0,0 +1,49 @@ +from urllib import parse +from requests import RequestException, Session, post +from .config import CLOUDPROXY_ENDPOINT + + +class FlareRequests(Session): + def request(self, method, url, params): + if not CLOUDPROXY_ENDPOINT: + return super().request(method, url, params) + + sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json() + + if "sessions" in sessions and len(sessions["sessions"]) > 0: + FLARESESSION = sessions["sessions"][0] + else: + response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"}) + session = response.json() + + if "session" in session: + FLARESESSION = session["session"] + else: + raise RequestException(response) + + try: + response = post( + CLOUDPROXY_ENDPOINT, + json={ + "cmd": f"request.{method.lower()}", + "session": FLARESESSION, + "url": url, + "postData": parse.urlencode(params), + }, + ) + solution = response.json() + + if "solution" in solution: + response.cookies = solution["solution"]["cookies"] + response.headers = solution["solution"]["headers"] + response.text = solution["solution"]["response"] + + return response + + raise RequestException(response) + except RequestException: + session = post( + CLOUDPROXY_ENDPOINT, {"cmd": "sessions.destroy", "session": FLARESESSION} + ) + + raise RequestException(solution) diff --git a/pynyaata/forms.py b/pynyaata/forms.py index 1b20f97..2a7813e 100644 --- a/pynyaata/forms.py +++ b/pynyaata/forms.py @@ -1,5 +1,5 @@ from flask_wtf import FlaskForm -from wtforms import HiddenField, StringField, SelectField +from wtforms import HiddenField, SelectField, StringField from wtforms.fields.html5 import SearchField, URLField from wtforms.validators import DataRequired diff --git a/pynyaata/utils.py b/pynyaata/utils.py index 55099a9..34ea733 100644 --- a/pynyaata/utils.py +++ b/pynyaata/utils.py @@ -2,7 +2,7 @@ import re from datetime import datetime from dateparser import parse -from .config import DB_ENABLED, BLACKLIST_WORDS +from .config import BLACKLIST_WORDS, DB_ENABLED def link_exist_in_db(href):