Reorder imports + readd FlareSolverr

.drone.yml

@@ -29,4 +29,3 @@ steps:
         - master
       event:
         - push
-

README.md

@@ -27,7 +27,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it
 
 ## Features
 
-* Search on [Nyaa.si](https://nyaa.si/), [Nyaa.net (codename Pantsu)](https://nyaa.net/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
+* Search on [Nyaa.si](https://nyaa.si/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
 * Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search
 * Color official and bad links
 * Add seeded links to a database
@@ -40,6 +40,15 @@ All is managed by environment variables.
 Please look into the `.env.dist` file to list all possible environment variables.
 You have to have a running database server to be able to access the admin panel.
 
+### Bypassing CloudFlare for YggTorrent
+
+YggTorrent use CloudFlare to protect them to DDoS attacks.
+This app will make abusive requests to their servers, and CloudFlare will try to detect if PyNyaaTa is a real human or not. *I think you have the answer to the question ...*
+Over time, CloudFlare will ask you systematically to prove yourself.
+To be able to see YggTorrent results, you have to have a FlareSolverr instance running.
+Please refer to their [documentation](https://github.com/FlareSolverr/FlareSolverr#installation).
+After that, change the `CLOUDPROXY_ENDPOINT` environment variable to refer to your CloudProxy instance.
+
 ## Links
 
 - Project homepage: https://nyaa.crystalyx.net/

pynyaata/__init__.py

@@ -1,14 +1,14 @@
-from asyncio import get_event_loop, set_event_loop, SelectorEventLoop
+from asyncio import SelectorEventLoop, get_event_loop, set_event_loop
 from functools import wraps
 from operator import attrgetter, itemgetter
 
-from flask import redirect, render_template, request, url_for, abort
+from flask import abort, redirect, render_template, request, url_for
 
 from . import utils
-from .config import app, auth, ADMIN_USERNAME, ADMIN_PASSWORD, DB_ENABLED, APP_PORT, IS_DEBUG, TRANSMISSION_ENABLED
+from .config import ADMIN_PASSWORD, ADMIN_USERNAME, APP_PORT, DB_ENABLED, IS_DEBUG, TRANSMISSION_ENABLED, app, auth
-from .connectors import get_instance, run_all, Nyaa
+from .connectors import Nyaa, get_instance, run_all
 from .connectors.core import ConnectorLang, ConnectorReturn
-from .forms import SearchForm, DeleteForm, EditForm, FolderDeleteForm, FolderEditForm
+from .forms import DeleteForm, EditForm, FolderDeleteForm, FolderEditForm, SearchForm
 
 if DB_ENABLED:
     from .config import db

pynyaata/config.py

@@ -17,6 +17,7 @@ APP_PORT = int(environ.get('FLASK_PORT', 5000))
 CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 60 * 60))
 REQUESTS_TIMEOUT = int(environ.get('REQUESTS_TIMEOUT', 5))
 BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',') if environ.get('BLACKLIST_WORDS', '') else []
+CLOUDPROXY_ENDPOINT = environ.get('CLOUDPROXY_ENDPOINT')
 DB_ENABLED = False
 REDIS_ENABLED = False
 TRANSMISSION_ENABLED = False

pynyaata/connectors/__init__.py

@@ -3,7 +3,7 @@ from asyncio import gather
 from .animeultime import AnimeUltime
 from .core import Other
 from .nyaa import Nyaa
-from .yggtorrent import YggTorrent, YggAnimation
+from .yggtorrent import YggAnimation, YggTorrent
 
 
 async def run_all(*args, **kwargs):

pynyaata/connectors/animeultime.py

@@ -2,8 +2,8 @@ from datetime import datetime, timedelta
 
 from bs4 import BeautifulSoup
 
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
+from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
-from ..utils import parse_date, link_exist_in_db
+from ..utils import link_exist_in_db, parse_date
 
 
 class AnimeUltime(ConnectorCore):

pynyaata/connectors/core.py

@@ -3,11 +3,11 @@ from enum import Enum
 from functools import wraps
 from json import dumps, loads
 
-import requests
-from requests import RequestException
 from redis.exceptions import RedisError
+from requests import RequestException, Session
 
-from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
+from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger
+from ..flarerequests import FlareRequests
 
 if REDIS_ENABLED:
     from ..config import cache
@@ -75,24 +75,22 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
     output = ''
     http_code = 500
     method = 'post' if (params is not None) else 'get'
+    request = FlareRequests() if cloudflare else Session()
     headers = {}
 
     if ajax:
         headers['X-Requested-With'] = 'XMLHttpRequest'
 
-    if cloudflare:
-        headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
-
     try:
         if method == 'post':
-            response = requests.post(
+            response = request.post(
                 url,
                 params,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers
             )
         else:
-            response = requests.get(
+            response = request.get(
                 url,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers

pynyaata/connectors/nyaa.py

@@ -1,7 +1,7 @@
 from bs4 import BeautifulSoup
 
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
+from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
-from ..utils import link_exist_in_db, check_blacklist_words, check_if_vf
+from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db
 
 
 class Nyaa(ConnectorCore):

pynyaata/connectors/yggtorrent.py

@@ -4,8 +4,8 @@ from urllib.parse import quote
 
 from bs4 import BeautifulSoup
 
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
+from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
-from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf
+from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db, parse_date
 
 
 class YggTorrent(ConnectorCore):

pynyaata/flarerequests.py

@@ -0,0 +1,49 @@
+from urllib import parse
+from requests import RequestException, Session, post
+from .config import CLOUDPROXY_ENDPOINT
+
+
+class FlareRequests(Session):
+    def request(self, method, url, params):
+        if not CLOUDPROXY_ENDPOINT:
+            return super().request(method, url, params)
+
+        sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
+
+        if "sessions" in sessions and len(sessions["sessions"]) > 0:
+            FLARESESSION = sessions["sessions"][0]
+        else:
+            response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
+            session = response.json()
+
+            if "session" in session:
+                FLARESESSION = session["session"]
+            else:
+                raise RequestException(response)
+
+        try:
+            response = post(
+                CLOUDPROXY_ENDPOINT,
+                json={
+                    "cmd": f"request.{method.lower()}",
+                    "session": FLARESESSION,
+                    "url": url,
+                    "postData": parse.urlencode(params),
+                },
+            )
+            solution = response.json()
+
+            if "solution" in solution:
+                response.cookies = solution["solution"]["cookies"]
+                response.headers = solution["solution"]["headers"]
+                response.text = solution["solution"]["response"]
+
+                return response
+
+            raise RequestException(response)
+        except RequestException:
+            session = post(
+                CLOUDPROXY_ENDPOINT, {"cmd": "sessions.destroy", "session": FLARESESSION}
+            )
+
+            raise RequestException(solution)

pynyaata/forms.py

@@ -1,5 +1,5 @@
 from flask_wtf import FlaskForm
-from wtforms import HiddenField, StringField, SelectField
+from wtforms import HiddenField, SelectField, StringField
 from wtforms.fields.html5 import SearchField, URLField
 from wtforms.validators import DataRequired
 

pynyaata/utils.py

@@ -2,7 +2,7 @@ import re
 from datetime import datetime
 from dateparser import parse
 
-from .config import DB_ENABLED, BLACKLIST_WORDS
+from .config import BLACKLIST_WORDS, DB_ENABLED
 
 
 def link_exist_in_db(href):