Reorder imports + readd FlareSolverr
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Michel Roux 2022-12-21 14:53:50 +00:00
parent 28b11257a4
commit 5b43615f02
12 changed files with 80 additions and 24 deletions

View File

@ -29,4 +29,3 @@ steps:
- master
event:
- push

View File

@ -27,7 +27,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it
## Features
* Search on [Nyaa.si](https://nyaa.si/), [Nyaa.net (codename Pantsu)](https://nyaa.net/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
* Search on [Nyaa.si](https://nyaa.si/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
* Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search
* Color official and bad links
* Add seeded links to a database
@ -40,6 +40,15 @@ All is managed by environment variables.
Please look into the `.env.dist` file to list all possible environment variables.
You have to have a running database server to be able to access the admin panel.
### Bypassing CloudFlare for YggTorrent
YggTorrent use CloudFlare to protect them to DDoS attacks.
This app will make abusive requests to their servers, and CloudFlare will try to detect if PyNyaaTa is a real human or not. *I think you have the answer to the question ...*
Over time, CloudFlare will ask you systematically to prove yourself.
To be able to see YggTorrent results, you have to have a FlareSolverr instance running.
Please refer to their [documentation](https://github.com/FlareSolverr/FlareSolverr#installation).
After that, change the `CLOUDPROXY_ENDPOINT` environment variable to refer to your CloudProxy instance.
## Links
- Project homepage: https://nyaa.crystalyx.net/

View File

@ -1,14 +1,14 @@
from asyncio import get_event_loop, set_event_loop, SelectorEventLoop
from asyncio import SelectorEventLoop, get_event_loop, set_event_loop
from functools import wraps
from operator import attrgetter, itemgetter
from flask import redirect, render_template, request, url_for, abort
from flask import abort, redirect, render_template, request, url_for
from . import utils
from .config import app, auth, ADMIN_USERNAME, ADMIN_PASSWORD, DB_ENABLED, APP_PORT, IS_DEBUG, TRANSMISSION_ENABLED
from .connectors import get_instance, run_all, Nyaa
from .config import ADMIN_PASSWORD, ADMIN_USERNAME, APP_PORT, DB_ENABLED, IS_DEBUG, TRANSMISSION_ENABLED, app, auth
from .connectors import Nyaa, get_instance, run_all
from .connectors.core import ConnectorLang, ConnectorReturn
from .forms import SearchForm, DeleteForm, EditForm, FolderDeleteForm, FolderEditForm
from .forms import DeleteForm, EditForm, FolderDeleteForm, FolderEditForm, SearchForm
if DB_ENABLED:
from .config import db

View File

@ -17,6 +17,7 @@ APP_PORT = int(environ.get('FLASK_PORT', 5000))
CACHE_TIMEOUT = int(environ.get('CACHE_TIMEOUT', 60 * 60))
REQUESTS_TIMEOUT = int(environ.get('REQUESTS_TIMEOUT', 5))
BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',') if environ.get('BLACKLIST_WORDS', '') else []
CLOUDPROXY_ENDPOINT = environ.get('CLOUDPROXY_ENDPOINT')
DB_ENABLED = False
REDIS_ENABLED = False
TRANSMISSION_ENABLED = False

View File

@ -3,7 +3,7 @@ from asyncio import gather
from .animeultime import AnimeUltime
from .core import Other
from .nyaa import Nyaa
from .yggtorrent import YggTorrent, YggAnimation
from .yggtorrent import YggAnimation, YggTorrent
async def run_all(*args, **kwargs):

View File

@ -2,8 +2,8 @@ from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
from ..utils import parse_date, link_exist_in_db
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import link_exist_in_db, parse_date
class AnimeUltime(ConnectorCore):

View File

@ -3,11 +3,11 @@ from enum import Enum
from functools import wraps
from json import dumps, loads
import requests
from requests import RequestException
from redis.exceptions import RedisError
from requests import RequestException, Session
from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger
from ..flarerequests import FlareRequests
if REDIS_ENABLED:
from ..config import cache
@ -75,24 +75,22 @@ def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
output = ''
http_code = 500
method = 'post' if (params is not None) else 'get'
request = FlareRequests() if cloudflare else Session()
headers = {}
if ajax:
headers['X-Requested-With'] = 'XMLHttpRequest'
if cloudflare:
headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
try:
if method == 'post':
response = requests.post(
response = request.post(
url,
params,
timeout=REQUESTS_TIMEOUT,
headers=headers
)
else:
response = requests.get(
response = request.get(
url,
timeout=REQUESTS_TIMEOUT,
headers=headers

View File

@ -1,7 +1,7 @@
from bs4 import BeautifulSoup
from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
from ..utils import link_exist_in_db, check_blacklist_words, check_if_vf
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db
class Nyaa(ConnectorCore):

View File

@ -4,8 +4,8 @@ from urllib.parse import quote
from bs4 import BeautifulSoup
from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db, parse_date
class YggTorrent(ConnectorCore):

49
pynyaata/flarerequests.py Normal file
View File

@ -0,0 +1,49 @@
from urllib import parse
from requests import RequestException, Session, post
from .config import CLOUDPROXY_ENDPOINT
class FlareRequests(Session):
def request(self, method, url, params):
if not CLOUDPROXY_ENDPOINT:
return super().request(method, url, params)
sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
if "sessions" in sessions and len(sessions["sessions"]) > 0:
FLARESESSION = sessions["sessions"][0]
else:
response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
session = response.json()
if "session" in session:
FLARESESSION = session["session"]
else:
raise RequestException(response)
try:
response = post(
CLOUDPROXY_ENDPOINT,
json={
"cmd": f"request.{method.lower()}",
"session": FLARESESSION,
"url": url,
"postData": parse.urlencode(params),
},
)
solution = response.json()
if "solution" in solution:
response.cookies = solution["solution"]["cookies"]
response.headers = solution["solution"]["headers"]
response.text = solution["solution"]["response"]
return response
raise RequestException(response)
except RequestException:
session = post(
CLOUDPROXY_ENDPOINT, {"cmd": "sessions.destroy", "session": FLARESESSION}
)
raise RequestException(solution)

View File

@ -1,5 +1,5 @@
from flask_wtf import FlaskForm
from wtforms import HiddenField, StringField, SelectField
from wtforms import HiddenField, SelectField, StringField
from wtforms.fields.html5 import SearchField, URLField
from wtforms.validators import DataRequired

View File

@ -2,7 +2,7 @@ import re
from datetime import datetime
from dateparser import parse
from .config import DB_ENABLED, BLACKLIST_WORDS
from .config import BLACKLIST_WORDS, DB_ENABLED
def link_exist_in_db(href):