Big Update ! PyNyaaTa doesn't require MySQL anymore /o/
All checks were successful
continuous-integration/drone/push Build is passing

This commit is contained in:
Michel Roux 2020-04-09 14:02:05 +02:00
parent 9b6cae5dc2
commit e5e1ebf875
8 changed files with 112 additions and 85 deletions

View File

@ -13,20 +13,16 @@ After a good rewrite in Python, it's time to show it to the public, and here it
### With Docker ### With Docker
- Install Docker: https://hub.docker.com/search/?type=edition&offering=community - Install Docker: https://hub.docker.com/search/?type=edition&offering=community
- Be sure to have a MySQL Server installed and running - Run `docker run -p 5000 xefir/pynyaata`
- Create a .env like [this one](.env.dist)
- Run `docker run --env-file .env -p 5000 xefir/pynyaata`
- The app is accessible at http://localhost:5000 - The app is accessible at http://localhost:5000
### Without Docker ### Without Docker
- Install Python 3: https://www.python.org/downloads/ - Install Python 3: https://www.python.org/downloads/
- Install Pip: https://pip.pypa.io/en/stable/installing/ - Install Pip: https://pip.pypa.io/en/stable/installing/
- Install MariaDB (or any MySQL server): https://mariadb.com/downloads/
- Clone this repository - Clone this repository
- Launch a terminal and move into the root of the cloned repository - Launch a terminal and move into the root of the cloned repository
- Run `pip install -r requirements.txt` - Run `pip install -r requirements.txt`
- Copy the `.env.dist` file to `.env` and ajust values to point to your MySQL server
- Run `python3 app.py` - Run `python3 app.py`
- The app is accessible at http://localhost:5000 - The app is accessible at http://localhost:5000
@ -43,6 +39,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it
All is managed by environment variables. All is managed by environment variables.
Please look into the `.env.dist` file to list all env variables possible. Please look into the `.env.dist` file to list all env variables possible.
You have to install MariaDB (or any MySQL server) to be able to access the admin panel.
## Links ## Links

50
app.py
View File

@ -1,11 +1,35 @@
from operator import attrgetter, itemgetter from operator import attrgetter, itemgetter
from time import sleep from time import sleep
from flask import redirect, render_template, request, url_for from flask import redirect, render_template, request, url_for, abort
from config import app, auth, db, ADMIN_USERNAME, ADMIN_PASSWORD, APP_PORT from config import app, auth, ADMIN_USERNAME, ADMIN_PASSWORD, APP_PORT
from connectors import * from connectors import *
from models import AnimeFolder, AnimeTitle, DeleteForm, SearchForm, EditForm from forms import SearchForm, DeleteForm, EditForm
if MYSQL_ENABLED:
from config import db
from models import AnimeFolder, AnimeTitle, AnimeLink
def clean_model(obj):
for attr in dir(obj):
if not attr.startswith('_') and getattr(obj, attr) is None:
try:
setattr(obj, attr, '')
except AttributeError:
pass
return obj
def mysql_required(f):
@wraps(f)
def decorated_function(*args, **kwargs):
if not MYSQL_ENABLED:
return abort(404)
return f(*args, **kwargs)
return decorated_function
@auth.verify_password @auth.verify_password
@ -17,6 +41,7 @@ def verify_password(username, password):
def boldify(name): def boldify(name):
query = request.args.get('q', '') query = request.args.get('q', '')
name = Connector.boldify(name, query) name = Connector.boldify(name, query)
if MYSQL_ENABLED:
for keyword in db.session.query(AnimeTitle.keyword.distinct()).all(): for keyword in db.session.query(AnimeTitle.keyword.distinct()).all():
if keyword[0].lower() != query.lower(): if keyword[0].lower() != query.lower():
name = Connector.boldify(name, keyword[0]) name = Connector.boldify(name, keyword[0])
@ -52,7 +77,8 @@ def search():
AnimeUltime(query).run(), AnimeUltime(query).run(),
] ]
return render_template('search.html', search_form=SearchForm(), connectors=results) return render_template('search.html', search_form=SearchForm(), connectors=results,
mysql_disabled=not MYSQL_ENABLED)
@app.route('/latest') @app.route('/latest')
@ -73,11 +99,13 @@ def latest(page=1):
result['self'] = Connector.get_instance(result['href'], '') result['self'] = Connector.get_instance(result['href'], '')
results.sort(key=itemgetter('date'), reverse=True) results.sort(key=itemgetter('date'), reverse=True)
return render_template('latest.html', search_form=SearchForm(), torrents=results, page=page) return render_template('latest.html', search_form=SearchForm(), torrents=results, page=page,
mysql_disabled=not MYSQL_ENABLED)
@app.route('/list') @app.route('/list')
@app.route('/list/<url_filters>') @app.route('/list/<url_filters>')
@mysql_required
def list_animes(url_filters='nyaa,yggtorrent'): def list_animes(url_filters='nyaa,yggtorrent'):
filters = None filters = None
for i, to_filter in enumerate(url_filters.split(',')): for i, to_filter in enumerate(url_filters.split(',')):
@ -99,6 +127,7 @@ def list_animes(url_filters='nyaa,yggtorrent'):
@app.route('/admin', methods=['GET', 'POST']) @app.route('/admin', methods=['GET', 'POST'])
@mysql_required
@auth.login_required @auth.login_required
def admin(): def admin():
folders = AnimeFolder.query.all() folders = AnimeFolder.query.all()
@ -128,18 +157,9 @@ def admin():
return render_template('admin/list.html', search_form=SearchForm(), folders=folders, action_form=form) return render_template('admin/list.html', search_form=SearchForm(), folders=folders, action_form=form)
def clean_model(obj):
for attr in dir(obj):
if not attr.startswith('_') and getattr(obj, attr) is None:
try:
setattr(obj, attr, '')
except AttributeError:
pass
return obj
@app.route('/admin/edit', methods=['GET', 'POST']) @app.route('/admin/edit', methods=['GET', 'POST'])
@app.route('/admin/edit/<int:link_id>', methods=['GET', 'POST']) @app.route('/admin/edit/<int:link_id>', methods=['GET', 'POST'])
@mysql_required
@auth.login_required @auth.login_required
def admin_edit(link_id=None): def admin_edit(link_id=None):
folders = AnimeFolder.query.all() folders = AnimeFolder.query.all()

View File

@ -5,32 +5,34 @@ from flask.cli import load_dotenv
from flask_httpauth import HTTPBasicAuth from flask_httpauth import HTTPBasicAuth
from flask_sqlalchemy import SQLAlchemy from flask_sqlalchemy import SQLAlchemy
# init DB and migration
load_dotenv() load_dotenv()
db_user = environ.get('MYSQL_USER')
db_password = environ.get('MYSQL_PASSWORD')
db_name = environ.get('MYSQL_DATABASE')
db_host = environ.get('MYSQL_SERVER')
if not db_host or not db_user or not db_password or not db_name:
print('Missing connection environment variables')
exit()
# load app constants
IS_DEBUG = environ.get('FLASK_ENV', 'production') == 'development' IS_DEBUG = environ.get('FLASK_ENV', 'production') == 'development'
ADMIN_USERNAME = environ.get('ADMIN_USERNAME', 'admin') ADMIN_USERNAME = environ.get('ADMIN_USERNAME', 'admin')
ADMIN_PASSWORD = environ.get('ADMIN_PASSWORD', 'secret') ADMIN_PASSWORD = environ.get('ADMIN_PASSWORD', 'secret')
APP_PORT = environ.get('FLASK_PORT', 5000) APP_PORT = environ.get('FLASK_PORT', 5000)
CACHE_TIMEOUT = environ.get('CACHE_TIMEOUT', 60 * 60) CACHE_TIMEOUT = environ.get('CACHE_TIMEOUT', 60 * 60)
BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',') BLACKLIST_WORDS = environ.get('BLACKLIST_WORDS', '').split(',')
MYSQL_ENABLED = False
app = Flask(__name__) app = Flask(__name__)
app.name = 'PyNyaaTa' app.name = 'PyNyaaTa'
app.secret_key = urandom(24).hex() app.secret_key = urandom(24).hex()
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://%s:%s@%s/%s?charset=utf8mb4' % (
db_user, db_password, db_host, db_name
)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SQLALCHEMY_ECHO'] = IS_DEBUG
app.url_map.strict_slashes = False app.url_map.strict_slashes = False
auth = HTTPBasicAuth() auth = HTTPBasicAuth()
db = SQLAlchemy(app)
db_host = environ.get('MYSQL_SERVER')
if db_host:
MYSQL_ENABLED = True
db_user = environ.get('MYSQL_USER')
db_password = environ.get('MYSQL_PASSWORD')
db_name = environ.get('MYSQL_DATABASE')
if not db_user or not db_password or not db_name:
print('Missing connection environment variables')
exit()
app.config['SQLALCHEMY_DATABASE_URI'] = 'mysql+pymysql://%s:%s@%s/%s?charset=utf8mb4' % (
db_user, db_password, db_host, db_name
)
app.config['SQLALCHEMY_TRACK_MODIFICATIONS'] = True
app.config['SQLALCHEMY_ECHO'] = IS_DEBUG
db = SQLAlchemy(app)

View File

@ -11,8 +11,7 @@ from bs4 import BeautifulSoup
from cloudscraper import create_scraper from cloudscraper import create_scraper
from requests import RequestException from requests import RequestException
from config import IS_DEBUG, CACHE_TIMEOUT, BLACKLIST_WORDS from config import IS_DEBUG, MYSQL_ENABLED, CACHE_TIMEOUT, BLACKLIST_WORDS
from models import AnimeLink
scraper = create_scraper() scraper = create_scraper()
@ -102,6 +101,13 @@ def curl_content(url, params=None, ajax=False):
return {'http_code': http_code, 'output': output} return {'http_code': http_code, 'output': output}
def link_exist_in_db(href):
if MYSQL_ENABLED:
from models import AnimeLink
return AnimeLink.query.filter_by(link=href).first()
return False
class Connector(ABC): class Connector(ABC):
@property @property
@abstractmethod @abstractmethod
@ -250,8 +256,7 @@ class Nyaa(Connector):
'seeds': check_seeds, 'seeds': check_seeds,
'leechs': tds[6].string, 'leechs': tds[6].string,
'downloads': check_downloads, 'downloads': check_downloads,
'class': self.color if AnimeLink.query.filter_by(link=href).first() else 'is-%s' % 'class': self.color if link_exist_in_db(href) else 'is-%s' % tr['class'][0]
tr['class'][0]
}) })
self.on_error = False self.on_error = False
@ -331,8 +336,7 @@ class Pantsu(Connector):
'seeds': check_seeds, 'seeds': check_seeds,
'leechs': tds[5].string, 'leechs': tds[5].string,
'downloads': check_downloads, 'downloads': check_downloads,
'class': self.color if AnimeLink.query.filter_by(link=href).first() else 'is-%s' % 'class': self.color if link_exist_in_db(href) else 'is-%s' % tr['class'][0]
tr['class'][0]
}) })
self.on_error = False self.on_error = False
@ -401,9 +405,7 @@ class YggTorrent(Connector):
'seeds': check_seeds, 'seeds': check_seeds,
'leechs': tds[8].string, 'leechs': tds[8].string,
'downloads': check_downloads, 'downloads': check_downloads,
'class': self.color if AnimeLink.query.filter_by( 'class': self.color if link_exist_in_db(quote(url['href'], '/+:')) else ''
link=quote(url['href'], '/+:')
).first() else ''
}) })
self.on_error = False self.on_error = False
@ -465,7 +467,7 @@ class AnimeUltime(Connector):
'name': url.get_text(), 'name': url.get_text(),
'type': tds[1].string, 'type': tds[1].string,
'date': datetime.fromtimestamp(0), 'date': datetime.fromtimestamp(0),
'class': self.color if AnimeLink.query.filter_by(link=href).first() else '' 'class': self.color if link_exist_in_db(href) else ''
}) })
else: else:
player = html.select('div.AUVideoPlayer') player = html.select('div.AUVideoPlayer')
@ -479,7 +481,7 @@ class AnimeUltime(Connector):
'name': name[0].string, 'name': name[0].string,
'type': ani_type[0].string.replace(':', ''), 'type': ani_type[0].string.replace(':', ''),
'date': datetime.fromtimestamp(0), 'date': datetime.fromtimestamp(0),
'class': self.color if AnimeLink.query.filter_by(link=href).first() else '' 'class': self.color if link_exist_in_db(href) else ''
}) })
self.on_error = False self.on_error = False
@ -513,7 +515,7 @@ class AnimeUltime(Connector):
'name': link.string, 'name': link.string,
'type': tds[4].string, 'type': tds[4].string,
'date': release_date, 'date': release_date,
'class': self.color if AnimeLink.query.filter_by(link=href).first() else '' 'class': self.color if link_exist_in_db(href) else ''
}) })
self.on_error = False self.on_error = False

26
forms.py Normal file
View File

@ -0,0 +1,26 @@
from flask_wtf import FlaskForm
from wtforms import BooleanField, HiddenField, StringField
from wtforms.fields.html5 import SearchField, URLField
from wtforms.validators import DataRequired
class SearchForm(FlaskForm):
q = SearchField('search', validators=[DataRequired()])
class DeleteForm(FlaskForm):
class Meta:
csrf = False
id = HiddenField('id', validators=[DataRequired()])
class EditForm(FlaskForm):
id = HiddenField('id')
folder = StringField('folder', validators=[DataRequired()])
name = StringField('name', validators=[DataRequired()])
link = URLField('link', validators=[DataRequired()])
season = StringField('season', validators=[DataRequired()])
comment = StringField('comment')
keyword = StringField('keyword')
is_vf = BooleanField('is_vf')

View File

@ -1,8 +1,3 @@
from flask_wtf import FlaskForm
from wtforms import BooleanField, HiddenField, StringField
from wtforms.fields.html5 import SearchField, URLField
from wtforms.validators import DataRequired
from config import db from config import db
@ -29,26 +24,4 @@ class AnimeLink(db.Model):
title_id = db.Column(db.Integer, db.ForeignKey('anime_title.id')) title_id = db.Column(db.Integer, db.ForeignKey('anime_title.id'))
class SearchForm(FlaskForm):
q = SearchField('search', validators=[DataRequired()])
class DeleteForm(FlaskForm):
class Meta:
csrf = False
id = HiddenField('id', validators=[DataRequired()])
class EditForm(FlaskForm):
id = HiddenField('id')
folder = StringField('folder', validators=[DataRequired()])
name = StringField('name', validators=[DataRequired()])
link = URLField('link', validators=[DataRequired()])
season = StringField('season', validators=[DataRequired()])
comment = StringField('comment')
keyword = StringField('keyword')
is_vf = BooleanField('is_vf')
db.create_all() db.create_all()

View File

@ -2,7 +2,8 @@
{% block title %}- Admin List{% endblock %} {% block title %}- Admin List{% endblock %}
{% block add_button %} {% block add_button %}
<a class="navbar-item has-tooltip-bottom" data-tooltip="Add entry" href="{{ url_for('admin_edit') }}"> <a class="navbar-item has-tooltip-bottom" data-tooltip="Add entry" href="{{ url_for('admin_edit') }}">
<i class="fa fa-plus"></i><i>&nbsp;</i><span class="is-hidden-mobile">Add entry</span> <i class="fa fa-plus"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Add entry</span>
</a> </a>
{% endblock %} {% endblock %}
{% block body %} {% block body %}

View File

@ -20,22 +20,28 @@
𝛑 😼 た 𝛑 😼 た
</a> </a>
<a class="navbar-item has-tooltip-bottom" data-tooltip="Latest torrents" href="{{ url_for('latest') }}"> <a class="navbar-item has-tooltip-bottom" data-tooltip="Latest torrents" href="{{ url_for('latest') }}">
<i class="fa fa-newspaper-o"></i><i>&nbsp;</i><span class="is-hidden-mobile">Latest torrents</span> <i class="fa fa-newspaper-o"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Latest torrents</span>
</a> </a>
{% if mysql_disabled %}
<a class="navbar-item has-tooltip-bottom" data-tooltip="My seeded torrents" <a class="navbar-item has-tooltip-bottom" data-tooltip="My seeded torrents"
href="{{ url_for('list_animes') }}"> href="{{ url_for('list_animes') }}">
<i class="fa fa-cloud-download"></i><i>&nbsp;</i><span class="is-hidden-mobile">My seeded torrents</span> <i class="fa fa-cloud-download"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">My seeded torrents</span>
</a> </a>
{% endif %}
{% block add_button %}{% endblock %} {% block add_button %}{% endblock %}
{% if request.args.get('q') %} {% if request.args.get('q') %}
<a class="navbar-item has-tooltip-bottom" data-tooltip="TVDB" <a class="navbar-item has-tooltip-bottom" data-tooltip="TVDB"
href="https://www.thetvdb.com/search?menu%5Btype%5D=TV&query={{ request.args.get('q') }}" href="https://www.thetvdb.com/search?menu%5Btype%5D=TV&query={{ request.args.get('q') }}"
target="_blank"> target="_blank">
<i class="fa fa-television"></i><i>&nbsp;</i><span class="is-hidden-mobile">TVDB</span> <i class="fa fa-television"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">TVDB</span>
</a> </a>
<a class="navbar-item has-tooltip-bottom" data-tooltip="Nautiljon" <a class="navbar-item has-tooltip-bottom" data-tooltip="Nautiljon"
href="https://www.nautiljon.com/search.php?q={{ request.args.get('q') }}" target="_blank"> href="https://www.nautiljon.com/search.php?q={{ request.args.get('q') }}" target="_blank">
<i class="fa fa-rss"></i><i>&nbsp;</i><span class="is-hidden-mobile">Nautiljon</span> <i class="fa fa-rss"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Nautiljon</span>
</a> </a>
{% endif %} {% endif %}
</div> </div>