From c2c3e3087969a1a44ee0256fdf80239083e31aed Mon Sep 17 00:00:00 2001 From: Michel Roux Date: Sat, 30 Jan 2021 19:40:36 +0100 Subject: [PATCH] VF is auto now (refacto) --- pynyaata/__init__.py | 6 ++++-- pynyaata/connectors/__init__.py | 2 +- pynyaata/connectors/animeultime.py | 12 ++++++++---- pynyaata/connectors/core.py | 12 ++++-------- pynyaata/connectors/nyaa.py | 15 +++++++++++++-- pynyaata/connectors/pantsu.py | 16 ++++++++++++++-- pynyaata/connectors/yggtorrent.py | 15 +++++++++++++-- pynyaata/forms.py | 3 +-- pynyaata/templates/admin/edit.html | 9 ++------- pynyaata/templates/latest.html | 2 +- pynyaata/templates/search.html | 2 +- pynyaata/utils.py | 4 ++++ 12 files changed, 66 insertions(+), 32 deletions(-) diff --git a/pynyaata/__init__.py b/pynyaata/__init__.py index d5095b3..0d84806 100644 --- a/pynyaata/__init__.py +++ b/pynyaata/__init__.py @@ -85,7 +85,7 @@ def latest(page=1): for torrent in torrents: results = results + torrent.data for result in results: - result['self'] = get_instance(result['href'], '') + result['self'] = get_instance(result['href']) results.sort(key=itemgetter('date'), reverse=True) return render_template('latest.html', search_form=SearchForm(), torrents=results, page=page) @@ -155,6 +155,8 @@ def admin_edit(link_id=None): form = EditForm(request.form) if form.validate_on_submit(): + # Instance for VF tag + instance = get_instance(form.link.data) # Folder folder = AnimeFolder.query.filter_by(name=form.folder.data).first() folder = folder if folder else AnimeFolder() @@ -177,7 +179,7 @@ def admin_edit(link_id=None): link.link = form.link.data link.season = form.season.data link.comment = form.comment.data - link.vf = form.is_vf.data + link.vf = instance.is_vf(form.link.data) db.session.add(link) db.session.commit() remove_garbage(link) diff --git a/pynyaata/connectors/__init__.py b/pynyaata/connectors/__init__.py index ad2d839..b76a4d3 100644 --- a/pynyaata/connectors/__init__.py +++ b/pynyaata/connectors/__init__.py @@ -20,7 +20,7 @@ async def run_all(*args, **kwargs): return list(await gather(*coroutines)) -def get_instance(url, query): +def get_instance(url, query=''): if 'nyaa.si' in url: return Nyaa(query) elif 'nyaa.net' in url: diff --git a/pynyaata/connectors/animeultime.py b/pynyaata/connectors/animeultime.py index f7347df..0163310 100644 --- a/pynyaata/connectors/animeultime.py +++ b/pynyaata/connectors/animeultime.py @@ -2,7 +2,7 @@ from datetime import datetime, timedelta from bs4 import BeautifulSoup -from .core import ConnectorCore, ConnectorReturn, ConnectorCache, ConnectorLang, curl_content +from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content from ..utils import parse_date, link_exist_in_db @@ -54,7 +54,7 @@ class AnimeUltime(ConnectorCore): if not any(href == d['href'] for d in self.data): self.data.append({ - 'lang': ConnectorLang.JP, + 'vf': self.is_vf(), 'href': href, 'name': url.get_text(), 'type': tds[1].get_text(), @@ -67,7 +67,7 @@ class AnimeUltime(ConnectorCore): href = '%s/file-0-1/%s' % (self.base_url, player[0]['data-serie']) self.data.append({ - 'lang': ConnectorLang.JP, + 'vf': self.is_vf(), 'href': '%s/file-0-1/%s' % (self.base_url, player[0]['data-serie']), 'name': name[0].get_text(), 'type': ani_type[0].get_text().replace(':', ''), @@ -96,7 +96,7 @@ class AnimeUltime(ConnectorCore): href = '%s/%s' % (self.base_url, link['href']) self.data.append({ - 'lang': ConnectorLang.JP, + 'vf': self.is_vf(), 'href': '%s/%s' % (self.base_url, link['href']), 'name': link.get_text(), 'type': tds[4].get_text(), @@ -105,3 +105,7 @@ class AnimeUltime(ConnectorCore): }) self.on_error = False + + @ConnectorCache.cache_data + def is_vf(self, url=''): + return False diff --git a/pynyaata/connectors/core.py b/pynyaata/connectors/core.py index 8dce83b..47cfc4d 100644 --- a/pynyaata/connectors/core.py +++ b/pynyaata/connectors/core.py @@ -1,4 +1,3 @@ -import re from abc import ABC, abstractmethod from datetime import datetime from enum import Enum @@ -80,7 +79,7 @@ def curl_content(url, params=None, ajax=False, debug=True): output = '' http_code = 500 method = 'post' if (params is not None) else 'get' - instance = get_instance(url, '') + instance = get_instance(url) if ajax: headers = {'X-Requested-With': 'XMLHttpRequest'} @@ -183,12 +182,9 @@ class ConnectorCore(ABC): def get_history(self): pass - @staticmethod - def get_lang(str_to_test): - if re.search('(vf|multi|french)', str_to_test, re.IGNORECASE): - return ConnectorLang.FR - else: - return ConnectorLang.JP + @abstractmethod + def is_vf(self, url): + pass async def run(self): if self.on_error: diff --git a/pynyaata/connectors/nyaa.py b/pynyaata/connectors/nyaa.py index c1ef269..06e9323 100644 --- a/pynyaata/connectors/nyaa.py +++ b/pynyaata/connectors/nyaa.py @@ -1,7 +1,7 @@ from bs4 import BeautifulSoup from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import parse_date, link_exist_in_db, check_blacklist_words +from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf class Nyaa(ConnectorCore): @@ -59,7 +59,7 @@ class Nyaa(ConnectorCore): href = self.base_url + url['href'] self.data.append({ - 'lang': self.get_lang(url_safe), + 'vf': check_if_vf(url_safe), 'href': href, 'name': url_safe, 'comment': str(urls[0]).replace('/view/', self.base_url + '/view/') if has_comment else '', @@ -74,3 +74,14 @@ class Nyaa(ConnectorCore): self.on_error = False self.is_more = valid_trs and valid_trs is not len(trs) - 1 + + @ConnectorCache.cache_data + def is_vf(self, url): + response = curl_content(url) + + if response['http_code'] == 200: + html = BeautifulSoup(response['output'], 'html.parser') + title = html.select('h3.panel-title') + return check_if_vf(title[0].get_text()) + + return False diff --git a/pynyaata/connectors/pantsu.py b/pynyaata/connectors/pantsu.py index 64abd3d..0c83950 100644 --- a/pynyaata/connectors/pantsu.py +++ b/pynyaata/connectors/pantsu.py @@ -1,7 +1,7 @@ from bs4 import BeautifulSoup from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import parse_date, link_exist_in_db, check_blacklist_words +from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf class Pantsu(ConnectorCore): @@ -51,7 +51,7 @@ class Pantsu(ConnectorCore): href = self.base_url + url['href'] self.data.append({ - 'lang': self.get_lang(url_safe), + 'vf': check_if_vf(url_safe), 'href': href, 'name': url_safe, 'comment': '', @@ -67,3 +67,15 @@ class Pantsu(ConnectorCore): self.on_error = False self.is_more = valid_trs and valid_trs is not len(trs) - 1 + + @ConnectorCache.cache_data + def is_vf(self, url): + response = curl_content(url) + + if response['http_code'] == 200: + html = BeautifulSoup(response['output'], 'html.parser') + title = html.select('h1.torrent-hr') + print(title, flush=True) + return check_if_vf(title[0].get_text()) + + return False diff --git a/pynyaata/connectors/yggtorrent.py b/pynyaata/connectors/yggtorrent.py index deef0b0..bf65e1f 100644 --- a/pynyaata/connectors/yggtorrent.py +++ b/pynyaata/connectors/yggtorrent.py @@ -5,7 +5,7 @@ from urllib.parse import quote from bs4 import BeautifulSoup from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content -from ..utils import parse_date, link_exist_in_db, check_blacklist_words +from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf class YggTorrent(ConnectorCore): @@ -58,7 +58,7 @@ class YggTorrent(ConnectorCore): valid_trs = valid_trs + 1 self.data.append({ - 'lang': self.get_lang(url_safe), + 'vf': check_if_vf(url_safe), 'href': url['href'], 'name': url_safe, 'comment': '%s' % @@ -77,6 +77,17 @@ class YggTorrent(ConnectorCore): self.on_error = False self.is_more = valid_trs and valid_trs is not len(trs) - 1 + @ConnectorCache.cache_data + def is_vf(self, url): + response = curl_content(url) + + if response['http_code'] == 200: + html = BeautifulSoup(response['output'], 'html.parser') + title = html.select('#title h1') + return check_if_vf(title[0].get_text()) + + return False + class YggAnimation(YggTorrent): title = 'YggAnimation' diff --git a/pynyaata/forms.py b/pynyaata/forms.py index 6c5966c..7048985 100644 --- a/pynyaata/forms.py +++ b/pynyaata/forms.py @@ -1,5 +1,5 @@ from flask_wtf import FlaskForm -from wtforms import BooleanField, HiddenField, StringField +from wtforms import HiddenField, StringField from wtforms.fields.html5 import SearchField, URLField from wtforms.validators import DataRequired @@ -23,4 +23,3 @@ class EditForm(FlaskForm): season = StringField('season', validators=[DataRequired()]) comment = StringField('comment') keyword = StringField('keyword', validators=[DataRequired()]) - is_vf = BooleanField('is_vf') diff --git a/pynyaata/templates/admin/edit.html b/pynyaata/templates/admin/edit.html index ae70b0e..360156b 100644 --- a/pynyaata/templates/admin/edit.html +++ b/pynyaata/templates/admin/edit.html @@ -53,7 +53,7 @@
-
+
{{ action_form.link(value=link.link, class='input', placeholder='Link') }}
@@ -69,7 +69,7 @@
-
+
{{ action_form.comment(value=link.comment, class='input', placeholder='Comment') }}
@@ -90,11 +90,6 @@
- -
diff --git a/pynyaata/templates/latest.html b/pynyaata/templates/latest.html index b76ef81..48c5f1f 100644 --- a/pynyaata/templates/latest.html +++ b/pynyaata/templates/latest.html @@ -27,7 +27,7 @@   - {{ torrent.lang.value }} + {{ torrent.vf|flagify }} {{ torrent.name|boldify|safe }} diff --git a/pynyaata/templates/search.html b/pynyaata/templates/search.html index 03fd674..d5f9014 100644 --- a/pynyaata/templates/search.html +++ b/pynyaata/templates/search.html @@ -32,7 +32,7 @@ {% for torrent in connector.data %} - {{ torrent.lang.value }} + {{ torrent.vf|flagify }} {{ torrent.name|boldify|safe }} diff --git a/pynyaata/utils.py b/pynyaata/utils.py index c4e1fed..4f3683f 100644 --- a/pynyaata/utils.py +++ b/pynyaata/utils.py @@ -45,3 +45,7 @@ def clean_model(obj): def check_blacklist_words(url): return any(word.lower() in url.lower() for word in BLACKLIST_WORDS) + + +def check_if_vf(title): + return any(word.lower() in title.lower() for word in ['vf', 'multi', 'french'])