Merge branch 'master' of ssh://patema.crystalyx.net:2222/Xefir/PyNyaaTa

Reviewed-on: #162

.drone.yml

@@ -1,32 +0,0 @@
-kind: pipeline
-name: default
-type: docker
-
-steps:
-  - name: flake8
-    image: python:slim
-    commands:
-      - pip install flake8
-      - flake8 pynyaata --ignore=E501
-  - name: docker
-    image: plugins/docker
-    settings:
-      repo: xefir/pynyaata
-      auto_tag: true
-      username:
-        from_secret: docker_username
-      password:
-        from_secret: docker_password
-  - name: pypi
-    image: plugins/pypi
-    settings:
-      username:
-        from_secret: pypi_username
-      password:
-        from_secret: pypi_password
-    when:
-      branch:
-        - master
-      event:
-        - push
-

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 120

.gitea/workflows/pynyaata.yml

@@ -0,0 +1,55 @@
+name: books
+on: [push]
+
+jobs:
+  flake8:
+    runs-on: ubuntu-latest
+    container: python
+    steps:
+      - run: apt-get update
+      - run: apt-get install -y git nodejs
+      - uses: actions/checkout@v4
+      - run: pip install flake8
+      - run: flake8 pynyaata
+
+  docker:
+    runs-on: ubuntu-latest
+    container: docker
+    needs: [lint]
+    steps:
+      - run: apk add git nodejs
+      - uses: actions/checkout@v4
+      - uses: docker/metadata-action@v5
+        id: meta
+        with:
+          images: xefir/pynyaata
+          tags: |
+            type=schedule
+            type=ref,event=tag
+            type=ref,event=pr
+            type=raw,value=latest,enable={{is_default_branch}}
+      - uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+      - uses: docker/build-push-action@v5
+        with:
+          push: ${{ gitea.ref == 'refs/heads/master' || startsWith(gitea.ref, 'refs/tags') }}
+          tags: ${{ steps.meta.outputs.tags }}
+          labels: ${{ steps.meta.outputs.labels }}
+
+  pypi:
+    runs-on: ubuntu-latest
+    container: python
+    needs: [lint]
+    if: ${{ gitea.ref == 'refs/heads/master' || startsWith(gitea.ref, 'refs/tags') }}
+    steps:
+      - run: apt-get update
+      - run: apt-get install -y git nodejs
+      - uses: actions/checkout@v4
+      - run: pip install twine
+      - run: python setup.py sdist
+      - run: twine upload dist/*
+        env:
+          TWINE_USERNAME: __token__
+          TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}

Dockerfile

@@ -1,4 +1,4 @@
-FROM python:3.10.6-slim
+FROM python:3.11.5
 
 COPY pynyaata /app/pynyaata
 COPY requirements.txt *.py /app/

README.md

@@ -27,7 +27,7 @@ After a good rewrite in Python, it's time to show it to the public, and here it
 
 ## Features
 
-* Search on [Nyaa.si](https://nyaa.si/), [Nyaa.net (codename Pantsu)](https://nyaa.net/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
+* Search on [Nyaa.si](https://nyaa.si/) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
 * Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search
 * Color official and bad links
 * Add seeded links to a database

get404.py

@@ -4,7 +4,7 @@ from pynyaata.models import AnimeLink
 links = AnimeLink.query.all()
 
 for link in links:
-    html = curl_content(link.link, debug=False, cloudflare=True)
+    html = curl_content(link.link, debug=False)
 
     if html['http_code'] != 200 and html['http_code'] != 500:
         print('(%d) %s %s : %s' % (

pynyaata/__init__.py

@@ -1,14 +1,14 @@
-from asyncio import get_event_loop, set_event_loop, SelectorEventLoop
+from asyncio import SelectorEventLoop, get_event_loop, set_event_loop
 from functools import wraps
 from operator import attrgetter, itemgetter
 
-from flask import redirect, render_template, request, url_for, abort
+from flask import abort, redirect, render_template, request, url_for
 
 from . import utils
-from .config import app, auth, ADMIN_USERNAME, ADMIN_PASSWORD, DB_ENABLED, APP_PORT, IS_DEBUG, TRANSMISSION_ENABLED
-from .connectors import get_instance, run_all, Nyaa
+from .config import ADMIN_PASSWORD, ADMIN_USERNAME, APP_PORT, DB_ENABLED, IS_DEBUG, TRANSMISSION_ENABLED, app, auth
+from .connectors import Nyaa, get_instance, run_all
 from .connectors.core import ConnectorLang, ConnectorReturn
-from .forms import SearchForm, DeleteForm, EditForm, FolderDeleteForm, FolderEditForm
+from .forms import DeleteForm, EditForm, FolderDeleteForm, FolderEditForm, SearchForm
 
 if DB_ENABLED:
     from .config import db
@@ -109,7 +109,7 @@ def latest(page=1):
 @app.route('/list')
 @app.route('/list/<url_filters>')
 @db_required
-def list_animes(url_filters='nyaa,yggtorrent'):
+def list_animes(url_filters='nyaa'):
     filters = None
     for i, to_filter in enumerate(url_filters.split(',')):
         if not i:

pynyaata/connectors/__init__.py

@@ -3,14 +3,11 @@ from asyncio import gather
 from .animeultime import AnimeUltime
 from .core import Other
 from .nyaa import Nyaa
-from .yggtorrent import YggTorrent, YggAnimation
 
 
 async def run_all(*args, **kwargs):
     coroutines = [Nyaa(*args, **kwargs).run(),
-                  AnimeUltime(*args, **kwargs).run(),
-                  YggTorrent(*args, **kwargs).run(),
-                  YggAnimation(*args, **kwargs).run()]
+                  AnimeUltime(*args, **kwargs).run()]
 
     return list(await gather(*coroutines))
 
@@ -20,7 +17,5 @@ def get_instance(url, query=''):
         return Nyaa(query)
     elif 'anime-ultime' in url:
         return AnimeUltime(query)
-    elif 'ygg' in url:
-        return YggTorrent(query)
     else:
         return Other(query)

pynyaata/connectors/animeultime.py

@@ -2,8 +2,8 @@ from datetime import datetime, timedelta
 
 from bs4 import BeautifulSoup
 
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
-from ..utils import parse_date, link_exist_in_db
+from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
+from ..utils import link_exist_in_db, parse_date
 
 
 class AnimeUltime(ConnectorCore):

pynyaata/connectors/core.py

@@ -3,17 +3,14 @@ from enum import Enum
 from functools import wraps
 from json import dumps, loads
 
-import requests
-from requests import RequestException
 from redis.exceptions import RedisError
+from requests import RequestException, Session
 
-from ..config import CACHE_TIMEOUT, REQUESTS_TIMEOUT, logger, REDIS_ENABLED
+from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger
 
 if REDIS_ENABLED:
     from ..config import cache
 
-cloudproxy_session = None
-
 
 class ConnectorReturn(Enum):
     SEARCH = 1
@@ -71,28 +68,26 @@ class Cache:
 ConnectorCache = Cache()
 
 
-def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
+def curl_content(url, params=None, ajax=False, debug=True):
     output = ''
     http_code = 500
     method = 'post' if (params is not None) else 'get'
+    request = Session()
     headers = {}
 
     if ajax:
         headers['X-Requested-With'] = 'XMLHttpRequest'
 
-    if cloudflare:
-        headers['User-Agent'] = 'Googlebot/2.1 (+http://www.google.com/bot.html)'
-
     try:
         if method == 'post':
-            response = requests.post(
+            response = request.post(
                 url,
                 params,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers
             )
         else:
-            response = requests.get(
+            response = request.get(
                 url,
                 timeout=REQUESTS_TIMEOUT,
                 headers=headers

pynyaata/connectors/nyaa.py

@@ -1,7 +1,7 @@
 from bs4 import BeautifulSoup
 
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
-from ..utils import link_exist_in_db, check_blacklist_words, check_if_vf
+from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
+from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db
 
 
 class Nyaa(ConnectorCore):

pynyaata/connectors/yggtorrent.py

@@ -1,98 +0,0 @@
-import re
-from datetime import datetime
-from urllib.parse import quote
-
-from bs4 import BeautifulSoup
-
-from .core import ConnectorCore, ConnectorReturn, ConnectorCache, curl_content
-from ..utils import parse_date, link_exist_in_db, check_blacklist_words, check_if_vf
-
-
-class YggTorrent(ConnectorCore):
-    color = 'is-success'
-    title = 'YggTorrent'
-    favicon = 'yggtorrent.png'
-    base_url = 'https://www5.yggtorrent.fi'
-    is_light = False
-    category = 2179
-
-    def get_full_search_url(self):
-        sort_type = 'size'
-        if self.return_type is ConnectorReturn.HISTORY:
-            sort_type = 'publish_date'
-        sort_page = '&page=%s' % (
-            (self.page - 1) * 50
-        ) if self.page > 1 else ''
-
-        return '%s/engine/search?name=%s&category=2145&sub_category=%s&do=search&order=desc&sort=%s%s' % (
-            self.base_url, self.query, self.category, sort_type, sort_page
-        )
-
-    def get_history(self):
-        self.search()
-
-    @ConnectorCache.cache_data
-    def search(self):
-        if self.category:
-            response = curl_content(
-                self.get_full_search_url(), cloudflare=True
-            )
-
-            if response['http_code'] == 200:
-                html = BeautifulSoup(response['output'], 'html.parser')
-                trs = html.select('table.table tr')
-                valid_trs = 0
-
-                for i, tr in enumerate(trs):
-                    if not i:
-                        continue
-
-                    tds = tr.findAll('td')
-                    check_downloads = int(tds[6].get_text())
-                    check_seeds = int(tds[7].get_text())
-
-                    if check_downloads or check_seeds:
-                        url = tds[1].a
-                        url_safe = url.get_text()
-
-                        if check_blacklist_words(url_safe):
-                            continue
-
-                        valid_trs = valid_trs + 1
-
-                        self.data.append({
-                            'vf': check_if_vf(url_safe),
-                            'href': url['href'],
-                            'name': url_safe,
-                            'comment': '<a href="%s#comm" target="_blank"><i class="fa fa-comments-o"></i>%s</a>' %
-                                       (url['href'], tds[3].decode_contents()),
-                            'link': '<a href="%s/engine/download_torrent?id=%s">'
-                                    '<i class="fa fa-fw fa-download"></i>'
-                                    '</a>' % (self.base_url,
-                                              re.search(r'/(\d+)', url['href']).group(1)),
-                            'size': tds[5].get_text(),
-                            'date': parse_date(datetime.fromtimestamp(int(tds[4].div.get_text()))),
-                            'seeds': check_seeds,
-                            'leechs': tds[8].get_text(),
-                            'downloads': check_downloads,
-                            'class': self.color if link_exist_in_db(quote(url['href'], '/+:')) else ''
-                        })
-
-                self.on_error = False
-                self.is_more = valid_trs and valid_trs != len(trs) - 1
-
-    @ConnectorCache.cache_data
-    def is_vf(self, url):
-        response = curl_content(url)
-
-        if response['http_code'] == 200:
-            html = BeautifulSoup(response['output'], 'html.parser')
-            title = html.select('#title h1')
-            return check_if_vf(title[0].get_text())
-
-        return False
-
-
-class YggAnimation(YggTorrent):
-    title = 'YggAnimation'
-    category = 2178

pynyaata/forms.py

@@ -1,5 +1,5 @@
 from flask_wtf import FlaskForm
-from wtforms import HiddenField, StringField, SelectField
+from wtforms import HiddenField, SelectField, StringField
 from wtforms.fields.html5 import SearchField, URLField
 from wtforms.validators import DataRequired
 

pynyaata/utils.py

@@ -2,7 +2,7 @@ import re
 from datetime import datetime
 from dateparser import parse
 
-from .config import DB_ENABLED, BLACKLIST_WORDS
+from .config import BLACKLIST_WORDS, DB_ENABLED
 
 
 def link_exist_in_db(href):

requirements.txt

@@ -1,13 +1,14 @@
-Flask==2.2.2
+Flask==2.3.3
 Flask-SQLAlchemy==2.5.1
-Flask-HTTPAuth==4.7.0
-Flask-WTF==1.0.1
+Flask-HTTPAuth==4.8.0
+Flask-WTF==1.1.2
 WTForms==2.3.3
-PyMySQL==1.0.2
-pg8000==1.29.1
-requests==2.28.1
-beautifulsoup4==4.11.1
-python-dotenv==0.20.0
-dateparser==1.1.1
-redis==4.3.4
-transmission-rpc==3.3.2
+PyMySQL==1.1.0
+pg8000==1.30.2
+requests==2.31.0
+charset-normalizer==3.2.0
+beautifulsoup4==4.12.2
+python-dotenv==1.0.0
+dateparser==1.1.8
+redis==5.0.1
+transmission-rpc==7.0.1