This commit is contained in:
Michel Roux 2022-10-26 13:06:04 +00:00
parent 46977c7e38
commit 434fbeb9fa
58 changed files with 686 additions and 5091 deletions

View File

@ -1,38 +0,0 @@
kind: pipeline
name: default
type: docker
steps:
- name: lint
image: python:3.7-slim
commands:
- pip install poetry
- poetry install
- poetry run flake8
- poetry run mypy .
- poetry run djlint .
- name: docker
image: plugins/docker
settings:
repo: xefir/pynyaata
auto_tag: true
username:
from_secret: docker_username
password:
from_secret: docker_password
- name: pypi
image: plugins/pypi
settings:
username:
from_secret: pypi_username
password:
from_secret: pypi_password
skip_build: true
when:
branch:
- master
event:
- push

View File

@ -1,9 +0,0 @@
FLASK_APP=run.py
FLASK_ENV=development
FLASK_PORT=5000
REDIS_SERVER=redis
ADMIN_USERNAME=admin
ADMIN_PASSWORD=secret
REQUESTS_TIMEOUT=5
CACHE_TIMEOUT=3600
BLACKLIST_WORDS=Chris44,Vol.,[zza],.ssa,.ass,Ref:rain

View File

@ -1,2 +1,2 @@
[flake8] [flake8]
max-line-length = 100 max-line-length = 111

10
.gitignore vendored
View File

@ -152,3 +152,13 @@ dmypy.json
# Cython debug symbols # Cython debug symbols
cython_debug/ cython_debug/
# PyCharm
# JetBrains specific template is maintained in a separate JetBrains.gitignore that can
# be found at https://github.com/github/gitignore/blob/main/Global/JetBrains.gitignore
# and can be added to the global gitignore or merged into this file. For a more nuclear
# option (not recommended) you can uncomment the following to ignore the entire idea folder.
.idea/
# Tests
mocks/*.html

View File

@ -1,12 +0,0 @@
FROM python:3.10.6-slim as build
WORKDIR /app
COPY . .
RUN pip install poetry && poetry build
FROM python:3.10.6-slim
COPY --from=build /app/dist /tmp/dist
RUN pip install /tmp/dist/*.whl && rm -rf /tmp/dist
CMD ["pynyaata"]

View File

@ -1,13 +0,0 @@
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
Version 2, December 2004
Copyright (C) 2004 Sam Hocevar <sam@hocevar.net>
Everyone is permitted to copy and distribute verbatim or modified
copies of this license document, and changing it is allowed as long
as the name is changed.
DO WHAT THE FUCK YOU WANT TO PUBLIC LICENSE
TERMS AND CONDITIONS FOR COPYING, DISTRIBUTION AND MODIFICATION
0. You just DO WHAT THE FUCK YOU WANT TO.

View File

@ -1,51 +0,0 @@
# π 😼た
> "PyNyaaTa", Xéfir's personal anime torrent search engine
[![Build Status](https://ci.crystalyx.net/api/badges/Xefir/PyNyaaTa/status.svg)](https://ci.crystalyx.net/Xefir/PyNyaaTa)
[![Docker Hub](https://img.shields.io/docker/pulls/xefir/pynyaata)](https://hub.docker.com/r/xefir/pynyaata)
I'm lazy, and I want to search across several VF and VOSTFR torrents databases in one click.
That's the starting point that build this app.
At first, it was a crappy PHP project without any good future.
After a good rewrite in Python, it's time to show it to the public, and here it is!
## Installing / Getting started
### With Docker
- Install Docker: https://hub.docker.com/search/?type=edition&offering=community
- Run `docker run -p 5000 xefir/pynyaata`
- The app is accessible at http://localhost:5000
### Without Docker
- Install Python 3: https://www.python.org/downloads/
- Install Pip: https://pip.pypa.io/en/stable/installing/
- Run `pip install pynyaata`
- Run `pynyaata`
- The app is accessible at http://localhost:5000
## Features
* Search on [Nyaa.si](https://nyaa.si/), [YggTorrent](https://duckduckgo.com/?q=yggtorrent) and [Anime-Ultime](http://www.anime-ultime.net/index-0-1)
* Provide useful links to [TheTVDB](https://www.thetvdb.com/) and [Nautiljon](https://www.nautiljon.com/) during a search
* Color official and bad links
* Add seeded links to a database
* Color seeded link on search
* Run a batch to list all dead link on database
## Configuration
All is managed by environment variables.
Please look into the `.env.dist` file to list all possible environment variables.
You have to have a running database server to be able to access the admin panel.
## Links
- Project homepage: https://nyaa.crystalyx.net/
- Source repository: https://git.crystalyx.net/Xefir/PyNyaaTa
- Issue tracker: https://git.crystalyx.net/Xefir/PyNyaaTa/issues
- My other projects: https://git.crystalyx.net/Xefir
- Docker hub: https://hub.docker.com/r/xefir/pynyaata
- Donations: https://paypal.me/Xefir

725
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,308 +0,0 @@
from asyncio import SelectorEventLoop, get_event_loop, set_event_loop
from functools import wraps
from operator import attrgetter, itemgetter
from flask import abort, redirect, render_template, request, url_for
from . import utils
from .config import (
ADMIN_PASSWORD,
ADMIN_USERNAME,
APP_PORT,
DB_ENABLED,
IS_DEBUG,
TRANSMISSION_ENABLED,
app,
auth,
)
from .connectors import Nyaa, get_instance, run_all
from .connectors.core import ConnectorLang, ConnectorReturn
from .forms import DeleteForm, EditForm, FolderDeleteForm, FolderEditForm, SearchForm
if DB_ENABLED:
from .config import db
from .models import AnimeFolder, AnimeTitle, AnimeLink
if TRANSMISSION_ENABLED:
from .config import transmission
def db_required(f):
@wraps(f)
def decorated_function(*args, **kwargs):
if not DB_ENABLED:
return abort(404)
return f(*args, **kwargs)
return decorated_function
def clean_titles():
db.engine.execute(
"""
DELETE
FROM anime_title
WHERE id IN (
SELECT anime_title.id
FROM anime_title
LEFT JOIN anime_link ON anime_title.id = anime_link.title_id
WHERE anime_link.id IS NULL
)
"""
)
@auth.verify_password
def verify_password(username, password):
return username == ADMIN_USERNAME and ADMIN_PASSWORD == password
@app.template_filter("boldify")
def boldify(name):
query = request.args.get("q", "")
name = utils.boldify(name, query)
if DB_ENABLED:
for keyword in db.session.query(AnimeTitle.keyword.distinct()).all():
if keyword[0].lower() != query.lower():
name = utils.boldify(name, keyword[0])
return name
@app.template_filter("flagify")
def flagify(is_vf):
return ConnectorLang.FR.value if is_vf else ConnectorLang.JP.value
@app.template_filter("colorify")
def colorify(model):
return get_instance(model.link, model.title.keyword).color
@app.context_processor
def inject_user():
return dict(db_disabled=not DB_ENABLED)
@app.route("/")
def home():
return render_template(
"layout.html", search_form=SearchForm(), title="Anime torrents search engine"
)
@app.route("/search")
def search():
query = request.args.get("q")
if not query:
return redirect(url_for("home"))
set_event_loop(SelectorEventLoop())
torrents = get_event_loop().run_until_complete(run_all(query))
return render_template("search.html", search_form=SearchForm(), connectors=torrents)
@app.route("/latest")
@app.route("/latest/<int:page>")
def latest(page=1):
set_event_loop(SelectorEventLoop())
torrents = get_event_loop().run_until_complete(
run_all("", return_type=ConnectorReturn.HISTORY, page=page)
)
results = []
for torrent in torrents:
results = results + torrent.data
for result in results:
result["self"] = get_instance(result["href"])
results.sort(key=itemgetter("date"), reverse=True)
return render_template(
"latest.html", search_form=SearchForm(), torrents=results, page=page
)
@app.route("/list")
@app.route("/list/<url_filters>")
@db_required
def list_animes(url_filters="nyaa,yggtorrent"):
filters = None
for i, to_filter in enumerate(url_filters.split(",")):
if not i:
filters = AnimeLink.link.contains(to_filter)
else:
filters = filters | AnimeLink.link.contains(to_filter)
titles = (
db.session.query(AnimeTitle, AnimeLink)
.join(AnimeLink)
.filter(filters)
.order_by(AnimeTitle.name)
.all()
)
results = {}
for title, link in titles:
if title.id not in results:
results[title.id] = [link]
else:
results[title.id].append(link)
return render_template("list.html", search_form=SearchForm(), titles=results)
@app.route("/admin", methods=["GET", "POST"])
@db_required
@auth.login_required
def admin():
form = DeleteForm(request.form)
if form.validate_on_submit():
link = AnimeLink.query.filter_by(id=int(form.id.data)).first()
if link:
form.message = "%s (%s) has been successfully deleted" % (
link.title.name,
link.season,
)
db.session.delete(link)
db.session.commit()
title = link.title
if title and not len(title.links):
db.session.delete(title)
db.session.commit()
else:
form._errors = {
"id": ["Id %s was not found in the database" % form.id.data]
}
folders = AnimeFolder.query.all()
for folder in folders:
for title in folder.titles:
title.links.sort(key=attrgetter("season"))
folder.titles.sort(key=attrgetter("name"))
return render_template(
"admin/list.html", search_form=SearchForm(), folders=folders, action_form=form
)
@app.route("/admin/folder", methods=["GET", "POST"])
@db_required
@auth.login_required
def folder_list():
form = FolderDeleteForm(request.form)
if form.validate_on_submit():
folder = AnimeFolder.query.filter_by(id=int(form.id.data)).first()
if folder:
form.message = "%s has been successfully deleted" % folder.name
db.session.delete(folder)
db.session.commit()
else:
form._errors = {
"id": ["Id %s was not found in the database" % form.id.data]
}
folders = AnimeFolder.query.all()
return render_template(
"admin/folder/list.html",
search_form=SearchForm(),
folders=folders,
action_form=form,
)
@app.route("/admin/folder/edit", methods=["GET", "POST"])
@app.route("/admin/folder/edit/<int:folder_id>", methods=["GET", "POST"])
@db_required
@auth.login_required
def folder_edit(folder_id=None):
folder = AnimeFolder.query.filter_by(id=folder_id).first()
folder = folder if folder else AnimeFolder()
form = FolderEditForm(
request.form, id=folder.id, name=folder.name, path=folder.path
)
if form.validate_on_submit():
# Folder
folder.name = form.name.data
folder.path = form.path.data
db.session.add(folder)
db.session.commit()
return redirect(url_for("folder_list"))
return render_template(
"admin/folder/edit.html", search_form=SearchForm(), action_form=form
)
@app.route("/admin/edit", methods=["GET", "POST"])
@app.route("/admin/edit/<int:link_id>", methods=["GET", "POST"])
@db_required
@auth.login_required
def admin_edit(link_id=None):
link = AnimeLink.query.filter_by(id=link_id).first()
link = link if link else AnimeLink()
folders = AnimeFolder.query.all()
form = EditForm(
request.form,
id=link.id,
folder=link.title.folder.id if link.title else None,
name=link.title.name if link.title else None,
link=link.link,
season=link.season,
comment=link.comment,
keyword=link.title.keyword if link.title else None,
)
form.folder.choices = [("", "")] + [(g.id, g.name) for g in folders]
if form.validate_on_submit():
# Instance for VF tag
instance = get_instance(form.link.data)
# Title
title = AnimeTitle.query.filter_by(id=link.title_id).first()
title = (
title if title else AnimeTitle.query.filter_by(name=form.name.data).first()
)
title = title if title else AnimeTitle()
title.folder_id = form.folder.data
title.name = form.name.data
title.keyword = form.keyword.data.lower()
db.session.add(title)
db.session.commit()
# Link
link.title_id = title.id
link.link = form.link.data
link.season = form.season.data
link.comment = form.comment.data
link.vf = instance.is_vf(form.link.data)
# Database
db.session.add(link)
db.session.commit()
clean_titles()
# Transmission
if TRANSMISSION_ENABLED and isinstance(instance, Nyaa):
if title.folder.path is not None and title.folder.path != "":
download_url = link.link.replace("/view/", "/download/") + ".torrent"
torrent_path = "%s/%s" % (title.folder.path, title.name)
torrent = transmission.add_torrent(
download_url, download_dir=torrent_path
)
transmission.move_torrent_data(torrent.id, torrent_path)
transmission.start_torrent(torrent.id)
return redirect(url_for("admin"))
return render_template(
"admin/edit.html", search_form=SearchForm(), folders=folders, action_form=form
)
def run():
app.run("0.0.0.0", APP_PORT, IS_DEBUG)

View File

@ -0,0 +1,101 @@
from datetime import datetime, timedelta
from typing import List
from bs4 import BeautifulSoup
import dateparser
from pydantic import HttpUrl, parse_obj_as
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError
class AnimeUltime(Bridge):
color = Color.WARNING
title = "Anime-Ultime"
base_url = parse_obj_as(HttpUrl, "http://www.anime-ultime.net")
favicon = parse_obj_as(HttpUrl, "http://www.anime-ultime.net/favicon.ico")
def search_url(self, query: str = "", page: int = 1) -> HttpUrl:
try:
page_date = datetime.now() - timedelta((page - 1) * 365 / 12)
except OverflowError:
page_date = datetime.fromtimestamp(0)
return parse_obj_as(
HttpUrl,
(
f"{self.base_url}"
f"{'search' if query else 'history'}-0-1/"
f"{page_date.strftime('%m%Y') if query else ''}"
),
)
def search(self, query: str = "", page: int = 1) -> List[RemoteFile]:
response = (
requests.post(self.search_url(query, page), {"search": query})
if query
else requests.get(self.search_url(query, page))
)
if response.status_code != 200:
raise HTTPError()
torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser")
title = html.select_one("div.title")
history = html.select_one("h1")
player = html.select_one("div.AUVideoPlayer")
tables = html.select("table.jtable")
if title and "Recherche" in title.get_text():
trs = html.select("table.jtable tr")
for i, tr in enumerate(trs):
if not i:
continue
tds = tr.find_all("td")
torrents.append(
RemoteFile(
id=tds[0].a["href"].split("/")[1].split("-")[0],
category=tds[1].get_text(),
name=tds[0].get_text(),
link=f"{self.base_url}{tds[0].a['href']}",
)
)
elif history and "Historique" in history.get_text():
h3s = html.findAll("h3")
for i, table in enumerate(tables):
for j, tr in enumerate(table.find_all("tr")):
if not j:
continue
tds = tr.find_all("td")
torrents.append(
RemoteFile(
id=tds[0].a["href"].split("/")[-2],
category=tds[4].get_text(),
name=tds[0].get_text(),
link=f"{self.base_url}{tds[0].a['href']}",
date=dateparser.parse(
h3s[i].get_text()[:-3], "%A %d %B %Y"
),
)
)
elif player and title and history and tables:
torrents.append(
RemoteFile(
id=player["data-serie"],
category=title.get_text(),
name=history.get_text(),
link=f"{self.base_url}file-0-1/{player['data-serie']}",
date=tables[0].find_all("tr")[1].find_all("td")[1].get_text(),
)
)
return torrents

76
pynyaata/bridge/nyaa.py Normal file
View File

@ -0,0 +1,76 @@
from datetime import datetime
from typing import List
from bs4 import BeautifulSoup
from pydantic import HttpUrl, parse_obj_as
from pynyaata.cache import cache_data
from pynyaata.constants import VF_WORDS
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError
from urllib.parse import urlencode
class Nyaa(Bridge):
color = Color.INFO
title = "Nyaa"
base_url = parse_obj_as(HttpUrl, "https://nyaa.si")
favicon = parse_obj_as(HttpUrl, "https://nyaa.si/static/favicon.png")
def search_url(self, query: str = "", page: int = 1) -> HttpUrl:
to_query = "|".join(map(lambda word: f"({query} {word})", VF_WORDS))
params = urlencode(
{
"f": 0,
"c": "1_3",
"q": to_query,
"s": "size" if query else "id",
"o": "desc",
"p": page,
}
)
return parse_obj_as(HttpUrl, f"{self.base_url}?{params}")
@cache_data
def search(self, query: str = "", page: int = 1) -> List[RemoteFile]:
response = requests.get(self.search_url(query, page))
if response.status_code != 200:
raise HTTPError()
torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser")
trs = html.select("table.torrent-list tr")
for i, tr in enumerate(trs):
if not i:
continue
tds = tr.find_all("td")
urls = tds[1].find_all("a")
links = tds[2].find_all("a")
torrents.append(
RemoteFile(
id=urls[1 if len(urls) > 1 else 0]["href"].split("/")[-1],
category=tds[0].a["title"],
color=Color[tr["class"][0].upper()],
name=urls[1 if len(urls) > 1 else 0].get_text(),
link=f"{self.base_url}{urls[1 if len(urls) > 1 else 0]['href']}",
comment=urls[0].get_text() if len(urls) > 1 else 0,
comment_url=f"{self.base_url}{urls[0]['href']}",
magnet=links[1]["href"],
torrent=f"{self.base_url}{links[0]['href']}",
size=tds[3].get_text(),
date=datetime.fromtimestamp(int(tds[4]["data-timestamp"])),
seeds=tds[5].get_text(),
leechs=tds[6].get_text(),
downloads=tds[7].get_text(),
nb_pages=html.select("ul.pagination li")[-2].get_text(),
)
)
return torrents

37
pynyaata/cache/__init__.py vendored Normal file
View File

@ -0,0 +1,37 @@
import logging
from functools import wraps
from os import environ
from pynyaata.cache.simple import SimpleCache
from pynyaata.types import Cache
from redis import RedisError
CACHE_TIMEOUT = int(environ.get("CACHE_TIMEOUT", 60 * 60))
REDIS_URL = environ.get("REDIS_URL", "")
client: Cache = SimpleCache()
if REDIS_URL:
try:
from pynyaata.cache.redis import RedisCache
client = RedisCache()
except RedisError as e:
logging.error(e)
def cache_data(f):
@wraps(f)
def wrapper(*args, **kwds):
bridge = args[0]
key = f"pynyaata.{bridge.__class__.__name__}.{f.__name__}.{bridge.query}.{bridge.page}"
ret = client.get(key)
if ret:
return ret
ret = f(*args, **kwds)
client.set(key, ret)
return ret
return wrapper

18
pynyaata/cache/redis.py vendored Normal file
View File

@ -0,0 +1,18 @@
from json import dumps, loads
from os import environ
from typing import List, Optional
from pynyaata.cache import CACHE_TIMEOUT
from pynyaata.types import Cache, RemoteFile
from redis import Redis
REDIS_URL = environ.get("REDIS_URL", "")
client = Redis.from_url(REDIS_URL)
class RedisCache(Cache):
def get(self, key: str) -> Optional[List[RemoteFile]]:
return loads(str(client.get(key)))
def set(self, key: str, data: List[RemoteFile]):
return client.set(key, dumps(data), CACHE_TIMEOUT)

21
pynyaata/cache/simple.py vendored Normal file
View File

@ -0,0 +1,21 @@
from datetime import datetime, timedelta
from typing import Dict, List, Optional, Tuple
from pynyaata.cache import CACHE_TIMEOUT
from pynyaata.types import Cache, RemoteFile
CACHE_DATA: Dict[str, Tuple[List[RemoteFile], datetime]] = {}
class SimpleCache(Cache):
def get(self, key: str) -> Optional[List[RemoteFile]]:
if key in CACHE_DATA:
data, timeout = CACHE_DATA[key]
if datetime.now() > timeout + timedelta(seconds=CACHE_TIMEOUT):
return data
else:
CACHE_DATA.pop(key)
return None
def set(self, key: str, data: List[RemoteFile]):
CACHE_DATA[key] = (data, datetime.now())

View File

@ -1,64 +0,0 @@
import logging
from os import environ, urandom
from flask import Flask
from flask.cli import load_dotenv
from flask_httpauth import HTTPBasicAuth
from flask_sqlalchemy import SQLAlchemy
from redis import Redis
from transmission_rpc.client import Client
load_dotenv()
IS_DEBUG = environ.get("FLASK_ENV", "production") == "development"
ADMIN_USERNAME = environ.get("ADMIN_USERNAME", "admin")
ADMIN_PASSWORD = environ.get("ADMIN_PASSWORD", "secret")
APP_PORT = int(environ.get("FLASK_PORT", 5000))
CACHE_TIMEOUT = int(environ.get("CACHE_TIMEOUT", 60 * 60))
REQUESTS_TIMEOUT = int(environ.get("REQUESTS_TIMEOUT", 5))
BLACKLIST_WORDS = (
environ.get("BLACKLIST_WORDS", "").split(",")
if environ.get("BLACKLIST_WORDS", "")
else []
)
DB_ENABLED = False
REDIS_ENABLED = False
TRANSMISSION_ENABLED = False
app = Flask(__name__)
app.name = "PyNyaaTa"
app.debug = IS_DEBUG
app.secret_key = urandom(24).hex()
app.url_map.strict_slashes = False
auth = HTTPBasicAuth()
logging.basicConfig(level=(logging.DEBUG if IS_DEBUG else logging.INFO))
logger = logging.getLogger(app.name)
db_uri = environ.get("DATABASE_URI")
if db_uri:
DB_ENABLED = True
app.config["SQLALCHEMY_DATABASE_URI"] = db_uri
app.config["SQLALCHEMY_TRACK_MODIFICATIONS"] = True
app.config["SQLALCHEMY_ECHO"] = IS_DEBUG
app.config["SQLALCHEMY_ENGINE_OPTIONS"] = {"pool_recycle": 200}
db = SQLAlchemy(app)
from .models import create_all
create_all()
cache_host = environ.get("REDIS_SERVER")
if cache_host:
REDIS_ENABLED = True
cache = Redis(cache_host)
transmission_host = environ.get("TRANSMISSION_SERVER")
if transmission_host:
TRANSMISSION_ENABLED = True
transmission_username = environ.get("TRANSMISSION_RPC_USERNAME")
transmission_password = environ.get("TRANSMISSION_RPC_PASSWORD")
transmission = Client(
username=transmission_username,
password=transmission_password,
host=transmission_host,
logger=logger,
)

View File

@ -1,28 +0,0 @@
from asyncio import gather
from .animeultime import AnimeUltime
from .core import Other
from .nyaa import Nyaa
from .yggtorrent import YggAnimation, YggTorrent
async def run_all(*args, **kwargs):
coroutines = [
Nyaa(*args, **kwargs).run(),
AnimeUltime(*args, **kwargs).run(),
YggTorrent(*args, **kwargs).run(),
YggAnimation(*args, **kwargs).run(),
]
return list(await gather(*coroutines))
def get_instance(url, query=""):
if "nyaa.si" in url:
return Nyaa(query)
elif "anime-ultime" in url:
return AnimeUltime(query)
elif "ygg" in url:
return YggTorrent(query)
else:
return Other(query)

View File

@ -1,114 +0,0 @@
from datetime import datetime, timedelta
from bs4 import BeautifulSoup
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import link_exist_in_db, parse_date
class AnimeUltime(ConnectorCore):
color = "is-warning"
title = "Anime-Ultime"
favicon = "animeultime.png"
base_url = "http://www.anime-ultime.net"
is_light = True
def get_full_search_url(self):
from_date = ""
sort_type = "search"
if self.return_type is ConnectorReturn.HISTORY:
try:
page_date = datetime.now() - timedelta((int(self.page) - 1) * 365 / 12)
except OverflowError:
page_date = datetime.fromtimestamp(0)
from_date = page_date.strftime("%m%Y")
sort_type = "history"
return "%s/%s-0-1/%s" % (self.base_url, sort_type, from_date)
@ConnectorCache.cache_data
def search(self):
response = curl_content(self.get_full_search_url(), {"search": self.query})
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
title = html.select("div.title")
player = html.select("div.AUVideoPlayer")
if len(title) > 0 and "Recherche" in title[0].get_text():
trs = html.select("table.jtable tr")
for i, tr in enumerate(trs):
if not i:
continue
tds = tr.findAll("td")
if len(tds) < 2:
continue
url = tds[0].a
href = "%s/%s" % (self.base_url, url["href"])
if not any(href == d["href"] for d in self.data):
self.data.append(
{
"vf": self.is_vf(),
"href": href,
"name": url.get_text(),
"type": tds[1].get_text(),
"class": self.color if link_exist_in_db(href) else "",
}
)
elif len(player) > 0:
name = html.select("h1")
ani_type = html.select("div.titre")
href = "%s/file-0-1/%s" % (self.base_url, player[0]["data-serie"])
self.data.append(
{
"vf": self.is_vf(),
"href": href,
"name": name[0].get_text(),
"type": ani_type[0].get_text().replace(":", ""),
"class": self.color if link_exist_in_db(href) else "",
}
)
self.on_error = False
@ConnectorCache.cache_data
def get_history(self):
response = curl_content(self.get_full_search_url())
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
tables = html.select("table.jtable")
h3s = html.findAll("h3")
for i, table in enumerate(tables):
for j, tr in enumerate(table.findAll("tr")):
if not j:
continue
tds = tr.findAll("td")
link = tds[0].a
href = "%s/%s" % (self.base_url, link["href"])
self.data.append(
{
"vf": self.is_vf(),
"href": href,
"name": link.get_text(),
"type": tds[4].get_text(),
"date": parse_date(h3s[i].string[:-3], "%A %d %B %Y"),
"class": self.color if link_exist_in_db(href) else "",
}
)
self.on_error = False
@ConnectorCache.cache_data
def is_vf(self, url=""):
return False

View File

@ -1,180 +0,0 @@
from abc import ABC, abstractmethod
from enum import Enum
from functools import wraps
from json import dumps, loads
from redis.exceptions import RedisError
import requests
from requests import RequestException
from ..config import CACHE_TIMEOUT, REDIS_ENABLED, REQUESTS_TIMEOUT, logger
if REDIS_ENABLED:
from ..config import cache
cloudproxy_session = None
class ConnectorReturn(Enum):
SEARCH = 1
HISTORY = 2
class ConnectorLang(Enum):
FR = "🇫🇷"
JP = "🇯🇵"
class Cache:
def cache_data(self, f):
@wraps(f)
def wrapper(*args, **kwds):
connector = args[0]
key = "pynyaata.%s.%s.%s.%s" % (
connector.__class__.__name__,
f.__name__,
connector.query,
connector.page,
)
if REDIS_ENABLED:
json = None
try:
json = cache.get(key)
except RedisError:
pass
if json:
data = loads(json)
connector.data = data["data"]
connector.is_more = data["is_more"]
connector.on_error = False
return
ret = f(*args, **kwds)
if not connector.on_error and REDIS_ENABLED:
try:
cache.set(
key,
dumps({"data": connector.data, "is_more": connector.is_more}),
CACHE_TIMEOUT,
)
except RedisError:
pass
return ret
return wrapper
ConnectorCache = Cache()
def curl_content(url, params=None, ajax=False, debug=True, cloudflare=False):
output = ""
http_code = 500
method = "post" if (params is not None) else "get"
headers = {}
if ajax:
headers["X-Requested-With"] = "XMLHttpRequest"
if cloudflare:
headers["User-Agent"] = "Googlebot/2.1 (+http://www.google.com/bot.html)"
try:
if method == "post":
response = requests.post(
url, params, timeout=REQUESTS_TIMEOUT, headers=headers
)
else:
response = requests.get(url, timeout=REQUESTS_TIMEOUT, headers=headers)
output = response.text
http_code = response.status_code
except RequestException as e:
if debug:
logger.exception(e)
return {"http_code": http_code, "output": output}
class ConnectorCore(ABC):
@property
@abstractmethod
def color(self):
pass
@property
@abstractmethod
def title(self):
pass
@property
@abstractmethod
def favicon(self):
pass
@property
@abstractmethod
def base_url(self):
pass
@property
@abstractmethod
def is_light(self):
pass
def __init__(self, query, page=1, return_type=ConnectorReturn.SEARCH):
self.query = query
self.data = []
self.is_more = False
self.on_error = True
self.page = page
self.return_type = return_type
@abstractmethod
def get_full_search_url(self):
pass
@abstractmethod
def search(self):
pass
@abstractmethod
def get_history(self):
pass
@abstractmethod
def is_vf(self, url):
pass
async def run(self):
if self.on_error:
if self.return_type is ConnectorReturn.SEARCH:
self.search()
elif self.return_type is ConnectorReturn.HISTORY:
self.get_history()
return self
class Other(ConnectorCore):
color = "is-danger"
title = "Other"
favicon = "blank.png"
base_url = ""
is_light = True
def get_full_search_url(self):
pass
def search(self):
pass
def get_history(self):
pass
def is_vf(self, url):
return False

View File

@ -1,106 +0,0 @@
from bs4 import BeautifulSoup
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db
class Nyaa(ConnectorCore):
color = "is-link"
title = "Nyaa"
favicon = "nyaa.png"
base_url = "https://nyaa.si"
is_light = False
def get_full_search_url(self):
sort_type = "size"
if self.return_type is ConnectorReturn.HISTORY:
sort_type = "id"
to_query = "(%s vf)|(%s vostfr)|(%s multi)|(%s french)" % (
self.query,
self.query,
self.query,
self.query,
)
return "%s/?f=0&c=1_3&s=%s&o=desc&q=%s&p=%s" % (
self.base_url,
sort_type,
to_query,
self.page,
)
def get_history(self):
self.search()
@ConnectorCache.cache_data
def search(self):
response = curl_content(self.get_full_search_url())
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
trs = html.select("table.torrent-list tr")
valid_trs = 0
for i, tr in enumerate(trs):
if not i:
continue
tds = tr.findAll("td")
check_downloads = int(tds[7].get_text())
check_seeds = int(tds[5].get_text())
if check_downloads or check_seeds:
urls = tds[1].findAll("a")
if len(urls) > 1:
url = urls[1]
has_comment = True
else:
url = urls[0]
has_comment = False
url_safe = url.get_text()
if check_blacklist_words(url_safe):
continue
valid_trs = valid_trs + 1
href = self.base_url + url["href"]
self.data.append(
{
"vf": check_if_vf(url_safe),
"href": href,
"name": url_safe,
"comment": str(urls[0]).replace(
"/view/", self.base_url + "/view/"
)
if has_comment
else "",
"link": tds[2]
.decode_contents()
.replace("/download/", self.base_url + "/download/"),
"size": tds[3].get_text(),
"date": tds[4].get_text(),
"seeds": check_seeds,
"leechs": tds[6].get_text(),
"downloads": check_downloads,
"class": self.color
if link_exist_in_db(href)
else "is-%s" % tr["class"][0],
}
)
self.on_error = False
self.is_more = valid_trs and valid_trs != len(trs) - 1
@ConnectorCache.cache_data
def is_vf(self, url):
response = curl_content(url)
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
title = html.select("h3.panel-title")
return check_if_vf(title[0].get_text())
return False

View File

@ -1,107 +0,0 @@
import re
from datetime import datetime
from urllib.parse import quote
from bs4 import BeautifulSoup
from .core import ConnectorCache, ConnectorCore, ConnectorReturn, curl_content
from ..utils import check_blacklist_words, check_if_vf, link_exist_in_db, parse_date
class YggTorrent(ConnectorCore):
color = "is-success"
title = "YggTorrent"
favicon = "yggtorrent.png"
base_url = "https://www5.yggtorrent.fi"
is_light = False
category = 2179
def get_full_search_url(self):
sort_type = "size"
if self.return_type is ConnectorReturn.HISTORY:
sort_type = "publish_date"
sort_page = "&page=%s" % ((self.page - 1) * 50) if self.page > 1 else ""
return (
"%s/engine/search?name=%s&category=2145&sub_category=%s&do=search&order=desc&sort=%s%s"
% (self.base_url, self.query, self.category, sort_type, sort_page)
)
def get_history(self):
self.search()
@ConnectorCache.cache_data
def search(self):
if self.category:
response = curl_content(self.get_full_search_url(), cloudflare=True)
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
trs = html.select("table.table tr")
valid_trs = 0
for i, tr in enumerate(trs):
if not i:
continue
tds = tr.findAll("td")
check_downloads = int(tds[6].get_text())
check_seeds = int(tds[7].get_text())
if check_downloads or check_seeds:
url = tds[1].a
url_safe = url.get_text()
if check_blacklist_words(url_safe):
continue
valid_trs = valid_trs + 1
self.data.append(
{
"vf": check_if_vf(url_safe),
"href": url["href"],
"name": url_safe,
"comment": (
'<a href="%s#comm" target="_blank">'
'<i class="fa fa-comments-o"></i>%s</a>'
)
% (url["href"], tds[3].decode_contents()),
"link": '<a href="%s/engine/download_torrent?id=%s">'
'<i class="fa fa-fw fa-download"></i>'
"</a>"
% (
self.base_url,
re.search(r"/(\d+)", url["href"]).group(1),
),
"size": tds[5].get_text(),
"date": parse_date(
datetime.fromtimestamp(int(tds[4].div.get_text()))
),
"seeds": check_seeds,
"leechs": tds[8].get_text(),
"downloads": check_downloads,
"class": self.color
if link_exist_in_db(quote(url["href"], "/+:"))
else "",
}
)
self.on_error = False
self.is_more = valid_trs and valid_trs != len(trs) - 1
@ConnectorCache.cache_data
def is_vf(self, url):
response = curl_content(url)
if response["http_code"] == 200:
html = BeautifulSoup(response["output"], "html.parser")
title = html.select("#title h1")
return check_if_vf(title[0].get_text())
return False
class YggAnimation(YggTorrent):
title = "YggAnimation"
category = 2178

3
pynyaata/constants.py Normal file
View File

@ -0,0 +1,3 @@
from os import environ
VF_WORDS = environ.get("VF_WORDS", "vf,vostfr,multi,french").split(",")

63
pynyaata/filters.py Normal file
View File

@ -0,0 +1,63 @@
from functools import wraps
from os import environ
from typing import List
from pynyaata.types import Color, RemoteFile
BLACKLIST_WORDS = environ.get("BLACKLIST_WORDS", "").split(",")
def duplicate(remotes: List[RemoteFile]) -> List[RemoteFile]:
processed_ids: List[int] = []
dedup_remotes: List[RemoteFile] = []
for remote in remotes:
if remote.id not in processed_ids:
dedup_remotes.append(remote)
processed_ids.append(remote.id)
return dedup_remotes
def inactive(remotes: List[RemoteFile]) -> List[RemoteFile]:
return list(
filter(
lambda remote: remote.seeds != 0 or remote.downloads != 0,
remotes,
)
)
def blacklist(remotes: List[RemoteFile]) -> List[RemoteFile]:
return list(
filter(
lambda remote: any(word in remote.name.lower() for word in BLACKLIST_WORDS),
remotes,
)
)
def danger(remotes: List[RemoteFile]) -> List[RemoteFile]:
return list(
filter(
lambda remote: remote.color != Color.DANGER,
remotes,
)
)
def filter_data(f):
@wraps(f)
def wrapper(*args, **kwds):
ret = f(*args, **kwds)
ret = duplicate(ret)
ret = inactive(ret)
ret = blacklist(ret)
ret = danger(ret)
return ret
return wrapper

View File

@ -1,38 +0,0 @@
from flask_wtf import FlaskForm
from wtforms import HiddenField, SelectField, StringField
from wtforms.fields.html5 import SearchField, URLField
from wtforms.validators import DataRequired
class SearchForm(FlaskForm):
q = SearchField("search", validators=[DataRequired()])
class DeleteForm(FlaskForm):
class Meta:
csrf = False
id = HiddenField("id", validators=[DataRequired()])
class EditForm(FlaskForm):
id = HiddenField("id")
folder = SelectField("folder", validators=[DataRequired()])
name = StringField("name", validators=[DataRequired()])
link = URLField("link", validators=[DataRequired()])
season = StringField("season", validators=[DataRequired()])
comment = StringField("comment")
keyword = StringField("keyword", validators=[DataRequired()])
class FolderEditForm(FlaskForm):
id = HiddenField("id")
name = StringField("name", validators=[DataRequired()])
path = StringField("path")
class FolderDeleteForm(FlaskForm):
class Meta:
csrf = False
id = HiddenField("id", validators=[DataRequired()])

View File

@ -1,15 +0,0 @@
from .connectors.core import curl_content
from .models import AnimeLink
links = AnimeLink.query.all()
for link in links:
html = curl_content(link.link, debug=False, cloudflare=True)
if html["http_code"] != 200 and html["http_code"] != 500:
print(
"(%d) %s %s : %s"
% (html["http_code"], link.title.name, link.season, link.link)
)
elif "darkgray" in str(html["output"]):
print("(darkgray) %s %s : %s" % (link.title.name, link.season, link.link))

View File

@ -1,31 +0,0 @@
from .config import db
class AnimeFolder(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(length=100), unique=True, nullable=False)
path = db.Column(db.String(length=100))
titles = db.relationship(
"AnimeTitle", backref="folder", cascade="all,delete-orphan"
)
class AnimeTitle(db.Model):
id = db.Column(db.Integer, primary_key=True)
name = db.Column(db.String(length=100), unique=True, nullable=False)
keyword = db.Column(db.Text(), nullable=False)
folder_id = db.Column(db.Integer, db.ForeignKey("anime_folder.id"))
links = db.relationship("AnimeLink", backref="title", cascade="all,delete-orphan")
class AnimeLink(db.Model):
id = db.Column(db.Integer, primary_key=True)
link = db.Column(db.Text(), nullable=False)
season = db.Column(db.Text(), nullable=False)
comment = db.Column(db.Text())
vf = db.Column(db.Boolean, nullable=False)
title_id = db.Column(db.Integer, db.ForeignKey("anime_title.id"))
def create_all():
db.create_all()

50
pynyaata/requests.py Normal file
View File

@ -0,0 +1,50 @@
from dns import rdatatype, resolver
from requests import Session, adapters
from urllib.parse import urlparse
from urllib3.util.connection import HAS_IPV6
DNS_RESOLVER = resolver.Resolver()
DNS_RESOLVER.cache = resolver.LRUCache() # type: ignore
class DNSAdapter(adapters.HTTPAdapter):
def __init__(self, nameservers):
self.nameservers = nameservers
super().__init__()
def resolve(self, host, nameservers):
DNS_RESOLVER.nameservers = nameservers
if HAS_IPV6:
try:
answers_v6 = DNS_RESOLVER.query(host, rdatatype.AAAA)
for rdata_v6 in answers_v6:
return f"[{str(rdata_v6)}]"
except resolver.NoAnswer:
pass
answers_v4 = DNS_RESOLVER.query(host, rdatatype.A)
for rdata_v4 in answers_v4:
return str(rdata_v4)
def send(self, request, **kwargs):
connection_pool_kwargs = self.poolmanager.connection_pool_kw
result = urlparse(request.url)
resolved_ip = self.resolve(result.hostname, self.nameservers)
request.url = request.url.replace(result.hostname, resolved_ip)
request.headers["Host"] = result.hostname
request.headers[
"User-Agent"
] = "Googlebot/2.1 (+http://www.google.com/bot.html)"
if result.scheme == "https":
connection_pool_kwargs["server_hostname"] = result.hostname
connection_pool_kwargs["assert_hostname"] = result.hostname
return super().send(request, **kwargs)
requests = Session()
requests.mount("http://", DNSAdapter(["1.1.1.1"]))
requests.mount("https://", DNSAdapter(["1.1.1.1"]))

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

File diff suppressed because one or more lines are too long

View File

@ -1,130 +0,0 @@
html {
height: 100%;
}
body {
position: relative;
min-height: 100%;
padding-bottom: 1rem;
}
footer {
position: absolute;
bottom: 0;
width: 100%;
text-align: center;
font-size: small;
margin: 0.5rem;
color: #7a7a7a;
}
section {
overflow-x: auto;
padding-top: 2rem !important;
}
nav, nav > div {
display: flex;
align-items: center;
}
a.navbar-item, a.navbar-item:hover {
color: whitesmoke;
}
a.navbar-item:hover {
background-color: #292929;
}
div.navbar-end {
flex-basis: min-content;
}
th.error {
color: red;
}
img.favicon {
width: 16px;
height: 16px;
position: relative;
top: 2px;
}
button.fa-button {
padding: 0;
cursor: pointer;
border: none;
}
label.checkbox {
margin: 1.2rem 1.2rem 0;
}
.hidden {
display: none;
}
.table td:last-child {
white-space: nowrap;
}
.table td:last-child form {
display: inline;
}
.table td.is-primary, .table tr.is-primary {
background-color: rgba(0, 209, 178, 0.2) !important;
border-color: rgba(0, 209, 178, 0.1);
white-space: nowrap;
}
.table td.is-link, .table tr.is-link {
background-color: rgba(50, 115, 220, 0.2) !important;
border-color: rgba(50, 115, 220, 0.1);
white-space: nowrap;
}
.table td.is-info, .table tr.is-info {
background-color: rgba(32, 156, 238, 0.2) !important;
border-color: rgba(32, 156, 238, 0.1);
white-space: nowrap;
}
.table td.is-success, .table tr.is-success {
background-color: rgba(35, 209, 96, 0.2) !important;
border-color: rgba(35, 209, 96, 0.1);
white-space: nowrap;
}
.table td.is-warning, .table tr.is-warning {
background-color: rgba(255, 221, 87, 0.2) !important;
border-color: rgba(255, 221, 87, 0.1);
white-space: nowrap;
}
.table td.is-danger, .table tr.is-danger {
background-color: rgba(255, 56, 96, 0.2) !important;
border-color: rgba(255, 56, 96, 0.1);
white-space: nowrap;
}
.quick-scroll {
margin-bottom: 1rem;
}
.table.is-hoverable tbody tr:not(.is-selected):hover {
background-color: #f1f1f1 !important;
}
@media (prefers-color-scheme: dark) {
.table.is-hoverable tbody tr:not(.is-selected):hover {
background-color: #333 !important;
}
.select > select {
border-color: #363636 !important;
background-color: #0a0a0a !important;
color: #dbdbdb !important;
}
}

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.1 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 119 B

Binary file not shown.

Before

Width:  |  Height:  |  Size: 134 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 2.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.0 KiB

Binary file not shown.

Before

Width:  |  Height:  |  Size: 3.0 KiB

File diff suppressed because it is too large Load Diff

Before

Width:  |  Height:  |  Size: 434 KiB

View File

@ -1,99 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Admin Edit {{ action_form.name.data }}{% endblock %}
{% block body %}
<form method="post">
{{ action_form.csrf_token }}
<div class="field is-horizontal">
<div class="field-body">
<div class="field column">
<div class="control is-expanded">
<div class="select is-fullwidth">
{{ action_form.folder }}
</div>
</div>
</div>
<div class="field column is-6">
<div class="control is-expanded">
<div class="select is-fullwidth">
{{ action_form.name(list='animes', class='input', placeholder='Name') }}
<datalist id="animes">
{% for folder in folders %}
{% for title in folder.titles %}
<option {{ 'selected' if title.name == action_form.name.data }}
data-folder="{{ title.folder.id }}" value="{{ title.name }}"
data-keyword="{{ title.keyword }}">
{% endfor %}
{% endfor %}
</datalist>
<script>
document.getElementById('name').oninput = function (choice) {
document.getElementById('animes').childNodes.forEach(function (option) {
if (option.value === choice.target.value) {
document.getElementById('folder').value = option.dataset.folder;
document.getElementById('keyword').value = option.dataset.keyword;
}
});
};
</script>
</div>
</div>
</div>
</div>
</div>
<div class="field is-horizontal">
<div class="field-body">
<div class="field column is-6">
<div class="control is-expanded">
{{ action_form.link(class='input', placeholder='Link') }}
</div>
</div>
<div class="field column">
<div class="control is-expanded">
{{ action_form.season(class='input', placeholder='Season') }}
</div>
</div>
</div>
</div>
<div class="field is-horizontal">
<div class="field-body">
<div class="field column is-6">
<div class="control is-expanded">
{{ action_form.comment(class='input', placeholder='Comment') }}
</div>
</div>
<div class="field column">
<div class="control is-expanded">
<div class="select is-fullwidth">
{{ action_form.keyword(list='keywords', class='input', placeholder='Keyword') }}
<datalist id="keywords">
{% for folder in folders %}
{% for title in folder.titles %}
<option {{ 'selected' if title.keyword == action_form.keyword.data }}
value="{{ title.keyword }}">
{% endfor %}
{% endfor %}
</datalist>
</div>
</div>
</div>
</div>
</div>
<div class="field is-horizontal">
<div class="field-body">
<div class="field column">
<div class="control is-expanded">
<input class="button is-info" type="submit">
</div>
</div>
</div>
</div>
</form>
{% endblock %}

View File

@ -1,29 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Folder Edit {{ action_form.name.data }}{% endblock %}
{% block body %}
<form method="post">
{{ action_form.csrf_token }}
<div class="field is-horizontal">
<div class="field-body">
<div class="field column is-5">
<div class="control is-expanded">
{{ action_form.name(class='input', placeholder='Name') }}
</div>
</div>
<div class="field column is-5">
<div class="control is-expanded">
{{ action_form.path(class='input', placeholder='Path') }}
</div>
</div>
<div class="field column">
<div class="control is-expanded">
<input class="button is-info" type="submit">
</div>
</div>
</div>
</div>
</form>
{% endblock %}

View File

@ -1,47 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Folder List{% endblock %}
{% block add_button %}
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Back"
href="{{ url_for('admin') }}">
<i class="fa fa-arrow-left"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Back</span>
</a>
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Add folder"
href="{{ url_for('folder_edit') }}">
<i class="fa fa-plus"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Add folder</span>
</a>
{% endblock %}
{% block body %}
<table class="table is-bordered is-striped is-narrow is-fullwidth is-hoverable is-size-7">
<thead>
<tr>
<th>Name</th>
<th>Path</th>
<th>Tools</th>
</tr>
</thead>
<tbody>
{% for folder in folders %}
<tr>
<td>{{ folder.name }}</td>
<td>{{ folder.path }}</td>
<td>
<a href="{{ url_for('folder_edit', folder_id=folder.id) }}">
<i class="fa fa-pencil"></i>
</a>
<i>&nbsp;</i>
<form method="post">
{{ action_form.id(value=folder.id) }}
<button class="fa fa-trash fa-button"
onclick="return confirm('Are you sure you want to delete this item ?')">
</button>
</form>
</td>
</tr>
{% endfor %}
</tbody>
</table>
{% endblock %}

View File

@ -1,85 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Admin List{% endblock %}
{% block add_button %}
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Add entry" href="{{ url_for('admin_edit') }}">
<i class="fa fa-plus"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Add entry</span>
</a>
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Manage folders" href="{{ url_for('folder_list') }}">
<i class="fa fa-folder-open"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Manage folders</span>
</a>
{% endblock %}
{% block body %}
<div class="level-right quick-scroll">
<span class="level-item">Quick Scroll :</span>
{% for folder in folders %}
{% if loop.index0 %}
<a class="level-item" href="#{{ folder.name }}">{{ folder.name }}</a>
{% endif %}
{% endfor %}
</div>
<table class="table is-bordered is-striped is-narrow is-fullwidth is-hoverable is-size-7">
<thead>
<tr>
<th>Name</th>
<th>Link</th>
<th>Season</th>
<th>Comment</th>
<th>Tools</th>
</tr>
</thead>
<tbody>
{% for folder in folders %}
{% if folder.titles|length > 0 %}
<th colspan="5" id="{{ folder.name }}">{{ folder.name }}</th>
{% for title in folder.titles %}
{% for link in title.links %}
<tr>
{% if not loop.index0 %}
<td rowspan="{{ title.links|length }}">
{{ title.name }}
</td>
{% endif %}
<td class="{{ link|colorify }}">
{{ link.vf|flagify }}
{{ link.link|urlize(30, target='_blank') }}
</td>
<td>
{{ link.season }}
</td>
<td>
{{ link.comment|urlize(target='_blank') }}
</td>
<td>
<a href="{{ url_for('search', q=link.title.keyword) }}" target="_blank">
<i class="fa fa-search"></i>
</a>
<i>&nbsp;</i>
<a href="{{ url_for('admin_edit', link_id=link.id) }}">
<i class="fa fa-pencil"></i>
</a>
<i>&nbsp;</i>
<form method="post">
{{ action_form.id(value=link.id) }}
<button class="fa fa-trash fa-button"
onclick="return confirm('Are you sure you want to delete this item ?')">
</button>
</form>
</td>
</tr>
{% endfor %}
{% endfor %}
{% endif %}
{% endfor %}
</tbody>
</table>
{% endblock %}

View File

@ -1,82 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Latest torrents{% endblock %}
{% block body %}
<table class="table is-bordered is-striped is-narrow is-fullwidth is-hoverable is-size-7">
<thead>
<tr>
<th>Name</th>
<th>Link</th>
<th>Size</th>
<th>Date</th>
<th>
<i class="fa fa-arrow-up"></i>
</th>
<th>
<i class="fa fa-arrow-down"></i>
</th>
<th>
<i class="fa fa fa-check"></i>
</th>
</tr>
</thead>
<tbody>
{% for torrent in torrents %}
<tr class="{{ torrent.class }}">
<td colspan="{{ '3' if torrent.self.is_light else '' }}">
<img class="favicon"
src="{{ url_for('static', filename='favicons/%s' % torrent.self.favicon) }}" alt="">
<i>&nbsp;</i>
{{ torrent.vf|flagify }}
<a href="{{ torrent.href }}" target="_blank">
{{ torrent.name|boldify|safe }}
</a>
</td>
{% if torrent.self.is_light %}
<td>
{{ torrent.date }}
</td>
<td colspan="3">
{{ torrent.type }}
</td>
{% else %}
<td>
{{ torrent.link|safe }}
{{ torrent.comment|safe }}
</td>
<td>
{{ torrent.size }}
</td>
<td>
{{ torrent.date }}
</td>
<td>
{{ torrent.seeds }}
</td>
<td>
{{ torrent.leechs }}
</td>
<td>
{{ torrent.downloads }}
</td>
{% endif %}
</tr>
{% endfor %}
</tbody>
</table>
<nav class="pagination is-right" role="navigation" aria-label="pagination">
{% if page > 1 %}
<a class="pagination-previous" href="{{ url_for('latest', page=(page - 1)) }}">
Previous
</a>
{% endif %}
<a class="pagination-next" href="{{ url_for('latest', page=(page + 1)) }}">
Next page
</a>
<ul class="pagination-list"></ul>
</nav>
{% endblock %}

View File

@ -1,98 +0,0 @@
<!doctype html>
<html lang="fr">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0, maximum-scale=2.0, minimum-scale=1.0">
<meta http-equiv="X-UA-Compatible" content="ie=edge">
<meta name="description" content="Xefir's anime search engine (っ^‿^)っ">
<title>PyNyaaTa - {% block title %}{% endblock %}</title>
<link rel="icon" href="{{ url_for('static', filename='favicons/favicon.ico') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='css/bulma.min.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='css/bulma-prefers-dark.min.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='css/bulma-tooltip.min.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='css/font-awesome.min.css') }}">
<link rel="stylesheet" href="{{ url_for('static', filename='css/styles.css') }}">
</head>
<body>
<nav class="navbar is-dark" role="navigation" aria-label="main navigation">
<div class="navbar-start">
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Home" href="{{ url_for('home') }}">
π 😼た
</a>
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Latest torrents" href="{{ url_for('latest') }}">
<i class="fa fa-newspaper-o"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Latest torrents</span>
</a>
{% if not db_disabled %}
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="My seeded torrents"
href="{{ url_for('list_animes') }}">
<i class="fa fa-cloud-download"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">My seeded torrents</span>
</a>
{% endif %}
{% block add_button %}{% endblock %}
{% if request.args.get('q') %}
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="TVDB"
href="https://www.thetvdb.com/search?menu%5Btype%5D=series&query={{ request.args.get('q') }}"
target="_blank">
<i class="fa fa-television"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">TVDB</span>
</a>
<a class="navbar-item has-tooltip-bottom has-tooltip-hidden-desktop" data-tooltip="Nautiljon"
href="https://www.nautiljon.com/search.php?q={{ request.args.get('q') }}" target="_blank">
<i class="fa fa-rss"></i><i>&nbsp;</i>
<span class="is-hidden-mobile">Nautiljon</span>
</a>
{% endif %}
</div>
<div class="navbar-end">
<form action="{{ url_for('search') }}" class="navbar-item">
<div class="field has-addons">
<div class="control">
{{ search_form.q(placeholder='Search ...', class='input', value=request.args.get('q', '')) }}
</div>
<div class="control">
<button type="submit" class="button is-info">
<i class="fa fa-search"></i>
</button>
</div>
</div>
</form>
</div>
</nav>
<section class="section" role="main">
{% if action_form %}
{% if action_form.errors %}
<div class="notification is-danger">
<button class="delete" onclick="this.parentNode.style.display = 'none'"></button>
<ul>
{% for field in action_form.errors %}
{% for error in action_form.errors[field] %}
<li>"{{ field }}" => {{ error }}</li>
{% endfor %}
{% endfor %}
</ul>
</div>
{% endif %}
{% if action_form.message %}
<div class="notification is-success">
<button class="delete" onclick="this.parentNode.style.display = 'none'"></button>
{{ action_form.message }}
</div>
{% endif %}
{% endif %}
{% block body %}{% endblock %}
</section>
<footer>
<b>PyNyaata</b> made by <i>Xéfir Destiny</i>.
This software is open source under <a target="_blank" href="http://www.wtfpl.net">WTFPL</a> license !
Please look at the <a target="_blank" href="https://git.crystalyx.net/Xefir/PyNyaaTa">source code</a>
or <a target="_blank" href="https://hub.docker.com/r/xefir/pynyaata">host it</a> yourself o/
</footer>
</body>
</html>

View File

@ -1,46 +0,0 @@
{% extends "layout.html" %}
{% block title %}- My seeded torrents{% endblock %}
{% block body %}
<table class="table is-bordered is-striped is-narrow is-fullwidth is-hoverable is-size-7">
<thead>
<tr>
<th>Name</th>
<th>Link</th>
<th>Season</th>
<th>Tools</th>
</tr>
</thead>
<tbody>
{% for title in titles.values() %}
{% for link in title %}
<tr>
{% if not loop.index0 %}
<td rowspan="{{ title|length }}">
{{ link.title.name }}
</td>
{% endif %}
<td class="{{ link|colorify }}">
{{ link.vf|flagify }}
{{ link.link|urlize(30, target='_blank') }}
</td>
<td>
{{ link.season }}
</td>
{% if not loop.index0 %}
<td rowspan="{{ title|length }}">
<a href="{{ url_for('search', q=link.title.keyword) }}" target="_blank">
<i class="fa fa-search"></i>
</a>
</td>
{% endif %}
</tr>
{% endfor %}
{% endfor %}
</tbody>
</table>
{% endblock %}

View File

@ -1,91 +0,0 @@
{% extends "layout.html" %}
{% block title %}- Search for "{{ request.args.get('q') }}"{% endblock %}
{% block body %}
<table class="table is-bordered is-striped is-narrow is-fullwidth is-hoverable">
<thead>
<tr>
<th>Name</th>
<th>
<i class="fa fa-comment"></i>
</th>
<th>Link</th>
<th>Size</th>
<th>Date</th>
<th>
<i class="fa fa-arrow-up"></i>
</th>
<th>
<i class="fa fa-arrow-down"></i>
</th>
<th>
<i class="fa fa fa-check"></i>
</th>
</tr>
</thead>
<tbody>
{% for connector in connectors %}
{% if connector.data|length > 0 or connector.is_more %}
<th colspan="8">{{ connector.title }}</th>
{% endif %}
{% for torrent in connector.data %}
<tr class="{{ torrent.class }}">
<td>
{{ torrent.vf|flagify }}
<a href="{{ torrent.href }}" target="_blank">
{{ torrent.name|boldify|safe }}
</a>
</td>
{% if connector.is_light %}
<td colspan="7">
{{ torrent.type }}
</td>
{% else %}
<td>
{{ torrent.comment|safe }}
</td>
<td>
{{ torrent.link|safe }}
</td>
<td>
{{ torrent.size }}
</td>
<td>
{{ torrent.date }}
</td>
<td>
{{ torrent.seeds }}
</td>
<td>
{{ torrent.leechs }}
</td>
<td>
{{ torrent.downloads }}
</td>
{% endif %}
</tr>
{% endfor %}
{% if connector.is_more %}
<tr>
<th colspan="8">
<a href="{{ connector.get_full_search_url() }}" target="_blank">More ...</a>
</th>
</tr>
{% endif %}
{% if connector.on_error %}
<tr class="is-danger">
<th colspan="8" class="error">
Error, can't grab data from {{ connector.title }}
<a href="{{ connector.get_full_search_url() }}" target="_blank">Go to the website -></a>
</th>
</tr>
{% endif %}
{% endfor %}
</tbody>
</table>
{% endblock %}

63
pynyaata/types.py Normal file
View File

@ -0,0 +1,63 @@
from abc import ABC, abstractmethod
from datetime import datetime
from enum import Enum
from typing import List, Optional
from pydantic import BaseModel, ByteSize, HttpUrl
class Color(str, Enum):
WHITE = "white"
BLACK = "black"
LIGHT = "light"
DARK = "dark"
PRIMARY = "primary"
LINK = "link"
INFO = "info"
SUCCESS = "success"
WARNING = "warning"
DANGER = "danger"
DEFAULT = "default"
class RemoteFile(BaseModel):
id: int
category: str
color: Optional[Color]
name: str
link: HttpUrl
comment: int = 0
comment_url: HttpUrl
magnet: Optional[str]
torrent: Optional[HttpUrl]
size: Optional[ByteSize]
date: Optional[datetime]
seeds: Optional[int]
leechs: Optional[int]
downloads: Optional[int]
nb_pages: int = 1
class Bridge(BaseModel, ABC):
color: Color
title: str
favicon: HttpUrl
base_url: HttpUrl
@abstractmethod
def search_url(self, query: str = "", page: int = 1) -> HttpUrl:
pass
@abstractmethod
def search(self, query: str = "", page: int = 1) -> List[RemoteFile]:
pass
class Cache(ABC):
@abstractmethod
def get(self, key: str) -> Optional[List[RemoteFile]]:
pass
@abstractmethod
def set(self, key: str, data: List[RemoteFile]):
pass

View File

@ -1,45 +0,0 @@
import re
from datetime import datetime
from dateparser import parse
from .config import BLACKLIST_WORDS, DB_ENABLED
def link_exist_in_db(href):
if DB_ENABLED:
from .models import AnimeLink
return AnimeLink.query.filter_by(link=href).first()
return False
def parse_date(str_to_parse, date_format=""):
if str_to_parse is None:
date_to_format = datetime.fromtimestamp(0)
elif isinstance(str_to_parse, datetime):
date_to_format = str_to_parse
else:
date = parse(str_to_parse, date_formats=[date_format])
if date:
date_to_format = date
else:
date_to_format = datetime.fromtimestamp(0)
return date_to_format.isoformat(" ", "minutes")
def boldify(str_to_replace, keyword):
if keyword:
return re.sub(
"(%s)" % keyword, r"<b>\1</b>", str_to_replace, flags=re.IGNORECASE
)
else:
return str_to_replace
def check_blacklist_words(url):
return any(word.lower() in url.lower() for word in BLACKLIST_WORDS)
def check_if_vf(title):
return any(word.lower() in title.lower() for word in ["vf", "multi", "french"])

View File

@ -1,16 +1,12 @@
[tool.poetry] [tool.poetry]
name = "pynyaata" name = "pynyaata"
version = "2.0.0" version = "2.0.0"
description = "π 😼た, Xéfir's personal animes torrent search engine" description = "π 😼た, Xéfir's personal anime torrent search engine"
authors = ["Xéfir Destiny <xefir@crystalyx.net>"] authors = ["Xéfir Destiny"]
license = "WTFPL" license = "WTFPL"
readme = "README.md" readme = "README.md"
homepage = "https://nyaa.crystalyx.net/" homepage = "https://nyaa.crystalyx.net/"
repository = "https://git.crystalyx.net/Xefir/PyNyaaTa" repository = "https://git.crystalyx.net/Xefir/PyNyaaTa"
classifiers = [
"Programming Language :: Python :: 3",
"Operating System :: OS Independent"
]
[tool.poetry.scripts] [tool.poetry.scripts]
@ -19,36 +15,27 @@ pynyaata = 'pynyaata:run'
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.7" python = "^3.7"
Flask = "^2.2.2" beautifulsoup4 = "4.11.1"
Flask-SQLAlchemy = "^2.5.1" dateparser = "1.1.1"
Flask-HTTPAuth = "^4.7.0" dnspython = "2.2.1"
Flask-WTF = "^1.0.1" Flask = "2.2.2"
WTForms = "^3.0.1" pydantic = "1.10.2"
PyMySQL = "^1.0.2" redis = "4.3.4"
pg8000 = "^1.29.1" requests = "2.28.1"
requests = "^2.28.1" urllib3 = "1.26.12"
beautifulsoup4 = "^4.11.1"
python-dotenv = "^0.20.0"
dateparser = "^1.1.1"
redis = "^4.3.4"
transmission-rpc = "^3.3.2"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
flake8 = "3.9.2" black = "22.10.0"
black = "^22.8.0" flake8-alphabetize = "0.0.17"
mypy = "^0.971" flake8-black = "0.3.3"
mypy = "0.982"
types-beautifulsoup4 = "4.11.6"
types-dateparser = "1.1.4"
types-redis = "4.3.21.1"
types-requests = "2.28.11.2"
djlint = "1.9.3" djlint = "1.9.3"
pytest = "^7.1.2" flake8 = "3.9.2"
pytest-cov = "^3.0.0"
flake8-black = "^0.3.3"
flake8-alphabetize = "^0.0.17"
types-dateparser = "^1.1.4"
types-redis = "^4.3.19"
types-requests = "^2.28.9"
Flask-HTTPAuth-stubs = "^0.1.5"
types-Flask-SQLAlchemy = "^2.5.9"
types-beautifulsoup4 = "^4.11.5"
[build-system] [build-system]

View File

@ -1,3 +0,0 @@
{
"$schema": "https://docs.renovatebot.com/renovate-schema.json"
}