From dbb758649612eb75fb85686f0b6b35812c55ce5e Mon Sep 17 00:00:00 2001 From: Michel Roux Date: Thu, 22 Dec 2022 00:01:21 +0000 Subject: [PATCH] Stash --- pynyaata/__init__.py | 2 ++ pynyaata/bridge/animeultime.py | 7 +++++++ pynyaata/bridge/nyaa.py | 8 +++++++- pynyaata/cache/__init__.py | 10 +++++++--- pynyaata/cache/redis.py | 14 +++++++++----- pynyaata/cache/simple.py | 3 +++ pynyaata/constants.py | 4 ---- pynyaata/filters.py | 4 ++-- pynyaata/session.py | 6 ++++-- pynyaata/types.py | 1 + 10 files changed, 42 insertions(+), 17 deletions(-) delete mode 100644 pynyaata/constants.py diff --git a/pynyaata/__init__.py b/pynyaata/__init__.py index e69de29..399d803 100644 --- a/pynyaata/__init__.py +++ b/pynyaata/__init__.py @@ -0,0 +1,2 @@ +def run(): + print("oui") diff --git a/pynyaata/bridge/animeultime.py b/pynyaata/bridge/animeultime.py index 191e8f2..a792f50 100644 --- a/pynyaata/bridge/animeultime.py +++ b/pynyaata/bridge/animeultime.py @@ -5,6 +5,8 @@ from bs4 import BeautifulSoup import dateparser from pydantic import HttpUrl, parse_obj_as +from pynyaata.cache import cache_data +from pynyaata.filters import filter_data from pynyaata.types import Bridge, Color, RemoteFile import requests @@ -31,6 +33,8 @@ class AnimeUltime(Bridge): ), ) + @cache_data + @filter_data def search(self, query: str = "", page: int = 1) -> List[RemoteFile]: response = ( requests.post(self.search_url(query, page), {"search": query}) @@ -59,6 +63,7 @@ class AnimeUltime(Bridge): torrents.append( RemoteFile( + bridge=self.__class__.__name__, id=tds[0].a["href"].split("/")[1].split("-")[0], category=tds[1].get_text(), name=tds[0].get_text(), @@ -77,6 +82,7 @@ class AnimeUltime(Bridge): torrents.append( RemoteFile( + bridge=self.__class__.__name__, id=tds[0].a["href"].split("/")[-2], category=tds[4].get_text(), name=tds[0].get_text(), @@ -89,6 +95,7 @@ class AnimeUltime(Bridge): elif player and title and history and tables: torrents.append( RemoteFile( + bridge=self.__class__.__name__, id=player["data-serie"], category=title.get_text(), name=history.get_text(), diff --git a/pynyaata/bridge/nyaa.py b/pynyaata/bridge/nyaa.py index dd40cc7..d8f13ca 100644 --- a/pynyaata/bridge/nyaa.py +++ b/pynyaata/bridge/nyaa.py @@ -1,4 +1,5 @@ from datetime import datetime +from os import getenv from typing import List from urllib.parse import urlencode @@ -6,12 +7,15 @@ from bs4 import BeautifulSoup from pydantic import HttpUrl, parse_obj_as from pynyaata.cache import cache_data -from pynyaata.constants import VF_WORDS +from pynyaata.filters import filter_data from pynyaata.types import Bridge, Color, RemoteFile import requests +VF_WORDS = getenv("VF_WORDS", "vf,vostfr,multi,fre").split(",") + + class Nyaa(Bridge): color = Color.INFO title = "Nyaa" @@ -34,6 +38,7 @@ class Nyaa(Bridge): return parse_obj_as(HttpUrl, f"{self.base_url}?{params}") @cache_data + @filter_data def search(self, query: str = "", page: int = 1) -> List[RemoteFile]: response = requests.get(self.search_url(query, page)) @@ -54,6 +59,7 @@ class Nyaa(Bridge): torrents.append( RemoteFile( + bridge=self.__class__.__name__, id=urls[1 if len(urls) > 1 else 0]["href"].split("/")[-1], category=tds[0].a["title"], color=Color[tr["class"][0].upper()], diff --git a/pynyaata/cache/__init__.py b/pynyaata/cache/__init__.py index 5610658..ed5ab5e 100644 --- a/pynyaata/cache/__init__.py +++ b/pynyaata/cache/__init__.py @@ -1,15 +1,19 @@ import logging from functools import wraps -from os import environ +from os import getenv +from typing import Optional + from pynyaata.cache.simple import SimpleCache from pynyaata.types import Cache + from redis import RedisError -CACHE_TIMEOUT = int(environ.get("CACHE_TIMEOUT", 60 * 60)) -REDIS_URL = environ.get("REDIS_URL", "") +CACHE_TIMEOUT = int(getenv("CACHE_TIMEOUT", 60 * 60)) +REDIS_URL: Optional[str] = getenv("REDIS_URL") client: Cache = SimpleCache() + if REDIS_URL: try: from pynyaata.cache.redis import RedisCache diff --git a/pynyaata/cache/redis.py b/pynyaata/cache/redis.py index b944cd1..b4fb507 100644 --- a/pynyaata/cache/redis.py +++ b/pynyaata/cache/redis.py @@ -1,18 +1,22 @@ from json import dumps, loads -from os import environ from typing import List, Optional -from pynyaata.cache import CACHE_TIMEOUT + +from pynyaata.cache import CACHE_TIMEOUT, REDIS_URL from pynyaata.types import Cache, RemoteFile -from redis import Redis + +from redis import ConnectionError, Redis -REDIS_URL = environ.get("REDIS_URL", "") +if not REDIS_URL: + raise ConnectionError(f"Invalid REDIS_URL: {REDIS_URL}") + client = Redis.from_url(REDIS_URL) class RedisCache(Cache): def get(self, key: str) -> Optional[List[RemoteFile]]: - return loads(str(client.get(key))) + data = client.get(key) + return loads(str(data)) if data else None def set(self, key: str, data: List[RemoteFile]): return client.set(key, dumps(data), CACHE_TIMEOUT) diff --git a/pynyaata/cache/simple.py b/pynyaata/cache/simple.py index 1ec010e..b6655c8 100644 --- a/pynyaata/cache/simple.py +++ b/pynyaata/cache/simple.py @@ -1,5 +1,6 @@ from datetime import datetime, timedelta from typing import Dict, List, Optional, Tuple + from pynyaata.cache import CACHE_TIMEOUT from pynyaata.types import Cache, RemoteFile @@ -11,10 +12,12 @@ class SimpleCache(Cache): def get(self, key: str) -> Optional[List[RemoteFile]]: if key in CACHE_DATA: data, timeout = CACHE_DATA[key] + if datetime.now() > timeout + timedelta(seconds=CACHE_TIMEOUT): return data else: CACHE_DATA.pop(key) + return None def set(self, key: str, data: List[RemoteFile]): diff --git a/pynyaata/constants.py b/pynyaata/constants.py deleted file mode 100644 index f027a30..0000000 --- a/pynyaata/constants.py +++ /dev/null @@ -1,4 +0,0 @@ -from os import environ - -VF_WORDS = environ.get("VF_WORDS", "vf,vostfr,multi,french").split(",") -CLOUDPROXY_ENDPOINT = environ.get("CLOUDPROXY_ENDPOINT") diff --git a/pynyaata/filters.py b/pynyaata/filters.py index 519c1cd..d32c314 100644 --- a/pynyaata/filters.py +++ b/pynyaata/filters.py @@ -1,11 +1,11 @@ from functools import wraps -from os import environ +from os import getenv from typing import List from pynyaata.types import Color, RemoteFile -BLACKLIST_WORDS = environ.get("BLACKLIST_WORDS", "").split(",") +BLACKLIST_WORDS = getenv("BLACKLIST_WORDS", "").split(",") def duplicate(remotes: List[RemoteFile]) -> List[RemoteFile]: diff --git a/pynyaata/session.py b/pynyaata/session.py index a1a431e..a41592a 100644 --- a/pynyaata/session.py +++ b/pynyaata/session.py @@ -1,11 +1,13 @@ from io import BytesIO +from os import getenv from urllib import parse -from pynyaata.constants import CLOUDPROXY_ENDPOINT - from requests import RequestException, Response, Session, post +CLOUDPROXY_ENDPOINT = getenv("CLOUDPROXY_ENDPOINT") + + class FlareRequests(Session): def request(self, method, url, params=None, **kwargs): if not CLOUDPROXY_ENDPOINT: diff --git a/pynyaata/types.py b/pynyaata/types.py index e358625..23ca567 100644 --- a/pynyaata/types.py +++ b/pynyaata/types.py @@ -21,6 +21,7 @@ class Color(str, Enum): class RemoteFile(BaseModel): + bridge: str id: int category: str color: Optional[Color]