diff --git a/poetry.lock b/poetry.lock index 94d41d9..772bd34 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1071,6 +1071,26 @@ tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] testing = ["argcomplete", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +[[package]] +name = "pytest-asyncio" +version = "0.20.3" +description = "Pytest support for asyncio" +category = "dev" +optional = false +python-versions = ">=3.7" +files = [ + {file = "pytest-asyncio-0.20.3.tar.gz", hash = "sha256:83cbf01169ce3e8eb71c6c278ccb0574d1a7a3bb8eaaf5e50e0ad342afb33b36"}, + {file = "pytest_asyncio-0.20.3-py3-none-any.whl", hash = "sha256:f129998b209d04fcc65c96fc85c11e5316738358909a8399e93be553d7656442"}, +] + +[package.dependencies] +pytest = ">=6.1.0" +typing-extensions = {version = ">=3.7.2", markers = "python_version < \"3.8\""} + +[package.extras] +docs = ["sphinx (>=5.3)", "sphinx-rtd-theme (>=1.0)"] +testing = ["coverage (>=6.2)", "flaky (>=3.5.0)", "hypothesis (>=5.7.1)", "mypy (>=0.931)", "pytest-trio (>=0.7.0)"] + [[package]] name = "pytest-cov" version = "3.0.0" @@ -1324,6 +1344,26 @@ urllib3 = ">=1.21.1,<1.27" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] +[[package]] +name = "requests-mock" +version = "1.10.0" +description = "Mock out responses from the requests package" +category = "dev" +optional = false +python-versions = "*" +files = [ + {file = "requests-mock-1.10.0.tar.gz", hash = "sha256:59c9c32419a9fb1ae83ec242d98e889c45bd7d7a65d48375cc243ec08441658b"}, + {file = "requests_mock-1.10.0-py2.py3-none-any.whl", hash = "sha256:2fdbb637ad17ee15c06f33d31169e71bf9fe2bdb7bc9da26185be0dd8d842699"}, +] + +[package.dependencies] +requests = ">=2.3,<3" +six = "*" + +[package.extras] +fixture = ["fixtures"] +test = ["fixtures", "mock", "purl", "pytest", "requests-futures", "sphinx", "testrepository (>=0.0.18)", "testtools"] + [[package]] name = "scramp" version = "1.4.4" @@ -1769,4 +1809,4 @@ testing = ["flake8 (<5)", "func-timeout", "jaraco.functools", "jaraco.itertools" [metadata] lock-version = "2.0" python-versions = "^3.7" -content-hash = "73237cfcb4d545f3e1e170c084fae328a7a253135bb99bdb8d5750dc627936c3" +content-hash = "a151e081694d505f2ff9e037e1b45e52ab63b1b824baf8e2253bd059c43bb739" diff --git a/pynyaata/cache/__init__.py b/pynyaata/cache/__init__.py index 20b74b8..07671e8 100644 --- a/pynyaata/cache/__init__.py +++ b/pynyaata/cache/__init__.py @@ -4,7 +4,7 @@ from os import getenv from typing import Optional from pynyaata.cache.simple import SimpleCache -from pynyaata.types import Cache +from pynyaata.types import Bridge, Cache from redis import RedisError @@ -24,19 +24,16 @@ if REDIS_URL: def cache_data(f): @wraps(f) - async def wrapper(*args, **kwargs): - bridge = args[0] - query = args[1] - page = args[2] + async def wrapper(bridge: Bridge, query: str = "", page: int = 1): key = f"pynyaata.{bridge.__class__.__name__}.{f.__name__}.{query}.{page}" - ret = client.get(key) + cached = client.get(key) - if ret: - return ret + if cached: + return cached - ret = await f(*args, **kwargs) - client.set(key, ret) + reals = await f(bridge, query, page) + client.set(key, reals) - return ret + return reals return wrapper diff --git a/pynyaata/filters.py b/pynyaata/filters.py index 141b5a9..74c61d8 100644 --- a/pynyaata/filters.py +++ b/pynyaata/filters.py @@ -5,9 +5,6 @@ from typing import List from pynyaata.types import Color, RemoteFile -BLACKLIST_WORDS = getenv("BLACKLIST_WORDS", "").split(",") - - def duplicate(remotes: List[RemoteFile]) -> List[RemoteFile]: processed_ids: List[int] = [] dedup_remotes: List[RemoteFile] = [] @@ -31,9 +28,13 @@ def inactive(remotes: List[RemoteFile]) -> List[RemoteFile]: def blacklist(remotes: List[RemoteFile]) -> List[RemoteFile]: + BLACKLIST_WORDS = getenv("BLACKLIST_WORDS", "").split(",") + return list( filter( - lambda remote: any(word in remote.name.lower() for word in BLACKLIST_WORDS), + lambda remote: not any( + word in remote.name.lower() for word in BLACKLIST_WORDS + ), remotes, ) ) diff --git a/pyproject.toml b/pyproject.toml index ff07df6..d82c903 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,6 +36,8 @@ flake8-alphabetize = "0.0.19" flake8-black = "0.3.6" mypy = "0.991" pytest = "7.2.0" +pytest-asyncio = "0.20.3" +requests-mock = "1.10.0" types-beautifulsoup4 = "4.11.6.2" types-dateparser = "1.1.4.4" types-humanfriendly = "10.0.1.3" diff --git a/tests/bridge/test_nyaa.py b/tests/bridge/test_nyaa.py new file mode 100644 index 0000000..d447a04 --- /dev/null +++ b/tests/bridge/test_nyaa.py @@ -0,0 +1,42 @@ +from typing import List + +from pynyaata.bridge.nyaa import Nyaa +from pynyaata.types import RemoteFile + +from pytest import mark +import requests +from requests_mock import Mocker + + +def test_search_url(): + assert ( + Nyaa().search_url() + == "https://nyaa.si?f=0&c=1_3&q=%28+vf%29%7C%28+vostfr%29%7C%28+multi%29%7C%28+fre%29&s=id&o=desc&p=1" + ) + + assert ( + Nyaa().search_url("", 2) + == "https://nyaa.si?f=0&c=1_3&q=%28+vf%29%7C%28+vostfr%29%7C%28+multi%29%7C%28+fre%29&s=id&o=desc&p=2" + ) + + assert ( + Nyaa().search_url("test", 1) + == "https://nyaa.si?f=0&c=1_3&q=%28test+vf%29%7C%28test+vostfr%29%7C%28test+multi%29%7C%28test+fre%29&s=size&o=desc&p=1" + ) + + assert ( + Nyaa().search_url("test", 2) + == "https://nyaa.si?f=0&c=1_3&q=%28test+vf%29%7C%28test+vostfr%29%7C%28test+multi%29%7C%28test+fre%29&s=size&o=desc&p=2" + ) + + +@mark.asyncio +async def test_search(requests_mock: Mocker): + requests_mock.real_http = True + requests_mock.get( + Nyaa().search_url(), text=requests.get("https://nyaa.si/user/Chaussette33").text + ) + + remotes: List[RemoteFile] = [] + + assert await Nyaa().search() == remotes diff --git a/tests/cache_test.py b/tests/cache_test.py index 4858262..b0d5c04 100644 --- a/tests/cache_test.py +++ b/tests/cache_test.py @@ -4,7 +4,7 @@ from pynyaata.cache import client from pynyaata.types import RemoteFile -class RemoteFileFactory(ModelFactory): +class RemoteFileFactory(ModelFactory[RemoteFile]): __model__ = RemoteFile diff --git a/tests/filters_test.py b/tests/filters_test.py new file mode 100644 index 0000000..df2cba8 --- /dev/null +++ b/tests/filters_test.py @@ -0,0 +1,43 @@ +from pydantic_factories import ModelFactory + +from pynyaata.filters import blacklist, danger, duplicate, inactive +from pynyaata.types import Color, RemoteFile + +from pytest import MonkeyPatch + + +class RemoteFileFactory(ModelFactory[RemoteFile]): + __model__ = RemoteFile + color = Color.DEFAULT + + +def test_blacklist(monkeypatch: MonkeyPatch): + monkeypatch.setenv("BLACKLIST_WORDS", "one,two") + remotes = RemoteFileFactory.batch(10) + remotes[0].name = "oui one non" + remotes[1].name = "non two oui" + + assert len(blacklist(remotes)) == 8 + + +def test_danger(): + remotes = RemoteFileFactory.batch(10) + remotes[0].color = Color.DANGER + + assert len(danger(remotes)) == 9 + + +def test_duplicate(): + remotes = RemoteFileFactory.batch(10) + remotes[0].id = 1 + remotes[1].id = 1 + + assert len(duplicate(remotes)) == 9 + + +def test_inactive(): + remotes = RemoteFileFactory.batch(10) + remotes[0].seeds = 0 + remotes[0].downloads = 0 + + assert len(inactive(remotes)) == 9