Remove database, Update deps, Add trusted, Update FlareSolverr

pynyaata/__init__.py

@@ -1,19 +1,16 @@
 import asyncio
 from datetime import datetime
 from os import getenv
-from secrets import token_hex
 
 from flask import Flask, redirect, render_template, request, url_for
 
 from pynyaata.bridge import search_all
-from pynyaata.forms import SearchForm
 from pynyaata.translations import current_lang, i18n
 
 from sentry_sdk import init
 from sentry_sdk.integrations.asyncio import AsyncioIntegration
 from sentry_sdk.integrations.flask import FlaskIntegration
 from sentry_sdk.integrations.redis import RedisIntegration
-from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
 
 
 SENTRY_DNS = getenv("SENTRY_DSN")
@@ -24,13 +21,11 @@ if SENTRY_DNS:
             AsyncioIntegration(),
             FlaskIntegration(),
             RedisIntegration(),
-            SqlalchemyIntegration(),
         ],
     )
 
 
 app = Flask(__name__)
-app.config["SECRET_KEY"] = token_hex()
 
 
 @app.context_processor
@@ -44,7 +39,6 @@ def utility_processor():
     return dict(
         _=i18n,
         current_lang=current_lang(),
-        search_form=SearchForm(),
         apocalypse_left=apocalypse_left,
     )
 

pynyaata/filters.py

@@ -36,7 +36,8 @@ def blacklist(remotes: List[RemoteFile]) -> List[RemoteFile]:
         return list(
             filter(
                 lambda remote: not any(
-                    word in remote.name.lower() for word in BLACKLIST_WORDS.split(",")
+                    word and word in remote.name.lower()
+                    for word in BLACKLIST_WORDS.split(",")
                 ),
                 remotes,
             )
@@ -52,6 +53,18 @@ def danger(remotes: List[RemoteFile]) -> List[RemoteFile]:
     )
 
 
+def trusted(remotes: List[RemoteFile]) -> List[RemoteFile]:
+    TRUSTED_WORDS = getenv("TRUSTED_WORDS", "")
+
+    for remote in remotes:
+        if any(
+            word and word in remote.name.lower() for word in TRUSTED_WORDS.split(",")
+        ):
+            remote.color = Color.PRIMARY
+
+    return remotes
+
+
 def filter_data(f):
     @wraps(f)
     def wrapper(*args, **kwargs):
@@ -61,6 +74,7 @@ def filter_data(f):
         ret = inactive(ret)
         ret = blacklist(ret)
         ret = danger(ret)
+        ret = trusted(ret)
 
         return ret
 

pynyaata/forms.py

@@ -1,6 +0,0 @@
-import flask_wtf  # type:ignore
-import wtforms  # type:ignore
-
-
-class SearchForm(flask_wtf.FlaskForm):
-    q = wtforms.SearchField("search", validators=[wtforms.validators.DataRequired()])

pynyaata/session.py

@@ -2,6 +2,7 @@ from io import BytesIO
 from os import getenv
 from urllib import parse
 
+from charset_normalizer import detect
 from requests import RequestException, Response, Session, post
 
 
@@ -16,13 +17,13 @@ class FlareRequests(Session):
         sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
 
         if "sessions" in sessions and len(sessions["sessions"]) > 0:
-            FLARESESSION = sessions["sessions"][0]
+            FLARE_SESSION = sessions["sessions"][0]
         else:
             response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
             session = response.json()
 
             if "session" in session:
-                FLARESESSION = session["session"]
+                FLARE_SESSION = session["session"]
             else:
                 raise RequestException(response)
 
@@ -32,7 +33,7 @@ class FlareRequests(Session):
 
         post_data = {
             "cmd": f"request.{method.lower()}",
-            "session": FLARESESSION,
+            "session": FLARE_SESSION,
             "url": url,
         }
 
@@ -47,27 +48,30 @@ class FlareRequests(Session):
 
             content = response.json()
 
-            if "solution" in content:
+            if "solution" in content and content["solution"]:
                 solution = content["solution"]
+                raw = solution["response"].encode()
+                encoding = detect(raw)
 
                 resolved = Response()
                 resolved.status_code = solution["status"]
                 resolved.headers = solution["headers"]
-                resolved.raw = BytesIO(solution["response"].encode())
+                resolved.raw = BytesIO(raw)
                 resolved.url = url
+                resolved.encoding = encoding["encoding"]
                 resolved.reason = content["status"]
                 resolved.cookies = solution["cookies"]
 
                 return resolved
 
-            raise RequestException(response)
+            raise RequestException(content["message"], response=response)
         except RequestException:
             session = post(
                 CLOUDPROXY_ENDPOINT,
-                json={"cmd": "sessions.destroy", "session": FLARESESSION},
+                json={"cmd": "sessions.destroy", "session": FLARE_SESSION},
             )
 
-            raise RequestException(solution)
+            raise RequestException(content["message"], response=response)
 
 
 requests = FlareRequests()

pyproject.toml

@@ -15,30 +15,26 @@ pynyaata = 'pynyaata:run'
 
 [tool.poetry.dependencies]
 python = "^3.7"
-beautifulsoup4 = "^4.11.1"
+beautifulsoup4 = "^4.12.0"
-flask = "^2.2.2"
+charset-normalizer = "^3.1.0"
-flask-httpauth = "^4.7.0"
+flask = "^2.2.3"
-flask-sqlalchemy = "^3.0.2"
+pydantic = "^1.10.7"
-flask-wtf = "^1.1.1"
+redis = "^4.5.3"
-pydantic = "^1.10.4"
-redis = "^4.4.2"
 requests = "^2.28.2"
-sentry-sdk = {extras = ["flask", "sqlalchemy"], version = "^1.14.0"}
+sentry-sdk = {extras = ["flask"], version = "^1.17.0"}
-sqlalchemy = {extras = ["pymysql", "postgresql-pg8000"], version = "^1.4.46"}
 
 
 [tool.poetry.group.dev.dependencies]
-black = "^22.12.0"
+black = "^23.1.0"
 flake8-alphabetize = "^0.0.19"
 flake8-black = "^0.3.6"
-mypy = "^0.991"
+mypy = "^1.1.1"
-pytest = "^7.2.1"
+pytest = "^7.2.2"
-pytest-asyncio = "^0.20.3"
+pytest-asyncio = "^0.21.0"
 requests-mock = "^1.10.0"
-sqlalchemy = {extras = ["mypy"], version = "^1.4.46"}
+types-beautifulsoup4 = "^4.12.0.0"
-types-beautifulsoup4 = "^4.11.6.4"
+types-redis = "^4.5.2.0"
-types-redis = "^4.4.0.3"
+types-requests = "^2.28.11.16"
-types-requests = "^2.28.11.8"
 djlint = "^1.9.3"
 flake8 = "^3.9.2"
 pydantic-factories = "^1.15.0"

tests/filters_test.py

@@ -1,6 +1,6 @@
 from pydantic_factories import ModelFactory
 
-from pynyaata.filters import blacklist, danger, duplicate, inactive
+from pynyaata.filters import blacklist, danger, duplicate, inactive, trusted
 from pynyaata.types import Color, RemoteFile
 
 from pytest import MonkeyPatch
@@ -41,3 +41,15 @@ def test_inactive():
     remotes[0].downloads = 0
 
     assert len(inactive(remotes)) == 9
+
+
+def test_trusted(monkeypatch: MonkeyPatch):
+    monkeypatch.setenv("TRUSTED_WORDS", "one,two")
+    remotes = RemoteFileFactory.batch(10)
+    remotes[0].name = "oui one non"
+    remotes[1].name = "non two oui"
+
+    alter_remotes = trusted(remotes)
+
+    assert alter_remotes[0].color == Color.PRIMARY
+    assert alter_remotes[1].color == Color.PRIMARY