Remove database, Update deps, Add trusted, Update FlareSolverr

This commit is contained in:
Michel Roux 2023-03-23 00:20:25 +01:00
parent 1e0f1a76af
commit 30c05d9eac
7 changed files with 333 additions and 618 deletions

859
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -1,19 +1,16 @@
import asyncio import asyncio
from datetime import datetime from datetime import datetime
from os import getenv from os import getenv
from secrets import token_hex
from flask import Flask, redirect, render_template, request, url_for from flask import Flask, redirect, render_template, request, url_for
from pynyaata.bridge import search_all from pynyaata.bridge import search_all
from pynyaata.forms import SearchForm
from pynyaata.translations import current_lang, i18n from pynyaata.translations import current_lang, i18n
from sentry_sdk import init from sentry_sdk import init
from sentry_sdk.integrations.asyncio import AsyncioIntegration from sentry_sdk.integrations.asyncio import AsyncioIntegration
from sentry_sdk.integrations.flask import FlaskIntegration from sentry_sdk.integrations.flask import FlaskIntegration
from sentry_sdk.integrations.redis import RedisIntegration from sentry_sdk.integrations.redis import RedisIntegration
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
SENTRY_DNS = getenv("SENTRY_DSN") SENTRY_DNS = getenv("SENTRY_DSN")
@ -24,13 +21,11 @@ if SENTRY_DNS:
AsyncioIntegration(), AsyncioIntegration(),
FlaskIntegration(), FlaskIntegration(),
RedisIntegration(), RedisIntegration(),
SqlalchemyIntegration(),
], ],
) )
app = Flask(__name__) app = Flask(__name__)
app.config["SECRET_KEY"] = token_hex()
@app.context_processor @app.context_processor
@ -44,7 +39,6 @@ def utility_processor():
return dict( return dict(
_=i18n, _=i18n,
current_lang=current_lang(), current_lang=current_lang(),
search_form=SearchForm(),
apocalypse_left=apocalypse_left, apocalypse_left=apocalypse_left,
) )

View File

@ -36,7 +36,8 @@ def blacklist(remotes: List[RemoteFile]) -> List[RemoteFile]:
return list( return list(
filter( filter(
lambda remote: not any( lambda remote: not any(
word in remote.name.lower() for word in BLACKLIST_WORDS.split(",") word and word in remote.name.lower()
for word in BLACKLIST_WORDS.split(",")
), ),
remotes, remotes,
) )
@ -52,6 +53,18 @@ def danger(remotes: List[RemoteFile]) -> List[RemoteFile]:
) )
def trusted(remotes: List[RemoteFile]) -> List[RemoteFile]:
TRUSTED_WORDS = getenv("TRUSTED_WORDS", "")
for remote in remotes:
if any(
word and word in remote.name.lower() for word in TRUSTED_WORDS.split(",")
):
remote.color = Color.PRIMARY
return remotes
def filter_data(f): def filter_data(f):
@wraps(f) @wraps(f)
def wrapper(*args, **kwargs): def wrapper(*args, **kwargs):
@ -61,6 +74,7 @@ def filter_data(f):
ret = inactive(ret) ret = inactive(ret)
ret = blacklist(ret) ret = blacklist(ret)
ret = danger(ret) ret = danger(ret)
ret = trusted(ret)
return ret return ret

View File

@ -1,6 +0,0 @@
import flask_wtf # type:ignore
import wtforms # type:ignore
class SearchForm(flask_wtf.FlaskForm):
q = wtforms.SearchField("search", validators=[wtforms.validators.DataRequired()])

View File

@ -2,6 +2,7 @@ from io import BytesIO
from os import getenv from os import getenv
from urllib import parse from urllib import parse
from charset_normalizer import detect
from requests import RequestException, Response, Session, post from requests import RequestException, Response, Session, post
@ -16,13 +17,13 @@ class FlareRequests(Session):
sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json() sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
if "sessions" in sessions and len(sessions["sessions"]) > 0: if "sessions" in sessions and len(sessions["sessions"]) > 0:
FLARESESSION = sessions["sessions"][0] FLARE_SESSION = sessions["sessions"][0]
else: else:
response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"}) response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
session = response.json() session = response.json()
if "session" in session: if "session" in session:
FLARESESSION = session["session"] FLARE_SESSION = session["session"]
else: else:
raise RequestException(response) raise RequestException(response)
@ -32,7 +33,7 @@ class FlareRequests(Session):
post_data = { post_data = {
"cmd": f"request.{method.lower()}", "cmd": f"request.{method.lower()}",
"session": FLARESESSION, "session": FLARE_SESSION,
"url": url, "url": url,
} }
@ -47,27 +48,30 @@ class FlareRequests(Session):
content = response.json() content = response.json()
if "solution" in content: if "solution" in content and content["solution"]:
solution = content["solution"] solution = content["solution"]
raw = solution["response"].encode()
encoding = detect(raw)
resolved = Response() resolved = Response()
resolved.status_code = solution["status"] resolved.status_code = solution["status"]
resolved.headers = solution["headers"] resolved.headers = solution["headers"]
resolved.raw = BytesIO(solution["response"].encode()) resolved.raw = BytesIO(raw)
resolved.url = url resolved.url = url
resolved.encoding = encoding["encoding"]
resolved.reason = content["status"] resolved.reason = content["status"]
resolved.cookies = solution["cookies"] resolved.cookies = solution["cookies"]
return resolved return resolved
raise RequestException(response) raise RequestException(content["message"], response=response)
except RequestException: except RequestException:
session = post( session = post(
CLOUDPROXY_ENDPOINT, CLOUDPROXY_ENDPOINT,
json={"cmd": "sessions.destroy", "session": FLARESESSION}, json={"cmd": "sessions.destroy", "session": FLARE_SESSION},
) )
raise RequestException(solution) raise RequestException(content["message"], response=response)
requests = FlareRequests() requests = FlareRequests()

View File

@ -15,30 +15,26 @@ pynyaata = 'pynyaata:run'
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.7" python = "^3.7"
beautifulsoup4 = "^4.11.1" beautifulsoup4 = "^4.12.0"
flask = "^2.2.2" charset-normalizer = "^3.1.0"
flask-httpauth = "^4.7.0" flask = "^2.2.3"
flask-sqlalchemy = "^3.0.2" pydantic = "^1.10.7"
flask-wtf = "^1.1.1" redis = "^4.5.3"
pydantic = "^1.10.4"
redis = "^4.4.2"
requests = "^2.28.2" requests = "^2.28.2"
sentry-sdk = {extras = ["flask", "sqlalchemy"], version = "^1.14.0"} sentry-sdk = {extras = ["flask"], version = "^1.17.0"}
sqlalchemy = {extras = ["pymysql", "postgresql-pg8000"], version = "^1.4.46"}
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
black = "^22.12.0" black = "^23.1.0"
flake8-alphabetize = "^0.0.19" flake8-alphabetize = "^0.0.19"
flake8-black = "^0.3.6" flake8-black = "^0.3.6"
mypy = "^0.991" mypy = "^1.1.1"
pytest = "^7.2.1" pytest = "^7.2.2"
pytest-asyncio = "^0.20.3" pytest-asyncio = "^0.21.0"
requests-mock = "^1.10.0" requests-mock = "^1.10.0"
sqlalchemy = {extras = ["mypy"], version = "^1.4.46"} types-beautifulsoup4 = "^4.12.0.0"
types-beautifulsoup4 = "^4.11.6.4" types-redis = "^4.5.2.0"
types-redis = "^4.4.0.3" types-requests = "^2.28.11.16"
types-requests = "^2.28.11.8"
djlint = "^1.9.3" djlint = "^1.9.3"
flake8 = "^3.9.2" flake8 = "^3.9.2"
pydantic-factories = "^1.15.0" pydantic-factories = "^1.15.0"

View File

@ -1,6 +1,6 @@
from pydantic_factories import ModelFactory from pydantic_factories import ModelFactory
from pynyaata.filters import blacklist, danger, duplicate, inactive from pynyaata.filters import blacklist, danger, duplicate, inactive, trusted
from pynyaata.types import Color, RemoteFile from pynyaata.types import Color, RemoteFile
from pytest import MonkeyPatch from pytest import MonkeyPatch
@ -41,3 +41,15 @@ def test_inactive():
remotes[0].downloads = 0 remotes[0].downloads = 0
assert len(inactive(remotes)) == 9 assert len(inactive(remotes)) == 9
def test_trusted(monkeypatch: MonkeyPatch):
monkeypatch.setenv("TRUSTED_WORDS", "one,two")
remotes = RemoteFileFactory.batch(10)
remotes[0].name = "oui one non"
remotes[1].name = "non two oui"
alter_remotes = trusted(remotes)
assert alter_remotes[0].color == Color.PRIMARY
assert alter_remotes[1].color == Color.PRIMARY