Remove database, Update deps, Add trusted, Update FlareSolverr
This commit is contained in:
parent
1e0f1a76af
commit
30c05d9eac
859
poetry.lock
generated
859
poetry.lock
generated
File diff suppressed because it is too large
Load Diff
@ -1,19 +1,16 @@
|
||||
import asyncio
|
||||
from datetime import datetime
|
||||
from os import getenv
|
||||
from secrets import token_hex
|
||||
|
||||
from flask import Flask, redirect, render_template, request, url_for
|
||||
|
||||
from pynyaata.bridge import search_all
|
||||
from pynyaata.forms import SearchForm
|
||||
from pynyaata.translations import current_lang, i18n
|
||||
|
||||
from sentry_sdk import init
|
||||
from sentry_sdk.integrations.asyncio import AsyncioIntegration
|
||||
from sentry_sdk.integrations.flask import FlaskIntegration
|
||||
from sentry_sdk.integrations.redis import RedisIntegration
|
||||
from sentry_sdk.integrations.sqlalchemy import SqlalchemyIntegration
|
||||
|
||||
|
||||
SENTRY_DNS = getenv("SENTRY_DSN")
|
||||
@ -24,13 +21,11 @@ if SENTRY_DNS:
|
||||
AsyncioIntegration(),
|
||||
FlaskIntegration(),
|
||||
RedisIntegration(),
|
||||
SqlalchemyIntegration(),
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
app = Flask(__name__)
|
||||
app.config["SECRET_KEY"] = token_hex()
|
||||
|
||||
|
||||
@app.context_processor
|
||||
@ -44,7 +39,6 @@ def utility_processor():
|
||||
return dict(
|
||||
_=i18n,
|
||||
current_lang=current_lang(),
|
||||
search_form=SearchForm(),
|
||||
apocalypse_left=apocalypse_left,
|
||||
)
|
||||
|
||||
|
@ -36,7 +36,8 @@ def blacklist(remotes: List[RemoteFile]) -> List[RemoteFile]:
|
||||
return list(
|
||||
filter(
|
||||
lambda remote: not any(
|
||||
word in remote.name.lower() for word in BLACKLIST_WORDS.split(",")
|
||||
word and word in remote.name.lower()
|
||||
for word in BLACKLIST_WORDS.split(",")
|
||||
),
|
||||
remotes,
|
||||
)
|
||||
@ -52,6 +53,18 @@ def danger(remotes: List[RemoteFile]) -> List[RemoteFile]:
|
||||
)
|
||||
|
||||
|
||||
def trusted(remotes: List[RemoteFile]) -> List[RemoteFile]:
|
||||
TRUSTED_WORDS = getenv("TRUSTED_WORDS", "")
|
||||
|
||||
for remote in remotes:
|
||||
if any(
|
||||
word and word in remote.name.lower() for word in TRUSTED_WORDS.split(",")
|
||||
):
|
||||
remote.color = Color.PRIMARY
|
||||
|
||||
return remotes
|
||||
|
||||
|
||||
def filter_data(f):
|
||||
@wraps(f)
|
||||
def wrapper(*args, **kwargs):
|
||||
@ -61,6 +74,7 @@ def filter_data(f):
|
||||
ret = inactive(ret)
|
||||
ret = blacklist(ret)
|
||||
ret = danger(ret)
|
||||
ret = trusted(ret)
|
||||
|
||||
return ret
|
||||
|
||||
|
@ -1,6 +0,0 @@
|
||||
import flask_wtf # type:ignore
|
||||
import wtforms # type:ignore
|
||||
|
||||
|
||||
class SearchForm(flask_wtf.FlaskForm):
|
||||
q = wtforms.SearchField("search", validators=[wtforms.validators.DataRequired()])
|
@ -2,6 +2,7 @@ from io import BytesIO
|
||||
from os import getenv
|
||||
from urllib import parse
|
||||
|
||||
from charset_normalizer import detect
|
||||
from requests import RequestException, Response, Session, post
|
||||
|
||||
|
||||
@ -16,13 +17,13 @@ class FlareRequests(Session):
|
||||
sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
|
||||
|
||||
if "sessions" in sessions and len(sessions["sessions"]) > 0:
|
||||
FLARESESSION = sessions["sessions"][0]
|
||||
FLARE_SESSION = sessions["sessions"][0]
|
||||
else:
|
||||
response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
|
||||
session = response.json()
|
||||
|
||||
if "session" in session:
|
||||
FLARESESSION = session["session"]
|
||||
FLARE_SESSION = session["session"]
|
||||
else:
|
||||
raise RequestException(response)
|
||||
|
||||
@ -32,7 +33,7 @@ class FlareRequests(Session):
|
||||
|
||||
post_data = {
|
||||
"cmd": f"request.{method.lower()}",
|
||||
"session": FLARESESSION,
|
||||
"session": FLARE_SESSION,
|
||||
"url": url,
|
||||
}
|
||||
|
||||
@ -47,27 +48,30 @@ class FlareRequests(Session):
|
||||
|
||||
content = response.json()
|
||||
|
||||
if "solution" in content:
|
||||
if "solution" in content and content["solution"]:
|
||||
solution = content["solution"]
|
||||
raw = solution["response"].encode()
|
||||
encoding = detect(raw)
|
||||
|
||||
resolved = Response()
|
||||
resolved.status_code = solution["status"]
|
||||
resolved.headers = solution["headers"]
|
||||
resolved.raw = BytesIO(solution["response"].encode())
|
||||
resolved.raw = BytesIO(raw)
|
||||
resolved.url = url
|
||||
resolved.encoding = encoding["encoding"]
|
||||
resolved.reason = content["status"]
|
||||
resolved.cookies = solution["cookies"]
|
||||
|
||||
return resolved
|
||||
|
||||
raise RequestException(response)
|
||||
raise RequestException(content["message"], response=response)
|
||||
except RequestException:
|
||||
session = post(
|
||||
CLOUDPROXY_ENDPOINT,
|
||||
json={"cmd": "sessions.destroy", "session": FLARESESSION},
|
||||
json={"cmd": "sessions.destroy", "session": FLARE_SESSION},
|
||||
)
|
||||
|
||||
raise RequestException(solution)
|
||||
raise RequestException(content["message"], response=response)
|
||||
|
||||
|
||||
requests = FlareRequests()
|
||||
|
@ -15,30 +15,26 @@ pynyaata = 'pynyaata:run'
|
||||
|
||||
[tool.poetry.dependencies]
|
||||
python = "^3.7"
|
||||
beautifulsoup4 = "^4.11.1"
|
||||
flask = "^2.2.2"
|
||||
flask-httpauth = "^4.7.0"
|
||||
flask-sqlalchemy = "^3.0.2"
|
||||
flask-wtf = "^1.1.1"
|
||||
pydantic = "^1.10.4"
|
||||
redis = "^4.4.2"
|
||||
beautifulsoup4 = "^4.12.0"
|
||||
charset-normalizer = "^3.1.0"
|
||||
flask = "^2.2.3"
|
||||
pydantic = "^1.10.7"
|
||||
redis = "^4.5.3"
|
||||
requests = "^2.28.2"
|
||||
sentry-sdk = {extras = ["flask", "sqlalchemy"], version = "^1.14.0"}
|
||||
sqlalchemy = {extras = ["pymysql", "postgresql-pg8000"], version = "^1.4.46"}
|
||||
sentry-sdk = {extras = ["flask"], version = "^1.17.0"}
|
||||
|
||||
|
||||
[tool.poetry.group.dev.dependencies]
|
||||
black = "^22.12.0"
|
||||
black = "^23.1.0"
|
||||
flake8-alphabetize = "^0.0.19"
|
||||
flake8-black = "^0.3.6"
|
||||
mypy = "^0.991"
|
||||
pytest = "^7.2.1"
|
||||
pytest-asyncio = "^0.20.3"
|
||||
mypy = "^1.1.1"
|
||||
pytest = "^7.2.2"
|
||||
pytest-asyncio = "^0.21.0"
|
||||
requests-mock = "^1.10.0"
|
||||
sqlalchemy = {extras = ["mypy"], version = "^1.4.46"}
|
||||
types-beautifulsoup4 = "^4.11.6.4"
|
||||
types-redis = "^4.4.0.3"
|
||||
types-requests = "^2.28.11.8"
|
||||
types-beautifulsoup4 = "^4.12.0.0"
|
||||
types-redis = "^4.5.2.0"
|
||||
types-requests = "^2.28.11.16"
|
||||
djlint = "^1.9.3"
|
||||
flake8 = "^3.9.2"
|
||||
pydantic-factories = "^1.15.0"
|
||||
|
@ -1,6 +1,6 @@
|
||||
from pydantic_factories import ModelFactory
|
||||
|
||||
from pynyaata.filters import blacklist, danger, duplicate, inactive
|
||||
from pynyaata.filters import blacklist, danger, duplicate, inactive, trusted
|
||||
from pynyaata.types import Color, RemoteFile
|
||||
|
||||
from pytest import MonkeyPatch
|
||||
@ -41,3 +41,15 @@ def test_inactive():
|
||||
remotes[0].downloads = 0
|
||||
|
||||
assert len(inactive(remotes)) == 9
|
||||
|
||||
|
||||
def test_trusted(monkeypatch: MonkeyPatch):
|
||||
monkeypatch.setenv("TRUSTED_WORDS", "one,two")
|
||||
remotes = RemoteFileFactory.batch(10)
|
||||
remotes[0].name = "oui one non"
|
||||
remotes[1].name = "non two oui"
|
||||
|
||||
alter_remotes = trusted(remotes)
|
||||
|
||||
assert alter_remotes[0].color == Color.PRIMARY
|
||||
assert alter_remotes[1].color == Color.PRIMARY
|
||||
|
Reference in New Issue
Block a user