Use FlareSolverr

This commit is contained in:
Michel Roux 2022-12-21 22:32:42 +00:00
parent 434fbeb9fa
commit 910cc4e692
7 changed files with 1008 additions and 1110 deletions

1968
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -5,10 +5,9 @@ from bs4 import BeautifulSoup
import dateparser
from pydantic import HttpUrl, parse_obj_as
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError
import requests
class AnimeUltime(Bridge):
@ -40,7 +39,7 @@ class AnimeUltime(Bridge):
)
if response.status_code != 200:
raise HTTPError()
raise requests.HTTPError(response)
torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser")
@ -83,7 +82,7 @@ class AnimeUltime(Bridge):
name=tds[0].get_text(),
link=f"{self.base_url}{tds[0].a['href']}",
date=dateparser.parse(
h3s[i].get_text()[:-3], "%A %d %B %Y"
h3s[i].get_text()[:-3], ["%A %d %B %Y"]
),
)
)

View File

@ -1,16 +1,15 @@
from datetime import datetime
from typing import List
from urllib.parse import urlencode
from bs4 import BeautifulSoup
from pydantic import HttpUrl, parse_obj_as
from pynyaata.cache import cache_data
from pynyaata.constants import VF_WORDS
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError
from urllib.parse import urlencode
import requests
class Nyaa(Bridge):
@ -39,7 +38,7 @@ class Nyaa(Bridge):
response = requests.get(self.search_url(query, page))
if response.status_code != 200:
raise HTTPError()
raise requests.HTTPError(response)
torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser")

View File

@ -1,3 +1,4 @@
from os import environ
VF_WORDS = environ.get("VF_WORDS", "vf,vostfr,multi,french").split(",")
CLOUDPROXY_ENDPOINT = environ.get("CLOUDPROXY_ENDPOINT")

View File

@ -1,50 +0,0 @@
from dns import rdatatype, resolver
from requests import Session, adapters
from urllib.parse import urlparse
from urllib3.util.connection import HAS_IPV6
DNS_RESOLVER = resolver.Resolver()
DNS_RESOLVER.cache = resolver.LRUCache() # type: ignore
class DNSAdapter(adapters.HTTPAdapter):
def __init__(self, nameservers):
self.nameservers = nameservers
super().__init__()
def resolve(self, host, nameservers):
DNS_RESOLVER.nameservers = nameservers
if HAS_IPV6:
try:
answers_v6 = DNS_RESOLVER.query(host, rdatatype.AAAA)
for rdata_v6 in answers_v6:
return f"[{str(rdata_v6)}]"
except resolver.NoAnswer:
pass
answers_v4 = DNS_RESOLVER.query(host, rdatatype.A)
for rdata_v4 in answers_v4:
return str(rdata_v4)
def send(self, request, **kwargs):
connection_pool_kwargs = self.poolmanager.connection_pool_kw
result = urlparse(request.url)
resolved_ip = self.resolve(result.hostname, self.nameservers)
request.url = request.url.replace(result.hostname, resolved_ip)
request.headers["Host"] = result.hostname
request.headers[
"User-Agent"
] = "Googlebot/2.1 (+http://www.google.com/bot.html)"
if result.scheme == "https":
connection_pool_kwargs["server_hostname"] = result.hostname
connection_pool_kwargs["assert_hostname"] = result.hostname
return super().send(request, **kwargs)
requests = Session()
requests.mount("http://", DNSAdapter(["1.1.1.1"]))
requests.mount("https://", DNSAdapter(["1.1.1.1"]))

63
pynyaata/session.py Normal file
View File

@ -0,0 +1,63 @@
from io import BytesIO
from urllib import parse
from pynyaata.constants import CLOUDPROXY_ENDPOINT
from requests import RequestException, Response, Session, post
class FlareRequests(Session):
def request(self, method, url, params=None, **kwargs):
if not CLOUDPROXY_ENDPOINT:
return super().request(method, url, params, **kwargs)
sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
if "sessions" in sessions and len(sessions["sessions"]) > 0:
FLARESESSION = sessions["sessions"][0]
else:
response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
session = response.json()
if "session" in session:
FLARESESSION = session["session"]
else:
raise RequestException(response)
post_data = {
"cmd": f"request.{method}",
"session": FLARESESSION,
"url": url,
}
if params:
post_data["postData"] = parse.urlencode(params)
try:
response = post(
CLOUDPROXY_ENDPOINT,
json=post_data,
)
solution = response.json()
if "solution" in solution:
resolved = Response()
resolved.raw = BytesIO(solution["solution"]["response"].encode())
resolved.status_code = solution["solution"]["status"]
resolved.headers = solution["solution"]["headers"]
resolved.url = url
resolved.reason = solution["status"]
resolved.cookies = solution["solution"]["cookies"]
return resolved
raise RequestException(response)
except RequestException:
session = post(
CLOUDPROXY_ENDPOINT,
json={"cmd": "sessions.destroy", "session": FLARESESSION},
)
raise RequestException(solution)

View File

@ -16,24 +16,22 @@ pynyaata = 'pynyaata:run'
[tool.poetry.dependencies]
python = "^3.7"
beautifulsoup4 = "4.11.1"
dateparser = "1.1.1"
dnspython = "2.2.1"
dateparser = "1.1.4"
Flask = "2.2.2"
pydantic = "1.10.2"
redis = "4.3.4"
redis = "4.4.0"
requests = "2.28.1"
urllib3 = "1.26.12"
[tool.poetry.group.dev.dependencies]
black = "22.10.0"
flake8-alphabetize = "0.0.17"
flake8-black = "0.3.3"
mypy = "0.982"
types-beautifulsoup4 = "4.11.6"
types-dateparser = "1.1.4"
types-redis = "4.3.21.1"
types-requests = "2.28.11.2"
black = "22.12.0"
flake8-alphabetize = "0.0.19"
flake8-black = "0.3.6"
mypy = "0.991"
types-beautifulsoup4 = "4.11.6.1"
types-dateparser = "1.1.4.4"
types-redis = "4.3.21.6"
types-requests = "2.28.11.6"
djlint = "1.9.3"
flake8 = "3.9.2"