Use FlareSolverr

This commit is contained in:
Michel Roux 2022-12-21 22:32:42 +00:00
parent 434fbeb9fa
commit 910cc4e692
7 changed files with 1008 additions and 1110 deletions

1968
poetry.lock generated

File diff suppressed because it is too large Load Diff

View File

@ -5,10 +5,9 @@ from bs4 import BeautifulSoup
import dateparser import dateparser
from pydantic import HttpUrl, parse_obj_as from pydantic import HttpUrl, parse_obj_as
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError import requests
class AnimeUltime(Bridge): class AnimeUltime(Bridge):
@ -40,7 +39,7 @@ class AnimeUltime(Bridge):
) )
if response.status_code != 200: if response.status_code != 200:
raise HTTPError() raise requests.HTTPError(response)
torrents: List[RemoteFile] = [] torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser") html = BeautifulSoup(response.content, "html.parser")
@ -83,7 +82,7 @@ class AnimeUltime(Bridge):
name=tds[0].get_text(), name=tds[0].get_text(),
link=f"{self.base_url}{tds[0].a['href']}", link=f"{self.base_url}{tds[0].a['href']}",
date=dateparser.parse( date=dateparser.parse(
h3s[i].get_text()[:-3], "%A %d %B %Y" h3s[i].get_text()[:-3], ["%A %d %B %Y"]
), ),
) )
) )

View File

@ -1,16 +1,15 @@
from datetime import datetime from datetime import datetime
from typing import List from typing import List
from urllib.parse import urlencode
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
from pydantic import HttpUrl, parse_obj_as from pydantic import HttpUrl, parse_obj_as
from pynyaata.cache import cache_data from pynyaata.cache import cache_data
from pynyaata.constants import VF_WORDS from pynyaata.constants import VF_WORDS
from pynyaata.requests import requests
from pynyaata.types import Bridge, Color, RemoteFile from pynyaata.types import Bridge, Color, RemoteFile
from requests import HTTPError import requests
from urllib.parse import urlencode
class Nyaa(Bridge): class Nyaa(Bridge):
@ -39,7 +38,7 @@ class Nyaa(Bridge):
response = requests.get(self.search_url(query, page)) response = requests.get(self.search_url(query, page))
if response.status_code != 200: if response.status_code != 200:
raise HTTPError() raise requests.HTTPError(response)
torrents: List[RemoteFile] = [] torrents: List[RemoteFile] = []
html = BeautifulSoup(response.content, "html.parser") html = BeautifulSoup(response.content, "html.parser")

View File

@ -1,3 +1,4 @@
from os import environ from os import environ
VF_WORDS = environ.get("VF_WORDS", "vf,vostfr,multi,french").split(",") VF_WORDS = environ.get("VF_WORDS", "vf,vostfr,multi,french").split(",")
CLOUDPROXY_ENDPOINT = environ.get("CLOUDPROXY_ENDPOINT")

View File

@ -1,50 +0,0 @@
from dns import rdatatype, resolver
from requests import Session, adapters
from urllib.parse import urlparse
from urllib3.util.connection import HAS_IPV6
DNS_RESOLVER = resolver.Resolver()
DNS_RESOLVER.cache = resolver.LRUCache() # type: ignore
class DNSAdapter(adapters.HTTPAdapter):
def __init__(self, nameservers):
self.nameservers = nameservers
super().__init__()
def resolve(self, host, nameservers):
DNS_RESOLVER.nameservers = nameservers
if HAS_IPV6:
try:
answers_v6 = DNS_RESOLVER.query(host, rdatatype.AAAA)
for rdata_v6 in answers_v6:
return f"[{str(rdata_v6)}]"
except resolver.NoAnswer:
pass
answers_v4 = DNS_RESOLVER.query(host, rdatatype.A)
for rdata_v4 in answers_v4:
return str(rdata_v4)
def send(self, request, **kwargs):
connection_pool_kwargs = self.poolmanager.connection_pool_kw
result = urlparse(request.url)
resolved_ip = self.resolve(result.hostname, self.nameservers)
request.url = request.url.replace(result.hostname, resolved_ip)
request.headers["Host"] = result.hostname
request.headers[
"User-Agent"
] = "Googlebot/2.1 (+http://www.google.com/bot.html)"
if result.scheme == "https":
connection_pool_kwargs["server_hostname"] = result.hostname
connection_pool_kwargs["assert_hostname"] = result.hostname
return super().send(request, **kwargs)
requests = Session()
requests.mount("http://", DNSAdapter(["1.1.1.1"]))
requests.mount("https://", DNSAdapter(["1.1.1.1"]))

63
pynyaata/session.py Normal file
View File

@ -0,0 +1,63 @@
from io import BytesIO
from urllib import parse
from pynyaata.constants import CLOUDPROXY_ENDPOINT
from requests import RequestException, Response, Session, post
class FlareRequests(Session):
def request(self, method, url, params=None, **kwargs):
if not CLOUDPROXY_ENDPOINT:
return super().request(method, url, params, **kwargs)
sessions = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}).json()
if "sessions" in sessions and len(sessions["sessions"]) > 0:
FLARESESSION = sessions["sessions"][0]
else:
response = post(CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"})
session = response.json()
if "session" in session:
FLARESESSION = session["session"]
else:
raise RequestException(response)
post_data = {
"cmd": f"request.{method}",
"session": FLARESESSION,
"url": url,
}
if params:
post_data["postData"] = parse.urlencode(params)
try:
response = post(
CLOUDPROXY_ENDPOINT,
json=post_data,
)
solution = response.json()
if "solution" in solution:
resolved = Response()
resolved.raw = BytesIO(solution["solution"]["response"].encode())
resolved.status_code = solution["solution"]["status"]
resolved.headers = solution["solution"]["headers"]
resolved.url = url
resolved.reason = solution["status"]
resolved.cookies = solution["solution"]["cookies"]
return resolved
raise RequestException(response)
except RequestException:
session = post(
CLOUDPROXY_ENDPOINT,
json={"cmd": "sessions.destroy", "session": FLARESESSION},
)
raise RequestException(solution)

View File

@ -16,24 +16,22 @@ pynyaata = 'pynyaata:run'
[tool.poetry.dependencies] [tool.poetry.dependencies]
python = "^3.7" python = "^3.7"
beautifulsoup4 = "4.11.1" beautifulsoup4 = "4.11.1"
dateparser = "1.1.1" dateparser = "1.1.4"
dnspython = "2.2.1"
Flask = "2.2.2" Flask = "2.2.2"
pydantic = "1.10.2" pydantic = "1.10.2"
redis = "4.3.4" redis = "4.4.0"
requests = "2.28.1" requests = "2.28.1"
urllib3 = "1.26.12"
[tool.poetry.group.dev.dependencies] [tool.poetry.group.dev.dependencies]
black = "22.10.0" black = "22.12.0"
flake8-alphabetize = "0.0.17" flake8-alphabetize = "0.0.19"
flake8-black = "0.3.3" flake8-black = "0.3.6"
mypy = "0.982" mypy = "0.991"
types-beautifulsoup4 = "4.11.6" types-beautifulsoup4 = "4.11.6.1"
types-dateparser = "1.1.4" types-dateparser = "1.1.4.4"
types-redis = "4.3.21.1" types-redis = "4.3.21.6"
types-requests = "2.28.11.2" types-requests = "2.28.11.6"
djlint = "1.9.3" djlint = "1.9.3"
flake8 = "3.9.2" flake8 = "3.9.2"