From b7a1fb40d0a9a2b25221c8dcf219ba7bcab4e4ed Mon Sep 17 00:00:00 2001 From: Michel Roux Date: Sat, 24 Dec 2022 10:47:15 +0000 Subject: [PATCH] Use FlareRequests for pygg --- .flake8 | 2 + Dockerfile | 2 +- commands/pygg.py | 132 +++++++++++++++++++++++++++-------------------- 3 files changed, 79 insertions(+), 57 deletions(-) create mode 100644 .flake8 diff --git a/.flake8 b/.flake8 new file mode 100644 index 0000000..7da1f96 --- /dev/null +++ b/.flake8 @@ -0,0 +1,2 @@ +[flake8] +max-line-length = 100 diff --git a/Dockerfile b/Dockerfile index d445b50..4b6859e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -4,7 +4,7 @@ RUN apt-get update && \ apt-get install -y \ vim p7zip* git rsync lftp speedtest-cli rename wget curl procps psmisc \ openssh-client transmission-cli yt-dlp \ - python3-requests python3-bs4 python3-dnspython && \ + python3-requests python3-bs4 && \ rm -rf /var/lib/apt/lists/* RUN curl -sSL https://raw.githubusercontent.com/tremc/tremc/master/tremc -o /usr/local/bin/tremc && \ curl -sSL https://github.com/drone/drone-cli/releases/latest/download/drone_linux_amd64.tar.gz | tar xvz && \ diff --git a/commands/pygg.py b/commands/pygg.py index a3f64d2..cf3ecfd 100644 --- a/commands/pygg.py +++ b/commands/pygg.py @@ -1,19 +1,82 @@ #!/usr/bin/env python3 import argparse +import io +import os import re -from urllib.parse import urlencode, urlparse +import urllib -from bs4 import BeautifulSoup -from dns import rdatatype, resolver -from requests import Session, adapters -from urllib3.util.connection import HAS_IPV6 +import bs4 +import requests -BLACKLIST_WORDS = ["dvd", "iso"] -YGGTORRENT_BASE_URL = "https://www5.yggtorrent.fi" + +YGGTORRENT_BASE_URL = "https://www6.yggtorrent.lol" +CLOUDPROXY_ENDPOINT = os.getenv("CLOUDPROXY_ENDPOINT") + + +class FlareRequests(requests.Session): + def request(self, method, url, params=None, **kwargs): + if not CLOUDPROXY_ENDPOINT: + return super().request(method, url, params, **kwargs) + + sessions = requests.post( + CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"} + ).json() + + if "sessions" in sessions and len(sessions["sessions"]) > 0: + FLARESESSION = sessions["sessions"][0] + else: + response = requests.post( + CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"} + ) + session = response.json() + + if "session" in session: + FLARESESSION = session["session"] + else: + raise requests.RequestException(response) + + post_data = { + "cmd": f"request.{method.lower()}", + "session": FLARESESSION, + "url": url, + } + + if params: + post_data["postData"] = urllib.parse.urlencode(params) + + try: + response = requests.post( + CLOUDPROXY_ENDPOINT, + json=post_data, + ) + + solution = response.json() + + if "solution" in solution: + resolved = requests.Response() + + resolved.raw = io.BytesIO(solution["solution"]["response"].encode()) + resolved.status_code = solution["solution"]["status"] + resolved.headers = solution["solution"]["headers"] + resolved.url = url + resolved.reason = solution["status"] + resolved.cookies = solution["solution"]["cookies"] + + return resolved + + raise requests.RequestException(response) + except requests.RequestException: + session = requests.post( + CLOUDPROXY_ENDPOINT, + json={"cmd": "sessions.destroy", "session": FLARESESSION}, + ) + + raise requests.RequestException(solution) parser = argparse.ArgumentParser() parser.add_argument("-u", "--uploader", action="append") +parser.add_argument("-b", "--blacklist", action="append", default=["dvd", "iso"]) parser.add_argument("-y", "--year", type=int) parser.add_argument("-s", "--size", type=int, default=10) parser.add_argument("-d", "--downloads", type=int, default=20) @@ -29,56 +92,13 @@ def parse_size(size): return int(float(number) * units[unit]) -DNS_RESOLVER = resolver.Resolver() -DNS_RESOLVER.cache = resolver.LRUCache() # type: ignore - - -class DNSAdapter(adapters.HTTPAdapter): - def __init__(self, nameservers): - self.nameservers = nameservers - super().__init__() - - def resolve(self, host, nameservers): - DNS_RESOLVER.nameservers = nameservers - - if HAS_IPV6: - try: - answers_v6 = DNS_RESOLVER.query(host, rdatatype.AAAA) - for rdata_v6 in answers_v6: - return f"[{str(rdata_v6)}]" - except resolver.NoAnswer: - pass - - answers_v4 = DNS_RESOLVER.query(host, rdatatype.A) - for rdata_v4 in answers_v4: - return str(rdata_v4) - - def send(self, request, **kwargs): - connection_pool_kwargs = self.poolmanager.connection_pool_kw - result = urlparse(request.url) - resolved_ip = self.resolve(result.hostname, self.nameservers) - request.url = request.url.replace(result.hostname, resolved_ip) - request.headers["Host"] = result.hostname - request.headers[ - "User-Agent" - ] = "Googlebot/2.1 (+http://www.google.com/bot.html)" - - if result.scheme == "https": - connection_pool_kwargs["server_hostname"] = result.hostname - connection_pool_kwargs["assert_hostname"] = result.hostname - - return super().send(request, **kwargs) - - -session = Session() -session.mount("http://", DNSAdapter(["1.1.1.1"])) -session.mount("https://", DNSAdapter(["1.1.1.1"])) +session = FlareRequests() def check_files(id): req = session.get(f"{YGGTORRENT_BASE_URL}/engine/get_files", params={"torrent": id}) files = req.json() - html = BeautifulSoup(files["html"], "html.parser") + html = bs4.BeautifulSoup(files["html"], "html.parser") trs = html.select("tr") return len(trs) == 1 and "mkv" in trs[0].get_text().lower() @@ -103,7 +123,7 @@ def search_ygg(query, multi, full): ygg_params["option_langue:multiple[]"] = "4" req = session.get(f"{YGGTORRENT_BASE_URL}/engine/search", params=ygg_params) - html = BeautifulSoup(req.text, "html.parser") + html = bs4.BeautifulSoup(req.text, "html.parser") trs = html.select("table.table tr") if len(trs) > 1: @@ -122,7 +142,7 @@ def search_ygg(query, multi, full): if int(downloads) < args.downloads: continue - if any(word.lower() in name for word in BLACKLIST_WORDS): + if any(word.lower() in name for word in args.blacklist): continue if args.year and str(args.year) not in name: @@ -148,13 +168,13 @@ query_string = {"query": args.query, "filters": "type:movie"} if args.year: query_string["filters"] += f" AND year:{args.year}" -tvdb = session.post( +tvdb = requests.post( "https://tvshowtime-dsn.algolia.net/1/indexes/TVDB/query", params={ "x-algolia-application-id": "tvshowtime", "x-algolia-api-key": "c9d5ec1316cec12f093754c69dd879d3", }, - json={"params": urlencode(query_string)}, + json={"params": urllib.parse.urlencode(query_string)}, ) tvdata = tvdb.json()