Xefir/dl
1
1
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Use FlareRequests for pygg
All checks were successful
continuous-integration/drone/push Build is passing
Details

Browse Source
This commit is contained in:
Michel Roux 2022-12-24 10:47:15 +00:00
parent cc40473b86
commit b7a1fb40d0
3 changed files with 79 additions and 57 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
.flake8 Normal file
View File

@ -0,0 +1,2 @@
[flake8]
max-line-length = 100

2
Dockerfile
View File

@ -4,7 +4,7 @@ RUN apt-get update && \
apt-get install -y \
vim p7zip* git rsync lftp speedtest-cli rename wget curl procps psmisc \
openssh-client transmission-cli yt-dlp \
python3-requests python3-bs4 python3-dnspython && \
python3-requests python3-bs4 && \
rm -rf /var/lib/apt/lists/*
RUN curl -sSL https://raw.githubusercontent.com/tremc/tremc/master/tremc -o /usr/local/bin/tremc && \
curl -sSL https://github.com/drone/drone-cli/releases/latest/download/drone_linux_amd64.tar.gz | tar xvz && \

132
commands/pygg.py
View File

@ -1,19 +1,82 @@
#!/usr/bin/env python3
import argparse
import io
import os
import re
from urllib.parse import urlencode, urlparse
import urllib
from bs4 import BeautifulSoup
from dns import rdatatype, resolver
from requests import Session, adapters
from urllib3.util.connection import HAS_IPV6
import bs4
import requests
BLACKLIST_WORDS = ["dvd", "iso"]
YGGTORRENT_BASE_URL = "https://www5.yggtorrent.fi"
YGGTORRENT_BASE_URL = "https://www6.yggtorrent.lol"
CLOUDPROXY_ENDPOINT = os.getenv("CLOUDPROXY_ENDPOINT")
class FlareRequests(requests.Session):
def request(self, method, url, params=None, **kwargs):
if not CLOUDPROXY_ENDPOINT:
return super().request(method, url, params, **kwargs)
sessions = requests.post(
CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.list"}
).json()
if "sessions" in sessions and len(sessions["sessions"]) > 0:
FLARESESSION = sessions["sessions"][0]
else:
response = requests.post(
CLOUDPROXY_ENDPOINT, json={"cmd": "sessions.create"}
)
session = response.json()
if "session" in session:
FLARESESSION = session["session"]
else:
raise requests.RequestException(response)
post_data = {
"cmd": f"request.{method.lower()}",
"session": FLARESESSION,
"url": url,
}
if params:
post_data["postData"] = urllib.parse.urlencode(params)
try:
response = requests.post(
CLOUDPROXY_ENDPOINT,
json=post_data,
)
solution = response.json()
if "solution" in solution:
resolved = requests.Response()
resolved.raw = io.BytesIO(solution["solution"]["response"].encode())
resolved.status_code = solution["solution"]["status"]
resolved.headers = solution["solution"]["headers"]
resolved.url = url
resolved.reason = solution["status"]
resolved.cookies = solution["solution"]["cookies"]
return resolved
raise requests.RequestException(response)
except requests.RequestException:
session = requests.post(
CLOUDPROXY_ENDPOINT,
json={"cmd": "sessions.destroy", "session": FLARESESSION},
)
raise requests.RequestException(solution)
parser = argparse.ArgumentParser()
parser.add_argument("-u", "--uploader", action="append")
parser.add_argument("-b", "--blacklist", action="append", default=["dvd", "iso"])
parser.add_argument("-y", "--year", type=int)
parser.add_argument("-s", "--size", type=int, default=10)
parser.add_argument("-d", "--downloads", type=int, default=20)
@ -29,56 +92,13 @@ def parse_size(size):
return int(float(number) * units[unit])
DNS_RESOLVER = resolver.Resolver()
DNS_RESOLVER.cache = resolver.LRUCache() # type: ignore
class DNSAdapter(adapters.HTTPAdapter):
def __init__(self, nameservers):
self.nameservers = nameservers
super().__init__()
def resolve(self, host, nameservers):
DNS_RESOLVER.nameservers = nameservers
if HAS_IPV6:
try:
answers_v6 = DNS_RESOLVER.query(host, rdatatype.AAAA)
for rdata_v6 in answers_v6:
return f"[{str(rdata_v6)}]"
except resolver.NoAnswer:
pass
answers_v4 = DNS_RESOLVER.query(host, rdatatype.A)
for rdata_v4 in answers_v4:
return str(rdata_v4)
def send(self, request, **kwargs):
connection_pool_kwargs = self.poolmanager.connection_pool_kw
result = urlparse(request.url)
resolved_ip = self.resolve(result.hostname, self.nameservers)
request.url = request.url.replace(result.hostname, resolved_ip)
request.headers["Host"] = result.hostname
request.headers[
"User-Agent"
] = "Googlebot/2.1 (+http://www.google.com/bot.html)"
if result.scheme == "https":
connection_pool_kwargs["server_hostname"] = result.hostname
connection_pool_kwargs["assert_hostname"] = result.hostname
return super().send(request, **kwargs)
session = Session()
session.mount("http://", DNSAdapter(["1.1.1.1"]))
session.mount("https://", DNSAdapter(["1.1.1.1"]))
session = FlareRequests()
def check_files(id):
req = session.get(f"{YGGTORRENT_BASE_URL}/engine/get_files", params={"torrent": id})
files = req.json()
html = BeautifulSoup(files["html"], "html.parser")
html = bs4.BeautifulSoup(files["html"], "html.parser")
trs = html.select("tr")
return len(trs) == 1 and "mkv" in trs[0].get_text().lower()
@ -103,7 +123,7 @@ def search_ygg(query, multi, full):
ygg_params["option_langue:multiple[]"] = "4"
req = session.get(f"{YGGTORRENT_BASE_URL}/engine/search", params=ygg_params)
html = BeautifulSoup(req.text, "html.parser")
html = bs4.BeautifulSoup(req.text, "html.parser")
trs = html.select("table.table tr")
if len(trs) > 1:
@ -122,7 +142,7 @@ def search_ygg(query, multi, full):
if int(downloads) < args.downloads:
continue
if any(word.lower() in name for word in BLACKLIST_WORDS):
if any(word.lower() in name for word in args.blacklist):
continue
if args.year and str(args.year) not in name:
@ -148,13 +168,13 @@ query_string = {"query": args.query, "filters": "type:movie"}
if args.year:
query_string["filters"] += f" AND year:{args.year}"
tvdb = session.post(
tvdb = requests.post(
"https://tvshowtime-dsn.algolia.net/1/indexes/TVDB/query",
params={
"x-algolia-application-id": "tvshowtime",
"x-algolia-api-key": "c9d5ec1316cec12f093754c69dd879d3",
},
json={"params": urlencode(query_string)},
json={"params": urllib.parse.urlencode(query_string)},
)
tvdata = tvdb.json()