parent
a778a6b947
commit
fff3b3e886
@ -18,8 +18,6 @@ parser.add_argument("-v", "--volume", type=int, choices=range(1, 10), required=T
|
||||
parser.add_argument("output", type=Path)
|
||||
args = parser.parse_args()
|
||||
|
||||
COOKIE: requests.CookieTypes = {}
|
||||
|
||||
|
||||
class MyChapterFactory(SimpleChapterFactory): # type: ignore
|
||||
def cleanup_html(self, content: bytes) -> HtmlElement:
|
||||
@ -40,28 +38,6 @@ class MyChapterFactory(SimpleChapterFactory): # type: ignore
|
||||
return etree
|
||||
|
||||
|
||||
def download(url: str) -> str:
|
||||
global COOKIE
|
||||
|
||||
if COOKIE:
|
||||
timeout = randint(10, 100)
|
||||
print(f"Wait {timeout} seconds...")
|
||||
sleep(2)
|
||||
|
||||
response = requests.get(
|
||||
url,
|
||||
cookies=COOKIE,
|
||||
impersonate=requests.BrowserType.firefox133.value,
|
||||
thread=None,
|
||||
curl_options=None,
|
||||
debug=False,
|
||||
)
|
||||
|
||||
COOKIE = response.cookies
|
||||
|
||||
return str(response.text)
|
||||
|
||||
|
||||
with open(f"{path.dirname(__file__)}/{args.lang}.json") as f:
|
||||
manifest = load(f)
|
||||
book = manifest[args.volume - 1]
|
||||
@ -75,8 +51,14 @@ epub = Epub(
|
||||
)
|
||||
|
||||
for url in book["chapters"]:
|
||||
text = download(url)
|
||||
etree = BeautifulSoup(text, "html.parser")
|
||||
markup = requests.get(
|
||||
url,
|
||||
impersonate=requests.BrowserType.firefox133.value,
|
||||
thread=None,
|
||||
curl_options=None,
|
||||
debug=False,
|
||||
)
|
||||
etree = BeautifulSoup(markup.text, "html.parser")
|
||||
title = etree.select_one("h1.entry-title").text # type: ignore
|
||||
content = etree.select("div.entry-content p")
|
||||
print(f"Chapter {title}...")
|
||||
@ -86,6 +68,9 @@ for url in book["chapters"]:
|
||||
continue
|
||||
chapter.content += elem.prettify()
|
||||
epub.add_chapter(chapter)
|
||||
timeout = randint(10, 90)
|
||||
print(f"Wait {timeout} seconds...")
|
||||
sleep(timeout)
|
||||
|
||||
with NamedTemporaryFile() as cover:
|
||||
response = requests.get(book["cover"], thread=None, curl_options=None, debug=False)
|
||||
|
Loading…
x
Reference in New Issue
Block a user