This repository has been archived on 2024-02-23. You can view files and clone it, but cannot push or open issues or pull requests.
Auberge_Vagabonde/twi.py
Michel Roux aa0a420b76
Some checks reported errors
continuous-integration/drone/push Build was killed
Fix metadata
2022-01-19 12:37:45 +00:00

54 lines
1.7 KiB
Python

import bs4
import executor
import logging
import pypub
import requests
def fetchVolume(title, metadata, volume_title, output_dir, links):
book = pypub.Epub(f"{title} - {volume_title}", **metadata)
for link in links:
chapter_response = requests.get(link["href"])
chapter_html = bs4.BeautifulSoup(chapter_response.content, "html.parser")
chapter_content = chapter_html.select_one("div.entry-content")
chapter_title = chapter_html.select_one("h1.entry-title").get_text().strip()
a_remove = chapter_content.find_all("a")
hr_remove = chapter_content.find_all("hr")
div_remove = chapter_content.find_all("div.tiled-gallery")
for removed in a_remove:
removed.decompose()
for removed in hr_remove:
removed.decompose()
for removed in div_remove:
removed.decompose()
logging.log(logging.INFO, f"{title} - {chapter_title}")
book.add_chapter(
pypub.create_chapter_from_string(chapter_content.prettify(), chapter_title)
)
logging.log(logging.INFO, f"{title} - {volume_title}")
book.create_epub(output_dir)
@executor.executor
def process(metadata, output_dir, url):
response = requests.get(url)
html = bs4.BeautifulSoup(response.content, "html.parser")
content = html.select("div.entry-content > p")
title = html.select_one("#site-title > span > a").get_text().strip()
volume_title = None
for paragraph in content:
if paragraph.strong is not None:
volume_title = paragraph.strong.get_text().strip()
else:
fetchVolume(
title, metadata, volume_title, output_dir, paragraph.find_all("a")
)