From bd60f3c0c556a287d99b59127be168c3e30a81e8 Mon Sep 17 00:00:00 2001 From: Michel Roux Date: Tue, 18 Jan 2022 21:53:50 +0000 Subject: [PATCH] Fix async + drone --- .drone.yml | 4 +--- executor.py | 12 +++++++----- gravesong.py | 2 +- twi.py | 16 +++++----------- 4 files changed, 14 insertions(+), 20 deletions(-) diff --git a/.drone.yml b/.drone.yml index acaf294..d44f3e7 100644 --- a/.drone.yml +++ b/.drone.yml @@ -4,10 +4,8 @@ type: docker steps: - name: epub - image: python:slim + image: python commands: - - apt-get update - - apt-get install -y git - pip install flake8 black - flake8 --ignore=E501 - black --check . diff --git a/executor.py b/executor.py index 29ed0ea..eb1768f 100644 --- a/executor.py +++ b/executor.py @@ -1,10 +1,12 @@ import asyncio +import functools -def executor(func): - async def wrapper(*args, **kwargs): - await asyncio.get_event_loop().run_in_executor( - None, lambda: func(*args, **kwargs) +def executor(f): + @functools.wraps(f) + async def wrapped(*args, **kwargs): + return await asyncio.get_running_loop().run_in_executor( + None, lambda: f(*args, **kwargs) ) - return wrapper + return wrapped diff --git a/gravesong.py b/gravesong.py index c560e22..2707fc3 100644 --- a/gravesong.py +++ b/gravesong.py @@ -31,7 +31,7 @@ def process(metadata, output_dir): book.add_chapter( pypub.create_chapter_from_string(chapter_content, chapter_title) ) - chapter_title = paragraph.get_text() + chapter_title = paragraph.get_text().strip() elif paragraph.name == "p": chapter_content += paragraph.prettify() diff --git a/twi.py b/twi.py index 35884ee..4336761 100644 --- a/twi.py +++ b/twi.py @@ -1,4 +1,3 @@ -import asyncio import bs4 import executor import logging @@ -13,7 +12,7 @@ def fetchVolume(title, metadata, volume_title, output_dir, links): chapter_response = requests.get(link["href"]) chapter_html = bs4.BeautifulSoup(chapter_response.content, "html.parser") chapter_content = chapter_html.select_one("div.entry-content") - chapter_title = chapter_html.select_one("h1.entry-title").get_text() + chapter_title = chapter_html.select_one("h1.entry-title").get_text().strip() a_remove = chapter_content.find_all("a") hr_remove = chapter_content.find_all("hr") @@ -41,19 +40,14 @@ def process(metadata, output_dir, url): html = bs4.BeautifulSoup(response.content, "html.parser") content = html.select("div.entry-content > p") - title = html.select_one("#site-title > span > a").get_text() + title = html.select_one("#site-title > span > a").get_text().strip() volume_title = None - tasks = [] for paragraph in content: if paragraph.strong is not None: - volume_title = paragraph.strong.get_text() + volume_title = paragraph.strong.get_text().strip() else: - tasks.append( - fetchVolume( - title, metadata, volume_title, output_dir, paragraph.find_all("a") - ) + fetchVolume( + title, metadata, volume_title, output_dir, paragraph.find_all("a") ) - - asyncio.run(asyncio.gather(*tasks))