Cleaning mess

app.py

@@ -1,4 +1,3 @@
-from logging import getLogger
 from operator import attrgetter, itemgetter
 from time import sleep
 
@@ -134,7 +133,7 @@ def clean_model(obj):
         if not attr.startswith('_') and getattr(obj, attr) is None:
             try:
                 setattr(obj, attr, '')
-            except Exception:
+            except AttributeError:
                 pass
     return obj
 

connectors.py

@@ -9,6 +9,7 @@ from urllib.parse import quote
 
 from bs4 import BeautifulSoup
 from cloudscraper import create_scraper
+from requests import RequestException
 
 from config import IS_DEBUG, CACHE_TIMEOUT, BLACKLIST_WORDS
 from models import AnimeLink
@@ -78,6 +79,29 @@ class Cache:
 ConnectorCache = Cache()
 
 
+def curl_content(url, params=None, ajax=False):
+    if ajax:
+        headers = {'X-Requested-With': 'XMLHttpRequest'}
+    else:
+        headers = {}
+
+    try:
+        if params is not None:
+            response = scraper.post(url, params, timeout=5, headers=headers)
+        else:
+            response = scraper.get(url, timeout=5, headers=headers)
+
+        output = response.text
+        http_code = response.status_code
+    except RequestException as e:
+        output = ''
+        http_code = 500
+        if IS_DEBUG:
+            getLogger().exception(e)
+
+    return {'http_code': http_code, 'output': output}
+
+
 class Connector(ABC):
     @property
     @abstractmethod
@@ -132,28 +156,6 @@ class Connector(ABC):
                 self.get_history()
         return self
 
-    def curl_content(self, url, params=None, ajax=False):
-        if ajax:
-            headers = {'X-Requested-With': 'XMLHttpRequest'}
-        else:
-            headers = {}
-
-        try:
-            if params is not None:
-                response = scraper.post(url, params, timeout=5, headers=headers)
-            else:
-                response = scraper.get(url, timeout=5, headers=headers)
-
-            output = response.text
-            http_code = response.status_code
-        except Exception as e:
-            output = ''
-            http_code = 500
-            if IS_DEBUG:
-                getLogger().exception(e)
-
-        return {'http_code': http_code, 'output': output}
-
     @staticmethod
     def get_instance(url, query):
         if 'nyaa.si' in url:
@@ -202,7 +204,7 @@ class Nyaa(Connector):
 
     @ConnectorCache.cache_data
     def search(self):
-        response = self.curl_content(self.get_full_search_url())
+        response = curl_content(self.get_full_search_url())
 
         if response['http_code'] is 200:
             html = BeautifulSoup(response['output'], 'html.parser')
@@ -274,7 +276,7 @@ class Pantsu(Connector):
 
     @ConnectorCache.cache_data
     def search(self):
-        response = self.curl_content(self.get_full_search_url())
+        response = curl_content(self.get_full_search_url())
 
         if response['http_code'] is 200:
             html = BeautifulSoup(response['output'], 'html.parser')
@@ -320,9 +322,8 @@ class Pantsu(Connector):
                         'href': href,
                         'name': url_safe,
                         'comment': '',
-                        'link': tds[2].decode_contents()
-                            .replace('icon-magnet', 'fa fa-fw fa-magnet')
-                            .replace('icon-floppy', 'fa fa-fw fa-download'),
+                        'link': tds[2].decode_contents().replace('icon-magnet', 'fa fa-fw fa-magnet').replace(
+                            'icon-floppy', 'fa fa-fw fa-download'),
                         'size': tds[3].string,
                         'date': formatted_date,
                         'seeds': check_seeds,
@@ -360,7 +361,7 @@ class YggTorrent(Connector):
     @ConnectorCache.cache_data
     def search(self):
         if self.category:
-            response = self.curl_content(self.get_full_search_url())
+            response = curl_content(self.get_full_search_url())
 
             if response['http_code'] is 200:
                 html = BeautifulSoup(response['output'], 'html.parser')
@@ -434,7 +435,7 @@ class AnimeUltime(Connector):
 
     @ConnectorCache.cache_data
     def search(self):
-        response = self.curl_content(self.get_full_search_url(), {'search': self.query})
+        response = curl_content(self.get_full_search_url(), {'search': self.query})
 
         if response['http_code'] is 200:
             html = BeautifulSoup(response['output'], 'html.parser')
@@ -482,7 +483,7 @@ class AnimeUltime(Connector):
 
     @ConnectorCache.cache_data
     def get_history(self):
-        response = self.curl_content(self.get_full_search_url())
+        response = curl_content(self.get_full_search_url())
 
         if response['http_code'] is 200:
             html = BeautifulSoup(response['output'], 'html.parser')

get404.py

@@ -1,13 +1,12 @@
 from config import app
-from connectors import Connector
+from connectors import curl_content
 from models import AnimeLink
 
 app.config['SQLALCHEMY_ECHO'] = False
 links = AnimeLink.query.all()
 
 for link in links:
-    connect = Connector.get_instance(link.link, link.title.keyword)
-    html = connect.curl_content(link.link)
+    html = curl_content(link.link)
 
     if html['http_code'] is not 200:
         print('(%d) %s %s : %s' % (html['http_code'], link.title.name, link.season, link.link))

requirements.txt

@@ -2,7 +2,8 @@ Flask==1.0.2
 Flask-SQLAlchemy==2.1
 SQLAlchemy==1.2.18
 Flask-HTTPAuth==3.2.4
-Flask-WTF===0.14.2
+Flask-WTF==0.14.2
+WTForms==2.2.1
 PyMySQL==0.9.3
 requests==2.21.0
 beautifulsoup4==4.7.1