juni/aikaterna-cogs
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[RSS] Different list unpacking

Browse Source
This commit is contained in:
aikaterna
2020-10-01 15:10:34 -07:00
parent eb0ff5bdeb
commit d6c052dcfd
2 changed files with 24 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
rss/info.json
View File

@@ -5,6 +5,6 @@
"description": "Read RSS feeds", "description": "Read RSS feeds",
"tags": ["rss"], "tags": ["rss"],
"permissions": ["embed_links"], "permissions": ["embed_links"],
"requirements": ["beautifulsoup4>=4.9.1", "feedparser>=6.0.0", "scipy", "webcolors==1.3"], "requirements": ["bs4", "feedparser>=6.0.0", "scipy", "webcolors==1.3"],
"min_bot_version" : "3.4.0" "min_bot_version" : "3.4.0"
} }

39
rss/rss.py
View File

@@ -1,7 +1,6 @@
import asyncio import asyncio
import aiohttp import aiohttp
from bs4 import BeautifulSoup, MarkupResemblesLocatorWarning from bs4 import BeautifulSoup
import contextlib
import copy import copy
import datetime import datetime
import discord import discord
@@ -26,7 +25,7 @@ from .tag_type import INTERNAL_TAGS, VALID_IMAGES, TagType
log = logging.getLogger("red.aikaterna.rss") log = logging.getLogger("red.aikaterna.rss")
__version__ = "1.1.16" __version__ = "1.1.17"
class RSS(commands.Cog): class RSS(commands.Cog):
@@ -154,11 +153,17 @@ class RSS(commands.Cog):
list_tags = ["value", "href"] list_tags = ["value", "href"]
for tag in list_tags: for tag in list_tags:
try: try:
with contextlib.suppress(MarkupResemblesLocatorWarning): url_check = await self._valid_url(list_item[tag], feed_check=False)
soup = BeautifulSoup(list_item, "html.parser") if not url_check:
# bs4 will cry if you try to give it a url to parse, so let's only
# parse non-url content
tag_content = BeautifulSoup(list_item[tag], "html.parser")
tag_content = self._add_generic_html_plaintext(tag_content)
else:
tag_content = list_item[tag]
list_html_content_counter += 1 list_html_content_counter += 1
name = f"{tag_name}_plaintext{str(list_html_content_counter).zfill(2)}" name = f"{tag_name}_plaintext{str(list_html_content_counter).zfill(2)}"
rss_object[name] = self._add_generic_html_plaintext(soup) rss_object[name] = tag_content
rss_object["is_special"].append(name) rss_object["is_special"].append(name)
except (KeyError, TypeError): except (KeyError, TypeError):
pass pass
@@ -390,7 +395,7 @@ class RSS(commands.Cog):
# the feed was deleted during a _get_current_feed execution # the feed was deleted during a _get_current_feed execution
pass pass
async def _valid_url(self, url: str): async def _valid_url(self, url: str, feed_check=True):
"""Helper for rss add.""" """Helper for rss add."""
try: try:
result = urlparse(url) result = urlparse(url)
@@ -399,19 +404,21 @@ class RSS(commands.Cog):
return False return False
if all([result.scheme, result.netloc, result.path]): if all([result.scheme, result.netloc, result.path]):
text = await self._get_url_content(url) if feed_check:
if not text: text = await self._get_url_content(url)
log.debug(f"no text from _get_url_content: {url}") if not text:
return False log.debug(f"no text from _get_url_content: {url}")
return False
rss = feedparser.parse(text) rss = feedparser.parse(text)
if rss.bozo: if rss.bozo:
log.debug(f"bozo feed at {url}") log.debug(f"bozo feed at {url}")
return False return False
else:
return True
else: else:
return True return True
else: else:
log.debug(f"failed to urlparse {url}")
return False return False
async def _validate_image(self, url: str): async def _validate_image(self, url: str):