[UrlFetch] Initial commit

README.md

@@ -67,6 +67,8 @@ tools - A collection of mod and admin tools, ported from my v2 version. Sitryk i
 
 ttt - A Tic Tac Toe cog originally for Red V2 by HizikiFW. This cog is licensed under the Apache-2.0 license.
 
+urlfetch - Fetch text from a URL. Mainly used for simple text API queries (not JSON).
+
 warcraftlogs - Fetch player info/metrics from the WarcraftLogs API for World of Warcraft Classic. Does not provide stats for non-Classic characters.
 
 wolfram - A v3 port of Paddo's abandoned Wolfram Alpha cog.

urlfetch/__init__.py

@@ -0,0 +1,5 @@
+from .urlfetch import UrlFetch
+
+
+def setup(bot):
+    bot.add_cog(UrlFetch(bot))

urlfetch/info.json

@@ -0,0 +1,8 @@
+{
+    "author": ["aikaterna"],
+    "install_msg": "Thanks for installing.",
+    "short": "Fetch text from a URL.",
+    "description": "Fetch text from a URL.",
+    "tags": ["api"],
+    "min_bot_version" : "3.4.0"
+}

urlfetch/urlfetch.py

@@ -0,0 +1,70 @@
+import aiohttp
+import logging
+from urllib.parse import urlparse
+
+from redbot.core import checks, commands, Config
+from redbot.core.utils.chat_formatting import box, pagify
+from redbot.core.utils.menus import menu, DEFAULT_CONTROLS
+
+log = logging.getLogger("red.aikaterna.urlfetch")
+
+
+__version__ = "1.0.0"
+
+
+class UrlFetch(commands.Cog):
+    """Grab stuff from a text API."""
+
+    def __init__(self, bot):
+        self.bot = bot
+
+    @commands.command()
+    async def urlfetch(self, ctx, url: str):
+        """
+        Input a URL to read.
+        """
+        valid_url = await self._valid_url(ctx, url)
+        if valid_url:
+            text = await self._get_url_content(url)
+            if text:
+                page_list = []
+                for page in pagify(text, delims=["\n"], page_length=1800):
+                    page_list.append(box(page))
+                if len(page_list) == 1:
+                    await ctx.send(box(page))
+                else:
+                    await menu(ctx, page_list, DEFAULT_CONTROLS)
+        else:
+            return
+
+    async def _get_url_content(self, url: str):
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as resp:
+                    text = await resp.text()
+            return text
+        except aiohttp.client_exceptions.ClientConnectorError:
+            log.error(f"aiohttp failure accessing feed at url:\n\t{url}", exc_info=True)
+            return None
+        except Exception:
+            log.error(f"General failure accessing feed at url:\n\t{url}", exc_info=True)
+            return None
+
+    async def _valid_url(self, ctx, url: str):
+        try:
+            result = urlparse(url)
+        except Exception as e:
+            log.exception(e, exc_info=e)
+            await ctx.send("There was an issue trying to fetch that feed. Please check your console for the error.")
+            return None
+
+        if all([result.scheme, result.netloc, result.path]):
+            text = await self._get_url_content(url)
+            if not text:
+                await ctx.send(f"No text present at: {url}")
+                return None
+            else:
+                return text
+        else:
+            await ctx.send(f"Url seems to be incomplete: {url}")
+            return None