From 663efc3f85370dca9e20239d48b77c7b5b21a507 Mon Sep 17 00:00:00 2001
From: aikaterna <20862007+aikaterna@users.noreply.github.com>
Date: Tue, 9 Feb 2021 15:58:42 -0800
Subject: [PATCH] [UrlFetch] Initial commit

---
 README.md            |  2 ++
 urlfetch/__init__.py |  5 ++++
 urlfetch/info.json   |  8 +++++
 urlfetch/urlfetch.py | 70 ++++++++++++++++++++++++++++++++++++++++++++
 4 files changed, 85 insertions(+)
 create mode 100644 urlfetch/__init__.py
 create mode 100644 urlfetch/info.json
 create mode 100644 urlfetch/urlfetch.py

diff --git a/README.md b/README.md
index 48b9367..7efd467 100644
--- a/README.md
+++ b/README.md
@@ -67,6 +67,8 @@ tools - A collection of mod and admin tools, ported from my v2 version. Sitryk i
 
 ttt - A Tic Tac Toe cog originally for Red V2 by HizikiFW. This cog is licensed under the Apache-2.0 license.
 
+urlfetch - Fetch text from a URL. Mainly used for simple text API queries (not JSON).
+
 warcraftlogs - Fetch player info/metrics from the WarcraftLogs API for World of Warcraft Classic. Does not provide stats for non-Classic characters.
 
 wolfram - A v3 port of Paddo's abandoned Wolfram Alpha cog.
diff --git a/urlfetch/__init__.py b/urlfetch/__init__.py
new file mode 100644
index 0000000..dc34bfe
--- /dev/null
+++ b/urlfetch/__init__.py
@@ -0,0 +1,5 @@
+from .urlfetch import UrlFetch
+
+
+def setup(bot):
+    bot.add_cog(UrlFetch(bot))
diff --git a/urlfetch/info.json b/urlfetch/info.json
new file mode 100644
index 0000000..c3ec513
--- /dev/null
+++ b/urlfetch/info.json
@@ -0,0 +1,8 @@
+{
+    "author": ["aikaterna"],
+    "install_msg": "Thanks for installing.",
+    "short": "Fetch text from a URL.",
+    "description": "Fetch text from a URL.",
+    "tags": ["api"],
+    "min_bot_version" : "3.4.0"
+}
diff --git a/urlfetch/urlfetch.py b/urlfetch/urlfetch.py
new file mode 100644
index 0000000..19b350b
--- /dev/null
+++ b/urlfetch/urlfetch.py
@@ -0,0 +1,70 @@
+import aiohttp
+import logging
+from urllib.parse import urlparse
+
+from redbot.core import checks, commands, Config
+from redbot.core.utils.chat_formatting import box, pagify
+from redbot.core.utils.menus import menu, DEFAULT_CONTROLS
+
+log = logging.getLogger("red.aikaterna.urlfetch")
+
+
+__version__ = "1.0.0"
+
+
+class UrlFetch(commands.Cog):
+    """Grab stuff from a text API."""
+
+    def __init__(self, bot):
+        self.bot = bot
+
+    @commands.command()
+    async def urlfetch(self, ctx, url: str):
+        """
+        Input a URL to read.
+        """
+        valid_url = await self._valid_url(ctx, url)
+        if valid_url:
+            text = await self._get_url_content(url)
+            if text:
+                page_list = []
+                for page in pagify(text, delims=["\n"], page_length=1800):
+                    page_list.append(box(page))
+                if len(page_list) == 1:
+                    await ctx.send(box(page))
+                else:
+                    await menu(ctx, page_list, DEFAULT_CONTROLS)
+        else:
+            return
+
+    async def _get_url_content(self, url: str):
+        try:
+            async with aiohttp.ClientSession() as session:
+                async with session.get(url) as resp:
+                    text = await resp.text()
+            return text
+        except aiohttp.client_exceptions.ClientConnectorError:
+            log.error(f"aiohttp failure accessing feed at url:\n\t{url}", exc_info=True)
+            return None
+        except Exception:
+            log.error(f"General failure accessing feed at url:\n\t{url}", exc_info=True)
+            return None
+
+    async def _valid_url(self, ctx, url: str):
+        try:
+            result = urlparse(url)
+        except Exception as e:
+            log.exception(e, exc_info=e)
+            await ctx.send("There was an issue trying to fetch that feed. Please check your console for the error.")
+            return None
+
+        if all([result.scheme, result.netloc, result.path]):
+            text = await self._get_url_content(url)
+            if not text:
+                await ctx.send(f"No text present at: {url}")
+                return None
+            else:
+                return text
+        else:
+            await ctx.send(f"Url seems to be incomplete: {url}")
+            return None