UPDATE: urls

functions.py

@@ -261,27 +261,24 @@ def listTables(username: str):
 
 def getLinks(url: str, timeout = 30):
     start = time.time()
-    def getLinksFromPage(url: str):
-        response = requests.get(url, verify=False)
-        htmlContent = response.content
-        soup = BeautifulSoup(htmlContent, "lxml")
-        anchorTags = soup.find_all("a")
-        allLinks = []
-        for tag in anchorTags:
-            if "href" in tag.attrs:
-                href = tag.attrs["href"]
-                parseObject = urlparse(href)      
-                if ((parseObject.scheme == "") | (parseObject.netloc == "")):
-                    fullUrl = urljoin(url, "/"+urljoin("/"+parseObject.path, urljoin("/"+parseObject.params, urljoin("/"+parseObject.query, "/"+parseObject.fragment))))
-                else: 
-                    fullUrl = href
-                if urlparse(fullUrl).netloc == urlparse(url).netloc:
-                    allLinks.append(fullUrl)
+    def getLinksFromPage(url: str) -> list:
+        response = requests.get(url)
+        soup = BeautifulSoup(response.content, "lxml")
+        anchors = soup.find_all("a")
+        links = []
+        for anchor in anchors:
+            if anchor.attrs["href"]:
+                if urlparse(anchor.get("href")).netloc == urlparse(url).netloc:
+                    newUrl = anchors.get("href")
+                elif anchor.get("href").startswith("/"):
+                    newUrl = urljoin(url + "/", anchor.get("href"))
                 else:
-                    continue
+                    pass
+                links.append(newUrl)
+                links = list(set(links))
             else:
                 continue
-        return allLinks
+        return links
     links = getLinksFromPage(url)
     uniqueLinks = set()
     for link in links: