UPDATE: adding urls

app.py

@@ -68,4 +68,10 @@ async def delete(chatbotName: str):
 
 @app.post("/listChatbots")
 async def delete(username: str):
- return listTables(username=username)
+ return listTables(username=username)
+
+@app.post("/getLinks")
+async def crawlUrl(baseUrl: str):
+ return {
+ "urls": getLinks(url=baseUrl, timeout=30)
+ }

functions.py

@@ -290,4 +290,8 @@ def getLinks(url: str, timeout = 30):
  break
  else:
  uniqueLinks = uniqueLinks.union(set(getLinksFromPage(link)))
- return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
+ return list(set([x[:len(x) - 1] if x[-1] == "/" else x for x in uniqueLinks]))
+
+
+def getTextLength(text: str):
+ return len(text)