awacke1 commited on
Commit
3b59fe8
·
1 Parent(s): 40082db

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +15 -8
app.py CHANGED
@@ -7,13 +7,17 @@ from bs4 import BeautifulSoup
7
 
8
  EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
9
 
 
 
 
 
10
  def download_file(url, local_filename):
11
  if url.startswith('http://') or url.startswith('https://'):
12
  try:
13
  with requests.get(url, stream=True) as r:
14
  r.raise_for_status()
15
  with open(local_filename, 'wb') as f:
16
- for chunk in r.iter_content(chunk_size=8192):
17
  f.write(chunk)
18
  return local_filename
19
  except requests.exceptions.HTTPError as err:
@@ -51,19 +55,22 @@ def show_download_links():
51
  def main():
52
  st.sidebar.title('Web Datasets Bulk Downloader')
53
  url = st.sidebar.text_input('Please enter a Web URL to bulk download text and files')
 
 
 
 
 
 
54
  if st.sidebar.button('📥 Get All the Content'):
55
  download_html_and_files(url)
56
  show_download_links()
57
- with open("history.txt", "a") as file:
58
- file.write(f"{url}\n")
59
  if st.sidebar.button('📂 Show Download Links'):
60
  show_download_links()
61
 
62
- with open("history.txt", "r") as file:
63
- history_urls = file.readlines()
64
- if history_urls:
65
- st.markdown("## History")
66
- st.text_area("URL history", value="".join(history_urls), height=200)
67
 
68
  if __name__ == "__main__":
69
  main()
 
7
 
8
  EXCLUDED_FILES = ['app.py', 'requirements.txt', 'pre-requirements.txt', 'packages.txt', 'README.md','.gitattributes', "backup.py","Dockerfile"]
9
 
10
+ # Create a history.txt file if it doesn't exist yet
11
+ with open("history.txt", "a+") as f:
12
+ f.close()
13
+
14
  def download_file(url, local_filename):
15
  if url.startswith('http://') or url.startswith('https://'):
16
  try:
17
  with requests.get(url, stream=True) as r:
18
  r.raise_for_status()
19
  with open(local_filename, 'wb') as f:
20
+ for chunk in r.iter_content(chunk_size=8192):
21
  f.write(chunk)
22
  return local_filename
23
  except requests.exceptions.HTTPError as err:
 
55
  def main():
56
  st.sidebar.title('Web Datasets Bulk Downloader')
57
  url = st.sidebar.text_input('Please enter a Web URL to bulk download text and files')
58
+
59
+ # Save the history of URL entered as a text file
60
+ if url:
61
+ with open("history.txt", "a") as f:
62
+ f.write(url + "\n")
63
+
64
  if st.sidebar.button('📥 Get All the Content'):
65
  download_html_and_files(url)
66
  show_download_links()
 
 
67
  if st.sidebar.button('📂 Show Download Links'):
68
  show_download_links()
69
 
70
+ # Display history as markdown
71
+ with open("history.txt", "r") as f:
72
+ history = f.read()
73
+ st.markdown(f"### History\n\n{history}")
 
74
 
75
  if __name__ == "__main__":
76
  main()