Maharshi Gor commited on
Commit
bd9e200
Β·
1 Parent(s): 98bfd66

Updated requirements

Browse files
Files changed (2) hide show
  1. requirements.txt +22 -8
  2. src/hf_datasets_utils.py +3 -2
requirements.txt CHANGED
@@ -1,20 +1,25 @@
1
  APScheduler
2
  black
3
- datasets
 
4
  gradio
5
- modelscope_studio
6
  gradio[oauth]
7
  gradio_leaderboard
8
  gradio_client
 
 
 
9
  huggingface-hub>=0.18.0
10
- matplotlib
11
- numpy<2.0.0
12
- pandas>=2.0.0
13
- python-dateutil
14
- tqdm
15
  transformers
16
  tokenizers>=0.15.0
17
  sentencepiece
 
 
 
 
 
 
 
18
  litellm
19
  diskcache
20
  openai
@@ -28,6 +33,15 @@ langchain-openai
28
  langchain-cohere
29
  langchain-deepseek
30
  json_repair
 
 
 
31
  inflect
32
  unidecode
33
- loguru
 
 
 
 
 
 
 
1
  APScheduler
2
  black
3
+
4
+ # UI
5
  gradio
 
6
  gradio[oauth]
7
  gradio_leaderboard
8
  gradio_client
9
+
10
+ # Hugging Face
11
+ datasets
12
  huggingface-hub>=0.18.0
 
 
 
 
 
13
  transformers
14
  tokenizers>=0.15.0
15
  sentencepiece
16
+
17
+ # Data Science
18
+ matplotlib
19
+ numpy<2.0.0
20
+ pandas>=2.0.0
21
+
22
+ # LLMs / APIs
23
  litellm
24
  diskcache
25
  openai
 
33
  langchain-cohere
34
  langchain-deepseek
35
  json_repair
36
+
37
+ # NLP
38
+ nltk
39
  inflect
40
  unidecode
41
+
42
+ # Misc
43
+ python-dateutil
44
+ tqdm
45
+ rich
46
+ loguru
47
+ tabulate
src/hf_datasets_utils.py CHANGED
@@ -7,10 +7,11 @@ api = HfApi()
7
  def check_and_create_dataset_repo(repo_id: str):
8
  try:
9
  api.repo_info(repo_id=repo_id, repo_type="dataset")
10
- print(f"{repo_id} exists")
11
  except Exception:
12
- print(f"Creating {repo_id}")
13
  api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
 
14
 
15
 
16
  def download_dataset_snapshot(repo_id, local_dir):
 
7
  def check_and_create_dataset_repo(repo_id: str):
8
  try:
9
  api.repo_info(repo_id=repo_id, repo_type="dataset")
10
+ print(f"πŸ’πŸ»β€β™‚οΈ Repo {repo_id} exists βœ…")
11
  except Exception:
12
+ print(f"⚠️ Repo {repo_id} doesn't exist, Creating...")
13
  api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
14
+ print("πŸ’πŸ»β€β™‚οΈ Repo created βœ…")
15
 
16
 
17
  def download_dataset_snapshot(repo_id, local_dir):