Maharshi Gor
commited on
Commit
Β·
bd9e200
1
Parent(s):
98bfd66
Updated requirements
Browse files- requirements.txt +22 -8
- src/hf_datasets_utils.py +3 -2
requirements.txt
CHANGED
@@ -1,20 +1,25 @@
|
|
1 |
APScheduler
|
2 |
black
|
3 |
-
|
|
|
4 |
gradio
|
5 |
-
modelscope_studio
|
6 |
gradio[oauth]
|
7 |
gradio_leaderboard
|
8 |
gradio_client
|
|
|
|
|
|
|
9 |
huggingface-hub>=0.18.0
|
10 |
-
matplotlib
|
11 |
-
numpy<2.0.0
|
12 |
-
pandas>=2.0.0
|
13 |
-
python-dateutil
|
14 |
-
tqdm
|
15 |
transformers
|
16 |
tokenizers>=0.15.0
|
17 |
sentencepiece
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
18 |
litellm
|
19 |
diskcache
|
20 |
openai
|
@@ -28,6 +33,15 @@ langchain-openai
|
|
28 |
langchain-cohere
|
29 |
langchain-deepseek
|
30 |
json_repair
|
|
|
|
|
|
|
31 |
inflect
|
32 |
unidecode
|
33 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
APScheduler
|
2 |
black
|
3 |
+
|
4 |
+
# UI
|
5 |
gradio
|
|
|
6 |
gradio[oauth]
|
7 |
gradio_leaderboard
|
8 |
gradio_client
|
9 |
+
|
10 |
+
# Hugging Face
|
11 |
+
datasets
|
12 |
huggingface-hub>=0.18.0
|
|
|
|
|
|
|
|
|
|
|
13 |
transformers
|
14 |
tokenizers>=0.15.0
|
15 |
sentencepiece
|
16 |
+
|
17 |
+
# Data Science
|
18 |
+
matplotlib
|
19 |
+
numpy<2.0.0
|
20 |
+
pandas>=2.0.0
|
21 |
+
|
22 |
+
# LLMs / APIs
|
23 |
litellm
|
24 |
diskcache
|
25 |
openai
|
|
|
33 |
langchain-cohere
|
34 |
langchain-deepseek
|
35 |
json_repair
|
36 |
+
|
37 |
+
# NLP
|
38 |
+
nltk
|
39 |
inflect
|
40 |
unidecode
|
41 |
+
|
42 |
+
# Misc
|
43 |
+
python-dateutil
|
44 |
+
tqdm
|
45 |
+
rich
|
46 |
+
loguru
|
47 |
+
tabulate
|
src/hf_datasets_utils.py
CHANGED
@@ -7,10 +7,11 @@ api = HfApi()
|
|
7 |
def check_and_create_dataset_repo(repo_id: str):
|
8 |
try:
|
9 |
api.repo_info(repo_id=repo_id, repo_type="dataset")
|
10 |
-
print(f"{repo_id} exists")
|
11 |
except Exception:
|
12 |
-
print(f"
|
13 |
api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
|
|
|
14 |
|
15 |
|
16 |
def download_dataset_snapshot(repo_id, local_dir):
|
|
|
7 |
def check_and_create_dataset_repo(repo_id: str):
|
8 |
try:
|
9 |
api.repo_info(repo_id=repo_id, repo_type="dataset")
|
10 |
+
print(f"ππ»ββοΈ Repo {repo_id} exists β
")
|
11 |
except Exception:
|
12 |
+
print(f"β οΈ Repo {repo_id} doesn't exist, Creating...")
|
13 |
api.create_repo(repo_id=repo_id, repo_type="dataset", exist_ok=True, private=True)
|
14 |
+
print("ππ»ββοΈ Repo created β
")
|
15 |
|
16 |
|
17 |
def download_dataset_snapshot(repo_id, local_dir):
|