Upload hf_utils.py with huggingface_hub
Browse files- hf_utils.py +8 -11
hf_utils.py
CHANGED
@@ -1,14 +1,13 @@
|
|
1 |
-
from datasets import (
|
2 |
-
disable_caching,
|
3 |
-
enable_caching,
|
4 |
-
is_caching_enabled,
|
5 |
-
)
|
6 |
from pathlib import Path
|
7 |
-
|
|
|
8 |
from datasets.utils.py_utils import get_imports
|
9 |
|
|
|
|
|
10 |
HF_CACHING_ENABLED = False
|
11 |
|
|
|
12 |
class HFCachingContextManager:
|
13 |
def __init__(self, should_cache):
|
14 |
self.should_cache = should_cache
|
@@ -30,15 +29,13 @@ class HFCachingContextManager:
|
|
30 |
else:
|
31 |
disable_caching()
|
32 |
|
33 |
-
def set_hf_caching(enabled):
|
34 |
-
return HFCachingContextManager(enabled)
|
35 |
|
36 |
def get_missing_imports(file, exclude=[]):
|
37 |
src_dir = Path(__file__).parent
|
38 |
-
python_files = get_all_files_in_dir(src_dir, file_extension=
|
39 |
# get only the file without the path and extension
|
40 |
required_modules = [Path(p).stem for p in python_files]
|
41 |
imports = get_imports(file)
|
42 |
-
imported_modules = [i[1] for i in imports if i[0] ==
|
43 |
missing_imports = [i for i in required_modules if i not in imported_modules and i not in exclude]
|
44 |
-
return missing_imports
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
from pathlib import Path
|
2 |
+
|
3 |
+
from datasets import disable_caching, enable_caching, is_caching_enabled
|
4 |
from datasets.utils.py_utils import get_imports
|
5 |
|
6 |
+
from .file_utils import get_all_files_in_dir
|
7 |
+
|
8 |
HF_CACHING_ENABLED = False
|
9 |
|
10 |
+
|
11 |
class HFCachingContextManager:
|
12 |
def __init__(self, should_cache):
|
13 |
self.should_cache = should_cache
|
|
|
29 |
else:
|
30 |
disable_caching()
|
31 |
|
|
|
|
|
32 |
|
33 |
def get_missing_imports(file, exclude=[]):
|
34 |
src_dir = Path(__file__).parent
|
35 |
+
python_files = get_all_files_in_dir(src_dir, file_extension=".py")
|
36 |
# get only the file without the path and extension
|
37 |
required_modules = [Path(p).stem for p in python_files]
|
38 |
imports = get_imports(file)
|
39 |
+
imported_modules = [i[1] for i in imports if i[0] == "internal"]
|
40 |
missing_imports = [i for i in required_modules if i not in imported_modules and i not in exclude]
|
41 |
+
return missing_imports
|