That1BrainCell commited on
Commit
3516cae
·
verified ·
1 Parent(s): 32bb13b

hf folder path change

Browse files
Files changed (1) hide show
  1. github_storage.py +77 -77
github_storage.py CHANGED
@@ -1,77 +1,77 @@
1
- import os
2
- from github import Github
3
- import base64
4
- import shutil
5
- import zipfile
6
- from io import BytesIO
7
-
8
-
9
- # Global Variables
10
-
11
- # HF ------------
12
- hf_folder_path = 'api/embeddings'
13
- zip_name = 'embeddings'
14
-
15
- # Github -------
16
- github_token = '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' # Replace with your GitHub token
17
- repo_name = 'AdityaMetkar/Patseer-Database' # Replace with your repository, e.g., 'octocat/Hello-World'
18
- folder_path = 'Manual Database/embeddings.zip' # Replace with the path to the folder in the repository
19
-
20
- # Authenticate to GitHub
21
- g = Github(github_token)
22
- repo = g.get_repo(repo_name)
23
-
24
-
25
-
26
- # Functions -------------------------------
27
- def zip_folder():
28
- shutil.make_archive(zip_name, 'zip', hf_folder_path)
29
- return zip_name + '.zip'
30
-
31
-
32
- def update_db():
33
-
34
- try:
35
- # Check if the file already exists in the repository
36
- existing_file = repo.get_contents(folder_path)
37
-
38
- compressed_zip = zip_folder()
39
- with open(compressed_zip, 'rb') as file:
40
- file_content = file.read()
41
-
42
- # Update the existing file
43
- repo.update_file(existing_file.path, "New DB Update", file_content, existing_file.sha)
44
- print(f"Updated {folder_path} in GitHub repository.")
45
-
46
- except Exception as e:
47
- print(f"Error: {e}")
48
-
49
-
50
- def download_db():
51
- if not os.path.exists(hf_folder_path):
52
- os.makedirs(hf_folder_path)
53
-
54
- file_content = repo.get_contents(folder_path)
55
-
56
- try:
57
- # Download the zip file content from GitHub
58
- file_content = repo.get_contents(folder_path)
59
- zip_data = base64.b64decode(file_content.content)
60
-
61
- # Extract the downloaded zip file directly to hf_folder_path using shutil
62
- with zipfile.ZipFile(BytesIO(zip_data)) as zip_ref:
63
- for file in zip_ref.namelist():
64
- zip_ref.extract(file, hf_folder_path)
65
-
66
- print(f"Successfully unzipped files to {hf_folder_path}")
67
-
68
- except Exception as e:
69
- print(f"Error: {e}")
70
-
71
-
72
-
73
-
74
- # Download the folder
75
- # download_folder()
76
- # update_db()
77
-
 
1
+ import os
2
+ from github import Github
3
+ import base64
4
+ import shutil
5
+ import zipfile
6
+ from io import BytesIO
7
+
8
+
9
+ # Global Variables
10
+
11
+ # HF ------------
12
+ hf_folder_path = '/home/user/app/embeddings'
13
+ zip_name = 'embeddings'
14
+
15
+ # Github -------
16
+ github_token = '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' # Replace with your GitHub token
17
+ repo_name = 'AdityaMetkar/Patseer-Database' # Replace with your repository, e.g., 'octocat/Hello-World'
18
+ folder_path = 'Manual Database/embeddings.zip' # Replace with the path to the folder in the repository
19
+
20
+ # Authenticate to GitHub
21
+ g = Github(github_token)
22
+ repo = g.get_repo(repo_name)
23
+
24
+
25
+
26
+ # Functions -------------------------------
27
+ def zip_folder():
28
+ shutil.make_archive(zip_name, 'zip', hf_folder_path)
29
+ return zip_name + '.zip'
30
+
31
+
32
+ def update_db():
33
+
34
+ try:
35
+ # Check if the file already exists in the repository
36
+ existing_file = repo.get_contents(folder_path)
37
+
38
+ compressed_zip = zip_folder()
39
+ with open(compressed_zip, 'rb') as file:
40
+ file_content = file.read()
41
+
42
+ # Update the existing file
43
+ repo.update_file(existing_file.path, "New DB Update", file_content, existing_file.sha)
44
+ print(f"Updated {folder_path} in GitHub repository.")
45
+
46
+ except Exception as e:
47
+ print(f"Error: {e}")
48
+
49
+
50
+ def download_db():
51
+ if not os.path.exists(hf_folder_path):
52
+ os.makedirs(hf_folder_path)
53
+
54
+ file_content = repo.get_contents(folder_path)
55
+
56
+ try:
57
+ # Download the zip file content from GitHub
58
+ file_content = repo.get_contents(folder_path)
59
+ zip_data = base64.b64decode(file_content.content)
60
+
61
+ # Extract the downloaded zip file directly to hf_folder_path using shutil
62
+ with zipfile.ZipFile(BytesIO(zip_data)) as zip_ref:
63
+ for file in zip_ref.namelist():
64
+ zip_ref.extract(file, hf_folder_path)
65
+
66
+ print(f"Successfully unzipped files to {hf_folder_path}")
67
+
68
+ except Exception as e:
69
+ print(f"Error: {e}")
70
+
71
+
72
+
73
+
74
+ # Download the folder
75
+ # download_folder()
76
+ # update_db()
77
+