Spaces:
Running
Running
more fixes
Browse files- backend/scripts/backup.py +21 -99
- backend/scripts/restore.py +19 -84
- backend/start.sh +30 -83
backend/scripts/backup.py
CHANGED
@@ -6,41 +6,29 @@ import datetime
|
|
6 |
import sqlite3
|
7 |
from pathlib import Path
|
8 |
from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
|
|
|
9 |
|
10 |
-
|
11 |
-
# 1) Determine DB location from your env.py (DATA_DIR), defaulting to /app/backend/data
|
12 |
-
###############################################################################
|
13 |
-
DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
|
14 |
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
15 |
|
16 |
-
|
17 |
-
|
18 |
-
###############################################################################
|
19 |
-
BACKUP_DIR = "/workspace/open_webui/db_backup"
|
20 |
TIMESTAMP_FILE_PATH = os.path.join(BACKUP_DIR, "last_backup_time.txt")
|
21 |
DB_GPG_PATH = os.path.join(BACKUP_DIR, "webui.db.gpg")
|
22 |
|
23 |
-
#
|
24 |
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
25 |
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
26 |
|
27 |
|
28 |
def ensure_directories():
|
29 |
-
"""
|
30 |
-
Create/verify /workspace/open_webui/db_backup.
|
31 |
-
"""
|
32 |
try:
|
33 |
os.makedirs(BACKUP_DIR, mode=0o755, exist_ok=True)
|
34 |
-
dir_stat = os.stat(BACKUP_DIR)
|
35 |
-
print(f"Backup directory {BACKUP_DIR} exists with permissions: {oct(dir_stat.st_mode)[-3:]}")
|
36 |
-
|
37 |
-
# Quick test to verify we can write
|
38 |
test_file = os.path.join(BACKUP_DIR, '.write_test')
|
39 |
with open(test_file, 'w') as f:
|
40 |
f.write('test')
|
41 |
os.remove(test_file)
|
42 |
print(f"Successfully verified write access to {BACKUP_DIR}")
|
43 |
-
|
44 |
return True
|
45 |
except Exception as e:
|
46 |
print(f"Error creating/verifying backup directory: {e}")
|
@@ -48,18 +36,11 @@ def ensure_directories():
|
|
48 |
|
49 |
|
50 |
def verify_database():
|
51 |
-
"""
|
52 |
-
Check that webui.db exists and passes PRAGMA integrity_check.
|
53 |
-
"""
|
54 |
if not os.path.exists(DB_FILE_PATH):
|
55 |
print(f"Database file not found at: {DB_FILE_PATH}")
|
56 |
return False
|
57 |
|
58 |
try:
|
59 |
-
file_stat = os.stat(DB_FILE_PATH)
|
60 |
-
print(f"Database file size: {file_stat.st_size:,} bytes")
|
61 |
-
print(f"Database file permissions: {oct(file_stat.st_mode)[-3:]}")
|
62 |
-
|
63 |
with sqlite3.connect(DB_FILE_PATH) as conn:
|
64 |
cursor = conn.cursor()
|
65 |
cursor.execute("PRAGMA integrity_check;")
|
@@ -68,80 +49,52 @@ def verify_database():
|
|
68 |
tables = cursor.fetchall()
|
69 |
|
70 |
if result.lower() == "ok" and len(tables) > 0:
|
71 |
-
print("Database
|
72 |
-
print(f"Found {len(tables)} tables in database.")
|
73 |
return True
|
74 |
else:
|
75 |
-
print("Database
|
76 |
-
if result.lower() != "ok":
|
77 |
-
print(f"Integrity check result: {result}")
|
78 |
-
if len(tables) == 0:
|
79 |
-
print("No tables found in database.")
|
80 |
return False
|
81 |
-
except sqlite3.Error as e:
|
82 |
-
print(f"SQLite error: {e}")
|
83 |
-
return False
|
84 |
except Exception as e:
|
85 |
-
print(f"
|
86 |
return False
|
87 |
|
88 |
|
89 |
def encrypt_database(passphrase):
|
90 |
-
"""
|
91 |
-
Encrypt /app/backend/data/webui.db to /workspace/open_webui/db_backup/webui.db.gpg.
|
92 |
-
"""
|
93 |
try:
|
94 |
-
|
95 |
-
|
96 |
-
# Make sure .gnupg is created with correct permissions
|
97 |
-
gnupg_dir = '/root/.gnupg'
|
98 |
-
os.makedirs(gnupg_dir, mode=0o700, exist_ok=True)
|
99 |
|
100 |
-
print("Running GPG encryption...")
|
101 |
encrypt_cmd = [
|
102 |
"gpg",
|
103 |
"--batch",
|
104 |
"--yes",
|
105 |
"--passphrase", passphrase,
|
106 |
-
"--pinentry-mode", "loopback",
|
107 |
"-c",
|
108 |
"--cipher-algo", "AES256",
|
109 |
"-o", DB_GPG_PATH,
|
110 |
DB_FILE_PATH
|
111 |
]
|
112 |
|
113 |
-
result = subprocess.run(
|
114 |
-
encrypt_cmd,
|
115 |
-
capture_output=True,
|
116 |
-
text=True
|
117 |
-
)
|
118 |
|
119 |
if result.returncode != 0:
|
120 |
-
print(f"GPG encryption failed
|
121 |
-
print(f"GPG stderr: {result.stderr}")
|
122 |
return False
|
123 |
|
124 |
if os.path.exists(DB_GPG_PATH):
|
125 |
-
size
|
126 |
-
print(f"Encryption successful. Encrypted file size: {size:,} bytes")
|
127 |
return True
|
128 |
-
|
129 |
-
print("GPG reported success, but webui.db.gpg not found.")
|
130 |
-
return False
|
131 |
except Exception as e:
|
132 |
-
print(f"Encryption
|
133 |
return False
|
134 |
|
135 |
|
136 |
def get_last_backup_time(repo_id, hf_token):
|
137 |
-
"""
|
138 |
-
Retrieve last backup timestamp from the HF repo (if it exists).
|
139 |
-
"""
|
140 |
try:
|
141 |
api = HfApi()
|
142 |
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
143 |
if REPO_TIMESTAMP_FILE not in files:
|
144 |
-
print("No timestamp file found in repository.")
|
145 |
return None
|
146 |
|
147 |
tmp_file = hf_hub_download(
|
@@ -159,9 +112,6 @@ def get_last_backup_time(repo_id, hf_token):
|
|
159 |
|
160 |
|
161 |
def save_timestamp_locally():
|
162 |
-
"""
|
163 |
-
Save the current UTC time to /workspace/open_webui/db_backup/last_backup_time.txt
|
164 |
-
"""
|
165 |
try:
|
166 |
now = datetime.datetime.now(datetime.timezone.utc)
|
167 |
os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
|
@@ -170,7 +120,6 @@ def save_timestamp_locally():
|
|
170 |
with open(temp_path, "w", encoding="utf-8") as f:
|
171 |
f.write(now.isoformat())
|
172 |
os.replace(temp_path, TIMESTAMP_FILE_PATH)
|
173 |
-
|
174 |
return True
|
175 |
except Exception as e:
|
176 |
print(f"Error saving timestamp: {e}")
|
@@ -178,26 +127,15 @@ def save_timestamp_locally():
|
|
178 |
|
179 |
|
180 |
def backup_db():
|
181 |
-
"""
|
182 |
-
Main backup process:
|
183 |
-
- Read passphrase, HF creds
|
184 |
-
- Create /workspace/open_webui/db_backup
|
185 |
-
- Possibly skip if threshold not met
|
186 |
-
- Verify DB
|
187 |
-
- Encrypt DB
|
188 |
-
- Save timestamp
|
189 |
-
- Upload to HF
|
190 |
-
"""
|
191 |
passphrase = os.environ.get("BACKUP_PASSPHRASE")
|
192 |
hf_token = os.environ.get("HF_TOKEN")
|
193 |
space_id = os.environ.get("SPACE_ID")
|
194 |
|
195 |
if not all([passphrase, hf_token, space_id]):
|
196 |
-
print("Error: Missing required environment variables (BACKUP_PASSPHRASE, HF_TOKEN, SPACE_ID)
|
197 |
return False
|
198 |
|
199 |
if not ensure_directories():
|
200 |
-
print("Failed to create or verify backup directories.")
|
201 |
return False
|
202 |
|
203 |
threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
|
@@ -209,39 +147,23 @@ def backup_db():
|
|
209 |
last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
|
210 |
elapsed = now - last_backup_dt
|
211 |
if elapsed.total_seconds() < threshold_minutes * 60:
|
212 |
-
print(f"Last backup was
|
213 |
-
print(f"Threshold is {threshold_minutes} minutes. Skipping backup.")
|
214 |
return True
|
215 |
-
else:
|
216 |
-
print("Backup threshold check disabled (0).")
|
217 |
|
218 |
-
# Verify DB
|
219 |
if not verify_database():
|
220 |
-
print("Database verification failed, aborting backup.")
|
221 |
return False
|
222 |
|
223 |
-
# Encrypt
|
224 |
if not encrypt_database(passphrase):
|
225 |
-
print("Database encryption failed, aborting backup.")
|
226 |
return False
|
227 |
|
228 |
-
# Save timestamp
|
229 |
if not save_timestamp_locally():
|
230 |
-
print("Warning: Failed to save timestamp
|
231 |
|
232 |
-
# Upload to HF
|
233 |
-
print("\nUploading to Hugging Face Spaces...")
|
234 |
try:
|
235 |
api = HfApi()
|
236 |
operations = [
|
237 |
-
CommitOperationAdd(
|
238 |
-
|
239 |
-
path_or_fileobj=DB_GPG_PATH
|
240 |
-
),
|
241 |
-
CommitOperationAdd(
|
242 |
-
path_in_repo=REPO_TIMESTAMP_FILE,
|
243 |
-
path_or_fileobj=TIMESTAMP_FILE_PATH
|
244 |
-
)
|
245 |
]
|
246 |
api.create_commit(
|
247 |
repo_id=space_id,
|
@@ -253,7 +175,7 @@ def backup_db():
|
|
253 |
print("Backup files uploaded successfully!")
|
254 |
return True
|
255 |
except Exception as e:
|
256 |
-
print(f"Error uploading
|
257 |
return False
|
258 |
|
259 |
|
|
|
6 |
import sqlite3
|
7 |
from pathlib import Path
|
8 |
from huggingface_hub import HfApi, hf_hub_download, CommitOperationAdd
|
9 |
+
from open_webui.env import DATA_DIR
|
10 |
|
11 |
+
# Application database path
|
|
|
|
|
|
|
12 |
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
13 |
|
14 |
+
# Backup paths
|
15 |
+
BACKUP_DIR = os.environ.get("BACKUP_DIR", "/tmp/open_webui/db_backup")
|
|
|
|
|
16 |
TIMESTAMP_FILE_PATH = os.path.join(BACKUP_DIR, "last_backup_time.txt")
|
17 |
DB_GPG_PATH = os.path.join(BACKUP_DIR, "webui.db.gpg")
|
18 |
|
19 |
+
# Hugging Face repo paths
|
20 |
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
21 |
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
22 |
|
23 |
|
24 |
def ensure_directories():
|
|
|
|
|
|
|
25 |
try:
|
26 |
os.makedirs(BACKUP_DIR, mode=0o755, exist_ok=True)
|
|
|
|
|
|
|
|
|
27 |
test_file = os.path.join(BACKUP_DIR, '.write_test')
|
28 |
with open(test_file, 'w') as f:
|
29 |
f.write('test')
|
30 |
os.remove(test_file)
|
31 |
print(f"Successfully verified write access to {BACKUP_DIR}")
|
|
|
32 |
return True
|
33 |
except Exception as e:
|
34 |
print(f"Error creating/verifying backup directory: {e}")
|
|
|
36 |
|
37 |
|
38 |
def verify_database():
|
|
|
|
|
|
|
39 |
if not os.path.exists(DB_FILE_PATH):
|
40 |
print(f"Database file not found at: {DB_FILE_PATH}")
|
41 |
return False
|
42 |
|
43 |
try:
|
|
|
|
|
|
|
|
|
44 |
with sqlite3.connect(DB_FILE_PATH) as conn:
|
45 |
cursor = conn.cursor()
|
46 |
cursor.execute("PRAGMA integrity_check;")
|
|
|
49 |
tables = cursor.fetchall()
|
50 |
|
51 |
if result.lower() == "ok" and len(tables) > 0:
|
52 |
+
print(f"Database verified: {len(tables)} tables found")
|
|
|
53 |
return True
|
54 |
else:
|
55 |
+
print("Database verification failed")
|
|
|
|
|
|
|
|
|
56 |
return False
|
|
|
|
|
|
|
57 |
except Exception as e:
|
58 |
+
print(f"Database verification error: {e}")
|
59 |
return False
|
60 |
|
61 |
|
62 |
def encrypt_database(passphrase):
|
|
|
|
|
|
|
63 |
try:
|
64 |
+
os.makedirs('/root/.gnupg', mode=0o700, exist_ok=True)
|
|
|
|
|
|
|
|
|
65 |
|
|
|
66 |
encrypt_cmd = [
|
67 |
"gpg",
|
68 |
"--batch",
|
69 |
"--yes",
|
70 |
"--passphrase", passphrase,
|
71 |
+
"--pinentry-mode", "loopback",
|
72 |
"-c",
|
73 |
"--cipher-algo", "AES256",
|
74 |
"-o", DB_GPG_PATH,
|
75 |
DB_FILE_PATH
|
76 |
]
|
77 |
|
78 |
+
result = subprocess.run(encrypt_cmd, capture_output=True, text=True)
|
|
|
|
|
|
|
|
|
79 |
|
80 |
if result.returncode != 0:
|
81 |
+
print(f"GPG encryption failed: {result.stderr}")
|
|
|
82 |
return False
|
83 |
|
84 |
if os.path.exists(DB_GPG_PATH):
|
85 |
+
print(f"Encryption successful. File size: {os.path.getsize(DB_GPG_PATH):,} bytes")
|
|
|
86 |
return True
|
87 |
+
return False
|
|
|
|
|
88 |
except Exception as e:
|
89 |
+
print(f"Encryption error: {e}")
|
90 |
return False
|
91 |
|
92 |
|
93 |
def get_last_backup_time(repo_id, hf_token):
|
|
|
|
|
|
|
94 |
try:
|
95 |
api = HfApi()
|
96 |
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
97 |
if REPO_TIMESTAMP_FILE not in files:
|
|
|
98 |
return None
|
99 |
|
100 |
tmp_file = hf_hub_download(
|
|
|
112 |
|
113 |
|
114 |
def save_timestamp_locally():
|
|
|
|
|
|
|
115 |
try:
|
116 |
now = datetime.datetime.now(datetime.timezone.utc)
|
117 |
os.makedirs(os.path.dirname(TIMESTAMP_FILE_PATH), exist_ok=True)
|
|
|
120 |
with open(temp_path, "w", encoding="utf-8") as f:
|
121 |
f.write(now.isoformat())
|
122 |
os.replace(temp_path, TIMESTAMP_FILE_PATH)
|
|
|
123 |
return True
|
124 |
except Exception as e:
|
125 |
print(f"Error saving timestamp: {e}")
|
|
|
127 |
|
128 |
|
129 |
def backup_db():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
130 |
passphrase = os.environ.get("BACKUP_PASSPHRASE")
|
131 |
hf_token = os.environ.get("HF_TOKEN")
|
132 |
space_id = os.environ.get("SPACE_ID")
|
133 |
|
134 |
if not all([passphrase, hf_token, space_id]):
|
135 |
+
print("Error: Missing required environment variables (BACKUP_PASSPHRASE, HF_TOKEN, SPACE_ID)")
|
136 |
return False
|
137 |
|
138 |
if not ensure_directories():
|
|
|
139 |
return False
|
140 |
|
141 |
threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
|
|
|
147 |
last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
|
148 |
elapsed = now - last_backup_dt
|
149 |
if elapsed.total_seconds() < threshold_minutes * 60:
|
150 |
+
print(f"Last backup was {elapsed.total_seconds()/60:.1f} min ago (threshold: {threshold_minutes})")
|
|
|
151 |
return True
|
|
|
|
|
152 |
|
|
|
153 |
if not verify_database():
|
|
|
154 |
return False
|
155 |
|
|
|
156 |
if not encrypt_database(passphrase):
|
|
|
157 |
return False
|
158 |
|
|
|
159 |
if not save_timestamp_locally():
|
160 |
+
print("Warning: Failed to save timestamp")
|
161 |
|
|
|
|
|
162 |
try:
|
163 |
api = HfApi()
|
164 |
operations = [
|
165 |
+
CommitOperationAdd(path_in_repo=REPO_DB_GPG_FILE, path_or_fileobj=DB_GPG_PATH),
|
166 |
+
CommitOperationAdd(path_in_repo=REPO_TIMESTAMP_FILE, path_or_fileobj=TIMESTAMP_FILE_PATH)
|
|
|
|
|
|
|
|
|
|
|
|
|
167 |
]
|
168 |
api.create_commit(
|
169 |
repo_id=space_id,
|
|
|
175 |
print("Backup files uploaded successfully!")
|
176 |
return True
|
177 |
except Exception as e:
|
178 |
+
print(f"Error uploading to HF: {e}")
|
179 |
return False
|
180 |
|
181 |
|
backend/scripts/restore.py
CHANGED
@@ -5,47 +5,32 @@ import subprocess
|
|
5 |
import datetime
|
6 |
import sqlite3
|
7 |
from pathlib import Path
|
8 |
-
|
9 |
from huggingface_hub import HfApi, hf_hub_download
|
|
|
10 |
|
11 |
-
|
12 |
-
# 1) Determine the *final* database file location.
|
13 |
-
# - We read DATA_DIR from env, defaulting to "/app/backend/data".
|
14 |
-
# - The final DB (decrypted) should be "/app/backend/data/webui.db" so
|
15 |
-
# the app will use the restored DB on startup.
|
16 |
-
###############################################################################
|
17 |
-
DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
|
18 |
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
19 |
|
20 |
-
|
21 |
-
|
22 |
-
# - A writable directory on Hugging Face is "/tmp/open_webui/db_backup".
|
23 |
-
###############################################################################
|
24 |
-
RESTORE_BACKUP_DIR = "/tmp/open_webui/db_backup"
|
25 |
DB_GPG_PATH = os.path.join(RESTORE_BACKUP_DIR, "webui.db.gpg")
|
26 |
TIMESTAMP_FILE_PATH = os.path.join(RESTORE_BACKUP_DIR, "last_backup_time.txt")
|
27 |
|
28 |
-
#
|
29 |
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
30 |
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
31 |
|
32 |
|
33 |
def check_requirements():
|
34 |
-
"""
|
35 |
-
Verify that GPG is installed and available in the system.
|
36 |
-
"""
|
37 |
try:
|
38 |
subprocess.run(["gpg", "--version"], check=True, capture_output=True)
|
39 |
return True
|
40 |
except (subprocess.CalledProcessError, FileNotFoundError):
|
41 |
-
print("Error: gpg is not installed or not in PATH
|
42 |
return False
|
43 |
|
44 |
|
45 |
def validate_secrets():
|
46 |
-
"""
|
47 |
-
Ensure all required environment variables are set: BACKUP_PASSPHRASE, HF_TOKEN, SPACE_ID.
|
48 |
-
"""
|
49 |
required_vars = ["BACKUP_PASSPHRASE", "HF_TOKEN", "SPACE_ID"]
|
50 |
missing = [var for var in required_vars if not os.environ.get(var)]
|
51 |
|
@@ -56,15 +41,8 @@ def validate_secrets():
|
|
56 |
|
57 |
|
58 |
def ensure_directories():
|
59 |
-
"""
|
60 |
-
Create necessary directories for database (DATA_DIR) and backup files (RESTORE_BACKUP_DIR).
|
61 |
-
We must ensure the final DB directory is writable so the decrypted DB can be placed there.
|
62 |
-
"""
|
63 |
try:
|
64 |
-
# Create the final DB folder (if not already existing)
|
65 |
os.makedirs(DATA_DIR, mode=0o755, exist_ok=True)
|
66 |
-
|
67 |
-
# Create the backup folder in /tmp
|
68 |
os.makedirs(RESTORE_BACKUP_DIR, mode=0o755, exist_ok=True)
|
69 |
return True
|
70 |
except Exception as e:
|
@@ -73,20 +51,15 @@ def ensure_directories():
|
|
73 |
|
74 |
|
75 |
def get_latest_backup_info(repo_id, hf_token):
|
76 |
-
"""
|
77 |
-
Check if a backup (webui.db.gpg) exists in the HF Space. If so, return True + timestamp (or None).
|
78 |
-
"""
|
79 |
api = HfApi()
|
80 |
try:
|
81 |
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
82 |
|
83 |
-
# Check if the encrypted DB is present
|
84 |
backup_exists = (REPO_DB_GPG_FILE in files)
|
85 |
if not backup_exists:
|
86 |
-
print("No backup file found in
|
87 |
return False, None
|
88 |
|
89 |
-
# Attempt to fetch the timestamp file
|
90 |
if REPO_TIMESTAMP_FILE in files:
|
91 |
try:
|
92 |
timestamp_file = hf_hub_download(
|
@@ -101,20 +74,15 @@ def get_latest_backup_info(repo_id, hf_token):
|
|
101 |
print(f"Found backup from: {timestamp} UTC")
|
102 |
return True, timestamp
|
103 |
except Exception as e:
|
104 |
-
print(f"Could not read timestamp
|
105 |
return True, None
|
106 |
-
|
107 |
-
print("No timestamp file found, but backup file exists.")
|
108 |
-
return True, None
|
109 |
except Exception as e:
|
110 |
print(f"Error checking repository: {e}")
|
111 |
return False, None
|
112 |
|
113 |
|
114 |
def download_backup(repo_id, hf_token):
|
115 |
-
"""
|
116 |
-
Download the encrypted database backup from Hugging Face to /tmp/open_webui/db_backup.
|
117 |
-
"""
|
118 |
try:
|
119 |
print("Downloading encrypted database backup...")
|
120 |
temp_file = hf_hub_download(
|
@@ -123,9 +91,8 @@ def download_backup(repo_id, hf_token):
|
|
123 |
filename=REPO_DB_GPG_FILE,
|
124 |
token=hf_token
|
125 |
)
|
126 |
-
# Move the downloaded file to DB_GPG_PATH
|
127 |
os.replace(temp_file, DB_GPG_PATH)
|
128 |
-
print("Backup downloaded successfully
|
129 |
return True
|
130 |
except Exception as e:
|
131 |
print(f"Error downloading backup: {e}")
|
@@ -133,20 +100,13 @@ def download_backup(repo_id, hf_token):
|
|
133 |
|
134 |
|
135 |
def decrypt_database(passphrase):
|
136 |
-
"""
|
137 |
-
Decrypt the database into DATA_DIR/webui.db so the app can use it.
|
138 |
-
The encrypted file is at /tmp/open_webui/db_backup/webui.db.gpg.
|
139 |
-
"""
|
140 |
if not os.path.exists(DB_GPG_PATH):
|
141 |
-
print("No encrypted backup found locally.
|
142 |
-
return True
|
143 |
|
144 |
try:
|
145 |
-
print("Decrypting database
|
146 |
-
|
147 |
-
# Ensure /root/.gnupg is set up
|
148 |
-
gnupg_dir = "/root/.gnupg"
|
149 |
-
os.makedirs(gnupg_dir, mode=0o700, exist_ok=True)
|
150 |
|
151 |
decrypt_cmd = [
|
152 |
"gpg",
|
@@ -161,18 +121,12 @@ def decrypt_database(passphrase):
|
|
161 |
print(f"Database decrypted successfully to {DB_FILE_PATH}")
|
162 |
return True
|
163 |
except subprocess.CalledProcessError as e:
|
164 |
-
print(f"
|
165 |
-
if e.stderr:
|
166 |
-
print(f"GPG error output: {e.stderr.decode(errors='ignore')}")
|
167 |
return False
|
168 |
|
169 |
|
170 |
def verify_database():
|
171 |
-
"""
|
172 |
-
Verify the integrity of the restored DB at DATA_DIR/webui.db using SQLite's built-in checks.
|
173 |
-
"""
|
174 |
if not os.path.exists(DB_FILE_PATH):
|
175 |
-
# If there's no DB yet, might be fresh start, not an immediate error
|
176 |
return True
|
177 |
|
178 |
try:
|
@@ -185,32 +139,17 @@ def verify_database():
|
|
185 |
tables = cursor.fetchall()
|
186 |
|
187 |
if result.lower() == "ok" and len(tables) > 0:
|
188 |
-
print("Database
|
189 |
-
print(f"Found {len(tables)} tables in database.")
|
190 |
return True
|
191 |
else:
|
192 |
-
print("Database
|
193 |
-
if result.lower() != "ok":
|
194 |
-
print(f"Integrity check result: {result}")
|
195 |
-
if len(tables) == 0:
|
196 |
-
print("No tables found in database.")
|
197 |
return False
|
198 |
-
except sqlite3.Error as e:
|
199 |
-
print(f"Database verification failed: {e}")
|
200 |
-
return False
|
201 |
except Exception as e:
|
202 |
-
print(f"
|
203 |
return False
|
204 |
|
205 |
|
206 |
def restore_db():
|
207 |
-
"""
|
208 |
-
Main restore function:
|
209 |
-
1. Check GPG + environment
|
210 |
-
2. Create directories
|
211 |
-
3. Check if remote backup exists, download + decrypt
|
212 |
-
4. Verify DB
|
213 |
-
"""
|
214 |
if not check_requirements() or not validate_secrets():
|
215 |
return False
|
216 |
|
@@ -224,21 +163,17 @@ def restore_db():
|
|
224 |
backup_exists, timestamp = get_latest_backup_info(space_id, hf_token)
|
225 |
if backup_exists:
|
226 |
if not download_backup(space_id, hf_token):
|
227 |
-
print("Failed to download backup.")
|
228 |
return False
|
229 |
|
230 |
if not decrypt_database(passphrase):
|
231 |
-
print("Failed to decrypt database.")
|
232 |
return False
|
233 |
|
234 |
if not verify_database():
|
235 |
-
print("Database integrity verification failed.")
|
236 |
-
# Remove the corrupted DB so we don’t keep a broken file
|
237 |
if os.path.exists(DB_FILE_PATH):
|
238 |
os.unlink(DB_FILE_PATH)
|
239 |
return False
|
240 |
else:
|
241 |
-
print("No backup found - starting with
|
242 |
|
243 |
print("Database restore completed successfully!")
|
244 |
return True
|
|
|
5 |
import datetime
|
6 |
import sqlite3
|
7 |
from pathlib import Path
|
|
|
8 |
from huggingface_hub import HfApi, hf_hub_download
|
9 |
+
from open_webui.env import DATA_DIR # Import actual DATA_DIR from application
|
10 |
|
11 |
+
# Application database path
|
|
|
|
|
|
|
|
|
|
|
|
|
12 |
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
13 |
|
14 |
+
# Restore paths
|
15 |
+
RESTORE_BACKUP_DIR = os.environ.get("BACKUP_DIR", "/tmp/open_webui/db_backup")
|
|
|
|
|
|
|
16 |
DB_GPG_PATH = os.path.join(RESTORE_BACKUP_DIR, "webui.db.gpg")
|
17 |
TIMESTAMP_FILE_PATH = os.path.join(RESTORE_BACKUP_DIR, "last_backup_time.txt")
|
18 |
|
19 |
+
# Hugging Face repo paths
|
20 |
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
21 |
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
22 |
|
23 |
|
24 |
def check_requirements():
|
|
|
|
|
|
|
25 |
try:
|
26 |
subprocess.run(["gpg", "--version"], check=True, capture_output=True)
|
27 |
return True
|
28 |
except (subprocess.CalledProcessError, FileNotFoundError):
|
29 |
+
print("Error: gpg is not installed or not in PATH")
|
30 |
return False
|
31 |
|
32 |
|
33 |
def validate_secrets():
|
|
|
|
|
|
|
34 |
required_vars = ["BACKUP_PASSPHRASE", "HF_TOKEN", "SPACE_ID"]
|
35 |
missing = [var for var in required_vars if not os.environ.get(var)]
|
36 |
|
|
|
41 |
|
42 |
|
43 |
def ensure_directories():
|
|
|
|
|
|
|
|
|
44 |
try:
|
|
|
45 |
os.makedirs(DATA_DIR, mode=0o755, exist_ok=True)
|
|
|
|
|
46 |
os.makedirs(RESTORE_BACKUP_DIR, mode=0o755, exist_ok=True)
|
47 |
return True
|
48 |
except Exception as e:
|
|
|
51 |
|
52 |
|
53 |
def get_latest_backup_info(repo_id, hf_token):
|
|
|
|
|
|
|
54 |
api = HfApi()
|
55 |
try:
|
56 |
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
57 |
|
|
|
58 |
backup_exists = (REPO_DB_GPG_FILE in files)
|
59 |
if not backup_exists:
|
60 |
+
print("No backup file found in repository")
|
61 |
return False, None
|
62 |
|
|
|
63 |
if REPO_TIMESTAMP_FILE in files:
|
64 |
try:
|
65 |
timestamp_file = hf_hub_download(
|
|
|
74 |
print(f"Found backup from: {timestamp} UTC")
|
75 |
return True, timestamp
|
76 |
except Exception as e:
|
77 |
+
print(f"Could not read timestamp: {e}")
|
78 |
return True, None
|
79 |
+
return True, None
|
|
|
|
|
80 |
except Exception as e:
|
81 |
print(f"Error checking repository: {e}")
|
82 |
return False, None
|
83 |
|
84 |
|
85 |
def download_backup(repo_id, hf_token):
|
|
|
|
|
|
|
86 |
try:
|
87 |
print("Downloading encrypted database backup...")
|
88 |
temp_file = hf_hub_download(
|
|
|
91 |
filename=REPO_DB_GPG_FILE,
|
92 |
token=hf_token
|
93 |
)
|
|
|
94 |
os.replace(temp_file, DB_GPG_PATH)
|
95 |
+
print("Backup downloaded successfully")
|
96 |
return True
|
97 |
except Exception as e:
|
98 |
print(f"Error downloading backup: {e}")
|
|
|
100 |
|
101 |
|
102 |
def decrypt_database(passphrase):
|
|
|
|
|
|
|
|
|
103 |
if not os.path.exists(DB_GPG_PATH):
|
104 |
+
print("No encrypted backup found locally. Starting with fresh DB")
|
105 |
+
return True
|
106 |
|
107 |
try:
|
108 |
+
print("Decrypting database...")
|
109 |
+
os.makedirs("/root/.gnupg", mode=0o700, exist_ok=True)
|
|
|
|
|
|
|
110 |
|
111 |
decrypt_cmd = [
|
112 |
"gpg",
|
|
|
121 |
print(f"Database decrypted successfully to {DB_FILE_PATH}")
|
122 |
return True
|
123 |
except subprocess.CalledProcessError as e:
|
124 |
+
print(f"Decryption failed: {e.stderr.decode(errors='ignore')}")
|
|
|
|
|
125 |
return False
|
126 |
|
127 |
|
128 |
def verify_database():
|
|
|
|
|
|
|
129 |
if not os.path.exists(DB_FILE_PATH):
|
|
|
130 |
return True
|
131 |
|
132 |
try:
|
|
|
139 |
tables = cursor.fetchall()
|
140 |
|
141 |
if result.lower() == "ok" and len(tables) > 0:
|
142 |
+
print(f"Database verified: {len(tables)} tables found")
|
|
|
143 |
return True
|
144 |
else:
|
145 |
+
print("Database verification failed")
|
|
|
|
|
|
|
|
|
146 |
return False
|
|
|
|
|
|
|
147 |
except Exception as e:
|
148 |
+
print(f"Database verification error: {e}")
|
149 |
return False
|
150 |
|
151 |
|
152 |
def restore_db():
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
153 |
if not check_requirements() or not validate_secrets():
|
154 |
return False
|
155 |
|
|
|
163 |
backup_exists, timestamp = get_latest_backup_info(space_id, hf_token)
|
164 |
if backup_exists:
|
165 |
if not download_backup(space_id, hf_token):
|
|
|
166 |
return False
|
167 |
|
168 |
if not decrypt_database(passphrase):
|
|
|
169 |
return False
|
170 |
|
171 |
if not verify_database():
|
|
|
|
|
172 |
if os.path.exists(DB_FILE_PATH):
|
173 |
os.unlink(DB_FILE_PATH)
|
174 |
return False
|
175 |
else:
|
176 |
+
print("No backup found - starting with fresh database")
|
177 |
|
178 |
print("Database restore completed successfully!")
|
179 |
return True
|
backend/start.sh
CHANGED
@@ -1,150 +1,97 @@
|
|
1 |
#!/usr/bin/env bash
|
2 |
|
3 |
-
|
4 |
-
|
5 |
-
ORIGINAL_DIR=$(pwd)
|
6 |
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
|
11 |
-
|
12 |
-
# Check if we're in the expected directory structure
|
13 |
-
if [[ "$(pwd)" != *"/app/backend" ]]; then
|
14 |
-
log_message "Warning: Unexpected working directory: $(pwd)"
|
15 |
-
log_message "Expected path to contain: /app/backend"
|
16 |
-
return 1
|
17 |
-
fi
|
18 |
-
return 0
|
19 |
-
}
|
20 |
-
|
21 |
-
log_message "Changing to script directory: $SCRIPT_DIR"
|
22 |
-
cd "$SCRIPT_DIR" || {
|
23 |
-
log_message "Error: Could not change to directory: $SCRIPT_DIR"
|
24 |
-
exit 1
|
25 |
-
}
|
26 |
-
|
27 |
-
validate_environment || {
|
28 |
-
log_message "Warning: Environment validation failed, but continuing..."
|
29 |
-
}
|
30 |
-
|
31 |
-
log_message "Working from directory: $(pwd)"
|
32 |
-
|
33 |
-
# ---------------------------------------------------------------------------
|
34 |
-
# IMPORTANT: We no longer rely on /app/backend/data or /app/backend/db_backup.
|
35 |
-
# We define external paths (matching our updated backup/restore scripts):
|
36 |
-
# ---------------------------------------------------------------------------
|
37 |
-
SCRIPTS_DIR="$SCRIPT_DIR/scripts"
|
38 |
-
DATA_DIR="/tmp/open_webui/data"
|
39 |
-
BACKUP_DIR="/tmp/open_webui/db_backup"
|
40 |
-
|
41 |
-
# Validate required environment variables for backup/restore operations
|
42 |
for var in "BACKUP_PASSPHRASE" "HF_TOKEN" "SPACE_ID"; do
|
43 |
if [ -z "${!var}" ]; then
|
44 |
-
echo "Error: $var is not set.
|
45 |
exit 1
|
46 |
fi
|
47 |
done
|
48 |
|
49 |
-
# Restore database from backup
|
50 |
-
|
51 |
-
python "$
|
52 |
restore_status=$?
|
53 |
if [ $restore_status -ne 0 ]; then
|
54 |
-
|
55 |
fi
|
56 |
|
57 |
-
# Handle WebUI secret key
|
58 |
KEY_FILE="$SCRIPT_DIR/.webui_secret_key"
|
59 |
PORT="${PORT:-8080}"
|
60 |
HOST="${HOST:-0.0.0.0}"
|
61 |
|
62 |
if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
|
63 |
-
log_message "Loading WEBUI_SECRET_KEY from file, not provided as an environment variable."
|
64 |
if ! [ -e "$KEY_FILE" ]; then
|
65 |
-
|
66 |
-
head -c 12 /dev/random | base64 > "$KEY_FILE" || {
|
67 |
-
log_message "Error: Could not generate secret key"
|
68 |
-
exit 1
|
69 |
-
}
|
70 |
fi
|
71 |
-
|
72 |
-
WEBUI_SECRET_KEY=$(cat "$KEY_FILE") || {
|
73 |
-
log_message "Error: Could not read secret key"
|
74 |
-
exit 1
|
75 |
-
}
|
76 |
fi
|
77 |
|
78 |
-
#
|
79 |
if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
|
80 |
-
log_message "USE_OLLAMA_DOCKER is set to true, starting ollama serve."
|
81 |
ollama serve &
|
82 |
fi
|
83 |
|
84 |
-
# Configure CUDA
|
85 |
if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
|
86 |
-
log_message "CUDA is enabled, configuring LD_LIBRARY_PATH"
|
87 |
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
|
88 |
fi
|
89 |
|
90 |
# Handle HuggingFace Space deployment
|
91 |
if [ -n "$SPACE_ID" ]; then
|
92 |
-
echo "Configuring
|
93 |
if [ -n "$ADMIN_USER_EMAIL" ] && [ -n "$ADMIN_USER_PASSWORD" ]; then
|
94 |
-
echo "Admin user configured, creating"
|
95 |
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
|
96 |
webui_pid=$!
|
97 |
-
|
98 |
while ! curl -s http://localhost:8080/health > /dev/null; do
|
99 |
sleep 1
|
100 |
done
|
101 |
-
|
102 |
curl \
|
103 |
-X POST "http://localhost:8080/api/v1/auths/signup" \
|
104 |
-H "accept: application/json" \
|
105 |
-H "Content-Type: application/json" \
|
106 |
-d "{ \"email\": \"${ADMIN_USER_EMAIL}\", \"password\": \"${ADMIN_USER_PASSWORD}\", \"name\": \"Admin\" }"
|
107 |
-
|
108 |
kill $webui_pid
|
109 |
fi
|
110 |
export WEBUI_URL=${SPACE_HOST}
|
111 |
fi
|
112 |
|
113 |
-
#
|
114 |
-
log_message "Starting main web server..."
|
115 |
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app \
|
116 |
--host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
|
117 |
WEBUI_PID=$!
|
118 |
|
119 |
-
# Configure backup schedule
|
120 |
-
BACKUP_INITIAL_WAIT="${BACKUP_INITIAL_WAIT:-500}"
|
121 |
-
BACKUP_INTERVAL="${BACKUP_INTERVAL:-21600}"
|
122 |
|
123 |
-
# Start
|
124 |
(
|
125 |
-
|
126 |
-
log_message "Initial wait: ${BACKUP_INITIAL_WAIT}s, Interval: ${BACKUP_INTERVAL}s"
|
127 |
-
|
128 |
sleep "$BACKUP_INITIAL_WAIT"
|
129 |
|
130 |
while true; do
|
131 |
-
|
132 |
-
|
133 |
-
# Run backup script with absolute path
|
134 |
BACKUP_THRESHOLD_MINUTES="${BACKUP_THRESHOLD_MINUTES:-120}" \
|
135 |
-
python "$
|
136 |
|
137 |
-
|
138 |
-
|
139 |
-
log_message "Warning: Backup failed with status $backup_status"
|
140 |
-
else
|
141 |
-
log_message "Backup completed successfully"
|
142 |
fi
|
143 |
|
144 |
-
log_message "Waiting ${BACKUP_INTERVAL} seconds until next backup..."
|
145 |
sleep "$BACKUP_INTERVAL"
|
146 |
done
|
147 |
) &
|
148 |
|
149 |
-
# Keep the container alive by waiting on the main web server
|
150 |
wait $WEBUI_PID
|
|
|
1 |
#!/usr/bin/env bash
|
2 |
|
3 |
+
SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
|
4 |
+
cd "$SCRIPT_DIR" || exit 1
|
|
|
5 |
|
6 |
+
# Configure paths and environment
|
7 |
+
export DATA_DIR="/app/backend/data"
|
8 |
+
export BACKUP_DIR="/tmp/open_webui/db_backup"
|
9 |
|
10 |
+
# Validate environment variables
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
11 |
for var in "BACKUP_PASSPHRASE" "HF_TOKEN" "SPACE_ID"; do
|
12 |
if [ -z "${!var}" ]; then
|
13 |
+
echo "Error: $var is not set. Required for database backup/restore."
|
14 |
exit 1
|
15 |
fi
|
16 |
done
|
17 |
|
18 |
+
# Restore database from backup
|
19 |
+
echo "Restoring database from backup..."
|
20 |
+
python "$SCRIPT_DIR/scripts/restore.py"
|
21 |
restore_status=$?
|
22 |
if [ $restore_status -ne 0 ]; then
|
23 |
+
echo "Warning: Database restore failed. Starting with empty database."
|
24 |
fi
|
25 |
|
26 |
+
# Handle WebUI secret key
|
27 |
KEY_FILE="$SCRIPT_DIR/.webui_secret_key"
|
28 |
PORT="${PORT:-8080}"
|
29 |
HOST="${HOST:-0.0.0.0}"
|
30 |
|
31 |
if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
|
|
|
32 |
if ! [ -e "$KEY_FILE" ]; then
|
33 |
+
head -c 12 /dev/random | base64 > "$KEY_FILE" || exit 1
|
|
|
|
|
|
|
|
|
34 |
fi
|
35 |
+
WEBUI_SECRET_KEY=$(cat "$KEY_FILE") || exit 1
|
|
|
|
|
|
|
|
|
36 |
fi
|
37 |
|
38 |
+
# Start Ollama if enabled
|
39 |
if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
|
|
|
40 |
ollama serve &
|
41 |
fi
|
42 |
|
43 |
+
# Configure CUDA if enabled
|
44 |
if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
|
|
|
45 |
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
|
46 |
fi
|
47 |
|
48 |
# Handle HuggingFace Space deployment
|
49 |
if [ -n "$SPACE_ID" ]; then
|
50 |
+
echo "Configuring HuggingFace Space deployment"
|
51 |
if [ -n "$ADMIN_USER_EMAIL" ] && [ -n "$ADMIN_USER_PASSWORD" ]; then
|
|
|
52 |
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
|
53 |
webui_pid=$!
|
54 |
+
|
55 |
while ! curl -s http://localhost:8080/health > /dev/null; do
|
56 |
sleep 1
|
57 |
done
|
58 |
+
|
59 |
curl \
|
60 |
-X POST "http://localhost:8080/api/v1/auths/signup" \
|
61 |
-H "accept: application/json" \
|
62 |
-H "Content-Type: application/json" \
|
63 |
-d "{ \"email\": \"${ADMIN_USER_EMAIL}\", \"password\": \"${ADMIN_USER_PASSWORD}\", \"name\": \"Admin\" }"
|
64 |
+
|
65 |
kill $webui_pid
|
66 |
fi
|
67 |
export WEBUI_URL=${SPACE_HOST}
|
68 |
fi
|
69 |
|
70 |
+
# Start the main web server
|
|
|
71 |
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app \
|
72 |
--host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
|
73 |
WEBUI_PID=$!
|
74 |
|
75 |
+
# Configure backup schedule
|
76 |
+
BACKUP_INITIAL_WAIT="${BACKUP_INITIAL_WAIT:-500}"
|
77 |
+
BACKUP_INTERVAL="${BACKUP_INTERVAL:-21600}"
|
78 |
|
79 |
+
# Start backup scheduler
|
80 |
(
|
81 |
+
echo "Starting backup scheduler (Initial wait: ${BACKUP_INITIAL_WAIT}s, Interval: ${BACKUP_INTERVAL}s)"
|
|
|
|
|
82 |
sleep "$BACKUP_INITIAL_WAIT"
|
83 |
|
84 |
while true; do
|
85 |
+
echo "Running scheduled backup"
|
|
|
|
|
86 |
BACKUP_THRESHOLD_MINUTES="${BACKUP_THRESHOLD_MINUTES:-120}" \
|
87 |
+
python "$SCRIPT_DIR/scripts/backup.py"
|
88 |
|
89 |
+
if [ $? -ne 0 ]; then
|
90 |
+
echo "Warning: Backup failed"
|
|
|
|
|
|
|
91 |
fi
|
92 |
|
|
|
93 |
sleep "$BACKUP_INTERVAL"
|
94 |
done
|
95 |
) &
|
96 |
|
|
|
97 |
wait $WEBUI_PID
|