Spaces:
Building
Building
restore latest backup files
Browse files- .gitattributes +1 -0
- .gitignore +2 -0
- Dockerfile +3 -0
- backend/requirements.txt +3 -12
- backend/scripts/backup.py +177 -0
- backend/scripts/db_crypt.sh +100 -0
- backend/scripts/restore.py +211 -0
- backend/start.sh +94 -47
- db_backup/last_backup_time.txt +1 -0
- vite.config.ts +1 -1
.gitattributes
CHANGED
@@ -1,3 +1,4 @@
|
|
1 |
*.sh text eol=lf
|
2 |
*.ttf filter=lfs diff=lfs merge=lfs -text
|
3 |
*.jpg filter=lfs diff=lfs merge=lfs -text
|
|
|
|
1 |
*.sh text eol=lf
|
2 |
*.ttf filter=lfs diff=lfs merge=lfs -text
|
3 |
*.jpg filter=lfs diff=lfs merge=lfs -text
|
4 |
+
db_backup/*.gpg filter=lfs diff=lfs merge=lfs -text
|
.gitignore
CHANGED
@@ -307,3 +307,5 @@ dist
|
|
307 |
cypress/videos
|
308 |
cypress/screenshots
|
309 |
.vscode/settings.json
|
|
|
|
|
|
307 |
cypress/videos
|
308 |
cypress/screenshots
|
309 |
.vscode/settings.json
|
310 |
+
|
311 |
+
webui.db
|
Dockerfile
CHANGED
@@ -104,6 +104,7 @@ RUN echo -n 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry
|
|
104 |
|
105 |
# Make sure the user has access to the app and root directory
|
106 |
RUN chown -R $UID:$GID /app $HOME
|
|
|
107 |
|
108 |
RUN if [ "$USE_OLLAMA" = "true" ]; then \
|
109 |
apt-get update && \
|
@@ -133,6 +134,8 @@ RUN if [ "$USE_OLLAMA" = "true" ]; then \
|
|
133 |
COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
|
134 |
|
135 |
RUN pip3 install --no-cache-dir uv && \
|
|
|
|
|
136 |
if [ "$USE_CUDA" = "true" ]; then \
|
137 |
# If you use CUDA the whisper and embedding model will be downloaded on first use
|
138 |
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
|
|
|
104 |
|
105 |
# Make sure the user has access to the app and root directory
|
106 |
RUN chown -R $UID:$GID /app $HOME
|
107 |
+
RUN apt-get update && apt-get install -y --no-install-recommends gnupg sqlite3
|
108 |
|
109 |
RUN if [ "$USE_OLLAMA" = "true" ]; then \
|
110 |
apt-get update && \
|
|
|
134 |
COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
|
135 |
|
136 |
RUN pip3 install --no-cache-dir uv && \
|
137 |
+
RUN pip3 install huggingface_hub
|
138 |
+
RUN pip3 install uv && \
|
139 |
if [ "$USE_CUDA" = "true" ]; then \
|
140 |
# If you use CUDA the whisper and embedding model will be downloaded on first use
|
141 |
pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
|
backend/requirements.txt
CHANGED
@@ -126,15 +126,6 @@ firecrawl-py==1.12.0
|
|
126 |
# Sougou API SDK(Tencentcloud SDK)
|
127 |
tencentcloud-sdk-python==3.0.1336
|
128 |
|
129 |
-
|
130 |
-
|
131 |
-
|
132 |
-
opentelemetry-exporter-otlp==1.31.1
|
133 |
-
opentelemetry-instrumentation==0.52b1
|
134 |
-
opentelemetry-instrumentation-fastapi==0.52b1
|
135 |
-
opentelemetry-instrumentation-sqlalchemy==0.52b1
|
136 |
-
opentelemetry-instrumentation-redis==0.52b1
|
137 |
-
opentelemetry-instrumentation-requests==0.52b1
|
138 |
-
opentelemetry-instrumentation-logging==0.52b1
|
139 |
-
opentelemetry-instrumentation-httpx==0.52b1
|
140 |
-
opentelemetry-instrumentation-aiohttp-client==0.52b1
|
|
|
126 |
# Sougou API SDK(Tencentcloud SDK)
|
127 |
tencentcloud-sdk-python==3.0.1336
|
128 |
|
129 |
+
|
130 |
+
gnupg
|
131 |
+
huggingface_hub
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
backend/scripts/backup.py
ADDED
@@ -0,0 +1,177 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
import os
|
3 |
+
import sys
|
4 |
+
import subprocess
|
5 |
+
import datetime
|
6 |
+
import sqlite3
|
7 |
+
from pathlib import Path
|
8 |
+
from io import BytesIO
|
9 |
+
from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
|
10 |
+
import tempfile
|
11 |
+
|
12 |
+
# Set up path to include the application module
|
13 |
+
SCRIPT_DIR = Path(__file__).parent.resolve()
|
14 |
+
BACKEND_DIR = SCRIPT_DIR.parent
|
15 |
+
sys.path.append(str(BACKEND_DIR))
|
16 |
+
|
17 |
+
# Database path (actual application database)
|
18 |
+
DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
|
19 |
+
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
20 |
+
|
21 |
+
# Hugging Face repo paths (virtual paths in HF storage)
|
22 |
+
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
23 |
+
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
24 |
+
|
25 |
+
|
26 |
+
def verify_database():
|
27 |
+
"""Verify database integrity."""
|
28 |
+
if not os.path.exists(DB_FILE_PATH):
|
29 |
+
print(f"Database file not found at: {DB_FILE_PATH}")
|
30 |
+
return False
|
31 |
+
|
32 |
+
try:
|
33 |
+
with sqlite3.connect(DB_FILE_PATH) as conn:
|
34 |
+
cursor = conn.cursor()
|
35 |
+
cursor.execute("PRAGMA integrity_check;")
|
36 |
+
result = cursor.fetchone()[0]
|
37 |
+
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
38 |
+
tables = cursor.fetchall()
|
39 |
+
|
40 |
+
if result.lower() == "ok" and len(tables) > 0:
|
41 |
+
print(f"Database verified: {len(tables)} tables found")
|
42 |
+
return True
|
43 |
+
print("Database verification failed")
|
44 |
+
return False
|
45 |
+
except Exception as e:
|
46 |
+
print(f"Database verification error: {e}")
|
47 |
+
return False
|
48 |
+
|
49 |
+
|
50 |
+
def encrypt_database_to_memory(passphrase):
|
51 |
+
"""Encrypt database directly to a memory buffer."""
|
52 |
+
try:
|
53 |
+
# Create a secure temporary directory for GPG
|
54 |
+
with tempfile.TemporaryDirectory(prefix='gpg_home_') as gpg_home:
|
55 |
+
os.chmod(gpg_home, 0o700)
|
56 |
+
|
57 |
+
encrypt_cmd = [
|
58 |
+
"gpg",
|
59 |
+
"--batch",
|
60 |
+
"--yes",
|
61 |
+
"--homedir", gpg_home,
|
62 |
+
"--passphrase", passphrase,
|
63 |
+
"--pinentry-mode", "loopback",
|
64 |
+
"-c",
|
65 |
+
"--cipher-algo", "AES256",
|
66 |
+
"-o", "-", # Output to stdout
|
67 |
+
DB_FILE_PATH
|
68 |
+
]
|
69 |
+
|
70 |
+
# Run GPG and capture output directly
|
71 |
+
result = subprocess.run(
|
72 |
+
encrypt_cmd,
|
73 |
+
capture_output=True,
|
74 |
+
check=True
|
75 |
+
)
|
76 |
+
|
77 |
+
if result.returncode != 0:
|
78 |
+
print(f"GPG encryption failed: {result.stderr.decode()}")
|
79 |
+
return None
|
80 |
+
|
81 |
+
return result.stdout
|
82 |
+
except subprocess.CalledProcessError as e:
|
83 |
+
print(f"GPG process error: {e.stderr.decode()}")
|
84 |
+
return None
|
85 |
+
except Exception as e:
|
86 |
+
print(f"Encryption error: {e}")
|
87 |
+
return None
|
88 |
+
|
89 |
+
|
90 |
+
def get_last_backup_time(repo_id, hf_token):
|
91 |
+
"""Get timestamp of last backup from HuggingFace."""
|
92 |
+
try:
|
93 |
+
api = HfApi()
|
94 |
+
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
95 |
+
if REPO_TIMESTAMP_FILE not in files:
|
96 |
+
return None
|
97 |
+
|
98 |
+
tmp_file = hf_hub_download(
|
99 |
+
repo_id=repo_id,
|
100 |
+
repo_type="space",
|
101 |
+
filename=REPO_TIMESTAMP_FILE,
|
102 |
+
token=hf_token
|
103 |
+
)
|
104 |
+
with open(tmp_file, "r", encoding="utf-8") as f:
|
105 |
+
stamp_str = f.read().strip()
|
106 |
+
return datetime.datetime.fromisoformat(stamp_str)
|
107 |
+
except Exception as e:
|
108 |
+
print(f"Error getting last backup time: {e}")
|
109 |
+
return None
|
110 |
+
|
111 |
+
|
112 |
+
def backup_db():
|
113 |
+
"""Main backup function using streaming approach."""
|
114 |
+
# Validate environment
|
115 |
+
passphrase = os.environ.get("BACKUP_PASSPHRASE")
|
116 |
+
hf_token = os.environ.get("HF_TOKEN")
|
117 |
+
space_id = os.environ.get("SPACE_ID")
|
118 |
+
|
119 |
+
if not all([passphrase, hf_token, space_id]):
|
120 |
+
print("Error: Missing required environment variables")
|
121 |
+
return False
|
122 |
+
|
123 |
+
# Check backup threshold
|
124 |
+
threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
|
125 |
+
if threshold_minutes > 0:
|
126 |
+
last_backup_dt = get_last_backup_time(space_id, hf_token)
|
127 |
+
if last_backup_dt is not None:
|
128 |
+
now = datetime.datetime.now(datetime.timezone.utc)
|
129 |
+
if not last_backup_dt.tzinfo:
|
130 |
+
last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
|
131 |
+
elapsed = now - last_backup_dt
|
132 |
+
if elapsed.total_seconds() < threshold_minutes * 60:
|
133 |
+
print(f"Last backup was {elapsed.total_seconds()/60:.1f} min ago (threshold: {threshold_minutes})")
|
134 |
+
return True
|
135 |
+
|
136 |
+
# Verify database before backup
|
137 |
+
if not verify_database():
|
138 |
+
return False
|
139 |
+
|
140 |
+
# Encrypt database to memory
|
141 |
+
print("Encrypting database...")
|
142 |
+
encrypted_data = encrypt_database_to_memory(passphrase)
|
143 |
+
if encrypted_data is None:
|
144 |
+
return False
|
145 |
+
print(f"Database encrypted successfully: {len(encrypted_data)} bytes")
|
146 |
+
|
147 |
+
# Generate timestamp
|
148 |
+
timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
149 |
+
timestamp_bytes = timestamp.encode('utf-8')
|
150 |
+
|
151 |
+
# Upload both files to HuggingFace
|
152 |
+
try:
|
153 |
+
api = HfApi()
|
154 |
+
api.upload_file(
|
155 |
+
path_or_fileobj=BytesIO(encrypted_data),
|
156 |
+
path_in_repo=REPO_DB_GPG_FILE,
|
157 |
+
repo_id=space_id,
|
158 |
+
repo_type="space",
|
159 |
+
token=hf_token
|
160 |
+
)
|
161 |
+
api.upload_file(
|
162 |
+
path_or_fileobj=BytesIO(timestamp_bytes),
|
163 |
+
path_in_repo=REPO_TIMESTAMP_FILE,
|
164 |
+
repo_id=space_id,
|
165 |
+
repo_type="space",
|
166 |
+
token=hf_token
|
167 |
+
)
|
168 |
+
print("Backup completed successfully!")
|
169 |
+
return True
|
170 |
+
except Exception as e:
|
171 |
+
print(f"Error uploading to HuggingFace: {e}")
|
172 |
+
return False
|
173 |
+
|
174 |
+
|
175 |
+
if __name__ == "__main__":
|
176 |
+
success = backup_db()
|
177 |
+
sys.exit(0 if success else 1)
|
backend/scripts/db_crypt.sh
ADDED
@@ -0,0 +1,100 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env bash
|
2 |
+
|
3 |
+
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
4 |
+
BACKEND_DIR="$(realpath "$SCRIPT_DIR/..")"
|
5 |
+
SPACE_NAME="thryyyyy/open-webui"
|
6 |
+
|
7 |
+
function check_requirements() {
|
8 |
+
if ! command -v gpg >/dev/null; then
|
9 |
+
echo "Error: gpg is not installed"
|
10 |
+
return 1
|
11 |
+
fi
|
12 |
+
}
|
13 |
+
|
14 |
+
function validate_secrets() {
|
15 |
+
if [ -z "$BACKUP_PASSPHRASE" ]; then
|
16 |
+
echo "Error: BACKUP_PASSPHRASE secret not set"
|
17 |
+
return 1
|
18 |
+
fi
|
19 |
+
|
20 |
+
if [ -z "$HF_TOKEN" ]; then
|
21 |
+
echo "Error: HF_TOKEN secret not set"
|
22 |
+
return 1
|
23 |
+
fi
|
24 |
+
}
|
25 |
+
|
26 |
+
function decrypt_database() {
|
27 |
+
validate_secrets || return 1
|
28 |
+
|
29 |
+
mkdir -p "$BACKEND_DIR/data"
|
30 |
+
|
31 |
+
if [ -f "$BACKEND_DIR/db_backup/webui.db.gpg" ]; then
|
32 |
+
echo "Decrypting database backup..."
|
33 |
+
gpg --batch --yes --passphrase "$BACKUP_PASSPHRASE" -d \
|
34 |
+
-o "$BACKEND_DIR/data/webui.db" "$BACKEND_DIR/db_backup/webui.db.gpg"
|
35 |
+
|
36 |
+
if [ $? -eq 0 ]; then
|
37 |
+
echo "Database decrypted successfully"
|
38 |
+
return 0
|
39 |
+
else
|
40 |
+
echo "Failed to decrypt database"
|
41 |
+
return 1
|
42 |
+
fi
|
43 |
+
else
|
44 |
+
echo "No encrypted backup found at db_backup/webui.db.gpg"
|
45 |
+
# Not an error, might be first run
|
46 |
+
return 0
|
47 |
+
fi
|
48 |
+
}
|
49 |
+
|
50 |
+
function encrypt_database() {
|
51 |
+
validate_secrets || return 1
|
52 |
+
|
53 |
+
if [ ! -f "$BACKEND_DIR/data/webui.db" ]; then
|
54 |
+
echo "Database not found at data/webui.db"
|
55 |
+
return 1
|
56 |
+
fi
|
57 |
+
|
58 |
+
mkdir -p "$BACKEND_DIR/db_backup"
|
59 |
+
|
60 |
+
echo "Encrypting database..."
|
61 |
+
gpg --batch --yes --passphrase "$BACKUP_PASSPHRASE" -c --cipher-algo AES256 \
|
62 |
+
-o "$BACKEND_DIR/db_backup/webui.db.gpg" "$BACKEND_DIR/data/webui.db"
|
63 |
+
|
64 |
+
if [ $? -eq 0 ]; then
|
65 |
+
echo "Database encrypted successfully"
|
66 |
+
cd "$BACKEND_DIR" || exit 1
|
67 |
+
|
68 |
+
# Configure Git for this operation
|
69 |
+
git config --local user.email "[email protected]"
|
70 |
+
git config --local user.name "Space Bot"
|
71 |
+
|
72 |
+
echo "Committing and pushing changes..."
|
73 |
+
git add db_backup/webui.db.gpg
|
74 |
+
git commit -m "Update encrypted database backup"
|
75 |
+
|
76 |
+
# Push using the token
|
77 |
+
REPO_URL="https://user:[email protected]/spaces/$SPACE_NAME"
|
78 |
+
if git push "$REPO_URL" main; then
|
79 |
+
echo "Successfully pushed backup to repository"
|
80 |
+
cd - >/dev/null
|
81 |
+
return 0
|
82 |
+
else
|
83 |
+
echo "Failed to push to repository"
|
84 |
+
cd - >/dev/null
|
85 |
+
return 1
|
86 |
+
fi
|
87 |
+
else
|
88 |
+
echo "Failed to encrypt database"
|
89 |
+
return 1
|
90 |
+
fi
|
91 |
+
}
|
92 |
+
|
93 |
+
# Check requirements first
|
94 |
+
check_requirements || exit 1
|
95 |
+
|
96 |
+
# If script is run directly, default to encryption
|
97 |
+
# (equivalent to fish's: if test (status filename) = (status -f))
|
98 |
+
if [ "${BASH_SOURCE[0]}" = "$0" ]; then
|
99 |
+
encrypt_database
|
100 |
+
fi
|
backend/scripts/restore.py
ADDED
@@ -0,0 +1,211 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
#!/usr/bin/env python3
|
2 |
+
import os
|
3 |
+
import sys
|
4 |
+
import subprocess
|
5 |
+
import datetime
|
6 |
+
import sqlite3
|
7 |
+
import tempfile
|
8 |
+
from pathlib import Path
|
9 |
+
from huggingface_hub import HfApi, hf_hub_download
|
10 |
+
|
11 |
+
# Set up path to include the application module
|
12 |
+
SCRIPT_DIR = Path(__file__).parent.resolve()
|
13 |
+
BACKEND_DIR = SCRIPT_DIR.parent
|
14 |
+
sys.path.append(str(BACKEND_DIR))
|
15 |
+
|
16 |
+
# Database path (actual application database)
|
17 |
+
DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
|
18 |
+
DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
|
19 |
+
|
20 |
+
# Hugging Face repo paths (virtual paths in HF storage)
|
21 |
+
REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
|
22 |
+
REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
|
23 |
+
|
24 |
+
|
25 |
+
def check_gpg():
|
26 |
+
"""Verify GPG is available."""
|
27 |
+
try:
|
28 |
+
subprocess.run(["gpg", "--version"], check=True, capture_output=True)
|
29 |
+
return True
|
30 |
+
except (subprocess.CalledProcessError, FileNotFoundError):
|
31 |
+
print("Error: gpg is not installed or not in PATH")
|
32 |
+
return False
|
33 |
+
|
34 |
+
|
35 |
+
def validate_environment():
|
36 |
+
"""Verify all required environment variables are set."""
|
37 |
+
required = ["BACKUP_PASSPHRASE", "HF_TOKEN", "SPACE_ID"]
|
38 |
+
missing = [var for var in required if not os.environ.get(var)]
|
39 |
+
if missing:
|
40 |
+
print(f"Error: Missing environment variables: {', '.join(missing)}")
|
41 |
+
return False
|
42 |
+
return True
|
43 |
+
|
44 |
+
|
45 |
+
def ensure_data_dir():
|
46 |
+
"""Ensure the database directory exists."""
|
47 |
+
try:
|
48 |
+
os.makedirs(DATA_DIR, mode=0o755, exist_ok=True)
|
49 |
+
return True
|
50 |
+
except Exception as e:
|
51 |
+
print(f"Error creating data directory: {e}")
|
52 |
+
return False
|
53 |
+
|
54 |
+
|
55 |
+
def get_latest_backup_info(repo_id, hf_token):
|
56 |
+
"""Check if backup exists and get its timestamp."""
|
57 |
+
api = HfApi()
|
58 |
+
try:
|
59 |
+
files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
|
60 |
+
|
61 |
+
if REPO_DB_GPG_FILE not in files:
|
62 |
+
print("No backup file found in repository")
|
63 |
+
return False, None
|
64 |
+
|
65 |
+
if REPO_TIMESTAMP_FILE in files:
|
66 |
+
try:
|
67 |
+
timestamp_file = hf_hub_download(
|
68 |
+
repo_id=repo_id,
|
69 |
+
repo_type="space",
|
70 |
+
filename=REPO_TIMESTAMP_FILE,
|
71 |
+
token=hf_token
|
72 |
+
)
|
73 |
+
with open(timestamp_file, "r", encoding="utf-8") as f:
|
74 |
+
timestamp_str = f.read().strip()
|
75 |
+
timestamp = datetime.datetime.fromisoformat(timestamp_str)
|
76 |
+
print(f"Found backup from: {timestamp} UTC")
|
77 |
+
return True, timestamp
|
78 |
+
except Exception as e:
|
79 |
+
print(f"Could not read timestamp: {e}")
|
80 |
+
return True, None
|
81 |
+
return True, None
|
82 |
+
except Exception as e:
|
83 |
+
print(f"Error checking repository: {e}")
|
84 |
+
return False, None
|
85 |
+
|
86 |
+
|
87 |
+
def decrypt_database_from_memory(encrypted_data, passphrase):
|
88 |
+
"""Decrypt database directly from memory."""
|
89 |
+
try:
|
90 |
+
# Create a secure temporary directory for GPG operations
|
91 |
+
with tempfile.TemporaryDirectory(prefix='gpg_home_') as gpg_home:
|
92 |
+
os.chmod(gpg_home, 0o700)
|
93 |
+
|
94 |
+
# Create a temporary file for the encrypted data
|
95 |
+
with tempfile.NamedTemporaryFile(mode='wb', suffix='.gpg', delete=False) as temp_encrypted:
|
96 |
+
temp_encrypted.write(encrypted_data)
|
97 |
+
temp_encrypted_path = temp_encrypted.name
|
98 |
+
|
99 |
+
try:
|
100 |
+
print(f"Decrypting database ({len(encrypted_data)} bytes)...")
|
101 |
+
decrypt_cmd = [
|
102 |
+
"gpg",
|
103 |
+
"--batch",
|
104 |
+
"--yes",
|
105 |
+
"--homedir", gpg_home,
|
106 |
+
"--passphrase", passphrase,
|
107 |
+
"--pinentry-mode", "loopback",
|
108 |
+
"-d",
|
109 |
+
"-o", DB_FILE_PATH,
|
110 |
+
temp_encrypted_path
|
111 |
+
]
|
112 |
+
|
113 |
+
result = subprocess.run(decrypt_cmd, capture_output=True, check=True)
|
114 |
+
|
115 |
+
if os.path.exists(DB_FILE_PATH) and os.path.getsize(DB_FILE_PATH) > 0:
|
116 |
+
print(f"Database decrypted successfully ({os.path.getsize(DB_FILE_PATH)} bytes)")
|
117 |
+
return True
|
118 |
+
else:
|
119 |
+
print("Error: Decrypted database is missing or empty")
|
120 |
+
return False
|
121 |
+
|
122 |
+
finally:
|
123 |
+
# Clean up the temporary encrypted file
|
124 |
+
if os.path.exists(temp_encrypted_path):
|
125 |
+
os.unlink(temp_encrypted_path)
|
126 |
+
|
127 |
+
except subprocess.CalledProcessError as e:
|
128 |
+
print(f"Decryption failed: {e.stderr.decode()}")
|
129 |
+
return False
|
130 |
+
except Exception as e:
|
131 |
+
print(f"Decryption error: {e}")
|
132 |
+
return False
|
133 |
+
|
134 |
+
|
135 |
+
def verify_database():
|
136 |
+
"""Verify the restored database integrity."""
|
137 |
+
if not os.path.exists(DB_FILE_PATH):
|
138 |
+
print(f"Error: Database file not found at {DB_FILE_PATH}")
|
139 |
+
return False
|
140 |
+
|
141 |
+
try:
|
142 |
+
print("Verifying database integrity...")
|
143 |
+
with sqlite3.connect(DB_FILE_PATH) as conn:
|
144 |
+
cursor = conn.cursor()
|
145 |
+
cursor.execute("PRAGMA integrity_check;")
|
146 |
+
result = cursor.fetchone()[0]
|
147 |
+
cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
|
148 |
+
tables = cursor.fetchall()
|
149 |
+
|
150 |
+
if result.lower() == "ok" and len(tables) > 0:
|
151 |
+
print(f"Database verified: {len(tables)} tables found")
|
152 |
+
return True
|
153 |
+
print("Database verification failed")
|
154 |
+
return False
|
155 |
+
except Exception as e:
|
156 |
+
print(f"Database verification error: {e}")
|
157 |
+
return False
|
158 |
+
|
159 |
+
|
160 |
+
def restore_db():
|
161 |
+
"""Main restore function using in-memory approach."""
|
162 |
+
if not check_gpg() or not validate_environment() or not ensure_data_dir():
|
163 |
+
return False
|
164 |
+
|
165 |
+
passphrase = os.environ["BACKUP_PASSPHRASE"]
|
166 |
+
hf_token = os.environ["HF_TOKEN"]
|
167 |
+
space_id = os.environ["SPACE_ID"]
|
168 |
+
|
169 |
+
backup_exists, timestamp = get_latest_backup_info(space_id, hf_token)
|
170 |
+
if not backup_exists:
|
171 |
+
print("No backup found - starting with fresh database")
|
172 |
+
return True
|
173 |
+
|
174 |
+
try:
|
175 |
+
print("Downloading encrypted database...")
|
176 |
+
encrypted_file = hf_hub_download(
|
177 |
+
repo_id=space_id,
|
178 |
+
repo_type="space",
|
179 |
+
filename=REPO_DB_GPG_FILE,
|
180 |
+
token=hf_token
|
181 |
+
)
|
182 |
+
|
183 |
+
# Read encrypted data into memory
|
184 |
+
with open(encrypted_file, 'rb') as f:
|
185 |
+
encrypted_data = f.read()
|
186 |
+
|
187 |
+
print(f"Downloaded encrypted data: {len(encrypted_data)} bytes")
|
188 |
+
|
189 |
+
if not decrypt_database_from_memory(encrypted_data, passphrase):
|
190 |
+
print("Failed to decrypt database")
|
191 |
+
return False
|
192 |
+
|
193 |
+
if not verify_database():
|
194 |
+
print("Failed to verify database")
|
195 |
+
if os.path.exists(DB_FILE_PATH):
|
196 |
+
os.unlink(DB_FILE_PATH)
|
197 |
+
return False
|
198 |
+
|
199 |
+
print("Database restore completed successfully!")
|
200 |
+
return True
|
201 |
+
|
202 |
+
except Exception as e:
|
203 |
+
print(f"Restore error: {e}")
|
204 |
+
if os.path.exists(DB_FILE_PATH):
|
205 |
+
os.unlink(DB_FILE_PATH)
|
206 |
+
return False
|
207 |
+
|
208 |
+
|
209 |
+
if __name__ == "__main__":
|
210 |
+
success = restore_db()
|
211 |
+
sys.exit(0 if success else 1)
|
backend/start.sh
CHANGED
@@ -1,68 +1,115 @@
|
|
1 |
#!/usr/bin/env bash
|
2 |
|
3 |
-
|
4 |
-
|
5 |
-
|
6 |
-
|
7 |
-
|
8 |
-
|
9 |
-
|
10 |
-
|
11 |
-
|
|
|
|
|
|
|
12 |
fi
|
|
|
13 |
|
14 |
-
|
|
|
|
|
|
|
|
|
|
|
15 |
fi
|
16 |
|
17 |
-
|
18 |
-
|
19 |
PORT="${PORT:-8080}"
|
20 |
HOST="${HOST:-0.0.0.0}"
|
21 |
-
if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
|
22 |
-
echo "Loading WEBUI_SECRET_KEY from file, not provided as an environment variable."
|
23 |
-
|
24 |
-
if ! [ -e "$KEY_FILE" ]; then
|
25 |
-
echo "Generating WEBUI_SECRET_KEY"
|
26 |
-
# Generate a random value to use as a WEBUI_SECRET_KEY in case the user didn't provide one.
|
27 |
-
echo $(head -c 12 /dev/random | base64) > "$KEY_FILE"
|
28 |
-
fi
|
29 |
|
30 |
-
|
31 |
-
|
|
|
|
|
|
|
32 |
fi
|
33 |
|
|
|
34 |
if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
|
35 |
-
echo "USE_OLLAMA is set to true, starting ollama serve."
|
36 |
ollama serve &
|
37 |
fi
|
38 |
|
|
|
39 |
if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
|
40 |
-
|
41 |
-
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
|
42 |
fi
|
43 |
|
44 |
-
#
|
45 |
if [ -n "$SPACE_ID" ]; then
|
46 |
-
|
47 |
-
|
48 |
-
|
49 |
-
|
50 |
-
|
51 |
-
|
52 |
-
|
53 |
-
|
54 |
-
|
55 |
-
|
56 |
-
|
57 |
-
|
58 |
-
|
59 |
-
|
60 |
-
|
61 |
-
|
62 |
-
|
63 |
-
|
64 |
-
|
65 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
66 |
fi
|
67 |
|
68 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
#!/usr/bin/env bash
|
2 |
|
3 |
+
# Ensure we're in the right directory and set up paths
|
4 |
+
SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
|
5 |
+
cd "$SCRIPT_DIR" || exit 1
|
6 |
+
|
7 |
+
export DATA_DIR="/app/backend/data"
|
8 |
+
export PYTHONPATH="/app/backend:${PYTHONPATH}"
|
9 |
+
|
10 |
+
# Validate required environment variables
|
11 |
+
for var in "BACKUP_PASSPHRASE" "HF_TOKEN" "SPACE_ID"; do
|
12 |
+
if [ -z "${!var}" ]; then
|
13 |
+
echo "Error: $var is not set. Required for database backup/restore."
|
14 |
+
exit 1
|
15 |
fi
|
16 |
+
done
|
17 |
|
18 |
+
# Restore database from backup
|
19 |
+
echo "Restoring database from backup..."
|
20 |
+
python "$SCRIPT_DIR/scripts/restore.py"
|
21 |
+
restore_status=$?
|
22 |
+
if [ $restore_status -ne 0 ]; then
|
23 |
+
echo "Warning: Database restore failed. Starting with empty database."
|
24 |
fi
|
25 |
|
26 |
+
# Handle WebUI secret key
|
27 |
+
KEY_FILE="$SCRIPT_DIR/.webui_secret_key"
|
28 |
PORT="${PORT:-8080}"
|
29 |
HOST="${HOST:-0.0.0.0}"
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
30 |
|
31 |
+
if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
|
32 |
+
if ! [ -e "$KEY_FILE" ]; then
|
33 |
+
head -c 12 /dev/random | base64 > "$KEY_FILE" || exit 1
|
34 |
+
fi
|
35 |
+
WEBUI_SECRET_KEY=$(cat "$KEY_FILE") || exit 1
|
36 |
fi
|
37 |
|
38 |
+
# Start Ollama if enabled
|
39 |
if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
|
|
|
40 |
ollama serve &
|
41 |
fi
|
42 |
|
43 |
+
# Configure CUDA if enabled
|
44 |
if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
|
45 |
+
export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
|
|
|
46 |
fi
|
47 |
|
48 |
+
# Handle HuggingFace Space deployment
|
49 |
if [ -n "$SPACE_ID" ]; then
|
50 |
+
echo "Configuring HuggingFace Space deployment"
|
51 |
+
if [ -n "$ADMIN_USER_EMAIL" ] && [ -n "$ADMIN_USER_PASSWORD" ]; then
|
52 |
+
# Start webui temporarily to create admin user
|
53 |
+
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
|
54 |
+
webui_pid=$!
|
55 |
+
|
56 |
+
# Wait for the server to be ready
|
57 |
+
while ! curl -s http://localhost:8080/health > /dev/null; do
|
58 |
+
echo "Waiting for WebUI to start..."
|
59 |
+
sleep 1
|
60 |
+
done
|
61 |
+
|
62 |
+
# Create the admin user
|
63 |
+
echo "Creating admin user: ${ADMIN_USER_EMAIL}"
|
64 |
+
curl \
|
65 |
+
-X POST "http://localhost:8080/api/v1/auths/signup" \
|
66 |
+
-H "accept: application/json" \
|
67 |
+
-H "Content-Type: application/json" \
|
68 |
+
-d "{ \"email\": \"${ADMIN_USER_EMAIL}\", \"password\": \"${ADMIN_USER_PASSWORD}\", \"name\": \"Admin\" }"
|
69 |
+
|
70 |
+
# Stop the temporary webui server
|
71 |
+
echo "Stopping temporary WebUI server..."
|
72 |
+
kill $webui_pid
|
73 |
+
wait $webui_pid 2>/dev/null # Wait for it to fully shut down
|
74 |
+
fi
|
75 |
+
# Set the public URL for Space deployments
|
76 |
+
export WEBUI_URL=${SPACE_HOST}
|
77 |
fi
|
78 |
|
79 |
+
# Start the main web server in the background
|
80 |
+
echo "Starting main WebUI server..."
|
81 |
+
WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app \
|
82 |
+
--host "$HOST" --port "$PORT" --forwarded-allow-ips '*' \
|
83 |
+
--workers "${UVICORN_WORKERS:-1}" &
|
84 |
+
WEBUI_PID=$!
|
85 |
+
|
86 |
+
# Configure and start backup scheduler in the background
|
87 |
+
BACKUP_INITIAL_WAIT="${BACKUP_INITIAL_WAIT:-100}"
|
88 |
+
BACKUP_INTERVAL="${BACKUP_INTERVAL:-300}"
|
89 |
+
|
90 |
+
(
|
91 |
+
echo "Starting backup scheduler (Initial wait: ${BACKUP_INITIAL_WAIT}s, Interval: ${BACKUP_INTERVAL}s)"
|
92 |
+
sleep "$BACKUP_INITIAL_WAIT"
|
93 |
+
|
94 |
+
while true; do
|
95 |
+
echo "Running scheduled backup"
|
96 |
+
# Pass relevant env vars to backup script if needed
|
97 |
+
BACKUP_THRESHOLD_MINUTES="${BACKUP_THRESHOLD_MINUTES:-120}" \
|
98 |
+
python "$SCRIPT_DIR/scripts/backup.py"
|
99 |
+
|
100 |
+
if [ $? -ne 0 ]; then
|
101 |
+
echo "Warning: Backup failed"
|
102 |
+
fi
|
103 |
+
|
104 |
+
sleep "$BACKUP_INTERVAL"
|
105 |
+
done
|
106 |
+
) &
|
107 |
+
|
108 |
+
# Wait for the main web server process to exit
|
109 |
+
# This keeps the script alive while the server runs and allows graceful shutdown
|
110 |
+
echo "Web server started (PID: ${WEBUI_PID}). Waiting for it to exit..."
|
111 |
+
wait $WEBUI_PID
|
112 |
+
echo "Web server process ${WEBUI_PID} exited."
|
113 |
+
|
114 |
+
# Optional: Add cleanup here if needed after the web server stops.
|
115 |
+
# The background backup loop will likely be terminated when the main script exits.
|
db_backup/last_backup_time.txt
ADDED
@@ -0,0 +1 @@
|
|
|
|
|
1 |
+
2025-03-22T14:02:37.983192+00:00
|
vite.config.ts
CHANGED
@@ -35,7 +35,7 @@ export default defineConfig({
|
|
35 |
APP_BUILD_HASH: JSON.stringify(process.env.APP_BUILD_HASH || 'dev-build')
|
36 |
},
|
37 |
build: {
|
38 |
-
sourcemap:
|
39 |
},
|
40 |
worker: {
|
41 |
format: 'es'
|
|
|
35 |
APP_BUILD_HASH: JSON.stringify(process.env.APP_BUILD_HASH || 'dev-build')
|
36 |
},
|
37 |
build: {
|
38 |
+
sourcemap: false
|
39 |
},
|
40 |
worker: {
|
41 |
format: 'es'
|