thryyyyy commited on
Commit
ffd30b5
·
1 Parent(s): 1538221

restore latest backup files

Browse files
.gitattributes CHANGED
@@ -1,3 +1,4 @@
1
  *.sh text eol=lf
2
  *.ttf filter=lfs diff=lfs merge=lfs -text
3
  *.jpg filter=lfs diff=lfs merge=lfs -text
 
 
1
  *.sh text eol=lf
2
  *.ttf filter=lfs diff=lfs merge=lfs -text
3
  *.jpg filter=lfs diff=lfs merge=lfs -text
4
+ db_backup/*.gpg filter=lfs diff=lfs merge=lfs -text
.gitignore CHANGED
@@ -307,3 +307,5 @@ dist
307
  cypress/videos
308
  cypress/screenshots
309
  .vscode/settings.json
 
 
 
307
  cypress/videos
308
  cypress/screenshots
309
  .vscode/settings.json
310
+
311
+ webui.db
Dockerfile CHANGED
@@ -104,6 +104,7 @@ RUN echo -n 00000000-0000-0000-0000-000000000000 > $HOME/.cache/chroma/telemetry
104
 
105
  # Make sure the user has access to the app and root directory
106
  RUN chown -R $UID:$GID /app $HOME
 
107
 
108
  RUN if [ "$USE_OLLAMA" = "true" ]; then \
109
  apt-get update && \
@@ -133,6 +134,8 @@ RUN if [ "$USE_OLLAMA" = "true" ]; then \
133
  COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
134
 
135
  RUN pip3 install --no-cache-dir uv && \
 
 
136
  if [ "$USE_CUDA" = "true" ]; then \
137
  # If you use CUDA the whisper and embedding model will be downloaded on first use
138
  pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
 
104
 
105
  # Make sure the user has access to the app and root directory
106
  RUN chown -R $UID:$GID /app $HOME
107
+ RUN apt-get update && apt-get install -y --no-install-recommends gnupg sqlite3
108
 
109
  RUN if [ "$USE_OLLAMA" = "true" ]; then \
110
  apt-get update && \
 
134
  COPY --chown=$UID:$GID ./backend/requirements.txt ./requirements.txt
135
 
136
  RUN pip3 install --no-cache-dir uv && \
137
+ RUN pip3 install huggingface_hub
138
+ RUN pip3 install uv && \
139
  if [ "$USE_CUDA" = "true" ]; then \
140
  # If you use CUDA the whisper and embedding model will be downloaded on first use
141
  pip3 install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/$USE_CUDA_DOCKER_VER --no-cache-dir && \
backend/requirements.txt CHANGED
@@ -126,15 +126,6 @@ firecrawl-py==1.12.0
126
  # Sougou API SDK(Tencentcloud SDK)
127
  tencentcloud-sdk-python==3.0.1336
128
 
129
- ## Trace
130
- opentelemetry-api==1.31.1
131
- opentelemetry-sdk==1.31.1
132
- opentelemetry-exporter-otlp==1.31.1
133
- opentelemetry-instrumentation==0.52b1
134
- opentelemetry-instrumentation-fastapi==0.52b1
135
- opentelemetry-instrumentation-sqlalchemy==0.52b1
136
- opentelemetry-instrumentation-redis==0.52b1
137
- opentelemetry-instrumentation-requests==0.52b1
138
- opentelemetry-instrumentation-logging==0.52b1
139
- opentelemetry-instrumentation-httpx==0.52b1
140
- opentelemetry-instrumentation-aiohttp-client==0.52b1
 
126
  # Sougou API SDK(Tencentcloud SDK)
127
  tencentcloud-sdk-python==3.0.1336
128
 
129
+
130
+ gnupg
131
+ huggingface_hub
 
 
 
 
 
 
 
 
 
backend/scripts/backup.py ADDED
@@ -0,0 +1,177 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ import os
3
+ import sys
4
+ import subprocess
5
+ import datetime
6
+ import sqlite3
7
+ from pathlib import Path
8
+ from io import BytesIO
9
+ from huggingface_hub import HfApi, HfFileSystem, hf_hub_download
10
+ import tempfile
11
+
12
+ # Set up path to include the application module
13
+ SCRIPT_DIR = Path(__file__).parent.resolve()
14
+ BACKEND_DIR = SCRIPT_DIR.parent
15
+ sys.path.append(str(BACKEND_DIR))
16
+
17
+ # Database path (actual application database)
18
+ DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
19
+ DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
20
+
21
+ # Hugging Face repo paths (virtual paths in HF storage)
22
+ REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
23
+ REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
24
+
25
+
26
+ def verify_database():
27
+ """Verify database integrity."""
28
+ if not os.path.exists(DB_FILE_PATH):
29
+ print(f"Database file not found at: {DB_FILE_PATH}")
30
+ return False
31
+
32
+ try:
33
+ with sqlite3.connect(DB_FILE_PATH) as conn:
34
+ cursor = conn.cursor()
35
+ cursor.execute("PRAGMA integrity_check;")
36
+ result = cursor.fetchone()[0]
37
+ cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
38
+ tables = cursor.fetchall()
39
+
40
+ if result.lower() == "ok" and len(tables) > 0:
41
+ print(f"Database verified: {len(tables)} tables found")
42
+ return True
43
+ print("Database verification failed")
44
+ return False
45
+ except Exception as e:
46
+ print(f"Database verification error: {e}")
47
+ return False
48
+
49
+
50
+ def encrypt_database_to_memory(passphrase):
51
+ """Encrypt database directly to a memory buffer."""
52
+ try:
53
+ # Create a secure temporary directory for GPG
54
+ with tempfile.TemporaryDirectory(prefix='gpg_home_') as gpg_home:
55
+ os.chmod(gpg_home, 0o700)
56
+
57
+ encrypt_cmd = [
58
+ "gpg",
59
+ "--batch",
60
+ "--yes",
61
+ "--homedir", gpg_home,
62
+ "--passphrase", passphrase,
63
+ "--pinentry-mode", "loopback",
64
+ "-c",
65
+ "--cipher-algo", "AES256",
66
+ "-o", "-", # Output to stdout
67
+ DB_FILE_PATH
68
+ ]
69
+
70
+ # Run GPG and capture output directly
71
+ result = subprocess.run(
72
+ encrypt_cmd,
73
+ capture_output=True,
74
+ check=True
75
+ )
76
+
77
+ if result.returncode != 0:
78
+ print(f"GPG encryption failed: {result.stderr.decode()}")
79
+ return None
80
+
81
+ return result.stdout
82
+ except subprocess.CalledProcessError as e:
83
+ print(f"GPG process error: {e.stderr.decode()}")
84
+ return None
85
+ except Exception as e:
86
+ print(f"Encryption error: {e}")
87
+ return None
88
+
89
+
90
+ def get_last_backup_time(repo_id, hf_token):
91
+ """Get timestamp of last backup from HuggingFace."""
92
+ try:
93
+ api = HfApi()
94
+ files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
95
+ if REPO_TIMESTAMP_FILE not in files:
96
+ return None
97
+
98
+ tmp_file = hf_hub_download(
99
+ repo_id=repo_id,
100
+ repo_type="space",
101
+ filename=REPO_TIMESTAMP_FILE,
102
+ token=hf_token
103
+ )
104
+ with open(tmp_file, "r", encoding="utf-8") as f:
105
+ stamp_str = f.read().strip()
106
+ return datetime.datetime.fromisoformat(stamp_str)
107
+ except Exception as e:
108
+ print(f"Error getting last backup time: {e}")
109
+ return None
110
+
111
+
112
+ def backup_db():
113
+ """Main backup function using streaming approach."""
114
+ # Validate environment
115
+ passphrase = os.environ.get("BACKUP_PASSPHRASE")
116
+ hf_token = os.environ.get("HF_TOKEN")
117
+ space_id = os.environ.get("SPACE_ID")
118
+
119
+ if not all([passphrase, hf_token, space_id]):
120
+ print("Error: Missing required environment variables")
121
+ return False
122
+
123
+ # Check backup threshold
124
+ threshold_minutes = int(os.environ.get("BACKUP_THRESHOLD_MINUTES", 120))
125
+ if threshold_minutes > 0:
126
+ last_backup_dt = get_last_backup_time(space_id, hf_token)
127
+ if last_backup_dt is not None:
128
+ now = datetime.datetime.now(datetime.timezone.utc)
129
+ if not last_backup_dt.tzinfo:
130
+ last_backup_dt = last_backup_dt.replace(tzinfo=datetime.timezone.utc)
131
+ elapsed = now - last_backup_dt
132
+ if elapsed.total_seconds() < threshold_minutes * 60:
133
+ print(f"Last backup was {elapsed.total_seconds()/60:.1f} min ago (threshold: {threshold_minutes})")
134
+ return True
135
+
136
+ # Verify database before backup
137
+ if not verify_database():
138
+ return False
139
+
140
+ # Encrypt database to memory
141
+ print("Encrypting database...")
142
+ encrypted_data = encrypt_database_to_memory(passphrase)
143
+ if encrypted_data is None:
144
+ return False
145
+ print(f"Database encrypted successfully: {len(encrypted_data)} bytes")
146
+
147
+ # Generate timestamp
148
+ timestamp = datetime.datetime.now(datetime.timezone.utc).isoformat()
149
+ timestamp_bytes = timestamp.encode('utf-8')
150
+
151
+ # Upload both files to HuggingFace
152
+ try:
153
+ api = HfApi()
154
+ api.upload_file(
155
+ path_or_fileobj=BytesIO(encrypted_data),
156
+ path_in_repo=REPO_DB_GPG_FILE,
157
+ repo_id=space_id,
158
+ repo_type="space",
159
+ token=hf_token
160
+ )
161
+ api.upload_file(
162
+ path_or_fileobj=BytesIO(timestamp_bytes),
163
+ path_in_repo=REPO_TIMESTAMP_FILE,
164
+ repo_id=space_id,
165
+ repo_type="space",
166
+ token=hf_token
167
+ )
168
+ print("Backup completed successfully!")
169
+ return True
170
+ except Exception as e:
171
+ print(f"Error uploading to HuggingFace: {e}")
172
+ return False
173
+
174
+
175
+ if __name__ == "__main__":
176
+ success = backup_db()
177
+ sys.exit(0 if success else 1)
backend/scripts/db_crypt.sh ADDED
@@ -0,0 +1,100 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env bash
2
+
3
+ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
4
+ BACKEND_DIR="$(realpath "$SCRIPT_DIR/..")"
5
+ SPACE_NAME="thryyyyy/open-webui"
6
+
7
+ function check_requirements() {
8
+ if ! command -v gpg >/dev/null; then
9
+ echo "Error: gpg is not installed"
10
+ return 1
11
+ fi
12
+ }
13
+
14
+ function validate_secrets() {
15
+ if [ -z "$BACKUP_PASSPHRASE" ]; then
16
+ echo "Error: BACKUP_PASSPHRASE secret not set"
17
+ return 1
18
+ fi
19
+
20
+ if [ -z "$HF_TOKEN" ]; then
21
+ echo "Error: HF_TOKEN secret not set"
22
+ return 1
23
+ fi
24
+ }
25
+
26
+ function decrypt_database() {
27
+ validate_secrets || return 1
28
+
29
+ mkdir -p "$BACKEND_DIR/data"
30
+
31
+ if [ -f "$BACKEND_DIR/db_backup/webui.db.gpg" ]; then
32
+ echo "Decrypting database backup..."
33
+ gpg --batch --yes --passphrase "$BACKUP_PASSPHRASE" -d \
34
+ -o "$BACKEND_DIR/data/webui.db" "$BACKEND_DIR/db_backup/webui.db.gpg"
35
+
36
+ if [ $? -eq 0 ]; then
37
+ echo "Database decrypted successfully"
38
+ return 0
39
+ else
40
+ echo "Failed to decrypt database"
41
+ return 1
42
+ fi
43
+ else
44
+ echo "No encrypted backup found at db_backup/webui.db.gpg"
45
+ # Not an error, might be first run
46
+ return 0
47
+ fi
48
+ }
49
+
50
+ function encrypt_database() {
51
+ validate_secrets || return 1
52
+
53
+ if [ ! -f "$BACKEND_DIR/data/webui.db" ]; then
54
+ echo "Database not found at data/webui.db"
55
+ return 1
56
+ fi
57
+
58
+ mkdir -p "$BACKEND_DIR/db_backup"
59
+
60
+ echo "Encrypting database..."
61
+ gpg --batch --yes --passphrase "$BACKUP_PASSPHRASE" -c --cipher-algo AES256 \
62
+ -o "$BACKEND_DIR/db_backup/webui.db.gpg" "$BACKEND_DIR/data/webui.db"
63
+
64
+ if [ $? -eq 0 ]; then
65
+ echo "Database encrypted successfully"
66
+ cd "$BACKEND_DIR" || exit 1
67
+
68
+ # Configure Git for this operation
69
+ git config --local user.email "[email protected]"
70
+ git config --local user.name "Space Bot"
71
+
72
+ echo "Committing and pushing changes..."
73
+ git add db_backup/webui.db.gpg
74
+ git commit -m "Update encrypted database backup"
75
+
76
+ # Push using the token
77
+ REPO_URL="https://user:[email protected]/spaces/$SPACE_NAME"
78
+ if git push "$REPO_URL" main; then
79
+ echo "Successfully pushed backup to repository"
80
+ cd - >/dev/null
81
+ return 0
82
+ else
83
+ echo "Failed to push to repository"
84
+ cd - >/dev/null
85
+ return 1
86
+ fi
87
+ else
88
+ echo "Failed to encrypt database"
89
+ return 1
90
+ fi
91
+ }
92
+
93
+ # Check requirements first
94
+ check_requirements || exit 1
95
+
96
+ # If script is run directly, default to encryption
97
+ # (equivalent to fish's: if test (status filename) = (status -f))
98
+ if [ "${BASH_SOURCE[0]}" = "$0" ]; then
99
+ encrypt_database
100
+ fi
backend/scripts/restore.py ADDED
@@ -0,0 +1,211 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ import os
3
+ import sys
4
+ import subprocess
5
+ import datetime
6
+ import sqlite3
7
+ import tempfile
8
+ from pathlib import Path
9
+ from huggingface_hub import HfApi, hf_hub_download
10
+
11
+ # Set up path to include the application module
12
+ SCRIPT_DIR = Path(__file__).parent.resolve()
13
+ BACKEND_DIR = SCRIPT_DIR.parent
14
+ sys.path.append(str(BACKEND_DIR))
15
+
16
+ # Database path (actual application database)
17
+ DATA_DIR = os.environ.get("DATA_DIR", "/app/backend/data")
18
+ DB_FILE_PATH = os.path.join(DATA_DIR, "webui.db")
19
+
20
+ # Hugging Face repo paths (virtual paths in HF storage)
21
+ REPO_DB_GPG_FILE = "db_backup/webui.db.gpg"
22
+ REPO_TIMESTAMP_FILE = "db_backup/last_backup_time.txt"
23
+
24
+
25
+ def check_gpg():
26
+ """Verify GPG is available."""
27
+ try:
28
+ subprocess.run(["gpg", "--version"], check=True, capture_output=True)
29
+ return True
30
+ except (subprocess.CalledProcessError, FileNotFoundError):
31
+ print("Error: gpg is not installed or not in PATH")
32
+ return False
33
+
34
+
35
+ def validate_environment():
36
+ """Verify all required environment variables are set."""
37
+ required = ["BACKUP_PASSPHRASE", "HF_TOKEN", "SPACE_ID"]
38
+ missing = [var for var in required if not os.environ.get(var)]
39
+ if missing:
40
+ print(f"Error: Missing environment variables: {', '.join(missing)}")
41
+ return False
42
+ return True
43
+
44
+
45
+ def ensure_data_dir():
46
+ """Ensure the database directory exists."""
47
+ try:
48
+ os.makedirs(DATA_DIR, mode=0o755, exist_ok=True)
49
+ return True
50
+ except Exception as e:
51
+ print(f"Error creating data directory: {e}")
52
+ return False
53
+
54
+
55
+ def get_latest_backup_info(repo_id, hf_token):
56
+ """Check if backup exists and get its timestamp."""
57
+ api = HfApi()
58
+ try:
59
+ files = api.list_repo_files(repo_id=repo_id, repo_type="space", token=hf_token)
60
+
61
+ if REPO_DB_GPG_FILE not in files:
62
+ print("No backup file found in repository")
63
+ return False, None
64
+
65
+ if REPO_TIMESTAMP_FILE in files:
66
+ try:
67
+ timestamp_file = hf_hub_download(
68
+ repo_id=repo_id,
69
+ repo_type="space",
70
+ filename=REPO_TIMESTAMP_FILE,
71
+ token=hf_token
72
+ )
73
+ with open(timestamp_file, "r", encoding="utf-8") as f:
74
+ timestamp_str = f.read().strip()
75
+ timestamp = datetime.datetime.fromisoformat(timestamp_str)
76
+ print(f"Found backup from: {timestamp} UTC")
77
+ return True, timestamp
78
+ except Exception as e:
79
+ print(f"Could not read timestamp: {e}")
80
+ return True, None
81
+ return True, None
82
+ except Exception as e:
83
+ print(f"Error checking repository: {e}")
84
+ return False, None
85
+
86
+
87
+ def decrypt_database_from_memory(encrypted_data, passphrase):
88
+ """Decrypt database directly from memory."""
89
+ try:
90
+ # Create a secure temporary directory for GPG operations
91
+ with tempfile.TemporaryDirectory(prefix='gpg_home_') as gpg_home:
92
+ os.chmod(gpg_home, 0o700)
93
+
94
+ # Create a temporary file for the encrypted data
95
+ with tempfile.NamedTemporaryFile(mode='wb', suffix='.gpg', delete=False) as temp_encrypted:
96
+ temp_encrypted.write(encrypted_data)
97
+ temp_encrypted_path = temp_encrypted.name
98
+
99
+ try:
100
+ print(f"Decrypting database ({len(encrypted_data)} bytes)...")
101
+ decrypt_cmd = [
102
+ "gpg",
103
+ "--batch",
104
+ "--yes",
105
+ "--homedir", gpg_home,
106
+ "--passphrase", passphrase,
107
+ "--pinentry-mode", "loopback",
108
+ "-d",
109
+ "-o", DB_FILE_PATH,
110
+ temp_encrypted_path
111
+ ]
112
+
113
+ result = subprocess.run(decrypt_cmd, capture_output=True, check=True)
114
+
115
+ if os.path.exists(DB_FILE_PATH) and os.path.getsize(DB_FILE_PATH) > 0:
116
+ print(f"Database decrypted successfully ({os.path.getsize(DB_FILE_PATH)} bytes)")
117
+ return True
118
+ else:
119
+ print("Error: Decrypted database is missing or empty")
120
+ return False
121
+
122
+ finally:
123
+ # Clean up the temporary encrypted file
124
+ if os.path.exists(temp_encrypted_path):
125
+ os.unlink(temp_encrypted_path)
126
+
127
+ except subprocess.CalledProcessError as e:
128
+ print(f"Decryption failed: {e.stderr.decode()}")
129
+ return False
130
+ except Exception as e:
131
+ print(f"Decryption error: {e}")
132
+ return False
133
+
134
+
135
+ def verify_database():
136
+ """Verify the restored database integrity."""
137
+ if not os.path.exists(DB_FILE_PATH):
138
+ print(f"Error: Database file not found at {DB_FILE_PATH}")
139
+ return False
140
+
141
+ try:
142
+ print("Verifying database integrity...")
143
+ with sqlite3.connect(DB_FILE_PATH) as conn:
144
+ cursor = conn.cursor()
145
+ cursor.execute("PRAGMA integrity_check;")
146
+ result = cursor.fetchone()[0]
147
+ cursor.execute("SELECT name FROM sqlite_master WHERE type='table';")
148
+ tables = cursor.fetchall()
149
+
150
+ if result.lower() == "ok" and len(tables) > 0:
151
+ print(f"Database verified: {len(tables)} tables found")
152
+ return True
153
+ print("Database verification failed")
154
+ return False
155
+ except Exception as e:
156
+ print(f"Database verification error: {e}")
157
+ return False
158
+
159
+
160
+ def restore_db():
161
+ """Main restore function using in-memory approach."""
162
+ if not check_gpg() or not validate_environment() or not ensure_data_dir():
163
+ return False
164
+
165
+ passphrase = os.environ["BACKUP_PASSPHRASE"]
166
+ hf_token = os.environ["HF_TOKEN"]
167
+ space_id = os.environ["SPACE_ID"]
168
+
169
+ backup_exists, timestamp = get_latest_backup_info(space_id, hf_token)
170
+ if not backup_exists:
171
+ print("No backup found - starting with fresh database")
172
+ return True
173
+
174
+ try:
175
+ print("Downloading encrypted database...")
176
+ encrypted_file = hf_hub_download(
177
+ repo_id=space_id,
178
+ repo_type="space",
179
+ filename=REPO_DB_GPG_FILE,
180
+ token=hf_token
181
+ )
182
+
183
+ # Read encrypted data into memory
184
+ with open(encrypted_file, 'rb') as f:
185
+ encrypted_data = f.read()
186
+
187
+ print(f"Downloaded encrypted data: {len(encrypted_data)} bytes")
188
+
189
+ if not decrypt_database_from_memory(encrypted_data, passphrase):
190
+ print("Failed to decrypt database")
191
+ return False
192
+
193
+ if not verify_database():
194
+ print("Failed to verify database")
195
+ if os.path.exists(DB_FILE_PATH):
196
+ os.unlink(DB_FILE_PATH)
197
+ return False
198
+
199
+ print("Database restore completed successfully!")
200
+ return True
201
+
202
+ except Exception as e:
203
+ print(f"Restore error: {e}")
204
+ if os.path.exists(DB_FILE_PATH):
205
+ os.unlink(DB_FILE_PATH)
206
+ return False
207
+
208
+
209
+ if __name__ == "__main__":
210
+ success = restore_db()
211
+ sys.exit(0 if success else 1)
backend/start.sh CHANGED
@@ -1,68 +1,115 @@
1
  #!/usr/bin/env bash
2
 
3
- SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )
4
- cd "$SCRIPT_DIR" || exit
5
-
6
- # Add conditional Playwright browser installation
7
- if [[ "${WEB_LOADER_ENGINE,,}" == "playwright" ]]; then
8
- if [[ -z "${PLAYWRIGHT_WS_URL}" ]]; then
9
- echo "Installing Playwright browsers..."
10
- playwright install chromium
11
- playwright install-deps chromium
 
 
 
12
  fi
 
13
 
14
- python -c "import nltk; nltk.download('punkt_tab')"
 
 
 
 
 
15
  fi
16
 
17
- KEY_FILE=.webui_secret_key
18
-
19
  PORT="${PORT:-8080}"
20
  HOST="${HOST:-0.0.0.0}"
21
- if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
22
- echo "Loading WEBUI_SECRET_KEY from file, not provided as an environment variable."
23
-
24
- if ! [ -e "$KEY_FILE" ]; then
25
- echo "Generating WEBUI_SECRET_KEY"
26
- # Generate a random value to use as a WEBUI_SECRET_KEY in case the user didn't provide one.
27
- echo $(head -c 12 /dev/random | base64) > "$KEY_FILE"
28
- fi
29
 
30
- echo "Loading WEBUI_SECRET_KEY from $KEY_FILE"
31
- WEBUI_SECRET_KEY=$(cat "$KEY_FILE")
 
 
 
32
  fi
33
 
 
34
  if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
35
- echo "USE_OLLAMA is set to true, starting ollama serve."
36
  ollama serve &
37
  fi
38
 
 
39
  if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
40
- echo "CUDA is enabled, appending LD_LIBRARY_PATH to include torch/cudnn & cublas libraries."
41
- export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
42
  fi
43
 
44
- # Check if SPACE_ID is set, if so, configure for space
45
  if [ -n "$SPACE_ID" ]; then
46
- echo "Configuring for HuggingFace Space deployment"
47
- if [ -n "$ADMIN_USER_EMAIL" ] && [ -n "$ADMIN_USER_PASSWORD" ]; then
48
- echo "Admin user configured, creating"
49
- WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
50
- webui_pid=$!
51
- echo "Waiting for webui to start..."
52
- while ! curl -s http://localhost:8080/health > /dev/null; do
53
- sleep 1
54
- done
55
- echo "Creating admin user..."
56
- curl \
57
- -X POST "http://localhost:8080/api/v1/auths/signup" \
58
- -H "accept: application/json" \
59
- -H "Content-Type: application/json" \
60
- -d "{ \"email\": \"${ADMIN_USER_EMAIL}\", \"password\": \"${ADMIN_USER_PASSWORD}\", \"name\": \"Admin\" }"
61
- echo "Shutting down webui..."
62
- kill $webui_pid
63
- fi
64
-
65
- export WEBUI_URL=${SPACE_HOST}
 
 
 
 
 
 
 
66
  fi
67
 
68
- WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" exec uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' --workers "${UVICORN_WORKERS:-1}"
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  #!/usr/bin/env bash
2
 
3
+ # Ensure we're in the right directory and set up paths
4
+ SCRIPT_DIR=$(dirname "$(readlink -f "${BASH_SOURCE[0]}")")
5
+ cd "$SCRIPT_DIR" || exit 1
6
+
7
+ export DATA_DIR="/app/backend/data"
8
+ export PYTHONPATH="/app/backend:${PYTHONPATH}"
9
+
10
+ # Validate required environment variables
11
+ for var in "BACKUP_PASSPHRASE" "HF_TOKEN" "SPACE_ID"; do
12
+ if [ -z "${!var}" ]; then
13
+ echo "Error: $var is not set. Required for database backup/restore."
14
+ exit 1
15
  fi
16
+ done
17
 
18
+ # Restore database from backup
19
+ echo "Restoring database from backup..."
20
+ python "$SCRIPT_DIR/scripts/restore.py"
21
+ restore_status=$?
22
+ if [ $restore_status -ne 0 ]; then
23
+ echo "Warning: Database restore failed. Starting with empty database."
24
  fi
25
 
26
+ # Handle WebUI secret key
27
+ KEY_FILE="$SCRIPT_DIR/.webui_secret_key"
28
  PORT="${PORT:-8080}"
29
  HOST="${HOST:-0.0.0.0}"
 
 
 
 
 
 
 
 
30
 
31
+ if test "$WEBUI_SECRET_KEY $WEBUI_JWT_SECRET_KEY" = " "; then
32
+ if ! [ -e "$KEY_FILE" ]; then
33
+ head -c 12 /dev/random | base64 > "$KEY_FILE" || exit 1
34
+ fi
35
+ WEBUI_SECRET_KEY=$(cat "$KEY_FILE") || exit 1
36
  fi
37
 
38
+ # Start Ollama if enabled
39
  if [[ "${USE_OLLAMA_DOCKER,,}" == "true" ]]; then
 
40
  ollama serve &
41
  fi
42
 
43
+ # Configure CUDA if enabled
44
  if [[ "${USE_CUDA_DOCKER,,}" == "true" ]]; then
45
+ export LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/local/lib/python3.11/site-packages/torch/lib:/usr/local/lib/python3.11/site-packages/nvidia/cudnn/lib"
 
46
  fi
47
 
48
+ # Handle HuggingFace Space deployment
49
  if [ -n "$SPACE_ID" ]; then
50
+ echo "Configuring HuggingFace Space deployment"
51
+ if [ -n "$ADMIN_USER_EMAIL" ] && [ -n "$ADMIN_USER_PASSWORD" ]; then
52
+ # Start webui temporarily to create admin user
53
+ WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' &
54
+ webui_pid=$!
55
+
56
+ # Wait for the server to be ready
57
+ while ! curl -s http://localhost:8080/health > /dev/null; do
58
+ echo "Waiting for WebUI to start..."
59
+ sleep 1
60
+ done
61
+
62
+ # Create the admin user
63
+ echo "Creating admin user: ${ADMIN_USER_EMAIL}"
64
+ curl \
65
+ -X POST "http://localhost:8080/api/v1/auths/signup" \
66
+ -H "accept: application/json" \
67
+ -H "Content-Type: application/json" \
68
+ -d "{ \"email\": \"${ADMIN_USER_EMAIL}\", \"password\": \"${ADMIN_USER_PASSWORD}\", \"name\": \"Admin\" }"
69
+
70
+ # Stop the temporary webui server
71
+ echo "Stopping temporary WebUI server..."
72
+ kill $webui_pid
73
+ wait $webui_pid 2>/dev/null # Wait for it to fully shut down
74
+ fi
75
+ # Set the public URL for Space deployments
76
+ export WEBUI_URL=${SPACE_HOST}
77
  fi
78
 
79
+ # Start the main web server in the background
80
+ echo "Starting main WebUI server..."
81
+ WEBUI_SECRET_KEY="$WEBUI_SECRET_KEY" uvicorn open_webui.main:app \
82
+ --host "$HOST" --port "$PORT" --forwarded-allow-ips '*' \
83
+ --workers "${UVICORN_WORKERS:-1}" &
84
+ WEBUI_PID=$!
85
+
86
+ # Configure and start backup scheduler in the background
87
+ BACKUP_INITIAL_WAIT="${BACKUP_INITIAL_WAIT:-100}"
88
+ BACKUP_INTERVAL="${BACKUP_INTERVAL:-300}"
89
+
90
+ (
91
+ echo "Starting backup scheduler (Initial wait: ${BACKUP_INITIAL_WAIT}s, Interval: ${BACKUP_INTERVAL}s)"
92
+ sleep "$BACKUP_INITIAL_WAIT"
93
+
94
+ while true; do
95
+ echo "Running scheduled backup"
96
+ # Pass relevant env vars to backup script if needed
97
+ BACKUP_THRESHOLD_MINUTES="${BACKUP_THRESHOLD_MINUTES:-120}" \
98
+ python "$SCRIPT_DIR/scripts/backup.py"
99
+
100
+ if [ $? -ne 0 ]; then
101
+ echo "Warning: Backup failed"
102
+ fi
103
+
104
+ sleep "$BACKUP_INTERVAL"
105
+ done
106
+ ) &
107
+
108
+ # Wait for the main web server process to exit
109
+ # This keeps the script alive while the server runs and allows graceful shutdown
110
+ echo "Web server started (PID: ${WEBUI_PID}). Waiting for it to exit..."
111
+ wait $WEBUI_PID
112
+ echo "Web server process ${WEBUI_PID} exited."
113
+
114
+ # Optional: Add cleanup here if needed after the web server stops.
115
+ # The background backup loop will likely be terminated when the main script exits.
db_backup/last_backup_time.txt ADDED
@@ -0,0 +1 @@
 
 
1
+ 2025-03-22T14:02:37.983192+00:00
vite.config.ts CHANGED
@@ -35,7 +35,7 @@ export default defineConfig({
35
  APP_BUILD_HASH: JSON.stringify(process.env.APP_BUILD_HASH || 'dev-build')
36
  },
37
  build: {
38
- sourcemap: true
39
  },
40
  worker: {
41
  format: 'es'
 
35
  APP_BUILD_HASH: JSON.stringify(process.env.APP_BUILD_HASH || 'dev-build')
36
  },
37
  build: {
38
+ sourcemap: false
39
  },
40
  worker: {
41
  format: 'es'