FABLESLIP commited on
Commit
ce9842f
·
verified ·
1 Parent(s): 4fb0ce4

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +40 -12
app.py CHANGED
@@ -133,24 +133,42 @@ def _log_warmup(msg: str):
133
  if len(warmup_state["logs"]) > 400:
134
  warmup_state["logs"] = warmup_state["logs"][-400:]
135
 
 
 
 
 
 
 
 
 
 
 
 
 
 
136
  def _download_one(repo_id: str, tries: int = 3) -> bool:
137
  """
138
- Télécharge un repo HF en réessayant si besoin.
139
  """
140
  cache_home = os.path.expanduser(os.getenv("HF_HOME", "/home/user/.cache/huggingface"))
141
  local_dir = os.path.join(cache_home, "models", repo_id.replace("/", "__"))
142
- # Déjà présent localement ? On log et on saute proprement.
 
143
  try:
144
  if _is_repo_cached(repo_id):
145
- _log_warmup(f"Déjà en cache: {repo_id} (skip)")
 
146
  return True
147
  except Exception:
148
- # En cas d'erreur d'audit, on continue le process normal
149
  pass
150
 
151
  for attempt in range(1, tries + 1):
152
  if warmup_stop.is_set():
 
153
  return False
 
 
 
154
  try:
155
  snapshot_download(
156
  repo_id,
@@ -158,9 +176,13 @@ def _download_one(repo_id: str, tries: int = 3) -> bool:
158
  local_dir_use_symlinks=False,
159
  resume_download=True,
160
  )
 
 
 
161
  return True
162
  except Exception as e:
163
- _log_warmup(f"{repo_id} -> tentative {attempt}/{tries} échouée: {e}")
 
164
  time.sleep(min(10, 2 * attempt))
165
  return False
166
 
@@ -178,33 +200,39 @@ def _warmup_thread(models: List[str]):
178
  })
179
  warmup_stop.clear()
180
  ok_count = 0
 
 
 
181
  for i, repo in enumerate(models):
182
  if warmup_stop.is_set():
183
- _log_warmup("Arrêt demandé par l’utilisateur.")
184
  break
 
185
  with warmup_lock:
186
  warmup_state["idx"] = i
187
  warmup_state["current"] = repo
188
  warmup_state["percent"] = int((i / max(1, len(models))) * 100)
189
- _log_warmup(f"Téléchargement: {repo}")
190
  ok = _download_one(repo)
191
  if ok:
192
  ok_count += 1
193
- _log_warmup(f"OK: {repo}")
194
  with warmup_lock:
195
  warmup_state["ok_count"] = ok_count
196
- else:
197
- _log_warmup(f"ÉCHEC: {repo}")
198
- # Met à jour la progression globale après ce repo (ok ou échec)
199
  with warmup_lock:
200
  warmup_state["percent"] = int(((i + 1) / max(1, len(models))) * 100)
201
 
 
 
 
 
202
  with warmup_lock:
203
  warmup_state["percent"] = 100
204
  warmup_state["done"] = True
205
  warmup_state["running"] = False
206
  warmup_state["ok_count"] = ok_count
207
- _log_warmup(f"Terminé: {ok_count}/{len(models)} modèles.")
 
208
  # <<<< HUGINFACE PATCH: WARMUP STATE+HELPERS END >>>>
209
  # ---------- Helpers ----------
210
 
 
133
  if len(warmup_state["logs"]) > 400:
134
  warmup_state["logs"] = warmup_state["logs"][-400:]
135
 
136
+ def _dir_size_bytes(path: str) -> int:
137
+ try:
138
+ total = 0
139
+ for root, _, files in os.walk(path):
140
+ for f in files:
141
+ try:
142
+ total += os.path.getsize(os.path.join(root, f))
143
+ except Exception:
144
+ pass
145
+ return total
146
+ except Exception:
147
+ return 0
148
+
149
  def _download_one(repo_id: str, tries: int = 3) -> bool:
150
  """
151
+ Télécharge un repo HF en réessayant si besoin, avec logs précis.
152
  """
153
  cache_home = os.path.expanduser(os.getenv("HF_HOME", "/home/user/.cache/huggingface"))
154
  local_dir = os.path.join(cache_home, "models", repo_id.replace("/", "__"))
155
+
156
+ # Cache déjà présent ?
157
  try:
158
  if _is_repo_cached(repo_id):
159
+ sz = _dir_size_bytes(local_dir)
160
+ _log_warmup(f"[CACHE] {repo_id} • {local_dir} • {sz/1e6:.1f} MB (skip)")
161
  return True
162
  except Exception:
 
163
  pass
164
 
165
  for attempt in range(1, tries + 1):
166
  if warmup_stop.is_set():
167
+ _log_warmup(f"[STOP] Abandon demandé avant téléchargement de {repo_id}")
168
  return False
169
+
170
+ t0 = time.time()
171
+ _log_warmup(f"[START] {repo_id} • tentative {attempt}/{tries} • {local_dir}")
172
  try:
173
  snapshot_download(
174
  repo_id,
 
176
  local_dir_use_symlinks=False,
177
  resume_download=True,
178
  )
179
+ dt = time.time() - t0
180
+ sz = _dir_size_bytes(local_dir)
181
+ _log_warmup(f"[DONE] {repo_id} • {dt:.1f}s • {sz/1e6:.1f} MB")
182
  return True
183
  except Exception as e:
184
+ dt = time.time() - t0
185
+ _log_warmup(f"[FAIL] {repo_id} • {dt:.1f}s • {type(e).__name__}: {e}")
186
  time.sleep(min(10, 2 * attempt))
187
  return False
188
 
 
200
  })
201
  warmup_stop.clear()
202
  ok_count = 0
203
+ t_global = time.time()
204
+ _log_warmup(f"[JOB] start • {len(models)} dépôts")
205
+
206
  for i, repo in enumerate(models):
207
  if warmup_stop.is_set():
208
+ _log_warmup("[STOP] Arrêt demandé par l’utilisateur — fin du job après ce point")
209
  break
210
+
211
  with warmup_lock:
212
  warmup_state["idx"] = i
213
  warmup_state["current"] = repo
214
  warmup_state["percent"] = int((i / max(1, len(models))) * 100)
215
+
216
  ok = _download_one(repo)
217
  if ok:
218
  ok_count += 1
 
219
  with warmup_lock:
220
  warmup_state["ok_count"] = ok_count
221
+ # progression après dépôt (OK ou FAIL)
 
 
222
  with warmup_lock:
223
  warmup_state["percent"] = int(((i + 1) / max(1, len(models))) * 100)
224
 
225
+ if warmup_stop.is_set():
226
+ _log_warmup("[STOP] Fin anticipée — demande reçue pendant le run")
227
+ break
228
+
229
  with warmup_lock:
230
  warmup_state["percent"] = 100
231
  warmup_state["done"] = True
232
  warmup_state["running"] = False
233
  warmup_state["ok_count"] = ok_count
234
+ _log_warmup(f"[JOB] done • {ok_count}/{len(models)} • {time.time()-t_global:.1f}s")
235
+
236
  # <<<< HUGINFACE PATCH: WARMUP STATE+HELPERS END >>>>
237
  # ---------- Helpers ----------
238