|
@@ -2,6 +2,7 @@ import json
|
|
|
import os
|
|
import os
|
|
|
import shlex
|
|
import shlex
|
|
|
import subprocess
|
|
import subprocess
|
|
|
|
|
+from concurrent.futures import ThreadPoolExecutor
|
|
|
|
|
|
|
|
from flask import Blueprint, current_app, flash, redirect, render_template, request, url_for
|
|
from flask import Blueprint, current_app, flash, redirect, render_template, request, url_for
|
|
|
|
|
|
|
@@ -128,55 +129,45 @@ def overview():
|
|
|
instance = current_app.config["INSTANCE_NAME"]
|
|
instance = current_app.config["INSTANCE_NAME"]
|
|
|
jobs = Job.query.order_by(Job.name).all()
|
|
jobs = Job.query.order_by(Job.name).all()
|
|
|
|
|
|
|
|
- locations = []
|
|
|
|
|
-
|
|
|
|
|
- # --- Local ---
|
|
|
|
|
- try:
|
|
|
|
|
- archives = _collect_local(backup_dir)
|
|
|
|
|
- groups, orphaned = _group_archives(archives, jobs, instance)
|
|
|
|
|
- locations.append({
|
|
|
|
|
- "type": "local", "label": "Local",
|
|
|
|
|
- "error": None, "groups": groups, "orphaned": orphaned,
|
|
|
|
|
- })
|
|
|
|
|
- except Exception as exc:
|
|
|
|
|
- locations.append({
|
|
|
|
|
- "type": "local", "label": "Local",
|
|
|
|
|
- "error": str(exc), "groups": [], "orphaned": [],
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ # Construire la liste des tâches de collecte dans l'ordre d'affichage souhaité
|
|
|
|
|
+ dests = Destination.query.filter_by(enabled=True).order_by(Destination.name).all()
|
|
|
|
|
+ insts = RemoteInstance.query.order_by(RemoteInstance.name).all()
|
|
|
|
|
|
|
|
- # --- SSH ---
|
|
|
|
|
- for dest in Destination.query.filter_by(enabled=True).order_by(Destination.name).all():
|
|
|
|
|
|
|
+ tasks = [
|
|
|
|
|
+ ({"type": "local", "label": "Local"},
|
|
|
|
|
+ lambda: _collect_local(backup_dir)),
|
|
|
|
|
+ ]
|
|
|
|
|
+ for dest in dests:
|
|
|
|
|
+ tasks.append((
|
|
|
|
|
+ {"type": "ssh", "label": f"SSH → {dest.remote_str}", "dest_id": dest.id},
|
|
|
|
|
+ lambda d=dest: _collect_ssh(d, data_dir),
|
|
|
|
|
+ ))
|
|
|
|
|
+ for inst in insts:
|
|
|
|
|
+ tasks.append((
|
|
|
|
|
+ {"type": "instance", "label": inst.name,
|
|
|
|
|
+ "instance_id": inst.id, "instance_url": inst.url},
|
|
|
|
|
+ lambda i=inst: _collect_instance(i),
|
|
|
|
|
+ ))
|
|
|
|
|
+
|
|
|
|
|
+ # Collecter toutes les destinations en parallèle pour éviter que les timeouts
|
|
|
|
|
+ # d'une destination lente ne bloquent les autres
|
|
|
|
|
+ def _run_task(args):
|
|
|
|
|
+ meta, fn = args
|
|
|
try:
|
|
try:
|
|
|
- archives = _collect_ssh(dest, data_dir)
|
|
|
|
|
- groups, orphaned = _group_archives(archives, jobs, instance)
|
|
|
|
|
- locations.append({
|
|
|
|
|
- "type": "ssh", "label": f"SSH → {dest.remote_str}",
|
|
|
|
|
- "dest_id": dest.id, "error": None,
|
|
|
|
|
- "groups": groups, "orphaned": orphaned,
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ return meta, fn(), None
|
|
|
except Exception as exc:
|
|
except Exception as exc:
|
|
|
- locations.append({
|
|
|
|
|
- "type": "ssh", "label": f"SSH → {dest.remote_str}",
|
|
|
|
|
- "dest_id": dest.id, "error": str(exc),
|
|
|
|
|
- "groups": [], "orphaned": [],
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ return meta, None, str(exc)
|
|
|
|
|
|
|
|
- # --- Instances fédérées ---
|
|
|
|
|
- for inst in RemoteInstance.query.order_by(RemoteInstance.name).all():
|
|
|
|
|
- try:
|
|
|
|
|
- archives = _collect_instance(inst)
|
|
|
|
|
|
|
+ with ThreadPoolExecutor(max_workers=min(len(tasks), 10)) as pool:
|
|
|
|
|
+ raw_results = list(pool.map(_run_task, tasks))
|
|
|
|
|
+
|
|
|
|
|
+ locations = []
|
|
|
|
|
+ for meta, archives, error in raw_results:
|
|
|
|
|
+ if error:
|
|
|
|
|
+ locations.append({**meta, "error": error, "groups": [], "orphaned": []})
|
|
|
|
|
+ else:
|
|
|
groups, orphaned = _group_archives(archives, jobs, instance)
|
|
groups, orphaned = _group_archives(archives, jobs, instance)
|
|
|
- locations.append({
|
|
|
|
|
- "type": "instance", "label": f"{inst.name}",
|
|
|
|
|
- "instance_id": inst.id, "instance_url": inst.url,
|
|
|
|
|
- "error": None, "groups": groups, "orphaned": orphaned,
|
|
|
|
|
- })
|
|
|
|
|
- except Exception as exc:
|
|
|
|
|
- locations.append({
|
|
|
|
|
- "type": "instance", "label": f"{inst.name}",
|
|
|
|
|
- "instance_id": inst.id, "instance_url": inst.url,
|
|
|
|
|
- "error": str(exc), "groups": [], "orphaned": [],
|
|
|
|
|
- })
|
|
|
|
|
|
|
+ locations.append({**meta, "error": None, "groups": groups, "orphaned": orphaned})
|
|
|
|
|
|
|
|
# --- Vue par job : pivot de locations → jobs ---
|
|
# --- Vue par job : pivot de locations → jobs ---
|
|
|
jobs_view = []
|
|
jobs_view = []
|