Ver Fonte

feat: rétention distante après push HTTP chunked

- _job_archive_prefix() extrait de _list_archives_for_job (réutilisable)
- apply_remote_retention(job, client) dans retention.py :
  filtre les archives distantes par préfixe du job local,
  applique la même politique count/daily via DELETE /api/v1/archives/<name>
- FederationClient.delete_archive() dans federation/client.py
- push_archive_to_instance() accepte job= optionnel et appelle
  apply_remote_retention() après chaque push réussi
- Le log du run trace les suppressions distantes

Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
Cedric Hansen há 1 mês atrás
pai
commit
8fb5bdf664
4 ficheiros alterados com 80 adições e 21 exclusões
  1. 10 0
      sources/federation/client.py
  2. 13 2
      sources/jobs/transfer.py
  3. 1 1
      sources/jobs/ynh_backup.py
  4. 56 18
      sources/retention.py

+ 10 - 0
sources/federation/client.py

@@ -117,6 +117,16 @@ class FederationClient:
         r.raise_for_status()
         return r.content
 
+    def delete_archive(self, archive_name):
+        r = requests.delete(
+            f"{self.base}/api/v1/archives/{archive_name}",
+            headers=self.headers,
+            timeout=self.timeout,
+            verify=True,
+        )
+        r.raise_for_status()
+        return r.json()
+
     def upload_cancel(self, upload_id):
         r = requests.delete(
             f"{self.base}/api/v1/archives/upload/{upload_id}",

+ 13 - 2
sources/jobs/transfer.py

@@ -98,7 +98,7 @@ def test_connection(destination, data_dir):
     return False, result.stderr.strip() or f"Connexion échouée (code {result.returncode})."
 
 
-def push_archive_to_instance(archive_name, instance, backup_dir):
+def push_archive_to_instance(archive_name, instance, backup_dir, job=None):
     """Pousse .tar + .info.json vers une instance fédérée via HTTP chunked. Retourne un log texte."""
     import hashlib
     from federation.client import FederationClient
@@ -150,7 +150,18 @@ def push_archive_to_instance(archive_name, instance, backup_dir):
                     subprocess.run(["sudo", "rm", "-rf", tmp_info], capture_output=True)
 
         client.upload_finish_with_info(upload_id, info_json_content)
-        return f"Transfert HTTP chunked vers {instance.name} ({instance.url}) — {total_size // (1024*1024)} Mo en {n} chunks."
+        log = f"Transfert HTTP chunked vers {instance.name} ({instance.url}) — {total_size // (1024*1024)} Mo en {n} chunks."
+
+        if job is not None:
+            from retention import apply_remote_retention
+            try:
+                deleted = apply_remote_retention(job, client)
+                if deleted:
+                    log += f"\nRétention distante : {len(deleted)} archive(s) supprimée(s) : {', '.join(deleted)}"
+            except Exception as exc:
+                log += f"\n⚠ Rétention distante échouée : {exc}"
+
+        return log
 
     finally:
         if os.path.exists(tmp_tar):

+ 1 - 1
sources/jobs/ynh_backup.py

@@ -69,7 +69,7 @@ def execute_job(job_id):
             if inst:
                 try:
                     from jobs.transfer import push_archive_to_instance
-                    transfer_log = push_archive_to_instance(archive_name, inst, backup_dir)
+                    transfer_log = push_archive_to_instance(archive_name, inst, backup_dir, job=job)
                     run.log_text += f"\n\nTransfert HTTP → {inst.name} :\n{transfer_log}"
                 except Exception as transfer_exc:
                     run.log_text += f"\n\n⚠ Transfert HTTP échoué vers {inst.name} :\n{transfer_exc}"

+ 56 - 18
sources/retention.py

@@ -24,40 +24,78 @@ def apply_retention(job, new_archive_name, backup_dir):
     return deleted
 
 
-def _list_archives_for_job(job, backup_dir):
-    """Liste les archives correspondant à ce job, triées par date (plus ancienne en premier)."""
-    from flask import current_app
-    instance = current_app.config["INSTANCE_NAME"]
-
+def _job_archive_prefix(job, instance_name):
+    """Retourne le préfixe des archives pour ce job (ex: jerry_nextcloud_)."""
     if job.type == "ynh_app":
         import json
         cfg = json.loads(job.config_json or "{}")
-        app_id = cfg.get("app_id", "")
-        prefix = f"{instance}_{app_id}_"
+        return f"{instance_name}_{cfg.get('app_id', '')}_"
     elif job.type == "ynh_system":
-        prefix = f"{instance}_system_"
+        return f"{instance_name}_system_"
     elif job.type in ("mysql", "postgresql"):
         import json
         cfg = json.loads(job.config_json or "{}")
-        dbname = cfg.get("database", "")
-        prefix = f"{instance}_{job.type}_{dbname}_"
+        return f"{instance_name}_{job.type}_{cfg.get('database', '')}_"
     elif job.type == "custom_dir":
-        import re
         label = re.sub(r'[^a-z0-9]+', '-', job.name.lower().strip()).strip('-')
-        prefix = f"{instance}_{label}_"
+        return f"{instance_name}_{label}_"
     else:
-        prefix = f"{instance}_{job.name.lower().replace(' ', '-')}_"
+        return f"{instance_name}_{job.name.lower().replace(' ', '-')}_"
 
-    from jobs.utils import sudo_listdir
-    archives = []
-    for fname in sudo_listdir(backup_dir):
-        if fname.startswith(prefix) and fname.endswith(".tar"):
-            archives.append(fname)
 
+def _list_archives_for_job(job, backup_dir):
+    """Liste les archives correspondant à ce job, triées par date (plus ancienne en premier)."""
+    from flask import current_app
+    instance = current_app.config["INSTANCE_NAME"]
+    prefix = _job_archive_prefix(job, instance)
+
+    from jobs.utils import sudo_listdir
+    archives = [
+        fname for fname in sudo_listdir(backup_dir)
+        if fname.startswith(prefix) and fname.endswith(".tar")
+    ]
     archives.sort(key=_extract_date)
     return archives
 
 
+def apply_remote_retention(job, client):
+    """Applique la rétention sur l'instance distante après un push.
+
+    Filtre les archives par le même préfixe que le job local et applique
+    la même politique (count/daily). Ne touche pas aux archives des autres jobs.
+    """
+    from flask import current_app
+    instance = current_app.config["INSTANCE_NAME"]
+    prefix = _job_archive_prefix(job, instance)
+
+    try:
+        remote_archives = client.get_archives()
+    except Exception:
+        return []
+
+    matching = sorted(
+        [a["name"] + ".tar" for a in remote_archives if a["name"].startswith(prefix)],
+        key=_extract_date,
+    )
+
+    if job.retention_mode == "count":
+        to_delete = _retention_count(matching, job.retention_value)
+    elif job.retention_mode == "daily":
+        to_delete = _retention_daily(matching, job.retention_value)
+    else:
+        return []
+
+    deleted = []
+    for archive_filename in to_delete:
+        base = os.path.splitext(archive_filename)[0]
+        try:
+            client.delete_archive(base)
+            deleted.append(base)
+        except Exception:
+            pass
+    return deleted
+
+
 def _extract_date(filename):
     match = re.search(r'(\d{8})', filename)
     if match: