import os import time import subprocess import datetime BACKUP_REPO = os.environ.get("BACKUP_REPO") HF_TOKEN = os.environ.get("HF_TOKEN") if not BACKUP_REPO or not HF_TOKEN: print("[Backup] Skipping: BACKUP_REPO or HF_TOKEN not set") exit(0) def run_backup(): env = os.environ.copy() env["HF_HOME"] = "/tmp/hf_cache" env["XDG_CACHE_HOME"] = "/tmp/xdg_cache" env["TMPDIR"] = "/tmp" env["HF_TOKEN"] = HF_TOKEN os.makedirs(env["HF_HOME"], exist_ok=True) os.makedirs(env["XDG_CACHE_HOME"], exist_ok=True) os.makedirs(env["TMPDIR"], exist_ok=True) local_path = '/home/vscode/workspace' # Upload with huggingface-cli cmd = [ "hf", "upload", BACKUP_REPO, local_path, "workspace/", "--repo-type", "dataset", ] process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, text=True, cwd="/tmp", ) for line in iter(process.stdout.readline, ''): print("[Backup]", line.strip()) cmd = [ "hf", "repo-files", "delete", BACKUP_REPO, "workspace/", "--repo-type", "dataset", ] process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, env=env, text=True, cwd="/tmp", ) for line in iter(process.stdout.readline, ''): print("[Backup]", line.strip()) process.wait() while True: try: run_backup() except Exception as e: print("[Backup] Failed:", e) time.sleep(45 * 60) # 45 minutes