"""Lädt alle Dateien der SharePoint-/OneDrive-Laufwerke herunter (nur GET). Quelle ist graph/drives_inventory.json (siehe drive_inventory.py). Wiederaufnehmbar: Dateien mit passender Grösse und Manifest-Eintrag werden übersprungen. Schreibt files/, manifest.jsonl und files_errors.json.""" import hashlib, json, os, re, sys, threading, time from concurrent.futures import ThreadPoolExecutor sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import m365_auth as auth def sanitize(s): return re.sub(r'[\x00-\x1f<>:"|?*\\]', "_", s).strip() or "_" def main(argv=None): p = auth.make_parser("Download all drive files (read-only)") p.add_argument("--max-files", type=int, default=0, help="limit the number of files (0 = all)") p.add_argument("--threads", type=int, default=4) args = p.parse_args(argv) import requests out = args.out inventory = json.load(open(os.path.join(out, "graph", "drives_inventory.json"))) files_dir = os.path.join(out, "files"); os.makedirs(files_dir, exist_ok=True) manifest = os.path.join(out, "manifest.jsonl") lock = threading.Lock() done = {} if os.path.exists(manifest): for line in open(manifest): try: j = json.loads(line); done[j["path"]] = j except Exception: pass tasks, errors = [], [] for drive_id, dr in inventory.items(): if not isinstance(dr["items"], list): errors.append((drive_id, dr["name"], "inventory faulty")); continue items = {i["id"]: i for i in dr["items"]} root = next((i["id"] for i in dr["items"] if i.get("name") == "root" and "folder" in i), None) m = re.search(r"https://[^/]+/(?:sites|teams|personal)/([^/]+)", dr.get("webUrl") or "") site = sanitize(m.group(1) if m else (dr.get("webUrl") or drive_id)[:40]) lib = sanitize(dr.get("name") or drive_id[-8:]) def rel_path(i, items=items, root=root): parts = [] while i and i in items and i != root: parts.append(sanitize(items[i]["name"])); i = items[i].get("parentReference", {}).get("id") return "/".join(reversed(parts)) for it in dr["items"]: if "file" in it and not it.get("deleted"): tasks.append((drive_id, it, f"{site}/{lib}/{rel_path(it['id'])}")) if args.max_files: tasks = tasks[:args.max_files] print(len(tasks), "files", sum(t[1].get("size", 0) for t in tasks) / 1e9, "GB", flush=True) counter = [0, 0] def work(t): drive_id, it, rel = t target = os.path.join(files_dir, rel) if rel in done and os.path.exists(target) and os.path.getsize(target) == it.get("size"): return os.makedirs(os.path.dirname(target), exist_ok=True) err = "" for attempt in range(6): try: r = auth.g("GET", f"/drives/{drive_id}/items/{it['id']}") if r.status_code in (429, 500, 502, 503, 504): time.sleep(int(r.headers.get("Retry-After", 5 * (attempt + 1)))); continue if r.status_code != 200: raise Exception(f"meta {r.status_code}") url = r.json().get("@microsoft.graph.downloadUrl") if not url: raise Exception("no downloadUrl") h = hashlib.sha256() with requests.get(url, stream=True, timeout=120) as s: if s.status_code in (429, 500, 502, 503, 504): time.sleep(int(s.headers.get("Retry-After", 5 * (attempt + 1)))); continue s.raise_for_status() with open(target + ".part", "wb") as f: for c in s.iter_content(1 << 20): f.write(c); h.update(c) os.replace(target + ".part", target) rec = {"path": rel, "size": os.path.getsize(target), "sha256": h.hexdigest(), "lastModified": it.get("lastModifiedDateTime"), "webUrl": it.get("webUrl")} with lock: open(manifest, "a").write(json.dumps(rec, ensure_ascii=False) + "\n"); counter[0] += 1 if counter[0] % 50 == 0: print(counter, flush=True) return except Exception as e: err = str(e)[:150]; time.sleep(3 * (attempt + 1)) with lock: counter[1] += 1; errors.append((rel, err)) with ThreadPoolExecutor(args.threads) as ex: list(ex.map(work, tasks)) auth.write_json(os.path.join(out, "files_errors.json"), errors) print("DONE", counter, len(errors)) return 0 if __name__ == "__main__": sys.exit(main())