"""Dateiversionen aller Laufwerke (nur GET), wiederaufnehmbar. Pro Datei: Metadaten aller Versionen nach versions/_meta//.json. Ältere Versionen (ohne die aktuelle, das ist der erste Eintrag; höchstens --max-versions neueste ältere) werden als Inhalt nach versions/// geladen. Das Manifest ist versions_manifest.jsonl.""" import hashlib, json, os, re, sys, threading, time from concurrent.futures import ThreadPoolExecutor sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import m365_auth as auth def sanitize(s): return re.sub(r'[^A-Za-z0-9._-]', "_", s) def main(argv=None): p = auth.make_parser("Download file versions of all drives (read-only)") p.add_argument("--max-versions", type=int, default=10, help="older versions per file (default 10)") p.add_argument("--max-files", type=int, default=0, help="limit the number of files (0 = all)") p.add_argument("--threads", type=int, default=int(os.environ.get("THREADS", "6"))) args = p.parse_args(argv) import requests out, max_v = args.out, args.max_versions vdir = os.path.join(out, "versions"); os.makedirs(os.path.join(vdir, "_meta"), exist_ok=True) manifest = os.path.join(out, "versions_manifest.jsonl") lock = threading.Lock(); done = set() if os.path.exists(manifest): for line in open(manifest): try: j = json.loads(line); done.add((j["drive"], j["item"], j["version"])) except Exception: pass inventory = json.load(open(os.path.join(out, "graph", "drives_inventory.json"))) tasks = [(did, it["id"], it["name"]) for did, dr in inventory.items() for it in dr["items"] if "file" in it] if args.max_files: tasks = tasks[:args.max_files] print(len(tasks), "files", flush=True) st = {"files": 0, "with_older": 0, "versions_meta": 0, "downloaded": 0, "bytes": 0, "skipped_limit": 0, "errors": 0} errors = [] def meta_error(t, msg): with lock: st["errors"] += 1; errors.append({"drive": t[0], "item": t[1], "name": t[2], "error": msg}) def work(t): did, iid, name = t; dd = sanitize(did) mp = f"{vdir}/_meta/{dd}/{sanitize(iid)}.json" if os.path.exists(mp): versions = json.load(open(mp))["versions"] else: versions, url = [], f"/drives/{did}/items/{iid}/versions" while url: r = auth.g("GET", url) if r.status_code != 200: return meta_error(t, f"versions {r.status_code} {r.text[:120]}") j = r.json(); versions += j.get("value", []); url = j.get("@odata.nextLink") os.makedirs(os.path.dirname(mp), exist_ok=True) json.dump({"drive": did, "item": iid, "name": name, "versions": versions}, open(mp, "w"), ensure_ascii=False) with lock: st["files"] += 1; st["versions_meta"] += len(versions) older = versions[1:] if older: with lock: st["with_older"] += 1 if len(older) > max_v: with lock: st["skipped_limit"] += len(older) - max_v for v in older[:max_v]: vid = v["id"] if (did, iid, vid) in done: continue target = f"{vdir}/{dd}/{sanitize(iid)}/{sanitize(vid)}" ok, err = False, "" for attempt in range(5): try: r = auth.g("GET", f"/drives/{did}/items/{iid}/versions/{vid}/content", allow_redirects=False) if r.status_code in (301, 302, 303, 307): url = r.headers["Location"] elif r.status_code == 200: url = None else: err = f"content {r.status_code}"; break os.makedirs(os.path.dirname(target), exist_ok=True); h = hashlib.sha256() if url is None: c = r.content; open(target + ".part", "wb").write(c); h.update(c) else: with requests.get(url, stream=True, timeout=120) as s: if s.status_code in (429, 500, 502, 503, 504): time.sleep(int(s.headers.get("Retry-After", 5 * (attempt + 1)))); continue s.raise_for_status() with open(target + ".part", "wb") as f: for c in s.iter_content(1 << 20): f.write(c); h.update(c) os.replace(target + ".part", target); ok = True; break except Exception as e: err = str(e)[:120]; time.sleep(3 * (attempt + 1)) if ok: rec = {"drive": did, "item": iid, "name": name, "version": vid, "size": os.path.getsize(target), "sha256": h.hexdigest(), "lastModifiedDateTime": v.get("lastModifiedDateTime"), "lastModifiedBy": (v.get("lastModifiedBy") or {}).get("user", {}).get("displayName"), "path": f"versions/{dd}/{sanitize(iid)}/{sanitize(vid)}"} with lock: open(manifest, "a").write(json.dumps(rec, ensure_ascii=False) + "\n") st["downloaded"] += 1; st["bytes"] += rec["size"] else: meta_error(t, f"version {vid}: {err}") counter = [0] def work_counted(t): work(t) with lock: counter[0] += 1 if counter[0] % 100 == 0: print(counter[0], st, flush=True) with ThreadPoolExecutor(args.threads) as ex: list(ex.map(work_counted, tasks)) auth.write_json(os.path.join(out, "versions_errors.json"), errors) auth.write_json(os.path.join(out, "versions_status.json"), {"max_older_versions_per_file": max_v, **st}) print("DONE", st) return 0 if __name__ == "__main__": sys.exit(main())