"""Vollständigkeitsprüfung: Inventar gegen Manifest, dazu 20 Stichproben mit SHA-256 (nur lokal, kein Netz). Schreibt graph/completeness_check.json.""" import hashlib, json, os, random, re, sys sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import m365_auth as auth def sanitize(s): return re.sub(r'[\x00-\x1f<>:"|?*\\]', "_", s).strip() or "_" def main(argv=None): p = auth.make_parser("Check the downloaded files against the inventory and manifest (local only)") args = p.parse_args(argv) B = args.out inventory = json.load(open(B + "/graph/drives_inventory.json")) manifest = {} for line in open(B + "/manifest.jsonl"): j = json.loads(line); manifest[j["path"]] = j files, folders = [], 0 for did, dr in inventory.items(): items = {i["id"]: i for i in dr["items"]} root = next((i["id"] for i in dr["items"] if i.get("name") == "root" and "folder" in i), None) m = re.search(r"https://[^/]+/(?:sites|teams|personal)/([^/]+)", dr.get("webUrl") or "") site = sanitize(m.group(1) if m else (dr.get("webUrl") or did)[:40]); lib = sanitize(dr.get("name") or did[-8:]) def rel_path(i, items=items, root=root): parts = [] while i and i in items and i != root: parts.append(sanitize(items[i]["name"])); i = items[i].get("parentReference", {}).get("id") return "/".join(reversed(parts)) for it in dr["items"]: if "file" in it: files.append((f"{site}/{lib}/{rel_path(it['id'])}", it, did)) elif "folder" in it: folders += 1 res = {"inventory_entries": sum(len(v["items"]) for v in inventory.values()), "inventory_files": len(files), "inventory_folders_incl_root": folders, "manifest_lines": sum(1 for _ in open(B + "/manifest.jsonl")), "manifest_unique": len(manifest)} missing, size_mismatch, paths = [], [], {} for rel, it, did in files: paths.setdefault(rel, []).append(it["id"]) target = B + "/files/" + rel if rel not in manifest or not os.path.exists(target): missing.append(rel) elif os.path.getsize(target) != it.get("size"): size_mismatch.append((rel, "size", os.path.getsize(target), it.get("size"))) res["path_collisions"] = {k: v for k, v in paths.items() if len(v) > 1} res["missing"] = missing; res["size_mismatch"] = size_mismatch random.seed(16) sample = random.sample(files, min(20, len(files))); sres = [] for rel, it, did in sample: target = B + "/files/" + rel h = hashlib.sha256(open(target, "rb").read()).hexdigest() hashes = it["file"].get("hashes") or {} sres.append({"path": rel, "size_local": os.path.getsize(target), "size_inventory": it.get("size"), "sha256_local": h, "sha256_manifest": manifest.get(rel, {}).get("sha256"), "hash_in_inventory": hashes.get("sha256Hash") or hashes.get("quickXorHash"), "ok": os.path.getsize(target) == it.get("size") and h == manifest.get(rel, {}).get("sha256")}) res["sample"] = sres auth.write_json(B + "/graph/completeness_check.json", res) print({k: (v if not isinstance(v, (list, dict)) else len(v)) for k, v in res.items()}, "sample ok:", sum(s["ok"] for s in sres)) return 0 if __name__ == "__main__": sys.exit(main())