Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
"""在融合 E 基线上调度 N/H/W 归并实验,保留严格输出与 8 GiB 内存门槛。"""
from __future__ import annotations
import argparse
import json
import shutil
import statistics
from pathlib import Path
from typing import Any
from analyze_det_fusion import diagnose_differences, execute_run, load_runs
from benchmark_det_corpus import read_json
from benchmark_mps_det_service import write_json
from det_shape_buckets import SHAPE_STRATEGIES, project_shape
NAMES = {
"A": "CPU 原模型",
"E": "MPS 全融合,原 N/H/W",
"F": "MPS 全融合,拆为 N=1",
"G": "MPS 全融合,拆分并补齐 N=4",
"H": "MPS 全融合,H/W 补齐到 64 倍数",
"I": "MPS 全融合,H/W 补齐到 128 倍数",
}
def shape_distribution(args: argparse.Namespace) -> dict[str, Any]:
"""在执行前统计准确网络形状和计算量,避免只凭桶数量估计速度。"""
documents = read_json(args.corpus / "manifest.json")["documents"]
result = {}
for policy in ("E", *SHAPE_STRATEGIES):
rows = []
for document in documents:
projected = [project_shape(call["shape"], policy) for call in document["calls"]]
shapes = {tuple(shape) for item in projected for shape in item["network_shapes"]}
rows.append(
{
"document": document["name"],
"unique_shapes": len(shapes),
"forward_calls": sum(len(item["network_shapes"]) for item in projected),
"processed_pixels": sum(item["processed_pixels"] for item in projected),
"original_pixels": sum(item["original_pixels"] for item in projected),
}
)
result[policy] = rows
return result
def collect_shape_summary(args: argparse.Namespace) -> dict[str, Any]:
"""使用同轮 A/E 配对验收新策略,烟测失败或内存超预算都不算有效收益。"""
runs = load_runs(args.output)
smoke = [entry for entry in runs if entry["directory"].startswith("smoke")]
formal = [entry for entry in runs if entry["directory"].startswith("round-")]
checks = {}
required = {
(call["box_thresh"], call["unclip_ratio"])
for document in read_json(args.corpus / "manifest.json")["documents"]
for call in document["calls"]
}
for policy in NAMES:
rows = [row for entry in smoke if entry["policy"] == policy for row in entry["records"]]
profiles = {tuple(model["profile"]) for row in rows for model in row["models"]}
checks[policy] = {
"inputs": sum(row["crops"] for row in rows),
"differences": sum(len(row["differences"]) for row in rows),
"exact": bool(rows) and all(row["reference_exact_crops"] == row["crops"] for row in rows),
"profiles_complete": profiles == required,
"det_seconds": sum(row["seconds"] for row in rows),
}
fingerprints = {
(
entry["environment"]["model_sha256"],
entry["environment"]["torch_commit"],
entry["environment"]["python"],
entry["environment"]["threads"],
entry["environment"]["interop_threads"],
entry["environment"]["git_head"],
entry["environment"]["script_sha256"],
entry["environment"]["shape_adapter_sha256"],
)
for entry in runs
}
matched = len(fingerprints) == 1
fp32 = all(model["dtype"] == "torch.float32" for entry in runs for row in entry["records"] for model in row["models"])
reuse = all(
len({model["net_id"] for row in entry["records"] for model in row["models"] if tuple(model["profile"]) == profile}) == 1
for entry in runs
for profile in {tuple(model["profile"]) for row in entry["records"] for model in row["models"]}
)
rounds: dict[str, dict[str, Any]] = {}
for entry in formal:
protocol = entry["protocol"]
if (
protocol["documents"] != "tender,python,114"
or protocol["cycles"] != 1
or protocol["limit_calls"]
or protocol["start_call"]
or protocol["call_indices"]
or protocol["save_maps"]
):
raise ValueError("正式形状对照必须完整单遍重放,不能混入诊断或生命周期数据")
env = entry["environment"]
rows = entry["records"]
loading = sum(item["loading_seconds"] for item in env["model_loads"])
fusion = sum(item["fusion"]["seconds"] for item in env["model_loads"])
cleanup = sum(row["cleanup_seconds"] for row in rows) + env["shutdown_seconds"]
det = sum(row["seconds"] for row in rows)
metrics = [metric for row in rows for metric in row["call_metrics"]]
rounds.setdefault(entry["directory"], {})[entry["policy"]] = {
"det_seconds": det,
"loading_seconds": loading,
"fusion_seconds": fusion,
"cleanup_seconds": cleanup,
"operational_seconds": det + loading + fusion + cleanup,
"peak_footprint": max(
*(row["peak_memory_diagnostics"]["phys_footprint"] for row in rows),
env["shutdown_memory_diagnostics_before"]["phys_footprint"],
env["shutdown_memory_diagnostics_after"]["phys_footprint"],
),
"exact": all(row["reference_exact_crops"] == row["crops"] for row in rows),
"forward_calls": sum(len(metric["network_shapes"]) for metric in metrics),
"pixel_ratio": sum(metric["shape_projection"]["processed_pixels"] for metric in metrics)
/ sum(metric["shape_projection"]["original_pixels"] for metric in metrics),
"bucket_padding_seconds": sum(metric["bucket_padding_seconds"] for metric in metrics),
}
baseline = all(checks[policy]["exact"] and checks[policy]["profiles_complete"] for policy in ("A", "E"))
baseline &= all(value["exact"] for group in rounds.values() for policy, value in group.items() if policy in ("A", "E"))
candidates = {}
for policy in SHAPE_STRATEGIES:
paired = [group for group in rounds.values() if {"A", "E", policy}.issubset(group)]
if not paired:
continue
seconds = statistics.median(group[policy]["det_seconds"] for group in paired)
cpu = statistics.median(group["A"]["det_seconds"] for group in paired)
original = statistics.median(group["E"]["det_seconds"] for group in paired)
peak = max(group[policy]["peak_footprint"] for group in paired)
exact = checks[policy]["exact"] and all(group[policy]["exact"] for group in paired)
gains = [1 - group[policy]["det_seconds"] / group["E"]["det_seconds"] for group in paired]
reduction = 1 - seconds / original
cpu_reduction = 1 - seconds / cpu
eligible = matched and fp32 and reuse and baseline and exact and peak <= 8 * 1024**3
candidates[policy] = {
"rounds": len(paired),
"median_det_seconds": seconds,
"matched_e_seconds": original,
"matched_cpu_seconds": cpu,
"reduction_vs_e": reduction,
"reduction_vs_a": cpu_reduction,
"round_gains": gains,
"peak_footprint": peak,
"outputs_exact": exact,
"eligible": eligible,
"research_pass": eligible
and len(paired) >= 2
and reduction >= 0.25
and sum(gain > 0 for gain in gains) > len(gains) / 2,
"repeat_needed": eligible
and (
reduction < 0.1
or (any(gain <= 0 for gain in gains) and any(gain > 0 for gain in gains))
or abs(reduction - 0.25) < 0.05
or abs(cpu_reduction - 0.25) < 0.05
),
"lifecycle_pass": None,
"default_candidate": False,
}
for entry in runs:
if entry["directory"] != "lifecycle":
continue
policy = entry["policy"]
rows = entry["records"]
if len(rows) != 6 or policy not in candidates:
raise ValueError("生命周期要求候选连续两遍三文档")
before = rows[2]["memory_after"] | {"footprint": rows[2]["memory_diagnostics_after"]["phys_footprint"]}
after = rows[5]["memory_after"] | {"footprint": rows[5]["memory_diagnostics_after"]["phys_footprint"]}
growth = {
key: {
"delta": after[key] - before[key],
"limit": max(before[key] * 0.1, 256 * 1024**2),
"first": before[key],
"second": after[key],
}
for key in before
}
peak = max(row["peak_memory_diagnostics"]["phys_footprint"] for row in rows)
passed = (
all(value["delta"] <= value["limit"] for value in growth.values())
and all(row["crops"] == row["reference_exact_crops"] for row in rows)
and peak <= 8 * 1024**3
)
candidates[policy].update(
{
"lifecycle_pass": passed,
"lifecycle_peak_footprint": peak,
"lifecycle_growth": growth,
"default_candidate": passed
and candidates[policy]["research_pass"]
and candidates[policy]["reduction_vs_a"] >= 0.25,
}
)
return {
"smoke": checks,
"rounds": rounds,
"candidates": candidates,
"baseline_exact": baseline,
"matched_environment": matched,
"fp32": fp32,
"models_reused": reuse,
"runs": runs,
}
def shape_experiment(args: argparse.Namespace) -> None:
"""按烟测、正反两轮、必要复测和生命周期顺序验收形状归并,输出失败者仅保留诊断证据。"""
if args.documents != "tender,python,114" or args.start_call or args.limit_calls or args.call_indices or args.save_maps:
raise ValueError("自动形状实验使用固定完整语料,定点调用请使用 run")
if args.output.exists() and any(args.output.iterdir()):
raise FileExistsError("请用全新输出目录保留上一轮证据")
args.output.mkdir(parents=True, exist_ok=True)
source = args.output / "executed-scripts"
source.mkdir()
for name in (
"benchmark_det_corpus.py",
"benchmark_mps_det_service.py",
"analyze_det_fusion.py",
"analyze_det_shapes.py",
"det_shape_buckets.py",
):
shutil.copy2(Path(__file__).parent / name, source / name)
write_json(args.output / "shape-distribution.json", shape_distribution(args))
state: dict[str, Any] = {"phase": "smoke", "footprint_budget": 8 * 1024**3, "baseline": "E", "corpus": str(args.corpus)}
write_json(args.output / "experiment.json", state)
execute_run(args, "smoke", "A,E,F,G,H,I", limit_calls=12)
summary = collect_shape_summary(args)
if not all(value["profiles_complete"] for value in summary["smoke"].values()):
manifest = read_json(args.corpus / "manifest.json")
seen = {tuple(model["profile"]) for entry in summary["runs"] for row in entry["records"] for model in row["models"]}
missing = {(call["box_thresh"], call["unclip_ratio"]) for doc in manifest["documents"] for call in doc["calls"]} - seen
for number, profile in enumerate(sorted(missing)):
doc, index = next(
(doc["name"], index)
for doc in manifest["documents"]
for index, call in enumerate(doc["calls"])
if (call["box_thresh"], call["unclip_ratio"]) == profile
)
execute_run(args, f"smoke-profile-{number + 1}", "A,E,F,G,H,I", documents=doc, call_indices=str(index))
summary = collect_shape_summary(args)
candidates = [
policy
for policy in SHAPE_STRATEGIES
if summary["smoke"][policy]["exact"] and summary["smoke"][policy]["profiles_complete"]
]
if not summary["baseline_exact"]:
state["phase"] = "baseline_rejected"
elif not candidates:
state["phase"] = "candidates_rejected_at_smoke"
else:
state.update({"phase": "formal", "formal_candidates": candidates})
write_json(args.output / "experiment.json", state)
policies = ["A", "E", *candidates]
execute_run(args, "round-1", ",".join(policies))
execute_run(args, "round-2", ",".join(reversed(policies)))
summary = collect_shape_summary(args)
valid = [policy for policy, value in summary["candidates"].items() if value["eligible"]]
best = min(valid, key=lambda policy: summary["candidates"][policy]["median_det_seconds"]) if valid else None
if best and summary["candidates"][best]["repeat_needed"]:
execute_run(args, "round-3", f"A,E,{best}")
summary = collect_shape_summary(args)
passed = [policy for policy, value in summary["candidates"].items() if value["research_pass"]]
if passed:
best = min(passed, key=lambda policy: summary["candidates"][policy]["median_det_seconds"])
state["lifecycle_policy"] = best
execute_run(args, "lifecycle", best, cycles=2)
state["phase"] = "complete"
write_json(args.output / "experiment.json", state)
diagnose_differences(args, baseline_policies=("A", "E"))
shape_summarize(args)
def shape_summarize(args: argparse.Namespace) -> None:
"""保存形状收益、真实算量、框差异和内存门槛,区分烟测淘汰与正式性能结论。"""
summary = collect_shape_summary(args)
summary["experiment"] = read_json(args.output / "experiment.json")
summary["distribution"] = read_json(args.output / "shape-distribution.json")
summary["probability_diagnostics"] = (
read_json(args.output / "probability-diagnostics.json")
if (args.output / "probability-diagnostics.json").exists()
else []
)
write_json(args.output / "summary.json", summary)
smoke_rows = []
for policy, value in summary["smoke"].items():
smoke_rows.append(
f"| {policy} | {NAMES[policy]} | {value['inputs']} | {value['differences']} | "
f"{value['exact']} | {value['det_seconds']:.3f} |"
)
distribution_rows = []
for policy, rows in summary["distribution"].items():
for row in rows:
distribution_rows.append(
f"| {policy} | {row['document']} | {row['unique_shapes']} | {row['forward_calls']} | "
f"{row['processed_pixels'] / row['original_pixels']:.3f} |"
)
performance = []
for round_name, group in summary["rounds"].items():
for policy, value in group.items():
performance.append(
f"| {round_name} | {policy} | {value['det_seconds']:.3f} | {value['operational_seconds']:.3f} | "
f"{value['peak_footprint'] / 1024**3:.2f} | {value['forward_calls']} | "
f"{value['pixel_ratio']:.3f} | {value['exact']} |"
)
gates = []
for policy, value in summary["candidates"].items():
gates.append(
f"| {policy} | {value['median_det_seconds']:.3f} | {value['reduction_vs_e']:.1%} | {value['reduction_vs_a']:.1%} | "
f"{value['peak_footprint'] / 1024**3:.2f} | {value['outputs_exact']} | {value['research_pass']} | "
f"{value['lifecycle_pass']} | {value['default_candidate']} |"
)
document_rows = []
for entry in summary["runs"]:
for row in entry["records"]:
document_rows.append(
f"| {entry['directory']} | {entry['policy']} | {row['label']} | {row['seconds']:.3f} | "
f"{len(row['shape_counts'])} | {row['peak_memory_diagnostics']['phys_footprint'] / 1024**3:.2f} | "
f"{row['reference_exact_crops']}/{row['crops']} |"
)
env = summary["runs"][0]["environment"]
failed = [policy for policy in SHAPE_STRATEGIES if not summary["smoke"][policy]["exact"]]
conclusions = []
for policy, value in summary["candidates"].items():
conclusions.append(
f"{policy}:det 中位数 {value['median_det_seconds']:.2f} 秒,相对 E 降低 {value['reduction_vs_e']:.1%},"
f"采样峰 footprint {value['peak_footprint'] / 1024**3:.2f} GiB,框一致 {value['outputs_exact']}。"
)
if value["lifecycle_pass"] is False:
growth = value["lifecycle_growth"]
conclusions.append(
f"{policy} 生命周期未通过:结束 footprint 从 {growth['footprint']['first'] / 1024**3:.2f} "
f"增至 {growth['footprint']['second'] / 1024**3:.2f} GiB,"
f"RSS 增长 {growth['rss']['delta'] / 1024**3:.2f} GiB;allocated/driver 的增长见 summary.json。"
)
lifetime_rows = []
for entry in summary["runs"]:
if entry["directory"] == "lifecycle":
for row in entry["records"]:
diagnostic = row["memory_diagnostics_after"]
lifetime_rows.append(
f"| {entry['policy']} | {row['label']} | {row['seconds']:.3f} | "
f"{row['peak_memory_diagnostics']['phys_footprint'] / 1024**3:.2f} | "
f"{diagnostic['phys_footprint'] / 1024**3:.2f} | {row['memory_after']['rss'] / 1024**3:.2f} | "
f"{row['memory_after']['mps_driver'] / 1024**3:.3f} |"
)
difference_counts = {}
unchanged_spatial_differences = 0
for entry in summary["runs"]:
for row in entry["records"]:
by_call = {metric["call_index"]: metric for metric in row["call_metrics"]}
for difference in row["differences"]:
counter = difference_counts.setdefault(entry["policy"], {"inputs": 0, "box_count_changes": 0})
counter["inputs"] += 1
counter["box_count_changes"] += len(difference["reference"] or []) != len(difference["actual"] or [])
if entry["policy"] in ("H", "I"):
metric = by_call[difference["call_index"]]
unchanged_spatial_differences += metric["shape"][2:] == metric["network_shapes"][0][2:]
write_json(args.output / "difference-counts.json", difference_counts)
count_description = (
";".join(
f"{policy} 有 {value['inputs']} 个差异输入,其中 {value['box_count_changes']} 个改变框数量"
for policy, value in difference_counts.items()
)
or "没有检测框差异"
)
defaults = [policy for policy, value in summary["candidates"].items() if value["default_candidate"]]
default_description = (
f"下一轮默认策略验证候选:{defaults}。" if defaults else "本轮没有满足默认策略要求的候选,保持 CPU 默认。"
)
report = f"""# 融合 E 基线上的 det 形状归并
{chr(10).join(conclusions)}
状态:{summary["experiment"]["phase"]}。烟测框一致性失败的候选:{failed},这些策略不进入正式性能验收。
仅改实验脚本;使用冻结的 2280 个普通文字 det 输入,不再解析 PDF。生产默认和原权重不变。
沿用完全一致的检测框门槛与 8 GiB 采样峰 footprint 预算;研究收益要求相对 E 至少降低 25%。
全部策略独立进程串行,正式两轮顺序互逆,每本文档执行原 MPS 清理;不保留跨文档图缓存、不预热。
环境一致:{summary["matched_environment"]},FP32:{summary["fp32"]},模型复用:{summary["models_reused"]}。
{env["cpu"]} / {env["platform"]} / Python {env["python"]} / Torch {env["torch"]}。
intra/inter-op {env["threads"]}/{env["interop_threads"]}。
原权重 SHA256:`{env["model_sha256"]}`。实际执行源码保存在 executed-scripts/。
## 实现和烟测
E/F/G/H/I 都融合同样的 36 组 Conv/BN,A 为 CPU 原模型。形状阶段的对照是 E,不将上一轮融合收益计为归并收益。
保留原 resize、归一化与原始坐标信息;F/G 只改变 N,G 用最后一个真实样本补齐并丢弃额外输出。
H/I 在归一化后的 CPU 输入右侧/底部补 0,不改变原图像素;先将输出回传 CPU,再裁切到原 H/W,最后执行原 DB 后处理。
Tiny 模型包含全局池化和 SE,裁切输出不能保证空间补齐等价;通过逐项检测框比较裁决。
记录原调用与实际网络形状、拆批次数、实际像素算量和补齐耗时(已计入 det)。原调用输出顺序不变。
| 策略 | 方案 | 烟测输入 | 差异输入 | 框一致 | 烟测 det 秒 |
| --- | --- | ---: | ---: | --- | ---: |
{chr(10).join(smoke_rows)}
## 全量形状预估
| 策略 | 文档 | 唯一 NCHW | forward 次数 | 像素算量相对 E |
| --- | --- | ---: | ---: | ---: |
{chr(10).join(distribution_rows)}
## 正式性能与验收
| 轮次 | 策略 | det 秒 | 加载+融合+det+清理秒 | 峰 footprint GiB | forward 次数 | 像素算量比 | 框一致 |
| --- | --- | ---: | ---: | ---: | ---: | ---: | --- |
{chr(10).join(performance) if performance else "| 未执行 | — | — | — | — | — | — | — |"}
| 候选 | det 中位数秒 | 相对 E 降低 | 相对 A 降低 | 峰 footprint GiB | 框一致 | 研究门槛 | 生命周期 | 默认策略候选 |
| --- | ---: | ---: | ---: | ---: | --- | --- | --- | --- |
{chr(10).join(gates) if gates else "| 无合格候选 | — | — | — | — | False | False | 未执行 | False |"}
速度取独立轮次三文档合计中位数;研究门槛为输出严格一致、≤8 GiB、相对 E 降低 ≥25% 且多数轮次同向。
通过者另作同进程两遍三文档生命周期验证;清理后 RSS/footprint/allocated/driver 增长均须 ≤max(10%,256 MiB)。
相对 CPU A 降低 ≥25% 且通过生命周期才可成为下一轮默认策略候选。
{default_description} 生命周期的内存增长不能直接等同于已确认泄漏。
## 生命周期
| 策略 | 文档 | det 秒 | 峰 footprint GiB | 清理后 footprint GiB | 清理后 RSS GiB | 清理后 driver GiB |
| --- | --- | ---: | ---: | ---: | ---: | ---: |
{chr(10).join(lifetime_rows) if lifetime_rows else "| 未执行 | — | — | — | — | — | — |"}
## 逐文档记录和差异证据
| 运行 | 策略 | 文档 | det 秒 | 实际唯一 NCHW | 峰 footprint GiB | 框一致输入 |
| --- | --- | --- | ---: | ---: | ---: | --- |
{chr(10).join(document_rows)}
所有差异的原 PNG、坐标和叠框位于各策略 review/ 与 results.json;CPU、E 及涉及候选的差异调用概率图保存在 diagnostics/。
probability-diagnostics.json 给出概率图最大/平均误差和跨 0.3 阈值像素数,仅用于诊断,不能放宽框一致性门槛。
difference-counts.json 另区分坐标变化与框数量变化:{count_description}。
H/I 未改变 H/W 的输入发生框差异 {unchanged_spatial_differences} 次;裁切最终概率图不能保证消除 padding 的影响。
det 包含真实前处理、补齐/拆批、张量传输、同步网络执行、输出恢复及后处理。读 PNG、写结果、采样与清理在外。
采样峰值不是内部瞬时峰值;footprint 包含压缩页记账,与 RSS/driver 重叠。全机 swap 不能全部归属 det。
仅代表 det 阶段,不推导完整 Basic/OCR 或 Standard/VLM。
```bash
.venv4/bin/python tools/benchmark_det_corpus.py shape-experiment --output NEW_OUTPUT
.venv4/bin/python tools/benchmark_det_corpus.py shape-summarize --output {args.output}
```

"""
(args.output / "report.md").write_text(report, encoding="utf-8")
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
fig, axes = plt.subplots(1, 2, figsize=(13, 4), constrained_layout=True)
plotted_phase = "round-1" if summary["rounds"] else "smoke"
for policy in NAMES:
entries = [entry for entry in summary["runs"] if entry["policy"] == policy and entry["directory"] == plotted_phase]
if not entries:
continue
chosen = entries[0]
rows = chosen["records"]
axes[0].plot(range(1, len(rows) + 1), [row["seconds"] for row in rows], "o-", label=f"{policy} ({chosen['directory']})")
samples = [metric["memory_diagnostics"]["phys_footprint"] / 1024**3 for row in rows for metric in row["call_metrics"]]
axes[1].plot(range(1, len(samples) + 1), samples, label=policy)
axes[0].set(xlabel="Document replay index", ylabel="det seconds", title=f"det performance: {plotted_phase}")
axes[1].set(xlabel="Original call index", ylabel="Sampled footprint GiB", title="Memory accounting")
axes[1].axhline(8, color="red", linestyle="--", label="8 GiB budget")
for axis in axes:
axis.legend(fontsize=8)
axis.grid(alpha=0.25)
fig.savefig(args.output / "performance-memory.png", dpi=160)
plt.close(fig)
print(
json.dumps(
{"phase": summary["experiment"]["phase"], "smoke": summary["smoke"], "candidates": summary["candidates"]},
ensure_ascii=False,
indent=2,
),
flush=True,
)