World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
"""抓合同链语料:按 edgar-manifest.json 从 SEC EDGAR 取原件。
一条链是一份原合同,加上后来改写它的补充协议、物业转让与终止协议。每份存原始 HTML,
旁边放一份纯文本便于阅读与比对。
**文件不进仓库。** 它们是各公司提交给 SEC 的文件,公开可取但版权不属于我们,与
`fetch-sec-filings.mjs` 抓的英伟达语料同一个处理:仓库里只有脚本和清单,跑一次就有。
用法:
SEC_USER_AGENT="你的名字 [email protected]" py -3 fetch.py [--with-redacted]
SEC 要求 User-Agent 带能联系上的地址,限速每秒 10 次;这个脚本每秒约 3 次。
CSG 与 Comcast 那条链金额涂黑、主协议约 4 MB,默认不抓,要 `--with-redacted`。
"""
import html
import json
import os
import re
import subprocess
import sys
import time
from pathlib import Path
HERE = Path(__file__).resolve().parent
OUT = HERE / "edgar"
UA = os.environ.get("SEC_USER_AGENT", "Utopia-bench ([email protected])")
def url_of(doc):
acc = doc["accession"].replace("-", "")
return f"https://www.sec.gov/Archives/edgar/data/{int(doc['cik'])}/{acc}/{doc['file']}"
def to_text(raw: bytes) -> str:
t = raw.decode("utf-8", "replace")
t = re.sub(r"(?is)<(script|style).*?</\1>", " ", t)
t = re.sub(r"(?i)<br\s*/?>|</(p|div|tr|li|h[1-6])>", "\n", t)
t = re.sub(r"<[^>]+>", " ", t)
t = html.unescape(t).replace("\xa0", " ")
t = re.sub(r"[ \t]+", " ", t)
return re.sub(r"\n\s*\n+", "\n\n", t).strip() + "\n"
def fetch(url: str) -> bytes:
# 走 curl 不走 urllib:与 fetch-sec-filings.mjs 同一个理由,本机的代理只有 curl 读
code = b""
for attempt in range(4):
r = subprocess.run(
["curl", "-sSL", "--max-time", "120", "-H", f"User-Agent: {UA}", "-w", "%{http_code}", url],
capture_output=True,
)
body, code = r.stdout[:-3], r.stdout[-3:]
if code == b"200" and body:
return body
time.sleep(3 + attempt * 5)
raise RuntimeError(f"failed after retries: {url} ({code!r})")
def main():
with_redacted = "--with-redacted" in sys.argv
manifest = json.loads((HERE / "edgar-manifest.json").read_text(encoding="utf-8"))
total = 0
for chain in manifest["chains"]:
if chain.get("redacted") and not with_redacted:
print(f"skip {chain['id']}(涂黑,要抓就加 --with-redacted)")
continue
folder = OUT / chain["id"]
folder.mkdir(parents=True, exist_ok=True)
for doc in chain["docs"]:
stem = f"{doc['seq']:02d}-{doc['role']}-{doc['filed']}"
dest = folder / f"{stem}.html"
if dest.exists() and dest.stat().st_size > 0:
total += 1
continue
body = fetch(url_of(doc))
dest.write_bytes(body)
(folder / f"{stem}.txt").write_text(to_text(body), encoding="utf-8")
total += 1
print(f"ok {chain['id']}/{stem} {len(body):>8} bytes")
time.sleep(0.35)
print(f"\n{total} 份文件在 {OUT}")
if __name__ == "__main__":
main()