Apache DataFusion SQL Query Engine
doc: Add an auto-generated dependency graph for internal crates (#19280)
Yongting You567ba7584049
diff --git a/.github/workflows/dependencies.yml b/.github/workflows/dependencies.ymlindex fef65870b6..f32eb7d2dd 100644--- a/.github/workflows/dependencies.yml+++ b/.github/workflows/dependencies.yml@@ -66,4 +66,4 @@ jobs:- name: Install cargo-macheterun: cargo install cargo-machete --version ^0.9 --locked- name: Detect unused dependencies- run: cargo machete --with-metadata\ No newline at end of file+ run: cargo machete --with-metadatadiff --git a/.github/workflows/docs.yaml b/.github/workflows/docs.yamlindex 3e2c48643c..b62055b13b 100644--- a/.github/workflows/docs.yaml+++ b/.github/workflows/docs.yaml@@ -51,6 +51,12 @@ jobs:python3 -m venv venvsource venv/bin/activatepip install -r docs/requirements.txt+ - name: Install dependency graph tooling+ run: |+ set -x+ sudo apt-get update+ sudo apt-get install -y graphviz+ cargo install cargo-depgraph --version ^1.6 --locked- name: Build docsrun: |diff --git a/.github/workflows/docs_pr.yaml b/.github/workflows/docs_pr.yamlindex 81eeb4039b..784a33d4c5 100644--- a/.github/workflows/docs_pr.yaml+++ b/.github/workflows/docs_pr.yaml@@ -54,10 +54,15 @@ jobs:python3 -m venv venvsource venv/bin/activatepip install -r docs/requirements.txt+ - name: Install dependency graph tooling+ run: |+ set -x+ sudo apt-get update+ sudo apt-get install -y graphviz+ cargo install cargo-depgraph --version ^1.6 --locked- name: Build docs html and check for warningsrun: |set -xsource venv/bin/activatecd docs./build.sh # fails on errors-diff --git a/docs/.gitignore b/docs/.gitignoreindex a3adddc690..e73866cc0f 100644--- a/docs/.gitignore+++ b/docs/.gitignore@@ -20,3 +20,7 @@ build/venv/.python-version__pycache__/++# Generated dependency graph artifacts (produced during docs CI)+source/_static/data/deps.dot+source/_static/data/deps.svgdiff --git a/docs/README.md b/docs/README.mdindex c3d87ee8e8..0340a3b8bf 100644--- a/docs/README.md+++ b/docs/README.md@@ -40,6 +40,11 @@ needing to create a virtual environment:uv run --with-requirements requirements.txt bash build.sh```+The docs build regenerates the workspace dependency graph via+`docs/scripts/generate_dependency_graph.sh`, so ensure `cargo`, `cargo-depgraph`+(`cargo install cargo-depgraph --version ^1.6 --locked`), and Graphviz `dot`+(`brew install graphviz` or `sudo apt-get install -y graphviz`) are available.+## Build & PreviewRun the provided script to build the HTML pages.diff --git a/docs/build.sh b/docs/build.shindex 9e4a118580..e12e3c1a5f 100755--- a/docs/build.sh+++ b/docs/build.sh@@ -18,7 +18,14 @@# under the License.#-set -e+set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+cd "${SCRIPT_DIR}"+rm -rf build 2> /dev/null+# Keep the workspace dependency graph in sync with the codebase.+scripts/generate_dependency_graph.sh+make htmldiff --git a/docs/scripts/generate_dependency_graph.sh b/docs/scripts/generate_dependency_graph.shnew file mode 100755index 0000000000..771f6f1932--- /dev/null+++ b/docs/scripts/generate_dependency_graph.sh@@ -0,0 +1,97 @@+#!/usr/bin/env bash+#+# Licensed to the Apache Software Foundation (ASF) under one+# or more contributor license agreements. See the NOTICE file+# distributed with this work for additional information+# regarding copyright ownership. The ASF licenses this file+# to you under the Apache License, Version 2.0 (the+# "License"); you may not use this file except in compliance+# with the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing,+# software distributed under the License is distributed on an+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+# KIND, either express or implied. See the License for the+# specific language governing permissions and limitations+# under the License.++# See `usage()` for details about this script.+#+# The key commands to generate the dependency graph SVG in this script are:+# cargo depgraph ... | dot -Tsvg > deps.svg+# See below for the exact command used.++set -euo pipefail++SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO_DIR="$(cd "${SCRIPT_DIR}/../.." && pwd)"+OUTPUT_DIR="${REPO_DIR}/docs/source/_static/data"+SVG_OUTPUT="${OUTPUT_DIR}/deps.svg"++usage() {+ cat <<EOF+Generate the workspace dependency graph SVG for the docs.++'deps.svg' is embedded in the DataFusion docs (Contributor Guide → Architecture → Workspace Dependency Graph).++Output:+ SVG: ${SVG_OUTPUT}++Usage: $(basename "$0")++Options:+ -h, --help Show this help message.+EOF+}++while [[ $# -gt 0 ]]; do+ case "$1" in+ -h|--help)+ usage+ exit 0+ ;;+ *)+ echo "Unknown option: $1" >&2+ usage+ exit 1+ ;;+ esac+ shift+done++if ! command -v cargo >/dev/null 2>&1; then+ echo "cargo is required to build the dependency graph." >&2+ exit 1+fi++if ! command -v cargo-depgraph > /dev/null 2>&1; then+ echo "cargo-depgraph is required (install with: cargo install cargo-depgraph)." >&2+ exit 1+fi++if ! command -v dot >/dev/null 2>&1; then+ echo "Graphviz 'dot' is required to render the SVG." >&2+ exit 1+fi++mkdir -p "${OUTPUT_DIR}"++(+ cd "${REPO_DIR}"+ # Ignore utility crates only used by internal scripts+ cargo depgraph \+ --workspace-only \+ --all-deps \+ --dedup-transitive-deps \+ --exclude gen,gen-common \+ | dot \+ -Grankdir=TB \+ -Gconcentrate=true \+ -Goverlap=false \+ -Tsvg \+ > "${SVG_OUTPUT}"+)++echo "Wrote dependency graph SVG to ${SVG_OUTPUT}"diff --git a/docs/source/contributor-guide/architecture/dependency-graph.md b/docs/source/contributor-guide/architecture/dependency-graph.mdnew file mode 100644index 0000000000..be3502f48b--- /dev/null+++ b/docs/source/contributor-guide/architecture/dependency-graph.md@@ -0,0 +1,180 @@+<!---+ Licensed to the Apache Software Foundation (ASF) under one+ or more contributor license agreements. See the NOTICE file+ distributed with this work for additional information+ regarding copyright ownership. The ASF licenses this file+ to you under the Apache License, Version 2.0 (the+ "License"); you may not use this file except in compliance+ with the License. You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing,+ software distributed under the License is distributed on an+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY+ KIND, either express or implied. See the License for the+ specific language governing permissions and limitations+ under the License.+-->++# Workspace Dependency Graph++This page shows the dependency relationships between DataFusion's workspace+crates. This only includes internal dependencies, external crates like `Arrow` are not included++The dependency graph is auto-generated by `docs/scripts/generate_dependency_graph.sh` to ensure it stays up-to-date, and the script now runs automatically as part of `docs/build.sh`.++## Dependency Graph for Workspace Crates++<!--+ Below is an embedded .svg file, with interactive functionalities like drag/zoom-in/etc.+ -->++```{raw} html+<div id="workspace-deps-wrapper" style="border:1px solid #d4d4d8;border-radius:10px;overflow:hidden;background:#fff;">+ <div id="workspace-deps-inline" style="min-height:760px;width:100%;background:#f8fafc;overflow:hidden;padding:0;margin:0;">+```++```{eval-rst}+.. raw:: html+ :file: ../../_static/data/deps.svg+```++```{raw} html+ </div>+ <div style="padding:10px 12px;background:#f1f5f9;border-top:1px solid #e5e7eb;display:flex;justify-content:space-between;align-items:center;flex-wrap:wrap;gap:8px;">+ <span style="color:#334155;font-size:0.95rem;">Interactive SVG (pan, zoom, search)</span>+ <div style="display:flex;align-items:center;gap:6px;">+ <button id="workspace-deps-zoom-out" type="button" style="padding:6px 10px;border:1px solid #cbd5e1;border-radius:6px;background:#fff;color:#334155;cursor:pointer;">−</button>+ <button id="workspace-deps-zoom-in" type="button" style="padding:6px 10px;border:1px solid #cbd5e1;border-radius:6px;background:#fff;color:#334155;cursor:pointer;">+</button>+ </div>+ <a href="../../_static/data/deps.svg" target="_blank" rel="noopener"+ style="font-weight:600;color:#2563eb;text-decoration:none;">Open SVG ↗</a>+ </div>+</div>+<script>+ (function () {+ const host = document.getElementById("workspace-deps-inline");+ if (!host) {+ return;+ }++ const svg = host.querySelector("svg");+ if (!svg) {+ host.textContent = "Unable to load dependency graph.";+ host.style.display = "flex";+ host.style.alignItems = "center";+ host.style.justifyContent = "center";+ host.style.background = "#f8fafc";+ return;+ }++ svg.removeAttribute("width");+ svg.removeAttribute("height");+ svg.style.width = "100%";+ svg.style.height = "100%";+ svg.style.cursor = "grab";+ svg.style.touchAction = "none";++ const rawViewBox = (svg.getAttribute("viewBox") || "").split(/\s+/).map(Number);+ if (rawViewBox.length !== 4 || rawViewBox.some((v) => Number.isNaN(v))) {+ return;+ }++ const initial = {+ x: rawViewBox[0],+ y: rawViewBox[1],+ width: rawViewBox[2],+ height: rawViewBox[3],+ };++ const state = { ...initial };+ const applyViewBox = () => {+ svg.setAttribute("viewBox", `${state.x} ${state.y} ${state.width} ${state.height}`);+ };++ let isPanning = false;+ let last = { x: 0, y: 0 };++ svg.addEventListener("pointerdown", (event) => {+ isPanning = true;+ last = { x: event.clientX, y: event.clientY };+ svg.setPointerCapture(event.pointerId);+ svg.style.cursor = "grabbing";+ });++ const endPan = (event) => {+ if (event && svg.hasPointerCapture(event.pointerId)) {+ svg.releasePointerCapture(event.pointerId);+ }+ isPanning = false;+ svg.style.cursor = "grab";+ };++ svg.addEventListener("pointerup", endPan);+ svg.addEventListener("pointerleave", endPan);+ svg.addEventListener("pointercancel", endPan);++ const zoomBy = (factor) => {+ const targetWidth = state.width * factor;+ const targetHeight = state.height * factor;+ const minSize = Math.max(initial.width * 0.05, 10);+ const maxSize = initial.width * 20;+ const clampedWidth = Math.min(Math.max(targetWidth, minSize), maxSize);+ const clampedHeight = Math.min(Math.max(targetHeight, minSize), maxSize);++ state.x += (state.width - clampedWidth) / 2;+ state.y += (state.height - clampedHeight) / 2;+ state.width = clampedWidth;+ state.height = clampedHeight;+ applyViewBox();+ };++ const normalizeDelta = (deltaY, deltaMode) => {+ // Make trackpad/wheel zoom feel smooth across devices.+ const multiplier = deltaMode === 1 ? 16 : deltaMode === 2 ? window.innerHeight : 1;+ return deltaY * multiplier;+ };++ svg.addEventListener("pointermove", (event) => {+ if (!isPanning) {+ return;+ }+ const scaleX = state.width / svg.clientWidth;+ const scaleY = state.height / svg.clientHeight;+ state.x -= (event.clientX - last.x) * scaleX;+ state.y -= (event.clientY - last.y) * scaleY;+ last = { x: event.clientX, y: event.clientY };+ applyViewBox();+ });++ svg.addEventListener("wheel", (event) => {+ event.preventDefault();++ const delta = normalizeDelta(event.deltaY, event.deltaMode);+ const factor = Math.exp(delta * 0.0015); // smaller magnitude for smoother scrolling+ zoomBy(factor);+ }, { passive: false });++ const zoomIn = document.getElementById("workspace-deps-zoom-in");+ const zoomOut = document.getElementById("workspace-deps-zoom-out");+ if (zoomIn) {+ zoomIn.addEventListener("click", () => zoomBy(0.9));+ }+ if (zoomOut) {+ zoomOut.addEventListener("click", () => zoomBy(1.1));+ }+ })();+</script>+```++### Legend++- black lines: normal dependency+- blue lines: dev-dependency+- green lines: build-dependency+- dotted lines: optional dependency (could be removed by disabling a cargo feature)++Transitive dependencies are intentionally ignored to keep the graph readable.++The dependency graph is generated through `cargo depgraph` by `docs/scripts/generate_dependency_graph.sh`.diff --git a/docs/source/index.rst b/docs/source/index.rstindex 181d54a664..ae210de099 100644--- a/docs/source/index.rst+++ b/docs/source/index.rst@@ -159,6 +159,7 @@ To get started, seecontributor-guide/communicationcontributor-guide/development_environmentcontributor-guide/architecture+ contributor-guide/architecture/dependency-graphcontributor-guide/testingcontributor-guide/api-healthcontributor-guide/howtos