Skip to content

Apache DataFusion SQL Query Engine

7.8 KB
Raw Blame
// SPDX-License-Identifier: Apache-2.0
// SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements

//! See `main.rs` for how to run it.
//!
//! The same problem as `composed_extension_codec.rs`, solved without composing
//! codecs: two plans from two independent crates in one tree.
//!
//! ```text
//! ParentExec      (from crate A)
//!     ChildExec   (from crate B)
//! ```
//!
//! With `ComposedPhysicalExtensionCodec` the decoding side has to register the
//! same codecs in the same *order* the encoder used, because the composed codec
//! writes the position of the encoding codec into the payload. Here each plan
//! names itself on the wire instead, so order does not matter and a name two
//! crates both claim is an error at registration rather than a wrong decode.

use std::fmt::Formatter;
use std::sync::Arc;

use datafusion::common::tree_node::TreeNodeRecursion;
use datafusion::common::{Result, internal_err};
use datafusion::execution::TaskContext;
use datafusion::physical_expr::PhysicalExpr;
use datafusion::physical_plan::proto::{
    ExecutionPlanDecodeCtx, ExecutionPlanEncodeCtx, ExtensionPlanFromProto,
};
use datafusion::physical_plan::{
    ChildrenPropertiesMode, DisplayAs, DisplayFormatType, ExecutionPlan, PlanProperties,
    ReplaceChildrenOptions, SendableRecordBatchStream,
};
use datafusion::prelude::{SessionConfig, SessionContext};
use datafusion_proto::physical_plan::{AsExecutionPlan, DefaultPhysicalExtensionCodec};
use datafusion_proto::protobuf::PhysicalPlanNode;
use datafusion_proto::protobuf::physical_plan_node::PhysicalPlanType;

pub fn extension_plan_registry() -> Result<()> {
    let plan: Arc<dyn ExecutionPlan> = Arc::new(ParentExec {
        input: Arc::new(ChildExec {}),
    });

    // Each crate exposes a function that adds its decoders to the session's
    // registry. The two crates need to know nothing about each other, and the
    // call order does not matter.
    let mut config = SessionConfig::new();
    crate_a::register(&mut config)?;
    crate_b::register(&mut config)?;
    let ctx = SessionContext::new_with_config(config);

    // `DefaultPhysicalExtensionCodec` decodes nothing at all, so whatever
    // survives the round trip came from the registry.
    let codec = DefaultPhysicalExtensionCodec {};
    let node = PhysicalPlanNode::try_from_physical_plan(Arc::clone(&plan), &codec)?;
    let decoded = node.try_into_physical_plan(ctx.task_ctx().as_ref(), &codec)?;

    assert_eq!(format!("{plan:?}"), format!("{decoded:?}"));

    // A session that does not register the plans cannot decode them, and the
    // error names what is missing rather than only what the codec said. Children
    // are decoded before their parent falls back to the codec, so the innermost
    // unregistered plan is the one reported:
    //
    //     No decoder is registered for the extension ExecutionPlan
    //     'example-crate-b.ChildExec'. Register the plan in the
    //     ProtoDecoderRegistry attached to the decoding session's
    //     SessionConfig, ... Registered extension plans: none
    let bare = SessionContext::new();
    let err = node
        .try_into_physical_plan(bare.task_ctx().as_ref(), &codec)
        .expect_err("an unregistered plan must not decode");
    assert!(err.to_string().contains("No decoder is registered"));
    assert!(err.to_string().contains(ChildExec::NAME));

    Ok(())
}

/// What the crate that owns `ParentExec` would export.
mod crate_a {
    use super::*;

    pub fn register(config: &mut SessionConfig) -> Result<()> {
        config.update_extension(ParentExec::register_proto)
    }
}

/// What the crate that owns `ChildExec` would export.
mod crate_b {
    use super::*;

    pub fn register(config: &mut SessionConfig) -> Result<()> {
        config.update_extension(ChildExec::register_proto)
    }
}

#[derive(Debug)]
struct ParentExec {
    input: Arc<dyn ExecutionPlan>,
}

impl ExtensionPlanFromProto for ParentExec {
    // Namespaced with the owning crate, so a collision with another crate is
    // an error at registration rather than a wrong decode.
    const NAME: &'static str = "example-crate-a.ParentExec";

    fn try_from_proto(
        node: &PhysicalPlanNode,
        ctx: &ExecutionPlanDecodeCtx<'_>,
    ) -> Result<Arc<dyn ExecutionPlan>> {
        let Some(PhysicalPlanType::Extension(extension)) = &node.physical_plan_type
        else {
            return internal_err!("PhysicalPlanNode is not an Extension");
        };
        let mut children = ctx.decode_children(&extension.inputs)?;
        if children.len() != 1 {
            return internal_err!("ParentExec expects exactly one child");
        }
        Ok(Arc::new(ParentExec {
            input: children.remove(0),
        }))
    }
}

#[derive(Debug)]
struct ChildExec {}

impl ExtensionPlanFromProto for ChildExec {
    const NAME: &'static str = "example-crate-b.ChildExec";

    fn try_from_proto(
        _node: &PhysicalPlanNode,
        _ctx: &ExecutionPlanDecodeCtx<'_>,
    ) -> Result<Arc<dyn ExecutionPlan>> {
        Ok(Arc::new(ChildExec {}))
    }
}

/// Both plans are serde-only: this example never executes them.
macro_rules! serde_only_body {
    ($name:ident) => {
        fn name(&self) -> &str {
            stringify!($name)
        }

        fn properties(&self) -> &Arc<PlanProperties> {
            unreachable!("this example only serializes")
        }

        fn apply_expressions(
            &self,
            _f: &mut dyn FnMut(&Arc<dyn PhysicalExpr>) -> Result<TreeNodeRecursion>,
        ) -> Result<TreeNodeRecursion> {
            Ok(TreeNodeRecursion::Continue)
        }

        fn with_new_children(
            self: Arc<Self>,
            children: Vec<Arc<dyn ExecutionPlan>>,
        ) -> Result<Arc<dyn ExecutionPlan>> {
            self.replace_children(
                children,
                ReplaceChildrenOptions::new(ChildrenPropertiesMode::Recompute),
            )
        }

        fn execute(
            &self,
            _partition: usize,
            _context: Arc<TaskContext>,
        ) -> Result<SendableRecordBatchStream> {
            internal_err!("{} is a serde-only example plan", stringify!($name))
        }

        /// `extension_node` stamps `Self::NAME`, so the name on the wire and
        /// the registry key cannot drift apart. Neither plan has state of its
        /// own, hence the empty payload.
        fn try_to_proto(
            &self,
            ctx: &ExecutionPlanEncodeCtx<'_>,
        ) -> Result<Option<PhysicalPlanNode>> {
            Ok(Some(ctx.extension_node::<Self>(vec![], self.children())?))
        }
    };
}

impl DisplayAs for ParentExec {
    fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
        write!(f, "ParentExec")
    }
}

impl ExecutionPlan for ParentExec {
    serde_only_body!(ParentExec);

    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
        vec![&self.input]
    }

    fn replace_children(
        self: Arc<Self>,
        mut children: Vec<Arc<dyn ExecutionPlan>>,
        _: ReplaceChildrenOptions,
    ) -> Result<Arc<dyn ExecutionPlan>> {
        if children.len() != 1 {
            return internal_err!("ParentExec expects exactly one child");
        }
        Ok(Arc::new(ParentExec {
            input: children.remove(0),
        }))
    }
}

impl DisplayAs for ChildExec {
    fn fmt_as(&self, _t: DisplayFormatType, f: &mut Formatter) -> std::fmt::Result {
        write!(f, "ChildExec")
    }
}

impl ExecutionPlan for ChildExec {
    serde_only_body!(ChildExec);

    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
        vec![]
    }

    fn replace_children(
        self: Arc<Self>,
        _children: Vec<Arc<dyn ExecutionPlan>>,
        _: ReplaceChildrenOptions,
    ) -> Result<Arc<dyn ExecutionPlan>> {
        Ok(self)
    }
}