Skip to content

SeaweedFS is a distributed storage system for object storage (S3), file systems, Iceberg table buckets, and Lance vector buckets, designed to handle billions of files with O(1) disk access and effortless horizontal scaling.

31.1 KB
Raw Blame
package engine

import (
	"context"
	"fmt"
	"math"
	"strconv"
	"strings"

	"github.com/seaweedfs/seaweedfs/weed/mq/topic"
	"github.com/seaweedfs/seaweedfs/weed/pb/schema_pb"
	"github.com/seaweedfs/seaweedfs/weed/query/sqltypes"
)

// AggregationSpec defines an aggregation function to be computed
type AggregationSpec struct {
	Function string // COUNT, SUM, AVG, MIN, MAX
	Column   string // Column name, or "*" for COUNT(*)
	Alias    string // Optional alias for the result column
	Distinct bool   // Support for DISTINCT keyword
}

// AggregationResult holds the computed result of an aggregation
type AggregationResult struct {
	Count int64
	Sum   float64
	Min   interface{}
	Max   interface{}
}

// AggregationStrategy represents the strategy for executing aggregations
type AggregationStrategy struct {
	CanUseFastPath   bool
	Reason           string
	UnsupportedSpecs []AggregationSpec
}

// TopicDataSources represents the data sources available for a topic
type TopicDataSources struct {
	ParquetFiles         map[string][]*ParquetFileStats // partitionPath -> parquet file stats
	ParquetRowCount      int64
	LiveLogRowCount      int64
	LiveLogFilesCount    int // Total count of live log files across all partitions
	PartitionsCount      int
	BrokerUnflushedCount int64
}

// FastPathOptimizer handles fast path aggregation optimization decisions
type FastPathOptimizer struct {
	engine *SQLEngine
}

// NewFastPathOptimizer creates a new fast path optimizer
func NewFastPathOptimizer(engine *SQLEngine) *FastPathOptimizer {
	return &FastPathOptimizer{engine: engine}
}

// DetermineStrategy analyzes aggregations and determines if fast path can be used
func (opt *FastPathOptimizer) DetermineStrategy(aggregations []AggregationSpec) AggregationStrategy {
	strategy := AggregationStrategy{
		CanUseFastPath:   true,
		Reason:           "all_aggregations_supported",
		UnsupportedSpecs: []AggregationSpec{},
	}

	for _, spec := range aggregations {
		if !opt.engine.canUseParquetStatsForAggregation(spec) {
			strategy.CanUseFastPath = false
			strategy.Reason = "unsupported_aggregation_functions"
			strategy.UnsupportedSpecs = append(strategy.UnsupportedSpecs, spec)
		}
	}

	return strategy
}

// CollectDataSourcesWithTimeFilter gathers information about available data sources for a topic
// with optional time filtering to skip irrelevant parquet files
func (opt *FastPathOptimizer) CollectDataSourcesWithTimeFilter(ctx context.Context, hybridScanner *HybridMessageScanner, startTimeNs, stopTimeNs int64) (*TopicDataSources, error) {
	dataSources := &TopicDataSources{
		ParquetFiles:      make(map[string][]*ParquetFileStats),
		ParquetRowCount:   0,
		LiveLogRowCount:   0,
		LiveLogFilesCount: 0,
		PartitionsCount:   0,
	}

	if isDebugMode(ctx) {
		fmt.Printf("Collecting data sources for: %s/%s\n", hybridScanner.topic.Namespace, hybridScanner.topic.Name)
	}

	// Discover partitions for the topic
	partitionPaths, err := opt.engine.discoverTopicPartitions(hybridScanner.topic.Namespace, hybridScanner.topic.Name)
	if err != nil {
		if isDebugMode(ctx) {
			fmt.Printf("ERROR: Partition discovery failed: %v\n", err)
		}
		return dataSources, DataSourceError{
			Source: "partition_discovery",
			Cause:  err,
		}
	}

	// DEBUG: Log discovered partitions
	if isDebugMode(ctx) {
		fmt.Printf("Discovered %d partitions: %v\n", len(partitionPaths), partitionPaths)
	}

	// Collect stats from each partition
	// Note: discoverTopicPartitions always returns absolute paths starting with "/topics/"
	for _, partitionPath := range partitionPaths {
		if isDebugMode(ctx) {
			fmt.Printf("\nProcessing partition: %s\n", partitionPath)
		}

		// Read parquet file statistics
		parquetStats, err := hybridScanner.ReadParquetStatistics(partitionPath)
		if err != nil {
			return dataSources, DataSourceError{
				Source: "parquet_statistics",
				Cause:  fmt.Errorf("partition %s: %w", partitionPath, err),
			}
		} else if len(parquetStats) == 0 {
			if isDebugMode(ctx) {
				fmt.Printf("  No parquet files found in partition\n")
			}
		} else {
			// Prune by time range using parquet column statistics
			filtered := pruneParquetFilesByTime(ctx, parquetStats, hybridScanner, startTimeNs, stopTimeNs)
			dataSources.ParquetFiles[partitionPath] = filtered
			partitionParquetRows := int64(0)
			for _, stat := range filtered {
				partitionParquetRows += stat.RowCount
				dataSources.ParquetRowCount += stat.RowCount
			}
			if isDebugMode(ctx) {
				fmt.Printf("  Found %d parquet files with %d total rows\n", len(filtered), partitionParquetRows)
			}
		}

		// Count live log files (excluding those converted to parquet)
		parquetSources := opt.engine.extractParquetSourceFiles(dataSources.ParquetFiles[partitionPath])
		liveLogCount, liveLogErr := opt.engine.countLiveLogRowsExcludingParquetSources(ctx, partitionPath, parquetSources)
		if liveLogErr != nil {
			return dataSources, DataSourceError{
				Source: "live_log_count",
				Cause:  fmt.Errorf("partition %s: %w", partitionPath, liveLogErr),
			}
		} else {
			dataSources.LiveLogRowCount += liveLogCount
			if isDebugMode(ctx) {
				fmt.Printf("  Found %d live log rows (excluding %d parquet sources)\n", liveLogCount, len(parquetSources))
			}
		}

		// Count live log files for partition with proper range values
		// Extract partition name from absolute path (e.g., "0000-2520" from "/topics/.../v2025.../0000-2520")
		partitionName := partitionPath[strings.LastIndex(partitionPath, "/")+1:]
		partitionParts := strings.Split(partitionName, "-")
		if len(partitionParts) == 2 {
			rangeStart, err1 := strconv.Atoi(partitionParts[0])
			rangeStop, err2 := strconv.Atoi(partitionParts[1])
			if err1 == nil && err2 == nil {
				partition := topic.Partition{
					RangeStart: int32(rangeStart),
					RangeStop:  int32(rangeStop),
				}
				liveLogFileCount, err := hybridScanner.countLiveLogFiles(partition)
				if err == nil {
					dataSources.LiveLogFilesCount += liveLogFileCount
				}

				// Count broker unflushed messages for this partition
				if hybridScanner.brokerClient != nil {
					entries, err := hybridScanner.brokerClient.GetUnflushedMessages(ctx, hybridScanner.topic.Namespace, hybridScanner.topic.Name, partition, 0)
					if err == nil {
						dataSources.BrokerUnflushedCount += int64(len(entries))
						if isDebugMode(ctx) {
							fmt.Printf("  Found %d unflushed broker messages\n", len(entries))
						}
					} else if isDebugMode(ctx) {
						fmt.Printf("  ERROR: Failed to get unflushed broker messages: %v\n", err)
					}
				}
			}
		}
	}

	dataSources.PartitionsCount = len(partitionPaths)

	if isDebugMode(ctx) {
		fmt.Printf("Data sources collected: %d partitions, %d parquet rows, %d live log rows, %d broker buffer rows\n",
			dataSources.PartitionsCount, dataSources.ParquetRowCount, dataSources.LiveLogRowCount, dataSources.BrokerUnflushedCount)
	}

	return dataSources, nil
}

// AggregationComputer handles the computation of aggregations using fast path
type AggregationComputer struct {
	engine *SQLEngine
}

// NewAggregationComputer creates a new aggregation computer
func NewAggregationComputer(engine *SQLEngine) *AggregationComputer {
	return &AggregationComputer{engine: engine}
}

// ComputeFastPathAggregations computes aggregations using parquet statistics and live log data
func (comp *AggregationComputer) ComputeFastPathAggregations(
	ctx context.Context,
	aggregations []AggregationSpec,
	dataSources *TopicDataSources,
	partitions []string,
) ([]AggregationResult, error) {

	aggResults := make([]AggregationResult, len(aggregations))

	for i, spec := range aggregations {
		switch spec.Function {
		case FuncCOUNT:
			if spec.Column == "*" {
				aggResults[i].Count = dataSources.ParquetRowCount + dataSources.LiveLogRowCount + dataSources.BrokerUnflushedCount
			} else {
				// For specific columns, we might need to account for NULLs in the future
				aggResults[i].Count = dataSources.ParquetRowCount + dataSources.LiveLogRowCount + dataSources.BrokerUnflushedCount
			}

		case FuncMIN:
			globalMin, err := comp.computeGlobalMin(spec, dataSources, partitions)
			if err != nil {
				return nil, AggregationError{
					Operation: spec.Function,
					Column:    spec.Column,
					Cause:     err,
				}
			}
			aggResults[i].Min = globalMin

		case FuncMAX:
			globalMax, err := comp.computeGlobalMax(spec, dataSources, partitions)
			if err != nil {
				return nil, AggregationError{
					Operation: spec.Function,
					Column:    spec.Column,
					Cause:     err,
				}
			}
			aggResults[i].Max = globalMax

		default:
			return nil, OptimizationError{
				Strategy: "fast_path_aggregation",
				Reason:   fmt.Sprintf("unsupported aggregation function: %s", spec.Function),
			}
		}
	}

	return aggResults, nil
}

// computeGlobalMin computes the global minimum value across all data sources
func (comp *AggregationComputer) computeGlobalMin(spec AggregationSpec, dataSources *TopicDataSources, partitions []string) (interface{}, error) {
	var globalMin interface{}
	var globalMinValue *schema_pb.Value
	hasParquetStats := false

	// Step 1: Get minimum from parquet statistics
	for _, fileStats := range dataSources.ParquetFiles {
		for _, fileStat := range fileStats {
			// Try case-insensitive column lookup
			var colStats *ParquetColumnStats
			var found bool

			// First try exact match
			if stats, exists := fileStat.ColumnStats[spec.Column]; exists {
				colStats = stats
				found = true
			} else {
				// Try case-insensitive lookup
				for colName, stats := range fileStat.ColumnStats {
					if strings.EqualFold(colName, spec.Column) {
						colStats = stats
						found = true
						break
					}
				}
			}

			if found && colStats != nil && colStats.MinValue != nil {
				if globalMinValue == nil || comp.engine.compareValues(colStats.MinValue, globalMinValue) < 0 {
					globalMinValue = colStats.MinValue
					extractedValue := comp.engine.extractRawValue(colStats.MinValue)
					if extractedValue != nil {
						globalMin = extractedValue
						hasParquetStats = true
					}
				}
			}
		}
	}

	// Step 2: Get minimum from live log data (only if no live logs or if we need to compare)
	if dataSources.LiveLogRowCount > 0 {
		for _, partition := range partitions {
			partitionParquetSources := make(map[string]bool)
			if partitionFileStats, exists := dataSources.ParquetFiles[partition]; exists {
				partitionParquetSources = comp.engine.extractParquetSourceFiles(partitionFileStats)
			}

			liveLogMin, _, err := comp.engine.computeLiveLogMinMax(partition, spec.Column, partitionParquetSources)
			if err != nil {
				return nil, err
			}

			if liveLogMin != nil {
				if globalMin == nil {
					globalMin = liveLogMin
				} else {
					liveLogSchemaValue := comp.engine.convertRawValueToSchemaValue(liveLogMin)
					if liveLogSchemaValue != nil && comp.engine.compareValues(liveLogSchemaValue, globalMinValue) < 0 {
						globalMin = liveLogMin
						globalMinValue = liveLogSchemaValue
					}
				}
			}
		}
	}

	// Step 3: Handle system columns if no regular data found
	if globalMin == nil && !hasParquetStats {
		globalMin = comp.engine.getSystemColumnGlobalMin(spec.Column, dataSources.ParquetFiles)
	}

	return globalMin, nil
}

// computeGlobalMax computes the global maximum value across all data sources
func (comp *AggregationComputer) computeGlobalMax(spec AggregationSpec, dataSources *TopicDataSources, partitions []string) (interface{}, error) {
	var globalMax interface{}
	var globalMaxValue *schema_pb.Value
	hasParquetStats := false

	// Step 1: Get maximum from parquet statistics
	for _, fileStats := range dataSources.ParquetFiles {
		for _, fileStat := range fileStats {
			// Try case-insensitive column lookup
			var colStats *ParquetColumnStats
			var found bool

			// First try exact match
			if stats, exists := fileStat.ColumnStats[spec.Column]; exists {
				colStats = stats
				found = true
			} else {
				// Try case-insensitive lookup
				for colName, stats := range fileStat.ColumnStats {
					if strings.EqualFold(colName, spec.Column) {
						colStats = stats
						found = true
						break
					}
				}
			}

			if found && colStats != nil && colStats.MaxValue != nil {
				if globalMaxValue == nil || comp.engine.compareValues(colStats.MaxValue, globalMaxValue) > 0 {
					globalMaxValue = colStats.MaxValue
					extractedValue := comp.engine.extractRawValue(colStats.MaxValue)
					if extractedValue != nil {
						globalMax = extractedValue
						hasParquetStats = true
					}
				}
			}
		}
	}

	// Step 2: Get maximum from live log data (only if live logs exist)
	if dataSources.LiveLogRowCount > 0 {
		for _, partition := range partitions {
			partitionParquetSources := make(map[string]bool)
			if partitionFileStats, exists := dataSources.ParquetFiles[partition]; exists {
				partitionParquetSources = comp.engine.extractParquetSourceFiles(partitionFileStats)
			}

			_, liveLogMax, err := comp.engine.computeLiveLogMinMax(partition, spec.Column, partitionParquetSources)
			if err != nil {
				return nil, err
			}

			if liveLogMax != nil {
				if globalMax == nil {
					globalMax = liveLogMax
				} else {
					liveLogSchemaValue := comp.engine.convertRawValueToSchemaValue(liveLogMax)
					if liveLogSchemaValue != nil && comp.engine.compareValues(liveLogSchemaValue, globalMaxValue) > 0 {
						globalMax = liveLogMax
						globalMaxValue = liveLogSchemaValue
					}
				}
			}
		}
	}

	// Step 3: Handle system columns if no regular data found
	if globalMax == nil && !hasParquetStats {
		globalMax = comp.engine.getSystemColumnGlobalMax(spec.Column, dataSources.ParquetFiles)
	}

	return globalMax, nil
}

// executeAggregationQuery handles SELECT queries with aggregation functions
func (e *SQLEngine) executeAggregationQuery(ctx context.Context, hybridScanner *HybridMessageScanner, aggregations []AggregationSpec, stmt *SelectStatement) (*QueryResult, error) {
	return e.executeAggregationQueryWithPlan(ctx, hybridScanner, aggregations, stmt, nil)
}

// executeAggregationQueryWithPlan handles SELECT queries with aggregation functions and populates execution plan
func (e *SQLEngine) executeAggregationQueryWithPlan(ctx context.Context, hybridScanner *HybridMessageScanner, aggregations []AggregationSpec, stmt *SelectStatement, plan *QueryExecutionPlan) (*QueryResult, error) {
	// Parse LIMIT and OFFSET for aggregation results (do this first)
	// Use -1 to distinguish "no LIMIT" from "LIMIT 0"
	limit := -1
	offset := 0
	if stmt.Limit != nil && stmt.Limit.Rowcount != nil {
		if limitExpr, ok := stmt.Limit.Rowcount.(*SQLVal); ok && limitExpr.Type == IntVal {
			if limit64, err := strconv.ParseInt(string(limitExpr.Val), 10, 64); err == nil {
				if limit64 > int64(math.MaxInt) || limit64 < 0 {
					return nil, fmt.Errorf("LIMIT value %d is out of range", limit64)
				}
				// Safe conversion after bounds check
				limit = int(limit64)
			}
		}
	}
	if stmt.Limit != nil && stmt.Limit.Offset != nil {
		if offsetExpr, ok := stmt.Limit.Offset.(*SQLVal); ok && offsetExpr.Type == IntVal {
			if offset64, err := strconv.ParseInt(string(offsetExpr.Val), 10, 64); err == nil {
				if offset64 > int64(math.MaxInt) || offset64 < 0 {
					return nil, fmt.Errorf("OFFSET value %d is out of range", offset64)
				}
				// Safe conversion after bounds check
				offset = int(offset64)
			}
		}
	}

	// Parse WHERE clause for filtering
	var predicate func(*schema_pb.RecordValue) bool
	var err error
	if stmt.Where != nil {
		predicate, err = e.buildPredicate(stmt.Where.Expr)
		if err != nil {
			return &QueryResult{Error: err}, err
		}
	}

	// Extract time filters and validate that WHERE clause contains only time-based predicates
	startTimeNs, stopTimeNs := int64(0), int64(0)
	onlyTimePredicates := true
	if stmt.Where != nil {
		startTimeNs, stopTimeNs, onlyTimePredicates = e.extractTimeFiltersWithValidation(stmt.Where.Expr)
	}

	// FAST PATH WITH TIME-BASED OPTIMIZATION:
	// Allow fast path only for queries without WHERE clause or with time-only WHERE clauses
	// This prevents incorrect results when non-time predicates are present
	canAttemptFastPath := stmt.Where == nil || onlyTimePredicates

	if canAttemptFastPath {
		if isDebugMode(ctx) {
			if stmt.Where == nil {
				fmt.Printf("\nFast path optimization attempt (no WHERE clause)...\n")
			} else {
				fmt.Printf("\nFast path optimization attempt (time-only WHERE clause)...\n")
			}
		}
		fastResult, canOptimize := e.tryFastParquetAggregationWithPlan(ctx, hybridScanner, aggregations, plan, startTimeNs, stopTimeNs, stmt)
		if canOptimize {
			if isDebugMode(ctx) {
				fmt.Printf("Fast path optimization succeeded!\n")
			}
			return fastResult, nil
		} else {
			if isDebugMode(ctx) {
				fmt.Printf("Fast path optimization failed, falling back to slow path\n")
			}
		}
	} else {
		if isDebugMode(ctx) {
			fmt.Printf("Fast path not applicable due to complex WHERE clause\n")
		}
	}

	// SLOW PATH: Fall back to full table scan
	if isDebugMode(ctx) {
		fmt.Printf("Using full table scan for aggregation (parquet optimization not applicable)\n")
	}

	// Extract columns needed for aggregations
	columnsNeeded := make(map[string]bool)
	for _, spec := range aggregations {
		if spec.Column != "*" {
			columnsNeeded[spec.Column] = true
		}
	}

	// Convert to slice
	var scanColumns []string
	if len(columnsNeeded) > 0 {
		scanColumns = make([]string, 0, len(columnsNeeded))
		for col := range columnsNeeded {
			scanColumns = append(scanColumns, col)
		}
	}
	// If no specific columns needed (COUNT(*) only), don't specify columns (scan all)

	// Build scan options for full table scan (aggregations need all data during scanning)
	hybridScanOptions := HybridScanOptions{
		StartTimeNs: startTimeNs,
		StopTimeNs:  stopTimeNs,
		Limit:       -1, // Use -1 to mean "no limit" - need all data for aggregation
		Offset:      0,  // No offset during scanning - OFFSET applies to final results
		Predicate:   predicate,
		Columns:     scanColumns, // Include columns needed for aggregation functions
	}

	// DEBUG: Log scan options for aggregation
	debugHybridScanOptions(ctx, hybridScanOptions, "AGGREGATION")

	// Execute the hybrid scan to get all matching records
	var results []HybridScanResult
	if plan != nil {
		// EXPLAIN mode - capture broker buffer stats
		var stats *HybridScanStats
		results, stats, err = hybridScanner.ScanWithStats(ctx, hybridScanOptions)
		if err != nil {
			return &QueryResult{Error: err}, err
		}

		// Populate plan with broker buffer information
		if stats != nil {
			plan.BrokerBufferQueried = stats.BrokerBufferQueried
			plan.BrokerBufferMessages = stats.BrokerBufferMessages
			plan.BufferStartIndex = stats.BufferStartIndex

			// Add broker_buffer to data sources if buffer was queried
			if stats.BrokerBufferQueried {
				// Check if broker_buffer is already in data sources
				hasBrokerBuffer := false
				for _, source := range plan.DataSources {
					if source == "broker_buffer" {
						hasBrokerBuffer = true
						break
					}
				}
				if !hasBrokerBuffer {
					plan.DataSources = append(plan.DataSources, "broker_buffer")
				}
			}
		}
	} else {
		// Normal mode - just get results
		results, err = hybridScanner.Scan(ctx, hybridScanOptions)
		if err != nil {
			return &QueryResult{Error: err}, err
		}
	}

	// DEBUG: Log scan results
	if isDebugMode(ctx) {
		fmt.Printf("AGGREGATION SCAN RESULTS: %d rows returned\n", len(results))
	}

	// Compute aggregations
	aggResults := e.computeAggregations(results, aggregations)

	// Build result set
	columns := make([]string, len(aggregations))
	row := make([]sqltypes.Value, len(aggregations))

	for i, spec := range aggregations {
		columns[i] = spec.Alias
		row[i] = e.formatAggregationResult(spec, aggResults[i])
	}

	// Apply OFFSET and LIMIT to aggregation results
	// Limit semantics: -1 = no limit, 0 = LIMIT 0 (empty), >0 = limit to N rows
	rows := [][]sqltypes.Value{row}
	if offset > 0 || limit >= 0 {
		// Handle LIMIT 0 first
		if limit == 0 {
			rows = [][]sqltypes.Value{}
		} else {
			// Apply OFFSET first
			if offset > 0 {
				if offset >= len(rows) {
					rows = [][]sqltypes.Value{}
				} else {
					rows = rows[offset:]
				}
			}

			// Apply LIMIT after OFFSET (only if limit > 0)
			if limit > 0 && len(rows) > limit {
				rows = rows[:limit]
			}
		}
	}

	result := &QueryResult{
		Columns: columns,
		Rows:    rows,
	}

	// Build execution tree for aggregation queries if plan is provided
	if plan != nil {
		// Populate detailed plan information for full scan (similar to fast path)
		e.populateFullScanPlanDetails(ctx, plan, hybridScanner, stmt)
		plan.RootNode = e.buildExecutionTree(plan, stmt)
	}

	return result, nil
}

// populateFullScanPlanDetails populates detailed plan information for full scan queries
// This provides consistency with fast path execution plan details
func (e *SQLEngine) populateFullScanPlanDetails(ctx context.Context, plan *QueryExecutionPlan, hybridScanner *HybridMessageScanner, stmt *SelectStatement) {
	// plan.Details is initialized at the start of the SELECT execution

	// Extract table information
	var database, tableName string
	if len(stmt.From) == 1 {
		if table, ok := stmt.From[0].(*AliasedTableExpr); ok {
			if tableExpr, ok := table.Expr.(TableName); ok {
				tableName = tableExpr.Name.String()
				if tableExpr.Qualifier != nil && tableExpr.Qualifier.String() != "" {
					database = tableExpr.Qualifier.String()
				}
			}
		}
	}

	// Use current database if not specified
	if database == "" {
		database = e.catalog.currentDatabase
		if database == "" {
			database = "default"
		}
	}

	// Discover partitions and populate file details
	if partitions, discoverErr := e.discoverTopicPartitions(database, tableName); discoverErr == nil {
		// Add partition paths to execution plan details
		plan.Details["partition_paths"] = partitions

		// Populate detailed file information using shared helper
		e.populatePlanFileDetails(ctx, plan, hybridScanner, partitions, stmt)
	} else {
		// Record discovery error to plan for better diagnostics
		plan.Details["error_partition_discovery"] = discoverErr.Error()
	}
}

// tryFastParquetAggregation attempts to compute aggregations using hybrid approach:
// - Use parquet metadata for parquet files
// - Count live log files for live data
// - Combine both for accurate results per partition
// Returns (result, canOptimize) where canOptimize=true means the hybrid fast path was used
func (e *SQLEngine) tryFastParquetAggregation(ctx context.Context, hybridScanner *HybridMessageScanner, aggregations []AggregationSpec) (*QueryResult, bool) {
	return e.tryFastParquetAggregationWithPlan(ctx, hybridScanner, aggregations, nil, 0, 0, nil)
}

// tryFastParquetAggregationWithPlan is the same as tryFastParquetAggregation but also populates execution plan if provided
// startTimeNs, stopTimeNs: optional time range filters for parquet file optimization (0 means no filtering)
// stmt: SELECT statement for column statistics pruning optimization (can be nil)
func (e *SQLEngine) tryFastParquetAggregationWithPlan(ctx context.Context, hybridScanner *HybridMessageScanner, aggregations []AggregationSpec, plan *QueryExecutionPlan, startTimeNs, stopTimeNs int64, stmt *SelectStatement) (*QueryResult, bool) {
	// Use the new modular components
	optimizer := NewFastPathOptimizer(e)
	computer := NewAggregationComputer(e)

	// Step 1: Determine strategy
	strategy := optimizer.DetermineStrategy(aggregations)
	if !strategy.CanUseFastPath {
		return nil, false
	}

	// Step 2: Collect data sources with time filtering for parquet file optimization
	dataSources, err := optimizer.CollectDataSourcesWithTimeFilter(ctx, hybridScanner, startTimeNs, stopTimeNs)
	if err != nil {
		return nil, false
	}

	// Build partition list for aggregation computer
	// Note: discoverTopicPartitions always returns absolute paths
	partitions, err := e.discoverTopicPartitions(hybridScanner.topic.Namespace, hybridScanner.topic.Name)
	if err != nil {
		return nil, false
	}

	// Debug: Show the hybrid optimization results (only in explain mode)
	if isDebugMode(ctx) && (dataSources.ParquetRowCount > 0 || dataSources.LiveLogRowCount > 0 || dataSources.BrokerUnflushedCount > 0) {
		partitionsWithLiveLogs := 0
		if dataSources.LiveLogRowCount > 0 || dataSources.BrokerUnflushedCount > 0 {
			partitionsWithLiveLogs = 1 // Simplified for now
		}
		fmt.Printf("Hybrid fast aggregation with deduplication: %d parquet rows + %d deduplicated live log rows + %d broker buffer rows from %d partitions\n",
			dataSources.ParquetRowCount, dataSources.LiveLogRowCount, dataSources.BrokerUnflushedCount, partitionsWithLiveLogs)
	}

	// Step 3: Compute aggregations using fast path
	aggResults, err := computer.ComputeFastPathAggregations(ctx, aggregations, dataSources, partitions)
	if err != nil {
		return nil, false
	}

	// Step 3.5: Validate fast path results (safety check)
	// For simple COUNT(*) queries, ensure we got a reasonable result
	if len(aggregations) == 1 && aggregations[0].Function == FuncCOUNT && aggregations[0].Column == "*" {
		totalRows := dataSources.ParquetRowCount + dataSources.LiveLogRowCount + dataSources.BrokerUnflushedCount
		countResult := aggResults[0].Count

		if isDebugMode(ctx) {
			fmt.Printf("Validating fast path: COUNT=%d, Sources=%d\n", countResult, totalRows)
		}

		if totalRows == 0 && countResult > 0 {
			// Fast path found data but data sources show 0 - this suggests a bug
			if isDebugMode(ctx) {
				fmt.Printf("Fast path validation failed: COUNT=%d but sources=0\n", countResult)
			}
			return nil, false
		}
		if totalRows > 0 && countResult == 0 {
			// Data sources show data but COUNT is 0 - this also suggests a bug
			if isDebugMode(ctx) {
				fmt.Printf("Fast path validation failed: sources=%d but COUNT=0\n", totalRows)
			}
			return nil, false
		}
		if countResult != totalRows {
			// Counts don't match - this suggests inconsistent logic
			if isDebugMode(ctx) {
				fmt.Printf("Fast path validation failed: COUNT=%d != sources=%d\n", countResult, totalRows)
			}
			return nil, false
		}
		if isDebugMode(ctx) {
			fmt.Printf("Fast path validation passed: COUNT=%d\n", countResult)
		}
	}

	// Step 4: Populate execution plan if provided (for EXPLAIN queries)
	if plan != nil {
		strategy := optimizer.DetermineStrategy(aggregations)
		builder := &ExecutionPlanBuilder{}

		// Create a minimal SELECT statement for the plan builder (avoid nil pointer)
		stmt := &SelectStatement{}

		// Build aggregation plan with fast path strategy
		aggPlan := builder.BuildAggregationPlan(stmt, aggregations, strategy, dataSources)

		// Copy relevant fields to the main plan
		plan.ExecutionStrategy = aggPlan.ExecutionStrategy
		plan.DataSources = aggPlan.DataSources
		plan.OptimizationsUsed = aggPlan.OptimizationsUsed
		plan.PartitionsScanned = aggPlan.PartitionsScanned
		plan.ParquetFilesScanned = aggPlan.ParquetFilesScanned
		plan.LiveLogFilesScanned = aggPlan.LiveLogFilesScanned
		plan.TotalRowsProcessed = aggPlan.TotalRowsProcessed
		plan.Aggregations = aggPlan.Aggregations

		// Indicate broker buffer participation for EXPLAIN tree rendering
		if dataSources.BrokerUnflushedCount > 0 {
			plan.BrokerBufferQueried = true
			plan.BrokerBufferMessages = int(dataSources.BrokerUnflushedCount)
		}

		// Merge details while preserving existing ones
		for key, value := range aggPlan.Details {
			plan.Details[key] = value
		}

		// Add file path information from the data collection
		plan.Details["partition_paths"] = partitions

		// Populate detailed file information using shared helper, including time filters for pruning
		plan.Details[PlanDetailStartTimeNs] = startTimeNs
		plan.Details[PlanDetailStopTimeNs] = stopTimeNs
		e.populatePlanFileDetails(ctx, plan, hybridScanner, partitions, stmt)

		// Update counts to match discovered live log files
		if liveLogFiles, ok := plan.Details["live_log_files"].([]string); ok {
			dataSources.LiveLogFilesCount = len(liveLogFiles)
			plan.LiveLogFilesScanned = len(liveLogFiles)
		}

		// Ensure PartitionsScanned is set so Statistics section appears
		if plan.PartitionsScanned == 0 && len(partitions) > 0 {
			plan.PartitionsScanned = len(partitions)
		}

		if isDebugMode(ctx) {
			fmt.Printf("Populated execution plan with fast path strategy\n")
		}
	}

	// Step 5: Build final query result
	columns := make([]string, len(aggregations))
	row := make([]sqltypes.Value, len(aggregations))

	for i, spec := range aggregations {
		columns[i] = spec.Alias
		row[i] = e.formatAggregationResult(spec, aggResults[i])
	}

	result := &QueryResult{
		Columns: columns,
		Rows:    [][]sqltypes.Value{row},
	}

	return result, true
}

// computeAggregations computes aggregation results from a full table scan
func (e *SQLEngine) computeAggregations(results []HybridScanResult, aggregations []AggregationSpec) []AggregationResult {
	aggResults := make([]AggregationResult, len(aggregations))

	for i, spec := range aggregations {
		switch spec.Function {
		case FuncCOUNT:
			if spec.Column == "*" {
				aggResults[i].Count = int64(len(results))
			} else {
				count := int64(0)
				for _, result := range results {
					if value := e.findColumnValue(result, spec.Column); value != nil && !e.isNullValue(value) {
						count++
					}
				}
				aggResults[i].Count = count
			}

		case FuncSUM:
			sum := float64(0)
			for _, result := range results {
				if value := e.findColumnValue(result, spec.Column); value != nil {
					if numValue := e.convertToNumber(value); numValue != nil {
						sum += *numValue
					}
				}
			}
			aggResults[i].Sum = sum

		case FuncAVG:
			sum := float64(0)
			count := int64(0)
			for _, result := range results {
				if value := e.findColumnValue(result, spec.Column); value != nil {
					if numValue := e.convertToNumber(value); numValue != nil {
						sum += *numValue
						count++
					}
				}
			}
			if count > 0 {
				aggResults[i].Sum = sum / float64(count) // Store average in Sum field
				aggResults[i].Count = count
			}

		case FuncMIN:
			var min interface{}
			var minValue *schema_pb.Value
			for _, result := range results {
				if value := e.findColumnValue(result, spec.Column); value != nil {
					if minValue == nil || e.compareValues(value, minValue) < 0 {
						minValue = value
						min = e.extractRawValue(value)
					}
				}
			}
			aggResults[i].Min = min

		case FuncMAX:
			var max interface{}
			var maxValue *schema_pb.Value
			for _, result := range results {
				if value := e.findColumnValue(result, spec.Column); value != nil {
					if maxValue == nil || e.compareValues(value, maxValue) > 0 {
						maxValue = value
						max = e.extractRawValue(value)
					}
				}
			}
			aggResults[i].Max = max
		}
	}

	return aggResults
}

// canUseParquetStatsForAggregation determines if an aggregation can be optimized with parquet stats
func (e *SQLEngine) canUseParquetStatsForAggregation(spec AggregationSpec) bool {
	switch spec.Function {
	case FuncCOUNT:
		return spec.Column == "*" || e.isSystemColumn(spec.Column) || e.isRegularColumn(spec.Column)
	case FuncMIN, FuncMAX:
		return e.isSystemColumn(spec.Column) || e.isRegularColumn(spec.Column)
	case FuncSUM, FuncAVG:
		// These require scanning actual values, not just min/max
		return false
	default:
		return false
	}
}

// debugHybridScanOptions logs the exact scan options being used
func debugHybridScanOptions(ctx context.Context, options HybridScanOptions, queryType string) {
	if isDebugMode(ctx) {
		fmt.Printf("\n=== HYBRID SCAN OPTIONS DEBUG (%s) ===\n", queryType)
		fmt.Printf("StartTimeNs: %d\n", options.StartTimeNs)
		fmt.Printf("StopTimeNs: %d\n", options.StopTimeNs)
		fmt.Printf("Limit: %d\n", options.Limit)
		fmt.Printf("Offset: %d\n", options.Offset)
		fmt.Printf("Predicate: %v\n", options.Predicate != nil)
		fmt.Printf("Columns: %v\n", options.Columns)
		fmt.Printf("==========================================\n")
	}
}