Skip to content

A reactive notebook for Python — run reproducible experiments, query with SQL, execute as a script, deploy as an app, and version with git. Stored as pure Python. All in a modern, AI-native editor.

23.1 KB
Raw Blame
/* Copyright 2026 Marimo. All rights reserved. */
"use no memo";

import { Popover as PopoverPrimitive } from "radix-ui";

const PopoverClose = PopoverPrimitive.Close;

import type { Column, ColumnDef } from "@tanstack/react-table";
import { isValid } from "date-fns";
import { useRef } from "react";
import { useLocale, useNumberFormatter } from "react-aria";
import { WithLocale } from "@/core/i18n/with-locale";
import type { DataType } from "@/core/kernel/messages";
import type { CalculateTopKRows } from "@/plugins/impl/DataTablePlugin";
import { cn } from "@/utils/cn";
import { exactDateTime } from "@/utils/dates";
import { Events } from "@/utils/events";
import { Logger } from "@/utils/Logger";
import { Maps } from "@/utils/maps";
import { maxFractionalDigits } from "@/utils/numbers";
import { Objects } from "@/utils/objects";
import { parseContent } from "@/utils/url-parser";
import { JsonOutput } from "../editor/output/JsonOutput";
import { CopyClipboardIcon } from "../icons/copy-icon";
import { Button } from "../ui/button";
import { Checkbox } from "../ui/checkbox";
import { Popover, PopoverContent, PopoverTrigger } from "../ui/popover";
import { Tooltip } from "../ui/tooltip";
import { DataTableColumnHeader } from "./column-header";
import type { ColumnChartSpecModel } from "./column-summary/chart-spec-model";
import { TableColumnSummary } from "./column-summary/column-summary";
import { COLUMN_WRAPPING_STYLES } from "./column-wrapping/feature";
import { DatePopover } from "./date-popover";
import type { FilterType } from "./filters";
import { getMimeValues, MimeCell } from "./mime-cell";
import {
  type DataTableSelection,
  extractTimezone,
  type FieldTypesWithExternalType,
  INDEX_COLUMN_NAME,
  isNumericType,
} from "./types";
import { SentinelCell, WhitespaceMarkers } from "./sentinel-cell";
import { highlightMatches } from "./search-highlight";
import { detectSentinel, splitLeadingTrailingWhitespace } from "./utils";
import { uniformSample } from "./uniformSample";
import { MarkdownUrlDetector, UrlDetector } from "./url-detector";

export const NAMELESS_COLUMN_PREFIX = "__m_column__";
// Artificial limit to display long strings
export const SELECT_ID = "__select__";
const MAX_STRING_LENGTH = 50;

function inferDataType(value: unknown): [type: DataType, displayType: string] {
  if (typeof value === "string") {
    return ["string", "string"];
  }
  if (typeof value === "number") {
    return ["number", "number"];
  }
  if (value instanceof Date) {
    return ["datetime", "datetime"];
  }
  if (typeof value === "boolean") {
    return ["boolean", "boolean"];
  }
  if (value == null) {
    return ["unknown", "object"];
  }
  return ["unknown", "object"];
}

export function inferFieldTypes<T>(items: T[]): FieldTypesWithExternalType {
  // No items
  if (items.length === 0) {
    return [];
  }

  // Not an object
  if (typeof items[0] !== "object") {
    return [];
  }

  const fieldTypes: Record<string, [DataType, string]> = {};

  // This can be slow for large datasets,
  // so only sample 10 evenly distributed rows
  uniformSample(items, 10).forEach((item) => {
    if (typeof item !== "object" || item === null) {
      return;
    }
    // We will be a bit defensive and assume values are not homogeneous.
    // If any is a mimetype, then we will treat it as a mimetype (i.e. not sortable)
    Object.entries(item).forEach(([key, value]) => {
      const currentValue = fieldTypes[key];
      if (!currentValue) {
        // Set for the first time
        fieldTypes[key] = inferDataType(value);
      }

      // If its not null, override the type
      if (value != null) {
        // This can be lossy as we infer take the last seen type
        fieldTypes[key] = inferDataType(value);
      }
    });
  });

  return Objects.entries(fieldTypes);
}

export function generateColumns<T>({
  rowHeaders,
  selection,
  fieldTypes,
  chartSpecModel,
  textJustifyColumns,
  wrappedColumns,
  headerTooltip,
  showDataTypes,
  calculateTopKRows,
  fractionDigitsByColumn,
  columnWidths,
}: {
  rowHeaders: FieldTypesWithExternalType;
  selection: DataTableSelection;
  fieldTypes: FieldTypesWithExternalType;
  chartSpecModel?: ColumnChartSpecModel<unknown>;
  textJustifyColumns?: Record<string, "left" | "center" | "right">;
  wrappedColumns?: string[];
  headerTooltip?: Record<string, string>;
  showDataTypes?: boolean;
  calculateTopKRows?: CalculateTopKRows;
  fractionDigitsByColumn?: Record<string, number>;
  columnWidths?: Record<string, number>;
}): ColumnDef<T>[] {
  // Row-headers are typically index columns
  const rowHeadersSet = new Set(rowHeaders.map(([columnName]) => columnName));
  const wrappedColumnsSet = new Set(wrappedColumns);

  const typesByColumn = Maps.keyBy(fieldTypes, (entry) => entry[0]);

  const getMeta = (key: string) => {
    const types = typesByColumn.get(key)?.[1];
    const isRowHeader = rowHeadersSet.has(key);

    if (isRowHeader || !types) {
      const types = rowHeaders.find(([columnName]) => columnName === key)?.[1];
      return {
        rowHeader: isRowHeader,
        dtype: types?.[1],
        dataType: types?.[0],
        minFractionDigits: fractionDigitsByColumn?.[key],
      };
    }

    return {
      rowHeader: isRowHeader,
      filterType: getFilterTypeForFieldType(types[0]),
      dtype: types[1],
      dataType: types[0],
      minFractionDigits: fractionDigitsByColumn?.[key],
    };
  };

  const getJustify = (key: string): "left" | "center" | "right" | undefined => {
    // Explicit user override takes precedence
    if (textJustifyColumns?.[key]) {
      return textJustifyColumns[key];
    }
    // Auto right-align numeric columns
    const dataType = getMeta(key).dataType;
    if (isNumericType(dataType)) {
      return "right";
    }
    return undefined;
  };

  const columnKeys: string[] = [
    ...rowHeadersSet,
    ...fieldTypes
      .filter(([columnName]) => {
        if (rowHeadersSet.has(columnName)) {
          Logger.warn(
            `Skipping field type for row header column ${columnName}`,
          );
          return false;
        }
        return true;
      })
      .map(([columnName]) => columnName),
  ];

  // Remove the index column if it exists
  const indexColumnIdx = columnKeys.indexOf(INDEX_COLUMN_NAME);
  if (indexColumnIdx !== -1) {
    columnKeys.splice(indexColumnIdx, 1);
  }

  const columns = columnKeys.map((key, idx): ColumnDef<T> => ({
    id: key || `${NAMELESS_COLUMN_PREFIX}${idx}`,
    // Use an accessorFn instead of an accessorKey because column names
    // may have periods in them ...
    // https://github.com/TanStack/table/issues/1671
    accessorFn: (row) => {
      return row[key as keyof T];
    },
    enableHiding: !rowHeadersSet.has(key) && key !== "",
    header: ({ column, table }) => {
      const stats = chartSpecModel?.getColumnStats(key);
      const dtype = column.columnDef.meta?.dtype;
      const headerTitle = headerTooltip?.[key];
      const headerJustify = textJustifyColumns?.[key];

      const dtypeHeader =
        showDataTypes && dtype ? (
          <div
            className={cn(
              "flex flex-row gap-1",
              headerJustify === "center" && "justify-center",
              headerJustify === "right" && "justify-end",
            )}
          >
            <span className="text-xs text-muted-foreground">{dtype}</span>
            {stats && typeof stats.nulls === "number" && stats.nulls > 0 && (
              <span className="text-xs text-muted-foreground">
                (nulls: {stats.nulls})
              </span>
            )}
          </div>
        ) : null;

      const headerName = (
        <span
          className={cn(
            "font-bold",
            headerTitle && "underline decoration-dotted",
          )}
        >
          {key === "" ? " " : key}
        </span>
      );

      const headerWithTooltip = headerTitle ? (
        <Tooltip content={headerTitle} delayDuration={300}>
          {headerName}
        </Tooltip>
      ) : (
        headerName
      );

      const dataTableColumnHeader = (
        <DataTableColumnHeader
          header={headerWithTooltip}
          subheader={dtypeHeader}
          column={column}
          justify={headerJustify}
          calculateTopKRows={calculateTopKRows}
          table={table}
        />
      );

      // Row headers have no summaries
      if (rowHeadersSet.has(key)) {
        return dataTableColumnHeader;
      }

      return (
        <div
          className={cn(
            "flex flex-col h-full pt-0.5 pb-3 justify-between items-start",
            headerJustify === "center" && "items-center",
            headerJustify === "right" && "items-end",
          )}
        >
          {dataTableColumnHeader}
          <TableColumnSummary columnId={key} />
        </div>
      );
    },

    cell: ({ column, renderValue, getValue, cell, table }) => {
      function selectCell() {
        if (selection !== "single-cell" && selection !== "multi-cell") {
          return;
        }

        cell.toggleSelected?.();
      }

      const justify = getJustify(key);
      const wrapped = wrappedColumnsSet.has(key);
      const isCellSelected = cell?.getIsSelected?.() || false;
      const canSelectCell =
        (selection === "single-cell" || selection === "multi-cell") &&
        !isCellSelected;

      const dataType = column.columnDef.meta?.dataType;
      const isNumeric = isNumericType(dataType);
      const cellStyles = getCellStyleClass({
        justify,
        wrapped,
        canSelectCell,
        isSelected: isCellSelected,
        isNumeric,
      });

      const renderedCell = renderCellValue({
        column,
        renderValue,
        getValue,
        selectCell,
        cellStyles,
        highlightPattern: table.options.meta?.highlightPattern,
      });

      // Row headers are bold
      if (rowHeadersSet.has(key)) {
        return <b>{renderedCell}</b>;
      }

      return renderedCell;
    },
    // Remove any default filtering
    filterFn: undefined,
    // Unnamed index columns and geometry columns are not sortable
    enableSorting: !!key && getMeta(key).dataType !== "geometry",
    meta: {
      ...getMeta(key),
      width: columnWidths?.[key],
    },
    // size seeds the width before measurement; minSize/maxSize pin
    // getSize() to the fixed width so the per-paint measurement writeback
    // cannot drift the header width or sticky-pin offsets.
    ...(columnWidths?.[key] !== undefined
      ? {
          size: columnWidths[key],
          minSize: columnWidths[key],
          maxSize: columnWidths[key],
        }
      : {}),
  }));

  if (selection === "single" || selection === "multi") {
    columns.unshift({
      id: SELECT_ID,
      maxSize: 40,
      header: ({ table }) =>
        selection === "multi" ? (
          <Checkbox
            data-testid="select-all-checkbox"
            checked={table.getIsAllPageRowsSelected()}
            onCheckedChange={(value) =>
              table.toggleAllPageRowsSelected(!!value)
            }
            aria-label="Select all"
            className="mx-1.5 my-4"
          />
        ) : null,
      cell: ({ row }) => (
        <Checkbox
          data-testid="select-row-checkbox"
          checked={row.getIsSelected()}
          onCheckedChange={(value) => row.toggleSelected(!!value)}
          aria-label="Select row"
          className="mx-2"
          onMouseDown={(e) => {
            // Prevent cell underneath from being selected
            e.stopPropagation();
          }}
        />
      ),
      enableSorting: false,
      enableHiding: false,
    });
  }

  return columns;
}

const PopoutColumn = ({
  cellStyles,
  selectCell,
  rawStringValue,
  edges,
  contentClassName,
  scrollable = false,
  wrapped,
  children,
  buttonText,
}: {
  cellStyles?: string;
  selectCell?: () => void;
  rawStringValue: string;
  // Edge whitespace shown as visible markers in the trigger; copy/title
  // still use `rawStringValue`. Middle is sliced from `rawStringValue`.
  edges?: { leading: string; trailing: string };
  contentClassName?: string;
  scrollable?: boolean;
  wrapped?: boolean;
  buttonText?: string;
  children: React.ReactNode;
}) => {
  const triggerRef = useRef<HTMLButtonElement>(null);
  const hasEdgeWhitespace =
    edges !== undefined &&
    (edges.leading.length > 0 || edges.trailing.length > 0);

  const displayText = hasEdgeWhitespace
    ? rawStringValue.slice(
        edges.leading.length,
        rawStringValue.length - edges.trailing.length,
      )
    : rawStringValue;

  return (
    <Popover>
      <PopoverTrigger
        ref={triggerRef}
        className={cn(cellStyles, "max-w-fit outline-hidden")}
        onClick={selectCell}
        onMouseDown={(e) => {
          // Prevent cell underneath from being selected
          e.stopPropagation();
        }}
      >
        <span
          className={cn(
            "cursor-pointer hover:text-link",
            wrapped && COLUMN_WRAPPING_STYLES,
          )}
          title={rawStringValue}
        >
          {edges ? <WhitespaceMarkers value={edges.leading} /> : null}
          {displayText}
          {edges ? <WhitespaceMarkers value={edges.trailing} /> : null}
        </span>
      </PopoverTrigger>
      <PopoverContent
        className={cn(
          contentClassName,
          scrollable && "flex flex-col overflow-hidden",
        )}
        align="start"
        alignOffset={10}
        onInteractOutside={(event) => {
          // Radix sees the shadow host as the target. Resolve the original
          // target so pointer-down does not dismiss before click toggles.
          const target = Events.composedTarget(event.detail.originalEvent);
          if (triggerRef.current?.contains(target)) {
            event.preventDefault();
          }
        }}
      >
        <div
          className={cn(
            "flex -mt-3 -mr-2",
            scrollable ? "shrink-0 justify-end" : "float-right",
          )}
        >
          <CopyClipboardIcon
            value={rawStringValue}
            className="size-3 hover:text-link"
            buttonClassName="flex size-5 items-center justify-center"
            tooltip={false}
          />
          <PopoverClose asChild={true}>
            <Button
              variant="link"
              size="xs"
              className={cn("size-5 p-0", !buttonText && "ml-1.5 mr-1")}
              aria-label="Close"
            >
              {buttonText ?? "Close"}
            </Button>
          </PopoverClose>
        </div>
        {scrollable ? (
          <div className="min-h-0 overflow-auto">{children}</div>
        ) : (
          children
        )}
      </PopoverContent>
    </Popover>
  );
};

/**
 * Parse a string that holds a JSON object or array document.
 * Returns `undefined` for everything else, including JSON scalars,
 * so ordinary text keeps its plain rendering.
 */
function parseJsonDocument(text: string): unknown {
  const trimmedText = text.trim();

  const first = trimmedText[0];
  const last = trimmedText[trimmedText.length - 1];
  const isBracketPair =
    (first === "{" && last === "}") || (first === "[" && last === "]");
  if (!isBracketPair) {
    return undefined;
  }
  try {
    return JSON.parse(trimmedText) as unknown;
  } catch {
    return undefined;
  }
}

function isPrimitiveOrNullish(value: unknown): boolean {
  if (value == null) {
    return true;
  }
  const isObject = typeof value === "object";
  return !isObject;
}

function getFilterTypeForFieldType(
  type: DataType | undefined,
): FilterType | undefined {
  if (type === undefined) {
    return undefined;
  }
  switch (type) {
    case "string":
      return "text";
    case "number":
      return "number";
    case "integer":
      return "number";
    case "date":
      return "date";
    case "datetime":
      return "datetime";
    case "time":
      return "time";
    case "boolean":
      return "boolean";
    case "geometry":
      return undefined;
    default:
      return undefined;
  }
}

function getCellStyleClass({
  justify = "left",
  wrapped,
  canSelectCell,
  isSelected,
  isNumeric = false,
}: {
  justify: "left" | "center" | "right" | undefined;
  wrapped: boolean | undefined;
  canSelectCell: boolean;
  isSelected: boolean;
  isNumeric?: boolean;
}): string {
  return cn(
    canSelectCell && "cursor-pointer",
    isSelected &&
      "relative before:absolute before:inset-0 before:bg-(--blue-3) before:rounded before:-z-10 before:mx-[-4px] before:my-[-2px]",
    "w-full",
    "text-left",
    "truncate",
    isNumeric && "tabular-nums",
    justify === "center" && "text-center",
    justify === "right" && "text-right",
    wrapped && `${COLUMN_WRAPPING_STYLES} wrap-break-word`,
  );
}

function renderAny(value: unknown): string {
  if (value == null) {
    return "";
  }

  try {
    return JSON.stringify(value);
  } catch {
    return String(value);
  }
}

function renderDate({
  value,
  dataType,
  dtype,
  displayValue,
  locale,
  highlightPattern,
}: {
  value: Date;
  dataType?: DataType;
  dtype?: string;
  displayValue?: string;
  locale: string;
  highlightPattern?: RegExp;
}): React.ReactNode {
  const type = dataType === "date" ? "date" : "datetime";
  const timezone = extractTimezone(dtype);

  const exactValue = displayValue ?? exactDateTime(value, timezone, locale);

  return (
    <DatePopover date={value} type={type}>
      {highlightMatches(exactValue, highlightPattern)}
    </DatePopover>
  );
}

export function renderCellValue<TData, TValue>({
  column,
  renderValue,
  getValue,
  selectCell,
  cellStyles,
  highlightPattern,
}: {
  column: Column<TData, TValue>;
  renderValue: () => TValue | null;
  getValue: () => TValue;
  selectCell?: () => void;
  cellStyles?: string;
  highlightPattern?: RegExp;
}) {
  const value = getValue();
  const format = column.getColumnFormatting?.();

  const dataType = column.columnDef.meta?.dataType;
  const dtype = column.columnDef.meta?.dtype;

  const isWrapped = column.getColumnWrapping?.() === "wrap";

  const textCell = (text: string) => (
    <div onClick={selectCell} className={cellStyles}>
      {highlightMatches(text, highlightPattern)}
    </div>
  );
  const dateCell = (date: Date, displayValue?: string) => (
    <div onClick={selectCell} className={cellStyles}>
      <WithLocale>
        {(locale) =>
          renderDate({
            value: date,
            dataType,
            dtype,
            displayValue,
            locale,
            highlightPattern,
          })
        }
      </WithLocale>
    </div>
  );

  // Sentinel values (null, whitespace, NaN, Infinity, NaT) rendered specially.
  // Empty strings are left as-is
  const sentinel = detectSentinel(value, dataType, dtype);
  if (sentinel && sentinel.type !== "empty-string") {
    return (
      <div onClick={selectCell} className={cellStyles}>
        <SentinelCell sentinel={sentinel} />
      </div>
    );
  }

  if (dataType === "datetime" && typeof value === "string") {
    try {
      const date = new Date(value);
      if (!format || !isValid(date)) {
        return textCell(value);
      }

      return dateCell(date, String(column.applyColumnFormatting(value)));
    } catch (error) {
      Logger.error("Error parsing datetime, fallback to string", error);
    }
  }

  if (value instanceof Date) {
    if (!isValid(value)) {
      return (
        <div onClick={selectCell} className={cellStyles}>
          {value.toString()}
        </div>
      );
    }

    // e.g. 2010-10-07 17:15:00
    return dateCell(
      value,
      format ? String(column.applyColumnFormatting(value)) : undefined,
    );
  }

  if (typeof value === "string") {
    const stringValue = format
      ? String(column.applyColumnFormatting(value))
      : String(renderValue());

    const { leading, middle, trailing } =
      splitLeadingTrailingWhitespace(stringValue);
    const hasEdgeWhitespace = leading.length > 0 || trailing.length > 0;

    // A str cell that holds a JSON document opens the same JSON viewer as
    // native list/struct cells.
    const jsonDocument = parseJsonDocument(middle);
    if (jsonDocument !== undefined) {
      return (
        <PopoutColumn
          cellStyles={cellStyles}
          selectCell={selectCell}
          rawStringValue={stringValue}
          edges={{ leading, trailing }}
          wrapped={isWrapped}
        >
          <JsonOutput
            data={jsonDocument}
            format="tree"
            valueTypes="json"
            className="max-h-64"
          />
        </PopoutColumn>
      );
    }

    // Parse only the inner content for URL detection so URLDetector doesn't
    // split on the whitespace padding.
    const parts = parseContent(hasEdgeWhitespace ? middle : stringValue);
    const allMarkup = parts.every((part) => part.type !== "text");
    if (allMarkup || stringValue.length < MAX_STRING_LENGTH || isWrapped) {
      return (
        <div
          onClick={selectCell}
          className={cn(cellStyles, isWrapped && COLUMN_WRAPPING_STYLES)}
        >
          <WhitespaceMarkers value={leading} />
          <UrlDetector parts={parts} highlightPattern={highlightPattern} />
          <WhitespaceMarkers value={trailing} />
        </div>
      );
    }

    return (
      <PopoutColumn
        cellStyles={cellStyles}
        selectCell={selectCell}
        rawStringValue={stringValue}
        edges={{ leading, trailing }}
        contentClassName="max-h-64 whitespace-pre-wrap wrap-break-word text-sm w-96"
        scrollable={true}
        wrapped={isWrapped}
        buttonText="X"
      >
        <MarkdownUrlDetector
          content={stringValue}
          parts={parseContent(stringValue)}
        />
      </PopoutColumn>
    );
  }

  if (format) {
    return textCell(String(column.applyColumnFormatting(value)));
  }

  // Format to the correct locale
  if (typeof value === "number") {
    return (
      <div onClick={selectCell} className={cellStyles}>
        <LocaleNumber
          value={value}
          minFractionDigits={column.columnDef.meta?.minFractionDigits}
          highlightPattern={highlightPattern}
        />
      </div>
    );
  }

  if (typeof value === "boolean") {
    return textCell(value ? "True" : "False");
  }

  if (isPrimitiveOrNullish(value)) {
    const rendered = renderValue();
    return textCell(rendered == null ? "" : String(rendered));
  }

  const mimeValues = getMimeValues(value);
  if (mimeValues) {
    return (
      <div onClick={selectCell} className={cellStyles}>
        {mimeValues.map((mimeValue, idx) => (
          <MimeCell key={idx} value={mimeValue} />
        ))}
      </div>
    );
  }

  if (Array.isArray(value) || typeof value === "object") {
    const rawStringValue = renderAny(value);
    return (
      <PopoutColumn
        cellStyles={cellStyles}
        selectCell={selectCell}
        rawStringValue={rawStringValue}
        wrapped={isWrapped}
      >
        <JsonOutput data={value} format="tree" className="max-h-64" />
      </PopoutColumn>
    );
  }

  return (
    <div onClick={selectCell} className={cellStyles}>
      {renderAny(value)}
    </div>
  );
}

export const LocaleNumber = ({
  value,
  minFractionDigits,
  highlightPattern,
}: {
  value: number;
  minFractionDigits?: number;
  highlightPattern?: RegExp;
}) => {
  const { locale } = useLocale();
  const format = useNumberFormatter({
    minimumFractionDigits: minFractionDigits,
    maximumFractionDigits: maxFractionalDigits(locale),
  });
  return highlightMatches(format.format(value), highlightPattern);
};