TimesFM (Time Series Foundation Model) is a pretrained time-series foundation model developed by Google Research for time-series forecasting.
Python
# Copyright 2026 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""TimesFM3 PyTorch model (inference only).
Supports:
- forward(): equivalent to Flax __call__, full-sequence forward pass.
- decode(): non-autoregressive and cached decoding with frozen stats and
configurable output_patch_len.
"""
from __future__ import annotations
import dataclasses
import math
from typing import Any
import torch
from huggingface_hub import PyTorchModelHubMixin
from torch import nn
from . import configs, dense, transformer, util
from . import cpm_revin_refine as cpm_revin_refine_lib
class TimesFM3Torch(
nn.Module,
PyTorchModelHubMixin,
repo_url="https://github.com/google-research/timesfm",
):
"""PyTorch inference-only implementation of TimesFM3.
Attributes:
input_patch_len: The length of each input patch.
output_patch_len: The length of the output forecast for each patch.
quantiles: A list of quantiles to predict.
residual_block_config: Configuration for the pre-transformer residual block.
transformer_config: Configuration for the stacked transformers.
use_variate_attention: Whether to use variate attention.
value_clip: Absolute value to clip input values to.
use_stitching: Whether to use stitching for predictions.
use_linear_detrending: Whether to apply linear detrending on context.
linear_detrending_threshold: Ratio threshold for applying linear detrending.
use_iterative_cpm_revin: Whether to use iterative RevIN refinement.
use_frozen_running_stats: Whether running stats freeze at context boundary.
"""
def __init__(
self,
input_patch_len: int = 32,
output_patch_len: int = 64,
quantiles: list[float] | None = None,
residual_block_config: configs.ResidualBlockConfig | dict[str, Any] | None = None,
transformer_config: configs.StackedTransformersConfig
| dict[str, Any]
| None = None,
use_variate_attention: bool = True,
value_clip: float = 1e20,
use_stitching: bool = True,
use_linear_detrending: bool = True,
linear_detrending_threshold: float = 0.5,
use_iterative_cpm_revin: bool = True,
use_frozen_running_stats: bool = False,
input_transform: str = "identity",
):
super().__init__()
if quantiles is None:
quantiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
if residual_block_config is None:
residual_block_config = configs.ResidualBlockConfig(
hidden_dims=1280,
output_dims=1280,
use_bias=False,
activation="relu",
)
elif isinstance(residual_block_config, dict):
residual_block_config = configs.ResidualBlockConfig(**residual_block_config)
if transformer_config is None:
transformer_config = configs.StackedTransformersConfig(
num_layers=20,
transformer=configs.TransformerConfig(
model_dims=1280,
hidden_dims=1280,
num_heads=16,
attention_norm="rms",
feedforward_norm="rms",
qk_norm="rms",
use_rope_seq=True,
use_rope_var=False,
use_bias=False,
ff_activation="relu",
deterministic=True,
),
)
elif isinstance(transformer_config, dict):
t_data = transformer_config.get("transformer", {})
if isinstance(t_data, dict):
t_cfg = configs.TransformerConfig(**t_data)
else:
t_cfg = t_data
tc_dict = dict(transformer_config)
tc_dict["transformer"] = t_cfg
transformer_config = configs.StackedTransformersConfig(**tc_dict)
if output_patch_len % input_patch_len != 0:
raise ValueError(
f"Output patch len {output_patch_len} must be a multiple of"
f" input patch len {input_patch_len}."
)
if residual_block_config.output_dims != transformer_config.transformer.model_dims:
raise ValueError("ResidualBlock output_dims must match Transformer model_dims.")
self.input_patch_len = input_patch_len
self.output_patch_len = output_patch_len
self.quantiles = quantiles
self.num_quantiles = len(quantiles)
self.rolls = output_patch_len // input_patch_len
self.residual_block_config = residual_block_config
self.transformer_config = transformer_config
self.use_variate_attention = use_variate_attention
self.value_clip = value_clip
self.use_stitching = use_stitching
self.use_linear_detrending = use_linear_detrending
self.linear_detrending_threshold = linear_detrending_threshold
self.use_iterative_cpm_revin = use_iterative_cpm_revin
self.use_frozen_running_stats = use_frozen_running_stats
self.input_transform = input_transform
if self.use_stitching:
if self.output_patch_len <= self.input_patch_len:
raise ValueError("use_stitching requires output_patch_len > input_patch_len")
self._stitching_extract_len = min(2 * self.input_patch_len, self.output_patch_len)
self.pre_transformer_resblock = dense.ResidualBlock(config=residual_block_config)
self.pre_transformer_resblock.set_input_dims(
2 * (input_patch_len + output_patch_len)
)
self.transformer_stack = transformer.StackedMixingTransformer(
config=transformer_config,
use_variate_attention=use_variate_attention,
)
self.output_head = nn.Linear(
transformer_config.transformer.model_dims,
output_patch_len * self.num_quantiles,
bias=True,
)
def to_dict(self) -> dict[str, Any]:
"""Returns a serializable dictionary of model configuration."""
return {
"input_patch_len": self.input_patch_len,
"output_patch_len": self.output_patch_len,
"quantiles": list(self.quantiles),
"residual_block_config": dataclasses.asdict(self.residual_block_config),
"transformer_config": dataclasses.asdict(self.transformer_config),
"use_variate_attention": self.use_variate_attention,
"value_clip": self.value_clip,
"use_stitching": self.use_stitching,
"use_linear_detrending": self.use_linear_detrending,
"linear_detrending_threshold": self.linear_detrending_threshold,
"use_iterative_cpm_revin": self.use_iterative_cpm_revin,
"use_frozen_running_stats": self.use_frozen_running_stats,
"input_transform": self.input_transform,
}
def save_pretrained(
self,
save_directory: str | Any,
*,
config: dict[str, Any] | None = None,
**kwargs: Any,
) -> str | None:
"""Saves model weights and config to a local directory or pushes to Hub."""
if config is None:
config = self.to_dict()
return super().save_pretrained(save_directory, config=config, **kwargs)
def _preprocess(
self,
values: torch.Tensor,
masks: torch.Tensor,
patch_is_target: torch.Tensor,
freeze_after: int | None = None,
patch_cpm_mask: torch.Tensor | None = None,
) -> tuple[
torch.Tensor,
torch.Tensor,
torch.Tensor,
tuple[torch.Tensor, torch.Tensor],
torch.Tensor,
]:
"""Applies preprocessing: RevIN, masking, future covariates, ResBlock.
Args:
values: (b, v, n, p).
masks: (b, v, n, p) bool. True=masked.
patch_is_target: (b, v, n) bool.
freeze_after: Optional patch index after which running stats freeze.
patch_cpm_mask: (b, n) bool or None. If given, target variates at True
positions are additionally masked (used for horizon CPM masking).
Returns:
(resblock_input, resblock_output, patch_mask, (running_mean, running_std),
running_n)
"""
running_n, running_mean, running_std = util.get_running_stats(values, masks)
if freeze_after is not None:
_, _, n, _ = values.shape
if 0 <= freeze_after < n - 1:
running_mean[:, :, freeze_after + 1 :] = running_mean[
:, :, freeze_after : freeze_after + 1
]
running_std[:, :, freeze_after + 1 :] = running_std[
:, :, freeze_after : freeze_after + 1
]
# Apply CPM mask: mask target variates at CPM positions.
if patch_cpm_mask is not None:
cpm_bvnp = patch_cpm_mask[:, None, :, None] # (b, 1, n, 1)
cpm_target_only = cpm_bvnp & patch_is_target.unsqueeze(-1)
masks = masks | cpm_target_only
values_bvnp = util.revin(values, running_mean, running_std, reverse=False)
values_bvnp = torch.where(masks, 0.0, values_bvnp)
# Roll values to get future covariate patches
values_fcov, wrap_mask = util.get_output_patch_via_roll(values, self.rolls)
values_fcov = util.revin(values_fcov, running_mean, running_std, reverse=False)
# Roll the (CPM-modified) masks for future covariate masking.
masks_fcov_raw, _ = util.get_output_patch_via_roll(masks, self.rolls)
masks_fcov = masks_fcov_raw | patch_is_target.unsqueeze(-1) | wrap_mask
values_fcov = torch.where(masks_fcov, 0.0, values_fcov)
values_cat = torch.cat([values_bvnp, values_fcov], dim=-1)
masks_cat = torch.cat([masks, masks_fcov], dim=-1)
resblock_input = torch.cat([values_cat, masks_cat.float()], dim=-1)
resblock_output = self.pre_transformer_resblock(resblock_input)
# Patch mask: a patch is fully masked if ALL points are masked
patch_mask_bvn = masks_cat.all(dim=3)
return (
resblock_input,
resblock_output,
patch_mask_bvn,
(running_mean, running_std),
running_n,
)
def forward(
self,
inputs: dict[str, Any],
freeze_after: int | None = None,
patch_cpm_mask: torch.Tensor | None = None,
return_aux_outputs: bool = False,
) -> dict[str, Any]:
"""Full-sequence forward pass (equivalent to Flax __call__).
Args:
inputs: Dictionary with keys: - "values": (b, v, n, p) float - "masks":
(b, v, n, p) bool - "patch_is_target": (b, v, n) bool
freeze_after: Optional patch index after which running stats freeze.
patch_cpm_mask: (b, n) bool or None. Horizon CPM mask.
return_aux_outputs: Whether to return auxiliary outputs.
Returns:
Dictionary with "logits" of shape (b, v, n, output_patch_len,
num_quantiles).
"""
values = inputs["values"]
values = torch.nan_to_num(values, nan=0.0)
values = torch.clamp(values, -self.value_clip, self.value_clip)
masks = inputs["masks"].bool()
patch_is_target = inputs["patch_is_target"]
_, _, _, p = values.shape
if p != self.input_patch_len:
raise ValueError(
f"Input patch_len {p} != model input_patch_len {self.input_patch_len}"
)
# Preprocessing & ResBlock
(
resblock_input,
transformer_input,
transformer_patch_mask,
revin_stats,
running_n,
) = self._preprocess(
values,
masks,
patch_is_target,
freeze_after=freeze_after,
patch_cpm_mask=patch_cpm_mask,
)
# Transformer
# At inference, only mask *leading* fully-masked patches (left-padding).
# Flax __call__ does: effective_patch_mask = cumprod(mask, axis=2) when
# not training. This keeps horizon patches (which are fully masked but
# come after valid context) visible to attention.
effective_patch_mask = torch.cumprod(transformer_patch_mask.int(), dim=2).bool()
transformer_output, _, seq_attn_mask = self.transformer_stack(
transformer_input,
effective_patch_mask,
)
# Output head
raw_logits = self.output_head(transformer_output)
revin_mean, revin_std = revin_stats
if self.use_iterative_cpm_revin and patch_cpm_mask is not None:
refined_mu, refined_sigma = cpm_revin_refine_lib.cpm_iterative_revin_refine(
raw_logits,
revin_n=running_n,
revin_mu=revin_mean,
revin_sigma=revin_std,
patch_cpm_mask=patch_cpm_mask,
median_q_idx=self.num_quantiles // 2,
rolls=self.rolls,
patch_len=self.input_patch_len,
num_quantiles=self.num_quantiles,
value_clip=self.value_clip,
)
cpm_bvn = patch_cpm_mask.unsqueeze(1) # (b, 1, n)
revin_mean = torch.where(cpm_bvn, refined_mu, revin_mean)
revin_std = torch.where(cpm_bvn, refined_sigma, revin_std)
revin_logits = util.revin(raw_logits, revin_mean, revin_std, reverse=True)
clipped_logits = torch.clamp(revin_logits, -self.value_clip, self.value_clip)
# Reshape: (b, v, n, o*q) -> (b, v, n, o, q)
b, v, n_patches = clipped_logits.shape[:3]
final_logits = clipped_logits.view(
b, v, n_patches, self.output_patch_len, self.num_quantiles
)
outputs = {"logits": final_logits, "revin_stats": revin_stats}
if return_aux_outputs:
outputs["__call__:resblock_input"] = resblock_input
outputs["__call__:transformer_input"] = transformer_input
outputs["__call__:seq_attn_mask"] = seq_attn_mask
outputs["__call__:transformer_output"] = transformer_output
return outputs
@torch.no_grad()
def decode(
self,
target: torch.Tensor,
horizon: int = 0,
past_only_covariates: torch.Tensor | None = None,
past_future_covariates: torch.Tensor | None = None,
target_mask: torch.Tensor | None = None,
past_only_mask: torch.Tensor | None = None,
past_future_mask: torch.Tensor | None = None,
mask: torch.Tensor | None = None,
return_aux_outputs: bool = False,
) -> Any:
"""Non-autoregressive single-pass decoding for TimesFM3.
Args:
target: (b, u, context_len).
horizon: Forecast horizon. Inferred if past_future_covariates given.
past_only_covariates: (b, v_po, context_len) or None.
past_future_covariates: (b, w, context_len+horizon) or None.
target_mask: (b, u, context_len) bool or None.
past_only_mask: (b, v_po, context_len) bool or None.
past_future_mask: (b, w, context_len+horizon) bool or None.
mask: (b, context_len) global bool mask or None.
return_aux_outputs: If True, return (logits, aux_dict).
Returns:
Logits of shape (b, num_variates, horizon, num_quantiles).
"""
device = target.device
batch_size, num_target, context = target.shape
if past_future_covariates is not None:
horizon = past_future_covariates.shape[-1] - context
if horizon <= 0:
raise ValueError("Decode function requires horizon > 0.")
# 1. Pad context to multiple of input_patch_len
ctx_padding = (
self.input_patch_len - (context % self.input_patch_len)
) % self.input_patch_len
if ctx_padding > 0:
target = torch.nn.functional.pad(target, (ctx_padding, 0))
if mask is not None:
mask = torch.nn.functional.pad(mask, (ctx_padding, 0), value=True)
if past_only_covariates is not None:
past_only_covariates = torch.nn.functional.pad(
past_only_covariates, (ctx_padding, 0)
)
if past_future_covariates is not None:
past_future_covariates = torch.nn.functional.pad(
past_future_covariates, (ctx_padding, 0)
)
if target_mask is not None:
target_mask = torch.nn.functional.pad(target_mask, (ctx_padding, 0), value=True)
if past_only_mask is not None:
past_only_mask = torch.nn.functional.pad(
past_only_mask, (ctx_padding, 0), value=True
)
if past_future_mask is not None:
past_future_mask = torch.nn.functional.pad(
past_future_mask, (ctx_padding, 0), value=True
)
context = context + ctx_padding
if mask is None:
mask = torch.zeros(batch_size, context, dtype=torch.bool, device=device)
if ctx_padding > 0:
mask[:, :ctx_padding] = True
# 2. Pad horizon
if self.use_stitching:
extract_len = self._stitching_extract_len
overlap = extract_len - self.input_patch_len
num_forecast_patches = max(
math.ceil((horizon - overlap) / self.input_patch_len), 1
)
num_horizon_patches = num_forecast_patches + self.rolls - 1
padded_horizon = num_horizon_patches * self.input_patch_len
hor_padding = padded_horizon - horizon
else:
hor_padding = (-horizon) % self.output_patch_len
padded_horizon = horizon + hor_padding
num_horizon_patches = padded_horizon // self.input_patch_len
num_context_patches = context // self.input_patch_len
# 3. Build context & horizon inputs
if target_mask is None:
target_mask = torch.zeros_like(target, dtype=torch.bool)
target_mask = target_mask | mask.unsqueeze(1)
all_ctx_vals = [target]
all_ctx_masks = [target_mask]
num_past_only = 0
if past_only_covariates is not None:
num_past_only = past_only_covariates.shape[1]
if past_only_mask is None:
past_only_mask = torch.zeros_like(past_only_covariates, dtype=torch.bool)
all_ctx_vals.append(past_only_covariates)
all_ctx_masks.append(past_only_mask | mask.unsqueeze(1))
if past_future_covariates is not None:
if past_future_mask is None:
past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)
all_ctx_vals.append(past_future_covariates[..., :context])
all_ctx_masks.append(past_future_mask[..., :context] | mask.unsqueeze(1))
ctx_vals = torch.cat(all_ctx_vals, dim=1)
ctx_masks = torch.cat(all_ctx_masks, dim=1)
if self.use_linear_detrending:
t_ctx = torch.arange(-(context - 1), 1, dtype=torch.float32, device=device)
t_ctx_bvc = t_ctx[None, None, :]
t_ctx_bvc_normalized = t_ctx_bvc / context
valid = ~ctx_masks
n_v = valid.float().sum(dim=-1, keepdim=True)
sum_t = torch.where(valid, t_ctx_bvc_normalized, 0.0).sum(dim=-1, keepdim=True)
sum_t2 = torch.where(valid, t_ctx_bvc_normalized**2, 0.0).sum(
dim=-1, keepdim=True
)
sum_y = torch.where(valid, ctx_vals, 0.0).sum(dim=-1, keepdim=True)
sum_ty = torch.where(valid, t_ctx_bvc_normalized * ctx_vals, 0.0).sum(
dim=-1, keepdim=True
)
det = n_v * sum_t2 - sum_t**2
safe_det = torch.where(det == 0.0, 1.0, det)
m_trend = torch.where(det == 0.0, 0.0, (n_v * sum_ty - sum_t * sum_y) / safe_det)
c_trend = torch.where(
det == 0.0,
torch.where(n_v > 0, sum_y / torch.clamp_min(n_v, 1.0), 0.0),
(sum_y - m_trend * sum_t) / torch.clamp_min(n_v, 1.0),
)
ctx_vals_detrended = ctx_vals - (m_trend * t_ctx_bvc_normalized + c_trend)
mean_y = sum_y / torch.clamp_min(n_v, 1.0)
sum_y2 = torch.where(valid, ctx_vals**2, 0.0).sum(dim=-1, keepdim=True)
var_orig = torch.clamp_min(sum_y2 / torch.clamp_min(n_v, 1.0) - mean_y**2, 0.0)
std_orig = torch.sqrt(var_orig)
sum_yd = torch.where(valid, ctx_vals_detrended, 0.0).sum(dim=-1, keepdim=True)
mean_yd = sum_yd / torch.clamp_min(n_v, 1.0)
sum_yd2 = torch.where(valid, ctx_vals_detrended**2, 0.0).sum(dim=-1, keepdim=True)
var_det = torch.clamp_min(sum_yd2 / torch.clamp_min(n_v, 1.0) - mean_yd**2, 0.0)
std_det = torch.sqrt(var_det)
apply_detrend = std_det < self.linear_detrending_threshold * std_orig
ctx_vals = torch.where(apply_detrend, ctx_vals_detrended, ctx_vals)
else:
num_variates = ctx_vals.shape[1]
m_trend = torch.zeros(
(batch_size, num_variates, 1), dtype=torch.float32, device=device
)
c_trend = torch.zeros(
(batch_size, num_variates, 1), dtype=torch.float32, device=device
)
apply_detrend = torch.zeros(
(batch_size, num_variates, 1), dtype=torch.bool, device=device
)
ctx_vals = torch.where(ctx_masks, 0.0, ctx_vals)
all_hor_vals = [
torch.zeros(batch_size, num_target, padded_horizon, device=device),
torch.zeros(batch_size, num_past_only, padded_horizon, device=device),
]
all_hor_masks = [
torch.ones(
batch_size,
num_target,
padded_horizon,
dtype=torch.bool,
device=device,
),
torch.ones(
batch_size,
num_past_only,
padded_horizon,
dtype=torch.bool,
device=device,
),
]
if past_future_covariates is not None:
if past_future_mask is None:
past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)
pf_future_vals = past_future_covariates[..., context : context + horizon]
pf_future_masks = past_future_mask[..., context : context + horizon]
if self.use_linear_detrending:
m_pf = m_trend[:, num_target + num_past_only :, :]
c_pf = c_trend[:, num_target + num_past_only :, :]
apply_detrend_pf = apply_detrend[:, num_target + num_past_only :, :]
t_hor_pf = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)[
None, None, :
]
t_hor_pf_normalized = t_hor_pf / context
pf_trend_hor = m_pf * t_hor_pf_normalized + c_pf
pf_future_vals = torch.where(
apply_detrend_pf, pf_future_vals - pf_trend_hor, pf_future_vals
)
pf_future_vals = torch.where(pf_future_masks, 0.0, pf_future_vals)
if hor_padding > 0:
pf_future_vals = torch.nn.functional.pad(pf_future_vals, (0, hor_padding))
pf_future_masks = torch.nn.functional.pad(
pf_future_masks, (0, hor_padding), value=True
)
all_hor_vals.append(pf_future_vals)
all_hor_masks.append(pf_future_masks)
hor_vals = torch.cat(all_hor_vals, dim=1)
hor_masks = torch.cat(all_hor_masks, dim=1)
all_vals = torch.cat([ctx_vals, hor_vals], dim=-1)
all_masks = torch.cat([ctx_masks, hor_masks], dim=-1)
num_variates = all_vals.shape[1]
patch_is_target = torch.zeros(
(batch_size, num_variates, num_context_patches + num_horizon_patches),
dtype=torch.bool,
device=device,
)
patch_is_target[:, : num_target + num_past_only, :] = True
# Reshape values & masks to patched shape (b, v, n, p)
values_bvnp = all_vals.reshape(batch_size, num_variates, -1, self.input_patch_len)
masks_bvnp = all_masks.reshape(batch_size, num_variates, -1, self.input_patch_len)
inputs = {
"values": values_bvnp,
"masks": masks_bvnp,
"patch_is_target": patch_is_target,
}
# Build horizon CPM mask: context=False, horizon=True.
num_total_patches = num_context_patches + num_horizon_patches
horizon_cpm_mask = torch.zeros(
batch_size, num_total_patches, dtype=torch.bool, device=device
)
horizon_cpm_mask[:, num_context_patches:] = True
freeze_after = num_context_patches - 1 if self.use_frozen_running_stats else None
forward_out = self.forward(
inputs,
freeze_after=freeze_after,
patch_cpm_mask=horizon_cpm_mask,
return_aux_outputs=return_aux_outputs,
)
logits = forward_out["logits"] # (b, v, n, output_patch_len, num_quantiles)
if self.use_stitching:
extract_len = self._stitching_extract_len
overlap = extract_len - self.input_patch_len
num_forecast_patches = max(
math.ceil((horizon - overlap) / self.input_patch_len), 1
)
forecast_indices = torch.arange(num_forecast_patches, device=device) + (
num_context_patches - 1
)
patch_preds = logits[:, :, forecast_indices, :extract_len, :]
horizon_logits = util.stitch_patches(
patch_preds,
self.input_patch_len,
)[:, :, :horizon, :]
else:
num_forecast_chunks = padded_horizon // self.output_patch_len
forecast_indices = torch.arange(
num_forecast_chunks, device=device
) * self.rolls + (num_context_patches - 1)
forecast_logits = logits[:, :, forecast_indices, :, :]
horizon_logits = forecast_logits.reshape(
batch_size, num_variates, -1, self.num_quantiles
)[:, :, :horizon, :]
if self.use_linear_detrending:
t_forecast = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)
t_forecast_normalized = t_forecast / context
trend_forecast = (
m_trend[:, :, 0, None] * t_forecast_normalized[None, None, :]
+ c_trend[:, :, 0, None]
)
trend_forecast = torch.where(apply_detrend[:, :, 0, None], trend_forecast, 0.0)
horizon_logits = horizon_logits + trend_forecast[:, :, :, None]
if return_aux_outputs:
return horizon_logits, forward_out
return horizon_logits