TimesFM (Time Series Foundation Model) is a pretrained time-series foundation model developed by Google Research for time-series forecasting.
Python
enable hugginface support
Rajat Sen29a5ca6531eb
diff --git a/pyproject.toml b/pyproject.tomlindex 743b737..041b25d 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -15,7 +15,7 @@ readme = "README.md"requires-python = ">=3.10"dependencies = ["numpy>=1.26.4",- "huggingface_hub[cli]>=0.23.0",+ "huggingface_hub>=0.28.0","safetensors>=0.5.3",]diff --git a/src/timesfm3/__init__.py b/src/timesfm3/__init__.pyindex b4a70e6..380acce 100644--- a/src/timesfm3/__init__.py+++ b/src/timesfm3/__init__.py@@ -14,22 +14,22 @@"""TimesFM3 PyTorch API."""-from .configs import ResidualBlockConfig-from .configs import StackedTransformersConfig-from .configs import TransformerConfig+from .configs import ResidualBlockConfig, StackedTransformersConfig, TransformerConfigfrom .model import TimesFM3Torch-from .timesfm3_forecaster import ForecastOutput-from .timesfm3_forecaster import ModelConfig-from .timesfm3_forecaster import TimesFM3Forecaster-from .timesfm3_forecaster import _ModelConfig+from .timesfm3_forecaster import (+ ForecastOutput,+ ModelConfig,+ TimesFM3Forecaster,+ _ModelConfig,+)__all__ = [- "TimesFM3Torch",- "TimesFM3Forecaster",- "ForecastOutput",- "_ModelConfig",- "ModelConfig",- "ResidualBlockConfig",- "TransformerConfig",- "StackedTransformersConfig",+ "ForecastOutput",+ "ModelConfig",+ "ResidualBlockConfig",+ "StackedTransformersConfig",+ "TimesFM3Forecaster",+ "TimesFM3Torch",+ "TransformerConfig",+ "_ModelConfig",]diff --git a/src/timesfm3/cpm_revin_refine.py b/src/timesfm3/cpm_revin_refine.pyindex 753225f..30603ff 100644--- a/src/timesfm3/cpm_revin_refine.py+++ b/src/timesfm3/cpm_revin_refine.py@@ -11,16 +11,16 @@ from . import utildef cpm_iterative_revin_refine(- raw_logits: torch.Tensor,- revin_n: torch.Tensor,- revin_mu: torch.Tensor,- revin_sigma: torch.Tensor,- patch_cpm_mask: torch.Tensor,- median_q_idx: int,- rolls: int,- patch_len: int,- num_quantiles: int,- value_clip: float = 1e9,+ raw_logits: torch.Tensor,+ revin_n: torch.Tensor,+ revin_mu: torch.Tensor,+ revin_sigma: torch.Tensor,+ patch_cpm_mask: torch.Tensor,+ median_q_idx: int,+ rolls: int,+ patch_len: int,+ num_quantiles: int,+ value_clip: float = 1e9,) -> tuple[torch.Tensor, torch.Tensor]:"""Refines RevIN stats at CPM-masked patches via iterative estimation.@@ -57,16 +57,16 @@ def cpm_iterative_revin_refine(# Reshape and slice raw_logits to keep only the median quantile.# (b, v, n, oq) -> (b, v, n, rolls, patch_len, num_quantiles)# -> (b, v, n, rolls, patch_len)- median_logits = raw_logits.reshape(- b, v, n_patches, rolls, patch_len, num_quantiles- )[:, :, :, :, :, median_q_idx]+ median_logits = raw_logits.reshape(b, v, n_patches, rolls, patch_len, num_quantiles)[+ :, :, :, :, :, median_q_idx+ ]# Initialise carry with zeros.carry_n = torch.zeros((b, v), dtype=torch.float32, device=device)carry_mu = torch.zeros((b, v), dtype=torch.float32, device=device)carry_sigma = torch.zeros((b, v), dtype=torch.float32, device=device)anchor_predicted_values = torch.zeros(- (b, v, rolls, patch_len), dtype=torch.float32, device=device+ (b, v, rolls, patch_len), dtype=torch.float32, device=device)block_offset = torch.zeros((b,), dtype=torch.long, device=device)@@ -84,16 +84,16 @@ def cpm_iterative_revin_refine(# Select the block_offset[b]-th patch for each batch elementoffset_onehot = torch.eq(- torch.arange(rolls, device=device).unsqueeze(0),- block_offset.unsqueeze(1),+ torch.arange(rolls, device=device).unsqueeze(0),+ block_offset.unsqueeze(1),).float()predicted_values_step = torch.einsum(- "br,bvrp->bvp", offset_onehot, anchor_predicted_values+ "br,bvrp->bvp", offset_onehot, anchor_predicted_values)# Update running stats with the estimated patch.new_n, new_mu, new_sigma = util.update_running_stats(- carry_n, carry_mu, carry_sigma, predicted_values_step, step_masks+ carry_n, carry_mu, carry_sigma, predicted_values_step, step_masks)out_n = torch.where(is_cpm, new_n, actual_n)@@ -102,25 +102,23 @@ def cpm_iterative_revin_refine(# Advance block_offset: +1 (mod rolls) for CPM, reset to 0 for non-CPM.new_block_offset = torch.where(- is_cpm.squeeze(-1),- (block_offset + 1) % rolls,- torch.zeros_like(block_offset),+ is_cpm.squeeze(-1),+ (block_offset + 1) % rolls,+ torch.zeros_like(block_offset),)should_update_anchor = torch.eq(new_block_offset, 0)# Pre-calculate predicted values for the new anchor.step_predicted_values = util.revin(- current_step_logits, out_mu, out_sigma, reverse=True- )- step_predicted_values = torch.clamp(- step_predicted_values, -value_clip, value_clip+ current_step_logits, out_mu, out_sigma, reverse=True)+ step_predicted_values = torch.clamp(step_predicted_values, -value_clip, value_clip)new_anchor_predicted_values = torch.where(- should_update_anchor.view(b, 1, 1, 1),- step_predicted_values,- anchor_predicted_values,+ should_update_anchor.view(b, 1, 1, 1),+ step_predicted_values,+ anchor_predicted_values,)carry_n = out_ndiff --git a/src/timesfm3/cpm_revin_refine_test.py b/src/timesfm3/cpm_revin_refine_test.pyindex 4b68719..c8be5aa 100644--- a/src/timesfm3/cpm_revin_refine_test.py+++ b/src/timesfm3/cpm_revin_refine_test.py@@ -15,6 +15,7 @@"""Tests for PyTorch cpm_iterative_revin_refine."""import unittest+import numpy as npimport torch@@ -22,7 +23,6 @@ from . import cpm_revin_refine as torch_libclass CpmRevinRefineTest(unittest.TestCase):-def test_shape(self):b, v, n = 2, 3, 8rolls, patch_len, num_q = 2, 2, 3@@ -34,15 +34,15 @@ class CpmRevinRefineTest(unittest.TestCase):patch_cpm_mask = torch.zeros((b, n), dtype=torch.bool)refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(- raw_logits=raw_logits,- revin_n=revin_n,- revin_mu=revin_mu,- revin_sigma=revin_sigma,- patch_cpm_mask=patch_cpm_mask,- median_q_idx=num_q // 2,- rolls=rolls,- patch_len=patch_len,- num_quantiles=num_q,+ raw_logits=raw_logits,+ revin_n=revin_n,+ revin_mu=revin_mu,+ revin_sigma=revin_sigma,+ patch_cpm_mask=patch_cpm_mask,+ median_q_idx=num_q // 2,+ rolls=rolls,+ patch_len=patch_len,+ num_quantiles=num_q,)self.assertEqual(refined_mu.shape, (b, v, n))self.assertEqual(refined_sigma.shape, (b, v, n))@@ -59,24 +59,24 @@ class CpmRevinRefineTest(unittest.TestCase):patch_cpm_mask = torch.zeros((b, n), dtype=torch.bool)refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(- raw_logits=raw_logits,- revin_n=revin_n,- revin_mu=revin_mu,- revin_sigma=revin_sigma,- patch_cpm_mask=patch_cpm_mask,- median_q_idx=num_q // 2,- rolls=rolls,- patch_len=patch_len,- num_quantiles=num_q,+ raw_logits=raw_logits,+ revin_n=revin_n,+ revin_mu=revin_mu,+ revin_sigma=revin_sigma,+ patch_cpm_mask=patch_cpm_mask,+ median_q_idx=num_q // 2,+ rolls=rolls,+ patch_len=patch_len,+ num_quantiles=num_q,)np.testing.assert_allclose(refined_mu.numpy(), revin_mu.numpy())np.testing.assert_allclose(refined_sigma.numpy(), revin_sigma.numpy())def test_cpm_mask_modifies_cpm_positions_only(self):for b, v, n, rolls, patch_len, num_q in [- (1, 1, 8, 2, 4, 3),- (2, 3, 16, 2, 32, 9),- (2, 2, 12, 4, 8, 5),+ (1, 1, 8, 2, 4, 3),+ (2, 3, 16, 2, 32, 9),+ (2, 2, 12, 4, 8, 5),]:torch.manual_seed(42)oq = rolls * patch_len * num_q@@ -93,23 +93,23 @@ class CpmRevinRefineTest(unittest.TestCase):median_q_idx = num_q // 2refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(- raw_logits=raw_logits,- revin_n=revin_n,- revin_mu=revin_mu,- revin_sigma=revin_sigma,- patch_cpm_mask=patch_cpm_mask,- median_q_idx=median_q_idx,- rolls=rolls,- patch_len=patch_len,- num_quantiles=num_q,+ raw_logits=raw_logits,+ revin_n=revin_n,+ revin_mu=revin_mu,+ revin_sigma=revin_sigma,+ patch_cpm_mask=patch_cpm_mask,+ median_q_idx=median_q_idx,+ rolls=rolls,+ patch_len=patch_len,+ num_quantiles=num_q,)# Non-CPM positions should be identical to inputsnp.testing.assert_allclose(- refined_mu[:, :, :4].numpy(), revin_mu[:, :, :4].numpy()+ refined_mu[:, :, :4].numpy(), revin_mu[:, :, :4].numpy())np.testing.assert_allclose(- refined_sigma[:, :, :4].numpy(), revin_sigma[:, :, :4].numpy()+ refined_sigma[:, :, :4].numpy(), revin_sigma[:, :, :4].numpy())# Refined outputs should be finitediff --git a/src/timesfm3/dense.py b/src/timesfm3/dense.pyindex f8b1c9d..f4c9933 100644--- a/src/timesfm3/dense.py+++ b/src/timesfm3/dense.py@@ -5,8 +5,7 @@ from __future__ import annotationsimport torchfrom torch import nn-from . import configs-from . import util+from . import configs, utilclass ResidualBlock(nn.Module):@@ -30,23 +29,23 @@ class ResidualBlock(nn.Module):# the first forward pass (using `set_input_dims()`) before the matrix# multiplication is evaluated, avoiding shape mismatch errors.self.hidden_layer = nn.Linear(- in_features=config.hidden_dims, # placeholder, set in first forward- out_features=config.hidden_dims,- bias=config.use_bias,+ in_features=config.hidden_dims, # placeholder, set in first forward+ out_features=config.hidden_dims,+ bias=config.use_bias,)self.output_layer = nn.Linear(- in_features=config.hidden_dims,- out_features=config.output_dims,- bias=config.use_bias,+ in_features=config.hidden_dims,+ out_features=config.output_dims,+ bias=config.use_bias,)if config.identity_skip:self.residual_layer = Noneelse:self.residual_layer = nn.Linear(- in_features=config.hidden_dims, # placeholder- out_features=config.output_dims,- bias=config.use_bias,+ in_features=config.hidden_dims, # placeholder+ out_features=config.output_dims,+ bias=config.use_bias,)self.activation = util.get_activation_fn(config.activation)@@ -67,16 +66,16 @@ class ResidualBlock(nn.Module):dtype = self.hidden_layer.weight.dtypeself.hidden_layer = nn.Linear(- input_dim, self.config.hidden_dims, bias=self.config.use_bias+ input_dim, self.config.hidden_dims, bias=self.config.use_bias).to(device=device, dtype=dtype)self.output_layer = nn.Linear(- self.config.hidden_dims,- self.config.output_dims,- bias=self.config.use_bias,+ self.config.hidden_dims,+ self.config.output_dims,+ bias=self.config.use_bias,).to(device=device, dtype=dtype)if self.residual_layer is not None:self.residual_layer = nn.Linear(- input_dim, self.config.output_dims, bias=self.config.use_bias+ input_dim, self.config.output_dims, bias=self.config.use_bias).to(device=device, dtype=dtype)if self.pre_norm is not None:self.pre_norm = nn.RMSNorm(input_dim).to(device=device, dtype=dtype)diff --git a/src/timesfm3/model.py b/src/timesfm3/model.pyindex 102a2fe..73638c6 100644--- a/src/timesfm3/model.py+++ b/src/timesfm3/model.py@@ -8,21 +8,24 @@ Supports:from __future__ import annotations+import dataclassesimport mathfrom typing import Anyimport torch+from huggingface_hub import PyTorchModelHubMixinfrom torch import nn-from . import configs+from . import configs, dense, transformer, utilfrom . import cpm_revin_refine as cpm_revin_refine_lib-from . import dense-from . import transformations-from . import transformer-from . import util-class TimesFM3Torch(nn.Module):+class TimesFM3Torch(+ nn.Module,+ PyTorchModelHubMixin,+ library_name="timesfm",+ repo_url="https://github.com/google-research/timesfm",+):"""PyTorch inference-only implementation of TimesFM3.Attributes:@@ -41,60 +44,70 @@ class TimesFM3Torch(nn.Module):"""def __init__(- self,- input_patch_len: int = 32,- output_patch_len: int = 64,- quantiles: list[float] | None = None,- residual_block_config: configs.ResidualBlockConfig | None = None,- transformer_config: configs.StackedTransformersConfig | None = None,- use_variate_attention: bool = True,- value_clip: float = 1e20,- use_stitching: bool = True,- use_linear_detrending: bool = True,- linear_detrending_threshold: float = 0.5,- use_iterative_cpm_revin: bool = True,- use_frozen_running_stats: bool = False,- input_transform: str = "identity",+ self,+ input_patch_len: int = 32,+ output_patch_len: int = 64,+ quantiles: list[float] | None = None,+ residual_block_config: configs.ResidualBlockConfig | dict[str, Any] | None = None,+ transformer_config: configs.StackedTransformersConfig+ | dict[str, Any]+ | None = None,+ use_variate_attention: bool = True,+ value_clip: float = 1e20,+ use_stitching: bool = True,+ use_linear_detrending: bool = True,+ linear_detrending_threshold: float = 0.5,+ use_iterative_cpm_revin: bool = True,+ use_frozen_running_stats: bool = False,+ input_transform: str = "identity",):super().__init__()if quantiles is None:quantiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]if residual_block_config is None:residual_block_config = configs.ResidualBlockConfig(- hidden_dims=1280,- output_dims=1280,- use_bias=False,- activation="relu",+ hidden_dims=1280,+ output_dims=1280,+ use_bias=False,+ activation="relu",)+ elif isinstance(residual_block_config, dict):+ residual_block_config = configs.ResidualBlockConfig(**residual_block_config)+if transformer_config is None:transformer_config = configs.StackedTransformersConfig(- num_layers=20,- transformer=configs.TransformerConfig(- model_dims=1280,- hidden_dims=1280,- num_heads=16,- attention_norm="rms",- feedforward_norm="rms",- qk_norm="rms",- use_rope_seq=True,- use_rope_var=False,- use_bias=False,- ff_activation="relu",- deterministic=True,- ),+ num_layers=20,+ transformer=configs.TransformerConfig(+ model_dims=1280,+ hidden_dims=1280,+ num_heads=16,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=False,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ ),)+ elif isinstance(transformer_config, dict):+ t_data = transformer_config.get("transformer", {})+ if isinstance(t_data, dict):+ t_cfg = configs.TransformerConfig(**t_data)+ else:+ t_cfg = t_data+ tc_dict = dict(transformer_config)+ tc_dict["transformer"] = t_cfg+ transformer_config = configs.StackedTransformersConfig(**tc_dict)+if output_patch_len % input_patch_len != 0:raise ValueError(- f"Output patch len {output_patch_len} must be a multiple of"- f" input patch len {input_patch_len}."- )- if (- residual_block_config.output_dims- != transformer_config.transformer.model_dims- ):- raise ValueError(- "ResidualBlock output_dims must match Transformer model_dims."+ f"Output patch len {output_patch_len} must be a multiple of"+ f" input patch len {input_patch_len}.")+ if residual_block_config.output_dims != transformer_config.transformer.model_dims:+ raise ValueError("ResidualBlock output_dims must match Transformer model_dims.")self.input_patch_len = input_patch_lenself.output_patch_len = output_patch_len@@ -114,40 +127,67 @@ class TimesFM3Torch(nn.Module):if self.use_stitching:if self.output_patch_len <= self.input_patch_len:- raise ValueError(- "use_stitching requires output_patch_len > input_patch_len"- )- self._stitching_extract_len = min(- 2 * self.input_patch_len, self.output_patch_len- )+ raise ValueError("use_stitching requires output_patch_len > input_patch_len")+ self._stitching_extract_len = min(2 * self.input_patch_len, self.output_patch_len)- self.pre_transformer_resblock = dense.ResidualBlock(- config=residual_block_config+ self.pre_transformer_resblock = dense.ResidualBlock(config=residual_block_config)+ self.pre_transformer_resblock.set_input_dims(+ 2 * (input_patch_len + output_patch_len))self.transformer_stack = transformer.StackedMixingTransformer(- config=transformer_config,- use_variate_attention=use_variate_attention,+ config=transformer_config,+ use_variate_attention=use_variate_attention,)self.output_head = nn.Linear(- transformer_config.transformer.model_dims,- output_patch_len * self.num_quantiles,- bias=True,+ transformer_config.transformer.model_dims,+ output_patch_len * self.num_quantiles,+ bias=True,)+ def to_dict(self) -> dict[str, Any]:+ """Returns a serializable dictionary of model configuration."""+ return {+ "input_patch_len": self.input_patch_len,+ "output_patch_len": self.output_patch_len,+ "quantiles": list(self.quantiles),+ "residual_block_config": dataclasses.asdict(self.residual_block_config),+ "transformer_config": dataclasses.asdict(self.transformer_config),+ "use_variate_attention": self.use_variate_attention,+ "value_clip": self.value_clip,+ "use_stitching": self.use_stitching,+ "use_linear_detrending": self.use_linear_detrending,+ "linear_detrending_threshold": self.linear_detrending_threshold,+ "use_iterative_cpm_revin": self.use_iterative_cpm_revin,+ "use_frozen_running_stats": self.use_frozen_running_stats,+ "input_transform": self.input_transform,+ }++ def save_pretrained(+ self,+ save_directory: str | Any,+ *,+ config: dict[str, Any] | None = None,+ **kwargs: Any,+ ) -> str | None:+ """Saves model weights and config to a local directory or pushes to Hub."""+ if config is None:+ config = self.to_dict()+ return super().save_pretrained(save_directory, config=config, **kwargs)+def _preprocess(- self,- values: torch.Tensor,- masks: torch.Tensor,- patch_is_target: torch.Tensor,- freeze_after: int | None = None,- patch_cpm_mask: torch.Tensor | None = None,+ self,+ values: torch.Tensor,+ masks: torch.Tensor,+ patch_is_target: torch.Tensor,+ freeze_after: int | None = None,+ patch_cpm_mask: torch.Tensor | None = None,) -> tuple[- torch.Tensor,- torch.Tensor,- torch.Tensor,- tuple[torch.Tensor, torch.Tensor],- torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ torch.Tensor,+ tuple[torch.Tensor, torch.Tensor],+ torch.Tensor,]:"""Applies preprocessing: RevIN, masking, future covariates, ResBlock.@@ -163,17 +203,15 @@ class TimesFM3Torch(nn.Module):(resblock_input, resblock_output, patch_mask, (running_mean, running_std),running_n)"""- running_n, running_mean, running_std = util.get_running_stats(- values, masks- )+ running_n, running_mean, running_std = util.get_running_stats(values, masks)if freeze_after is not None:_, _, n, _ = values.shapeif 0 <= freeze_after < n - 1:running_mean[:, :, freeze_after + 1 :] = running_mean[- :, :, freeze_after : freeze_after + 1+ :, :, freeze_after : freeze_after + 1]running_std[:, :, freeze_after + 1 :] = running_std[- :, :, freeze_after : freeze_after + 1+ :, :, freeze_after : freeze_after + 1]# Apply CPM mask: mask target variates at CPM positions.@@ -187,9 +225,7 @@ class TimesFM3Torch(nn.Module):# Roll values to get future covariate patchesvalues_fcov, wrap_mask = util.get_output_patch_via_roll(values, self.rolls)- values_fcov = util.revin(- values_fcov, running_mean, running_std, reverse=False- )+ values_fcov = util.revin(values_fcov, running_mean, running_std, reverse=False)# Roll the (CPM-modified) masks for future covariate masking.masks_fcov_raw, _ = util.get_output_patch_via_roll(masks, self.rolls)@@ -206,19 +242,19 @@ class TimesFM3Torch(nn.Module):patch_mask_bvn = masks_cat.all(dim=3)return (- resblock_input,- resblock_output,- patch_mask_bvn,- (running_mean, running_std),- running_n,+ resblock_input,+ resblock_output,+ patch_mask_bvn,+ (running_mean, running_std),+ running_n,)def forward(- self,- inputs: dict[str, Any],- freeze_after: int | None = None,- patch_cpm_mask: torch.Tensor | None = None,- return_aux_outputs: bool = False,+ self,+ inputs: dict[str, Any],+ freeze_after: int | None = None,+ patch_cpm_mask: torch.Tensor | None = None,+ return_aux_outputs: bool = False,) -> dict[str, Any]:"""Full-sequence forward pass (equivalent to Flax __call__).@@ -242,22 +278,22 @@ class TimesFM3Torch(nn.Module):_, _, _, p = values.shapeif p != self.input_patch_len:raise ValueError(- f"Input patch_len {p} != model input_patch_len {self.input_patch_len}"+ f"Input patch_len {p} != model input_patch_len {self.input_patch_len}")# Preprocessing & ResBlock(- resblock_input,- transformer_input,- transformer_patch_mask,- revin_stats,- running_n,+ resblock_input,+ transformer_input,+ transformer_patch_mask,+ revin_stats,+ running_n,) = self._preprocess(- values,- masks,- patch_is_target,- freeze_after=freeze_after,- patch_cpm_mask=patch_cpm_mask,+ values,+ masks,+ patch_is_target,+ freeze_after=freeze_after,+ patch_cpm_mask=patch_cpm_mask,)# Transformer@@ -265,12 +301,10 @@ class TimesFM3Torch(nn.Module):# Flax __call__ does: effective_patch_mask = cumprod(mask, axis=2) when# not training. This keeps horizon patches (which are fully masked but# come after valid context) visible to attention.- effective_patch_mask = torch.cumprod(- transformer_patch_mask.int(), dim=2- ).bool()+ effective_patch_mask = torch.cumprod(transformer_patch_mask.int(), dim=2).bool()transformer_output, _, seq_attn_mask = self.transformer_stack(- transformer_input,- effective_patch_mask,+ transformer_input,+ effective_patch_mask,)# Output head@@ -278,33 +312,29 @@ class TimesFM3Torch(nn.Module):revin_mean, revin_std = revin_statsif self.use_iterative_cpm_revin and patch_cpm_mask is not None:- refined_mu, refined_sigma = (- cpm_revin_refine_lib.cpm_iterative_revin_refine(- raw_logits,- revin_n=running_n,- revin_mu=revin_mean,- revin_sigma=revin_std,- patch_cpm_mask=patch_cpm_mask,- median_q_idx=self.num_quantiles // 2,- rolls=self.rolls,- patch_len=self.input_patch_len,- num_quantiles=self.num_quantiles,- value_clip=self.value_clip,- )+ refined_mu, refined_sigma = cpm_revin_refine_lib.cpm_iterative_revin_refine(+ raw_logits,+ revin_n=running_n,+ revin_mu=revin_mean,+ revin_sigma=revin_std,+ patch_cpm_mask=patch_cpm_mask,+ median_q_idx=self.num_quantiles // 2,+ rolls=self.rolls,+ patch_len=self.input_patch_len,+ num_quantiles=self.num_quantiles,+ value_clip=self.value_clip,)cpm_bvn = patch_cpm_mask.unsqueeze(1) # (b, 1, n)revin_mean = torch.where(cpm_bvn, refined_mu, revin_mean)revin_std = torch.where(cpm_bvn, refined_sigma, revin_std)revin_logits = util.revin(raw_logits, revin_mean, revin_std, reverse=True)- clipped_logits = torch.clamp(- revin_logits, -self.value_clip, self.value_clip- )+ clipped_logits = torch.clamp(revin_logits, -self.value_clip, self.value_clip)# Reshape: (b, v, n, o*q) -> (b, v, n, o, q)b, v, n_patches = clipped_logits.shape[:3]final_logits = clipped_logits.view(- b, v, n_patches, self.output_patch_len, self.num_quantiles+ b, v, n_patches, self.output_patch_len, self.num_quantiles)outputs = {"logits": final_logits, "revin_stats": revin_stats}@@ -319,16 +349,16 @@ class TimesFM3Torch(nn.Module):@torch.no_grad()def decode(- self,- target: torch.Tensor,- horizon: int = 0,- past_only_covariates: torch.Tensor | None = None,- past_future_covariates: torch.Tensor | None = None,- target_mask: torch.Tensor | None = None,- past_only_mask: torch.Tensor | None = None,- past_future_mask: torch.Tensor | None = None,- mask: torch.Tensor | None = None,- return_aux_outputs: bool = False,+ self,+ target: torch.Tensor,+ horizon: int = 0,+ past_only_covariates: torch.Tensor | None = None,+ past_future_covariates: torch.Tensor | None = None,+ target_mask: torch.Tensor | None = None,+ past_only_mask: torch.Tensor | None = None,+ past_future_mask: torch.Tensor | None = None,+ mask: torch.Tensor | None = None,+ return_aux_outputs: bool = False,) -> Any:"""Non-autoregressive single-pass decoding for TimesFM3.@@ -356,7 +386,7 @@ class TimesFM3Torch(nn.Module):# 1. Pad context to multiple of input_patch_lenctx_padding = (- self.input_patch_len - (context % self.input_patch_len)+ self.input_patch_len - (context % self.input_patch_len)) % self.input_patch_lenif ctx_padding > 0:target = torch.nn.functional.pad(target, (ctx_padding, 0))@@ -364,23 +394,21 @@ class TimesFM3Torch(nn.Module):mask = torch.nn.functional.pad(mask, (ctx_padding, 0), value=True)if past_only_covariates is not None:past_only_covariates = torch.nn.functional.pad(- past_only_covariates, (ctx_padding, 0)+ past_only_covariates, (ctx_padding, 0))if past_future_covariates is not None:past_future_covariates = torch.nn.functional.pad(- past_future_covariates, (ctx_padding, 0)+ past_future_covariates, (ctx_padding, 0))if target_mask is not None:- target_mask = torch.nn.functional.pad(- target_mask, (ctx_padding, 0), value=True- )+ target_mask = torch.nn.functional.pad(target_mask, (ctx_padding, 0), value=True)if past_only_mask is not None:past_only_mask = torch.nn.functional.pad(- past_only_mask, (ctx_padding, 0), value=True+ past_only_mask, (ctx_padding, 0), value=True)if past_future_mask is not None:past_future_mask = torch.nn.functional.pad(- past_future_mask, (ctx_padding, 0), value=True+ past_future_mask, (ctx_padding, 0), value=True)context = context + ctx_padding@@ -394,7 +422,7 @@ class TimesFM3Torch(nn.Module):extract_len = self._stitching_extract_lenoverlap = extract_len - self.input_patch_lennum_forecast_patches = max(- math.ceil((horizon - overlap) / self.input_patch_len), 1+ math.ceil((horizon - overlap) / self.input_patch_len), 1)num_horizon_patches = num_forecast_patches + self.rolls - 1padded_horizon = num_horizon_patches * self.input_patch_len@@ -416,16 +444,12 @@ class TimesFM3Torch(nn.Module):if past_only_covariates is not None:num_past_only = past_only_covariates.shape[1]if past_only_mask is None:- past_only_mask = torch.zeros_like(- past_only_covariates, dtype=torch.bool- )+ past_only_mask = torch.zeros_like(past_only_covariates, dtype=torch.bool)all_ctx_vals.append(past_only_covariates)all_ctx_masks.append(past_only_mask | mask.unsqueeze(1))if past_future_covariates is not None:if past_future_mask is None:- past_future_mask = torch.zeros_like(- past_future_covariates, dtype=torch.bool- )+ past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)all_ctx_vals.append(past_future_covariates[..., :context])all_ctx_masks.append(past_future_mask[..., :context] | mask.unsqueeze(1))@@ -433,57 +457,41 @@ class TimesFM3Torch(nn.Module):ctx_masks = torch.cat(all_ctx_masks, dim=1)if self.use_linear_detrending:- t_ctx = torch.arange(- -(context - 1), 1, dtype=torch.float32, device=device- )+ t_ctx = torch.arange(-(context - 1), 1, dtype=torch.float32, device=device)t_ctx_bvc = t_ctx[None, None, :]t_ctx_bvc_normalized = t_ctx_bvc / contextvalid = ~ctx_masksn_v = valid.float().sum(dim=-1, keepdim=True)- sum_t = torch.where(valid, t_ctx_bvc_normalized, 0.0).sum(- dim=-1, keepdim=True- )+ sum_t = torch.where(valid, t_ctx_bvc_normalized, 0.0).sum(dim=-1, keepdim=True)sum_t2 = torch.where(valid, t_ctx_bvc_normalized**2, 0.0).sum(- dim=-1, keepdim=True+ dim=-1, keepdim=True)sum_y = torch.where(valid, ctx_vals, 0.0).sum(dim=-1, keepdim=True)sum_ty = torch.where(valid, t_ctx_bvc_normalized * ctx_vals, 0.0).sum(- dim=-1, keepdim=True+ dim=-1, keepdim=True)det = n_v * sum_t2 - sum_t**2safe_det = torch.where(det == 0.0, 1.0, det)- m_trend = torch.where(- det == 0.0, 0.0, (n_v * sum_ty - sum_t * sum_y) / safe_det- )+ m_trend = torch.where(det == 0.0, 0.0, (n_v * sum_ty - sum_t * sum_y) / safe_det)c_trend = torch.where(- det == 0.0,- torch.where(n_v > 0, sum_y / torch.clamp_min(n_v, 1.0), 0.0),- (sum_y - m_trend * sum_t) / torch.clamp_min(n_v, 1.0),+ det == 0.0,+ torch.where(n_v > 0, sum_y / torch.clamp_min(n_v, 1.0), 0.0),+ (sum_y - m_trend * sum_t) / torch.clamp_min(n_v, 1.0),)- ctx_vals_detrended = ctx_vals - (- m_trend * t_ctx_bvc_normalized + c_trend- )+ ctx_vals_detrended = ctx_vals - (m_trend * t_ctx_bvc_normalized + c_trend)mean_y = sum_y / torch.clamp_min(n_v, 1.0)sum_y2 = torch.where(valid, ctx_vals**2, 0.0).sum(dim=-1, keepdim=True)- var_orig = torch.clamp_min(- sum_y2 / torch.clamp_min(n_v, 1.0) - mean_y**2, 0.0- )+ var_orig = torch.clamp_min(sum_y2 / torch.clamp_min(n_v, 1.0) - mean_y**2, 0.0)std_orig = torch.sqrt(var_orig)- sum_yd = torch.where(valid, ctx_vals_detrended, 0.0).sum(- dim=-1, keepdim=True- )+ sum_yd = torch.where(valid, ctx_vals_detrended, 0.0).sum(dim=-1, keepdim=True)mean_yd = sum_yd / torch.clamp_min(n_v, 1.0)- sum_yd2 = torch.where(valid, ctx_vals_detrended**2, 0.0).sum(- dim=-1, keepdim=True- )- var_det = torch.clamp_min(- sum_yd2 / torch.clamp_min(n_v, 1.0) - mean_yd**2, 0.0- )+ sum_yd2 = torch.where(valid, ctx_vals_detrended**2, 0.0).sum(dim=-1, keepdim=True)+ var_det = torch.clamp_min(sum_yd2 / torch.clamp_min(n_v, 1.0) - mean_yd**2, 0.0)std_det = torch.sqrt(var_det)apply_detrend = std_det < self.linear_detrending_threshold * std_orig@@ -491,64 +499,60 @@ class TimesFM3Torch(nn.Module):else:num_variates = ctx_vals.shape[1]m_trend = torch.zeros(- (batch_size, num_variates, 1), dtype=torch.float32, device=device+ (batch_size, num_variates, 1), dtype=torch.float32, device=device)c_trend = torch.zeros(- (batch_size, num_variates, 1), dtype=torch.float32, device=device+ (batch_size, num_variates, 1), dtype=torch.float32, device=device)apply_detrend = torch.zeros(- (batch_size, num_variates, 1), dtype=torch.bool, device=device+ (batch_size, num_variates, 1), dtype=torch.bool, device=device)ctx_vals = torch.where(ctx_masks, 0.0, ctx_vals)all_hor_vals = [- torch.zeros(batch_size, num_target, padded_horizon, device=device),- torch.zeros(batch_size, num_past_only, padded_horizon, device=device),+ torch.zeros(batch_size, num_target, padded_horizon, device=device),+ torch.zeros(batch_size, num_past_only, padded_horizon, device=device),]all_hor_masks = [- torch.ones(- batch_size,- num_target,- padded_horizon,- dtype=torch.bool,- device=device,- ),- torch.ones(- batch_size,- num_past_only,- padded_horizon,- dtype=torch.bool,- device=device,- ),+ torch.ones(+ batch_size,+ num_target,+ padded_horizon,+ dtype=torch.bool,+ device=device,+ ),+ torch.ones(+ batch_size,+ num_past_only,+ padded_horizon,+ dtype=torch.bool,+ device=device,+ ),]if past_future_covariates is not None:if past_future_mask is None:- past_future_mask = torch.zeros_like(- past_future_covariates, dtype=torch.bool- )+ past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)pf_future_vals = past_future_covariates[..., context : context + horizon]pf_future_masks = past_future_mask[..., context : context + horizon]if self.use_linear_detrending:m_pf = m_trend[:, num_target + num_past_only :, :]c_pf = c_trend[:, num_target + num_past_only :, :]apply_detrend_pf = apply_detrend[:, num_target + num_past_only :, :]- t_hor_pf = torch.arange(- 1, horizon + 1, dtype=torch.float32, device=device- )[None, None, :]+ t_hor_pf = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)[+ None, None, :+ ]t_hor_pf_normalized = t_hor_pf / contextpf_trend_hor = m_pf * t_hor_pf_normalized + c_pfpf_future_vals = torch.where(- apply_detrend_pf, pf_future_vals - pf_trend_hor, pf_future_vals+ apply_detrend_pf, pf_future_vals - pf_trend_hor, pf_future_vals)pf_future_vals = torch.where(pf_future_masks, 0.0, pf_future_vals)if hor_padding > 0:- pf_future_vals = torch.nn.functional.pad(- pf_future_vals, (0, hor_padding)- )+ pf_future_vals = torch.nn.functional.pad(pf_future_vals, (0, hor_padding))pf_future_masks = torch.nn.functional.pad(- pf_future_masks, (0, hor_padding), value=True+ pf_future_masks, (0, hor_padding), value=True)all_hor_vals.append(pf_future_vals)all_hor_masks.append(pf_future_masks)@@ -561,41 +565,35 @@ class TimesFM3Torch(nn.Module):num_variates = all_vals.shape[1]patch_is_target = torch.zeros(- (batch_size, num_variates, num_context_patches + num_horizon_patches),- dtype=torch.bool,- device=device,+ (batch_size, num_variates, num_context_patches + num_horizon_patches),+ dtype=torch.bool,+ device=device,)patch_is_target[:, : num_target + num_past_only, :] = True# Reshape values & masks to patched shape (b, v, n, p)- values_bvnp = all_vals.reshape(- batch_size, num_variates, -1, self.input_patch_len- )- masks_bvnp = all_masks.reshape(- batch_size, num_variates, -1, self.input_patch_len- )+ values_bvnp = all_vals.reshape(batch_size, num_variates, -1, self.input_patch_len)+ masks_bvnp = all_masks.reshape(batch_size, num_variates, -1, self.input_patch_len)inputs = {- "values": values_bvnp,- "masks": masks_bvnp,- "patch_is_target": patch_is_target,+ "values": values_bvnp,+ "masks": masks_bvnp,+ "patch_is_target": patch_is_target,}# Build horizon CPM mask: context=False, horizon=True.num_total_patches = num_context_patches + num_horizon_patcheshorizon_cpm_mask = torch.zeros(- batch_size, num_total_patches, dtype=torch.bool, device=device+ batch_size, num_total_patches, dtype=torch.bool, device=device)horizon_cpm_mask[:, num_context_patches:] = True- freeze_after = (- num_context_patches - 1 if self.use_frozen_running_stats else None- )+ freeze_after = num_context_patches - 1 if self.use_frozen_running_stats else Noneforward_out = self.forward(- inputs,- freeze_after=freeze_after,- patch_cpm_mask=horizon_cpm_mask,- return_aux_outputs=return_aux_outputs,+ inputs,+ freeze_after=freeze_after,+ patch_cpm_mask=horizon_cpm_mask,+ return_aux_outputs=return_aux_outputs,)logits = forward_out["logits"] # (b, v, n, output_patch_len, num_quantiles)@@ -603,38 +601,34 @@ class TimesFM3Torch(nn.Module):extract_len = self._stitching_extract_lenoverlap = extract_len - self.input_patch_lennum_forecast_patches = max(- math.ceil((horizon - overlap) / self.input_patch_len), 1+ math.ceil((horizon - overlap) / self.input_patch_len), 1+ )+ forecast_indices = torch.arange(num_forecast_patches, device=device) + (+ num_context_patches - 1)- forecast_indices = torch.arange(- num_forecast_patches, device=device- ) + (num_context_patches - 1)patch_preds = logits[:, :, forecast_indices, :extract_len, :]horizon_logits = util.stitch_patches(- patch_preds,- self.input_patch_len,+ patch_preds,+ self.input_patch_len,)[:, :, :horizon, :]else:num_forecast_chunks = padded_horizon // self.output_patch_lenforecast_indices = torch.arange(- num_forecast_chunks, device=device+ num_forecast_chunks, device=device) * self.rolls + (num_context_patches - 1)forecast_logits = logits[:, :, forecast_indices, :, :]horizon_logits = forecast_logits.reshape(- batch_size, num_variates, -1, self.num_quantiles+ batch_size, num_variates, -1, self.num_quantiles)[:, :, :horizon, :]if self.use_linear_detrending:- t_forecast = torch.arange(- 1, horizon + 1, dtype=torch.float32, device=device- )+ t_forecast = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)t_forecast_normalized = t_forecast / contexttrend_forecast = (- m_trend[:, :, 0, None] * t_forecast_normalized[None, None, :]- + c_trend[:, :, 0, None]- )- trend_forecast = torch.where(- apply_detrend[:, :, 0, None], trend_forecast, 0.0+ m_trend[:, :, 0, None] * t_forecast_normalized[None, None, :]+ + c_trend[:, :, 0, None])+ trend_forecast = torch.where(apply_detrend[:, :, 0, None], trend_forecast, 0.0)horizon_logits = horizon_logits + trend_forecast[:, :, :, None]if return_aux_outputs:diff --git a/src/timesfm3/model_test.py b/src/timesfm3/model_test.pyindex ffefd46..e36d893 100644--- a/src/timesfm3/model_test.py+++ b/src/timesfm3/model_test.py@@ -15,7 +15,7 @@"""Tests for PyTorch TimesFM3Torch model."""import unittest-import numpy as np+import torchfrom . import configs@@ -23,57 +23,56 @@ from . import model as torch_model_libclass TimesFM3TorchTest(unittest.TestCase):-def setUp(self):super().setUp()self.resblock_config = configs.ResidualBlockConfig(- hidden_dims=32,- output_dims=32,- use_bias=False,- activation="relu",- dropout=0.0,+ hidden_dims=32,+ output_dims=32,+ use_bias=False,+ activation="relu",+ dropout=0.0,)self.transformer_config = configs.StackedTransformersConfig(- num_layers=2,- use_remat=False,- transformer=configs.TransformerConfig(- model_dims=32,- hidden_dims=32,- num_heads=4,- attention_norm="rms",- feedforward_norm="rms",- qk_norm="rms",- use_rope_seq=True,- use_rope_var=True,- use_bias=False,- ff_activation="relu",- deterministic=True,- use_sdpa=True,- ),+ num_layers=2,+ use_remat=False,+ transformer=configs.TransformerConfig(+ model_dims=32,+ hidden_dims=32,+ num_heads=4,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=True,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ use_sdpa=True,+ ),)self.model = torch_model_lib.TimesFM3Torch(- input_patch_len=8,- output_patch_len=16,- quantiles=[0.1, 0.5, 0.9],- residual_block_config=self.resblock_config,- transformer_config=self.transformer_config,- use_stitching=True,- use_linear_detrending=True,- use_iterative_cpm_revin=True,- use_frozen_running_stats=False,+ input_patch_len=8,+ output_patch_len=16,+ quantiles=[0.1, 0.5, 0.9],+ residual_block_config=self.resblock_config,+ transformer_config=self.transformer_config,+ use_stitching=True,+ use_linear_detrending=True,+ use_iterative_cpm_revin=True,+ use_frozen_running_stats=False,)self.model.eval()def test_forward_pass_training_dict(self):b, v, n, p = 2, 3, 4, 8inputs = {- "values": torch.randn(b, v, n, p),- "masks": torch.zeros(b, v, n, p, dtype=torch.bool),- "patch_segment_ids": torch.zeros(b, n, dtype=torch.long),- "patch_positions": torch.arange(n).unsqueeze(0).repeat(b, 1),- "patch_is_target": torch.ones(b, v, n, dtype=torch.bool),- "patch_is_past_only": torch.zeros(b, v, n, dtype=torch.bool),- "patch_is_past_future_covariate": torch.zeros(b, v, n, dtype=torch.bool),+ "values": torch.randn(b, v, n, p),+ "masks": torch.zeros(b, v, n, p, dtype=torch.bool),+ "patch_segment_ids": torch.zeros(b, n, dtype=torch.long),+ "patch_positions": torch.arange(n).unsqueeze(0).repeat(b, 1),+ "patch_is_target": torch.ones(b, v, n, dtype=torch.bool),+ "patch_is_past_only": torch.zeros(b, v, n, dtype=torch.bool),+ "patch_is_past_future_covariate": torch.zeros(b, v, n, dtype=torch.bool),}with torch.no_grad():out = self.model(inputs)@@ -101,16 +100,40 @@ class TimesFM3TorchTest(unittest.TestCase):with torch.no_grad():out = self.model.decode(- target=target,- horizon=horizon,- past_only_covariates=po_cov,- past_future_covariates=pf_cov,- mask=mask,+ target=target,+ horizon=horizon,+ past_only_covariates=po_cov,+ past_future_covariates=pf_cov,+ mask=mask,)# Total variates: 3 targets + 3 past_only + 3 past_future = 9self.assertEqual(out.shape, (b, 9, horizon, 3))self.assertTrue(torch.isfinite(out).all().item())+ def test_save_and_from_pretrained(self):+ import json+ import os+ import tempfile++ with tempfile.TemporaryDirectory() as tmpdir:+ self.model.save_pretrained(tmpdir)+ config_path = os.path.join(tmpdir, "config.json")+ self.assertTrue(os.path.exists(config_path))+ with open(config_path) as f:+ config_dict = json.load(f)+ self.assertEqual(config_dict["input_patch_len"], 8)+ self.assertEqual(config_dict["output_patch_len"], 16)+ self.assertIn("transformer_config", config_dict)++ loaded_model = torch_model_lib.TimesFM3Torch.from_pretrained(tmpdir)+ loaded_model.eval()++ target = torch.randn(2, 1, 32)+ with torch.no_grad():+ orig_out = self.model.decode(target=target, horizon=16)+ loaded_out = loaded_model.decode(target=target, horizon=16)+ self.assertTrue(torch.allclose(orig_out, loaded_out, atol=1e-5))+if __name__ == "__main__":unittest.main()diff --git a/src/timesfm3/normalization.py b/src/timesfm3/normalization.pyindex 25fcf91..6f20c5a 100644--- a/src/timesfm3/normalization.py+++ b/src/timesfm3/normalization.py@@ -28,8 +28,8 @@ class PerDimScale(nn.Module):def forward(self, x: torch.Tensor) -> torch.Tensor:"""Applies per-dim scaling to the last dimension of x."""return (- x- * _RECIPROCAL_OF_SOFTPLUS_0- / math.sqrt(self.num_dims)- * torch.nn.functional.softplus(self.per_dim_scale)+ x+ * _RECIPROCAL_OF_SOFTPLUS_0+ / math.sqrt(self.num_dims)+ * torch.nn.functional.softplus(self.per_dim_scale))diff --git a/src/timesfm3/primitives_test.py b/src/timesfm3/primitives_test.pyindex 6f76493..c7fdc9d 100644--- a/src/timesfm3/primitives_test.py+++ b/src/timesfm3/primitives_test.py@@ -16,6 +16,7 @@import mathimport unittest+import numpy as npimport torch@@ -59,19 +60,19 @@ class TransformationsTest(unittest.TestCase):def test_max_output(self):value_clip = 10.0self.assertAlmostEqual(- torch_trans.max_output("signed_log", value_clip).item(),- math.log1p(value_clip),- places=5,+ torch_trans.max_output("signed_log", value_clip).item(),+ math.log1p(value_clip),+ places=5,)self.assertAlmostEqual(- torch_trans.max_output("signed_sqrt", value_clip).item(),- math.sqrt(value_clip),- places=5,+ torch_trans.max_output("signed_sqrt", value_clip).item(),+ math.sqrt(value_clip),+ places=5,)self.assertAlmostEqual(- torch_trans.max_output("identity", value_clip).item(),- value_clip,- places=5,+ torch_trans.max_output("identity", value_clip).item(),+ value_clip,+ places=5,)@@ -89,7 +90,7 @@ class NormalizationTest(unittest.TestCase):# at init it approximately scales by 1 / sqrt(num_dims).expected_scale = 1.0 / math.sqrt(num_dims)np.testing.assert_allclose(- y.detach().numpy(), (x * expected_scale).numpy(), atol=1e-4+ y.detach().numpy(), (x * expected_scale).numpy(), atol=1e-4)@@ -100,12 +101,12 @@ class ResidualBlockTest(unittest.TestCase):input_dim = 8hidden_dims = 16config = configs.ResidualBlockConfig(- hidden_dims=hidden_dims,- output_dims=input_dim,- use_bias=True,- activation="relu",- identity_skip=True,- prenorm="none",+ hidden_dims=hidden_dims,+ output_dims=input_dim,+ use_bias=True,+ activation="relu",+ identity_skip=True,+ prenorm="none",)mod = torch_dense.ResidualBlock(config=config)x = torch.randn(2, input_dim)@@ -117,12 +118,12 @@ class ResidualBlockTest(unittest.TestCase):hidden_dims = 16output_dims = 12config = configs.ResidualBlockConfig(- hidden_dims=hidden_dims,- output_dims=output_dims,- use_bias=True,- activation="relu",- identity_skip=False,- prenorm="none",+ hidden_dims=hidden_dims,+ output_dims=output_dims,+ use_bias=True,+ activation="relu",+ identity_skip=False,+ prenorm="none",)mod = torch_dense.ResidualBlock(config=config)x = torch.randn(2, input_dim)@@ -133,12 +134,12 @@ class ResidualBlockTest(unittest.TestCase):input_dim = 8hidden_dims = 16config = configs.ResidualBlockConfig(- hidden_dims=hidden_dims,- output_dims=input_dim,- use_bias=False,- activation="relu",- identity_skip=True,- prenorm="rms",+ hidden_dims=hidden_dims,+ output_dims=input_dim,+ use_bias=False,+ activation="relu",+ identity_skip=True,+ prenorm="rms",)mod = torch_dense.ResidualBlock(config=config)x = torch.randn(2, input_dim)@@ -174,12 +175,12 @@ class UtilTest(unittest.TestCase):head_dim = 8caches = torch_util.DecodeCache.init_decode_cache(- num_layers=num_layers,- batch_size=batch_size,- num_variates=num_variates,- num_total_input_patches=num_total_input_patches,- num_heads=num_heads,- head_dim=head_dim,+ num_layers=num_layers,+ batch_size=batch_size,+ num_variates=num_variates,+ num_total_input_patches=num_total_input_patches,+ num_heads=num_heads,+ head_dim=head_dim,)self.assertEqual(len(caches), num_layers)@@ -189,12 +190,12 @@ class UtilTest(unittest.TestCase):self.assertEqual(cache.next_index.shape, (leading_size,))self.assertEqual(cache.num_front_masked.shape, (leading_size,))self.assertEqual(- cache.key.shape,- (leading_size, num_total_input_patches, num_heads, head_dim),+ cache.key.shape,+ (leading_size, num_total_input_patches, num_heads, head_dim),)self.assertEqual(- cache.value.shape,- (leading_size, num_total_input_patches, num_heads, head_dim),+ cache.value.shape,+ (leading_size, num_total_input_patches, num_heads, head_dim),)np.testing.assert_allclose(cache.next_index.numpy(), 0)np.testing.assert_allclose(cache.num_front_masked.numpy(), 0)@@ -206,16 +207,16 @@ class UtilTest(unittest.TestCase):x = torch.tensor([[[1.0, 2.0, 3.0]]])mask = torch.zeros_like(x, dtype=torch.bool)new_n, new_mu, new_sigma = torch_util.update_running_stats(- torch.tensor([[n]]),- torch.tensor([[mu]]),- torch.tensor([[sigma]]),- x,- mask,+ torch.tensor([[n]]),+ torch.tensor([[mu]]),+ torch.tensor([[sigma]]),+ x,+ mask,)self.assertAlmostEqual(new_n[0, 0].item(), 3.0)self.assertAlmostEqual(new_mu[0, 0].item(), 2.0)self.assertAlmostEqual(- new_sigma[0, 0].item(), torch.std(x, unbiased=False).item(), places=6+ new_sigma[0, 0].item(), torch.std(x, unbiased=False).item(), places=6)def test_update_running_stats_multiple_batches_no_mask(self):@@ -225,23 +226,21 @@ class UtilTest(unittest.TestCase):mask2 = torch.zeros_like(x2, dtype=torch.bool)n1, mu1, sigma1 = torch_util.update_running_stats(- torch.tensor([[0.0]]),- torch.tensor([[0.0]]),- torch.tensor([[0.0]]),- x1,- mask1,- )- n2, mu2, sigma2 = torch_util.update_running_stats(- n1, mu1, sigma1, x2, mask2+ torch.tensor([[0.0]]),+ torch.tensor([[0.0]]),+ torch.tensor([[0.0]]),+ x1,+ mask1,)+ n2, mu2, sigma2 = torch_util.update_running_stats(n1, mu1, sigma1, x2, mask2)self.assertAlmostEqual(n2[0, 0].item(), 6.0)self.assertAlmostEqual(mu2[0, 0].item(), 3.5)concat_x = torch.cat([x1, x2], dim=-1)self.assertAlmostEqual(- sigma2[0, 0].item(),- torch.std(concat_x, unbiased=False).item(),- places=6,+ sigma2[0, 0].item(),+ torch.std(concat_x, unbiased=False).item(),+ places=6,)def test_update_running_stats_with_mask(self):@@ -249,19 +248,19 @@ class UtilTest(unittest.TestCase):x = torch.tensor([[[1.0, 100.0, 2.0, 3.0]]])mask = torch.tensor([[[False, True, False, False]]])new_n, new_mu, new_sigma = torch_util.update_running_stats(- torch.tensor([[n]]),- torch.tensor([[mu]]),- torch.tensor([[sigma]]),- x,- mask,+ torch.tensor([[n]]),+ torch.tensor([[mu]]),+ torch.tensor([[sigma]]),+ x,+ mask,)self.assertAlmostEqual(new_n[0, 0].item(), 3.0)self.assertAlmostEqual(new_mu[0, 0].item(), 2.0)valid_x = torch.tensor([1.0, 2.0, 3.0])self.assertAlmostEqual(- new_sigma[0, 0].item(),- torch.std(valid_x, unbiased=False).item(),- places=6,+ new_sigma[0, 0].item(),+ torch.std(valid_x, unbiased=False).item(),+ places=6,)def test_revin_no_reverse(self):@@ -270,9 +269,7 @@ class UtilTest(unittest.TestCase):sigma = torch.tensor([[1.0]])normalized_x = torch_util.revin(x, mu, sigma, reverse=False)expected = torch.tensor([[[-1.0, 0.0, 1.0]]])- np.testing.assert_allclose(- normalized_x.numpy(), expected.numpy(), atol=1e-6- )+ np.testing.assert_allclose(normalized_x.numpy(), expected.numpy(), atol=1e-6)def test_revin_reverse(self):x_normalized = torch.tensor([[[-1.0, 0.0, 1.0]]])@@ -280,9 +277,7 @@ class UtilTest(unittest.TestCase):sigma = torch.tensor([[1.0]])denormalized_x = torch_util.revin(x_normalized, mu, sigma, reverse=True)expected = torch.tensor([[[1.0, 2.0, 3.0]]])- np.testing.assert_allclose(- denormalized_x.numpy(), expected.numpy(), atol=1e-6- )+ np.testing.assert_allclose(denormalized_x.numpy(), expected.numpy(), atol=1e-6)def test_revin_near_zero_sigma(self):x = torch.tensor([[[2.0, 2.0, 2.0]]])@@ -290,29 +285,23 @@ class UtilTest(unittest.TestCase):sigma = torch.tensor([[1e-7]])normalized_x = torch_util.revin(x, mu, sigma, reverse=False)expected = torch.tensor([[[0.0, 0.0, 0.0]]])- np.testing.assert_allclose(- normalized_x.numpy(), expected.numpy(), atol=1e-6- )+ np.testing.assert_allclose(normalized_x.numpy(), expected.numpy(), atol=1e-6)def test_get_output_patch_via_roll(self):x = torch.tensor([[[[1, 2], [3, 4], [5, 6], [7, 8]]]], dtype=torch.float32)rolls = 2expected_output = torch.tensor(- [[[[3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 1, 2], [1, 2, 3, 4]]]],- dtype=torch.float32,+ [[[[3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 1, 2], [1, 2, 3, 4]]]],+ dtype=torch.float32,)output, _ = torch_util.get_output_patch_via_roll(x, rolls)- np.testing.assert_allclose(- output.numpy(), expected_output.numpy(), atol=1e-6- )+ np.testing.assert_allclose(output.numpy(), expected_output.numpy(), atol=1e-6)def test_get_running_stats_single_segment(self):values = torch.tensor([[[[1, 3], [5, 7]]]], dtype=torch.float32)masks = torch.zeros_like(values, dtype=torch.bool)segment_ids = torch.tensor([[0, 0]], dtype=torch.int32)- _, mu, sigma = torch_util.get_running_stats(- values, masks, segment_ids=segment_ids- )+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)expected_mu = torch.tensor([[[2.0, 4.0]]])expected_sigma = torch.tensor([[[1.0, 2.236068]]])@@ -320,14 +309,10 @@ class UtilTest(unittest.TestCase):np.testing.assert_allclose(sigma.numpy(), expected_sigma.numpy(), atol=1e-6)def test_get_running_stats_multiple_segments(self):- values = torch.tensor(- [[[[1, 3], [10, 10], [5, 7], [4, 6]]]], dtype=torch.float32- )+ values = torch.tensor([[[[1, 3], [10, 10], [5, 7], [4, 6]]]], dtype=torch.float32)masks = torch.zeros_like(values, dtype=torch.bool)segment_ids = torch.tensor([[0, 0, 1, 1]], dtype=torch.int32)- _, mu, sigma = torch_util.get_running_stats(- values, masks, segment_ids=segment_ids- )+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)expected_mu = torch.tensor([[[2.0, 6.0, 6.0, 5.5]]])expected_sigma = torch.tensor([[[1.0, 4.062019, 1.0, 1.118034]]])@@ -335,17 +320,13 @@ class UtilTest(unittest.TestCase):np.testing.assert_allclose(sigma.numpy(), expected_sigma.numpy(), atol=1e-6)def test_get_running_stats_with_masks(self):- values = torch.tensor(- [[[[1, 10, 3, 0], [100, 5, 200, 7]]]], dtype=torch.float32- )+ values = torch.tensor([[[[1, 10, 3, 0], [100, 5, 200, 7]]]], dtype=torch.float32)masks = torch.tensor(- [[[[False, True, False, True], [True, False, True, False]]]],- dtype=torch.bool,+ [[[[False, True, False, True], [True, False, True, False]]]],+ dtype=torch.bool,)segment_ids = torch.tensor([[0, 0]], dtype=torch.int32)- _, mu, sigma = torch_util.get_running_stats(- values, masks, segment_ids=segment_ids- )+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)expected_mu = torch.tensor([[[2.0, 4.0]]])expected_sigma = torch.tensor([[[1.0, 2.236068]]])diff --git a/src/timesfm3/timesfm3_forecaster.py b/src/timesfm3/timesfm3_forecaster.pyindex 61c0e37..c16da1c 100644--- a/src/timesfm3/timesfm3_forecaster.py+++ b/src/timesfm3/timesfm3_forecaster.py@@ -4,17 +4,16 @@ from __future__ import annotationsimport dataclassesimport gc-import loggingimport mathimport os-from typing import Any, Iterator+from collections.abc import Iterator+from typing import Anyimport numpy as npimport torch-from . import configs+from . import configs, utilfrom . import model as torch_model_lib-from . import util_MAX_CONTEXT_LENGTH = 16384_SIGMA_THRESHOLD: float = 1e-7@@ -25,8 +24,8 @@ _GC_MEMORY_THRESHOLD: float = 0.9class _ModelConfig:"""Configuration for a PyTorch TimesFM3 forecaster."""- # Path to checkpoint file (.pth or .safetensors).- checkpoint_path: str = "~/models/tfm3/timesfm3_torch_1944000.safetensors"+ # Path to checkpoint file (.pth or .safetensors) or Hugging Face repo ID.+ checkpoint_path: str = "google/timesfm-3.0-pytorch"# Batch size to use for inference.per_core_batch_size: int = 4@@ -39,7 +38,7 @@ class _ModelConfig:# Quantiles to predict.quantiles: list[float] = dataclasses.field(- default_factory=lambda: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]+ default_factory=lambda: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9])# Median quantile index for the forecast.@@ -97,8 +96,8 @@ class ForecastOutput:def try_gc(- device: torch.device | str | None = None,- gc_memory_threshold: float = _GC_MEMORY_THRESHOLD,+ device: torch.device | str | None = None,+ gc_memory_threshold: float = _GC_MEMORY_THRESHOLD,) -> None:"""Trigger Python GC and empty CUDA cache if memory exceeds threshold."""if device is not None and torch.cuda.is_available():@@ -193,26 +192,28 @@ class _Query:def context_length(self) -> int:return self.targets.shape[-1]- def format(self, global_context: int) -> tuple[- int,- np.ndarray,- np.ndarray,- np.ndarray | None,- np.ndarray | None,- bool,+ def format(+ self, global_context: int+ ) -> tuple[+ int,+ np.ndarray,+ np.ndarray,+ np.ndarray | None,+ np.ndarray | None,+ bool,]:"""Formats and left-pads/truncates the query to global_context length."""targets = np.atleast_2d(self.targets)masks = np.zeros((self.context_length,), dtype=bool)past_only_covariates = (- np.atleast_2d(self.past_only_covariates)- if self.past_only_covariates is not None- else None+ np.atleast_2d(self.past_only_covariates)+ if self.past_only_covariates is not None+ else None)past_future_covariates = (- np.atleast_2d(self.past_future_covariates)- if self.past_future_covariates is not None- else None+ np.atleast_2d(self.past_future_covariates)+ if self.past_future_covariates is not None+ else None)if self.context_length > global_context:@@ -222,99 +223,99 @@ class _Query:past_only_covariates = past_only_covariates[:, -global_context:]if past_future_covariates is not None:past_future_covariates = past_future_covariates[- :, -(global_context + self.horizon) :+ :, -(global_context + self.horizon) :]elif self.context_length < global_context:pad_len = global_context - self.context_lengthtargets = np.pad(- targets,- [(0, 0), (pad_len, 0)],- mode="constant",- constant_values=0.0,+ targets,+ [(0, 0), (pad_len, 0)],+ mode="constant",+ constant_values=0.0,)masks = np.pad(- masks,- [(pad_len, 0)],- mode="constant",- constant_values=True,+ masks,+ [(pad_len, 0)],+ mode="constant",+ constant_values=True,)if self.padded:masks = np.ones_like(masks, dtype=bool)if past_only_covariates is not None:past_only_covariates = np.pad(- past_only_covariates,- [(0, 0), (pad_len, 0)],- mode="constant",- constant_values=0.0,+ past_only_covariates,+ [(0, 0), (pad_len, 0)],+ mode="constant",+ constant_values=0.0,)if past_future_covariates is not None:past_future_covariates = np.pad(- past_future_covariates,- [(0, 0), (pad_len, 0)],- mode="constant",- constant_values=0.0,+ past_future_covariates,+ [(0, 0), (pad_len, 0)],+ mode="constant",+ constant_values=0.0,)return (- self.horizon,- targets,- masks,- past_only_covariates,- past_future_covariates,- self.padded,+ self.horizon,+ targets,+ masks,+ past_only_covariates,+ past_future_covariates,+ self.padded,)def _make_torch_model(- config: _ModelConfig,+ config: _ModelConfig,) -> torch_model_lib.TimesFM3Torch:"""Builds a PyTorch model using ModelConfig."""resblock_config = (- config.residual_block_config- if config.residual_block_config is not None- else configs.ResidualBlockConfig(- hidden_dims=1280,- output_dims=1280,- use_bias=False,- activation="relu",- )+ config.residual_block_config+ if config.residual_block_config is not None+ else configs.ResidualBlockConfig(+ hidden_dims=1280,+ output_dims=1280,+ use_bias=False,+ activation="relu",+ ))transformer_config = (- config.transformer_config- if config.transformer_config is not None- else configs.StackedTransformersConfig(- num_layers=20,- transformer=configs.TransformerConfig(- model_dims=1280,- hidden_dims=1280,- num_heads=16,- attention_norm="rms",- feedforward_norm="rms",- qk_norm="rms",- use_rope_seq=True,- use_rope_var=True,- use_bias=False,- ff_activation="relu",- deterministic=True,- ),- )+ config.transformer_config+ if config.transformer_config is not None+ else configs.StackedTransformersConfig(+ num_layers=20,+ transformer=configs.TransformerConfig(+ model_dims=1280,+ hidden_dims=1280,+ num_heads=16,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=True,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ ),+ ))t_model = torch_model_lib.TimesFM3Torch(- input_patch_len=config.input_patch_length,- output_patch_len=config.output_patch_length,- quantiles=config.quantiles,- use_variate_attention=config.use_variate_attention,- value_clip=config.value_clip,- input_transform=config.input_transform,- use_stitching=config.use_stitching,- use_linear_detrending=config.use_linear_detrending,- linear_detrending_threshold=config.linear_detrending_threshold,- use_iterative_cpm_revin=config.use_iterative_cpm_revin,- use_frozen_running_stats=config.use_frozen_running_stats,- residual_block_config=resblock_config,- transformer_config=transformer_config,+ input_patch_len=config.input_patch_length,+ output_patch_len=config.output_patch_length,+ quantiles=config.quantiles,+ use_variate_attention=config.use_variate_attention,+ value_clip=config.value_clip,+ input_transform=config.input_transform,+ use_stitching=config.use_stitching,+ use_linear_detrending=config.use_linear_detrending,+ linear_detrending_threshold=config.linear_detrending_threshold,+ use_iterative_cpm_revin=config.use_iterative_cpm_revin,+ use_frozen_running_stats=config.use_frozen_running_stats,+ residual_block_config=resblock_config,+ transformer_config=transformer_config,)t_model.eval()input_dim = 2 * (t_model.input_patch_len + t_model.output_patch_len)@@ -334,12 +335,27 @@ class TimesFM3Forecaster:self.config = configself._init_model()+ @classmethod+ def from_pretrained(+ cls,+ pretrained_model_name_or_path: str = "google/timesfm-3.0-pytorch",+ device: str | None = None,+ **kwargs: Any,+ ) -> TimesFM3Forecaster:+ """Instantiates a TimesFM3Forecaster from a pretrained HF repo or directory."""+ config = _ModelConfig(+ checkpoint_path=pretrained_model_name_or_path,+ device=device,+ **kwargs,+ )+ return cls(config=config)+@propertydef global_context(self) -> int:"""Max context length rounded up to the nearest input patch boundary."""return (- math.ceil(_MAX_CONTEXT_LENGTH / self.config.input_patch_length)- * self.config.input_patch_length+ math.ceil(_MAX_CONTEXT_LENGTH / self.config.input_patch_length)+ * self.config.input_patch_length)def _init_model(self):@@ -349,87 +365,117 @@ class TimesFM3Forecaster:else:self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")- self.model = _make_torch_model(self.config)-checkpoint_path = os.path.expanduser(self.config.checkpoint_path)- if os.path.exists(checkpoint_path):++ # If checkpoint_path is a directory with config.json or a Hugging Face repo ID:+ is_local_dir = os.path.isdir(checkpoint_path)+ is_local_file = os.path.isfile(checkpoint_path)++ if is_local_dir or not is_local_file:+ # Load via PyTorchModelHubMixin.from_pretrained (downloads config.json and weights)+ self.model = torch_model_lib.TimesFM3Torch.from_pretrained(+ checkpoint_path,+ )+ # Synchronize forecaster config with the loaded model config+ median_q_idx = self.config.median_quantile_index+ if median_q_idx >= len(self.model.quantiles):+ median_q_idx = len(self.model.quantiles) // 2++ self.config = dataclasses.replace(+ self.config,+ input_patch_length=self.model.input_patch_len,+ output_patch_length=self.model.output_patch_len,+ quantiles=list(self.model.quantiles),+ median_quantile_index=median_q_idx,+ residual_block_config=self.model.residual_block_config,+ transformer_config=self.model.transformer_config,+ use_variate_attention=self.model.use_variate_attention,+ value_clip=self.model.value_clip,+ use_stitching=self.model.use_stitching,+ use_linear_detrending=self.model.use_linear_detrending,+ linear_detrending_threshold=self.model.linear_detrending_threshold,+ use_iterative_cpm_revin=self.model.use_iterative_cpm_revin,+ use_frozen_running_stats=self.model.use_frozen_running_stats,+ input_transform=self.model.input_transform,+ )+ else:+ # Local file (.safetensors or .pth / .pt)+ self.model = _make_torch_model(self.config)if checkpoint_path.endswith(".safetensors"):state_dict = util.load_safetensors(checkpoint_path, device=self.device)self.model.load_state_dict(state_dict)- elif checkpoint_path.endswith(".pth") or checkpoint_path.endswith(".pt"):+ elif checkpoint_path.endswith((".pth", ".pt")):state_dict = torch.load(checkpoint_path, map_location=self.device)self.model.load_state_dict(state_dict)else:raise ValueError(- f"Unsupported checkpoint path format: {checkpoint_path}. "- "Expected .safetensors or .pth / .pt file."+ f"Unsupported checkpoint path format: {checkpoint_path}. "+ "Expected .safetensors or .pth / .pt file.")self.model.to(self.device)self.model.eval()def predict(- self,- context: np.ndarray,- horizon: int,- past_only_covariates: np.ndarray | None = None,- past_future_covariates: np.ndarray | None = None,- ts_id: str | None = None,- return_quantiles: bool = False,- use_symmetric_averaging: bool = False,- make_positive: bool = False,- use_znorm: bool = False,- padding_mode: str = "none",+ self,+ context: np.ndarray,+ horizon: int,+ past_only_covariates: np.ndarray | None = None,+ past_future_covariates: np.ndarray | None = None,+ ts_id: str | None = None,+ return_quantiles: bool = False,+ use_symmetric_averaging: bool = False,+ make_positive: bool = False,+ use_znorm: bool = False,+ padding_mode: str = "none",) -> ForecastOutput:"""Convenience wrapper: runs inference on a single time series."""results = list(- self.predict_batch(- contexts=[context],- horizon=horizon,- past_only_covariates=[past_only_covariates],- past_future_covariates=[past_future_covariates],- ts_ids=[ts_id] if ts_id is not None else None,- return_quantiles=return_quantiles,- use_symmetric_averaging=use_symmetric_averaging,- make_positive=make_positive,- use_znorm=use_znorm,- padding_mode=padding_mode,- )+ self.predict_batch(+ contexts=[context],+ horizon=horizon,+ past_only_covariates=[past_only_covariates],+ past_future_covariates=[past_future_covariates],+ ts_ids=[ts_id] if ts_id is not None else None,+ return_quantiles=return_quantiles,+ use_symmetric_averaging=use_symmetric_averaging,+ make_positive=make_positive,+ use_znorm=use_znorm,+ padding_mode=padding_mode,+ ))return results[0]def predict_batch(- self,- contexts: list[np.ndarray],- horizon: int,- past_only_covariates: list[np.ndarray | None] | None = None,- past_future_covariates: list[np.ndarray | None] | None = None,- ts_ids: list[str] | None = None,- return_quantiles: bool = False,- use_symmetric_averaging: bool = False,- make_positive: bool = False,- use_znorm: bool = False,- padding_mode: str = "none",+ self,+ contexts: list[np.ndarray],+ horizon: int,+ past_only_covariates: list[np.ndarray | None] | None = None,+ past_future_covariates: list[np.ndarray | None] | None = None,+ ts_ids: list[str] | None = None,+ return_quantiles: bool = False,+ use_symmetric_averaging: bool = False,+ make_positive: bool = False,+ use_znorm: bool = False,+ padding_mode: str = "none",) -> Iterator[ForecastOutput]:"""Runs inference on a batch of time series with optional covariates."""global_horizon = (- math.ceil(horizon / self.config.output_patch_length)- * self.config.output_patch_length+ math.ceil(horizon / self.config.output_patch_length)+ * self.config.output_patch_length)num_original_ts = len(contexts)- original_ts_ids = (- list(ts_ids) if ts_ids is not None else [None] * num_original_ts- )+ original_ts_ids = list(ts_ids) if ts_ids is not None else [None] * num_original_tspo_cov_list = (- list(past_only_covariates)- if past_only_covariates is not None- else [None] * num_original_ts+ list(past_only_covariates)+ if past_only_covariates is not None+ else [None] * num_original_ts)pf_cov_list = (- list(past_future_covariates)- if past_future_covariates is not None- else [None] * num_original_ts+ list(past_future_covariates)+ if past_future_covariates is not None+ else [None] * num_original_ts)contexts_2d: list[np.ndarray] = []@@ -466,9 +512,9 @@ class TimesFM3Forecaster:for idx, ctx in enumerate(contexts_2d):if ctx.shape[0] != num_targets_in:raise ValueError(- "All contexts must have the same number of target variates, but"- f" contexts[0] has {num_targets_in} and contexts[{idx}] has"- f" {ctx.shape[0]}."+ "All contexts must have the same number of target variates, but"+ f" contexts[0] has {num_targets_in} and contexts[{idx}] has"+ f" {ctx.shape[0]}.")is_univariate = num_targets_in == 1@@ -546,12 +592,12 @@ class TimesFM3Forecaster:queries: list[_Query] = []for idx, ctx in enumerate(contexts_2d):queries.append(- _Query(- horizon=global_horizon,- targets=ctx,- past_only_covariates=po_2d[idx],- past_future_covariates=pf_2d[idx],- )+ _Query(+ horizon=global_horizon,+ targets=ctx,+ past_only_covariates=po_2d[idx],+ past_future_covariates=pf_2d[idx],+ ))if not queries:@@ -572,48 +618,48 @@ class TimesFM3Forecaster:formatted_batch = [q.format(self.global_context) for q in query_batch](- batched_hor,- batched_tgt,- batched_mask,- batched_po,- batched_pf,- _,+ batched_hor,+ batched_tgt,+ batched_mask,+ batched_po,+ batched_pf,+ _,) = tuple(list(w) for w in zip(*formatted_batch))tgt_torch = torch.from_numpy(np.stack(batched_tgt, axis=0)).to(- self.device, dtype=torch.float32+ self.device, dtype=torch.float32)mask_torch = torch.from_numpy(np.stack(batched_mask, axis=0)).to(- self.device, dtype=torch.bool+ self.device, dtype=torch.bool)po_torch = Noneif any(po is not None for po in batched_po):po_arrs = [- po if po is not None else np.zeros_like(batched_tgt[j])- for j, po in enumerate(batched_po)+ po if po is not None else np.zeros_like(batched_tgt[j])+ for j, po in enumerate(batched_po)]po_torch = torch.from_numpy(np.stack(po_arrs, axis=0)).to(- self.device, dtype=torch.float32+ self.device, dtype=torch.float32)pf_torch = Noneif any(pf is not None for pf in batched_pf):pf_arrs = [- pf if pf is not None else np.zeros_like(batched_tgt[j])- for j, pf in enumerate(batched_pf)+ pf if pf is not None else np.zeros_like(batched_tgt[j])+ for j, pf in enumerate(batched_pf)]pf_torch = torch.from_numpy(np.stack(pf_arrs, axis=0)).to(- self.device, dtype=torch.float32+ self.device, dtype=torch.float32)with torch.no_grad():out_logits = self.model.decode(- target=tgt_torch,- horizon=batched_hor[0],- past_only_covariates=po_torch,- past_future_covariates=pf_torch,- mask=mask_torch,+ target=tgt_torch,+ horizon=batched_hor[0],+ past_only_covariates=po_torch,+ past_future_covariates=pf_torch,+ mask=mask_torch,)ys.append(out_logits.cpu().numpy())try_gc(self.device)@@ -621,11 +667,9 @@ class TimesFM3Forecaster:all_raw_outputs = np.concatenate(ys, axis=0)num_relevant_outputs = (- 2 * num_original_ts if use_symmetric_averaging else num_original_ts+ 2 * num_original_ts if use_symmetric_averaging else num_original_ts)- all_raw_outputs = all_raw_outputs[- :num_relevant_outputs, :num_targets_in, :, :- ]+ all_raw_outputs = all_raw_outputs[:num_relevant_outputs, :num_targets_in, :, :]if use_symmetric_averaging:ys_pos = all_raw_outputs[0::2]@@ -659,13 +703,13 @@ class TimesFM3Forecaster:if is_univariate:raw = raw[0]yield ForecastOutput(- ts_id=original_ts_ids[i],- forecast=raw[:horizon, self.config.median_quantile_index],- quantiles=raw[:horizon, :] if return_quantiles else None,+ ts_id=original_ts_ids[i],+ forecast=raw[:horizon, self.config.median_quantile_index],+ quantiles=raw[:horizon, :] if return_quantiles else None,)else:yield ForecastOutput(- ts_id=original_ts_ids[i],- forecast=raw[:, :horizon, self.config.median_quantile_index],- quantiles=raw[:, :horizon, :] if return_quantiles else None,+ ts_id=original_ts_ids[i],+ forecast=raw[:, :horizon, self.config.median_quantile_index],+ quantiles=raw[:, :horizon, :] if return_quantiles else None,)diff --git a/src/timesfm3/timesfm3_forecaster_test.py b/src/timesfm3/timesfm3_forecaster_test.pyindex aa1d8bb..e9dc229 100644--- a/src/timesfm3/timesfm3_forecaster_test.py+++ b/src/timesfm3/timesfm3_forecaster_test.py@@ -16,6 +16,7 @@import unittestfrom unittest import mock+import numpy as npimport torch@@ -32,13 +33,13 @@ class FakeModel(torch.nn.Module):self.quantiles = quantilesdef decode(- self,- target,- autoregressive_index=0,- horizon=8,- past_only_covariates=None,- past_future_covariates=None,- mask=None,+ self,+ target,+ autoregressive_index=0,+ horizon=8,+ past_only_covariates=None,+ past_future_covariates=None,+ mask=None,):del autoregressive_index, past_only_covariates, past_future_covariates, maskb, v, _ = target.shape@@ -47,15 +48,14 @@ class FakeModel(torch.nn.Module):class TimesFM3ForecasterTest(unittest.TestCase):-def setUp(self):super().setUp()self.config = timesfm3_forecaster._ModelConfig(- checkpoint_path="/fake/path",- per_core_batch_size=2,- input_patch_length=8,- output_patch_length=8,- median_quantile_index=1,+ checkpoint_path="/fake/path",+ per_core_batch_size=2,+ input_patch_length=8,+ output_patch_length=8,+ median_quantile_index=1,)def test_strip_leading_nans_1d(self):@@ -64,15 +64,19 @@ class TimesFM3ForecasterTest(unittest.TestCase):np.testing.assert_array_equal(res, np.array([1.0, 2.0, np.nan]))def test_strip_leading_nans_2d(self):- arr = np.array([+ arr = np.array(+ [[np.nan, 1.0, 2.0],[np.nan, np.nan, 3.0],- ])+ ]+ )res = timesfm3_forecaster.strip_leading_nans(arr)- expected = np.array([+ expected = np.array(+ [[1.0, 2.0],[np.nan, 3.0],- ])+ ]+ )np.testing.assert_array_equal(res, expected)def test_linear_interpolation(self):@@ -93,7 +97,7 @@ class TimesFM3ForecasterTest(unittest.TestCase):self.assertFalse(timesfm3_forecaster._is_nonnegative(arr_neg))@mock.patch.object(- timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True+ timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True)def test_univariate_predict(self, _):forecaster = timesfm3_forecaster.TimesFM3Forecaster(self.config)@@ -106,30 +110,78 @@ class TimesFM3ForecasterTest(unittest.TestCase):np.testing.assert_array_equal(out.forecast, np.full((4,), 2.0))@mock.patch.object(- timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True+ timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True)def test_multivariate_predict_with_covariates(self, _):forecaster = timesfm3_forecaster.TimesFM3Forecaster(self.config)forecaster.model = FakeModel()forecaster.device = torch.device("cpu")- ctx = np.array([+ ctx = np.array(+ [[1.0, 2.0, 3.0, 4.0],[5.0, 6.0, 7.0, 8.0],- ])+ ]+ )po_cov = np.array([1.0, 1.0, 1.0, 1.0])pf_cov = np.array([2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0])out = forecaster.predict(- ctx,- horizon=4,- past_only_covariates=po_cov,- past_future_covariates=pf_cov,- return_quantiles=True,+ ctx,+ horizon=4,+ past_only_covariates=po_cov,+ past_future_covariates=pf_cov,+ return_quantiles=True,)self.assertEqual(out.forecast.shape, (2, 4))self.assertEqual(out.quantiles.shape, (2, 4, 3))+ def test_from_pretrained_local_directory(self):+ import tempfile++ from . import configs+ from . import model as torch_model_lib++ resblock_config = configs.ResidualBlockConfig(+ hidden_dims=16,+ output_dims=16,+ use_bias=False,+ activation="relu",+ )+ transformer_config = configs.StackedTransformersConfig(+ num_layers=1,+ transformer=configs.TransformerConfig(+ model_dims=16,+ hidden_dims=16,+ num_heads=2,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=False,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ ),+ )+ model = torch_model_lib.TimesFM3Torch(+ input_patch_len=8,+ output_patch_len=16,+ quantiles=[0.1, 0.5, 0.9],+ residual_block_config=resblock_config,+ transformer_config=transformer_config,+ )+ with tempfile.TemporaryDirectory() as tmpdir:+ model.save_pretrained(tmpdir)+ forecaster = timesfm3_forecaster.TimesFM3Forecaster.from_pretrained(+ tmpdir, device="cpu"+ )+ self.assertEqual(forecaster.config.input_patch_length, 8)+ self.assertEqual(forecaster.config.output_patch_length, 16)+ ctx = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])+ out = forecaster.predict(ctx, horizon=8)+ self.assertEqual(out.forecast.shape, (8,))+if __name__ == "__main__":unittest.main()diff --git a/src/timesfm3/transformations.py b/src/timesfm3/transformations.pyindex 6167dc8..c25da83 100644--- a/src/timesfm3/transformations.py+++ b/src/timesfm3/transformations.py@@ -6,7 +6,8 @@ Each transformation is a callable with signature (x, reverse=False) -> x'.from __future__ import annotations-from typing import Callable, Protocol+from collections.abc import Callable+from typing import Protocolimport torch@@ -14,8 +15,7 @@ import torchclass TransformFn(Protocol):"""Protocol for a reversible transformation function."""- def __call__(self, x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor:- ...+ def __call__(self, x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor: ...def signed_log(x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor:@@ -51,12 +51,12 @@ def _max_output_identity(value_clip: float) -> torch.Tensor:_REGISTRY: dict[- str,- tuple[TransformFn, Callable[[float], torch.Tensor]],+ str,+ tuple[TransformFn, Callable[[float], torch.Tensor]],] = {- "signed_log": (signed_log, _max_output_signed_log),- "signed_sqrt": (signed_sqrt, _max_output_signed_sqrt),- "identity": (identity, _max_output_identity),+ "signed_log": (signed_log, _max_output_signed_log),+ "signed_sqrt": (signed_sqrt, _max_output_signed_sqrt),+ "identity": (identity, _max_output_identity),}diff --git a/src/timesfm3/transformer.py b/src/timesfm3/transformer.pyindex db8e57e..9811fd3 100644--- a/src/timesfm3/transformer.py+++ b/src/timesfm3/transformer.py@@ -12,20 +12,18 @@ from __future__ import annotationsimport mathimport torch-from torch import nnimport torch.nn.functional as F+from torch import nn-from . import configs-from . import normalization-from . import util+from . import configs, normalization, utildef make_attn_mask(- query_length: int,- num_all_masked_kv: torch.Tensor,- query_index_offset: torch.Tensor | None = None,- kv_length: int = 0,- causal: bool = True,+ query_length: int,+ num_all_masked_kv: torch.Tensor,+ query_index_offset: torch.Tensor | None = None,+ kv_length: int = 0,+ causal: bool = True,) -> torch.Tensor:"""Makes attention mask. True = attend, False = mask.@@ -54,7 +52,7 @@ def make_attn_mask(def make_segment_mask(- segment_ids: torch.Tensor,+ segment_ids: torch.Tensor,) -> torch.Tensor:"""Makes a segment mask from segment ids.@@ -75,10 +73,10 @@ class RotaryPositionalEmbedding(nn.Module):"""def __init__(- self,- embedding_dims: int,- min_timescale: int = 1,- max_timescale: int = 10000,+ self,+ embedding_dims: int,+ min_timescale: int = 1,+ max_timescale: int = 10000,):super().__init__()self.embedding_dims = embedding_dims@@ -86,21 +84,14 @@ class RotaryPositionalEmbedding(nn.Module):self.max_timescale = max_timescalehalf_dim = embedding_dims // 2- fraction = (- 2.0- * torch.arange(half_dim, dtype=torch.float32)- / embedding_dims- )- timescale = (- min_timescale- * (max_timescale / min_timescale) ** fraction- )+ fraction = 2.0 * torch.arange(half_dim, dtype=torch.float32) / embedding_dims+ timescale = min_timescale * (max_timescale / min_timescale) ** fractionself.register_buffer("timescale", timescale, persistent=False)def forward(- self,- inputs: torch.Tensor,- position: torch.Tensor | None = None,+ self,+ inputs: torch.Tensor,+ position: torch.Tensor | None = None,) -> torch.Tensor:"""Applies rotary positional embeddings.@@ -113,15 +104,15 @@ class RotaryPositionalEmbedding(nn.Module):"""if self.embedding_dims != inputs.shape[-1]:raise ValueError(- "The embedding dims of the rotary position embedding "- "must match the hidden dimension of the inputs."+ "The embedding dims of the rotary position embedding "+ "must match the hidden dimension of the inputs.")timescale = self.timescale.to(inputs.device)if position is None:seq_length = inputs.shape[1]position = torch.arange(- seq_length, device=inputs.device, dtype=torch.float32+ seq_length, device=inputs.device, dtype=torch.float32).unsqueeze(0)if inputs.dim() == 4:@@ -152,17 +143,17 @@ class MultiHeadAttention(nn.Module):"""def __init__(- self,- num_heads: int,- in_features: int,- use_per_dim_scale: bool = True,- use_rotary_position_embeddings: bool = True,- causal_attention: bool = True,- use_bias: bool = False,- qk_norm: str = "rms",- v_norm: str = "none",- use_sdpa: bool = False,- rescale_logits: bool = False,+ self,+ num_heads: int,+ in_features: int,+ use_per_dim_scale: bool = True,+ use_rotary_position_embeddings: bool = True,+ causal_attention: bool = True,+ use_bias: bool = False,+ qk_norm: str = "rms",+ v_norm: str = "none",+ use_sdpa: bool = False,+ rescale_logits: bool = False,):super().__init__()self.num_heads = num_heads@@ -204,7 +195,7 @@ class MultiHeadAttention(nn.Module):# RoPEif use_rotary_position_embeddings:self.rotary_position_embedding = RotaryPositionalEmbedding(- embedding_dims=self.head_dim+ embedding_dims=self.head_dim)else:self.rotary_position_embedding = None@@ -216,13 +207,13 @@ class MultiHeadAttention(nn.Module):self.per_dim_scale = Nonedef forward(- self,- inputs_q: torch.Tensor,- *,- segment_ids: torch.Tensor | None = None,- segment_pos: torch.Tensor | None = None,- decode_cache: util.DecodeCache | None = None,- patch_mask: torch.Tensor | None = None,+ self,+ inputs_q: torch.Tensor,+ *,+ segment_ids: torch.Tensor | None = None,+ segment_pos: torch.Tensor | None = None,+ decode_cache: util.DecodeCache | None = None,+ patch_mask: torch.Tensor | None = None,) -> tuple[torch.Tensor, util.DecodeCache | None, torch.Tensor]:"""Applies multi-head attention.@@ -243,25 +234,21 @@ class MultiHeadAttention(nn.Module):device = inputs_q.deviceif patch_mask is None:- patch_mask = torch.zeros(- batch_size, n_patches, dtype=torch.bool, device=device- )+ patch_mask = torch.zeros(batch_size, n_patches, dtype=torch.bool, device=device)# Project Q, K, V and reshape to (b, n, h, hd)query = self.query_proj(inputs_q).view(- batch_size, n_patches, self.num_heads, self.head_dim+ batch_size, n_patches, self.num_heads, self.head_dim)key = self.key_proj(inputs_q).view(- batch_size, n_patches, self.num_heads, self.head_dim+ batch_size, n_patches, self.num_heads, self.head_dim)value = self.value_proj(inputs_q).view(- batch_size, n_patches, self.num_heads, self.head_dim+ batch_size, n_patches, self.num_heads, self.head_dim)if decode_cache is None:- num_front_masked = torch.sum(- torch.cumprod(patch_mask.int(), dim=-1), dim=-1- )+ num_front_masked = torch.sum(torch.cumprod(patch_mask.int(), dim=-1), dim=-1)next_index = torch.zeros_like(num_front_masked, dtype=torch.int32)else:num_front_masked = decode_cache.num_front_masked@@ -270,9 +257,9 @@ class MultiHeadAttention(nn.Module):# Apply RoPEif self.rotary_position_embedding is not None:if segment_pos is None:- position = torch.arange(- n_patches, device=device, dtype=torch.int32- ).unsqueeze(0) + next_index.unsqueeze(-1)+ position = torch.arange(n_patches, device=device, dtype=torch.int32).unsqueeze(+ 0+ ) + next_index.unsqueeze(-1)else:position = segment_posquery = self.rotary_position_embedding(query, position)@@ -302,25 +289,25 @@ class MultiHeadAttention(nn.Module):value = decode_cache.valuedecode_cache = util.DecodeCache(- next_index=next_index + n_patches,- num_front_masked=num_front_masked,- key=key,- value=value,+ next_index=next_index + n_patches,+ num_front_masked=num_front_masked,+ key=key,+ value=value,)attn_mask = make_attn_mask(- query_length=n_patches,- num_all_masked_kv=num_front_masked,- query_index_offset=next_index,- kv_length=cache_size,- causal=self.causal_attention,+ query_length=n_patches,+ num_all_masked_kv=num_front_masked,+ query_index_offset=next_index,+ kv_length=cache_size,+ causal=self.causal_attention,)else:# Training / full-sequence modeattn_mask = make_attn_mask(- query_length=n_patches,- num_all_masked_kv=torch.zeros_like(num_front_masked),- causal=self.causal_attention,+ query_length=n_patches,+ num_all_masked_kv=torch.zeros_like(num_front_masked),+ causal=self.causal_attention,)# Apply patch_mask to K/V positionsattn_mask = attn_mask & (~patch_mask[:, None, None, :])@@ -345,20 +332,20 @@ class MultiHeadAttention(nn.Module):# → no internal division → net scale = √d.attn_scale = math.sqrt(self.head_dim)x = F.scaled_dot_product_attention(- query,- key,- value,- attn_mask=attn_mask.expand(-1, self.num_heads, -1, -1),- scale=attn_scale,+ query,+ key,+ value,+ attn_mask=attn_mask.expand(-1, self.num_heads, -1, -1),+ scale=attn_scale,)else:# --- Manual attention path ---# Convert mask: True=attend -> 0.0, False=mask -> -1e9 (matching Flax# MEA bias).float_mask = torch.where(- attn_mask.expand(-1, self.num_heads, -1, -1),- torch.tensor(0.0, device=device),- torch.tensor(-1e9, device=device),+ attn_mask.expand(-1, self.num_heads, -1, -1),+ torch.tensor(0.0, device=device),+ torch.tensor(-1e9, device=device),)if self.rescale_logits:# MEA=False equivalent: Q*√d then divide by √d → net scale = 1.0.@@ -366,9 +353,8 @@ class MultiHeadAttention(nn.Module):# internally, so the pre-multiplication and rescaling cancel out.query = query * math.sqrt(self.head_dim)attn_logits = (- torch.matmul(query, key.transpose(-2, -1))- / math.sqrt(self.head_dim)- + float_mask+ torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(self.head_dim)+ + float_mask)else:# MEA=True equivalent: Q*√d, no internal division → net scale = √d.@@ -402,9 +388,9 @@ class MixingTransformer(nn.Module):"""def __init__(- self,- config: configs.TransformerConfig,- use_variate_attention: bool = True,+ self,+ config: configs.TransformerConfig,+ use_variate_attention: bool = True,):super().__init__()self.config = config@@ -419,54 +405,50 @@ class MixingTransformer(nn.Module):# net scale=1.0).rescale_logits = not getattr(config, "use_memory_efficient_attention", False)self.seq_attn = MultiHeadAttention(+ num_heads=config.num_heads,+ in_features=config.model_dims,+ use_per_dim_scale=True,+ use_rotary_position_embeddings=config.use_rope_seq,+ qk_norm=config.qk_norm,+ v_norm=getattr(config, "v_norm", "none"),+ causal_attention=getattr(config, "causal_attention", True),+ use_bias=config.use_bias,+ use_sdpa=config.use_sdpa,+ rescale_logits=rescale_logits,+ )++ # Variate attention norms + module+ if use_variate_attention:+ self.pre_var_attn_ln = nn.RMSNorm(config.model_dims)+ self.post_var_attn_ln = nn.RMSNorm(config.model_dims)+ self.var_attn = MultiHeadAttention(num_heads=config.num_heads,in_features=config.model_dims,use_per_dim_scale=True,- use_rotary_position_embeddings=config.use_rope_seq,+ use_rotary_position_embeddings=config.use_rope_var,qk_norm=config.qk_norm,v_norm=getattr(config, "v_norm", "none"),- causal_attention=getattr(config, "causal_attention", True),+ causal_attention=False,use_bias=config.use_bias,use_sdpa=config.use_sdpa,rescale_logits=rescale_logits,- )-- # Variate attention norms + module- if use_variate_attention:- self.pre_var_attn_ln = nn.RMSNorm(config.model_dims)- self.post_var_attn_ln = nn.RMSNorm(config.model_dims)- self.var_attn = MultiHeadAttention(- num_heads=config.num_heads,- in_features=config.model_dims,- use_per_dim_scale=True,- use_rotary_position_embeddings=config.use_rope_var,- qk_norm=config.qk_norm,- v_norm=getattr(config, "v_norm", "none"),- causal_attention=False,- use_bias=config.use_bias,- use_sdpa=config.use_sdpa,- rescale_logits=rescale_logits,)# FFN norms + layersself.pre_ff_ln = nn.RMSNorm(config.model_dims)self.post_ff_ln = nn.RMSNorm(config.model_dims)- self.ff0 = nn.Linear(- config.model_dims, config.hidden_dims, bias=config.use_bias- )- self.ff1 = nn.Linear(- config.hidden_dims, config.model_dims, bias=config.use_bias- )+ self.ff0 = nn.Linear(config.model_dims, config.hidden_dims, bias=config.use_bias)+ self.ff1 = nn.Linear(config.hidden_dims, config.model_dims, bias=config.use_bias)self.activation = util.get_activation_fn(config.ff_activation)def forward(- self,- input_embeddings: torch.Tensor,- patch_mask: torch.Tensor,- segment_ids: torch.Tensor | None = None,- segment_pos: torch.Tensor | None = None,- decode_cache: util.DecodeCache | None = None,- var_segment_pos: torch.Tensor | None = None,+ self,+ input_embeddings: torch.Tensor,+ patch_mask: torch.Tensor,+ segment_ids: torch.Tensor | None = None,+ segment_pos: torch.Tensor | None = None,+ decode_cache: util.DecodeCache | None = None,+ var_segment_pos: torch.Tensor | None = None,) -> tuple[torch.Tensor, util.DecodeCache | None, torch.Tensor]:"""Forward pass.@@ -501,11 +483,11 @@ class MixingTransformer(nn.Module):seq_seg_pos_flat = seg_pos_bvn.reshape(b * v, n)seq_attn_out_flat, decode_cache, seq_attn_mask = self.seq_attn(- seq_attn_in_flat,- segment_ids=seq_seg_ids_flat,- segment_pos=seq_seg_pos_flat,- decode_cache=decode_cache,- patch_mask=patch_mask_flat,+ seq_attn_in_flat,+ segment_ids=seq_seg_ids_flat,+ segment_pos=seq_seg_pos_flat,+ decode_cache=decode_cache,+ patch_mask=patch_mask_flat,)seq_attn_out = seq_attn_out_flat.view(b, v, n, d)h1 = self.post_seq_attn_ln(seq_attn_out) + input_embeddings@@ -519,10 +501,10 @@ class MixingTransformer(nn.Module):var_patch_mask = patch_mask.permute(0, 2, 1).reshape(b * n, v)var_attn_out_flat, _, _ = self.var_attn(- var_attn_in_flat,- segment_pos=var_segment_pos,- decode_cache=None,- patch_mask=var_patch_mask,+ var_attn_in_flat,+ segment_pos=var_segment_pos,+ decode_cache=None,+ patch_mask=var_patch_mask,)# (b*n, v, d) -> (b, n, v, d) -> (b, v, n, d)var_attn_out = var_attn_out_flat.view(b, n, v, d).permute(0, 2, 1, 3)@@ -541,28 +523,30 @@ class StackedMixingTransformer(nn.Module):"""Stacked MixingTransformer layers."""def __init__(- self,- config: configs.StackedTransformersConfig,- use_variate_attention: bool = True,+ self,+ config: configs.StackedTransformersConfig,+ use_variate_attention: bool = True,):super().__init__()self.config = config- self.layers = nn.ModuleList([+ self.layers = nn.ModuleList(+ [MixingTransformer(- config=config.transformer,- use_variate_attention=use_variate_attention,+ config=config.transformer,+ use_variate_attention=use_variate_attention,)for _ in range(config.num_layers)- ])+ ]+ )def forward(- self,- input_embeddings: torch.Tensor,- patch_mask: torch.Tensor,- segment_ids: torch.Tensor | None = None,- segment_pos: torch.Tensor | None = None,- decode_cache: list[util.DecodeCache] | None = None,- var_segment_pos: torch.Tensor | None = None,+ self,+ input_embeddings: torch.Tensor,+ patch_mask: torch.Tensor,+ segment_ids: torch.Tensor | None = None,+ segment_pos: torch.Tensor | None = None,+ decode_cache: list[util.DecodeCache] | None = None,+ var_segment_pos: torch.Tensor | None = None,) -> tuple[torch.Tensor, list[util.DecodeCache] | None, list[torch.Tensor]]:"""Forward pass through all layers.@@ -586,12 +570,12 @@ class StackedMixingTransformer(nn.Module):for i, layer in enumerate(self.layers):output, layer_cache, layer_mask = layer(- output,- patch_mask,- segment_ids,- segment_pos,- decode_cache[i], # pyrefly: ignore[unsupported-operation]- var_segment_pos,+ output,+ patch_mask,+ segment_ids,+ segment_pos,+ decode_cache[i], # pyrefly: ignore[unsupported-operation]+ var_segment_pos,)new_caches.append(layer_cache)attn_masks.append(layer_mask)diff --git a/src/timesfm3/transformer_test.py b/src/timesfm3/transformer_test.pyindex e482eb6..2498cdf 100644--- a/src/timesfm3/transformer_test.py+++ b/src/timesfm3/transformer_test.py@@ -15,7 +15,7 @@"""Tests for TimesFM3 PyTorch transformer layers."""import unittest-import numpy as np+import torchfrom . import configs@@ -24,12 +24,11 @@ from . import util as torch_utilclass MaskTest(unittest.TestCase):-def test_make_attn_mask_causal(self):query_len = 4num_masked = torch.tensor([1])mask = torch_trans.make_attn_mask(- query_length=query_len, num_all_masked_kv=num_masked, causal=True+ query_length=query_len, num_all_masked_kv=num_masked, causal=True)# Shape: (1, 1, 4, 4)self.assertEqual(mask.shape, (1, 1, 4, 4))@@ -49,7 +48,6 @@ class MaskTest(unittest.TestCase):class RotaryEmbeddingTest(unittest.TestCase):-def test_rope_forward(self):rope = torch_trans.RotaryPositionalEmbedding(embedding_dims=16)x = torch.randn(2, 8, 16)@@ -65,57 +63,53 @@ class RotaryEmbeddingTest(unittest.TestCase):class MultiHeadAttentionTest(unittest.TestCase):-def test_mha_forward_and_cache(self):mha = torch_trans.MultiHeadAttention(- num_heads=4,- in_features=64,- qk_norm="rms",- use_bias=False,- use_sdpa=True,+ num_heads=4,+ in_features=64,+ qk_norm="rms",+ use_bias=False,+ use_sdpa=True,)b, n, d = 2, 8, 64x = torch.randn(b, n, d)patch_mask = torch.zeros(b, n, dtype=torch.bool)# Full forward pass- out, cache, mask = mha(x, patch_mask=patch_mask)+ out, cache, _ = mha(x, patch_mask=patch_mask)self.assertEqual(out.shape, (b, n, d))self.assertIsNone(cache)# Decode mode with cachedecode_cache = torch_util.DecodeCache(- next_index=torch.zeros(b, dtype=torch.int32),- num_front_masked=torch.zeros(b, dtype=torch.int32),- key=torch.zeros(b, 16, 4, 16),- value=torch.zeros(b, 16, 4, 16),+ next_index=torch.zeros(b, dtype=torch.int32),+ num_front_masked=torch.zeros(b, dtype=torch.int32),+ key=torch.zeros(b, 16, 4, 16),+ value=torch.zeros(b, 16, 4, 16),)q_x = torch.randn(b, 1, d)q_mask = torch.zeros(b, 1, dtype=torch.bool)- out_step, updated_cache, _ = mha(- q_x, patch_mask=q_mask, decode_cache=decode_cache- )+ out_step, updated_cache, _ = mha(q_x, patch_mask=q_mask, decode_cache=decode_cache)self.assertEqual(out_step.shape, (b, 1, d))self.assertIsNotNone(updated_cache)self.assertEqual(updated_cache.next_index[0].item(), 1)class MixingTransformerTest(unittest.TestCase):-def test_mixing_transformer_forward(self):cfg = configs.TransformerConfig(- model_dims=64,- hidden_dims=64,- num_heads=4,- attention_norm="rms",- feedforward_norm="rms",- qk_norm="rms",- use_rope_seq=True,- use_rope_var=True,- use_bias=False,- ff_activation="relu",- deterministic=True,- use_sdpa=True,+ model_dims=64,+ hidden_dims=64,+ num_heads=4,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=True,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ use_sdpa=True,)layer = torch_trans.MixingTransformer(config=cfg, use_variate_attention=True)b, v, n, d = 2, 3, 8, 64@@ -128,35 +122,34 @@ class MixingTransformerTest(unittest.TestCase):class StackedMixingTransformerTest(unittest.TestCase):-def test_stacked_transformer(self):sub_cfg = configs.TransformerConfig(- model_dims=64,- hidden_dims=64,- num_heads=4,- attention_norm="rms",- feedforward_norm="rms",- qk_norm="rms",- use_rope_seq=True,- use_rope_var=True,- use_bias=False,- ff_activation="relu",- deterministic=True,- use_sdpa=True,+ model_dims=64,+ hidden_dims=64,+ num_heads=4,+ attention_norm="rms",+ feedforward_norm="rms",+ qk_norm="rms",+ use_rope_seq=True,+ use_rope_var=True,+ use_bias=False,+ ff_activation="relu",+ deterministic=True,+ use_sdpa=True,)stack_cfg = configs.StackedTransformersConfig(- num_layers=2,- use_remat=False,- transformer=sub_cfg,+ num_layers=2,+ use_remat=False,+ transformer=sub_cfg,)stack = torch_trans.StackedMixingTransformer(- config=stack_cfg, use_variate_attention=True+ config=stack_cfg, use_variate_attention=True)b, v, n, d = 2, 2, 4, 64inputs = torch.randn(b, v, n, d)patch_mask = torch.zeros(b, v, n, dtype=torch.bool)- out, caches, _ = stack(inputs, patch_mask=patch_mask)+ out, _, _ = stack(inputs, patch_mask=patch_mask)self.assertEqual(out.shape, (b, v, n, d))diff --git a/src/timesfm3/util.py b/src/timesfm3/util.pyindex ba6347b..5c303fe 100644--- a/src/timesfm3/util.py+++ b/src/timesfm3/util.py@@ -4,12 +4,11 @@ from __future__ import annotationsimport dataclassesimport os-import tempfile-from typing import Callable+from collections.abc import Callable-from safetensors import torch as safetensors_torchimport torchimport torch.nn.functional as F+from safetensors import torch as safetensors_torch_TOLERANCE = 1e-6@@ -39,15 +38,15 @@ class DecodeCache:@classmethoddef init_decode_cache(- cls,- num_layers: int,- batch_size: int,- num_variates: int,- num_total_input_patches: int,- num_heads: int,- head_dim: int,- device: torch.device | None = None,- ) -> list["DecodeCache"]:+ cls,+ num_layers: int,+ batch_size: int,+ num_variates: int,+ num_total_input_patches: int,+ num_heads: int,+ head_dim: int,+ device: torch.device | None = None,+ ) -> list[DecodeCache]:"""Initializes a list of decode caches for stacked layers.Args:@@ -64,38 +63,34 @@ class DecodeCache:"""leading_size = batch_size * num_variatesreturn [- cls(- next_index=torch.zeros(- leading_size, dtype=torch.int32, device=device- ),- num_front_masked=torch.zeros(- leading_size, dtype=torch.int32, device=device- ),- key=torch.zeros(- leading_size,- num_total_input_patches,- num_heads,- head_dim,- device=device,- ),- value=torch.zeros(- leading_size,- num_total_input_patches,- num_heads,- head_dim,- device=device,- ),- )- for _ in range(num_layers)+ cls(+ next_index=torch.zeros(leading_size, dtype=torch.int32, device=device),+ num_front_masked=torch.zeros(leading_size, dtype=torch.int32, device=device),+ key=torch.zeros(+ leading_size,+ num_total_input_patches,+ num_heads,+ head_dim,+ device=device,+ ),+ value=torch.zeros(+ leading_size,+ num_total_input_patches,+ num_heads,+ head_dim,+ device=device,+ ),+ )+ for _ in range(num_layers)]def update_running_stats(- n: torch.Tensor,- mu: torch.Tensor,- sigma: torch.Tensor,- x: torch.Tensor,- mask: torch.Tensor,+ n: torch.Tensor,+ mu: torch.Tensor,+ sigma: torch.Tensor,+ x: torch.Tensor,+ mask: torch.Tensor,) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:"""Updates running stats with a new patch of data.@@ -120,45 +115,43 @@ def update_running_stats(# std of valid elements in patchx_diff_sq = torch.where(- is_legit, (x - inc_mu.unsqueeze(-1)) ** 2, torch.zeros_like(x)+ is_legit, (x - inc_mu.unsqueeze(-1)) ** 2, torch.zeros_like(x))inc_var = torch.where(- inc_n == 0,- torch.zeros_like(inc_sum),- x_diff_sq.sum(dim=-1) / inc_n,+ inc_n == 0,+ torch.zeros_like(inc_sum),+ x_diff_sq.sum(dim=-1) / inc_n,)inc_sigma = torch.sqrt(inc_var)new_n = n + inc_nnew_mu = torch.where(- new_n == 0,- torch.zeros_like(mu),- (n * mu + inc_mu * inc_n) / new_n,+ new_n == 0,+ torch.zeros_like(mu),+ (n * mu + inc_mu * inc_n) / new_n,)new_sigma = torch.sqrt(- torch.where(- new_n == 0,- torch.zeros_like(sigma),- (- n * sigma * sigma- + inc_n * inc_sigma * inc_sigma- + n * (mu - new_mu) * (mu - new_mu)- + inc_n * (inc_mu - new_mu) * (inc_mu - new_mu)- )- / new_n,+ torch.where(+ new_n == 0,+ torch.zeros_like(sigma),+ (+ n * sigma * sigma+ + inc_n * inc_sigma * inc_sigma+ + n * (mu - new_mu) * (mu - new_mu)+ + inc_n * (inc_mu - new_mu) * (inc_mu - new_mu))+ / new_n,+ ))return new_n, new_mu, new_sigmadef get_running_stats(- values: torch.Tensor,- masks: torch.Tensor,- *,- segment_ids: torch.Tensor | None = None,- initial_stats: (- tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None- ) = None,+ values: torch.Tensor,+ masks: torch.Tensor,+ *,+ segment_ids: torch.Tensor | None = None,+ initial_stats: (tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None) = None,) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:"""Computes cumulative running statistics patch-by-patch.@@ -204,24 +197,24 @@ def get_running_stats(cur_sigma = torch.where(reset, init_sigma, cur_sigma)cur_n, cur_mu, cur_sigma = update_running_stats(- cur_n, cur_mu, cur_sigma, values[:, :, i, :], masks[:, :, i, :]+ cur_n, cur_mu, cur_sigma, values[:, :, i, :], masks[:, :, i, :])all_n.append(cur_n)all_mu.append(cur_mu)all_sigma.append(cur_sigma)return (- torch.stack(all_n, dim=2),- torch.stack(all_mu, dim=2),- torch.stack(all_sigma, dim=2),+ torch.stack(all_n, dim=2),+ torch.stack(all_mu, dim=2),+ torch.stack(all_sigma, dim=2),)def revin(- x: torch.Tensor,- mu: torch.Tensor,- sigma: torch.Tensor,- reverse: bool = False,+ x: torch.Tensor,+ mu: torch.Tensor,+ sigma: torch.Tensor,+ reverse: bool = False,) -> torch.Tensor:"""Reversible per-instance normalization.@@ -251,7 +244,7 @@ def revin(def get_output_patch_via_roll(- x: torch.Tensor, rolls: int+ x: torch.Tensor, rolls: int) -> tuple[torch.Tensor, torch.Tensor]:"""Creates labels of output_patch length by rolling the patched inputs.@@ -274,7 +267,7 @@ def get_output_patch_via_roll(for i in range(rolls):rolling_mat[:, :, :, i + 1, :] = torch.roll(- rolling_mat[:, :, :, i, :], shifts=-1, dims=2+ rolling_mat[:, :, :, i, :], shifts=-1, dims=2)# Take [1:] along the roll axis and flatten@@ -290,30 +283,30 @@ def get_output_patch_via_roll(_ACTIVATIONS: dict[str, Callable[[torch.Tensor], torch.Tensor]] = {- "relu": F.relu,- "swish": F.silu,- "silu": F.silu,- "swiglu": F.silu,- "none": lambda x: x,+ "relu": F.relu,+ "swish": F.silu,+ "silu": F.silu,+ "swiglu": F.silu,+ "none": lambda x: x,}def get_activation_fn(- activation_name: str,+ activation_name: str,) -> Callable[[torch.Tensor], torch.Tensor]:"""Returns the activation function for the given name."""try:return _ACTIVATIONS[activation_name]except KeyError:raise ValueError(- f"Activation: {activation_name} not supported. Supported "- f"activations: {list(_ACTIVATIONS.keys())}"+ f"Activation: {activation_name} not supported. Supported "+ f"activations: {list(_ACTIVATIONS.keys())}") from Nonedef load_safetensors(- pytorch_safetensors_path: str,- device: str | torch.device = "cpu",+ pytorch_safetensors_path: str,+ device: str | torch.device = "cpu",) -> dict[str, torch.Tensor]:"""Loads a PyTorch state dict from a safetensors file.@@ -329,8 +322,8 @@ def load_safetensors(def stitch_patches(- patch_preds: torch.Tensor,- patch_len: int,+ patch_preds: torch.Tensor,+ patch_len: int,) -> torch.Tensor:"""Stitches overlapping patch predictions.@@ -354,7 +347,7 @@ def stitch_patches(return patch_preds[:, :, 0, :, :]stitch_weights = torch.linspace(- 1.0, 0.0, overlap, device=patch_preds.device, dtype=patch_preds.dtype+ 1.0, 0.0, overlap, device=patch_preds.device, dtype=patch_preds.dtype)stitch_weights = stitch_weights[None, None, None, :, None]@@ -367,8 +360,7 @@ def stitch_patches(next_overlaps = next_patches[:, :, :, :overlap, :]stitched_overlaps = (- stitch_weights * prev_overlaps- + (1.0 - stitch_weights) * next_overlaps+ stitch_weights * prev_overlaps + (1.0 - stitch_weights) * next_overlaps)middles = next_patches[:, :, :, overlap:patch_len, :]@@ -380,4 +372,3 @@ def stitch_patches(tail = patch_preds[:, :, -1, patch_len:, :]return torch.cat([first_chunk, mid, tail], dim=2)-diff --git a/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb b/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynbindex 7fb5851..c9bf45a 100644--- a/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb+++ b/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb@@ -1,273 +1,270 @@{- "cells": [- {- "cell_type": "markdown",- "metadata": {},- "source": [- "# TimesFM 3.0 PyTorch Inference Demo\n",- "\n",- "This interactive notebook demonstrates zero-shot time series forecasting using **TimesFM 3.0** (PyTorch).\n",- "\n",- "### Key Features Highlighted:\n",- "1. **Promotion Example with Dynamic Covariates**: Context length = 2048, Forecast Horizon = 512 with dynamic past-future promotion indicator.\n",- "2. **Synthetic Curves (Horizon > 64)**: Evaluating long horizons across sine waves and linear trend extrapolation.\n",- "3. **GPU Acceleration**: Automatic CUDA inference with quantile uncertainty intervals."- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## 1. Setup and Imports"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "import os\n",- "import numpy as np\n",- "import matplotlib.pyplot as plt\n",- "import torch\n",- "\n",- "from timesfm3 import TimesFM3Forecaster, ModelConfig\n",- "\n",- "# Set random seeds for reproducibility\n",- "np.random.seed(42)\n",- "torch.manual_seed(42)\n",- "\n",- "print(f\"PyTorch version: {torch.__version__}\")\n",- "print(f\"CUDA available: {torch.cuda.is_available()}\")"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## 2. Initialize TimesFM 3.0 Forecaster\n",- "\n",- "We instantiate `TimesFM3Forecaster` with the default 1280-dim, 20-layer, 16-head checkpoint."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "config = ModelConfig(\n",- " checkpoint_path=\"~/models/tfm3/timesfm3_torch_1944000.safetensors\",\n",- " input_patch_length=32,\n",- " output_patch_length=64,\n",- " per_core_batch_size=2,\n",- " use_sdpa=True,\n",- ")\n",- "\n",- "forecaster = TimesFM3Forecaster(config)\n",- "print(f\"TimesFM3 forecaster loaded successfully on device: {forecaster.device}\")"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## 3. Scenario 1: Promotion Example with Dynamic Covariates\n",- "\n",- "- **Context Length**: 2048\n",- "- **Forecast Horizon**: 512\n",- "- **Past-Future Covariates**: Binary promotion covariate (0 or 1) spanning the full length (2560 steps).\n",- "- **Target Multiplier**: Target value is boosted by 5x during active promotions."- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "context_len = 2048\n",- "horizon_len = 512\n",- "total_len = context_len + horizon_len\n",- "time = np.arange(total_len)\n",- "\n",- "# Generate promotion indicator\n",- "promotions = np.zeros(total_len, dtype=np.float32)\n",- "# Promotion in context\n",- "promotions[1130:1180] = 1.0\n",- "# Promotions in forecast horizon\n",- "promotions[2250:2290] = 1.0\n",- "promotions[2320:2355] = 1.0\n",- "\n",- "pfc_np = np.array([promotions], dtype=np.float32)\n",- "\n",- "# Generate base target: periodic sine + noise\n",- "base_target = 2.0 + np.sin(time / 20.0) + np.random.randn(total_len) * 0.15\n",- "# Apply 5x promo boost\n",- "target_full = (base_target * np.where(promotions == 1.0, 5.0, 1.0)).astype(np.float32)\n",- "\n",- "context = target_full[:context_len]\n",- "actual_horizon = target_full[context_len:]\n",- "\n",- "print(f\"Context shape: {context.shape}\")\n",- "print(f\"Horizon length: {horizon_len}\")\n",- "print(f\"Past-Future Covariates shape: {pfc_np.shape}\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Run zero-shot forecast with dynamic past-future covariates\n",- "torch_out = forecaster.predict(\n",- " context=context,\n",- " horizon=horizon_len,\n",- " past_future_covariates=pfc_np,\n",- " return_quantiles=True,\n",- ")\n",- "\n",- "print(\"Forecast generated successfully!\")\n",- "print(f\"Point forecast shape: {torch_out.forecast.shape}\")\n",- "print(f\"Quantiles shape: {torch_out.quantiles.shape}\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Plot Promotion Example\n",- "time_context = np.arange(context_len)\n",- "time_horizon = np.arange(context_len, total_len)\n",- "\n",- "fig, axes = plt.subplots(2, 1, figsize=(16, 8), sharex=True)\n",- "\n",- "# Subplot 0: Target, Actual Horizon, and PyTorch Forecast\n",- "axes[0].plot(time_context, context, label=\"Context (Input)\", color=\"black\", alpha=0.7, linewidth=1.2)\n",- "axes[0].plot(time_horizon, actual_horizon, label=\"Actual Horizon (Ground Truth)\", linestyle=\"--\", color=\"gray\", alpha=0.8)\n",- "axes[0].plot(time_horizon, torch_out.forecast, label=\"TimesFM3 Median Forecast (p50)\", color=\"#ff7f0e\", linewidth=2.5)\n",- "if torch_out.quantiles is not None:\n",- " axes[0].fill_between(\n",- " time_horizon,\n",- " torch_out.quantiles[:, 0],\n",- " torch_out.quantiles[:, -1],\n",- " color=\"salmon\",\n",- " alpha=0.3,\n",- " label=\"p10-p90 Prediction Interval\",\n",- " )\n",- "axes[0].set_title(\"TimesFM 3.0 Promotion Example Forecast (Context 2048, Horizon 512)\", fontsize=14)\n",- "axes[0].set_ylabel(\"Target Value\", fontsize=12)\n",- "axes[0].legend(loc=\"upper left\")\n",- "axes[0].grid(True, alpha=0.3)\n",- "\n",- "# Subplot 1: Past-Future Promotion Covariate\n",- "axes[1].plot(time, promotions, label=\"Promotion Indicator (0 or 1)\", color=\"green\", drawstyle=\"steps-mid\", linewidth=1.5)\n",- "axes[1].fill_between(time, 0, promotions, color=\"green\", alpha=0.2)\n",- "axes[1].set_title(\"Past-Future Covariate: Promotions Timeline\", fontsize=12)\n",- "axes[1].set_ylabel(\"Promotion Active\", fontsize=12)\n",- "axes[1].set_xlabel(\"Time Step (t)\", fontsize=12)\n",- "axes[1].set_yticks([0, 1])\n",- "axes[1].legend(loc=\"upper left\")\n",- "axes[1].grid(True, alpha=0.3)\n",- "\n",- "plt.tight_layout()\n",- "plt.show()"- ]- },- {- "cell_type": "markdown",- "metadata": {},- "source": [- "## 4. Scenario 2: Long Horizon Synthetic Curves (Horizon 128 > 64)\n",- "\n",- "Evaluates rolling patch autoregressive decoding across:\n",- "1. **Sine Curve** (periodic continuation)\n",- "2. **Linear Trend** (linear detrending extrapolation)"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "synth_context_len = 256\n",- "synth_horizon_len = 128\n",- "synth_total_len = synth_context_len + synth_horizon_len\n",- "t_synth = np.linspace(0, 8 * np.pi, synth_total_len, dtype=np.float32)\n",- "\n",- "# Series 0: Sine wave\n",- "s_sine = np.sin(t_synth)\n",- "# Series 1: Linear trend line\n",- "s_line = np.arange(synth_total_len, dtype=np.float32) * 0.05 + 2.0\n",- "\n",- "contexts = [s_sine[:synth_context_len], s_line[:synth_context_len]]\n",- "ground_truths = [s_sine[synth_context_len:], s_line[synth_context_len:]]\n",- "\n",- "# Batch predict\n",- "synth_outputs = list(forecaster.predict_batch(contexts, horizon=synth_horizon_len))\n",- "\n",- "# Compute metrics\n",- "sine_mae = np.mean(np.abs(synth_outputs[0].forecast - ground_truths[0]))\n",- "line_mae = np.mean(np.abs(synth_outputs[1].forecast - ground_truths[1]))\n",- "\n",- "print(f\"Sine Curve Mean Absolute Error: {sine_mae:.6f}\")\n",- "print(f\"Linear Trend Mean Absolute Error: {line_mae:.6f}\")"- ]- },- {- "cell_type": "code",- "execution_count": null,- "metadata": {},- "outputs": [],- "source": [- "# Plot Synthetic Curves\n",- "fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n",- "t_ctx = np.arange(synth_context_len)\n",- "t_hor = np.arange(synth_context_len, synth_total_len)\n",- "titles = [\n",- " f\"Sine Curve Forecast (MAE: {sine_mae:.6f})\",\n",- " f\"Linear Trend Forecast (MAE: {line_mae:.6f})\",\n",- "]\n",- "\n",- "for i, ax in enumerate(axes):\n",- " ax.plot(t_ctx, contexts[i], label=\"Context (Input)\", color=\"black\", linewidth=1.5)\n",- " ax.plot(t_hor, ground_truths[i], label=\"Ground Truth\", color=\"gray\", linestyle=\"--\", alpha=0.7)\n",- " ax.plot(t_hor, synth_outputs[i].forecast, label=\"TimesFM3 Forecast\", color=\"#ff7f0e\", linewidth=2.5)\n",- " ax.set_title(titles[i], fontsize=13)\n",- " ax.set_ylabel(\"Value\", fontsize=11)\n",- " ax.legend(loc=\"upper left\")\n",- " ax.grid(True, alpha=0.3)\n",- "\n",- "axes[1].set_xlabel(\"Time Step\", fontsize=12)\n",- "plt.tight_layout()\n",- "plt.show()"- ]- }- ],- "metadata": {- "kernelspec": {- "display_name": ".venv",- "language": "python",- "name": "python3"- },- "language_info": {- "codemirror_mode": {- "name": "ipython",- "version": 3- },- "file_extension": ".py",- "mimetype": "text/x-python",- "name": "python",- "nbconvert_exporter": "python",- "pygments_lexer": "ipython3",- "version": "3.12.3"- }+ "cells": [+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "# TimesFM 3.0 PyTorch Inference Demo\n",+ "\n",+ "This interactive notebook demonstrates zero-shot time series forecasting using **TimesFM 3.0** (PyTorch).\n",+ "\n",+ "### Key Features Highlighted:\n",+ "1. **Promotion Example with Dynamic Covariates**: Context length = 2048, Forecast Horizon = 512 with dynamic past-future promotion indicator.\n",+ "2. **Synthetic Curves (Horizon > 64)**: Evaluating long horizons across sine waves and linear trend extrapolation.\n",+ "3. **GPU Acceleration**: Automatic CUDA inference with quantile uncertainty intervals."+ ]},- "nbformat": 4,- "nbformat_minor": 4+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 1. Setup and Imports"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "import os\n",+ "import numpy as np\n",+ "import matplotlib.pyplot as plt\n",+ "import torch\n",+ "\n",+ "from timesfm3 import TimesFM3Forecaster\n",+ "\n",+ "# Set random seeds for reproducibility\n",+ "np.random.seed(42)\n",+ "torch.manual_seed(42)\n",+ "\n",+ "print(f\"PyTorch version: {torch.__version__}\")\n",+ "print(f\"CUDA available: {torch.cuda.is_available()}\")\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 2. Initialize TimesFM 3.0 Forecaster\n",+ "\n",+ "We instantiate `TimesFM3Forecaster` from the default Hugging Face pretrained checkpoint (`google/timesfm-3.0-pytorch`).\n"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Load TimesFM 3.0 directly from Hugging Face Hub (default: \"google/timesfm-3.0-pytorch\")\n",+ "forecaster = TimesFM3Forecaster.from_pretrained(\n",+ " \"google/timesfm-3.0-pytorch\",\n",+ " per_core_batch_size=4,\n",+ ")\n",+ "\n",+ "print(f\"TimesFM3 forecaster loaded successfully on device: {forecaster.device}\")\n"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 3. Scenario 1: Promotion Example with Dynamic Covariates\n",+ "\n",+ "- **Context Length**: 2048\n",+ "- **Forecast Horizon**: 512\n",+ "- **Past-Future Covariates**: Binary promotion covariate (0 or 1) spanning the full length (2560 steps).\n",+ "- **Target Multiplier**: Target value is boosted by 5x during active promotions."+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "context_len = 2048\n",+ "horizon_len = 512\n",+ "total_len = context_len + horizon_len\n",+ "time = np.arange(total_len)\n",+ "\n",+ "# Generate promotion indicator\n",+ "promotions = np.zeros(total_len, dtype=np.float32)\n",+ "# Promotion in context\n",+ "promotions[1130:1180] = 1.0\n",+ "# Promotions in forecast horizon\n",+ "promotions[2250:2290] = 1.0\n",+ "promotions[2320:2355] = 1.0\n",+ "\n",+ "pfc_np = np.array([promotions], dtype=np.float32)\n",+ "\n",+ "# Generate base target: periodic sine + noise\n",+ "base_target = 2.0 + np.sin(time / 20.0) + np.random.randn(total_len) * 0.15\n",+ "# Apply 5x promo boost\n",+ "target_full = (base_target * np.where(promotions == 1.0, 5.0, 1.0)).astype(np.float32)\n",+ "\n",+ "context = target_full[:context_len]\n",+ "actual_horizon = target_full[context_len:]\n",+ "\n",+ "print(f\"Context shape: {context.shape}\")\n",+ "print(f\"Horizon length: {horizon_len}\")\n",+ "print(f\"Past-Future Covariates shape: {pfc_np.shape}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Run zero-shot forecast with dynamic past-future covariates\n",+ "torch_out = forecaster.predict(\n",+ " context=context,\n",+ " horizon=horizon_len,\n",+ " past_future_covariates=pfc_np,\n",+ " return_quantiles=True,\n",+ ")\n",+ "\n",+ "print(\"Forecast generated successfully!\")\n",+ "print(f\"Point forecast shape: {torch_out.forecast.shape}\")\n",+ "print(f\"Quantiles shape: {torch_out.quantiles.shape}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Plot Promotion Example\n",+ "time_context = np.arange(context_len)\n",+ "time_horizon = np.arange(context_len, total_len)\n",+ "\n",+ "fig, axes = plt.subplots(2, 1, figsize=(16, 8), sharex=True)\n",+ "\n",+ "# Subplot 0: Target, Actual Horizon, and PyTorch Forecast\n",+ "axes[0].plot(time_context, context, label=\"Context (Input)\", color=\"black\", alpha=0.7, linewidth=1.2)\n",+ "axes[0].plot(time_horizon, actual_horizon, label=\"Actual Horizon (Ground Truth)\", linestyle=\"--\", color=\"gray\", alpha=0.8)\n",+ "axes[0].plot(time_horizon, torch_out.forecast, label=\"TimesFM3 Median Forecast (p50)\", color=\"#ff7f0e\", linewidth=2.5)\n",+ "if torch_out.quantiles is not None:\n",+ " axes[0].fill_between(\n",+ " time_horizon,\n",+ " torch_out.quantiles[:, 0],\n",+ " torch_out.quantiles[:, -1],\n",+ " color=\"salmon\",\n",+ " alpha=0.3,\n",+ " label=\"p10-p90 Prediction Interval\",\n",+ " )\n",+ "axes[0].set_title(\"TimesFM 3.0 Promotion Example Forecast (Context 2048, Horizon 512)\", fontsize=14)\n",+ "axes[0].set_ylabel(\"Target Value\", fontsize=12)\n",+ "axes[0].legend(loc=\"upper left\")\n",+ "axes[0].grid(True, alpha=0.3)\n",+ "\n",+ "# Subplot 1: Past-Future Promotion Covariate\n",+ "axes[1].plot(time, promotions, label=\"Promotion Indicator (0 or 1)\", color=\"green\", drawstyle=\"steps-mid\", linewidth=1.5)\n",+ "axes[1].fill_between(time, 0, promotions, color=\"green\", alpha=0.2)\n",+ "axes[1].set_title(\"Past-Future Covariate: Promotions Timeline\", fontsize=12)\n",+ "axes[1].set_ylabel(\"Promotion Active\", fontsize=12)\n",+ "axes[1].set_xlabel(\"Time Step (t)\", fontsize=12)\n",+ "axes[1].set_yticks([0, 1])\n",+ "axes[1].legend(loc=\"upper left\")\n",+ "axes[1].grid(True, alpha=0.3)\n",+ "\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ },+ {+ "cell_type": "markdown",+ "metadata": {},+ "source": [+ "## 4. Scenario 2: Long Horizon Synthetic Curves (Horizon 128 > 64)\n",+ "\n",+ "Evaluates rolling patch autoregressive decoding across:\n",+ "1. **Sine Curve** (periodic continuation)\n",+ "2. **Linear Trend** (linear detrending extrapolation)"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "synth_context_len = 256\n",+ "synth_horizon_len = 128\n",+ "synth_total_len = synth_context_len + synth_horizon_len\n",+ "t_synth = np.linspace(0, 8 * np.pi, synth_total_len, dtype=np.float32)\n",+ "\n",+ "# Series 0: Sine wave\n",+ "s_sine = np.sin(t_synth)\n",+ "# Series 1: Linear trend line\n",+ "s_line = np.arange(synth_total_len, dtype=np.float32) * 0.05 + 2.0\n",+ "\n",+ "contexts = [s_sine[:synth_context_len], s_line[:synth_context_len]]\n",+ "ground_truths = [s_sine[synth_context_len:], s_line[synth_context_len:]]\n",+ "\n",+ "# Batch predict\n",+ "synth_outputs = list(forecaster.predict_batch(contexts, horizon=synth_horizon_len))\n",+ "\n",+ "# Compute metrics\n",+ "sine_mae = np.mean(np.abs(synth_outputs[0].forecast - ground_truths[0]))\n",+ "line_mae = np.mean(np.abs(synth_outputs[1].forecast - ground_truths[1]))\n",+ "\n",+ "print(f\"Sine Curve Mean Absolute Error: {sine_mae:.6f}\")\n",+ "print(f\"Linear Trend Mean Absolute Error: {line_mae:.6f}\")"+ ]+ },+ {+ "cell_type": "code",+ "execution_count": null,+ "metadata": {},+ "outputs": [],+ "source": [+ "# Plot Synthetic Curves\n",+ "fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n",+ "t_ctx = np.arange(synth_context_len)\n",+ "t_hor = np.arange(synth_context_len, synth_total_len)\n",+ "titles = [\n",+ " f\"Sine Curve Forecast (MAE: {sine_mae:.6f})\",\n",+ " f\"Linear Trend Forecast (MAE: {line_mae:.6f})\",\n",+ "]\n",+ "\n",+ "for i, ax in enumerate(axes):\n",+ " ax.plot(t_ctx, contexts[i], label=\"Context (Input)\", color=\"black\", linewidth=1.5)\n",+ " ax.plot(t_hor, ground_truths[i], label=\"Ground Truth\", color=\"gray\", linestyle=\"--\", alpha=0.7)\n",+ " ax.plot(t_hor, synth_outputs[i].forecast, label=\"TimesFM3 Forecast\", color=\"#ff7f0e\", linewidth=2.5)\n",+ " ax.set_title(titles[i], fontsize=13)\n",+ " ax.set_ylabel(\"Value\", fontsize=11)\n",+ " ax.legend(loc=\"upper left\")\n",+ " ax.grid(True, alpha=0.3)\n",+ "\n",+ "axes[1].set_xlabel(\"Time Step\", fontsize=12)\n",+ "plt.tight_layout()\n",+ "plt.show()"+ ]+ }+ ],+ "metadata": {+ "kernelspec": {+ "display_name": ".venv",+ "language": "python",+ "name": "python3"+ },+ "language_info": {+ "codemirror_mode": {+ "name": "ipython",+ "version": 3+ },+ "file_extension": ".py",+ "mimetype": "text/x-python",+ "name": "python",+ "nbconvert_exporter": "python",+ "pygments_lexer": "ipython3",+ "version": "3.12.3"+ }+ },+ "nbformat": 4,+ "nbformat_minor": 4}