Skip to content

TimesFM (Time Series Foundation Model) is a pretrained time-series foundation model developed by Google Research for time-series forecasting.

Python

enable hugginface support

Rajat Sen29a5ca6531eb
diff --git a/pyproject.toml b/pyproject.toml
index 743b737..041b25d 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -15,7 +15,7 @@ readme = "README.md"
requires-python = ">=3.10"
dependencies = [
"numpy>=1.26.4",
- "huggingface_hub[cli]>=0.23.0",
+ "huggingface_hub>=0.28.0",
"safetensors>=0.5.3",
]
diff --git a/src/timesfm3/__init__.py b/src/timesfm3/__init__.py
index b4a70e6..380acce 100644
--- a/src/timesfm3/__init__.py
+++ b/src/timesfm3/__init__.py
@@ -14,22 +14,22 @@
"""TimesFM3 PyTorch API."""
-from .configs import ResidualBlockConfig
-from .configs import StackedTransformersConfig
-from .configs import TransformerConfig
+from .configs import ResidualBlockConfig, StackedTransformersConfig, TransformerConfig
from .model import TimesFM3Torch
-from .timesfm3_forecaster import ForecastOutput
-from .timesfm3_forecaster import ModelConfig
-from .timesfm3_forecaster import TimesFM3Forecaster
-from .timesfm3_forecaster import _ModelConfig
+from .timesfm3_forecaster import (
+ ForecastOutput,
+ ModelConfig,
+ TimesFM3Forecaster,
+ _ModelConfig,
+)
__all__ = [
- "TimesFM3Torch",
- "TimesFM3Forecaster",
- "ForecastOutput",
- "_ModelConfig",
- "ModelConfig",
- "ResidualBlockConfig",
- "TransformerConfig",
- "StackedTransformersConfig",
+ "ForecastOutput",
+ "ModelConfig",
+ "ResidualBlockConfig",
+ "StackedTransformersConfig",
+ "TimesFM3Forecaster",
+ "TimesFM3Torch",
+ "TransformerConfig",
+ "_ModelConfig",
]
diff --git a/src/timesfm3/cpm_revin_refine.py b/src/timesfm3/cpm_revin_refine.py
index 753225f..30603ff 100644
--- a/src/timesfm3/cpm_revin_refine.py
+++ b/src/timesfm3/cpm_revin_refine.py
@@ -11,16 +11,16 @@ from . import util
def cpm_iterative_revin_refine(
- raw_logits: torch.Tensor,
- revin_n: torch.Tensor,
- revin_mu: torch.Tensor,
- revin_sigma: torch.Tensor,
- patch_cpm_mask: torch.Tensor,
- median_q_idx: int,
- rolls: int,
- patch_len: int,
- num_quantiles: int,
- value_clip: float = 1e9,
+ raw_logits: torch.Tensor,
+ revin_n: torch.Tensor,
+ revin_mu: torch.Tensor,
+ revin_sigma: torch.Tensor,
+ patch_cpm_mask: torch.Tensor,
+ median_q_idx: int,
+ rolls: int,
+ patch_len: int,
+ num_quantiles: int,
+ value_clip: float = 1e9,
) -> tuple[torch.Tensor, torch.Tensor]:
"""Refines RevIN stats at CPM-masked patches via iterative estimation.
@@ -57,16 +57,16 @@ def cpm_iterative_revin_refine(
# Reshape and slice raw_logits to keep only the median quantile.
# (b, v, n, oq) -> (b, v, n, rolls, patch_len, num_quantiles)
# -> (b, v, n, rolls, patch_len)
- median_logits = raw_logits.reshape(
- b, v, n_patches, rolls, patch_len, num_quantiles
- )[:, :, :, :, :, median_q_idx]
+ median_logits = raw_logits.reshape(b, v, n_patches, rolls, patch_len, num_quantiles)[
+ :, :, :, :, :, median_q_idx
+ ]
# Initialise carry with zeros.
carry_n = torch.zeros((b, v), dtype=torch.float32, device=device)
carry_mu = torch.zeros((b, v), dtype=torch.float32, device=device)
carry_sigma = torch.zeros((b, v), dtype=torch.float32, device=device)
anchor_predicted_values = torch.zeros(
- (b, v, rolls, patch_len), dtype=torch.float32, device=device
+ (b, v, rolls, patch_len), dtype=torch.float32, device=device
)
block_offset = torch.zeros((b,), dtype=torch.long, device=device)
@@ -84,16 +84,16 @@ def cpm_iterative_revin_refine(
# Select the block_offset[b]-th patch for each batch element
offset_onehot = torch.eq(
- torch.arange(rolls, device=device).unsqueeze(0),
- block_offset.unsqueeze(1),
+ torch.arange(rolls, device=device).unsqueeze(0),
+ block_offset.unsqueeze(1),
).float()
predicted_values_step = torch.einsum(
- "br,bvrp->bvp", offset_onehot, anchor_predicted_values
+ "br,bvrp->bvp", offset_onehot, anchor_predicted_values
)
# Update running stats with the estimated patch.
new_n, new_mu, new_sigma = util.update_running_stats(
- carry_n, carry_mu, carry_sigma, predicted_values_step, step_masks
+ carry_n, carry_mu, carry_sigma, predicted_values_step, step_masks
)
out_n = torch.where(is_cpm, new_n, actual_n)
@@ -102,25 +102,23 @@ def cpm_iterative_revin_refine(
# Advance block_offset: +1 (mod rolls) for CPM, reset to 0 for non-CPM.
new_block_offset = torch.where(
- is_cpm.squeeze(-1),
- (block_offset + 1) % rolls,
- torch.zeros_like(block_offset),
+ is_cpm.squeeze(-1),
+ (block_offset + 1) % rolls,
+ torch.zeros_like(block_offset),
)
should_update_anchor = torch.eq(new_block_offset, 0)
# Pre-calculate predicted values for the new anchor.
step_predicted_values = util.revin(
- current_step_logits, out_mu, out_sigma, reverse=True
- )
- step_predicted_values = torch.clamp(
- step_predicted_values, -value_clip, value_clip
+ current_step_logits, out_mu, out_sigma, reverse=True
)
+ step_predicted_values = torch.clamp(step_predicted_values, -value_clip, value_clip)
new_anchor_predicted_values = torch.where(
- should_update_anchor.view(b, 1, 1, 1),
- step_predicted_values,
- anchor_predicted_values,
+ should_update_anchor.view(b, 1, 1, 1),
+ step_predicted_values,
+ anchor_predicted_values,
)
carry_n = out_n
diff --git a/src/timesfm3/cpm_revin_refine_test.py b/src/timesfm3/cpm_revin_refine_test.py
index 4b68719..c8be5aa 100644
--- a/src/timesfm3/cpm_revin_refine_test.py
+++ b/src/timesfm3/cpm_revin_refine_test.py
@@ -15,6 +15,7 @@
"""Tests for PyTorch cpm_iterative_revin_refine."""
import unittest
+
import numpy as np
import torch
@@ -22,7 +23,6 @@ from . import cpm_revin_refine as torch_lib
class CpmRevinRefineTest(unittest.TestCase):
-
def test_shape(self):
b, v, n = 2, 3, 8
rolls, patch_len, num_q = 2, 2, 3
@@ -34,15 +34,15 @@ class CpmRevinRefineTest(unittest.TestCase):
patch_cpm_mask = torch.zeros((b, n), dtype=torch.bool)
refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(
- raw_logits=raw_logits,
- revin_n=revin_n,
- revin_mu=revin_mu,
- revin_sigma=revin_sigma,
- patch_cpm_mask=patch_cpm_mask,
- median_q_idx=num_q // 2,
- rolls=rolls,
- patch_len=patch_len,
- num_quantiles=num_q,
+ raw_logits=raw_logits,
+ revin_n=revin_n,
+ revin_mu=revin_mu,
+ revin_sigma=revin_sigma,
+ patch_cpm_mask=patch_cpm_mask,
+ median_q_idx=num_q // 2,
+ rolls=rolls,
+ patch_len=patch_len,
+ num_quantiles=num_q,
)
self.assertEqual(refined_mu.shape, (b, v, n))
self.assertEqual(refined_sigma.shape, (b, v, n))
@@ -59,24 +59,24 @@ class CpmRevinRefineTest(unittest.TestCase):
patch_cpm_mask = torch.zeros((b, n), dtype=torch.bool)
refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(
- raw_logits=raw_logits,
- revin_n=revin_n,
- revin_mu=revin_mu,
- revin_sigma=revin_sigma,
- patch_cpm_mask=patch_cpm_mask,
- median_q_idx=num_q // 2,
- rolls=rolls,
- patch_len=patch_len,
- num_quantiles=num_q,
+ raw_logits=raw_logits,
+ revin_n=revin_n,
+ revin_mu=revin_mu,
+ revin_sigma=revin_sigma,
+ patch_cpm_mask=patch_cpm_mask,
+ median_q_idx=num_q // 2,
+ rolls=rolls,
+ patch_len=patch_len,
+ num_quantiles=num_q,
)
np.testing.assert_allclose(refined_mu.numpy(), revin_mu.numpy())
np.testing.assert_allclose(refined_sigma.numpy(), revin_sigma.numpy())
def test_cpm_mask_modifies_cpm_positions_only(self):
for b, v, n, rolls, patch_len, num_q in [
- (1, 1, 8, 2, 4, 3),
- (2, 3, 16, 2, 32, 9),
- (2, 2, 12, 4, 8, 5),
+ (1, 1, 8, 2, 4, 3),
+ (2, 3, 16, 2, 32, 9),
+ (2, 2, 12, 4, 8, 5),
]:
torch.manual_seed(42)
oq = rolls * patch_len * num_q
@@ -93,23 +93,23 @@ class CpmRevinRefineTest(unittest.TestCase):
median_q_idx = num_q // 2
refined_mu, refined_sigma = torch_lib.cpm_iterative_revin_refine(
- raw_logits=raw_logits,
- revin_n=revin_n,
- revin_mu=revin_mu,
- revin_sigma=revin_sigma,
- patch_cpm_mask=patch_cpm_mask,
- median_q_idx=median_q_idx,
- rolls=rolls,
- patch_len=patch_len,
- num_quantiles=num_q,
+ raw_logits=raw_logits,
+ revin_n=revin_n,
+ revin_mu=revin_mu,
+ revin_sigma=revin_sigma,
+ patch_cpm_mask=patch_cpm_mask,
+ median_q_idx=median_q_idx,
+ rolls=rolls,
+ patch_len=patch_len,
+ num_quantiles=num_q,
)
# Non-CPM positions should be identical to inputs
np.testing.assert_allclose(
- refined_mu[:, :, :4].numpy(), revin_mu[:, :, :4].numpy()
+ refined_mu[:, :, :4].numpy(), revin_mu[:, :, :4].numpy()
)
np.testing.assert_allclose(
- refined_sigma[:, :, :4].numpy(), revin_sigma[:, :, :4].numpy()
+ refined_sigma[:, :, :4].numpy(), revin_sigma[:, :, :4].numpy()
)
# Refined outputs should be finite
diff --git a/src/timesfm3/dense.py b/src/timesfm3/dense.py
index f8b1c9d..f4c9933 100644
--- a/src/timesfm3/dense.py
+++ b/src/timesfm3/dense.py
@@ -5,8 +5,7 @@ from __future__ import annotations
import torch
from torch import nn
-from . import configs
-from . import util
+from . import configs, util
class ResidualBlock(nn.Module):
@@ -30,23 +29,23 @@ class ResidualBlock(nn.Module):
# the first forward pass (using `set_input_dims()`) before the matrix
# multiplication is evaluated, avoiding shape mismatch errors.
self.hidden_layer = nn.Linear(
- in_features=config.hidden_dims, # placeholder, set in first forward
- out_features=config.hidden_dims,
- bias=config.use_bias,
+ in_features=config.hidden_dims, # placeholder, set in first forward
+ out_features=config.hidden_dims,
+ bias=config.use_bias,
)
self.output_layer = nn.Linear(
- in_features=config.hidden_dims,
- out_features=config.output_dims,
- bias=config.use_bias,
+ in_features=config.hidden_dims,
+ out_features=config.output_dims,
+ bias=config.use_bias,
)
if config.identity_skip:
self.residual_layer = None
else:
self.residual_layer = nn.Linear(
- in_features=config.hidden_dims, # placeholder
- out_features=config.output_dims,
- bias=config.use_bias,
+ in_features=config.hidden_dims, # placeholder
+ out_features=config.output_dims,
+ bias=config.use_bias,
)
self.activation = util.get_activation_fn(config.activation)
@@ -67,16 +66,16 @@ class ResidualBlock(nn.Module):
dtype = self.hidden_layer.weight.dtype
self.hidden_layer = nn.Linear(
- input_dim, self.config.hidden_dims, bias=self.config.use_bias
+ input_dim, self.config.hidden_dims, bias=self.config.use_bias
).to(device=device, dtype=dtype)
self.output_layer = nn.Linear(
- self.config.hidden_dims,
- self.config.output_dims,
- bias=self.config.use_bias,
+ self.config.hidden_dims,
+ self.config.output_dims,
+ bias=self.config.use_bias,
).to(device=device, dtype=dtype)
if self.residual_layer is not None:
self.residual_layer = nn.Linear(
- input_dim, self.config.output_dims, bias=self.config.use_bias
+ input_dim, self.config.output_dims, bias=self.config.use_bias
).to(device=device, dtype=dtype)
if self.pre_norm is not None:
self.pre_norm = nn.RMSNorm(input_dim).to(device=device, dtype=dtype)
diff --git a/src/timesfm3/model.py b/src/timesfm3/model.py
index 102a2fe..73638c6 100644
--- a/src/timesfm3/model.py
+++ b/src/timesfm3/model.py
@@ -8,21 +8,24 @@ Supports:
from __future__ import annotations
+import dataclasses
import math
from typing import Any
import torch
+from huggingface_hub import PyTorchModelHubMixin
from torch import nn
-from . import configs
+from . import configs, dense, transformer, util
from . import cpm_revin_refine as cpm_revin_refine_lib
-from . import dense
-from . import transformations
-from . import transformer
-from . import util
-class TimesFM3Torch(nn.Module):
+class TimesFM3Torch(
+ nn.Module,
+ PyTorchModelHubMixin,
+ library_name="timesfm",
+ repo_url="https://github.com/google-research/timesfm",
+):
"""PyTorch inference-only implementation of TimesFM3.
Attributes:
@@ -41,60 +44,70 @@ class TimesFM3Torch(nn.Module):
"""
def __init__(
- self,
- input_patch_len: int = 32,
- output_patch_len: int = 64,
- quantiles: list[float] | None = None,
- residual_block_config: configs.ResidualBlockConfig | None = None,
- transformer_config: configs.StackedTransformersConfig | None = None,
- use_variate_attention: bool = True,
- value_clip: float = 1e20,
- use_stitching: bool = True,
- use_linear_detrending: bool = True,
- linear_detrending_threshold: float = 0.5,
- use_iterative_cpm_revin: bool = True,
- use_frozen_running_stats: bool = False,
- input_transform: str = "identity",
+ self,
+ input_patch_len: int = 32,
+ output_patch_len: int = 64,
+ quantiles: list[float] | None = None,
+ residual_block_config: configs.ResidualBlockConfig | dict[str, Any] | None = None,
+ transformer_config: configs.StackedTransformersConfig
+ | dict[str, Any]
+ | None = None,
+ use_variate_attention: bool = True,
+ value_clip: float = 1e20,
+ use_stitching: bool = True,
+ use_linear_detrending: bool = True,
+ linear_detrending_threshold: float = 0.5,
+ use_iterative_cpm_revin: bool = True,
+ use_frozen_running_stats: bool = False,
+ input_transform: str = "identity",
):
super().__init__()
if quantiles is None:
quantiles = [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
if residual_block_config is None:
residual_block_config = configs.ResidualBlockConfig(
- hidden_dims=1280,
- output_dims=1280,
- use_bias=False,
- activation="relu",
+ hidden_dims=1280,
+ output_dims=1280,
+ use_bias=False,
+ activation="relu",
)
+ elif isinstance(residual_block_config, dict):
+ residual_block_config = configs.ResidualBlockConfig(**residual_block_config)
+
if transformer_config is None:
transformer_config = configs.StackedTransformersConfig(
- num_layers=20,
- transformer=configs.TransformerConfig(
- model_dims=1280,
- hidden_dims=1280,
- num_heads=16,
- attention_norm="rms",
- feedforward_norm="rms",
- qk_norm="rms",
- use_rope_seq=True,
- use_rope_var=False,
- use_bias=False,
- ff_activation="relu",
- deterministic=True,
- ),
+ num_layers=20,
+ transformer=configs.TransformerConfig(
+ model_dims=1280,
+ hidden_dims=1280,
+ num_heads=16,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=False,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ ),
)
+ elif isinstance(transformer_config, dict):
+ t_data = transformer_config.get("transformer", {})
+ if isinstance(t_data, dict):
+ t_cfg = configs.TransformerConfig(**t_data)
+ else:
+ t_cfg = t_data
+ tc_dict = dict(transformer_config)
+ tc_dict["transformer"] = t_cfg
+ transformer_config = configs.StackedTransformersConfig(**tc_dict)
+
if output_patch_len % input_patch_len != 0:
raise ValueError(
- f"Output patch len {output_patch_len} must be a multiple of"
- f" input patch len {input_patch_len}."
- )
- if (
- residual_block_config.output_dims
- != transformer_config.transformer.model_dims
- ):
- raise ValueError(
- "ResidualBlock output_dims must match Transformer model_dims."
+ f"Output patch len {output_patch_len} must be a multiple of"
+ f" input patch len {input_patch_len}."
)
+ if residual_block_config.output_dims != transformer_config.transformer.model_dims:
+ raise ValueError("ResidualBlock output_dims must match Transformer model_dims.")
self.input_patch_len = input_patch_len
self.output_patch_len = output_patch_len
@@ -114,40 +127,67 @@ class TimesFM3Torch(nn.Module):
if self.use_stitching:
if self.output_patch_len <= self.input_patch_len:
- raise ValueError(
- "use_stitching requires output_patch_len > input_patch_len"
- )
- self._stitching_extract_len = min(
- 2 * self.input_patch_len, self.output_patch_len
- )
+ raise ValueError("use_stitching requires output_patch_len > input_patch_len")
+ self._stitching_extract_len = min(2 * self.input_patch_len, self.output_patch_len)
- self.pre_transformer_resblock = dense.ResidualBlock(
- config=residual_block_config
+ self.pre_transformer_resblock = dense.ResidualBlock(config=residual_block_config)
+ self.pre_transformer_resblock.set_input_dims(
+ 2 * (input_patch_len + output_patch_len)
)
self.transformer_stack = transformer.StackedMixingTransformer(
- config=transformer_config,
- use_variate_attention=use_variate_attention,
+ config=transformer_config,
+ use_variate_attention=use_variate_attention,
)
self.output_head = nn.Linear(
- transformer_config.transformer.model_dims,
- output_patch_len * self.num_quantiles,
- bias=True,
+ transformer_config.transformer.model_dims,
+ output_patch_len * self.num_quantiles,
+ bias=True,
)
+ def to_dict(self) -> dict[str, Any]:
+ """Returns a serializable dictionary of model configuration."""
+ return {
+ "input_patch_len": self.input_patch_len,
+ "output_patch_len": self.output_patch_len,
+ "quantiles": list(self.quantiles),
+ "residual_block_config": dataclasses.asdict(self.residual_block_config),
+ "transformer_config": dataclasses.asdict(self.transformer_config),
+ "use_variate_attention": self.use_variate_attention,
+ "value_clip": self.value_clip,
+ "use_stitching": self.use_stitching,
+ "use_linear_detrending": self.use_linear_detrending,
+ "linear_detrending_threshold": self.linear_detrending_threshold,
+ "use_iterative_cpm_revin": self.use_iterative_cpm_revin,
+ "use_frozen_running_stats": self.use_frozen_running_stats,
+ "input_transform": self.input_transform,
+ }
+
+ def save_pretrained(
+ self,
+ save_directory: str | Any,
+ *,
+ config: dict[str, Any] | None = None,
+ **kwargs: Any,
+ ) -> str | None:
+ """Saves model weights and config to a local directory or pushes to Hub."""
+ if config is None:
+ config = self.to_dict()
+ return super().save_pretrained(save_directory, config=config, **kwargs)
+
def _preprocess(
- self,
- values: torch.Tensor,
- masks: torch.Tensor,
- patch_is_target: torch.Tensor,
- freeze_after: int | None = None,
- patch_cpm_mask: torch.Tensor | None = None,
+ self,
+ values: torch.Tensor,
+ masks: torch.Tensor,
+ patch_is_target: torch.Tensor,
+ freeze_after: int | None = None,
+ patch_cpm_mask: torch.Tensor | None = None,
) -> tuple[
- torch.Tensor,
- torch.Tensor,
- torch.Tensor,
- tuple[torch.Tensor, torch.Tensor],
- torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ torch.Tensor,
+ tuple[torch.Tensor, torch.Tensor],
+ torch.Tensor,
]:
"""Applies preprocessing: RevIN, masking, future covariates, ResBlock.
@@ -163,17 +203,15 @@ class TimesFM3Torch(nn.Module):
(resblock_input, resblock_output, patch_mask, (running_mean, running_std),
running_n)
"""
- running_n, running_mean, running_std = util.get_running_stats(
- values, masks
- )
+ running_n, running_mean, running_std = util.get_running_stats(values, masks)
if freeze_after is not None:
_, _, n, _ = values.shape
if 0 <= freeze_after < n - 1:
running_mean[:, :, freeze_after + 1 :] = running_mean[
- :, :, freeze_after : freeze_after + 1
+ :, :, freeze_after : freeze_after + 1
]
running_std[:, :, freeze_after + 1 :] = running_std[
- :, :, freeze_after : freeze_after + 1
+ :, :, freeze_after : freeze_after + 1
]
# Apply CPM mask: mask target variates at CPM positions.
@@ -187,9 +225,7 @@ class TimesFM3Torch(nn.Module):
# Roll values to get future covariate patches
values_fcov, wrap_mask = util.get_output_patch_via_roll(values, self.rolls)
- values_fcov = util.revin(
- values_fcov, running_mean, running_std, reverse=False
- )
+ values_fcov = util.revin(values_fcov, running_mean, running_std, reverse=False)
# Roll the (CPM-modified) masks for future covariate masking.
masks_fcov_raw, _ = util.get_output_patch_via_roll(masks, self.rolls)
@@ -206,19 +242,19 @@ class TimesFM3Torch(nn.Module):
patch_mask_bvn = masks_cat.all(dim=3)
return (
- resblock_input,
- resblock_output,
- patch_mask_bvn,
- (running_mean, running_std),
- running_n,
+ resblock_input,
+ resblock_output,
+ patch_mask_bvn,
+ (running_mean, running_std),
+ running_n,
)
def forward(
- self,
- inputs: dict[str, Any],
- freeze_after: int | None = None,
- patch_cpm_mask: torch.Tensor | None = None,
- return_aux_outputs: bool = False,
+ self,
+ inputs: dict[str, Any],
+ freeze_after: int | None = None,
+ patch_cpm_mask: torch.Tensor | None = None,
+ return_aux_outputs: bool = False,
) -> dict[str, Any]:
"""Full-sequence forward pass (equivalent to Flax __call__).
@@ -242,22 +278,22 @@ class TimesFM3Torch(nn.Module):
_, _, _, p = values.shape
if p != self.input_patch_len:
raise ValueError(
- f"Input patch_len {p} != model input_patch_len {self.input_patch_len}"
+ f"Input patch_len {p} != model input_patch_len {self.input_patch_len}"
)
# Preprocessing & ResBlock
(
- resblock_input,
- transformer_input,
- transformer_patch_mask,
- revin_stats,
- running_n,
+ resblock_input,
+ transformer_input,
+ transformer_patch_mask,
+ revin_stats,
+ running_n,
) = self._preprocess(
- values,
- masks,
- patch_is_target,
- freeze_after=freeze_after,
- patch_cpm_mask=patch_cpm_mask,
+ values,
+ masks,
+ patch_is_target,
+ freeze_after=freeze_after,
+ patch_cpm_mask=patch_cpm_mask,
)
# Transformer
@@ -265,12 +301,10 @@ class TimesFM3Torch(nn.Module):
# Flax __call__ does: effective_patch_mask = cumprod(mask, axis=2) when
# not training. This keeps horizon patches (which are fully masked but
# come after valid context) visible to attention.
- effective_patch_mask = torch.cumprod(
- transformer_patch_mask.int(), dim=2
- ).bool()
+ effective_patch_mask = torch.cumprod(transformer_patch_mask.int(), dim=2).bool()
transformer_output, _, seq_attn_mask = self.transformer_stack(
- transformer_input,
- effective_patch_mask,
+ transformer_input,
+ effective_patch_mask,
)
# Output head
@@ -278,33 +312,29 @@ class TimesFM3Torch(nn.Module):
revin_mean, revin_std = revin_stats
if self.use_iterative_cpm_revin and patch_cpm_mask is not None:
- refined_mu, refined_sigma = (
- cpm_revin_refine_lib.cpm_iterative_revin_refine(
- raw_logits,
- revin_n=running_n,
- revin_mu=revin_mean,
- revin_sigma=revin_std,
- patch_cpm_mask=patch_cpm_mask,
- median_q_idx=self.num_quantiles // 2,
- rolls=self.rolls,
- patch_len=self.input_patch_len,
- num_quantiles=self.num_quantiles,
- value_clip=self.value_clip,
- )
+ refined_mu, refined_sigma = cpm_revin_refine_lib.cpm_iterative_revin_refine(
+ raw_logits,
+ revin_n=running_n,
+ revin_mu=revin_mean,
+ revin_sigma=revin_std,
+ patch_cpm_mask=patch_cpm_mask,
+ median_q_idx=self.num_quantiles // 2,
+ rolls=self.rolls,
+ patch_len=self.input_patch_len,
+ num_quantiles=self.num_quantiles,
+ value_clip=self.value_clip,
)
cpm_bvn = patch_cpm_mask.unsqueeze(1) # (b, 1, n)
revin_mean = torch.where(cpm_bvn, refined_mu, revin_mean)
revin_std = torch.where(cpm_bvn, refined_sigma, revin_std)
revin_logits = util.revin(raw_logits, revin_mean, revin_std, reverse=True)
- clipped_logits = torch.clamp(
- revin_logits, -self.value_clip, self.value_clip
- )
+ clipped_logits = torch.clamp(revin_logits, -self.value_clip, self.value_clip)
# Reshape: (b, v, n, o*q) -> (b, v, n, o, q)
b, v, n_patches = clipped_logits.shape[:3]
final_logits = clipped_logits.view(
- b, v, n_patches, self.output_patch_len, self.num_quantiles
+ b, v, n_patches, self.output_patch_len, self.num_quantiles
)
outputs = {"logits": final_logits, "revin_stats": revin_stats}
@@ -319,16 +349,16 @@ class TimesFM3Torch(nn.Module):
@torch.no_grad()
def decode(
- self,
- target: torch.Tensor,
- horizon: int = 0,
- past_only_covariates: torch.Tensor | None = None,
- past_future_covariates: torch.Tensor | None = None,
- target_mask: torch.Tensor | None = None,
- past_only_mask: torch.Tensor | None = None,
- past_future_mask: torch.Tensor | None = None,
- mask: torch.Tensor | None = None,
- return_aux_outputs: bool = False,
+ self,
+ target: torch.Tensor,
+ horizon: int = 0,
+ past_only_covariates: torch.Tensor | None = None,
+ past_future_covariates: torch.Tensor | None = None,
+ target_mask: torch.Tensor | None = None,
+ past_only_mask: torch.Tensor | None = None,
+ past_future_mask: torch.Tensor | None = None,
+ mask: torch.Tensor | None = None,
+ return_aux_outputs: bool = False,
) -> Any:
"""Non-autoregressive single-pass decoding for TimesFM3.
@@ -356,7 +386,7 @@ class TimesFM3Torch(nn.Module):
# 1. Pad context to multiple of input_patch_len
ctx_padding = (
- self.input_patch_len - (context % self.input_patch_len)
+ self.input_patch_len - (context % self.input_patch_len)
) % self.input_patch_len
if ctx_padding > 0:
target = torch.nn.functional.pad(target, (ctx_padding, 0))
@@ -364,23 +394,21 @@ class TimesFM3Torch(nn.Module):
mask = torch.nn.functional.pad(mask, (ctx_padding, 0), value=True)
if past_only_covariates is not None:
past_only_covariates = torch.nn.functional.pad(
- past_only_covariates, (ctx_padding, 0)
+ past_only_covariates, (ctx_padding, 0)
)
if past_future_covariates is not None:
past_future_covariates = torch.nn.functional.pad(
- past_future_covariates, (ctx_padding, 0)
+ past_future_covariates, (ctx_padding, 0)
)
if target_mask is not None:
- target_mask = torch.nn.functional.pad(
- target_mask, (ctx_padding, 0), value=True
- )
+ target_mask = torch.nn.functional.pad(target_mask, (ctx_padding, 0), value=True)
if past_only_mask is not None:
past_only_mask = torch.nn.functional.pad(
- past_only_mask, (ctx_padding, 0), value=True
+ past_only_mask, (ctx_padding, 0), value=True
)
if past_future_mask is not None:
past_future_mask = torch.nn.functional.pad(
- past_future_mask, (ctx_padding, 0), value=True
+ past_future_mask, (ctx_padding, 0), value=True
)
context = context + ctx_padding
@@ -394,7 +422,7 @@ class TimesFM3Torch(nn.Module):
extract_len = self._stitching_extract_len
overlap = extract_len - self.input_patch_len
num_forecast_patches = max(
- math.ceil((horizon - overlap) / self.input_patch_len), 1
+ math.ceil((horizon - overlap) / self.input_patch_len), 1
)
num_horizon_patches = num_forecast_patches + self.rolls - 1
padded_horizon = num_horizon_patches * self.input_patch_len
@@ -416,16 +444,12 @@ class TimesFM3Torch(nn.Module):
if past_only_covariates is not None:
num_past_only = past_only_covariates.shape[1]
if past_only_mask is None:
- past_only_mask = torch.zeros_like(
- past_only_covariates, dtype=torch.bool
- )
+ past_only_mask = torch.zeros_like(past_only_covariates, dtype=torch.bool)
all_ctx_vals.append(past_only_covariates)
all_ctx_masks.append(past_only_mask | mask.unsqueeze(1))
if past_future_covariates is not None:
if past_future_mask is None:
- past_future_mask = torch.zeros_like(
- past_future_covariates, dtype=torch.bool
- )
+ past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)
all_ctx_vals.append(past_future_covariates[..., :context])
all_ctx_masks.append(past_future_mask[..., :context] | mask.unsqueeze(1))
@@ -433,57 +457,41 @@ class TimesFM3Torch(nn.Module):
ctx_masks = torch.cat(all_ctx_masks, dim=1)
if self.use_linear_detrending:
- t_ctx = torch.arange(
- -(context - 1), 1, dtype=torch.float32, device=device
- )
+ t_ctx = torch.arange(-(context - 1), 1, dtype=torch.float32, device=device)
t_ctx_bvc = t_ctx[None, None, :]
t_ctx_bvc_normalized = t_ctx_bvc / context
valid = ~ctx_masks
n_v = valid.float().sum(dim=-1, keepdim=True)
- sum_t = torch.where(valid, t_ctx_bvc_normalized, 0.0).sum(
- dim=-1, keepdim=True
- )
+ sum_t = torch.where(valid, t_ctx_bvc_normalized, 0.0).sum(dim=-1, keepdim=True)
sum_t2 = torch.where(valid, t_ctx_bvc_normalized**2, 0.0).sum(
- dim=-1, keepdim=True
+ dim=-1, keepdim=True
)
sum_y = torch.where(valid, ctx_vals, 0.0).sum(dim=-1, keepdim=True)
sum_ty = torch.where(valid, t_ctx_bvc_normalized * ctx_vals, 0.0).sum(
- dim=-1, keepdim=True
+ dim=-1, keepdim=True
)
det = n_v * sum_t2 - sum_t**2
safe_det = torch.where(det == 0.0, 1.0, det)
- m_trend = torch.where(
- det == 0.0, 0.0, (n_v * sum_ty - sum_t * sum_y) / safe_det
- )
+ m_trend = torch.where(det == 0.0, 0.0, (n_v * sum_ty - sum_t * sum_y) / safe_det)
c_trend = torch.where(
- det == 0.0,
- torch.where(n_v > 0, sum_y / torch.clamp_min(n_v, 1.0), 0.0),
- (sum_y - m_trend * sum_t) / torch.clamp_min(n_v, 1.0),
+ det == 0.0,
+ torch.where(n_v > 0, sum_y / torch.clamp_min(n_v, 1.0), 0.0),
+ (sum_y - m_trend * sum_t) / torch.clamp_min(n_v, 1.0),
)
- ctx_vals_detrended = ctx_vals - (
- m_trend * t_ctx_bvc_normalized + c_trend
- )
+ ctx_vals_detrended = ctx_vals - (m_trend * t_ctx_bvc_normalized + c_trend)
mean_y = sum_y / torch.clamp_min(n_v, 1.0)
sum_y2 = torch.where(valid, ctx_vals**2, 0.0).sum(dim=-1, keepdim=True)
- var_orig = torch.clamp_min(
- sum_y2 / torch.clamp_min(n_v, 1.0) - mean_y**2, 0.0
- )
+ var_orig = torch.clamp_min(sum_y2 / torch.clamp_min(n_v, 1.0) - mean_y**2, 0.0)
std_orig = torch.sqrt(var_orig)
- sum_yd = torch.where(valid, ctx_vals_detrended, 0.0).sum(
- dim=-1, keepdim=True
- )
+ sum_yd = torch.where(valid, ctx_vals_detrended, 0.0).sum(dim=-1, keepdim=True)
mean_yd = sum_yd / torch.clamp_min(n_v, 1.0)
- sum_yd2 = torch.where(valid, ctx_vals_detrended**2, 0.0).sum(
- dim=-1, keepdim=True
- )
- var_det = torch.clamp_min(
- sum_yd2 / torch.clamp_min(n_v, 1.0) - mean_yd**2, 0.0
- )
+ sum_yd2 = torch.where(valid, ctx_vals_detrended**2, 0.0).sum(dim=-1, keepdim=True)
+ var_det = torch.clamp_min(sum_yd2 / torch.clamp_min(n_v, 1.0) - mean_yd**2, 0.0)
std_det = torch.sqrt(var_det)
apply_detrend = std_det < self.linear_detrending_threshold * std_orig
@@ -491,64 +499,60 @@ class TimesFM3Torch(nn.Module):
else:
num_variates = ctx_vals.shape[1]
m_trend = torch.zeros(
- (batch_size, num_variates, 1), dtype=torch.float32, device=device
+ (batch_size, num_variates, 1), dtype=torch.float32, device=device
)
c_trend = torch.zeros(
- (batch_size, num_variates, 1), dtype=torch.float32, device=device
+ (batch_size, num_variates, 1), dtype=torch.float32, device=device
)
apply_detrend = torch.zeros(
- (batch_size, num_variates, 1), dtype=torch.bool, device=device
+ (batch_size, num_variates, 1), dtype=torch.bool, device=device
)
ctx_vals = torch.where(ctx_masks, 0.0, ctx_vals)
all_hor_vals = [
- torch.zeros(batch_size, num_target, padded_horizon, device=device),
- torch.zeros(batch_size, num_past_only, padded_horizon, device=device),
+ torch.zeros(batch_size, num_target, padded_horizon, device=device),
+ torch.zeros(batch_size, num_past_only, padded_horizon, device=device),
]
all_hor_masks = [
- torch.ones(
- batch_size,
- num_target,
- padded_horizon,
- dtype=torch.bool,
- device=device,
- ),
- torch.ones(
- batch_size,
- num_past_only,
- padded_horizon,
- dtype=torch.bool,
- device=device,
- ),
+ torch.ones(
+ batch_size,
+ num_target,
+ padded_horizon,
+ dtype=torch.bool,
+ device=device,
+ ),
+ torch.ones(
+ batch_size,
+ num_past_only,
+ padded_horizon,
+ dtype=torch.bool,
+ device=device,
+ ),
]
if past_future_covariates is not None:
if past_future_mask is None:
- past_future_mask = torch.zeros_like(
- past_future_covariates, dtype=torch.bool
- )
+ past_future_mask = torch.zeros_like(past_future_covariates, dtype=torch.bool)
pf_future_vals = past_future_covariates[..., context : context + horizon]
pf_future_masks = past_future_mask[..., context : context + horizon]
if self.use_linear_detrending:
m_pf = m_trend[:, num_target + num_past_only :, :]
c_pf = c_trend[:, num_target + num_past_only :, :]
apply_detrend_pf = apply_detrend[:, num_target + num_past_only :, :]
- t_hor_pf = torch.arange(
- 1, horizon + 1, dtype=torch.float32, device=device
- )[None, None, :]
+ t_hor_pf = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)[
+ None, None, :
+ ]
t_hor_pf_normalized = t_hor_pf / context
pf_trend_hor = m_pf * t_hor_pf_normalized + c_pf
pf_future_vals = torch.where(
- apply_detrend_pf, pf_future_vals - pf_trend_hor, pf_future_vals
+ apply_detrend_pf, pf_future_vals - pf_trend_hor, pf_future_vals
)
pf_future_vals = torch.where(pf_future_masks, 0.0, pf_future_vals)
if hor_padding > 0:
- pf_future_vals = torch.nn.functional.pad(
- pf_future_vals, (0, hor_padding)
- )
+ pf_future_vals = torch.nn.functional.pad(pf_future_vals, (0, hor_padding))
pf_future_masks = torch.nn.functional.pad(
- pf_future_masks, (0, hor_padding), value=True
+ pf_future_masks, (0, hor_padding), value=True
)
all_hor_vals.append(pf_future_vals)
all_hor_masks.append(pf_future_masks)
@@ -561,41 +565,35 @@ class TimesFM3Torch(nn.Module):
num_variates = all_vals.shape[1]
patch_is_target = torch.zeros(
- (batch_size, num_variates, num_context_patches + num_horizon_patches),
- dtype=torch.bool,
- device=device,
+ (batch_size, num_variates, num_context_patches + num_horizon_patches),
+ dtype=torch.bool,
+ device=device,
)
patch_is_target[:, : num_target + num_past_only, :] = True
# Reshape values & masks to patched shape (b, v, n, p)
- values_bvnp = all_vals.reshape(
- batch_size, num_variates, -1, self.input_patch_len
- )
- masks_bvnp = all_masks.reshape(
- batch_size, num_variates, -1, self.input_patch_len
- )
+ values_bvnp = all_vals.reshape(batch_size, num_variates, -1, self.input_patch_len)
+ masks_bvnp = all_masks.reshape(batch_size, num_variates, -1, self.input_patch_len)
inputs = {
- "values": values_bvnp,
- "masks": masks_bvnp,
- "patch_is_target": patch_is_target,
+ "values": values_bvnp,
+ "masks": masks_bvnp,
+ "patch_is_target": patch_is_target,
}
# Build horizon CPM mask: context=False, horizon=True.
num_total_patches = num_context_patches + num_horizon_patches
horizon_cpm_mask = torch.zeros(
- batch_size, num_total_patches, dtype=torch.bool, device=device
+ batch_size, num_total_patches, dtype=torch.bool, device=device
)
horizon_cpm_mask[:, num_context_patches:] = True
- freeze_after = (
- num_context_patches - 1 if self.use_frozen_running_stats else None
- )
+ freeze_after = num_context_patches - 1 if self.use_frozen_running_stats else None
forward_out = self.forward(
- inputs,
- freeze_after=freeze_after,
- patch_cpm_mask=horizon_cpm_mask,
- return_aux_outputs=return_aux_outputs,
+ inputs,
+ freeze_after=freeze_after,
+ patch_cpm_mask=horizon_cpm_mask,
+ return_aux_outputs=return_aux_outputs,
)
logits = forward_out["logits"] # (b, v, n, output_patch_len, num_quantiles)
@@ -603,38 +601,34 @@ class TimesFM3Torch(nn.Module):
extract_len = self._stitching_extract_len
overlap = extract_len - self.input_patch_len
num_forecast_patches = max(
- math.ceil((horizon - overlap) / self.input_patch_len), 1
+ math.ceil((horizon - overlap) / self.input_patch_len), 1
+ )
+ forecast_indices = torch.arange(num_forecast_patches, device=device) + (
+ num_context_patches - 1
)
- forecast_indices = torch.arange(
- num_forecast_patches, device=device
- ) + (num_context_patches - 1)
patch_preds = logits[:, :, forecast_indices, :extract_len, :]
horizon_logits = util.stitch_patches(
- patch_preds,
- self.input_patch_len,
+ patch_preds,
+ self.input_patch_len,
)[:, :, :horizon, :]
else:
num_forecast_chunks = padded_horizon // self.output_patch_len
forecast_indices = torch.arange(
- num_forecast_chunks, device=device
+ num_forecast_chunks, device=device
) * self.rolls + (num_context_patches - 1)
forecast_logits = logits[:, :, forecast_indices, :, :]
horizon_logits = forecast_logits.reshape(
- batch_size, num_variates, -1, self.num_quantiles
+ batch_size, num_variates, -1, self.num_quantiles
)[:, :, :horizon, :]
if self.use_linear_detrending:
- t_forecast = torch.arange(
- 1, horizon + 1, dtype=torch.float32, device=device
- )
+ t_forecast = torch.arange(1, horizon + 1, dtype=torch.float32, device=device)
t_forecast_normalized = t_forecast / context
trend_forecast = (
- m_trend[:, :, 0, None] * t_forecast_normalized[None, None, :]
- + c_trend[:, :, 0, None]
- )
- trend_forecast = torch.where(
- apply_detrend[:, :, 0, None], trend_forecast, 0.0
+ m_trend[:, :, 0, None] * t_forecast_normalized[None, None, :]
+ + c_trend[:, :, 0, None]
)
+ trend_forecast = torch.where(apply_detrend[:, :, 0, None], trend_forecast, 0.0)
horizon_logits = horizon_logits + trend_forecast[:, :, :, None]
if return_aux_outputs:
diff --git a/src/timesfm3/model_test.py b/src/timesfm3/model_test.py
index ffefd46..e36d893 100644
--- a/src/timesfm3/model_test.py
+++ b/src/timesfm3/model_test.py
@@ -15,7 +15,7 @@
"""Tests for PyTorch TimesFM3Torch model."""
import unittest
-import numpy as np
+
import torch
from . import configs
@@ -23,57 +23,56 @@ from . import model as torch_model_lib
class TimesFM3TorchTest(unittest.TestCase):
-
def setUp(self):
super().setUp()
self.resblock_config = configs.ResidualBlockConfig(
- hidden_dims=32,
- output_dims=32,
- use_bias=False,
- activation="relu",
- dropout=0.0,
+ hidden_dims=32,
+ output_dims=32,
+ use_bias=False,
+ activation="relu",
+ dropout=0.0,
)
self.transformer_config = configs.StackedTransformersConfig(
- num_layers=2,
- use_remat=False,
- transformer=configs.TransformerConfig(
- model_dims=32,
- hidden_dims=32,
- num_heads=4,
- attention_norm="rms",
- feedforward_norm="rms",
- qk_norm="rms",
- use_rope_seq=True,
- use_rope_var=True,
- use_bias=False,
- ff_activation="relu",
- deterministic=True,
- use_sdpa=True,
- ),
+ num_layers=2,
+ use_remat=False,
+ transformer=configs.TransformerConfig(
+ model_dims=32,
+ hidden_dims=32,
+ num_heads=4,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=True,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ use_sdpa=True,
+ ),
)
self.model = torch_model_lib.TimesFM3Torch(
- input_patch_len=8,
- output_patch_len=16,
- quantiles=[0.1, 0.5, 0.9],
- residual_block_config=self.resblock_config,
- transformer_config=self.transformer_config,
- use_stitching=True,
- use_linear_detrending=True,
- use_iterative_cpm_revin=True,
- use_frozen_running_stats=False,
+ input_patch_len=8,
+ output_patch_len=16,
+ quantiles=[0.1, 0.5, 0.9],
+ residual_block_config=self.resblock_config,
+ transformer_config=self.transformer_config,
+ use_stitching=True,
+ use_linear_detrending=True,
+ use_iterative_cpm_revin=True,
+ use_frozen_running_stats=False,
)
self.model.eval()
def test_forward_pass_training_dict(self):
b, v, n, p = 2, 3, 4, 8
inputs = {
- "values": torch.randn(b, v, n, p),
- "masks": torch.zeros(b, v, n, p, dtype=torch.bool),
- "patch_segment_ids": torch.zeros(b, n, dtype=torch.long),
- "patch_positions": torch.arange(n).unsqueeze(0).repeat(b, 1),
- "patch_is_target": torch.ones(b, v, n, dtype=torch.bool),
- "patch_is_past_only": torch.zeros(b, v, n, dtype=torch.bool),
- "patch_is_past_future_covariate": torch.zeros(b, v, n, dtype=torch.bool),
+ "values": torch.randn(b, v, n, p),
+ "masks": torch.zeros(b, v, n, p, dtype=torch.bool),
+ "patch_segment_ids": torch.zeros(b, n, dtype=torch.long),
+ "patch_positions": torch.arange(n).unsqueeze(0).repeat(b, 1),
+ "patch_is_target": torch.ones(b, v, n, dtype=torch.bool),
+ "patch_is_past_only": torch.zeros(b, v, n, dtype=torch.bool),
+ "patch_is_past_future_covariate": torch.zeros(b, v, n, dtype=torch.bool),
}
with torch.no_grad():
out = self.model(inputs)
@@ -101,16 +100,40 @@ class TimesFM3TorchTest(unittest.TestCase):
with torch.no_grad():
out = self.model.decode(
- target=target,
- horizon=horizon,
- past_only_covariates=po_cov,
- past_future_covariates=pf_cov,
- mask=mask,
+ target=target,
+ horizon=horizon,
+ past_only_covariates=po_cov,
+ past_future_covariates=pf_cov,
+ mask=mask,
)
# Total variates: 3 targets + 3 past_only + 3 past_future = 9
self.assertEqual(out.shape, (b, 9, horizon, 3))
self.assertTrue(torch.isfinite(out).all().item())
+ def test_save_and_from_pretrained(self):
+ import json
+ import os
+ import tempfile
+
+ with tempfile.TemporaryDirectory() as tmpdir:
+ self.model.save_pretrained(tmpdir)
+ config_path = os.path.join(tmpdir, "config.json")
+ self.assertTrue(os.path.exists(config_path))
+ with open(config_path) as f:
+ config_dict = json.load(f)
+ self.assertEqual(config_dict["input_patch_len"], 8)
+ self.assertEqual(config_dict["output_patch_len"], 16)
+ self.assertIn("transformer_config", config_dict)
+
+ loaded_model = torch_model_lib.TimesFM3Torch.from_pretrained(tmpdir)
+ loaded_model.eval()
+
+ target = torch.randn(2, 1, 32)
+ with torch.no_grad():
+ orig_out = self.model.decode(target=target, horizon=16)
+ loaded_out = loaded_model.decode(target=target, horizon=16)
+ self.assertTrue(torch.allclose(orig_out, loaded_out, atol=1e-5))
+
if __name__ == "__main__":
unittest.main()
diff --git a/src/timesfm3/normalization.py b/src/timesfm3/normalization.py
index 25fcf91..6f20c5a 100644
--- a/src/timesfm3/normalization.py
+++ b/src/timesfm3/normalization.py
@@ -28,8 +28,8 @@ class PerDimScale(nn.Module):
def forward(self, x: torch.Tensor) -> torch.Tensor:
"""Applies per-dim scaling to the last dimension of x."""
return (
- x
- * _RECIPROCAL_OF_SOFTPLUS_0
- / math.sqrt(self.num_dims)
- * torch.nn.functional.softplus(self.per_dim_scale)
+ x
+ * _RECIPROCAL_OF_SOFTPLUS_0
+ / math.sqrt(self.num_dims)
+ * torch.nn.functional.softplus(self.per_dim_scale)
)
diff --git a/src/timesfm3/primitives_test.py b/src/timesfm3/primitives_test.py
index 6f76493..c7fdc9d 100644
--- a/src/timesfm3/primitives_test.py
+++ b/src/timesfm3/primitives_test.py
@@ -16,6 +16,7 @@
import math
import unittest
+
import numpy as np
import torch
@@ -59,19 +60,19 @@ class TransformationsTest(unittest.TestCase):
def test_max_output(self):
value_clip = 10.0
self.assertAlmostEqual(
- torch_trans.max_output("signed_log", value_clip).item(),
- math.log1p(value_clip),
- places=5,
+ torch_trans.max_output("signed_log", value_clip).item(),
+ math.log1p(value_clip),
+ places=5,
)
self.assertAlmostEqual(
- torch_trans.max_output("signed_sqrt", value_clip).item(),
- math.sqrt(value_clip),
- places=5,
+ torch_trans.max_output("signed_sqrt", value_clip).item(),
+ math.sqrt(value_clip),
+ places=5,
)
self.assertAlmostEqual(
- torch_trans.max_output("identity", value_clip).item(),
- value_clip,
- places=5,
+ torch_trans.max_output("identity", value_clip).item(),
+ value_clip,
+ places=5,
)
@@ -89,7 +90,7 @@ class NormalizationTest(unittest.TestCase):
# at init it approximately scales by 1 / sqrt(num_dims).
expected_scale = 1.0 / math.sqrt(num_dims)
np.testing.assert_allclose(
- y.detach().numpy(), (x * expected_scale).numpy(), atol=1e-4
+ y.detach().numpy(), (x * expected_scale).numpy(), atol=1e-4
)
@@ -100,12 +101,12 @@ class ResidualBlockTest(unittest.TestCase):
input_dim = 8
hidden_dims = 16
config = configs.ResidualBlockConfig(
- hidden_dims=hidden_dims,
- output_dims=input_dim,
- use_bias=True,
- activation="relu",
- identity_skip=True,
- prenorm="none",
+ hidden_dims=hidden_dims,
+ output_dims=input_dim,
+ use_bias=True,
+ activation="relu",
+ identity_skip=True,
+ prenorm="none",
)
mod = torch_dense.ResidualBlock(config=config)
x = torch.randn(2, input_dim)
@@ -117,12 +118,12 @@ class ResidualBlockTest(unittest.TestCase):
hidden_dims = 16
output_dims = 12
config = configs.ResidualBlockConfig(
- hidden_dims=hidden_dims,
- output_dims=output_dims,
- use_bias=True,
- activation="relu",
- identity_skip=False,
- prenorm="none",
+ hidden_dims=hidden_dims,
+ output_dims=output_dims,
+ use_bias=True,
+ activation="relu",
+ identity_skip=False,
+ prenorm="none",
)
mod = torch_dense.ResidualBlock(config=config)
x = torch.randn(2, input_dim)
@@ -133,12 +134,12 @@ class ResidualBlockTest(unittest.TestCase):
input_dim = 8
hidden_dims = 16
config = configs.ResidualBlockConfig(
- hidden_dims=hidden_dims,
- output_dims=input_dim,
- use_bias=False,
- activation="relu",
- identity_skip=True,
- prenorm="rms",
+ hidden_dims=hidden_dims,
+ output_dims=input_dim,
+ use_bias=False,
+ activation="relu",
+ identity_skip=True,
+ prenorm="rms",
)
mod = torch_dense.ResidualBlock(config=config)
x = torch.randn(2, input_dim)
@@ -174,12 +175,12 @@ class UtilTest(unittest.TestCase):
head_dim = 8
caches = torch_util.DecodeCache.init_decode_cache(
- num_layers=num_layers,
- batch_size=batch_size,
- num_variates=num_variates,
- num_total_input_patches=num_total_input_patches,
- num_heads=num_heads,
- head_dim=head_dim,
+ num_layers=num_layers,
+ batch_size=batch_size,
+ num_variates=num_variates,
+ num_total_input_patches=num_total_input_patches,
+ num_heads=num_heads,
+ head_dim=head_dim,
)
self.assertEqual(len(caches), num_layers)
@@ -189,12 +190,12 @@ class UtilTest(unittest.TestCase):
self.assertEqual(cache.next_index.shape, (leading_size,))
self.assertEqual(cache.num_front_masked.shape, (leading_size,))
self.assertEqual(
- cache.key.shape,
- (leading_size, num_total_input_patches, num_heads, head_dim),
+ cache.key.shape,
+ (leading_size, num_total_input_patches, num_heads, head_dim),
)
self.assertEqual(
- cache.value.shape,
- (leading_size, num_total_input_patches, num_heads, head_dim),
+ cache.value.shape,
+ (leading_size, num_total_input_patches, num_heads, head_dim),
)
np.testing.assert_allclose(cache.next_index.numpy(), 0)
np.testing.assert_allclose(cache.num_front_masked.numpy(), 0)
@@ -206,16 +207,16 @@ class UtilTest(unittest.TestCase):
x = torch.tensor([[[1.0, 2.0, 3.0]]])
mask = torch.zeros_like(x, dtype=torch.bool)
new_n, new_mu, new_sigma = torch_util.update_running_stats(
- torch.tensor([[n]]),
- torch.tensor([[mu]]),
- torch.tensor([[sigma]]),
- x,
- mask,
+ torch.tensor([[n]]),
+ torch.tensor([[mu]]),
+ torch.tensor([[sigma]]),
+ x,
+ mask,
)
self.assertAlmostEqual(new_n[0, 0].item(), 3.0)
self.assertAlmostEqual(new_mu[0, 0].item(), 2.0)
self.assertAlmostEqual(
- new_sigma[0, 0].item(), torch.std(x, unbiased=False).item(), places=6
+ new_sigma[0, 0].item(), torch.std(x, unbiased=False).item(), places=6
)
def test_update_running_stats_multiple_batches_no_mask(self):
@@ -225,23 +226,21 @@ class UtilTest(unittest.TestCase):
mask2 = torch.zeros_like(x2, dtype=torch.bool)
n1, mu1, sigma1 = torch_util.update_running_stats(
- torch.tensor([[0.0]]),
- torch.tensor([[0.0]]),
- torch.tensor([[0.0]]),
- x1,
- mask1,
- )
- n2, mu2, sigma2 = torch_util.update_running_stats(
- n1, mu1, sigma1, x2, mask2
+ torch.tensor([[0.0]]),
+ torch.tensor([[0.0]]),
+ torch.tensor([[0.0]]),
+ x1,
+ mask1,
)
+ n2, mu2, sigma2 = torch_util.update_running_stats(n1, mu1, sigma1, x2, mask2)
self.assertAlmostEqual(n2[0, 0].item(), 6.0)
self.assertAlmostEqual(mu2[0, 0].item(), 3.5)
concat_x = torch.cat([x1, x2], dim=-1)
self.assertAlmostEqual(
- sigma2[0, 0].item(),
- torch.std(concat_x, unbiased=False).item(),
- places=6,
+ sigma2[0, 0].item(),
+ torch.std(concat_x, unbiased=False).item(),
+ places=6,
)
def test_update_running_stats_with_mask(self):
@@ -249,19 +248,19 @@ class UtilTest(unittest.TestCase):
x = torch.tensor([[[1.0, 100.0, 2.0, 3.0]]])
mask = torch.tensor([[[False, True, False, False]]])
new_n, new_mu, new_sigma = torch_util.update_running_stats(
- torch.tensor([[n]]),
- torch.tensor([[mu]]),
- torch.tensor([[sigma]]),
- x,
- mask,
+ torch.tensor([[n]]),
+ torch.tensor([[mu]]),
+ torch.tensor([[sigma]]),
+ x,
+ mask,
)
self.assertAlmostEqual(new_n[0, 0].item(), 3.0)
self.assertAlmostEqual(new_mu[0, 0].item(), 2.0)
valid_x = torch.tensor([1.0, 2.0, 3.0])
self.assertAlmostEqual(
- new_sigma[0, 0].item(),
- torch.std(valid_x, unbiased=False).item(),
- places=6,
+ new_sigma[0, 0].item(),
+ torch.std(valid_x, unbiased=False).item(),
+ places=6,
)
def test_revin_no_reverse(self):
@@ -270,9 +269,7 @@ class UtilTest(unittest.TestCase):
sigma = torch.tensor([[1.0]])
normalized_x = torch_util.revin(x, mu, sigma, reverse=False)
expected = torch.tensor([[[-1.0, 0.0, 1.0]]])
- np.testing.assert_allclose(
- normalized_x.numpy(), expected.numpy(), atol=1e-6
- )
+ np.testing.assert_allclose(normalized_x.numpy(), expected.numpy(), atol=1e-6)
def test_revin_reverse(self):
x_normalized = torch.tensor([[[-1.0, 0.0, 1.0]]])
@@ -280,9 +277,7 @@ class UtilTest(unittest.TestCase):
sigma = torch.tensor([[1.0]])
denormalized_x = torch_util.revin(x_normalized, mu, sigma, reverse=True)
expected = torch.tensor([[[1.0, 2.0, 3.0]]])
- np.testing.assert_allclose(
- denormalized_x.numpy(), expected.numpy(), atol=1e-6
- )
+ np.testing.assert_allclose(denormalized_x.numpy(), expected.numpy(), atol=1e-6)
def test_revin_near_zero_sigma(self):
x = torch.tensor([[[2.0, 2.0, 2.0]]])
@@ -290,29 +285,23 @@ class UtilTest(unittest.TestCase):
sigma = torch.tensor([[1e-7]])
normalized_x = torch_util.revin(x, mu, sigma, reverse=False)
expected = torch.tensor([[[0.0, 0.0, 0.0]]])
- np.testing.assert_allclose(
- normalized_x.numpy(), expected.numpy(), atol=1e-6
- )
+ np.testing.assert_allclose(normalized_x.numpy(), expected.numpy(), atol=1e-6)
def test_get_output_patch_via_roll(self):
x = torch.tensor([[[[1, 2], [3, 4], [5, 6], [7, 8]]]], dtype=torch.float32)
rolls = 2
expected_output = torch.tensor(
- [[[[3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 1, 2], [1, 2, 3, 4]]]],
- dtype=torch.float32,
+ [[[[3, 4, 5, 6], [5, 6, 7, 8], [7, 8, 1, 2], [1, 2, 3, 4]]]],
+ dtype=torch.float32,
)
output, _ = torch_util.get_output_patch_via_roll(x, rolls)
- np.testing.assert_allclose(
- output.numpy(), expected_output.numpy(), atol=1e-6
- )
+ np.testing.assert_allclose(output.numpy(), expected_output.numpy(), atol=1e-6)
def test_get_running_stats_single_segment(self):
values = torch.tensor([[[[1, 3], [5, 7]]]], dtype=torch.float32)
masks = torch.zeros_like(values, dtype=torch.bool)
segment_ids = torch.tensor([[0, 0]], dtype=torch.int32)
- _, mu, sigma = torch_util.get_running_stats(
- values, masks, segment_ids=segment_ids
- )
+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)
expected_mu = torch.tensor([[[2.0, 4.0]]])
expected_sigma = torch.tensor([[[1.0, 2.236068]]])
@@ -320,14 +309,10 @@ class UtilTest(unittest.TestCase):
np.testing.assert_allclose(sigma.numpy(), expected_sigma.numpy(), atol=1e-6)
def test_get_running_stats_multiple_segments(self):
- values = torch.tensor(
- [[[[1, 3], [10, 10], [5, 7], [4, 6]]]], dtype=torch.float32
- )
+ values = torch.tensor([[[[1, 3], [10, 10], [5, 7], [4, 6]]]], dtype=torch.float32)
masks = torch.zeros_like(values, dtype=torch.bool)
segment_ids = torch.tensor([[0, 0, 1, 1]], dtype=torch.int32)
- _, mu, sigma = torch_util.get_running_stats(
- values, masks, segment_ids=segment_ids
- )
+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)
expected_mu = torch.tensor([[[2.0, 6.0, 6.0, 5.5]]])
expected_sigma = torch.tensor([[[1.0, 4.062019, 1.0, 1.118034]]])
@@ -335,17 +320,13 @@ class UtilTest(unittest.TestCase):
np.testing.assert_allclose(sigma.numpy(), expected_sigma.numpy(), atol=1e-6)
def test_get_running_stats_with_masks(self):
- values = torch.tensor(
- [[[[1, 10, 3, 0], [100, 5, 200, 7]]]], dtype=torch.float32
- )
+ values = torch.tensor([[[[1, 10, 3, 0], [100, 5, 200, 7]]]], dtype=torch.float32)
masks = torch.tensor(
- [[[[False, True, False, True], [True, False, True, False]]]],
- dtype=torch.bool,
+ [[[[False, True, False, True], [True, False, True, False]]]],
+ dtype=torch.bool,
)
segment_ids = torch.tensor([[0, 0]], dtype=torch.int32)
- _, mu, sigma = torch_util.get_running_stats(
- values, masks, segment_ids=segment_ids
- )
+ _, mu, sigma = torch_util.get_running_stats(values, masks, segment_ids=segment_ids)
expected_mu = torch.tensor([[[2.0, 4.0]]])
expected_sigma = torch.tensor([[[1.0, 2.236068]]])
diff --git a/src/timesfm3/timesfm3_forecaster.py b/src/timesfm3/timesfm3_forecaster.py
index 61c0e37..c16da1c 100644
--- a/src/timesfm3/timesfm3_forecaster.py
+++ b/src/timesfm3/timesfm3_forecaster.py
@@ -4,17 +4,16 @@ from __future__ import annotations
import dataclasses
import gc
-import logging
import math
import os
-from typing import Any, Iterator
+from collections.abc import Iterator
+from typing import Any
import numpy as np
import torch
-from . import configs
+from . import configs, util
from . import model as torch_model_lib
-from . import util
_MAX_CONTEXT_LENGTH = 16384
_SIGMA_THRESHOLD: float = 1e-7
@@ -25,8 +24,8 @@ _GC_MEMORY_THRESHOLD: float = 0.9
class _ModelConfig:
"""Configuration for a PyTorch TimesFM3 forecaster."""
- # Path to checkpoint file (.pth or .safetensors).
- checkpoint_path: str = "~/models/tfm3/timesfm3_torch_1944000.safetensors"
+ # Path to checkpoint file (.pth or .safetensors) or Hugging Face repo ID.
+ checkpoint_path: str = "google/timesfm-3.0-pytorch"
# Batch size to use for inference.
per_core_batch_size: int = 4
@@ -39,7 +38,7 @@ class _ModelConfig:
# Quantiles to predict.
quantiles: list[float] = dataclasses.field(
- default_factory=lambda: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
+ default_factory=lambda: [0.1, 0.2, 0.3, 0.4, 0.5, 0.6, 0.7, 0.8, 0.9]
)
# Median quantile index for the forecast.
@@ -97,8 +96,8 @@ class ForecastOutput:
def try_gc(
- device: torch.device | str | None = None,
- gc_memory_threshold: float = _GC_MEMORY_THRESHOLD,
+ device: torch.device | str | None = None,
+ gc_memory_threshold: float = _GC_MEMORY_THRESHOLD,
) -> None:
"""Trigger Python GC and empty CUDA cache if memory exceeds threshold."""
if device is not None and torch.cuda.is_available():
@@ -193,26 +192,28 @@ class _Query:
def context_length(self) -> int:
return self.targets.shape[-1]
- def format(self, global_context: int) -> tuple[
- int,
- np.ndarray,
- np.ndarray,
- np.ndarray | None,
- np.ndarray | None,
- bool,
+ def format(
+ self, global_context: int
+ ) -> tuple[
+ int,
+ np.ndarray,
+ np.ndarray,
+ np.ndarray | None,
+ np.ndarray | None,
+ bool,
]:
"""Formats and left-pads/truncates the query to global_context length."""
targets = np.atleast_2d(self.targets)
masks = np.zeros((self.context_length,), dtype=bool)
past_only_covariates = (
- np.atleast_2d(self.past_only_covariates)
- if self.past_only_covariates is not None
- else None
+ np.atleast_2d(self.past_only_covariates)
+ if self.past_only_covariates is not None
+ else None
)
past_future_covariates = (
- np.atleast_2d(self.past_future_covariates)
- if self.past_future_covariates is not None
- else None
+ np.atleast_2d(self.past_future_covariates)
+ if self.past_future_covariates is not None
+ else None
)
if self.context_length > global_context:
@@ -222,99 +223,99 @@ class _Query:
past_only_covariates = past_only_covariates[:, -global_context:]
if past_future_covariates is not None:
past_future_covariates = past_future_covariates[
- :, -(global_context + self.horizon) :
+ :, -(global_context + self.horizon) :
]
elif self.context_length < global_context:
pad_len = global_context - self.context_length
targets = np.pad(
- targets,
- [(0, 0), (pad_len, 0)],
- mode="constant",
- constant_values=0.0,
+ targets,
+ [(0, 0), (pad_len, 0)],
+ mode="constant",
+ constant_values=0.0,
)
masks = np.pad(
- masks,
- [(pad_len, 0)],
- mode="constant",
- constant_values=True,
+ masks,
+ [(pad_len, 0)],
+ mode="constant",
+ constant_values=True,
)
if self.padded:
masks = np.ones_like(masks, dtype=bool)
if past_only_covariates is not None:
past_only_covariates = np.pad(
- past_only_covariates,
- [(0, 0), (pad_len, 0)],
- mode="constant",
- constant_values=0.0,
+ past_only_covariates,
+ [(0, 0), (pad_len, 0)],
+ mode="constant",
+ constant_values=0.0,
)
if past_future_covariates is not None:
past_future_covariates = np.pad(
- past_future_covariates,
- [(0, 0), (pad_len, 0)],
- mode="constant",
- constant_values=0.0,
+ past_future_covariates,
+ [(0, 0), (pad_len, 0)],
+ mode="constant",
+ constant_values=0.0,
)
return (
- self.horizon,
- targets,
- masks,
- past_only_covariates,
- past_future_covariates,
- self.padded,
+ self.horizon,
+ targets,
+ masks,
+ past_only_covariates,
+ past_future_covariates,
+ self.padded,
)
def _make_torch_model(
- config: _ModelConfig,
+ config: _ModelConfig,
) -> torch_model_lib.TimesFM3Torch:
"""Builds a PyTorch model using ModelConfig."""
resblock_config = (
- config.residual_block_config
- if config.residual_block_config is not None
- else configs.ResidualBlockConfig(
- hidden_dims=1280,
- output_dims=1280,
- use_bias=False,
- activation="relu",
- )
+ config.residual_block_config
+ if config.residual_block_config is not None
+ else configs.ResidualBlockConfig(
+ hidden_dims=1280,
+ output_dims=1280,
+ use_bias=False,
+ activation="relu",
+ )
)
transformer_config = (
- config.transformer_config
- if config.transformer_config is not None
- else configs.StackedTransformersConfig(
- num_layers=20,
- transformer=configs.TransformerConfig(
- model_dims=1280,
- hidden_dims=1280,
- num_heads=16,
- attention_norm="rms",
- feedforward_norm="rms",
- qk_norm="rms",
- use_rope_seq=True,
- use_rope_var=True,
- use_bias=False,
- ff_activation="relu",
- deterministic=True,
- ),
- )
+ config.transformer_config
+ if config.transformer_config is not None
+ else configs.StackedTransformersConfig(
+ num_layers=20,
+ transformer=configs.TransformerConfig(
+ model_dims=1280,
+ hidden_dims=1280,
+ num_heads=16,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=True,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ ),
+ )
)
t_model = torch_model_lib.TimesFM3Torch(
- input_patch_len=config.input_patch_length,
- output_patch_len=config.output_patch_length,
- quantiles=config.quantiles,
- use_variate_attention=config.use_variate_attention,
- value_clip=config.value_clip,
- input_transform=config.input_transform,
- use_stitching=config.use_stitching,
- use_linear_detrending=config.use_linear_detrending,
- linear_detrending_threshold=config.linear_detrending_threshold,
- use_iterative_cpm_revin=config.use_iterative_cpm_revin,
- use_frozen_running_stats=config.use_frozen_running_stats,
- residual_block_config=resblock_config,
- transformer_config=transformer_config,
+ input_patch_len=config.input_patch_length,
+ output_patch_len=config.output_patch_length,
+ quantiles=config.quantiles,
+ use_variate_attention=config.use_variate_attention,
+ value_clip=config.value_clip,
+ input_transform=config.input_transform,
+ use_stitching=config.use_stitching,
+ use_linear_detrending=config.use_linear_detrending,
+ linear_detrending_threshold=config.linear_detrending_threshold,
+ use_iterative_cpm_revin=config.use_iterative_cpm_revin,
+ use_frozen_running_stats=config.use_frozen_running_stats,
+ residual_block_config=resblock_config,
+ transformer_config=transformer_config,
)
t_model.eval()
input_dim = 2 * (t_model.input_patch_len + t_model.output_patch_len)
@@ -334,12 +335,27 @@ class TimesFM3Forecaster:
self.config = config
self._init_model()
+ @classmethod
+ def from_pretrained(
+ cls,
+ pretrained_model_name_or_path: str = "google/timesfm-3.0-pytorch",
+ device: str | None = None,
+ **kwargs: Any,
+ ) -> TimesFM3Forecaster:
+ """Instantiates a TimesFM3Forecaster from a pretrained HF repo or directory."""
+ config = _ModelConfig(
+ checkpoint_path=pretrained_model_name_or_path,
+ device=device,
+ **kwargs,
+ )
+ return cls(config=config)
+
@property
def global_context(self) -> int:
"""Max context length rounded up to the nearest input patch boundary."""
return (
- math.ceil(_MAX_CONTEXT_LENGTH / self.config.input_patch_length)
- * self.config.input_patch_length
+ math.ceil(_MAX_CONTEXT_LENGTH / self.config.input_patch_length)
+ * self.config.input_patch_length
)
def _init_model(self):
@@ -349,87 +365,117 @@ class TimesFM3Forecaster:
else:
self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
- self.model = _make_torch_model(self.config)
-
checkpoint_path = os.path.expanduser(self.config.checkpoint_path)
- if os.path.exists(checkpoint_path):
+
+ # If checkpoint_path is a directory with config.json or a Hugging Face repo ID:
+ is_local_dir = os.path.isdir(checkpoint_path)
+ is_local_file = os.path.isfile(checkpoint_path)
+
+ if is_local_dir or not is_local_file:
+ # Load via PyTorchModelHubMixin.from_pretrained (downloads config.json and weights)
+ self.model = torch_model_lib.TimesFM3Torch.from_pretrained(
+ checkpoint_path,
+ )
+ # Synchronize forecaster config with the loaded model config
+ median_q_idx = self.config.median_quantile_index
+ if median_q_idx >= len(self.model.quantiles):
+ median_q_idx = len(self.model.quantiles) // 2
+
+ self.config = dataclasses.replace(
+ self.config,
+ input_patch_length=self.model.input_patch_len,
+ output_patch_length=self.model.output_patch_len,
+ quantiles=list(self.model.quantiles),
+ median_quantile_index=median_q_idx,
+ residual_block_config=self.model.residual_block_config,
+ transformer_config=self.model.transformer_config,
+ use_variate_attention=self.model.use_variate_attention,
+ value_clip=self.model.value_clip,
+ use_stitching=self.model.use_stitching,
+ use_linear_detrending=self.model.use_linear_detrending,
+ linear_detrending_threshold=self.model.linear_detrending_threshold,
+ use_iterative_cpm_revin=self.model.use_iterative_cpm_revin,
+ use_frozen_running_stats=self.model.use_frozen_running_stats,
+ input_transform=self.model.input_transform,
+ )
+ else:
+ # Local file (.safetensors or .pth / .pt)
+ self.model = _make_torch_model(self.config)
if checkpoint_path.endswith(".safetensors"):
state_dict = util.load_safetensors(checkpoint_path, device=self.device)
self.model.load_state_dict(state_dict)
- elif checkpoint_path.endswith(".pth") or checkpoint_path.endswith(".pt"):
+ elif checkpoint_path.endswith((".pth", ".pt")):
state_dict = torch.load(checkpoint_path, map_location=self.device)
self.model.load_state_dict(state_dict)
else:
raise ValueError(
- f"Unsupported checkpoint path format: {checkpoint_path}. "
- "Expected .safetensors or .pth / .pt file."
+ f"Unsupported checkpoint path format: {checkpoint_path}. "
+ "Expected .safetensors or .pth / .pt file."
)
self.model.to(self.device)
self.model.eval()
def predict(
- self,
- context: np.ndarray,
- horizon: int,
- past_only_covariates: np.ndarray | None = None,
- past_future_covariates: np.ndarray | None = None,
- ts_id: str | None = None,
- return_quantiles: bool = False,
- use_symmetric_averaging: bool = False,
- make_positive: bool = False,
- use_znorm: bool = False,
- padding_mode: str = "none",
+ self,
+ context: np.ndarray,
+ horizon: int,
+ past_only_covariates: np.ndarray | None = None,
+ past_future_covariates: np.ndarray | None = None,
+ ts_id: str | None = None,
+ return_quantiles: bool = False,
+ use_symmetric_averaging: bool = False,
+ make_positive: bool = False,
+ use_znorm: bool = False,
+ padding_mode: str = "none",
) -> ForecastOutput:
"""Convenience wrapper: runs inference on a single time series."""
results = list(
- self.predict_batch(
- contexts=[context],
- horizon=horizon,
- past_only_covariates=[past_only_covariates],
- past_future_covariates=[past_future_covariates],
- ts_ids=[ts_id] if ts_id is not None else None,
- return_quantiles=return_quantiles,
- use_symmetric_averaging=use_symmetric_averaging,
- make_positive=make_positive,
- use_znorm=use_znorm,
- padding_mode=padding_mode,
- )
+ self.predict_batch(
+ contexts=[context],
+ horizon=horizon,
+ past_only_covariates=[past_only_covariates],
+ past_future_covariates=[past_future_covariates],
+ ts_ids=[ts_id] if ts_id is not None else None,
+ return_quantiles=return_quantiles,
+ use_symmetric_averaging=use_symmetric_averaging,
+ make_positive=make_positive,
+ use_znorm=use_znorm,
+ padding_mode=padding_mode,
+ )
)
return results[0]
def predict_batch(
- self,
- contexts: list[np.ndarray],
- horizon: int,
- past_only_covariates: list[np.ndarray | None] | None = None,
- past_future_covariates: list[np.ndarray | None] | None = None,
- ts_ids: list[str] | None = None,
- return_quantiles: bool = False,
- use_symmetric_averaging: bool = False,
- make_positive: bool = False,
- use_znorm: bool = False,
- padding_mode: str = "none",
+ self,
+ contexts: list[np.ndarray],
+ horizon: int,
+ past_only_covariates: list[np.ndarray | None] | None = None,
+ past_future_covariates: list[np.ndarray | None] | None = None,
+ ts_ids: list[str] | None = None,
+ return_quantiles: bool = False,
+ use_symmetric_averaging: bool = False,
+ make_positive: bool = False,
+ use_znorm: bool = False,
+ padding_mode: str = "none",
) -> Iterator[ForecastOutput]:
"""Runs inference on a batch of time series with optional covariates."""
global_horizon = (
- math.ceil(horizon / self.config.output_patch_length)
- * self.config.output_patch_length
+ math.ceil(horizon / self.config.output_patch_length)
+ * self.config.output_patch_length
)
num_original_ts = len(contexts)
- original_ts_ids = (
- list(ts_ids) if ts_ids is not None else [None] * num_original_ts
- )
+ original_ts_ids = list(ts_ids) if ts_ids is not None else [None] * num_original_ts
po_cov_list = (
- list(past_only_covariates)
- if past_only_covariates is not None
- else [None] * num_original_ts
+ list(past_only_covariates)
+ if past_only_covariates is not None
+ else [None] * num_original_ts
)
pf_cov_list = (
- list(past_future_covariates)
- if past_future_covariates is not None
- else [None] * num_original_ts
+ list(past_future_covariates)
+ if past_future_covariates is not None
+ else [None] * num_original_ts
)
contexts_2d: list[np.ndarray] = []
@@ -466,9 +512,9 @@ class TimesFM3Forecaster:
for idx, ctx in enumerate(contexts_2d):
if ctx.shape[0] != num_targets_in:
raise ValueError(
- "All contexts must have the same number of target variates, but"
- f" contexts[0] has {num_targets_in} and contexts[{idx}] has"
- f" {ctx.shape[0]}."
+ "All contexts must have the same number of target variates, but"
+ f" contexts[0] has {num_targets_in} and contexts[{idx}] has"
+ f" {ctx.shape[0]}."
)
is_univariate = num_targets_in == 1
@@ -546,12 +592,12 @@ class TimesFM3Forecaster:
queries: list[_Query] = []
for idx, ctx in enumerate(contexts_2d):
queries.append(
- _Query(
- horizon=global_horizon,
- targets=ctx,
- past_only_covariates=po_2d[idx],
- past_future_covariates=pf_2d[idx],
- )
+ _Query(
+ horizon=global_horizon,
+ targets=ctx,
+ past_only_covariates=po_2d[idx],
+ past_future_covariates=pf_2d[idx],
+ )
)
if not queries:
@@ -572,48 +618,48 @@ class TimesFM3Forecaster:
formatted_batch = [q.format(self.global_context) for q in query_batch]
(
- batched_hor,
- batched_tgt,
- batched_mask,
- batched_po,
- batched_pf,
- _,
+ batched_hor,
+ batched_tgt,
+ batched_mask,
+ batched_po,
+ batched_pf,
+ _,
) = tuple(list(w) for w in zip(*formatted_batch))
tgt_torch = torch.from_numpy(np.stack(batched_tgt, axis=0)).to(
- self.device, dtype=torch.float32
+ self.device, dtype=torch.float32
)
mask_torch = torch.from_numpy(np.stack(batched_mask, axis=0)).to(
- self.device, dtype=torch.bool
+ self.device, dtype=torch.bool
)
po_torch = None
if any(po is not None for po in batched_po):
po_arrs = [
- po if po is not None else np.zeros_like(batched_tgt[j])
- for j, po in enumerate(batched_po)
+ po if po is not None else np.zeros_like(batched_tgt[j])
+ for j, po in enumerate(batched_po)
]
po_torch = torch.from_numpy(np.stack(po_arrs, axis=0)).to(
- self.device, dtype=torch.float32
+ self.device, dtype=torch.float32
)
pf_torch = None
if any(pf is not None for pf in batched_pf):
pf_arrs = [
- pf if pf is not None else np.zeros_like(batched_tgt[j])
- for j, pf in enumerate(batched_pf)
+ pf if pf is not None else np.zeros_like(batched_tgt[j])
+ for j, pf in enumerate(batched_pf)
]
pf_torch = torch.from_numpy(np.stack(pf_arrs, axis=0)).to(
- self.device, dtype=torch.float32
+ self.device, dtype=torch.float32
)
with torch.no_grad():
out_logits = self.model.decode(
- target=tgt_torch,
- horizon=batched_hor[0],
- past_only_covariates=po_torch,
- past_future_covariates=pf_torch,
- mask=mask_torch,
+ target=tgt_torch,
+ horizon=batched_hor[0],
+ past_only_covariates=po_torch,
+ past_future_covariates=pf_torch,
+ mask=mask_torch,
)
ys.append(out_logits.cpu().numpy())
try_gc(self.device)
@@ -621,11 +667,9 @@ class TimesFM3Forecaster:
all_raw_outputs = np.concatenate(ys, axis=0)
num_relevant_outputs = (
- 2 * num_original_ts if use_symmetric_averaging else num_original_ts
+ 2 * num_original_ts if use_symmetric_averaging else num_original_ts
)
- all_raw_outputs = all_raw_outputs[
- :num_relevant_outputs, :num_targets_in, :, :
- ]
+ all_raw_outputs = all_raw_outputs[:num_relevant_outputs, :num_targets_in, :, :]
if use_symmetric_averaging:
ys_pos = all_raw_outputs[0::2]
@@ -659,13 +703,13 @@ class TimesFM3Forecaster:
if is_univariate:
raw = raw[0]
yield ForecastOutput(
- ts_id=original_ts_ids[i],
- forecast=raw[:horizon, self.config.median_quantile_index],
- quantiles=raw[:horizon, :] if return_quantiles else None,
+ ts_id=original_ts_ids[i],
+ forecast=raw[:horizon, self.config.median_quantile_index],
+ quantiles=raw[:horizon, :] if return_quantiles else None,
)
else:
yield ForecastOutput(
- ts_id=original_ts_ids[i],
- forecast=raw[:, :horizon, self.config.median_quantile_index],
- quantiles=raw[:, :horizon, :] if return_quantiles else None,
+ ts_id=original_ts_ids[i],
+ forecast=raw[:, :horizon, self.config.median_quantile_index],
+ quantiles=raw[:, :horizon, :] if return_quantiles else None,
)
diff --git a/src/timesfm3/timesfm3_forecaster_test.py b/src/timesfm3/timesfm3_forecaster_test.py
index aa1d8bb..e9dc229 100644
--- a/src/timesfm3/timesfm3_forecaster_test.py
+++ b/src/timesfm3/timesfm3_forecaster_test.py
@@ -16,6 +16,7 @@
import unittest
from unittest import mock
+
import numpy as np
import torch
@@ -32,13 +33,13 @@ class FakeModel(torch.nn.Module):
self.quantiles = quantiles
def decode(
- self,
- target,
- autoregressive_index=0,
- horizon=8,
- past_only_covariates=None,
- past_future_covariates=None,
- mask=None,
+ self,
+ target,
+ autoregressive_index=0,
+ horizon=8,
+ past_only_covariates=None,
+ past_future_covariates=None,
+ mask=None,
):
del autoregressive_index, past_only_covariates, past_future_covariates, mask
b, v, _ = target.shape
@@ -47,15 +48,14 @@ class FakeModel(torch.nn.Module):
class TimesFM3ForecasterTest(unittest.TestCase):
-
def setUp(self):
super().setUp()
self.config = timesfm3_forecaster._ModelConfig(
- checkpoint_path="/fake/path",
- per_core_batch_size=2,
- input_patch_length=8,
- output_patch_length=8,
- median_quantile_index=1,
+ checkpoint_path="/fake/path",
+ per_core_batch_size=2,
+ input_patch_length=8,
+ output_patch_length=8,
+ median_quantile_index=1,
)
def test_strip_leading_nans_1d(self):
@@ -64,15 +64,19 @@ class TimesFM3ForecasterTest(unittest.TestCase):
np.testing.assert_array_equal(res, np.array([1.0, 2.0, np.nan]))
def test_strip_leading_nans_2d(self):
- arr = np.array([
+ arr = np.array(
+ [
[np.nan, 1.0, 2.0],
[np.nan, np.nan, 3.0],
- ])
+ ]
+ )
res = timesfm3_forecaster.strip_leading_nans(arr)
- expected = np.array([
+ expected = np.array(
+ [
[1.0, 2.0],
[np.nan, 3.0],
- ])
+ ]
+ )
np.testing.assert_array_equal(res, expected)
def test_linear_interpolation(self):
@@ -93,7 +97,7 @@ class TimesFM3ForecasterTest(unittest.TestCase):
self.assertFalse(timesfm3_forecaster._is_nonnegative(arr_neg))
@mock.patch.object(
- timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True
+ timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True
)
def test_univariate_predict(self, _):
forecaster = timesfm3_forecaster.TimesFM3Forecaster(self.config)
@@ -106,30 +110,78 @@ class TimesFM3ForecasterTest(unittest.TestCase):
np.testing.assert_array_equal(out.forecast, np.full((4,), 2.0))
@mock.patch.object(
- timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True
+ timesfm3_forecaster.TimesFM3Forecaster, "_init_model", autospec=True
)
def test_multivariate_predict_with_covariates(self, _):
forecaster = timesfm3_forecaster.TimesFM3Forecaster(self.config)
forecaster.model = FakeModel()
forecaster.device = torch.device("cpu")
- ctx = np.array([
+ ctx = np.array(
+ [
[1.0, 2.0, 3.0, 4.0],
[5.0, 6.0, 7.0, 8.0],
- ])
+ ]
+ )
po_cov = np.array([1.0, 1.0, 1.0, 1.0])
pf_cov = np.array([2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0, 2.0])
out = forecaster.predict(
- ctx,
- horizon=4,
- past_only_covariates=po_cov,
- past_future_covariates=pf_cov,
- return_quantiles=True,
+ ctx,
+ horizon=4,
+ past_only_covariates=po_cov,
+ past_future_covariates=pf_cov,
+ return_quantiles=True,
)
self.assertEqual(out.forecast.shape, (2, 4))
self.assertEqual(out.quantiles.shape, (2, 4, 3))
+ def test_from_pretrained_local_directory(self):
+ import tempfile
+
+ from . import configs
+ from . import model as torch_model_lib
+
+ resblock_config = configs.ResidualBlockConfig(
+ hidden_dims=16,
+ output_dims=16,
+ use_bias=False,
+ activation="relu",
+ )
+ transformer_config = configs.StackedTransformersConfig(
+ num_layers=1,
+ transformer=configs.TransformerConfig(
+ model_dims=16,
+ hidden_dims=16,
+ num_heads=2,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=False,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ ),
+ )
+ model = torch_model_lib.TimesFM3Torch(
+ input_patch_len=8,
+ output_patch_len=16,
+ quantiles=[0.1, 0.5, 0.9],
+ residual_block_config=resblock_config,
+ transformer_config=transformer_config,
+ )
+ with tempfile.TemporaryDirectory() as tmpdir:
+ model.save_pretrained(tmpdir)
+ forecaster = timesfm3_forecaster.TimesFM3Forecaster.from_pretrained(
+ tmpdir, device="cpu"
+ )
+ self.assertEqual(forecaster.config.input_patch_length, 8)
+ self.assertEqual(forecaster.config.output_patch_length, 16)
+ ctx = np.array([1.0, 2.0, 3.0, 4.0, 5.0, 6.0, 7.0, 8.0])
+ out = forecaster.predict(ctx, horizon=8)
+ self.assertEqual(out.forecast.shape, (8,))
+
if __name__ == "__main__":
unittest.main()
diff --git a/src/timesfm3/transformations.py b/src/timesfm3/transformations.py
index 6167dc8..c25da83 100644
--- a/src/timesfm3/transformations.py
+++ b/src/timesfm3/transformations.py
@@ -6,7 +6,8 @@ Each transformation is a callable with signature (x, reverse=False) -> x'.
from __future__ import annotations
-from typing import Callable, Protocol
+from collections.abc import Callable
+from typing import Protocol
import torch
@@ -14,8 +15,7 @@ import torch
class TransformFn(Protocol):
"""Protocol for a reversible transformation function."""
- def __call__(self, x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor:
- ...
+ def __call__(self, x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor: ...
def signed_log(x: torch.Tensor, *, reverse: bool = False) -> torch.Tensor:
@@ -51,12 +51,12 @@ def _max_output_identity(value_clip: float) -> torch.Tensor:
_REGISTRY: dict[
- str,
- tuple[TransformFn, Callable[[float], torch.Tensor]],
+ str,
+ tuple[TransformFn, Callable[[float], torch.Tensor]],
] = {
- "signed_log": (signed_log, _max_output_signed_log),
- "signed_sqrt": (signed_sqrt, _max_output_signed_sqrt),
- "identity": (identity, _max_output_identity),
+ "signed_log": (signed_log, _max_output_signed_log),
+ "signed_sqrt": (signed_sqrt, _max_output_signed_sqrt),
+ "identity": (identity, _max_output_identity),
}
diff --git a/src/timesfm3/transformer.py b/src/timesfm3/transformer.py
index db8e57e..9811fd3 100644
--- a/src/timesfm3/transformer.py
+++ b/src/timesfm3/transformer.py
@@ -12,20 +12,18 @@ from __future__ import annotations
import math
import torch
-from torch import nn
import torch.nn.functional as F
+from torch import nn
-from . import configs
-from . import normalization
-from . import util
+from . import configs, normalization, util
def make_attn_mask(
- query_length: int,
- num_all_masked_kv: torch.Tensor,
- query_index_offset: torch.Tensor | None = None,
- kv_length: int = 0,
- causal: bool = True,
+ query_length: int,
+ num_all_masked_kv: torch.Tensor,
+ query_index_offset: torch.Tensor | None = None,
+ kv_length: int = 0,
+ causal: bool = True,
) -> torch.Tensor:
"""Makes attention mask. True = attend, False = mask.
@@ -54,7 +52,7 @@ def make_attn_mask(
def make_segment_mask(
- segment_ids: torch.Tensor,
+ segment_ids: torch.Tensor,
) -> torch.Tensor:
"""Makes a segment mask from segment ids.
@@ -75,10 +73,10 @@ class RotaryPositionalEmbedding(nn.Module):
"""
def __init__(
- self,
- embedding_dims: int,
- min_timescale: int = 1,
- max_timescale: int = 10000,
+ self,
+ embedding_dims: int,
+ min_timescale: int = 1,
+ max_timescale: int = 10000,
):
super().__init__()
self.embedding_dims = embedding_dims
@@ -86,21 +84,14 @@ class RotaryPositionalEmbedding(nn.Module):
self.max_timescale = max_timescale
half_dim = embedding_dims // 2
- fraction = (
- 2.0
- * torch.arange(half_dim, dtype=torch.float32)
- / embedding_dims
- )
- timescale = (
- min_timescale
- * (max_timescale / min_timescale) ** fraction
- )
+ fraction = 2.0 * torch.arange(half_dim, dtype=torch.float32) / embedding_dims
+ timescale = min_timescale * (max_timescale / min_timescale) ** fraction
self.register_buffer("timescale", timescale, persistent=False)
def forward(
- self,
- inputs: torch.Tensor,
- position: torch.Tensor | None = None,
+ self,
+ inputs: torch.Tensor,
+ position: torch.Tensor | None = None,
) -> torch.Tensor:
"""Applies rotary positional embeddings.
@@ -113,15 +104,15 @@ class RotaryPositionalEmbedding(nn.Module):
"""
if self.embedding_dims != inputs.shape[-1]:
raise ValueError(
- "The embedding dims of the rotary position embedding "
- "must match the hidden dimension of the inputs."
+ "The embedding dims of the rotary position embedding "
+ "must match the hidden dimension of the inputs."
)
timescale = self.timescale.to(inputs.device)
if position is None:
seq_length = inputs.shape[1]
position = torch.arange(
- seq_length, device=inputs.device, dtype=torch.float32
+ seq_length, device=inputs.device, dtype=torch.float32
).unsqueeze(0)
if inputs.dim() == 4:
@@ -152,17 +143,17 @@ class MultiHeadAttention(nn.Module):
"""
def __init__(
- self,
- num_heads: int,
- in_features: int,
- use_per_dim_scale: bool = True,
- use_rotary_position_embeddings: bool = True,
- causal_attention: bool = True,
- use_bias: bool = False,
- qk_norm: str = "rms",
- v_norm: str = "none",
- use_sdpa: bool = False,
- rescale_logits: bool = False,
+ self,
+ num_heads: int,
+ in_features: int,
+ use_per_dim_scale: bool = True,
+ use_rotary_position_embeddings: bool = True,
+ causal_attention: bool = True,
+ use_bias: bool = False,
+ qk_norm: str = "rms",
+ v_norm: str = "none",
+ use_sdpa: bool = False,
+ rescale_logits: bool = False,
):
super().__init__()
self.num_heads = num_heads
@@ -204,7 +195,7 @@ class MultiHeadAttention(nn.Module):
# RoPE
if use_rotary_position_embeddings:
self.rotary_position_embedding = RotaryPositionalEmbedding(
- embedding_dims=self.head_dim
+ embedding_dims=self.head_dim
)
else:
self.rotary_position_embedding = None
@@ -216,13 +207,13 @@ class MultiHeadAttention(nn.Module):
self.per_dim_scale = None
def forward(
- self,
- inputs_q: torch.Tensor,
- *,
- segment_ids: torch.Tensor | None = None,
- segment_pos: torch.Tensor | None = None,
- decode_cache: util.DecodeCache | None = None,
- patch_mask: torch.Tensor | None = None,
+ self,
+ inputs_q: torch.Tensor,
+ *,
+ segment_ids: torch.Tensor | None = None,
+ segment_pos: torch.Tensor | None = None,
+ decode_cache: util.DecodeCache | None = None,
+ patch_mask: torch.Tensor | None = None,
) -> tuple[torch.Tensor, util.DecodeCache | None, torch.Tensor]:
"""Applies multi-head attention.
@@ -243,25 +234,21 @@ class MultiHeadAttention(nn.Module):
device = inputs_q.device
if patch_mask is None:
- patch_mask = torch.zeros(
- batch_size, n_patches, dtype=torch.bool, device=device
- )
+ patch_mask = torch.zeros(batch_size, n_patches, dtype=torch.bool, device=device)
# Project Q, K, V and reshape to (b, n, h, hd)
query = self.query_proj(inputs_q).view(
- batch_size, n_patches, self.num_heads, self.head_dim
+ batch_size, n_patches, self.num_heads, self.head_dim
)
key = self.key_proj(inputs_q).view(
- batch_size, n_patches, self.num_heads, self.head_dim
+ batch_size, n_patches, self.num_heads, self.head_dim
)
value = self.value_proj(inputs_q).view(
- batch_size, n_patches, self.num_heads, self.head_dim
+ batch_size, n_patches, self.num_heads, self.head_dim
)
if decode_cache is None:
- num_front_masked = torch.sum(
- torch.cumprod(patch_mask.int(), dim=-1), dim=-1
- )
+ num_front_masked = torch.sum(torch.cumprod(patch_mask.int(), dim=-1), dim=-1)
next_index = torch.zeros_like(num_front_masked, dtype=torch.int32)
else:
num_front_masked = decode_cache.num_front_masked
@@ -270,9 +257,9 @@ class MultiHeadAttention(nn.Module):
# Apply RoPE
if self.rotary_position_embedding is not None:
if segment_pos is None:
- position = torch.arange(
- n_patches, device=device, dtype=torch.int32
- ).unsqueeze(0) + next_index.unsqueeze(-1)
+ position = torch.arange(n_patches, device=device, dtype=torch.int32).unsqueeze(
+ 0
+ ) + next_index.unsqueeze(-1)
else:
position = segment_pos
query = self.rotary_position_embedding(query, position)
@@ -302,25 +289,25 @@ class MultiHeadAttention(nn.Module):
value = decode_cache.value
decode_cache = util.DecodeCache(
- next_index=next_index + n_patches,
- num_front_masked=num_front_masked,
- key=key,
- value=value,
+ next_index=next_index + n_patches,
+ num_front_masked=num_front_masked,
+ key=key,
+ value=value,
)
attn_mask = make_attn_mask(
- query_length=n_patches,
- num_all_masked_kv=num_front_masked,
- query_index_offset=next_index,
- kv_length=cache_size,
- causal=self.causal_attention,
+ query_length=n_patches,
+ num_all_masked_kv=num_front_masked,
+ query_index_offset=next_index,
+ kv_length=cache_size,
+ causal=self.causal_attention,
)
else:
# Training / full-sequence mode
attn_mask = make_attn_mask(
- query_length=n_patches,
- num_all_masked_kv=torch.zeros_like(num_front_masked),
- causal=self.causal_attention,
+ query_length=n_patches,
+ num_all_masked_kv=torch.zeros_like(num_front_masked),
+ causal=self.causal_attention,
)
# Apply patch_mask to K/V positions
attn_mask = attn_mask & (~patch_mask[:, None, None, :])
@@ -345,20 +332,20 @@ class MultiHeadAttention(nn.Module):
# → no internal division → net scale = √d.
attn_scale = math.sqrt(self.head_dim)
x = F.scaled_dot_product_attention(
- query,
- key,
- value,
- attn_mask=attn_mask.expand(-1, self.num_heads, -1, -1),
- scale=attn_scale,
+ query,
+ key,
+ value,
+ attn_mask=attn_mask.expand(-1, self.num_heads, -1, -1),
+ scale=attn_scale,
)
else:
# --- Manual attention path ---
# Convert mask: True=attend -> 0.0, False=mask -> -1e9 (matching Flax
# MEA bias).
float_mask = torch.where(
- attn_mask.expand(-1, self.num_heads, -1, -1),
- torch.tensor(0.0, device=device),
- torch.tensor(-1e9, device=device),
+ attn_mask.expand(-1, self.num_heads, -1, -1),
+ torch.tensor(0.0, device=device),
+ torch.tensor(-1e9, device=device),
)
if self.rescale_logits:
# MEA=False equivalent: Q*√d then divide by √d → net scale = 1.0.
@@ -366,9 +353,8 @@ class MultiHeadAttention(nn.Module):
# internally, so the pre-multiplication and rescaling cancel out.
query = query * math.sqrt(self.head_dim)
attn_logits = (
- torch.matmul(query, key.transpose(-2, -1))
- / math.sqrt(self.head_dim)
- + float_mask
+ torch.matmul(query, key.transpose(-2, -1)) / math.sqrt(self.head_dim)
+ + float_mask
)
else:
# MEA=True equivalent: Q*√d, no internal division → net scale = √d.
@@ -402,9 +388,9 @@ class MixingTransformer(nn.Module):
"""
def __init__(
- self,
- config: configs.TransformerConfig,
- use_variate_attention: bool = True,
+ self,
+ config: configs.TransformerConfig,
+ use_variate_attention: bool = True,
):
super().__init__()
self.config = config
@@ -419,54 +405,50 @@ class MixingTransformer(nn.Module):
# net scale=1.0).
rescale_logits = not getattr(config, "use_memory_efficient_attention", False)
self.seq_attn = MultiHeadAttention(
+ num_heads=config.num_heads,
+ in_features=config.model_dims,
+ use_per_dim_scale=True,
+ use_rotary_position_embeddings=config.use_rope_seq,
+ qk_norm=config.qk_norm,
+ v_norm=getattr(config, "v_norm", "none"),
+ causal_attention=getattr(config, "causal_attention", True),
+ use_bias=config.use_bias,
+ use_sdpa=config.use_sdpa,
+ rescale_logits=rescale_logits,
+ )
+
+ # Variate attention norms + module
+ if use_variate_attention:
+ self.pre_var_attn_ln = nn.RMSNorm(config.model_dims)
+ self.post_var_attn_ln = nn.RMSNorm(config.model_dims)
+ self.var_attn = MultiHeadAttention(
num_heads=config.num_heads,
in_features=config.model_dims,
use_per_dim_scale=True,
- use_rotary_position_embeddings=config.use_rope_seq,
+ use_rotary_position_embeddings=config.use_rope_var,
qk_norm=config.qk_norm,
v_norm=getattr(config, "v_norm", "none"),
- causal_attention=getattr(config, "causal_attention", True),
+ causal_attention=False,
use_bias=config.use_bias,
use_sdpa=config.use_sdpa,
rescale_logits=rescale_logits,
- )
-
- # Variate attention norms + module
- if use_variate_attention:
- self.pre_var_attn_ln = nn.RMSNorm(config.model_dims)
- self.post_var_attn_ln = nn.RMSNorm(config.model_dims)
- self.var_attn = MultiHeadAttention(
- num_heads=config.num_heads,
- in_features=config.model_dims,
- use_per_dim_scale=True,
- use_rotary_position_embeddings=config.use_rope_var,
- qk_norm=config.qk_norm,
- v_norm=getattr(config, "v_norm", "none"),
- causal_attention=False,
- use_bias=config.use_bias,
- use_sdpa=config.use_sdpa,
- rescale_logits=rescale_logits,
)
# FFN norms + layers
self.pre_ff_ln = nn.RMSNorm(config.model_dims)
self.post_ff_ln = nn.RMSNorm(config.model_dims)
- self.ff0 = nn.Linear(
- config.model_dims, config.hidden_dims, bias=config.use_bias
- )
- self.ff1 = nn.Linear(
- config.hidden_dims, config.model_dims, bias=config.use_bias
- )
+ self.ff0 = nn.Linear(config.model_dims, config.hidden_dims, bias=config.use_bias)
+ self.ff1 = nn.Linear(config.hidden_dims, config.model_dims, bias=config.use_bias)
self.activation = util.get_activation_fn(config.ff_activation)
def forward(
- self,
- input_embeddings: torch.Tensor,
- patch_mask: torch.Tensor,
- segment_ids: torch.Tensor | None = None,
- segment_pos: torch.Tensor | None = None,
- decode_cache: util.DecodeCache | None = None,
- var_segment_pos: torch.Tensor | None = None,
+ self,
+ input_embeddings: torch.Tensor,
+ patch_mask: torch.Tensor,
+ segment_ids: torch.Tensor | None = None,
+ segment_pos: torch.Tensor | None = None,
+ decode_cache: util.DecodeCache | None = None,
+ var_segment_pos: torch.Tensor | None = None,
) -> tuple[torch.Tensor, util.DecodeCache | None, torch.Tensor]:
"""Forward pass.
@@ -501,11 +483,11 @@ class MixingTransformer(nn.Module):
seq_seg_pos_flat = seg_pos_bvn.reshape(b * v, n)
seq_attn_out_flat, decode_cache, seq_attn_mask = self.seq_attn(
- seq_attn_in_flat,
- segment_ids=seq_seg_ids_flat,
- segment_pos=seq_seg_pos_flat,
- decode_cache=decode_cache,
- patch_mask=patch_mask_flat,
+ seq_attn_in_flat,
+ segment_ids=seq_seg_ids_flat,
+ segment_pos=seq_seg_pos_flat,
+ decode_cache=decode_cache,
+ patch_mask=patch_mask_flat,
)
seq_attn_out = seq_attn_out_flat.view(b, v, n, d)
h1 = self.post_seq_attn_ln(seq_attn_out) + input_embeddings
@@ -519,10 +501,10 @@ class MixingTransformer(nn.Module):
var_patch_mask = patch_mask.permute(0, 2, 1).reshape(b * n, v)
var_attn_out_flat, _, _ = self.var_attn(
- var_attn_in_flat,
- segment_pos=var_segment_pos,
- decode_cache=None,
- patch_mask=var_patch_mask,
+ var_attn_in_flat,
+ segment_pos=var_segment_pos,
+ decode_cache=None,
+ patch_mask=var_patch_mask,
)
# (b*n, v, d) -> (b, n, v, d) -> (b, v, n, d)
var_attn_out = var_attn_out_flat.view(b, n, v, d).permute(0, 2, 1, 3)
@@ -541,28 +523,30 @@ class StackedMixingTransformer(nn.Module):
"""Stacked MixingTransformer layers."""
def __init__(
- self,
- config: configs.StackedTransformersConfig,
- use_variate_attention: bool = True,
+ self,
+ config: configs.StackedTransformersConfig,
+ use_variate_attention: bool = True,
):
super().__init__()
self.config = config
- self.layers = nn.ModuleList([
+ self.layers = nn.ModuleList(
+ [
MixingTransformer(
- config=config.transformer,
- use_variate_attention=use_variate_attention,
+ config=config.transformer,
+ use_variate_attention=use_variate_attention,
)
for _ in range(config.num_layers)
- ])
+ ]
+ )
def forward(
- self,
- input_embeddings: torch.Tensor,
- patch_mask: torch.Tensor,
- segment_ids: torch.Tensor | None = None,
- segment_pos: torch.Tensor | None = None,
- decode_cache: list[util.DecodeCache] | None = None,
- var_segment_pos: torch.Tensor | None = None,
+ self,
+ input_embeddings: torch.Tensor,
+ patch_mask: torch.Tensor,
+ segment_ids: torch.Tensor | None = None,
+ segment_pos: torch.Tensor | None = None,
+ decode_cache: list[util.DecodeCache] | None = None,
+ var_segment_pos: torch.Tensor | None = None,
) -> tuple[torch.Tensor, list[util.DecodeCache] | None, list[torch.Tensor]]:
"""Forward pass through all layers.
@@ -586,12 +570,12 @@ class StackedMixingTransformer(nn.Module):
for i, layer in enumerate(self.layers):
output, layer_cache, layer_mask = layer(
- output,
- patch_mask,
- segment_ids,
- segment_pos,
- decode_cache[i], # pyrefly: ignore[unsupported-operation]
- var_segment_pos,
+ output,
+ patch_mask,
+ segment_ids,
+ segment_pos,
+ decode_cache[i], # pyrefly: ignore[unsupported-operation]
+ var_segment_pos,
)
new_caches.append(layer_cache)
attn_masks.append(layer_mask)
diff --git a/src/timesfm3/transformer_test.py b/src/timesfm3/transformer_test.py
index e482eb6..2498cdf 100644
--- a/src/timesfm3/transformer_test.py
+++ b/src/timesfm3/transformer_test.py
@@ -15,7 +15,7 @@
"""Tests for TimesFM3 PyTorch transformer layers."""
import unittest
-import numpy as np
+
import torch
from . import configs
@@ -24,12 +24,11 @@ from . import util as torch_util
class MaskTest(unittest.TestCase):
-
def test_make_attn_mask_causal(self):
query_len = 4
num_masked = torch.tensor([1])
mask = torch_trans.make_attn_mask(
- query_length=query_len, num_all_masked_kv=num_masked, causal=True
+ query_length=query_len, num_all_masked_kv=num_masked, causal=True
)
# Shape: (1, 1, 4, 4)
self.assertEqual(mask.shape, (1, 1, 4, 4))
@@ -49,7 +48,6 @@ class MaskTest(unittest.TestCase):
class RotaryEmbeddingTest(unittest.TestCase):
-
def test_rope_forward(self):
rope = torch_trans.RotaryPositionalEmbedding(embedding_dims=16)
x = torch.randn(2, 8, 16)
@@ -65,57 +63,53 @@ class RotaryEmbeddingTest(unittest.TestCase):
class MultiHeadAttentionTest(unittest.TestCase):
-
def test_mha_forward_and_cache(self):
mha = torch_trans.MultiHeadAttention(
- num_heads=4,
- in_features=64,
- qk_norm="rms",
- use_bias=False,
- use_sdpa=True,
+ num_heads=4,
+ in_features=64,
+ qk_norm="rms",
+ use_bias=False,
+ use_sdpa=True,
)
b, n, d = 2, 8, 64
x = torch.randn(b, n, d)
patch_mask = torch.zeros(b, n, dtype=torch.bool)
# Full forward pass
- out, cache, mask = mha(x, patch_mask=patch_mask)
+ out, cache, _ = mha(x, patch_mask=patch_mask)
self.assertEqual(out.shape, (b, n, d))
self.assertIsNone(cache)
# Decode mode with cache
decode_cache = torch_util.DecodeCache(
- next_index=torch.zeros(b, dtype=torch.int32),
- num_front_masked=torch.zeros(b, dtype=torch.int32),
- key=torch.zeros(b, 16, 4, 16),
- value=torch.zeros(b, 16, 4, 16),
+ next_index=torch.zeros(b, dtype=torch.int32),
+ num_front_masked=torch.zeros(b, dtype=torch.int32),
+ key=torch.zeros(b, 16, 4, 16),
+ value=torch.zeros(b, 16, 4, 16),
)
q_x = torch.randn(b, 1, d)
q_mask = torch.zeros(b, 1, dtype=torch.bool)
- out_step, updated_cache, _ = mha(
- q_x, patch_mask=q_mask, decode_cache=decode_cache
- )
+ out_step, updated_cache, _ = mha(q_x, patch_mask=q_mask, decode_cache=decode_cache)
self.assertEqual(out_step.shape, (b, 1, d))
self.assertIsNotNone(updated_cache)
self.assertEqual(updated_cache.next_index[0].item(), 1)
class MixingTransformerTest(unittest.TestCase):
-
def test_mixing_transformer_forward(self):
cfg = configs.TransformerConfig(
- model_dims=64,
- hidden_dims=64,
- num_heads=4,
- attention_norm="rms",
- feedforward_norm="rms",
- qk_norm="rms",
- use_rope_seq=True,
- use_rope_var=True,
- use_bias=False,
- ff_activation="relu",
- deterministic=True,
- use_sdpa=True,
+ model_dims=64,
+ hidden_dims=64,
+ num_heads=4,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=True,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ use_sdpa=True,
)
layer = torch_trans.MixingTransformer(config=cfg, use_variate_attention=True)
b, v, n, d = 2, 3, 8, 64
@@ -128,35 +122,34 @@ class MixingTransformerTest(unittest.TestCase):
class StackedMixingTransformerTest(unittest.TestCase):
-
def test_stacked_transformer(self):
sub_cfg = configs.TransformerConfig(
- model_dims=64,
- hidden_dims=64,
- num_heads=4,
- attention_norm="rms",
- feedforward_norm="rms",
- qk_norm="rms",
- use_rope_seq=True,
- use_rope_var=True,
- use_bias=False,
- ff_activation="relu",
- deterministic=True,
- use_sdpa=True,
+ model_dims=64,
+ hidden_dims=64,
+ num_heads=4,
+ attention_norm="rms",
+ feedforward_norm="rms",
+ qk_norm="rms",
+ use_rope_seq=True,
+ use_rope_var=True,
+ use_bias=False,
+ ff_activation="relu",
+ deterministic=True,
+ use_sdpa=True,
)
stack_cfg = configs.StackedTransformersConfig(
- num_layers=2,
- use_remat=False,
- transformer=sub_cfg,
+ num_layers=2,
+ use_remat=False,
+ transformer=sub_cfg,
)
stack = torch_trans.StackedMixingTransformer(
- config=stack_cfg, use_variate_attention=True
+ config=stack_cfg, use_variate_attention=True
)
b, v, n, d = 2, 2, 4, 64
inputs = torch.randn(b, v, n, d)
patch_mask = torch.zeros(b, v, n, dtype=torch.bool)
- out, caches, _ = stack(inputs, patch_mask=patch_mask)
+ out, _, _ = stack(inputs, patch_mask=patch_mask)
self.assertEqual(out.shape, (b, v, n, d))
diff --git a/src/timesfm3/util.py b/src/timesfm3/util.py
index ba6347b..5c303fe 100644
--- a/src/timesfm3/util.py
+++ b/src/timesfm3/util.py
@@ -4,12 +4,11 @@ from __future__ import annotations
import dataclasses
import os
-import tempfile
-from typing import Callable
+from collections.abc import Callable
-from safetensors import torch as safetensors_torch
import torch
import torch.nn.functional as F
+from safetensors import torch as safetensors_torch
_TOLERANCE = 1e-6
@@ -39,15 +38,15 @@ class DecodeCache:
@classmethod
def init_decode_cache(
- cls,
- num_layers: int,
- batch_size: int,
- num_variates: int,
- num_total_input_patches: int,
- num_heads: int,
- head_dim: int,
- device: torch.device | None = None,
- ) -> list["DecodeCache"]:
+ cls,
+ num_layers: int,
+ batch_size: int,
+ num_variates: int,
+ num_total_input_patches: int,
+ num_heads: int,
+ head_dim: int,
+ device: torch.device | None = None,
+ ) -> list[DecodeCache]:
"""Initializes a list of decode caches for stacked layers.
Args:
@@ -64,38 +63,34 @@ class DecodeCache:
"""
leading_size = batch_size * num_variates
return [
- cls(
- next_index=torch.zeros(
- leading_size, dtype=torch.int32, device=device
- ),
- num_front_masked=torch.zeros(
- leading_size, dtype=torch.int32, device=device
- ),
- key=torch.zeros(
- leading_size,
- num_total_input_patches,
- num_heads,
- head_dim,
- device=device,
- ),
- value=torch.zeros(
- leading_size,
- num_total_input_patches,
- num_heads,
- head_dim,
- device=device,
- ),
- )
- for _ in range(num_layers)
+ cls(
+ next_index=torch.zeros(leading_size, dtype=torch.int32, device=device),
+ num_front_masked=torch.zeros(leading_size, dtype=torch.int32, device=device),
+ key=torch.zeros(
+ leading_size,
+ num_total_input_patches,
+ num_heads,
+ head_dim,
+ device=device,
+ ),
+ value=torch.zeros(
+ leading_size,
+ num_total_input_patches,
+ num_heads,
+ head_dim,
+ device=device,
+ ),
+ )
+ for _ in range(num_layers)
]
def update_running_stats(
- n: torch.Tensor,
- mu: torch.Tensor,
- sigma: torch.Tensor,
- x: torch.Tensor,
- mask: torch.Tensor,
+ n: torch.Tensor,
+ mu: torch.Tensor,
+ sigma: torch.Tensor,
+ x: torch.Tensor,
+ mask: torch.Tensor,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Updates running stats with a new patch of data.
@@ -120,45 +115,43 @@ def update_running_stats(
# std of valid elements in patch
x_diff_sq = torch.where(
- is_legit, (x - inc_mu.unsqueeze(-1)) ** 2, torch.zeros_like(x)
+ is_legit, (x - inc_mu.unsqueeze(-1)) ** 2, torch.zeros_like(x)
)
inc_var = torch.where(
- inc_n == 0,
- torch.zeros_like(inc_sum),
- x_diff_sq.sum(dim=-1) / inc_n,
+ inc_n == 0,
+ torch.zeros_like(inc_sum),
+ x_diff_sq.sum(dim=-1) / inc_n,
)
inc_sigma = torch.sqrt(inc_var)
new_n = n + inc_n
new_mu = torch.where(
- new_n == 0,
- torch.zeros_like(mu),
- (n * mu + inc_mu * inc_n) / new_n,
+ new_n == 0,
+ torch.zeros_like(mu),
+ (n * mu + inc_mu * inc_n) / new_n,
)
new_sigma = torch.sqrt(
- torch.where(
- new_n == 0,
- torch.zeros_like(sigma),
- (
- n * sigma * sigma
- + inc_n * inc_sigma * inc_sigma
- + n * (mu - new_mu) * (mu - new_mu)
- + inc_n * (inc_mu - new_mu) * (inc_mu - new_mu)
- )
- / new_n,
+ torch.where(
+ new_n == 0,
+ torch.zeros_like(sigma),
+ (
+ n * sigma * sigma
+ + inc_n * inc_sigma * inc_sigma
+ + n * (mu - new_mu) * (mu - new_mu)
+ + inc_n * (inc_mu - new_mu) * (inc_mu - new_mu)
)
+ / new_n,
+ )
)
return new_n, new_mu, new_sigma
def get_running_stats(
- values: torch.Tensor,
- masks: torch.Tensor,
- *,
- segment_ids: torch.Tensor | None = None,
- initial_stats: (
- tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None
- ) = None,
+ values: torch.Tensor,
+ masks: torch.Tensor,
+ *,
+ segment_ids: torch.Tensor | None = None,
+ initial_stats: (tuple[torch.Tensor, torch.Tensor, torch.Tensor] | None) = None,
) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
"""Computes cumulative running statistics patch-by-patch.
@@ -204,24 +197,24 @@ def get_running_stats(
cur_sigma = torch.where(reset, init_sigma, cur_sigma)
cur_n, cur_mu, cur_sigma = update_running_stats(
- cur_n, cur_mu, cur_sigma, values[:, :, i, :], masks[:, :, i, :]
+ cur_n, cur_mu, cur_sigma, values[:, :, i, :], masks[:, :, i, :]
)
all_n.append(cur_n)
all_mu.append(cur_mu)
all_sigma.append(cur_sigma)
return (
- torch.stack(all_n, dim=2),
- torch.stack(all_mu, dim=2),
- torch.stack(all_sigma, dim=2),
+ torch.stack(all_n, dim=2),
+ torch.stack(all_mu, dim=2),
+ torch.stack(all_sigma, dim=2),
)
def revin(
- x: torch.Tensor,
- mu: torch.Tensor,
- sigma: torch.Tensor,
- reverse: bool = False,
+ x: torch.Tensor,
+ mu: torch.Tensor,
+ sigma: torch.Tensor,
+ reverse: bool = False,
) -> torch.Tensor:
"""Reversible per-instance normalization.
@@ -251,7 +244,7 @@ def revin(
def get_output_patch_via_roll(
- x: torch.Tensor, rolls: int
+ x: torch.Tensor, rolls: int
) -> tuple[torch.Tensor, torch.Tensor]:
"""Creates labels of output_patch length by rolling the patched inputs.
@@ -274,7 +267,7 @@ def get_output_patch_via_roll(
for i in range(rolls):
rolling_mat[:, :, :, i + 1, :] = torch.roll(
- rolling_mat[:, :, :, i, :], shifts=-1, dims=2
+ rolling_mat[:, :, :, i, :], shifts=-1, dims=2
)
# Take [1:] along the roll axis and flatten
@@ -290,30 +283,30 @@ def get_output_patch_via_roll(
_ACTIVATIONS: dict[str, Callable[[torch.Tensor], torch.Tensor]] = {
- "relu": F.relu,
- "swish": F.silu,
- "silu": F.silu,
- "swiglu": F.silu,
- "none": lambda x: x,
+ "relu": F.relu,
+ "swish": F.silu,
+ "silu": F.silu,
+ "swiglu": F.silu,
+ "none": lambda x: x,
}
def get_activation_fn(
- activation_name: str,
+ activation_name: str,
) -> Callable[[torch.Tensor], torch.Tensor]:
"""Returns the activation function for the given name."""
try:
return _ACTIVATIONS[activation_name]
except KeyError:
raise ValueError(
- f"Activation: {activation_name} not supported. Supported "
- f"activations: {list(_ACTIVATIONS.keys())}"
+ f"Activation: {activation_name} not supported. Supported "
+ f"activations: {list(_ACTIVATIONS.keys())}"
) from None
def load_safetensors(
- pytorch_safetensors_path: str,
- device: str | torch.device = "cpu",
+ pytorch_safetensors_path: str,
+ device: str | torch.device = "cpu",
) -> dict[str, torch.Tensor]:
"""Loads a PyTorch state dict from a safetensors file.
@@ -329,8 +322,8 @@ def load_safetensors(
def stitch_patches(
- patch_preds: torch.Tensor,
- patch_len: int,
+ patch_preds: torch.Tensor,
+ patch_len: int,
) -> torch.Tensor:
"""Stitches overlapping patch predictions.
@@ -354,7 +347,7 @@ def stitch_patches(
return patch_preds[:, :, 0, :, :]
stitch_weights = torch.linspace(
- 1.0, 0.0, overlap, device=patch_preds.device, dtype=patch_preds.dtype
+ 1.0, 0.0, overlap, device=patch_preds.device, dtype=patch_preds.dtype
)
stitch_weights = stitch_weights[None, None, None, :, None]
@@ -367,8 +360,7 @@ def stitch_patches(
next_overlaps = next_patches[:, :, :, :overlap, :]
stitched_overlaps = (
- stitch_weights * prev_overlaps
- + (1.0 - stitch_weights) * next_overlaps
+ stitch_weights * prev_overlaps + (1.0 - stitch_weights) * next_overlaps
)
middles = next_patches[:, :, :, overlap:patch_len, :]
@@ -380,4 +372,3 @@ def stitch_patches(
tail = patch_preds[:, :, -1, patch_len:, :]
return torch.cat([first_chunk, mid, tail], dim=2)
-
diff --git a/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb b/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb
index 7fb5851..c9bf45a 100644
--- a/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb
+++ b/timesfm-forecasting/notebooks/promotion_and_synthetic_parity_demo.ipynb
@@ -1,273 +1,270 @@
{
- "cells": [
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "# TimesFM 3.0 PyTorch Inference Demo\n",
- "\n",
- "This interactive notebook demonstrates zero-shot time series forecasting using **TimesFM 3.0** (PyTorch).\n",
- "\n",
- "### Key Features Highlighted:\n",
- "1. **Promotion Example with Dynamic Covariates**: Context length = 2048, Forecast Horizon = 512 with dynamic past-future promotion indicator.\n",
- "2. **Synthetic Curves (Horizon > 64)**: Evaluating long horizons across sine waves and linear trend extrapolation.\n",
- "3. **GPU Acceleration**: Automatic CUDA inference with quantile uncertainty intervals."
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 1. Setup and Imports"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "import os\n",
- "import numpy as np\n",
- "import matplotlib.pyplot as plt\n",
- "import torch\n",
- "\n",
- "from timesfm3 import TimesFM3Forecaster, ModelConfig\n",
- "\n",
- "# Set random seeds for reproducibility\n",
- "np.random.seed(42)\n",
- "torch.manual_seed(42)\n",
- "\n",
- "print(f\"PyTorch version: {torch.__version__}\")\n",
- "print(f\"CUDA available: {torch.cuda.is_available()}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 2. Initialize TimesFM 3.0 Forecaster\n",
- "\n",
- "We instantiate `TimesFM3Forecaster` with the default 1280-dim, 20-layer, 16-head checkpoint."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "config = ModelConfig(\n",
- " checkpoint_path=\"~/models/tfm3/timesfm3_torch_1944000.safetensors\",\n",
- " input_patch_length=32,\n",
- " output_patch_length=64,\n",
- " per_core_batch_size=2,\n",
- " use_sdpa=True,\n",
- ")\n",
- "\n",
- "forecaster = TimesFM3Forecaster(config)\n",
- "print(f\"TimesFM3 forecaster loaded successfully on device: {forecaster.device}\")"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 3. Scenario 1: Promotion Example with Dynamic Covariates\n",
- "\n",
- "- **Context Length**: 2048\n",
- "- **Forecast Horizon**: 512\n",
- "- **Past-Future Covariates**: Binary promotion covariate (0 or 1) spanning the full length (2560 steps).\n",
- "- **Target Multiplier**: Target value is boosted by 5x during active promotions."
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "context_len = 2048\n",
- "horizon_len = 512\n",
- "total_len = context_len + horizon_len\n",
- "time = np.arange(total_len)\n",
- "\n",
- "# Generate promotion indicator\n",
- "promotions = np.zeros(total_len, dtype=np.float32)\n",
- "# Promotion in context\n",
- "promotions[1130:1180] = 1.0\n",
- "# Promotions in forecast horizon\n",
- "promotions[2250:2290] = 1.0\n",
- "promotions[2320:2355] = 1.0\n",
- "\n",
- "pfc_np = np.array([promotions], dtype=np.float32)\n",
- "\n",
- "# Generate base target: periodic sine + noise\n",
- "base_target = 2.0 + np.sin(time / 20.0) + np.random.randn(total_len) * 0.15\n",
- "# Apply 5x promo boost\n",
- "target_full = (base_target * np.where(promotions == 1.0, 5.0, 1.0)).astype(np.float32)\n",
- "\n",
- "context = target_full[:context_len]\n",
- "actual_horizon = target_full[context_len:]\n",
- "\n",
- "print(f\"Context shape: {context.shape}\")\n",
- "print(f\"Horizon length: {horizon_len}\")\n",
- "print(f\"Past-Future Covariates shape: {pfc_np.shape}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Run zero-shot forecast with dynamic past-future covariates\n",
- "torch_out = forecaster.predict(\n",
- " context=context,\n",
- " horizon=horizon_len,\n",
- " past_future_covariates=pfc_np,\n",
- " return_quantiles=True,\n",
- ")\n",
- "\n",
- "print(\"Forecast generated successfully!\")\n",
- "print(f\"Point forecast shape: {torch_out.forecast.shape}\")\n",
- "print(f\"Quantiles shape: {torch_out.quantiles.shape}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Plot Promotion Example\n",
- "time_context = np.arange(context_len)\n",
- "time_horizon = np.arange(context_len, total_len)\n",
- "\n",
- "fig, axes = plt.subplots(2, 1, figsize=(16, 8), sharex=True)\n",
- "\n",
- "# Subplot 0: Target, Actual Horizon, and PyTorch Forecast\n",
- "axes[0].plot(time_context, context, label=\"Context (Input)\", color=\"black\", alpha=0.7, linewidth=1.2)\n",
- "axes[0].plot(time_horizon, actual_horizon, label=\"Actual Horizon (Ground Truth)\", linestyle=\"--\", color=\"gray\", alpha=0.8)\n",
- "axes[0].plot(time_horizon, torch_out.forecast, label=\"TimesFM3 Median Forecast (p50)\", color=\"#ff7f0e\", linewidth=2.5)\n",
- "if torch_out.quantiles is not None:\n",
- " axes[0].fill_between(\n",
- " time_horizon,\n",
- " torch_out.quantiles[:, 0],\n",
- " torch_out.quantiles[:, -1],\n",
- " color=\"salmon\",\n",
- " alpha=0.3,\n",
- " label=\"p10-p90 Prediction Interval\",\n",
- " )\n",
- "axes[0].set_title(\"TimesFM 3.0 Promotion Example Forecast (Context 2048, Horizon 512)\", fontsize=14)\n",
- "axes[0].set_ylabel(\"Target Value\", fontsize=12)\n",
- "axes[0].legend(loc=\"upper left\")\n",
- "axes[0].grid(True, alpha=0.3)\n",
- "\n",
- "# Subplot 1: Past-Future Promotion Covariate\n",
- "axes[1].plot(time, promotions, label=\"Promotion Indicator (0 or 1)\", color=\"green\", drawstyle=\"steps-mid\", linewidth=1.5)\n",
- "axes[1].fill_between(time, 0, promotions, color=\"green\", alpha=0.2)\n",
- "axes[1].set_title(\"Past-Future Covariate: Promotions Timeline\", fontsize=12)\n",
- "axes[1].set_ylabel(\"Promotion Active\", fontsize=12)\n",
- "axes[1].set_xlabel(\"Time Step (t)\", fontsize=12)\n",
- "axes[1].set_yticks([0, 1])\n",
- "axes[1].legend(loc=\"upper left\")\n",
- "axes[1].grid(True, alpha=0.3)\n",
- "\n",
- "plt.tight_layout()\n",
- "plt.show()"
- ]
- },
- {
- "cell_type": "markdown",
- "metadata": {},
- "source": [
- "## 4. Scenario 2: Long Horizon Synthetic Curves (Horizon 128 > 64)\n",
- "\n",
- "Evaluates rolling patch autoregressive decoding across:\n",
- "1. **Sine Curve** (periodic continuation)\n",
- "2. **Linear Trend** (linear detrending extrapolation)"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "synth_context_len = 256\n",
- "synth_horizon_len = 128\n",
- "synth_total_len = synth_context_len + synth_horizon_len\n",
- "t_synth = np.linspace(0, 8 * np.pi, synth_total_len, dtype=np.float32)\n",
- "\n",
- "# Series 0: Sine wave\n",
- "s_sine = np.sin(t_synth)\n",
- "# Series 1: Linear trend line\n",
- "s_line = np.arange(synth_total_len, dtype=np.float32) * 0.05 + 2.0\n",
- "\n",
- "contexts = [s_sine[:synth_context_len], s_line[:synth_context_len]]\n",
- "ground_truths = [s_sine[synth_context_len:], s_line[synth_context_len:]]\n",
- "\n",
- "# Batch predict\n",
- "synth_outputs = list(forecaster.predict_batch(contexts, horizon=synth_horizon_len))\n",
- "\n",
- "# Compute metrics\n",
- "sine_mae = np.mean(np.abs(synth_outputs[0].forecast - ground_truths[0]))\n",
- "line_mae = np.mean(np.abs(synth_outputs[1].forecast - ground_truths[1]))\n",
- "\n",
- "print(f\"Sine Curve Mean Absolute Error: {sine_mae:.6f}\")\n",
- "print(f\"Linear Trend Mean Absolute Error: {line_mae:.6f}\")"
- ]
- },
- {
- "cell_type": "code",
- "execution_count": null,
- "metadata": {},
- "outputs": [],
- "source": [
- "# Plot Synthetic Curves\n",
- "fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n",
- "t_ctx = np.arange(synth_context_len)\n",
- "t_hor = np.arange(synth_context_len, synth_total_len)\n",
- "titles = [\n",
- " f\"Sine Curve Forecast (MAE: {sine_mae:.6f})\",\n",
- " f\"Linear Trend Forecast (MAE: {line_mae:.6f})\",\n",
- "]\n",
- "\n",
- "for i, ax in enumerate(axes):\n",
- " ax.plot(t_ctx, contexts[i], label=\"Context (Input)\", color=\"black\", linewidth=1.5)\n",
- " ax.plot(t_hor, ground_truths[i], label=\"Ground Truth\", color=\"gray\", linestyle=\"--\", alpha=0.7)\n",
- " ax.plot(t_hor, synth_outputs[i].forecast, label=\"TimesFM3 Forecast\", color=\"#ff7f0e\", linewidth=2.5)\n",
- " ax.set_title(titles[i], fontsize=13)\n",
- " ax.set_ylabel(\"Value\", fontsize=11)\n",
- " ax.legend(loc=\"upper left\")\n",
- " ax.grid(True, alpha=0.3)\n",
- "\n",
- "axes[1].set_xlabel(\"Time Step\", fontsize=12)\n",
- "plt.tight_layout()\n",
- "plt.show()"
- ]
- }
- ],
- "metadata": {
- "kernelspec": {
- "display_name": ".venv",
- "language": "python",
- "name": "python3"
- },
- "language_info": {
- "codemirror_mode": {
- "name": "ipython",
- "version": 3
- },
- "file_extension": ".py",
- "mimetype": "text/x-python",
- "name": "python",
- "nbconvert_exporter": "python",
- "pygments_lexer": "ipython3",
- "version": "3.12.3"
- }
+ "cells": [
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "# TimesFM 3.0 PyTorch Inference Demo\n",
+ "\n",
+ "This interactive notebook demonstrates zero-shot time series forecasting using **TimesFM 3.0** (PyTorch).\n",
+ "\n",
+ "### Key Features Highlighted:\n",
+ "1. **Promotion Example with Dynamic Covariates**: Context length = 2048, Forecast Horizon = 512 with dynamic past-future promotion indicator.\n",
+ "2. **Synthetic Curves (Horizon > 64)**: Evaluating long horizons across sine waves and linear trend extrapolation.\n",
+ "3. **GPU Acceleration**: Automatic CUDA inference with quantile uncertainty intervals."
+ ]
},
- "nbformat": 4,
- "nbformat_minor": 4
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 1. Setup and Imports"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "import os\n",
+ "import numpy as np\n",
+ "import matplotlib.pyplot as plt\n",
+ "import torch\n",
+ "\n",
+ "from timesfm3 import TimesFM3Forecaster\n",
+ "\n",
+ "# Set random seeds for reproducibility\n",
+ "np.random.seed(42)\n",
+ "torch.manual_seed(42)\n",
+ "\n",
+ "print(f\"PyTorch version: {torch.__version__}\")\n",
+ "print(f\"CUDA available: {torch.cuda.is_available()}\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 2. Initialize TimesFM 3.0 Forecaster\n",
+ "\n",
+ "We instantiate `TimesFM3Forecaster` from the default Hugging Face pretrained checkpoint (`google/timesfm-3.0-pytorch`).\n"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Load TimesFM 3.0 directly from Hugging Face Hub (default: \"google/timesfm-3.0-pytorch\")\n",
+ "forecaster = TimesFM3Forecaster.from_pretrained(\n",
+ " \"google/timesfm-3.0-pytorch\",\n",
+ " per_core_batch_size=4,\n",
+ ")\n",
+ "\n",
+ "print(f\"TimesFM3 forecaster loaded successfully on device: {forecaster.device}\")\n"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 3. Scenario 1: Promotion Example with Dynamic Covariates\n",
+ "\n",
+ "- **Context Length**: 2048\n",
+ "- **Forecast Horizon**: 512\n",
+ "- **Past-Future Covariates**: Binary promotion covariate (0 or 1) spanning the full length (2560 steps).\n",
+ "- **Target Multiplier**: Target value is boosted by 5x during active promotions."
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "context_len = 2048\n",
+ "horizon_len = 512\n",
+ "total_len = context_len + horizon_len\n",
+ "time = np.arange(total_len)\n",
+ "\n",
+ "# Generate promotion indicator\n",
+ "promotions = np.zeros(total_len, dtype=np.float32)\n",
+ "# Promotion in context\n",
+ "promotions[1130:1180] = 1.0\n",
+ "# Promotions in forecast horizon\n",
+ "promotions[2250:2290] = 1.0\n",
+ "promotions[2320:2355] = 1.0\n",
+ "\n",
+ "pfc_np = np.array([promotions], dtype=np.float32)\n",
+ "\n",
+ "# Generate base target: periodic sine + noise\n",
+ "base_target = 2.0 + np.sin(time / 20.0) + np.random.randn(total_len) * 0.15\n",
+ "# Apply 5x promo boost\n",
+ "target_full = (base_target * np.where(promotions == 1.0, 5.0, 1.0)).astype(np.float32)\n",
+ "\n",
+ "context = target_full[:context_len]\n",
+ "actual_horizon = target_full[context_len:]\n",
+ "\n",
+ "print(f\"Context shape: {context.shape}\")\n",
+ "print(f\"Horizon length: {horizon_len}\")\n",
+ "print(f\"Past-Future Covariates shape: {pfc_np.shape}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Run zero-shot forecast with dynamic past-future covariates\n",
+ "torch_out = forecaster.predict(\n",
+ " context=context,\n",
+ " horizon=horizon_len,\n",
+ " past_future_covariates=pfc_np,\n",
+ " return_quantiles=True,\n",
+ ")\n",
+ "\n",
+ "print(\"Forecast generated successfully!\")\n",
+ "print(f\"Point forecast shape: {torch_out.forecast.shape}\")\n",
+ "print(f\"Quantiles shape: {torch_out.quantiles.shape}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Plot Promotion Example\n",
+ "time_context = np.arange(context_len)\n",
+ "time_horizon = np.arange(context_len, total_len)\n",
+ "\n",
+ "fig, axes = plt.subplots(2, 1, figsize=(16, 8), sharex=True)\n",
+ "\n",
+ "# Subplot 0: Target, Actual Horizon, and PyTorch Forecast\n",
+ "axes[0].plot(time_context, context, label=\"Context (Input)\", color=\"black\", alpha=0.7, linewidth=1.2)\n",
+ "axes[0].plot(time_horizon, actual_horizon, label=\"Actual Horizon (Ground Truth)\", linestyle=\"--\", color=\"gray\", alpha=0.8)\n",
+ "axes[0].plot(time_horizon, torch_out.forecast, label=\"TimesFM3 Median Forecast (p50)\", color=\"#ff7f0e\", linewidth=2.5)\n",
+ "if torch_out.quantiles is not None:\n",
+ " axes[0].fill_between(\n",
+ " time_horizon,\n",
+ " torch_out.quantiles[:, 0],\n",
+ " torch_out.quantiles[:, -1],\n",
+ " color=\"salmon\",\n",
+ " alpha=0.3,\n",
+ " label=\"p10-p90 Prediction Interval\",\n",
+ " )\n",
+ "axes[0].set_title(\"TimesFM 3.0 Promotion Example Forecast (Context 2048, Horizon 512)\", fontsize=14)\n",
+ "axes[0].set_ylabel(\"Target Value\", fontsize=12)\n",
+ "axes[0].legend(loc=\"upper left\")\n",
+ "axes[0].grid(True, alpha=0.3)\n",
+ "\n",
+ "# Subplot 1: Past-Future Promotion Covariate\n",
+ "axes[1].plot(time, promotions, label=\"Promotion Indicator (0 or 1)\", color=\"green\", drawstyle=\"steps-mid\", linewidth=1.5)\n",
+ "axes[1].fill_between(time, 0, promotions, color=\"green\", alpha=0.2)\n",
+ "axes[1].set_title(\"Past-Future Covariate: Promotions Timeline\", fontsize=12)\n",
+ "axes[1].set_ylabel(\"Promotion Active\", fontsize=12)\n",
+ "axes[1].set_xlabel(\"Time Step (t)\", fontsize=12)\n",
+ "axes[1].set_yticks([0, 1])\n",
+ "axes[1].legend(loc=\"upper left\")\n",
+ "axes[1].grid(True, alpha=0.3)\n",
+ "\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ },
+ {
+ "cell_type": "markdown",
+ "metadata": {},
+ "source": [
+ "## 4. Scenario 2: Long Horizon Synthetic Curves (Horizon 128 > 64)\n",
+ "\n",
+ "Evaluates rolling patch autoregressive decoding across:\n",
+ "1. **Sine Curve** (periodic continuation)\n",
+ "2. **Linear Trend** (linear detrending extrapolation)"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "synth_context_len = 256\n",
+ "synth_horizon_len = 128\n",
+ "synth_total_len = synth_context_len + synth_horizon_len\n",
+ "t_synth = np.linspace(0, 8 * np.pi, synth_total_len, dtype=np.float32)\n",
+ "\n",
+ "# Series 0: Sine wave\n",
+ "s_sine = np.sin(t_synth)\n",
+ "# Series 1: Linear trend line\n",
+ "s_line = np.arange(synth_total_len, dtype=np.float32) * 0.05 + 2.0\n",
+ "\n",
+ "contexts = [s_sine[:synth_context_len], s_line[:synth_context_len]]\n",
+ "ground_truths = [s_sine[synth_context_len:], s_line[synth_context_len:]]\n",
+ "\n",
+ "# Batch predict\n",
+ "synth_outputs = list(forecaster.predict_batch(contexts, horizon=synth_horizon_len))\n",
+ "\n",
+ "# Compute metrics\n",
+ "sine_mae = np.mean(np.abs(synth_outputs[0].forecast - ground_truths[0]))\n",
+ "line_mae = np.mean(np.abs(synth_outputs[1].forecast - ground_truths[1]))\n",
+ "\n",
+ "print(f\"Sine Curve Mean Absolute Error: {sine_mae:.6f}\")\n",
+ "print(f\"Linear Trend Mean Absolute Error: {line_mae:.6f}\")"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": null,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "# Plot Synthetic Curves\n",
+ "fig, axes = plt.subplots(2, 1, figsize=(14, 8), sharex=True)\n",
+ "t_ctx = np.arange(synth_context_len)\n",
+ "t_hor = np.arange(synth_context_len, synth_total_len)\n",
+ "titles = [\n",
+ " f\"Sine Curve Forecast (MAE: {sine_mae:.6f})\",\n",
+ " f\"Linear Trend Forecast (MAE: {line_mae:.6f})\",\n",
+ "]\n",
+ "\n",
+ "for i, ax in enumerate(axes):\n",
+ " ax.plot(t_ctx, contexts[i], label=\"Context (Input)\", color=\"black\", linewidth=1.5)\n",
+ " ax.plot(t_hor, ground_truths[i], label=\"Ground Truth\", color=\"gray\", linestyle=\"--\", alpha=0.7)\n",
+ " ax.plot(t_hor, synth_outputs[i].forecast, label=\"TimesFM3 Forecast\", color=\"#ff7f0e\", linewidth=2.5)\n",
+ " ax.set_title(titles[i], fontsize=13)\n",
+ " ax.set_ylabel(\"Value\", fontsize=11)\n",
+ " ax.legend(loc=\"upper left\")\n",
+ " ax.grid(True, alpha=0.3)\n",
+ "\n",
+ "axes[1].set_xlabel(\"Time Step\", fontsize=12)\n",
+ "plt.tight_layout()\n",
+ "plt.show()"
+ ]
+ }
+ ],
+ "metadata": {
+ "kernelspec": {
+ "display_name": ".venv",
+ "language": "python",
+ "name": "python3"
+ },
+ "language_info": {
+ "codemirror_mode": {
+ "name": "ipython",
+ "version": 3
+ },
+ "file_extension": ".py",
+ "mimetype": "text/x-python",
+ "name": "python",
+ "nbconvert_exporter": "python",
+ "pygments_lexer": "ipython3",
+ "version": "3.12.3"
+ }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 4
}