Skip to content

Looking Ahead: Modeling Future Evolution in Autoregressive Video Generation

Log attention backend selection and FA4 fallback

HELLORPG2bde021e5272
diff --git a/src/utils/attention.py b/src/utils/attention.py
index e2a3421..b556b0e 100644
--- a/src/utils/attention.py
+++ b/src/utils/attention.py
@@ -4,13 +4,18 @@ import warnings
import torch
from transformers.utils.import_utils import is_flash_attn_4_available
+from src.utils.attention_logging import log_attention_backend, warn_flash_attention_fallback
+
FLASH_ATTN_4_AVAILABLE = is_flash_attn_4_available()
if FLASH_ATTN_4_AVAILABLE:
try:
import flash_attn.cute
- except ImportError:
+ except ImportError as exc:
# Installed but not importable, e.g. on Windows, where its quack dependency imports fcntl.
FLASH_ATTN_4_AVAILABLE = False
+ warn_flash_attention_fallback(type(exc).__name__, str(exc))
+
+log_attention_backend(FLASH_ATTN_4_AVAILABLE)
__all__ = [
diff --git a/src/utils/attention_logging.py b/src/utils/attention_logging.py
new file mode 100644
index 0000000..2de5db9
--- /dev/null
+++ b/src/utils/attention_logging.py
@@ -0,0 +1,24 @@
+"""Report attention backend selection without changing backend availability."""
+
+import logging
+from functools import cache, lru_cache
+
+logger = logging.getLogger(__name__)
+
+
+@lru_cache(maxsize=2)
+def log_attention_backend(flash_attn_available: bool) -> None:
+ """Log each selected backend once per process across both attention modules."""
+ backend = "FA4" if flash_attn_available else "PyTorch SDPA"
+ logger.info("[attention] backend=%s", backend)
+
+
+@cache
+def warn_flash_attention_fallback(error_type: str, error_message: str) -> None:
+ """Report each distinct import failure once, without retaining its traceback."""
+ logger.warning(
+ "[attention] FA4 import failed (%s: %s); falling back to PyTorch SDPA. "
+ "Performance and numerical results may differ.",
+ error_type,
+ error_message,
+ )
diff --git a/src/wan/modules/attention.py b/src/wan/modules/attention.py
index bb003b0..73a6802 100644
--- a/src/wan/modules/attention.py
+++ b/src/wan/modules/attention.py
@@ -2,13 +2,19 @@
# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.
import torch
from transformers.utils.import_utils import is_flash_attn_4_available
+
+from src.utils.attention_logging import log_attention_backend, warn_flash_attention_fallback
+
FLASH_ATTN_4_AVAILABLE = is_flash_attn_4_available()
if FLASH_ATTN_4_AVAILABLE:
try:
import flash_attn.cute
- except ImportError:
+ except ImportError as exc:
# Installed but not importable, e.g. on Windows, where its quack dependency imports fcntl.
FLASH_ATTN_4_AVAILABLE = False
+ warn_flash_attention_fallback(type(exc).__name__, str(exc))
+
+log_attention_backend(FLASH_ATTN_4_AVAILABLE)
import warnings
diff --git a/tests/test_attention_logging.py b/tests/test_attention_logging.py
new file mode 100644
index 0000000..773a948
--- /dev/null
+++ b/tests/test_attention_logging.py
@@ -0,0 +1,111 @@
+"""Backend reporting must be visible, deduplicated and independent of RNG state."""
+
+import importlib.util
+import random
+import unittest
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import numpy as np
+import torch
+from transformers.utils import import_utils
+
+from src.utils.attention_logging import (
+ log_attention_backend,
+ warn_flash_attention_fallback,
+)
+
+ROOT = Path(__file__).parent.parent
+ATTENTION_PATHS = ("src/utils/attention.py", "src/wan/modules/attention.py")
+LOGGER = "src.utils.attention_logging"
+
+
+class AttentionLoggingTests(unittest.TestCase):
+ def setUp(self):
+ log_attention_backend.cache_clear()
+ warn_flash_attention_fallback.cache_clear()
+
+ def tearDown(self):
+ log_attention_backend.cache_clear()
+ warn_flash_attention_fallback.cache_clear()
+
+ def import_attention_modules(self, *, available, error=None):
+ """Exercise both real module initializers with controlled FA4 availability."""
+ import builtins
+
+ original_import = builtins.__import__
+
+ def import_with_fa4(name, globals=None, locals=None, fromlist=(), level=0):
+ if name == "flash_attn.cute":
+ if error is not None:
+ raise type(error)(str(error))
+ return SimpleNamespace(cute=SimpleNamespace())
+ return original_import(name, globals, locals, fromlist, level)
+
+ modules = []
+ with (
+ patch.object(import_utils, "is_flash_attn_4_available", return_value=available),
+ patch("builtins.__import__", side_effect=import_with_fa4),
+ ):
+ for index, relative in enumerate(ATTENTION_PATHS):
+ spec = importlib.util.spec_from_file_location(f"attention_logging_test_{index}", ROOT / relative)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ modules.append(module)
+ return modules
+
+ def test_fa4_logs_once_across_both_modules(self):
+ with self.assertLogs(LOGGER, level="INFO") as captured:
+ modules = self.import_attention_modules(available=True)
+ self.assertTrue(all(module.FLASH_ATTN_4_AVAILABLE for module in modules))
+ self.assertEqual([record.getMessage() for record in captured.records], ["[attention] backend=FA4"])
+ self.assertEqual(captured.records[0].levelname, "INFO")
+
+ def test_unavailable_fa4_reports_sdpa_without_warning(self):
+ with self.assertLogs(LOGGER, level="INFO") as captured:
+ modules = self.import_attention_modules(available=False)
+ self.assertFalse(any(module.FLASH_ATTN_4_AVAILABLE for module in modules))
+ self.assertEqual([record.getMessage() for record in captured.records], ["[attention] backend=PyTorch SDPA"])
+ self.assertEqual(captured.records[0].levelname, "INFO")
+
+ def test_import_failures_warn_once_and_preserve_rng(self):
+ for missing in ("fcntl", "triton"):
+ with self.subTest(missing=missing):
+ log_attention_backend.cache_clear()
+ warn_flash_attention_fallback.cache_clear()
+ cuda_states = torch.cuda.get_rng_state_all() if torch.cuda.is_available() else []
+ python_state = random.getstate()
+ numpy_state = np.random.get_state()
+ cpu_state = torch.get_rng_state().clone()
+ with self.assertLogs(LOGGER, level="INFO") as captured:
+ modules = self.import_attention_modules(
+ available=True, error=ModuleNotFoundError(f"No module named '{missing}'")
+ )
+ self.assertFalse(any(module.FLASH_ATTN_4_AVAILABLE for module in modules))
+ warnings = [record for record in captured.records if record.levelname == "WARNING"]
+ self.assertEqual(len(warnings), 1)
+ self.assertIn(f"ModuleNotFoundError: No module named '{missing}'", warnings[0].getMessage())
+ self.assertIn("falling back to PyTorch SDPA", warnings[0].getMessage())
+ self.assertIn("numerical results may differ", warnings[0].getMessage())
+ self.assertEqual(captured.records[-1].getMessage(), "[attention] backend=PyTorch SDPA")
+ self.assertEqual(len(captured.records), 2)
+ self.assertEqual(random.getstate(), python_state)
+ self.assertTrue(torch.equal(torch.get_rng_state(), cpu_state))
+ current_numpy_state = np.random.get_state()
+ self.assertEqual(current_numpy_state[0], numpy_state[0])
+ np.testing.assert_array_equal(current_numpy_state[1], numpy_state[1])
+ self.assertEqual(current_numpy_state[2:], numpy_state[2:])
+ if cuda_states:
+ self.assertTrue(all(
+ torch.equal(before, after)
+ for before, after in zip(cuda_states, torch.cuda.get_rng_state_all(), strict=True)
+ ))
+
+ def test_runtime_errors_still_propagate(self):
+ with self.assertRaisesRegex(RuntimeError, "kernel error"):
+ self.import_attention_modules(available=True, error=RuntimeError("kernel error"))
+
+
+if __name__ == "__main__":
+ unittest.main()