from __future__ import annotations import torch import torch.nn as nn import torch.nn.functional as F from layers.layers import MixerLayer, TimeBatchNorm2d, feature_to_time, time_to_feature class TSMixer(nn.Module): """TSMixer model for time series forecasting. This model uses a series of mixer layers to process time series data, followed by a linear transformation to project the output to the desired prediction length. Attributes: mixer_layers: Sequential container of mixer layers. temporal_projection: Linear layer for temporal projection. Args: sequence_length: Length of the input time series sequence. prediction_length: Desired length of the output prediction sequence. input_channels: Number of input channels. output_channels: Number of output channels. Defaults to None. activation_fn: Activation function to use. Defaults to "relu". num_blocks: Number of mixer blocks. Defaults to 2. dropout_rate: Dropout rate for regularization. Defaults to 0.1. ff_dim: Dimension of feedforward network inside mixer layer. Defaults to 64. normalize_before: Whether to apply layer normalization before or after mixer layer. norm_type: Type of normalization to use. "batch" or "layer". Defaults to "batch". """ def __init__( self, sequence_length: int, prediction_length: int, input_channels: int, output_channels: int = 1, activation_fn: str = "relu", num_blocks: int = 2, dropout_rate: float = 0.1, ff_dim: int = 64, normalize_before: bool = True, norm_type: str = "batch", ): super().__init__() # Transform activation_fn to callable activation_fn = getattr(F, activation_fn) # Transform norm_type to callable assert norm_type in { "batch", "layer", }, f"Invalid norm_type: {norm_type}, must be one of batch, layer." # norm_type = TimeBatchNorm2d if norm_type == "batch" else nn.LayerNorm norm_type = TimeBatchNorm2d if input_channels == 1 else nn.LayerNorm # Build mixer layers self.mixer_layers = self._build_mixer( num_blocks, input_channels, output_channels, ff_dim=ff_dim, activation_fn=activation_fn, dropout_rate=dropout_rate, sequence_length=sequence_length, normalize_before=normalize_before, norm_type=norm_type, ) # Temporal projection layer self.temporal_projection = nn.Linear(sequence_length, prediction_length) self.line = nn.Linear(input_channels,output_channels) def _build_mixer( self, num_blocks: int, input_channels: int, output_channels: int, **kwargs ): """Build the mixer blocks for the model. Args: num_blocks (int): Number of mixer blocks to be built. input_channels (int): Number of input channels for the first block. output_channels (int): Number of output channels for the last block. **kwargs: Additional keyword arguments for mixer layer configuration. Returns: nn.Sequential: Sequential container of mixer layers. """ channels = [input_channels] * (num_blocks - 1) + [output_channels] return nn.Sequential( *[ MixerLayer(input_channels=in_ch, output_channels=out_ch, **kwargs) for in_ch, out_ch in zip(channels[:-1], channels[1:]) ] ) def forward(self, x_hist: torch.Tensor) -> torch.Tensor: """Forward pass of the TSMixer model. Args: x_hist (torch.Tensor): Input time series tensor. Returns: torch.Tensor: The output tensor after processing by the model. """ x = self.mixer_layers(x_hist) x_temp = feature_to_time(x) x_temp = self.temporal_projection(x_temp) x = time_to_feature(x_temp) x = self.line(x) B , L , C = x.shape x = x.reshape(B,-1) return x if __name__ == "__main__": m = TSMixer(288, 288, 17, output_channels=1) x = torch.randn(32, 288, 17) y = m(x) print(y.shape)