Swish and SiLU

You will learn

How x times sigmoid(beta x) gates itself, and how one parameter moves it from a smooth curve to ReLU.

Try it

Hold x at -1.28 and raise beta from 1 to 8; then find the beta at which the dip is gone at this scale.

specs/ml/activation/silu_swish_activation.t27

// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// SiLU derivative | φ² + 1/φ² = 3 | TRINITY

// #8980: forward_batch returned its input unchanged, beta was never read, and the
// invariants asserted `true` over an undeclared `valid_input`. SiLU is now
// x * sigmoid(beta * x) (Swish; beta = 1 is SiLU) on scalars and in place on slices.

module SiluSwish;
    use base::types;
    use math::constants;

    // ═══════════════════════════════════════════════════════════
    // 1. Constants
    // ═══════════════════════════════════════════════════════════

    const DEFAULT_BETA : f32 = 1.0;

    // ═══════════════════════════════════════════════════════════
    // 2. Types
    // ═══════════════════════════════════════════════════════════

    pub const SiLUConfig = struct {
        beta : f32,
    };

    // ═══════════════════════════════════════════════════════════
    // 3. Core Functions
    // ═══════════════════════════════════════════════════════════

    // sigmoid(z) = 1 / (1 + exp(-z))
    fn sigmoid(z: f32) -> f32 {
        return 1.0 / (1.0 + exp(-z));
    }

    // Swish(x) = x * sigmoid(beta * x); beta = 1 is SiLU
    fn forward(x: f32, beta: f32) -> f32 {
        return x * sigmoid(beta * x);
    }

    // forward on each element IN PLACE; returns the same slice
    fn forward_batch(input: []f32, beta: f32) -> []f32 {
        var i : usize = 0;
        while (i < input.len) {
            input[i] = forward(input[i], beta);
            i = i + 1;
        }
        return input;
    }

    // d/dx x sigmoid(beta x) = s + beta x s (1 - s), s = sigmoid(beta x)
    fn derivative(x: f32, beta: f32) -> f32 {
        const s = sigmoid(beta * x);
        return s + beta * x * s * (1.0 - s);
    }

    fn batch_at(a: f32, b: f32, c: f32, index: usize) -> f32 {
        var input : [3]f32 = [a, b, c];
        const output = forward_batch(&input, DEFAULT_BETA);
        return output[index];
    }

    fn valid_input() -> f32 {
        return 2.0;
    }

    // ═══════════════════════════════════════════════════════════
    // TDD: Tests (from .tri behaviors)
    // ═══════════════════════════════════════════════════════════

    test "forward_basic_case"
        given input = 0.0, beta = 1.0
        when result = forward(input, beta)
        then result == 0.0  // SiLU(0) = 0 * sigmoid(0) = 0

    test "forward_positive_case"
        given input = 1.0, beta = 1.0
        when result = forward(input, beta)
        then result > 0.0  // SiLU(1) = 1 * sigmoid(1) > 0

    test "forward_negative_case"
        given input = -1.0, beta = 1.0
        when result = forward(input, beta)
        then result < 0.0  // SiLU(-1) = -1 * sigmoid(-1) < 0

    test "forward_batch_basic_case"
        given beta = 1.0
        when result = batch_at(1.0, 2.0, 3.0, 2)
        then @abs(result - forward(3.0, beta)) < 0.0000001 and @abs(batch_at(1.0, 2.0, 3.0, 0) - 0.7310586) < 0.000001

    test "forward_batch_zero_case"
        given beta = 1.0
        when result = batch_at(0.0, 0.0, 0.0, 0)
        then result == 0.0 and batch_at(0.0, 0.0, 0.0, 1) == 0.0 and batch_at(0.0, 0.0, 0.0, 2) == 0.0  // SiLU(0) = 0

    test "derivative_basic_case"
        given input = 0.0, beta = 1.0
        when result = derivative(input, beta)
        then result > 0.0  // SiLU'(0) = sigmoid(0) + 0 * sigmoid(0) * (1 - sigmoid(0)) = sigmoid(0) > 0

    test "derivative_positive_case"
        given input = 1.0, beta = 1.0
        when result = derivative(input, beta)
        then result > 0.0  // SiLU derivative should be positive for positive inputs

    test "derivative_negative_case"
        given input = -1.0, beta = 1.0
        when result = derivative(input, beta)
        then result > 0.0  // SiLU derivative should be positive for negative inputs too

    // ═══════════════════════════════════════════════════════════
    // TDD: Invariants (from .tri constraints)
    // ═══════════════════════════════════════════════════════════

    // beta > 0
    invariant silu_swish_constraint_0
        assert DEFAULT_BETA > 0.0

    // name: silu_formula -- x * sigmoid(x), so SiLU(x) - SiLU(-x) = x
    invariant silu_swish_constraint_1
        assert @abs(forward(valid_input(), DEFAULT_BETA) - forward(-valid_input(), DEFAULT_BETA) - valid_input()) < 0.000001

    // name: derivative -- sigmoid(0) = 0.5 at the origin
    invariant silu_swish_constraint_2
        assert derivative(0.0, DEFAULT_BETA) == 0.5

    // "Elfwing et al. (2017) - Sigmoid-Weighted Linear Units for Neural Network Function Approximation":
    // SiLU dips below zero left of the origin
    invariant silu_swish_constraint_3
        assert forward(-valid_input(), DEFAULT_BETA) < 0.0

    // "Ramachandran et al. (2017) - Searching for Activation Functions (Swish)": a large beta
    // approaches ReLU
    invariant silu_swish_constraint_4
        assert @abs(forward(valid_input(), 50.0) - valid_input()) < 0.000001

Open the lesson's spec in the player ↗

All lessons

• Module 1 · One neuron

  A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.

  1. 01 ReLU: keep the positive part
  2. 02 The sigmoid: a number into a probability
  3. 03 tanh and saturation

• Module 2 · New bends

  The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.

  1. 04 GELU: a dip below zero
  2. 05 Swish and SiLU
  3. 06 ELU: a floor, not a cliff

• Module 3 · A layer

  What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.

  1. 07 Softmax and temperature
  2. 08 Embeddings
  3. 09 Dropout

• Module 4 · Pictures

  Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.

  1. 10 Patches of a picture
  2. 11 Max pooling
  3. 12 Batch normalisation

• Module 5 · The error

  How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.

  1. 13 Mean squared error
  2. 14 Cross-entropy
  3. 15 The Huber loss

• Module 6 · Downhill

  Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.

  1. 16 Gradient descent
  2. 17 RMSProp
  3. 18 Adam

• Module 7 · Learning

  Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.

  1. 19 Backpropagation
  2. 20 Learning XOR
  3. 21 LAMB and large batches

• Module 8 · Memory and attention

  Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.

  1. 22 A recurrent cell
  2. 23 LSTM gates
  3. 24 Attention

• Module 9 · The transformer and the reward

  The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching.

  1. 25 Positional encoding
  2. 26 An encoder block
  3. 27 PPO and the reward