ReLU: keep the positive part

You will learn

What ReLU does to a positive and to a negative input, and why a slope of 0 stops a unit from learning.

Try it

Put x at -2 and read the slope with and without the leak; then find the slope at x = 1.5.

specs/ml/activation/relu_activation.t27

// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// ReLU produces sparse gradients | φ² + 1/φ² = 3 | TRINITY

// #8980: the tests called forward and backward with one or two of their three or
// four arguments, two asserted only `result != undefined`, and the invariants
// asserted `true`. Each test now runs the functions on buffers and checks values.

module Relu;
    use base::types;
    use math::constants;

    // ═══════════════════════════════════════════════════════════
    // 1. Constants
    // ═══════════════════════════════════════════════════════════

    const ZERO : f32 = 0.0;

    // ═══════════════════════════════════════════════════════════
    // 2. Types
    // ═══════════════════════════════════════════════════════════

    pub const ReLUConfig = struct {
        negative_slope : f32,
        inplace : bool,
    };

    // ═══════════════════════════════════════════════════════════
    // 3. Core Functions
    // ═══════════════════════════════════════════════════════════

    // forward(input: []const f32, output: []f32, config: ReLUConfig) → void
    fn forward(input: []const f32, output: []f32, config: ReLUConfig) -> void {
        // Values at or below ZERO take the negative branch, values above ZERO pass through
        // For negative_slope = 0.0, this gives max(0, x) behavior
        var i : usize = 0;
        while (i < input.len) : (i += 1) {
            if (input[i] > ZERO) {
                output[i] = input[i];  // Positive branch: identity
            } else {
                output[i] = input[i] * config.negative_slope;  // Negative branch: scaled
            }
        }
    }

    // backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) → void
    fn backward(grad_output: []const f32, input: []const f32, grad_input: []f32, config: ReLUConfig) -> void {
        // The gradient gate is on the forward input, not grad_output - this makes gradients sparse
        var i : usize = 0;
        while (i < input.len) : (i += 1) {
            if (input[i] > ZERO) {
                grad_input[i] = grad_output[i];  // Positive branch: identity gradient
            } else {
                grad_input[i] = grad_output[i] * config.negative_slope;  // Negative branch: scaled gradient
            }
        }
    }

    // Note: config.inplace is declared but currently unused by these functions.
    // The buffers are separate parameters, so the aliasing case is not handled.

    // ═══════════════════════════════════════════════════════════
    // TDD: Tests (from .tri behaviors)
    // ═══════════════════════════════════════════════════════════

    fn cfg(slope: f32) -> ReLUConfig {
        return ReLUConfig{ .negative_slope = slope, .inplace = false };
    }

    // output[index] of forward over (a, b, c)
    fn forward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
        const input : [3]f32 = [a, b, c];
        var output : [3]f32 = [9.0] ** 3;
        forward(&input, &output, cfg(slope));
        return output[index];
    }

    // grad_input[index] of backward over (a, b, c), every grad_output 1
    fn backward_at(a: f32, b: f32, c: f32, slope: f32, index: usize) -> f32 {
        const grad_output : [3]f32 = [1.0, 1.0, 1.0];
        const input : [3]f32 = [a, b, c];
        var grad_input : [3]f32 = [9.0] ** 3;
        backward(&grad_output, &input, &grad_input, cfg(slope));
        return grad_input[index];
    }

    // The shared fixture: one negative, zero, one positive.
    fn valid_input() -> f32 {
        return -2.0;
    }

    // max(0, x) on (-2, 0, 2)
    test "forward_basic_case"
        given negative = forward_at(-2.0, 0.0, 2.0, 0.0, 0)
        when positive = forward_at(-2.0, 0.0, 2.0, 0.0, 2)
        then negative == 0.0 and forward_at(-2.0, 0.0, 2.0, 0.0, 1) == 0.0 and positive == 2.0

    test "forward_positive_identity"
        given slope = 0.0
        when first = forward_at(1.0, 2.0, 3.0, slope, 0)
        then first == 1.0 and forward_at(1.0, 2.0, 3.0, slope, 1) == 2.0 and forward_at(1.0, 2.0, 3.0, slope, 2) == 3.0

    test "forward_negative_scaling"
        given slope = 0.5
        when first = forward_at(-1.0, -2.0, -3.0, slope, 0)
        then first == -0.5 and forward_at(-1.0, -2.0, -3.0, slope, 1) == -1.0 and forward_at(-1.0, -2.0, -3.0, slope, 2) == -1.5

    test "forward_zero_negative_slope"
        given slope = 0.0
        when first = forward_at(-1.0, 0.0, 1.0, slope, 0)
        then first == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 1) == 0.0 and forward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0

    // grad_output * negative_slope for input <= ZERO (0 included), grad_output above it
    test "backward_input_gate"
        given slope = 0.5
        when first = backward_at(-1.0, 0.0, 1.0, slope, 0)
        then first == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 1) == 0.5 and backward_at(-1.0, 0.0, 1.0, slope, 2) == 1.0

    // Plain ReLU: the gradient is 0 left of zero and passes through right of it
    test "backward_basic_case"
        given negative = backward_at(-2.0, 0.0, 2.0, 0.0, 0)
        when positive = backward_at(-2.0, 0.0, 2.0, 0.0, 2)
        then negative == 0.0 and positive == 1.0

    // ═══════════════════════════════════════════════════════════
    // TDD: Invariants (from .tri constraints)
    // ═══════════════════════════════════════════════════════════

    // input.len == output.len: forward writes every element of its buffer (none keeps the 9.0 fill)
    invariant relu_constraint_0
        assert forward_at(valid_input(), 0.0, 1.0, 0.0, 0) != 9.0 and forward_at(valid_input(), 0.0, 1.0, 0.0, 2) != 9.0

    // A finite input gives a finite output: ReLU never exceeds its input's magnitude
    invariant relu_constraint_1
        assert @abs(forward_at(valid_input(), 0.0, 1.0, 0.5, 0)) <= @abs(valid_input())

Open the lesson's spec in the player ↗

All lessons

• Module 1 · One neuron

  A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.

  1. 01 ReLU: keep the positive part
  2. 02 The sigmoid: a number into a probability
  3. 03 tanh and saturation

• Module 2 · New bends

  The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.

  1. 04 GELU: a dip below zero
  2. 05 Swish and SiLU
  3. 06 ELU: a floor, not a cliff

• Module 3 · A layer

  What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.

  1. 07 Softmax and temperature
  2. 08 Embeddings
  3. 09 Dropout

• Module 4 · Pictures

  Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.

  1. 10 Patches of a picture
  2. 11 Max pooling
  3. 12 Batch normalisation

• Module 5 · The error

  How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.

  1. 13 Mean squared error
  2. 14 Cross-entropy
  3. 15 The Huber loss

• Module 6 · Downhill

  Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.

  1. 16 Gradient descent
  2. 17 RMSProp
  3. 18 Adam

• Module 7 · Learning

  Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.

  1. 19 Backpropagation
  2. 20 Learning XOR
  3. 21 LAMB and large batches

• Module 8 · Memory and attention

  Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.

  1. 22 A recurrent cell
  2. 23 LSTM gates
  3. 24 Attention

• Module 9 · The transformer and the reward

  The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching.

  1. 25 Positional encoding
  2. 26 An encoder block
  3. 27 PPO and the reward