t27.aiРусский

tanh and saturation

You will learn

Why tanh is centred on zero, and where its slope falls so low that a unit stops learning.

tanh is a sigmoid stretched to -1..1 and centred on 0, so a layer of tanh units passes on values that average near zero, and its slope reaches 1 at the centre. It still saturates: at 2.5 the output is 0.9866 and the slope only 0.0266. The red band marks every input where the slope is under 0.1, beyond about 1.8 on either side, and a unit whose inputs live there barely learns. The spec tanh_activation.t27 computes the curve, the slope and its own batch, tanh of 0, 1 and 2: 0, 0.762 and 0.964.

Try it

Drag x until the band starts and read the slope there; then compare the slope at 0 and at 2.5.

Open the interactive lesson →

tanh: where a neuron stops learning
tanh: where a neuron stops learning ↗

Drag along tanh: it is centred on zero, and where it flattens out its slope drops under 0.1. The red band shows every input where the gradient has all but stopped.

specs/ml/activation/tanh_activation.t27

// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// Derivative (can compute from output) | φ² + 1/φ² = 3 | TRINITY

// #8980: `math::exp` named a module this spec never imports, and
// forward_batch allocated with `[]f32{input.len}`. tanh is now computed in the
// form that cannot overflow: exp(2|x|) reaches inf in f32 at |x| = 45, and
// inf / inf is NaN.

module Tanh;
    use base::types;
    use math::constants;

    // ═══════════════════════════════════════════════════════════
    // 2. Types
    // ═══════════════════════════════════════════════════════════

    pub const TanhConfig = struct {
        none : void,
    };

    // ═══════════════════════════════════════════════════════════
    // 3. Core Functions
    // ═══════════════════════════════════════════════════════════

    // tanh(x) = (1 - e^(-2|x|)) / (1 + e^(-2|x|)), with the sign of x
    fn forward(x: f32) -> f32 {
        if (x < 0.0) {
            return -forward(-x);
        }
        const t = exp(-2.0 * x);
        return (1.0 - t) / (1.0 + t);
    }

    // forward_batch: output[i] = tanh(input[i])
    fn forward_batch(input: []const f32, output: []f32) -> void {
        var i : usize = 0;
        while (i < input.len) {
            output[i] = forward(input[i]);
            i = i + 1;
        }
    }

    // tanh'(x) = 1 - tanh(x)^2
    fn derivative(x: f32) -> f32 {
        const tanh_x = forward(x);
        return 1.0 - (tanh_x * tanh_x);
    }

    // Given y = tanh(x), the derivative is 1 - y^2
    fn derivative_from_output(tanh_x: f32) -> f32 {
        return 1.0 - (tanh_x * tanh_x);
    }

    fn batch_at(index: usize) -> f32 {
        const input : [3]f32 = [0.0, 1.0, 2.0];
        var output : [3]f32 = [0.0] ** 3;
        forward_batch(&input, &output);
        return output[index];
    }

    // ═══════════════════════════════════════════════════════════
    // TDD: Tests (from .tri behaviors)
    // ═══════════════════════════════════════════════════════════

    // tanh(0) = 0, tanh(1) = 0.7615942, tanh is odd
    test forward_basic_case
        given input = 0.0
        when result = forward(input)
        then result == 0.0 and @abs(forward(1.0) - 0.7615942) < 0.000001 and forward(-1.0) == -forward(1.0)

    // exp(2x) is inf in f32 here; the stable form still gives 1, not NaN
    test forward_saturates_without_overflow
        given input = 100.0
        when result = forward(input)
        then result == 1.0 and forward(-100.0) == -1.0

    test forward_batch_basic_case
        given first = batch_at(0)
        when second = batch_at(1)
        then first == 0.0 and second > 0.0 and second < 1.0 and batch_at(2) > second

    test derivative_basic_case
        given input = 0.0
        when result = derivative(input)
        then result == 1.0

    test derivative_from_output_basic_case
        given input = 0.5
        when tanh_result = forward(input)
        then derivative_from_output(tanh_result) > 0.0 and derivative_from_output(tanh_result) < 1.0 and derivative_from_output(tanh_result) == derivative(input)

    test derivative_matches_finite_difference
        given slope = (forward(0.5 + 0.001) - forward(0.5 - 0.001)) / 0.002
        when result = derivative(0.5)
        then @abs(result - slope) < 0.001

Open the lesson's spec in the player ↗

All lessons