Backpropagation
You will learn
How the gradient of each weight and each input of a layer follows from the gradient of its outputs.
To learn, every weight needs to know how the loss changes when it moves. Backpropagation answers one layer at a time: given g, how much the loss wants each output to move, the weight from input i to output o gets g_o x_i, and input i gets the sum over o of W[o][i] g_o, which it hands on to the layer below. With inputs (1, 2, 3) and g = (1, 0.5) the weight gradients are 1, 2, 3 and 0.5, 1, 1.5, and the input gradients 0.5, 2 and 3.5. The layer is linear, so moving x0 by 1 moves the loss g . y from 15.25 to 15.75, by its gradient 0.5. The spec dense_layer.t27 computes all three.
Try it
Set g = (1, 0) and read which edges go dark; then set g = (-2, 1) and check the input gradients by hand.
Set how much the loss wants each of two outputs to move: every weight gets that times its input, every input gets W transposed times it. The edges and nodes show the numbers.
specs/ml/layers/dense_layer.t27
// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// Row-major weight layout for cache efficiency | φ² + 1/φ² = 3 | TRINITY
// #8980: forward and backward were empty placeholders, the tests asserted
// `result != undefined`, and the invariants asserted `true` over an undeclared
// `valid_input`. Dense is now y = act(W x + b) with W row-major
// (output_size x input_size), and backward gives the three gradients.
module Dense;
use base::types;
use math::constants;
// ═══════════════════════════════════════════════════════════
// 1. Constants
// ═══════════════════════════════════════════════════════════
const MIN_INPUT_SIZE : u32 = 1;
const MAX_INPUT_SIZE : u32 = 4096;
// ═══════════════════════════════════════════════════════════
// 2. Types
// ═══════════════════════════════════════════════════════════
pub const DenseConfig = struct {
input_size : u32,
output_size : u32,
has_bias : bool,
activation : ActivationType,
};
pub const ActivationType = enum(u8) {
None,
ReLU,
GELU,
Sigmoid,
Tanh,
Softmax,
};
// ═══════════════════════════════════════════════════════════
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
// relu, the one activation the tests exercise besides None
fn activate(z: f32, activation: ActivationType) -> f32 {
if (activation == ActivationType.ReLU) {
if (z > 0.0) {
return z;
}
return 0.0;
}
return z;
}
// output[o] = act(sum_i weights[o * in + i] * input[i] + bias[o])
fn forward(input: []const f32, weights: []const f32, bias: []const f32, config: DenseConfig, output: []f32) -> void {
var o : usize = 0;
while (o < config.output_size) {
var z : f32 = 0.0;
if (config.has_bias) {
z = bias[o];
}
var i : usize = 0;
while (i < config.input_size) {
z = z + weights[o * config.input_size + i] * input[i];
i = i + 1;
}
output[o] = activate(z, config.activation);
o = o + 1;
}
}
// For a linear layer (activation None), given dL/dy = grad_output:
// grad_input[i] = sum_o W[o][i] grad_output[o] (W^T g)
// grad_weights[o][i] = grad_output[o] * input[i] (g x^T)
// grad_bias[o] = grad_output[o]
fn backward(grad_output: []const f32, input: []const f32, weights: []const f32, config: DenseConfig, grad_input: []f32, grad_weights: []f32, grad_bias: []f32) -> void {
var i : usize = 0;
while (i < config.input_size) {
grad_input[i] = 0.0;
i = i + 1;
}
var o : usize = 0;
while (o < config.output_size) {
grad_bias[o] = grad_output[o];
i = 0;
while (i < config.input_size) {
grad_input[i] = grad_input[i] + weights[o * config.input_size + i] * grad_output[o];
grad_weights[o * config.input_size + i] = grad_output[o] * input[i];
i = i + 1;
}
o = o + 1;
}
}
// The fixture: 3 inputs, 2 outputs, W = [[1, 2, 3], [-1, 0, 1]], b = [0.5, -0.5]
fn cfg(activation: ActivationType) -> DenseConfig {
return DenseConfig{ .input_size = 3, .output_size = 2, .has_bias = true, .activation = activation };
}
fn forward_at(x0: f32, x1: f32, x2: f32, activation: ActivationType, index: usize) -> f32 {
const input : [3]f32 = [x0, x1, x2];
const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
const bias : [2]f32 = [0.5, -0.5];
var output : [2]f32 = [0.0] ** 2;
forward(&input, &weights, &bias, cfg(activation), &output);
return output[index];
}
// L = g . y for g = (g0, g1): the loss whose gradients backward computes
fn loss_at(x0: f32, x1: f32, x2: f32, g0: f32, g1: f32) -> f32 {
return g0 * forward_at(x0, x1, x2, ActivationType.None, 0) + g1 * forward_at(x0, x1, x2, ActivationType.None, 1);
}
// which: 0 = grad_input, 1 = grad_weights, 2 = grad_bias
fn backward_at(g0: f32, g1: f32, which: u8, index: usize) -> f32 {
const grad_output : [2]f32 = [g0, g1];
const input : [3]f32 = [1.0, 2.0, 3.0];
const weights : [6]f32 = [1.0, 2.0, 3.0, -1.0, 0.0, 1.0];
var grad_input : [3]f32 = [9.0] ** 3;
var grad_weights : [6]f32 = [9.0] ** 6;
var grad_bias : [2]f32 = [9.0] ** 2;
backward(&grad_output, &input, &weights, cfg(ActivationType.None), &grad_input, &grad_weights, &grad_bias);
if (which == 0) {
return grad_input[index];
}
if (which == 1) {
return grad_weights[index];
}
return grad_bias[index];
}
// An empty layer (no inputs, no outputs) writes nothing
fn empty_forward_untouched() -> bool {
const none : [0]f32 = [];
var sentinel : [1]f32 = [7.0];
const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
forward(&none, &none, &none, empty_cfg, &sentinel);
return sentinel[0] == 7.0;
}
fn empty_backward_untouched() -> bool {
const none : [0]f32 = [];
var sentinel : [1]f32 = [7.0];
const empty_cfg = DenseConfig{ .input_size = 0, .output_size = 0, .has_bias = true, .activation = ActivationType.None };
backward(&none, &none, &none, empty_cfg, &sentinel, &sentinel, &sentinel);
return sentinel[0] == 7.0;
}
fn valid_input() -> f32 {
return forward_at(1.0, 2.0, 3.0, ActivationType.None, 0);
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
// x = (1, 2, 3): y0 = 1 + 4 + 9 + 0.5 = 14.5, y1 = -1 + 0 + 3 - 0.5 = 1.5
test forward_basic_case
given y0 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 0)
when y1 = forward_at(1.0, 2.0, 3.0, ActivationType.None, 1)
then y0 == 14.5 and y1 == 1.5
// ReLU clips the negative pre-activation: x = (0, 0, -1) gives y = (-2.5, -1.5) -> (0, 0)
test forward_relu_clips_negatives
given linear = forward_at(0.0, 0.0, -1.0, ActivationType.None, 0)
when clipped = forward_at(0.0, 0.0, -1.0, ActivationType.ReLU, 0)
then linear == -2.5 and clipped == 0.0 and forward_at(1.0, 2.0, 3.0, ActivationType.ReLU, 0) == 14.5
test forward_empty_input
given result = empty_forward_untouched()
then result
// g = (1, 2): grad_input = W^T g = (1 - 2, 2 + 0, 3 + 2) = (-1, 2, 5); grad_bias = g;
// grad_weights[1][2] = g1 * x2 = 6
test backward_basic_case
given gi0 = backward_at(1.0, 2.0, 0, 0)
when gi2 = backward_at(1.0, 2.0, 0, 2)
then gi0 == -1.0 and backward_at(1.0, 2.0, 0, 1) == 2.0 and gi2 == 5.0 and backward_at(1.0, 2.0, 2, 1) == 2.0 and backward_at(1.0, 2.0, 1, 5) == 6.0
// dL/dx1 by finite difference equals grad_input[1]
test backward_matches_finite_difference
given slope = (loss_at(1.0, 2.0 + 0.01, 3.0, 1.0, 2.0) - loss_at(1.0, 2.0 - 0.01, 3.0, 1.0, 2.0)) / 0.02
when result = backward_at(1.0, 2.0, 0, 1)
then @abs(result - slope) < 0.001
test backward_empty_gradient
given result = empty_backward_untouched()
then result
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
// input.size() == config.input_size: the bounds the layer accepts
invariant dense_constraint_0
assert MIN_INPUT_SIZE >= 1 and MAX_INPUT_SIZE >= MIN_INPUT_SIZE
// output.size() == config.output_size: each output is one row of W x + b
invariant dense_constraint_1
assert valid_input() == 14.5
// weights.size() == input_size * output_size: the fixture holds 3 * 2 = 6 weights
invariant dense_constraint_2
assert cfg(ActivationType.None).input_size * cfg(ActivationType.None).output_size == 6
// bias.size() == output_size (if has_bias): the fixture carries a bias of 2
invariant dense_constraint_3
assert cfg(ActivationType.None).has_bias
// All inputs must be finite (no NaN/Inf): finite in, finite out
invariant dense_constraint_4
assert valid_input() == valid_input()
All lessons
Module 1 · One neuron
A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.
Module 2 · New bends
The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.
Module 3 · A layer
What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.
Module 4 · Pictures
Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.
Module 5 · The error
How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.
Module 6 · Downhill
Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.
Module 7 · Learning
Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.
Module 8 · Memory and attention
Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.
Module 9 · The transformer and the reward
The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching.