A recurrent cell
You will learn
How a recurrent cell carries a state from step to step, and why that memory fades.
A sentence comes one word at a time, and a recurrent cell carries a state h from one step to the next: h' = tanh(W_x x + W_h h + b). In the spec's tiny cell W_x is 1 and W_h 0.5, so a single pulse gives 0.7616, 0.3634 and 0.1797: the memory of it roughly halves each step. A second pulse at step 3 lifts h to 0.7968, a little above where the single pulse began, because the old memory is still there. After 8 steps only 0.0466 is left. That fading is why plain cells forget long contexts. The spec rnn_cell.t27 computes each step.
Try it
Load one pulse and read how many steps it takes to fall under 0.1; then make the input alternate.
Tap the inputs of eight steps: the cell feeds its own last output back, h' = tanh(x + 0.5 h), so every pulse echoes on and fades by about half a step. Two pulses add up.
specs/ml/recurrent/rnn_cell.t27
// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// Basic RNN recurrence equation | φ² + 1/φ² = 3 | TRINITY
// #8980: written against undeclared `matmul`, `add`, `zeros`, `len` and `append`,
// it applied tanh twice (h = tanh(tanh(z))), and the tests asserted
// `result != undefined`. It is now the Elman cell,
// h' = tanh(W_x x + W_h h + b), with W_x hidden x input and W_h hidden x hidden
// (row-major), stepping over buffers the caller owns.
module RnnCell;
use base::types;
use math::constants;
// ═══════════════════════════════════════════════════════════
// 1. Constants
// ═══════════════════════════════════════════════════════════
const DEFAULT_INPUT_SIZE : u32 = 128;
const DEFAULT_HIDDEN_SIZE : u32 = 128;
// ═══════════════════════════════════════════════════════════
// 2. Types
// ═══════════════════════════════════════════════════════════
pub const RNNCellConfig = struct {
input_size : u32,
hidden_size : u32,
};
pub const RNNState = struct {
hidden : []f32,
cell : []f32,
};
pub const RNNParams = struct {
w_x : []f32,
w_h : []f32,
b : []f32,
};
// ═══════════════════════════════════════════════════════════
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
// new_hidden[i] = tanh(sum_j W_x[i][j] x[j] + sum_k W_h[i][k] h[k] + b[i])
fn forward_step(input: []const f32, hidden: []const f32, params: RNNParams, config: RNNCellConfig, new_hidden: []f32) -> void {
var i : usize = 0;
while (i < config.hidden_size) {
var z : f32 = params.b[i];
var j : usize = 0;
while (j < config.input_size) {
z = z + params.w_x[i * config.input_size + j] * input[j];
j = j + 1;
}
var k : usize = 0;
while (k < config.hidden_size) {
z = z + params.w_h[i * config.hidden_size + k] * hidden[k];
k = k + 1;
}
new_hidden[i] = tanh(z);
i = i + 1;
}
}
// inputs: steps x input_size; outputs: steps x hidden_size, row t = h after step t.
// The state starts at initial_state and each step reads the previous row.
fn forward_sequence(inputs: []const f32, steps: usize, initial_state: []const f32, params: RNNParams, config: RNNCellConfig, outputs: []f32) -> void {
const n_in : usize = config.input_size;
const n_h : usize = config.hidden_size;
var t : usize = 0;
while (t < steps) {
if (t == 0) {
forward_step(inputs[0..n_in], initial_state, params, config, outputs[0..n_h]);
} else {
forward_step(inputs[t * n_in .. (t + 1) * n_in], outputs[(t - 1) * n_h .. t * n_h], params, config, outputs[t * n_h .. (t + 1) * n_h]);
}
t = t + 1;
}
}
// The first hidden_size entries of `buffer`, zeroed
fn init_state(buffer: []f32, hidden_size: u32) -> []f32 {
var i : usize = 0;
while (i < hidden_size) {
buffer[i] = 0.0;
i = i + 1;
}
return buffer[0..hidden_size];
}
// A 1-in, 1-hidden cell with W_x = 1, W_h = 0.5, b = 0: h' = tanh(x + 0.5 h)
fn tiny() -> RNNCellConfig {
return RNNCellConfig{ .input_size = 1, .hidden_size = 1 };
}
fn tiny_step(x: f32, h: f32) -> f32 {
var w_x : [1]f32 = [1.0];
var w_h : [1]f32 = [0.5];
var b : [1]f32 = [0.0];
const input : [1]f32 = [x];
const hidden : [1]f32 = [h];
var out : [1]f32 = [9.0];
forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &out);
return out[0];
}
// The sequence (1, 0, 0) from h = 0: h after step t
fn tiny_sequence(t: usize) -> f32 {
var w_x : [1]f32 = [1.0];
var w_h : [1]f32 = [0.5];
var b : [1]f32 = [0.0];
const inputs : [3]f32 = [1.0, 0.0, 0.0];
const h0 : [1]f32 = [0.0];
var outputs : [3]f32 = [9.0] ** 3;
forward_sequence(&inputs, 3, &h0, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, tiny(), &outputs);
return outputs[t];
}
// A 2-in, 2-hidden cell: h' = tanh(W_x x + W_h h + b) with W_x = [[1, -1], [0.5, 0.5]],
// W_h = [[0.1, 0], [0, 0.1]], b = (0, -0.5), at x = (1, 2), h = (1, -1)
fn wide_step(index: usize) -> f32 {
var w_x : [4]f32 = [1.0, -1.0, 0.5, 0.5];
var w_h : [4]f32 = [0.1, 0.0, 0.0, 0.1];
var b : [2]f32 = [0.0, -0.5];
const input : [2]f32 = [1.0, 2.0];
const hidden : [2]f32 = [1.0, -1.0];
var out : [2]f32 = [9.0] ** 2;
forward_step(&input, &hidden, RNNParams{ .w_x = &w_x, .w_h = &w_h, .b = &b }, RNNCellConfig{ .input_size = 2, .hidden_size = 2 }, &out);
return out[index];
}
fn init_len(hidden_size: u32) -> usize {
var buffer : [64]f32 = [9.0] ** 64;
const h = init_state(&buffer, hidden_size);
if (h[hidden_size - 1] != 0.0) {
return 0;
}
return h.len;
}
fn valid_input() -> f32 {
return tiny_step(1.0, 0.0);
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
// z = (1 - 2 + 0.1, 0.5 + 1 - 0.1 - 0.5) = (-0.9, 0.9): h' = (tanh(-0.9), tanh(0.9))
test forward_step_basic_case
given h0 = wide_step(0)
when h1 = wide_step(1)
then @abs(h0 + 0.7162979) < 0.000001 and @abs(h1 - 0.7162979) < 0.000001 and @abs(tiny_step(1.0, 0.0) - 0.7615942) < 0.000001
// The cell remembers: after the input pulse, h decays through the recurrent weight,
// h1 = tanh(1), h2 = tanh(0.5 h1), h3 = tanh(0.5 h2)
test forward_sequence_basic_case
given h1 = tiny_sequence(0)
when h3 = tiny_sequence(2)
then @abs(h1 - 0.7615942) < 0.000001 and @abs(tiny_sequence(1) - tanh(0.5 * h1)) < 0.000001 and @abs(h3 - tanh(0.5 * tiny_sequence(1))) < 0.000001 and h3 > 0.0 and h3 < tiny_sequence(1)
test init_state_basic_case
given hidden_size = 64
when result = init_len(hidden_size)
then result == hidden_size
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
// input_size >= 1
invariant rnn_cell_constraint_0
assert DEFAULT_INPUT_SIZE >= 1
// hidden_size >= 1
invariant rnn_cell_constraint_1
assert DEFAULT_HIDDEN_SIZE >= 1
// name: rnn_formula -- one tanh, not two: tanh(1) = 0.7615942, tanh(tanh(1)) = 0.6420
invariant rnn_cell_constraint_2
assert valid_input() > 0.76 and valid_input() < 0.77
// "Elman (1990) - Finding Structure in Time": the state carries the past forward
invariant rnn_cell_constraint_3
assert tiny_step(0.0, 1.0) > 0.0
// "Goodfellow et al. (2016) - Sequence Modeling with RNNs and LSTMs": tanh bounds the state
invariant rnn_cell_constraint_4
assert tiny_step(100.0, 1.0) <= 1.0 and tiny_step(-100.0, -1.0) >= -1.0
All lessons
Module 1 · One neuron
A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.
Module 2 · New bends
The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.
Module 3 · A layer
What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.
Module 4 · Pictures
Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.
Module 5 · The error
How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.
Module 6 · Downhill
Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.
Module 7 · Learning
Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.
Module 8 · Memory and attention
Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.
Module 9 · The transformer and the reward
The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching.