Cross-entropy
You will learn
Why cross-entropy charges a confident wrong answer so much, and why that is what trains a classifier.
A classifier outputs a probability, and cross-entropy charges -log of the probability it gave to the true answer. Predictions 0.8, 0.3 and 0.9 against answers 1, 0 and 1 cost 0.2284 on average. Being sure and right is nearly free: 0.99, 0.01 and 0.99 cost 0.0101. Being sure and wrong is expensive: 0.01, 0.99 and 0.01 cost 4.6052, and toward 0 the cost grows without bound, which is why the spec clips p to [1e-7, 1 - 1e-7]. Its gradient is largest exactly where the model is sure and wrong. The spec binary_crossentropy_loss.t27 computes the loss and the gradient, from probabilities or from logits.
Try it
Drag prediction 0 toward 0.01 and watch the curve; then read the gradient bars at 0.5, 0.5 and 0.5.
Set three predicted probabilities against the answers 1, 0, 1: the loss is -log of the chance given to the truth. Drag toward a confident mistake and the cost climbs without bound.
specs/ml/loss/binary_crossentropy_loss.t27
// SPDX-License-Identifier: Apache-2.0
// t27/specs/
// Gradient w.r.t predictions | φ² + 1/φ² = 3 | TRINITY
// #8980: never compiled (`new f32[n]`, a float loop index, `predictions.size`,
// an empty enum), and it approximated sigmoid and log with polynomials that
// were 0.3 off at x = 1. The probability gradient also had the wrong sign on its
// second term: d/dp of -(t log p + (1-t) log(1-p)) is -t/p + (1-t)/(1-p). It now
// uses exp and log, writes gradients into a caller's buffer, and reduces by mean.
module BinaryCe;
use base::types;
use math::constants;
// ═══════════════════════════════════════════════════════════
// 1. Constants
// ═══════════════════════════════════════════════════════════
const DEFAULT_EPSILON : f32 = 1e-7;
// ═══════════════════════════════════════════════════════════
// 2. Types
// ═══════════════════════════════════════════════════════════
pub const BinaryCEConfig = struct {
from_logits : bool,
epsilon : f32,
reduction : Reduction,
};
pub const Reduction = enum(u8) {
Mean,
Sum,
};
// ═══════════════════════════════════════════════════════════
// 3. Core Functions
// ═══════════════════════════════════════════════════════════
fn sigmoid(x: f32) -> f32 {
if (x >= 0.0) {
return 1.0 / (1.0 + exp(-x));
}
const e = exp(x);
return e / (1.0 + e);
}
// p clipped into [eps, 1 - eps] so log stays finite
fn clip(p: f32, eps: f32) -> f32 {
if (p < eps) {
return eps;
}
if (p > 1.0 - eps) {
return 1.0 - eps;
}
return p;
}
// -(t log p + (1 - t) log(1 - p)) for one element
fn element_loss(pred: f32, target: f32, config: BinaryCEConfig) -> f32 {
var p = pred;
if (config.from_logits) {
p = sigmoid(pred);
}
p = clip(p, config.epsilon);
return -(target * log(p) + (1.0 - target) * log(1.0 - p));
}
// mean (or sum) of element_loss
fn forward(predictions: []const f32, targets: []const f32, config: BinaryCEConfig) -> f32 {
var loss : f32 = 0.0;
var i : usize = 0;
while (i < predictions.len) {
loss = loss + element_loss(predictions[i], targets[i], config);
i = i + 1;
}
if (config.reduction == Reduction.Sum) {
return loss;
}
return loss / @as(f32, @floatFromInt(predictions.len));
}
// A batch of rows of width `width`, flattened row-major: the mean over every element
fn forward_batch(predictions: []const f32, width: usize, targets: []const f32, config: BinaryCEConfig) -> f32 {
var total : f32 = 0.0;
var i : usize = 0;
while (i < predictions.len) {
total = total + element_loss(predictions[i], targets[i], config);
i = i + 1;
}
_ = width;
return total / @as(f32, @floatFromInt(predictions.len));
}
// Per-element gradient of the (unreduced) loss:
// from logits: sigmoid(z) - t
// from probabilities: -t / p + (1 - t) / (1 - p)
fn gradient(predictions: []const f32, targets: []const f32, config: BinaryCEConfig, grads: []f32) -> void {
var i : usize = 0;
while (i < predictions.len) {
const t = targets[i];
if (config.from_logits) {
grads[i] = sigmoid(predictions[i]) - t;
} else {
const p = clip(predictions[i], config.epsilon);
grads[i] = -t / p + (1.0 - t) / (1.0 - p);
}
i = i + 1;
}
}
fn cfg(from_logits: bool) -> BinaryCEConfig {
return BinaryCEConfig{ .from_logits = from_logits, .epsilon = DEFAULT_EPSILON, .reduction = Reduction.Mean };
}
// The fixture: predictions (a, b, c) against targets (1, 0, 1)
fn loss_of(a: f32, b: f32, c: f32, from_logits: bool) -> f32 {
const predictions : [3]f32 = [a, b, c];
const targets : [3]f32 = [1.0, 0.0, 1.0];
return forward(&predictions, &targets, cfg(from_logits));
}
fn grad_of(a: f32, b: f32, c: f32, from_logits: bool, index: usize) -> f32 {
const predictions : [3]f32 = [a, b, c];
const targets : [3]f32 = [1.0, 0.0, 1.0];
var grads : [3]f32 = [9.0] ** 3;
gradient(&predictions, &targets, cfg(from_logits), &grads);
return grads[index];
}
fn batch_loss() -> f32 {
const predictions : [4]f32 = [0.8, 0.3, 0.9, 0.1];
const targets : [4]f32 = [1.0, 0.0, 1.0, 0.0];
return forward_batch(&predictions, 2, &targets, cfg(false));
}
fn valid_input() -> f32 {
return loss_of(0.8, 0.3, 0.9, false);
}
// ═══════════════════════════════════════════════════════════
// TDD: Tests (from .tri behaviors)
// ═══════════════════════════════════════════════════════════
test sigmoid_basic_case
given input = 0.0
when result = sigmoid(input)
then @abs(result - 0.5) < 1e-6
test sigmoid_positive_input
given input = 1.0
when result = sigmoid(input)
then result > 0.5 and result < 1.0 and @abs(result - 0.7310586) < 0.000001
test sigmoid_negative_input
given input = -1.0
when result = sigmoid(input)
then result > 0.0 and result < 0.5 and @abs(result + sigmoid(1.0) - 1.0) < 0.000001
// -(ln 0.8 + ln 0.7 + ln 0.9) / 3 = 0.2283930
test forward_basic_case
given result = loss_of(0.8, 0.3, 0.9, false)
then result > 0.0 and @abs(result - 0.2283930) < 0.000001
// From logits (2, -1, 3) it is the same loss as from their sigmoids
test forward_from_logits_case
given result = loss_of(2.0, -1.0, 3.0, true)
when probs = loss_of(sigmoid(2.0), sigmoid(-1.0), sigmoid(3.0), false)
then result > 0.0 and @abs(result - probs) < 0.000001
// (-ln 0.8 - ln 0.7 - ln 0.9 - ln 0.9) / 4 = 0.1976349
test forward_batch_basic_case
given result = batch_loss()
then result > 0.0 and @abs(result - 0.1976349) < 0.000001
// t = 1 pulls p up (negative gradient), t = 0 pushes it down (positive)
test gradient_basic_case
given g0 = grad_of(0.8, 0.3, 0.9, false, 0)
when g1 = grad_of(0.8, 0.3, 0.9, false, 1)
then g0 < 0.0 and g1 > 0.0 and grad_of(0.8, 0.3, 0.9, false, 2) < 0.0 and @abs(g0 + 1.25) < 0.000001 and @abs(g1 - 1.0 / 0.7) < 0.000001
// From logits: sigmoid(z) - t, and it matches a finite difference of the loss
test gradient_from_logits_case
given g1 = grad_of(2.0, -1.0, 3.0, true, 1)
when slope = 3.0 * (loss_of(2.0, -1.0 + 0.001, 3.0, true) - loss_of(2.0, -1.0 - 0.001, 3.0, true)) / 0.002
then @abs(g1 - sigmoid(-1.0)) < 0.000001 and @abs(g1 - slope) < 0.001
// ═══════════════════════════════════════════════════════════
// TDD: Invariants (from .tri constraints)
// ═══════════════════════════════════════════════════════════
// targets[i] in {0, 1}: a perfect prediction costs (almost) nothing
invariant binary_ce_constraint_0
assert loss_of(1.0, 0.0, 1.0, false) < 0.000001
// 0 < epsilon << 1
invariant binary_ce_constraint_1
assert DEFAULT_EPSILON > 0.0 and DEFAULT_EPSILON < 0.001
// name: binary_ce -- positive for an imperfect prediction
invariant binary_ce_constraint_2
assert valid_input() > 0.0
// name: gradient -- zero where the logit already gives the target
invariant binary_ce_constraint_3
assert @abs(grad_of(30.0, -30.0, 30.0, true, 0)) < 0.000001
// "Bishop (2006) - Pattern Recognition and Machine Learning": the loss falls as p nears t
invariant binary_ce_constraint_4
assert loss_of(0.9, 0.1, 0.9, false) < loss_of(0.6, 0.4, 0.6, false)
// "Goodfellow et al. (2016) - Deep Learning": clipping keeps a certain wrong answer finite
invariant binary_ce_constraint_5
assert loss_of(0.0, 1.0, 0.0, false) < 20.0
All lessons
Module 1 · One neuron
A neuron sums its inputs and bends the sum: ReLU, the sigmoid and tanh, and where each one stops passing a gradient back.
Module 2 · New bends
The smooth curves of current models: GELU with its dip, Swish with its knob, and ELU with a floor instead of a cliff.
Module 3 · A layer
What sits around the neurons: softmax turns scores into a choice, an embedding turns a word into numbers, dropout switches units off.
Module 4 · Pictures
Reading a picture in patches: a window averaged over its patch, max pooling, and batch normalisation of features on different scales.
Module 5 · The error
How wrong a guess is, as one number: mean squared error, cross-entropy for probabilities, and the Huber loss for outliers.
Module 6 · Downhill
Walking down the loss: gradient descent with momentum, RMSProp that ignores the gradient's size, and Adam that combines them.
Module 7 · Learning
Where the gradients come from and what they achieve: backpropagation through a layer, a network that learns XOR, and LAMB for large batches.
Module 8 · Memory and attention
Networks that read a sequence: a recurrent cell and its fading echo, the gates of an LSTM, and attention over every position at once.
Module 9 · The transformer and the reward
The parts of a transformer and of an agent: positions as clocks, one encoder block, and PPO, which learns from reward without lurching.