Multiply, then add

You will learn

What a multiply-accumulate does with two signed products, and how the swap from lesson 12 breaks it.

Try it

In the recording, find the test that expects 512 and the test the planted bug breaks; then in the spec frame change the XOR in smul to OR yourself and see whether the browser catches it.

specs/ternary/gft_signed_mac.t27

module GftSignedMac;
// #1764 + GF-T: a spec-first SIGNED GF-T16 MAC (y = a1*b1 + a2*b2) with
// round-to-nearest-even, bit-exact to the ideal oracle gft16_ref.py (with sign).
// This is the piece real NN inference needs -- negative weights/activations and
// cancellation in the accumulator.
//
// Signed GF-T16: [ sign(1) : offset(7) : mant(9) ] (17 bits, held in a u32),
// magnitude value = (1+mant/512)*2^(offset-40). Signed mul = sign XOR + RNE
// magnitude mul. Signed add = same-sign RNE add, or (different sign) subtract the
// smaller magnitude from the larger: align, subtract with G guard bits, left-
// normalize, round to nearest even; result takes the larger operand's sign, or is
// zero on exact cancellation.
// All intermediates non-negative small ints -> i32 (signed shift/compare/mul on
// positives == unsigned, bit-identical to the oracle).

// --- RNE magnitude multiply (matches gft_mul_rne) ---
fn magmul(a: i32, b: i32) -> i32 {
    var ao : i32 = a >> 9;
    var am : i32 = a & 511;
    var bo : i32 = b >> 9;
    var bm : i32 = b & 511;
    var prod : i32 = (512 + am) * (512 + bm);
    var carry : i32 = 0;
    if (prod >= 524288) { carry = 1; }
    var q : i32 = prod >> 9;
    var r : i32 = prod & 511;
    var half : i32 = 256;
    if (carry == 1) { q = prod >> 10; r = prod & 1023; half = 512; }
    var mant : i32 = q - 512;
    if (r > half) { mant = mant + 1; }
    if (r == half) { if ((q & 1) == 1) { mant = mant + 1; } }
    var s : i32 = ao + bo + carry;
    var off : i32 = 0;
    if (s >= 40) { var rr : i32 = s - 40; if (rr >= 80) { off = 80; } else { off = rr; } }
    if (mant >= 512) { mant = 0; off = off + 1; if (off >= 80) { off = 80; } }
    return (off << 9) | mant;
}

// --- RNE magnitude same-sign add (matches gft_add_rne) ---
fn magadd(a: i32, b: i32) -> i32 {
    var ao : i32 = a >> 9;
    var am : i32 = a & 511;
    var bo : i32 = b >> 9;
    var bm : i32 = b & 511;
    var ho : i32 = bo; var hm : i32 = bm; var lo : i32 = ao; var lm : i32 = am;
    if (ao >= bo) { ho = ao; hm = am; lo = bo; lm = bm; }
    var hs : i32 = 512 + hm;
    var ls : i32 = 512 + lm;
    var d : i32 = ho - lo;
    if (d > 11) { d = 11; }
    var losh : i32 = ls >> d;
    var rem : i32 = ls - (losh << d);
    var s : i32 = hs + losh;
    var off : i32 = ho;
    var mant : i32 = s - 512;
    if (s >= 1024) {
        var g : i32 = s & 1;
        var pre : i32 = s >> 1;
        mant = pre - 512;
        if (g == 1) { if (rem > 0) { mant = mant + 1; } else { if ((pre & 1) == 1) { mant = mant + 1; } } }
        off = ho + 1; if (off >= 80) { off = 80; }
    } else {
        var t : i32 = rem << 1;
        var hf : i32 = 1 << d;
        if (t > hf) { mant = mant + 1; }
        else { if (t == hf) { if ((s & 1) == 1) { mant = mant + 1; } } }
    }
    if (mant >= 512) { mant = 0; off = off + 1; if (off >= 80) { off = 80; } }
    return (off << 9) | mant;
}

// --- RNE magnitude subtract, hi >= lo. Returns 0 on exact cancellation. G=14 guard bits. ---
fn magsub(hi: i32, lo: i32) -> i32 {
    if (hi == lo) { return 0; }
    var ho : i32 = hi >> 9; var hm : i32 = hi & 511;
    var lo_o : i32 = lo >> 9; var lm : i32 = lo & 511;
    var d : i32 = ho - lo_o; var hs : i32 = (512 + hm) << 14;
    var la : i32 = 0; var sticky : i32 = 0;
    if (d >= 26) { la = 0; sticky = 1; }
    else { var ls : i32 = (512 + lm) << 14; la = ls >> d; if ((ls - (la << d)) > 0) { sticky = 1; } }
    var diff : i32 = hs - la; var off : i32 = ho;
    var cap : i32 = 12; if (off - 1 < cap) { cap = off - 1; } if (cap < 0) { cap = 0; }
    var sh : i32 = 0;
    if (diff != 0) {
        var t : i32 = diff;
        if (t < 65536) { if (sh + 8 <= cap) { t = t << 8; sh = sh + 8; } }
        if (t < 1048576) { if (sh + 4 <= cap) { t = t << 4; sh = sh + 4; } }
        if (t < 4194304) { if (sh + 2 <= cap) { t = t << 2; sh = sh + 2; } }
        if (t < 8388608) { if (sh + 1 <= cap) { t = t << 1; sh = sh + 1; } }
    }
    diff = diff << sh; off = off - sh;
    var q : i32 = diff >> 14; var rem : i32 = diff - (q << 14); var half : i32 = 8192; var mant : i32 = q - 512;
    if (rem > half) { mant = mant + 1; }
    else { if (rem == half) { if (sticky == 1) { mant = mant + 1; } else { if ((q & 1) == 1) { mant = mant + 1; } } } }
    if (mant >= 512) { mant = 0; off = off + 1; if (off >= 80) { off = 80; } }
    return (off << 9) | mant;
}

fn smul(a: u32, b: u32) -> u32 {
    var sa : i32 = (a >> 16) as i32;
    var sb : i32 = (b >> 16) as i32;
    var mag : i32 = magmul((a & 65535) as i32, (b & 65535) as i32);
    return (((sa ^ sb) << 16) | mag) as u32;
}

fn sadd(a: u32, b: u32) -> u32 {
    var sa : i32 = (a >> 16) as i32;
    var ma : i32 = (a & 65535) as i32;
    var sb : i32 = (b >> 16) as i32;
    var mb : i32 = (b & 65535) as i32;
    if (sa == sb) { return ((sa << 16) | magadd(ma, mb)) as u32; }
    // different signs: a top-level r==0 guard (NOT nested inside the ma>=mb
    // branch -- gen-verilog does not lower that as an early return, so it fell
    // through to (sign<<16)|0 = a wrong negative zero on exact cancellation).
    var bsign : i32 = sa;
    var r : i32 = magsub(ma, mb);
    if (ma < mb) { r = magsub(mb, ma); bsign = sb; }
    if (r == 0) { return 0; }
    return ((bsign << 16) | r) as u32;
}

// The signed MAC: y = a1*b1 + a2*b2.
fn on_comb(a1: u32, b1: u32, a2: u32, b2: u32) -> u32 {
    return sadd(smul(a1, b1), smul(a2, b2));
}

// +1.0 = offset 40 mant 0 sign 0 = 20480. -1.0 = sign 1 = 0x15000 = 86016.
// (+1)*(+1) + (-1)*(-1) = 1 + 1 = 2.0 (+) = offset 41 = 20992.
test pp { assert_eq(on_comb(20480, 20480, 86016, 86016), 20992); }
// (+1)*(+1) + (+1)*(-1) = 1 - 1 = 0.
test cancel { assert_eq(on_comb(20480, 20480, 20480, 86016), 0); }
endmodule

Open the lesson's spec in the player ↗

All lessons

• Module 1 · Lab: our own research

  A number format of our own, an honest scoreboard, and a model's tables multiplied on the board.

  1. 01 T27: a number format of our own
  2. 02 An honest scoreboard
  3. 03 A model's tables on the board

• Module 2 · AI numbers: the MX block

  How AI chips keep weights in a few bits: one shared scale per block, the scale byte itself, and what one outlier does to its neighbours.

  1. 04 One scale for a block of weights
  2. 05 The scale byte on a real machine
  3. 06 One outlier, many zeros

• Module 3 · Ternary weights

  Weights that are only minus, zero or plus a scale, the five rules a ternary alphabet must pass, and a test pass that checked nothing.

  1. 07 Weights of minus phi, zero and plus phi
  2. 08 A pass that checked nothing
  3. 09 Five rules for a weight alphabet

• Module 4 · The Ternary Network Float

  A rule the compiler enforces before any test runs, and a 17-bit float whose exponent is four balanced trits.

  1. 10 Blocked is not failed
  2. 11 A float whose exponent is four trits
  3. 12 Negate twice, get the number back

• Module 5 · Arithmetic on signed numbers

  Multiply two signed numbers, add them when their signs differ, and do both at once in a multiply-accumulate.

  1. 13 A sign is one bit
  2. 14 Opposite signs subtract
  3. 15 Multiply, then add

• Module 6 · Parts of a neuron

  A ReLU that bends at zero, a power of two for softmax, and an argmax that names the answer.

  1. 16 A bend at zero
  2. 17 Two to the x
  3. 18 Pick the largest

• Module 7 · Learning from a mistake

  A loss that prices a wrong guess in bits, one step that moves a weight against its gradient, and the hidden layer that XOR needs.

  1. 19 A loss in bits
  2. 20 One step downhill
  3. 21 XOR needs a bend

• Module 8 · BitNet: ternary networks

  A threshold that squeezes a sum back to three values, one neuron that becomes a different function when its weights change, and a neuron that reads its inputs 27 trits at a time.

  1. 22 Back to three values
  2. 23 Same neuron, new weights
  3. 24 A neuron in chunks

• Module 9 · The ternary MAC as a chip

  The 27-trit dot product as wires with no register, the same sum added into a register on every clock, and a small whole network to close the course.

  1. 25 A dot product made of wires
  2. 26 Add it up on every clock
  3. 27 A whole network