// Copyright (C) 2026 Kiyotsugu Arai
// SPDX-License-Identifier: LGPL-3.0-or-later

// MpnOps.hpp
// Low-level operation functions over raw limb arrays
// Equivalent to GMP's mpn_*. Does not use any Int objects.

#pragma once

#include <cstdint>
#include <cstddef>
#include <cstring>
#include <cassert>
#include <algorithm>
#include <bit>
#include <vector>
#ifdef SANGI_TOOM8_PROFILE
#include <chrono>
#endif
#include <math/core/mp/Int/compiler_intrinsics.hpp>
#include <math/core/mp/ThreadPool.hpp>
#include <math/core/mp/Int/UInt128.hpp>
#include <math/core/mp/Int/ScratchArena.hpp>
#include <math/core/mp/Int/PrimeNtt.hpp>
#include <math/core/mp/Int/DoubleFft.hpp>

namespace sangi {
namespace mpn {

// ================================================================
// BMI2 + ADX assembly optimization (MSVC x64 only)
// ================================================================
// When SANGI_INT_HAS_ASM is defined, hand-written assembly versions of
// addmul_1/mul_1 using MULX+ADCX/ADOX are invoked.
// Falls back to the intrinsics version when the CPU does not support BMI2+ADX.

#ifdef SANGI_INT_HAS_ASM
extern "C" uint64_t mpn_addmul_1_mulx(uint64_t* rp, const uint64_t* ap,
                    size_t n, uint64_t b);
extern "C" uint64_t mpn_mul_1_mulx(uint64_t* rp, const uint64_t* ap,
                  size_t n, uint64_t b);
extern "C" uint64_t mpn_submul_1_mulx(uint64_t* rp, const uint64_t* ap,
                    size_t n, uint64_t b);
extern "C" void mpn_mul_basecase_mulx(uint64_t* rp, const uint64_t* ap,
                    size_t an, const uint64_t* bp, size_t bn);
// sqr_basecase: exploits symmetry, all performs inlined (push/pop removed)
extern "C" void mpn_sqr_basecase_mulx(uint64_t* rp, const uint64_t* ap, size_t n);
// add_n / sub_n: BMI2/ADX not required, only basic x86-64 ADC/SBB used
extern "C" uint64_t mpn_add_n_asm(uint64_t* rp, const uint64_t* ap,
                  const uint64_t* bp, size_t n);
extern "C" uint64_t mpn_sub_n_asm(uint64_t* rp, const uint64_t* ap,
                  const uint64_t* bp, size_t n);
// add_n / sub_n small-size specialization (n=1..4): no loops, no push/pop
extern "C" uint64_t mpn_add_n_small_asm(uint64_t* rp, const uint64_t* ap,
                     const uint64_t* bp, size_t n);
extern "C" uint64_t mpn_sub_n_small_asm(uint64_t* rp, const uint64_t* ap,
                     const uint64_t* bp, size_t n);
// mul_basecase small-size specialization (nxn, n=1..7): fully unrolled MULX
//  n=1..4: simple ADD/ADC unroll
//  n=5..7: cyclic accumulator (N=n+1) + ADCX/ADOX dual chain
extern "C" void mpn_mul_small_asm(uint64_t* rp, const uint64_t* ap,
                  const uint64_t* bp, size_t n);
// mul_basecase 8x8 specialization: cyclic 9-reg accumulator + ADCX/ADOX dual chain
extern "C" void mpn_mul_8x8_asm(uint64_t* rp, const uint64_t* ap,
                 const uint64_t* bp);
// addmul_1 small-size specialization (n=1..4): no push/pop
extern "C" uint64_t mpn_addmul_1_small_asm(uint64_t* rp, const uint64_t* ap,
                      size_t n, uint64_t b);
// div_basecase 3-by-2 ASM (submul inline-fused)
extern "C" void mpn_sbpi1_div_qr_asm(uint64_t* qp, uint64_t* ap, size_t qn,
                   const uint64_t* dp, size_t dn,
                   uint64_t dinv3);
// lshift / rshift: fast shift via SHLD/SHRD instructions
extern "C" uint64_t mpn_lshift_asm(uint64_t* rp, const uint64_t* ap,
                  size_t n, unsigned shift);
extern "C" uint64_t mpn_rshift_asm(uint64_t* rp, const uint64_t* ap,
                  size_t n, unsigned shift);
// Montgomery CIOS multiplication / REDC (addmul_1 inlined, push/pop removed)
extern "C" void mpn_mont_mul_mulx(uint64_t* rp, const uint64_t* ap,
                  const uint64_t* bp, size_t n,
                  const uint64_t* mp, uint64_t m_inv,
                  uint64_t* scratch);
extern "C" void mpn_mont_redc_mulx(uint64_t* rp, uint64_t* tp, size_t tn,
                  const uint64_t* mp, size_t n,
                  uint64_t m_inv);

namespace detail {
  // Check both BMI2 (bit 8) and ADX (bit 19) via CPUID
  inline bool detect_bmi2_adx() {
    int info[4];
#if defined(_MSC_VER)
    __cpuidex(info, 7, 0);
#elif defined(__GNUC__) || defined(__clang__)
    __asm__ __volatile__(
      "cpuid"
      : "=a"(info[0]), "=b"(info[1]), "=c"(info[2]), "=d"(info[3])
      : "a"(7), "c"(0)
    );
#endif
    bool bmi2 = (info[1] >> 8) & 1;  // EBX bit 8
    bool adx = (info[1] >> 19) & 1; // EBX bit 19
    return bmi2 && adx;
  }

  // Thread-safe one-time detection (C++11 magic statics)
  inline bool has_bmi2_adx() {
    static const bool result = detect_bmi2_adx();
    return result;
  }
} // namespace detail
#endif // SANGI_INT_HAS_ASM

// ================================================================
// multiplication algorithm thresholds (per-platform tuning)
// ================================================================
// Measured 2026-04-27 (Zen 3 / AMD Ryzen 5000 series):
//  tests/test-Int/{bench_threshold, bench_mul_threshold, bench_toom6,
//          bench_toom8, bench_mul_small}.cpp via sweep.
//
// The notable difference is PRIME_NTT_DIRECT_THRESHOLD:
//  In bench_mul_small, Linux gcc has the NTT path slower than
//  Windows MSVC (n=1300 at 2.02x vs 1.37x GMP ratio), so the
//  advantageous range for Toom-8 fallback is wider.
//
// Other thresholds (Karatsuba/Toom-3/4/6/8) differ within +/-2-5% noise across both,
// so values are shared (future divergence remains possible).
//
// Re-measurement is mandatory when porting to other hardware.
#if defined(_MSC_VER) && !defined(__clang__)
  // ---- Windows MSVC ----
  constexpr size_t KARATSUBA_THRESHOLD    = 22;
  constexpr size_t SQR_KARATSUBA_THRESHOLD  = 74;
  // PRIME_NTT_DIRECT_THRESHOLD: in bench_mul_small, the NTT path at n=1300-1700
  // runs at 1.18-1.37x of GMP. In bench_toom8, Toom-8 beats NTT, but in
  // multiply() context NTT is practically competitive at n=1500-1700 (Toom-8 fallback
  // is counterproductive at some sizes). Thus Windows MSVC keeps the current value.
  constexpr size_t PRIME_NTT_DIRECT_THRESHOLD   = 1250;
  constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 1250;
#elif defined(__GNUC__) || defined(__clang__)
  // ---- Linux gcc / clang ----
  constexpr size_t KARATSUBA_THRESHOLD    = 22;
  constexpr size_t SQR_KARATSUBA_THRESHOLD  = 74;
  // Linux NTT is relatively slower than Windows; in bench, n=1300 is at 2.02x GMP.
  // Extending Toom-8 fallback to n=2999 yields n=1300: -36%, n=1500: -26%,
  // n=1700: -19%, n=2000: -6% improvement (bench_mul_small measurement).
  constexpr size_t PRIME_NTT_DIRECT_THRESHOLD   = 3000;
  constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 3000;
#else
  // ---- Unknown platform: conservative values ----
  constexpr size_t KARATSUBA_THRESHOLD    = 22;
  constexpr size_t SQR_KARATSUBA_THRESHOLD  = 74;
  constexpr size_t PRIME_NTT_DIRECT_THRESHOLD   = 1250;
  constexpr size_t SQR_PRIME_NTT_DIRECT_THRESHOLD = 1250;
#endif

// ================================================================
// Utilities
// ================================================================

// Return the actual size with leading zeros stripped
inline size_t normalized_size(const uint64_t* a, size_t n) {
  while (n > 0 && a[n - 1] == 0) --n;
  return n;
}

// Compare: positive if a > b, negative if a < b, 0 if equal
// Precondition: a, b are normalized (no leading zeros)
inline int cmp(const uint64_t* a, size_t an, const uint64_t* b, size_t bn) {
  if (an != bn) return (an > bn) ? 1 : -1;
  for (size_t i = an; i > 0; --i) {
    if (a[i - 1] != b[i - 1])
      return (a[i - 1] > b[i - 1]) ? 1 : -1;
  }
  return 0;
}

// ================================================================
// Addition / Subtraction
// ================================================================
// Note: add/sub/add_1/sub_1 intentionally use manual carry detection.
// MSVC's _addcarry_u64/_subborrow_u64 generate CF<->general-register conversions
// per iteration, making pure add/sub loops slower than manual code.
// In addmul_1/mul_1/submul_1, MUL-instruction savings exceed the conversion cost,
// so intrinsics are used. (verified by PERF-6 benchmark 2026-02)

// r = a + b, returns carry (0 or 1)
// Precondition: an >= bn, r size >= an
// r may alias a or b (in-place)
inline uint64_t add(uint64_t* r, const uint64_t* a, size_t an,
          const uint64_t* b, size_t bn) {
#ifdef SANGI_INT_HAS_ASM
  // ASM path: small-size specialization (n=1..4) + generic 8x unroll (n>4)
  uint64_t carry;
  if (bn == 0) {
    carry = 0;
  } else if (bn <= 4) {
    carry = mpn_add_n_small_asm(r, a, b, bn);
  } else {
    carry = mpn_add_n_asm(r, a, b, bn);
  }
  // Carry propagation for the remainder (an > bn)
  for (size_t i = bn; i < an; i++) {
    uint64_t sum = a[i] + carry;
    carry = (sum < a[i]) ? 1ULL : 0ULL;
    r[i] = sum;
    if (!carry) {
      if (r != a) { for (i++; i < an; i++) r[i] = a[i]; }
      return 0;
    }
  }
  return carry;
#else
  // Non-ASM path: C++ manual carry detection
  if (bn <= 2) {
    uint64_t carry = 0;
    for (size_t i = 0; i < bn; ++i) {
      uint64_t sum = a[i] + b[i];
      uint64_t c1 = (sum < a[i]) ? 1ULL : 0ULL;
      uint64_t sum2 = sum + carry;
      uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL;
      r[i] = sum2;
      carry = c1 + c2;
    }
    for (size_t i = bn; i < an; ++i) {
      uint64_t sum = a[i] + carry;
      carry = (sum < a[i]) ? 1ULL : 0ULL;
      r[i] = sum;
      if (!carry) {
        if (r != a) { for (++i; i < an; ++i) r[i] = a[i]; }
        return 0;
      }
    }
    return carry;
  }
#if defined(_MSC_VER) && defined(_M_X64)
  unsigned char carry = 0;
  size_t i = 0;
  for (; i < bn; i++) {
    carry = _addcarry_u64(carry, a[i], b[i], &r[i]);
  }
  for (; i < an; i++) {
    carry = _addcarry_u64(carry, a[i], 0, &r[i]);
    if (!carry) {
      if (r != a) {
        for (i++; i < an; i++) r[i] = a[i];
      }
      return 0;
    }
  }
  return carry;
#else
  uint64_t carry = 0;
  size_t i = 0;
  for (; i < bn; i++) {
    uint64_t sum = a[i] + b[i];
    uint64_t c1 = (sum < a[i]) ? 1ULL : 0ULL;
    uint64_t sum2 = sum + carry;
    uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL;
    r[i] = sum2;
    carry = c1 + c2;
  }
  for (; i < an; i++) {
    uint64_t sum = a[i] + carry;
    carry = (sum < a[i]) ? 1ULL : 0ULL;
    r[i] = sum;
  }
  return carry;
#endif // _MSC_VER
#endif // SANGI_INT_HAS_ASM
}

// r = a - b, returns borrow (0 or 1)
// Precondition: an >= bn, a >= b (absolute value), r size >= an
inline uint64_t sub(uint64_t* r, const uint64_t* a, size_t an,
          const uint64_t* b, size_t bn) {
#ifdef SANGI_INT_HAS_ASM
  // ASM path: small-size specialization (n=1..4) + generic 8x unroll (n>4)
  uint64_t borrow;
  if (bn == 0) {
    borrow = 0;
  } else if (bn <= 4) {
    borrow = mpn_sub_n_small_asm(r, a, b, bn);
  } else {
    borrow = mpn_sub_n_asm(r, a, b, bn);
  }
  // Borrow propagation for the remainder (an > bn)
  for (size_t i = bn; i < an; i++) {
    uint64_t diff = a[i] - borrow;
    borrow = (a[i] < borrow) ? 1ULL : 0ULL;
    r[i] = diff;
    if (!borrow) {
      if (r != a) { for (i++; i < an; i++) r[i] = a[i]; }
      return 0;
    }
  }
  return borrow;
#else
  // Non-ASM path: C++ manual borrow detection
  if (bn <= 2) {
    uint64_t borrow = 0;
    for (size_t i = 0; i < bn; ++i) {
      uint64_t diff = a[i] - b[i];
      uint64_t b1 = (a[i] < b[i]) ? 1ULL : 0ULL;
      uint64_t diff2 = diff - borrow;
      uint64_t b2 = (diff < borrow) ? 1ULL : 0ULL;
      r[i] = diff2;
      borrow = b1 + b2;
    }
    for (size_t i = bn; i < an; ++i) {
      uint64_t diff = a[i] - borrow;
      borrow = (a[i] < borrow) ? 1ULL : 0ULL;
      r[i] = diff;
      if (!borrow) {
        if (r != a) { for (++i; i < an; ++i) r[i] = a[i]; }
        return 0;
      }
    }
    return borrow;
  }
#if defined(_MSC_VER) && defined(_M_X64)
  unsigned char borrow = 0;
  size_t i = 0;
  for (; i < bn; i++) {
    borrow = _subborrow_u64(borrow, a[i], b[i], &r[i]);
  }
  for (; i < an; i++) {
    borrow = _subborrow_u64(borrow, a[i], 0, &r[i]);
    if (!borrow) {
      if (r != a) {
        for (i++; i < an; i++) r[i] = a[i];
      }
      return 0;
    }
  }
  return borrow;
#else
  uint64_t borrow = 0;
  size_t i = 0;
  for (; i < bn; i++) {
    uint64_t diff = a[i] - b[i];
    uint64_t b1 = (a[i] < b[i]) ? 1ULL : 0ULL;
    uint64_t diff2 = diff - borrow;
    uint64_t b2 = (diff < borrow) ? 1ULL : 0ULL;
    r[i] = diff2;
    borrow = b1 + b2;
  }
  for (; i < an; i++) {
    uint64_t diff = a[i] - borrow;
    borrow = (a[i] < borrow) ? 1ULL : 0ULL;
    r[i] = diff;
  }
  return borrow;
#endif // _MSC_VER
#endif // SANGI_INT_HAS_ASM
}

// r[0..n-1] += b (single limb), returns carry
inline uint64_t add_1(uint64_t* r, size_t n, uint64_t b) {
  if (n == 0) return b;
#if defined(_MSC_VER) && defined(_M_X64)
  unsigned char carry = _addcarry_u64(0, r[0], b, &r[0]);
  for (size_t i = 1; i < n && carry; i++) {
    carry = _addcarry_u64(carry, r[i], 0, &r[i]);
  }
  return carry;
#else
  for (size_t i = 0; i < n; i++) {
    uint64_t old = r[i];
    r[i] += b;
    b = (r[i] < old) ? 1ULL : 0ULL;
    if (b == 0) break;
  }
  return b;
#endif
}

// r[0..n-1] -= b (single limb), returns borrow
inline uint64_t sub_1(uint64_t* r, size_t n, uint64_t b) {
  if (n == 0) return b;
#if defined(_MSC_VER) && defined(_M_X64)
  unsigned char borrow = _subborrow_u64(0, r[0], b, &r[0]);
  for (size_t i = 1; i < n && borrow; i++) {
    borrow = _subborrow_u64(borrow, r[i], 0, &r[i]);
  }
  return borrow;
#else
  for (size_t i = 0; i < n; i++) {
    uint64_t old = r[i];
    r[i] -= b;
    b = (old < b) ? 1ULL : 0ULL;
    if (b == 0) break;
  }
  return b;
#endif
}

// r = a - b - borrow_in (with initial borrow), returns final borrow
// Equivalent to GMP mpn_sub_nc
inline uint64_t sub_nc(uint64_t* r, const uint64_t* a, size_t n,
            const uint64_t* b, size_t bn, uint64_t borrow_in) {
  uint64_t borrow = sub(r, a, n, b, bn);
  if (borrow_in) borrow += sub_1(r, n, borrow_in);
  return borrow;
}

// r = a - 2*b, returns total borrow (0, 1, or 2)
// Single pass: subtract while left-shifting b[i]
inline uint64_t sublsh1_n(uint64_t* r, const uint64_t* a,
              const uint64_t* b, size_t n) {
#if defined(_MSC_VER) && defined(_M_X64)
  // MSVC intrinsics: single pass via _subborrow_u64
  unsigned char borrow = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t b2 = b[i] << 1;     // low part of 2*b[i]
    uint64_t b2_hi = b[i] >> 63;   // carry of 2*b[i]
    // r[i] = a[i] - b2 - borrow
    borrow = _subborrow_u64(borrow, a[i], b2, &r[i]);
    // Add b2_hi (carry from MSB of b[i]) as extra borrow
    borrow += static_cast<unsigned char>(b2_hi);
  }
  return borrow;
#else
  uint64_t borrow = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t b2 = b[i] << 1;
    uint64_t b2_hi = b[i] >> 63;
    uint64_t diff = a[i] - b2;
    uint64_t bw1 = (a[i] < b2) ? 1ULL : 0ULL;
    uint64_t diff2 = diff - borrow;
    uint64_t bw2 = (diff < borrow) ? 1ULL : 0ULL;
    r[i] = diff2;
    borrow = bw1 + bw2 + b2_hi;
  }
  return borrow;
#endif
}

// r = 2*b - a, returns sign: -1 (negative), 0 (exact), +1 (carry)
// Single pass: subtract while left-shifting b[i]
inline int64_t rsblsh1_n(uint64_t* r, const uint64_t* a,
             const uint64_t* b, size_t n) {
#if defined(_MSC_VER) && defined(_M_X64)
  // Single pass: compute 2*b[i] - a[i]
  unsigned char borrow = 0;
  uint64_t shift_carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t b2 = (b[i] << 1) | shift_carry;
    shift_carry = b[i] >> 63;
    borrow = _subborrow_u64(borrow, b2, a[i], &r[i]);
  }
  // shift_carry is the top carry of 2*b (0 or 1)
  // borrow is the subtraction borrow (0 or 1)
  return static_cast<int64_t>(shift_carry) - static_cast<int64_t>(borrow);
#else
  uint64_t borrow = 0;
  uint64_t shift_carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t b2 = (b[i] << 1) | shift_carry;
    shift_carry = b[i] >> 63;
    uint64_t diff = b2 - a[i];
    uint64_t bw1 = (b2 < a[i]) ? 1ULL : 0ULL;
    uint64_t diff2 = diff - borrow;
    uint64_t bw2 = (diff < borrow) ? 1ULL : 0ULL;
    r[i] = diff2;
    borrow = bw1 + bw2;
  }
  return static_cast<int64_t>(shift_carry) - static_cast<int64_t>(borrow);
#endif
}

// r += a << shift (shift=1..63), returns carry (shift bits + add carry)
// addmul_1(r, a, n, 2^shift) fast version: no MULX needed, only shift + add
inline uint64_t addlsh_n(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) {
  if (n == 0) return 0;
  unsigned rs = 64 - shift;
#if defined(_MSC_VER) && defined(_M_X64)
  uint64_t shift_carry = 0;
  unsigned char adc = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t shifted = (a[i] << shift) | shift_carry;
    shift_carry = a[i] >> rs;
    adc = _addcarry_u64(adc, r[i], shifted, &r[i]);
  }
  return shift_carry + adc;
#else
  uint64_t shift_carry = 0;
  uint64_t add_carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t shifted = (a[i] << shift) | shift_carry;
    shift_carry = a[i] >> rs;
    uint64_t sum = r[i] + shifted;
    uint64_t c1 = (sum < shifted) ? 1ULL : 0ULL;
    uint64_t sum2 = sum + add_carry;
    uint64_t c2 = (sum2 < sum) ? 1ULL : 0ULL;
    r[i] = sum2;
    add_carry = c1 + c2;
  }
  return shift_carry + add_carry;
#endif
}

// r -= a << shift (shift=1..63), returns borrow
// submul_1(r, a, n, 2^shift) fast version: no MULX needed
inline uint64_t sublsh_n(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) {
  if (n == 0) return 0;
  unsigned rs = 64 - shift;
#if defined(_MSC_VER) && defined(_M_X64)
  uint64_t shift_carry = 0;
  unsigned char borrow = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t shifted = (a[i] << shift) | shift_carry;
    shift_carry = a[i] >> rs;
    borrow = _subborrow_u64(borrow, r[i], shifted, &r[i]);
  }
  return shift_carry + borrow;
#else
  uint64_t shift_carry = 0;
  uint64_t sub_borrow = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t shifted = (a[i] << shift) | shift_carry;
    shift_carry = a[i] >> rs;
    uint64_t diff = r[i] - shifted;
    uint64_t b1 = (r[i] < shifted) ? 1ULL : 0ULL;
    uint64_t diff2 = diff - sub_borrow;
    uint64_t b2 = (diff < sub_borrow) ? 1ULL : 0ULL;
    r[i] = diff2;
    sub_borrow = b1 + b2;
  }
  return shift_carry + sub_borrow;
#endif
}

// r += a * b (single limb), returns carry
// addmul_1: inner loop of basecase multiplication
inline uint64_t addmul_1(uint64_t* r, const uint64_t* a, size_t an, uint64_t b) {
#ifdef SANGI_INT_HAS_ASM
  if (detail::has_bmi2_adx()) {
    if (an <= 4) return mpn_addmul_1_small_asm(r, a, an, b);
    return mpn_addmul_1_mulx(r, a, an, b);
  }
#endif
  uint64_t carry = 0;
#if defined(_MSC_VER) && defined(_M_X64)
  // MSVC x64: _umul128 + _addcarry_u64 intrinsics
  // 4x loop unroll to encourage pipelined parallel execution of MUL
  size_t i = 0;
  for (; i + 4 <= an; i += 4) {
    uint64_t hi0, hi1, hi2, hi3;
    uint64_t lo0 = _umul128(a[i + 0], b, &hi0);
    uint64_t lo1 = _umul128(a[i + 1], b, &hi1);
    uint64_t lo2 = _umul128(a[i + 2], b, &hi2);
    uint64_t lo3 = _umul128(a[i + 3], b, &hi3);

    // c1, c2 are independent. Do not chain (do not mix c1 into r[i])
    unsigned char c1, c2;
    c1 = _addcarry_u64(0, lo0, carry, &lo0);
    c2 = _addcarry_u64(0, lo0, r[i + 0], &r[i + 0]);
    carry = hi0 + c1 + c2;

    c1 = _addcarry_u64(0, lo1, carry, &lo1);
    c2 = _addcarry_u64(0, lo1, r[i + 1], &r[i + 1]);
    carry = hi1 + c1 + c2;

    c1 = _addcarry_u64(0, lo2, carry, &lo2);
    c2 = _addcarry_u64(0, lo2, r[i + 2], &r[i + 2]);
    carry = hi2 + c1 + c2;

    c1 = _addcarry_u64(0, lo3, carry, &lo3);
    c2 = _addcarry_u64(0, lo3, r[i + 3], &r[i + 3]);
    carry = hi3 + c1 + c2;
  }
  for (; i < an; i++) {
    uint64_t hi;
    uint64_t lo = _umul128(a[i], b, &hi);
    unsigned char c1 = _addcarry_u64(0, lo, carry, &lo);
    unsigned char c2 = _addcarry_u64(0, lo, r[i], &r[i]);
    carry = hi + c1 + c2;
  }
#elif defined(__SIZEOF_INT128__)
  // GCC/Clang: let the compiler generate optimal instructions via __uint128_t
  __uint128_t cy = carry;
  size_t i = 0;
  for (; i + 4 <= an; i += 4) {
    cy += (__uint128_t)a[i+0] * b + r[i+0]; r[i+0] = (uint64_t)cy; cy >>= 64;
    cy += (__uint128_t)a[i+1] * b + r[i+1]; r[i+1] = (uint64_t)cy; cy >>= 64;
    cy += (__uint128_t)a[i+2] * b + r[i+2]; r[i+2] = (uint64_t)cy; cy >>= 64;
    cy += (__uint128_t)a[i+3] * b + r[i+3]; r[i+3] = (uint64_t)cy; cy >>= 64;
  }
  for (; i < an; i++) {
    cy += (__uint128_t)a[i] * b + r[i]; r[i] = (uint64_t)cy; cy >>= 64;
  }
  return (uint64_t)cy;
#else
  for (size_t i = 0; i < an; i++) {
    UInt128 prod = UInt128::multiply(a[i], b);
    uint64_t lo = prod.low + carry;
    uint64_t c1 = (lo < prod.low) ? 1ULL : 0ULL;
    uint64_t lo2 = lo + r[i];
    uint64_t c2 = (lo2 < lo) ? 1ULL : 0ULL;
    r[i] = lo2;
    carry = prod.high + c1 + c2;
  }
#endif
  return carry;
}

// r -= a * b (single limb), returns borrow
// submul_1: inner loop of schoolbook division
// ADCX/ADOX dual carry-chain cannot be used (SUB clobbers OF)
// 4x loop unroll to encourage pipelined parallel execution of MUL
inline uint64_t submul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b) {
#ifdef SANGI_INT_HAS_ASM
  if (detail::has_bmi2_adx()) {
    return mpn_submul_1_mulx(r, a, n, b);
  }
#endif
  uint64_t carry = 0;
#if defined(_MSC_VER) && defined(_M_X64)
  size_t i = 0;
  for (; i + 4 <= n; i += 4) {
    uint64_t hi0, hi1, hi2, hi3;
    uint64_t lo0 = _umul128(a[i + 0], b, &hi0);
    uint64_t lo1 = _umul128(a[i + 1], b, &hi1);
    uint64_t lo2 = _umul128(a[i + 2], b, &hi2);
    uint64_t lo3 = _umul128(a[i + 3], b, &hi3);

    unsigned char c;
    c = _addcarry_u64(0, lo0, carry, &lo0);
    carry = hi0 + c;
    c = _subborrow_u64(0, r[i + 0], lo0, &r[i + 0]);
    carry += c;

    c = _addcarry_u64(0, lo1, carry, &lo1);
    carry = hi1 + c;
    c = _subborrow_u64(0, r[i + 1], lo1, &r[i + 1]);
    carry += c;

    c = _addcarry_u64(0, lo2, carry, &lo2);
    carry = hi2 + c;
    c = _subborrow_u64(0, r[i + 2], lo2, &r[i + 2]);
    carry += c;

    c = _addcarry_u64(0, lo3, carry, &lo3);
    carry = hi3 + c;
    c = _subborrow_u64(0, r[i + 3], lo3, &r[i + 3]);
    carry += c;
  }
  for (; i < n; i++) {
    uint64_t hi;
    uint64_t lo = _umul128(a[i], b, &hi);
    unsigned char c;
    c = _addcarry_u64(0, lo, carry, &lo);
    carry = hi + c;
    c = _subborrow_u64(0, r[i], lo, &r[i]);
    carry += c;
  }
#elif defined(__SIZEOF_INT128__)
  __uint128_t cy = 0;
  for (size_t i = 0; i < n; i++) {
    cy += (__uint128_t)a[i] * b;
    uint64_t pl = (uint64_t)cy;
    uint64_t prev = r[i];
    r[i] = prev - pl;
    cy >>= 64;
    cy += (prev < pl) ? 1ULL : 0ULL;
  }
  return (uint64_t)cy;
#else
  for (size_t i = 0; i < n; i++) {
    UInt128 prod = UInt128::multiply(a[i], b);
    uint64_t pl = prod.low + carry;
    uint64_t c1 = (pl < prod.low) ? 1ULL : 0ULL;
    uint64_t prev = r[i];
    r[i] = prev - pl;
    uint64_t c2 = (prev < pl) ? 1ULL : 0ULL;
    carry = prod.high + c1 + c2;
  }
#endif
  return carry;
}

// ================================================================
// multiplication
// ================================================================

// Forward declaration (mul_basecase uses mul_1)
inline uint64_t mul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b);

// Basecase multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// r must not overlap a or b
inline void mul_basecase(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }

#ifdef SANGI_INT_HAS_ASM
  if (detail::has_bmi2_adx()) {
    // n*n small-size specialization:
    //  n=1..4: ADD/ADC unroll, n=5..7: ADCX/ADOX dual chain
    if (an == bn && bn <= 7) {
      mpn_mul_small_asm(r, a, b, bn);
      return;
    }
    // 8x8 specialization: cyclic 9-reg accumulator + ADCX/ADOX dual chain
    if (an == 8 && bn == 8) {
      mpn_mul_8x8_asm(r, a, b);
      return;
    }
    // Generic asm version: all performs handled with a single push/pop
    mpn_mul_basecase_mulx(r, a, an, b, bn);
    return;
  }
#endif
  // Non-ASM path: small-size specialization - shared by GCC/MSVC
  // * Codex feedback #2: the most frequent region in real-world SBO 9-limb usage
#if defined(__GNUC__) && defined(__SIZEOF_INT128__)
  // GCC/Clang: expand 1x1, 2x2 via __uint128_t
  typedef unsigned __int128 u128;
  if (bn == 1 && an == 1) {
    u128 p = (u128)a[0] * b[0];
    r[0] = (uint64_t)p; r[1] = (uint64_t)(p >> 64);
    return;
  }
  if (bn == 2 && an == 2) {
    u128 p00 = (u128)a[0] * b[0];
    u128 p01 = (u128)a[0] * b[1];
    u128 p10 = (u128)a[1] * b[0];
    u128 p11 = (u128)a[1] * b[1];
    r[0] = (uint64_t)p00;
    u128 mid = (p00 >> 64) + (uint64_t)p01 + (uint64_t)p10;
    r[1] = (uint64_t)mid;
    u128 hi = (mid >> 64) + (p01 >> 64) + (p10 >> 64) + (uint64_t)p11;
    r[2] = (uint64_t)hi;
    r[3] = (uint64_t)(hi >> 64) + (uint64_t)(p11 >> 64);
    return;
  }
#elif defined(_MSC_VER) && defined(_M_X64)
  if (bn == 1) {
    if (an == 1) {
      uint64_t hi;
      r[0] = _umul128(a[0], b[0], &hi);
      r[1] = hi;
      return;
    }
    if (an == 2) {
      uint64_t h0, h1;
      r[0] = _umul128(a[0], b[0], &h0);
      r[1] = _umul128(a[1], b[0], &h1);
      unsigned char c = _addcarry_u64(0, r[1], h0, &r[1]);
      r[2] = h1 + c;
      return;
    }
  }
#endif
  // First row: write directly via mul_1 (lighter than memset + addmul_1)
  r[an] = mul_1(r, a, an, b[0]);
  // Remaining performs: accumulate via addmul_1
  for (size_t j = 1; j < bn; j++) {
    if (b[j] == 0) { r[an + j] = 0; continue; }
    r[an + j] = addmul_1(r + j, a, an, b[j]);
  }
}

// Short multiplication: computes only the top rn words of the product
// rp[0..rn-1] = top rn words of a[0..an-1] * b[0..bn-1]
// rn is supplied by the caller including 2 guard words
// Requirement: an >= 1, bn >= 1, rn >= 1, rn <= an+bn
inline void mulhigh_basecase(uint64_t* rp,
               const uint64_t* ap, size_t an,
               const uint64_t* bp, size_t bn,
               size_t rn) {
  if (an < bn) { std::swap(ap, bp); std::swap(an, bn); }

  size_t total = an + bn;
  // If rn >= total then it equals the full product -> fall back to normal version
  if (rn >= total) {
    mul_basecase(rp, ap, an, bp, bn);
    return;
  }

  size_t lo = total - rn; // number of low columns to skip

  // Zero-initialize the output buffer
  std::memset(rp, 0, rn * sizeof(uint64_t));

  // Write the first contributing row directly with mul_1 (lighter than addmul_1)
  bool first_row = true;

  for (size_t j = 0; j < bn; j++) {
    if (bp[j] == 0) continue;

    // Row j contributes to columns j..j+an-1
    // Compute only columns >= lo -> adjust starting position in a
    size_t i_start = (j >= lo) ? 0 : (lo - j);
    if (i_start >= an) continue; // this row contributes nothing

    size_t count = an - i_start;
    size_t r_pos = j + i_start - lo; // position within the output buffer

    uint64_t cy;
    if (first_row) {
      cy = mul_1(rp + r_pos, ap + i_start, count, bp[j]);
      first_row = false;
    } else {
      cy = addmul_1(rp + r_pos, ap + i_start, count, bp[j]);
    }

    // Propagate carry
    size_t cy_pos = r_pos + count;
    if (cy != 0 && cy_pos < rn) {
      add_1(rp + cy_pos, rn - cy_pos, cy);
    }
  }
}

// ================================================================
// mulhigh_n: fast computation of the top n limbs (skips the low half of the product)
// ================================================================

// scratch size for mulhigh_n
// Since multiply is forward-declared, allocate conservatively here
inline size_t mulhigh_n_scratch_size(size_t n);

// mulhigh_n: compute the top n limbs of a[0..n-1] * b[0..n-1]
// rp[0..n-1] = floor(a * b / B^n) (approximate, at most O(1) error)
// Used for newton-division quotient estimation. Errors are absorbed by the correction loop.
//
// Algorithm: Karatsuba decomposition a=a1*B^h+a0, b=b1*B^h+b0;
//  skip a0*b0, compute only the upper parts of a1*b1 and
//  cross=(a0+a1)*(b0+b1)-a1*b1. multiplication cost ~ 2/3 * M(n).
inline void mulhigh_n(uint64_t* rp,
            const uint64_t* ap, const uint64_t* bp,
            size_t n, uint64_t* scratch);

// Compute the scratch size required by Karatsuba
inline size_t mul_karatsuba_scratch_size(size_t n) {
  return (n < 16) ? 128 : 8 * n + 64;
}

// Karatsuba multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// r must not overlap a or b
// scratch: temporary buffer (size >= mul_karatsuba_scratch_size(max(an,bn)))
inline void mul_karatsuba(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }
  if (bn < KARATSUBA_THRESHOLD) {
    mul_basecase(r, a, an, b, bn);
    return;
  }

  size_t half = (an + 1) / 2;

  const uint64_t* a0 = a;
  size_t a0n = std::min(half, an);
  const uint64_t* a1 = a + half;
  size_t a1n = (an > half) ? an - half : 0;

  const uint64_t* b0 = b;
  size_t b0n = std::min(half, bn);
  const uint64_t* b1 = b + half;
  size_t b1n = (bn > half) ? bn - half : 0;

  a0n = normalized_size(a0, a0n);
  a1n = normalized_size(a1, a1n);
  b0n = normalized_size(b0, b0n);
  b1n = normalized_size(b1, b1n);

  size_t rn = an + bn;

  uint64_t* s = scratch;
  uint64_t* t = scratch + (half + 1);
  uint64_t* middle = scratch + 2 * (half + 1);
  size_t middle_max = 2 * (half + 2);
  uint64_t* rec_scratch = scratch + 2 * (half + 1) + middle_max;

  size_t v0n = 0, vinfn = 0;
  if (a0n > 0 && b0n > 0) {
    mul_karatsuba(r, a0, a0n, b0, b0n, rec_scratch);
    v0n = a0n + b0n;
  }

  if (a1n > 0 && b1n > 0) {
    mul_karatsuba(r + 2 * half, a1, a1n, b1, b1n, rec_scratch);
    vinfn = a1n + b1n;
  }

  // Zero out the gap region (between v0 and vinf, and after vinf)
  if (v0n < 2 * half)
    std::memset(r + v0n, 0, (2 * half - v0n) * sizeof(uint64_t));
  if (2 * half + vinfn < rn)
    std::memset(r + 2 * half + vinfn, 0, (rn - 2 * half - vinfn) * sizeof(uint64_t));

  size_t sn, tn;
  if (a0n >= a1n) {
    if (a1n > 0) {
      uint64_t carry = add(s, a0, a0n, a1, a1n);
      sn = a0n;
      if (carry) { s[sn] = carry; sn++; }
    } else {
      std::memcpy(s, a0, a0n * sizeof(uint64_t));
      sn = a0n;
    }
  } else {
    uint64_t carry = add(s, a1, a1n, a0, a0n);
    sn = a1n;
    if (carry) { s[sn] = carry; sn++; }
  }

  if (b0n >= b1n) {
    if (b1n > 0) {
      uint64_t carry = add(t, b0, b0n, b1, b1n);
      tn = b0n;
      if (carry) { t[tn] = carry; tn++; }
    } else {
      std::memcpy(t, b0, b0n * sizeof(uint64_t));
      tn = b0n;
    }
  } else {
    uint64_t carry = add(t, b1, b1n, b0, b0n);
    tn = b1n;
    if (carry) { t[tn] = carry; tn++; }
  }

  size_t mn = 0;
  if (sn > 0 && tn > 0) {
    mul_karatsuba(middle, s, sn, t, tn, rec_scratch);
    mn = normalized_size(middle, sn + tn);
  }

  {
    size_t t0n = normalized_size(r, a0n + b0n);
    if (t0n > 0 && mn > 0) {
      sub(middle, middle, mn, r, t0n);
      mn = normalized_size(middle, mn);
    }
  }
  {
    // Get the normalized size of vinf. a1n + b1n may exceed the output buffer tail (rn - 2*half)
    // (when bn < half, b1n=0 but a1n > rn-2*half).
    // Regions beyond rn are uninitialized and must not be read.
    size_t vinf_max = std::min(a1n + b1n, rn - 2 * half);
    size_t t2n = normalized_size(r + 2 * half, vinf_max);
    if (t2n > 0 && mn > 0) {
      sub(middle, middle, mn, r + 2 * half, t2n);
      mn = normalized_size(middle, mn);
    }
  }

  if (mn > 0) {
    uint64_t carry = add(r + half, r + half, rn - half, middle, mn);
    (void)carry;
  }
}

// ================================================================
// Exact Division (raw-limb version)
// ================================================================

// 3 exact division: r[0..n-1] = a[0..n-1] / 3
inline size_t divexact_by3(uint64_t* r, const uint64_t* a, size_t n) {
  static constexpr uint64_t INV3 = 0xAAAAAAAAAAAAAAABULL;
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * INV3;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, 3);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// Right-shift by 3 bits
inline size_t shift_right_3(uint64_t* r, const uint64_t* a, size_t n) {
  if (n == 0) return 0;
  for (size_t i = 0; i < n - 1; i++) {
    r[i] = (a[i] >> 3) | (a[i + 1] << 61);
  }
  r[n - 1] = a[n - 1] >> 3;
  return normalized_size(r, n);
}

// 5 exact division: r[0..n-1] = a[0..n-1] / 5
inline size_t divexact_by5(uint64_t* r, const uint64_t* a, size_t n) {
  static constexpr uint64_t INV5 = 0xCCCCCCCCCCCCCCCDULL;
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * INV5;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, 5);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// 7 exact division: r[0..n-1] = a[0..n-1] / 7
inline size_t divexact_by7(uint64_t* r, const uint64_t* a, size_t n) {
  static constexpr uint64_t INV7 = 0x6DB6DB6DB6DB6DB7ULL;
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * INV7;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, 7);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// 11 exact division: r[0..n-1] = a[0..n-1] / 11
// Used for the c11 divisor in Toom-8 interpolation (59875200 = 2^7 * 3^5 * 5^2 * 7 * 11)
inline size_t divexact_by11(uint64_t* r, const uint64_t* a, size_t n) {
  static constexpr uint64_t INV11 = 0x2E8BA2E8BA2E8BA3ULL;
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * INV11;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, 11);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// 13 exact division: r[0..n-1] = a[0..n-1] / 13
// Used for the c13 divisor in Toom-8 interpolation (18681062400 = 2^10 * 3^6 * 5^2 * 7 * 11 * 13)
inline size_t divexact_by13(uint64_t* r, const uint64_t* a, size_t n) {
  static constexpr uint64_t INV13 = 0x4EC4EC4EC4EC4EC5ULL;
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * INV13;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, 13);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// Hensel inverse of an odd 64-bit constant (d * inv ≡ 1 mod 2^64)
// newton iteration: inv <- inv * (2 - d * inv); converges to 64-bit precision in 6 steps
constexpr uint64_t hensel_inverse_u64(uint64_t d) {
  uint64_t inv = 1;
  for (int i = 0; i < 6; i++) inv = inv * (2 - d * inv);
  return inv;
}

// Exact division by an arbitrary odd 64-bit constant d: r = a / d
// inv is the Hensel inverse of d (precomputed by hensel_inverse_u64).
// Used in Toom-8 interpolation to replace chained divexact_by_{3,5,7,11,13} with a single pass
inline size_t divexact_by_odd(uint64_t* r, const uint64_t* a, size_t n,
               uint64_t d, uint64_t inv) {
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    uint64_t ai_minus_carry = a[i] - carry;
    uint64_t borrow = (a[i] < carry) ? 1ULL : 0ULL;
    uint64_t yi = ai_minus_carry * inv;
    r[i] = yi;
    UInt128 prod = UInt128::multiply(yi, d);
    carry = prod.high + borrow;
  }
  return normalized_size(r, n);
}

// 24 exact division: r = a / 24 = (a >> 3) / 3
inline size_t divexact_by24(uint64_t* r, const uint64_t* a, size_t n) {
  size_t sn = shift_right_3(r, a, n);
  return divexact_by3(r, r, sn);
}

// Right-shift by 2 bits
inline size_t shift_right_2(uint64_t* r, const uint64_t* a, size_t n) {
  if (n == 0) return 0;
  for (size_t i = 0; i < n - 1; i++) {
    r[i] = (a[i] >> 2) | (a[i + 1] << 62);
  }
  r[n - 1] = a[n - 1] >> 2;
  return normalized_size(r, n);
}

// 12 exact division: r = a / 12 = (a >> 2) / 3
inline size_t divexact_by12(uint64_t* r, const uint64_t* a, size_t n) {
  size_t sn = shift_right_2(r, a, n);
  return divexact_by3(r, r, sn);
}

// 120 exact division: r = a / 120 = (a >> 3) / 15
// Folds /3 + /5 into a single pass via the Hensel inverse of 15 = 3*5 (3 passes -> 2 passes)
// Used in c5 computation for Toom-4/6/8 and in sqr_toomcook4
inline size_t divexact_by120(uint64_t* r, const uint64_t* a, size_t n) {
  size_t sn = shift_right_3(r, a, n);
  constexpr uint64_t D_15 = 15ULL;
  constexpr uint64_t INV_15 = hensel_inverse_u64(D_15);
  return divexact_by_odd(r, r, sn, D_15, INV_15);
}

// ================================================================
// Left shift (small bit count)
// ================================================================

// r = a << shift, returns overflow word
//
// Note: when r overlaps a with r > a (forward overlap), the low->high processing of a+a / _addcarry
// overwrites a[i+k] that will be read later (reading back at index i+k yields 0).
// The shift=1 fast path is valid only for "same location (r == a)" or "no overlap (r >= a+n or r+n <= a)".
// Forward overlap (a < r < a+n) is handled high->low by the generic loop.
inline uint64_t lshift(uint64_t* r, const uint64_t* a, size_t n, unsigned shift) {
  if (n == 0 || shift == 0) return 0;
  // Forward overlap (r > a, and r < a + n): high->low processing required
  bool forward_overlap = (r > a) && (r < a + n);
#ifdef SANGI_INT_HAS_ASM
  if (shift == 1 && !forward_overlap) {
    // shift=1 optimized via a+a (ADC chain) - only when no overlap or in-place
    return mpn_add_n_asm(r, a, a, n);
  }
  if (!forward_overlap) {
    // Fast shift via SHLD instruction (shift 2-63) - likewise only when no overlap or in-place
    return mpn_lshift_asm(r, a, n, shift);
  }
  // forward_overlap: fall back to high->low loop
#elif defined(_MSC_VER) && defined(_M_X64)
  if (shift == 1 && !forward_overlap) {
    unsigned char c = 0;
    for (size_t i = 0; i < n; i++) {
      c = _addcarry_u64(c, a[i], a[i], &r[i]);
    }
    return c;
  }
#endif
  if (forward_overlap) {
    // high->low: r[i] depends only on a[i+1] (or a[i]); writing in reverse order is overlap-safe.
    unsigned rs = 64 - shift;
    uint64_t carry = a[n - 1] >> rs; // return value (top overflow)
    for (size_t i = n - 1; i > 0; i--) {
      r[i] = (a[i] << shift) | (a[i - 1] >> rs);
    }
    r[0] = a[0] << shift;
    return carry;
  }
  uint64_t carry = 0;
  unsigned rs = 64 - shift;
  for (size_t i = 0; i < n; i++) {
    uint64_t v = a[i];
    r[i] = (v << shift) | carry;
    carry = v >> rs;
  }
  return carry;
}

// ================================================================
// Right-shift by 1 bit (for /2 division)
// ================================================================

// r = a >> 1, returns normalized size
// r may alias a (in-place)
inline size_t rshift_1(uint64_t* r, const uint64_t* a, size_t n) {
  if (n == 0) return 0;
#ifdef SANGI_INT_HAS_ASM
  mpn_rshift_asm(r, a, n, 1);
#else
  for (size_t i = 0; i < n - 1; i++) {
    r[i] = (a[i] >> 1) | (a[i + 1] << 63);
  }
  r[n - 1] = a[n - 1] >> 1;
#endif
  return normalized_size(r, n);
}

// ================================================================
// Single-limb multiplication
// ================================================================

// r = a * b (single limb), returns carry
inline uint64_t mul_1(uint64_t* r, const uint64_t* a, size_t n, uint64_t b) {
#ifdef SANGI_INT_HAS_ASM
  if (detail::has_bmi2_adx()) {
    return mpn_mul_1_mulx(r, a, n, b);
  }
#endif
  uint64_t carry = 0;
#if defined(_MSC_VER) && defined(_M_X64)
  for (size_t i = 0; i < n; i++) {
    uint64_t hi;
    uint64_t lo = _umul128(a[i], b, &hi);
    unsigned char c = _addcarry_u64(0, lo, carry, &r[i]);
    carry = hi + c;
  }
#elif defined(__SIZEOF_INT128__)
  __uint128_t cy = 0;
  for (size_t i = 0; i < n; i++) {
    cy += (__uint128_t)a[i] * b;
    r[i] = (uint64_t)cy;
    cy >>= 64;
  }
  return (uint64_t)cy;
#else
  for (size_t i = 0; i < n; i++) {
    UInt128 prod = UInt128::multiply(a[i], b);
    uint64_t lo = prod.low + carry;
    uint64_t c = (lo < prod.low) ? 1ULL : 0ULL;
    r[i] = lo;
    carry = prod.high + c;
  }
#endif
  return carry;
}

// ================================================================
// Toom-Cook-3 helpers
// ================================================================

// r = a + b (size order arbitrary)
// r requires space of max(an,bn)+1 limbs
// Returns the normalized size
inline size_t add_any(uint64_t* r,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn) {
  if (an == 0) { if (bn > 0) std::memcpy(r, b, bn * sizeof(uint64_t)); return bn; }
  if (bn == 0) { if (an > 0) std::memcpy(r, a, an * sizeof(uint64_t)); return an; }
  if (an < bn) { std::swap(a, b); std::swap(an, bn); }
  uint64_t carry = add(r, a, an, b, bn);
  if (carry) { r[an] = carry; return an + 1; }
  return an;
}

// r = |a - b|, sets sign (+1: a>=b, -1: a<b, 0: a==b)
// r requires space of max(an,bn) limbs
// r may alias a
// Returns the normalized size
inline size_t abs_sub(uint64_t* r, int& sign,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn) {
  an = normalized_size(a, an);
  bn = normalized_size(b, bn);
  int c = cmp(a, an, b, bn);
  if (c == 0) { sign = 0; return 0; }
  if (c > 0) {
    sign = 1;
    sub(r, a, an, b, bn);
    return normalized_size(r, an);
  } else {
    sign = -1;
    sub(r, b, bn, a, an);
    return normalized_size(r, bn);
  }
}

// ================================================================
// Toom-Cook-3 multiplication
// ================================================================

constexpr size_t TOOMCOOK3_THRESHOLD = 140;

inline size_t mul_toomcook3_scratch_size(size_t n) {
  if (n < TOOMCOOK3_THRESHOLD) return mul_karatsuba_scratch_size(n);
  return 30 * n + 256;
}

// Toom-Cook-3 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// Evaluation points {0, 1, -1, 2, ∞} (GMP style)
// interpolation uses only /2 (shift) and /3 (divexact_by3); no /24 needed
// r must not overlap a or b
inline void mul_toomcook3(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }
  if (bn < TOOMCOOK3_THRESHOLD) {
    mul_karatsuba(r, a, an, b, bn, scratch);
    return;
  }

  size_t k = (an + 2) / 3;
  size_t rn = an + bn;

  // --- 3-way split (pointer arithmetic only, no copying) ---
  const uint64_t* a0 = a;
  size_t a0n = normalized_size(a0, std::min(k, an));
  const uint64_t* a1 = a + k;
  size_t a1n = (an > k) ? normalized_size(a1, std::min(k, an - k)) : 0;
  const uint64_t* a2 = a + 2 * k;
  size_t a2n = (an > 2 * k) ? normalized_size(a2, an - 2 * k) : 0;

  const uint64_t* b0 = b;
  size_t b0n = normalized_size(b0, std::min(k, bn));
  const uint64_t* b1 = b + k;
  size_t b1n = (bn > k) ? normalized_size(b1, std::min(k, bn - k)) : 0;
  const uint64_t* b2 = b + 2 * k;
  size_t b2n = (bn > 2 * k) ? normalized_size(b2, bn - 2 * k) : 0;

  // --- Scratch layout ---
  size_t blk = 2 * (k + 4); // max size of each buffer
  uint64_t* v1_buf   = scratch;        // v(1) product
  uint64_t* vm1_buf   = scratch + blk;     // v(-1) product
  uint64_t* v2_buf   = scratch + 2 * blk;   // v(2) product
  uint64_t* tmp1    = scratch + 3 * blk;   // evaluation temporary 1
  uint64_t* tmp2    = scratch + 4 * blk;   // evaluation temporary 2
  uint64_t* interp_buf = scratch + 5 * blk;   // interpolation workspace
  uint64_t* rec_scratch = scratch + 6 * blk;   // recursion scratch

  size_t v1n = 0, vm1n = 0, v2n = 0;
  int vm1_sign = 0;

  // ============================================================
  // pointwise multiplication (5 recursive calls)
  // ============================================================

  // Point 0: v0 = a0 * b0 → r[0..]
  size_t v0n = 0;
  if (a0n > 0 && b0n > 0) {
    mul_toomcook3(r, a0, a0n, b0, b0n, rec_scratch);
    v0n = normalized_size(r, std::min(a0n + b0n, rn));
  }

  // Point ∞: vinf = a2 * b2 → r[4k..]
  size_t vinfn = 0;
  size_t vinf_off = 4 * k;
  if (a2n > 0 && b2n > 0 && vinf_off < rn) {
    mul_toomcook3(r + vinf_off, a2, a2n, b2, b2n, rec_scratch);
    vinfn = normalized_size(r + vinf_off, std::min(a2n + b2n, rn - vinf_off));
  }

  // Zero out the gap region (between v0 and vinf, and after vinf)
  {
    size_t gap_start = v0n;
    size_t gap_end = std::min(vinf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(vinf_off + vinfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // Point 1: v1 = (a0+a1+a2) * (b0+b1+b2)
  {
    size_t ean = add_any(tmp1, a0, a0n, a1, a1n);
    ean = add_any(tmp1, tmp1, ean, a2, a2n);

    size_t ebn = add_any(tmp2, b0, b0n, b1, b1n);
    ebn = add_any(tmp2, tmp2, ebn, b2, b2n);

    if (ean > 0 && ebn > 0) {
      mul_toomcook3(v1_buf, tmp1, ean, tmp2, ebn, rec_scratch);
      v1n = normalized_size(v1_buf, ean + ebn);
    }
  }

  // Point -1: vm1 = (a0-a1+a2) * (b0-b1+b2) [signed]
  {
    int ea_sign = 1, eb_sign = 1;

    // ea = (a0 + a2) - a1
    size_t t_n = add_any(tmp1, a0, a0n, a2, a2n);
    size_t ean = abs_sub(tmp1, ea_sign, tmp1, t_n, a1, a1n);

    // eb = (b0 + b2) - b1
    t_n = add_any(tmp2, b0, b0n, b2, b2n);
    size_t ebn = abs_sub(tmp2, eb_sign, tmp2, t_n, b1, b1n);

    vm1_sign = ea_sign * eb_sign;

    if (ean > 0 && ebn > 0) {
      mul_toomcook3(vm1_buf, tmp1, ean, tmp2, ebn, rec_scratch);
      vm1n = normalized_size(vm1_buf, ean + ebn);
    }
    if (vm1n == 0) vm1_sign = 0;
  }

  // Point 2: v2 = (a0+2*a1+4*a2) * (b0+2*b1+4*b2)
  {
    // ea2 = a0 + 2*a1 + 4*a2
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    size_t ean = a0n;
    if (a1n > 0) {
      uint64_t ov = lshift(interp_buf, a1, a1n, 1);
      size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp1, tmp1, ean, interp_buf, tn);
    }
    if (a2n > 0) {
      uint64_t ov = lshift(interp_buf, a2, a2n, 2);
      size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp1, tmp1, ean, interp_buf, tn);
    }

    // eb2 = b0 + 2*b1 + 4*b2
    if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t));
    size_t ebn = b0n;
    if (b1n > 0) {
      uint64_t ov = lshift(interp_buf, b1, b1n, 1);
      size_t tn = b1n; if (ov) { interp_buf[tn] = ov; tn++; }
      ebn = add_any(tmp2, tmp2, ebn, interp_buf, tn);
    }
    if (b2n > 0) {
      uint64_t ov = lshift(interp_buf, b2, b2n, 2);
      size_t tn = b2n; if (ov) { interp_buf[tn] = ov; tn++; }
      ebn = add_any(tmp2, tmp2, ebn, interp_buf, tn);
    }

    if (ean > 0 && ebn > 0) {
      mul_toomcook3(v2_buf, tmp1, ean, tmp2, ebn, rec_scratch);
      v2n = normalized_size(v2_buf, ean + ebn);
    }
  }

  // ============================================================
  // interpolation (GMP style: /2 and /3 only)
  // ============================================================
  //
  // v0 = c0, vinf = c4 (already placed directly in r)
  // v1 = c0+c1+c2+c3+c4
  // vm1 = c0-c1+c2-c3+c4 (signed)
  // v2 = c0+2c1+4c2+8c3+16c4
  //
  // Step 1: A = v1 + vm1 = 2(c0+c2+c4) [always non-negative]
  // Step 2: B = v1 - vm1 = 2(c1+c3)   [always non-negative]
  // Step 3: c2 = A/2 - v0 - vinf
  // Step 4: C = B/2 = c1+c3
  // Step 5: D = v2 - v0 - 4*c2 - 16*vinf = 2(c1+4c3)
  // Step 6: E = D/2 = c1+4c3
  // Step 7: c3 = (E - C) / 3
  // Step 8: c1 = C - c3

  // Step 1: A = v1 ± vm1 → tmp1
  size_t An;
  if (vm1_sign >= 0) {
    An = add_any(tmp1, v1_buf, v1n, vm1_buf, vm1n);
  } else {
    // vm1 < 0 → A = v1 - |vm1|, guarantee: v1 >= |vm1|
    std::memcpy(tmp1, v1_buf, v1n * sizeof(uint64_t));
    An = v1n;
    if (vm1n > 0) { sub(tmp1, tmp1, An, vm1_buf, vm1n); An = normalized_size(tmp1, An); }
  }

  // Step 2: B = v1 ∓ vm1 → tmp2
  size_t Bn;
  if (vm1_sign >= 0) {
    // B = v1 - vm1, guarantee: v1 >= vm1
    std::memcpy(tmp2, v1_buf, v1n * sizeof(uint64_t));
    Bn = v1n;
    if (vm1n > 0) { sub(tmp2, tmp2, Bn, vm1_buf, vm1n); Bn = normalized_size(tmp2, Bn); }
  } else {
    // vm1 < 0 → B = v1 + |vm1|
    Bn = add_any(tmp2, v1_buf, v1n, vm1_buf, vm1n);
  }

  // Step 3: c2 = A/2 - v0 - vinf → tmp1
  if (An > 0) An = rshift_1(tmp1, tmp1, An);
  if (v0n > 0 && An > 0) { sub(tmp1, tmp1, An, r, v0n); An = normalized_size(tmp1, An); }
  if (vinfn > 0 && An > 0) { sub(tmp1, tmp1, An, r + 4 * k, vinfn); An = normalized_size(tmp1, An); }
  size_t c2n = An;
  uint64_t* c2_ptr = tmp1;

  // Step 4: C = B/2 → tmp2
  if (Bn > 0) Bn = rshift_1(tmp2, tmp2, Bn);
  size_t Cn = Bn;

  // Step 5: D = v2 - v0 - 4*c2 - 16*vinf → interp_buf
  if (v2n > 0) std::memcpy(interp_buf, v2_buf, v2n * sizeof(uint64_t));
  size_t Dn = v2n;
  // D -= v0
  if (v0n > 0 && Dn > 0) { sub(interp_buf, interp_buf, Dn, r, v0n); Dn = normalized_size(interp_buf, Dn); }
  // D -= 4*c2
  if (c2n > 0 && Dn > 0) {
    uint64_t ov = lshift(v1_buf, c2_ptr, c2n, 2);
    size_t tn = c2n; if (ov) { v1_buf[tn] = ov; tn++; }
    sub(interp_buf, interp_buf, Dn, v1_buf, tn);
    Dn = normalized_size(interp_buf, Dn);
  }
  // D -= 16*vinf
  if (vinfn > 0 && Dn > 0) {
    uint64_t ov = lshift(v1_buf, r + 4 * k, vinfn, 4);
    size_t tn = vinfn; if (ov) { v1_buf[tn] = ov; tn++; }
    sub(interp_buf, interp_buf, Dn, v1_buf, tn);
    Dn = normalized_size(interp_buf, Dn);
  }

  // Step 6: E = D/2 → interp_buf
  if (Dn > 0) Dn = rshift_1(interp_buf, interp_buf, Dn);

  // Step 7: c3 = (E - C) / 3 → vm1_buf
  size_t c3n = 0;
  if (Dn > 0 || Cn > 0) {
    if (Dn >= Cn) {
      if (Cn > 0) sub(v1_buf, interp_buf, Dn, tmp2, Cn);
      else std::memcpy(v1_buf, interp_buf, Dn * sizeof(uint64_t));
    } else {
      // Dn < Cn: extend interp_buf to Cn limbs (upper part zero)
      // Use Cn to satisfy the sub precondition an >= bn
      for (size_t i = Dn; i < Cn; ++i) interp_buf[i] = 0;
      sub(v1_buf, interp_buf, Cn, tmp2, Cn);
    }
    size_t Fn = normalized_size(v1_buf, std::max(Dn, Cn));
    if (Fn > 0) c3n = divexact_by3(vm1_buf, v1_buf, Fn);
  }
  uint64_t* c3_ptr = vm1_buf;

  // Step 8: c1 = C - c3 → v2_buf
  size_t c1n = 0;
  if (Cn > 0) {
    if (c3n > 0) {
      sub(v2_buf, tmp2, Cn, c3_ptr, c3n);
      c1n = normalized_size(v2_buf, Cn);
    } else {
      std::memcpy(v2_buf, tmp2, Cn * sizeof(uint64_t));
      c1n = Cn;
    }
  }
  uint64_t* c1_ptr = v2_buf;

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k)
  // ============================================================
  // c0 placed at r[0..], c4 placed at r[4k..]

  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    uint64_t carry = add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
    (void)carry;
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    uint64_t carry = add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
    (void)carry;
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    uint64_t carry = add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
    (void)carry;
  }
}

// ================================================================
// Toom-Cook-4,2 (2:1 unbalanced specialization)
// ================================================================
//
// A(x) = a3·x³ + a2·x² + a1·x + a0 (4 chunks, deg 3)
// B(x) = b1·x + b0         (2 chunks, deg 1)
// C(x) = A(x)*B(x) has deg 4 -> 5 evaluation points needed
//
// Evaluation points {0, 1, -1, 2, inf}, interpolation uses only /2 (rshift_1) and /3 (divexact_by3).
//
// Applicability: 5/3·bn ≤ an ≤ 5/2·bn (an/bn ≈ 2:1, 1.67~2.5)
// effect: mul(2049, 1025) goes from 2 x Toom-8(1025)=394us to 5 x Toom-4(513)=205us
//
// Math verification:
//  Bodrato 5-point interpolation:
//   v2 ← (v2 - vm1)/3  = c1+c2+3c3+5c4
//   vm1 ← (v1 - vm1)/2  = c1+c3
//   v1 ← v1 - v0     = c1+c2+c3+c4
//   v2 ← (v2 - v1)/2   = c3+2c4
//   v2 ← v2 - 2·vinf   = c3
//   v1 ← v1 - vm1 - vinf = c2
//   vm1 ← vm1 - v2    = c1
//  result: r0=v0, r1=vm1, r2=v1, r3=v2, r4=vinf
constexpr size_t TOOMCOOK42_THRESHOLD = 280; // bn minimum (sub-mult becomes Toom-3+)

// Forward declaration (must be defined before multiply, mutually referenced)
inline void multiply(uint64_t* r,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn,
           uint64_t* scratch);
inline size_t multiply_scratch_size(size_t an, size_t bn);
// conservative recursion scratch (avoids FFT early-zero, defined below).
// Toom-N used for rec_scratch size computation.
inline size_t multiply_rec_scratch_size(size_t an, size_t bn);

inline size_t mul_toomcook42_scratch_size(size_t an, size_t bn) {
  if (an < bn) std::swap(an, bn);
  size_t n_a = (an + 3) / 4;
  size_t blk = 2 * (n_a + 4);
  // 3 sub-product regions (v1,vm1,v2) + 4 eval/interp temps (n_a+4 each) + recursion scratch
  // rec_scratch uses multiply_rec_scratch_size to avoid FFT early-zero
  return 3 * blk + 4 * (n_a + 4) + multiply_rec_scratch_size(n_a + 2, n_a + 2);
}

// Toom-4,2 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// Precondition: an >= bn, 5/3·bn ≤ an ≤ 5/2·bn, bn >= TOOMCOOK42_THRESHOLD
// r must not overlap a or b
inline void mul_toomcook42(uint64_t* r,
              const uint64_t* a, size_t an,
              const uint64_t* b, size_t bn,
              uint64_t* scratch) {
  // Assumes: an >= bn (guaranteed by caller)
  size_t n_a = (an + 3) / 4;   // chunk size
  size_t rn = an + bn;

  // --- 4-way split (A) ---
  const uint64_t* a0 = a;
  size_t a0n = normalized_size(a0, std::min(n_a, an));
  const uint64_t* a1 = a + n_a;
  size_t a1n = (an > n_a) ? normalized_size(a1, std::min(n_a, an - n_a)) : 0;
  const uint64_t* a2 = a + 2 * n_a;
  size_t a2n = (an > 2 * n_a) ? normalized_size(a2, std::min(n_a, an - 2 * n_a)) : 0;
  const uint64_t* a3 = a + 3 * n_a;
  size_t a3n = (an > 3 * n_a) ? normalized_size(a3, an - 3 * n_a) : 0;

  // --- 2-way split (B) ---
  const uint64_t* b0 = b;
  size_t b0n = normalized_size(b0, std::min(n_a, bn));
  const uint64_t* b1 = b + n_a;
  size_t b1n = (bn > n_a) ? normalized_size(b1, bn - n_a) : 0;

  // --- Scratch layout ---
  size_t blk = 2 * (n_a + 4);
  uint64_t* v1_buf   = scratch;                  // C(1)
  uint64_t* vm1_buf   = scratch + blk;               // C(-1)
  uint64_t* v2_buf   = scratch + 2 * blk;             // C(2)
  uint64_t* tmp_a    = scratch + 3 * blk;             // A evaluation
  uint64_t* tmp_b    = scratch + 3 * blk + (n_a + 4);       // B evaluation
  uint64_t* interp_buf = scratch + 3 * blk + 2 * (n_a + 4);     // interpolation tmp1
  uint64_t* interp_buf2 = scratch + 3 * blk + 3 * (n_a + 4);     // interpolation tmp2
  uint64_t* rec_scratch = scratch + 3 * blk + 4 * (n_a + 4);     // recursion scratch

  // ============================================================
  // Point 0: v0 = a0 * b0 → r[0..]
  // Point ∞: vinf = a3 * b1 → r[4*n_a..]
  // ============================================================
  size_t v0n = 0;
  if (a0n > 0 && b0n > 0) {
    multiply(r, a0, a0n, b0, b0n, rec_scratch);
    v0n = normalized_size(r, std::min(a0n + b0n, rn));
  }

  size_t vinfn = 0;
  size_t vinf_off = 4 * n_a;
  if (a3n > 0 && b1n > 0 && vinf_off < rn) {
    multiply(r + vinf_off, a3, a3n, b1, b1n, rec_scratch);
    vinfn = normalized_size(r + vinf_off, std::min(a3n + b1n, rn - vinf_off));
  }

  // Zero out the gap region (between v0 and vinf, and after vinf)
  {
    size_t gap_start = v0n;
    size_t gap_end = std::min(vinf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(vinf_off + vinfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // ============================================================
  // Point 1: v1 = (a0+a1+a2+a3) * (b0+b1)
  // ============================================================
  size_t v1n = 0;
  {
    // ea = a0 + a1 + a2 + a3
    size_t ean = add_any(tmp_a, a0, a0n, a1, a1n);
    ean = add_any(tmp_a, tmp_a, ean, a2, a2n);
    ean = add_any(tmp_a, tmp_a, ean, a3, a3n);

    // eb = b0 + b1
    size_t ebn = add_any(tmp_b, b0, b0n, b1, b1n);

    if (ean > 0 && ebn > 0) {
      multiply(v1_buf, tmp_a, ean, tmp_b, ebn, rec_scratch);
      v1n = normalized_size(v1_buf, ean + ebn);
    }
  }

  // ============================================================
  // Point -1: vm1 = (a0-a1+a2-a3) * (b0-b1) [signed]
  // ============================================================
  size_t vm1n = 0;
  int vm1_sign = 0;
  {
    int ea_sign = 1, eb_sign = 1;

    // ea = (a0+a2) - (a1+a3)
    size_t pos_n = add_any(interp_buf, a0, a0n, a2, a2n);
    size_t neg_n = add_any(interp_buf2, a1, a1n, a3, a3n);
    size_t ean = abs_sub(tmp_a, ea_sign, interp_buf, pos_n, interp_buf2, neg_n);

    // eb = b0 - b1
    size_t ebn = abs_sub(tmp_b, eb_sign, b0, b0n, b1, b1n);

    vm1_sign = ea_sign * eb_sign;

    if (ean > 0 && ebn > 0) {
      multiply(vm1_buf, tmp_a, ean, tmp_b, ebn, rec_scratch);
      vm1n = normalized_size(vm1_buf, ean + ebn);
    }
    if (vm1n == 0) vm1_sign = 0;
  }

  // ============================================================
  // Point 2: v2 = (a0+2*a1+4*a2+8*a3) * (b0+2*b1)
  // ============================================================
  size_t v2n = 0;
  {
    // ea2 = a0 + 2*a1 + 4*a2 + 8*a3
    if (a0n > 0) std::memcpy(tmp_a, a0, a0n * sizeof(uint64_t));
    size_t ean = a0n;
    if (a1n > 0) {
      uint64_t ov = lshift(interp_buf, a1, a1n, 1);
      size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn);
    }
    if (a2n > 0) {
      uint64_t ov = lshift(interp_buf, a2, a2n, 2);
      size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn);
    }
    if (a3n > 0) {
      uint64_t ov = lshift(interp_buf, a3, a3n, 3);
      size_t tn = a3n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp_a, tmp_a, ean, interp_buf, tn);
    }

    // eb2 = b0 + 2*b1
    if (b0n > 0) std::memcpy(tmp_b, b0, b0n * sizeof(uint64_t));
    size_t ebn = b0n;
    if (b1n > 0) {
      uint64_t ov = lshift(interp_buf, b1, b1n, 1);
      size_t tn = b1n; if (ov) { interp_buf[tn] = ov; tn++; }
      ebn = add_any(tmp_b, tmp_b, ebn, interp_buf, tn);
    }

    if (ean > 0 && ebn > 0) {
      multiply(v2_buf, tmp_a, ean, tmp_b, ebn, rec_scratch);
      v2n = normalized_size(v2_buf, ean + ebn);
    }
  }

  // ============================================================
  // Bodrato 5-point interpolation
  // ============================================================
  //
  // Step 1: v2 ← (v2 - vm1) / 3
  // (when vm1 is negative,v2 - (-|vm1|) = v2 + |vm1|)
  {
    if (vm1_sign >= 0) {
      if (vm1n > 0 && v2n >= vm1n) {
        sub(v2_buf, v2_buf, v2n, vm1_buf, vm1n);
        v2n = normalized_size(v2_buf, v2n);
      }
    } else {
      v2n = add_any(v2_buf, v2_buf, v2n, vm1_buf, vm1n);
    }
    if (v2n > 0) v2n = divexact_by3(v2_buf, v2_buf, v2n);
  }

  // Step 2: vm1 ← (v1 - vm1) / 2
  // (when vm1 is negative,v1 - (-|vm1|) = v1 + |vm1|)
  {
    if (vm1_sign >= 0) {
      // vm1 = v1 - vm1 (guarantee: v1 >= vm1)
      if (vm1n > 0) {
        std::memcpy(interp_buf, v1_buf, v1n * sizeof(uint64_t));
        sub(interp_buf, interp_buf, v1n, vm1_buf, vm1n);
        vm1n = normalized_size(interp_buf, v1n);
        std::memcpy(vm1_buf, interp_buf, vm1n * sizeof(uint64_t));
      } else {
        std::memcpy(vm1_buf, v1_buf, v1n * sizeof(uint64_t));
        vm1n = v1n;
      }
    } else {
      // vm1 = v1 + |vm1|
      vm1n = add_any(vm1_buf, v1_buf, v1n, vm1_buf, vm1n);
    }
    if (vm1n > 0) vm1n = rshift_1(vm1_buf, vm1_buf, vm1n);
  }

  // Step 3: v1 ← v1 - v0
  if (v0n > 0 && v1n > 0) {
    sub(v1_buf, v1_buf, v1n, r, v0n);
    v1n = normalized_size(v1_buf, v1n);
  }

  // Step 4: v2 ← (v2 - v1) / 2
  if (v1n > 0 && v2n > 0) {
    sub(v2_buf, v2_buf, v2n, v1_buf, v1n);
    v2n = normalized_size(v2_buf, v2n);
  }
  if (v2n > 0) v2n = rshift_1(v2_buf, v2_buf, v2n);

  // Step 5: v2 ← v2 - 2 * vinf
  if (vinfn > 0 && v2n > 0) {
    uint64_t ov = lshift(interp_buf, r + vinf_off, vinfn, 1);
    size_t tn = vinfn; if (ov) { interp_buf[tn] = ov; tn++; }
    sub(v2_buf, v2_buf, v2n, interp_buf, tn);
    v2n = normalized_size(v2_buf, v2n);
  }

  // Step 6: v1 ← v1 - vm1 - vinf
  if (vm1n > 0 && v1n > 0) {
    sub(v1_buf, v1_buf, v1n, vm1_buf, vm1n);
    v1n = normalized_size(v1_buf, v1n);
  }
  if (vinfn > 0 && v1n > 0) {
    sub(v1_buf, v1_buf, v1n, r + vinf_off, vinfn);
    v1n = normalized_size(v1_buf, v1n);
  }

  // Step 7: vm1 ← vm1 - v2
  if (v2n > 0 && vm1n > 0) {
    sub(vm1_buf, vm1_buf, vm1n, v2_buf, v2n);
    vm1n = normalized_size(vm1_buf, vm1n);
  }

  // ============================================================
  // Compose: r += vm1·B^k + v1·B^{2k} + v2·B^{3k}
  // (r0 = c0 in r, r4 = c4 in r already)
  // ============================================================
  if (vm1n > 0 && n_a < rn) {
    size_t space = rn - n_a;
    add(r + n_a, r + n_a, space, vm1_buf, std::min(vm1n, space));
  }
  if (v1n > 0 && 2 * n_a < rn) {
    size_t space = rn - 2 * n_a;
    add(r + 2 * n_a, r + 2 * n_a, space, v1_buf, std::min(v1n, space));
  }
  if (v2n > 0 && 3 * n_a < rn) {
    size_t space = rn - 3 * n_a;
    add(r + 3 * n_a, r + 3 * n_a, space, v2_buf, std::min(v2n, space));
  }
}

// ================================================================
// Toom-Cook-4 multiplication
// ================================================================

// Toom-4: at 200+ limbs comparable to ~10% faster than Toom-3 (2026-03-11 sweep, 200-1400 limbs).
// Up to ~10% advantage near 600 limbs. Below 200, Toom-3 is lighter.
constexpr size_t TOOMCOOK4_THRESHOLD = 200;

inline size_t mul_toomcook4_scratch_size(size_t n) {
  if (n < TOOMCOOK4_THRESHOLD) return mul_toomcook3_scratch_size(n);
  return 50 * n + 512;
}

// Toom-Cook-4 multiplication: r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// Evaluation points {0, 1, -1, 2, -2, 3, ∞}
// interpolation uses only /2 (shift), /3 (divexact_by3), /5 (divexact_by5), /12, /120
// r must not overlap a or b
//
// optimization: Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size
inline void mul_toomcook4(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }
  if (bn < TOOMCOOK4_THRESHOLD) {
    mul_toomcook3(r, a, an, b, bn, scratch);
    return;
  }

  size_t k = (an + 3) / 4;
  size_t rn = an + bn;

  // --- Zero-pad coefficients to k limbs ---
  // Enables fixed-size evaluation based on addmul_1
  uint64_t* pa = scratch;      // 4*k limbs (a coefficients)
  uint64_t* pb = scratch + 4 * k;  // 4*k limbs (b coefficients)

  auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) {
    size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0;
    if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t));
    if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t));
  };
  for (size_t i = 0; i < 4; i++) pad_coeff(pa + i * k, a, an, i * k);
  for (size_t i = 0; i < 4; i++) pad_coeff(pb + i * k, b, bn, i * k);

  uint64_t* pa0 = pa;
  uint64_t* pa1 = pa + k;
  uint64_t* pa2 = pa + 2 * k;
  uint64_t* pa3 = pa + 3 * k;
  uint64_t* pb0 = pb;
  uint64_t* pb1 = pb + k;
  uint64_t* pb2 = pb + 2 * k;
  uint64_t* pb3 = pb + 3 * k;

  // --- Scratch layout (after padding) ---
  size_t blk = 2 * k + 4; // each product buffer (max 2k+2 limbs + margin)
  uint64_t* w1_buf   = scratch + 8 * k;
  uint64_t* wm1_buf   = w1_buf + blk;
  uint64_t* w2_buf   = wm1_buf + blk;
  uint64_t* wm2_buf   = w2_buf + blk;
  uint64_t* w3_buf   = wm2_buf + blk;
  uint64_t* ea_even   = w3_buf + blk;    // k+2 limbs (a even terms)
  uint64_t* ea_odd   = ea_even + k + 2;  // k+2 limbs (a odd terms)
  uint64_t* eb_even   = ea_odd + k + 2;   // k+2 limbs (b even terms)
  uint64_t* eb_odd   = eb_even + k + 2;  // k+2 limbs (b odd terms)
  uint64_t* eval_tmp  = eb_odd + k + 2;   // k+2 limbs (evaluation result)
  uint64_t* rec_scratch = eval_tmp + k + 2;

  size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0, w3n = 0;
  int wm1_sign = 0, wm2_sign = 0;

  // ============================================================
  // Pointwise multiplication (7 recursive calls)
  // Evaluation: batched fixed k-limb via addmul_1
  // ============================================================

  // Point 0: w0 = a0 * b0 → r[0..] (invoked with actual size)
  size_t a0n_real = std::min(k, an);
  size_t b0n_real = std::min(k, bn);
  size_t w0n = 0;
  if (a0n_real > 0 && b0n_real > 0) {
    mul_toomcook4(r, a, a0n_real, b, b0n_real, rec_scratch);
    w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn));
  }

  // Point ∞: winf = a3 * b3 → r[6k..] (invoked with actual size)
  size_t a3n_real = (an > 3 * k) ? an - 3 * k : 0;
  size_t b3n_real = (bn > 3 * k) ? bn - 3 * k : 0;
  size_t winfn = 0;
  size_t winf_off = 6 * k;
  if (a3n_real > 0 && b3n_real > 0 && winf_off < rn) {
    mul_toomcook4(r + winf_off, a + 3 * k, a3n_real, b + 3 * k, b3n_real, rec_scratch);
    winfn = normalized_size(r + winf_off, std::min(a3n_real + b3n_real, rn - winf_off));
  }

  // Zero out gap region
  {
    size_t gap_start = w0n;
    size_t gap_end = std::min(winf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(winf_off + winfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // +/-1 sharing: even1 = a0+a2, odd1 = a1+a3
  // a(1) = even1 + odd1, a(-1) = |even1 - odd1|
  {
    // a side: even1/odd1
    std::memcpy(ea_even, pa0, k * sizeof(uint64_t));
    ea_even[k] = add(ea_even, ea_even, k, pa2, k);
    std::memcpy(ea_odd, pa1, k * sizeof(uint64_t));
    ea_odd[k] = add(ea_odd, ea_odd, k, pa3, k);

    // b side: even1/odd1
    std::memcpy(eb_even, pb0, k * sizeof(uint64_t));
    eb_even[k] = add(eb_even, eb_even, k, pb2, k);
    std::memcpy(eb_odd, pb1, k * sizeof(uint64_t));
    eb_odd[k] = add(eb_odd, eb_odd, k, pb3, k);
  }

  // Point 1: w1 = a(1) * b(1) = (even1+odd1) * (even1+odd1)
  {
    // a(1) → eval_tmp
    std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t));
    eval_tmp[k + 1] = 0;
    uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1);
    if (cy) eval_tmp[k + 1] = cy;
    size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k)));

    // b(1) → wm1_buf (temporary use, overwritten at Point -1)
    std::memcpy(wm1_buf, eb_even, (k + 1) * sizeof(uint64_t));
    wm1_buf[k + 1] = 0;
    cy = add(wm1_buf, wm1_buf, k + 1, eb_odd, k + 1);
    if (cy) wm1_buf[k + 1] = cy;
    size_t ebn = (wm1_buf[k + 1] ? k + 2 : (wm1_buf[k] ? k + 1 : normalized_size(wm1_buf, k)));

    if (ean > 0 && ebn > 0) {
      mul_toomcook4(w1_buf, eval_tmp, ean, wm1_buf, ebn, rec_scratch);
      w1n = normalized_size(w1_buf, ean + ebn);
    }
  }

  // Point -1: wm1 = a(-1) * b(-1) = |even1-odd1| * |even1-odd1|
  {
    int ea_sign = 1, eb_sign = 1;

    // a(-1) = |ea_even - ea_odd| → eval_tmp
    int c = cmp(ea_even, k + 1, ea_odd, k + 1);
    if (c >= 0) {
      sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1);
    } else {
      sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1);
      ea_sign = -1;
    }
    size_t ean = normalized_size(eval_tmp, k + 1);

    // b(-1) = |eb_even - eb_odd| → w2_buf (temporary use)
    c = cmp(eb_even, k + 1, eb_odd, k + 1);
    if (c >= 0) {
      sub(w2_buf, eb_even, k + 1, eb_odd, k + 1);
    } else {
      sub(w2_buf, eb_odd, k + 1, eb_even, k + 1);
      eb_sign = -1;
    }
    size_t ebn = normalized_size(w2_buf, k + 1);

    wm1_sign = ea_sign * eb_sign;
    if (ean > 0 && ebn > 0) {
      mul_toomcook4(wm1_buf, eval_tmp, ean, w2_buf, ebn, rec_scratch);
      wm1n = normalized_size(wm1_buf, ean + ebn);
    }
    if (wm1n == 0) wm1_sign = 0;
  }

  // +/-2 sharing: even2 = a0+4*a2, odd2 = 2*a1+8*a3
  // a(2) = even2 + odd2, a(-2) = |even2 - odd2|
  {
    // a side: even2/odd2 → ea_even/ea_odd (reused)
    std::memcpy(ea_even, pa0, k * sizeof(uint64_t));
    ea_even[k] = addmul_1(ea_even, pa2, k, 4);
    ea_odd[k] = mul_1(ea_odd, pa1, k, 2);
    ea_odd[k] += addmul_1(ea_odd, pa3, k, 8);

    // b side: even2/odd2 → eb_even/eb_odd (reused)
    std::memcpy(eb_even, pb0, k * sizeof(uint64_t));
    eb_even[k] = addmul_1(eb_even, pb2, k, 4);
    eb_odd[k] = mul_1(eb_odd, pb1, k, 2);
    eb_odd[k] += addmul_1(eb_odd, pb3, k, 8);
  }

  // Point 2: w2 = a(2) * b(2) = (even2+odd2) * (even2+odd2)
  {
    // a(2) → eval_tmp
    std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t));
    eval_tmp[k + 1] = 0;
    uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1);
    if (cy) eval_tmp[k + 1] = cy;
    size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k)));

    // b(2) → wm2_buf (temporary use,overwritten at Point -2)
    std::memcpy(wm2_buf, eb_even, (k + 1) * sizeof(uint64_t));
    wm2_buf[k + 1] = 0;
    cy = add(wm2_buf, wm2_buf, k + 1, eb_odd, k + 1);
    if (cy) wm2_buf[k + 1] = cy;
    size_t ebn = (wm2_buf[k + 1] ? k + 2 : (wm2_buf[k] ? k + 1 : normalized_size(wm2_buf, k)));

    if (ean > 0 && ebn > 0) {
      mul_toomcook4(w2_buf, eval_tmp, ean, wm2_buf, ebn, rec_scratch);
      w2n = normalized_size(w2_buf, ean + ebn);
    }
  }

  // Point -2: wm2 = a(-2) * b(-2) = |even2-odd2| * |even2-odd2|
  {
    int ea_sign = 1, eb_sign = 1;

    // a(-2) = |ea_even - ea_odd| → eval_tmp
    int c = cmp(ea_even, k + 1, ea_odd, k + 1);
    if (c >= 0) {
      sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1);
    } else {
      sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1);
      ea_sign = -1;
    }
    size_t ean = normalized_size(eval_tmp, k + 1);

    // b(-2) = |eb_even - eb_odd| → w3_buf (temporary use)
    c = cmp(eb_even, k + 1, eb_odd, k + 1);
    if (c >= 0) {
      sub(w3_buf, eb_even, k + 1, eb_odd, k + 1);
    } else {
      sub(w3_buf, eb_odd, k + 1, eb_even, k + 1);
      eb_sign = -1;
    }
    size_t ebn = normalized_size(w3_buf, k + 1);

    wm2_sign = ea_sign * eb_sign;
    if (ean > 0 && ebn > 0) {
      mul_toomcook4(wm2_buf, eval_tmp, ean, w3_buf, ebn, rec_scratch);
      wm2n = normalized_size(wm2_buf, ean + ebn);
    }
    if (wm2n == 0) wm2_sign = 0;
  }

  // Point 3: w3 = a(3) * b(3)
  // a(3) = a0 + 3*a1 + 9*a2 + 27*a3 [addmul_1]
  {
    std::memcpy(eval_tmp, pa0, k * sizeof(uint64_t));
    eval_tmp[k] = 0;
    eval_tmp[k] += addmul_1(eval_tmp, pa1, k, 3);
    eval_tmp[k] += addmul_1(eval_tmp, pa2, k, 9);
    eval_tmp[k] += addmul_1(eval_tmp, pa3, k, 27);
    size_t ean = k + (eval_tmp[k] ? 1 : 0);

    // b(3) → ea_even (temporary use,from here on ea_even no longer needed)
    std::memcpy(ea_even, pb0, k * sizeof(uint64_t));
    ea_even[k] = 0;
    ea_even[k] += addmul_1(ea_even, pb1, k, 3);
    ea_even[k] += addmul_1(ea_even, pb2, k, 9);
    ea_even[k] += addmul_1(ea_even, pb3, k, 27);
    size_t ebn = k + (ea_even[k] ? 1 : 0);

    if (ean > 0 && ebn > 0) {
      mul_toomcook4(w3_buf, eval_tmp, ean, ea_even, ebn, rec_scratch);
      w3n = normalized_size(w3_buf, ean + ebn);
    }
  }

  // ============================================================
  // interpolation
  // ============================================================
  // interpolation temporaries: reuse rec_scratch region (no longer needed after evaluation recursion)
  // Each buffer needs at most 2k+4 limbs
  uint64_t* tmp1    = rec_scratch;
  uint64_t* tmp2    = rec_scratch + blk;
  uint64_t* interp_buf = rec_scratch + 2 * blk;
  uint64_t* interp_buf2 = rec_scratch + 3 * blk;
  //
  // c0 = w0 (r[0..] already placed)
  // c6 = winf (r[6k..] already placed)
  //
  // Step 1-4: separate even/odd using +/-1, +/-2 symmetry
  //  t1 = (w1 + wm1) / 2 = c0 + c2 + c4 + c6
  //  t2 = (w1 - wm1) / 2 = c1 + c3 + c5
  //  t3 = (w2 + wm2) / 2 = c0 + 4c2 + 16c4 + 64c6
  //  t4 = (w2 - wm2) / 2 = 2c1 + 8c3 + 32c5

  // Step 1: t1 = w1 ± wm1 → tmp1
  size_t t1n;
  if (wm1_sign >= 0) {
    t1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n);
  } else {
    std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t));
    t1n = w1n;
    if (wm1n > 0) { sub(tmp1, tmp1, t1n, wm1_buf, wm1n); t1n = normalized_size(tmp1, t1n); }
  }
  // t1 /= 2
  if (t1n > 0) t1n = rshift_1(tmp1, tmp1, t1n);

  // Step 2: t2 = w1 ∓ wm1 → tmp2
  size_t t2n;
  if (wm1_sign >= 0) {
    std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t));
    t2n = w1n;
    if (wm1n > 0) { sub(tmp2, tmp2, t2n, wm1_buf, wm1n); t2n = normalized_size(tmp2, t2n); }
  } else {
    t2n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n);
  }
  // t2 /= 2
  if (t2n > 0) t2n = rshift_1(tmp2, tmp2, t2n);

  // Step 3: t3 = w2 ± wm2 → interp_buf
  size_t t3n;
  if (wm2_sign >= 0) {
    t3n = add_any(interp_buf, w2_buf, w2n, wm2_buf, wm2n);
  } else {
    std::memcpy(interp_buf, w2_buf, w2n * sizeof(uint64_t));
    t3n = w2n;
    if (wm2n > 0) { sub(interp_buf, interp_buf, t3n, wm2_buf, wm2n); t3n = normalized_size(interp_buf, t3n); }
  }
  // t3 /= 2
  if (t3n > 0) t3n = rshift_1(interp_buf, interp_buf, t3n);

  // Step 4: t4 = w2 ∓ wm2 → interp_buf2
  size_t t4n;
  if (wm2_sign >= 0) {
    std::memcpy(interp_buf2, w2_buf, w2n * sizeof(uint64_t));
    t4n = w2n;
    if (wm2n > 0) { sub(interp_buf2, interp_buf2, t4n, wm2_buf, wm2n); t4n = normalized_size(interp_buf2, t4n); }
  } else {
    t4n = add_any(interp_buf2, w2_buf, w2n, wm2_buf, wm2n);
  }
  // t4 /= 2
  if (t4n > 0) t4n = rshift_1(interp_buf2, interp_buf2, t4n);

  // Variable layout so far:
  //  tmp1    = t1 (c0+c2+c4+c6)
  //  tmp2    = t2 (c1+c3+c5)
  //  interp_buf = t3 (c0+4c2+16c4+64c6)
  //  interp_buf2= t4 (2c1+8c3+32c5)
  //  w1_buf, wm1_buf, w2_buf, wm2_buf are reusable

  // Step 9: t5 = t1 - w0 - winf = c2 + c4 → w1_buf
  if (t1n > 0) std::memcpy(w1_buf, tmp1, t1n * sizeof(uint64_t));
  size_t t5n = t1n;
  if (w0n > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r, w0n); t5n = normalized_size(w1_buf, t5n); }
  if (winfn > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r + 6 * k, winfn); t5n = normalized_size(w1_buf, t5n); }

  // Step 10: t6 = t3 - w0 - 64*winf = 4c2 + 16c4 → wm1_buf
  if (t3n > 0) std::memcpy(wm1_buf, interp_buf, t3n * sizeof(uint64_t));
  size_t t6n = t3n;
  if (w0n > 0 && t6n > 0) { sub(wm1_buf, wm1_buf, t6n, r, w0n); t6n = normalized_size(wm1_buf, t6n); }
  if (winfn > 0 && t6n > 0) {
    uint64_t bw = submul_1(wm1_buf, r + winf_off, winfn, 64);
    if (bw > 0 && t6n > winfn) sub_1(wm1_buf + winfn, t6n - winfn, bw);
    t6n = normalized_size(wm1_buf, t6n);
  }

  // Step 11: t6 = t6 - 4*t5 = 12*c4 → wm1_buf
  if (t5n > 0 && t6n > 0) {
    uint64_t bw = submul_1(wm1_buf, w1_buf, t5n, 4);
    if (bw > 0 && t6n > t5n) sub_1(wm1_buf + t5n, t6n - t5n, bw);
    t6n = normalized_size(wm1_buf, t6n);
  }

  // Step 12: c4 = t6 / 12 → wm2_buf
  size_t c4n = 0;
  if (t6n > 0) {
    c4n = divexact_by12(wm2_buf, wm1_buf, t6n);
  }
  uint64_t* c4_ptr = wm2_buf;

  // Step 13: c2 = t5 - c4 → w1_buf (t5 already there)
  size_t c2n = t5n;
  if (c4n > 0 && c2n > 0) {
    sub(w1_buf, w1_buf, c2n, c4_ptr, c4n);
    c2n = normalized_size(w1_buf, c2n);
  }
  uint64_t* c2_ptr = w1_buf;

  // Step 14: t8 = t4 - 2*t2 = 6c3+30c5 → wm1_buf
  if (t4n > 0) std::memcpy(wm1_buf, interp_buf2, t4n * sizeof(uint64_t));
  size_t t8n = t4n;
  if (t2n > 0 && t8n > 0) {
    uint64_t bw = submul_1(wm1_buf, tmp2, t2n, 2);
    if (bw > 0 && t8n > t2n) sub_1(wm1_buf + t2n, t8n - t2n, bw);
    t8n = normalized_size(wm1_buf, t8n);
  }

  // Step 15: t9 = w3 - w0 - 9*c2 - 81*c4 - 729*winf → interp_buf
  if (w3n > 0) std::memcpy(interp_buf, w3_buf, w3n * sizeof(uint64_t));
  size_t t9n = w3n;

  // t9 -= w0
  if (w0n > 0 && t9n > 0) {
    sub(interp_buf, interp_buf, t9n, r, w0n);
    t9n = normalized_size(interp_buf, t9n);
  }
  // t9 -= 9*c2 [submul_1: single pass]
  if (c2n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, c2_ptr, c2n, 9);
    if (bw > 0 && t9n > c2n) sub_1(interp_buf + c2n, t9n - c2n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }
  // t9 -= 81*c4 [submul_1: single pass]
  if (c4n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, c4_ptr, c4n, 81);
    if (bw > 0 && t9n > c4n) sub_1(interp_buf + c4n, t9n - c4n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }
  // t9 -= 729*winf [submul_1: single pass]
  if (winfn > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, r + winf_off, winfn, 729);
    if (bw > 0 && t9n > winfn) sub_1(interp_buf + winfn, t9n - winfn, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 16: t9 -= 3*t2 [submul_1: single pass]
  if (t2n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, tmp2, t2n, 3);
    if (bw > 0 && t9n > t2n) sub_1(interp_buf + t2n, t9n - t2n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 17: t9 -= 4*t8 [submul_1: single pass]
  if (t8n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, wm1_buf, t8n, 4);
    if (bw > 0 && t9n > t8n) sub_1(interp_buf + t8n, t9n - t8n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 18: c5 = t9 / 120 → w3_buf
  size_t c5n = 0;
  if (t9n > 0) {
    c5n = divexact_by120(w3_buf, interp_buf, t9n);
  }
  uint64_t* c5_ptr = w3_buf;

  // Step 19: t8 = t8 - 30*c5 = 6*c3 → wm1_buf [submul_1: single pass]
  if (c5n > 0 && t8n > 0) {
    uint64_t bw = submul_1(wm1_buf, c5_ptr, c5n, 30);
    if (bw > 0 && t8n > c5n) sub_1(wm1_buf + c5n, t8n - c5n, bw);
    t8n = normalized_size(wm1_buf, t8n);
  }

  // Step 20: c3 = t8 / 6 = (t8 >> 1) / 3 → interp_buf
  size_t c3n = 0;
  if (t8n > 0) {
    size_t sn = rshift_1(wm1_buf, wm1_buf, t8n); // /2 in-place
    c3n = divexact_by3(interp_buf, wm1_buf, sn);  // /3
  }
  uint64_t* c3_ptr = interp_buf;

  // Step 21: c1 = t2 - c3 - c5 → interp_buf2
  size_t c1n = t2n;
  if (c1n > 0) {
    std::memcpy(interp_buf2, tmp2, t2n * sizeof(uint64_t));
    if (c3n > 0) { sub(interp_buf2, interp_buf2, c1n, c3_ptr, c3n); c1n = normalized_size(interp_buf2, c1n); }
    if (c5n > 0 && c1n > 0) { sub(interp_buf2, interp_buf2, c1n, c5_ptr, c5n); c1n = normalized_size(interp_buf2, c1n); }
  }
  uint64_t* c1_ptr = interp_buf2;

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) + c4*B^(4k) + c5*B^(5k)
  // ============================================================
  // c0 placed at r[0..], c6 placed at r[6k..]

  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    uint64_t carry = add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
    (void)carry;
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    uint64_t carry = add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
    (void)carry;
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    uint64_t carry = add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
    (void)carry;
  }
  if (c4n > 0 && 4 * k < rn) {
    size_t space = rn - 4 * k;
    uint64_t carry = add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space));
    (void)carry;
  }
  if (c5n > 0 && 5 * k < rn) {
    size_t space = rn - 5 * k;
    uint64_t carry = add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space));
    (void)carry;
  }
}

// ================================================================
// Toom-Cook-6 multiplication
// ================================================================
// 6-way split, evaluation points {0, ±1, ±2, ±3, ±4, ±5, ∞},12 recursive multiplications
// Product degree 10 → 11 coefficients (c0..c10)
// Separate even/odd via +/- symmetry; even 4x4 + odd 5x5 Vandermonde interpolation

// Toom-6: 1-7% faster than Toom-4 at 200-2500 limbs (Zen 3 measurement)
// * Previously TOOM4==TOOM6==200 left the Toom-4 branch dead (Codex pointed out)
// Toom-6 threshold - history
// - 2026-03-11: 600 (initial, 1-7% faster than Toom-4)
// - 2026-04-26: 750 (after Toom-4 ASM tuning, Toom-4 became faster in the n=600 range, so raised)
// - 2026-04-26 (this session): even with pre-pad + addmul_1, Toom-4 retains its advantage (13-19% at n=448-512)
//                unchanged, threshold 750 retained. Toom-6 alone is 2-3% faster,
//                mainly effective via the Toom-8 -> Toom-6 recursion path (mul(6000,3000) -3.7%).
//
// Applies the optimization proven on Toom-8 to Toom-6 (eval single-pass):
//  old: a_even = a0 + lshift(a2,sh) + lshift(a4,sh*2) (two-pass lshift -> add per coefficient)
//  new: a_even = a0; a_even[k] += addmul_1(c_sq,a2); a_even[k] += addmul_1(c_4,a4)
//    (single-pass fixed k-limb evaluation + addmul_1 fused multiply-add)
constexpr size_t TOOMCOOK6_THRESHOLD = 750; // Toom-4: 200-749, Toom-6: 750-799, Toom-8: 800+

inline size_t mul_toomcook6_scratch_size(size_t n) {
  if (n < TOOMCOOK6_THRESHOLD) return mul_toomcook4_scratch_size(n);
  // 12*k pre-pad (pa+pb) + 15*blk (10 w/wm + 5 tmp) + recursive scratch
  // 12*k ≈ 2*n, 15*blk = 15*(2*(n/6+10)) ≈ 5*n + 300
  // 80*n is sufficient including recursion (even with old layout, margin 70+%)
  return 80 * n + 1024;
}

inline void mul_toomcook6(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }
  if (bn < TOOMCOOK6_THRESHOLD) {
    mul_toomcook4(r, a, an, b, bn, scratch);
    return;
  }

  size_t k = (an + 5) / 6;
  size_t rn = an + bn;

  // --- Zero-pad coefficients to k limbs (for fixed-length evaluation, same as Toom-8) ---
  uint64_t* pa = scratch;       // 6*k limbs
  uint64_t* pb = scratch + 6 * k;   // 6*k limbs
  auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) {
    size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0;
    if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t));
    if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t));
  };
  for (size_t i = 0; i < 6; i++) pad_coeff(pa + i * k, a, an, i * k);
  for (size_t i = 0; i < 6; i++) pad_coeff(pb + i * k, b, bn, i * k);

  // --- Scratch layout (shift after pre-pad 12*k) ---
  // Each +/-k product buffer + evaluation temporary + interpolation workspace + recursion scratch
  size_t blk = 2 * (k + 10);
  uint64_t* base    = scratch + 12 * k;
  uint64_t* w1_buf   = base;           // v(1) product
  uint64_t* wm1_buf  = base + blk;        // v(-1) product
  uint64_t* w2_buf   = base + 2 * blk;      // v(2) product
  uint64_t* wm2_buf  = base + 3 * blk;      // v(-2) product
  uint64_t* w3_buf   = base + 4 * blk;      // v(3) product
  uint64_t* wm3_buf  = base + 5 * blk;      // v(-3) product
  uint64_t* w4_buf   = base + 6 * blk;      // v(4) product
  uint64_t* wm4_buf  = base + 7 * blk;      // v(-4) product
  uint64_t* w5_buf   = base + 8 * blk;      // v(5) product
  uint64_t* wm5_buf  = base + 9 * blk;      // v(-5) product
  uint64_t* tmp1    = base + 10 * blk;      // evaluation temporary / for e1, E1, c1
  uint64_t* tmp2    = base + 11 * blk;      // evaluation temporary / for o1
  uint64_t* tmp3    = base + 12 * blk;      // evaluation temporary
  uint64_t* tmp4    = base + 13 * blk;      // evaluation temporary
  uint64_t* tmp5    = base + 14 * blk;      // evaluation temporary / interpolation workspace
  uint64_t* rec_scratch = base + 15 * blk;     // recursion scratch

  size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0;
  size_t w3n = 0, wm3n = 0, w4n = 0, wm4n = 0;
  size_t w5n = 0, wm5n = 0;
  int wm1_sign = 0, wm2_sign = 0, wm3_sign = 0, wm4_sign = 0, wm5_sign = 0;

  // ============================================================
  // Evaluation / multiplication (12 recursive calls)
  // Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size
  // ============================================================

  // Point 0: v0 = a0 * b0 → r[0..] (invoked with actual size; padded value not used)
  size_t a0n_real = std::min(k, an);
  size_t b0n_real = std::min(k, bn);
  a0n_real = normalized_size(a, a0n_real);
  b0n_real = normalized_size(b, b0n_real);
  size_t w0n = 0;
  if (a0n_real > 0 && b0n_real > 0) {
    mul_toomcook6(r, a, a0n_real, b, b0n_real, rec_scratch);
    w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn));
  }

  // Point ∞: vinf = a5 * b5 → r[10k..] (invoked with actual size)
  size_t a5n_real = (an > 5 * k) ? an - 5 * k : 0;
  size_t b5n_real = (bn > 5 * k) ? bn - 5 * k : 0;
  a5n_real = normalized_size(a + 5 * k, a5n_real);
  b5n_real = normalized_size(b + 5 * k, b5n_real);
  size_t winfn = 0;
  size_t winf_off = 10 * k;
  if (a5n_real > 0 && b5n_real > 0 && winf_off < rn) {
    mul_toomcook6(r + winf_off, a + 5 * k, a5n_real, b + 5 * k, b5n_real, rec_scratch);
    winfn = normalized_size(r + winf_off, std::min(a5n_real + b5n_real, rn - winf_off));
  }

  // Zero out gap region
  {
    size_t gap_start = w0n;
    size_t gap_end = std::min(winf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(winf_off + winfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // Evaluation at +/-1, +/-2, +/-3, +/-4, +/-5 - even/odd split + fixed k-limb pre-pad + addmul_1 (same as Toom-8)
  // a(t) = (a0 + a2*t² + a4*t⁴) + t*(a1 + a3*t² + a5*t⁴)
  // a(-t) = (a0 + a2*t² + a4*t⁴) - t*(a1 + a3*t² + a5*t⁴)
  //
  // Even part ea_even is fixed k+1 limbs (high [k] <= 2*625 = 1250, k=5 is the max)
  // Odd part ea_odd: (a1 + c_sq*a3 + c_4*a5) times k_factor -> k+1 or k+2 limbs
  auto eval_even_pad = [k](uint64_t* dst, const uint64_t* p,
               uint64_t c_sq, uint64_t c_4) {
    // dst = p[0..k-1] + c_sq * p[2k..3k-1] + c_4 * p[4k..5k-1]
    std::memcpy(dst, p, k * sizeof(uint64_t));
    dst[k] = 0;
    dst[k] += addmul_1(dst, p + 2 * k, k, c_sq);
    dst[k] += addmul_1(dst, p + 4 * k, k, c_4);
  };
  auto eval_odd_pad = [k](uint64_t* dst, const uint64_t* p,
              uint64_t c_sq, uint64_t c_4,
              uint64_t k_val, unsigned k_sh) -> size_t {
    // Step 1: a_odd = p[k..2k-1] + c_sq * p[3k..4k-1] + c_4 * p[5k..6k-1]
    std::memcpy(dst, p + 1 * k, k * sizeof(uint64_t));
    dst[k] = 0;
    dst[k] += addmul_1(dst, p + 3 * k, k, c_sq);
    dst[k] += addmul_1(dst, p + 5 * k, k, c_4);
    size_t sz = k + 1;
    // Step 2: multiply by k (shift if k=2^sh, otherwise mul_1)
    if (k_sh > 0) {
      uint64_t ov = lshift(dst, dst, sz, k_sh);
      if (ov) { dst[sz] = ov; sz++; }
    } else if (k_val != 1) {
      uint64_t ov = mul_1(dst, dst, sz, k_val);
      if (ov) { dst[sz] = ov; sz++; }
    }
    return sz;
  };

  // Constant table: c_sq = k^2, c_4 = k^4, (k_val, k_sh) for the *k factor
  struct PtCoef { uint64_t sq, c4, kv; unsigned sh_k; };
  static constexpr PtCoef COEFS[6] = {
    {0,0,0,0},    // index 0 unused
    {1,1,1,0},    // k=1: *1
    {4,16,2,1},    // k=2: shift 1
    {9,81,3,0},    // k=3: *3
    {16,256,4,2},   // k=4: shift 2
    {25,625,5,0},   // k=5: *5
  };
  uint64_t* w_bufs[6] = {nullptr, w1_buf, w2_buf, w3_buf, w4_buf, w5_buf};
  uint64_t* wm_bufs[6] = {nullptr, wm1_buf, wm2_buf, wm3_buf, wm4_buf, wm5_buf};
  size_t w_sz_arr[6] = {0};
  size_t wm_sz_arr[6] = {0};
  int wm_sign_arr[6] = {0};

  for (int kk = 1; kk <= 5; kk++) {
    const PtCoef& cf = COEFS[kk];

    // ea_even → tmp1 (k+1 limb)
    eval_even_pad(tmp1, pa, cf.sq, cf.c4);
    size_t aen = (tmp1[k] ? k + 1 : normalized_size(tmp1, k));

    // ea_odd → tmp3 (k+1 or k+2 limbs)
    size_t aon = eval_odd_pad(tmp3, pa, cf.sq, cf.c4, cf.kv, cf.sh_k);

    // eb_even → tmp2, eb_odd → tmp4
    eval_even_pad(tmp2, pb, cf.sq, cf.c4);
    size_t ben = (tmp2[k] ? k + 1 : normalized_size(tmp2, k));
    size_t bon = eval_odd_pad(tmp4, pb, cf.sq, cf.c4, cf.kv, cf.sh_k);

    // a(k) → tmp5, a(-k) → tmp1 (overwrite)
    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    // b(k) -> wm{kk}_buf (temp), b(-k) -> tmp2 (overwrite)
    size_t bp_n = add_any(wm_bufs[kk], tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    // v(k) = a(k) * b(k) → w{kk}_buf
    if (ap_n > 0 && bp_n > 0) {
      mul_toomcook6(w_bufs[kk], tmp5, ap_n, wm_bufs[kk], bp_n, rec_scratch);
      w_sz_arr[kk] = normalized_size(w_bufs[kk], ap_n + bp_n);
    }
    // v(-k) = a(-k) * b(-k) -> wm{kk}_buf (overwritten)
    wm_sign_arr[kk] = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      mul_toomcook6(wm_bufs[kk], tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm_sz_arr[kk] = normalized_size(wm_bufs[kk], am_n + bm_n);
    }
    if (wm_sz_arr[kk] == 0) wm_sign_arr[kk] = 0;
  }

  // Copy into named variables used in interpolation (so interpolation code stays unchanged)
  w1n = w_sz_arr[1]; w2n = w_sz_arr[2]; w3n = w_sz_arr[3]; w4n = w_sz_arr[4]; w5n = w_sz_arr[5];
  wm1n = wm_sz_arr[1]; wm2n = wm_sz_arr[2]; wm3n = wm_sz_arr[3]; wm4n = wm_sz_arr[4]; wm5n = wm_sz_arr[5];
  wm1_sign = wm_sign_arr[1]; wm2_sign = wm_sign_arr[2]; wm3_sign = wm_sign_arr[3];
  wm4_sign = wm_sign_arr[4]; wm5_sign = wm_sign_arr[5];

  // ============================================================
  // interpolation - even/odd split via +/- symmetry -> Vandermonde solve
  // ============================================================
  //
  // interpolation optimization helper (2026-04-26 session 5):
  //  The 4-stage "mul_1 -> tmp buffer write -> sub -> normalized_size" is collapsed into
  //  a 2-stage "submul_1 -> high borrow propagation -> normalized_size" (single-pass).
  //  Reduces memory bandwidth (eliminates tmp routing); leverages ASM submul_1 (MULX/ADCX/ADOX 8x unrolled).

  // dst[0..dst_n-1] -= mul_const * src[0..src_n-1],returns new dst size
  auto fused_submul = [](uint64_t* dst, size_t dst_n,
              const uint64_t* src, size_t src_n, uint64_t mul_const) -> size_t {
    if (src_n == 0 || dst_n == 0 || src_n > dst_n) return dst_n;
    uint64_t borrow = submul_1(dst, src, src_n, mul_const);
    if (borrow && src_n < dst_n) sub_1(dst + src_n, dst_n - src_n, borrow);
    return normalized_size(dst, dst_n);
  };
  // dst[0..dst_n-1] -= (src[0..src_n-1] << shift),returns new dst size
  auto fused_sublsh = [](uint64_t* dst, size_t dst_n,
              const uint64_t* src, size_t src_n, unsigned shift) -> size_t {
    if (src_n == 0 || dst_n == 0 || src_n > dst_n) return dst_n;
    uint64_t borrow = sublsh_n(dst, src, src_n, shift);
    if (borrow && src_n < dst_n) sub_1(dst + src_n, dst_n - src_n, borrow);
    return normalized_size(dst, dst_n);
  };
  (void)fused_submul; (void)fused_sublsh;
  //
  // c0 = v0 (r[0..]), c10 = vinf (r[10k..])
  //
  // ± coupling:
  //  e_k = (v(k) + v(-k)) / 2 = c0 + c2*k² + c4*k⁴ + c6*k⁶ + c8*k⁸ + c10*k¹⁰
  //  o_k = (v(k) - v(-k)) / (2k) = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸
  //
  // Even: E_k = e_k - c0 - c10*k¹⁰ → 4x4 Vandermonde (k=1,2,3,4) for c2,c4,c6,c8
  // Odd: o_k for k=1,2,3,4,5 → 5x5 Vandermonde for c1,c3,c5,c7,c9

  // Buffer reuse: after evaluation completes, w*_buf is usable for interpolation
  // e_k → w(2k-1)_buf (reused),o_k → wm(2k-1)_buf (reused)
  // However, rename to manage buffer sizes

  // ---- Step 1: +/- coupling (even/odd split) ----

  // e1 = (v1 + vm1) / 2, o1 = (v1 - vm1) / 2
  // Buffer layout: e1 -> tmp1, o1 -> tmp2
  size_t e1n, o1n;
  {
    // e1 = (w1 + wm1) / 2
    if (wm1_sign >= 0) {
      e1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n);
    } else {
      std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t));
      e1n = w1n;
      if (wm1n > 0) { sub(tmp1, tmp1, e1n, wm1_buf, wm1n); e1n = normalized_size(tmp1, e1n); }
    }
    if (e1n > 0) e1n = rshift_1(tmp1, tmp1, e1n);

    // o1 = (w1 - wm1) / 2
    if (wm1_sign >= 0) {
      std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t));
      o1n = w1n;
      if (wm1n > 0) { sub(tmp2, tmp2, o1n, wm1_buf, wm1n); o1n = normalized_size(tmp2, o1n); }
    } else {
      o1n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n);
    }
    if (o1n > 0) o1n = rshift_1(tmp2, tmp2, o1n);
  }
  // e1 in tmp1, o1 in tmp2

  // e2 = (w2 + wm2) / 2, o2 = (w2 - wm2) / 4
  size_t e2n, o2n;
  {
    if (wm2_sign >= 0) {
      e2n = add_any(w1_buf, w2_buf, w2n, wm2_buf, wm2n);
    } else {
      std::memcpy(w1_buf, w2_buf, w2n * sizeof(uint64_t));
      e2n = w2n;
      if (wm2n > 0) { sub(w1_buf, w1_buf, e2n, wm2_buf, wm2n); e2n = normalized_size(w1_buf, e2n); }
    }
    if (e2n > 0) e2n = rshift_1(w1_buf, w1_buf, e2n);

    if (wm2_sign >= 0) {
      std::memcpy(wm1_buf, w2_buf, w2n * sizeof(uint64_t));
      o2n = w2n;
      if (wm2n > 0) { sub(wm1_buf, wm1_buf, o2n, wm2_buf, wm2n); o2n = normalized_size(wm1_buf, o2n); }
    } else {
      o2n = add_any(wm1_buf, w2_buf, w2n, wm2_buf, wm2n);
    }
    // /4 = >>2
    if (o2n > 0) o2n = shift_right_2(wm1_buf, wm1_buf, o2n);
  }
  // e2 in w1_buf, o2 in wm1_buf

  // e3 = (w3 + wm3) / 2, o3 = (w3 - wm3) / 6
  size_t e3n, o3n;
  {
    if (wm3_sign >= 0) {
      e3n = add_any(w2_buf, w3_buf, w3n, wm3_buf, wm3n);
    } else {
      std::memcpy(w2_buf, w3_buf, w3n * sizeof(uint64_t));
      e3n = w3n;
      if (wm3n > 0) { sub(w2_buf, w2_buf, e3n, wm3_buf, wm3n); e3n = normalized_size(w2_buf, e3n); }
    }
    if (e3n > 0) e3n = rshift_1(w2_buf, w2_buf, e3n);

    if (wm3_sign >= 0) {
      std::memcpy(wm2_buf, w3_buf, w3n * sizeof(uint64_t));
      o3n = w3n;
      if (wm3n > 0) { sub(wm2_buf, wm2_buf, o3n, wm3_buf, wm3n); o3n = normalized_size(wm2_buf, o3n); }
    } else {
      o3n = add_any(wm2_buf, w3_buf, w3n, wm3_buf, wm3n);
    }
    // /6 = >>1 then /3
    if (o3n > 0) {
      o3n = rshift_1(wm2_buf, wm2_buf, o3n);
      o3n = divexact_by3(wm2_buf, wm2_buf, o3n);
    }
  }
  // e3 in w2_buf, o3 in wm2_buf

  // e4 = (w4 + wm4) / 2, o4 = (w4 - wm4) / 8
  size_t e4n, o4n;
  {
    if (wm4_sign >= 0) {
      e4n = add_any(w3_buf, w4_buf, w4n, wm4_buf, wm4n);
    } else {
      std::memcpy(w3_buf, w4_buf, w4n * sizeof(uint64_t));
      e4n = w4n;
      if (wm4n > 0) { sub(w3_buf, w3_buf, e4n, wm4_buf, wm4n); e4n = normalized_size(w3_buf, e4n); }
    }
    if (e4n > 0) e4n = rshift_1(w3_buf, w3_buf, e4n);

    if (wm4_sign >= 0) {
      std::memcpy(wm3_buf, w4_buf, w4n * sizeof(uint64_t));
      o4n = w4n;
      if (wm4n > 0) { sub(wm3_buf, wm3_buf, o4n, wm4_buf, wm4n); o4n = normalized_size(wm3_buf, o4n); }
    } else {
      o4n = add_any(wm3_buf, w4_buf, w4n, wm4_buf, wm4n);
    }
    // /8 = >>3
    if (o4n > 0) o4n = shift_right_3(wm3_buf, wm3_buf, o4n);
  }
  // e4 in w3_buf, o4 in wm3_buf

  // e5 = (w5 + wm5) / 2, o5 = (w5 - wm5) / 10
  size_t e5n, o5n;
  {
    if (wm5_sign >= 0) {
      e5n = add_any(w4_buf, w5_buf, w5n, wm5_buf, wm5n);
    } else {
      std::memcpy(w4_buf, w5_buf, w5n * sizeof(uint64_t));
      e5n = w5n;
      if (wm5n > 0) { sub(w4_buf, w4_buf, e5n, wm5_buf, wm5n); e5n = normalized_size(w4_buf, e5n); }
    }
    if (e5n > 0) e5n = rshift_1(w4_buf, w4_buf, e5n);

    if (wm5_sign >= 0) {
      std::memcpy(wm4_buf, w5_buf, w5n * sizeof(uint64_t));
      o5n = w5n;
      if (wm5n > 0) { sub(wm4_buf, wm4_buf, o5n, wm5_buf, wm5n); o5n = normalized_size(wm4_buf, o5n); }
    } else {
      o5n = add_any(wm4_buf, w5_buf, w5n, wm5_buf, wm5n);
    }
    // /10 = >>1 then /5
    if (o5n > 0) {
      o5n = rshift_1(wm4_buf, wm4_buf, o5n);
      o5n = divexact_by5(wm4_buf, wm4_buf, o5n);
    }
  }
  // e5 in w4_buf, o5 in wm4_buf

  // ---- Buffer layout summary ----
  // e1: tmp1, e2: w1_buf, e3: w2_buf, e4: w3_buf, e5: w4_buf
  // o1: tmp2, o2: wm1_buf, o3: wm2_buf, o4: wm3_buf, o5: wm4_buf
  // free buffers: w5_buf, wm5_buf, tmp3, tmp4, tmp5

  // ---- Step 2: Solve even system ----
  // E_k = e_k - c0 - c10 * k^10 for k=1,2,3,4
  // E1 = e1 - c0 - c10     → tmp1 overwrite
  // E2 = e2 - c0 - 1024*c10   → w1_buf overwrite
  // E3 = e3 - c0 - 59049*c10  → w2_buf overwrite
  // E4 = e4 - c0 - 1048576*c10 → w3_buf overwrite

  // E1
  size_t E1n = e1n;
  if (w0n > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r, w0n); E1n = normalized_size(tmp1, E1n); }
  if (winfn > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r + winf_off, winfn); E1n = normalized_size(tmp1, E1n); }

  // E2: e2 - c0 - 1024*c10
  size_t E2n = e2n;
  if (w0n > 0 && E2n > 0) { sub(w1_buf, w1_buf, E2n, r, w0n); E2n = normalized_size(w1_buf, E2n); }
  E2n = fused_sublsh(w1_buf, E2n, r + winf_off, winfn, 10); // -= c10 << 10

  // E3: e3 - c0 - 59049*c10 (3^10 = 59049)
  size_t E3n = e3n;
  if (w0n > 0 && E3n > 0) { sub(w2_buf, w2_buf, E3n, r, w0n); E3n = normalized_size(w2_buf, E3n); }
  E3n = fused_submul(w2_buf, E3n, r + winf_off, winfn, 59049); // -= 59049*c10

  // E4: e4 - c0 - 1048576*c10 (4^10 = 2^20 = 1048576)
  size_t E4n = e4n;
  if (w0n > 0 && E4n > 0) { sub(w3_buf, w3_buf, E4n, r, w0n); E4n = normalized_size(w3_buf, E4n); }
  E4n = fused_sublsh(w3_buf, E4n, r + winf_off, winfn, 20); // -= c10 << 20

  // Even Vandermonde solve (4x4, vars c2,c4,c6,c8, nodes u=1,4,9,16):
  // E1 = c2 + c4 + c6 + c8
  // E2 = 4c2 + 16c4 + 64c6 + 256c8
  // E3 = 9c2 + 81c4 + 729c6 + 6561c8
  // E4 = 16c2 + 256c4 + 4096c6 + 65536c8
  //
  // A = E2 - 4*E1    = 12c4 + 60c6 + 252c8
  // B = E3 - 9*E1    = 72c4 + 720c6 + 6552c8
  // C = E4 - 16*E1   = 240c4 + 4080c6 + 65520c8
  // D = B - 6*A     = 360c6 + 5040c8
  // E_ = C - 20*A    = 2880c6 + 60480c8
  // H = E_ - 8*D     = 20160*c8

  // Save E1 to tmp5 (for c2 computation)
  size_t saved_E1n = E1n;
  if (E1n > 0) std::memcpy(tmp5, tmp1, E1n * sizeof(uint64_t));

  // A = E2 - 4*E1 → w5_buf
  size_t An;
  {
    std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t));
    An = E2n;
    An = fused_sublsh(w5_buf, An, tmp1, E1n, 2); // -= E1 << 2
  }

  // B = E3 - 9*E1 → wm5_buf
  size_t Bn;
  {
    std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t));
    Bn = E3n;
    Bn = fused_submul(wm5_buf, Bn, tmp1, E1n, 9); // -= 9*E1
  }

  // C = E4 - 16*E1 → tmp3
  size_t Cn;
  {
    std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t));
    Cn = E4n;
    Cn = fused_sublsh(tmp3, Cn, tmp1, E1n, 4); // -= E1 << 4
  }
  // A in w5_buf, B in wm5_buf, C in tmp3
  // E1 in tmp1 (no longer needed after this)

  // D = B - 6*A → tmp4
  size_t Dn;
  {
    std::memcpy(tmp4, wm5_buf, Bn * sizeof(uint64_t));
    Dn = Bn;
    Dn = fused_submul(tmp4, Dn, w5_buf, An, 6); // -= 6*A
  }

  // E_ = C - 20*A → tmp1
  size_t E_n;
  {
    std::memcpy(tmp1, tmp3, Cn * sizeof(uint64_t));
    E_n = Cn;
    E_n = fused_submul(tmp1, E_n, w5_buf, An, 20); // -= 20*A
  }

  // H = E_ - 8*D → tmp3
  size_t Hn;
  {
    std::memcpy(tmp3, tmp1, E_n * sizeof(uint64_t));
    Hn = E_n;
    Hn = fused_sublsh(tmp3, Hn, tmp4, Dn, 3); // -= D << 3
  }

  // c8 = H / 20160 = >>6, /315 (20160 = 2^6 * 315, 315 = 3^2 * 5 * 7)
  size_t c8n = 0;
  if (Hn > 0) {
    for (size_t i = 0; i + 1 < Hn; i++) tmp3[i] = (tmp3[i] >> 6) | (tmp3[i + 1] << 58);
    tmp3[Hn - 1] >>= 6;
    c8n = normalized_size(tmp3, Hn);
    constexpr uint64_t D_315 = 315ULL;
    constexpr uint64_t INV_315 = hensel_inverse_u64(D_315);
    c8n = divexact_by_odd(tmp3, tmp3, c8n, D_315, INV_315);
  }
  uint64_t* c8_ptr = tmp3;

  // c6 = (D - 5040*c8) / 360
  // 5040 = 7*720 = 7*16*45 = 2^4 * 3^2 * 5 * 7
  size_t c6n = Dn;
  // tmp4 holds D
  c6n = fused_submul(tmp4, c6n, c8_ptr, c8n, 5040); // -= 5040*c8
  // /360 = >>3, /45 (45 = 3^2 * 5)
  if (c6n > 0) {
    c6n = shift_right_3(tmp4, tmp4, c6n);
    constexpr uint64_t D_45 = 45ULL;
    constexpr uint64_t INV_45 = hensel_inverse_u64(D_45);
    c6n = divexact_by_odd(tmp4, tmp4, c6n, D_45, INV_45);
  }
  uint64_t* c6_ptr = tmp4;

  // c4 = (A - 60*c6 - 252*c8) / 12
  size_t c4n = An;
  // w5_buf holds A
  c4n = fused_submul(w5_buf, c4n, c6_ptr, c6n, 60);  // -= 60*c6
  c4n = fused_submul(w5_buf, c4n, c8_ptr, c8n, 252); // -= 252*c8
  if (c4n > 0) c4n = divexact_by12(w5_buf, w5_buf, c4n);
  uint64_t* c4_ptr = w5_buf;

  // c2 = E1 - c4 - c6 - c8 (E1 saved in tmp5)
  size_t c2n = saved_E1n;
  if (saved_E1n > 0) std::memcpy(wm5_buf, tmp5, saved_E1n * sizeof(uint64_t));
  if (c4n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c4_ptr, c4n); c2n = normalized_size(wm5_buf, c2n); }
  if (c6n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c6_ptr, c6n); c2n = normalized_size(wm5_buf, c2n); }
  if (c8n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c8_ptr, c8n); c2n = normalized_size(wm5_buf, c2n); }
  uint64_t* c2_ptr = wm5_buf;

  // ---- Step 3: Solve odd system ----
  // o_k = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸ for k=1,2,3,4,5
  // 5x5 Vandermonde (nodes u=1,4,9,16,25)
  //
  // o1 in tmp2, o2 in wm1_buf, o3 in wm2_buf, o4 in wm3_buf, o5 in wm4_buf
  //
  // D1 = o2 - o1 = 3c3 + 15c5 + 63c7 + 255c9
  // D2 = o3 - o1 = 8c3 + 80c5 + 728c7 + 6560c9
  // D3 = o4 - o1 = 15c3 + 255c5 + 4095c7 + 65535c9
  // D4 = o5 - o1 = 24c3 + 624c5 + 15624c7 + 390624c9

  // D1 = o2 - o1 → w1_buf
  size_t D1n;
  {
    std::memcpy(w1_buf, wm1_buf, o2n * sizeof(uint64_t));
    D1n = o2n;
    if (o1n > 0 && D1n > 0) {
      sub(w1_buf, w1_buf, D1n, tmp2, o1n);
      D1n = normalized_size(w1_buf, D1n);
    }
  }

  // D2 = o3 - o1 → w2_buf
  size_t D2n;
  {
    std::memcpy(w2_buf, wm2_buf, o3n * sizeof(uint64_t));
    D2n = o3n;
    if (o1n > 0 && D2n > 0) {
      sub(w2_buf, w2_buf, D2n, tmp2, o1n);
      D2n = normalized_size(w2_buf, D2n);
    }
  }

  // D3 = o4 - o1 → w3_buf
  size_t D3n;
  {
    std::memcpy(w3_buf, wm3_buf, o4n * sizeof(uint64_t));
    D3n = o4n;
    if (o1n > 0 && D3n > 0) {
      sub(w3_buf, w3_buf, D3n, tmp2, o1n);
      D3n = normalized_size(w3_buf, D3n);
    }
  }

  // D4 = o5 - o1 → w4_buf (e5 was there but we've consumed it)
  size_t D4n;
  {
    std::memcpy(w4_buf, wm4_buf, o5n * sizeof(uint64_t));
    D4n = o5n;
    if (o1n > 0 && D4n > 0) {
      sub(w4_buf, w4_buf, D4n, tmp2, o1n);
      D4n = normalized_size(w4_buf, D4n);
    }
  }

  // F1 = 3*D2 - 8*D1 = 120c5 + 1680c7 + 17640c9 → wm1_buf
  size_t F1n;
  {
    if (D2n > 0) {
      uint64_t ov = mul_1(wm1_buf, w2_buf, D2n, 3);
      F1n = D2n; if (ov) { wm1_buf[F1n] = ov; F1n++; }
    } else { F1n = 0; }
    F1n = fused_sublsh(wm1_buf, F1n, w1_buf, D1n, 3); // -= D1 << 3
  }

  // F2 = 3*D3 - 15*D1 = 540c5 + 11340c7 + 192780c9 → wm2_buf
  size_t F2n;
  {
    if (D3n > 0) {
      uint64_t ov = mul_1(wm2_buf, w3_buf, D3n, 3);
      F2n = D3n; if (ov) { wm2_buf[F2n] = ov; F2n++; }
    } else { F2n = 0; }
    F2n = fused_submul(wm2_buf, F2n, w1_buf, D1n, 15); // -= 15*D1
  }

  // F3 = 3*D4 - 24*D1 = 1512c5 + 45360c7 + 1165752c9 → wm3_buf
  size_t F3n;
  {
    if (D4n > 0) {
      uint64_t ov = mul_1(wm3_buf, w4_buf, D4n, 3);
      F3n = D4n; if (ov) { wm3_buf[F3n] = ov; F3n++; }
    } else { F3n = 0; }
    F3n = fused_submul(wm3_buf, F3n, w1_buf, D1n, 24); // -= 24*D1
  }

  // G1 = 2*F2 - 9*F1 = 7560c7 + 226800c9 → wm4_buf
  size_t G1n;
  {
    if (F2n > 0) {
      uint64_t ov = lshift(wm4_buf, wm2_buf, F2n, 1);
      G1n = F2n; if (ov) { wm4_buf[G1n] = ov; G1n++; }
    } else { G1n = 0; }
    G1n = fused_submul(wm4_buf, G1n, wm1_buf, F1n, 9); // -= 9*F1
  }

  // G2 = 5*F3 - 63*F1 = 120960c7 + 4717440c9 → w2_buf
  size_t G2n;
  {
    if (F3n > 0) {
      uint64_t ov = mul_1(w2_buf, wm3_buf, F3n, 5);
      G2n = F3n; if (ov) { w2_buf[G2n] = ov; G2n++; }
    } else { G2n = 0; }
    G2n = fused_submul(w2_buf, G2n, wm1_buf, F1n, 63); // -= 63*F1
  }

  // H_odd = G2 - 16*G1 = 1088640*c9 → w3_buf
  size_t H_odd_n;
  {
    std::memcpy(w3_buf, w2_buf, G2n * sizeof(uint64_t));
    H_odd_n = G2n;
    H_odd_n = fused_sublsh(w3_buf, H_odd_n, wm4_buf, G1n, 4); // -= G1 << 4
  }

  // c9 = H_odd / 1088640 = >>7, /3^5, /5, /7
  // 1088640 = 2^7 * 3^5 * 5 * 7 ... wait let me recheck.
  // 1088640 = 1088640.
  // 1088640 / 128 = 8505
  // 8505 / 3 = 2835, /3 = 945, /3 = 315, /3 = 105, /3 = 35, /5 = 7, /7 = 1
  // So 1088640 = 2^7 * 3^5 * 5 * 7
  size_t c9n = H_odd_n;
  if (c9n > 0) {
    // >>7 then /8505 (8505 = 3^5 * 5 * 7)
    for (size_t i = 0; i + 1 < c9n; i++) w3_buf[i] = (w3_buf[i] >> 7) | (w3_buf[i + 1] << 57);
    w3_buf[c9n - 1] >>= 7;
    c9n = normalized_size(w3_buf, c9n);
    constexpr uint64_t D_8505 = 8505ULL;
    constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505);
    c9n = divexact_by_odd(w3_buf, w3_buf, c9n, D_8505, INV_8505);
  }
  uint64_t* c9_ptr = w3_buf;

  // c7 = (G1 - 226800*c9) / 7560
  // 7560 = 2^3 * 3^3 * 5 * 7
  size_t c7n = G1n;
  // wm4_buf holds G1
  c7n = fused_submul(wm4_buf, c7n, c9_ptr, c9n, 226800); // -= 226800*c9
  if (c7n > 0) {
    // /7560 = >>3, /945 (945 = 3^3 * 5 * 7)
    c7n = shift_right_3(wm4_buf, wm4_buf, c7n);
    constexpr uint64_t D_945 = 945ULL;
    constexpr uint64_t INV_945 = hensel_inverse_u64(D_945);
    c7n = divexact_by_odd(wm4_buf, wm4_buf, c7n, D_945, INV_945);
  }
  uint64_t* c7_ptr = wm4_buf;

  // c5 = (F1 - 1680*c7 - 17640*c9) / 120
  size_t c5n = F1n;
  // wm1_buf holds F1
  c5n = fused_submul(wm1_buf, c5n, c7_ptr, c7n, 1680);  // -= 1680*c7
  c5n = fused_submul(wm1_buf, c5n, c9_ptr, c9n, 17640); // -= 17640*c9
  if (c5n > 0) c5n = divexact_by120(wm1_buf, wm1_buf, c5n);
  uint64_t* c5_ptr = wm1_buf;

  // c3 = (D1 - 15*c5 - 63*c7 - 255*c9) / 3
  size_t c3n = D1n;
  // w1_buf holds D1
  c3n = fused_submul(w1_buf, c3n, c5_ptr, c5n, 15);  // -= 15*c5
  c3n = fused_submul(w1_buf, c3n, c7_ptr, c7n, 63);  // -= 63*c7
  c3n = fused_submul(w1_buf, c3n, c9_ptr, c9n, 255); // -= 255*c9
  if (c3n > 0) c3n = divexact_by3(w1_buf, w1_buf, c3n);
  uint64_t* c3_ptr = w1_buf;

  // c1 = o1 - c3 - c5 - c7 - c9
  size_t c1n = o1n;
  // tmp2 holds o1
  if (c3n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c3_ptr, c3n); c1n = normalized_size(tmp2, c1n); }
  if (c5n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c5_ptr, c5n); c1n = normalized_size(tmp2, c1n); }
  if (c7n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c7_ptr, c7n); c1n = normalized_size(tmp2, c1n); }
  if (c9n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c9_ptr, c9n); c1n = normalized_size(tmp2, c1n); }
  uint64_t* c1_ptr = tmp2;

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + ... + c9*B^(9k)
  // ============================================================
  // c0 placed at r[0..], c10 placed at r[10k..]

  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
  }
  if (c4n > 0 && 4 * k < rn) {
    size_t space = rn - 4 * k;
    add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space));
  }
  if (c5n > 0 && 5 * k < rn) {
    size_t space = rn - 5 * k;
    add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space));
  }
  if (c6n > 0 && 6 * k < rn) {
    size_t space = rn - 6 * k;
    add(r + 6 * k, r + 6 * k, space, c6_ptr, std::min(c6n, space));
  }
  if (c7n > 0 && 7 * k < rn) {
    size_t space = rn - 7 * k;
    add(r + 7 * k, r + 7 * k, space, c7_ptr, std::min(c7n, space));
  }
  if (c8n > 0 && 8 * k < rn) {
    size_t space = rn - 8 * k;
    add(r + 8 * k, r + 8 * k, space, c8_ptr, std::min(c8n, space));
  }
  if (c9n > 0 && 9 * k < rn) {
    size_t space = rn - 9 * k;
    add(r + 9 * k, r + 9 * k, space, c9_ptr, std::min(c9n, space));
  }
}

// ================================================================
// Toom-Cook-8,4 multiplication (2:1 unbalanced specialization, deg-10 product)
// ================================================================
//
// A(x) = a7·x⁷ + ... + a0 (8 chunks, deg 7)
// B(x) = b3·x³ + ... + b0 (4 chunks, deg 3)
// C(x) = A*B has deg 10 -> same interpolation structure as Toom-6 (12 evaluation points {0,+/-1..+/-5,inf})
//
// Only the evaluation changes:
//  A_even(t²) = a0 + a2·t² + a4·t⁴ + a6·t⁶ (4 terms)
//  A_odd (t²) = a1 + a3·t² + a5·t⁴ + a7·t⁶ (4 terms)
//  B_even(t²) = b0 + b2·t²          (2 terms)
//  B_odd (t²) = b1 + b3·t²          (2 terms)
//  vinf = a7*b3 (Toom-6 uses a5*b5)
//
// interpolation + compose is identical to mul_toomcook6 (depends only on deg(C))
//
// Applicability: bn >= TOOMCOOK84_THRESHOLD (1000) with ratio 2:1
// effect: mul(2049,1025) replaces 5xToom-4(513) with 11xToom-3(257), ~10% improvement expected
// Toom-8,4 dispatch threshold - clear improvement at bn >= 1200 (measured):
//  bn=1025: Toom-4,2=202us, Toom-8,4=208us (marginal loss, within variance)
//  bn=1500: Toom-4,2=373us, Toom-8,4=365us (-2%, slight improvement)
//  bn=3000: Toom-4,2=1126us, Toom-8,4=1025us (**-9%, significant improvement**)
// Use Toom-8,4 for bn >= 1200 and Toom-4,2 for bn < 1200.
// Toom-8,4 has small sub-mults via 11xToom-3(n_a) or 11xToom-4(n_a),
// and at large n is advantageous in both cache efficiency and arithmetic cost.
constexpr size_t TOOMCOOK84_THRESHOLD = 1200;

inline size_t mul_toomcook84_scratch_size(size_t an, size_t bn) {
  if (an < bn) std::swap(an, bn);
  size_t k = (an + 7) / 8;
  // Same scratch layout as Toom-6 + margin (A evaluation has one more term, so blk +2)
  size_t blk = 2 * (k + 12);
  // rec_scratch uses multiply_rec_scratch_size to avoid FFT early-zero
  return 15 * blk + multiply_rec_scratch_size(k + 4, k + 4);
}

inline void mul_toomcook84(uint64_t* r,
              const uint64_t* a, size_t an,
              const uint64_t* b, size_t bn,
              uint64_t* scratch) {
  // Assumes: an >= bn, 2*bn-1 ≤ an ≤ 5*bn/2-1 (guaranteed by caller)

  size_t k = (an + 7) / 8;
  size_t rn = an + bn;

  // --- 8-way split (A) ---
  const uint64_t* a0 = a;
  size_t a0n = normalized_size(a0, std::min(k, an));
  const uint64_t* a1 = a + k;
  size_t a1n = (an > k) ? normalized_size(a1, std::min(k, an - k)) : 0;
  const uint64_t* a2 = a + 2 * k;
  size_t a2n = (an > 2 * k) ? normalized_size(a2, std::min(k, an - 2 * k)) : 0;
  const uint64_t* a3 = a + 3 * k;
  size_t a3n = (an > 3 * k) ? normalized_size(a3, std::min(k, an - 3 * k)) : 0;
  const uint64_t* a4 = a + 4 * k;
  size_t a4n = (an > 4 * k) ? normalized_size(a4, std::min(k, an - 4 * k)) : 0;
  const uint64_t* a5 = a + 5 * k;
  size_t a5n = (an > 5 * k) ? normalized_size(a5, std::min(k, an - 5 * k)) : 0;
  const uint64_t* a6 = a + 6 * k;
  size_t a6n = (an > 6 * k) ? normalized_size(a6, std::min(k, an - 6 * k)) : 0;
  const uint64_t* a7 = a + 7 * k;
  size_t a7n = (an > 7 * k) ? normalized_size(a7, an - 7 * k) : 0;

  // --- 4-way split (B) ---
  // Assumes bn ~ an/2. The natural b-chunk size is ceil(bn/4) ~ k (an=2*bn => k=bn/4)
  const uint64_t* b0 = b;
  size_t b0n = normalized_size(b0, std::min(k, bn));
  const uint64_t* b1 = b + k;
  size_t b1n = (bn > k) ? normalized_size(b1, std::min(k, bn - k)) : 0;
  const uint64_t* b2 = b + 2 * k;
  size_t b2n = (bn > 2 * k) ? normalized_size(b2, std::min(k, bn - 2 * k)) : 0;
  const uint64_t* b3 = b + 3 * k;
  size_t b3n = (bn > 3 * k) ? normalized_size(b3, bn - 3 * k) : 0;

  // --- Scratch layout (same as Toom-6) ---
  size_t blk = 2 * (k + 12);
  uint64_t* w1_buf   = scratch;
  uint64_t* wm1_buf  = scratch + blk;
  uint64_t* w2_buf   = scratch + 2 * blk;
  uint64_t* wm2_buf  = scratch + 3 * blk;
  uint64_t* w3_buf   = scratch + 4 * blk;
  uint64_t* wm3_buf  = scratch + 5 * blk;
  uint64_t* w4_buf   = scratch + 6 * blk;
  uint64_t* wm4_buf  = scratch + 7 * blk;
  uint64_t* w5_buf   = scratch + 8 * blk;
  uint64_t* wm5_buf  = scratch + 9 * blk;
  uint64_t* tmp1    = scratch + 10 * blk;
  uint64_t* tmp2    = scratch + 11 * blk;
  uint64_t* tmp3    = scratch + 12 * blk;
  uint64_t* tmp4    = scratch + 13 * blk;
  uint64_t* tmp5    = scratch + 14 * blk;
  uint64_t* rec_scratch = scratch + 15 * blk;

  size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0;
  size_t w3n = 0, wm3n = 0, w4n = 0, wm4n = 0;
  size_t w5n = 0, wm5n = 0;
  int wm1_sign = 0, wm2_sign = 0, wm3_sign = 0, wm4_sign = 0, wm5_sign = 0;

  // ============================================================
  // Evaluation / multiplication (12 recursive calls)
  // ============================================================

  // Point 0: v0 = a0 * b0 → r[0..]
  size_t w0n = 0;
  if (a0n > 0 && b0n > 0) {
    multiply(r, a0, a0n, b0, b0n, rec_scratch);
    w0n = normalized_size(r, std::min(a0n + b0n, rn));
  }

  // Point ∞: vinf = a7 * b3 → r[10k..]
  size_t winfn = 0;
  size_t winf_off = 10 * k;
  if (a7n > 0 && b3n > 0 && winf_off < rn) {
    multiply(r + winf_off, a7, a7n, b3, b3n, rec_scratch);
    winfn = normalized_size(r + winf_off, std::min(a7n + b3n, rn - winf_off));
  }

  // Zero out gap region
  {
    size_t gap_start = w0n;
    size_t gap_end = std::min(winf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(winf_off + winfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // Point ±1: A(±1) = (a0+a2+a4+a6) ± (a1+a3+a5+a7), B(±1) = (b0+b2) ± (b1+b3)
  {
    // a_even = a0 + a2 + a4 + a6
    size_t aen = add_any(tmp1, a0, a0n, a2, a2n);
    aen = add_any(tmp1, tmp1, aen, a4, a4n);
    aen = add_any(tmp1, tmp1, aen, a6, a6n);

    // a_odd = a1 + a3 + a5 + a7
    size_t aon = add_any(tmp3, a1, a1n, a3, a3n);
    aon = add_any(tmp3, tmp3, aon, a5, a5n);
    aon = add_any(tmp3, tmp3, aon, a7, a7n);

    // b_even = b0 + b2
    size_t ben = add_any(tmp2, b0, b0n, b2, b2n);

    // b_odd = b1 + b3
    size_t bon = add_any(tmp4, b1, b1n, b3, b3n);

    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    size_t bp_n = add_any(wm1_buf, tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    if (ap_n > 0 && bp_n > 0) {
      multiply(w1_buf, tmp5, ap_n, wm1_buf, bp_n, rec_scratch);
      w1n = normalized_size(w1_buf, ap_n + bp_n);
    }

    wm1_sign = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      multiply(wm1_buf, tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm1n = normalized_size(wm1_buf, am_n + bm_n);
    }
    if (wm1n == 0) wm1_sign = 0;
  }

  // Point ±2: A_even = a0+4a2+16a4+64a6, A_odd_half = a1+4a3+16a5+64a7
  //      B_even = b0+4b2,      B_odd_half = b1+4b3
  {
    size_t aen = a0n;
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    if (a2n > 0) {
      uint64_t ov = lshift(tmp5, a2, a2n, 2);
      size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a4n > 0) {
      uint64_t ov = lshift(tmp5, a4, a4n, 4);
      size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a6n > 0) {
      uint64_t ov = lshift(tmp5, a6, a6n, 6);
      size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }

    size_t aon = a1n;
    if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t));
    if (a3n > 0) {
      uint64_t ov = lshift(tmp5, a3, a3n, 2);
      size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a5n > 0) {
      uint64_t ov = lshift(tmp5, a5, a5n, 4);
      size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a7n > 0) {
      uint64_t ov = lshift(tmp5, a7, a7n, 6);
      size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    // a_odd = 2 * a_odd_half
    if (aon > 0) {
      uint64_t ov = lshift(tmp3, tmp3, aon, 1);
      if (ov) { tmp3[aon] = ov; aon++; }
    }

    size_t ben = b0n;
    if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t));
    if (b2n > 0) {
      uint64_t ov = lshift(tmp5, b2, b2n, 2);
      size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; }
      ben = add_any(tmp2, tmp2, ben, tmp5, tn);
    }

    size_t bon = b1n;
    if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t));
    if (b3n > 0) {
      uint64_t ov = lshift(tmp5, b3, b3n, 2);
      size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; }
      bon = add_any(tmp4, tmp4, bon, tmp5, tn);
    }
    // b_odd = 2 * b_odd_half
    if (bon > 0) {
      uint64_t ov = lshift(tmp4, tmp4, bon, 1);
      if (ov) { tmp4[bon] = ov; bon++; }
    }

    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    size_t bp_n = add_any(wm2_buf, tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    if (ap_n > 0 && bp_n > 0) {
      multiply(w2_buf, tmp5, ap_n, wm2_buf, bp_n, rec_scratch);
      w2n = normalized_size(w2_buf, ap_n + bp_n);
    }
    wm2_sign = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      multiply(wm2_buf, tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm2n = normalized_size(wm2_buf, am_n + bm_n);
    }
    if (wm2n == 0) wm2_sign = 0;
  }

  // Point ±3: A_even = a0+9a2+81a4+729a6, A_odd_third = a1+9a3+81a5+729a7
  //      B_even = b0+9b2,      B_odd_third = b1+9b3
  {
    size_t aen = a0n;
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    if (a2n > 0) {
      uint64_t ov = mul_1(tmp5, a2, a2n, 9);
      size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a4n > 0) {
      uint64_t ov = mul_1(tmp5, a4, a4n, 81);
      size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a6n > 0) {
      uint64_t ov = mul_1(tmp5, a6, a6n, 729);
      size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }

    size_t aon = a1n;
    if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t));
    if (a3n > 0) {
      uint64_t ov = mul_1(tmp5, a3, a3n, 9);
      size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a5n > 0) {
      uint64_t ov = mul_1(tmp5, a5, a5n, 81);
      size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a7n > 0) {
      uint64_t ov = mul_1(tmp5, a7, a7n, 729);
      size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (aon > 0) {
      uint64_t ov = mul_1(tmp3, tmp3, aon, 3);
      if (ov) { tmp3[aon] = ov; aon++; }
    }

    size_t ben = b0n;
    if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t));
    if (b2n > 0) {
      uint64_t ov = mul_1(tmp5, b2, b2n, 9);
      size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; }
      ben = add_any(tmp2, tmp2, ben, tmp5, tn);
    }

    size_t bon = b1n;
    if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t));
    if (b3n > 0) {
      uint64_t ov = mul_1(tmp5, b3, b3n, 9);
      size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; }
      bon = add_any(tmp4, tmp4, bon, tmp5, tn);
    }
    if (bon > 0) {
      uint64_t ov = mul_1(tmp4, tmp4, bon, 3);
      if (ov) { tmp4[bon] = ov; bon++; }
    }

    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    size_t bp_n = add_any(wm3_buf, tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    if (ap_n > 0 && bp_n > 0) {
      multiply(w3_buf, tmp5, ap_n, wm3_buf, bp_n, rec_scratch);
      w3n = normalized_size(w3_buf, ap_n + bp_n);
    }
    wm3_sign = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      multiply(wm3_buf, tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm3n = normalized_size(wm3_buf, am_n + bm_n);
    }
    if (wm3n == 0) wm3_sign = 0;
  }

  // Point ±4: A_even = a0+16a2+256a4+4096a6, A_odd_quarter = a1+16a3+256a5+4096a7
  //      B_even = b0+16b2,       B_odd_quarter = b1+16b3
  {
    size_t aen = a0n;
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    if (a2n > 0) {
      uint64_t ov = lshift(tmp5, a2, a2n, 4);
      size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a4n > 0) {
      uint64_t ov = lshift(tmp5, a4, a4n, 8);
      size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a6n > 0) {
      uint64_t ov = lshift(tmp5, a6, a6n, 12);
      size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }

    size_t aon = a1n;
    if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t));
    if (a3n > 0) {
      uint64_t ov = lshift(tmp5, a3, a3n, 4);
      size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a5n > 0) {
      uint64_t ov = lshift(tmp5, a5, a5n, 8);
      size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a7n > 0) {
      uint64_t ov = lshift(tmp5, a7, a7n, 12);
      size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (aon > 0) {
      uint64_t ov = lshift(tmp3, tmp3, aon, 2); // *4
      if (ov) { tmp3[aon] = ov; aon++; }
    }

    size_t ben = b0n;
    if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t));
    if (b2n > 0) {
      uint64_t ov = lshift(tmp5, b2, b2n, 4);
      size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; }
      ben = add_any(tmp2, tmp2, ben, tmp5, tn);
    }

    size_t bon = b1n;
    if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t));
    if (b3n > 0) {
      uint64_t ov = lshift(tmp5, b3, b3n, 4);
      size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; }
      bon = add_any(tmp4, tmp4, bon, tmp5, tn);
    }
    if (bon > 0) {
      uint64_t ov = lshift(tmp4, tmp4, bon, 2); // *4
      if (ov) { tmp4[bon] = ov; bon++; }
    }

    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    size_t bp_n = add_any(wm4_buf, tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    if (ap_n > 0 && bp_n > 0) {
      multiply(w4_buf, tmp5, ap_n, wm4_buf, bp_n, rec_scratch);
      w4n = normalized_size(w4_buf, ap_n + bp_n);
    }
    wm4_sign = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      multiply(wm4_buf, tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm4n = normalized_size(wm4_buf, am_n + bm_n);
    }
    if (wm4n == 0) wm4_sign = 0;
  }

  // Point ±5: A_even = a0+25a2+625a4+15625a6, A_odd_fifth = a1+25a3+625a5+15625a7
  //      B_even = b0+25b2,        B_odd_fifth = b1+25b3
  {
    size_t aen = a0n;
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    if (a2n > 0) {
      uint64_t ov = mul_1(tmp5, a2, a2n, 25);
      size_t tn = a2n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a4n > 0) {
      uint64_t ov = mul_1(tmp5, a4, a4n, 625);
      size_t tn = a4n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }
    if (a6n > 0) {
      uint64_t ov = mul_1(tmp5, a6, a6n, 15625);
      size_t tn = a6n; if (ov) { tmp5[tn] = ov; tn++; }
      aen = add_any(tmp1, tmp1, aen, tmp5, tn);
    }

    size_t aon = a1n;
    if (a1n > 0) std::memcpy(tmp3, a1, a1n * sizeof(uint64_t));
    if (a3n > 0) {
      uint64_t ov = mul_1(tmp5, a3, a3n, 25);
      size_t tn = a3n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a5n > 0) {
      uint64_t ov = mul_1(tmp5, a5, a5n, 625);
      size_t tn = a5n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (a7n > 0) {
      uint64_t ov = mul_1(tmp5, a7, a7n, 15625);
      size_t tn = a7n; if (ov) { tmp5[tn] = ov; tn++; }
      aon = add_any(tmp3, tmp3, aon, tmp5, tn);
    }
    if (aon > 0) {
      uint64_t ov = mul_1(tmp3, tmp3, aon, 5);
      if (ov) { tmp3[aon] = ov; aon++; }
    }

    size_t ben = b0n;
    if (b0n > 0) std::memcpy(tmp2, b0, b0n * sizeof(uint64_t));
    if (b2n > 0) {
      uint64_t ov = mul_1(tmp5, b2, b2n, 25);
      size_t tn = b2n; if (ov) { tmp5[tn] = ov; tn++; }
      ben = add_any(tmp2, tmp2, ben, tmp5, tn);
    }

    size_t bon = b1n;
    if (b1n > 0) std::memcpy(tmp4, b1, b1n * sizeof(uint64_t));
    if (b3n > 0) {
      uint64_t ov = mul_1(tmp5, b3, b3n, 25);
      size_t tn = b3n; if (ov) { tmp5[tn] = ov; tn++; }
      bon = add_any(tmp4, tmp4, bon, tmp5, tn);
    }
    if (bon > 0) {
      uint64_t ov = mul_1(tmp4, tmp4, bon, 5);
      if (ov) { tmp4[bon] = ov; bon++; }
    }

    size_t ap_n = add_any(tmp5, tmp1, aen, tmp3, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(tmp1, am_sign, tmp1, aen, tmp3, aon);

    size_t bp_n = add_any(wm5_buf, tmp2, ben, tmp4, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(tmp2, bm_sign, tmp2, ben, tmp4, bon);

    if (ap_n > 0 && bp_n > 0) {
      multiply(w5_buf, tmp5, ap_n, wm5_buf, bp_n, rec_scratch);
      w5n = normalized_size(w5_buf, ap_n + bp_n);
    }
    wm5_sign = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      multiply(wm5_buf, tmp1, am_n, tmp2, bm_n, rec_scratch);
      wm5n = normalized_size(wm5_buf, am_n + bm_n);
    }
    if (wm5n == 0) wm5_sign = 0;
  }

  // ============================================================
  // interpolation - completely identical to Toom-6 (Bodrato 12-point with deg(C)=10)
  // Below is a verbatim copy of mul_toomcook6 (line 2780-3375)
  // ============================================================

  // ---- Step 1: +/- coupling (even/odd split) ----
  size_t e1n, o1n;
  {
    if (wm1_sign >= 0) {
      e1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n);
    } else {
      std::memcpy(tmp1, w1_buf, w1n * sizeof(uint64_t));
      e1n = w1n;
      if (wm1n > 0) { sub(tmp1, tmp1, e1n, wm1_buf, wm1n); e1n = normalized_size(tmp1, e1n); }
    }
    if (e1n > 0) e1n = rshift_1(tmp1, tmp1, e1n);

    if (wm1_sign >= 0) {
      std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t));
      o1n = w1n;
      if (wm1n > 0) { sub(tmp2, tmp2, o1n, wm1_buf, wm1n); o1n = normalized_size(tmp2, o1n); }
    } else {
      o1n = add_any(tmp2, w1_buf, w1n, wm1_buf, wm1n);
    }
    if (o1n > 0) o1n = rshift_1(tmp2, tmp2, o1n);
  }

  size_t e2n, o2n;
  {
    if (wm2_sign >= 0) {
      e2n = add_any(w1_buf, w2_buf, w2n, wm2_buf, wm2n);
    } else {
      std::memcpy(w1_buf, w2_buf, w2n * sizeof(uint64_t));
      e2n = w2n;
      if (wm2n > 0) { sub(w1_buf, w1_buf, e2n, wm2_buf, wm2n); e2n = normalized_size(w1_buf, e2n); }
    }
    if (e2n > 0) e2n = rshift_1(w1_buf, w1_buf, e2n);

    if (wm2_sign >= 0) {
      std::memcpy(wm1_buf, w2_buf, w2n * sizeof(uint64_t));
      o2n = w2n;
      if (wm2n > 0) { sub(wm1_buf, wm1_buf, o2n, wm2_buf, wm2n); o2n = normalized_size(wm1_buf, o2n); }
    } else {
      o2n = add_any(wm1_buf, w2_buf, w2n, wm2_buf, wm2n);
    }
    if (o2n > 0) o2n = shift_right_2(wm1_buf, wm1_buf, o2n);
  }

  size_t e3n, o3n;
  {
    if (wm3_sign >= 0) {
      e3n = add_any(w2_buf, w3_buf, w3n, wm3_buf, wm3n);
    } else {
      std::memcpy(w2_buf, w3_buf, w3n * sizeof(uint64_t));
      e3n = w3n;
      if (wm3n > 0) { sub(w2_buf, w2_buf, e3n, wm3_buf, wm3n); e3n = normalized_size(w2_buf, e3n); }
    }
    if (e3n > 0) e3n = rshift_1(w2_buf, w2_buf, e3n);

    if (wm3_sign >= 0) {
      std::memcpy(wm2_buf, w3_buf, w3n * sizeof(uint64_t));
      o3n = w3n;
      if (wm3n > 0) { sub(wm2_buf, wm2_buf, o3n, wm3_buf, wm3n); o3n = normalized_size(wm2_buf, o3n); }
    } else {
      o3n = add_any(wm2_buf, w3_buf, w3n, wm3_buf, wm3n);
    }
    if (o3n > 0) {
      o3n = rshift_1(wm2_buf, wm2_buf, o3n);
      o3n = divexact_by3(wm2_buf, wm2_buf, o3n);
    }
  }

  size_t e4n, o4n;
  {
    if (wm4_sign >= 0) {
      e4n = add_any(w3_buf, w4_buf, w4n, wm4_buf, wm4n);
    } else {
      std::memcpy(w3_buf, w4_buf, w4n * sizeof(uint64_t));
      e4n = w4n;
      if (wm4n > 0) { sub(w3_buf, w3_buf, e4n, wm4_buf, wm4n); e4n = normalized_size(w3_buf, e4n); }
    }
    if (e4n > 0) e4n = rshift_1(w3_buf, w3_buf, e4n);

    if (wm4_sign >= 0) {
      std::memcpy(wm3_buf, w4_buf, w4n * sizeof(uint64_t));
      o4n = w4n;
      if (wm4n > 0) { sub(wm3_buf, wm3_buf, o4n, wm4_buf, wm4n); o4n = normalized_size(wm3_buf, o4n); }
    } else {
      o4n = add_any(wm3_buf, w4_buf, w4n, wm4_buf, wm4n);
    }
    if (o4n > 0) o4n = shift_right_3(wm3_buf, wm3_buf, o4n);
  }

  size_t e5n, o5n;
  {
    if (wm5_sign >= 0) {
      e5n = add_any(w4_buf, w5_buf, w5n, wm5_buf, wm5n);
    } else {
      std::memcpy(w4_buf, w5_buf, w5n * sizeof(uint64_t));
      e5n = w5n;
      if (wm5n > 0) { sub(w4_buf, w4_buf, e5n, wm5_buf, wm5n); e5n = normalized_size(w4_buf, e5n); }
    }
    if (e5n > 0) e5n = rshift_1(w4_buf, w4_buf, e5n);

    if (wm5_sign >= 0) {
      std::memcpy(wm4_buf, w5_buf, w5n * sizeof(uint64_t));
      o5n = w5n;
      if (wm5n > 0) { sub(wm4_buf, wm4_buf, o5n, wm5_buf, wm5n); o5n = normalized_size(wm4_buf, o5n); }
    } else {
      o5n = add_any(wm4_buf, w5_buf, w5n, wm5_buf, wm5n);
    }
    if (o5n > 0) {
      o5n = rshift_1(wm4_buf, wm4_buf, o5n);
      o5n = divexact_by5(wm4_buf, wm4_buf, o5n);
    }
  }
  (void)e5n; // e5 not used in the Even system (4 equations for 4 unknowns is sufficient)

  // ---- Step 2: Even system ----
  size_t E1n = e1n;
  if (w0n > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r, w0n); E1n = normalized_size(tmp1, E1n); }
  if (winfn > 0 && E1n > 0) { sub(tmp1, tmp1, E1n, r + winf_off, winfn); E1n = normalized_size(tmp1, E1n); }

  size_t E2n = e2n;
  if (w0n > 0 && E2n > 0) { sub(w1_buf, w1_buf, E2n, r, w0n); E2n = normalized_size(w1_buf, E2n); }
  if (winfn > 0 && E2n > 0) {
    uint64_t ov = lshift(w5_buf, r + winf_off, winfn, 10);
    size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; }
    sub(w1_buf, w1_buf, E2n, w5_buf, tn);
    E2n = normalized_size(w1_buf, E2n);
  }

  size_t E3n = e3n;
  if (w0n > 0 && E3n > 0) { sub(w2_buf, w2_buf, E3n, r, w0n); E3n = normalized_size(w2_buf, E3n); }
  if (winfn > 0 && E3n > 0) {
    uint64_t ov = mul_1(w5_buf, r + winf_off, winfn, 59049);
    size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; }
    sub(w2_buf, w2_buf, E3n, w5_buf, tn);
    E3n = normalized_size(w2_buf, E3n);
  }

  size_t E4n = e4n;
  if (w0n > 0 && E4n > 0) { sub(w3_buf, w3_buf, E4n, r, w0n); E4n = normalized_size(w3_buf, E4n); }
  if (winfn > 0 && E4n > 0) {
    uint64_t ov = lshift(w5_buf, r + winf_off, winfn, 20);
    size_t tn = winfn; if (ov) { w5_buf[tn] = ov; tn++; }
    sub(w3_buf, w3_buf, E4n, w5_buf, tn);
    E4n = normalized_size(w3_buf, E4n);
  }

  size_t saved_E1n = E1n;
  if (E1n > 0) std::memcpy(tmp5, tmp1, E1n * sizeof(uint64_t));

  size_t An;
  {
    if (E1n > 0) {
      uint64_t ov = lshift(wm5_buf, tmp1, E1n, 2);
      size_t tn = E1n; if (ov) { wm5_buf[tn] = ov; tn++; }
      std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t));
      An = E2n;
      sub(w5_buf, w5_buf, An, wm5_buf, tn);
      An = normalized_size(w5_buf, An);
    } else {
      std::memcpy(w5_buf, w1_buf, E2n * sizeof(uint64_t));
      An = E2n;
    }
  }

  size_t Bn;
  {
    if (E1n > 0) {
      uint64_t ov = mul_1(tmp3, tmp1, E1n, 9);
      size_t tn = E1n; if (ov) { tmp3[tn] = ov; tn++; }
      std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t));
      Bn = E3n;
      sub(wm5_buf, wm5_buf, Bn, tmp3, tn);
      Bn = normalized_size(wm5_buf, Bn);
    } else {
      std::memcpy(wm5_buf, w2_buf, E3n * sizeof(uint64_t));
      Bn = E3n;
    }
  }

  size_t Cn;
  {
    if (E1n > 0) {
      uint64_t ov = lshift(tmp4, tmp1, E1n, 4);
      size_t tn = E1n; if (ov) { tmp4[tn] = ov; tn++; }
      std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t));
      Cn = E4n;
      sub(tmp3, tmp3, Cn, tmp4, tn);
      Cn = normalized_size(tmp3, Cn);
    } else {
      std::memcpy(tmp3, w3_buf, E4n * sizeof(uint64_t));
      Cn = E4n;
    }
  }

  size_t Dn;
  {
    std::memcpy(tmp4, wm5_buf, Bn * sizeof(uint64_t));
    Dn = Bn;
    if (An > 0) {
      uint64_t ov = mul_1(tmp1, w5_buf, An, 6);
      size_t tn = An; if (ov) { tmp1[tn] = ov; tn++; }
      sub(tmp4, tmp4, Dn, tmp1, tn);
      Dn = normalized_size(tmp4, Dn);
    }
  }

  size_t E_n;
  {
    std::memcpy(tmp1, tmp3, Cn * sizeof(uint64_t));
    E_n = Cn;
    if (An > 0) {
      uint64_t ov = mul_1(tmp3, w5_buf, An, 20);
      size_t tn = An; if (ov) { tmp3[tn] = ov; tn++; }
      sub(tmp1, tmp1, E_n, tmp3, tn);
      E_n = normalized_size(tmp1, E_n);
    }
  }

  size_t Hn;
  {
    std::memcpy(tmp3, tmp1, E_n * sizeof(uint64_t));
    Hn = E_n;
    if (Dn > 0) {
      uint64_t ov = lshift(tmp1, tmp4, Dn, 3);
      size_t tn = Dn; if (ov) { tmp1[tn] = ov; tn++; }
      sub(tmp3, tmp3, Hn, tmp1, tn);
      Hn = normalized_size(tmp3, Hn);
    }
  }

  size_t c8n = 0;
  if (Hn > 0) {
    for (size_t i = 0; i + 1 < Hn; i++) tmp3[i] = (tmp3[i] >> 6) | (tmp3[i + 1] << 58);
    tmp3[Hn - 1] >>= 6;
    c8n = normalized_size(tmp3, Hn);
    constexpr uint64_t D_315 = 315ULL;
    constexpr uint64_t INV_315 = hensel_inverse_u64(D_315);
    c8n = divexact_by_odd(tmp3, tmp3, c8n, D_315, INV_315);
  }
  uint64_t* c8_ptr = tmp3;

  size_t c6n = Dn;
  if (c8n > 0 && c6n > 0) {
    uint64_t ov = mul_1(tmp1, c8_ptr, c8n, 5040);
    size_t tn = c8n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(tmp4, tmp4, c6n, tmp1, tn);
    c6n = normalized_size(tmp4, c6n);
  }
  if (c6n > 0) {
    c6n = shift_right_3(tmp4, tmp4, c6n);
    constexpr uint64_t D_45 = 45ULL;
    constexpr uint64_t INV_45 = hensel_inverse_u64(D_45);
    c6n = divexact_by_odd(tmp4, tmp4, c6n, D_45, INV_45);
  }
  uint64_t* c6_ptr = tmp4;

  size_t c4n = An;
  if (c6n > 0 && c4n > 0) {
    uint64_t ov = mul_1(tmp1, c6_ptr, c6n, 60);
    size_t tn = c6n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(w5_buf, w5_buf, c4n, tmp1, tn);
    c4n = normalized_size(w5_buf, c4n);
  }
  if (c8n > 0 && c4n > 0) {
    uint64_t ov = mul_1(tmp1, c8_ptr, c8n, 252);
    size_t tn = c8n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(w5_buf, w5_buf, c4n, tmp1, tn);
    c4n = normalized_size(w5_buf, c4n);
  }
  if (c4n > 0) c4n = divexact_by12(w5_buf, w5_buf, c4n);
  uint64_t* c4_ptr = w5_buf;

  size_t c2n = saved_E1n;
  if (saved_E1n > 0) std::memcpy(wm5_buf, tmp5, saved_E1n * sizeof(uint64_t));
  if (c4n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c4_ptr, c4n); c2n = normalized_size(wm5_buf, c2n); }
  if (c6n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c6_ptr, c6n); c2n = normalized_size(wm5_buf, c2n); }
  if (c8n > 0 && c2n > 0) { sub(wm5_buf, wm5_buf, c2n, c8_ptr, c8n); c2n = normalized_size(wm5_buf, c2n); }
  uint64_t* c2_ptr = wm5_buf;

  // ---- Step 4: Odd system ----
  size_t D1n;
  {
    std::memcpy(w1_buf, wm1_buf, o2n * sizeof(uint64_t));
    D1n = o2n;
    if (o1n > 0 && D1n > 0) {
      sub(w1_buf, w1_buf, D1n, tmp2, o1n);
      D1n = normalized_size(w1_buf, D1n);
    }
  }

  size_t D2n;
  {
    std::memcpy(w2_buf, wm2_buf, o3n * sizeof(uint64_t));
    D2n = o3n;
    if (o1n > 0 && D2n > 0) {
      sub(w2_buf, w2_buf, D2n, tmp2, o1n);
      D2n = normalized_size(w2_buf, D2n);
    }
  }

  size_t D3n;
  {
    std::memcpy(w3_buf, wm3_buf, o4n * sizeof(uint64_t));
    D3n = o4n;
    if (o1n > 0 && D3n > 0) {
      sub(w3_buf, w3_buf, D3n, tmp2, o1n);
      D3n = normalized_size(w3_buf, D3n);
    }
  }

  size_t D4n;
  {
    std::memcpy(w4_buf, wm4_buf, o5n * sizeof(uint64_t));
    D4n = o5n;
    if (o1n > 0 && D4n > 0) {
      sub(w4_buf, w4_buf, D4n, tmp2, o1n);
      D4n = normalized_size(w4_buf, D4n);
    }
  }

  size_t F1n;
  {
    if (D2n > 0) {
      uint64_t ov = mul_1(wm1_buf, w2_buf, D2n, 3);
      F1n = D2n; if (ov) { wm1_buf[F1n] = ov; F1n++; }
    } else { F1n = 0; }
    if (D1n > 0 && F1n > 0) {
      uint64_t ov = lshift(tmp1, w1_buf, D1n, 3);
      size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(wm1_buf, wm1_buf, F1n, tmp1, tn);
      F1n = normalized_size(wm1_buf, F1n);
    }
  }

  size_t F2n;
  {
    if (D3n > 0) {
      uint64_t ov = mul_1(wm2_buf, w3_buf, D3n, 3);
      F2n = D3n; if (ov) { wm2_buf[F2n] = ov; F2n++; }
    } else { F2n = 0; }
    if (D1n > 0 && F2n > 0) {
      uint64_t ov = mul_1(tmp1, w1_buf, D1n, 15);
      size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(wm2_buf, wm2_buf, F2n, tmp1, tn);
      F2n = normalized_size(wm2_buf, F2n);
    }
  }

  size_t F3n;
  {
    if (D4n > 0) {
      uint64_t ov = mul_1(wm3_buf, w4_buf, D4n, 3);
      F3n = D4n; if (ov) { wm3_buf[F3n] = ov; F3n++; }
    } else { F3n = 0; }
    if (D1n > 0 && F3n > 0) {
      uint64_t ov = mul_1(tmp1, w1_buf, D1n, 24);
      size_t tn = D1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(wm3_buf, wm3_buf, F3n, tmp1, tn);
      F3n = normalized_size(wm3_buf, F3n);
    }
  }

  size_t G1n;
  {
    if (F2n > 0) {
      uint64_t ov = lshift(wm4_buf, wm2_buf, F2n, 1);
      G1n = F2n; if (ov) { wm4_buf[G1n] = ov; G1n++; }
    } else { G1n = 0; }
    if (F1n > 0 && G1n > 0) {
      uint64_t ov = mul_1(tmp1, wm1_buf, F1n, 9);
      size_t tn = F1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(wm4_buf, wm4_buf, G1n, tmp1, tn);
      G1n = normalized_size(wm4_buf, G1n);
    }
  }

  size_t G2n;
  {
    if (F3n > 0) {
      uint64_t ov = mul_1(w2_buf, wm3_buf, F3n, 5);
      G2n = F3n; if (ov) { w2_buf[G2n] = ov; G2n++; }
    } else { G2n = 0; }
    if (F1n > 0 && G2n > 0) {
      uint64_t ov = mul_1(tmp1, wm1_buf, F1n, 63);
      size_t tn = F1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(w2_buf, w2_buf, G2n, tmp1, tn);
      G2n = normalized_size(w2_buf, G2n);
    }
  }

  size_t H_odd_n;
  {
    std::memcpy(w3_buf, w2_buf, G2n * sizeof(uint64_t));
    H_odd_n = G2n;
    if (G1n > 0 && H_odd_n > 0) {
      uint64_t ov = lshift(tmp1, wm4_buf, G1n, 4);
      size_t tn = G1n; if (ov) { tmp1[tn] = ov; tn++; }
      sub(w3_buf, w3_buf, H_odd_n, tmp1, tn);
      H_odd_n = normalized_size(w3_buf, H_odd_n);
    }
  }

  size_t c9n = H_odd_n;
  if (c9n > 0) {
    for (size_t i = 0; i + 1 < c9n; i++) w3_buf[i] = (w3_buf[i] >> 7) | (w3_buf[i + 1] << 57);
    w3_buf[c9n - 1] >>= 7;
    c9n = normalized_size(w3_buf, c9n);
    constexpr uint64_t D_8505 = 8505ULL;
    constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505);
    c9n = divexact_by_odd(w3_buf, w3_buf, c9n, D_8505, INV_8505);
  }
  uint64_t* c9_ptr = w3_buf;

  size_t c7n = G1n;
  if (c9n > 0 && c7n > 0) {
    uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 226800);
    size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(wm4_buf, wm4_buf, c7n, tmp1, tn);
    c7n = normalized_size(wm4_buf, c7n);
  }
  if (c7n > 0) {
    c7n = shift_right_3(wm4_buf, wm4_buf, c7n);
    constexpr uint64_t D_945 = 945ULL;
    constexpr uint64_t INV_945 = hensel_inverse_u64(D_945);
    c7n = divexact_by_odd(wm4_buf, wm4_buf, c7n, D_945, INV_945);
  }
  uint64_t* c7_ptr = wm4_buf;

  size_t c5n = F1n;
  if (c7n > 0 && c5n > 0) {
    uint64_t ov = mul_1(tmp1, c7_ptr, c7n, 1680);
    size_t tn = c7n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(wm1_buf, wm1_buf, c5n, tmp1, tn);
    c5n = normalized_size(wm1_buf, c5n);
  }
  if (c9n > 0 && c5n > 0) {
    uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 17640);
    size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(wm1_buf, wm1_buf, c5n, tmp1, tn);
    c5n = normalized_size(wm1_buf, c5n);
  }
  if (c5n > 0) c5n = divexact_by120(wm1_buf, wm1_buf, c5n);
  uint64_t* c5_ptr = wm1_buf;

  size_t c3n = D1n;
  if (c5n > 0 && c3n > 0) {
    uint64_t ov = mul_1(tmp1, c5_ptr, c5n, 15);
    size_t tn = c5n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(w1_buf, w1_buf, c3n, tmp1, tn);
    c3n = normalized_size(w1_buf, c3n);
  }
  if (c7n > 0 && c3n > 0) {
    uint64_t ov = mul_1(tmp1, c7_ptr, c7n, 63);
    size_t tn = c7n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(w1_buf, w1_buf, c3n, tmp1, tn);
    c3n = normalized_size(w1_buf, c3n);
  }
  if (c9n > 0 && c3n > 0) {
    uint64_t ov = mul_1(tmp1, c9_ptr, c9n, 255);
    size_t tn = c9n; if (ov) { tmp1[tn] = ov; tn++; }
    sub(w1_buf, w1_buf, c3n, tmp1, tn);
    c3n = normalized_size(w1_buf, c3n);
  }
  if (c3n > 0) c3n = divexact_by3(w1_buf, w1_buf, c3n);
  uint64_t* c3_ptr = w1_buf;

  size_t c1n = o1n;
  if (c3n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c3_ptr, c3n); c1n = normalized_size(tmp2, c1n); }
  if (c5n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c5_ptr, c5n); c1n = normalized_size(tmp2, c1n); }
  if (c7n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c7_ptr, c7n); c1n = normalized_size(tmp2, c1n); }
  if (c9n > 0 && c1n > 0) { sub(tmp2, tmp2, c1n, c9_ptr, c9n); c1n = normalized_size(tmp2, c1n); }
  uint64_t* c1_ptr = tmp2;

  // ============================================================
  // Compose: r += c1*B^k + c2*B^{2k} + ... + c9*B^{9k}
  // ============================================================
  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
  }
  if (c4n > 0 && 4 * k < rn) {
    size_t space = rn - 4 * k;
    add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space));
  }
  if (c5n > 0 && 5 * k < rn) {
    size_t space = rn - 5 * k;
    add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space));
  }
  if (c6n > 0 && 6 * k < rn) {
    size_t space = rn - 6 * k;
    add(r + 6 * k, r + 6 * k, space, c6_ptr, std::min(c6n, space));
  }
  if (c7n > 0 && 7 * k < rn) {
    size_t space = rn - 7 * k;
    add(r + 7 * k, r + 7 * k, space, c7_ptr, std::min(c7n, space));
  }
  if (c8n > 0 && 8 * k < rn) {
    size_t space = rn - 8 * k;
    add(r + 8 * k, r + 8 * k, space, c8_ptr, std::min(c8n, space));
  }
  if (c9n > 0 && 9 * k < rn) {
    size_t space = rn - 9 * k;
    add(r + 9 * k, r + 9 * k, space, c9_ptr, std::min(c9n, space));
  }
}

// ================================================================
// Toom-Cook-8 multiplication
// ================================================================
// 8-way split,Evaluation points {0, ±1, ±2, ±3, ±4, ±5, ±6, ±7, ∞},15 recursive multiplications
// Product degree 14 → 15 coefficients (c0..c14)
// Separate even/odd via +/- symmetry; even 6x6 + odd 6x6 Vandermonde interpolation (c0, c14 obtained directly)
//
// interpolation divisors (final):
//  Even: c12 div = 239500800 = 2^9*3^5*5^2*7*11 (requires divexact_by11)
//  Odd: c13 div = 18681062400 = 2^10*3^6*5^2*7*11*13 (requires divexact_by11, divexact_by13)
//
// 2026-04-19 benchmark (Zen 3): after pre-pad + addmul_1 evaluation optimization
//  Initial implementation (lambda + variable length): 2-14% slower than Toom-6 at n<4000, 13% faster at n=4000
//  After optimization (fixed k-limb pre-pad + addmul_1 chain):
//   n=1000: T8/T6 = 0.987 (3% faster)
//   n=2000: T8/T6 = 0.981 (2% faster)
//   crossover achieved at n=1000-2500 where it matches or slightly beats Toom-6
//  524K div effect: invert total multiplication 684us -> 609us (11% improvement), mul(1025,1025) 9% faster
//
// 2026-04-19 additional optimization: chained divexact_by_odd folding
//  bench-toom8-profile (SANGI_TOOM8_PROFILE) revealed that interp accounts for 38% at n=1025.
//  Consolidates back-sub chained divexact_by_{3,5,7,11,13} into a single Hensel-inverse divexact_by_odd:
//   c12: >>9 + 10×chained → >>9 + divexact_by_odd(467775)  2 passes
//   c10: >>7 + 8×chained  → >>7 + divexact_by_odd(14175)
//   c8: >>6 + 5×chained  → >>6 + divexact_by_odd(315)
//   c6: >>3 + 3×chained  → >>3 + divexact_by_odd(45)
//   c13: >>10 + 12×chained → >>10 + divexact_by_odd(18243225)
//   c11: >>7 + 10×chained → >>7 + divexact_by_odd(467775)
//   c9: >>7 + 8×chained  → >>7 + divexact_by_odd(8505)
//   c7: >>3 + 6×chained  → >>3 + divexact_by_odd(945)
//  effect (Zen 3, n=1025): Toom-8 total 125.5 → 102.2 us (-18.6%),interp 48.3 → 24.2 us (-49.9%)
//  524K div: invert total multiplication 610.5 us, GMP ratio 2.00 -> 1.87x (-6.5%)
// Toom-6 -> Toom-8 switching threshold (Zen 3 measurement, 2026-04-19b)
// 2026-04-19b: after Toom-8 interp optimization (-18.6%) + Toom-6 interp optimization (-7%),
// crossover moves forward from n~1000 to n~750. Toom-8 is 4.5% faster at n=800, 7% faster at n=900.
// The old value 1000 predates the Toom-8 interp optimization and is now too conservative.
constexpr size_t TOOMCOOK8_THRESHOLD = 800;

inline size_t mul_toomcook8_scratch_size(size_t n) {
  if (n < TOOMCOOK8_THRESHOLD) return mul_toomcook6_scratch_size(n);
  // 16k (pre-pad) + 14*(2k+10) w/wm + 4*(k+2)+4*(k+3) eval + 8*(2k+10) tmp + rec = ~68*k + margin
  return 200 * n + 8192;
}

#ifdef SANGI_TOOM8_PROFILE
// Phase accumulator (thread_local nanoseconds)
// Reset via `sangi_toom8_profile_reset()`, read via `sangi_toom8_profile_*()`.
struct SangiToom8ProfileData {
  uint64_t setup_ns = 0;
  uint64_t eval_ns = 0;  // addmul_1 / add_any / abs_sub (no recursion)
  uint64_t rec_ns = 0;   // 16 recursive mul_toomcook8 calls
  uint64_t interp_ns = 0; // ± coupling + even/odd elim + back-sub
  uint64_t compose_ns = 0;
  uint64_t toplevel_calls = 0;
};
inline SangiToom8ProfileData& sangi_toom8_profile() {
  static thread_local SangiToom8ProfileData d;
  return d;
}
inline void sangi_toom8_profile_reset() { sangi_toom8_profile() = {}; }
#define SANGI_T8P_TICK() auto _t8p_t = std::chrono::high_resolution_clock::now()
#define SANGI_T8P_ACCUM(field) do { \
  if (is_toplevel) { \
    auto _t8p_e = std::chrono::high_resolution_clock::now(); \
    sangi_toom8_profile().field += \
      (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(_t8p_e - _t8p_t).count(); \
    _t8p_t = _t8p_e; \
  } \
} while (0)
#else
#define SANGI_T8P_TICK() do{}while(0)
#define SANGI_T8P_ACCUM(field) do{}while(0)
#endif

inline void mul_toomcook8(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an < bn) {
    std::swap(a, b);
    std::swap(an, bn);
  }
  if (bn < TOOMCOOK8_THRESHOLD) {
    mul_toomcook6(r, a, an, b, bn, scratch);
    return;
  }

#ifdef SANGI_TOOM8_PROFILE
  // Only accumulate at top-level (not in recursion). Detected by checking if this is the
  // first call in the stack. Simple heuristic: use a static thread_local flag.
  static thread_local bool _t8p_in_toplevel = false;
  bool is_toplevel = !_t8p_in_toplevel;
  if (is_toplevel) { _t8p_in_toplevel = true; sangi_toom8_profile().toplevel_calls++; }
  struct _T8PScopeGuard {
    bool& flag; bool set_by_us;
    ~_T8PScopeGuard() { if (set_by_us) flag = false; }
  } _t8p_guard{_t8p_in_toplevel, is_toplevel};
#else
  constexpr bool is_toplevel = false;
  (void)is_toplevel;
#endif
  SANGI_T8P_TICK();

  size_t k = (an + 7) / 8;
  size_t rn = an + bn;

  // --- Zero-pad coefficients to k limbs (for fixed-length evaluation) ---
  uint64_t* pa = scratch;       // 8*k limbs
  uint64_t* pb = scratch + 8 * k;   // 8*k limbs
  auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) {
    size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0;
    if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t));
    if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t));
  };
  for (size_t i = 0; i < 8; i++) pad_coeff(pa + i * k, a, an, i * k);
  for (size_t i = 0; i < 8; i++) pad_coeff(pb + i * k, b, bn, i * k);

  // --- Scratch layout ---
  // Each w/wm buffer: at most 2*(k+3) = 2k+6 limbs (a(k) limited to k+3, product is 2k+6)
  size_t blk = 2 * k + 10;
  uint64_t* base = scratch + 16 * k;
  uint64_t* w_buf[8];
  uint64_t* wm_buf[8];
  for (int i = 1; i <= 7; i++) {
    w_buf[i] = base + (2 * (i - 1))   * blk;
    wm_buf[i] = base + (2 * (i - 1) + 1) * blk;
  }
  // 4 slots of k+2 limb buffers for evaluation + 4 slots of k+3 limb buffers for a(k)/a(-k)/b(k)/b(-k)
  uint64_t* eval_area = base + 14 * blk;
  uint64_t* ea_even = eval_area;            // k+2
  uint64_t* ea_odd = ea_even + (k + 2);        // k+2
  uint64_t* eb_even = ea_odd + (k + 2);        // k+2
  uint64_t* eb_odd = eb_even + (k + 2);        // k+2
  uint64_t* ap_buf = eb_odd + (k + 2);        // k+3 (a(k))
  uint64_t* am_buf = ap_buf + (k + 3);        // k+3 (a(-k))
  uint64_t* bp_buf = am_buf + (k + 3);        // k+3 (b(k))
  uint64_t* bm_buf = bp_buf + (k + 3);        // k+3 (b(-k))
  // 8 slots of tmp buffers used in interpolation (compatible with old tmp1..tmp8)
  uint64_t* tmp1 = bm_buf + (k + 3);
  uint64_t* tmp2 = tmp1 + blk;
  uint64_t* tmp3 = tmp2 + blk;
  uint64_t* tmp4 = tmp3 + blk;
  uint64_t* tmp5 = tmp4 + blk;
  uint64_t* tmp6 = tmp5 + blk;
  uint64_t* tmp7 = tmp6 + blk;
  uint64_t* tmp8 = tmp7 + blk;
  uint64_t* rec_scratch = tmp8 + blk;

  size_t w_sz[8] = {0};
  size_t wm_sz[8] = {0};
  int wm_sign[8] = {0};

  SANGI_T8P_ACCUM(setup_ns);

  // ============================================================
  // Evaluation / multiplication (15 recursive calls)
  // Fixed k-limb evaluation + addmul_1 reduces temporary buffers and normalized_size
  // ============================================================

  // Point 0: v0 = a0 * b0 → r[0..] (invoked with actual size)
  size_t a0n_real = std::min(k, an);
  size_t b0n_real = std::min(k, bn);
  a0n_real = normalized_size(a, a0n_real);
  b0n_real = normalized_size(b, b0n_real);
  size_t w0n = 0;
  if (a0n_real > 0 && b0n_real > 0) {
    SANGI_T8P_ACCUM(eval_ns);
    mul_toomcook8(r, a, a0n_real, b, b0n_real, rec_scratch);
    SANGI_T8P_ACCUM(rec_ns);
    w0n = normalized_size(r, std::min(a0n_real + b0n_real, rn));
  }

  // Point ∞: vinf = a7 * b7 → r[14k..] (invoked with actual size)
  size_t a7n_real = (an > 7 * k) ? an - 7 * k : 0;
  size_t b7n_real = (bn > 7 * k) ? bn - 7 * k : 0;
  a7n_real = normalized_size(a + 7 * k, a7n_real);
  b7n_real = normalized_size(b + 7 * k, b7n_real);
  size_t winfn = 0;
  size_t winf_off = 14 * k;
  if (a7n_real > 0 && b7n_real > 0 && winf_off < rn) {
    SANGI_T8P_ACCUM(eval_ns);
    mul_toomcook8(r + winf_off, a + 7 * k, a7n_real, b + 7 * k, b7n_real, rec_scratch);
    SANGI_T8P_ACCUM(rec_ns);
    winfn = normalized_size(r + winf_off, std::min(a7n_real + b7n_real, rn - winf_off));
  }

  // Zero out gap region
  {
    size_t gap_start = w0n;
    size_t gap_end = std::min(winf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(winf_off + winfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // Evaluate points +/-1..+/-7 via even/odd split + addmul_1
  // c_sq, c_4, c_6: constants multiplying a2, a4, a6
  // k_val, k_sh: constants/shift for multiplying the odd part by k (shift if k_sh > 0)
  // max(c_6) = 117649 < 2^17, max of 4-term sum < 2^20 -> ea_even fits in k+1 limbs
  // a_odd is also k+1 limbs. After multiplying by k it becomes k+2 limbs.
  // a(k), a(-k) are k+2 limbs (ea_even + ea_odd is at most k+1 limbs + overflow)
  // In practice a k+3 limb buffer is allocated (with margin)
  auto eval_even_pad = [k](uint64_t* dst, const uint64_t* p,
               uint64_t c_sq, uint64_t c_4, uint64_t c_6) {
    // dst = p[0..k-1] + c_sq * p[2k..3k-1] + c_4 * p[4k..5k-1] + c_6 * p[6k..7k-1]
    // dst is k+2 limbs, all zeroed first
    std::memcpy(dst, p, k * sizeof(uint64_t));
    dst[k] = 0; dst[k + 1] = 0;
    dst[k] += addmul_1(dst, p + 2 * k, k, c_sq);
    dst[k] += addmul_1(dst, p + 4 * k, k, c_4);
    dst[k] += addmul_1(dst, p + 6 * k, k, c_6);
    // dst[k+1] = 0 (no overflow possible, max ea_even[k] < 2^20)
  };
  auto eval_odd_pad = [k](uint64_t* dst, const uint64_t* p,
              uint64_t c_sq, uint64_t c_4, uint64_t c_6,
              uint64_t k_val, unsigned k_sh) -> size_t {
    // Step 1: compute a_odd = a1 + c_sq*a3 + c_4*a5 + c_6*a7 (k+1 limb)
    std::memcpy(dst, p + 1 * k, k * sizeof(uint64_t));
    dst[k] = 0;
    dst[k] += addmul_1(dst, p + 3 * k, k, c_sq);
    dst[k] += addmul_1(dst, p + 5 * k, k, c_4);
    dst[k] += addmul_1(dst, p + 7 * k, k, c_6);
    size_t sz = k + 1;
    // Step 2: multiply by k (k_val or shift k_sh)
    if (k_sh > 0) {
      uint64_t ov = lshift(dst, dst, sz, k_sh);
      if (ov) { dst[sz] = ov; sz++; }
    } else if (k_val != 1) {
      uint64_t ov = mul_1(dst, dst, sz, k_val);
      if (ov) { dst[sz] = ov; sz++; }
    }
    return sz;
  };

  // Constant table (reduced size): k_sq, k_4, k_6, k_val, k_sh
  struct PtCoef { uint64_t sq, c4, c6, kv; unsigned sh_k; };
  static constexpr PtCoef COEFS[8] = {
    {0,0,0,0,0},             // index 0 unused
    {1,1,1,1,0},             // k=1
    {4,16,64,2,1},            // k=2 (k_factor shift 1)
    {9,81,729,3,0},           // k=3
    {16,256,4096,4,2},          // k=4 (k_factor shift 2)
    {25,625,15625,5,0},         // k=5
    {36,1296,46656,6,0},         // k=6
    {49,2401,117649,7,0}         // k=7
  };

  for (int kk = 1; kk <= 7; kk++) {
    const PtCoef& cf = COEFS[kk];
    // a_even (k+2 limbs, with margin), a_odd*k (at most k+2 limbs)
    eval_even_pad(ea_even, pa, cf.sq, cf.c4, cf.c6);
    size_t aon = eval_odd_pad(ea_odd, pa, cf.sq, cf.c4, cf.c6, cf.kv, cf.sh_k);
    eval_even_pad(eb_even, pb, cf.sq, cf.c4, cf.c6);
    size_t bon = eval_odd_pad(eb_odd, pb, cf.sq, cf.c4, cf.c6, cf.kv, cf.sh_k);

    // ea_even up to k+1 limbs (k+2-th is 0), aon is k+1..k+2 limbs
    size_t aen = (ea_even[k] ? k + 1 : normalized_size(ea_even, k));

    // a(k) = aen + aon, a(-k) = |aen - aon| (signed)
    size_t ap_n = add_any(ap_buf, ea_even, aen, ea_odd, aon);
    int am_sign = 1;
    size_t am_n = abs_sub(am_buf, am_sign, ea_even, aen, ea_odd, aon);

    // b(k), b(-k)
    size_t ben = (eb_even[k] ? k + 1 : normalized_size(eb_even, k));
    size_t bp_n = add_any(bp_buf, eb_even, ben, eb_odd, bon);
    int bm_sign = 1;
    size_t bm_n = abs_sub(bm_buf, bm_sign, eb_even, ben, eb_odd, bon);

    SANGI_T8P_ACCUM(eval_ns);

    // v(k) = a(k) * b(k) → w_buf[kk]
    if (ap_n > 0 && bp_n > 0) {
      mul_toomcook8(w_buf[kk], ap_buf, ap_n, bp_buf, bp_n, rec_scratch);
      w_sz[kk] = normalized_size(w_buf[kk], ap_n + bp_n);
    }
    // v(-k) = a(-k) * b(-k) → wm_buf[kk]
    wm_sign[kk] = am_sign * bm_sign;
    if (am_n > 0 && bm_n > 0) {
      mul_toomcook8(wm_buf[kk], am_buf, am_n, bm_buf, bm_n, rec_scratch);
      wm_sz[kk] = normalized_size(wm_buf[kk], am_n + bm_n);
    }
    if (wm_sz[kk] == 0) wm_sign[kk] = 0;

    SANGI_T8P_ACCUM(rec_ns);
  }

  // interpolation helper: dst = src * c (c is a constant; returns 0 if sn == 0)
  auto mul_const = [](uint64_t* dst, const uint64_t* src, size_t sn,
            uint64_t c, unsigned /*sh_unused*/) -> size_t {
    if (sn == 0) return 0;
    if (c == 1) {
      if (dst != src) std::memcpy(dst, src, sn * sizeof(uint64_t));
      return sn;
    }
    uint64_t ov = mul_1(dst, src, sn, c);
    if (ov) { dst[sn] = ov; return sn + 1; }
    return sn;
  };

  // ============================================================
  // interpolation - even/odd split via +/- symmetry
  // ============================================================
  //
  // c0 = v0 (r[0..]), c14 = vinf (r[14k..])
  //
  // e_k = (v(k) + v(-k)) / 2   = c0 + c2*k² + c4*k⁴ + c6*k⁶ + c8*k⁸ + c10*k¹⁰ + c12*k¹² + c14*k¹⁴
  // o_k = (v(k) - v(-k)) / (2k) = c1 + c3*k² + c5*k⁴ + c7*k⁶ + c9*k⁸ + c11*k¹⁰ + c13*k¹²
  //
  // E_k = e_k - c0 - c14*k^14 -> 6x6 Vandermonde (k=1..6) for c2..c12, E7 unused
  // Odd: o_k for k=1..7 -> eliminate c3..c13; compute c1 from o1 last

  // ---- Step 1: store +/- coupling in tmp buffers ----
  // Store e_k in the tmp buffer group (e_bufs), o_k in a separate buffer group
  // To save space, w_buf and wm_buf are reused
  // Layout: e1..e7 in tmp1..tmp7, o1..o7 in wm_buf[1]..wm_buf[7]
  //
  // However, v(-k) must be finished before wm_buf[k] is overwritten
  // Processing order: compute (e_k, o_k) for each k -> free wm_buf[k] and w_buf[k]

  // Strategy: write e_k, o_k into dedicated buffers
  // Layout:
  //  e_k → tmp_e[k] (k=1..7) - store in w_buf[k] (no longer needed)
  //  o_k → tmp_o[k] (k=1..7) - store in wm_buf[k] (no longer needed)
  //  i.e. overwrite w_buf with e_k and wm_buf with o_k

  size_t e_sz[8] = {0};
  size_t o_sz[8] = {0};
  // Use tmp buffers as workspace (tmp1..tmp8)

  for (int kk = 1; kk <= 7; kk++) {
    // e_k = (v(k) + v(-k)) / 2 → tmp1 (temp)
    size_t ek_n;
    if (wm_sign[kk] >= 0) {
      ek_n = add_any(tmp1, w_buf[kk], w_sz[kk], wm_buf[kk], wm_sz[kk]);
    } else {
      std::memcpy(tmp1, w_buf[kk], w_sz[kk] * sizeof(uint64_t));
      ek_n = w_sz[kk];
      if (wm_sz[kk] > 0) {
        sub(tmp1, tmp1, ek_n, wm_buf[kk], wm_sz[kk]);
        ek_n = normalized_size(tmp1, ek_n);
      }
    }
    if (ek_n > 0) ek_n = rshift_1(tmp1, tmp1, ek_n);

    // o_k = (v(k) - v(-k)) / (2k) → tmp2 (temp)
    size_t ok_n;
    if (wm_sign[kk] >= 0) {
      std::memcpy(tmp2, w_buf[kk], w_sz[kk] * sizeof(uint64_t));
      ok_n = w_sz[kk];
      if (wm_sz[kk] > 0) {
        sub(tmp2, tmp2, ok_n, wm_buf[kk], wm_sz[kk]);
        ok_n = normalized_size(tmp2, ok_n);
      }
    } else {
      ok_n = add_any(tmp2, w_buf[kk], w_sz[kk], wm_buf[kk], wm_sz[kk]);
    }
    // Divide by 2*kk
    if (ok_n > 0) {
      if (kk == 1) {
        ok_n = rshift_1(tmp2, tmp2, ok_n);     // /2
      } else if (kk == 2) {
        ok_n = shift_right_2(tmp2, tmp2, ok_n);  // /4
      } else if (kk == 3) {
        ok_n = rshift_1(tmp2, tmp2, ok_n);     // /2
        ok_n = divexact_by3(tmp2, tmp2, ok_n);   // /3 → /6 total
      } else if (kk == 4) {
        ok_n = shift_right_3(tmp2, tmp2, ok_n);  // /8
      } else if (kk == 5) {
        ok_n = rshift_1(tmp2, tmp2, ok_n);     // /2
        ok_n = divexact_by5(tmp2, tmp2, ok_n);   // /5 → /10 total
      } else if (kk == 6) {
        ok_n = shift_right_2(tmp2, tmp2, ok_n);  // /4
        ok_n = divexact_by3(tmp2, tmp2, ok_n);   // /3 → /12 total
      } else if (kk == 7) {
        ok_n = rshift_1(tmp2, tmp2, ok_n);     // /2
        ok_n = divexact_by7(tmp2, tmp2, ok_n);   // /7 → /14 total
      }
    }

    // Store e_k and o_k in w_buf[kk] and wm_buf[kk]
    if (ek_n > 0) std::memcpy(w_buf[kk], tmp1, ek_n * sizeof(uint64_t));
    e_sz[kk] = ek_n;
    if (ok_n > 0) std::memcpy(wm_buf[kk], tmp2, ok_n * sizeof(uint64_t));
    o_sz[kk] = ok_n;
  }

  // Bug fix for kk=4: we divided by 2 (rshift_1) + shift_right_3 = /16, but we wanted /8
  // Correct logic: after /2, we need /4 more for kk=4. Let me rewrite the division step properly.
  // (Done above; will fix after dry run)

  // ---- Step 2: E_k = e_k - c0 - c14*k^14 (k=1..6) ----
  // E_k ∈ tmp1..tmp6. Note: we only need E1..E6 (E7 unused for 6x6 system).
  // c14 * k^14 constants: k=1: 1, k=2: 16384, k=3: 4782969, k=4: 268435456,
  //           k=5: 6103515625, k=6: 78364164096
  static constexpr uint64_t K14[7] = {
    0, 1ULL, 16384ULL, 4782969ULL, 268435456ULL,
    6103515625ULL, 78364164096ULL
  };

  size_t E_sz[7] = {0}; // index 1..6 used
  uint64_t* E_buf[7];
  E_buf[1] = tmp1; E_buf[2] = tmp2; E_buf[3] = tmp3;
  E_buf[4] = tmp4; E_buf[5] = tmp5; E_buf[6] = tmp6;

  for (int kk = 1; kk <= 6; kk++) {
    // Copy e_k to E_buf[kk]
    std::memcpy(E_buf[kk], w_buf[kk], e_sz[kk] * sizeof(uint64_t));
    size_t sz = e_sz[kk];
    // Subtract c0 (= r[0..w0n-1])
    if (w0n > 0 && sz > 0) {
      sub(E_buf[kk], E_buf[kk], sz, r, w0n);
      sz = normalized_size(E_buf[kk], sz);
    }
    // Subtract c14 * k^14
    if (winfn > 0 && sz > 0) {
      uint64_t k14 = K14[kk];
      if (k14 == 1) {
        sub(E_buf[kk], E_buf[kk], sz, r + winf_off, winfn);
      } else {
        size_t tn = mul_const(tmp7, r + winf_off, winfn, k14, 0);
        sub(E_buf[kk], E_buf[kk], sz, tmp7, tn);
      }
      sz = normalized_size(E_buf[kk], sz);
    }
    E_sz[kk] = sz;
  }

  // ---- Step 3: Even 6x6 elimination tree ----
  // Level 1: A_k = E_k - k² * E1 for k=2..6
  //  A2..A6 → reuse E_buf slots (E2..E6 overwritten)
  //  c4 coeffs: 12, 72, 240, 600, 1260 (all mults of 12)
  // Level 2: B_k = A_k - ratio * A2 for k=3..6, ratios: 6, 20, 50, 105
  //  c6 coeffs: 360, 2880, 12600, 40320 (all mults of 360)
  // Level 3: C_k = B_k - ratio * B3 for k=4..6, ratios: 8, 35, 112
  //  c8 coeffs: 20160, 201600, 1088640 (all mults of 20160)
  // Level 4: D_k = C_k - ratio * C4 for k=5..6, ratios: 10, 54
  //  c10 coeffs: 1814400, 21772800 (12:1)
  // Level 5: F6 = D6 - 12 * D5 → c12 coef only
  // Divisors: c12 /239500800, c10 /1814400, c8 /20160, c6 /360, c4 /12

  // Level 1: A_k = E_k - k²*E1 (k=2..6)
  {
    // Save E1 for later use in c2 computation
    // E1 stays at tmp1 = E_buf[1]
    for (int kk = 2; kk <= 6; kk++) {
      uint64_t k_sq = (uint64_t)kk * kk;
      if (E_sz[1] > 0 && E_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, E_buf[1], E_sz[1], k_sq, 0);
        sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn);
        E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]);
      }
    }
  }
  // Now E_buf[2..6] contain A2..A6.

  // Level 2: B_k = A_k - ratio * A2 for k=3..6
  {
    static constexpr uint64_t B_ratio[7] = {0, 0, 0, 6, 20, 50, 105};
    for (int kk = 3; kk <= 6; kk++) {
      if (E_sz[2] > 0 && E_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, E_buf[2], E_sz[2], B_ratio[kk], 0);
        sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn);
        E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]);
      }
    }
  }
  // Now E_buf[3..6] contain B3..B6.

  // Level 3: C_k = B_k - ratio * B3 for k=4..6
  {
    static constexpr uint64_t C_ratio[7] = {0, 0, 0, 0, 8, 35, 112};
    for (int kk = 4; kk <= 6; kk++) {
      if (E_sz[3] > 0 && E_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, E_buf[3], E_sz[3], C_ratio[kk], 0);
        sub(E_buf[kk], E_buf[kk], E_sz[kk], tmp7, tn);
        E_sz[kk] = normalized_size(E_buf[kk], E_sz[kk]);
      }
    }
  }

  // Level 4: D_k = C_k - ratio * C4 for k=5..6
  {
    if (E_sz[4] > 0 && E_sz[5] > 0) {
      size_t tn = mul_const(tmp7, E_buf[4], E_sz[4], 10, 0);
      sub(E_buf[5], E_buf[5], E_sz[5], tmp7, tn);
      E_sz[5] = normalized_size(E_buf[5], E_sz[5]);
    }
    if (E_sz[4] > 0 && E_sz[6] > 0) {
      size_t tn = mul_const(tmp7, E_buf[4], E_sz[4], 54, 0);
      sub(E_buf[6], E_buf[6], E_sz[6], tmp7, tn);
      E_sz[6] = normalized_size(E_buf[6], E_sz[6]);
    }
  }

  // Level 5: F6 = D6 - 12 * D5
  {
    if (E_sz[5] > 0 && E_sz[6] > 0) {
      size_t tn = mul_const(tmp7, E_buf[5], E_sz[5], 12, 0);
      sub(E_buf[6], E_buf[6], E_sz[6], tmp7, tn);
      E_sz[6] = normalized_size(E_buf[6], E_sz[6]);
    }
  }

  // c12 = F6 / 239500800 = F6 >> 9 / 467775
  // 239500800 = 2^9 * 467775 (467775 = 3^5 * 5^2 * 7 * 11)
  size_t c12_n = E_sz[6];
  uint64_t* c12_ptr = E_buf[6];
  if (c12_n > 0) {
    for (size_t i = 0; i + 1 < c12_n; i++)
      c12_ptr[i] = (c12_ptr[i] >> 9) | (c12_ptr[i + 1] << 55);
    c12_ptr[c12_n - 1] >>= 9;
    c12_n = normalized_size(c12_ptr, c12_n);
    constexpr uint64_t D_467775 = 467775ULL;
    constexpr uint64_t INV_467775 = hensel_inverse_u64(D_467775);
    c12_n = divexact_by_odd(c12_ptr, c12_ptr, c12_n, D_467775, INV_467775);
  }

  // c10 = (D5 - 99792000*c12) / 1814400
  // 99792000 = 2^7 * 3^4 * 5^3 * 7 * 11 (= 55 * 1814400)
  size_t c10_n = E_sz[5];
  uint64_t* c10_ptr = E_buf[5];
  if (c12_n > 0 && c10_n > 0) {
    // 99792000 doesn't fit in uint64_t... wait, 99792000 < 2^27, fits.
    size_t tn = mul_const(tmp7, c12_ptr, c12_n, 99792000ULL, 0);
    sub(c10_ptr, c10_ptr, c10_n, tmp7, tn);
    c10_n = normalized_size(c10_ptr, c10_n);
  }
  if (c10_n > 0) {
    // /1814400 = 2^7 * 14175 (14175 = 3^4 * 5^2 * 7)
    for (size_t i = 0; i + 1 < c10_n; i++)
      c10_ptr[i] = (c10_ptr[i] >> 7) | (c10_ptr[i + 1] << 57);
    c10_ptr[c10_n - 1] >>= 7;
    c10_n = normalized_size(c10_ptr, c10_n);
    constexpr uint64_t D_14175 = 14175ULL;
    constexpr uint64_t INV_14175 = hensel_inverse_u64(D_14175);
    c10_n = divexact_by_odd(c10_ptr, c10_ptr, c10_n, D_14175, INV_14175);
  }

  // c8 = (C4 - 604800*c10 - 12640320*c12) / 20160
  size_t c8_n = E_sz[4];
  uint64_t* c8_ptr = E_buf[4];
  if (c10_n > 0 && c8_n > 0) {
    size_t tn = mul_const(tmp7, c10_ptr, c10_n, 604800ULL, 0);
    sub(c8_ptr, c8_ptr, c8_n, tmp7, tn);
    c8_n = normalized_size(c8_ptr, c8_n);
  }
  if (c12_n > 0 && c8_n > 0) {
    size_t tn = mul_const(tmp7, c12_ptr, c12_n, 12640320ULL, 0);
    sub(c8_ptr, c8_ptr, c8_n, tmp7, tn);
    c8_n = normalized_size(c8_ptr, c8_n);
  }
  if (c8_n > 0) {
    // /20160 = 2^6 * 315 (315 = 3^2 * 5 * 7)
    for (size_t i = 0; i + 1 < c8_n; i++)
      c8_ptr[i] = (c8_ptr[i] >> 6) | (c8_ptr[i + 1] << 58);
    c8_ptr[c8_n - 1] >>= 6;
    c8_n = normalized_size(c8_ptr, c8_n);
    constexpr uint64_t D_315 = 315ULL;
    constexpr uint64_t INV_315 = hensel_inverse_u64(D_315);
    c8_n = divexact_by_odd(c8_ptr, c8_ptr, c8_n, D_315, INV_315);
  }

  // c6 = (B3 - 5040*c8 - 52920*c10 - 506880*c12) / 360
  size_t c6_n = E_sz[3];
  uint64_t* c6_ptr = E_buf[3];
  if (c8_n > 0 && c6_n > 0) {
    size_t tn = mul_const(tmp7, c8_ptr, c8_n, 5040ULL, 0);
    sub(c6_ptr, c6_ptr, c6_n, tmp7, tn);
    c6_n = normalized_size(c6_ptr, c6_n);
  }
  if (c10_n > 0 && c6_n > 0) {
    size_t tn = mul_const(tmp7, c10_ptr, c10_n, 52920ULL, 0);
    sub(c6_ptr, c6_ptr, c6_n, tmp7, tn);
    c6_n = normalized_size(c6_ptr, c6_n);
  }
  if (c12_n > 0 && c6_n > 0) {
    size_t tn = mul_const(tmp7, c12_ptr, c12_n, 506880ULL, 0);
    sub(c6_ptr, c6_ptr, c6_n, tmp7, tn);
    c6_n = normalized_size(c6_ptr, c6_n);
  }
  if (c6_n > 0) {
    // /360 = 2^3 * 45 (45 = 3^2 * 5)
    c6_n = shift_right_3(c6_ptr, c6_ptr, c6_n);
    constexpr uint64_t D_45 = 45ULL;
    constexpr uint64_t INV_45 = hensel_inverse_u64(D_45);
    c6_n = divexact_by_odd(c6_ptr, c6_ptr, c6_n, D_45, INV_45);
  }

  // c4 = (A2 - 60*c6 - 252*c8 - 1020*c10 - 4092*c12) / 12
  size_t c4_n = E_sz[2];
  uint64_t* c4_ptr = E_buf[2];
  if (c6_n > 0 && c4_n > 0) {
    size_t tn = mul_const(tmp7, c6_ptr, c6_n, 60ULL, 0);
    sub(c4_ptr, c4_ptr, c4_n, tmp7, tn);
    c4_n = normalized_size(c4_ptr, c4_n);
  }
  if (c8_n > 0 && c4_n > 0) {
    size_t tn = mul_const(tmp7, c8_ptr, c8_n, 252ULL, 0);
    sub(c4_ptr, c4_ptr, c4_n, tmp7, tn);
    c4_n = normalized_size(c4_ptr, c4_n);
  }
  if (c10_n > 0 && c4_n > 0) {
    size_t tn = mul_const(tmp7, c10_ptr, c10_n, 1020ULL, 0);
    sub(c4_ptr, c4_ptr, c4_n, tmp7, tn);
    c4_n = normalized_size(c4_ptr, c4_n);
  }
  if (c12_n > 0 && c4_n > 0) {
    size_t tn = mul_const(tmp7, c12_ptr, c12_n, 4092ULL, 0);
    sub(c4_ptr, c4_ptr, c4_n, tmp7, tn);
    c4_n = normalized_size(c4_ptr, c4_n);
  }
  if (c4_n > 0) c4_n = divexact_by12(c4_ptr, c4_ptr, c4_n);

  // c2 = E1 - c4 - c6 - c8 - c10 - c12
  size_t c2_n = E_sz[1];
  uint64_t* c2_ptr = E_buf[1];
  if (c4_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c4_ptr, c4_n); c2_n = normalized_size(c2_ptr, c2_n); }
  if (c6_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c6_ptr, c6_n); c2_n = normalized_size(c2_ptr, c2_n); }
  if (c8_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c8_ptr, c8_n); c2_n = normalized_size(c2_ptr, c2_n); }
  if (c10_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c10_ptr, c10_n); c2_n = normalized_size(c2_ptr, c2_n); }
  if (c12_n > 0 && c2_n > 0) { sub(c2_ptr, c2_ptr, c2_n, c12_ptr, c12_n); c2_n = normalized_size(c2_ptr, c2_n); }

  // At this point, even coefficients c2..c12 are in E_buf[1..6]
  // c2 = E_buf[1], c4 = E_buf[2], c6 = E_buf[3], c8 = E_buf[4], c10 = E_buf[5], c12 = E_buf[6]
  // We now need to free tmp1..tmp6 for odd system, but they ARE E_buf[1..6]
  // Solution: copy c2..c12 to w_buf slots (w_buf[1..6]) which are no longer needed as e_k storage
  // Actually w_buf[1..7] still hold e_k. Let me reassign storage.

  // Preserve c2..c12 by moving them to separate buffers.
  // After this point, we need: e_k (for odd unused), o_k (k=1..7, in wm_buf), c2..c12.
  // Strategy: move c2..c12 into tmp1..tmp6 (already there) -- but we need tmp buffers for odd elim.
  // Let's move c2..c12 to w_buf[1..6] (overwriting e_k, which is no longer needed now that even is done).

  if (c2_n > 0) std::memcpy(w_buf[1], c2_ptr, c2_n * sizeof(uint64_t)); else w_buf[1] = c2_ptr;
  if (c4_n > 0) std::memcpy(w_buf[2], c4_ptr, c4_n * sizeof(uint64_t)); else w_buf[2] = c4_ptr;
  if (c6_n > 0) std::memcpy(w_buf[3], c6_ptr, c6_n * sizeof(uint64_t)); else w_buf[3] = c6_ptr;
  if (c8_n > 0) std::memcpy(w_buf[4], c8_ptr, c8_n * sizeof(uint64_t)); else w_buf[4] = c8_ptr;
  if (c10_n > 0) std::memcpy(w_buf[5], c10_ptr, c10_n * sizeof(uint64_t)); else w_buf[5] = c10_ptr;
  if (c12_n > 0) std::memcpy(w_buf[6], c12_ptr, c12_n * sizeof(uint64_t)); else w_buf[6] = c12_ptr;

  c2_ptr = w_buf[1];
  c4_ptr = w_buf[2];
  c6_ptr = w_buf[3];
  c8_ptr = w_buf[4];
  c10_ptr = w_buf[5];
  c12_ptr = w_buf[6];

  // ---- Step 4: Odd system ----
  // D_k = o_k - o1 for k=2..7 (stored in wm_buf[2..7])
  // o1 is in wm_buf[1], o_k in wm_buf[k]

  size_t D_sz[8] = {0}; // D_sz[2..7] used
  uint64_t* D_buf[8];
  // D_k uses tmp1..tmp6 (6 slots)
  D_buf[2] = tmp1; D_buf[3] = tmp2; D_buf[4] = tmp3;
  D_buf[5] = tmp4; D_buf[6] = tmp5; D_buf[7] = tmp6;

  for (int kk = 2; kk <= 7; kk++) {
    std::memcpy(D_buf[kk], wm_buf[kk], o_sz[kk] * sizeof(uint64_t));
    D_sz[kk] = o_sz[kk];
    if (o_sz[1] > 0 && D_sz[kk] > 0) {
      sub(D_buf[kk], D_buf[kk], D_sz[kk], wm_buf[1], o_sz[1]);
      D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]);
    }
  }

  // Level 1: F_k = 3*D_k - (k²-1)*D2 for k=3..7
  // D_k c3 coefficient = k²-1. After F_k = 3*D_k - (k²-1)*D2, c3 is eliminated and all coefs *3.
  {
    // Save D2 for F computation (D2 = D_buf[2])
    // F2 is not computed (D2 is the pivot)
    static constexpr uint64_t F_ratio[8] = {0, 0, 0, 8, 15, 24, 35, 48};
    for (int kk = 3; kk <= 7; kk++) {
      // 3*D_k
      if (D_sz[kk] > 0) {
        uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], 3);
        if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; }
      }
      // - (k²-1)*D2
      if (D_sz[2] > 0 && D_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, D_buf[2], D_sz[2], F_ratio[kk], 0);
        sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn);
        D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]);
      }
    }
  }
  // D_buf[3..7] now contain F3..F7. D_buf[2] still contains D2 (used later for c3).

  // Level 2: G_k eliminate c5 using F3 (F3 c5 = 120)
  // F_k c5: F3=120, F4=540, F5=1512, F6=3360, F7=6480
  // Using (a_k, b_k) scaling:
  //  G4 = 2*F4 - 9*F3  (GCD-reduced)
  //  G5 = 5*F5 - 63*F3
  //  G6 = 1*F6 - 28*F3
  //  G7 = 1*F7 - 54*F3
  {
    static constexpr uint64_t G_a[8] = {0, 0, 0, 0, 2, 5, 1, 1};
    static constexpr uint64_t G_b[8] = {0, 0, 0, 0, 9, 63, 28, 54};
    for (int kk = 4; kk <= 7; kk++) {
      // F_k *= G_a[kk]
      if (G_a[kk] != 1 && D_sz[kk] > 0) {
        uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], G_a[kk]);
        if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; }
      }
      // F_k -= G_b[kk] * F3
      if (D_sz[3] > 0 && D_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, D_buf[3], D_sz[3], G_b[kk], 0);
        sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn);
        D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]);
      }
    }
  }
  // D_buf[4..7] now contain G4..G7. D_buf[3] still contains F3 (used later for c5).

  // Level 3: H_k eliminate c7 using G4 (G4 c7 = 7560)
  // G_k c7: G4=7560, G5=120960, G6=90720, G7=259200
  //  H5 = 1*G5 - 16*G4 (120960 = 16 * 7560)
  //  H6 = 1*G6 - 12*G4 (90720 = 12 * 7560)
  //  H7 = 7*G7 - 240*G4 (GCD-reduced: G7 c7 = 259200, GCD(7560, 259200)=360+ actual GCD)
  //           Actually 259200/7560 = 34.28... let me verify
  //           7*259200 = 1814400, 240*7560 = 1814400 ✓
  {
    static constexpr uint64_t H_a[8] = {0, 0, 0, 0, 0, 1, 1, 7};
    static constexpr uint64_t H_b[8] = {0, 0, 0, 0, 0, 16, 12, 240};
    for (int kk = 5; kk <= 7; kk++) {
      if (H_a[kk] != 1 && D_sz[kk] > 0) {
        uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], H_a[kk]);
        if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; }
      }
      if (D_sz[4] > 0 && D_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, D_buf[4], D_sz[4], H_b[kk], 0);
        sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn);
        D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]);
      }
    }
  }
  // D_buf[5..7] now contain H5..H7. D_buf[4] still contains G4 (used later for c7).

  // Level 4: I_k eliminate c9 using H5 (H5 c9 = 1088640)
  //  I6 = 3*H6 - 5*H5
  //  I7 = 1*H7 - 55*H5
  {
    static constexpr uint64_t I_a[8] = {0, 0, 0, 0, 0, 0, 3, 1};
    static constexpr uint64_t I_b[8] = {0, 0, 0, 0, 0, 0, 5, 55};
    for (int kk = 6; kk <= 7; kk++) {
      if (I_a[kk] != 1 && D_sz[kk] > 0) {
        uint64_t ov = mul_1(D_buf[kk], D_buf[kk], D_sz[kk], I_a[kk]);
        if (ov) { D_buf[kk][D_sz[kk]] = ov; D_sz[kk]++; }
      }
      if (D_sz[5] > 0 && D_sz[kk] > 0) {
        size_t tn = mul_const(tmp7, D_buf[5], D_sz[5], I_b[kk], 0);
        sub(D_buf[kk], D_buf[kk], D_sz[kk], tmp7, tn);
        D_sz[kk] = normalized_size(D_buf[kk], D_sz[kk]);
      }
    }
  }

  // Level 5: J7 = I7 - 24*I6 → c13 only
  {
    if (D_sz[6] > 0 && D_sz[7] > 0) {
      size_t tn = mul_const(tmp7, D_buf[6], D_sz[6], 24, 0);
      sub(D_buf[7], D_buf[7], D_sz[7], tmp7, tn);
      D_sz[7] = normalized_size(D_buf[7], D_sz[7]);
    }
  }

  // c13 = J7 / 18681062400 = J7 >> 10 / 18243225
  // 18681062400 = 2^10 * 18243225 (18243225 = 3^6 * 5^2 * 7 * 11 * 13)
  size_t c13_n = D_sz[7];
  uint64_t* c13_ptr = D_buf[7];
  if (c13_n > 0) {
    for (size_t i = 0; i + 1 < c13_n; i++)
      c13_ptr[i] = (c13_ptr[i] >> 10) | (c13_ptr[i + 1] << 54);
    c13_ptr[c13_n - 1] >>= 10;
    c13_n = normalized_size(c13_ptr, c13_n);
    constexpr uint64_t D_18243225 = 18243225ULL;
    constexpr uint64_t INV_18243225 = hensel_inverse_u64(D_18243225);
    c13_n = divexact_by_odd(c13_ptr, c13_ptr, c13_n, D_18243225, INV_18243225);
  }

  // c11 = (I6 - 5448643200*c13) / 59875200
  // 5448643200 doesn't fit... wait, 5448643200 < 2^33, fits in uint64_t
  // 59875200 = 2^7 * 3^5 * 5^2 * 7 * 11
  size_t c11_n = D_sz[6];
  uint64_t* c11_ptr = D_buf[6];
  if (c13_n > 0 && c11_n > 0) {
    size_t tn = mul_const(tmp7, c13_ptr, c13_n, 5448643200ULL, 0);
    sub(c11_ptr, c11_ptr, c11_n, tmp7, tn);
    c11_n = normalized_size(c11_ptr, c11_n);
  }
  if (c11_n > 0) {
    // /59875200 = 2^7 * 467775 (467775 = 3^5 * 5^2 * 7 * 11)
    for (size_t i = 0; i + 1 < c11_n; i++)
      c11_ptr[i] = (c11_ptr[i] >> 7) | (c11_ptr[i + 1] << 57);
    c11_ptr[c11_n - 1] >>= 7;
    c11_n = normalized_size(c11_ptr, c11_n);
    constexpr uint64_t D_467775 = 467775ULL;
    constexpr uint64_t INV_467775 = hensel_inverse_u64(D_467775);
    c11_n = divexact_by_odd(c11_ptr, c11_ptr, c11_n, D_467775, INV_467775);
  }

  // c9 = (H5 - 59875200*c11 - 2179457280*c13) / 1088640
  // 1088640 = 2^7 * 3^5 * 5 * 7
  size_t c9_n = D_sz[5];
  uint64_t* c9_ptr = D_buf[5];
  if (c11_n > 0 && c9_n > 0) {
    size_t tn = mul_const(tmp7, c11_ptr, c11_n, 59875200ULL, 0);
    sub(c9_ptr, c9_ptr, c9_n, tmp7, tn);
    c9_n = normalized_size(c9_ptr, c9_n);
  }
  if (c13_n > 0 && c9_n > 0) {
    size_t tn = mul_const(tmp7, c13_ptr, c13_n, 2179457280ULL, 0);
    sub(c9_ptr, c9_ptr, c9_n, tmp7, tn);
    c9_n = normalized_size(c9_ptr, c9_n);
  }
  if (c9_n > 0) {
    // /1088640 = 2^7 * 8505 (8505 = 3^5 * 5 * 7)
    for (size_t i = 0; i + 1 < c9_n; i++)
      c9_ptr[i] = (c9_ptr[i] >> 7) | (c9_ptr[i + 1] << 57);
    c9_ptr[c9_n - 1] >>= 7;
    c9_n = normalized_size(c9_ptr, c9_n);
    constexpr uint64_t D_8505 = 8505ULL;
    constexpr uint64_t INV_8505 = hensel_inverse_u64(D_8505);
    c9_n = divexact_by_odd(c9_ptr, c9_ptr, c9_n, D_8505, INV_8505);
  }

  // c7 = (G4 - 226800*c9 - 4740120*c11 - 86486400*c13) / 7560
  // 7560 = 2^3 * 3^3 * 5 * 7
  size_t c7_n = D_sz[4];
  uint64_t* c7_ptr = D_buf[4];
  if (c9_n > 0 && c7_n > 0) {
    size_t tn = mul_const(tmp7, c9_ptr, c9_n, 226800ULL, 0);
    sub(c7_ptr, c7_ptr, c7_n, tmp7, tn);
    c7_n = normalized_size(c7_ptr, c7_n);
  }
  if (c11_n > 0 && c7_n > 0) {
    size_t tn = mul_const(tmp7, c11_ptr, c11_n, 4740120ULL, 0);
    sub(c7_ptr, c7_ptr, c7_n, tmp7, tn);
    c7_n = normalized_size(c7_ptr, c7_n);
  }
  if (c13_n > 0 && c7_n > 0) {
    size_t tn = mul_const(tmp7, c13_ptr, c13_n, 86486400ULL, 0);
    sub(c7_ptr, c7_ptr, c7_n, tmp7, tn);
    c7_n = normalized_size(c7_ptr, c7_n);
  }
  if (c7_n > 0) {
    // /7560 = 2^3 * 945 (945 = 3^3 * 5 * 7)
    c7_n = shift_right_3(c7_ptr, c7_ptr, c7_n);
    constexpr uint64_t D_945 = 945ULL;
    constexpr uint64_t INV_945 = hensel_inverse_u64(D_945);
    c7_n = divexact_by_odd(c7_ptr, c7_ptr, c7_n, D_945, INV_945);
  }

  // c5 = (F3 - 1680*c7 - 17640*c9 - 168960*c11 - 1561560*c13) / 120
  // 120 = 2^3 * 3 * 5
  size_t c5_n = D_sz[3];
  uint64_t* c5_ptr = D_buf[3];
  if (c7_n > 0 && c5_n > 0) {
    size_t tn = mul_const(tmp7, c7_ptr, c7_n, 1680ULL, 0);
    sub(c5_ptr, c5_ptr, c5_n, tmp7, tn);
    c5_n = normalized_size(c5_ptr, c5_n);
  }
  if (c9_n > 0 && c5_n > 0) {
    size_t tn = mul_const(tmp7, c9_ptr, c9_n, 17640ULL, 0);
    sub(c5_ptr, c5_ptr, c5_n, tmp7, tn);
    c5_n = normalized_size(c5_ptr, c5_n);
  }
  if (c11_n > 0 && c5_n > 0) {
    size_t tn = mul_const(tmp7, c11_ptr, c11_n, 168960ULL, 0);
    sub(c5_ptr, c5_ptr, c5_n, tmp7, tn);
    c5_n = normalized_size(c5_ptr, c5_n);
  }
  if (c13_n > 0 && c5_n > 0) {
    size_t tn = mul_const(tmp7, c13_ptr, c13_n, 1561560ULL, 0);
    sub(c5_ptr, c5_ptr, c5_n, tmp7, tn);
    c5_n = normalized_size(c5_ptr, c5_n);
  }
  if (c5_n > 0) c5_n = divexact_by120(c5_ptr, c5_ptr, c5_n);

  // c3 = (D2 - 15*c5 - 63*c7 - 255*c9 - 1023*c11 - 4095*c13) / 3
  size_t c3_n = D_sz[2];
  uint64_t* c3_ptr = D_buf[2];
  if (c5_n > 0 && c3_n > 0) {
    size_t tn = mul_const(tmp7, c5_ptr, c5_n, 15ULL, 0);
    sub(c3_ptr, c3_ptr, c3_n, tmp7, tn);
    c3_n = normalized_size(c3_ptr, c3_n);
  }
  if (c7_n > 0 && c3_n > 0) {
    size_t tn = mul_const(tmp7, c7_ptr, c7_n, 63ULL, 0);
    sub(c3_ptr, c3_ptr, c3_n, tmp7, tn);
    c3_n = normalized_size(c3_ptr, c3_n);
  }
  if (c9_n > 0 && c3_n > 0) {
    size_t tn = mul_const(tmp7, c9_ptr, c9_n, 255ULL, 0);
    sub(c3_ptr, c3_ptr, c3_n, tmp7, tn);
    c3_n = normalized_size(c3_ptr, c3_n);
  }
  if (c11_n > 0 && c3_n > 0) {
    size_t tn = mul_const(tmp7, c11_ptr, c11_n, 1023ULL, 0);
    sub(c3_ptr, c3_ptr, c3_n, tmp7, tn);
    c3_n = normalized_size(c3_ptr, c3_n);
  }
  if (c13_n > 0 && c3_n > 0) {
    size_t tn = mul_const(tmp7, c13_ptr, c13_n, 4095ULL, 0);
    sub(c3_ptr, c3_ptr, c3_n, tmp7, tn);
    c3_n = normalized_size(c3_ptr, c3_n);
  }
  if (c3_n > 0) c3_n = divexact_by3(c3_ptr, c3_ptr, c3_n);

  // c1 = o1 - c3 - c5 - c7 - c9 - c11 - c13
  // o1 in wm_buf[1]
  size_t c1_n = o_sz[1];
  uint64_t* c1_ptr = tmp8;
  if (o_sz[1] > 0) std::memcpy(c1_ptr, wm_buf[1], o_sz[1] * sizeof(uint64_t));
  if (c3_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c3_ptr, c3_n); c1_n = normalized_size(c1_ptr, c1_n); }
  if (c5_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c5_ptr, c5_n); c1_n = normalized_size(c1_ptr, c1_n); }
  if (c7_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c7_ptr, c7_n); c1_n = normalized_size(c1_ptr, c1_n); }
  if (c9_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c9_ptr, c9_n); c1_n = normalized_size(c1_ptr, c1_n); }
  if (c11_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c11_ptr, c11_n); c1_n = normalized_size(c1_ptr, c1_n); }
  if (c13_n > 0 && c1_n > 0) { sub(c1_ptr, c1_ptr, c1_n, c13_ptr, c13_n); c1_n = normalized_size(c1_ptr, c1_n); }

  SANGI_T8P_ACCUM(interp_ns);

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + ... + c13*B^(13k)
  // (c0 placed at r[0..], c14 placed at r[14k..])
  // ============================================================

  auto add_coef = [&](size_t cn, const uint64_t* cp, size_t offset) {
    if (cn > 0 && offset < rn) {
      size_t space = rn - offset;
      add(r + offset, r + offset, space, cp, std::min(cn, space));
    }
  };
  add_coef(c1_n, c1_ptr, k);
  add_coef(c2_n, c2_ptr, 2 * k);
  add_coef(c3_n, c3_ptr, 3 * k);
  add_coef(c4_n, c4_ptr, 4 * k);
  add_coef(c5_n, c5_ptr, 5 * k);
  add_coef(c6_n, c6_ptr, 6 * k);
  add_coef(c7_n, c7_ptr, 7 * k);
  add_coef(c8_n, c8_ptr, 8 * k);
  add_coef(c9_n, c9_ptr, 9 * k);
  add_coef(c10_n, c10_ptr, 10 * k);
  add_coef(c11_n, c11_ptr, 11 * k);
  add_coef(c12_n, c12_ptr, 12 * k);
  add_coef(c13_n, c13_ptr, 13 * k);

  SANGI_T8P_ACCUM(compose_ns);
}

// ============================================================================
// Schoenhage-Strassen FFT multiplication - forward declarations and threshold constants
// ============================================================================
// multiply -> mul_fft switching threshold (limb count)
// NTT-based FFT multiplication. Performs convolution over Z/(B^F+1)Z.
// The internal pointwise multiplication uses F ~ 2*sqrt(N) << N, so no re-entry guard is needed.
constexpr size_t FFT_THRESHOLD = 3000; // Fermat NTT (for fallback)

// Prime NTT (3 primes + CRT) switching threshold (MUL)
// 2026-03-11 sweep: faster than Toom-3 at n>=1400
constexpr size_t PRIME_NTT_THRESHOLD = 1400;

// Prime NTT threshold for SQR (unused, reference value)
// Benchmark (2026-04-11): TC3/NTT crossover ~1300 limbs
// SQR_PRIME_NTT_DIRECT_THRESHOLD (1250) is already set near this point
constexpr size_t SQR_PRIME_NTT_THRESHOLD = 1300;

// DOUBLE_FFT (double-precision FFT multiplication) switching threshold
//
// History:
//  YC-8e post-optimization benchmark (2026-03-14):
//   MUL 1100: FFT 124us vs Toom 134us vs NTT 180us -> FFT fastest
//   MUL 1200: FFT 139us vs Toom 217us vs NTT 186us -> FFT fastest
//   At the time it was enabled with the sweet spot at 1050-1250 limbs.
//
//  2026-04-27 re-measurement (after Toom-8 interp/eval -49.9%/-18.6%):
//   MUL 1100: old sangi/gmp 1.34x (Win) / 1.64x (Linux) →
//        with DOUBLE_FFT disabled: 1.16x (Win) / 1.12x (Linux)
//   MUL 1200: 1.21x→1.18x (Win), 1.43x→1.22x (Linux)
//   SQR 1100: via DOUBLE_FFT 107us vs Toom-8 92us (DOUBLE_FFT is 16% slower)
//   SQR 1200: DOUBLE_FFT 107us vs Toom-8 114us (DOUBLE_FFT 6% faster, washes out)
//
// Conclusion: Toom-8 beats DOUBLE_FFT in the practical range. Threshold effectively disabled.
// If DOUBLE_FFT is accelerated in the future, re-enabling remains an option (implementation kept in DoubleFft.hpp).
//
// PRIME_NTT_DIRECT_THRESHOLD is defined per-platform at the top of the file
constexpr size_t DOUBLE_FFT_THRESHOLD     = 999999; // disabled
constexpr size_t DOUBLE_FFT_MAX_THRESHOLD   = 999999;
constexpr size_t SQR_DOUBLE_FFT_THRESHOLD   = 999999; // disabled
constexpr size_t SQR_DOUBLE_FFT_MAX_THRESHOLD = 999999;

// Forward declarations of mul_fft / sqr_fft (implementations at the end of the file)
inline void mul_fft(uint64_t* rp,
          const uint64_t* ap, size_t an,
          const uint64_t* bp, size_t bn);
inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an);

// ============================================================================
// Generic multiplication dispatcher (handles unbalanced multiplication)
// ============================================================================

// Conservative scratch size for recursion: used for Toom-N rec_scratch computation.
// Unlike multiply_scratch_size, does not return 0 even when the top-level shape hits the FFT/NTT path,
// so that partial chunks (which can take any shape - Toom-N or unbalanced dispatch outside the FFT range)
// are accounted for, returning at least the Toom-N fallback scratch.
//
// Background: heap-buffer-overflow found by ASan (in mul_toomcook84(12499, 5000),
//    sub-mult vinf = mul(1558, 311) wrote 16K limbs beyond rec_scratch=0).
//    The issue was that Toom-N scratch calculation using multiply_scratch_size(k+4, k+4)
//    for rec_scratch relied on the assumption that sub-mult shape is preserved.
inline size_t multiply_rec_scratch_size(size_t an, size_t bn) {
  if (an < bn) std::swap(an, bn);
  if (bn == 0) return 0;
  if (bn < KARATSUBA_THRESHOLD) return 0;
  // Skip the early-zero return for FFT/NTT to ensure the Toom-N fallback is allocated.
  // Toom-8,4 / Toom-4,2 unbalanced
  if (bn >= TOOMCOOK84_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) {
    return mul_toomcook84_scratch_size(an, bn);
  }
  if (bn >= TOOMCOOK42_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) {
    return mul_toomcook42_scratch_size(an, bn);
  }
  // unbalanced chunking
  {
    size_t chunk_trigger;
    if (bn >= TOOMCOOK8_THRESHOLD)   chunk_trigger = (5 * bn + 2) / 3;
    else if (bn >= TOOMCOOK6_THRESHOLD) chunk_trigger = (9 * bn + 4) / 5;
    else if (bn >= TOOMCOOK4_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3;
    else if (bn >= TOOMCOOK3_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3;
    else                chunk_trigger = 2 * bn;
    if (an >= chunk_trigger) {
      return 2 * bn + multiply_rec_scratch_size(bn, bn);
    }
  }
  if (bn >= TOOMCOOK8_THRESHOLD) return mul_toomcook8_scratch_size(an);
  if (bn >= TOOMCOOK6_THRESHOLD) return mul_toomcook6_scratch_size(an);
  if (bn >= TOOMCOOK4_THRESHOLD) return mul_toomcook4_scratch_size(an);
  if (bn >= TOOMCOOK3_THRESHOLD) return mul_toomcook3_scratch_size(an);
  return mul_karatsuba_scratch_size(an);
}

// scratch size for multiply: takes both operand sizes into account
// FFT (mul_fft) allocates scratch internally, so
// it returns 0 for FFT-target sizes.
inline size_t multiply_scratch_size(size_t an, size_t bn) {
  if (an < bn) std::swap(an, bn);
  if (bn == 0) return 0;
  if (bn < KARATSUBA_THRESHOLD) return 0;
  if (bn >= PRIME_NTT_DIRECT_THRESHOLD && an < 2 * bn) return 0; // NTT allocates automatically
  if (bn >= DOUBLE_FFT_THRESHOLD && bn < DOUBLE_FFT_MAX_THRESHOLD && an < 2 * bn) return 0; // double_fft / NTT allocates automatically
  // Toom-8,4 (2:1 unbalanced specialization, deg-10 product) - takes priority over Toom-4,2
  if (bn >= TOOMCOOK84_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) {
    return mul_toomcook84_scratch_size(an, bn);
  }
  // Toom-4,2 (2:1 unbalanced specialization) - synced with the dispatch condition in multiply()
  if (bn >= TOOMCOOK42_THRESHOLD && an + 1 >= 2 * bn && 2 * (an + 1) <= 5 * bn) {
    return mul_toomcook42_scratch_size(an, bn);
  }
  // Unbalanced chunking: synced with multiply() dispatch (chunk_trigger).
  // Note: old code chunked only when `an >= 2*bn`, but multiply()
  //    chunks from 1.67-1.80*bn in the Toom-N size range, so
  //    multiply_scratch_size and the dispatch in multiply had diverged.
  {
    size_t chunk_trigger;
    if (bn >= TOOMCOOK8_THRESHOLD)   chunk_trigger = (5 * bn + 2) / 3;
    else if (bn >= TOOMCOOK6_THRESHOLD) chunk_trigger = (9 * bn + 4) / 5;
    else if (bn >= TOOMCOOK4_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3;
    else if (bn >= TOOMCOOK3_THRESHOLD) chunk_trigger = (5 * bn + 2) / 3;
    else                chunk_trigger = 2 * bn;
    if (an >= chunk_trigger) {
      // tmp product buffer (2*bn) + sub-mult scratch for each chunk
      // sub-mult shape (bn, residual) may fall into the Toom-N path outside the FFT range,
      // so use multiply_rec_scratch_size (avoid FFT short-circuit 0).
      return 2 * bn + multiply_rec_scratch_size(bn, bn);
    }
  }
  if (bn >= TOOMCOOK8_THRESHOLD) return mul_toomcook8_scratch_size(an);
  if (bn >= TOOMCOOK6_THRESHOLD) return mul_toomcook6_scratch_size(an);
  if (bn >= TOOMCOOK4_THRESHOLD) return mul_toomcook4_scratch_size(an);
  if (bn >= TOOMCOOK3_THRESHOLD) return mul_toomcook3_scratch_size(an);
  return mul_karatsuba_scratch_size(an);
}

// Forward declaration (mutually referenced with mul_unbalanced)
inline void multiply(uint64_t* r,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn,
           uint64_t* scratch);

// Unbalanced multiplication: split the larger operand into chunks and accumulate
// Precondition: an >= 2*bn, an >= bn >= KARATSUBA_THRESHOLD
inline void mul_unbalanced(uint64_t* r,
              const uint64_t* a, size_t an,
              const uint64_t* b, size_t bn,
              uint64_t* scratch) {
  size_t chunk = bn;
  size_t rn = an + bn;
  std::memset(r, 0, rn * sizeof(uint64_t));

  size_t prod_buf_size = chunk + bn; // at most 2*bn words
  uint64_t* tmp = scratch;
  uint64_t* sub_scratch = scratch + prod_buf_size;

  // First chunk: write directly to r
  size_t first_an = std::min(chunk, an);
  multiply(r, a, first_an, b, bn, sub_scratch);

  // Remaining chunks: multiply into tmp, then add to r
  for (size_t offset = chunk; offset < an; offset += chunk) {
    size_t this_an = std::min(chunk, an - offset);
    size_t this_prod_n = this_an + bn;
    std::memset(tmp, 0, this_prod_n * sizeof(uint64_t));
    multiply(tmp, a + offset, this_an, b, bn, sub_scratch);
    size_t actual_n = normalized_size(tmp, this_prod_n);
    if (actual_n > 0) {
      add(r + offset, r + offset, rn - offset, tmp, actual_n);
    }
  }
}

// Generic multiplication: chooses the algorithm automatically by size
// r[0..an+bn-1] = a[0..an-1] * b[0..bn-1]
// r must not overlap a or b
inline void multiply(uint64_t* r,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn,
           uint64_t* scratch) {
  if (an < bn) { std::swap(a, b); std::swap(an, bn); }

  if (bn == 0) {
    std::memset(r, 0, an * sizeof(uint64_t));
    return;
  }
  if (bn < KARATSUBA_THRESHOLD) {
    mul_basecase(r, a, an, b, bn);
    return;
  }
  if (bn >= PRIME_NTT_DIRECT_THRESHOLD && an < 2 * bn) {
    prime_ntt::mul_prime_ntt(r, a, an, b, bn);
    return;
  }
  if (bn >= DOUBLE_FFT_THRESHOLD && bn < DOUBLE_FFT_MAX_THRESHOLD && an < 2 * bn) {
    if (double_fft::mul_double_fft(r, a, an, b, bn))
      return;
    prime_ntt::mul_prime_ntt(r, a, an, b, bn);
    return;
  }
  // ===== Toom-8,4 (2:1 unbalanced specialization, deg-10 product) =====
  // At large sizes, 11xToom-3(n/4) is faster than 5xToom-4(n/2)
  if (bn >= TOOMCOOK84_THRESHOLD &&
    an + 1 >= 2 * bn &&
    2 * (an + 1) <= 5 * bn) {
    mul_toomcook84(r, a, an, b, bn, scratch);
    return;
  }
  // ===== Toom-4,2 (2:1 unbalanced specialization) =====
  // Applicability: 2*bn-1 <= an <= 5/2*bn-1 (an/bn is 2.0-2.5, b1 <= n_a guaranteed)
  // n_a = ceil(an/4), b1 = bn - n_a. If b1 exceeds n_a the buffer overflows.
  // effect: mul(2049,1025) replaces 2xToom-8(1025) with 5xToom-4(513) (~50% reduction expected)
  if (bn >= TOOMCOOK42_THRESHOLD &&
    an + 1 >= 2 * bn &&    // an >= 2*bn - 1 (b1 ≤ n_a guarantee)
    2 * (an + 1) <= 5 * bn) { // an <= 5*bn/2 - 1 (a3 < n_a guarantee)
    mul_toomcook42(r, a, an, b, bn, scratch);
    return;
  }
  // ===== Extended trigger for unbalanced chunking =====
  // Previously only `an >= 2*bn` was chunked, but Toom-k (k=3,4,6,8)
  // splits b into k parts and degrades where parts below 1/k become zero-padded.
  // Concrete example: mul(2049,1025) under Toom-8 8-way-splits b (257 limbs each), but
  // bn=1025 fills only 4 parts; the remaining 4 parts are zero -> about half of
  // the 15 recursive multiplications are wasted on zeros. Splitting into 2 x Toom-8(1025,1025)
  // via `mul_unbalanced` is faster.
  //
  // Heuristics:
  //  Toom-8: chunking is advantageous when an/bn > 5/3 (~1.67)
  //  Toom-6: an/bn > 9/5 (=1.80) - threshold raised because degradation is mild
  //  Toom-4/3: an/bn > 5/3 - degradation is severe for Toom-4, lowering the threshold helps
  {
    size_t chunk_trigger;
    if (bn >= TOOMCOOK8_THRESHOLD) {
      // Toom-8: 3*an >= 5*bn (an >= 1.67*bn)
      chunk_trigger = (5 * bn + 2) / 3;
    } else if (bn >= TOOMCOOK6_THRESHOLD) {
      // Toom-6: an >= 1.8*bn (chunking cost higher for smaller bn)
      chunk_trigger = (9 * bn + 4) / 5;
    } else if (bn >= TOOMCOOK4_THRESHOLD) {
      // Toom-4: an >= 1.67*bn (with 4-way split of b, bn < 3/4*an degrades)
      chunk_trigger = (5 * bn + 2) / 3;
    } else if (bn >= TOOMCOOK3_THRESHOLD) {
      // Toom-3: an >= 1.67*bn (same)
      chunk_trigger = (5 * bn + 2) / 3;
    } else {
      // Karatsuba and below: as before, an >= 2*bn
      chunk_trigger = 2 * bn;
    }
    if (an >= chunk_trigger) {
      mul_unbalanced(r, a, an, b, bn, scratch);
      return;
    }
  }
  if (bn >= TOOMCOOK8_THRESHOLD) {
    mul_toomcook8(r, a, an, b, bn, scratch);
    return;
  }
  if (bn >= TOOMCOOK6_THRESHOLD) {
    mul_toomcook6(r, a, an, b, bn, scratch);
    return;
  }
  if (bn >= TOOMCOOK4_THRESHOLD) {
    mul_toomcook4(r, a, an, b, bn, scratch);
    return;
  }
  if (bn >= TOOMCOOK3_THRESHOLD) {
    mul_toomcook3(r, a, an, b, bn, scratch);
    return;
  }
  mul_karatsuba(r, a, an, b, bn, scratch);
}

// ================================================================
// mulhigh_n implementation
// ================================================================

// mulhigh_n threshold
// BASECASE: below this, use mulhigh_basecase (~50% savings via column skip)
// KARATSUBA_LIMIT: above this, use full multiply + top extraction (Karatsuba recursion is inefficient in NTT range)
// Measured by bench_mulhigh.cpp:
//  basecase: 0.74x at n=52, 0.97x at n=128
//  karatsuba: 0.80-1.00x at n=128..400, slower at n>=1400 (NTT range)
constexpr size_t MULHIGH_BASECASE_THRESHOLD = 128;
constexpr size_t MULHIGH_KARATSUBA_LIMIT = 2000;

inline size_t mulhigh_n_scratch_size(size_t n) {
  if (n < MULHIGH_BASECASE_THRESHOLD) return 0;
  if (n >= MULHIGH_KARATSUBA_LIMIT) {
    // full multiply + top extraction
    return 2 * n + multiply_scratch_size(n, n);
  }
  // Karatsuba split: full multiply of a1*b1 + recursive mulhigh for cross
  size_t h = n / 2;
  size_t hn = n - h;
  // Phase 1: multiply(a1,b1) → 2*hn limbs + multiply scratch
  size_t mul_need = 2 * hn + multiply_scratch_size(hn, hn);
  // Phase 2 (scratch reuse): cross result h limbs + recursive mulhigh(h)
  size_t cross_need = h + mulhigh_n_scratch_size(h);
  return std::max(mul_need, cross_need);
}

inline void mulhigh_n(uint64_t* rp,
            const uint64_t* ap, const uint64_t* bp,
            size_t n, uint64_t* scratch) {
  if (n == 0) return;

  // Small size: mulhigh_basecase skips low columns, saving ~50%
  if (n < MULHIGH_BASECASE_THRESHOLD) {
    mulhigh_basecase(rp, ap, n, bp, n, n);
    return;
  }

  // Large size (NTT/Toom range): full multiply + top extraction
  // Karatsuba recursion at fan-out 2^depth x basecase(n/2^depth) is actually slower
  if (n >= MULHIGH_KARATSUBA_LIMIT) {
    uint64_t* prod = scratch;
    uint64_t* mul_work = scratch + 2 * n;
    multiply(prod, ap, n, bp, n, mul_work);
    std::memcpy(rp, prod + n, n * sizeof(uint64_t));
    return;
  }

  // ================================================================
  // Karatsuba-based mulhigh (128 <= n < 500)
  //
  // Split: a = a1*B^h + a0, b = b1*B^h + b0 (h = n/2)
  // a*b = a1*b1*B^(2h) + (a1*b0 + a0*b1)*B^h + a0*b0
  //
  // Contributions to the top n limbs:
  //  a1*b1: all 2*hn limbs (positions 2h..2n-1)
  //  a1*b0, a0*b1: only top h limbs (positions n..n+h-1)
  //  a0*b0: carry only -> skip (error <= 1)
  //
  // Cost: M(n/2) + 2*T(n/2) ~ 0.67*M(n) (one level)
  // Error: <= 3 (a0*b0 carry + 2 cross approximations) - absorbed by mu_div_qr correction loop
  // ================================================================

  size_t h = n / 2;
  size_t hn = n - h; // h (even n) or h+1 (odd n)

  // --- Step 1: compute a1*b1 exactly ---
  uint64_t* prod = scratch;
  uint64_t* mul_work = scratch + 2 * hn;
  multiply(prod, ap + h, hn, bp + h, hn, mul_work);

  // prod is 2*hn limbs. Corresponds to positions 2h..2n-1 of the full product.
  //  even n: 2h=n -> prod[0..n-1] = rp[0..n-1]
  //  odd n: 2h=n-1 -> prod[0] is outside the output, prod[1..n] = rp[0..n-1]
  if (hn == h) {
    std::memcpy(rp, prod, n * sizeof(uint64_t));
  } else {
    std::memcpy(rp, prod + 1, n * sizeof(uint64_t));
  }

  // --- Step 2: cross terms (top h limbs only) ---
  uint64_t* cross = scratch;
  uint64_t* cross_work = scratch + h;

  // Cross 1: top h limbs of a1*b0
  // For odd n, (h+1)xh -> approximate with the top h limbs of a1 (a1[0] contributes only a carry <= 1)
  mulhigh_n(cross, ap + h + (hn - h), bp, h, cross_work);
  uint64_t cy = add(rp, rp, h, cross, h);

  // Cross 2: top h limbs of a0*b1 (symmetric)
  mulhigh_n(cross, ap, bp + h + (hn - h), h, cross_work);
  cy += add(rp, rp, h, cross, h);

  if (cy) {
    add_1(rp + h, hn, cy);
  }
}

// Forward declaration: square / square_scratch_size (definitions near end of this file)
inline size_t square_scratch_size(size_t n);
inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch);

// ================================================================
// multiply_high / square_high: top-only product with keep parameter
//  - 2026-04-27: foundational API (full mul + extract)
//  - 2026-04-28: Karatsuba-high recursion (mid range 64 <= bn < 1400)
// ================================================================
//
// Generalizes the existing mulhigh_n (balanced nxn -> top n limbs):
// - arbitrary (an, bn) shape
// - arbitrary keep <= an+bn (returns the top keep limbs)
// - used in x^(n-1) computation of mpn_pow_for_root to compute only the required top limbs
//
// Error: rp[0] (lowest limb) has +/-C error (C <= 4). Upper limbs are exact (carries propagated).
// Use: nthRoot newton's divide_q_approx references only the top d2n limbs of the divisor, so
//    this is faster than full multiply + extract. In extreme non-balanced cases with drop >> keep
//    it is particularly effective (n=3 final square: drop ~ 2*xn-keep, h ~ xn-keep/2 -> done with only
//    a1^2 and cross terms, skipping all of a0^2).
//
// Strategy:
//  keep ≥ an + bn:    full multiply + zero pad
//  bn < BASECASE (= 64): schoolbook column-skip (reuse mulhigh_basecase)
//  bn >= BASECASE and h>0: Karatsuba-high recursion (drop a0*b0)
//  Otherwise / large NTT range: full multiply + extract
//
// Karatsuba-high split:
//  drop = an + bn - keep, h = drop / 2 (requires h < bn)
//  a = a1·B^h + a0, b = b1·B^h + b0
//  a*b = a1*b1·B^(2h) + (a1*b0 + a0*b1)·B^h + a0*b0
//  a0*b0 (size <= 2h <= drop) cannot reach the keep region -> drop (error <= 1)
//  Compute only the required portions of a1*b1, a1*b0, a0*b1 recursively via mulhigh

constexpr size_t MULHIGH_KEEP_BASECASE_THRESHOLD = 64;   // column-skip schoolbook upper bound
constexpr size_t MULHIGH_KEEP_KARATSUBA_LIMIT  = 1400;  // above this, use full mul (NTT range)

inline size_t multiply_high_scratch_size(size_t an, size_t bn, size_t keep) {
  if (an < bn) std::swap(an, bn); // an >= bn
  size_t total = an + bn;
  if (keep >= total) return multiply_scratch_size(an, bn);
  if (keep == 0 || bn == 0) return 0;

  size_t drop = total - keep;
  size_t h = drop / 2;

  // Karatsuba-high recursion guard:
  //  In shallow truncation (drop < an), recursion overhead exceeds the savings.
  //  = 2*M(a1n) + cross_recursion ~ 3*M(a1n); higher than M(an, bn) when a1n > an/2.
  //  -> apply recursion as deep truncation only when drop >= an.
  if (bn < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= bn ||
    an >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < an) {
    if (an < MULHIGH_KEEP_BASECASE_THRESHOLD) return 0;
    return total + multiply_scratch_size(an, bn);
  }

  // Karatsuba-high recursion (deep truncation)
  size_t a1n = an - h;
  size_t b1n = bn - h;
  size_t a1b0_keep = (an + h > drop) ? (an + h - drop) : 0;
  size_t a0b1_keep = (bn + h > drop) ? (bn + h - drop) : 0;

  size_t s1 = multiply_high_scratch_size(a1n, b1n, keep);
  size_t s2 = (a1b0_keep > 0) ? (a1b0_keep + multiply_high_scratch_size(a1n, h, a1b0_keep)) : 0;
  size_t s3 = (a0b1_keep > 0) ? (a0b1_keep + multiply_high_scratch_size(h, b1n, a0b1_keep)) : 0;
  return std::max({s1, s2, s3});
}

// rp[0..keep-1] = top keep limbs of a*b
// keep <= an + bn required
inline void multiply_high(uint64_t* rp,
              const uint64_t* ap, size_t an,
              const uint64_t* bp, size_t bn,
              size_t keep,
              uint64_t* scratch) {
  if (an < bn) { std::swap(ap, bp); std::swap(an, bn); }
  if (an == 0 || bn == 0 || keep == 0) {
    if (keep > 0) std::memset(rp, 0, keep * sizeof(uint64_t));
    return;
  }
  size_t total = an + bn;
  if (keep >= total) {
    multiply(rp, ap, an, bp, bn, scratch);
    if (keep > total) std::memset(rp + total, 0, (keep - total) * sizeof(uint64_t));
    return;
  }

  size_t drop = total - keep;
  size_t h = drop / 2;

  // Non-recursive path: includes shallow truncation (drop < an) - region where recursion is unfavorable
  if (bn < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= bn ||
    an >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < an) {
    if (an < MULHIGH_KEEP_BASECASE_THRESHOLD) {
      mulhigh_basecase(rp, ap, an, bp, bn, keep);
      return;
    }
    // small bn / invalid h / NTT range / shallow truncation -> full mul + extract
    uint64_t* prod = scratch;
    uint64_t* mul_sc = scratch + total;
    multiply(prod, ap, an, bp, bn, mul_sc);
    std::memcpy(rp, prod + (total - keep), keep * sizeof(uint64_t));
    return;
  }

  // ---- Karatsuba-high recursion (deep truncation: drop ≥ an) ----
  // a0[0..h-1], a1[0..a1n-1] = ap+h..ap+an-1
  // b0[0..h-1], b1[0..b1n-1] = bp+h..bp+bn-1
  const uint64_t* a0 = ap;
  const uint64_t* a1 = ap + h;
  size_t a1n = an - h;
  const uint64_t* b0 = bp;
  const uint64_t* b1 = bp + h;
  size_t b1n = bn - h;

  // Output position relationships:
  //  a1*b1 (size a1n+b1n = total-2h >= keep) sits at abs pos [2h, total-1].
  //   The portion entering the keep region [drop, total-1] is the top keep limbs.
  //  a1*b0 (size an) at abs pos [h, h+an-1]. Top (an+h-drop) limbs enter the keep region.
  //  a0*b1 (size bn) at abs pos [h, h+bn-1]. Top (bn+h-drop) limbs enter the keep region.
  //  a0*b0 (size 2h <= drop) lies entirely in the drop region -> ignore (error <= 1 carry).
  size_t a1b0_keep = (an + h > drop) ? (an + h - drop) : 0;
  size_t a0b1_keep = (bn + h > drop) ? (bn + h - drop) : 0;

  // Step 1: rp = top keep of a1*b1 (recursive multiply_high)
  multiply_high(rp, a1, a1n, b1, b1n, keep, scratch);

  // Step 2: add the top a1b0_keep limbs of a1*b0 to rp[0..a1b0_keep-1]
  if (a1b0_keep > 0) {
    uint64_t* cross = scratch;
    uint64_t* cross_sc = scratch + a1b0_keep;
    multiply_high(cross, a1, a1n, b0, h, a1b0_keep, cross_sc);
    uint64_t cy = add(rp, rp, a1b0_keep, cross, a1b0_keep);
    if (cy && a1b0_keep < keep) add_1(rp + a1b0_keep, keep - a1b0_keep, cy);
  }

  // Step 3: a0*b1 -> same as above
  if (a0b1_keep > 0) {
    uint64_t* cross = scratch;
    uint64_t* cross_sc = scratch + a0b1_keep;
    multiply_high(cross, a0, h, b1, b1n, a0b1_keep, cross_sc);
    uint64_t cy = add(rp, rp, a0b1_keep, cross, a0b1_keep);
    if (cy && a0b1_keep < keep) add_1(rp + a0b1_keep, keep - a0b1_keep, cy);
  }
}

// rp[0..keep-1] = top keep limbs of a²
// keep <= 2*n required
inline size_t square_high_scratch_size(size_t n, size_t keep) {
  size_t total = 2 * n;
  if (keep >= total) return square_scratch_size(n);
  if (keep == 0 || n == 0) return 0;

  size_t drop = total - keep;
  size_t h = drop / 2;

  // shallow truncation guard: recursion is unfavorable when drop < n (no better than full sq + extract)
  if (n < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= n ||
    n >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < n) {
    if (n < MULHIGH_KEEP_BASECASE_THRESHOLD) return 0;
    return total + square_scratch_size(n);
  }

  size_t a1n = n - h;
  size_t visible_len = (n + h > drop) ? (n + h - drop + 1) : 0;

  size_t s1 = square_high_scratch_size(a1n, keep);
  size_t s2 = (visible_len > 0) ? (visible_len + multiply_high_scratch_size(a1n, h, visible_len)) : 0;
  return std::max(s1, s2);
}

inline void square_high(uint64_t* rp,
             const uint64_t* ap, size_t n,
             size_t keep,
             uint64_t* scratch) {
  if (n == 0 || keep == 0) {
    if (keep > 0) std::memset(rp, 0, keep * sizeof(uint64_t));
    return;
  }
  size_t total = 2 * n;
  if (keep >= total) {
    square(rp, ap, n, scratch);
    if (keep > total) std::memset(rp + total, 0, (keep - total) * sizeof(uint64_t));
    return;
  }

  size_t drop = total - keep;
  size_t h = drop / 2;

  // Non-recursive path: includes shallow truncation (drop < n) - region where recursion is unfavorable
  if (n < MULHIGH_KEEP_BASECASE_THRESHOLD || h == 0 || h >= n ||
    n >= MULHIGH_KEEP_KARATSUBA_LIMIT || drop < n) {
    if (n < MULHIGH_KEEP_BASECASE_THRESHOLD) {
      // Column-skip basecase exploiting symmetry is not implemented; reuse mulhigh_basecase
      mulhigh_basecase(rp, ap, n, ap, n, keep);
      return;
    }
    uint64_t* prod = scratch;
    uint64_t* sq_sc = scratch + total;
    square(prod, ap, n, sq_sc);
    std::memcpy(rp, prod + (total - keep), keep * sizeof(uint64_t));
    return;
  }

  // ---- Karatsuba-high square (deep truncation: drop ≥ n) ----
  // a = a1·B^h + a0
  // a² = a1²·B^(2h) + 2·a1·a0·B^h + a0²
  //  a0^2 (size 2h <= drop) - completely dropped (carry +/-1 error)
  //  2*a1*a0 - visible_len = (h+n-drop+1) limbs in the keep region (1 limb is the <<1 carry-out)
  //  a1^2 (size 2*a1n) - write the top keep limbs to rp (recursive square_high)
  //
  // Processing of 2*a1*a0:
  //  Take the top (visible_len) limbs of a1*a0 (size n); after <<1, the new cross[0]
  //  corresponds to abs position drop-1 (one limb below keep).
  //  - cross[1..visible_len-1] corresponds to abs [drop..drop+visible_len-2] (= rp[0..visible_len-2])
  //  - lsh_carry corresponds to abs drop+visible_len-1 (= rp[visible_len-1])
  //  So drop cross[0], add cross + 1 to rp[0..visible_len-2],
  //  and add lsh_carry to rp[visible_len-1].
  const uint64_t* a0 = ap;
  const uint64_t* a1 = ap + h;
  size_t a1n = n - h;

  // Step 1: rp = top keep of a1²
  square_high(rp, a1, a1n, keep, scratch);

  // Step 2: 2·a1·a0
  if (n + h > drop) { // visible_len > 0
    size_t visible_len = n + h - drop + 1;
    uint64_t* cross = scratch;
    uint64_t* cross_sc = scratch + visible_len;
    // Take the top `visible_len` limbs of a1*a0 (size n)
    multiply_high(cross, a1, a1n, a0, h, visible_len, cross_sc);

    // 2 * cross via lshift by 1 bit
    // cross[0] corresponds to abs (drop-1) (below keep) - drop it
    // cross[1..visible_len-1] corresponds to abs [drop..drop+visible_len-2]
    // lsh_carry corresponds to abs (drop+visible_len-1)
    uint64_t lsh_carry = lshift(cross, cross, visible_len, 1);

    // Add (visible_len - 1) limbs from cross + 1 to rp[0..visible_len-2]
    size_t useful = visible_len - 1; // effective limbs of cross + 1
    size_t add_n = (useful <= keep) ? useful : keep;
    uint64_t cy = (add_n > 0) ? add(rp, rp, add_n, cross + 1, add_n) : 0;

    // Add lsh_carry to rp[useful] (= rp[visible_len-1])
    if (visible_len - 1 < keep) {
      uint64_t total_cy = cy + lsh_carry;
      if (total_cy) add_1(rp + (visible_len - 1), keep - (visible_len - 1), total_cy);
    }
  }
}

// ============================================================================
// Squaring (dedicated square that exploits a*a symmetry)
// ============================================================================

// sqr_basecase: r[0..2n-1] = a[0..n-1]²
// Symmetry use: off-diagonal performs n(n-1)/2 multiplications + doubling, diagonal performs n multiplications
// No scratch needed. r must not overlap a.
inline void sqr_basecase(uint64_t* r, const uint64_t* a, size_t n) {
  if (n == 0) return;
#if defined(_MSC_VER) && defined(_M_X64)
  // n=1: single 128-bit multiplication (avoids ASM push/pop)
  if (n == 1) {
    r[0] = _umul128(a[0], a[0], &r[1]);
    return;
  }
#endif
#ifdef SANGI_INT_HAS_ASM
  if (detail::has_bmi2_adx()) {
    mpn_sqr_basecase_mulx(r, a, n);
    return;
  }
#endif
  std::memset(r, 0, 2 * n * sizeof(uint64_t));

  // Step 1: off-diagonal — Σ_{i<j} a_i * a_j * B^(i+j)
  for (size_t i = 0; i + 1 < n; i++) {
    r[n + i] = addmul_1(r + 2 * i + 1, a + i + 1, n - i - 1, a[i]);
  }

  // Step 2: double the whole result (left-shift by 1 bit)
  uint64_t overflow = lshift(r, r, 2 * n, 1);
  (void)overflow; // the top must always be 0 (2*off_diag < 2*a_max^2*n^2 < B^(2n))

  // Step 3: diagonal - add a_i^2 to r[2i], r[2i+1]
#if defined(_MSC_VER) && defined(_M_X64)
  // 2x unroll: issue two independent MULs in advance and fold via ADC chain
  unsigned char carry = 0;
  size_t i = 0;
  for (; i + 2 <= n; i += 2) {
    uint64_t hi0, hi1;
    uint64_t lo0 = _umul128(a[i], a[i], &hi0);
    uint64_t lo1 = _umul128(a[i + 1], a[i + 1], &hi1);
    carry = _addcarry_u64(carry, r[2 * i], lo0, &r[2 * i]);
    carry = _addcarry_u64(carry, r[2 * i + 1], hi0, &r[2 * i + 1]);
    carry = _addcarry_u64(carry, r[2 * i + 2], lo1, &r[2 * i + 2]);
    carry = _addcarry_u64(carry, r[2 * i + 3], hi1, &r[2 * i + 3]);
  }
  for (; i < n; i++) {
    uint64_t hi;
    uint64_t lo = _umul128(a[i], a[i], &hi);
    carry = _addcarry_u64(carry, r[2 * i], lo, &r[2 * i]);
    carry = _addcarry_u64(carry, r[2 * i + 1], hi, &r[2 * i + 1]);
  }
#else
  uint64_t carry = 0;
  for (size_t i = 0; i < n; i++) {
    UInt128 sq = UInt128::multiply(a[i], a[i]);
    uint64_t lo = sq.low;
    uint64_t hi = sq.high;

    // r[2i] += lo + carry
    uint64_t s = r[2 * i] + carry;
    uint64_t c1 = (s < r[2 * i]) ? 1ULL : 0ULL;
    uint64_t t = s + lo;
    c1 += (t < s) ? 1ULL : 0ULL;
    r[2 * i] = t;

    // r[2i+1] += hi + c1
    s = r[2 * i + 1] + hi;
    uint64_t c2 = (s < r[2 * i + 1]) ? 1ULL : 0ULL;
    t = s + c1;
    c2 += (t < s) ? 1ULL : 0ULL;
    r[2 * i + 1] = t;

    carry = c2;
  }
#endif
}

// Column-Comba squaring: r[0..2n-1] = a[0..n-1]²
// Accumulate all partial products per column. No doubling pass required.
// Holds a 3-word accumulator (c2:c1:c0) in registers.
// No scratch needed. r must not overlap a.
inline void sqr_comba(uint64_t* r, const uint64_t* a, size_t n) {
  if (n == 0) return;
#if defined(_MSC_VER) && defined(_M_X64)
  uint64_t c0 = 0, c1 = 0, c2 = 0;
  for (size_t col = 0; col < 2 * n - 1; ++col) {
    size_t i_min = (col >= n) ? col - n + 1 : 0;
    size_t i_max = col / 2; // i <= i_max for off-diagonal (i < j)

    for (size_t i = i_min; i <= i_max; ++i) {
      size_t j = col - i;
      if (j >= n) continue;
      uint64_t hi, lo;
      lo = _umul128(a[i], a[j], &hi);
      if (i < j) {
        // Off-diagonal: add 2*(hi:lo)
        unsigned char top = static_cast<unsigned char>(hi >> 63);
        hi = (hi << 1) | (lo >> 63);
        lo <<= 1;
        unsigned char carry;
        carry = _addcarry_u64(0, c0, lo, &c0);
        carry = _addcarry_u64(carry, c1, hi, &c1);
        _addcarry_u64(carry, c2, top, &c2);
      } else {
        // Diagonal: add (hi:lo)
        unsigned char carry;
        carry = _addcarry_u64(0, c0, lo, &c0);
        carry = _addcarry_u64(carry, c1, hi, &c1);
        _addcarry_u64(carry, c2, 0, &c2);
      }
    }
    r[col] = c0;
    c0 = c1;
    c1 = c2;
    c2 = 0;
  }
  r[2 * n - 1] = c0;
#else
  // Fallback: use the standard sqr_basecase
  sqr_basecase(r, a, n);
#endif
}

// sqr_karatsuba scratch size
inline size_t sqr_karatsuba_scratch_size(size_t n) {
  return (n < 16) ? 128 : 8 * n + 64;
}

// sqr_karatsuba: r[0..2n-1] = a[0..n-1]²
// a² = a0² + ((a0+a1)² - a0² - a1²)·B^half + a1²·B^(2·half)
// 3 recursive squarings
inline void sqr_karatsuba(uint64_t* r,
              const uint64_t* a, size_t n,
              uint64_t* scratch) {
  if (n < SQR_KARATSUBA_THRESHOLD) {
    sqr_basecase(r, a, n);
    return;
  }

  size_t half = (n + 1) / 2;

  const uint64_t* a0 = a;
  size_t a0n = std::min(half, n);
  const uint64_t* a1 = a + half;
  size_t a1n = (n > half) ? n - half : 0;

  a0n = normalized_size(a0, a0n);
  a1n = normalized_size(a1, a1n);

  size_t rn = 2 * n;
  std::memset(r, 0, rn * sizeof(uint64_t));

  uint64_t* s = scratch;          // a0 + a1: half+1 limbs
  uint64_t* middle = scratch + (half + 1); // result of (a0+a1)^2: 2*(half+1)+2 limbs
  size_t middle_max = 2 * (half + 2);
  uint64_t* rec_scratch = scratch + (half + 1) + middle_max;

  // r[0..] = a0²
  if (a0n > 0)
    sqr_karatsuba(r, a0, a0n, rec_scratch);

  // r[2*half..] = a1²
  if (a1n > 0)
    sqr_karatsuba(r + 2 * half, a1, a1n, rec_scratch);

  // s = a0 + a1
  size_t sn;
  std::memset(s, 0, (half + 1) * sizeof(uint64_t));
  if (a0n >= a1n) {
    if (a1n > 0) {
      uint64_t carry = add(s, a0, a0n, a1, a1n);
      sn = a0n;
      if (carry) { s[sn] = carry; sn++; }
    } else {
      std::memcpy(s, a0, a0n * sizeof(uint64_t));
      sn = a0n;
    }
  } else {
    uint64_t carry = add(s, a1, a1n, a0, a0n);
    sn = a1n;
    if (carry) { s[sn] = carry; sn++; }
  }

  // middle = (a0+a1)²
  std::memset(middle, 0, middle_max * sizeof(uint64_t));
  if (sn > 0)
    sqr_karatsuba(middle, s, sn, rec_scratch);
  size_t mn = normalized_size(middle, 2 * sn);

  // middle -= a0² (= r[0..2*a0n-1])
  {
    size_t t0n = normalized_size(r, a0n + a0n);
    if (t0n > 0 && mn > 0) {
      sub(middle, middle, mn, r, t0n);
      mn = normalized_size(middle, mn);
    }
  }
  // middle -= a1² (= r[2*half..])
  {
    size_t t2n = normalized_size(r + 2 * half, a1n + a1n);
    if (t2n > 0 && mn > 0) {
      sub(middle, middle, mn, r + 2 * half, t2n);
      mn = normalized_size(middle, mn);
    }
  }

  // r[half..] += middle
  if (mn > 0) {
    uint64_t carry = add(r + half, r + half, rn - half, middle, mn);
    (void)carry;
  }
}

// Toom-Cook-3 threshold for squaring (higher than the 80 for multiplication)
// sqr_basecase's symmetry use makes the advantageous region of sqr_karatsuba wider
// Benchmark (2026-04-11): TC3/Kara roughly tied at 48-200 limbs (diff < 5%)
// TC3 reliably wins from ~80 limbs onward
constexpr size_t SQR_TOOMCOOK3_THRESHOLD = 160;

// sqr_toomcook3 scratch size
inline size_t sqr_toomcook3_scratch_size(size_t n) {
  if (n < SQR_TOOMCOOK3_THRESHOLD) return sqr_karatsuba_scratch_size(n);
  return 30 * n + 256; // same conservative size as mul_toomcook3
}

// Forward declaration (mutual recursion between sqr_toomcook3 and square)
inline size_t square_scratch_size(size_t n);
inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch);

// sqr_toomcook3: r[0..2n-1] = a[0..n-1]²
// Evaluation points {0, 1, -1, 2, inf} - evaluate only one polynomial (uses symmetry)
// 5 recursive squarings + interpolation
inline void sqr_toomcook3(uint64_t* r,
              const uint64_t* a, size_t n,
              uint64_t* scratch) {
  if (n < SQR_TOOMCOOK3_THRESHOLD) {
    sqr_karatsuba(r, a, n, scratch);
    return;
  }

  size_t k = (n + 2) / 3;
  size_t rn = 2 * n;

  // 3-way split
  const uint64_t* a0 = a;
  size_t a0n = normalized_size(a0, std::min(k, n));
  const uint64_t* a1 = a + k;
  size_t a1n = (n > k) ? normalized_size(a1, std::min(k, n - k)) : 0;
  const uint64_t* a2 = a + 2 * k;
  size_t a2n = (n > 2 * k) ? normalized_size(a2, n - 2 * k) : 0;

  // Scratch layout (smaller than the multiplication version: no b-side buffers needed)
  size_t blk = 2 * (k + 4);
  uint64_t* v1_buf   = scratch;
  uint64_t* vm1_buf   = scratch + blk;
  uint64_t* v2_buf   = scratch + 2 * blk;
  uint64_t* tmp1    = scratch + 3 * blk;   // evaluation temporary
  uint64_t* interp_buf = scratch + 4 * blk;   // interpolation/evaluation workspace
  uint64_t* rec_scratch = scratch + 5 * blk;

  size_t v1n = 0, vm1n = 0, v2n = 0;

  std::memset(r, 0, rn * sizeof(uint64_t));

  // Point 0: v0 = a0² → r[0..]
  if (a0n > 0)
    square(r, a0, a0n, rec_scratch);
  size_t v0n = normalized_size(r, std::min(2 * a0n, rn));

  // Point ∞: vinf = a2² → r[4k..]
  size_t vinfn = 0;
  if (a2n > 0 && 4 * k < rn) {
    square(r + 4 * k, a2, a2n, rec_scratch);
    vinfn = normalized_size(r + 4 * k, std::min(2 * a2n, rn - 4 * k));
  }

  // Point 1: v1 = (a0+a1+a2)²
  {
    std::memset(tmp1, 0, blk * sizeof(uint64_t));
    size_t ean = add_any(tmp1, a0, a0n, a1, a1n);
    ean = add_any(tmp1, tmp1, ean, a2, a2n);

    std::memset(v1_buf, 0, blk * sizeof(uint64_t));
    if (ean > 0)
      square(v1_buf, tmp1, ean, rec_scratch);
    v1n = normalized_size(v1_buf, 2 * ean);
  }

  // Point -1: vm1 = (a0-a1+a2)^2 [always non-negative - sign not neededed since it is a square]
  {
    std::memset(tmp1, 0, blk * sizeof(uint64_t));
    size_t t_n = add_any(tmp1, a0, a0n, a2, a2n);
    int ea_sign = 1;
    size_t ean = abs_sub(tmp1, ea_sign, tmp1, t_n, a1, a1n);

    std::memset(vm1_buf, 0, blk * sizeof(uint64_t));
    if (ean > 0)
      square(vm1_buf, tmp1, ean, rec_scratch);
    vm1n = normalized_size(vm1_buf, 2 * ean);
  }

  // Point 2: v2 = (a0+2*a1+4*a2)²
  {
    std::memset(tmp1, 0, blk * sizeof(uint64_t));
    if (a0n > 0) std::memcpy(tmp1, a0, a0n * sizeof(uint64_t));
    size_t ean = a0n;
    if (a1n > 0) {
      std::memset(interp_buf, 0, blk * sizeof(uint64_t));
      uint64_t ov = lshift(interp_buf, a1, a1n, 1);
      size_t tn = a1n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp1, tmp1, ean, interp_buf, tn);
    }
    if (a2n > 0) {
      std::memset(interp_buf, 0, blk * sizeof(uint64_t));
      uint64_t ov = lshift(interp_buf, a2, a2n, 2);
      size_t tn = a2n; if (ov) { interp_buf[tn] = ov; tn++; }
      ean = add_any(tmp1, tmp1, ean, interp_buf, tn);
    }

    std::memset(v2_buf, 0, blk * sizeof(uint64_t));
    if (ean > 0)
      square(v2_buf, tmp1, ean, rec_scratch);
    v2n = normalized_size(v2_buf, 2 * ean);
  }

  // ============================================================
  // interpolation (same structure as the original mul_toomcook3; vm1_sign is always >= 0)
  // ============================================================
  // Buffer layout:
  //  tmp1: A → c2  (Step 1,3)
  //  interp_buf: B -> C -> c1 (Step 2,4,8) - used as tmp2
  //  v1_buf: temp buffer (Step 5,7)
  //  vm1_buf: c3       (Step 7)
  //  v2_buf: D -> E (Step 5,6) - eventually free

  uint64_t* tmp2 = interp_buf; // reuse interp_buf as tmp2

  // Step 1: A = v1 + vm1 -> tmp1 (vm1_sign >= 0, so always added)
  std::memset(tmp1, 0, blk * sizeof(uint64_t));
  size_t An = add_any(tmp1, v1_buf, v1n, vm1_buf, vm1n);

  // Step 2: B = v1 - vm1 → tmp2 (v1 >= vm1 guarantee: f(1)² >= f(-1)²)
  std::memset(tmp2, 0, blk * sizeof(uint64_t));
  if (v1n > 0) std::memcpy(tmp2, v1_buf, v1n * sizeof(uint64_t));
  size_t Bn = v1n;
  if (vm1n > 0) { sub(tmp2, tmp2, Bn, vm1_buf, vm1n); Bn = normalized_size(tmp2, Bn); }

  // Step 3: c2 = A/2 - v0 - vinf → tmp1
  if (An > 0) An = rshift_1(tmp1, tmp1, An);
  if (v0n > 0 && An > 0) { sub(tmp1, tmp1, An, r, v0n); An = normalized_size(tmp1, An); }
  if (vinfn > 0 && An > 0) { sub(tmp1, tmp1, An, r + 4 * k, vinfn); An = normalized_size(tmp1, An); }
  size_t c2n = An;
  uint64_t* c2_ptr = tmp1;

  // Step 4: C = B/2 → tmp2
  if (Bn > 0) Bn = rshift_1(tmp2, tmp2, Bn);
  size_t Cn = Bn;

  // Step 5: D = v2 - v0 - 4*c2 - 16*vinf -> use v2_buf as interp_buf
  // (accumulate D in the location equivalent to interp_buf, matching the original code)
  {
    // Initial value of D = v2 (kept as-is in v2_buf)
    size_t Dn = v2n;
    // D -= v0
    if (v0n > 0 && Dn > 0) { sub(v2_buf, v2_buf, Dn, r, v0n); Dn = normalized_size(v2_buf, Dn); }
    // D -= 4*c2
    if (c2n > 0 && Dn > 0) {
      std::memset(v1_buf, 0, blk * sizeof(uint64_t));
      uint64_t ov = lshift(v1_buf, c2_ptr, c2n, 2);
      size_t tn = c2n; if (ov) { v1_buf[tn] = ov; tn++; }
      sub(v2_buf, v2_buf, Dn, v1_buf, tn);
      Dn = normalized_size(v2_buf, Dn);
    }
    // D -= 16*vinf
    if (vinfn > 0 && Dn > 0) {
      std::memset(v1_buf, 0, blk * sizeof(uint64_t));
      uint64_t ov = lshift(v1_buf, r + 4 * k, vinfn, 4);
      size_t tn = vinfn; if (ov) { v1_buf[tn] = ov; tn++; }
      sub(v2_buf, v2_buf, Dn, v1_buf, tn);
      Dn = normalized_size(v2_buf, Dn);
    }

    // Step 6: E = D/2 → v2_buf
    if (Dn > 0) Dn = rshift_1(v2_buf, v2_buf, Dn);

    // Step 7: c3 = (E - C) / 3 → vm1_buf
    size_t c3n_local = 0;
    if (Dn > 0 || Cn > 0) {
      std::memset(v1_buf, 0, blk * sizeof(uint64_t));
      if (Dn >= Cn) {
        if (Cn > 0) sub(v1_buf, v2_buf, Dn, tmp2, Cn);
        else std::memcpy(v1_buf, v2_buf, Dn * sizeof(uint64_t));
      } else {
        // Dn < Cn: extend v2_buf to Cn limbs (zero-fill upper part)
        for (size_t i = Dn; i < Cn; ++i) v2_buf[i] = 0;
        sub(v1_buf, v2_buf, Cn, tmp2, Cn);
      }
      size_t Fn = normalized_size(v1_buf, std::max(Dn, Cn));
      if (Fn > 0) c3n_local = divexact_by3(vm1_buf, v1_buf, Fn);
    }
    v2n = c3n_local; // reuse v2n as c3n
  }
  size_t c3n = v2n;
  uint64_t* c3_ptr = vm1_buf;

  // Step 8: c1 = C - c3 → v2_buf
  size_t c1n = 0;
  if (Cn > 0) {
    if (c3n > 0) {
      sub(v2_buf, tmp2, Cn, c3_ptr, c3n);
      c1n = normalized_size(v2_buf, Cn);
    } else {
      std::memcpy(v2_buf, tmp2, Cn * sizeof(uint64_t));
      c1n = Cn;
    }
  }
  uint64_t* c1_ptr = v2_buf;

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k)
  // ============================================================
  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
  }
}

// Toom-Cook-4 threshold for squaring
// mul_toomcook4 is comparable to ~10% faster than Toom-3 at 200+ limbs
// For sqr, symmetry widens the basecase/karatsuba advantage range, so the threshold is set higher
constexpr size_t SQR_TOOMCOOK4_THRESHOLD = 400;

inline size_t sqr_toomcook4_scratch_size(size_t n) {
  if (n < SQR_TOOMCOOK4_THRESHOLD) return sqr_toomcook3_scratch_size(n);
  return 50 * n + 512;
}

// sqr_toomcook4: r[0..2n-1] = a[0..n-1]²
// Evaluation points {0, 1, -1, 2, -2, 3, inf}, 7 recursive squarings
// Square-specialized version of mul_toomcook4:
//  - evaluation only on a(t) (since b=a, half the work)
//  - square at each point (faster than mul)
//  - wm1_sign = wm2_sign = +1 (squares are always positive)
inline void sqr_toomcook4(uint64_t* r,
             const uint64_t* a, size_t n,
             uint64_t* scratch) {
  if (n < SQR_TOOMCOOK4_THRESHOLD) {
    sqr_toomcook3(r, a, n, scratch);
    return;
  }

  size_t k = (n + 3) / 4;
  size_t rn = 2 * n;

  // --- Zero-pad coefficients to k limbs ---
  uint64_t* pa = scratch; // 4*k limbs (only a coefficients; no b needed)

  auto pad_coeff = [k](uint64_t* dst, const uint64_t* src, size_t src_len, size_t offset) {
    size_t actual = (src_len > offset) ? std::min(k, src_len - offset) : 0;
    if (actual > 0) std::memcpy(dst, src + offset, actual * sizeof(uint64_t));
    if (actual < k) std::memset(dst + actual, 0, (k - actual) * sizeof(uint64_t));
  };
  for (size_t i = 0; i < 4; i++) pad_coeff(pa + i * k, a, n, i * k);

  uint64_t* pa0 = pa;
  uint64_t* pa1 = pa + k;
  uint64_t* pa2 = pa + 2 * k;
  uint64_t* pa3 = pa + 3 * k;

  // --- Scratch layout ---
  size_t blk = 2 * k + 4;
  uint64_t* w1_buf   = scratch + 4 * k;     // starts from 4*k since b is not neededed
  uint64_t* wm1_buf   = w1_buf + blk;
  uint64_t* w2_buf   = wm1_buf + blk;
  uint64_t* wm2_buf   = w2_buf + blk;
  uint64_t* w3_buf   = wm2_buf + blk;
  uint64_t* ea_even   = w3_buf + blk;
  uint64_t* ea_odd   = ea_even + k + 2;
  uint64_t* eval_tmp  = ea_odd + k + 2;
  uint64_t* rec_scratch = eval_tmp + k + 2;

  size_t w1n = 0, wm1n = 0, w2n = 0, wm2n = 0, w3n = 0;

  // ============================================================
  // Pointwise squaring (7 recursive calls)
  // ============================================================

  // Point 0: w0 = a0² → r[0..]
  size_t a0n_real = normalized_size(pa0, std::min(k, n));
  size_t w0n = 0;
  if (a0n_real > 0) {
    square(r, pa0, a0n_real, rec_scratch);
    w0n = normalized_size(r, 2 * a0n_real);
  }

  // Point ∞: winf = a3² → r[6k..]
  size_t a3n_real = (n > 3 * k) ? n - 3 * k : 0;
  a3n_real = normalized_size(pa3, a3n_real);
  size_t winfn = 0;
  size_t winf_off = 6 * k;
  if (a3n_real > 0 && winf_off < rn) {
    square(r + winf_off, pa3, a3n_real, rec_scratch);
    winfn = normalized_size(r + winf_off, std::min(2 * a3n_real, rn - winf_off));
  }

  // Zero out gap region
  {
    size_t gap_start = w0n;
    size_t gap_end = std::min(winf_off, rn);
    if (gap_start < gap_end)
      std::memset(r + gap_start, 0, (gap_end - gap_start) * sizeof(uint64_t));
    size_t tail_start = std::min(winf_off + winfn, rn);
    if (tail_start < rn)
      std::memset(r + tail_start, 0, (rn - tail_start) * sizeof(uint64_t));
  }

  // +/-1 sharing: even1 = a0+a2, odd1 = a1+a3
  {
    std::memcpy(ea_even, pa0, k * sizeof(uint64_t));
    ea_even[k] = add(ea_even, ea_even, k, pa2, k);
    std::memcpy(ea_odd, pa1, k * sizeof(uint64_t));
    ea_odd[k] = add(ea_odd, ea_odd, k, pa3, k);
  }

  // Point 1: w1 = a(1)² = (even1+odd1)²
  {
    std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t));
    eval_tmp[k + 1] = 0;
    uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1);
    if (cy) eval_tmp[k + 1] = cy;
    size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k)));

    if (ean > 0) {
      square(w1_buf, eval_tmp, ean, rec_scratch);
      w1n = normalized_size(w1_buf, 2 * ean);
    }
  }

  // Point -1: wm1 = a(-1)^2 = (even1-odd1)^2 -> always positive (wm1_sign = +1)
  {
    int c = cmp(ea_even, k + 1, ea_odd, k + 1);
    if (c >= 0) {
      sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1);
    } else {
      sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1);
    }
    size_t ean = normalized_size(eval_tmp, k + 1);

    if (ean > 0) {
      square(wm1_buf, eval_tmp, ean, rec_scratch);
      wm1n = normalized_size(wm1_buf, 2 * ean);
    }
  }

  // +/-2 sharing: even2 = a0+4*a2, odd2 = 2*a1+8*a3
  {
    std::memcpy(ea_even, pa0, k * sizeof(uint64_t));
    ea_even[k] = addmul_1(ea_even, pa2, k, 4);
    ea_odd[k] = mul_1(ea_odd, pa1, k, 2);
    ea_odd[k] += addmul_1(ea_odd, pa3, k, 8);
  }

  // Point 2: w2 = a(2)² = (even2+odd2)²
  {
    std::memcpy(eval_tmp, ea_even, (k + 1) * sizeof(uint64_t));
    eval_tmp[k + 1] = 0;
    uint64_t cy = add(eval_tmp, eval_tmp, k + 1, ea_odd, k + 1);
    if (cy) eval_tmp[k + 1] = cy;
    size_t ean = (eval_tmp[k + 1] ? k + 2 : (eval_tmp[k] ? k + 1 : normalized_size(eval_tmp, k)));

    if (ean > 0) {
      square(w2_buf, eval_tmp, ean, rec_scratch);
      w2n = normalized_size(w2_buf, 2 * ean);
    }
  }

  // Point -2: wm2 = a(-2)^2 = (even2-odd2)^2 -> always positive (wm2_sign = +1)
  {
    int c = cmp(ea_even, k + 1, ea_odd, k + 1);
    if (c >= 0) {
      sub(eval_tmp, ea_even, k + 1, ea_odd, k + 1);
    } else {
      sub(eval_tmp, ea_odd, k + 1, ea_even, k + 1);
    }
    size_t ean = normalized_size(eval_tmp, k + 1);

    if (ean > 0) {
      square(wm2_buf, eval_tmp, ean, rec_scratch);
      wm2n = normalized_size(wm2_buf, 2 * ean);
    }
  }

  // Point 3: w3 = a(3)² = (a0+3a1+9a2+27a3)²
  {
    std::memcpy(eval_tmp, pa0, k * sizeof(uint64_t));
    eval_tmp[k] = 0;
    eval_tmp[k] += addmul_1(eval_tmp, pa1, k, 3);
    eval_tmp[k] += addmul_1(eval_tmp, pa2, k, 9);
    eval_tmp[k] += addmul_1(eval_tmp, pa3, k, 27);
    size_t ean = k + (eval_tmp[k] ? 1 : 0);

    if (ean > 0) {
      square(w3_buf, eval_tmp, ean, rec_scratch);
      w3n = normalized_size(w3_buf, 2 * ean);
    }
  }

  // ============================================================
  // Interpolation (same as mul_toomcook4 but with wm1_sign=+1, wm2_sign=+1 fixed)
  // ============================================================
  uint64_t* tmp1    = rec_scratch;
  uint64_t* tmp2    = rec_scratch + blk;
  uint64_t* interp_buf = rec_scratch + 2 * blk;
  uint64_t* interp_buf2 = rec_scratch + 3 * blk;

  // Step 1: t1 = (w1 + wm1) / 2 = c0+c2+c4+c6
  size_t t1n = add_any(tmp1, w1_buf, w1n, wm1_buf, wm1n);
  if (t1n > 0) t1n = rshift_1(tmp1, tmp1, t1n);

  // Step 2: t2 = (w1 - wm1) / 2 = c1+c3+c5
  size_t t2n;
  {
    std::memcpy(tmp2, w1_buf, w1n * sizeof(uint64_t));
    t2n = w1n;
    if (wm1n > 0) { sub(tmp2, tmp2, t2n, wm1_buf, wm1n); t2n = normalized_size(tmp2, t2n); }
    if (t2n > 0) t2n = rshift_1(tmp2, tmp2, t2n);
  }

  // Step 3: t3 = (w2 + wm2) / 2 = c0+4c2+16c4+64c6
  size_t t3n = add_any(interp_buf, w2_buf, w2n, wm2_buf, wm2n);
  if (t3n > 0) t3n = rshift_1(interp_buf, interp_buf, t3n);

  // Step 4: t4 = (w2 - wm2) / 2 = 2c1+8c3+32c5
  size_t t4n;
  {
    std::memcpy(interp_buf2, w2_buf, w2n * sizeof(uint64_t));
    t4n = w2n;
    if (wm2n > 0) { sub(interp_buf2, interp_buf2, t4n, wm2_buf, wm2n); t4n = normalized_size(interp_buf2, t4n); }
    if (t4n > 0) t4n = rshift_1(interp_buf2, interp_buf2, t4n);
  }

  // Step 9: t5 = t1 - w0 - winf = c2+c4
  if (t1n > 0) std::memcpy(w1_buf, tmp1, t1n * sizeof(uint64_t));
  size_t t5n = t1n;
  if (w0n > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r, w0n); t5n = normalized_size(w1_buf, t5n); }
  if (winfn > 0 && t5n > 0) { sub(w1_buf, w1_buf, t5n, r + winf_off, winfn); t5n = normalized_size(w1_buf, t5n); }

  // Step 10: t6 = t3 - w0 - 64*winf = 4c2+16c4
  if (t3n > 0) std::memcpy(wm1_buf, interp_buf, t3n * sizeof(uint64_t));
  size_t t6n = t3n;
  if (w0n > 0 && t6n > 0) { sub(wm1_buf, wm1_buf, t6n, r, w0n); t6n = normalized_size(wm1_buf, t6n); }
  if (winfn > 0 && t6n > 0) {
    uint64_t bw = submul_1(wm1_buf, r + winf_off, winfn, 64);
    if (bw > 0 && t6n > winfn) sub_1(wm1_buf + winfn, t6n - winfn, bw);
    t6n = normalized_size(wm1_buf, t6n);
  }

  // Step 11: t6 = t6 - 4*t5 = 12*c4
  if (t5n > 0 && t6n > 0) {
    uint64_t bw = submul_1(wm1_buf, w1_buf, t5n, 4);
    if (bw > 0 && t6n > t5n) sub_1(wm1_buf + t5n, t6n - t5n, bw);
    t6n = normalized_size(wm1_buf, t6n);
  }

  // Step 12: c4 = t6 / 12
  size_t c4n = 0;
  if (t6n > 0) c4n = divexact_by12(wm2_buf, wm1_buf, t6n);
  uint64_t* c4_ptr = wm2_buf;

  // Step 13: c2 = t5 - c4
  size_t c2n = t5n;
  if (c4n > 0 && c2n > 0) {
    sub(w1_buf, w1_buf, c2n, c4_ptr, c4n);
    c2n = normalized_size(w1_buf, c2n);
  }
  uint64_t* c2_ptr = w1_buf;

  // Step 14: t8 = t4 - 2*t2 = 6c3+30c5
  if (t4n > 0) std::memcpy(wm1_buf, interp_buf2, t4n * sizeof(uint64_t));
  size_t t8n = t4n;
  if (t2n > 0 && t8n > 0) {
    uint64_t bw = submul_1(wm1_buf, tmp2, t2n, 2);
    if (bw > 0 && t8n > t2n) sub_1(wm1_buf + t2n, t8n - t2n, bw);
    t8n = normalized_size(wm1_buf, t8n);
  }

  // Step 15: t9 = w3 - w0 - 9*c2 - 81*c4 - 729*winf
  if (w3n > 0) std::memcpy(interp_buf, w3_buf, w3n * sizeof(uint64_t));
  size_t t9n = w3n;
  if (w0n > 0 && t9n > 0) { sub(interp_buf, interp_buf, t9n, r, w0n); t9n = normalized_size(interp_buf, t9n); }
  if (c2n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, c2_ptr, c2n, 9);
    if (bw > 0 && t9n > c2n) sub_1(interp_buf + c2n, t9n - c2n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }
  if (c4n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, c4_ptr, c4n, 81);
    if (bw > 0 && t9n > c4n) sub_1(interp_buf + c4n, t9n - c4n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }
  if (winfn > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, r + winf_off, winfn, 729);
    if (bw > 0 && t9n > winfn) sub_1(interp_buf + winfn, t9n - winfn, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 16: t9 -= 3*t2
  if (t2n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, tmp2, t2n, 3);
    if (bw > 0 && t9n > t2n) sub_1(interp_buf + t2n, t9n - t2n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 17: t9 -= 4*t8
  if (t8n > 0 && t9n > 0) {
    uint64_t bw = submul_1(interp_buf, wm1_buf, t8n, 4);
    if (bw > 0 && t9n > t8n) sub_1(interp_buf + t8n, t9n - t8n, bw);
    t9n = normalized_size(interp_buf, t9n);
  }

  // Step 18: c5 = t9 / 120
  size_t c5n = 0;
  if (t9n > 0) c5n = divexact_by120(w3_buf, interp_buf, t9n);
  uint64_t* c5_ptr = w3_buf;

  // Step 19: t8 -= 30*c5 → 6*c3
  if (c5n > 0 && t8n > 0) {
    uint64_t bw = submul_1(wm1_buf, c5_ptr, c5n, 30);
    if (bw > 0 && t8n > c5n) sub_1(wm1_buf + c5n, t8n - c5n, bw);
    t8n = normalized_size(wm1_buf, t8n);
  }

  // Step 20: c3 = t8 / 6
  size_t c3n = 0;
  if (t8n > 0) {
    size_t sn = rshift_1(wm1_buf, wm1_buf, t8n);
    c3n = divexact_by3(interp_buf, wm1_buf, sn);
  }
  uint64_t* c3_ptr = interp_buf;

  // Step 21: c1 = t2 - c3 - c5
  size_t c1n = t2n;
  if (c1n > 0) {
    std::memcpy(interp_buf2, tmp2, t2n * sizeof(uint64_t));
    if (c3n > 0) { sub(interp_buf2, interp_buf2, c1n, c3_ptr, c3n); c1n = normalized_size(interp_buf2, c1n); }
    if (c5n > 0 && c1n > 0) { sub(interp_buf2, interp_buf2, c1n, c5_ptr, c5n); c1n = normalized_size(interp_buf2, c1n); }
  }
  uint64_t* c1_ptr = interp_buf2;

  // ============================================================
  // Assembly: r += c1*B^k + c2*B^(2k) + c3*B^(3k) + c4*B^(4k) + c5*B^(5k)
  // ============================================================
  if (c1n > 0 && k < rn) {
    size_t space = rn - k;
    add(r + k, r + k, space, c1_ptr, std::min(c1n, space));
  }
  if (c2n > 0 && 2 * k < rn) {
    size_t space = rn - 2 * k;
    add(r + 2 * k, r + 2 * k, space, c2_ptr, std::min(c2n, space));
  }
  if (c3n > 0 && 3 * k < rn) {
    size_t space = rn - 3 * k;
    add(r + 3 * k, r + 3 * k, space, c3_ptr, std::min(c3n, space));
  }
  if (c4n > 0 && 4 * k < rn) {
    size_t space = rn - 4 * k;
    add(r + 4 * k, r + 4 * k, space, c4_ptr, std::min(c4n, space));
  }
  if (c5n > 0 && 5 * k < rn) {
    size_t space = rn - 5 * k;
    add(r + 5 * k, r + 5 * k, space, c5_ptr, std::min(c5n, space));
  }
}

// square scratch size
inline size_t square_scratch_size(size_t n) {
  if (n < SQR_KARATSUBA_THRESHOLD) return 0;
  if (n >= SQR_PRIME_NTT_DIRECT_THRESHOLD) return 0; // NTT allocates automatically
  if (n >= SQR_DOUBLE_FFT_THRESHOLD && n < SQR_DOUBLE_FFT_MAX_THRESHOLD) return 0; // double_fft / NTT allocates automatically
  if (n >= SQR_TOOMCOOK4_THRESHOLD) return sqr_toomcook4_scratch_size(n);
  if (n >= SQR_TOOMCOOK3_THRESHOLD) return sqr_toomcook3_scratch_size(n);
  return sqr_karatsuba_scratch_size(n);
}

// Generic squaring: selects the algorithm automatically by size
// r[0..2n-1] = a[0..n-1]²
// r must not overlap a
inline void square(uint64_t* r, const uint64_t* a, size_t n, uint64_t* scratch) {
  if (n == 0) { return; }
  if (n < SQR_KARATSUBA_THRESHOLD) {
    sqr_basecase(r, a, n);
    return;
  }
  if (n >= SQR_PRIME_NTT_DIRECT_THRESHOLD) {
    prime_ntt::sqr_prime_ntt(r, a, n);
    return;
  }
  if (n >= SQR_DOUBLE_FFT_THRESHOLD && n < SQR_DOUBLE_FFT_MAX_THRESHOLD) {
    if (double_fft::sqr_double_fft(r, a, n))
      return;
    prime_ntt::sqr_prime_ntt(r, a, n);
    return;
  }
  if (n >= SQR_TOOMCOOK4_THRESHOLD) {
    sqr_toomcook4(r, a, n, scratch);
    return;
  }
  if (n >= SQR_TOOMCOOK3_THRESHOLD) {
    sqr_toomcook3(r, a, n, scratch);
    return;
  }
  sqr_karatsuba(r, a, n, scratch);
}

// ============================================================================
// Division Operations (Burnikel-Ziegler / Schoolbook)
// ============================================================================

// BZ recursion threshold (limb count). At or below this, use schoolbook (div_basecase)
// 2026-05-08 Phase E-1 redetermination: 64 -> 24 (reflects sbpi1 speedup after Phase A)
//  bench-div-threshold-e1e2: BZ is 12% faster from bn=24 and 31% faster at bn=64
constexpr size_t BZ_THRESHOLD = 24;

// Svoboda Division threshold (limb count).
// Note: the original "double divisor -> qhat = upper limb" scheme has quotient-estimation error exceeding 1, hence inaccurate.
// Disabled until replaced by Moeller-Granlund 3/2-based quotient estimation.
constexpr size_t SVOBODA_THRESHOLD = BZ_THRESHOLD + 1; // effectively disabled

// mu-division (Newton inverse iteration) threshold (limb count).
// For unbalanced (an > 2*bn+1), the threshold is determined dynamically by the an/bn ratio.
// For balanced (an <= 2*bn+1), BZ is O(M(n) log n) whereas
// mu_div_qr is O(M(n)), so mu wins at large bn.
//
// 2026-05-08 Phase E-2 redetermination (crossover measurement in bench-div-threshold-e1e2):
//  LOW 100 -> 150 (MU wins by 7% at bn=150; BZ wins at bn=100)
//  MID 200 -> 500 (MU wins by 2% at bn=500; BZ wins through bn=300)
//  HIGH 2500 (no change; k=3 unmeasured)
//  BALANCED 3000 -> 5000 (BZ ties through bn=4000; MU wins by 9% at bn=8000)
constexpr size_t MU_DIV_THRESHOLD_LOW = 150; // for high ratio k>=8
constexpr size_t MU_DIV_THRESHOLD_MID = 500; // for mid ratio k=4-7
constexpr size_t MU_DIV_THRESHOLD_HIGH = 2500; // for low ratio k=3
constexpr size_t MU_DIV_BALANCED_THRESHOLD = 5000; // balanced mu threshold (BZ -> MU crossover)

// invert_approx: floor of Newton recursion. For n <= INV_NEWTON_THRESHOLD,
// directly compute the inverse using div_basecase (schoolbook inversion).
constexpr size_t INV_NEWTON_THRESHOLD = 10;

inline size_t mu_div_threshold(size_t an, size_t bn) {
  size_t k = an / bn; // integer ratio
  if (k >= 8) return MU_DIV_THRESHOLD_LOW;
  if (k >= 4) return MU_DIV_THRESHOLD_MID;
  return MU_DIV_THRESHOLD_HIGH;
}

// --------------------------------------------------------------------------
// invert_limb: inverse of a normalized divisor
// --------------------------------------------------------------------------
// v = floor((B² - 1) / d) - B (B = 2^64)
// Precondition: MSB of d is set (d >= 2^63)
// Replaces the hardware division instruction (~35-90 cycles) inside div_basecase / divmod_1
// with multiplication-based quotient estimation (~10-15 cycles)
inline uint64_t invert_limb(uint64_t d) {
  // v = floor([~d, ~0] / d)
  // ~d < d (since d >= 2^63, ~d = 2^64 - 1 - d < 2^63 <= d)
#if defined(_MSC_VER) && defined(_M_X64)
  uint64_t dummy;
  return _udiv128(~d, ~uint64_t(0), d, &dummy);
#elif defined(__SIZEOF_INT128__)
  __uint128_t num = (static_cast<__uint128_t>(~d) << 64) | ~uint64_t(0);
  return static_cast<uint64_t>(num / d);
#else
  auto [v, rem] = UInt128::divmod_fast(~d, ~uint64_t(0), d);
  return v;
#endif
}

// --------------------------------------------------------------------------
// udiv_qrnnd_preinv: 2-limb / 1-limb division using a precomputed inverse
// --------------------------------------------------------------------------
// [u1, u0] / d → (quotient, remainder)
// Precondition: u1 < d, MSB of d is set, dinv = invert_limb(d)
// Algorithm by Moeller & Granlund (2011)
// Even on q1 + 1 overflow, correctness is recovered via the remainder operation
inline std::pair<uint64_t, uint64_t> udiv_qrnnd_preinv(
    uint64_t u1, uint64_t u0, uint64_t d, uint64_t dinv) {
  // (q1, q0) = u1 * dinv + (u1, u0)
  uint64_t p_hi;
  uint64_t p_lo = _umul128(u1, dinv, &p_hi);
  uint64_t q0 = p_lo + u0;
  uint64_t q1 = p_hi + u1 + (q0 < p_lo);
  q1 += 1; // tentative quotient (may wrap to 0; handled below)

  uint64_t r = u0 - q1 * d; // mod B

  if (r > q0) {
    // q1 was too large by 1 (or wrapped from overflow)
    q1--;
    r += d;
  }
  if (r >= d) {
    q1++;
    r -= d;
  }
  return {q1, r};
}

// --------------------------------------------------------------------------
// invert_pi1: inverse computation for 3-by-2 division
// --------------------------------------------------------------------------
// Returns floor((B^3 - 1) / (d1*B + d0)) - B.
// Equivalent to GMP's invert_pi1 macro. Start from invert_limb(d1) and correct via d0.
inline uint64_t invert_pi1(uint64_t d1, uint64_t d0) {
  uint64_t v = invert_limb(d1); // floor((B^2-1)/d1) - B
  uint64_t p = d1 * v;
  p += d0;
  if (p < d0) {
    v--;
    uint64_t mask = (p >= d1) ? UINT64_MAX : 0;
    p -= d1;
    v += mask;   // mask is -1 (subtract) or 0
    p -= mask & d1;
  }
  // {t1, t0} = d0 * v
  UInt128 t = UInt128::multiply(d0, v);
  p += t.high;
  if (p < t.high) {
    v--;
    if (p >= d1) {
      if (p > d1 || t.low >= d0)
        v--;
    }
  }
  return v;
}

// --------------------------------------------------------------------------
// udiv_qrnnd_3by2: Moeller-Granlund 3-by-2 quotient estimation
// --------------------------------------------------------------------------
// {u2, u1, u0} / {d1, d0} → quotient q, remainder {r1, r0}
// Precondition: MSB of d1 is set, u2 < d1 || (u2 == d1 && u1 <= d0 is undefined)
// dinv = invert_pi1(d1, d0) (3-by-2 inverse)
inline void udiv_qrnnd_3by2(uint64_t& q_out, uint64_t& r1_out, uint64_t& r0_out,
               uint64_t u2, uint64_t u1, uint64_t u0,
               uint64_t d1, uint64_t d0, uint64_t dinv) {
  // {q1, q0} = u2 * dinv + {u2, u1}
  uint64_t p_hi;
  uint64_t p_lo = _umul128(u2, dinv, &p_hi);
  uint64_t q0 = p_lo + u1;
  uint64_t q1 = p_hi + u2 + (q0 < p_lo);

  // r1 = u1 - d1*q1, {r1, r0} = {r1, u0} - {d1, d0}
  uint64_t r1 = u1 - d1 * q1;
  uint64_t r0 = u0 - d0;
  r1 -= d1 + (u0 < d0);

  // {r1, r0} -= d0 * q1
  uint64_t t_hi;
  uint64_t t_lo = _umul128(d0, q1, &t_hi);
  uint64_t old_r0 = r0;
  r0 -= t_lo;
  r1 -= t_hi + (r0 > old_r0);

  q1++;

  // Conditional correction (branchless)
  uint64_t mask = static_cast<uint64_t>(0) - static_cast<uint64_t>(r1 >= q0);
  q1 += mask;
  old_r0 = r0;
  r0 += mask & d0;
  r1 += (mask & d1) + (r0 < old_r0);

  if (r1 >= d1 && (r1 > d1 || r0 >= d0)) {
    q1++;
    old_r0 = r0;
    r0 -= d0;
    r1 -= d1 + (r0 > old_r0);
  }

  q_out = q1;
  r1_out = r1;
  r0_out = r0;
}

// Forward declaration (used from the n=2 base case of invert_approx)
inline void div_basecase(uint64_t* q, uint64_t* a, size_t an,
             const uint64_t* b, size_t bn);

// Forward declaration (B-3: used from invert_approx Step (a); definition in the second half of this file, line ~7960)
inline size_t mulmod_bnm1_next_size(size_t n);
inline size_t mulmod_bnm1_scratch_size(size_t rn, size_t an, size_t bn);
inline void mulmod_bnm1(uint64_t* rp, size_t rn,
             const uint64_t* ap, size_t an,
             const uint64_t* bp, size_t bn,
             uint64_t* scratch);

// --------------------------------------------------------------------------
// invert_approx: N-word approximate inverse (Newton inverse iteration)
// --------------------------------------------------------------------------
// d[0..n-1]: normalized divisor (MSB of d[n-1] is set)
// inv[0..n-1]: output inverse (does not include the implicit leading 1)
//  Interpret as I = B^n + inv, with D * I ~ B^(2n) (error < D)
// scratch: invert_approx_scratch_size(n) limbs
// Precondition: n >= 1, MSB of d[n-1] is set

inline size_t invert_approx_scratch_size(size_t n) {
  if (n <= INV_NEWTON_THRESHOLD) {
    // base case: directly compute floor((B^(2n)-1)/D) via div_basecase
    // num: 2n+1 limbs, q: n+1 limbs → 3n+4
    return 3 * n + 4;
  }
  size_t h = (n + 2) / 2; // ceil(n/2) + (n%2==0) guarantees 2h > n
  size_t mul_nh = multiply_scratch_size(n, h);
  size_t mul_hh = multiply_scratch_size(h, h);
  size_t mul_sz = std::max(mul_nh, mul_hh);
  size_t rec_sz = invert_approx_scratch_size(h);
  // newton step: xp (n+h) + err (h) + multiply_scratch (mul_sz)
  // The recursive call completes before the Newton step, so scratch is sharable
  size_t newton_sz = n + 2 * h + mul_sz;
  return std::max(newton_sz, rec_sz);
}

#ifdef SANGI_INVERT_PROFILE
// Per-level phase accumulator (thread_local nanoseconds)
// Each level in the recursion has its own bucket so we can see h-dependent
// behavior. Global totals are computed by summing levels.
struct SangiInvertProfileLevel {
  size_t n = 0;
  uint64_t rec_ns = 0;  // time in recursive invert_approx_inner call
  uint64_t mul_a_ns = 0; // Step (a): multiply(d, n, inv_h, h) — n×h
  uint64_t sign_ns = 0;  // Steps (b)-(c): sign dispatch, err computation
  uint64_t mul_d_ns = 0; // Step (d): multiply(err, h, inv_h, h) — h×h
  uint64_t corr_ns = 0;  // Step (e): add correction to inv
  uint64_t base_ns = 0;  // base case time
  uint64_t calls = 0;
};
struct SangiInvertProfileData {
  SangiInvertProfileLevel levels[32];
  uint64_t toplevel_calls = 0;
};
inline SangiInvertProfileData& sangi_invert_profile() {
  static thread_local SangiInvertProfileData d;
  return d;
}
inline void sangi_invert_profile_reset() { sangi_invert_profile() = {}; }
#define SANGI_IVP_TICK() auto _ivp_t = std::chrono::high_resolution_clock::now()
#define SANGI_IVP_ACCUM(field) do { \
  auto _ivp_e = std::chrono::high_resolution_clock::now(); \
  uint64_t _dt = (uint64_t)std::chrono::duration_cast<std::chrono::nanoseconds>(_ivp_e - _ivp_t).count(); \
  if (_ivp_level < 32) { \
    sangi_invert_profile().levels[_ivp_level].field += _dt; \
    sangi_invert_profile().levels[_ivp_level].n = n; \
  } \
  _ivp_t = _ivp_e; \
} while (0)
#else
#define SANGI_IVP_TICK() do{}while(0)
#define SANGI_IVP_ACCUM(field) do{}while(0)
#endif

// Internal implementation (follows GMP mpn_ni_invertappr)
// Truncated addition of D + positive/negative remainder class branch + h*h correction multiplication.
// Result has an error within +/-1 relative to the true inverse.
// Internal implementation: return value is the cy flag (1 or 2).
// cy=1: inverse is +/-0 relative to the true value (exact floor)
// cy=2: inverse may be +1 relative to the true value
inline int invert_approx_inner(uint64_t* inv,
                const uint64_t* d, size_t n,
                uint64_t* scratch) {
#ifdef SANGI_INVERT_PROFILE
  static thread_local int _ivp_depth = 0;
  int _ivp_level = _ivp_depth;
  bool _ivp_is_top = (_ivp_depth == 0);
  if (_ivp_is_top) sangi_invert_profile().toplevel_calls++;
  if (_ivp_level < 32) sangi_invert_profile().levels[_ivp_level].calls++;
  _ivp_depth++;
  struct _IvpGuard { int& d; ~_IvpGuard(){ d--; } } _ivp_guard{_ivp_depth};
#endif
  SANGI_IVP_TICK();
  // ================ Base case: n = 1 ================
  if (n == 1) {
    inv[0] = invert_limb(d[0]);
    SANGI_IVP_ACCUM(base_ns);
    return 1;
  }

  // ================ Base case: n <= INV_NEWTON_THRESHOLD ================
  // I = floor((β^(2n) - 1) / D), inv = I - β^n.
  // D is normalized (MSB set), so beta^n/2 <= D < beta^n and beta^n < I <= 2*beta^n - 1.
  // To satisfy the div_basecase precondition a_high < b, subtract beta^n first:
  //  Setting num = (beta^(2n) - 1) - D * beta^n yields num[n..2n-1] = (beta^n - 1) - D < D.
  //  floor(num / D) = floor((β^(2n)-1)/D) - β^n = inv.
  if (n <= INV_NEWTON_THRESHOLD) {
    uint64_t* num = scratch;
    uint64_t* q = scratch + 2 * n + 1;
    // num[0..2n-1] = β^(2n) - 1 (allbit 1)
    std::memset(num, 0xFF, 2 * n * sizeof(uint64_t));
    // num[n..2n-1] -= D → (β^n - 1) - D < D
    sub(num + n, num + n, n, d, n);
    // div_basecase forsentinel
    num[2 * n] = 0;
    std::memset(q, 0, (n + 1) * sizeof(uint64_t));
    div_basecase(q, num, 2 * n, d, n);
    std::memcpy(inv, q, n * sizeof(uint64_t));
    SANGI_IVP_ACCUM(base_ns);
    return 1; // div_basecase returns the exact quotient
  }

  // ================ Recursive case (n > INV_NEWTON_THRESHOLD) ================
  // h = ceil(n/2) + (n%2==0): for even n take h = n/2+1, guaranteeing 2h > n.
  // Newton's error amplification is eps_new ~ eps_old^2 * beta^{n-2h}.
  // Setting 2h > n yields beta^{n-2h} < 1, so the error always converges.
  size_t h = (n + 2) / 2;
  size_t l = n - h;

  // Step 1: recursively compute the h-limb inverse of the upper h limbs
  // Stored in inv[l..n-1]
  int cy_rec = invert_approx_inner(inv + l, d + l, h, scratch);
  SANGI_IVP_ACCUM(rec_ns);
  // I_h = B^h + inv[l..n-1]

  // Step 2: extend from h to n limbs via the Newton step (GMP style)
  // scratch layout:
  //  xp[0..n+h-1]: D × I_h product (n+h limbs)
  //  err[0..h-1]:  error (h limbs)
  //  mul_work[...]: multiplication scratch
  uint64_t* xp = scratch;
  uint64_t* err_buf = scratch + n + h;
  uint64_t* mul_work = scratch + n + 2 * h;

  // (a) xp = D × I_h (n × h → n+h limbs)
  //   X3 attempt (2026-05-08, B-3): tried replacing with mulmod_bnm1(rn=next_size(n+h+1)),
  //   but mul_a time degraded to 1.10-1.52x across n=513-8193 -> reverted.
  //   The M(n)/2 cost is a theoretical value in the NTT range; in the Toom range, CRT decomposition overhead
  //   exceeds the unbalanced multiply optimization (same pattern as the X1 revert).
  //   A custom truncated Toom (computing the low n+1 limbs) is the real solution (~25% expected,
  //   2-4 weeks of effort); low priority, re-evaluate later.
  multiply(xp, d, n, inv + l, h, mul_work);
  SANGI_IVP_ACCUM(mul_a_ns);

  // (b) xp[h..n] += D[0..l] (truncated addition: only the low l+1 limbs of D)
  //   As in GMP, the contribution from D[l+1..n-1] * B^{n+1} is ignored.
  //   This yields T mod B^{n+1} in xp[0..n].
  //   Carry is ignored (absorbed as part of T mod B^{n+1}).
  (void)add(xp + h, xp + h, l + 1, d, l + 1);

  // (c) Normalize based on xp[n] (GMP positive/negative remainder class branch)
  //   Since T ~ B^{n+h}, xp[n] is 0,1 (positive) or UINT64_MAX,MAX-1 (negative).
  int ret_cy;
  if (xp[n] < 2) {
    // === Positive remainder class ===
    // cy is "how much to subtract from I_h". Includes +1 for the truncation.
    uint64_t cy = xp[n];
    // cy++ adds the truncation amount (GMP: "Remember we truncated")
    if (cy++) {
      // xp[n] was 1 -> subtract D from xp[0..n-1]
      uint64_t borrow = sub(xp, xp, n, d, n);
      if (!borrow) {
        // no borrow -> xp >= D still -> subtract once more
        sub(xp, xp, n, d, n);
        ++cy;
      }
    }
    // 1 <= cy <= 3
    // Final check: if xp > D, subtract once
    if (cmp(xp, n, d, n) > 0) {
      sub(xp, xp, n, d, n);
      ++cy;
    }
    // 1 <= cy <= 4
    // Subtract cy from I_h
    sub_1(inv + l, h, cy);

    // error = D_h - xp_upper (h limbs)
    // D_h = d[l..n-1], xp_upper = xp[l..n-1]
    // Borrow from the lower part: 1 if xp[0..l-1] > d[0..l-1]
    sub(err_buf, d + l, h, xp + l, h);
    int _borrow_low = (cmp(xp, l, d, l) > 0) ? 1 : 0;
    if (_borrow_low) {
      sub_1(err_buf, h, 1);
    }
    // Positive branch: propagate cy_rec (Newton doubles precision; does not amplify error)
    ret_cy = cy_rec;
  } else {
    // === Negative remainder class (xp[n] >= UINT64_MAX - 1) ===
    // Subtract the truncation flag (1) from xp
    sub_1(xp, n + 1, 1);

    if (xp[n] != UINT64_MAX) {
      // Increment I_h by 1
      add_1(inv + l, h, 1);
      // Add D to xp (a carry should be produced, restoring xp[n] = UINT64_MAX)
      add(xp, xp, n, d, n);
    }

    // error = ~xp[l..n-1] (ones complement ~ B^h - 1 - xp_upper)
    for (size_t i = 0; i < h; i++) {
      err_buf[i] = ~xp[l + i];
    }
    // Negative branch: complement operation is exact -> cy=1
    ret_cy = 1;
  }
  SANGI_IVP_ACCUM(sign_ns);

  // (d) correctmultiplication: err × I_h (h × h → 2h limbs)
  //   X1 attempt (2026-04-20): tried replacing with mulhigh_n(h), but around h=1025
  //   mulhigh_n decomposes into multiply(513,513) + 2*mulhigh_n(512),
  //   falling to Toom-4. Meanwhile full multiply(1025,1025) can use Toom-8,
  //   so net it was 7-10% slower -> reverted. Focus on Step(a)-side X3.
  multiply(xp, err_buf, h, inv + l, h, mul_work);
  // xp[0..2h-1] = err × I_h
  SANGI_IVP_ACCUM(mul_d_ns);

  // (e) Add the implicit B^h component: delta = err + (err * I_h) >> h
  //   Split into 2 parts in GMP style:
  //   Part 1: xp[h..3h-n-1] += err[0..2h-n-1] (2h-n limbs)
  uint64_t cy_corr = add(xp + h, xp + h, 2 * h - n, err_buf, 2 * h - n);

  //   Part 2: inv[0..l-1] = xp[3h-n..2h-1] + err[2h-n..h-1] + cy (l limbs)
  uint64_t cy2 = add(inv, xp + 3 * h - n, l, err_buf + 2 * h - n, l);
  if (cy_corr) cy2 += add_1(inv, l, 1);

  // Propagate carry to inv[l..n-1]
  if (cy2) add_1(inv + l, h, cy2);
  SANGI_IVP_ACCUM(corr_ns);

  return ret_cy;
}

// Public API: computes the approximate inverse, guaranteeing D * I <= beta^(2n).
// Error upper bound: beta^(2n) - D * I <= D (handled by mu_div_qr quotient correction loop)
// DIV-1c: cy is inductively always 1 (base case=1, positive branch=cy_rec, negative branch=1).
//     The sub_1(inv, n, 2) case does not occur.
inline void invert_approx(uint64_t* inv,
              const uint64_t* d, size_t n,
              uint64_t* scratch) {
  [[maybe_unused]] int cy = invert_approx_inner(inv, d, n, scratch);
  assert(cy == 1 && "invert_approx_inner should always return cy=1");

  // Subtract 1 from inv to bias toward underestimation,
  // guaranteeing D * (B^n + inv) <= B^{2n}.
  // For D ~ beta^n (inv ~ 0), sub_1(1) underflows, so
  // check the borrow and fall back to inv = 0 (I = beta^n, a safe underestimate).
  uint64_t borrow = sub_1(inv, n, 1ULL);
  if (borrow) {
    std::memset(inv, 0, n * sizeof(uint64_t));
  }
}

// Forward declaration (called from div_basecase)
inline void div_basecase_svoboda(uint64_t* q, uint64_t* a, size_t an,
                 const uint64_t* b, size_t bn);

// --------------------------------------------------------------------------
// div_basecase: mpn port of Knuth Algorithm D (preinv-optimized version)
// --------------------------------------------------------------------------
// Precondition:
//  - bn >= 2
//  - MSB of b[bn-1] is set (normalization)
//  - a[] has an+1 limbs of space (uses a[an] as a sentinel)
//  - Caller must set a[an] = 0 beforehand
// result:
//  - q[0 .. an-bn] holds the quotient
//  - a[0 .. bn-1] holds the remainder (overwrites a)
inline void div_basecase(uint64_t* q, uint64_t* a, size_t an,
             const uint64_t* b, size_t bn) {
  // Svoboda: if both divisor and quotient digits exceed the threshold, use Svoboda Division
  if (bn >= SVOBODA_THRESHOLD && (an - bn) >= SVOBODA_THRESHOLD) {
    div_basecase_svoboda(q, a, an, b, bn);
    return;
  }

  const uint64_t d1 = b[bn - 1];      // topmost limb (MSB set)
  const uint64_t d0 = b[bn - 2];      // second limb

  if (bn == 2) {
    // ── divrem_2 passes (bn == 2) ──
    // Equivalent to GMP mpn_divrem_2: obtains the exact quotient via 3-by-2.
    // bn-2 = 0, so submul is not neededed. Holds the remainder {n1, n0} in registers.
    const uint64_t dinv3 = invert_pi1(d1, d0);

    uint64_t n1 = 0;
    uint64_t n0 = a[an - 1];

    for (size_t j = an - 2; ; ) {
      uint64_t q_hat;
      udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0, a[j], d1, d0, dinv3);
      q[j] = q_hat;
      if (j == 0) break;
      --j;
    }
    a[0] = n0;
    a[1] = n1;
  } else if (bn >= 7) {
    // ── Möller-Granlund 3-by-2 passes (bn >= 7) ──
    const uint64_t dinv3 = invert_pi1(d1, d0);

#ifdef SANGI_INT_HAS_ASM
    if (detail::has_bmi2_adx()) {
      mpn_sbpi1_div_qr_asm(q, a, an - bn, b, bn, dinv3);
      return;
    }
#endif

    for (size_t j = an - bn; ; ) {
      uint64_t q_hat;
      const uint64_t u2 = a[j + bn];
      const uint64_t u1 = a[j + bn - 1];

      if ((u2 == d1) && u1 == d0) {
        q_hat = UINT64_MAX;
        uint64_t borrow = submul_1(a + j, b, bn, q_hat);
        uint64_t prev = a[j + bn];
        a[j + bn] = prev - borrow;
        if (prev < borrow) {
          --q_hat;
          add(a + j, a + j, bn + 1, b, bn);
        }
      } else {
        uint64_t n1, n0;
        udiv_qrnnd_3by2(q_hat, n1, n0, u2, u1,
                a[j + bn - 2], d1, d0, dinv3);

        uint64_t cy = submul_1(a + j, b, bn - 2, q_hat);

        uint64_t cy0 = (n0 < cy) ? 1 : 0;
        n0 -= cy;
        uint64_t cy1 = (n1 < cy0) ? 1 : 0;
        n1 -= cy0;

        a[j + bn - 2] = n0;

        if (cy1) {
          n1 += d1 + add(a + j, a + j, bn - 1, b, bn - 1);
          --q_hat;
        }
        a[j + bn - 1] = n1;
        a[j + bn] = 0;
      }

      q[j] = q_hat;
      if (j == 0) break;
      --j;
    }
  } else {
    // ── 2-by-single pass (bn = 3..6) ──
    // For small bn, 3-by-2 overhead dominates, so the conventional scheme is used.
    const uint64_t dinv = invert_limb(d1);

    for (size_t j = an - bn; ; ) {
      uint64_t q_hat;
      const uint64_t u2 = a[j + bn];
      const uint64_t u1 = a[j + bn - 1];
      if (u2 >= d1) {
        q_hat = UINT64_MAX;
      } else {
        auto [qh, rh] = udiv_qrnnd_preinv(u2, u1, d1, dinv);
        q_hat = qh;
      }

      uint64_t borrow = submul_1(a + j, b, bn, q_hat);
      uint64_t prev = a[j + bn];
      a[j + bn] = prev - borrow;
      if (prev < borrow) {
        --q_hat;
        uint64_t cy = add(a + j, a + j, bn + 1, b, bn);
        if (!cy) {
          --q_hat;
          add(a + j, a + j, bn + 1, b, bn);
        }
      }

      q[j] = q_hat;
      if (j == 0) break;
      --j;
    }
  }
}

// --------------------------------------------------------------------------
// div_basecase_svoboda: Svoboda Division (simplifies quotient estimation to top-limb reads)
// --------------------------------------------------------------------------
// Double the divisor so that V'[bn] = 1 and set quotient estimate qhat = a[j+bn].
// Removes udiv_qrnnd_preinv (~12 cycles/digit); in exchange, submul_1 becomes bn+1 limbs.
// Precondition:
//  - bn >= 2, MSB of b[bn-1] is set (normalization)
//  - a[] has an+2 limbs of space (a[an] sentinel + a[an+1] Svoboda sentinel)
//  - Caller must set a[an] = 0, a[an+1] = 0 beforehand (overwritten by lshift)
// result:
//  - q[0 .. an-bn] holds the quotient
//  - a[0 .. bn-1] holds the remainder (overwrites a)
inline void div_basecase_svoboda(uint64_t* q, uint64_t* a, size_t an,
                 const uint64_t* b, size_t bn) {
  // bp = 2*b (bn+1 limbs, bp[bn] = 1)
  uint64_t bp[BZ_THRESHOLD + 1];
  bp[bn] = lshift(bp, b, bn, 1);

  // a = 2*a (in-place)
  a[an] = lshift(a, a, an, 1);
  a[an + 1] = 0; // Svoboda foradditional sentinel

  for (size_t j = an - bn; ; ) {
    // Quotient estimation: read the topmost limb (core of Svoboda)
    uint64_t q_hat = a[j + bn];

    // Multiply-subtract: a[j .. j+bn] -= bp[0..bn] * q_hat
    uint64_t borrow = submul_1(a + j, bp, bn + 1, q_hat);
    uint64_t prev = a[j + bn + 1];
    a[j + bn + 1] = prev - borrow;
    if (prev < borrow) {
      // add-back: q_hat was 1 too large (Svoboda guarantee: at most once)
      --q_hat;
      add(a + j, a + j, bn + 2, bp, bn + 1);
    }

    q[j] = q_hat;
    if (j == 0) break;
    --j;
  }

  // Svoboda scale removal: remainder = a[0..bn] / 2
  // Doubled remainder 2r is bn+1 limbs (a[bn] is 0 or 1). Halving fits in bn limbs.
  for (size_t i = 0; i < bn - 1; i++) {
    a[i] = (a[i] >> 1) | (a[i + 1] << 63);
  }
  a[bn - 1] = (a[bn - 1] >> 1) | (a[bn] << 63);
}

// --------------------------------------------------------------------------
// BZ recursion core: forward declaration
// --------------------------------------------------------------------------
inline void div_2n_by_n(uint64_t* q, uint64_t* r,
            const uint64_t* a,
            const uint64_t* b, size_t n,
            uint64_t* scratch);

// div_3n_by_2n: core of BZ
// Divide A[3*half] by B[n] = [B1, B0] (half limbs each)
// q[0..half-1] holds the quotient, r[0..n-1] holds the remainder
// scratch: for multiply + working buffer
inline void div_3n_by_2n(uint64_t* q, uint64_t* r,
             const uint64_t* a,    // 3*half limbs
             const uint64_t* b,    // n = 2*half limbs
             size_t n,
             uint64_t* scratch) {
  const size_t half = n / 2;
  // A = [A2, A1, A0] each half limbs
  const uint64_t* a0 = a;
  const uint64_t* a1 = a + half;
  // a2 = a + 2*half (A2 is passed to div_2n_by_n via a1)
  // B = [B1, B0] each half limbs
  const uint64_t* b0 = b;
  const uint64_t* b1 = b + half;

  // (Q_hat, R1) = div_2n_by_n([A2, A1], B1)
  // A2,A1 → 2*half limbs, B1 → half limbs
  uint64_t* r1 = scratch;         // half limbs
  uint64_t* q_hat = q;          // half limbs (output)
  uint64_t* sub_scratch = scratch + half; // for recursion

  const uint64_t* a2 = a + 2 * half; // A2 (top half of the 3*half-limb dividend)

  // BUGFIX (cas-c5cc deep): Burnikel-Ziegler "D3n2n" overflow case.
  // The 2n/n sub-quotient Q_hat can reach β^half (one limb beyond the half-limb
  // output buffer) precisely when A2 >= B1.  div_2n_by_n only produces `half`
  // quotient limbs, so on overflow it silently dropped the top limb, yielding a
  // value-dependent wrong quotient for operands whose limbs hit A2 == B1 (e.g.
  // power-of-10-structured (10^3723+7)/(10^1773+3)).
  // The classic BZ fix clamps Q_hat = β^half - 1 and forms R1 = [A2,A1] - Q_hat*B1.
  // Under the D3n2n precondition A < B*β^half we have A2 <= B1, so the overflow
  // case is exactly A2 == B1 and then R1 = [A2,A1] - (β^half-1)*B1 = A1 + B1
  // (which may carry one limb beyond `half`, tracked below in r_top).
  uint64_t r_top;
  if (cmp(a2, half, b1, half) < 0) {
    // normal case: a_top = [A1, A2] (2*half limbs) is contiguous within a
    div_2n_by_n(q_hat, r1, a1, b1, half, sub_scratch);
    r_top = 0;
  } else {
    // overflow case: Q_hat = β^half - 1 (all ones); R1 = A1 + B1
    std::memset(q_hat, 0xFF, half * sizeof(uint64_t));
    r_top = add(r1, a1, half, b1, half); // r1 = A1 + B1, carry-out into r_top
  }

  // D = Q_hat * B0 (multiplication: half × half → 2*half limbs)
  uint64_t* d = scratch + half; // 2*half limbs
  uint64_t* mul_scratch = d + 2 * half;
  std::memset(d, 0, 2 * half * sizeof(uint64_t));
  multiply(d, q_hat, half, b0, half, mul_scratch);

  // R = [R1, A0] - D, held as an (n+1)-limb value [r_top : r[0..n-1]]
  // [R1, A0] is n limbs (A0 low, R1 high); r_top is R1's overflow carry (0/1).
  std::memcpy(r, a0, half * sizeof(uint64_t));  // lower half = A0
  std::memcpy(r + half, r1, half * sizeof(uint64_t)); // upper half = R1 (low half)

  // r -= D ; keep the running top limb (signed) so the overflow carry is honored
  uint64_t borrow = sub(r, r, n, d, n);
  int64_t net_top = (int64_t)r_top - (int64_t)borrow;

  // Correction: while the (n+1)-limb remainder is negative, Q_hat--, r += B.
  // (In the non-overflow case r_top == 0 this is the original `while (borrow)` loop.)
  while (net_top < 0) {
    // Subtract 1 from Q_hat (multi-limb decrement)
    for (size_t i = 0; i < half; i++) {
      if (q_hat[i] != 0) { q_hat[i]--; break; }
      q_hat[i] = UINT64_MAX;
    }
    // r += B
    uint64_t carry = add(r, r, n, b, n);
    net_top += (int64_t)carry;
  }
}

// div_2n_by_n: divide A[2n] by B[n]
// q[0..n-1] holds the quotient, r[0..n-1] holds the remainder
inline void div_2n_by_n(uint64_t* q, uint64_t* r,
            const uint64_t* a,    // 2n limbs
            const uint64_t* b,    // n limbs
            size_t n,
            uint64_t* scratch) {
  // base case: schoolbook
  if (n < BZ_THRESHOLD) {
    // Strip leading zeros of the divisor (can occur with odd padding in BZ recursion)
    size_t bn_real = normalized_size(b, n);
    if (bn_real == 0) {
      // Zero division - set quotient and remainder to zero
      std::memset(q, 0, n * sizeof(uint64_t));
      std::memset(r, 0, n * sizeof(uint64_t));
      return;
    }

    // effective size of the dividend
    size_t an_real = normalized_size(a, 2 * n);
    if (an_real == 0) {
      std::memset(q, 0, n * sizeof(uint64_t));
      std::memset(r, 0, n * sizeof(uint64_t));
      return;
    }

    // If dividend < divisor: quotient=0, remainder=dividend
    // Can occur in BZ recursion when the upper part becomes smaller than the divisor
    if (an_real < bn_real || (an_real == bn_real && cmp(a, an_real, b, bn_real) < 0)) {
      std::memset(q, 0, n * sizeof(uint64_t));
      std::memset(r, 0, n * sizeof(uint64_t));
      std::memcpy(r, a, an_real * sizeof(uint64_t));
      return;
    }

    // Normalization: set the MSB of the divisor
    unsigned shift = std::countl_zero(b[bn_real - 1]);

    // scratch layout:
    //  nb:  bn_real limbs   (normalized divisor)
    //  tmp_a: an_real + 2 limbs (normalized dividend + sentinel + Svoboda sentinel)
    //  tmp_q: an_real - bn_real + 1 limbs (quotient)
    uint64_t* nb = scratch;
    uint64_t* tmp_a = nb + bn_real;
    uint64_t* tmp_q = tmp_a + an_real + 2;

    size_t nan;
    if (shift > 0) {
      lshift(nb, b, bn_real, shift);
      tmp_a[an_real] = lshift(tmp_a, a, an_real, shift);
      nan = an_real + (tmp_a[an_real] ? 1 : 0);
      tmp_a[nan] = 0;
      div_basecase(tmp_q, tmp_a, nan, nb, bn_real);
    } else {
      // * shift==0: skip nb copy, use b directly
      std::memcpy(tmp_a, a, an_real * sizeof(uint64_t));
      tmp_a[an_real] = 0;
      nan = an_real;
      tmp_a[nan] = 0;
      div_basecase(tmp_q, tmp_a, nan, b, bn_real);
    }

    // Copy the quotient (n limbs, zero-fill the excess)
    size_t qn = nan - bn_real + 1;
    std::memset(q, 0, n * sizeof(uint64_t));
    std::memcpy(q, tmp_q, std::min(qn, n) * sizeof(uint64_t));

    // Denormalize the remainder
    if (shift > 0) {
      for (size_t i = 0; i < bn_real - 1; i++) {
        tmp_a[i] = (tmp_a[i] >> shift) | (tmp_a[i + 1] << (64 - shift));
      }
      tmp_a[bn_real - 1] >>= shift;
    }
    std::memset(r, 0, n * sizeof(uint64_t));
    std::memcpy(r, tmp_a, bn_real * sizeof(uint64_t));
    return;
  }

  // When n is odd: shift left by 1 word to make it even
  // (placing 0 in the upper position would cause B1 to start with zero in the BZ split and break)
  // A/B = (A*2^64)/(B*2^64) - quotient is the same; restore the remainder by 1-word right shift
  if (n & 1) {
    size_t nn = n + 1; // even
    // B' = B * 2^64 = [0, b[0], b[1], ..., b[n-1]] (nn limbs; MSB remains b[n-1])
    uint64_t* bp = scratch;
    bp[0] = 0;
    std::memcpy(bp + 1, b, n * sizeof(uint64_t));
    // A' = A * 2^64 = [0, a[0], a[1], ..., a[2n-1], 0] (2*nn = 2n+2 limbs)
    uint64_t* ap = scratch + nn;
    ap[0] = 0;
    std::memcpy(ap + 1, a, 2 * n * sizeof(uint64_t));
    ap[2 * n + 1] = 0; // top limb
    // Recurse (even version)
    uint64_t* qp = scratch + nn + 2 * nn; // nn limbs
    uint64_t* rp = qp + nn;        // nn limbs
    uint64_t* sub_scratch = rp + nn;
    div_2n_by_n(qp, rp, ap, bp, nn, sub_scratch);
    // Quotient: the low n limbs of qp (the upper qp[n] is usually 0)
    std::memcpy(q, qp, n * sizeof(uint64_t));
    // Remainder: rp is (A mod B)*2^64, so right-shift by 1 word
    std::memcpy(r, rp + 1, n * sizeof(uint64_t));
    return;
  }

  // n is even -> BZ split
  const size_t half = n / 2;
  // A = [A3, A2, A1, A0] each half limbs (little-endian)
  const uint64_t* a0 = a;
  // a1 = a + half (A1 is passed directly to div_3n_by_2n as a + half)
  // (Q1, R1) = div_3n_by_2n([A3, A2, A1], B)
  // [A1, A2, A3] spans 3*half limbs starting at a + half
  uint64_t* q1 = q + half;     // upper half limbs of Q
  uint64_t* r1_tmp = scratch;    // n limbs (remainder)
  uint64_t* sub_scratch1 = scratch + n;

  div_3n_by_2n(q1, r1_tmp, a + half, b, n, sub_scratch1);

  // (Q0, R0) = div_3n_by_2n([R1, A0], B)
  // [A0, R1] = A0 (half limbs) + R1 (n limbs) → 3*half limbs
  uint64_t* combined = scratch + n; // 3*half limbs
  std::memcpy(combined, a0, half * sizeof(uint64_t));
  std::memcpy(combined + half, r1_tmp, n * sizeof(uint64_t));
  uint64_t* q0 = q;         // lower half limbs of Q
  uint64_t* sub_scratch2 = combined + 3 * half;
  div_3n_by_2n(q0, r, combined, b, n, sub_scratch2);
}

// --------------------------------------------------------------------------
// div_unbalanced: handle the an > 2*bn case via chunk-split processing
// --------------------------------------------------------------------------
inline size_t div_unbalanced(uint64_t* q, uint64_t* r,
               const uint64_t* a, size_t an,
               const uint64_t* b, size_t bn,
               uint64_t* scratch) {
  // Process chunks of bn limbs each from the top
  // Each step: divide 2*bn limbs (previous remainder bn limbs + next chunk bn limbs) by bn
  size_t qn = an - bn; // quotient size (max)

  // First chunk: topmost (an mod bn) + bn limbs
  // -> handles the case where an is not a multiple of bn
  size_t first_chunk = an % bn;
  if (first_chunk == 0) first_chunk = bn;
  size_t pos = an; // processing position (top, advances downward)

  // remainder buffer (within scratch): initially the topmost chunk of a
  uint64_t* rem = scratch;     // bn + 1 limbs
  uint64_t* div_scratch = scratch + bn + 1;

  // First chunk: a[pos - first_chunk .. pos - 1]
  pos -= first_chunk;
  std::memcpy(rem, a + pos, first_chunk * sizeof(uint64_t));
  if (first_chunk < bn) {
    std::memset(rem + first_chunk, 0, (bn - first_chunk) * sizeof(uint64_t));
  }

  // If the first chunk is shorter than bn: upper part of the quotient is 0
  if (first_chunk < bn) {
    // The upper first_chunk limbs of a are less than b (bn limbs), so the quotient is 0
    // keep as-is in rem and move to the next chunk
    q[qn] = 0;
  } else {
    // first_chunk == bn: if rem >= b, subtract once and set the topmost quotient bit
    // (b is normalized with MSB set, so rem < 2*b -> at most once)
    if (cmp(rem, bn, b, bn) >= 0) {
      sub(rem, rem, bn, b, bn);
      q[qn] = 1;
    } else {
      q[qn] = 0;
    }
  }

  // Main loop: process chunks downward
  while (pos > 0) {
    size_t chunk = (pos >= bn) ? bn : pos;
    pos -= chunk;

    // Assemble the dividend: [a[pos..pos+chunk-1], rem[0..bn-1]]
    // -> combined[0..chunk+bn-1] = a[pos..] (low) + rem (high)
    uint64_t* combined = div_scratch;
    std::memcpy(combined, a + pos, chunk * sizeof(uint64_t));
    std::memcpy(combined + chunk, rem, bn * sizeof(uint64_t));
    size_t cn = chunk + bn;
    size_t real_cn = normalized_size(combined, cn);

    if (real_cn <= bn) {
      // dividend < divisor -> quotient = 0, remainder = combined
      std::memset(q + pos, 0, chunk * sizeof(uint64_t));
      std::memcpy(rem, combined, bn * sizeof(uint64_t));
    } else {
      // Divide via div_2n_by_n (pad cn to 2*bn)
      uint64_t* padded_a = div_scratch + cn + 1;
      std::memcpy(padded_a, combined, cn * sizeof(uint64_t));
      if (cn < 2 * bn) {
        std::memset(padded_a + cn, 0, (2 * bn - cn) * sizeof(uint64_t));
      }
      uint64_t* temp_q = padded_a + 2 * bn;    // bn limbs
      uint64_t* inner_scratch = temp_q + bn;
      div_2n_by_n(temp_q, rem, padded_a, b, bn, inner_scratch);
      // Copy only the necessary portion of temp_q into q (to prevent buffer overflow)
      size_t q_copy = std::min(chunk, qn + 1 - pos);
      std::memcpy(q + pos, temp_q, q_copy * sizeof(uint64_t));
    }
  }

  // Output the remainder
  std::memcpy(r, rem, bn * sizeof(uint64_t));
  return normalized_size(q, qn + 1);
}

// ============================================================================
// DC Division (Divide-and-Conquer with Preinverted reciprocal)
// ============================================================================
// Equivalent to GMP mpn_dcpi1_div_qr. Same DC principle as BZ, but:
//  1. in-place operation (directly modifies np, no extra copies)
//  2. precomputed 3-by-2 inverse (dinv) shared across all recursion levels
//  3. operates on normalized data (no renormalization needed)
// Eliminates the normalization/copy overhead that BZ incurs at its base case,
// speeding up mid-size division (DC_DIV_QR_THRESHOLD <= dn < MU).

// DC recursion base-case threshold. At or below this, use schoolbook (sbpi1_div_qr).
// 20-30 is a typical optimal value (tune via benchmark later).
constexpr size_t DC_DIV_QR_THRESHOLD = 20;

// --------------------------------------------------------------------------
// sbpi1_div_qr: in-place Schoolbook division (preinverted 3-by-2 reciprocal)
// --------------------------------------------------------------------------
// Divide np[0..nn-1] by dp[0..dn-1].
// Precondition: dp is normalized (MSB of dp[dn-1] is set), dn >= 2
//    Requires np[qn..nn-1] < dp[0..dn-1] (qh==0 case)
//    np needs nn+1 limbs of space (for the sentinel)
// dinv = invert_pi1(dp[dn-1], dp[dn-2])
// result: qp[0..qn-1] = quotient (qn = nn - dn), np[0..dn-1] = remainder
// Return value: topmost quotient limb (0 or 1)
inline uint64_t sbpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn,
               const uint64_t* dp, size_t dn, uint64_t dinv) {
  size_t qn = nn - dn;

  // Topmost quotient: if np[qn..nn-1] >= dp, subtract once and set qh=1
  uint64_t qh = 0;
  if (cmp(np + qn, dn, dp, dn) >= 0) {
    qh = 1;
    sub(np + qn, np + qn, dn, dp, dn);
  }

  const uint64_t d1 = dp[dn - 1];
  const uint64_t d0 = dp[dn - 2];

  if (dn == 2) {
    // divrem_2 path: completes via 3-by-2 only (no submul needed)
    uint64_t n1 = np[qn + 1];
    uint64_t n0 = np[qn];

    for (size_t j = qn; j-- > 0; ) {
      uint64_t q_hat;
      udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0, np[j], d1, d0, dinv);
      qp[j] = q_hat;
    }
    np[0] = n0;
    np[1] = n1;
  } else {
    // Möller-Granlund 3-by-2 main loop
#ifdef SANGI_INT_HAS_ASM
    if (detail::has_bmi2_adx()) {
      mpn_sbpi1_div_qr_asm(qp, np, qn, dp, dn, dinv);
      return qh;
    }
#endif
    for (size_t i = qn; i-- > 0; ) {
      uint64_t q_hat;
      uint64_t n1 = np[i + dn];
      uint64_t n0 = np[i + dn - 1];

      if (n1 == d1 && n0 == d0) {
        q_hat = UINT64_MAX;
        uint64_t borrow = submul_1(np + i, dp, dn, q_hat);
        uint64_t prev = np[i + dn];
        np[i + dn] = prev - borrow;
        if (prev < borrow) {
          --q_hat;
          add(np + i, np + i, dn + 1, dp, dn);
        }
      } else {
        udiv_qrnnd_3by2(q_hat, n1, n0, n1, n0,
                np[i + dn - 2], d1, d0, dinv);

        uint64_t cy = submul_1(np + i, dp, dn - 2, q_hat);

        uint64_t cy0 = (n0 < cy) ? 1 : 0;
        n0 -= cy;
        uint64_t cy1 = (n1 < cy0) ? 1 : 0;
        n1 -= cy0;

        np[i + dn - 2] = n0;

        if (cy1) {
          n1 += d1 + add(np + i, np + i, dn - 1, dp, dn - 1);
          --q_hat;
        }
        np[i + dn - 1] = n1;
        np[i + dn] = 0;
      }

      qp[i] = q_hat;
    }
  }

  return qh;
}

// sbpi1_div_q: schoolbook quotient-only version (with divisor truncation)
// Follows GMP mpn_sbpi1_div_q.
// Instead of computing the remainder, gradually reduce the divisor in the latter half of the loop.
// Reduces total submul work by ~25%.
inline uint64_t sbpi1_div_q(uint64_t* qp, uint64_t* np, size_t nn,
               const uint64_t* dp, size_t dn, uint64_t dinv) {
  size_t qn = nn - dn;
  // When qn + 1 < dn, use only the upper (qn+1) limbs of the divisor
  if (qn + 1 < dn) {
    dp += dn - (qn + 1);
    dn = qn + 1;
  }

  np += nn;
  uint64_t qh = (cmp(np - dn, dn, dp, dn) >= 0) ? 1 : 0;
  if (qh)
    sub(np - dn, np - dn, dn, dp, dn);

  qp += qn;

  const uint64_t d1 = dp[dn - 1];
  const uint64_t d0 = dp[dn - 2];
  size_t dn2 = dn - 2; // submul size (dn minus 2 for d1,d0)

  np -= 2;
  uint64_t n1 = np[1];

  // Phase 1: normal loop (qn - dn limbs)
  // Execute submul over all dn limbs
  for (size_t i = qn; i > dn2 + 2; i--) {
    np--;
    uint64_t q_hat;
    if (n1 == d1 && np[1] == d0) {
      q_hat = UINT64_MAX;
      submul_1(np - dn2, dp, dn2 + 2, q_hat);
      n1 = np[1];
    } else {
      uint64_t n0;
      udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv);
      uint64_t cy = submul_1(np - dn2, dp, dn2, q_hat);
      uint64_t cy0 = (n0 < cy) ? 1 : 0;
      n0 -= cy;
      uint64_t cy1 = (n1 < cy0) ? 1 : 0;
      n1 -= cy0;
      np[0] = n0;
      if (cy1) {
        n1 += d1 + add(np - dn2, np - dn2, dn2 + 1, dp, dn2 + 1);
        --q_hat;
      }
    }
    *--qp = q_hat;
  }

  // Phase 2: truncation loop (remaining dn limbs)
  // In each iteration, dn2-- and dp++ to reduce the submul size
  uint64_t flag = ~uint64_t(0);
  while (dn2 > 0) {
    np--;
    uint64_t q_hat;
    if (n1 >= (d1 & flag)) {
      q_hat = UINT64_MAX;
      uint64_t cy = submul_1(np - dn2, dp, dn2 + 2, q_hat);
      if (n1 != cy) {
        if (n1 < (cy & flag)) {
          q_hat--;
          add(np - dn2, np - dn2, dn2 + 2, dp, dn2 + 2);
        } else {
          flag = 0;
        }
      }
      n1 = np[1];
    } else {
      uint64_t n0;
      udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv);
      uint64_t cy = submul_1(np - dn2, dp, dn2, q_hat);
      uint64_t cy0 = (n0 < cy) ? 1 : 0;
      n0 -= cy;
      uint64_t cy1 = (n1 < cy0) ? 1 : 0;
      n1 -= cy0;
      np[0] = n0;
      if (cy1) {
        n1 += d1 + add(np - dn2, np - dn2, dn2 + 1, dp, dn2 + 1);
        q_hat--;
      }
    }
    *--qp = q_hat;
    dn2--;
    dp++;
  }

  // last 1 limb: dn2 == 0, submul not neededed
  np--;
  {
    uint64_t q_hat;
    if (n1 >= (d1 & flag)) {
      q_hat = UINT64_MAX;
      uint64_t cy = submul_1(np, dp, 2, q_hat);
      if (n1 != cy) {
        if (n1 < (cy & flag)) {
          q_hat--;
          add(np, np, 2, dp, 2);
        } else {
          flag = 0;
        }
      }
      n1 = np[1];
    } else {
      uint64_t n0;
      udiv_qrnnd_3by2(q_hat, n1, n0, n1, np[1], np[0], d1, d0, dinv);
      np[0] = n0;
    }
    *--qp = q_hat;
  }

  return qh;
}

// Forward declaration
inline uint64_t dcpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn,
               const uint64_t* dp, size_t dn,
               uint64_t dinv, uint64_t* scratch);
inline uint64_t dcpi1_div_qr_n(uint64_t* qp, uint64_t* np,
                const uint64_t* dp, size_t n,
                uint64_t dinv, uint64_t* scratch);
inline uint64_t sbpi1_divappr_q(uint64_t* qp, uint64_t* np, size_t nn,
                 const uint64_t* dp, size_t dn, uint64_t dinv);

// --------------------------------------------------------------------------
// dcpi1_divappr_q_n: DC approximate quotient (2n by n, +/-1 error in the lowest quotient limb)
// --------------------------------------------------------------------------
// Follows GMP mpn_dcpi1_divappr_q_n.
// Upper half: div_qr (exact), lower half: divappr (approximate, skips remainder computation)
// Completely removes the remainder-computation cost in the lower half of DC division.
constexpr size_t DC_DIVAPPR_Q_THRESHOLD = DC_DIV_QR_THRESHOLD; // Reuse the same threshold

inline uint64_t dcpi1_divappr_q_n(uint64_t* qp, uint64_t* np,
                  const uint64_t* dp, size_t n,
                  uint64_t dinv, uint64_t* scratch) {
  if (n < DC_DIV_QR_THRESHOLD) {
    return sbpi1_div_qr(qp, np, 2 * n, dp, n, dinv);
  }

  const size_t lo = n >> 1;
  const size_t hi = n - lo;

  // Step 1: upper quotient (exact div_qr)
  uint64_t qh;
  if (hi < DC_DIV_QR_THRESHOLD)
    qh = sbpi1_div_qr(qp + lo, np + 2 * lo, 2 * hi, dp + lo, hi, dinv);
  else
    qh = dcpi1_div_qr_n(qp + lo, np + 2 * lo, dp + lo, hi, dinv, scratch);

  // Step 2: subtract Q_high * D_low from the remainder
  multiply(scratch, qp + lo, hi, dp, lo, scratch + n);
  uint64_t cy = sub(np + lo, np + lo, n, scratch, n);
  if (qh)
    cy += sub(np + n, np + n, lo, dp, lo);
  while (cy) {
    qh -= sub_1(qp + lo, hi, 1);
    cy -= add(np + lo, np + lo, n, dp, n);
  }

  // Step 3: lower quotient (approximate - divappr, skips remainder computation)
  uint64_t ql;
  if (lo < DC_DIVAPPR_Q_THRESHOLD)
    ql = sbpi1_divappr_q(qp, np + hi, 2 * lo, dp + hi, lo, dinv);
  else
    ql = dcpi1_divappr_q_n(qp, np + hi, dp + hi, lo, dinv, scratch);

  if (ql) {
    // ql != 0 -> quotient overflows -> set all bits to 1
    for (size_t i = 0; i < lo; i++) qp[i] = UINT64_MAX;
  }

  return qh;
}

// sbpi1_divappr_q: schoolbook approximate quotient (with divisor truncation)
// Same as sbpi1_div_q, but does not guarantee precision in the lowest 1 limb.
// (Implementation is identical to sbpi1_div_q - divisor truncation produces the approximation automatically)
inline uint64_t sbpi1_divappr_q(uint64_t* qp, uint64_t* np, size_t nn,
                 const uint64_t* dp, size_t dn, uint64_t dinv) {
  return sbpi1_div_q(qp, np, nn, dp, dn, dinv);
}

// --------------------------------------------------------------------------
// dcpi1_div_qr_n: balanced DC division (2n by n)
// --------------------------------------------------------------------------
// Divide np[0..2n-1] by dp[0..n-1] (in-place).
// Precondition: dp normalized; np[n..2n-1] < dp[0..n-1] is not guaranteed (adjusted internally)
// dinv = invert_pi1(dp[n-1], dp[n-2])
// result: qp[0..n-1] = quotient, np[0..n-1] = remainder
// scratch: dcpi1_div_qr_n_scratch_size(n) limbs or more
// Return value: topmost quotient limb (0 or 1)
inline uint64_t dcpi1_div_qr_n(uint64_t* qp, uint64_t* np,
                const uint64_t* dp, size_t n,
                uint64_t dinv, uint64_t* scratch) {
  // Base case: schoolbook
  if (n < DC_DIV_QR_THRESHOLD) {
    return sbpi1_div_qr(qp, np, 2 * n, dp, n, dinv);
  }

  const size_t lo = n >> 1;    // floor(n/2)
  const size_t hi = n - lo;    // ceil(n/2), hi >= lo

  // np[n..2n-1] >= dp -> subtract once and set qh=1
  uint64_t qh = 0;
  if (cmp(np + n, n, dp, n) >= 0) {
    qh = 1;
    sub(np + n, np + n, n, dp, n);
  }

  // ── Step 1: upper quotient ──
  // Divide np[2*lo..2*n-1] (2*hi limbs) by dp[lo..n-1] (hi limbs)
  uint64_t cy1;
  if (hi < DC_DIV_QR_THRESHOLD) {
    cy1 = sbpi1_div_qr(qp + lo, np + 2 * lo, 2 * hi, dp + lo, hi, dinv);
  } else {
    cy1 = dcpi1_div_qr_n(qp + lo, np + 2 * lo, dp + lo, hi, dinv, scratch);
  }
  // qp[lo..lo+hi-1] = upper quotient, np[2*lo..2*lo+hi-1] = remainder

  // BUGFIX (cas-c5cc deep #2): cy1 is the upper-quotient's overflow limb, which sits
  // at position lo+hi == n -- i.e. the qh (top quotient) slot -- so it must be folded
  // into qh.  Previously it was only handled on the remainder side (the cy1 sub below);
  // its quotient contribution was dropped.  As a result, when cy1 == 1 and the Step-2
  // correction loop borrowed past the top of the upper quotient, `qh -= b` drove qh
  // from 0 to (uint64_t)-1.  That bogus -1 was then returned and consumed by the parent
  // as cy1/cy3, where `if (cy3)` treated it as a true flag and performed a spurious
  // correction -> quotient 1 too small (silent-wrong on balanced power-of-10 divisions,
  // e.g. (10^3046+7)/(10^1523+3)).  Folding cy1 into qh keeps qh in {0,1}.
  // (cy3, the lower-quotient overflow, is always an over-estimate that the Step-4
  //  correction removes, so it needs no analogous quotient carry -- verified by fuzz.)
  qh += cy1;

  // ── Step 2: correct the lower part ──
  // Subtract (Q_H + cy1*B^hi) * D_low from np[lo..]
  // GMP style: perform the product subtraction first, then add the cy1 borrow.
  // Subtracting cy1 first introduced a bug where the borrow propagated into a zero region and was lost.
  {
    uint64_t* tp = scratch;

    // qp[lo..lo+hi-1] × dp[0..lo-1] → tp[0..n-1]
    multiply(tp, qp + lo, hi, dp, lo, tp + n);

    // Subtract product from np[lo..lo+n-1]
    uint64_t borrow = sub(np + lo, np + lo, n, tp, n);

    // Handle cy1: subtract dp[0..lo-1] from np[n..n+lo-1]
    if (cy1) {
      borrow += sub(np + n, np + n, lo, dp, lo);
    }

    // Correction loop: on borrow, add dp back and decrement the quotient
    while (borrow) {
      borrow -= add(np + lo, np + lo, n, dp, n);
      uint64_t b = 1;
      for (size_t k = lo; k < lo + hi && b; k++) {
        uint64_t old = qp[k];
        qp[k] = old - b;
        b = (qp[k] > old) ? 1 : 0;
      }
      qh -= b;
    }
  }

  // ── Step 3: lower quotient ──
  // Divide np[lo..lo+n-1] (= lo+hi limbs = n limbs) by dp[lo..n-1] (hi limbs)
  // -> qp[0..lo-1] = lower quotient (lo limbs), np[lo..lo+hi-1] = remainder
  // Note: ASM sbpi1 also writes to qp[qn] (off-by-one design).
  // Since qn = hi = lo in Step 3, qp[lo] (Step 1's upper quotient) gets overwritten.
  // → Save and restore qp[lo] to protect it.
  uint64_t cy3;
  {
    uint64_t saved_qp_lo = qp[lo]; // Save Step 1's lowest upper-quotient limb
    if (lo == hi) {
      // n is even: lo == hi -> balanced (2*hi by hi) -> DC recursion
      if (hi < DC_DIV_QR_THRESHOLD) {
        cy3 = sbpi1_div_qr(qp, np + lo, 2 * hi, dp + lo, hi, dinv);
      } else {
        cy3 = dcpi1_div_qr_n(qp, np + lo, dp + lo, hi, dinv, scratch);
      }
    } else {
      // n is odd: lo < hi -> (lo+hi) by hi is non-balanced -> schoolbook
      cy3 = sbpi1_div_qr(qp, np + lo, lo + hi, dp + lo, hi, dinv);
    }
    qp[lo] = saved_qp_lo; // Restore the upper quotient
  }
  // qp[0..lo-1] = lower quotient, np[lo..lo+hi-1] = remainder

  // ── Step 4: correct the lower part (again) ──
  // GMP style: perform the product subtraction first, then add the cy3 borrow.
  {
    uint64_t* tp = scratch;

    // qp[0..lo-1] × dp[0..lo-1] → tp[0..2*lo-1]
    multiply(tp, qp, lo, dp, lo, tp + 2 * lo);

    // Subtract product from np[0..2*lo-1]
    uint64_t borrow = sub(np, np, 2 * lo, tp, 2 * lo);

    // When hi > lo, propagate borrow to np[2*lo..n-1]
    if (borrow && 2 * lo < n) {
      for (size_t k = 2 * lo; k < n && borrow; k++) {
        uint64_t old = np[k];
        np[k] = old - borrow;
        borrow = (np[k] > old) ? 1 : 0;
      }
    }

    // Handle cy3: subtract dp[0..lo-1] from np[lo..lo+lo-1] = np[lo..2*lo-1]
    if (cy3) {
      borrow += sub(np + lo, np + lo, lo, dp, lo);
    }

    // Correction: on borrow, add dp back and decrement the quotient
    while (borrow) {
      borrow -= add(np, np, n, dp, n);
      uint64_t b = 1;
      for (size_t k = 0; k < lo && b; k++) {
        uint64_t old = qp[k];
        qp[k] = old - b;
        b = (qp[k] > old) ? 1 : 0;
      }
    }
  }

  return qh;
}

// --------------------------------------------------------------------------
// dcpi1_div_qr: general DC division (nn by dn)
// --------------------------------------------------------------------------
// Divide np[0..nn-1] by dp[0..dn-1] (in-place).
// Precondition: dp normalized, nn >= dn >= DC_DIV_QR_THRESHOLD, dn >= 2
// dinv = invert_pi1(dp[dn-1], dp[dn-2])
// result: qp[0..nn-dn-1] = quotient, np[0..dn-1] = remainder
// scratch: dcpi1_div_qr_scratch_size(nn, dn) limbs or more
// Return value: topmost quotient limb (0 or 1)
inline uint64_t dcpi1_div_qr(uint64_t* qp, uint64_t* np, size_t nn,
               const uint64_t* dp, size_t dn,
               uint64_t dinv, uint64_t* scratch) {
  size_t qn = nn - dn;

  // Base case
  if (qn + 1 < DC_DIV_QR_THRESHOLD) {
    return sbpi1_div_qr(qp, np, nn, dp, dn, dinv);
  }

  // Unbalanced: qn > dn -> process from the top using chunks of size dn
  if (qn > dn) {
    // First chunk: take 2*dn limbs from the top of nn
    uint64_t qh = 0;

    // Topmost excess (nn mod dn)
    size_t first_qn = qn % dn;
    if (first_qn == 0) first_qn = dn;

    // First chunk: divide np[nn-chunk_nn..nn-1] (first_qn + dn limbs) by dp (dn limbs)
    // Quotient (first_qn limbs) goes at qp[qn - first_qn .. qn-1], identical for the
    // sbpi1 base case and the dcpi1 recursion (both have the same qp/np contract).
    size_t chunk_nn = first_qn + dn;
    if (first_qn + 1 < DC_DIV_QR_THRESHOLD) {
      // BUGFIX (cas-c5cc): the qp offset must be qp + qn - first_qn, matching the
      // else branch. The previous expression added an extra +dn (it reduced to
      // qp + dn + qn - first_qn), writing first_qn quotient limbs at indices
      // [qn-first_qn+dn .. qn+dn-1] and overrunning the qp buffer (size qn+1) by
      // up to dn-1 limbs -> intermittent heap-buffer-overflow on large divisions.
      qh = sbpi1_div_qr(qp + qn - first_qn,
                np + nn - chunk_nn, chunk_nn, dp, dn, dinv);
    } else {
      // Divide np[nn - chunk_nn .. nn-1] by dp
      // Quotient at qp[qn - first_qn .. qn-1] (first_qn limbs)
      qh = dcpi1_div_qr(qp + qn - first_qn, np + nn - chunk_nn,
                chunk_nn, dp, dn, dinv, scratch);
    }
    // np[nn-chunk_nn..nn-chunk_nn+dn-1] = remainder

    // Remaining chunks: process in dn-limb steps
    size_t q_pos = qn - first_qn;
    size_t n_pos = nn - chunk_nn;

    while (q_pos >= dn) {
      q_pos -= dn;
      n_pos -= dn;
      // Divide np[n_pos..n_pos+2*dn-1] by dp (dn limbs)
      // Upper dn limbs are the previous remainder (np[n_pos+dn..n_pos+2*dn-1])
      // Lower dn limbs are np[n_pos..n_pos+dn-1]
      uint64_t qh2 = dcpi1_div_qr_n(qp + q_pos, np + n_pos, dp, dn,
                      dinv, scratch);
      // BUGFIX (cas-c5cc): qp[q_pos + dn] already holds a valid quotient limb of
      // the block above (the first chunk wrote qp[qn-first_qn..qn-1], whose lowest
      // limb is qp[q_pos+dn] on the first iteration). The 2*dn-by-dn block has its
      // high dn limbs equal to the previous remainder, which is < dp, so the block
      // quotient fits in exactly dn limbs and the carry qh2 is always 0. The old
      // `qp[q_pos + dn] = qh2;` therefore clobbered a real quotient limb with 0,
      // producing wrong quotients for unbalanced large divisions. Propagate the
      // carry by addition instead (a no-op while qh2 == 0, matching GMP, which does
      // not store it at all).
      if (qh2) {
        uint64_t c = qh2;
        for (size_t k = q_pos + dn; c != 0; ++k) {
          uint64_t old = qp[k];
          qp[k] = old + c;
          c = (qp[k] < old) ? 1 : 0;
        }
      }
    }

    // Remaining: q_pos < dn limbs of the quotient still missing -> partial division
    if (q_pos > 0) {
      // Divide np[0..q_pos+dn-1] (q_pos+dn limbs) by dp (dn limbs)
      if (q_pos + 1 < DC_DIV_QR_THRESHOLD) {
        uint64_t qh3 = sbpi1_div_qr(qp, np, q_pos + dn, dp, dn, dinv);
        qp[q_pos] = qh3;
      } else {
        uint64_t qh3 = dcpi1_div_qr(qp, np, q_pos + dn, dp, dn,
                       dinv, scratch);
        qp[q_pos] = qh3;
      }
    }

    return qh;
  }

  // Balanced or nearly balanced: qn <= dn
  if (qn < dn) {
    // Partial division: divide using the upper (qn+1) limbs of dp and correct
    // Divide the upper part of np by dp[dn-qn-1..dn-1] (qn+1 limbs)
    size_t partial_dn = qn + 1;
    uint64_t qh;

    if (partial_dn < DC_DIV_QR_THRESHOLD) {
      qh = sbpi1_div_qr(qp, np + dn - partial_dn,
                nn - (dn - partial_dn), dp + dn - partial_dn,
                partial_dn, dinv);
    } else if (qn + 1 <= partial_dn) {
      qh = dcpi1_div_qr_n(qp, np + dn - partial_dn,
                dp + dn - partial_dn, partial_dn,
                dinv, scratch);
    } else {
      qh = dcpi1_div_qr(qp, np + dn - partial_dn,
                nn - (dn - partial_dn), dp + dn - partial_dn,
                partial_dn, dinv, scratch);
    }

    // Correct using the lower dp limbs: q * dp[0..dn-partial_dn-1]
    if (dn - partial_dn > 0) {
      uint64_t* tp = scratch;
      size_t lo_dn = dn - partial_dn;
      multiply(tp, qp, qn, dp, lo_dn, tp + qn + lo_dn);

      if (qh) {
        add(tp + qn, tp + qn, lo_dn, dp, lo_dn);
      }

      uint64_t borrow = sub(np, np, qn + lo_dn, tp, qn + lo_dn);

      // Propagate borrow to the upper part
      if (borrow) {
        for (size_t k = qn + lo_dn; k < dn && borrow; k++) {
          uint64_t old = np[k];
          np[k] = old - borrow;
          borrow = (np[k] > old) ? 1 : 0;
        }
      }

      // Correction: remainder < 0
      while (borrow) {
        add(np, np, dn, dp, dn);
        uint64_t b = 1;
        for (size_t k = 0; k < qn && b; k++) {
          uint64_t old = qp[k];
          qp[k] = old - b;
          b = (qp[k] > old) ? 1 : 0;
        }
        qh -= b;
        borrow = 0; // at most once
      }
    }

    return qh;
  }

  // qn == dn → balanced
  return dcpi1_div_qr_n(qp, np, dp, dn, dinv, scratch);
}

// DC division for scratch size
inline size_t dcpi1_div_qr_scratch_size(size_t nn, size_t dn) {
  // Each level needs multiplication scratch + product buffer
  // max: n limbs (product) + multiply_scratch_size(n, n)
  // Halves with recursion, so geometric series sum ~ 2n
  // Use multiply_rec_scratch_size to guard against sub-mult shape falling outside the FFT range
  size_t n = std::max(nn - dn, dn);
  return 2 * n + multiply_rec_scratch_size(n, n) + 64; // leave some margin
}

// --------------------------------------------------------------------------
// mulmod_bnm1: CRT-based cyclic multiplication (mod beta^n - 1)
// --------------------------------------------------------------------------
// Computes (A * B) mod (beta^n - 1). Result is n limbs.
// Since beta^n == 1 (mod beta^n - 1), limbs at positions >= n wrap back to the lower part.
//
// Use beta^n - 1 = (beta^h - 1)(beta^h + 1) (h = n/2) for CRT decomposition:
//  xm = A*B mod (beta^h - 1) - recursive call
//  xp = A*B mod (beta^h + 1) - negacyclic multiplication
//  CRT composition: cost ~ 2*M(h) ~ M(n)/2
//
// Follows GMP mpn_mulmod_bnm1.

constexpr size_t MULMOD_BNM1_THRESHOLD = 16;
constexpr size_t MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD = 20;

// Round n up to a CRT-friendly (even) size
inline size_t mulmod_bnm1_next_size(size_t n) {
  if (n < MULMOD_BNM1_THRESHOLD) return n;
  if (n < 4 * (MULMOD_BNM1_THRESHOLD - 1) + 1) return (n + 1) & ~size_t(1);
  if (n < 8 * (MULMOD_BNM1_THRESHOLD - 1) + 1) return (n + 3) & ~size_t(3);
  return (n + 7) & ~size_t(7);
}

inline size_t mulmod_bnm1_scratch_size(size_t rn, size_t an, size_t bn) {
  // base case: full product + multiply workspace
  size_t bc = (an + bn) + multiply_scratch_size(an, bn);
  if ((rn & 1) != 0 || rn < MULMOD_BNM1_THRESHOLD) return bc;
  // CRT recursion: at each level fixed 5h+3 + max(bnp1, recursive)
  // bnp1 scratch = 2*h + multiply_scratch_size(h, h)
  // conservativeupper bound: 8*rn + multiply_rec_scratch_size(rn/2, rn/2) + 16
  // (avoid FFT early-zero)
  size_t crt = 8 * rn + multiply_rec_scratch_size(rn / 2, rn / 2) + 16;
  return std::max(bc, crt);
}

// neg: r = 0 - a (mod beta^n). Return value: 1 if a != 0 (borrow)
inline uint64_t neg(uint64_t* r, const uint64_t* a, size_t n) {
  size_t i = 0;
  while (i < n && a[i] == 0) { r[i] = 0; i++; }
  if (i == n) return 0;
  r[i] = ~a[i] + 1; // = -a[i] (uint64 wrap)
  i++;
  for (; i < n; i++) r[i] = ~a[i];
  return 1;
}

// mulmod_bnp1_bc: negacyclic multiplication (base case)
// r[0..n] = (a[0..n] × b[0..n]) mod (β^n + 1)
// a[n], b[n] ∈ {0, 1}. r[n] ∈ {0, 1}.
// scratch: 2*n + multiply_scratch_size(n, n) limbs
inline void mulmod_bnp1_bc(uint64_t* r, const uint64_t* a,
              const uint64_t* b, size_t n,
              uint64_t* scratch) {
  uint64_t cy;
  if (a[n] | b[n]) {
    // a[n]=1 → a mod (β^n+1) = -1, product = -b (or +1 if both high)
    if (a[n])
      cy = b[n] + neg(r, b, n);
    else
      cy = neg(r, a, n);
  } else {
    // full multiplication -> negacyclic fold
    uint64_t* tp = scratch;
    uint64_t* mw = scratch + 2 * n;
    std::memset(tp, 0, 2 * n * sizeof(uint64_t));
    multiply(tp, a, n, b, n, mw);
    cy = sub(r, tp, n, tp + n, n);
  }
  r[n] = 0;
  if (cy) add_1(r, n + 1, cy);
}

// base case mulmod_bnm1: full multiplication + fold
inline void mulmod_bnm1_bc(uint64_t* r, const uint64_t* a, size_t an,
              const uint64_t* b, size_t bn, size_t rn,
              uint64_t* scratch) {
  size_t pn = an + bn;
  uint64_t* prod = scratch;
  uint64_t* mul_work = scratch + pn;
  std::memset(prod, 0, pn * sizeof(uint64_t));
  multiply(prod, a, an, b, bn, mul_work);
  if (pn <= rn) {
    std::memcpy(r, prod, pn * sizeof(uint64_t));
    if (pn < rn) std::memset(r + pn, 0, (rn - pn) * sizeof(uint64_t));
  } else {
    std::memcpy(r, prod, rn * sizeof(uint64_t));
    size_t high_n = pn - rn;
    uint64_t carry = add(r, r, rn, prod + rn, high_n);
    while (carry > 0) carry = add_1(r, rn, carry);
  }
}

// r[0..rn-1] = (a[0..an-1] × b[0..bn-1]) mod (β^rn - 1)
// Precondition: 0 < bn <= an <= rn, an + bn > rn/2
// r must not overlap a or b
// scratch: mulmod_bnm1_scratch_size(rn, an, bn) limbs
inline void mulmod_bnm1(uint64_t* r, size_t rn,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  // base case: rn is odd or small
  if ((rn & 1) != 0 || rn < MULMOD_BNM1_THRESHOLD) {
    if (an + bn <= rn) {
      // If the product fits in rn: full multiplication suffices
      uint64_t* prod = scratch;
      uint64_t* mw = scratch + an + bn;
      std::memset(prod, 0, (an + bn) * sizeof(uint64_t));
      multiply(prod, a, an, b, bn, mw);
      std::memcpy(r, prod, (an + bn) * sizeof(uint64_t));
      if (an + bn < rn) std::memset(r + an + bn, 0, (rn - an - bn) * sizeof(uint64_t));
    } else {
      mulmod_bnm1_bc(r, a, an, b, bn, rn, scratch);
    }
    return;
  }

  // CRT decomposition: rn = 2*h
  size_t h = rn >> 1;
  uint64_t cy;

  // scratch layout (follows GMP):
  //  xp[0..2h+1]: mod (β^h+1) product + tempbuffer
  //  sp1[0..]: storage for ap1/bp1 (after xp)
  uint64_t* xp = scratch;
  uint64_t* sp1 = scratch + 2 * h + 2;

#define a0 a
#define a1 (a + h)
#define b0 b
#define b1 (b + h)

  // ====== mod (β^h - 1) side: xm = A*B mod (β^h - 1) ======
  {
    const uint64_t* am1;
    const uint64_t* bm1;
    size_t anm, bnm;
    uint64_t* so;

    bm1 = b0;
    bnm = bn;
    if (an > h) {
      // am1 = a0 + a1 mod (β^h - 1)
      am1 = xp;
      cy = add(xp, a0, h, a1, an - h);
      if (cy) add_1(xp, h, cy);
      anm = h;
      so = xp + h;
      if (bn > h) {
        // bm1 = b0 + b1 mod (β^h - 1)
        bm1 = so;
        cy = add(so, b0, h, b1, bn - h);
        if (cy) add_1(so, h, cy);
        bnm = h;
        so += h;
      }
    } else {
      so = xp;
      am1 = a0;
      anm = an;
    }

    // Recursive call: output the result into r[0..h-1]
    mulmod_bnm1(r, h, am1, anm, bm1, bnm, so);
  }

  // ====== mod (β^h + 1) side: xp = A*B mod (β^h + 1) ======
  {
    const uint64_t* ap1;
    const uint64_t* bp1;
    size_t anp, bnp;

    bp1 = b0;
    bnp = bn;
    if (an > h) {
      ap1 = sp1;
      cy = sub(sp1, a0, h, a1, an - h);
      sp1[h] = 0;
      if (cy) add_1(sp1, h + 1, cy);
      anp = h + ap1[h];
      if (bn > h) {
        bp1 = sp1 + h + 1;
        cy = sub(sp1 + h + 1, b0, h, b1, bn - h);
        sp1[2 * h + 1] = 0;
        if (cy) add_1(sp1 + h + 1, h + 1, cy);
        bnp = h + bp1[h];
      }
    } else {
      ap1 = a0;
      anp = an;
    }

    // mulmod_bnp1: negacyclic multiplication
    // Input is (h+1) limbs (high 0 or 1); output is xp[0..h]
    // bnp1_scratch is placed after sp1 (sp1 + 2*(h+1))
    uint64_t* bnp1_scratch = sp1 + 2 * (h + 1);
    if (bp1 == b0) {
      // bp1 is short (no folding): full multiplication + negacyclic fold
      // ap1 may also be short (when an <= h)
      size_t a1n = std::min(anp, h); // exclude high bit
      size_t b1n = std::min(bnp, h);
      uint64_t* tp2 = bnp1_scratch;
      size_t pn2 = a1n + b1n;
      uint64_t* mw = tp2 + pn2;
      std::memset(tp2, 0, pn2 * sizeof(uint64_t));
      if (a1n >= b1n)
        multiply(tp2, ap1, a1n, bp1, b1n, mw);
      else
        multiply(tp2, bp1, b1n, ap1, a1n, mw);
      // negacyclic fold
      if (pn2 > h) {
        cy = sub(xp, tp2, h, tp2 + h, pn2 - h);
      } else {
        std::memcpy(xp, tp2, pn2 * sizeof(uint64_t));
        if (pn2 < h) std::memset(xp + pn2, 0, (h - pn2) * sizeof(uint64_t));
        cy = 0;
      }
      xp[h] = 0;
      if (cy) add_1(xp, h + 1, cy);
    } else {
      // both (h+1) limbs: mulmod_bnp1_bc
      mulmod_bnp1_bc(xp, ap1, bp1, h, bnp1_scratch);
    }
  }

  // ====== CRT composition ======
  // xm = r[0..h-1] (mod β^h - 1)
  // xp = xp[0..h] (mod beta^h + 1, normalization: xp[h] in {0,1})
  //
  // CRT: x = -xp * β^h + (β^h + 1) * [(xp + xm)/2 mod (β^h - 1)]
  //
  // Step 1: r[0..h-1] = (xm + xp) / 2 mod (β^h - 1)
  cy = xp[h] + add(r, r, h, xp, h);
  cy += (r[0] & 1);
  // 1-bit right shift (fixed size h, carry-in = cy)
#ifdef SANGI_INT_HAS_ASM
  mpn_rshift_asm(r, r, h, 1);
#else
  for (size_t i = 0; i < h - 1; i++)
    r[i] = (r[i] >> 1) | (r[i + 1] << 63);
  r[h - 1] = r[h - 1] >> 1;
#endif
  uint64_t hi = (cy & 1) << 63;
  cy >>= 1;
  r[h - 1] |= hi;
  if (cy) add_1(r, h, cy);

  // Step 2: r[h..2h-1] = r[0..h-1] - xp[0..h-1]
  cy = xp[h] + sub(r + h, r, h, xp, h);
  if (cy) sub_1(r, 2 * h, cy);

#undef a0
#undef a1
#undef b0
#undef b1
}

// --------------------------------------------------------------------------
// fold_mod_bnm1: fold an array to n limbs (mod beta^n - 1)
// --------------------------------------------------------------------------
// r[0..n-1] = a[0..an-1] mod (β^n - 1)
// r may equal a (in-place). Precondition: an <= 2*n.
inline void fold_mod_bnm1(uint64_t* r, size_t n,
              const uint64_t* a, size_t an) {
  if (an <= n) {
    if (r != a) std::memcpy(r, a, an * sizeof(uint64_t));
    if (an < n) std::memset(r + an, 0, (n - an) * sizeof(uint64_t));
    return;
  }
  if (r != a) std::memcpy(r, a, n * sizeof(uint64_t));
  size_t high_n = an - n;
  uint64_t carry = add(r, r, n, a + n, high_n);
  while (carry > 0) {
    carry = add_1(r, n, carry);
  }
}

// --------------------------------------------------------------------------
// mu_div_qr: O(M(n)) division using Newton inverse iteration
// --------------------------------------------------------------------------
// Follows GMP mpn_mu_div_qr / mpn_preinv_mu_div_qr.
// Computes the inverse at size in ~ qn/2 (balanced) and processes the quotient in chunks.
// In each chunk, estimate in-limb quotient, subtract Q*D, and update the remainder.
//
// Precondition:
//  - bn >= 2, MSB of b[bn-1] is set (normalization)
//  - an >= bn
//  - a[] is overwritten as work space (the remainder is left in a[0..bn-1])
// result:
//  - q[0..an-bn] holds the quotient
//  - a[0..bn-1] holds the remainder (the caller copies it to r[])
// Return value: normalized size of the quotient
// scratch: mu_div_qr_scratch_size(an, bn) limbs

// Inverse-size selection (follows GMP mpn_mu_div_qr_choose_in)
// NTT-friendly adjustment: tune in so that the internal multiplication of invert_approx(in+1) does not cross a power-of-2 boundary.
// Minor adjustment of in. Optimal when h = ceil((in+1)/2) is a power of 2.
inline size_t mu_div_qr_choose_in(size_t qn, size_t bn) {
  size_t in;
  if (qn > bn) {
    size_t blocks = (qn - 1) / bn + 1;
    in = (qn - 1) / blocks + 1;
  } else if (3 * qn > bn) {
    in = (qn - 1) / 2 + 1;
  } else {
    in = qn;
  }

  // NTT-friendly adjustment has limited effect, so it was removed.
  // A fundamental solution requires a mixed-radix NTT implementation.

  return in;
}

inline size_t mu_div_qr_scratch_size(size_t an, size_t bn) {
  size_t qn = (an > bn) ? an - bn : 1;
  size_t in = mu_div_qr_choose_in(qn, bn);

  // scratch layout: ip[0..in-1] + work[0..]
  // work is shared between inverse computation and the preinv loop

  // work during inverse computation:
  size_t inv_work = 2 * (in + 1) + invert_approx_scratch_size(in + 1);

  // preinv loopof the time work:
  //  rp[0..bn-1]: partial remainder
  //  tp[0..tn-1]: product/mulmod buffer
  //  mul_work / mulmod_scratch: workspace
  size_t mul_in_in = multiply_scratch_size(in, in);
  size_t mul_bn_in = multiply_scratch_size(bn, in);
  size_t mul_sz = std::max(mul_in_in, mul_bn_in);
  size_t tn_full = bn + in + 1;
  size_t preinv_full = bn + tn_full + mul_sz;

  // mulmod path: tp[tn_mod] + mulmod_scratch
  size_t preinv_mulmod = 0;
  if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) {
    size_t tn_mod = mulmod_bnm1_next_size(bn + 1);
    size_t mm_scratch = mulmod_bnm1_scratch_size(tn_mod, bn, in);
    preinv_mulmod = bn + tn_mod + mm_scratch;
  }
  size_t preinv_work = std::max(preinv_full, preinv_mulmod);

  return in + std::max(inv_work, preinv_work);
}

// Forward declaration
inline size_t mu_div_qr(uint64_t* q, uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch);

// preinv_mu_div_qr: chunk-based division loop using a precomputed inverse
// (follows GMP mpn_preinv_mu_div_qr)
//
// np[0..nn-1]: dividend (read-only)
// dp[0..dn-1]: divisor
// ip[0..in-1]: approximate inverse (with implicit MSB 1)
// qp[0..nn-dn-1]: quotient output
// rp[0..dn-1]: remainder output
// scratch: tp + mul_work
// Return value: qh (topmost bit of the quotient, 0 or 1)
inline uint64_t preinv_mu_div_qr(uint64_t* qp, uint64_t* rp,
                  const uint64_t* np, size_t nn,
                  const uint64_t* dp, size_t dn,
                  const uint64_t* ip, size_t in,
                  uint64_t* scratch) {
  size_t qn = nn - dn;

  uint64_t* tp = scratch;
  size_t tn = dn + in + 1;
  uint64_t* mul_work = scratch + tn;

  np += qn;
  qp += qn;

  uint64_t qh = (cmp(np, dn, dp, dn) >= 0) ? 1 : 0;
  if (qh)
    sub(rp, np, dn, dp, dn);
  else
    std::memcpy(rp, np, dn * sizeof(uint64_t));

  while (qn > 0) {
    if (qn < in) {
      ip += in - qn;
      in = qn;
    }
    np -= in;
    qp -= in;

    // ====== Step 1: quotient estimation (GMP style: mul_n + top extraction) ======
    // GMP: mpn_mul_n(tp, rp + dn - in, ip, in) → qp = tp[in..2in-1] + rp
    // Faster than mulhigh_n (avoids Karatsuba mulhigh recursion overhead)
    multiply(tp, rp + dn - in, in, ip, in, mul_work);
    uint64_t cy_q = add(qp, tp + in, in, rp + dn - in, in);

    qn -= in;

    // ====== Step 2: Q_chunk × D ======
    if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) {
      size_t tn_mod = mulmod_bnm1_next_size(dn + 1);
      mulmod_bnm1(tp, tn_mod, dp, dn, qp, in, mul_work);
      size_t wn = dn + in - tn_mod;
      if (wn > 0) {
        uint64_t cy_wrap = sub(tp, tp, wn, rp + dn - wn, wn);
        if (tn_mod > wn)
          cy_wrap = sub_1(tp + wn, tn_mod - wn, cy_wrap);
        int cx = (cmp(rp + dn - in, tn_mod - dn, tp + dn, tn_mod - dn) < 0) ? 1 : 0;
        if (cx != (int)cy_wrap) {
          if (cx) add_1(tp, tn_mod, 1);
          else  sub_1(tp, tn_mod, 1);
        }
      }
    } else {
      std::memset(tp, 0, tn * sizeof(uint64_t));
      if (dn >= in) multiply(tp, dp, dn, qp, in, mul_work);
      else     multiply(tp, qp, in, dp, dn, mul_work);
    }

    if (cy_q) {
      sub(rp, rp, dn, dp, dn);
      qp[in] += 1;
    }

    uint64_t r = rp[dn - in] - tp[dn];

    // ====== Step 3: new partial remainder ======
    uint64_t cy;
    if (dn != in) {
      cy = sub(tp, np, in, tp, in);
      cy = sub_nc(tp + in, rp, dn - in, tp + in, dn - in, cy);
      std::memcpy(rp, tp, dn * sizeof(uint64_t));
    } else {
      cy = sub(rp, np, in, tp, in);
    }

    // ====== Step 4: correct ======
    r -= cy;
    while (r != 0) {
      add_1(qp, in, 1);
      uint64_t sub_cy = sub(rp, rp, dn, dp, dn);
      r -= sub_cy;
    }
    if (cmp(rp, dn, dp, dn) >= 0) {
      add_1(qp, in, 1);
      sub(rp, rp, dn, dp, dn);
    }
  }

  return qh;
}

inline size_t mu_div_qr(uint64_t* q, uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  size_t qn = an - bn;
  size_t in = mu_div_qr_choose_in(qn, bn);

  // scratch layout:
  //  ip[0..in-1]: inverse
  //  work[0..]: after inverse computation, reused for the preinv loop
  uint64_t* ip = scratch;
  uint64_t* work = scratch + in;

  // ====== Step 1: compute the in-limb inverse ======
  // Compute the inverse of the upper (in+1) limbs of D and use the upper in limbs
  // (GMP mpn_mu_div_qr2 style)
  //
  // work layout (temporary use, overwritten in Step 2):
  //  ip_full[0..in]:  (in+1)-limb inverse output
  //  tp_inv[0..in]:   (in+1)-limb divisor copy
  //  inv_scratch[..]:  invert_approx workspace
  {
    uint64_t* ip_full = work;
    uint64_t* tp_inv = work + in + 1;
    uint64_t* inv_scratch = tp_inv + in + 1;

    if (bn == in) {
      // Full inverse: inverse of all of D
      // Compute the (in+1)-limb inverse of (1, D[0..in-1])
      tp_inv[0] = 1;
      std::memcpy(tp_inv + 1, b, in * sizeof(uint64_t));
      invert_approx(ip_full, tp_inv, in + 1, inv_scratch);
      // Discard the lowest limb and copy to ip[0..in-1]
      std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t));
    } else {
      // Partial inverse: inverse of the upper (in+1) limbs of D
      // Inverse of D[bn-in-1..bn-1] + 1 (round up for a safe underestimate)
      std::memcpy(tp_inv, b + bn - (in + 1), (in + 1) * sizeof(uint64_t));
      uint64_t cy = add_1(tp_inv, in + 1, 1);
      if (cy) {
        // overflow: upper part of D is all ones -> inverse ~ 0
        std::memset(ip, 0, in * sizeof(uint64_t));
      } else {
        invert_approx(ip_full, tp_inv, in + 1, inv_scratch);
        std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t));
      }
    }
  }

  // ====== Step 2: divide via the preinv loop ======
  uint64_t* rp = work;
  uint64_t* preinv_scratch = work + bn;

  uint64_t qh = preinv_mu_div_qr(q, rp, a, an, b, bn, ip, in, preinv_scratch);

  std::memcpy(a, rp, bn * sizeof(uint64_t));

  if (qh) {
    q[qn] = qh;
    return normalized_size(q, qn + 1);
  }
  return normalized_size(q, qn);
}

// --------------------------------------------------------------------------
// mu_div_q: quotient-only Newton-inverse division (no remainder needed)
// --------------------------------------------------------------------------
// Skips the remainder computation (Steps 3+4) in the last chunk of preinv_mu_div_qr.
// ~15% speedup for balanced 2n/n division (equivalent to GMP mpn_mu_div_q).

// --------------------------------------------------------------------------
// preinv_mu_div_q: quotient-only version of the preinv loop
// --------------------------------------------------------------------------
// Differences from preinv_mu_div_qr:
//  (1) In the last chunk, do not compute the remainder (rp) - track only the borrow chain
//  (2) Replace the memcpy(rp, tp) in intermediate chunks with a pointer swap
//  (3) Skip sub(rp, rp, dn, dp, dn) in the final Step 4 - quotient addition only

inline uint64_t preinv_mu_div_q(uint64_t* qp, uint64_t* rp,
                 const uint64_t* np, size_t nn,
                 const uint64_t* dp, size_t dn,
                 const uint64_t* ip, size_t in,
                 uint64_t* scratch) {
  size_t qn = nn - dn;

  uint64_t* tp = scratch;
  size_t tn = dn + in + 1;
  uint64_t* mul_work = scratch + tn;

  np += qn;
  qp += qn;

  uint64_t qh = (cmp(np, dn, dp, dn) >= 0) ? 1 : 0;
  if (qh)
    sub(rp, np, dn, dp, dn);
  else
    std::memcpy(rp, np, dn * sizeof(uint64_t));

  while (qn > 0) {
    if (qn < in) {
      ip += in - qn;
      in = qn;
    }
    np -= in;
    qp -= in;

    // Step 1: quotient estimation (GMP style: full multiply + top extraction)
    multiply(tp, rp + dn - in, in, ip, in, mul_work);
    uint64_t cy_q = add(qp, tp + in, in, rp + dn - in, in);

    qn -= in;

    // Step 2: Q×D
    if (in >= MUL_TO_MULMOD_BNM1_FOR_2NXN_THRESHOLD) {
      size_t tn_mod = mulmod_bnm1_next_size(dn + 1);
      mulmod_bnm1(tp, tn_mod, dp, dn, qp, in, mul_work);
      size_t wn = dn + in - tn_mod;
      if (wn > 0) {
        uint64_t cy_wrap = sub(tp, tp, wn, rp + dn - wn, wn);
        if (tn_mod > wn)
          cy_wrap = sub_1(tp + wn, tn_mod - wn, cy_wrap);
        int cx = (cmp(rp + dn - in, tn_mod - dn, tp + dn, tn_mod - dn) < 0) ? 1 : 0;
        if (cx != (int)cy_wrap) {
          if (cx) add_1(tp, tn_mod, 1);
          else  sub_1(tp, tn_mod, 1);
        }
      }
    } else {
      std::memset(tp, 0, tn * sizeof(uint64_t));
      if (dn >= in) multiply(tp, dp, dn, qp, in, mul_work);
      else     multiply(tp, qp, in, dp, dn, mul_work);
    }

    if (cy_q) {
      sub(rp, rp, dn, dp, dn);
      qp[in] += 1;
    }

    uint64_t r = rp[dn - in] - tp[dn];

    if (qn == 0) {
      // ====== Last chunk: borrow chain only (skip remainder write) ======
      uint64_t cy = 0;
      for (size_t i = 0; i < in; i++) {
        uint64_t a_val = np[i], b_val = tp[i];
        uint64_t t = a_val - b_val;
        uint64_t bw1 = (a_val < b_val) ? 1 : 0;
        uint64_t t2 = t - cy;
        uint64_t bw2 = (t < cy) ? 1 : 0;
        cy = bw1 + bw2;
        (void)t2;
      }
      if (dn != in) {
        for (size_t i = 0; i < dn - in; i++) {
          uint64_t a_val = rp[i], b_val = tp[in + i];
          uint64_t t = a_val - b_val;
          uint64_t bw1 = (a_val < b_val) ? 1 : 0;
          uint64_t t2 = t - cy;
          uint64_t bw2 = (t < cy) ? 1 : 0;
          cy = bw1 + bw2;
          (void)t2;
        }
      }

      r -= cy;
      while (r != 0) {
        add_1(qp, in, 1);
        r--;
      }
      if (cy) {
        sub_1(qp, in, 1);
      } else {
        uint64_t cy2;
        if (dn != in) {
          cy2 = sub(tp, np, in, tp, in);
          for (size_t i = 0; i < dn - in; i++) {
            uint64_t a_val = rp[i], b_val = tp[in + i];
            uint64_t diff = a_val - b_val;
            uint64_t bw1 = (a_val < b_val) ? 1 : 0;
            uint64_t diff2 = diff - cy2;
            uint64_t bw2 = (diff < cy2) ? 1 : 0;
            tp[in + i] = diff2;
            cy2 = bw1 + bw2;
          }
        } else {
          cy2 = sub(tp, np, in, tp, in);
        }
        if (cmp(tp, dn, dp, dn) >= 0) {
          add_1(qp, in, 1);
        }
      }
      break;
    }

    // Intermediate chunks: complete remainder computation
    uint64_t cy;
    if (dn != in) {
      cy = sub(tp, np, in, tp, in);
      cy = sub_nc(tp + in, rp, dn - in, tp + in, dn - in, cy);
      std::memcpy(rp, tp, dn * sizeof(uint64_t));
    } else {
      cy = sub(rp, np, in, tp, in);
    }

    r -= cy;
    while (r != 0) {
      add_1(qp, in, 1);
      uint64_t sub_cy = sub(rp, rp, dn, dp, dn);
      r -= sub_cy;
    }
    if (cmp(rp, dn, dp, dn) >= 0) {
      add_1(qp, in, 1);
      sub(rp, rp, dn, dp, dn);
    }
  }

  return qh;
}

// mu_div_q: quotient-only Newton-inverse division
inline size_t mu_div_q_scratch_size(size_t an, size_t bn) {
  // preinv_mu_div_q uses the same scratch layout as preinv_mu_div_qr
  return mu_div_qr_scratch_size(an, bn);
}

inline size_t mu_div_q(uint64_t* q, const uint64_t* a, size_t an,
            const uint64_t* b, size_t bn,
            uint64_t* scratch) {
  size_t qn = an - bn;
  size_t in = mu_div_qr_choose_in(qn, bn);

  uint64_t* ip = scratch;
  uint64_t* work = scratch + in;

  // Inverse computation (identical to mu_div_qr)
  {
    uint64_t* ip_full = work;
    uint64_t* tp_inv = work + in + 1;
    uint64_t* inv_scratch = tp_inv + in + 1;
    if (bn == in) {
      tp_inv[0] = 1;
      std::memcpy(tp_inv + 1, b, in * sizeof(uint64_t));
      invert_approx(ip_full, tp_inv, in + 1, inv_scratch);
      std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t));
    } else {
      std::memcpy(tp_inv, b + bn - (in + 1), (in + 1) * sizeof(uint64_t));
      uint64_t cy = add_1(tp_inv, in + 1, 1);
      if (cy) {
        std::memset(ip, 0, in * sizeof(uint64_t));
      } else {
        invert_approx(ip_full, tp_inv, in + 1, inv_scratch);
        std::memcpy(ip, ip_full + 1, in * sizeof(uint64_t));
      }
    }
  }

  // preinv loop (quotient-only version)
  uint64_t* rp = work;
  uint64_t* preinv_scratch = work + bn;

  uint64_t qh = preinv_mu_div_q(q, rp, a, an, b, bn, ip, in, preinv_scratch);

  if (qh) {
    q[qn] = qh;
    return normalized_size(q, qn + 1);
  }
  return normalized_size(q, qn);
}

// --------------------------------------------------------------------------
// divide_scratch_size / divide: generic division dispatcher
// --------------------------------------------------------------------------

inline size_t divide_scratch_size(size_t an, size_t bn) {
  if (bn < 2) return 0;
  // normalization copy: bn (nb) + an + 2 (na + sentinel) + work
  size_t norm_sz = bn + an + 8; // +8: Svoboda sentinel + margin

  // DC/BZ recursion has log2(bn/DC_DIV_QR_THRESHOLD) levels.
  // DC needs ~n limbs (product) + multiply scratch per level.
  // Safe upper bound: 30*bn + mul_sz (allocated generously)
  size_t mul_sz = multiply_scratch_size(bn, bn);
  size_t bz_sz = norm_sz + 30 * bn + mul_sz;

  if (bn >= MU_DIV_BALANCED_THRESHOLD) {
    // Handles both mu-division (balanced + unbalanced) and BZ
    size_t mu_sz = norm_sz + mu_div_qr_scratch_size(an, bn);
    return std::max(bz_sz, mu_sz);
  }

  return bz_sz;
}

// Generic division
// q[0..an-bn] holds the quotient, r[0..bn-1] holds the remainder
// Return value: normalized size of the quotient
// scratch: divide_scratch_size(an, bn) limbs
inline size_t divide(uint64_t* q, uint64_t* r,
           const uint64_t* a, size_t an,
           const uint64_t* b, size_t bn,
           uint64_t* scratch) {
  // an < bn → quotient = 0, remainder = a
  if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) {
    std::memcpy(r, a, an * sizeof(uint64_t));
    if (an < bn) std::memset(r + an, 0, (bn - an) * sizeof(uint64_t));
    return 0;
  }

  // 1-limb divisor should be handled by the caller (fast path)
  // Here we assume bn >= 2

  // --- Normalization: set the MSB of b's topmost limb ---
  unsigned shift = 0;
  {
    uint64_t hi = b[bn - 1];
    shift = std::countl_zero(hi);
  }

  // Create a normalized copy within scratch
  uint64_t* nb = scratch;      // bn limbs
  uint64_t* na = scratch + bn;   // an + 2 limbs (+1 for the Svoboda sentinel)
  uint64_t* work = scratch + bn + an + 2; // remaining scratch

  if (shift > 0) {
    lshift(nb, b, bn, shift);
    na[an] = lshift(na, a, an, shift);
  } else {
    std::memcpy(nb, b, bn * sizeof(uint64_t));
    std::memcpy(na, a, an * sizeof(uint64_t));
    na[an] = 0;
  }

  size_t nan = an + (na[an] ? 1 : 0); // effective size after normalization
  size_t qn;

  if (bn < BZ_THRESHOLD) {
    // small divisor: always schoolbook
    na[nan] = 0; // sentinel for div_basecase
    div_basecase(q, na, nan, nb, bn);
    qn = normalized_size(q, nan - bn + 1);
  } else if (bn >= MU_DIV_BALANCED_THRESHOLD) {
    // Large divisor: Newton inverse iteration (handles both balanced and unbalanced)
    // Speedup from BZ O(M(n) log n) to mu O(M(n))
    qn = mu_div_qr(q, na, nan, nb, bn, work);
  } else if (nan > 2 * bn + 1 && bn >= mu_div_threshold(nan, bn)) {
    qn = mu_div_qr(q, na, nan, nb, bn, work);
  } else if (nan == 2 * bn && bn < BZ_THRESHOLD) {
    // ── balanced in-place schoolbook (small to mid size) ──
    // sbpi1_div_qr: applies directly to normalized data, eliminating BZ's renormalization/copying
    uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]);
    na[nan] = 0; // sentinel
    uint64_t qh = sbpi1_div_qr(q, na, nan, nb, bn, dinv);
    q[nan - bn] = qh;
    qn = normalized_size(q, nan - bn + 1);
  } else if (nan == 2 * bn) {
    // ── balanced DC division (in-place, BZ replace) ──
    // dcpi1_div_qr_n: applies directly to normalized data
    uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]);
    na[nan] = 0; // sentinel
    uint64_t qh = dcpi1_div_qr_n(q, na, nb, bn, dinv, work);
    q[bn] = qh;
    qn = normalized_size(q, nan - bn + 1);
  } else if (nan < 2 * bn) {
    if (nan - bn < BZ_THRESHOLD) {
      // Small quotient (< BZ_THRESHOLD): schoolbook
      na[nan] = 0;
      div_basecase(q, na, nan, nb, bn);
      qn = normalized_size(q, nan - bn + 1);
    } else {
      // Large quotient but nan < 2*bn: zero-pad and process via BZ
      std::memset(na + nan, 0, (2 * bn - nan) * sizeof(uint64_t));
      uint64_t* bz_q = work;      // bn limbs
      uint64_t* bz_r = work + bn;   // bn limbs
      uint64_t* bz_scratch = work + 2 * bn;
      div_2n_by_n(bz_q, bz_r, na, nb, bn, bz_scratch);
      std::memcpy(q, bz_q, (nan - bn + 1) * sizeof(uint64_t));
      std::memcpy(na, bz_r, bn * sizeof(uint64_t));
      qn = normalized_size(q, nan - bn + 1);
    }
  } else {
    // heavily unbalanced: chunk split
    qn = div_unbalanced(q, na, na, nan, nb, bn, work);
  }

  // Denormalize the remainder (shift < 64, processed inline)
  if (shift > 0) {
    unsigned rsh = shift;
    unsigned lsh = 64 - rsh;
    for (size_t i = 0; i < bn - 1; i++) {
      r[i] = (na[i] >> rsh) | (na[i + 1] << lsh);
    }
    r[bn - 1] = na[bn - 1] >> rsh;
  } else {
    std::memcpy(r, na, bn * sizeof(uint64_t));
  }

  return qn;
}

// Forward declaration (because divide_q is defined before divmod_1)
inline uint64_t divmod_1(uint64_t* q, const uint64_t* a, size_t an, uint64_t d);

// --------------------------------------------------------------------------
// divide_q: quotient-only division (skips remainder)
// --------------------------------------------------------------------------
// When the quotient is much smaller than the divisor (qn + 5 <= bn), apply truncation optimization:
// Approximate division using only the upper (qn+EXTRA) limbs of the divisor, then fix +/-1 via a correction multiplication.
// Effective for q = a / x^(n-1) in nthRoot PD loops etc. (qn ~ bn/(n-1)).

inline size_t divide_q_scratch_size(size_t an, size_t bn) {
  if (bn < 2) return 0;
  // BUGFIX (cas-c5cc deep #5): divide_q now delegates to mpn::divide. It needs a
  // bn-limb buffer for the discarded remainder plus mpn::divide's own scratch.
  return bn + divide_scratch_size(an, bn) + 8;
}

// Quotient-only division: q[0..an-bn] = floor(a / b)
// Does not compute the remainder. scratch: divide_q_scratch_size(an, bn) limbs
inline size_t divide_q(uint64_t* q,
            const uint64_t* a, size_t an,
            const uint64_t* b, size_t bn,
            uint64_t* scratch) {
  // a < b → q = 0
  if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) {
    return 0;
  }
  // 1-limb divisor
  if (bn == 1) {
    divmod_1(q, a, an, b[0]);
    return normalized_size(q, an);
  }

  // BUGFIX (cas-c5cc deep #5): the bespoke quotient-only fast paths below
  // (truncation optimization, balanced divappr, and the dcpi1_div_qr fallback)
  // produced value-dependent silently-wrong quotients for ~1-6% of inputs — both
  // exact and inexact — corrupting Rational reduction (IntOps::divUnchecked
  // cofactors) and IntSqrt Newton steps.  Delegate to the fully-verified
  // mpn::divide (which selects schoolbook / BZ / MU internally) and discard the
  // remainder.  The only optimization lost vs the old code is skipping the O(bn)
  // remainder denormalization — a micro-saving in a path that was simply wrong.
  {
    uint64_t* r_tmp = scratch;        // bn limbs (discarded remainder)
    uint64_t* inner = scratch + bn;   // divide_scratch_size(an, bn) limbs
    return divide(q, r_tmp, a, an, b, bn, inner);
  }

#if 0  // ---- superseded by the delegation above (kept for reference) ----
  size_t qn = an - bn + 1;

  // ====== Truncation optimization ======
  // When quotient qn is much smaller than divisor bn, divide using only the upper (qn+EXTRA) limbs of the divisor.
  // Round up by +1 on the divisor, so q_trial <= q (safe underestimate).
  // Error is at most +2, so a correction multiplication fixes it.
  constexpr size_t EXTRA = 3;
  if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) {
    size_t d2n = qn + EXTRA;
    size_t k = bn - d2n; // number of dropped low limbs

    // Phase 1: truncated division
    uint64_t* d2 = scratch;
    std::memcpy(d2, b + k, d2n * sizeof(uint64_t));
    uint64_t cy = add_1(d2, d2n, 1); // round up -> q is an underestimate

    if (!cy) {
      const uint64_t* a2 = a + k;
      size_t a2n = an - k;

      uint64_t* r_tmp  = d2 + d2n;      // d2n + 2 limbs
      uint64_t* inner_sc = r_tmp + d2n + 2;

      size_t qtn = divide(q, r_tmp, a2, a2n, d2, d2n, inner_sc);
      qtn = normalized_size(q, qtn);
      if (qtn == 0) return 0;

      // Phase 2: correction multiplication - t = q_trial * b, compare with a
      uint64_t* prod = scratch; // Phase 1 data (reused)
      size_t prod_n = qtn + bn;
      uint64_t* mul_sc = prod + prod_n + 1;

      if (qtn >= bn)
        multiply(prod, q, qtn, b, bn, mul_sc);
      else
        multiply(prod, b, bn, q, qtn, mul_sc);
      prod[prod_n] = 0; // sentinel
      prod_n = normalized_size(prod, prod_n);

      // Compare a and prod
      int c = cmp(a, an, prod, prod_n);
      if (c < 0) {
        // Overestimate (rarely from rounding up): q--
        sub_1(q, qtn, 1);
        return normalized_size(q, qtn);
      }

      // Underestimate check: if a - prod >= b then q++
      // diff = a - prod (prod buffer (reused))
      uint64_t* diff = prod; // an <= prod_n + 1 <= prod buffer size
      // Since a >= prod is guaranteed, an >= prod_n
      sub(diff, a, an, prod, prod_n);
      size_t diff_n = normalized_size(diff, an);

      // At most two corrections suffice (within the EXTRA=3 error range)
      for (int adj = 0; adj < 3 && cmp(diff, diff_n, b, bn) >= 0; adj++) {
        add_1(q, qtn + 1, 1);
        sub(diff, diff, diff_n, b, bn);
        diff_n = normalized_size(diff, diff_n);
      }
      return normalized_size(q, qtn + 1);
    }
    // d2 overflow (all-ones divisor): fall through to the normal path
  }

  // ====== MU quotient-only path: Newton inverse when bn >= MU_DIV_BALANCED_THRESHOLD ======
  if (bn >= MU_DIV_BALANCED_THRESHOLD) {
    unsigned shift = std::countl_zero(b[bn - 1]);
    uint64_t* nb = scratch;
    uint64_t* na = scratch + bn;
    uint64_t* work = scratch + bn + an + 2;

    if (shift > 0) {
      lshift(nb, b, bn, shift);
      na[an] = lshift(na, a, an, shift);
    } else {
      std::memcpy(nb, b, bn * sizeof(uint64_t));
      std::memcpy(na, a, an * sizeof(uint64_t));
      na[an] = 0;
    }
    size_t nan = an + (na[an] ? 1 : 0);

    uint64_t qh = mu_div_q(q, na, nan, nb, bn, work);
    size_t ret_qn = nan - bn;
    if (qh) {
      q[ret_qn] = qh;
      return normalized_size(q, ret_qn + 1);
    }
    return normalized_size(q, ret_qn);
  }

  // ====== schoolbook quotient-only path: sbpi1_div_q (with divisor truncation) ======
  // Below BZ_THRESHOLD use schoolbook; at or above BZ_THRESHOLD fall through to DC
  {
    unsigned shift = std::countl_zero(b[bn - 1]);
    uint64_t* nb = scratch;
    uint64_t* na = scratch + bn;

    if (shift > 0) {
      lshift(nb, b, bn, shift);
      na[an] = lshift(na, a, an, shift);
    } else {
      std::memcpy(nb, b, bn * sizeof(uint64_t));
      std::memcpy(na, a, an * sizeof(uint64_t));
      na[an] = 0;
    }
    size_t nan = an + (na[an] ? 1 : 0);

    if (bn < BZ_THRESHOLD || nan - bn < BZ_THRESHOLD) {
      // BUGFIX (cas-c5cc deep #3): use the EXACT schoolbook sbpi1_div_qr, not the
      // approximate sbpi1_div_q.  sbpi1_div_q uses divisor truncation / the GMP
      // "flag" mechanism and can return a quotient that is too small (verified ~20%
      // of power-of-10-structured quotient-only divisions, e.g. (10^40-1)/(10^20-1)
      // -> 92233720368547758080 instead of 10^20+1).  Unlike the divappr block below
      // (which verifies and corrects), this branch returned the raw approximate
      // quotient with no correction -> silent-wrong.  divide_q is reached via
      // IntOps::divUnchecked (Rational reduction, gcd cofactors), so this poisoned
      // Rational operator/ on operands sharing a factor.  sbpi1_div_qr is the exact
      // base case used throughout the DC/BZ recursions; the remainder it leaves in
      // na is simply discarded here.
      uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]);
      na[nan] = 0; // sentinel
      uint64_t qh = sbpi1_div_qr(q, na, nan, nb, bn, dinv);
      if (qh) {
        q[nan - bn] = qh;
        return normalized_size(q, nan - bn + 1);
      }
      return normalized_size(q, nan - bn);
    }

    // DC quotient-only: divappr_q + verification (GMP mpn_dcpi1_div_q style)
    uint64_t* work = na + nan + 2;
    uint64_t dinv = invert_pi1(nb[bn - 1], nb[bn - 2]);
    size_t ret_qn = nan - bn;

    // divappr optimization only for EXACTLY balanced 2*bn / bn.
    // BUGFIX (cas-c5cc): the previous block (a) accepted nan == 2*bn+1, which is
    // outside dcpi1_divappr_q_n's 2n/n contract; (b) read/verified the quotient from
    // wp+2, but dcpi1_divappr_q_n(wp, wp+1, ...) writes the quotient to wp[0..ret_qn-1]
    // (qh is the high limb), so wp+2 was the wrong address entirely; (c) gated on a
    // bogus limb (wp[1]) and corrected by at most 1, while the divappr quotient can be
    // up to 2 ulps too large. Net effect: silent-wrong (off-by-small) quotients for
    // balanced large divisions. Fix: restrict to nan == 2*bn, materialize qh as the
    // top quotient limb, and verify/correct the whole candidate in a bounded loop.
    if (nan == 2 * bn && bn >= 6) {
      uint64_t* wp = work;
      uint64_t* tp = work + ret_qn + 2;
      std::memmove(wp + 1, na, nan * sizeof(uint64_t));
      wp[0] = 0;

      uint64_t qh = dcpi1_divappr_q_n(wp, wp + 1, nb, bn, dinv, tp);

      // dcpi1_divappr_q_n wrote quotient limbs at wp[0..ret_qn-1]; qh is the carry-out.
      wp[ret_qn] = qh;
      size_t cand_qn = normalized_size(wp, ret_qn + 1);

      uint64_t* prod = tp;
      uint64_t* mul_sc = tp + cand_qn + bn + 2;
      size_t prod_n = 0;
      if (cand_qn != 0) {
        if (cand_qn >= bn)
          multiply(prod, wp, cand_qn, nb, bn, mul_sc);
        else
          multiply(prod, nb, bn, wp, cand_qn, mul_sc);
        prod_n = normalized_size(prod, cand_qn + bn);
      }

      // The approximate quotient is from above (overestimate by <= 2). Correct in a
      // bounded loop: while Q*D > dividend, Q-- (and prod -= D).
      for (int adj = 0; adj < 3 && cmp(prod, prod_n, na, nan) > 0; ++adj) {
        sub_1(wp, ret_qn + 1, 1);
        prod_n = prod_n - sub(prod, prod, prod_n, nb, bn);
        prod_n = normalized_size(prod, prod_n);
      }

      std::memcpy(q, wp, ret_qn * sizeof(uint64_t));
      if (wp[ret_qn]) {
        q[ret_qn] = wp[ret_qn];
        return normalized_size(q, ret_qn + 1);
      }
      return normalized_size(q, ret_qn);
    }

    // non-balanced: fall back to dcpi1_div_qr
    na[nan] = 0;
    uint64_t qh = dcpi1_div_qr(q, na, nan, nb, bn, dinv, work);
    q[ret_qn] = qh;
    return normalized_size(q, ret_qn + 1);
  }
#endif  // ---- end superseded divide_q fast paths ----
}

// ============================================================================
// mpn_divappr_q: quotient-only approximate division - handles non-balanced shapes (equivalent to GMP mpn_divappr_q)
// ============================================================================
//
// New API extracted for nthRoot candidate B. Uses the truncated-divisor scheme to make divappr_q_n
// usable in non-balanced shapes (a2n < 2*bn, typically a2n ~ 2*bn - 4).
//
// input:
//  a (an limbs): dividend
//  b (bn limbs): divisor (normalized - MSB of b[bn-1] set)
//  bn ≤ an < 2*bn
// output:
//  q (qn = an - bn + 1 limbs): approximate value of the quotient
//  Return value: top overflow limb (usually 0)
// Error: |q - floor(a/b)| <= 2 for the quotient (toward underestimate, guaranteed by verification multiplication)

inline size_t mpn_divappr_q_scratch_size(size_t an, size_t bn) {
  if (bn < 2 || an < bn) return 0;
  // Layout: nd(bn) | na_padded(2*bn) | qq(bn+1) | prod(qn+bn+2) | dc_sc / mul_sc
  size_t qn = an - bn + 1;
  size_t pad = (an < 2 * bn) ? (2 * bn - an) : 0;
  size_t na_size = 2 * bn; // padded to 2*bn
  size_t prod_size = qn + bn + 2;
  size_t dc_sc = dcpi1_div_qr_scratch_size(2 * bn, bn);
  size_t mul_sc = multiply_scratch_size(std::max(qn, bn), std::min(qn, bn));
  return bn + na_size + (bn + 1) + prod_size + std::max(dc_sc, mul_sc) + 8;
}

inline size_t mpn_divappr_q(uint64_t* q,
               const uint64_t* a, size_t an,
               const uint64_t* b, size_t bn,
               uint64_t* scratch) {
  if (bn < 2 || an < bn) return 0;

  size_t qn = an - bn + 1;

  // Layout
  uint64_t* nd = scratch;       // bn limbs (= b)
  uint64_t* na = nd + bn;       // 2*bn limbs (a + zero pad at top)
  uint64_t* qq = na + 2 * bn;     // bn limbs (raw quotient from divappr)
  uint64_t* prod = qq + bn;      // qn + bn + 2 limbs (verify product)
  uint64_t* sc = prod + qn + bn + 2; // dc / mul scratch

  // Divisor copy (already normalized as a precondition)
  std::memcpy(nd, b, bn * sizeof(uint64_t));

  // Dividend: align a at the top and zero-pad to 2*bn limbs
  // a (an limbs) → na[0..an-1] = a, na[an..2*bn-1] = 0
  std::memcpy(na, a, an * sizeof(uint64_t));
  if (an < 2 * bn) {
    std::memset(na + an, 0, (2 * bn - an) * sizeof(uint64_t));
  }

  uint64_t dinv = invert_pi1(nd[bn - 1], nd[bn - 2]);
  uint64_t qh = dcpi1_divappr_q_n(qq, na, nd, bn, dinv, sc);

  // qq[0..bn-1] is the divappr quotient. The actual quotient size is qn (<= bn).
  // qq[qn..bn-1] and qh are "almost zero" (non-zero indicates an overshoot) -> handled by the verification loop.
  std::memcpy(q, qq, qn * sizeof(uint64_t));

  // === Lightweight verification: compare Q*b vs a using only the upper limbs ===
  // Full M(qn, bn) verification is expensive. Here we use that both "qh != 0" and "qq[qn..bn-1] != 0"
  // indicate overflow, and correct only when needed.
  // Otherwise, rely on divappr's +/-1 precision (absorbed by the divide_q_approx <= 2 contract).
  bool overflow = (qh != 0);
  for (size_t i = qn; i < bn && !overflow; i++) {
    if (qq[i] != 0) overflow = true;
  }

  if (overflow) {
    // Fix overflow: correct exactly via verification multiplication
    if (qn >= bn)
      multiply(prod, q, qn, b, bn, sc);
    else
      multiply(prod, b, bn, q, qn, sc);
    // Add the upper part from qh and qq[qn..bn-1] into prod
    if (qh) {
      // Add the effect of q + qh*B^qn into prod: prod += qh * b * B^qn
      // Simplification: handle via mass decrement (decrease from max value q)
      std::memset(q, 0xFF, qn * sizeof(uint64_t)); // Reset q to B^qn - 1
      multiply(prod, q, qn, b, bn, sc);
    }
    size_t prod_n = normalized_size(prod, qn + bn);
    int adj = 0;
    while (cmp(prod, prod_n, a, an) > 0 && adj < 64) {
      sub_1(q, qn, 1);
      sub(prod, prod, prod_n, b, bn);
      prod_n = normalized_size(prod, prod_n);
      adj++;
    }
  }

  return 0;
}

// Approximate-quotient division: q ~ floor(a / b), error <= 2 (toward underestimate)
// Used when subsequent steps (e.g. Newton iteration) can absorb the error.
// Skips the correction multiplication, so significantly faster than divide_q.
// scratch: divide_q_approx_scratch_size(an, bn) limbs
//
// nthRoot candidate B (2026-04-27): perform the internal division after truncation via mpn_divappr_q
// (quotient-only + verified correction). Saves the M(n)/2 of the remainder computation.
inline size_t divide_q_approx_scratch_size(size_t an, size_t bn) {
  if (bn < 2) return 0;
  if (an < bn) {
    // Degenerate case (called from upper-bound estimates such as pd_step_scratch_size)
    return divide_scratch_size(bn, bn) + bn + 2;
  }
  size_t qn = an - bn + 1;
  constexpr size_t EXTRA = 3;
  size_t full_sz = divide_scratch_size(an, bn) + bn + 2;
  if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) {
    size_t d2n = qn + EXTRA;
    size_t a2n = an - (bn - d2n);
    // new path: d2 + mpn_divappr_q scratch
    size_t new_sz = (d2n + 2) + mpn_divappr_q_scratch_size(a2n, d2n);
    // fallback: d2 + r_tmp + divide scratch
    size_t old_sz = (d2n + 2) + (d2n + 2) + divide_scratch_size(a2n, d2n);
    size_t trunc_sz = std::max(new_sz, old_sz);
    return std::max(full_sz, trunc_sz);
  }
  return full_sz;
}

inline size_t divide_q_approx(uint64_t* q,
                const uint64_t* a, size_t an,
                const uint64_t* b, size_t bn,
                uint64_t* scratch) {
  if (an < bn || (an == bn && cmp(a, an, b, bn) < 0)) {
    return 0;
  }
  if (bn == 1) {
    divmod_1(q, a, an, b[0]);
    return normalized_size(q, an);
  }

  size_t qn = an - bn + 1;

  // Truncation: no correction multiplication (error <= 2)
  constexpr size_t EXTRA = 3;
  if (qn + EXTRA + 2 <= bn && bn >= BZ_THRESHOLD) {
    size_t d2n = qn + EXTRA;
    size_t k = bn - d2n;

    uint64_t* d2 = scratch;
    std::memcpy(d2, b + k, d2n * sizeof(uint64_t));
    uint64_t cy = add_1(d2, d2n, 1);

    if (!cy) {
      const uint64_t* a2 = a + k;
      size_t a2n = an - k;

      // Strategy A applicability: d2n is large enough for DC-range divappr precision.
      // If d2n is too small, schoolbook divappr has large error, and
      // verification load grows; restrict to the safe range (d2n >= BZ_THRESHOLD).
      if (d2n >= BZ_THRESHOLD) {
        uint64_t* dq_sc = d2 + d2n + 2;
        // d2 is normalized as a precondition, but the MSB may be lost after +1.
        // mpn_divappr_q expects the divisor to be normalized internally, so verify d2.
        if ((d2[d2n - 1] >> 63) & 1) {
          mpn_divappr_q(q, a2, a2n, d2, d2n, dq_sc);
          return normalized_size(q, a2n - d2n + 1);
        }
      }

      // fallback: normal divide (q + r)
      uint64_t* r_tmp  = d2 + d2n;
      uint64_t* inner_sc = r_tmp + d2n + 2;
      size_t qtn = divide(q, r_tmp, a2, a2n, d2, d2n, inner_sc);
      return normalized_size(q, qtn);
    }
  }

  // normal path
  uint64_t* r_tmp = scratch;
  uint64_t* div_sc = scratch + bn + 2;
  return divide(q, r_tmp, a, an, b, bn, div_sc);
}

// ============================================================================
// GCD Operations (Binary GCD / Stein's Algorithm)
// ============================================================================

// Count trailing zeros in a limb array (bits, not limbs)
// Returns the total number of trailing zero bits across all limbs
// Used for extracting common power-of-2 factor in binary GCD
inline size_t ctz_limb_array(const uint64_t* a, size_t n) {
  for (size_t i = 0; i < n; i++) {
    if (a[i] != 0) {
      return i * 64 + std::countr_zero(a[i]);
    }
  }
  // All limbs are zero
  return n * 64;
}

// Right shift by arbitrary number of bits
// Returns normalized size after shift
inline size_t rshift(uint64_t* r, const uint64_t* a, size_t n, size_t shift_bits) {
  if (shift_bits == 0 || n == 0) {
    if (r != a) {
      for (size_t i = 0; i < n; i++) r[i] = a[i];
    }
    return n;
  }

  size_t limb_shift = shift_bits / 64;
  size_t bit_shift = shift_bits % 64;

  if (limb_shift >= n) {
    // Shifted everything away
    return 0;
  }

  size_t remaining = n - limb_shift;

  if (bit_shift == 0) {
    // Pure limb shift
    for (size_t i = 0; i < remaining; i++) {
      r[i] = a[i + limb_shift];
    }
    return normalized_size(r, remaining);
  }

  // Combined shift: limb shift + bit shift
#ifdef SANGI_INT_HAS_ASM
  mpn_rshift_asm(r, a + limb_shift, remaining, static_cast<unsigned>(bit_shift));
#else
  for (size_t i = 0; i < remaining - 1; i++) {
    uint64_t low = a[i + limb_shift] >> bit_shift;
    uint64_t high = a[i + limb_shift + 1] << (64 - bit_shift);
    r[i] = low | high;
  }
  r[remaining - 1] = a[n - 1] >> bit_shift;
#endif

  return normalized_size(r, remaining);
}

// Left shift by arbitrary number of bits
// Returns normalized size after shift
// r must have space for at least n + (shift_bits + 63) / 64 limbs
// In-place safe (r == a)
inline size_t lshift_arbitrary(uint64_t* r, const uint64_t* a, size_t n, size_t shift_bits) {
  if (shift_bits == 0 || n == 0) {
    if (r != a) {
      for (size_t i = 0; i < n; i++) r[i] = a[i];
    }
    return n;
  }

  size_t limb_shift = shift_bits / 64;
  size_t bit_shift = shift_bits % 64;

  if (bit_shift == 0) {
    // Pure limb shift — reverse order for in-place safety
    for (size_t i = n; i-- > 0; ) {
      r[i + limb_shift] = a[i];
    }
  } else {
    // Combined shift — reverse order for in-place safety
    // Carry must be computed before any writes (a may alias r)
    uint64_t top_carry = a[n - 1] >> (64 - bit_shift);
    if (top_carry != 0) {
      r[n + limb_shift] = top_carry;
    }
    for (size_t i = n - 1; i > 0; i--) {
      r[i + limb_shift] = (a[i] << bit_shift) | (a[i - 1] >> (64 - bit_shift));
    }
    r[limb_shift] = a[0] << bit_shift;

    // Clear lower limbs (after data is moved, safe for in-place)
    for (size_t i = 0; i < limb_shift; i++) {
      r[i] = 0;
    }

    if (top_carry != 0) {
      return n + limb_shift + 1;
    }
    return n + limb_shift;
  }

  // Clear lower limbs for pure limb shift
  for (size_t i = 0; i < limb_shift; i++) {
    r[i] = 0;
  }
  return n + limb_shift;
}

// ============================================================================
// Multi-limb Exact Division (Hensel Lifting / Jebelean 1993)
// ============================================================================
//
// q = a / b, with the precondition that b divides a exactly (only usable when remainder = 0 is guaranteed).
// Determines the quotient from low limbs via Hensel lifting, so no quotient estimation or correction is needed.
// Significantly lighter than the usual divide() (only division -> submul_1).
//
// Use: num/g, den/g in Rational::reduce() (after applying multi-limb gcd).

// scratch size: divexact_scratch_size(an, bn) limbs
inline size_t divexact_scratch_size(size_t an, size_t bn) {
  // a_buf (an + 2 padding) + b_buf (bn + 2 padding)
  return an + bn + 4;
}

// q has space for qn = an - bn + 1 limbs.
// Return value: normalized size of q (actual size with leading zeros stripped).
// Precondition: a >= b (an >= bn and a is non-zero).
inline size_t divexact(uint64_t* q,
            const uint64_t* a, size_t an,
            const uint64_t* b, size_t bn,
            uint64_t* scratch) {
  if (bn == 0 || an < bn) return 0;

  // Factor out the common power-of-2 (right-shift a and b by b's trailing zeros)
  size_t shift_bits = ctz_limb_array(b, bn);

  uint64_t* a_buf = scratch;       // an + 2 limbs
  uint64_t* b_buf = scratch + an + 2;  // bn + 2 limbs

  const uint64_t* b_use;
  size_t b_use_n;
  size_t a_use_n;

  if (shift_bits > 0) {
    a_use_n = rshift(a_buf, a, an, shift_bits);
    b_use_n = rshift(b_buf, b, bn, shift_bits);
    b_use = b_buf;
  } else {
    // Copy a into a_buf (as a mutable working buffer)
    std::memcpy(a_buf, a, an * sizeof(uint64_t));
    a_use_n = an;
    b_use = b;
    b_use_n = bn;
  }

  if (a_use_n < b_use_n) return 0;

  size_t qn = a_use_n - b_use_n + 1;

  // 1-limb divisor: call divexact_by_odd directly (b is already odd)
  if (b_use_n == 1) {
    uint64_t inv = hensel_inverse_u64(b_use[0]);
    return divexact_by_odd(q, a_buf, a_use_n, b_use[0], inv);
  }

  // Hensel lifting: determine the quotient from low limbs
  // In each iteration, q[i] = a[i] * b0_inv mod 2^64 -> submul_1(a+i, b, bn, q[i])
  // → a[i] becomes 0 and the borrow propagates into upper limbs.
  uint64_t b0_inv = hensel_inverse_u64(b_use[0]);

  for (size_t i = 0; i < qn; i++) {
    uint64_t qi = a_buf[i] * b0_inv;
    q[i] = qi;

    // submul_1: a_buf[i..i+b_use_n-1] -= qi * b_use, returns borrow at top
    uint64_t borrow = submul_1(a_buf + i, b_use, b_use_n, qi);

    // Propagate borrow into upper limbs a_buf[i+b_use_n..a_use_n-1]
    // (In the final iteration i = qn-1, i + b_use_n == a_use_n, so the borrow should be 0)
    if (i + b_use_n < a_use_n && borrow > 0) {
      size_t j = i + b_use_n;
      while (j < a_use_n && borrow > 0) {
        uint64_t old = a_buf[j];
        a_buf[j] = old - borrow;
        borrow = (old < borrow) ? 1ULL : 0ULL;
        j++;
      }
    }
  }

  return normalized_size(q, qn);
}

// Binary GCD (Stein's algorithm) - division-free GCD
// Input: a[0..an-1], b[0..bn-1] (normalized, non-zero)
// Output: r[0..return_value-1] = gcd(a, b)
// scratch: temporary buffer, size >= 3 * max(an, bn)
// Returns: size of result
// Algorithm: Cohen "Computational Algebraic Number Theory" p.14
inline size_t gcd_binary(uint64_t* r,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn,
             uint64_t* scratch) {
  if (an == 0 || bn == 0) {
    // gcd(0, b) = b, gcd(a, 0) = a
    if (an == 0) {
      for (size_t i = 0; i < bn; i++) r[i] = b[i];
      return bn;
    } else {
      for (size_t i = 0; i < an; i++) r[i] = a[i];
      return an;
    }
  }

  size_t max_size = (an > bn) ? an : bn;

  // Allocate scratch buffers
  uint64_t* a_work = scratch;
  uint64_t* b_work = scratch + max_size;
  uint64_t* temp = scratch + 2 * max_size;

  // Copy inputs to work buffers
  for (size_t i = 0; i < an; i++) a_work[i] = a[i];
  for (size_t i = 0; i < bn; i++) b_work[i] = b[i];
  size_t a_size = an;
  size_t b_size = bn;

  // Step 1: Count trailing zeros
  size_t k_a = ctz_limb_array(a_work, a_size);
  size_t k_b = ctz_limb_array(b_work, b_size);
  size_t k = (k_a < k_b) ? k_a : k_b; // Common factor = 2^k

  // Step 2: Remove trailing zeros from both
  if (k_a > 0) {
    a_size = rshift(a_work, a_work, a_size, k_a);
  }
  if (k_b > 0) {
    b_size = rshift(b_work, b_work, b_size, k_b);
  }

  // Step 3: Binary GCD loop
  while (true) {
    // Compare a_work and b_work
    int c = cmp(a_work, a_size, b_work, b_size);

    if (c == 0) {
      // a_work == b_work, we're done
      break;
    }

    // Subtract smaller from larger
    int sign_dummy;
    size_t diff_size;
    if (c > 0) {
      // a_work > b_work: a_work = a_work - b_work
      diff_size = abs_sub(temp, sign_dummy, a_work, a_size, b_work, b_size);
      for (size_t i = 0; i < diff_size; i++) a_work[i] = temp[i];
      a_size = diff_size;

      // Remove trailing zeros from difference
      size_t shift = ctz_limb_array(a_work, a_size);
      if (shift > 0) {
        a_size = rshift(a_work, a_work, a_size, shift);
      }
    } else {
      // b_work > a_work: b_work = b_work - a_work
      diff_size = abs_sub(temp, sign_dummy, b_work, b_size, a_work, a_size);
      for (size_t i = 0; i < diff_size; i++) b_work[i] = temp[i];
      b_size = diff_size;

      // Remove trailing zeros from difference
      size_t shift = ctz_limb_array(b_work, b_size);
      if (shift > 0) {
        b_size = rshift(b_work, b_work, b_size, shift);
      }
    }

    // Check for zero (shouldn't happen in normal cases)
    if (a_size == 0 || b_size == 0) {
      break;
    }
  }

  // Step 4: Restore common factor by left shift
  size_t result_size;
  if (k > 0) {
    result_size = lshift_arbitrary(r, a_work, a_size, k);
  } else {
    for (size_t i = 0; i < a_size; i++) r[i] = a_work[i];
    result_size = a_size;
  }

  return result_size;
}

// Calculate scratch size needed for gcd_binary
inline size_t gcd_binary_scratch_size(size_t an, size_t bn) {
  size_t max_size = (an > bn) ? an : bn;
  return 3 * max_size;
}

// ============================================================================
// Lehmer GCD (equivalent to GMP mpn_gcd)
// Construct a transformation matrix from the top 2 limbs via hgcd2 and apply it in batch to the n-limb vector
// Complexity: O(n^2 / 64) - ~64x faster than Binary GCD's O(n^2)
// ============================================================================

// 1-limb GCD (precondition: both are odd)
// Equivalent to GMP mpn_gcd_11
inline uint64_t gcd_11(uint64_t u, uint64_t v) {
  // u, v are both odd
  // GMP-style branchless Binary GCD
  // Remove redundant LSB representation (implicit lowest bit)
  u >>= 1;
  v >>= 1;

  while (u != v) {
    uint64_t t = u - v;
    // vgtu = (u < v) ? ~0 : 0 (sign-extend the sign bit to all bits)
    uint64_t vgtu = static_cast<uint64_t>(static_cast<int64_t>(t) >> 63);
    // v = min(u, v)
    v += (vgtu & t);
    // u = |u - v|
    u = (t ^ vgtu) - vgtu;
    // Strip trailing zeros (ctz(t) is the same as ctz(|t|))
    int c = std::countr_zero(t);
    // (u >> 1) >> c: the extra 1-bit shift can run independently of ctz
    u = (u >> 1) >> c;
  }
  return (u << 1) + 1;
}

// 2-limb GCD: gcd({u1,u0}, {v1,v0})
// Precondition: both are odd
// Equivalent to GMP mpn_gcd_22
struct DoubleLimb {
  uint64_t d0, d1; // little-endian: d0 = low, d1 = high
};

inline DoubleLimb gcd_22(uint64_t u1, uint64_t u0, uint64_t v1, uint64_t v0) {
  // GMP-style branchless 2-limb Binary GCD
  // Implicit LSB: shift right by 1 bit
  u0 = (u0 >> 1) | (u1 << 63);
  u1 >>= 1;
  v0 = (v0 >> 1) | (v1 << 63);
  v1 >>= 1;

  while (u1 || v1) {
    // sub_ddmmss: (t1, t0) = (u1, u0) - (v1, v0)
    uint64_t borrow = (u0 < v0) ? 1ULL : 0ULL;
    uint64_t t0 = u0 - v0;
    uint64_t t1 = u1 - v1 - borrow;
    // vgtu: all bits 1 if u < v, otherwise 0
    uint64_t vgtu = static_cast<uint64_t>(static_cast<int64_t>(t1) >> 63);

    if (t0 == 0) {
      if (t1 == 0) {
        // u == v: GCD discover
        DoubleLimb g;
        g.d1 = (u1 << 1) | (u0 >> 63);
        g.d0 = (u0 << 1) | 1;
        return g;
      }
      int c = std::countr_zero(t1);
      // v1 = min(u1, v1) via branchless
      v1 += (vgtu & t1);
      // u0 = |u1 - v1|
      u0 = (t1 ^ vgtu) - vgtu;
      u0 >>= c + 1;
      u1 = 0;
    } else {
      int c = std::countr_zero(t0) + 1;
      // V <-- min(U, V) via branchless add
      uint64_t add0 = vgtu & t0;
      uint64_t add1 = vgtu & t1;
      uint64_t carry2 = 0;
      v0 += add0;
      carry2 = (v0 < add0) ? 1ULL : 0ULL;
      v1 += add1 + carry2;

      // U <-- |U - V|
      u0 = (t0 ^ vgtu) - vgtu;
      u1 = t1 ^ vgtu;

      if (c == 64) {
        u0 = u1;
        u1 = 0;
      } else {
        u0 = (u0 >> c) | (u1 << (64 - c));
        u1 >>= c;
      }
    }
  }

  // One operand has high limb 0 -> transition loop for the case where the value is near the MSB
  while ((v0 | u0) & (1ULL << 63)) {
    uint64_t t0 = u0 - v0;
    uint64_t vgtu = static_cast<uint64_t>(-(u0 < v0)); // borrow → mask
    if (t0 == 0) {
      DoubleLimb g;
      g.d1 = u0 >> 63;
      g.d0 = (u0 << 1) | 1;
      return g;
    }
    v0 += (vgtu & t0);
    u0 = (t0 ^ vgtu) - vgtu;
    int c = std::countr_zero(t0);
    u0 = (u0 >> 1) >> c;
  }

  // Fall through to 1-limb GCD
  DoubleLimb g;
  g.d0 = gcd_11((u0 << 1) + 1, (v0 << 1) + 1);
  g.d1 = 0;
  return g;
}

// 2x2 transformation matrix (result of hgcd2)
struct HgcdMatrix1 {
  uint64_t u[2][2]; // u[row][col]
};

// Forward declaration (hgcd2_div2/div1 defined below)
inline uint64_t hgcd2_div2(uint64_t r[2], uint64_t n1, uint64_t n0, uint64_t d1, uint64_t d0);
inline DoubleLimb hgcd2_div1(uint64_t ah, uint64_t bh);

// =============================================================================
// Lehmer Jacobi (based on GMP's BITS state machine)
// =============================================================================
// Ported from GMP's jacobi.c / hgcd2_jacobi.c / hgcd_jacobi.c / gmp-impl.h.
//
// state encoding (5 bit):
//  bit 0: e (sign, 0=+1, 1=-1)
//  bit 1-4: encodes (a mod 4, b mod 4, denominator) into 13 values via decode_table[13]
//  State space 0-25 + BITS_FAIL=31
//
// Invariant: simply calling bits = jacobi_update(bits, d, q & 3) immediately after each Euclid step
// automatically handles reciprocity / 2-stripping / sign. This is not a flip-per-step:
// reciprocity flip happens only at denominator (d) switching with a==b==3 mod 4; 2-strip flip happens
// only when the denominator side is even, depending on q (Schoenhage's rule).

// jacobi_table[208]: output of gen-jacobitab.c (GMP) (Niels Moeller 2010)
//  index: (bits << 3) + (denominator << 2) + (q & 3)
inline constexpr uint8_t jacobi_table[208] = {
  0, 0, 0, 0, 0, 12, 8, 4, 1, 1, 1, 1, 1, 13, 9, 5,
  2, 2, 2, 2, 2, 6, 10, 14, 3, 3, 3, 3, 3, 7, 11, 15,
  4, 16, 6, 18, 4, 0, 12, 8, 5, 17, 7, 19, 5, 1, 13, 9,
  6, 18, 4, 16, 6, 10, 14, 2, 7, 19, 5, 17, 7, 11, 15, 3,
  8, 10, 9, 11, 8, 4, 0, 12, 9, 11, 8, 10, 9, 5, 1, 13,
  10, 9, 11, 8, 10, 14, 2, 6, 11, 8, 10, 9, 11, 15, 3, 7,
  12, 22, 24, 20, 12, 8, 4, 0, 13, 23, 25, 21, 13, 9, 5, 1,
  25, 21, 13, 23, 14, 2, 6, 10, 24, 20, 12, 22, 15, 3, 7, 11,
  16, 6, 18, 4, 16, 16, 16, 16, 17, 7, 19, 5, 17, 17, 17, 17,
  18, 4, 16, 6, 18, 22, 19, 23, 19, 5, 17, 7, 19, 23, 18, 22,
  20, 12, 22, 24, 20, 20, 20, 20, 21, 13, 23, 25, 21, 21, 21, 21,
  22, 24, 20, 12, 22, 19, 23, 18, 23, 25, 21, 13, 23, 18, 22, 19,
  24, 20, 12, 22, 15, 3, 7, 11, 25, 21, 13, 23, 14, 2, 6, 10,
};

constexpr unsigned BITS_FAIL = 31;

// Initial bits: a, b passed as mod 4 (b is odd by precondition, so b & 2 extracts bit 1)
// s is the initial sign (0 or 1). Computes (a/b) Jacobi.
inline unsigned jacobi_init(unsigned a, unsigned b, unsigned s) {
  return ((a & 3) << 2) + (b & 2) + s;
}

// Update bits after one Euclid step.
// denominator: 1 = b is the denominator (a is reduced), 0 = a is the denominator (b is reduced)
// q: quotient (only mod 4 used)
inline unsigned jacobi_update(unsigned bits, unsigned denominator, unsigned q) {
  return jacobi_table[(bits << 3) + (denominator << 2) + (q & 3)];
}

// bits in the end state -> +/-1 (caller returns 0 on BITS_FAIL)
inline int jacobi_finish(unsigned bits) {
  return 1 - 2 * (bits & 1);
}

// hgcd2_jacobi: batch Euclid steps over the top 2 limbs; track sign via the bits state machine.
// Ported from GMP mpn/generic/hgcd2_jacobi.c.
//
// Input (ah, al, bh, bl): top 2 limbs of a, b
// In/out *bitsp: jacobi state (initialized by jacobi_init())
// Output M: transformation matrix (det=1, M_stored^{-1} = [[u11,-u01],[-u10,u00]])
// Return value: 1 = progress made, 0 = none (M, bits unchanged)
//
// Call jacobi_update(bits, d, q&3) after each Euclid step.
// d=1: a is reduced (b is denominator), d=0: b is reduced (a is denominator).
inline int hgcd2_jacobi(uint64_t ah, uint64_t al, uint64_t bh, uint64_t bl,
            HgcdMatrix1* M, unsigned* bitsp) {
  uint64_t u00, u01, u10, u11;
  unsigned bits = *bitsp;

  if (ah < 2 || bh < 2) return 0;

  auto sub2 = [](uint64_t& xh, uint64_t& xl, uint64_t yh, uint64_t yl) {
    uint64_t borrow = (xl < yl) ? 1ULL : 0ULL;
    xh = xh - yh - borrow;
    xl = xl - yl;
  };

  // Initial 1 sub (q=1)
  if (ah > bh || (ah == bh && al > bl)) {
    sub2(ah, al, bh, bl);
    if (ah < 2) return 0;
    u00 = u01 = u11 = 1; u10 = 0;
    bits = jacobi_update(bits, 1, 1);
  } else {
    sub2(bh, bl, ah, al);
    if (bh < 2) return 0;
    u00 = u10 = u11 = 1; u01 = 0;
    bits = jacobi_update(bits, 0, 1);
  }

  if (ah < bh) goto sub_a;

  // 2-limb loop: alternately reduce a and b
  for (;;) {
    // ah >= bh
    if (ah == bh) goto done;

    // a -= q*b
    sub2(ah, al, bh, bl);
    if (ah < 2) goto done;

    if (ah <= bh) {
      // q = 1 (one sub suffices)
      u01 += u00;
      u11 += u10;
      bits = jacobi_update(bits, 1, 1);
    } else {
      uint64_t r[2];
      uint64_t q = hgcd2_div2(r, ah, al, bh, bl);
      al = r[0]; ah = r[1];
      if (ah < 2) {
        // GMP: "A is too small, but q is correct."
        // Record the div result q as-is (not q+1; drop the initial sub to allocate headroom)
        u01 += q * u00;
        u11 += q * u10;
        bits = jacobi_update(bits, 1, q & 3);
        goto done;
      }
      q++; // Normal path: +1 for the initial sub
      u01 += q * u00;
      u11 += q * u10;
      bits = jacobi_update(bits, 1, q & 3);
    }

  sub_a:
    // bh >= ah
    if (ah == bh) goto done;

    // b -= q*a
    sub2(bh, bl, ah, al);
    if (bh < 2) goto done;

    if (bh <= ah) {
      // q = 1
      u00 += u01;
      u10 += u11;
      bits = jacobi_update(bits, 0, 1);
    } else {
      uint64_t r[2];
      uint64_t q = hgcd2_div2(r, bh, bl, ah, al);
      bl = r[0]; bh = r[1];
      if (bh < 2) {
        u00 += q * u01;
        u10 += q * u11;
        bits = jacobi_update(bits, 0, q & 3);
        goto done;
      }
      q++;
      u00 += q * u01;
      u10 += q * u11;
      bits = jacobi_update(bits, 0, q & 3);
    }
  }

done:
  M->u[0][0] = u00; M->u[0][1] = u01;
  M->u[1][0] = u10; M->u[1][1] = u11;
  *bitsp = bits;
  return 1;
}

// 2-limb division: (ah,al) / (bh,bl) -> return the quotient and store the remainder in r
// Equivalent to div2 in GMP's hgcd2-div.h
inline uint64_t hgcd2_div2(uint64_t r[2],
              uint64_t n1, uint64_t n0,
              uint64_t d1, uint64_t d0) {
  // GMP Method 2 style: branchless bitwise division
  // Compute the quotient and remainder of (n1,n0) / (d1,d0)
  uint64_t q = 0;

  int ncnt = std::countl_zero(n1);
  int dcnt = std::countl_zero(d1);
  int cnt = dcnt - ncnt;

  // Left-shift d to align digits with n
  // d1 = (d1 << cnt) + (d0 >> 1 >> (63 - cnt))
  // d0 <<= cnt
  if (cnt > 0) {
    d1 = (d1 << cnt) | (d0 >> (64 - cnt));
    d0 <<= cnt;
  }

  // Branchless loop: cnt+1 iterations
  do {
    uint64_t mask;
    q <<= 1;
    // mask = (n >= d) ? ~0 : 0 (branchless)
    if (n1 == d1)
      mask = static_cast<uint64_t>(-static_cast<int64_t>(n0 >= d0));
    else
      mask = static_cast<uint64_t>(-static_cast<int64_t>(n1 > d1));

    q -= mask; // q += 1 when mask is ~0

    // n -= d & mask (conditional subtraction)
    uint64_t sub0 = mask & d0;
    uint64_t sub1 = mask & d1;
    uint64_t borrow = (n0 < sub0) ? 1ULL : 0ULL;
    n0 -= sub0;
    n1 -= sub1 + borrow;

    // d >>= 1
    d0 = (d1 << 63) | (d0 >> 1);
    d1 >>= 1;
  } while (cnt--);

  r[0] = n0;
  r[1] = n1;
  return q;
}

// 1-limb division: ah / bh -> quotient and remainder
inline DoubleLimb hgcd2_div1(uint64_t ah, uint64_t bh) {
  DoubleLimb result;
  result.d1 = ah / bh;  // quotient
  result.d0 = ah % bh;  // remainder
  return result;
}

// hgcd2: execute multiple Euclidean steps over the top 2 limbs and construct a transformation matrix
// Equivalent to GMP mpn_hgcd2
// Return value: 1 = progress made (matrix valid), 0 = no progress
inline int hgcd2(uint64_t ah, uint64_t al, uint64_t bh, uint64_t bl,
         HgcdMatrix1* M) {
  uint64_t u00, u01, u10, u11;

  if (ah < 2 || bh < 2)
    return 0;

  // Initial subtraction guarantees a >= b
  if (ah > bh || (ah == bh && al > bl)) {
    // a -= b
    uint64_t borrow = (al < bl) ? 1ULL : 0ULL;
    ah = ah - bh - borrow;
    al = al - bl;
    if (ah < 2) return 0;
    u00 = u01 = u11 = 1;
    u10 = 0;
  } else {
    // b -= a
    uint64_t borrow = (bl < al) ? 1ULL : 0ULL;
    bh = bh - ah - borrow;
    bl = bl - al;
    if (bh < 2) return 0;
    u00 = u10 = u11 = 1;
    u01 = 0;
  }

  if (ah < bh)
    goto sub_a;

  // Double-precision loop
  for (;;) {
    if (ah == bh)
      goto done;

    if (ah < (1ULL << 32)) {
      // Reduce to the upper half and switch to the single-precision loop
      ah = (ah << 32) | (al >> 32);
      bh = (bh << 32) | (bl >> 32);
      break;
    }

    // a -= q*b, update column 2 of the matrix
    {
      uint64_t borrow = (al < bl) ? 1ULL : 0ULL;
      ah = ah - bh - borrow;
      al = al - bl;
    }

    if (ah < 2) goto done;

    if (ah <= bh) {
      // q = 1
      u01 += u00;
      u11 += u10;
    } else {
      uint64_t r[2];
      uint64_t q = hgcd2_div2(r, ah, al, bh, bl);
      al = r[0]; ah = r[1];
      if (ah < 2) {
        u01 += q * u00;
        u11 += q * u10;
        goto done;
      }
      q++;
      u01 += q * u00;
      u11 += q * u10;
    }
  sub_a:
    if (ah == bh)
      goto done;

    if (bh < (1ULL << 32)) {
      ah = (ah << 32) | (al >> 32);
      bh = (bh << 32) | (bl >> 32);
      goto sub_a1;
    }

    // b -= q*a, update column 1 of the matrix
    {
      uint64_t borrow = (bl < al) ? 1ULL : 0ULL;
      bh = bh - ah - borrow;
      bl = bl - al;
    }

    if (bh < 2) goto done;

    if (bh <= ah) {
      u00 += u01;
      u10 += u11;
    } else {
      uint64_t r[2];
      uint64_t q = hgcd2_div2(r, bh, bl, ah, al);
      bl = r[0]; bh = r[1];
      if (bh < 2) {
        u00 += q * u01;
        u10 += q * u11;
        goto done;
      }
      q++;
      u00 += q * u01;
      u10 += q * u11;
    }
  }

  // Single-precision loop (upper half only)
  for (;;) {
    ah -= bh;
    if (ah < (1ULL << 33))
      break;

    if (ah <= bh) {
      u01 += u00;
      u11 += u10;
    } else {
      DoubleLimb rq = hgcd2_div1(ah, bh);
      uint64_t q = rq.d1;
      ah = rq.d0;
      if (ah < (1ULL << 33)) {
        u01 += q * u00;
        u11 += q * u10;
        break;
      }
      q++;
      u01 += q * u00;
      u11 += q * u10;
    }
  sub_a1:
    bh -= ah;
    if (bh < (1ULL << 33))
      break;

    if (bh <= ah) {
      u00 += u01;
      u10 += u11;
    } else {
      DoubleLimb rq = hgcd2_div1(bh, ah);
      uint64_t q = rq.d1;
      bh = rq.d0;
      if (bh < (1ULL << 33)) {
        u00 += q * u01;
        u10 += q * u11;
        break;
      }
      q++;
      u00 += q * u01;
      u10 += q * u11;
    }
  }

done:
  M->u[0][0] = u00; M->u[0][1] = u01;
  M->u[1][0] = u10; M->u[1][1] = u11;
  return 1;
}

// Apply the inverse of matrix M to the n-limb vector (ap, bp)
// Follows GMP mpn_matrix22_mul1_inverse_vector
//
// The matrix M_stored returned by hgcd2 is the inverse transform:
//  original = M_stored × reduced (det(M_stored) = 1)
//  M_stored^{-1} = [[u11, -u01], [-u10, u00]]
//
// Computation:
//  rp = u11 * ap - u01 * bp (carry and borrow are always equal)
//  bp_new = u00 * bp - u10 * ap (same as above)
//
// rp requires a separate buffer from ap; bp is updated in-place
inline size_t hgcd_mul_matrix1_vector(const HgcdMatrix1* M,
                    uint64_t* rp, const uint64_t* ap,
                    uint64_t* bp, size_t n) {
  uint64_t u00 = M->u[0][0], u01 = M->u[0][1];
  uint64_t u10 = M->u[1][0], u11 = M->u[1][1];

  // rp = u11 * ap - u01 * bp (guarantees carry == borrow)
  mul_1(rp, ap, n, u11);
  submul_1(rp, bp, n, u01);

  // bp = u00 * bp - u10 * ap (same as above)
  mul_1(bp, bp, n, u00);
  submul_1(bp, ap, n, u10);

  // GMP-style normalization: decrement length by 1 if the topmost limb is zero in both
  n -= (rp[n - 1] | bp[n - 1]) == 0;
  return n;
}

// ============================================================================
// Recursive HGCD (Half-GCD) - quasi-linear GCD in O(M(n) log n)
// Equivalent to GMP mpn_hgcd (Moeller 2008)
//
// overview:
//  hgcd(ap, bp, n, M, tp) reduces the n-limb (ap, bp) to at most n/2+1 limbs;
//  M is the accumulated product of Euclidean steps.
//
//  Following the convention of the matrix stored by hgcd2:
//   M^{-1} = [[M11, -M01], [-M10, M00]] (det(M) = 1)
//   new a = M11 * old_a - M01 * old_b
//   new b = M00 * old_b - M10 * old_a
//
// GCD entry point:
//  hgcd_reduce() is called from gcd_lehmer() and reduces quickly via HGCD.
//  Below HGCD_THRESHOLD, the conventional Lehmer loop runs as-is.
// ============================================================================

// HGCD threshold: below this, use only Lehmer (hgcd2)
// Set sufficiently large to maintain the small-size advantage over GMP (0.7-0.8x)
constexpr size_t HGCD_THRESHOLD = 126;

// multi-precision 2x2 transformation matrix
// M = [[p[0][0], p[0][1]], [p[1][0], p[1][1]]]
// det(M) = 1 (always)
struct HgcdMatrix {
  size_t alloc;    // max limb count per entry
  size_t n;      // max limb count currently in use
  uint64_t* p[2][2]; // pointers to matrix entries
};

// Initialize HgcdMatrix (identity matrix, allocated from arena)
// Zero-initialize all buffers, then set diagonal elements to 1
inline void hgcd_matrix_init(HgcdMatrix* M, size_t max_alloc) {
  M->alloc = max_alloc;
  M->n = 1;
  auto& arena = getThreadArena();
  for (int i = 0; i < 2; i++)
    for (int j = 0; j < 2; j++) {
      M->p[i][j] = arena.alloc_limbs(max_alloc);
      std::memset(M->p[i][j], 0, max_alloc * sizeof(uint64_t));
      if (i == j) M->p[i][j][0] = 1;
    }
}

// M = M * M1 (multi-precision * single-limb matrix)
// M1 is the output of hgcd2 (HgcdMatrix1)
// tp needs size >= M->n + 1
//
// Computation:
//  [M00' M01'] = [M00 M01] × [u00 u01]
//  [M10' M11']  [M10 M11]  [u10 u11]
//
// For each row, save M[row][0] to tp, then update
inline void hgcd_matrix_mul_1(HgcdMatrix* M, const HgcdMatrix1* M1,
                uint64_t* tp) {
  size_t n = M->n;
  uint64_t u00 = M1->u[0][0], u01 = M1->u[0][1];
  uint64_t u10 = M1->u[1][0], u11 = M1->u[1][1];

  // Row 0: [M00', M01'] = [M00, M01] × [[u00, u01], [u10, u11]]
  uint64_t c0 = mul_1(tp, M->p[0][0], n, u00);
  uint64_t c1 = addmul_1(tp, M->p[0][1], n, u10);
  tp[n] = c0 + c1;

  uint64_t c2 = mul_1(M->p[0][1], M->p[0][1], n, u11);
  uint64_t c3 = addmul_1(M->p[0][1], M->p[0][0], n, u01);
  M->p[0][1][n] = c2 + c3;

  std::memcpy(M->p[0][0], tp, (n + 1) * sizeof(uint64_t));

  // Row 1
  c0 = mul_1(tp, M->p[1][0], n, u00);
  c1 = addmul_1(tp, M->p[1][1], n, u10);
  tp[n] = c0 + c1;

  c2 = mul_1(M->p[1][1], M->p[1][1], n, u11);
  c3 = addmul_1(M->p[1][1], M->p[1][0], n, u01);
  M->p[1][1][n] = c2 + c3;

  std::memcpy(M->p[1][0], tp, (n + 1) * sizeof(uint64_t));

  // Update size
  n++;
  if (M->p[0][0][n-1] | M->p[0][1][n-1] | M->p[1][0][n-1] | M->p[1][1][n-1])
    M->n = n;
}

// M = M * M2 (multi-precision * multi-precision matrix, 8-mul version - for small sizes / reference implementation)
// tp layout: [save: n1] [t1: rn] [t2: rn] [mul_scratch]
inline void hgcd_matrix_mul_8way(HgcdMatrix* M, const HgcdMatrix* M2,
               uint64_t* tp) {
  size_t n1 = M->n;
  size_t n2 = M2->n;
  size_t rn = n1 + n2;

  uint64_t* save = tp;
  uint64_t* t1 = tp + n1;
  uint64_t* t2 = tp + n1 + rn;
  uint64_t* ms = tp + n1 + 2 * rn;

  size_t new_n = 0;
  for (int row = 0; row < 2; row++) {
    std::memcpy(save, M->p[row][0], n1 * sizeof(uint64_t));

    for (int col = 0; col < 2; col++) {
      multiply(t1, save, n1, M2->p[0][col], n2, ms);
      multiply(t2, M->p[row][1], n1, M2->p[1][col], n2, ms);
      size_t s1 = normalized_size(t1, rn);
      size_t s2 = normalized_size(t2, rn);

      if (s1 >= s2) {
        std::memcpy(M->p[row][col], t1, s1 * sizeof(uint64_t));
        if (s2 > 0) {
          uint64_t carry = add(M->p[row][col], M->p[row][col], s1, t2, s2);
          if (carry) { M->p[row][col][s1] = carry; s1++; }
        }
      } else {
        std::memcpy(M->p[row][col], t2, s2 * sizeof(uint64_t));
        uint64_t carry = add(M->p[row][col], M->p[row][col], s2, t1, s1);
        if (carry) { M->p[row][col][s2] = carry; s1 = s2 + 1; }
        else s1 = s2;
      }
      if (s1 > new_n) new_n = s1;
    }
  }
  M->n = new_n;
}

// ----------------------------------------------------------------------------
// Strassen 2x2 matrix multiplication (7 multiplies, dedicated to HGCD)
//
// Computation (Strassen 1969):
//  P0 = (A00 + A11)·(B00 + B11)
//  P1 = (A10 + A11)·B00
//  P2 = A00·(B01 - B11)
//  P3 = A11·(B10 - B00)
//  P4 = (A00 + A01)·B11
//  P5 = (A10 - A00)·(B00 + B01)
//  P6 = (A01 - A11)·(B10 + B11)
//  C00 = P0 + P3 - P4 + P6
//  C01 = P2 + P4
//  C10 = P1 + P3
//  C11 = P0 - P1 + P2 + P5
//
// Note: HGCD matrix entries are non-negative, but Strassen intermediates P2/P3/P5/P6 involve subtraction
// and become signed. Final Cij is mathematically non-negative (product of non-negative matrices).
//
// Required scratch size: hgcd_matrix_mul_strassen_scratch_size(n1, n2)
// ----------------------------------------------------------------------------

inline size_t hgcd_matrix_mul_strassen_scratch_size(size_t n1, size_t n2) {
  // u: n1+1, v: n2+1, P0..P6: 7×(n1+n2+2), tmp: n1+n2+2,
  // mul_scratch: multiply_scratch_size((n1+1) × (n2+1))
  size_t prod = n1 + n2 + 2;
  size_t mx = std::max(n1, n2) + 1;
  size_t mn = std::min(n1, n2) + 1;
  size_t ms = multiply_scratch_size(mx, mn);
  return (n1 + 1) + (n2 + 1) + 8 * prod + ms;
}

inline void hgcd_matrix_mul_strassen(HgcdMatrix* M, const HgcdMatrix* M2,
                    uint64_t* tp) {
  size_t n1 = M->n;
  size_t n2 = M2->n;

  // Scratch layout
  size_t op1_alloc = n1 + 1;
  size_t op2_alloc = n2 + 1;
  size_t prod_alloc = n1 + n2 + 2;

  uint64_t* u = tp;                // M-side operand (size ≤ n1+1)
  uint64_t* v = u + op1_alloc;          // M2-side operand (size ≤ n2+1)
  uint64_t* P[7];
  P[0] = v + op2_alloc;
  for (int i = 1; i < 7; i++) P[i] = P[i-1] + prod_alloc;
  uint64_t* tmp = P[6] + prod_alloc;       // for the |src|>|acc| case in signed addition
  uint64_t* ms = tmp + prod_alloc;        // multiply scratch

  size_t Pn[7];
  int Psign[7];

  // ---- Common subroutine (lambda) ----

  // dst = a + b (non-negative addition), returns size
  auto add_unsigned = [](uint64_t* dst,
              const uint64_t* a, size_t an,
              const uint64_t* b, size_t bn) -> size_t {
    an = normalized_size(a, an);
    bn = normalized_size(b, bn);
    if (an == 0) { if (bn) std::memcpy(dst, b, bn * sizeof(uint64_t)); return bn; }
    if (bn == 0) { std::memcpy(dst, a, an * sizeof(uint64_t)); return an; }
    if (an >= bn) {
      std::memcpy(dst, a, an * sizeof(uint64_t));
      uint64_t cy = add(dst, dst, an, b, bn);
      if (cy) { dst[an] = cy; return an + 1; }
      return an;
    } else {
      std::memcpy(dst, b, bn * sizeof(uint64_t));
      uint64_t cy = add(dst, dst, bn, a, an);
      if (cy) { dst[bn] = cy; return bn + 1; }
      return bn;
    }
  };

  // dst = a - b (signed), writes size into *rn and returns the sign
  auto sub_signed = [](uint64_t* dst, size_t* rn,
             const uint64_t* a, size_t an,
             const uint64_t* b, size_t bn) -> int {
    an = normalized_size(a, an);
    bn = normalized_size(b, bn);
    if (an == 0 && bn == 0) { *rn = 0; return 0; }
    if (bn == 0) { std::memcpy(dst, a, an * sizeof(uint64_t)); *rn = an; return +1; }
    if (an == 0) { std::memcpy(dst, b, bn * sizeof(uint64_t)); *rn = bn; return -1; }
    int c = (an != bn) ? (an > bn ? 1 : -1) : cmp(a, an, b, bn);
    if (c == 0) { *rn = 0; return 0; }
    if (c > 0) { sub(dst, a, an, b, bn); *rn = normalized_size(dst, an); return +1; }
    sub(dst, b, bn, a, an); *rn = normalized_size(dst, bn); return -1;
  };

  // P[i] = up * vp; sign is us*vs
  auto do_mul = [&](int idx,
           const uint64_t* up, size_t un, int us,
           const uint64_t* vp, size_t vn, int vs) {
    if (us == 0 || vs == 0 || un == 0 || vn == 0) {
      Pn[idx] = 0; Psign[idx] = 0; return;
    }
    size_t pn = un + vn;
    if (un >= vn) multiply(P[idx], up, un, vp, vn, ms);
    else     multiply(P[idx], vp, vn, up, un, ms);
    Pn[idx] = normalized_size(P[idx], pn);
    Psign[idx] = us * vs;
  };

  // signed accumulate: acc <- acc + sign * src
  // acc.d is a buffer with alloc capacity. tmp_buf is a backup for saving src.
  struct Acc { uint64_t* d; size_t n; int s; };
  auto acc_apply = [&](Acc& acc, size_t alloc,
             const uint64_t* src, size_t src_n, int src_s) {
    if (src_s == 0 || src_n == 0) return;
    if (acc.s == 0) {
      std::memcpy(acc.d, src, src_n * sizeof(uint64_t));
      if (src_n < alloc)
        std::memset(acc.d + src_n, 0, (alloc - src_n) * sizeof(uint64_t));
      acc.n = src_n; acc.s = src_s; return;
    }
    if (acc.s == src_s) {
      // Same sign: |acc| += |src|
      if (acc.n >= src_n) {
        uint64_t cy = add(acc.d, acc.d, acc.n, src, src_n);
        if (cy) acc.d[acc.n++] = cy;
        else  acc.n = normalized_size(acc.d, acc.n);
      } else {
        std::memset(acc.d + acc.n, 0, (src_n - acc.n) * sizeof(uint64_t));
        uint64_t cy = add(acc.d, acc.d, src_n, src, src_n);
        acc.n = src_n;
        if (cy) acc.d[acc.n++] = cy;
        else  acc.n = normalized_size(acc.d, acc.n);
      }
    } else {
      // Different signs: compare |acc| and |src|, subtract the smaller from the larger
      int c;
      if (acc.n != src_n) c = (acc.n > src_n) ? 1 : -1;
      else c = cmp(acc.d, acc.n, src, src_n);
      if (c == 0) { acc.n = 0; acc.s = 0; return; }
      if (c > 0) {
        sub(acc.d, acc.d, acc.n, src, src_n);
        acc.n = normalized_size(acc.d, acc.n);
      } else {
        // |src| > |acc|: acc.d ← src - acc,sign flip
        // src and acc.d are separate buffers, but go through tmp because acc.d is overwritten
        std::memcpy(tmp, src, src_n * sizeof(uint64_t));
        sub(acc.d, tmp, src_n, acc.d, acc.n);
        acc.n = normalized_size(acc.d, src_n);
        acc.s = src_s;
      }
    }
  };

  // ---- Compute the 7 products ----

  // P0 = (A00 + A11) · (B00 + B11)
  {
    size_t un = add_unsigned(u, M->p[0][0], n1, M->p[1][1], n1);
    size_t vn = add_unsigned(v, M2->p[0][0], n2, M2->p[1][1], n2);
    do_mul(0, u, un, +1, v, vn, +1);
  }
  // P1 = (A10 + A11) · B00
  {
    size_t un = add_unsigned(u, M->p[1][0], n1, M->p[1][1], n1);
    size_t vn = normalized_size(M2->p[0][0], n2);
    do_mul(1, u, un, +1, M2->p[0][0], vn, +1);
  }
  // P2 = A00 · (B01 - B11)
  {
    size_t vn; int vs = sub_signed(v, &vn, M2->p[0][1], n2, M2->p[1][1], n2);
    size_t un = normalized_size(M->p[0][0], n1);
    do_mul(2, M->p[0][0], un, +1, v, vn, vs);
  }
  // P3 = A11 · (B10 - B00)
  {
    size_t vn; int vs = sub_signed(v, &vn, M2->p[1][0], n2, M2->p[0][0], n2);
    size_t un = normalized_size(M->p[1][1], n1);
    do_mul(3, M->p[1][1], un, +1, v, vn, vs);
  }
  // P4 = (A00 + A01) · B11
  {
    size_t un = add_unsigned(u, M->p[0][0], n1, M->p[0][1], n1);
    size_t vn = normalized_size(M2->p[1][1], n2);
    do_mul(4, u, un, +1, M2->p[1][1], vn, +1);
  }
  // P5 = (A10 - A00) · (B00 + B01)
  {
    size_t un; int us = sub_signed(u, &un, M->p[1][0], n1, M->p[0][0], n1);
    size_t vn = add_unsigned(v, M2->p[0][0], n2, M2->p[0][1], n2);
    do_mul(5, u, un, us, v, vn, +1);
  }
  // P6 = (A01 - A11) · (B10 + B11)
  {
    size_t un; int us = sub_signed(u, &un, M->p[0][1], n1, M->p[1][1], n1);
    size_t vn = add_unsigned(v, M2->p[1][0], n2, M2->p[1][1], n2);
    do_mul(6, u, un, us, v, vn, +1);
  }

  // ---- Assemble C00..C11 and write them back to M->p ----
  // Each Cij is mathematically non-negative (product of HGCD matrices).
  size_t alloc = M->alloc;
  Acc acc;
  size_t new_n = 0;

  // C00 = P0 + P3 - P4 + P6 → M->p[0][0]
  acc.d = M->p[0][0]; acc.n = 0; acc.s = 0;
  acc_apply(acc, alloc, P[0], Pn[0], +Psign[0]);
  acc_apply(acc, alloc, P[3], Pn[3], +Psign[3]);
  acc_apply(acc, alloc, P[4], Pn[4], -Psign[4]);
  acc_apply(acc, alloc, P[6], Pn[6], +Psign[6]);
  if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t));
  if (acc.n > new_n) new_n = acc.n;

  // C01 = P2 + P4 → M->p[0][1]
  acc.d = M->p[0][1]; acc.n = 0; acc.s = 0;
  acc_apply(acc, alloc, P[2], Pn[2], +Psign[2]);
  acc_apply(acc, alloc, P[4], Pn[4], +Psign[4]);
  if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t));
  if (acc.n > new_n) new_n = acc.n;

  // C10 = P1 + P3 → M->p[1][0]
  acc.d = M->p[1][0]; acc.n = 0; acc.s = 0;
  acc_apply(acc, alloc, P[1], Pn[1], +Psign[1]);
  acc_apply(acc, alloc, P[3], Pn[3], +Psign[3]);
  if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t));
  if (acc.n > new_n) new_n = acc.n;

  // C11 = P0 - P1 + P2 + P5 → M->p[1][1]
  acc.d = M->p[1][1]; acc.n = 0; acc.s = 0;
  acc_apply(acc, alloc, P[0], Pn[0], +Psign[0]);
  acc_apply(acc, alloc, P[1], Pn[1], -Psign[1]);
  acc_apply(acc, alloc, P[2], Pn[2], +Psign[2]);
  acc_apply(acc, alloc, P[5], Pn[5], +Psign[5]);
  if (acc.n < alloc) std::memset(acc.d + acc.n, 0, (alloc - acc.n) * sizeof(uint64_t));
  if (acc.n > new_n) new_n = acc.n;

  M->n = (new_n > 0) ? new_n : 1;
}

// Dispatch: 8-way for small sizes, Strassen for large sizes
inline void hgcd_matrix_mul(HgcdMatrix* M, const HgcdMatrix* M2,
               uint64_t* tp) {
  // Strassen adoption threshold (measured):
  //  30 limb: 1.5-2% regression for 16K-65K bit GCD (in the Toom range, 18 add + memcpy
  //  outweighs the 8->7 mul gain)
  //  80 limb: keeps -5% at 1M bit while showing no regression at small sizes
  constexpr size_t STRASSEN_THRESHOLD = 80;
  if (M->n >= STRASSEN_THRESHOLD && M2->n >= STRASSEN_THRESHOLD) {
    hgcd_matrix_mul_strassen(M, M2, tp);
  } else {
    hgcd_matrix_mul_8way(M, M2, tp);
  }
}

// Update matrix column: M[i][col] += q * M[i][col^1]
// Used to track the matrix after a Euclidean division step
// col = 1: step for a <- a - q*b; col = 0: step for b <- b - q*a
// qp[0..qn-1]: quotient (multi-precision)
// tp: scratch buffer (used in the multi-precision q case)
inline void hgcd_matrix_update_q(HgcdMatrix* M, const uint64_t* qp, size_t qn,
                 int col, uint64_t* tp) {
  int other = col ^ 1;
  size_t mn = M->n;

  // Zero-extend all entries to the maximum possible size
  size_t max_new = mn + qn;
  for (int i = 0; i < 2; i++)
    for (int j = 0; j < 2; j++)
      std::memset(M->p[i][j] + mn, 0, (max_new + 1 - mn) * sizeof(uint64_t));

  size_t new_n = mn;
  for (int row = 0; row < 2; row++) {
    if (qn == 1) {
      // Single limb: process efficiently via addmul_1
      uint64_t cy = addmul_1(M->p[row][col], M->p[row][other], mn, qp[0]);
      if (cy) {
        M->p[row][col][mn] += cy;
        if (M->p[row][col][mn] != 0 && mn + 1 > new_n)
          new_n = mn + 1;
      }
    } else {
      // Multi-precision q: multiply -> add
      size_t prod_n = mn + qn;
      uint64_t* ms = tp + prod_n;
      if (qn >= mn)
        multiply(tp, qp, qn, M->p[row][other], mn, ms);
      else
        multiply(tp, M->p[row][other], mn, qp, qn, ms);

      uint64_t cy = add(M->p[row][col], M->p[row][col], max_new, tp, prod_n);
      if (cy) {
        M->p[row][col][max_new] = cy;
        if (max_new + 1 > new_n) new_n = max_new + 1;
      } else {
        if (prod_n > new_n) new_n = prod_n;
      }
    }
  }

  // Normalization: find the topmost non-zero position
  while (new_n > 1) {
    bool any = false;
    for (int i = 0; i < 2 && !any; i++)
      for (int j = 0; j < 2 && !any; j++)
        if (M->p[i][j][new_n - 1] != 0) any = true;
    if (any) break;
    new_n--;
  }
  M->n = new_n;
}

// Apply M^{-1} to a full-size vector (follows GMP mpn_hgcd_matrix_adjust)
//
// Called after HGCD(ap+p, bp+p, nn).
//  ap[0..p-1]:    unchanged lower part (a_lo)
//  bp[0..p-1]:    unchanged lower part (b_lo)
//  ap[p..n-1]:    HGCD-reduced upper part (a_hi') - n = p + nn
//  bp[p..n-1]:    HGCD-reduced upper part (b_hi')
//
// Computation (det(M) = 1, M^{-1} = [[M11,-M01],[-M10,M00]]):
//  new_a = M11 * a_lo + B^p * a_hi' - M01 * b_lo
//  new_b = M00 * b_lo + B^p * b_hi' - M10 * a_lo
//
// tp size: 2 * (p + M.n) + multiply_scratch
inline size_t hgcd_matrix_adjust(const HgcdMatrix* M, size_t n,
                 uint64_t* ap, uint64_t* bp,
                 size_t p, uint64_t* tp) {
  size_t mn = M->n;
  size_t prod_n = mn + p;

  uint64_t* t0 = tp;
  uint64_t* t1 = tp + prod_n;
  uint64_t* ms = tp + 2 * prod_n;

  uint64_t ah, bh, cy;

  // Compute the two products that depend on a_lo before overwriting a
  if (mn >= p) multiply(t0, M->p[1][1], mn, ap, p, ms);
  else     multiply(t0, ap, p, M->p[1][1], mn, ms);

  if (mn >= p) multiply(t1, M->p[1][0], mn, ap, p, ms);
  else     multiply(t1, ap, p, M->p[1][0], mn, ms);

  // --- new_a = M11 * a_lo + B^p * a_hi' - M01 * b_lo ---
  std::memcpy(ap, t0, p * sizeof(uint64_t));
  ah = add(ap + p, ap + p, n - p, t0 + p, mn);

  if (mn >= p) multiply(t0, M->p[0][1], mn, bp, p, ms);
  else     multiply(t0, bp, p, M->p[0][1], mn, ms);
  cy = sub(ap, ap, n, t0, prod_n);
  ah -= cy;

  // --- new_b = M00 * b_lo + B^p * b_hi' - M10 * a_lo ---
  if (mn >= p) multiply(t0, M->p[0][0], mn, bp, p, ms);
  else     multiply(t0, bp, p, M->p[0][0], mn, ms);
  std::memcpy(bp, t0, p * sizeof(uint64_t));
  bh = add(bp + p, bp + p, n - p, t0 + p, mn);

  cy = sub(bp, bp, n, t1, prod_n);
  bh -= cy;

  if (ah > 0 || bh > 0) {
    ap[n] = ah;
    bp[n] = bh;
    n++;
  } else {
    if (ap[n - 1] == 0 && bp[n - 1] == 0)
      n--;
  }

  return n;
}

// Forward declaration
inline size_t hgcd(uint64_t* ap, uint64_t* bp, size_t n,
          HgcdMatrix* M, uint64_t* tp);
inline size_t hgcd_jacobi(uint64_t* ap, uint64_t* bp, size_t n,
              HgcdMatrix* M, uint64_t* tp, unsigned* bitsp);

// Single step: try hgcd2, fall back to division on failure
// GMP  mpn_hgcd_step + mpn_gcd_subdiv_step corresponds to
// return value: reducesubsequentsize (0 = no progressor GCD discover)
inline size_t hgcd_step(size_t n, uint64_t* ap, uint64_t* bp,
             size_t s, HgcdMatrix* M, uint64_t* tp) {
  HgcdMatrix1 M1;
  uint64_t mask = ap[n-1] | bp[n-1];

  if (mask == 0) {
    n--;
    return n;
  }

  // Follows GMP: extract top 2 limbs (3 branches)
  {
  uint64_t uh, ul, vh, vl;
  if (n == s + 1) {
    // Just before threshold: skip hgcd2 when mask < 4,
    // otherwise use raw limbs without shift normalization (conservative)
    if (mask < 4)
      goto sub;
    uh = ap[n-1]; ul = ap[n-2];
    vh = bp[n-1]; vl = bp[n-2];
  } else if (mask >> 63) {
    // MSB is set: no shift needed
    uh = ap[n-1]; ul = ap[n-2];
    vh = bp[n-1]; vl = bp[n-2];
  } else {
    // shiftnormalization
    int shift = std::countl_zero(mask);
    if (n >= 3) {
      uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift));
      ul = (ap[n-2] << shift) | (ap[n-3] >> (64 - shift));
      vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift));
      vl = (bp[n-2] << shift) | (bp[n-3] >> (64 - shift));
    } else {
      uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift));
      ul = ap[n-2] << shift;
      vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift));
      vl = bp[n-2] << shift;
    }
  }

  // Try constructing the transformation matrix via hgcd2
  if (hgcd2(uh, ul, vh, vl, &M1)) {
    n = hgcd_mul_matrix1_vector(&M1, tp, ap, bp, n);
    std::memcpy(ap, tp, n * sizeof(uint64_t));
    hgcd_matrix_mul_1(M, &M1, tp);
    return n;
  }
  } // End of block (uh, ul, vh, vl scope)

  // hgcd2 failed (or skipped) -> fallback following GMP gcd_subdiv_step
  // 1. subtraction (q=1) -> matrix update
  // 2. division -> matrix update
  // s threshold check prevents overshoot
sub:
  {
    size_t an = normalized_size(ap, n);
    size_t bn = normalized_size(bp, n);
    if (an == 0 || bn == 0) return 0;

    // Arrange a < b (pointer swap, equivalent to GMP MP_PTR_SWAP)
    uint64_t* lp = ap; // smaller one
    uint64_t* rp = bp; // larger one
    size_t ln = an, rn = bn;
    int col = 0; // bp of the sidereduce → col=0

    if (ln == rn) {
      int c = cmp(lp, ln, rp, rn);
      if (c == 0) return 0; // GCD discover
      if (c > 0) { std::swap(lp, rp); col ^= 1; }
    } else if (ln > rn) {
      std::swap(lp, rp); std::swap(ln, rn); col ^= 1;
    }
    // lp[0:ln] < rp[0:rn] (ln <= rn)

    // If the smaller is <= s, no progress
    if (ln <= s) return 0;

    // Step 1: subtraction rp -= lp (large -= small)
    sub(rp, rp, rn, lp, ln);
    rn = normalized_size(rp, rn);
    if (rn == 0) return 0; // lp | rp → GCD = lp

    // s check: revert to original if result <= s -> no progress
    if (rn <= s) {
      uint64_t cy = add(rp, lp, ln, rp, rn);
      if (cy) rp[ln] = cy;
      return 0;
    }

    // q=1 matrix update
    {
      M->p[0][col][M->n] = 0;
      M->p[1][col][M->n] = 0;
      uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, 1);
      if (cy) M->p[0][col][M->n] = cy;
      cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, 1);
      if (cy) M->p[1][col][M->n] = cy;
      size_t new_mn = M->n;
      if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++;
      M->n = new_mn;
    }

    // Re-arrange a < b (after subtraction, the order may change)
    ln = normalized_size(lp, n);
    rn = normalized_size(rp, n);
    if (ln == 0 || rn == 0) goto step_done;

    if (ln == rn) {
      int c = cmp(lp, ln, rp, rn);
      if (c == 0) goto step_done; // GCD discover
      if (c > 0) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; }
    } else if (ln > rn) {
      std::swap(lp, rp); std::swap(ln, rn); col ^= 1;
    }

    // Step 2: division rp /= lp
    {
      auto& arena = getThreadArena();
      size_t div_mark = arena.mark();

      size_t qn = rn - ln + 1;
      uint64_t* qp = arena.alloc_limbs(qn + 4);
      uint64_t* dividend = arena.alloc_limbs(rn + 4);
      std::memcpy(dividend, rp, rn * sizeof(uint64_t));

      size_t ds = divide_scratch_size(rn, ln);
      uint64_t* work = arena.alloc_limbs(ds + 4);

      divide(qp, rp, dividend, rn, lp, ln, work);
      // rp[0:ln-1] = remainder.upper stale limb clear.
      if (ln < n)
        std::memset(rp + ln, 0, (n - ln) * sizeof(uint64_t));

      size_t rem_n = normalized_size(rp, ln);
      qn = normalized_size(qp, qn);

      // s check: if remainder <= s, adjust the quotient
      if (rem_n <= s) {
        if (qn == 0) {
          // contradiction (rp > lp yet q=0)
          arena.rewind(div_mark);
          goto step_done;
        }
        // Decrement quotient by 1 and add divisor to remainder (r' = r + lp > s)
        sub_1(qp, qn, 1);
        qn = normalized_size(qp, qn);
        if (rem_n > 0) {
          uint64_t cy = add(rp, lp, ln, rp, rem_n);
          if (cy) rp[ln] = cy;
        } else {
          std::memcpy(rp, lp, ln * sizeof(uint64_t));
        }
      }

      // Matrix update (only when q > 0)
      qn = normalized_size(qp, qn);
      if (qn > 0) {
        if (qn == 1) {
          M->p[0][col][M->n] = 0;
          M->p[1][col][M->n] = 0;
          uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, qp[0]);
          if (cy) M->p[0][col][M->n] = cy;
          cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, qp[0]);
          if (cy) M->p[1][col][M->n] = cy;
          size_t new_mn = M->n;
          if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++;
          M->n = new_mn;
        } else {
          size_t mn = M->n;
          size_t prod_n = mn + qn;
          size_t ms_sz = multiply_scratch_size(
            std::max(mn, qn), std::min(mn, qn));
          uint64_t* update_tp = arena.alloc_limbs(prod_n + ms_sz + 16);
          hgcd_matrix_update_q(M, qp, qn, col, update_tp);
        }
      }
      arena.rewind(div_mark);
    }
  }

step_done:
  // Recompute size and normalize (clear stale limbs)
  {
    size_t an = normalized_size(ap, n);
    size_t bn = normalized_size(bp, n);
    n = std::max(an, bn);
    if (n == 0) return 0;
    if (an < n) std::memset(ap + an, 0, (n - an) * sizeof(uint64_t));
    if (bn < n) std::memset(bp + bn, 0, (n - bn) * sizeof(uint64_t));
  }
  return n;
}

// Recursive HGCD body (Moeller 2008 4-phase algorithm)
// Follows GMP mpn_hgcd
//
// Phase 1: upper ceil(n/2) limb recursion → M construct
// Phase 2: additional reduction via hgcd_step (target: near 3n/4)
// Phase 3: second recursion -> construct M2, M = M * M2
// Phase 4: final hgcd_step loop (target: s = n/2 + 1)
inline size_t hgcd(uint64_t* ap, uint64_t* bp, size_t n,
          HgcdMatrix* M, uint64_t* tp) {
  size_t s = n / 2 + 1;
  size_t n_orig = n;
  bool success = false;

  // base case: hgcd_step loop only
  if (n < HGCD_THRESHOLD) {
    while (n > s) {
      size_t nn = hgcd_step(n, ap, bp, s, M, tp);
      if (nn == 0) break;
      n = nn;
      success = true;
    }
    return success ? n : 0;
  }

  auto& arena = getThreadArena();

  // ===== Phase 1: upper ceil(n/2) limb recursion =====
  // Follows GMP: p = floor(n/2), recursion size = n - p = ceil(n/2)
  size_t p = n / 2;
  size_t nn = hgcd(ap + p, bp + p, n - p, M, tp);

  if (nn > 0) {
    // Follows GMP: when nn > 0, always execute adjust (pass p + nn)
    size_t mark = arena.mark();
    size_t mn = M->n;
    size_t prod_n = mn + p;
    size_t ms_size = multiply_scratch_size(
      std::max(mn, p), std::min(mn, p));
    size_t adj_tp_size = 2 * prod_n + ms_size;
    uint64_t* adj_tp = arena.alloc_limbs(adj_tp_size + 16);
    n = hgcd_matrix_adjust(M, p + nn, ap, bp, p, adj_tp);
    arena.rewind(mark);
    success = true;
  }

  // normalization
  while (n > s && (ap[n-1] | bp[n-1]) == 0) n--;
  if (n <= s) return success ? n : 0;

  // ===== Phase 2: hgcd_step additionalreduce =====
  // Follows GMP: intermediate target n2 = 3*n_orig/4 + 1 (prepares for Phase 3 recursion)
  {
    size_t n2 = 3 * (n_orig / 4) + 1;
    if (n2 < s) n2 = s;

    size_t mark = arena.mark();
    uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4);
    while (n > n2) {
      size_t step_n = hgcd_step(n, ap, bp, s, M, ltp);
      if (step_n == 0) break;
      n = step_n;
      success = true;
    }
    arena.rewind(mark);
  }
  if (n <= s) return success ? n : 0;

  // ===== Phase 3: second recursion (GMP-conforming split point) =====
  // p2 = 2*s - n + 1,recursionsize = n - p2 = 2*(n-s) - 1
  if (n > s + 2) {
    size_t p2 = 2 * s - n + 1;
    size_t nn2 = n - p2;
    size_t mark = arena.mark();
    HgcdMatrix M2;
    hgcd_matrix_init(&M2, 2 * nn2 + 4); // nn2+2 is insufficient: M grows in Phases 2/4

    uint64_t* rec_tp = arena.alloc_limbs(std::max(nn2, M2.alloc) + 4);
    size_t nn_result = hgcd(ap + p2, bp + p2, nn2, &M2, rec_tp);

    if (nn_result > 0) {
      // Follows GMP: when nn_result > 0, always adjust + matrix multiplication (pass p2 + nn_result)
      size_t mn2 = M2.n;
      size_t prod_n2 = mn2 + p2;
      size_t ms2 = multiply_scratch_size(
        std::max(mn2, p2), std::min(mn2, p2));
      size_t adj2_size = 2 * prod_n2 + ms2;
      uint64_t* adj2_tp = arena.alloc_limbs(adj2_size + 16);
      n = hgcd_matrix_adjust(&M2, p2 + nn_result, ap, bp, p2, adj2_tp);

      // M = M × M2
      size_t n1 = M->n, n2m = M2.n;
      size_t rn = n1 + n2m;
      size_t mul_ms = multiply_scratch_size(n1, n2m);
      // 8-way scratch: n1 + 2*rn + ms
      // Strassen scratch: hgcd_matrix_mul_strassen_scratch_size(n1, n2m)
      size_t scratch_8 = n1 + 2 * rn + mul_ms;
      size_t scratch_st = hgcd_matrix_mul_strassen_scratch_size(n1, n2m);
      size_t mul_tp_size = std::max(scratch_8, scratch_st);
      uint64_t* mul_tp = arena.alloc_limbs(mul_tp_size + 16);
      hgcd_matrix_mul(M, &M2, mul_tp);
      success = true;
    }
    arena.rewind(mark);
  }

  // normalization
  while (n > s && (ap[n-1] | bp[n-1]) == 0) n--;
  if (n <= s) return success ? n : 0;

  // ===== Phase 4: final hgcd_step loop =====
  {
    size_t mark = arena.mark();
    uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4);
    while (n > s) {
      size_t step_n = hgcd_step(n, ap, bp, s, M, ltp);
      if (step_n == 0) break;
      n = step_n;
      success = true;
    }
    arena.rewind(mark);
  }

  return success ? n : 0;
}

// =============================================================================
// hgcd_step_jacobi: Jacobi version of hgcd_step (based on bits state machine)
// Ported from GMP mpn/generic/hgcd_jacobi.c::hgcd_jacobi_step
// =============================================================================
inline size_t hgcd_step_jacobi(size_t n, uint64_t* ap, uint64_t* bp,
                size_t s, HgcdMatrix* M, uint64_t* tp,
                unsigned* bitsp) {
  HgcdMatrix1 M1;
  uint64_t mask = ap[n-1] | bp[n-1];
  if (mask == 0) { n--; return n; }

  {
    uint64_t uh, ul, vh, vl;
    if (n == s + 1) {
      if (mask < 4) goto sub;
      uh = ap[n-1]; ul = ap[n-2];
      vh = bp[n-1]; vl = bp[n-2];
    } else if (mask >> 63) {
      uh = ap[n-1]; ul = ap[n-2];
      vh = bp[n-1]; vl = bp[n-2];
    } else {
      int shift = std::countl_zero(mask);
      if (n >= 3) {
        uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift));
        ul = (ap[n-2] << shift) | (ap[n-3] >> (64 - shift));
        vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift));
        vl = (bp[n-2] << shift) | (bp[n-3] >> (64 - shift));
      } else {
        uh = (ap[n-1] << shift) | (ap[n-2] >> (64 - shift));
        ul = ap[n-2] << shift;
        vh = (bp[n-1] << shift) | (bp[n-2] >> (64 - shift));
        vl = bp[n-2] << shift;
      }
    }

    if (hgcd2_jacobi(uh, ul, vh, vl, &M1, bitsp)) {
      n = hgcd_mul_matrix1_vector(&M1, tp, ap, bp, n);
      std::memcpy(ap, tp, n * sizeof(uint64_t));
      hgcd_matrix_mul_1(M, &M1, tp);
      return n;
    }
  }

sub:
  {
    size_t an = normalized_size(ap, n);
    size_t bn = normalized_size(bp, n);
    if (an == 0 || bn == 0) return 0;

    uint64_t* lp = ap;
    uint64_t* rp = bp;
    size_t ln = an, rn = bn;
    int col = 0;
    int d = 0; // 1 if rp == ap, 0 if rp == bp

    if (ln == rn) {
      int c = cmp(lp, ln, rp, rn);
      if (c == 0) return 0;
      if (c > 0) { std::swap(lp, rp); col ^= 1; d ^= 1; }
    } else if (ln > rn) {
      std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1;
    }

    if (ln <= s) return 0;

    // Step 1: subtract rp -= lp (q=1)
    sub(rp, rp, rn, lp, ln);
    rn = normalized_size(rp, rn);
    if (rn == 0) return 0;

    if (rn <= s) {
      // Rollback (no commit)
      uint64_t cy = add(rp, lp, ln, rp, rn);
      if (cy) rp[ln] = cy;
      return 0;
    }

    // Commit step 1 (q=1, d): bits + matrix
    *bitsp = jacobi_update(*bitsp, d, 1);
    {
      M->p[0][col][M->n] = 0;
      M->p[1][col][M->n] = 0;
      uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, 1);
      if (cy) M->p[0][col][M->n] = cy;
      cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, 1);
      if (cy) M->p[1][col][M->n] = cy;
      size_t new_mn = M->n;
      if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++;
      M->n = new_mn;
    }

    // Re-arrange a < b
    ln = normalized_size(lp, n);
    rn = normalized_size(rp, n);
    if (ln == 0 || rn == 0) goto step_done;

    if (ln == rn) {
      int c = cmp(lp, ln, rp, rn);
      if (c == 0) goto step_done;
      if (c > 0) { std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1; }
    } else if (ln > rn) {
      std::swap(lp, rp); std::swap(ln, rn); col ^= 1; d ^= 1;
    }

    // Step 2: divide rp by lp
    {
      auto& arena = getThreadArena();
      size_t div_mark = arena.mark();

      size_t qn = rn - ln + 1;
      uint64_t* qp = arena.alloc_limbs(qn + 4);
      uint64_t* dividend = arena.alloc_limbs(rn + 4);
      std::memcpy(dividend, rp, rn * sizeof(uint64_t));

      size_t ds = divide_scratch_size(rn, ln);
      uint64_t* work = arena.alloc_limbs(ds + 4);

      divide(qp, rp, dividend, rn, lp, ln, work);
      if (ln < n)
        std::memset(rp + ln, 0, (n - ln) * sizeof(uint64_t));

      size_t rem_n = normalized_size(rp, ln);
      qn = normalized_size(qp, qn);

      if (rem_n <= s) {
        if (qn == 0) {
          arena.rewind(div_mark);
          goto step_done;
        }
        sub_1(qp, qn, 1);
        qn = normalized_size(qp, qn);
        if (rem_n > 0) {
          uint64_t cy = add(rp, lp, ln, rp, rem_n);
          if (cy) rp[ln] = cy;
        } else {
          std::memcpy(rp, lp, ln * sizeof(uint64_t));
        }
      }

      qn = normalized_size(qp, qn);
      if (qn > 0) {
        // Commit step 2 (q, d): bits + matrix
        *bitsp = jacobi_update(*bitsp, d, qp[0] & 3);
        if (qn == 1) {
          M->p[0][col][M->n] = 0;
          M->p[1][col][M->n] = 0;
          uint64_t cy = addmul_1(M->p[0][col], M->p[0][col^1], M->n, qp[0]);
          if (cy) M->p[0][col][M->n] = cy;
          cy = addmul_1(M->p[1][col], M->p[1][col^1], M->n, qp[0]);
          if (cy) M->p[1][col][M->n] = cy;
          size_t new_mn = M->n;
          if (M->p[0][col][new_mn] || M->p[1][col][new_mn]) new_mn++;
          M->n = new_mn;
        } else {
          size_t mn = M->n;
          size_t prod_n = mn + qn;
          size_t ms_sz = multiply_scratch_size(
            std::max(mn, qn), std::min(mn, qn));
          uint64_t* update_tp = arena.alloc_limbs(prod_n + ms_sz + 16);
          hgcd_matrix_update_q(M, qp, qn, col, update_tp);
        }
      }
      arena.rewind(div_mark);
    }
  }

step_done:
  {
    size_t an = normalized_size(ap, n);
    size_t bn = normalized_size(bp, n);
    n = std::max(an, bn);
    if (n == 0) return 0;
    if (an < n) std::memset(ap + an, 0, (n - an) * sizeof(uint64_t));
    if (bn < n) std::memset(bp + bn, 0, (n - bn) * sizeof(uint64_t));
  }
  return n;
}

// =============================================================================
// hgcd_jacobi: Jacobi version of hgcd (4-phase recursion, based on bits state machine)
// Ported from GMP mpn/generic/hgcd_jacobi.c::mpn_hgcd_jacobi
// =============================================================================
inline size_t hgcd_jacobi(uint64_t* ap, uint64_t* bp, size_t n,
              HgcdMatrix* M, uint64_t* tp, unsigned* bitsp) {
  size_t s = n / 2 + 1;
  size_t n_orig = n;
  bool success = false;

  if (n < HGCD_THRESHOLD) {
    while (n > s) {
      size_t nn = hgcd_step_jacobi(n, ap, bp, s, M, tp, bitsp);
      if (nn == 0) break;
      n = nn;
      success = true;
    }
    return success ? n : 0;
  }

  auto& arena = getThreadArena();

  // Phase 1
  size_t p = n / 2;
  size_t nn = hgcd_jacobi(ap + p, bp + p, n - p, M, tp, bitsp);
  if (nn > 0) {
    size_t mark = arena.mark();
    size_t mn = M->n;
    size_t prod_n = mn + p;
    size_t ms_size = multiply_scratch_size(
      std::max(mn, p), std::min(mn, p));
    size_t adj_tp_size = 2 * prod_n + ms_size;
    uint64_t* adj_tp = arena.alloc_limbs(adj_tp_size + 16);
    n = hgcd_matrix_adjust(M, p + nn, ap, bp, p, adj_tp);
    arena.rewind(mark);
    success = true;
  }

  while (n > s && (ap[n-1] | bp[n-1]) == 0) n--;
  if (n <= s) return success ? n : 0;

  // Phase 2
  {
    size_t n2 = 3 * (n_orig / 4) + 1;
    if (n2 < s) n2 = s;
    size_t mark = arena.mark();
    uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4);
    while (n > n2) {
      size_t step_n = hgcd_step_jacobi(n, ap, bp, s, M, ltp, bitsp);
      if (step_n == 0) break;
      n = step_n;
      success = true;
    }
    arena.rewind(mark);
  }
  if (n <= s) return success ? n : 0;

  // Phase 3
  if (n > s + 2) {
    size_t p2 = 2 * s - n + 1;
    size_t nn2 = n - p2;
    size_t mark = arena.mark();
    HgcdMatrix M2;
    hgcd_matrix_init(&M2, 2 * nn2 + 4);

    uint64_t* rec_tp = arena.alloc_limbs(std::max(nn2, M2.alloc) + 4);
    size_t nn_result = hgcd_jacobi(ap + p2, bp + p2, nn2, &M2, rec_tp, bitsp);

    if (nn_result > 0) {
      size_t mn2 = M2.n;
      size_t prod_n2 = mn2 + p2;
      size_t ms2 = multiply_scratch_size(
        std::max(mn2, p2), std::min(mn2, p2));
      size_t adj2_size = 2 * prod_n2 + ms2;
      uint64_t* adj2_tp = arena.alloc_limbs(adj2_size + 16);
      n = hgcd_matrix_adjust(&M2, p2 + nn_result, ap, bp, p2, adj2_tp);

      size_t n1 = M->n, n2m = M2.n;
      size_t rn = n1 + n2m;
      size_t mul_ms = multiply_scratch_size(n1, n2m);
      size_t scratch_8 = n1 + 2 * rn + mul_ms;
      size_t scratch_st = hgcd_matrix_mul_strassen_scratch_size(n1, n2m);
      size_t mul_tp_size = std::max(scratch_8, scratch_st);
      uint64_t* mul_tp = arena.alloc_limbs(mul_tp_size + 16);
      hgcd_matrix_mul(M, &M2, mul_tp);
      success = true;
    }
    arena.rewind(mark);
  }

  while (n > s && (ap[n-1] | bp[n-1]) == 0) n--;
  if (n <= s) return success ? n : 0;

  // Phase 4
  {
    size_t mark = arena.mark();
    uint64_t* ltp = arena.alloc_limbs(std::max(n, M->alloc) + 4);
    while (n > s) {
      size_t step_n = hgcd_step_jacobi(n, ap, bp, s, M, ltp, bitsp);
      if (step_n == 0) break;
      n = step_n;
      success = true;
    }
    arena.rewind(mark);
  }

  return success ? n : 0;
}

// HGCD wrapper for GCD
// Called from gcd_lehmer; quickly reduces (up, vp) via HGCD
// return value: newsize
inline size_t hgcd_reduce(uint64_t* up, uint64_t* vp, size_t n) {
  auto& arena = getThreadArena();
  size_t mark = arena.mark();

  // normalization
  size_t un = normalized_size(up, n);
  size_t vn = normalized_size(vp, n);
  if (un == 0 || vn == 0) {
    arena.rewind(mark);
    return std::max(un, vn);
  }
  n = std::max(un, vn);
  if (n < HGCD_THRESHOLD) {
    arena.rewind(mark);
    return n;
  }

  // pad to the same size
  if (un < n) std::memset(up + un, 0, (n - un) * sizeof(uint64_t));
  if (vn < n) std::memset(vp + vn, 0, (n - vn) * sizeof(uint64_t));

  // up >= vp guarantee
  if (cmp(up, n, vp, n) < 0) {
    for (size_t i = 0; i < n; i++) std::swap(up[i], vp[i]);
  }

  // Split point: p = n/2 (benchmark shows faster than n/3)
  size_t p = n / 2;

  // Initialize the HGCD matrix
  HgcdMatrix M;
  hgcd_matrix_init(&M, 2 * (n - p) + 4);

  // HGCD for scratch
  uint64_t* tp = arena.alloc_limbs(std::max(n - p, M.alloc) + 4);

  // upper n-p limb  HGCD execute
  size_t nn = hgcd(up + p, vp + p, n - p, &M, tp);

  // Reflect the matrix result into the full vector
  if (nn > 0) {
    // Follows GMP: pass p + nn
    size_t mn = M.n;
    size_t prod_n = mn + p;
    size_t ms_sz = multiply_scratch_size(
      std::max(mn, p), std::min(mn, p));
    size_t adj_size = 2 * prod_n + ms_sz;
    uint64_t* adj_tp = arena.alloc_limbs(adj_size + 16);
    n = hgcd_matrix_adjust(&M, p + nn, up, vp, p, adj_tp);
  }

  // normalization
  n = std::max(normalized_size(up, n), normalized_size(vp, n));

  arena.rewind(mark);
  return n;
}

// GCD subdiv step: fallback when hgcd2 fails
// once subtraction + division a, b reduce
// Return value: new size (0 = GCD found, result stored in gp)
inline size_t gcd_subdiv_step(uint64_t* ap, uint64_t* bp, size_t n,
                uint64_t* gp, size_t* gn,
                uint64_t* tp) {
  size_t an = n, bn = n;
  // normalization
  while (an > 0 && ap[an - 1] == 0) an--;
  while (bn > 0 && bp[bn - 1] == 0) bn--;

  if (an == 0) {
    for (size_t i = 0; i < bn; i++) gp[i] = bp[i];
    *gn = bn;
    return 0;
  }
  if (bn == 0) {
    for (size_t i = 0; i < an; i++) gp[i] = ap[i];
    *gn = an;
    return 0;
  }

  int swapped = 0;

  // a < b guarantee
  if (an == bn) {
    int c = cmp(ap, an, bp, bn);
    if (c == 0) {
      // a == b → GCD = a
      for (size_t i = 0; i < an; i++) gp[i] = ap[i];
      *gn = an;
      return 0;
    }
    if (c > 0) {
      std::swap(ap, bp);
      swapped = 1;
    }
  } else if (an > bn) {
    std::swap(ap, bp);
    std::swap(an, bn);
    swapped = 1;
  }

  // b -= a (b > a)
  sub(bp, bp, bn, ap, an);
  bn = normalized_size(bp, bn);
  if (bn == 0) {
    for (size_t i = 0; i < an; i++) gp[i] = ap[i];
    *gn = an;
    return 0;
  }

  // guarantee a < b again
  if (an == bn) {
    int c = cmp(ap, an, bp, bn);
    if (c == 0) {
      for (size_t i = 0; i < an; i++) gp[i] = ap[i];
      *gn = an;
      return 0;
    }
    if (c > 0) {
      std::swap(ap, bp);
      swapped ^= 1;
    }
  } else if (an > bn) {
    std::swap(ap, bp);
    std::swap(an, bn);
    swapped ^= 1;
  }

  // b = b mod a (division)
  // tp quotientbuffer as used
  size_t qn = bn - an + 1;

  // divide has the form divide(q, r, a, an, b, bn, scratch)
  // Here divide bp by a: bp mod ap
  // tp quotient ,bp remainder 
  // Use the first qn limbs of tp as the quotient, the rest as scratch
  // however tp sufficiently largePrecondition
  uint64_t* qp = tp;
  uint64_t* work = tp + qn;

  // Copy bp to a sufficiently large temp buffer (divide overwrites the dividend)
  uint64_t* dividend = work;
  for (size_t i = 0; i < bn; i++) dividend[i] = bp[i];
  work += bn;

  divide(qp, bp, dividend, bn, ap, an, work);
  bn = normalized_size(bp, an); // remainder an limb or below

  if (bn == 0) {
    // divides evenly -> GCD = ap
    for (size_t i = 0; i < an; i++) gp[i] = ap[i];
    *gn = an;
    return 0;
  }

  // Normalize ap and bp and return the size
  // an is unchanged; bn is the remainder size
  return an;
}

// Lehmer GCD scratch size
inline size_t gcd_lehmer_scratch_size(size_t an, size_t bn) {
  size_t n = (an > bn) ? an : bn;
  // Lehmer loop for:
  //  tp buffer (output of hgcd_mul_matrix1_vector; swapped with up): n+1 limbs
  //  subdiv_scratch (for gcd_subdiv_step): quotient(n) + dividend copy(n) + divide scratch
  //  tp and subdiv_scratch are separate (after swap, up points to the tp region)
  size_t tp_size = n + 1;
  size_t div_scratch = divide_scratch_size(n, n);
  size_t subdiv_scratch = n + n + div_scratch;
  size_t lehmer_total = tp_size + subdiv_scratch;
  // For the initial unbalanced division: qn + an + divide_scratch(an, bn)
  size_t init_total = 0;
  if (an > bn) {
    size_t qn = an - bn + 1;
    size_t init_div = divide_scratch_size(an, bn);
    init_total = qn + an + init_div;
  }
  return (lehmer_total > init_total) ? lehmer_total : init_total;
}

// Lehmer GCD main loop
// Equivalent to GMP mpn_gcd (no HGCD recursion, Lehmer only)
// Input: up[0..usize-1] and vp[0..n-1] are overwritten
// Precondition: usize >= n, n >= 1, vp[n-1] > 0, bothand odd
// output: gp[0..return_value-1] = gcd(up, vp)
inline size_t gcd_lehmer(uint64_t* gp,
             uint64_t* up, size_t usize,
             uint64_t* vp, size_t n,
             uint64_t* scratch) {
  size_t gn = 0;

  // Initial unbalanced division: when usize > n, up = up mod vp
  if (usize > n) {
    size_t qn = usize - n + 1;
    uint64_t* qp = scratch;
    uint64_t* tmp = scratch + qn;
    // Copy up to a sufficiently large temp buffer
    for (size_t i = 0; i < usize; i++) tmp[i] = up[i];
    uint64_t* work = tmp + usize;
    divide(qp, up, tmp, usize, vp, n, work);
    // up remainder (n limb or below)
    // zero check
    bool zero = true;
    for (size_t i = 0; i < n; i++) {
      if (up[i] != 0) { zero = false; break; }
    }
    if (zero) {
      for (size_t i = 0; i < n; i++) gp[i] = vp[i];
      return n;
    }
  }

  // Lehmer loop: while n > 2
  // scratch layout: [tp: n+1] [subdiv_scratch: ...]
  // tp is the output of hgcd_mul_matrix1_vector; pointer-swapped with up
  // subdiv_scratch is dedicated to gcd_subdiv_step (separated from the tp region)
  uint64_t* tp = scratch;
  uint64_t* subdiv_scratch = scratch + n + 1;

  while (n > 2) {
    // For large sizes, reduce quickly via HGCD
    if (n >= HGCD_THRESHOLD) {
      size_t new_n = hgcd_reduce(up, vp, n);
      if (new_n < n) {
        n = new_n;
        // Zero check after HGCD
        size_t un2 = normalized_size(up, n);
        size_t vn2 = normalized_size(vp, n);
        if (un2 == 0) {
          for (size_t i = 0; i < vn2; i++) gp[i] = vp[i];
          return vn2;
        }
        if (vn2 == 0) {
          for (size_t i = 0; i < un2; i++) gp[i] = up[i];
          return un2;
        }
        continue;
      }
      // When HGCD made no progress, fall through to Lehmer
    }

    HgcdMatrix1 M_mat;
    uint64_t uh, ul, vh, vl;
    uint64_t mask = up[n - 1] | vp[n - 1];

    if (mask == 0) {
      // both topmost 0 → normalization
      n--;
      continue;
    }

    // Normalize and extract the top 2 limbs
    if (mask >> 63) {
      // topmost bit is set -> use as-is
      uh = up[n - 1]; ul = up[n - 2];
      vh = vp[n - 1]; vl = vp[n - 2];
    } else {
      // left shift normalization
      int shift = std::countl_zero(mask);
      if (n >= 3) {
        uh = (up[n - 1] << shift) | (up[n - 2] >> (64 - shift));
        ul = (up[n - 2] << shift) | (up[n - 3] >> (64 - shift));
        vh = (vp[n - 1] << shift) | (vp[n - 2] >> (64 - shift));
        vl = (vp[n - 2] << shift) | (vp[n - 3] >> (64 - shift));
      } else {
        uh = (up[n - 1] << shift) | (up[n - 2] >> (64 - shift));
        ul = up[n - 2] << shift;
        vh = (vp[n - 1] << shift) | (vp[n - 2] >> (64 - shift));
        vl = vp[n - 2] << shift;
      }
    }

    // hgcd2 transformation matrix construct
    if (hgcd2(uh, ul, vh, vl, &M_mat)) {
      // matrix n-limb vector apply
      // tp is used as rp; up is read as ap
      n = hgcd_mul_matrix1_vector(&M_mat, tp, up, vp, n);
      // GMP (similarly) pointerswap (O(1))
      // tp holds the new a value; vp is updated in-place
      std::swap(up, tp);
      // hgcd_mul_matrix1_vector within normalized
      if (n == 0) {
        gp[0] = 0;
        return 0;
      }
    } else {
      // hgcd2 failed -> subdiv_step (uses scratch separated from the tp region)
      n = gcd_subdiv_step(up, vp, n, gp, &gn, subdiv_scratch);
      if (n == 0) return gn;
    }
  }

  // base case: n <= 2
  // normalization
  size_t un = normalized_size(up, n);
  size_t vn = normalized_size(vp, n);
  if (un == 0) {
    for (size_t i = 0; i < vn; i++) gp[i] = vp[i];
    return vn;
  }
  if (vn == 0) {
    for (size_t i = 0; i < un; i++) gp[i] = up[i];
    return un;
  }

  // make odd
  if ((up[0] & 1) == 0) {
    std::swap(up, vp);
    std::swap(un, vn);
  }
  // make vp odd
  if ((vp[0] & 1) == 0) {
    // trailing zeros remove
    size_t tz = ctz_limb_array(vp, vn);
    if (tz > 0) {
      vn = rshift(vp, vp, vn, tz);
    }
  }

  if (n <= 1 || (un <= 1 && vn <= 1)) {
    uint64_t u0 = up[0];
    uint64_t v0 = vp[0];
    // make both odd
    if ((u0 & 1) == 0) {
      u0 >>= std::countr_zero(u0);
    }
    if ((v0 & 1) == 0) {
      v0 >>= std::countr_zero(v0);
    }
    gp[0] = gcd_11(u0, v0);
    return 1;
  }

  // n == 2
  uint64_t u0 = up[0], u1 = (un > 1) ? up[1] : 0;
  uint64_t v0 = vp[0], v1 = (vn > 1) ? vp[1] : 0;

  // if v is even, shift to make it odd
  if (v0 == 0) {
    v0 = v1; v1 = 0;
  }
  if ((v0 & 1) == 0) {
    int cnt = std::countr_zero(v0);
    v0 = (v0 >> cnt) | (v1 << (64 - cnt));
    v1 >>= cnt;
  }

  DoubleLimb g = gcd_22(u1, u0, v1, v0);
  gp[0] = g.d0;
  if (g.d1 > 0) {
    gp[1] = g.d1;
    return 2;
  }
  return 1;
}

// ============================================================================
// mod_34lsub1: compute N mod (2^48-1) without division
// GMP  mpn_mod_34lsub1 corresponds to.IsSquare fastfiltering used.
// ============================================================================

// 64-bit limb for: B1=16, B2=32, B3=48
// 2^48-1 prime factor: 3, 5, 7, 13, 17, 97, 241, 257, 673
// Return value is not the exact remainder, but a value congruent to N mod (2^48-1)
inline uint64_t mod_34lsub1(const uint64_t* p, size_t n) {
  constexpr unsigned B1 = 16;
  constexpr unsigned B2 = 32;
  constexpr unsigned B3 = 48;
  constexpr uint64_t M1 = (1ULL << B1) - 1; // 0xFFFF
  constexpr uint64_t M2 = (1ULL << B2) - 1; // 0xFFFFFFFF
  constexpr uint64_t M3 = (1ULL << B3) - 1; // 0xFFFFFFFFFFFF

  uint64_t a0 = 0, a1 = 0, a2 = 0;
  uint64_t c0 = 0, c1 = 0, c2 = 0;

  size_t i = 0;
  // Process 3 limbs at a time
  while (i + 3 <= n) {
    // a0 += p[i]  with carry tracking
    uint64_t s0 = a0 + p[i];
    c0 += (s0 < a0) ? 1 : 0;
    a0 = s0;

    uint64_t s1 = a1 + p[i + 1];
    c1 += (s1 < a1) ? 1 : 0;
    a1 = s1;

    uint64_t s2 = a2 + p[i + 2];
    c2 += (s2 < a2) ? 1 : 0;
    a2 = s2;

    i += 3;
  }
  // handle the tail
  if (i < n) {
    uint64_t s0 = a0 + p[i];
    c0 += (s0 < a0) ? 1 : 0;
    a0 = s0;
    i++;
    if (i < n) {
      uint64_t s1 = a1 + p[i];
      c1 += (s1 < a1) ? 1 : 0;
      a1 = s1;
    }
  }

  // PARTS0(x) = (x & M3) + (x >> B3)
  // PARTS1(x) = ((x & M2) << B1) + (x >> B2)
  // PARTS2(x) = ((x & M1) << B2) + (x >> B1)
  auto PARTS0 = [&](uint64_t x) -> uint64_t { return (x & M3) + (x >> B3); };
  auto PARTS1 = [&](uint64_t x) -> uint64_t { return ((x & M2) << B1) + (x >> B2); };
  auto PARTS2 = [&](uint64_t x) -> uint64_t { return ((x & M1) << B2) + (x >> B1); };

  return PARTS0(a0) + PARTS1(a1) + PARTS2(a2)
     + PARTS1(c0) + PARTS2(c1) + PARTS0(c2);
}

// ============================================================================
// Square root (Zimmermann recursive algorithm, mpn level)
// ============================================================================

// Inverse square-root table (from GMP sqrtrem.c)
// invsqrttab[i] ≈ 256/sqrt((i+128)/256) - 256
static constexpr unsigned char invsqrttab[384] = {
  0xff,0xfd,0xfb,0xf9,0xf7,0xf5,0xf3,0xf2,
  0xf0,0xee,0xec,0xea,0xe9,0xe7,0xe5,0xe4,
  0xe2,0xe0,0xdf,0xdd,0xdb,0xda,0xd8,0xd7,
  0xd5,0xd4,0xd2,0xd1,0xcf,0xce,0xcc,0xcb,
  0xc9,0xc8,0xc6,0xc5,0xc4,0xc2,0xc1,0xc0,
  0xbe,0xbd,0xbc,0xba,0xb9,0xb8,0xb7,0xb5,
  0xb4,0xb3,0xb2,0xb0,0xaf,0xae,0xad,0xac,
  0xaa,0xa9,0xa8,0xa7,0xa6,0xa5,0xa4,0xa3,
  0xa2,0xa0,0x9f,0x9e,0x9d,0x9c,0x9b,0x9a,
  0x99,0x98,0x97,0x96,0x95,0x94,0x93,0x92,
  0x91,0x90,0x8f,0x8e,0x8d,0x8c,0x8c,0x8b,
  0x8a,0x89,0x88,0x87,0x86,0x85,0x84,0x83,
  0x83,0x82,0x81,0x80,0x7f,0x7e,0x7e,0x7d,
  0x7c,0x7b,0x7a,0x79,0x79,0x78,0x77,0x76,
  0x76,0x75,0x74,0x73,0x72,0x72,0x71,0x70,
  0x6f,0x6f,0x6e,0x6d,0x6d,0x6c,0x6b,0x6a,
  0x6a,0x69,0x68,0x68,0x67,0x66,0x66,0x65,
  0x64,0x64,0x63,0x62,0x62,0x61,0x60,0x60,
  0x5f,0x5e,0x5e,0x5d,0x5c,0x5c,0x5b,0x5a,
  0x5a,0x59,0x59,0x58,0x57,0x57,0x56,0x56,
  0x55,0x54,0x54,0x53,0x53,0x52,0x52,0x51,
  0x50,0x50,0x4f,0x4f,0x4e,0x4e,0x4d,0x4d,
  0x4c,0x4b,0x4b,0x4a,0x4a,0x49,0x49,0x48,
  0x48,0x47,0x47,0x46,0x46,0x45,0x45,0x44,
  0x44,0x43,0x43,0x42,0x42,0x41,0x41,0x40,
  0x40,0x3f,0x3f,0x3e,0x3e,0x3d,0x3d,0x3c,
  0x3c,0x3b,0x3b,0x3a,0x3a,0x39,0x39,0x39,
  0x38,0x38,0x37,0x37,0x36,0x36,0x35,0x35,
  0x35,0x34,0x34,0x33,0x33,0x32,0x32,0x32,
  0x31,0x31,0x30,0x30,0x2f,0x2f,0x2f,0x2e,
  0x2e,0x2d,0x2d,0x2d,0x2c,0x2c,0x2b,0x2b,
  0x2b,0x2a,0x2a,0x29,0x29,0x29,0x28,0x28,
  0x27,0x27,0x27,0x26,0x26,0x26,0x25,0x25,
  0x24,0x24,0x24,0x23,0x23,0x23,0x22,0x22,
  0x21,0x21,0x21,0x20,0x20,0x20,0x1f,0x1f,
  0x1f,0x1e,0x1e,0x1e,0x1d,0x1d,0x1d,0x1c,
  0x1c,0x1b,0x1b,0x1b,0x1a,0x1a,0x1a,0x19,
  0x19,0x19,0x18,0x18,0x18,0x18,0x17,0x17,
  0x17,0x16,0x16,0x16,0x15,0x15,0x15,0x14,
  0x14,0x14,0x13,0x13,0x13,0x12,0x12,0x12,
  0x12,0x11,0x11,0x11,0x10,0x10,0x10,0x0f,
  0x0f,0x0f,0x0f,0x0e,0x0e,0x0e,0x0d,0x0d,
  0x0d,0x0c,0x0c,0x0c,0x0c,0x0b,0x0b,0x0b,
  0x0a,0x0a,0x0a,0x0a,0x09,0x09,0x09,0x09,
  0x08,0x08,0x08,0x07,0x07,0x07,0x07,0x06,
  0x06,0x06,0x06,0x05,0x05,0x05,0x04,0x04,
  0x04,0x04,0x03,0x03,0x03,0x03,0x02,0x02,
  0x02,0x02,0x01,0x01,0x01,0x01,0x00,0x00
};

// sqrtrem1: 1-limb square root (no division needed, multiplication only)
// Precondition: a0 >= 2^62 (at least one of the top 2 bits is 1)
// return value: floor(sqrt(a0)), *rp = a0 - result^2
inline uint64_t sqrtrem1(uint64_t* rp, uint64_t a0) {
  unsigned abits = static_cast<unsigned>(a0 >> 55);
  uint64_t x0 = 0x100ULL | invsqrttab[abits - 0x80];

  // newton iteration 1: 8-bit → ~16-bit (1/√a approximate)
  uint64_t a1 = a0 >> 31;
  int64_t t = static_cast<int64_t>(
    static_cast<uint64_t>(0x2000000000000ULL) - 0x30000ULL - a1 * x0 * x0
  ) >> 16;
  x0 = (x0 << 16) + (static_cast<int64_t>(x0 * static_cast<uint64_t>(t)) >> 18);

  // newton iteration 2: ~16-bit → ~32-bit (√a approximate)
  uint64_t t2 = x0 * (a0 >> 24);
  uint64_t t3 = t2 >> 25;
  t = static_cast<int64_t>((a0 << 14) - t3 * t3 - 0x10000000000ULL) >> 24;
  x0 = t2 + (static_cast<int64_t>(x0 * static_cast<uint64_t>(t)) >> 15);
  x0 >>= 32;

  // finalcorrect (at most once increment)
  uint64_t x2 = x0 * x0;
  if (x2 + 2 * x0 <= a0 - 1) {
    x2 += 2 * x0 + 1;
    x0++;
  }

  *rp = a0 - x2;
  return x0;
}

// sqrtrem2: 2-limb square root (extension of sqrtrem1)
// Precondition: np[1] >= 2^62
// sp[0] = floor(sqrt(np[0..1])), rp[0] = remainderlower
// return value: remainderuppercarry (0 or 1)
// rp may be the same pointer as np (in-place)
inline int sqrtrem2(uint64_t* sp, uint64_t* rp, const uint64_t* np) {
  constexpr unsigned Prec = 32;

  uint64_t np0 = np[0];
  uint64_t sp0 = sqrtrem1(rp, np[1]);
  uint64_t rp0 = rp[0];

  // remainderand np0 upperpartial composition → sp0 division
  rp0 = (rp0 << (Prec - 1)) + (np0 >> (Prec + 1));
  uint64_t q = rp0 / sp0;
  // Correction when q reaches 2^Prec
  q -= q >> Prec;
  uint64_t u = rp0 - q * sp0;
  sp0 = (sp0 << Prec) | q;

  int cc = static_cast<int>(u >> (Prec - 1));
  rp0 = ((u << (Prec + 1)) & UINT64_MAX)
    + (np0 & ((1ULL << (Prec + 1)) - 1));

  // q^2 subtraction
  uint64_t q2 = q * q;
  cc -= (rp0 < q2) ? 1 : 0;
  rp0 -= q2;

  // correct (at most once)
  if (cc < 0) {
    rp0 += sp0;
    cc += (rp0 < sp0) ? 1 : 0;
    --sp0;
    rp0 += sp0;
    cc += (rp0 < sp0) ? 1 : 0;
  }

  rp[0] = rp0;
  sp[0] = sp0;
  return cc;
}

// divmod_1: multi-precision / 1-limb division (preinv-optimized version)
// q[0..an-1] = a[0..an-1] / d, return value = remainder
// Normalize, then process each limb based on inverse multiplication
inline uint64_t divmod_1(uint64_t* q, const uint64_t* a, size_t an, uint64_t d) {
  if (an == 0) return 0;

  unsigned shift = std::countl_zero(d);
  uint64_t d_norm = d << shift;
  uint64_t dinv = invert_limb(d_norm);

  if (shift == 0) {
    // d alreadynormalized (MSB set)
    uint64_t r = 0;
    for (size_t i = an; i-- > 0; ) {
      auto [qq, rr] = udiv_qrnnd_preinv(r, a[i], d_norm, dinv);
      q[i] = qq;
      r = rr;
    }
    return r;
  } else {
    // Normalize d (left shift): also shift the dividend in the same step
    // To remove the in-loop branch (i > 0 ? ...), separate out the final element
    unsigned rshift = 64 - shift;
    uint64_t r = a[an - 1] >> rshift;
    for (size_t i = an; i-- > 1; ) {
      uint64_t n1 = (a[i] << shift) | (a[i - 1] >> rshift);
      auto [qq, rr] = udiv_qrnnd_preinv(r, n1, d_norm, dinv);
      q[i] = qq;
      r = rr;
    }
    {
      uint64_t n1 = a[0] << shift;
      auto [qq, rr] = udiv_qrnnd_preinv(r, n1, d_norm, dinv);
      q[0] = qq;
      r = rr;
    }
    return r >> shift; // Denormalize the remainder
  }
}

// scratch size needed by dc_sqrtrem (recursive)
inline size_t dc_sqrtrem_scratch_size(size_t n) {
  if (n <= 1) return 0;

  size_t l = n / 2;
  size_t h = n - l;

  // q_buf(l+2) + op_scratch (divide or square)
  size_t div_sz = (h >= 2) ? divide_scratch_size(n, h) : (n + 4);
  size_t sqr_sz = square_scratch_size(l);
  size_t level_scratch = (l + 2) + std::max(div_sz, sqr_sz);
  size_t rec_scratch = dc_sqrtrem_scratch_size(h);
  return std::max(level_scratch, rec_scratch);
}

// dc_sqrtrem: Zimmermann recursive square root
// Input: np[0..2n-1] (normalized: np[2n-1] >= 2^62), modified in-place
// output: sp[0..n-1] = floor(sqrt(np)) lowerpartial
// Return value: remainder carry c (remainder stored in np[0..n-1], full remainder = c*B^n + np[0..n-1])
// scratch: dc_sqrtrem_scratch_size(n) limbs
inline int dc_sqrtrem(uint64_t* sp, uint64_t* np, size_t n, uint64_t* scratch) {
  // --- base case: n = 1 (2-limb input) ---
  if (n == 1) {
    return sqrtrem2(sp, np, np); // in-place: rp = np
  }

  size_t l = n / 2;
  size_t h = n - l;

  // === Step 1: recursively compute the square root of the top 2h limbs ===
  int q;
  if (h == 1) {
    q = sqrtrem2(sp + l, np + 2 * l, np + 2 * l);
  } else {
    q = dc_sqrtrem(sp + l, np + 2 * l, h, scratch);
  }
  // sp[l..l+h-1] = sqrt, np[2l..2l+h-1] = remainder, q = remaindercarry

  // === Step 2: remaindercarry adjust ===
  if (q != 0) {
    // Subtract s' from the remainder (absorb the carry)
    sub(np + 2 * l, np + 2 * l, h, sp + l, h);
  }

  // === Step 3: np[l..l+n-1]  sp[l..l+h-1] division ===
  // dividend: [a1, adjustedremainder] = np[l..l+n-1] (n limbs)
  // divisor: sp[l..l+h-1] (h limbs)
  // quotient: q_buf[0..l] (max l+1 limbs)
  // remainder: np[l..l+h-1] overwrite
  uint64_t* q_buf = scratch;
  uint64_t* op_scratch = scratch + l + 2;

  if (h == 1) {
    // 1-limb divisor: divmod_1 is used
    uint64_t rem = divmod_1(q_buf, np + l, n, sp[l]);
    np[l] = rem;
    q_buf[n] = 0;
  } else {
    // h >= 2: divide is used
    // divide takes a as const, so np+l can be passed directly
    // Return remainder to np+l (divide requires a separate buffer for r)
    // → Receive the remainder in the op_scratch region first, then copy back
    uint64_t* r_tmp = op_scratch;
    uint64_t* div_work = op_scratch + h;
    size_t qn = divide(q_buf, r_tmp, np + l, n, sp + l, h, div_work);
    // Write the remainder back to np[l..l+h-1]
    std::memcpy(np + l, r_tmp, h * sizeof(uint64_t));
    // Zero-fill the upper part of the quotient
    for (size_t i = qn; i <= l; i++) q_buf[i] = 0;
  }

  // === Step 4: quotient process ===
  // q (carry) += quotient topmost
  q += static_cast<int>(q_buf[l]);
  // quotient even/odd save
  int c_parity = static_cast<int>(q_buf[0] & 1);
  // sp[0..l-1] = q_buf[0..l-1] >> 1 (divide the quotient by 2)
  for (size_t i = 0; i < l - 1; i++) {
    sp[i] = (q_buf[i] >> 1) | (q_buf[i + 1] << 63);
  }
  sp[l - 1] = (q_buf[l - 1] >> 1) | (static_cast<uint64_t>(q) << 63);
  q >>= 1;

  // === Step 5: adjust the remainder when the quotient was odd ===
  int c = 0;
  if (c_parity != 0) {
    c = static_cast<int>(add(np + l, np + l, h, sp + l, h));
  }

  // === Step 6: compute sp[0..l-1]^2 and subtract from the remainder ===
  // Use np[n..n+2l-1] as a temp buffer for the square result
  uint64_t* sq_buf = np + n;
  std::memset(sq_buf, 0, 2 * l * sizeof(uint64_t));
  square(sq_buf, sp, l, op_scratch);

  // np[0..2l-1] -= sq_buf[0..2l-1]
  uint64_t borrow = sub(np, np, 2 * l, sq_buf, 2 * l);
  int b = static_cast<int>(q) + static_cast<int>(borrow);
  if (l == h) {
    c -= b;
  } else {
    // l < h (h = l+1): np[2l] fromborrow propagation
    c -= static_cast<int>(sub_1(np + 2 * l, h, static_cast<uint64_t>(b)));
  }

  // === Step 7: correct (remainder negativecase) ===
  if (c < 0) {
    // sp[l..l+h-1] carry addition
    uint64_t q_carry = add_1(sp + l, h, static_cast<uint64_t>(q));
    // np += 2 * sp (addmul_1  sp * 2 addition)
    uint64_t am_carry = addmul_1(np, sp, n, 2) + 2 * q_carry;
    c += static_cast<int>(am_carry);
    // np -= 1
    c -= static_cast<int>(sub_1(np, n, 1));
    // sp -= 1
    q = static_cast<int>(q - sub_1(sp, n, 1));
  }

  return c;
}

// scratch size needed by sqrtrem
inline size_t sqrtrem_scratch_size(size_t an) {
  if (an <= 2) return 16;
  size_t sn = (an + 1) / 2; // root size
  // Working buffer: np_buf(2*sn+2) + dc_scratch + rem_compute(2*sn + mul_scratch)
  size_t dc_sz = dc_sqrtrem_scratch_size(sn);
  size_t sqr_sz = square_scratch_size(sn);
  return 2 * sn + 2 + dc_sz + 2 * sn + 2 + sqr_sz;
}

// floor(sqrt(a[0..an-1])); store remainder in rp (rp=nullptr allowed)
// return value: sqrt  normalized size
// Precondition: an >= 1, a[an-1] != 0
// scratch: sqrtrem_scratch_size(an) limbs
inline size_t sqrtrem(uint64_t* sp, uint64_t* rp,
           const uint64_t* ap, size_t an,
           uint64_t* scratch) {
  // --- 1-limb base case ---
  if (an == 1) {
    uint64_t val = ap[0];
    // Normalize: ensure at least one of the top 2 bits is 1
    unsigned shift = std::countl_zero(val);
    shift &= ~1u; // round down to even
    uint64_t norm_val = val << shift;
    uint64_t rem;
    uint64_t s = sqrtrem1(&rem, norm_val);
    // denormalization
    s >>= (shift / 2);
    sp[0] = s;
    if (rp) rp[0] = val - s * s;
    return (s > 0) ? 1 : 0;
  }

  // --- 2-limb base case ---
  if (an == 2) {
    // Normalize: ensure at least one of the top 2 bits of np[1] is 1
    unsigned shift = std::countl_zero(ap[1]);
    shift &= ~1u;
    uint64_t np_buf[2];
    if (shift > 0) {
      np_buf[1] = (ap[1] << shift) | (ap[0] >> (64 - shift));
      np_buf[0] = ap[0] << shift;
    } else {
      np_buf[0] = ap[0];
      np_buf[1] = ap[1];
    }
    sqrtrem2(sp, scratch, np_buf);
    // denormalization
    sp[0] >>= (shift / 2);
    if (rp) {
      UInt128 sq_final = UInt128::multiply(sp[0], sp[0]);
      rp[0] = ap[0] - sq_final.low;
      uint64_t borrow = (ap[0] < sq_final.low) ? 1ULL : 0ULL;
      rp[1] = ap[1] - sq_final.high - borrow;
    }
    return (sp[0] > 0) ? 1 : 0;
  }

  // --- General case: an >= 3, Zimmermann recursion ---
  size_t sn = (an + 1) / 2; // root size (limbs)

  // scratch layout:
  //  np_buf[0..2*sn+1]: input copy (padded + normalized)
  //  dc_scratch: dc_sqrtrem for
  //  sq_buf[0..2*sn+1] + mul_work: remainderfor computation
  uint64_t* np_buf = scratch;
  uint64_t* dc_scratch = np_buf + 2 * sn + 2;
  // (sq_buf is unnecessary because sp^2 recomputation is eliminated)

  // Copy input into np_buf (pad when limb count is odd)
  std::memset(np_buf, 0, (2 * sn + 2) * sizeof(uint64_t));
  std::memcpy(np_buf, ap, an * sizeof(uint64_t));
  // If an is odd, np_buf[an] is 0 (already padded)

  // Normalize: ensure at least one of the top 2 bits of np_buf[2*sn-1] is 1
  unsigned total_shift = 0;
  {
    uint64_t top = np_buf[2 * sn - 1];
    if (top == 0) {
      // padding topmost 0 case (an odd)
      // np_buf[2*sn-2] is the effective topmost
      // A large shift is needed to make element 2*sn-1 non-zero
      // However an is odd -> 2*sn = an+1 -> np_buf[an] = 0 is normal
      // top = np_buf[2*sn-2] = ap[an-1] (which is != 0)
      // In this case shift by 64 bits + a further bit-level shift
      unsigned clz2 = std::countl_zero(np_buf[2 * sn - 2]);
      total_shift = 64 + (clz2 & ~1u);
    } else {
      unsigned clz = std::countl_zero(top);
      total_shift = clz & ~1u;
    }
  }
  if (total_shift > 0) {
    // total_shift bitleft shift
    size_t limb_shift = total_shift / 64;
    unsigned bit_shift = total_shift % 64;
    if (limb_shift > 0) {
      // limb-unit shift (move toward upper)
      for (size_t i = 2 * sn - 1; i >= limb_shift; i--) {
        np_buf[i] = np_buf[i - limb_shift];
      }
      for (size_t i = 0; i < limb_shift; i++) {
        np_buf[i] = 0;
      }
    }
    if (bit_shift > 0) {
      lshift(np_buf, np_buf, 2 * sn, bit_shift);
    }
  }

  // dc_sqrtrem invocation
  uint64_t* sp_work = sp; // write the result directly into sp
  std::memset(sp_work, 0, sn * sizeof(uint64_t));
  dc_sqrtrem(sp_work, np_buf, sn, dc_scratch);

  // denormalization: root  total_shift/2 bitright shift
  unsigned root_shift = total_shift / 2;
  if (root_shift > 0) {
    rshift(sp_work, sp_work, sn, root_shift);
  }
  size_t sp_n = normalized_size(sp_work, sn);

  // Remainder computation: denormalize the dc_sqrtrem remainder (np_buf) and return
  // * Codex feedback #1: eliminate sp^2 recomputation - use the dc_sqrtrem remainder directly
  if (rp) {
    // np_buf[0..2*sn-1] retains the normalized-input remainder
    // total_shift bitright shift originalinput againstremainder restore
    if (total_shift > 0) {
      size_t limb_shift = total_shift / 64;
      unsigned bit_shift = total_shift % 64;
      if (bit_shift > 0) {
        rshift(np_buf, np_buf, 2 * sn, bit_shift);
      }
      if (limb_shift > 0) {
        for (size_t i = 0; i + limb_shift < 2 * sn; i++) {
          np_buf[i] = np_buf[i + limb_shift];
        }
        for (size_t i = 2 * sn - limb_shift; i < 2 * sn; i++) {
          np_buf[i] = 0;
        }
      }
    }
    // Copy into rp (remainder is at most sn limbs)
    size_t rem_n = std::min((size_t)sn, an);
    std::memcpy(rp, np_buf, rem_n * sizeof(uint64_t));
    if (rem_n < an) std::memset(rp + rem_n, 0, (an - rem_n) * sizeof(uint64_t));
  }

  return sp_n;
}

// sqrtrem_check_exact: compute sqrt and return whether the remainder is zero
// Skip the M(n) squaring; check the dc_sqrtrem remainder directly
// Return value: {normalized size of sqrt, true if remainder is zero}
// scratch: sqrtrem_scratch_size(an) limbs (squaring not needed, but same size for safety)
inline std::pair<size_t, bool> sqrtrem_check_exact(
  uint64_t* sp, const uint64_t* ap, size_t an, uint64_t* scratch) {

  // --- 1-limb base case ---
  if (an == 1) {
    uint64_t val = ap[0];
    unsigned shift = std::countl_zero(val);
    shift &= ~1u;
    uint64_t norm_val = val << shift;
    uint64_t rem;
    uint64_t s = sqrtrem1(&rem, norm_val);
    s >>= (shift / 2);
    sp[0] = s;
    // Remainder-zero check: s^2 == val
    return {(s > 0) ? 1u : 0u, s * s == val};
  }

  // --- 2-limb base case ---
  if (an == 2) {
    unsigned shift = std::countl_zero(ap[1]);
    shift &= ~1u;
    uint64_t np_buf[2];
    if (shift > 0) {
      np_buf[1] = (ap[1] << shift) | (ap[0] >> (64 - shift));
      np_buf[0] = ap[0] << shift;
    } else {
      np_buf[0] = ap[0]; np_buf[1] = ap[1];
    }
    sqrtrem2(sp, scratch, np_buf);
    sp[0] >>= (shift / 2);
    UInt128 sq = UInt128::multiply(sp[0], sp[0]);
    bool exact = (sq.low == ap[0] && sq.high == ap[1]);
    return {(sp[0] > 0) ? 1u : 0u, exact};
  }

  // --- General case: an >= 3 ---
  size_t sn = (an + 1) / 2;

  uint64_t* np_buf = scratch;
  uint64_t* dc_scratch = np_buf + 2 * sn + 2;

  std::memset(np_buf, 0, (2 * sn + 2) * sizeof(uint64_t));
  std::memcpy(np_buf, ap, an * sizeof(uint64_t));

  // normalization
  unsigned total_shift = 0;
  {
    uint64_t top = np_buf[2 * sn - 1];
    if (top == 0) {
      unsigned clz2 = std::countl_zero(np_buf[2 * sn - 2]);
      total_shift = 64 + (clz2 & ~1u);
    } else {
      unsigned clz = std::countl_zero(top);
      total_shift = clz & ~1u;
    }
  }
  if (total_shift > 0) {
    size_t limb_shift = total_shift / 64;
    unsigned bit_shift = total_shift % 64;
    if (limb_shift > 0) {
      for (size_t i = 2 * sn - 1; i >= limb_shift; i--)
        np_buf[i] = np_buf[i - limb_shift];
      for (size_t i = 0; i < limb_shift; i++)
        np_buf[i] = 0;
    }
    if (bit_shift > 0) {
      lshift(np_buf, np_buf, 2 * sn, bit_shift);
    }
  }

  uint64_t* sp_work = sp;
  std::memset(sp_work, 0, sn * sizeof(uint64_t));
  int carry = dc_sqrtrem(sp_work, np_buf, sn, dc_scratch);

  // dc_sqrtrem remainder = carry * B^sn + np_buf[0..sn-1]
  // normalizationinput a' = a << total_shift perfect square ⟺ a perfect square
  // (When total_shift is even, 2^{total_shift} is a perfect square)
  // Therefore: carry == 0 and np_buf[0..sn-1] all zero <-> remainder is zero
  bool exact = (carry == 0);
  if (exact) {
    for (size_t i = 0; i < sn; i++) {
      if (np_buf[i] != 0) { exact = false; break; }
    }
  }

  // denormalization
  unsigned root_shift = total_shift / 2;
  if (root_shift > 0) {
    rshift(sp_work, sp_work, sn, root_shift);
  }
  size_t sp_n = normalized_size(sp_work, sn);

  return {sp_n, exact};
}

// ============================================================================
// mpn_cbrtrem: Block-recursive cube root (dedicated to n=3)
// ============================================================================
//
// 2026-04-28: implemented modeled on GMP rootrem.c (Zimmermann block-by-block extraction).
//
// Algorithm:
//  Carry state {S, R, W=3*S^2}; at each step extend root from c to d limbs (b=d-c)
//  1. N = R·B^b + T2 (T2 next b limbs of A)
//  2. Q = floor(N / W), clamp to B^b - 1
//  3. D = N - W·Q
//  4. S' = S·B^b + Q
//  5. R' = D·B^(2b) + T1·B^b + T0 - 3·S·Q²·B^b - Q³
//  6. negative R repair: if R' < 0, S' -= 1, R' += 3·S'² + 3·S' + 1
//  7. W' = 3·S'²
//
// Difference from PD Newton (current sangi): instead of full pow + divide each iteration,
// carry-forward the remainder and extract only the "next block".
// Expected speedup: similar to GMP (1.5-2x ratio).

namespace cbrt_detail {

// 1-limb cube root: floor(cbrt(a))
inline uint64_t cbrt1(uint64_t a) {
  if (a == 0) return 0;
  // double approximation + correction
  double da = static_cast<double>(a);
  uint64_t s = static_cast<uint64_t>(std::cbrt(da));
  // Correct ±1
  while (s > 0) {
    // Check s³ > a
    // Use 128-bit safe: s³ might overflow uint64_t for large s (s > ~2.6M)
    // For 1-limb a, s ≤ cbrt(2^64) ≈ 2^21.3 ≈ 2.6M, so s³ ≤ 2^64 fits in uint64_t
    // Use UInt128 for safety on edge
    UInt128 s2 = UInt128::multiply(s, s);
    // s³ has up to 64 bits if s ≤ 2^21 (2^63 ≈ 9.2e18, cbrt ≈ 2.1e6)
    // For s up to ~2^21.3, s² has up to 43 bits, s³ has up to 64 bits
    // Compute s² · s using UInt128
    // s² fits in s2.low usually
    if (s2.high != 0) { s--; continue; }
    UInt128 s3 = UInt128::multiply(s2.low, s);
    if (s3.high > 0 || s3.low > a) { s--; continue; }
    break;
  }
  // Try s+1
  {
    uint64_t sp1 = s + 1;
    if (sp1 != 0) {
      UInt128 sp1_2 = UInt128::multiply(sp1, sp1);
      if (sp1_2.high == 0) {
        UInt128 sp1_3 = UInt128::multiply(sp1_2.low, sp1);
        if (sp1_3.high == 0 && sp1_3.low <= a) {
          s = sp1;
        }
      }
    }
  }
  return s;
}

// Helper: load truncated A block into dst (zero-pad if out of range)
inline void load_trunc_block(uint64_t* dst, const uint64_t* ap, size_t an,
               size_t off, size_t len) {
  for (size_t i = 0; i < len; i++) {
    size_t idx = off + i;
    dst[i] = (idx < an) ? ap[idx] : 0;
  }
}

// PD precision schedule (GMP rootrem.c style, limb-approximation):
// Brent-Zimmermann strict for cube root: c >= floor(d/2) + 1 (= ⌈(d+log2(3))/2⌉).
// Walking back from target sn: next_c = floor(d/2) + 1.
// Terminate at sn=2 (when sn>=2) so that base_cbrtrem produces 2-limb seed,
// which makes step 1 BZ-strict (c=2,b=1: b ≤ c-1 ✓). For sn=1 (small input),
// base produces 1-limb directly, no refinement.
inline void build_cbrt_schedule(size_t sn, size_t* sched, size_t& ns) {
  sched[0] = sn;
  ns = 0;
  size_t target_min = (sn >= 2) ? 2 : 1;
  while (sched[ns] > target_min && ns < 62) {
    size_t x = sched[ns];
    size_t next = x / 2 + 1;    // BZ-strict for k=3
    if (next >= x) next = x - 1;  // ensure progress
    if (next < target_min) next = target_min;
    sched[ns + 1] = next;
    ns++;
  }
  // Reverse
  for (size_t i = 0, j = ns; i < j; i++, j--) std::swap(sched[i], sched[j]);
}

// Base case: small a (≤ 6 limbs), produce sn limbs of root + remainder
// Uses standard PD newton or basic search
inline size_t base_cbrtrem(uint64_t* sp, uint64_t* rp,
               const uint64_t* ap, size_t an) {
  if (an == 0) { sp[0] = 0; rp[0] = 0; return 0; }
  if (an == 1) {
    uint64_t s = cbrt1(ap[0]);
    sp[0] = s;
    UInt128 s2 = UInt128::multiply(s, s);
    UInt128 s3;
    if (s2.high == 0) {
      s3 = UInt128::multiply(s2.low, s);
    } else {
      s3.low = 0; s3.high = ~0ULL; // overflow shouldn't happen for 1-limb a
    }
    rp[0] = ap[0] - s3.low;
    return (s > 0) ? 1 : 0;
  }
  // Multi-limb base: use double seed + newton iterations
  // Convert top limbs to double
  double da = (an >= 2) ?
    ((double)ap[an-1] * std::pow(2.0, 64.0) + (double)ap[an-2]) :
    (double)ap[0];
  if (an > 2) {
    // shift by remaining limbs
    da *= std::pow(2.0, 64.0 * (an - 2));
  }
  double sd = std::cbrt(da);
  uint64_t s = static_cast<uint64_t>(std::min(sd, std::pow(2.0, 64.0) - 1.0));

  // For multi-limb root (an > 3), can't fit in 1 limb
  // For now, only handle 1-limb root case (an ≤ 3)
  // an ≤ 6 → root ≤ 2 limbs
  size_t sn = (an + 2) / 3;
  if (sn == 1) {
    // double seed -> coarse improvement via integer Newton -> finalize via binary search
    // an <= 3 limbs -> root <= 1 limb. Double precision 53 bit -> ULP error ~2^11
    // binary search guaranteed to converge within 64 iterations

    // Step 1: roughly match via 2-limb integer Newton (s_new = (2*s + a/s^2)/3)
    for (int newton = 0; newton < 32; newton++) {
      if (s == 0) break;
      UInt128 s2 = UInt128::multiply(s, s);
      // a / s²: an=3 limb / s²=1-2 limb
      uint64_t a3[3] = {0, 0, 0};
      for (size_t i = 0; i < an; i++) a3[i] = ap[i];
      uint64_t quot[3] = {0, 0, 0};
      if (s2.high == 0 && s2.low != 0) {
        // 1-limb divisor
        divmod_1(quot, a3, 3, s2.low);
      } else if (s2.high != 0) {
        // 2-limb divisor: use UInt128 long division for top portion
        // a3[2]:a3[1] / s² gives top quotient digit, then refine
        // Simplification: skip newton if s is large (near 2^64)
        break;
      }
      // s_new = (2*s + quot[0]) / 3 (truncation OK)
      uint64_t two_s_lo, two_s_hi;
      two_s_lo = s << 1;
      two_s_hi = s >> 63;
      uint64_t sum_lo;
      unsigned char cc = _addcarry_u64(0, two_s_lo, quot[0], &sum_lo);
      uint64_t sum_hi = two_s_hi + quot[1] + cc;
      uint64_t s_buf[2] = {sum_lo, sum_hi};
      divmod_1(s_buf, s_buf, 2, 3ULL);
      uint64_t s_new = s_buf[0];
      if (s_new == s) break;
      s = s_new;
    }

    // Step 2: binary search over a 2^k interval (search for the correct value around s)
    // Search the range s +/- 2^11 (matches double-seed precision)
    // Speedup: first compute s^3 and compare, then decide direction and do binary search if needed
    auto compute_s3 = [](uint64_t s_val, uint64_t* s3_buf3) {
      UInt128 s2 = UInt128::multiply(s_val, s_val);
      UInt128 lo3 = UInt128::multiply(s2.low, s_val);
      UInt128 hi3;
      if (s2.high != 0) {
        hi3 = UInt128::multiply(s2.high, s_val);
        s3_buf3[0] = lo3.low;
        unsigned char c = _addcarry_u64(0, lo3.high, hi3.low, &s3_buf3[1]);
        s3_buf3[2] = hi3.high + c;
      } else {
        s3_buf3[0] = lo3.low;
        s3_buf3[1] = lo3.high;
        s3_buf3[2] = 0;
      }
    };
    auto cmp_s3_a = [&](uint64_t s_val) -> int {
      uint64_t s3_buf[3] = {0, 0, 0};
      compute_s3(s_val, s3_buf);
      uint64_t a_buf[3] = {0, 0, 0};
      for (size_t i = 0; i < an; i++) a_buf[i] = ap[i];
      for (size_t i = 3; i-- > 0; ) {
        if (s3_buf[i] > a_buf[i]) return 1;
        if (s3_buf[i] < a_buf[i]) return -1;
      }
      return 0;
    };

    // Binary search to find max s such that s³ ≤ a
    // Bracket: s might be slightly off from true root, expand range
    uint64_t lo, hi;
    int cmp_s = cmp_s3_a(s);
    if (cmp_s == 0) {
      // exact
    } else if (cmp_s < 0) {
      // s³ < a: search [s, s + range]
      lo = s;
      uint64_t range = 1;
      hi = s;
      while (range != 0 && hi != ~0ULL) {
        uint64_t new_hi = (hi > ~0ULL - range) ? ~0ULL : (hi + range);
        if (cmp_s3_a(new_hi) > 0) { hi = new_hi; break; }
        lo = new_hi;
        hi = new_hi;
        if (range > (1ULL << 40)) break; // safety
        range <<= 1;
      }
      // Binary search [lo, hi] for max s with s³ ≤ a
      while (lo + 1 < hi) {
        uint64_t mid = lo + (hi - lo) / 2;
        if (cmp_s3_a(mid) <= 0) lo = mid;
        else hi = mid;
      }
      s = lo;
    } else {
      // s³ > a: search [s - range, s]
      uint64_t range = 1;
      lo = s;
      while (range != 0 && lo > 0) {
        uint64_t new_lo = (lo > range) ? (lo - range) : 0;
        if (cmp_s3_a(new_lo) <= 0) { lo = new_lo; break; }
        lo = new_lo;
        if (range > (1ULL << 40)) break;
        range <<= 1;
      }
      hi = s;
      while (lo + 1 < hi) {
        uint64_t mid = lo + (hi - lo) / 2;
        if (cmp_s3_a(mid) <= 0) lo = mid;
        else hi = mid;
      }
      s = lo;
    }

    // Verify and compute remainder
    sp[0] = s;
    if (s == 0) {
      for (size_t i = 0; i < an; i++) rp[i] = ap[i];
      return 0;
    }
    uint64_t s3_final[3] = {0, 0, 0};
    compute_s3(s, s3_final);
    uint64_t a_buf[3] = {0, 0, 0};
    for (size_t i = 0; i < an; i++) a_buf[i] = ap[i];
    unsigned char borrow = 0;
    for (size_t i = 0; i < an; i++) {
      uint64_t a_lo = a_buf[i];
      uint64_t s_lo = s3_final[i];
      uint64_t diff;
      unsigned char b = _subborrow_u64(borrow, a_lo, s_lo, &diff);
      rp[i] = diff;
      borrow = b;
    }
    return 1;

    // (legacy linear search code below — unreachable)
    for (int iter = 0; iter < 32; iter++) {
      // Compute s³
      uint64_t s3_buf[3] = {0, 0, 0};
      UInt128 s2 = UInt128::multiply(s, s);
      // s² = s2.high * B + s2.low. Multiply by s.
      UInt128 lo3 = UInt128::multiply(s2.low, s);
      UInt128 hi3 = UInt128::multiply(s2.high, s);
      s3_buf[0] = lo3.low;
      // s3_buf[1] = lo3.high + hi3.low (with carry)
      unsigned char c = _addcarry_u64(0, lo3.high, hi3.low, &s3_buf[1]);
      s3_buf[2] = hi3.high + c;

      // Compare s3_buf with ap (an limbs). Treat ap as length-3 by zero-pad.
      uint64_t a_buf[3] = {0, 0, 0};
      for (size_t i = 0; i < an; i++) a_buf[i] = ap[i];

      // Compare s3 vs a_buf (3 limbs each)
      int cmp_r = 0;
      for (size_t i = 3; i-- > 0; ) {
        if (s3_buf[i] > a_buf[i]) { cmp_r = 1; break; }
        if (s3_buf[i] < a_buf[i]) { cmp_r = -1; break; }
      }
      if (cmp_r == 0) {
        sp[0] = s; rp[0] = 0;
        return s > 0 ? 1 : 0;
      }
      if (cmp_r > 0) {
        if (s == 0) break;
        s--;
        continue;
      }
      // s³ < a, check (s+1)³
      if (s + 1 == 0) break; // overflow
      uint64_t sp1 = s + 1;
      uint64_t sp13_buf[3] = {0, 0, 0};
      UInt128 sp1_2 = UInt128::multiply(sp1, sp1);
      UInt128 lo_p = UInt128::multiply(sp1_2.low, sp1);
      UInt128 hi_p = UInt128::multiply(sp1_2.high, sp1);
      sp13_buf[0] = lo_p.low;
      unsigned char c2 = _addcarry_u64(0, lo_p.high, hi_p.low, &sp13_buf[1]);
      sp13_buf[2] = hi_p.high + c2;

      int cmp_p = 0;
      for (size_t i = 3; i-- > 0; ) {
        if (sp13_buf[i] > a_buf[i]) { cmp_p = 1; break; }
        if (sp13_buf[i] < a_buf[i]) { cmp_p = -1; break; }
      }
      if (cmp_p > 0) {
        // s³ ≤ a < (s+1)³: s is correct
        sp[0] = s;
        // Compute remainder = a - s³
        unsigned char borrow = 0;
        for (size_t i = 0; i < an; i++) {
          uint64_t a_lo = a_buf[i];
          uint64_t s_lo = s3_buf[i];
          uint64_t diff;
          unsigned char b = _subborrow_u64(borrow, a_lo, s_lo, &diff);
          rp[i] = diff;
          borrow = b;
        }
        size_t rn = normalized_size(rp, an);
        return s > 0 ? 1 : 0;
      }
      s = sp1;
    }
    sp[0] = s; rp[0] = 0;
    return s > 0 ? 1 : 0;
  }
  // sn == 2: an in [4, 6], 2-limb result
  // newton iteration: s_new = (2·s + a/s²) / 3
  if (sn == 2) {
    // Initial seed via double precision on top 2 limbs
    double da_top = (double)ap[an - 1] * std::pow(2.0, 64.0) + (double)ap[an - 2];
    double cbrt_top = std::cbrt(da_top);
    double scale_exp = 64.0 * (double)(an - 2) / 3.0;
    double sd = cbrt_top * std::pow(2.0, scale_exp);

    uint64_t s_buf[3] = {0, 0, 0};
    const double B_dbl = std::pow(2.0, 64.0);
    if (sd < 1.0) sd = 1.0;
    if (sd >= B_dbl) {
      double s_hi = std::floor(sd / B_dbl);
      double s_lo = sd - s_hi * B_dbl;
      s_buf[1] = (uint64_t)s_hi;
      s_buf[0] = (uint64_t)s_lo;
    } else {
      s_buf[0] = (uint64_t)sd;
    }
    size_t slen = (s_buf[1] != 0) ? 2 : (s_buf[0] != 0 ? 1 : 1);

    // newton refinement (max 12 iters; quadratic convergence so few needed)
    for (int iter = 0; iter < 12; iter++) {
      if (slen == 0) break;

      // s²
      uint64_t s2_buf[6] = {0};
      std::vector<uint64_t> sqr_sc(square_scratch_size(slen));
      square(s2_buf, s_buf, slen, sqr_sc.data());
      size_t s2len = normalized_size(s2_buf, 2 * slen);
      if (s2len == 0) break;

      // q = a / s²
      uint64_t q_buf[8] = {0};
      size_t qlen = 0;
      if (an >= s2len) {
        std::vector<uint64_t> div_sc(divide_q_scratch_size(an, s2len));
        qlen = divide_q(q_buf, ap, an, s2_buf, s2len, div_sc.data());
        qlen = normalized_size(q_buf, qlen);
      }

      // 2s
      uint64_t two_s[3] = {0};
      for (size_t i = 0; i < slen; i++) two_s[i] = s_buf[i];
      uint64_t cy_two = mul_1(two_s, two_s, slen, 2ULL);
      size_t two_s_len = slen;
      if (cy_two) { two_s[slen] = cy_two; two_s_len = slen + 1; }

      // sum = 2s + q
      uint64_t sum_buf[8] = {0};
      size_t sum_len = 0;
      if (qlen >= two_s_len) {
        for (size_t i = 0; i < qlen; i++) sum_buf[i] = q_buf[i];
        sum_len = qlen;
        if (two_s_len > 0) {
          uint64_t cy = add(sum_buf, sum_buf, sum_len, two_s, two_s_len);
          if (cy) { sum_buf[sum_len] = cy; sum_len++; }
        }
      } else {
        for (size_t i = 0; i < two_s_len; i++) sum_buf[i] = two_s[i];
        sum_len = two_s_len;
        if (qlen > 0) {
          uint64_t cy = add(sum_buf, sum_buf, sum_len, q_buf, qlen);
          if (cy) { sum_buf[sum_len] = cy; sum_len++; }
        }
      }
      sum_len = normalized_size(sum_buf, sum_len);

      // s_new = sum / 3
      uint64_t s_new_buf[8] = {0};
      if (sum_len > 0) divmod_1(s_new_buf, sum_buf, sum_len, 3ULL);
      size_t s_new_len = normalized_size(s_new_buf, sum_len);

      // Truncate to 2 limbs
      if (s_new_len > 2) s_new_len = 2;

      // Convergence check
      bool same = (s_new_len == slen);
      if (same) {
        for (size_t i = 0; i < slen; i++) {
          if (s_buf[i] != s_new_buf[i]) { same = false; break; }
        }
      }

      // Update s
      for (size_t i = 0; i < 3; i++) s_buf[i] = (i < s_new_len) ? s_new_buf[i] : 0;
      slen = (s_new_len > 0) ? s_new_len : 1;

      if (same) break;
    }

    // ±1 verify: ensure s³ ≤ a < (s+1)³
    auto compute_cube = [&](const uint64_t* sv, size_t svlen, uint64_t* s3_out, size_t* s3_len_out) {
      if (svlen == 0) { *s3_len_out = 0; return; }
      uint64_t s2_t[6] = {0};
      std::vector<uint64_t> sqr_sc(square_scratch_size(svlen));
      square(s2_t, sv, svlen, sqr_sc.data());
      size_t s2len_t = normalized_size(s2_t, 2 * svlen);
      std::vector<uint64_t> mul_sc(multiply_scratch_size(s2len_t, svlen));
      multiply(s3_out, s2_t, s2len_t, sv, svlen, mul_sc.data());
      *s3_len_out = normalized_size(s3_out, s2len_t + svlen);
    };

    uint64_t s3_buf[8] = {0};
    size_t s3len = 0;
    compute_cube(s_buf, slen, s3_buf, &s3len);

    // While s³ > a, s--
    while (slen > 0) {
      bool ok = (an > s3len) || (an == s3len && cmp(ap, an, s3_buf, s3len) >= 0);
      if (ok) break;
      sub_1(s_buf, slen, 1ULL);
      slen = normalized_size(s_buf, slen);
      for (size_t i = 0; i < 8; i++) s3_buf[i] = 0;
      if (slen > 0) compute_cube(s_buf, slen, s3_buf, &s3len);
      else s3len = 0;
    }

    // While (s+1)³ ≤ a, s++
    for (int up = 0; up < 4; up++) {
      uint64_t sp1[3] = {0, 0, 0};
      for (size_t i = 0; i < slen; i++) sp1[i] = s_buf[i];
      size_t sp1_len = (slen == 0) ? 1 : slen;
      if (slen == 0) sp1[0] = 1;
      else {
        uint64_t cy = add_1(sp1, sp1_len, 1ULL);
        if (cy) { sp1[sp1_len] = cy; sp1_len++; }
      }
      if (sp1_len > 2) break;

      uint64_t sp1_3[8] = {0};
      size_t sp1_3_len = 0;
      compute_cube(sp1, sp1_len, sp1_3, &sp1_3_len);

      bool sp1_ok = (an > sp1_3_len) ||
             (an == sp1_3_len && cmp(ap, an, sp1_3, sp1_3_len) >= 0);
      if (!sp1_ok) break;

      // Increment s
      for (size_t i = 0; i < 3; i++) s_buf[i] = (i < sp1_len) ? sp1[i] : 0;
      slen = sp1_len;
      for (size_t i = 0; i < 8; i++) s3_buf[i] = (i < sp1_3_len) ? sp1_3[i] : 0;
      s3len = sp1_3_len;
    }

    // Output: sp[0..1], remainder rp
    sp[0] = (slen > 0) ? s_buf[0] : 0;
    sp[1] = (slen > 1) ? s_buf[1] : 0;
    for (size_t i = 0; i < an; i++) rp[i] = ap[i];
    if (s3len > 0) sub(rp, rp, an, s3_buf, s3len);

    return slen;
  }

  // sn >= 3: not implemented in base case. Caller should ensure sn ≤ 2 in base.
  sp[0] = 0;
  return 0;
}

} // namespace cbrt_detail

inline size_t cbrtrem_scratch_size(size_t an) {
  size_t sn = (an + 2) / 3;
  if (sn < 2) return 16;
  // R: 2*sn+1, W: 2*sn+1, T: 2*sn+2, D: 2*sn+2 (used as Q + div scratch + q3 etc)
  // div scratch upper bound
  size_t div_sz = (2 * sn >= 2) ? divide_scratch_size(2 * sn + 1, 2 * sn) : 16;
  // mul/sqr scratch
  size_t mul_sz = multiply_scratch_size(2 * sn, sn);
  size_t sq_sz = square_scratch_size(sn);
  return (2 * sn + 1) + (2 * sn + 1) + (2 * sn + 2) +
      std::max({div_sz, mul_sz, sq_sz}) + 32;
}

// For diagnostics: per-step clamp / repair statistics
struct CbrtRemDiag {
  size_t total_steps = 0;
  size_t total_clamps = 0;
  size_t total_repairs = 0;
  size_t max_repair_iters = 0;
  // Per-step detail (last 16 steps)
  size_t step_b[16] = {0};
  size_t step_c[16] = {0};
  int step_clamp[16] = {0};
  size_t step_repair[16] = {0};
  size_t recorded = 0;
};
inline thread_local CbrtRemDiag g_cbrtrem_diag = {};

// mpn_cbrtrem: floor(cbrt(a))
inline size_t cbrtrem(uint64_t* sp, uint64_t* rp, size_t* rp_n_out,
            const uint64_t* ap, size_t an,
            uint64_t* scratch) {
  g_cbrtrem_diag = {};
  if (an == 0 || (an == 1 && ap[0] == 0)) {
    if (rp_n_out) *rp_n_out = 0;
    return 0;
  }

  size_t sn = (an + 2) / 3; // ceil(an/3)
  size_t tn = 3 * sn;

  // Build schedule
  size_t sched[64];
  size_t ns = 0;
  cbrt_detail::build_cbrt_schedule(sn, sched, ns);
  // sched[0..ns] = base→target

  // Buffer layout
  // R always uses scratch (internally needs 2*sn+1 limbs, finally copied to rp)
  // Old implementation: R = rp ? rp : scratch, but if the user passed a small rp,
  // internal memset  heap-buffer-overflow → ASan detect (2026-05-08)
  uint64_t* S = sp;
  uint64_t* R = scratch;
  uint64_t* W = scratch + (2 * sn + 1);
  uint64_t* T = W + (2 * sn + 1);
  uint64_t* D = T + (2 * sn + 2); // D used as Q buffer + div scratch + q^2/q^3

  // ----- Base case -----
  size_t c0 = sched[0];
  uint64_t base_buf[8]; // Up to 6 limbs for base input (3*c0 ≤ 6)
  cbrt_detail::load_trunc_block(base_buf, ap, an, tn - 3 * c0, 3 * c0);
  uint64_t base_rem[8];
  size_t base_an = normalized_size(base_buf, 3 * c0);
  if (base_an == 0) base_an = 1;
  size_t slen = cbrt_detail::base_cbrtrem(S, base_rem, base_buf, base_an);
  // slen should be ≤ c0
  // Place remainder in R
  std::memset(R, 0, (2 * sn + 1) * sizeof(uint64_t));
  size_t rlen = (2 * c0 + 1 < 8) ? (2 * c0 + 1) : 8;
  rlen = std::min(rlen, base_an);
  for (size_t i = 0; i < rlen; i++) R[i] = base_rem[i];
  rlen = normalized_size(R, rlen);

  // Compute W = 3 * S^2
  std::memset(W, 0, (2 * sn + 1) * sizeof(uint64_t));
  if (slen > 0) {
    std::vector<uint64_t> sqr_sc(square_scratch_size(slen));
    square(T, S, slen, sqr_sc.data());
    size_t s2len = normalized_size(T, 2 * slen);
    uint64_t cy = mul_1(W, T, s2len, 3ULL);
    if (cy) W[s2len] = cy;
  }
  size_t wlen = normalized_size(W, 2 * sn + 1);

  // ----- Refinement loop (GMP rootrem.c style) -----
  // Each step:
  //  1. Q = floor(N / W) where N = R*B^b + T2 - divide_q (quotient only)
  //  2. Clamp if Q overflows (rare with strict schedule)
  //  3. S = S·B^b + Q (in-place)
  //  4. Recompute S² and S³ from scratch (GMP  mpn_pow_1 equivalent)
  //  5. R = A_trunc - S³ with at most ±1 correction (Q -= 1 if overshot)
  //  6. W = 3*S^2 (recompute)
  for (size_t step = 1; step <= ns; step++) {
    size_t c = sched[step - 1];
    size_t d = sched[step];
    size_t b = d - c;
    size_t off = tn - 3 * d;
    g_cbrtrem_diag.total_steps++;
    size_t diag_idx = (step <= 16) ? step - 1 : 15;
    g_cbrtrem_diag.step_c[diag_idx] = c;
    g_cbrtrem_diag.step_b[diag_idx] = b;
    if (step >= ns - 7 || step <= 8) g_cbrtrem_diag.recorded = std::max(g_cbrtrem_diag.recorded, diag_idx + 1);
    size_t this_step_repair_iters = 0;
    bool this_step_clamped = false;

    // (1) Build N = R·B^b + T2 in T buffer (T2 = ap[off+2b..off+3b-1])
    std::memset(T, 0, (2 * d + 2) * sizeof(uint64_t));
    cbrt_detail::load_trunc_block(T, ap, an, off + 2 * b, b);
    for (size_t i = 0; i < rlen; i++) T[b + i] = R[i];
    size_t nlen = normalized_size(T, b + rlen);

    // (2) Q = floor(N / W) - divide_q (quotient only, no remainder computation)
    uint64_t* Q = D;
    size_t qlen = 0;
    if (nlen >= wlen && wlen >= 1) {
      if (wlen == 1 && W[0] != 0) {
        divmod_1(Q, T, nlen, W[0]);
        qlen = nlen;
      } else if (wlen >= 2) {
        size_t div_sc_sz = divide_q_scratch_size(nlen, wlen);
        std::vector<uint64_t> div_sc(div_sc_sz);
        qlen = divide_q(Q, T, nlen, W, wlen, div_sc.data());
      }
      qlen = normalized_size(Q, qlen);
    }

    // (2b) Clamp: Q < B^b (BZ-strict schedule  rare)
    bool clamped = false;
    if (qlen > b) {
      for (size_t i = 0; i < b; i++) Q[i] = ~0ULL;
      qlen = b;
      clamped = true;
      g_cbrtrem_diag.total_clamps++;
      this_step_clamped = true;
    }

    // (3) S = S·B^b + Q (in-place)
    for (size_t i = slen; i-- > 0; ) S[i + b] = S[i];
    for (size_t i = 0; i < b; i++) S[i] = (i < qlen) ? Q[i] : 0;
    slen = slen + b;
    slen = normalized_size(S, slen);

    // (4) Recompute S² and S³ from scratch (GMP  mpn_pow_1 equivalent)
    std::vector<uint64_t> S2(2 * slen + 2, 0);
    size_t s2len = 0;
    std::vector<uint64_t> S3(3 * slen + 2, 0);
    size_t s3len = 0;
    if (slen > 0) {
      std::vector<uint64_t> sqr_sc(square_scratch_size(slen));
      square(S2.data(), S, slen, sqr_sc.data());
      s2len = normalized_size(S2.data(), 2 * slen);
      std::vector<uint64_t> mul_sc(multiply_scratch_size(s2len, slen));
      multiply(S3.data(), S2.data(), s2len, S, slen, mul_sc.data());
      s3len = normalized_size(S3.data(), s2len + slen);
    }

    // (5) R = A_trunc - S³ with at most ±1 correction
    std::vector<uint64_t> A_trunc(3 * d, 0);
    cbrt_detail::load_trunc_block(A_trunc.data(), ap, an, off, 3 * d);
    size_t at_len = normalized_size(A_trunc.data(), 3 * d);

    // Check: S³ ≤ A_trunc?
    bool nonneg = (at_len > s3len) ||
           (at_len == s3len && cmp(A_trunc.data(), at_len, S3.data(), s3len) >= 0);

    // Under the BZ-strict precondition, correction should not be needed. Major overshoots can occur only on clamp.
    // First try simple 1-correction, then smart bulk if that fails
    if (!nonneg) {
      g_cbrtrem_diag.total_repairs++;
      size_t repair_iters = 0;
      const int simple_iters_cap = 4;
      int max_repair = 64;
      while (max_repair-- > 0) {
        repair_iters++;
        if (slen == 0) break;

        bool use_smart = ((int)repair_iters > simple_iters_cap);
        if (!use_smart) {
          sub_1(S, slen, 1);
          slen = normalized_size(S, slen);
        } else {
          // Smart bulk: delta = ceil((S³ - A_trunc) / (3·S²))
          std::vector<uint64_t> overshoot(s3len + 1, 0);
          for (size_t i = 0; i < s3len; i++) overshoot[i] = S3[i];
          if (at_len > 0) sub(overshoot.data(), overshoot.data(), s3len, A_trunc.data(), at_len);
          size_t over_len = normalized_size(overshoot.data(), s3len);

          std::vector<uint64_t> three_s2(s2len + 2, 0);
          for (size_t i = 0; i < s2len; i++) three_s2[i] = S2[i];
          uint64_t cy_ts2 = mul_1(three_s2.data(), three_s2.data(), s2len, 3ULL);
          if (cy_ts2) three_s2[s2len] = cy_ts2;
          size_t ts2_len = normalized_size(three_s2.data(), s2len + 1);

          std::vector<uint64_t> delta(slen + 2, 0);
          size_t delta_len = 0;
          if (ts2_len == 0 || over_len == 0 || over_len < ts2_len ||
            (over_len == ts2_len && cmp(overshoot.data(), over_len, three_s2.data(), ts2_len) < 0)) {
            delta[0] = 1; delta_len = 1;
          } else {
            size_t qmax = over_len - ts2_len + 1;
            std::vector<uint64_t> q_buf(qmax + 1, 0);
            std::vector<uint64_t> r_buf(ts2_len + 1, 0);
            size_t qq_len = 0;
            if (ts2_len == 1) {
              uint64_t rr = divmod_1(q_buf.data(), overshoot.data(), over_len, three_s2[0]);
              qq_len = over_len;
              r_buf[0] = rr;
            } else {
              std::vector<uint64_t> div_sc(divide_scratch_size(over_len, ts2_len));
              qq_len = divide(q_buf.data(), r_buf.data(), overshoot.data(), over_len,
                      three_s2.data(), ts2_len, div_sc.data());
            }
            qq_len = normalized_size(q_buf.data(), qq_len);
            size_t rr_len = normalized_size(r_buf.data(), ts2_len);
            if (rr_len > 0 && qq_len > 0) {
              uint64_t cy = add_1(q_buf.data(), qq_len, 1ULL);
              if (cy) { q_buf[qq_len] = cy; qq_len++; }
            } else if (rr_len > 0 && qq_len == 0) {
              q_buf[0] = 1; qq_len = 1;
            }
            if (qq_len == 0) {
              delta[0] = 1; delta_len = 1;
            } else {
              size_t copy_len = std::min(qq_len, slen + 1);
              for (size_t i = 0; i < copy_len; i++) delta[i] = q_buf[i];
              delta_len = copy_len;
            }
          }

          if (delta_len > slen) { std::memset(S, 0, slen * sizeof(uint64_t)); slen = 0; break; }
          uint64_t bo_s = sub(S, S, slen, delta.data(), delta_len);
          if (bo_s) { std::memset(S, 0, slen * sizeof(uint64_t)); slen = 0; break; }
          slen = normalized_size(S, slen);
        }

        if (slen == 0) break;

        // Recompute S^2 and S^3
        S2.assign(2 * slen + 2, 0);
        std::vector<uint64_t> sqr_sc(square_scratch_size(slen));
        square(S2.data(), S, slen, sqr_sc.data());
        s2len = normalized_size(S2.data(), 2 * slen);
        S3.assign(3 * slen + 2, 0);
        std::vector<uint64_t> mul_sc(multiply_scratch_size(s2len, slen));
        multiply(S3.data(), S2.data(), s2len, S, slen, mul_sc.data());
        s3len = normalized_size(S3.data(), s2len + slen);

        nonneg = (at_len > s3len) ||
             (at_len == s3len && cmp(A_trunc.data(), at_len, S3.data(), s3len) >= 0);
        if (nonneg) break;
      }
      if (repair_iters > g_cbrtrem_diag.max_repair_iters)
        g_cbrtrem_diag.max_repair_iters = repair_iters;
      this_step_repair_iters = repair_iters;
    }

    // R = A_trunc - S³
    std::memset(R, 0, (2 * d + 2) * sizeof(uint64_t));
    if (slen > 0) {
      for (size_t i = 0; i < at_len; i++) R[i] = A_trunc[i];
      if (s3len > 0) sub(R, R, at_len, S3.data(), s3len);
      rlen = normalized_size(R, at_len);
    } else {
      rlen = 0;
    }

    size_t diag_idx2 = (step <= 16) ? step - 1 : 15;
    g_cbrtrem_diag.step_clamp[diag_idx2] = this_step_clamped ? 1 : 0;
    g_cbrtrem_diag.step_repair[diag_idx2] = this_step_repair_iters;

    // (6) W = 3·S² (post-correction S² (reused))
    std::memset(W, 0, (2 * sn + 1) * sizeof(uint64_t));
    if (slen > 0 && s2len > 0) {
      uint64_t cy3 = mul_1(W, S2.data(), s2len, 3ULL);
      if (cy3 && s2len < 2 * sn + 1) W[s2len] = cy3;
      wlen = normalized_size(W, 2 * sn + 1);
    } else {
      wlen = 0;
    }
  }

  // Final normalize
  slen = normalized_size(S, sn);
  // Copy remainder from internal R to rp (rp needs only rlen limbs)
  if (rp) {
    for (size_t i = 0; i < rlen; i++) rp[i] = R[i];
  }
  if (rp_n_out) *rp_n_out = rlen;
  return slen;
}

// ============================================================================
// Schönhage-Strassen FFT multiplication
// ============================================================================
//
// NTT-based Schoenhage-Strassen FFT. Performs convolution over Z/(B^F+1)Z.
// B = 2^64 (1 word).
//
// Split the input into M=2^l pieces of K words each,
// use omega = B^(2F/M) as the M-th root of unity, then NTT -> pointwise multiplication -> inverse NTT -> carry propagation.
//
// Pointwise multiplication size F ~ 2N/M << N, so via multiply() 
// recursive FFT calls naturally shrink the size, so no re-entry guard is needed.
//
// references:
//  - Schönhage, Strassen: "Schnelle Multiplikation großer Zahlen" (1971)
//  - Knuth TAOCP Vol.2 §4.3.3.C
// ============================================================================

namespace fft_detail {

// Two's complement negation: rp = -ap mod B^n = B^n - ap
// return value: 1 (ap != 0), 0 (ap == 0)
inline uint64_t neg_n(uint64_t* rp, const uint64_t* ap, size_t n) {
  size_t i = 0;
  while (i < n && ap[i] == 0) {
    rp[i] = 0;
    i++;
  }
  if (i == n) return 0; // all zero
  rp[i] = (~ap[i]) + 1; // negate first non-zero limb
  for (++i; i < n; i++) {
    rp[i] = ~ap[i]; // complement remaining
  }
  return 1;
}

// ================================================================
// FFT parameter selection
// ================================================================

struct FftParams {
  size_t M;  // piececount (= 2^l)
  size_t K;  // word count per piece
  size_t F;  // word count of the residue ring (B^F+1)
  size_t l;  // log2(M)
};

// Relative cost estimate for F*F multiplication. Reflects the actual multiplication thresholds.
// basecase O(F²), Karatsuba O(F^1.585), TC3 O(F^1.465), TC4 O(F^1.404)
inline size_t fft_pointwise_cost(size_t F) {
  if (F < KARATSUBA_THRESHOLD) return F * F;      // basecase
  if (F < TOOMCOOK3_THRESHOLD) return F * F * 3 / 4;  // Karatsuba ~25% saved
  if (F < TOOMCOOK4_THRESHOLD) return F * F * 9 / 16;  // TC3 ~44% saved
  return F * F * 7 / 16;                 // TC4 ~56% saved
}

inline FftParams fft_choose_params(size_t an, size_t bn) {
  size_t N = an + bn;
  // Search for optimal l: M=2^l pieces, K words each, residue ring of F words
  // Bit-level twiddle: because 2^(128F/M) becomes an integer bit shift
  // F % (M/128) == 0 required (any F is fine when M <= 128)
  // Cost = M * pointwise_multiply(F) + M * l * butterfly_cost(F)

  // Initial estimate for l: M ~ sqrt(N)
  size_t l0 = 1;
  while ((1ULL << (2 * l0)) < N) ++l0;

  // Pick the best in the range l0-2 to l0+4
  // (With bit-level twiddle, smaller F can make larger l optimal)
  size_t best_cost = SIZE_MAX;
  FftParams best = {};
  size_t lo = (l0 > 4) ? l0 - 2 : 3;

  for (size_t l = lo; l <= l0 + 4; ++l) {
    size_t M = 1ULL << l;
    size_t K = (N + M - 1) / M;
    if (K < 1) continue;
    size_t min_F = 2 * K + (l + 63) / 64 + 1;
    // Quantize F to a multiple of M/128 (no constraint when M <= 128)
    size_t quant = (M <= 128) ? 1 : (M >> 7);
    size_t F = ((min_F + quant - 1) / quant) * quant;

    // pointwise: M F*F multiplications
    // butterfly: M*l times, each O(F) word operations (add_mod + sub_mod + bitshift_mod)
    // pw_cost >> 5: correctly reflects multiplication cost when F is large (Toom-4 range)
    size_t pw_cost = fft_pointwise_cost(F);
    size_t cost = M * (pw_cost >> 5) + M * l * (F >> 2);

    if (cost < best_cost) {
      best_cost = cost;
      best = {M, K, F, l};
    }
  }
  return best;
}

// ================================================================
// mod (B^F + 1) arithmetic
// ================================================================
// Each element is F+1 words. a[F] is 0 or 1.
// When a[F] == 1, a[0..F-1] are all 0 (value = B^F).

// (a + b) mod (B^F+1)
// r, a, b: F+1 words. r may equal a.
inline void fft_add_mod(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t F) {
  uint64_t cy = add(r, a, F, b, F);
  cy += a[F] + b[F];
  r[F] = 0;
  // total = cy * B^F + r. Since B^F == -1 mod (B^F+1), total == r - cy.
  // Subtract cy (0..3) from r; if negative, add B^F+1 to normalize.
  if (cy > 0) {
    uint64_t bw = sub_1(r, F, cy);
    if (bw) {
      // negative → (B^F+1) addition: r + 1 (sub_1 borrow subsequent r = B^F + r_old - cy)
      r[F] = add_1(r, F, 1);
    }
  }
}

// (a - b) mod (B^F+1)
// r, a, b: F+1 words. r may equal a.
inline void fft_sub_mod(uint64_t* r, const uint64_t* a, const uint64_t* b, size_t F) {
  uint64_t borrow = sub(r, a, F, b, F);
  // hi = a[F] - b[F] - borrow (-2..1)
  int64_t hi = (int64_t)a[F] - (int64_t)b[F] - (int64_t)borrow;
  r[F] = 0;
  // total = hi * B^F + r. Since B^F == -1, total == r - hi mod (B^F+1).
  if (hi > 0) {
    // subtract hi from r
    uint64_t bw = sub_1(r, F, (uint64_t)hi);
    if (bw) {
      r[F] = add_1(r, F, 1);
    }
  } else if (hi < 0) {
    // r  |hi| addition
    uint64_t cy = add_1(r, F, (uint64_t)(-hi));
    if (cy) {
      // overflow -> reduce B^F + r': r' - 1, B^F if borrow
      uint64_t bw = sub_1(r, F, 1);
      if (bw) {
        r[F] = add_1(r, F, 1);
      }
    }
  }
}

// -a mod (B^F+1)
inline void fft_negate_mod(uint64_t* r, const uint64_t* a, size_t F) {
  if (a[F] != 0) {
    // a = B^F → -B^F mod (B^F+1) = 1
    std::memset(r, 0, F * sizeof(uint64_t));
    r[0] = 1;
    r[F] = 0;
    return;
  }
  // if a[0..F-1] are all 0, result is 0
  uint64_t nonzero = neg_n(r, a, F);
  if (nonzero) {
    // -a mod (B^F+1) = (B^F+1) - a = (B^F - a) + 1 = neg_n + 1
    // when carry=1 and a=1 -> result is B^F (r[F]=1, r[0..F-1]=0)
    r[F] = add_1(r, F, 1);
  } else {
    r[F] = 0;
  }
}

// a * B^j mod (B^F+1), j in word units (0 <= j < 2F)
// r != a (not in-place). r, a: F+1 words.
inline void fft_shift_mod(uint64_t* r, const uint64_t* a, size_t F, size_t j) {
  j %= (2 * F);
  bool negate = (j >= F);
  if (negate) j -= F;

  if (j == 0) {
    if (negate) {
      fft_negate_mod(r, a, F);
    } else {
      std::memcpy(r, a, (F + 1) * sizeof(uint64_t));
    }
    return;
  }

  // Handling of a[F]: a[F]*B^F == -a[F] mod (B^F+1)
  // Therefore a[F]*B^(F+j) == -a[F]*B^j

  // B^j * a[0..F-1] mod B^F+1:
  //  upper: a[0..F-j-1] -> place into r[j..F-1]
  //  lower: a[F-j..F-1] -> negate and place into r[0..j-1] (B^F == -1)
  //
  // r[j..F-1] = a[0..F-j-1]
  std::memcpy(r + j, a, (F - j) * sizeof(uint64_t));
  // r[0..j-1] = -a[F-j..F-1] mod B^j
  uint64_t borrow_from_neg = neg_n(r, a + (F - j), j);

  r[F] = 0;

  // borrow_from_neg == 0 means a[F-j..F-1] were all 0
  // borrow_from_neg == 1 means the negation happened -> subtract 1 from r[j..F-1] (borrow propagation)
  if (borrow_from_neg) {
    uint64_t bw = sub_1(r + j, F - j, 1);
    // if bw occurred -> r[j..F-1] were all 0 -> wrap around
    // Correct by adding B^F+1: +1 to all of r[0..F-1], carry -> r[F]
    if (bw) {
      r[F] = add_1(r, F, 1);
    }
  }

  // Handling of a[F]: a[F]*B^(F+j) == -a[F]*B^j
  // When a[F] == 1, a[0..F-1] are all 0, so by the above r is all 0
  // -B^j mod (B^F+1): subtract 1 from r[j..F-1] (subtract B^j)
  if (a[F] != 0) {
    uint64_t bw = sub_1(r + j, F - j, 1);
    if (bw) {
      r[F] += add_1(r, F, 1);
    }
  }

  // Normalize: r[F] >= 2 -> r -= (B^F+1)
  while (r[F] >= 2) {
    sub_1(r, F, 1);
    r[F] -= 1;
  }
  // When r[F] == 1, r[0..F-1] should all be 0 (normalization check)

  if (negate) {
    // in-place negate mod (B^F+1)
    if (r[F] != 0) {
      // r = B^F → -B^F mod (B^F+1) = 1
      std::memset(r, 0, F * sizeof(uint64_t));
      r[0] = 1;
      r[F] = 0;
    } else {
      uint64_t nonzero = neg_n(r, r, F);
      if (nonzero) {
        // (B^F+1) - r_old = neg_n(r_old) + 1
        // when carry=1 and r_old=1 -> result is B^F
        r[F] = add_1(r, F, 1);
      }
    }
  }
}

// a * 2^total_bits mod (B^F+1), r != a (not in-place)
// Bit-level twiddle: 2-stage word shift + tail bit shift
// r, a: F+1 word
inline void fft_bitshift_mod(uint64_t* r, const uint64_t* a,
               size_t F, size_t total_bits) {
  total_bits %= (128 * F); // period = 2*64*F bits
  size_t word_shift = total_bits / 64;
  size_t bit_shift = total_bits % 64;

  // Step 1: word shift (existing function)
  fft_shift_mod(r, a, F, word_shift);

  // Step 2: bit shift in-place
  if (bit_shift == 0) return;

  if (r[F] != 0) {
    // r = B^F → r * 2^s ≡ -(2^s) mod (B^F+1)
    // = (B^F + 1) - 2^s (s >= 1)
    // r[0] = 1 - 2^s (two's complement wrap), r[1..F-1] = UINT64_MAX, r[F] = 0
    r[0] = (uint64_t)1 - ((uint64_t)1 << bit_shift);
    std::memset(r + 1, 0xFF, (F - 1) * sizeof(uint64_t));
    r[F] = 0;
    return;
  }

  // r[F] == 0: lshift + overflow fold
  // overflow * B^F ≡ -overflow mod (B^F+1) → r -= overflow
  uint64_t overflow = lshift(r, r, F, (unsigned)bit_shift);
  if (overflow) {
    uint64_t bw = sub_1(r, F, overflow);
    if (bw) r[F] = add_1(r, F, 1);
  }
}

// a * b mod (B^F+1): full F*F multiplication -> fold
// r, a, b: F+1 words. scratch: 2*F + multiply_scratch_size(F,F) words
inline void fft_mulpoint(uint64_t* r, const uint64_t* a, const uint64_t* b,
             size_t F, uint64_t* scratch) {
  // Special case: a[F]!=0 or b[F]!=0
  if (a[F] != 0 && b[F] != 0) {
    // a = b = B^F → a*b = B^(2F) ≡ 1 mod (B^F+1)
    std::memset(r, 0, (F + 1) * sizeof(uint64_t));
    r[0] = 1;
    return;
  }
  if (a[F] != 0) {
    // a = B^F → a*b = B^F * b ≡ -b mod (B^F+1)
    fft_negate_mod(r, b, F);
    return;
  }
  if (b[F] != 0) {
    fft_negate_mod(r, a, F);
    return;
  }

  // General: full multiplication -> fold
  uint64_t* prod = scratch;
  uint64_t* mul_scratch = scratch + 2 * F;
  multiply(prod, a, F, b, F, mul_scratch);

  // fold: r[0..F-1] = prod[0..F-1] - prod[F..2F-1] mod (B^F+1)
  // Since B^F == -1, subtract the upper F words from the lower
  uint64_t borrow = sub(r, prod, F, prod + F, F);
  r[F] = 0;
  if (borrow) {
    // negative -> add (B^F+1): adding 1 to the sub result completes the (B^F+1) addition
    // when carry=1, result is B^F (r[0..F-1]=0, r[F]=1)
    r[F] = add_1(r, F, 1);
  }
}

// a^2 mod (B^F+1): full F*F squaring -> fold
inline void fft_sqrpoint(uint64_t* r, const uint64_t* a,
             size_t F, uint64_t* scratch) {
  if (a[F] != 0) {
    // a = B^F → a² = B^(2F) ≡ 1 mod (B^F+1)
    std::memset(r, 0, (F + 1) * sizeof(uint64_t));
    r[0] = 1;
    return;
  }

  uint64_t* prod = scratch;
  uint64_t* sqr_scratch = scratch + 2 * F;
  square(prod, a, F, sqr_scratch);

  uint64_t borrow = sub(r, prod, F, prod + F, F);
  r[F] = 0;
  if (borrow) {
    r[F] = add_1(r, F, 1);
  }
}

// Divide by M = 2^l mod (B^F+1)
// B^F+1 is odd -> 2^(-1) exists. Each halving: if odd, add B^F+1 to make even, then right-shift by 1 bit.
inline void fft_div_by_power_of_2(uint64_t* a, size_t F, size_t l) {
  for (size_t i = 0; i < l; i++) {
    if (a[0] & 1) {
      // a += (B^F + 1) → a[0..F-1] += 1, a[F] += 1
      uint64_t cy = add_1(a, F, 1);
      a[F] += 1 + cy;
    }
    // a[F] is at most 2 (because the addition runs only when a[F]=0, with cy in {0,1}).
    // The right shift is applied to all F+1 words, so even a[F]=2 is processed correctly:
    //  a[F]=2, a[0..F-1]=0 -> right shift -> a[F]=1, a[0..F-1]=0 = B^F
    // previously while loop (sub_1 + a[F]-=1)  borrow unprocess 
    // corrupting 2*B^F into 2*B^F-1 (odd).
#ifdef SANGI_INT_HAS_ASM
    mpn_rshift_asm(a, a, F + 1, 1);
#else
    for (size_t w = 0; w < F; w++) {
      a[w] = (a[w] >> 1) | (a[w + 1] << 63);
    }
    a[F] = a[F] >> 1;
#endif
  }
}

// ================================================================
// FFT Forward / Inverse NTT
// ================================================================

// Forward NTT (DIF — Decimation in Frequency)
// data: M elements of (F+1) words each, placed contiguously
// temp: F+1-word temporary buffer
inline void fft_forward(uint64_t* data, size_t M, size_t F, size_t l, uint64_t* temp) {
  size_t stride = F + 1;
  for (int s = (int)l - 1; s >= 0; --s) {
    size_t half = 1ULL << s;
    size_t block = half << 1;
    for (size_t k = 0; k < M; k += block) {
      for (size_t m = 0; m < half; ++m) {
        uint64_t* u = data + (k + m) * stride;
        uint64_t* v = data + (k + m + half) * stride;
        // twiddle: omega^(m * M/block) = 2^(m * 128F / block) (in bits)
        size_t tw_bits = m * (128 * F) / block;

        // DIF butterfly: temp = u - v; u = u + v; v = shift(temp, tw_bits)
        fft_sub_mod(temp, u, v, F);
        fft_add_mod(u, u, v, F);
        if (tw_bits != 0)
          fft_bitshift_mod(v, temp, F, tw_bits);
        else
          std::memcpy(v, temp, stride * sizeof(uint64_t));
      }
    }
  }
}

// Inverse NTT (DIT — Decimation in Time)
// data: M elements of (F+1) words each
// temp: F+1-word temporary buffer
inline void fft_inverse(uint64_t* data, size_t M, size_t F, size_t l, uint64_t* temp) {
  size_t stride = F + 1;
  for (size_t s = 0; s < l; ++s) {
    size_t half = 1ULL << s;
    size_t block = half << 1;
    for (size_t k = 0; k < M; k += block) {
      for (size_t m = 0; m < half; ++m) {
        uint64_t* u = data + (k + m) * stride;
        uint64_t* v = data + (k + m + half) * stride;
        size_t tw_bits = m * (128 * F) / block;
        size_t inv_tw_bits = (tw_bits == 0) ? 0 : (128 * F - tw_bits);

        // DIT butterfly: temp = shift(v, inv_tw); v = u - temp; u = u + temp
        if (inv_tw_bits != 0)
          fft_bitshift_mod(temp, v, F, inv_tw_bits);
        else
          std::memcpy(temp, v, stride * sizeof(uint64_t));
        fft_sub_mod(v, u, temp, F);
        fft_add_mod(u, u, temp, F);
      }
    }
  }
  // M = 2^l division
  for (size_t i = 0; i < M; ++i) {
    fft_div_by_power_of_2(data + i * stride, F, l);
  }
}

// ================================================================
// Input split / result assembly
// ================================================================

// Split input a[0..an-1] into M K-word pieces; zero-fill each to F+1 words
inline void fft_split(uint64_t* data, const uint64_t* a, size_t an,
            size_t M, size_t K, size_t F) {
  size_t stride = F + 1;
  for (size_t i = 0; i < M; ++i) {
    uint64_t* dst = data + i * stride;
    size_t src_off = i * K;
    size_t copy_len = 0;
    if (src_off < an) {
      copy_len = std::min(K, an - src_off);
      std::memcpy(dst, a + src_off, copy_len * sizeof(uint64_t));
    }
    // zero-fill the remainder
    std::memset(dst + copy_len, 0, (stride - copy_len) * sizeof(uint64_t));
  }
}

// Assemble the result from coefficients after the inverse NTT
// Accumulate each c_i into rp[i*K..]. c_i is at most F+1 words.
// full_rn: internal buffer size (M*K + F or more). Prevents truncation.
// The caller copies actual_rn words from rp.
inline void fft_recompose(uint64_t* rp, size_t full_rn, const uint64_t* data,
              size_t M, size_t K, size_t F) {
  size_t stride = F + 1;
  std::memset(rp, 0, full_rn * sizeof(uint64_t));

  for (size_t i = 0; i < M; ++i) {
    const uint64_t* coeff = data + i * stride;
    size_t base = i * K;
    if (base >= full_rn) break;

    // coefficients validwordcount
    size_t cn = F + 1;
    while (cn > 0 && coeff[cn - 1] == 0) --cn;
    if (cn == 0) continue;

    // Add into rp[base..] (no truncation: full_rn is sufficiently large)
    size_t space = full_rn - base;
    size_t add_len = std::min(cn, space);
    add(rp + base, rp + base, space, coeff, add_len);
  }
}

// ================================================================
// mul_fft / sqr_fft
// ================================================================

// FFT multiplication: rp[0..an+bn-1] = ap[0..an-1] * bp[0..bn-1]
// thread_local buffer avoids heap allocation. Since F < FFT_THRESHOLD, recursive FFT does not occur.
// NTT parallelization threshold: when both operands are >= this, execute forward NTT in parallel
static constexpr size_t FFT_PARALLEL_THRESHOLD = 8000;

inline void mul_fft(uint64_t* rp,
          const uint64_t* ap, size_t an,
          const uint64_t* bp, size_t bn) {
  if (an < bn) { std::swap(ap, bp); std::swap(an, bn); }

  auto p = fft_choose_params(an, bn);
  size_t stride = p.F + 1;
  size_t data_sz = p.M * stride;
  size_t mul_scratch_sz = 2 * p.F + multiply_scratch_size(p.F, p.F);

  bool parallel = (bn >= FFT_PARALLEL_THRESHOLD);
  // When parallel, allocate 2 temps (one for each forward NTT)
  size_t temp_count = parallel ? 2 : 1;
  size_t total = 2 * data_sz + mul_scratch_sz + stride * temp_count;

  thread_local std::vector<uint64_t> work;
  if (work.size() < total) work.resize(total);

  uint64_t* data_a = work.data();
  uint64_t* data_b = work.data() + data_sz;
  uint64_t* scratch = work.data() + 2 * data_sz;
  uint64_t* temp = scratch + mul_scratch_sz;
  uint64_t* temp2 = parallel ? temp + stride : nullptr;

  fft_split(data_a, ap, an, p.M, p.K, p.F);
  fft_split(data_b, bp, bn, p.M, p.K, p.F);

  if (parallel) {
    // Execute the two forward NTTs in parallel (separate temp buffers)
    auto future_b = sangi::threadPool().submit([&]() {
      fft_forward(data_b, p.M, p.F, p.l, temp2);
    });
    fft_forward(data_a, p.M, p.F, p.l, temp);
    future_b.get();
  } else {
    fft_forward(data_a, p.M, p.F, p.l, temp);
    fft_forward(data_b, p.M, p.F, p.l, temp);
  }

  for (size_t i = 0; i < p.M; ++i) {
    fft_mulpoint(data_a + i * stride, data_a + i * stride,
           data_b + i * stride, p.F, scratch);
  }

  fft_inverse(data_a, p.M, p.F, p.l, temp);

  // Reuse data_b as the recompose buffer (no longer needed after pointwise)
  size_t rn = an + bn;
  size_t full_rn = p.M * p.K + p.F;
  uint64_t* recomp_buf = data_b;
  fft_recompose(recomp_buf, full_rn, data_a, p.M, p.K, p.F);
  std::memcpy(rp, recomp_buf, rn * sizeof(uint64_t));
}

// FFT squaring: rp[0..2n-1] = ap[0..n-1]^2
inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an) {
  auto p = fft_choose_params(an, an);
  size_t stride = p.F + 1;
  size_t data_sz = p.M * stride;
  size_t sqr_scratch_sz = 2 * p.F + square_scratch_size(p.F);
  size_t full_rn = p.M * p.K + p.F;
  size_t total = data_sz + sqr_scratch_sz + stride + full_rn;

  thread_local std::vector<uint64_t> work;
  if (work.size() < total) work.resize(total);

  uint64_t* data_a = work.data();
  uint64_t* scratch = work.data() + data_sz;
  uint64_t* temp = scratch + sqr_scratch_sz;
  uint64_t* recomp_buf = temp + stride;

  fft_split(data_a, ap, an, p.M, p.K, p.F);

  fft_forward(data_a, p.M, p.F, p.l, temp);

  for (size_t i = 0; i < p.M; ++i) {
    fft_sqrpoint(data_a + i * stride, data_a + i * stride, p.F, scratch);
  }

  fft_inverse(data_a, p.M, p.F, p.l, temp);

  size_t rn = 2 * an;
  fft_recompose(recomp_buf, full_rn, data_a, p.M, p.K, p.F);
  std::memcpy(rp, recomp_buf, rn * sizeof(uint64_t));
}

} // namespace fft_detail

// Public interface: wrapper calling fft_detail::
inline void mul_fft(uint64_t* rp,
          const uint64_t* ap, size_t an,
          const uint64_t* bp, size_t bn) {
  fft_detail::mul_fft(rp, ap, an, bp, bn);
}

inline void sqr_fft(uint64_t* rp, const uint64_t* ap, size_t an) {
  fft_detail::sqr_fft(rp, ap, an);
}

} // namespace mpn
} // namespace sangi
