1#![allow(clippy::float_cmp)]
137#![allow(clippy::inline_always)]
142#![cfg_attr(
146 not(any(target_arch = "aarch64", target_arch = "x86", target_arch = "x86_64")),
147 allow(dead_code)
148)]
149
150use pulp::Simd;
151
152use crate::dd::DoubleDouble;
153use crate::eft::two_prod;
154use crate::kernels::{ColumnUpdate, Compensated, LANES, reference};
155
156pub(crate) trait Lanes: Copy {
158 type V: Copy;
160 fn splat(self, x: f64) -> Self::V;
162 fn load(self, a: [f64; LANES]) -> Self::V;
164 fn store(self, v: Self::V) -> [f64; LANES];
166 fn add(self, a: Self::V, b: Self::V) -> Self::V;
168 fn sub(self, a: Self::V, b: Self::V) -> Self::V;
170 fn mul(self, a: Self::V, b: Self::V) -> Self::V;
172 fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V;
174 fn neg(self, a: Self::V) -> Self::V;
176 fn abs(self, a: Self::V) -> Self::V;
178 fn all_finite(self, a: Self::V, b: Self::V) -> bool;
180}
181
182#[derive(Clone, Copy, Debug)]
184pub(crate) struct X4<S: Simd>(S);
185
186impl<S: Simd> X4<S> {
187 #[inline(always)]
189 #[cfg_attr(
190 not(any(target_arch = "x86", target_arch = "x86_64")),
191 allow(dead_code)
192 )]
193 pub(crate) fn new(simd: S) -> Self {
194 const { assert!(S::F64_LANES == LANES) };
195 Self(simd)
196 }
197}
198
199impl<S: Simd> Lanes for X4<S> {
200 type V = S::f64s;
201
202 #[inline(always)]
203 fn splat(self, x: f64) -> Self::V {
204 self.0.splat_f64s(x)
205 }
206 #[inline(always)]
207 fn load(self, a: [f64; LANES]) -> Self::V {
208 pulp::cast(a)
209 }
210 #[inline(always)]
211 fn store(self, v: Self::V) -> [f64; LANES] {
212 pulp::cast(v)
213 }
214 #[inline(always)]
215 fn add(self, a: Self::V, b: Self::V) -> Self::V {
216 self.0.add_f64s(a, b)
217 }
218 #[inline(always)]
219 fn sub(self, a: Self::V, b: Self::V) -> Self::V {
220 self.0.sub_f64s(a, b)
221 }
222 #[inline(always)]
223 fn mul(self, a: Self::V, b: Self::V) -> Self::V {
224 self.0.mul_f64s(a, b)
225 }
226 #[inline(always)]
227 fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V {
228 self.0.mul_add_f64s(a, b, c)
229 }
230 #[inline(always)]
231 fn neg(self, a: Self::V) -> Self::V {
232 self.0.neg_f64s(a)
233 }
234 #[inline(always)]
235 fn abs(self, a: Self::V) -> Self::V {
236 self.0.abs_f64s(a)
237 }
238 #[inline(always)]
239 fn all_finite(self, a: Self::V, b: Self::V) -> bool {
240 let s = self.0;
242 let z = s.add_f64s(s.sub_f64s(a, a), s.sub_f64s(b, b));
243 s.reduce_sum_f64s(z) == 0.0
244 }
245}
246
247#[derive(Clone, Copy, Debug)]
250pub(crate) struct X2<S: Simd>(S);
251
252impl<S: Simd> X2<S> {
253 #[inline(always)]
255 #[cfg_attr(not(target_arch = "aarch64"), allow(dead_code))]
256 pub(crate) fn new(simd: S) -> Self {
257 const { assert!(2 * S::F64_LANES == LANES) };
258 Self(simd)
259 }
260}
261
262impl<S: Simd> Lanes for X2<S> {
263 type V = [S::f64s; 2];
264
265 #[inline(always)]
266 fn splat(self, x: f64) -> Self::V {
267 [self.0.splat_f64s(x); 2]
268 }
269 #[inline(always)]
270 fn load(self, a: [f64; LANES]) -> Self::V {
271 [pulp::cast([a[0], a[1]]), pulp::cast([a[2], a[3]])]
272 }
273 #[inline(always)]
274 fn store(self, v: Self::V) -> [f64; LANES] {
275 let (lo, hi): ([f64; 2], [f64; 2]) = (pulp::cast(v[0]), pulp::cast(v[1]));
276 [lo[0], lo[1], hi[0], hi[1]]
277 }
278 #[inline(always)]
279 fn add(self, a: Self::V, b: Self::V) -> Self::V {
280 [self.0.add_f64s(a[0], b[0]), self.0.add_f64s(a[1], b[1])]
281 }
282 #[inline(always)]
283 fn sub(self, a: Self::V, b: Self::V) -> Self::V {
284 [self.0.sub_f64s(a[0], b[0]), self.0.sub_f64s(a[1], b[1])]
285 }
286 #[inline(always)]
287 fn mul(self, a: Self::V, b: Self::V) -> Self::V {
288 [self.0.mul_f64s(a[0], b[0]), self.0.mul_f64s(a[1], b[1])]
289 }
290 #[inline(always)]
291 fn mul_add(self, a: Self::V, b: Self::V, c: Self::V) -> Self::V {
292 [
293 self.0.mul_add_f64s(a[0], b[0], c[0]),
294 self.0.mul_add_f64s(a[1], b[1], c[1]),
295 ]
296 }
297 #[inline(always)]
298 fn neg(self, a: Self::V) -> Self::V {
299 [self.0.neg_f64s(a[0]), self.0.neg_f64s(a[1])]
300 }
301 #[inline(always)]
302 fn abs(self, a: Self::V) -> Self::V {
303 [self.0.abs_f64s(a[0]), self.0.abs_f64s(a[1])]
304 }
305 #[inline(always)]
306 fn all_finite(self, a: Self::V, b: Self::V) -> bool {
307 let s = self.0;
309 let z0 = s.add_f64s(s.sub_f64s(a[0], a[0]), s.sub_f64s(b[0], b[0]));
310 let z1 = s.add_f64s(s.sub_f64s(a[1], a[1]), s.sub_f64s(b[1], b[1]));
311 s.reduce_sum_f64s(s.add_f64s(z0, z1)) == 0.0
312 }
313}
314
315type Dq<L> = (<L as Lanes>::V, <L as Lanes>::V);
317
318#[inline(always)]
320fn two_sum_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
321 let s = l.add(a, b);
322 let a1 = l.sub(s, b);
323 let b1 = l.sub(s, a1);
324 let da = l.sub(a, a1);
325 let db = l.sub(b, b1);
326 (s, l.add(da, db))
327}
328
329#[inline(always)]
331fn fast_two_sum_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
332 let s = l.add(a, b);
333 let z = l.sub(s, a);
334 (s, l.sub(b, z))
335}
336
337#[inline(always)]
339fn two_prod_q<L: Lanes>(l: L, a: L::V, b: L::V) -> (L::V, L::V) {
340 let p = l.mul(a, b);
341 (p, l.mul_add(a, b, l.neg(p)))
342}
343
344#[inline(always)]
346fn add_q<L: Lanes>(l: L, x: Dq<L>, y: Dq<L>) -> Dq<L> {
347 let (sh, sl) = two_sum_q(l, x.0, y.0);
348 let (th, tl) = two_sum_q(l, x.1, y.1);
349 let c = l.add(sl, th);
350 let (vh, vl) = fast_two_sum_q(l, sh, c);
351 let w = l.add(tl, vl);
352 fast_two_sum_q(l, vh, w)
353}
354
355#[inline(always)]
357fn mul_q<L: Lanes>(l: L, x: Dq<L>, y: Dq<L>) -> Dq<L> {
358 let (ch, cl1) = two_prod_q(l, x.0, y.0);
359 let tl0 = l.mul(x.1, y.1);
360 let tl1 = l.mul_add(x.0, y.1, tl0);
361 let cl2 = l.mul_add(x.1, y.0, tl1);
362 let cl3 = l.add(cl1, cl2);
363 fast_two_sum_q(l, ch, cl3)
364}
365
366#[inline(always)]
368fn mul_f64_q<L: Lanes>(l: L, x: Dq<L>, y: L::V) -> Dq<L> {
369 let (ch, cl1) = two_prod_q(l, x.0, y);
370 let cl3 = l.mul_add(x.1, y, cl1);
371 fast_two_sum_q(l, ch, cl3)
372}
373
374#[inline(always)]
376fn load_dd<L: Lanes>(l: L, c: &[DoubleDouble; LANES]) -> Dq<L> {
377 (
378 l.load([c[0].hi(), c[1].hi(), c[2].hi(), c[3].hi()]),
379 l.load([c[0].lo(), c[1].lo(), c[2].lo(), c[3].lo()]),
380 )
381}
382
383#[inline(always)]
386fn store_dd<L: Lanes>(l: L, z: Dq<L>) -> [DoubleDouble; LANES] {
387 let (h, lo) = (l.store(z.0), l.store(z.1));
388 [
389 DoubleDouble::from_raw(h[0], lo[0]),
390 DoubleDouble::from_raw(h[1], lo[1]),
391 DoubleDouble::from_raw(h[2], lo[2]),
392 DoubleDouble::from_raw(h[3], lo[3]),
393 ]
394}
395
396#[inline(always)]
398fn splat_dd<L: Lanes>(l: L, x: DoubleDouble) -> Dq<L> {
399 (l.splat(x.hi()), l.splat(x.lo()))
400}
401
402#[inline(always)]
404pub(crate) fn sum2<L: Lanes>(l: L, x: &[f64]) -> Compensated {
405 let (chunks, tail) = x.as_chunks::<LANES>();
406 let zero = l.splat(0.0);
407 let (mut p, mut s) = (zero, zero);
408 for c in chunks {
409 let (pn, q) = two_sum_q(l, p, l.load(*c));
410 p = pn;
411 s = l.add(s, l.add(q, zero));
412 }
413 let mut acc = Compensated {
414 p: l.store(p),
415 s: l.store(s),
416 e: [0.0; LANES],
417 };
418 for (j, &xi) in tail.iter().enumerate() {
419 acc.push(j, xi, 0.0);
420 }
421 acc
422}
423
424#[inline(always)]
427pub(crate) fn dot2<L: Lanes, const BOUND: bool>(l: L, x: &[f64], y: &[f64]) -> Compensated {
428 debug_assert_eq!(x.len(), y.len(), "dot2: the lengths differ");
429 let (cx, tx) = x.as_chunks::<LANES>();
430 let (cy, ty) = y.as_chunks::<LANES>();
431 let zero = l.splat(0.0);
432 let (mut p, mut s, mut e) = (zero, zero, zero);
433 for (a, b) in cx.iter().zip(cy) {
434 let (g, r) = two_prod_q(l, l.load(*a), l.load(*b));
435 let (pn, q) = two_sum_q(l, p, g);
436 let t = l.add(q, r);
437 p = pn;
438 s = l.add(s, t);
439 if BOUND {
440 e = l.add(e, l.abs(t));
441 }
442 }
443 let mut acc = Compensated {
444 p: l.store(p),
445 s: l.store(s),
446 e: l.store(e),
447 };
448 for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
449 let (h, r) = two_prod(xi, yi);
450 acc.push(j, h, r);
451 }
452 acc
453}
454
455#[inline(always)]
457pub(crate) fn dot<L: Lanes>(l: L, x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
458 debug_assert_eq!(x.len(), y.len(), "dot: the lengths differ");
459 let (cx, tx) = x.as_chunks::<LANES>();
460 let (cy, ty) = y.as_chunks::<LANES>();
461 let mut a = splat_dd(l, DoubleDouble::ZERO);
462 for (p, q) in cx.iter().zip(cy) {
463 a = add_q(l, a, mul_q(l, load_dd(l, p), load_dd(l, q)));
464 }
465 if !l.all_finite(a.0, a.1) {
466 return reference::dot(x, y);
467 }
468 let mut acc = store_dd(l, a);
469 for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
470 acc[j] += xi * yi;
471 }
472 ((acc[0] + acc[1]) + acc[2]) + acc[3]
473}
474
475#[inline(always)]
478#[cfg_attr(not(feature = "sparse"), allow(dead_code))]
479pub(crate) fn dot4<L: Lanes>(l: L, x: &[DoubleDouble], y: &[DoubleDouble]) -> DoubleDouble {
480 debug_assert_eq!(x.len(), y.len(), "dot4: the lengths differ");
481 let (cx, tx) = x.as_chunks::<LANES>();
482 let (cy, ty) = y.as_chunks::<LANES>();
483 let mut a = splat_dd(l, DoubleDouble::ZERO);
484 for (p, q) in cx.iter().zip(cy) {
485 a = add_q(l, a, mul_q(l, load_dd(l, p), load_dd(l, q)));
486 }
487 if !l.all_finite(a.0, a.1) {
488 return reference::dot4(x, y);
489 }
490 let acc = store_dd(l, a);
491 let mut t = DoubleDouble::ZERO;
492 for (&p, &q) in tx.iter().zip(ty) {
493 t += p * q;
494 }
495 (acc[0] + acc[1]) + (acc[2] + acc[3]) + t
496}
497
498#[inline(always)]
500pub(crate) fn dot_mixed<L: Lanes>(l: L, x: &[DoubleDouble], y: &[f64]) -> DoubleDouble {
501 debug_assert_eq!(x.len(), y.len(), "dot_mixed: the lengths differ");
502 let (cx, tx) = x.as_chunks::<LANES>();
503 let (cy, ty) = y.as_chunks::<LANES>();
504 let mut a = splat_dd(l, DoubleDouble::ZERO);
505 for (p, q) in cx.iter().zip(cy) {
506 a = add_q(l, a, mul_f64_q(l, load_dd(l, p), l.load(*q)));
507 }
508 if !l.all_finite(a.0, a.1) {
509 return reference::dot_mixed(x, y);
510 }
511 let mut acc = store_dd(l, a);
512 for (j, (&xi, &yi)) in tx.iter().zip(ty).enumerate() {
513 acc[j] += xi * yi;
514 }
515 ((acc[0] + acc[1]) + acc[2]) + acc[3]
516}
517
518#[inline(always)]
520pub(crate) fn axpy<L: Lanes>(
521 l: L,
522 alpha: DoubleDouble,
523 x: &[DoubleDouble],
524 y: &mut [DoubleDouble],
525) {
526 debug_assert_eq!(x.len(), y.len(), "axpy: the lengths differ");
527 let (cx, tx) = x.as_chunks::<LANES>();
528 let (cy, ty) = y.as_chunks_mut::<LANES>();
529 let al = splat_dd(l, alpha);
530 for (p, q) in cx.iter().zip(cy) {
531 let z = add_q(l, load_dd(l, q), mul_q(l, al, load_dd(l, p)));
532 if l.all_finite(z.0, z.1) {
533 *q = store_dd(l, z);
534 } else {
535 reference::axpy(alpha, p, q);
536 }
537 }
538 reference::axpy(alpha, tx, ty);
539}
540
541impl<L: Lanes> ColumnUpdate for L {
543 #[inline(always)]
544 fn column_sub(self, c: &mut [DoubleDouble], a: &[DoubleDouble], t: DoubleDouble) {
545 debug_assert_eq!(c.len(), a.len(), "column_sub: the lengths differ");
546 let l = self;
547 let (cc, tc) = c.as_chunks_mut::<LANES>();
548 let (ca, ta) = a.as_chunks::<LANES>();
549 let tq = splat_dd(l, t);
550 for (q, p) in cc.iter_mut().zip(ca) {
551 let prod = mul_q(l, load_dd(l, p), tq);
552 let z = add_q(l, load_dd(l, q), (l.neg(prod.0), l.neg(prod.1)));
553 if l.all_finite(z.0, z.1) {
554 *q = store_dd(l, z);
555 } else {
556 reference::Reference.column_sub(q, p, t);
557 }
558 }
559 reference::Reference.column_sub(tc, ta, t);
560 }
561}
562
563#[cfg(test)]
564mod tests {
565 use super::*;
566
567 fn lanes_follow_f64<L: Lanes>(l: L) {
570 let a = [1.0 + f64::EPSILON, -3.5e200, 1e-310, f64::INFINITY];
571 let b = [1.0 - f64::EPSILON / 2.0, 2.25e150, -7.0, 1.0];
572 let c = [-1.0, f64::MAX, 3e-320, f64::NAN];
573 let (va, vb, vc) = (l.load(a), l.load(b), l.load(c));
574 let same = |v: L::V, f: &dyn Fn(usize) -> f64| {
575 for (k, got) in l.store(v).into_iter().enumerate() {
576 let want = f(k);
577 assert!(
578 got.to_bits() == want.to_bits() || (got.is_nan() && want.is_nan()),
579 "lane {k}: {got:e} != {want:e}"
580 );
581 }
582 };
583 same(va, &|k| a[k]);
584 same(l.splat(0.1), &|_| 0.1);
585 same(l.add(va, vb), &|k| a[k] + b[k]);
586 same(l.sub(va, vb), &|k| a[k] - b[k]);
587 same(l.mul(va, vb), &|k| a[k] * b[k]);
588 same(l.mul_add(va, vb, vc), &|k| a[k].mul_add(b[k], c[k]));
589 same(l.neg(vc), &|k| -c[k]);
590 same(l.abs(vb), &|k| b[k].abs());
591 let fused = l.store(l.mul_add(va, vb, l.splat(-1.0)))[0];
594 assert_eq!(fused, a[0].mul_add(b[0], -1.0));
595 assert_ne!(fused, a[0] * b[0] - 1.0);
596 assert!(l.all_finite(l.splat(1.0), l.load([0.0, -0.0, f64::MAX, 5e-324])));
597 for bad in [f64::INFINITY, f64::NEG_INFINITY, f64::NAN] {
598 for k in 0..LANES {
599 let mut v = [1.0; LANES];
600 v[k] = bad;
601 assert!(!l.all_finite(l.load(v), l.splat(0.0)));
602 assert!(!l.all_finite(l.splat(0.0), l.load(v)));
603 }
604 }
605 }
606
607 #[test]
608 fn the_native_backend_follows_f64() {
609 #[cfg(target_arch = "aarch64")]
610 if let Some(token) = pulp::aarch64::Neon::try_new() {
611 token.vectorize(|| lanes_follow_f64(X2::new(token)));
612 }
613 #[cfg(any(target_arch = "x86", target_arch = "x86_64"))]
614 if let Some(token) = pulp::x86::V3::try_new() {
615 token.vectorize(|| lanes_follow_f64(X4::new(token)));
616 }
617 }
618}