Lines Matching refs:EQ
56 template <class Type, PetscInt BS, PetscInt EQ>
59 …const PetscInt M = (EQ) ? 1 : bs / BS; /* If EQ, then M=1 enables compiler's const-propagation */ in d_Pack()
69 template <class Type, class Op, PetscInt BS, PetscInt EQ>
72 const PetscInt M = (EQ) ? 1 : bs / BS, MBS = M * BS; in d_UnpackAndOp()
82 template <class Type, class Op, PetscInt BS, PetscInt EQ>
85 const PetscInt M = (EQ) ? 1 : bs / BS, MBS = M * BS; in d_FetchAndOp()
95 template <class Type, class Op, PetscInt BS, PetscInt EQ>
98 const PetscInt M = (EQ) ? 1 : bs / BS, MBS = M * BS; in d_ScatterAndOp()
130 template <class Type, class Op, PetscInt BS, PetscInt EQ>
133 const PetscInt M = (EQ) ? 1 : bs / BS, MBS = M * BS; in d_FetchAndOpLocal()
275 template <typename Type, PetscInt BS, PetscInt EQ>
285 …PetscCall(PetscCUPMLaunchKernel1D(count, 0, link->stream, kernels::d_Pack<Type, BS, EQ>, link->bs,… in Pack()
291 template <typename Type, class Op, PetscInt BS, PetscInt EQ>
301 …aunchKernel1D(count, 0, link->stream, kernels::d_UnpackAndOp<Type, Op, BS, EQ>, link->bs, count, s… in UnpackAndOp()
307 template <typename Type, class Op, PetscInt BS, PetscInt EQ>
314 …PetscCall(PetscCUPMLaunchKernel1D(count, 0, link->stream, kernels::d_FetchAndOp<Type, Op, BS, EQ>,… in FetchAndOp()
319 template <typename Type, class Op, PetscInt BS, PetscInt EQ>
355 …unchKernel1D(count, 0, link->stream, kernels::d_ScatterAndOp<Type, Op, BS, EQ>, link->bs, count, s… in ScatterAndOp()
361 template <typename Type, PetscInt BS, PetscInt EQ>
370 …PetscCall(ScatterAndOp<Type, kernels::Insert<Type>, BS, EQ>(link, count, srcStart, srcOpt, srcIdx,… in ScatterAndInsert()
376 template <typename Type, class Op, PetscInt BS, PetscInt EQ>
384 …hKernel1D(count, 0, link->stream, kernels::d_FetchAndOpLocal<Type, Op, BS, EQ>, link->bs, count, r… in FetchAndOpLocal()
392 template <typename Type, PetscInt BS, PetscInt EQ>
396 link->d_Pack = Pack<Type, BS, EQ>; in PackInit_RealType()
397 link->d_UnpackAndInsert = UnpackAndOp<Type, kernels::Insert<Type>, BS, EQ>; in PackInit_RealType()
398 link->d_UnpackAndAdd = UnpackAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_RealType()
399 link->d_UnpackAndMult = UnpackAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_RealType()
400 link->d_UnpackAndMin = UnpackAndOp<Type, kernels::Min<Type>, BS, EQ>; in PackInit_RealType()
401 link->d_UnpackAndMax = UnpackAndOp<Type, kernels::Max<Type>, BS, EQ>; in PackInit_RealType()
402 link->d_FetchAndAdd = FetchAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_RealType()
405 link->d_ScatterAndInsert = ScatterAndInsert<Type, BS, EQ>; /* Has special optimizations */ in PackInit_RealType()
406 link->d_ScatterAndAdd = ScatterAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_RealType()
407 link->d_ScatterAndMult = ScatterAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_RealType()
408 link->d_ScatterAndMin = ScatterAndOp<Type, kernels::Min<Type>, BS, EQ>; in PackInit_RealType()
409 link->d_ScatterAndMax = ScatterAndOp<Type, kernels::Max<Type>, BS, EQ>; in PackInit_RealType()
410 link->d_FetchAndAddLocal = FetchAndOpLocal<Type, kernels::Add<Type>, BS, EQ>; in PackInit_RealType()
413 link->da_UnpackAndInsert = UnpackAndOp<Type, AtomicInsert<Type>, BS, EQ>; in PackInit_RealType()
414 link->da_UnpackAndAdd = UnpackAndOp<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_RealType()
415 link->da_UnpackAndMult = UnpackAndOp<Type, AtomicMult<Type>, BS, EQ>; in PackInit_RealType()
416 link->da_UnpackAndMin = UnpackAndOp<Type, AtomicMin<Type>, BS, EQ>; in PackInit_RealType()
417 link->da_UnpackAndMax = UnpackAndOp<Type, AtomicMax<Type>, BS, EQ>; in PackInit_RealType()
418 link->da_FetchAndAdd = FetchAndOp<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_RealType()
420 link->da_ScatterAndInsert = ScatterAndOp<Type, AtomicInsert<Type>, BS, EQ>; in PackInit_RealType()
421 link->da_ScatterAndAdd = ScatterAndOp<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_RealType()
422 link->da_ScatterAndMult = ScatterAndOp<Type, AtomicMult<Type>, BS, EQ>; in PackInit_RealType()
423 link->da_ScatterAndMin = ScatterAndOp<Type, AtomicMin<Type>, BS, EQ>; in PackInit_RealType()
424 link->da_ScatterAndMax = ScatterAndOp<Type, AtomicMax<Type>, BS, EQ>; in PackInit_RealType()
425 link->da_FetchAndAddLocal = FetchAndOpLocal<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_RealType()
430 template <typename Type, PetscInt BS, PetscInt EQ, PetscInt size /*sizeof(Type)*/>
434 link->da_UnpackAndInsert = UnpackAndOp<Type, AtomicInsert<Type>, BS, EQ>; in Init()
435 link->da_UnpackAndAdd = UnpackAndOp<Type, AtomicAdd<Type>, BS, EQ>; in Init()
436 link->da_UnpackAndMult = UnpackAndOp<Type, AtomicMult<Type>, BS, EQ>; in Init()
437 link->da_UnpackAndMin = UnpackAndOp<Type, AtomicMin<Type>, BS, EQ>; in Init()
438 link->da_UnpackAndMax = UnpackAndOp<Type, AtomicMax<Type>, BS, EQ>; in Init()
439 link->da_UnpackAndLAND = UnpackAndOp<Type, AtomicLAND<Type>, BS, EQ>; in Init()
440 link->da_UnpackAndLOR = UnpackAndOp<Type, AtomicLOR<Type>, BS, EQ>; in Init()
441 link->da_UnpackAndLXOR = UnpackAndOp<Type, AtomicLXOR<Type>, BS, EQ>; in Init()
442 link->da_UnpackAndBAND = UnpackAndOp<Type, AtomicBAND<Type>, BS, EQ>; in Init()
443 link->da_UnpackAndBOR = UnpackAndOp<Type, AtomicBOR<Type>, BS, EQ>; in Init()
444 link->da_UnpackAndBXOR = UnpackAndOp<Type, AtomicBXOR<Type>, BS, EQ>; in Init()
445 link->da_FetchAndAdd = FetchAndOp<Type, AtomicAdd<Type>, BS, EQ>; in Init()
447 link->da_ScatterAndInsert = ScatterAndOp<Type, AtomicInsert<Type>, BS, EQ>; in Init()
448 link->da_ScatterAndAdd = ScatterAndOp<Type, AtomicAdd<Type>, BS, EQ>; in Init()
449 link->da_ScatterAndMult = ScatterAndOp<Type, AtomicMult<Type>, BS, EQ>; in Init()
450 link->da_ScatterAndMin = ScatterAndOp<Type, AtomicMin<Type>, BS, EQ>; in Init()
451 link->da_ScatterAndMax = ScatterAndOp<Type, AtomicMax<Type>, BS, EQ>; in Init()
452 link->da_ScatterAndLAND = ScatterAndOp<Type, AtomicLAND<Type>, BS, EQ>; in Init()
453 link->da_ScatterAndLOR = ScatterAndOp<Type, AtomicLOR<Type>, BS, EQ>; in Init()
454 link->da_ScatterAndLXOR = ScatterAndOp<Type, AtomicLXOR<Type>, BS, EQ>; in Init()
455 link->da_ScatterAndBAND = ScatterAndOp<Type, AtomicBAND<Type>, BS, EQ>; in Init()
456 link->da_ScatterAndBOR = ScatterAndOp<Type, AtomicBOR<Type>, BS, EQ>; in Init()
457 link->da_ScatterAndBXOR = ScatterAndOp<Type, AtomicBXOR<Type>, BS, EQ>; in Init()
458 link->da_FetchAndAddLocal = FetchAndOpLocal<Type, AtomicAdd<Type>, BS, EQ>; in Init()
464 template <typename Type, PetscInt BS, PetscInt EQ>
465 struct SfInterface<T>::PackInit_IntegerType_Atomic<Type, BS, EQ, 1> {
470 template <typename Type, PetscInt BS, PetscInt EQ>
473 link->d_Pack = Pack<Type, BS, EQ>; in PackInit_IntegerType()
474 link->d_UnpackAndInsert = UnpackAndOp<Type, kernels::Insert<Type>, BS, EQ>; in PackInit_IntegerType()
475 link->d_UnpackAndAdd = UnpackAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_IntegerType()
476 link->d_UnpackAndMult = UnpackAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_IntegerType()
477 link->d_UnpackAndMin = UnpackAndOp<Type, kernels::Min<Type>, BS, EQ>; in PackInit_IntegerType()
478 link->d_UnpackAndMax = UnpackAndOp<Type, kernels::Max<Type>, BS, EQ>; in PackInit_IntegerType()
479 link->d_UnpackAndLAND = UnpackAndOp<Type, kernels::LAND<Type>, BS, EQ>; in PackInit_IntegerType()
480 link->d_UnpackAndLOR = UnpackAndOp<Type, kernels::LOR<Type>, BS, EQ>; in PackInit_IntegerType()
481 link->d_UnpackAndLXOR = UnpackAndOp<Type, kernels::LXOR<Type>, BS, EQ>; in PackInit_IntegerType()
482 link->d_UnpackAndBAND = UnpackAndOp<Type, kernels::BAND<Type>, BS, EQ>; in PackInit_IntegerType()
483 link->d_UnpackAndBOR = UnpackAndOp<Type, kernels::BOR<Type>, BS, EQ>; in PackInit_IntegerType()
484 link->d_UnpackAndBXOR = UnpackAndOp<Type, kernels::BXOR<Type>, BS, EQ>; in PackInit_IntegerType()
485 link->d_FetchAndAdd = FetchAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_IntegerType()
487 link->d_ScatterAndInsert = ScatterAndInsert<Type, BS, EQ>; in PackInit_IntegerType()
488 link->d_ScatterAndAdd = ScatterAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_IntegerType()
489 link->d_ScatterAndMult = ScatterAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_IntegerType()
490 link->d_ScatterAndMin = ScatterAndOp<Type, kernels::Min<Type>, BS, EQ>; in PackInit_IntegerType()
491 link->d_ScatterAndMax = ScatterAndOp<Type, kernels::Max<Type>, BS, EQ>; in PackInit_IntegerType()
492 link->d_ScatterAndLAND = ScatterAndOp<Type, kernels::LAND<Type>, BS, EQ>; in PackInit_IntegerType()
493 link->d_ScatterAndLOR = ScatterAndOp<Type, kernels::LOR<Type>, BS, EQ>; in PackInit_IntegerType()
494 link->d_ScatterAndLXOR = ScatterAndOp<Type, kernels::LXOR<Type>, BS, EQ>; in PackInit_IntegerType()
495 link->d_ScatterAndBAND = ScatterAndOp<Type, kernels::BAND<Type>, BS, EQ>; in PackInit_IntegerType()
496 link->d_ScatterAndBOR = ScatterAndOp<Type, kernels::BOR<Type>, BS, EQ>; in PackInit_IntegerType()
497 link->d_ScatterAndBXOR = ScatterAndOp<Type, kernels::BXOR<Type>, BS, EQ>; in PackInit_IntegerType()
498 link->d_FetchAndAddLocal = FetchAndOpLocal<Type, kernels::Add<Type>, BS, EQ>; in PackInit_IntegerType()
499 PackInit_IntegerType_Atomic<Type, BS, EQ, sizeof(Type)>::Init(link); in PackInit_IntegerType()
504 template <typename Type, PetscInt BS, PetscInt EQ>
507 link->d_Pack = Pack<Type, BS, EQ>; in PackInit_ComplexType()
508 link->d_UnpackAndInsert = UnpackAndOp<Type, kernels::Insert<Type>, BS, EQ>; in PackInit_ComplexType()
509 link->d_UnpackAndAdd = UnpackAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_ComplexType()
510 link->d_UnpackAndMult = UnpackAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_ComplexType()
511 link->d_FetchAndAdd = FetchAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_ComplexType()
513 link->d_ScatterAndInsert = ScatterAndInsert<Type, BS, EQ>; in PackInit_ComplexType()
514 link->d_ScatterAndAdd = ScatterAndOp<Type, kernels::Add<Type>, BS, EQ>; in PackInit_ComplexType()
515 link->d_ScatterAndMult = ScatterAndOp<Type, kernels::Mult<Type>, BS, EQ>; in PackInit_ComplexType()
516 link->d_FetchAndAddLocal = FetchAndOpLocal<Type, kernels::Add<Type>, BS, EQ>; in PackInit_ComplexType()
518 link->da_UnpackAndInsert = UnpackAndOp<Type, AtomicInsert<Type>, BS, EQ>; in PackInit_ComplexType()
519 link->da_UnpackAndAdd = UnpackAndOp<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_ComplexType()
523 link->da_ScatterAndInsert = ScatterAndOp<Type, AtomicInsert<Type>, BS, EQ>; in PackInit_ComplexType()
524 link->da_ScatterAndAdd = ScatterAndOp<Type, AtomicAdd<Type>, BS, EQ>; in PackInit_ComplexType()
555 template <typename Type, PetscInt BS, PetscInt EQ>
558 link->d_Pack = Pack<Type, BS, EQ>; in PackInit_DumbType()
559 link->d_UnpackAndInsert = UnpackAndOp<Type, kernels::Insert<Type>, BS, EQ>; in PackInit_DumbType()
560 link->d_ScatterAndInsert = ScatterAndInsert<Type, BS, EQ>; in PackInit_DumbType()