Lines Matching +full:- +full:- +full:ceed
1 // Copyright (c) 2017-2026, Lawrence Livermore National Security, LLC and other CEED contributors.
2 // All Rights Reserved. See the top-level LICENSE and NOTICE files for details.
4 // SPDX-License-Identifier: BSD-2-Clause
6 // This file is part of CEED: http://github.com/ceed
8 #include <ceed.h>
9 #include <ceed/backend.h>
10 #include <ceed/jit-tools.h>
14 #include "../hip/ceed-hip-common.h"
15 #include "../hip/ceed-hip-compile.h"
17 #include "../cuda/ceed-cuda-common.h"
18 #include "../cuda/ceed-cuda-compile.h"
20 #include "ceed-magma-common.h"
21 #include "ceed-magma.h"
23 #include "ceed-magma-gemm-nontensor.h"
24 #include "ceed-magma-gemm-selector.h"
26 //------------------------------------------------------------------------------
27 // Basis apply - tensor
28 //------------------------------------------------------------------------------
31 Ceed ceed; in CeedBasisApplyCore_Magma() local
38 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisApplyCore_Magma()
39 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisApplyCore_Magma()
55 …else CeedCheck(e_mode == CEED_EVAL_WEIGHT, ceed, CEED_ERROR_BACKEND, "An input vector is required … in CeedBasisApplyCore_Magma()
66 // E-vector ordering -------------- Q-vector ordering in CeedBasisApplyCore_Magma()
71 // --- Define strides for NOTRANSPOSE mode: --- in CeedBasisApplyCore_Magma()
72 // Input (d_u) is E-vector, output (d_v) is Q-vector in CeedBasisApplyCore_Magma()
81 // Input (d_u) is Q-vector, output (d_v) is E-vector in CeedBasisApplyCore_Magma()
117 …void *args[] = {&impl->d_interp_1d, &d_u, &u_elem_stride, &u_comp_stride, &d_v, &v_elem_stride, … in CeedBasisApplyCore_Magma()
120 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, apply_add ? impl->InterpTransposeAdd : impl->Int… in CeedBasisApplyCore_Magma()
123 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Interp, NULL, grid, num_threads, num_t_col… in CeedBasisApplyCore_Magma()
132 // d_u is (P^dim x nc), column-major layout (nc = num_comp) in CeedBasisApplyCore_Magma()
133 // d_v is (Q^dim x nc x dim), column-major layout (nc = num_comp) in CeedBasisApplyCore_Magma()
136 // E-vector ordering -------------- Q-vector ordering in CeedBasisApplyCore_Magma()
142 // --- Define strides for NOTRANSPOSE mode: --- in CeedBasisApplyCore_Magma()
143 // Input (d_u) is E-vector, output (d_v) is Q-vector in CeedBasisApplyCore_Magma()
155 // Input (d_u) is Q-vector, output (d_v) is E-vector in CeedBasisApplyCore_Magma()
194 …void *args[] = {&impl->d_interp_1d, &impl->d_grad_1d, &d_u, &u_elem_stride, &u_comp_str… in CeedBasisApplyCore_Magma()
198 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, apply_add ? impl->GradTransposeAdd : impl->GradT… in CeedBasisApplyCore_Magma()
201 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Grad, NULL, grid, num_threads, num_t_col, … in CeedBasisApplyCore_Magma()
205 …CeedCheck(t_mode != CEED_TRANSPOSE, ceed, CEED_ERROR_BACKEND, "CEED_EVAL_WEIGHT incompatible with … in CeedBasisApplyCore_Magma()
206 …CeedCheck(impl->d_q_weight_1d, ceed, CEED_ERROR_BACKEND, "%s not supported; q_weight_1d not set", … in CeedBasisApplyCore_Magma()
231 void *args[] = {&impl->d_q_weight_1d, &d_v, &elem_dofs_size, &num_elem}; in CeedBasisApplyCore_Magma()
233 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Weight, NULL, grid, num_threads, num_t_col… in CeedBasisApplyCore_Magma()
238 return CeedError(ceed, CEED_ERROR_BACKEND, "%s not supported", CeedEvalModes[e_mode]); in CeedBasisApplyCore_Magma()
240 … return CeedError(ceed, CEED_ERROR_BACKEND, "CEED_EVAL_NONE does not make sense in this context"); in CeedBasisApplyCore_Magma()
245 ceed_magma_queue_sync(data->queue); in CeedBasisApplyCore_Magma()
252 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisApplyCore_Magma()
266 //------------------------------------------------------------------------------
267 // Basis apply - tensor AtPoints
268 //------------------------------------------------------------------------------
274 //------------------------------------------------------------------------------
275 // Basis apply - non-tensor
276 //------------------------------------------------------------------------------
279 Ceed ceed; in CeedBasisApplyNonTensorCore_Magma() local
286 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisApplyNonTensorCore_Magma()
287 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisApplyNonTensorCore_Magma()
298 …else CeedCheck(e_mode == CEED_EVAL_WEIGHT, ceed, CEED_ERROR_BACKEND, "An input vector is required … in CeedBasisApplyNonTensorCore_Magma()
304 …X_P && Q <= MAGMA_NONTENSOR_CUSTOM_KERNEL_MAX_Q && (e_mode != CEED_EVAL_WEIGHT || !impl->Weight)) { in CeedBasisApplyNonTensorCore_Magma()
306 CeedInt diff = abs(n_array[iN] - N), idiff; in CeedBasisApplyNonTensorCore_Magma()
309 idiff = abs(n_array[in] - N); in CeedBasisApplyNonTensorCore_Magma()
316 if (!impl->NB_interp[iN]) { in CeedBasisApplyNonTensorCore_Magma()
319 Ceed ceed_delegate; in CeedBasisApplyNonTensorCore_Magma()
340 impl->NB_interp[iN] = nontensor_rtc_get_nb(arch, 'n', q_comp_interp, P, Q, n_array[iN]); in CeedBasisApplyNonTensorCore_Magma()
341 impl->NB_interp_t[iN] = nontensor_rtc_get_nb(arch, 't', q_comp_interp, P, Q, n_array[iN]); in CeedBasisApplyNonTensorCore_Magma()
342 impl->NB_deriv[iN] = nontensor_rtc_get_nb(arch, 'n', q_comp_deriv, P, Q, n_array[iN]); in CeedBasisApplyNonTensorCore_Magma()
343 impl->NB_deriv_t[iN] = nontensor_rtc_get_nb(arch, 't', q_comp_deriv, P, Q, n_array[iN]); in CeedBasisApplyNonTensorCore_Magma()
345 // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data in CeedBasisApplyNonTensorCore_Magma()
346 CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate)); in CeedBasisApplyNonTensorCore_Magma()
349 …CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-interp-deriv-nonte… in CeedBasisApplyNonTensorCore_Magma()
350 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n"); in CeedBasisApplyNonTensorCore_Magma()
351 …CeedCallBackend(CeedLoadSourceAndInitializeBuffer(ceed, basis_kernel_path, &num_file_paths, &file_… in CeedBasisApplyNonTensorCore_Magma()
352 if (!impl->Weight) { in CeedBasisApplyNonTensorCore_Magma()
353 …CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h… in CeedBasisApplyNonTensorCore_Magma()
354 …CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, weight_kernel_path, &num_file_paths, &file… in CeedBasisApplyNonTensorCore_Magma()
356 …CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n"… in CeedBasisApplyNonTensorCore_Magma()
357 …CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[iN], 8, "BASIS_… in CeedBasisApplyNonTensorCore_Magma()
358 …_Q_COMP_DERIV", q_comp_deriv, "BASIS_P", P, "BASIS_Q", Q, "BASIS_NB_INTERP_N", impl->NB_interp[iN], in CeedBasisApplyNonTensorCore_Magma()
359 …"BASIS_NB_INTERP_T", impl->NB_interp_t[iN], "BASIS_NB_DERIV_N", impl->NB_deriv[iN], "BASIS_NB_DERI… in CeedBasisApplyNonTensorCore_Magma()
360 impl->NB_deriv_t[iN])); in CeedBasisApplyNonTensorCore_Magma()
361 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_interp_nontensor_n", &impl->Inte… in CeedBasisApplyNonTensorCore_Magma()
362 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_interp_nontensor_t", &impl->Inte… in CeedBasisApplyNonTensorCore_Magma()
363 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_interp_nontensor_ta", &impl->Int… in CeedBasisApplyNonTensorCore_Magma()
364 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_deriv_nontensor_n", &impl->Deriv… in CeedBasisApplyNonTensorCore_Magma()
365 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_deriv_nontensor_t", &impl->Deriv… in CeedBasisApplyNonTensorCore_Magma()
366 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_deriv_nontensor_ta", &impl->Deri… in CeedBasisApplyNonTensorCore_Magma()
367 if (!impl->Weight) { in CeedBasisApplyNonTensorCore_Magma()
368 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[iN], "magma_weight_nontensor", &impl->Weight… in CeedBasisApplyNonTensorCore_Magma()
387 d_b = impl->d_interp; in CeedBasisApplyNonTensorCore_Magma()
390 d_b = impl->d_grad; in CeedBasisApplyNonTensorCore_Magma()
393 d_b = impl->d_div; in CeedBasisApplyNonTensorCore_Magma()
396 d_b = impl->d_curl; in CeedBasisApplyNonTensorCore_Magma()
401 …return CeedError(ceed, CEED_ERROR_BACKEND, "%s does not make sense in this context", CeedEvalModes… in CeedBasisApplyNonTensorCore_Magma()
410 Kernel = apply_add ? impl->InterpTransposeAdd[iN] : impl->InterpTranspose[iN]; in CeedBasisApplyNonTensorCore_Magma()
411 NB = impl->NB_interp_t[iN]; in CeedBasisApplyNonTensorCore_Magma()
413 Kernel = impl->Interp[iN]; in CeedBasisApplyNonTensorCore_Magma()
414 NB = impl->NB_interp[iN]; in CeedBasisApplyNonTensorCore_Magma()
418 Kernel = apply_add ? impl->DerivTransposeAdd[iN] : impl->DerivTranspose[iN]; in CeedBasisApplyNonTensorCore_Magma()
419 NB = impl->NB_deriv_t[iN]; in CeedBasisApplyNonTensorCore_Magma()
421 Kernel = impl->Deriv[iN]; in CeedBasisApplyNonTensorCore_Magma()
422 NB = impl->NB_deriv[iN]; in CeedBasisApplyNonTensorCore_Magma()
432 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, Kernel, NULL, grid, M, num_t_col, 1, shared_mem,… in CeedBasisApplyNonTensorCore_Magma()
437 …ns, MagmaNoTrans, P, N, Q, 1.0, d_b + d * P * Q, P, d_u + d * N * Q, Q, beta, d_v, P, data->queue); in CeedBasisApplyNonTensorCore_Magma()
439 …ans, MagmaNoTrans, Q, N, P, 1.0, d_b + d * P * Q, P, d_u, P, 0.0, d_v + d * N * Q, Q, data->queue); in CeedBasisApplyNonTensorCore_Magma()
444 …CeedCheck(t_mode != CEED_TRANSPOSE, ceed, CEED_ERROR_BACKEND, "CEED_EVAL_WEIGHT incompatible with … in CeedBasisApplyNonTensorCore_Magma()
445 …CeedCheck(impl->d_q_weight, ceed, CEED_ERROR_BACKEND, "%s not supported; q_weight not set", CeedEv… in CeedBasisApplyNonTensorCore_Magma()
449 void *args[] = {&num_elem, &impl->d_q_weight, &d_v}; in CeedBasisApplyNonTensorCore_Magma()
451 …CeedCallBackend(CeedRunKernelDimSharedMagma(ceed, impl->Weight, NULL, grid, Q, num_t_col, 1, share… in CeedBasisApplyNonTensorCore_Magma()
455 ceed_magma_queue_sync(data->queue); in CeedBasisApplyNonTensorCore_Magma()
462 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisApplyNonTensorCore_Magma()
478 //------------------------------------------------------------------------------
480 //------------------------------------------------------------------------------
482 Ceed ceed; in CeedBasisDestroy_Magma() local
485 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisDestroy_Magma()
488 CeedCallHip(ceed, hipModuleUnload(impl->module)); in CeedBasisDestroy_Magma()
490 CeedCallCuda(ceed, cuModuleUnload(impl->module)); in CeedBasisDestroy_Magma()
492 CeedCallBackend(magma_free(impl->d_interp_1d)); in CeedBasisDestroy_Magma()
493 CeedCallBackend(magma_free(impl->d_grad_1d)); in CeedBasisDestroy_Magma()
494 if (impl->d_q_weight_1d) CeedCallBackend(magma_free(impl->d_q_weight_1d)); in CeedBasisDestroy_Magma()
496 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisDestroy_Magma()
500 //------------------------------------------------------------------------------
501 // Destroy non-tensor basis
502 //------------------------------------------------------------------------------
504 Ceed ceed; in CeedBasisDestroyNonTensor_Magma() local
507 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisDestroyNonTensor_Magma()
510 if (impl->module[in]) { in CeedBasisDestroyNonTensor_Magma()
512 CeedCallHip(ceed, hipModuleUnload(impl->module[in])); in CeedBasisDestroyNonTensor_Magma()
514 CeedCallCuda(ceed, cuModuleUnload(impl->module[in])); in CeedBasisDestroyNonTensor_Magma()
518 CeedCallBackend(magma_free(impl->d_interp)); in CeedBasisDestroyNonTensor_Magma()
519 CeedCallBackend(magma_free(impl->d_grad)); in CeedBasisDestroyNonTensor_Magma()
520 CeedCallBackend(magma_free(impl->d_div)); in CeedBasisDestroyNonTensor_Magma()
521 CeedCallBackend(magma_free(impl->d_curl)); in CeedBasisDestroyNonTensor_Magma()
522 if (impl->d_q_weight) CeedCallBackend(magma_free(impl->d_q_weight)); in CeedBasisDestroyNonTensor_Magma()
524 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisDestroyNonTensor_Magma()
528 //------------------------------------------------------------------------------
530 //------------------------------------------------------------------------------
533 Ceed ceed, ceed_delegate; in CeedBasisCreateTensorH1_Magma() local
542 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisCreateTensorH1_Magma()
543 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisCreateTensorH1_Magma()
548 CeedCallBackend(magma_malloc((void **)&impl->d_q_weight_1d, Q_1d * sizeof(q_weight_1d[0]))); in CeedBasisCreateTensorH1_Magma()
549 …magma_setvector(Q_1d, sizeof(q_weight_1d[0]), q_weight_1d, 1, impl->d_q_weight_1d, 1, data->queue); in CeedBasisCreateTensorH1_Magma()
551 CeedCallBackend(magma_malloc((void **)&impl->d_interp_1d, Q_1d * P_1d * sizeof(interp_1d[0]))); in CeedBasisCreateTensorH1_Magma()
552 …magma_setvector(Q_1d * P_1d, sizeof(interp_1d[0]), interp_1d, 1, impl->d_interp_1d, 1, data->queue… in CeedBasisCreateTensorH1_Magma()
553 CeedCallBackend(magma_malloc((void **)&impl->d_grad_1d, Q_1d * P_1d * sizeof(grad_1d[0]))); in CeedBasisCreateTensorH1_Magma()
554 magma_setvector(Q_1d * P_1d, sizeof(grad_1d[0]), grad_1d, 1, impl->d_grad_1d, 1, data->queue); in CeedBasisCreateTensorH1_Magma()
556 // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data in CeedBasisCreateTensorH1_Magma()
557 CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate)); in CeedBasisCreateTensorH1_Magma()
562 char *interp_kernel_name_base = "ceed/jit-source/magma/magma-basis-interp"; in CeedBasisCreateTensorH1_Magma()
566 …snprintf(interp_kernel_name, interp_kernel_name_len, "%s-%" CeedInt_FMT "d.h", interp_kernel_name_… in CeedBasisCreateTensorH1_Magma()
567 CeedCallBackend(CeedGetJitAbsolutePath(ceed, interp_kernel_name, &interp_kernel_path)); in CeedBasisCreateTensorH1_Magma()
569 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n"); in CeedBasisCreateTensorH1_Magma()
570 …CeedCallBackend(CeedLoadSourceAndInitializeBuffer(ceed, interp_kernel_path, &num_file_paths, &file… in CeedBasisCreateTensorH1_Magma()
572 char *grad_kernel_name_base = "ceed/jit-source/magma/magma-basis-grad"; in CeedBasisCreateTensorH1_Magma()
576 …snprintf(grad_kernel_name, grad_kernel_name_len, "%s-%" CeedInt_FMT "d.h", grad_kernel_name_base, … in CeedBasisCreateTensorH1_Magma()
577 CeedCallBackend(CeedGetJitAbsolutePath(ceed, grad_kernel_name, &grad_kernel_path)); in CeedBasisCreateTensorH1_Magma()
579 …CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, grad_kernel_path, &num_file_paths, &file_p… in CeedBasisCreateTensorH1_Magma()
581 char *weight_kernel_name_base = "ceed/jit-source/magma/magma-basis-weight"; in CeedBasisCreateTensorH1_Magma()
585 …snprintf(weight_kernel_name, weight_kernel_name_len, "%s-%" CeedInt_FMT "d.h", weight_kernel_name_… in CeedBasisCreateTensorH1_Magma()
586 CeedCallBackend(CeedGetJitAbsolutePath(ceed, weight_kernel_name, &weight_kernel_path)); in CeedBasisCreateTensorH1_Magma()
588 …CeedCallBackend(CeedLoadSourceToInitializedBuffer(ceed, weight_kernel_path, &num_file_paths, &file… in CeedBasisCreateTensorH1_Magma()
589 …CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n"… in CeedBasisCreateTensorH1_Magma()
590 …CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module, 5, "BASIS_DIM"… in CeedBasisCreateTensorH1_Magma()
594 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_1d_kernel", &impl->Interp)); in CeedBasisCreateTensorH1_Magma()
595 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_1d_kernel", &impl->InterpTra… in CeedBasisCreateTensorH1_Magma()
596 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpta_1d_kernel", &impl->InterpTr… in CeedBasisCreateTensorH1_Magma()
597 CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_1d_kernel", &impl->Grad)); in CeedBasisCreateTensorH1_Magma()
598 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_1d_kernel", &impl->GradTranspo… in CeedBasisCreateTensorH1_Magma()
599 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradta_1d_kernel", &impl->GradTransp… in CeedBasisCreateTensorH1_Magma()
600 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_1d_kernel", &impl->Weight)); in CeedBasisCreateTensorH1_Magma()
603 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_2d_kernel", &impl->Interp)); in CeedBasisCreateTensorH1_Magma()
604 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_2d_kernel", &impl->InterpTra… in CeedBasisCreateTensorH1_Magma()
605 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpta_2d_kernel", &impl->InterpTr… in CeedBasisCreateTensorH1_Magma()
606 CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_2d_kernel", &impl->Grad)); in CeedBasisCreateTensorH1_Magma()
607 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_2d_kernel", &impl->GradTranspo… in CeedBasisCreateTensorH1_Magma()
608 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradta_2d_kernel", &impl->GradTransp… in CeedBasisCreateTensorH1_Magma()
609 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_2d_kernel", &impl->Weight)); in CeedBasisCreateTensorH1_Magma()
612 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpn_3d_kernel", &impl->Interp)); in CeedBasisCreateTensorH1_Magma()
613 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpt_3d_kernel", &impl->InterpTra… in CeedBasisCreateTensorH1_Magma()
614 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_interpta_3d_kernel", &impl->InterpTr… in CeedBasisCreateTensorH1_Magma()
615 CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradn_3d_kernel", &impl->Grad)); in CeedBasisCreateTensorH1_Magma()
616 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradt_3d_kernel", &impl->GradTranspo… in CeedBasisCreateTensorH1_Magma()
617 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_gradta_3d_kernel", &impl->GradTransp… in CeedBasisCreateTensorH1_Magma()
618 … CeedCallBackend(CeedGetKernelMagma(ceed, impl->module, "magma_weight_3d_kernel", &impl->Weight)); in CeedBasisCreateTensorH1_Magma()
630 CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApply_Magma)); in CeedBasisCreateTensorH1_Magma()
631 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "ApplyAdd", CeedBasisApplyAdd_Magma)); in CeedBasisCreateTensorH1_Magma()
632 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "ApplyAtPoints", CeedBasisApplyAtPoin… in CeedBasisCreateTensorH1_Magma()
633 CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroy_Magma)); in CeedBasisCreateTensorH1_Magma()
634 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisCreateTensorH1_Magma()
639 //------------------------------------------------------------------------------
640 // Create non-tensor H^1
641 //------------------------------------------------------------------------------
644 Ceed ceed; in CeedBasisCreateH1_Magma() local
648 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisCreateH1_Magma()
649 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisCreateH1_Magma()
654 CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0]))); in CeedBasisCreateH1_Magma()
655 magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue); in CeedBasisCreateH1_Magma()
661 …CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * size… in CeedBasisCreateH1_Magma()
662 …um_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue); in CeedBasisCreateH1_Magma()
668 …CeedCallBackend(magma_malloc((void **)&impl->d_grad, num_qpts * num_nodes * q_comp_grad * sizeof(g… in CeedBasisCreateH1_Magma()
669 …vector(num_qpts * num_nodes * q_comp_grad, sizeof(grad[0]), grad, 1, impl->d_grad, 1, data->queue); in CeedBasisCreateH1_Magma()
674 Ceed ceed_delegate; in CeedBasisCreateH1_Magma()
678 // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data in CeedBasisCreateH1_Magma()
679 CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate)); in CeedBasisCreateH1_Magma()
682 …CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h… in CeedBasisCreateH1_Magma()
683 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n"); in CeedBasisCreateH1_Magma()
684 CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source)); in CeedBasisCreateH1_Magma()
685 …CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n"… in CeedBasisCreateH1_Magma()
686 …CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q… in CeedBasisCreateH1_Magma()
687 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight)… in CeedBasisCreateH1_Magma()
696 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magm… in CeedBasisCreateH1_Magma()
697 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "ApplyAdd", CeedBasisApplyAddNonTenso… in CeedBasisCreateH1_Magma()
698 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_… in CeedBasisCreateH1_Magma()
699 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisCreateH1_Magma()
703 //------------------------------------------------------------------------------
704 // Create non-tensor H(div)
705 //------------------------------------------------------------------------------
708 Ceed ceed; in CeedBasisCreateHdiv_Magma() local
712 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisCreateHdiv_Magma()
713 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisCreateHdiv_Magma()
718 CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0]))); in CeedBasisCreateHdiv_Magma()
719 magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue); in CeedBasisCreateHdiv_Magma()
725 …CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * size… in CeedBasisCreateHdiv_Magma()
726 …um_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue); in CeedBasisCreateHdiv_Magma()
732 …CeedCallBackend(magma_malloc((void **)&impl->d_div, num_qpts * num_nodes * q_comp_div * sizeof(div… in CeedBasisCreateHdiv_Magma()
733 …_setvector(num_qpts * num_nodes * q_comp_div, sizeof(div[0]), div, 1, impl->d_div, 1, data->queue); in CeedBasisCreateHdiv_Magma()
738 Ceed ceed_delegate; in CeedBasisCreateHdiv_Magma()
742 // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data in CeedBasisCreateHdiv_Magma()
743 CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate)); in CeedBasisCreateHdiv_Magma()
746 …CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h… in CeedBasisCreateHdiv_Magma()
747 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n"); in CeedBasisCreateHdiv_Magma()
748 CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source)); in CeedBasisCreateHdiv_Magma()
749 …CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n"… in CeedBasisCreateHdiv_Magma()
750 …CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q… in CeedBasisCreateHdiv_Magma()
751 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight)… in CeedBasisCreateHdiv_Magma()
760 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magm… in CeedBasisCreateHdiv_Magma()
761 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "ApplyAdd", CeedBasisApplyAddNonTenso… in CeedBasisCreateHdiv_Magma()
762 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_… in CeedBasisCreateHdiv_Magma()
763 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisCreateHdiv_Magma()
767 //------------------------------------------------------------------------------
768 // Create non-tensor H(curl)
769 //------------------------------------------------------------------------------
772 Ceed ceed; in CeedBasisCreateHcurl_Magma() local
776 CeedCallBackend(CeedBasisGetCeed(basis, &ceed)); in CeedBasisCreateHcurl_Magma()
777 CeedCallBackend(CeedGetData(ceed, &data)); in CeedBasisCreateHcurl_Magma()
782 CeedCallBackend(magma_malloc((void **)&impl->d_q_weight, num_qpts * sizeof(q_weight[0]))); in CeedBasisCreateHcurl_Magma()
783 magma_setvector(num_qpts, sizeof(q_weight[0]), q_weight, 1, impl->d_q_weight, 1, data->queue); in CeedBasisCreateHcurl_Magma()
789 …CeedCallBackend(magma_malloc((void **)&impl->d_interp, num_qpts * num_nodes * q_comp_interp * size… in CeedBasisCreateHcurl_Magma()
790 …um_qpts * num_nodes * q_comp_interp, sizeof(interp[0]), interp, 1, impl->d_interp, 1, data->queue); in CeedBasisCreateHcurl_Magma()
796 …CeedCallBackend(magma_malloc((void **)&impl->d_curl, num_qpts * num_nodes * q_comp_curl * sizeof(c… in CeedBasisCreateHcurl_Magma()
797 …vector(num_qpts * num_nodes * q_comp_curl, sizeof(curl[0]), curl, 1, impl->d_curl, 1, data->queue); in CeedBasisCreateHcurl_Magma()
802 Ceed ceed_delegate; in CeedBasisCreateHcurl_Magma()
806 // The RTC compilation code expects a Ceed with the common Ceed_Cuda or Ceed_Hip data in CeedBasisCreateHcurl_Magma()
807 CeedCallBackend(CeedGetDelegate(ceed, &ceed_delegate)); in CeedBasisCreateHcurl_Magma()
810 …CeedCallBackend(CeedGetJitAbsolutePath(ceed, "ceed/jit-source/magma/magma-basis-weight-nontensor.h… in CeedBasisCreateHcurl_Magma()
811 CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source -----\n"); in CeedBasisCreateHcurl_Magma()
812 CeedCallBackend(CeedLoadSourceToBuffer(ceed, weight_kernel_path, &basis_kernel_source)); in CeedBasisCreateHcurl_Magma()
813 …CeedDebug256(ceed, CEED_DEBUG_COLOR_SUCCESS, "----- Loading Basis Kernel Source Complete! -----\n"… in CeedBasisCreateHcurl_Magma()
814 …CeedCallBackend(CeedCompileMagma(ceed_delegate, basis_kernel_source, &impl->module[0], 1, "BASIS_Q… in CeedBasisCreateHcurl_Magma()
815 …CeedCallBackend(CeedGetKernelMagma(ceed, impl->module[0], "magma_weight_nontensor", &impl->Weight)… in CeedBasisCreateHcurl_Magma()
824 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Apply", CeedBasisApplyNonTensor_Magm… in CeedBasisCreateHcurl_Magma()
825 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "ApplyAdd", CeedBasisApplyAddNonTenso… in CeedBasisCreateHcurl_Magma()
826 …CeedCallBackend(CeedSetBackendFunction(ceed, "Basis", basis, "Destroy", CeedBasisDestroyNonTensor_… in CeedBasisCreateHcurl_Magma()
827 CeedCallBackend(CeedDestroy(&ceed)); in CeedBasisCreateHcurl_Magma()
831 //------------------------------------------------------------------------------