yigid/openairinterface5g
1
0
Fork 0
mirror of https://gitlab.eurecom.fr/oai/openairinterface5g.git synced 2026-07-13 04:30:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

use_gpu/ENABLE_CUDA/USE_CUDA + sims

Browse Source 
Remove double nr_dlsim eval

Remove CUDA from normal LDPC coding, make standalone

nr_dlsim seems to work.
ldpctest: need to implement memcpy() in _cuda.c
nr_ulsim fails

fixup ldpctest: make it run

Initial version of ldpc_decoder_cuda done. Change configuration to GH200

modify -h option in dlsim and ulsim. First scratch for nrLDPC GPU decoder tutorial

Change code

add missing initialization on jetson for nr_ulsim. Bug fixed

add buffer memory management; remove an extra synchronization in cuda decoder

fix for ulsim via cuda encoder on jetson

fix for 256QAM

Move LDPC CUDA to separate subdirectory

Only functional change: remove USE_CUDA

Rename constant for clarity

ldpc_cuda: add tests to verify it works

Signed-off-by: Robert Schmidt <robert.schmidt@openairinterface.org>

Implement fallback CPU

Remove use_gpu

Signed-off-by: Robert Schmidt <robert.schmidt@openairinterface.org>
This commit is contained in:
Raymond Knopp
2026-05-29 17:44:50 +02:00
parent 9a6b332912
commit a51770341e
44 changed files with 23724 additions and 228 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
cmake_targets/build_oai
View File

@@ -18,7 +18,7 @@ HWs=""
BUILD_DIR=ran_build
CMAKE_BUILD_TYPE="RelWithDebInfo"
CMAKE_CMD="$CMAKE"
OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal websrv oai_iqplayer imscope imscope_record"
OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal ldpc_cuda websrv oai_iqplayer imscope imscope_record"
TARGET_LIST=""
BUILD_TOOL_OPT="-j$(nproc)"

3
openair1/PHY/CODING/nrLDPC_coding/CMakeLists.txt
View File

@@ -3,3 +3,6 @@
add_subdirectory(nrLDPC_coding_segment)
add_subdirectory(nrLDPC_coding_aal)
add_boolean_option(ENABLE_LDPC_CUDA OFF "Build support for CUDA" OFF)
if (ENABLE_LDPC_CUDA)
add_subdirectory(nrLDPC_coding_cuda)
endif()

71
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/CMakeLists.txt Normal file
View File

@@ -0,0 +1,71 @@
enable_language(CUDA)
# it would be possible to auto-detect compute capailities: https://stackoverflow.com/a/68223399
#include(FindCUDA/select_compute_arch)
#CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
# Nvidia recommends to build against all targeted archs: https://forums.developer.nvidia.com/t/261162
# this is for cmake 3.16 (OAI minimum), starting at cmake 3.18 there is another syntax, see below
SET(CUDA_VERBOSE_BUILD ON)
# version using NVLink-C2C in Grace Hopper
add_library(ldpc_cuda MODULE
# main entry
nrLDPC_coding_cuda.c
nrLDPC_decoder_cuda.c
ldpc_encoder_cuda32.c
ldpc_input32.c
nrLDPC_coding_segment_encoder_cuda.c
nrLDPC_coding_segment_decoder_cuda.c
# .cu files
nrLDPC_decoder_BG1_cuda.cu
ldpc_BG1_Zc384_32bit.cu
ldpc_input.cu
ldpc_deinterleave.cu
ldpc_rate_matching.cu
)
set_target_properties(ldpc_cuda PROPERTIES CUDA_ARCHITECTURES "80;87;90")
# we need to find the right include directory, but ti seems that
# enable_language(CUDA) does not make this available(?), so also use the CUDA package
find_package(CUDA REQUIRED)
target_include_directories(ldpc_cuda PRIVATE ${CUDA_INCLUDE_DIRS})
target_link_libraries(ldpc_cuda ldpc_gen_HEADERS log_headers)
set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
add_dependencies(ldpctest ldpc_cuda)
add_dependencies(nr-softmodem ldpc_cuda)
add_dependencies(nr-uesoftmodem ldpc_cuda)
add_dependencies(nr_ulsim ldpc_cuda)
add_dependencies(nr_ulschsim ldpc_cuda)
add_dependencies(nr_dlsim ldpc_cuda)
add_dependencies(nr_dlschsim ldpc_cuda)
if(ENABLE_PHYSIM_TESTS)
# this is a list of physical simulators designed to test the functioning of
# ldpc_cuda. Of course, one could also modify
# openair1/SIMULATION/tests/CMakeLists.txt to run them with ldpc_cuda
add_physim_test(ldpctest.ldpc_cuda.simple "ldpctest: simple CUDA load check" ldpctest -v _cuda -l8448 -s10 -n300)
add_physim_test(ldpctest.ldpc_cuda.r13 "ldpctest: base graph 1, rate 1/3" ldpctest -v _cuda -r1 -n300 -S64 -s4 -i4)
add_physim_test(ldpctest.ldpc_cuda.r23 "ldpctest: base graph 1, rate 2/3" ldpctest -v _cuda -r2 -n300 -S64 -s4 -i4)
add_physim_test(ldpctest.ldpc_cuda.r89 "ldpctest: base graph 1, rate 8/9" ldpctest -v _cuda -r22 -d25 -n300 -S64 -s4 -i4)
add_physim_test(nr_dlsim.ldpc_cuda.9seg_r13 "nr_dlsim: 9 segments, rate 1/3" nr_dlsim -n100 -s20 -e13 -R273 -b273 -x1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_dlsim.ldpc_cuda.18seg_r23 "nr_dlsim: 18 segments, rate 2/3" nr_dlsim -n100 -s20 -e22 -R273 -b273 -x1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_dlsim.ldpc_cuda.50seg_r89 "nr_dlsim: 50 segments, rate 8/9" nr_dlsim -n100 -s40 -e27 -R273 -b273 -z2 -x2 -y2 -P --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_dlsim.ldpc_cuda.2t2r_mcs25_64qam "nr_dlsim: 2T2R/2 layers, MCS 25, 64QAM, 100 MHz 30 kHz SCS)" nr_dlsim -n100 -R273 -b273 -P -e 25 -s30 -x2 -y2 -z2 --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_dlsim.ldpc_cuda.1t1r_mcs25_256qam "nr_dlsim: 1T1R/1 layer, MCS 25, 256QAM, 100 MHz 30 kHz SCS)" nr_dlsim -n100 -R273 -b273 -P -e 25 -s30 -q 1 --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_dlsim.ldpc_cuda.1t1r_mcs25_256qam_bw "nr_dlsim: 1T1R/1 layer, MCS 25 256QAM, 40 MHz 30 kHz SCS" nr_dlsim -n100 -R106 -b106 -P -e 25 -s30 -q 1 --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_ulsim.ldpc_cuda.9seg_r13 "nr_ulsim: 9 segments, rate 1/3" nr_ulsim -n100 -s20 -m13 -R273 -r273 -W1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_ulsim.ldpc_cuda.18seg_r23 "nr_ulsim: 18 segments, rate 2/3" nr_ulsim -n100 -s20 -m22 -R273 -r273 -W1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
add_physim_test(nr_ulsim.ldpc_cuda.50seg_r89 "nr_ulsim: 50 segments, rate 8/9" nr_ulsim -n100 -s40 -m27 -R273 -r273 -W2 -I9 -y2 -z2 -P --loader.ldpc.shlibversion _cuda)
endif()

216
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_BG1_Zc384_32bit.cu Normal file
View File

@@ -0,0 +1,216 @@
#include <stdio.h>
#include <stdint.h>
#include <cuda_runtime.h>
// generated code for Zc=384, byte encoding
__global__ void ldpc_BG1_Zc384_worker(uint32_t *c[4],uint32_t *d[4]) {
uint32_t *c32=c[blockIdx.x];
uint32_t *d32=d[blockIdx.x]+8448-2*384;
int i2 = threadIdx.x;
int i1 = blockIdx.y;
// copy 20 c values to d
if (i1<20) d[blockIdx.x][(i1*384) + i2] = c32[4*384+(2*384*i1)+i2];
if (i2 < 384) {
c32+=i2;
d32+=i2;
switch(i1) {
case 0:
d32[0]=c32[307]^c32[76]^c32[205]^c32[276]^c32[787]^c32[1018]^c32[855]^c32[1586]^c32[1612]^c32[1864]^c32[2673]^c32[2377]^c32[2304]^c32[3360]^c32[3404]^c32[3347]^c32[4021]^c32[3984]^c32[4096]^c32[4824]^c32[4769]^c32[4807]^c32[5707]^c32[5643]^c32[5529]^c32[6475]^c32[6304]^c32[6200]^c32[7229]^c32[7090]^c32[6975]^c32[7968]^c32[7809]^c32[7812]^c32[8557]^c32[8743]^c32[8753]^c32[9233]^c32[9558]^c32[9447]^c32[10341]^c32[10184]^c32[10325]^c32[10969]^c32[10840]^c32[10964]^c32[11735]^c32[11619]^c32[11573]^c32[12394]^c32[12642]^c32[12592]^c32[13170]^c32[13187]^c32[13356]^c32[14066]^c32[14064]^c32[14095]^c32[14772]^c32[14923]^c32[14797]^c32[15690]^c32[15373]^c32[15399]^c32[16474]^c32[16240]^c32[16485];
break;
case 1:
d32[384]=c32[307]^c32[308]^c32[77]^c32[206]^c32[277]^c32[787]^c32[788]^c32[1019]^c32[856]^c32[1586]^c32[1587]^c32[1613]^c32[1865]^c32[2673]^c32[2674]^c32[2378]^c32[2305]^c32[3361]^c32[3405]^c32[3348]^c32[4021]^c32[4022]^c32[3985]^c32[4097]^c32[4824]^c32[4825]^c32[4770]^c32[4808]^c32[5708]^c32[5644]^c32[5530]^c32[6476]^c32[6305]^c32[6201]^c32[7229]^c32[7230]^c32[7091]^c32[6976]^c32[7968]^c32[7969]^c32[7810]^c32[7813]^c32[8557]^c32[8558]^c32[8744]^c32[8754]^c32[9233]^c32[9234]^c32[9559]^c32[9448]^c32[10341]^c32[10342]^c32[10185]^c32[10326]^c32[10970]^c32[10841]^c32[10965]^c32[11735]^c32[11736]^c32[11620]^c32[11574]^c32[12394]^c32[12395]^c32[12643]^c32[12593]^c32[13171]^c32[13188]^c32[13357]^c32[14066]^c32[14067]^c32[14065]^c32[14096]^c32[14772]^c32[14773]^c32[14924]^c32[14798]^c32[15690]^c32[15691]^c32[15374]^c32[15400]^c32[16474]^c32[16475]^c32[16241]^c32[16486];
break;
case 2:
d32[768]=c32[308]^c32[77]^c32[205]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1018]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1864]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3404]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4096]^c32[4097]^c32[4825]^c32[4769]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5643]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6304]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6975]^c32[6976]^c32[7969]^c32[7809]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10184]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10840]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11573]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13187]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14064]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14797]^c32[14798]^c32[15691]^c32[15373]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
break;
case 3:
d32[1152]=c32[308]^c32[77]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4097]^c32[4825]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6976]^c32[7969]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14798]^c32[15691]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
break;
case 4:
d32[1536]=c32[332]^c32[949];
break;
case 5:
d32[1920]=c32[80]^c32[233]^c32[362]^c32[49]^c32[195]^c32[944]^c32[791]^c32[1012]^c32[782]^c32[1743]^c32[1769]^c32[1637]^c32[2446]^c32[2534]^c32[2461]^c32[2419]^c32[3133]^c32[3177]^c32[3120]^c32[4178]^c32[4141]^c32[3869]^c32[4981]^c32[4926]^c32[4964]^c32[5480]^c32[5416]^c32[5686]^c32[6248]^c32[6461]^c32[6357]^c32[7002]^c32[7247]^c32[7132]^c32[7741]^c32[7966]^c32[7969]^c32[8714]^c32[8516]^c32[8526]^c32[9390]^c32[9331]^c32[9220]^c32[9382]^c32[10114]^c32[10341]^c32[10098]^c32[11126]^c32[10997]^c32[11121]^c32[11892]^c32[11776]^c32[11730]^c32[12551]^c32[12415]^c32[12365]^c32[12529]^c32[13327]^c32[13344]^c32[13129]^c32[13839]^c32[13837]^c32[13868]^c32[14929]^c32[14696]^c32[14954]^c32[15463]^c32[15530]^c32[15556]^c32[16247]^c32[16397]^c32[16258]^c32[16179];
break;
case 6:
d32[2304]=c32[278]^c32[4865]^c32[7681]^c32[8799]^c32[10076]^c32[13309]^c32[13842]^c32[15585];
break;
case 7:
d32[2688]=c32[9]^c32[830]^c32[3388]^c32[5709]^c32[6434]^c32[10866];
break;
case 8:
d32[3072]=c32[374]^c32[94]^c32[143]^c32[247]^c32[272]^c32[375]^c32[376]^c32[343]^c32[62]^c32[63]^c32[307]^c32[854]^c32[958]^c32[1085]^c32[804]^c32[805]^c32[922]^c32[1025]^c32[1026]^c32[947]^c32[1653]^c32[1757]^c32[1679]^c32[1783]^c32[1547]^c32[1650]^c32[1651]^c32[2356]^c32[2460]^c32[2444]^c32[2548]^c32[2371]^c32[2474]^c32[2475]^c32[2469]^c32[3427]^c32[3147]^c32[3087]^c32[3190]^c32[3191]^c32[3414]^c32[3133]^c32[3134]^c32[4088]^c32[4192]^c32[4051]^c32[4155]^c32[4163]^c32[3882]^c32[3883]^c32[4891]^c32[4611]^c32[4836]^c32[4939]^c32[4940]^c32[4874]^c32[4977]^c32[4978]^c32[5390]^c32[5494]^c32[5710]^c32[5429]^c32[5430]^c32[5596]^c32[5699]^c32[5700]^c32[6158]^c32[6262]^c32[6371]^c32[6474]^c32[6475]^c32[6267]^c32[6370]^c32[6371]^c32[6912]^c32[7016]^c32[7157]^c32[7261]^c32[7042]^c32[7145]^c32[7146]^c32[8035]^c32[7755]^c32[7876]^c32[7979]^c32[7980]^c32[7879]^c32[7982]^c32[7983]^c32[8624]^c32[8728]^c32[8810]^c32[8530]^c32[8820]^c32[8539]^c32[8540]^c32[9300]^c32[9404]^c32[9241]^c32[9345]^c32[9514]^c32[9233]^c32[9234]^c32[9234]^c32[10024]^c32[10128]^c32[10251]^c32[10354]^c32[10355]^c32[10008]^c32[10111]^c32[10112]^c32[11036]^c32[10756]^c32[10907]^c32[11010]^c32[11011]^c32[11031]^c32[11134]^c32[11135]^c32[11802]^c32[11522]^c32[11686]^c32[11790]^c32[11640]^c32[11743]^c32[11744]^c32[12461]^c32[12565]^c32[12325]^c32[12429]^c32[12659]^c32[12378]^c32[12379]^c32[12327]^c32[13237]^c32[13341]^c32[13254]^c32[13357]^c32[13358]^c32[13423]^c32[13142]^c32[13143]^c32[14133]^c32[13853]^c32[14131]^c32[13850]^c32[13851]^c32[14162]^c32[13881]^c32[13882]^c32[14839]^c32[14943]^c32[14606]^c32[14710]^c32[14864]^c32[14967]^c32[14968]^c32[14816]^c32[15373]^c32[15477]^c32[15440]^c32[15543]^c32[15544]^c32[15466]^c32[15569]^c32[15570]^c32[16157]^c32[16261]^c32[16307]^c32[16411]^c32[16168]^c32[16271]^c32[16272]^c32[16496];
break;
case 9:
d32[3456]=c32[366]^c32[1000]^c32[8001]^c32[8581]^c32[10041]^c32[13359]^c32[13887]^c32[15442];
break;
case 10:
d32[3840]=c32[869]^c32[1875]^c32[3346]^c32[5487]^c32[6527]^c32[11106];
break;
case 11:
d32[4224]=c32[257]^c32[38]^c32[39]^c32[26]^c32[192]^c32[155]^c32[321]^c32[226]^c32[8]^c32[48]^c32[1121]^c32[902]^c32[903]^c32[968]^c32[1134]^c32[805]^c32[971]^c32[870]^c32[1536]^c32[1701]^c32[1702]^c32[1562]^c32[1728]^c32[1814]^c32[1596]^c32[2623]^c32[2404]^c32[2405]^c32[2327]^c32[2493]^c32[2638]^c32[2420]^c32[3310]^c32[3092]^c32[3354]^c32[3136]^c32[3297]^c32[3079]^c32[3971]^c32[4136]^c32[4137]^c32[3934]^c32[4100]^c32[4046]^c32[4212]^c32[4774]^c32[4939]^c32[4940]^c32[4719]^c32[4885]^c32[4757]^c32[4923]^c32[5657]^c32[5439]^c32[5593]^c32[5759]^c32[5479]^c32[5645]^c32[6425]^c32[6207]^c32[6254]^c32[6420]^c32[6150]^c32[6316]^c32[7179]^c32[6960]^c32[6961]^c32[7040]^c32[7206]^c32[6925]^c32[7091]^c32[7918]^c32[7699]^c32[7700]^c32[7759]^c32[7925]^c32[7762]^c32[7928]^c32[8507]^c32[8672]^c32[8673]^c32[8693]^c32[8475]^c32[8703]^c32[8485]^c32[9567]^c32[9348]^c32[9349]^c32[9508]^c32[9290]^c32[9397]^c32[9563]^c32[9224]^c32[10291]^c32[10072]^c32[10073]^c32[10134]^c32[10300]^c32[10275]^c32[10057]^c32[10919]^c32[11085]^c32[10790]^c32[10956]^c32[10914]^c32[11080]^c32[11685]^c32[11850]^c32[11851]^c32[11569]^c32[11735]^c32[11523]^c32[11689]^c32[12344]^c32[12509]^c32[12510]^c32[12592]^c32[12374]^c32[12542]^c32[12324]^c32[12335]^c32[13120]^c32[13286]^c32[13137]^c32[13303]^c32[13306]^c32[13088]^c32[14016]^c32[14181]^c32[14182]^c32[14014]^c32[14180]^c32[14045]^c32[13827]^c32[14722]^c32[14887]^c32[14888]^c32[14873]^c32[14655]^c32[14747]^c32[14913]^c32[15640]^c32[15421]^c32[15422]^c32[15707]^c32[15489]^c32[15733]^c32[15515]^c32[16424]^c32[16205]^c32[16206]^c32[16190]^c32[16356]^c32[16435]^c32[16217]^c32[16316];
break;
case 12:
d32[4608]=c32[77]^c32[954]^c32[7854]^c32[8680]^c32[10034]^c32[13898];
break;
case 13:
d32[4992]=c32[293]^c32[294]^c32[63]^c32[192]^c32[263]^c32[313]^c32[773]^c32[774]^c32[1005]^c32[842]^c32[1572]^c32[1573]^c32[1599]^c32[1851]^c32[2659]^c32[2660]^c32[2364]^c32[2675]^c32[2481]^c32[3347]^c32[3391]^c32[3334]^c32[4007]^c32[4008]^c32[3971]^c32[4083]^c32[4810]^c32[4811]^c32[4756]^c32[4794]^c32[5694]^c32[5630]^c32[5516]^c32[5642]^c32[6462]^c32[6291]^c32[6187]^c32[7215]^c32[7216]^c32[7077]^c32[6962]^c32[7954]^c32[7955]^c32[7796]^c32[7799]^c32[8543]^c32[8544]^c32[8730]^c32[8740]^c32[9219]^c32[9220]^c32[9545]^c32[9434]^c32[10327]^c32[10328]^c32[10171]^c32[10312]^c32[10956]^c32[10827]^c32[10951]^c32[11721]^c32[11722]^c32[11606]^c32[11560]^c32[12380]^c32[12381]^c32[12629]^c32[12579]^c32[13157]^c32[13174]^c32[13343]^c32[14052]^c32[14053]^c32[14051]^c32[14082]^c32[14758]^c32[14759]^c32[14910]^c32[14784]^c32[15676]^c32[15677]^c32[15360]^c32[15386]^c32[15475]^c32[16460]^c32[16461]^c32[16227]^c32[16472];
break;
case 14:
d32[5376]=c32[142]^c32[9464]^c32[11657]^c32[12377]^c32[13403]^c32[16140];
break;
case 15:
d32[5760]=c32[193]^c32[346]^c32[91]^c32[161]^c32[162]^c32[241]^c32[1057]^c32[904]^c32[1124]^c32[1125]^c32[770]^c32[1856]^c32[1882]^c32[1750]^c32[2559]^c32[2647]^c32[2573]^c32[2574]^c32[3246]^c32[3290]^c32[3232]^c32[3233]^c32[3907]^c32[3870]^c32[3982]^c32[4710]^c32[4655]^c32[4692]^c32[4693]^c32[5593]^c32[5529]^c32[5414]^c32[5415]^c32[6361]^c32[6190]^c32[6469]^c32[6470]^c32[7115]^c32[6976]^c32[7245]^c32[7854]^c32[7695]^c32[7697]^c32[7698]^c32[7890]^c32[8827]^c32[8629]^c32[8638]^c32[8639]^c32[9503]^c32[9444]^c32[9332]^c32[9333]^c32[10227]^c32[10070]^c32[10210]^c32[10211]^c32[10302]^c32[10855]^c32[11110]^c32[10849]^c32[10850]^c32[11621]^c32[11889]^c32[11843]^c32[12664]^c32[12528]^c32[12477]^c32[12478]^c32[13056]^c32[13073]^c32[13241]^c32[13242]^c32[13952]^c32[13950]^c32[13980]^c32[13981]^c32[13879]^c32[14658]^c32[14809]^c32[14683]^c32[15576]^c32[15643]^c32[15668]^c32[15669]^c32[16360]^c32[16510]^c32[16370]^c32[16371];
break;
case 16:
d32[6144]=c32[364]^c32[133]^c32[262]^c32[333]^c32[844]^c32[1075]^c32[912]^c32[781]^c32[1643]^c32[1669]^c32[1537]^c32[2346]^c32[2434]^c32[2361]^c32[2642]^c32[3417]^c32[3077]^c32[3404]^c32[4078]^c32[4041]^c32[4153]^c32[4881]^c32[4826]^c32[4864]^c32[5380]^c32[5700]^c32[5586]^c32[6148]^c32[6361]^c32[6257]^c32[7286]^c32[7147]^c32[7032]^c32[8025]^c32[7866]^c32[7869]^c32[8614]^c32[8800]^c32[8810]^c32[8505]^c32[9290]^c32[9231]^c32[9504]^c32[10014]^c32[10241]^c32[9998]^c32[11026]^c32[10897]^c32[11021]^c32[11792]^c32[11676]^c32[11630]^c32[12451]^c32[12315]^c32[12649]^c32[13227]^c32[13244]^c32[13413]^c32[14123]^c32[14121]^c32[14152]^c32[14829]^c32[14596]^c32[14854]^c32[15363]^c32[15430]^c32[15456]^c32[15649]^c32[16147]^c32[16297]^c32[16158];
break;
case 17:
d32[6528]=c32[260]^c32[11055]^c32[12369]^c32[13414]^c32[16503];
break;
case 18:
d32[6912]=c32[898]^c32[9379]^c32[10264]^c32[13956]^c32[14596];
break;
case 19:
d32[7296]=c32[145]^c32[981]^c32[5720]^c32[6386]^c32[7877];
break;
case 20:
d32[7680]=c32[366]^c32[135]^c32[264]^c32[335]^c32[187]^c32[846]^c32[1077]^c32[914]^c32[1645]^c32[1671]^c32[1539]^c32[2348]^c32[2436]^c32[2363]^c32[2510]^c32[3419]^c32[3079]^c32[3406]^c32[4080]^c32[4043]^c32[4155]^c32[4883]^c32[4828]^c32[4866]^c32[5382]^c32[5702]^c32[5588]^c32[6150]^c32[6363]^c32[6259]^c32[7288]^c32[7149]^c32[7034]^c32[7176]^c32[8027]^c32[7868]^c32[7871]^c32[8616]^c32[8802]^c32[8812]^c32[8789]^c32[9292]^c32[9233]^c32[9506]^c32[10016]^c32[10243]^c32[10000]^c32[11028]^c32[10899]^c32[11023]^c32[11794]^c32[11678]^c32[11632]^c32[12453]^c32[12317]^c32[12651]^c32[13229]^c32[13246]^c32[13415]^c32[14125]^c32[14123]^c32[14154]^c32[14831]^c32[14598]^c32[14856]^c32[15365]^c32[15432]^c32[15458]^c32[16149]^c32[16299]^c32[16160];
break;
case 21:
d32[8064]=c32[973]^c32[3942]^c32[12616]^c32[15573]^c32[16225];
break;
case 22:
d32[8448]=c32[30]^c32[9227]^c32[10217]^c32[13078];
break;
case 23:
d32[8832]=c32[792]^c32[1625]^c32[7741]^c32[13851];
break;
case 24:
d32[9216]=c32[157]^c32[310]^c32[55]^c32[126]^c32[298]^c32[1021]^c32[868]^c32[1089]^c32[1820]^c32[1846]^c32[1714]^c32[2523]^c32[2611]^c32[2538]^c32[2462]^c32[3210]^c32[3254]^c32[3197]^c32[3307]^c32[3871]^c32[4218]^c32[3946]^c32[4674]^c32[4619]^c32[4657]^c32[5557]^c32[5493]^c32[5379]^c32[6325]^c32[6154]^c32[6434]^c32[7079]^c32[6940]^c32[7209]^c32[7818]^c32[8043]^c32[8046]^c32[8791]^c32[8593]^c32[8603]^c32[8787]^c32[9467]^c32[9408]^c32[9297]^c32[10191]^c32[10034]^c32[10175]^c32[10819]^c32[11074]^c32[10814]^c32[11585]^c32[11853]^c32[11807]^c32[12628]^c32[12492]^c32[12442]^c32[13404]^c32[13421]^c32[13206]^c32[13916]^c32[13914]^c32[13945]^c32[14622]^c32[14773]^c32[14647]^c32[15540]^c32[15607]^c32[15633]^c32[16324]^c32[16474]^c32[16335];
break;
case 25:
d32[9600]=c32[840]^c32[4625]^c32[5759]^c32[11064];
break;
case 26:
d32[9984]=c32[71]^c32[1617]^c32[3148]^c32[11656];
break;
case 27:
d32[10368]=c32[962]^c32[4802]^c32[6245];
break;
case 28:
d32[10752]=c32[222]^c32[3091]^c32[14836]^c32[16402];
break;
case 29:
d32[11136]=c32[2]^c32[155]^c32[284]^c32[354]^c32[355]^c32[866]^c32[1097]^c32[933]^c32[934]^c32[1020]^c32[1665]^c32[1691]^c32[1559]^c32[2368]^c32[2456]^c32[2382]^c32[2383]^c32[3439]^c32[3099]^c32[3425]^c32[3426]^c32[4100]^c32[4063]^c32[4175]^c32[4903]^c32[4848]^c32[4885]^c32[4886]^c32[5402]^c32[5722]^c32[5607]^c32[5608]^c32[6170]^c32[6383]^c32[6278]^c32[6279]^c32[6924]^c32[7169]^c32[7054]^c32[8047]^c32[7888]^c32[7890]^c32[7891]^c32[8636]^c32[8822]^c32[8831]^c32[8448]^c32[9312]^c32[9253]^c32[9525]^c32[9526]^c32[10036]^c32[10263]^c32[10019]^c32[10020]^c32[11048]^c32[10919]^c32[11042]^c32[11043]^c32[10757]^c32[11814]^c32[11698]^c32[11652]^c32[12473]^c32[12337]^c32[12670]^c32[12671]^c32[13249]^c32[13266]^c32[13434]^c32[13435]^c32[14145]^c32[14143]^c32[14173]^c32[14174]^c32[13971]^c32[14851]^c32[14618]^c32[14876]^c32[15385]^c32[15452]^c32[15477]^c32[15478]^c32[16169]^c32[16319]^c32[16179]^c32[16180];
break;
case 30:
d32[11520]=c32[14]^c32[167]^c32[295]^c32[296]^c32[366]^c32[367]^c32[159]^c32[878]^c32[1108]^c32[1109]^c32[945]^c32[946]^c32[1677]^c32[1703]^c32[1570]^c32[1571]^c32[2380]^c32[2468]^c32[2394]^c32[2395]^c32[3451]^c32[3110]^c32[3111]^c32[3437]^c32[3438]^c32[4112]^c32[4075]^c32[4186]^c32[4187]^c32[4915]^c32[4859]^c32[4860]^c32[4897]^c32[4898]^c32[5414]^c32[5733]^c32[5734]^c32[5619]^c32[5620]^c32[6182]^c32[6394]^c32[6395]^c32[6290]^c32[6291]^c32[6936]^c32[7181]^c32[7065]^c32[7066]^c32[8059]^c32[7899]^c32[7900]^c32[7902]^c32[7903]^c32[7909]^c32[8648]^c32[8450]^c32[8459]^c32[8460]^c32[9324]^c32[9265]^c32[9537]^c32[9538]^c32[10048]^c32[10274]^c32[10275]^c32[10031]^c32[10032]^c32[10244]^c32[11060]^c32[10930]^c32[10931]^c32[11054]^c32[11055]^c32[11826]^c32[11710]^c32[11663]^c32[11664]^c32[12485]^c32[12349]^c32[12298]^c32[12299]^c32[13261]^c32[13277]^c32[13278]^c32[13062]^c32[13063]^c32[14157]^c32[14154]^c32[14155]^c32[14185]^c32[14186]^c32[14863]^c32[14630]^c32[14887]^c32[14888]^c32[15397]^c32[15463]^c32[15464]^c32[15489]^c32[15490]^c32[16181]^c32[16331]^c32[16191]^c32[16192];
break;
case 31:
d32[11904]=c32[181]^c32[180]^c32[334]^c32[333]^c32[79]^c32[78]^c32[150]^c32[148]^c32[149]^c32[1045]^c32[1044]^c32[892]^c32[891]^c32[1113]^c32[1111]^c32[1112]^c32[868]^c32[1844]^c32[1843]^c32[1870]^c32[1869]^c32[1738]^c32[1737]^c32[2547]^c32[2546]^c32[2635]^c32[2634]^c32[2562]^c32[2560]^c32[2561]^c32[3234]^c32[3233]^c32[3278]^c32[3277]^c32[3221]^c32[3219]^c32[3220]^c32[3895]^c32[3894]^c32[3858]^c32[3857]^c32[3970]^c32[3969]^c32[4698]^c32[4697]^c32[4643]^c32[4642]^c32[4681]^c32[4679]^c32[4680]^c32[5581]^c32[5580]^c32[5517]^c32[5516]^c32[5403]^c32[5401]^c32[5402]^c32[5591]^c32[6349]^c32[6348]^c32[6178]^c32[6177]^c32[6458]^c32[6456]^c32[6457]^c32[7103]^c32[7102]^c32[6964]^c32[6963]^c32[7233]^c32[7232]^c32[7842]^c32[7841]^c32[7683]^c32[7682]^c32[7686]^c32[7684]^c32[7685]^c32[8815]^c32[8814]^c32[8617]^c32[8616]^c32[8627]^c32[8625]^c32[8626]^c32[9491]^c32[9490]^c32[9432]^c32[9431]^c32[9321]^c32[9319]^c32[9320]^c32[10215]^c32[10214]^c32[10058]^c32[10057]^c32[10199]^c32[10197]^c32[10198]^c32[10843]^c32[10842]^c32[11098]^c32[11097]^c32[10838]^c32[10836]^c32[10837]^c32[11609]^c32[11608]^c32[11877]^c32[11876]^c32[11831]^c32[11830]^c32[12652]^c32[12651]^c32[12516]^c32[12515]^c32[12466]^c32[12464]^c32[12465]^c32[13428]^c32[13427]^c32[13061]^c32[13060]^c32[13230]^c32[13228]^c32[13229]^c32[13940]^c32[13939]^c32[13938]^c32[13937]^c32[13969]^c32[13967]^c32[13968]^c32[14646]^c32[14645]^c32[14797]^c32[14796]^c32[14671]^c32[14670]^c32[15564]^c32[15563]^c32[15631]^c32[15630]^c32[15657]^c32[15655]^c32[15656]^c32[16348]^c32[16347]^c32[16498]^c32[16497]^c32[16359]^c32[16357]^c32[16358];
break;
case 32:
d32[12288]=c32[211]^c32[364]^c32[108]^c32[109]^c32[179]^c32[180]^c32[102]^c32[1075]^c32[921]^c32[922]^c32[1142]^c32[1143]^c32[1874]^c32[1900]^c32[1767]^c32[1768]^c32[2577]^c32[2665]^c32[2591]^c32[2592]^c32[3264]^c32[3307]^c32[3308]^c32[3250]^c32[3251]^c32[3925]^c32[3888]^c32[3999]^c32[4000]^c32[4728]^c32[4672]^c32[4673]^c32[4710]^c32[4711]^c32[5611]^c32[5546]^c32[5547]^c32[5432]^c32[5433]^c32[6379]^c32[6207]^c32[6208]^c32[6487]^c32[6488]^c32[7133]^c32[6994]^c32[7262]^c32[7263]^c32[7872]^c32[7712]^c32[7713]^c32[7715]^c32[7716]^c32[8461]^c32[8647]^c32[8656]^c32[8657]^c32[9521]^c32[9462]^c32[9350]^c32[9351]^c32[9417]^c32[10245]^c32[10087]^c32[10088]^c32[10228]^c32[10229]^c32[10873]^c32[11127]^c32[11128]^c32[10867]^c32[10868]^c32[10927]^c32[11639]^c32[11523]^c32[11860]^c32[11861]^c32[12298]^c32[12546]^c32[12495]^c32[12496]^c32[13074]^c32[13090]^c32[13091]^c32[13259]^c32[13260]^c32[13970]^c32[13967]^c32[13968]^c32[13998]^c32[13999]^c32[14676]^c32[14827]^c32[14700]^c32[14701]^c32[15594]^c32[15660]^c32[15661]^c32[15686]^c32[15687]^c32[16378]^c32[16144]^c32[16388]^c32[16389];
break;
case 33:
d32[12672]=c32[1091]^c32[1544]^c32[8809]^c32[16233];
break;
case 34:
d32[13056]=c32[230]^c32[5524]^c32[11722]^c32[13368];
break;
case 35:
d32[13440]=c32[189]^c32[342]^c32[87]^c32[158]^c32[1053]^c32[900]^c32[1121]^c32[1088]^c32[1852]^c32[1878]^c32[1746]^c32[2555]^c32[2643]^c32[2570]^c32[3242]^c32[3286]^c32[3229]^c32[3903]^c32[3866]^c32[3978]^c32[4706]^c32[4651]^c32[4689]^c32[4943]^c32[5589]^c32[5525]^c32[5411]^c32[6357]^c32[6186]^c32[6466]^c32[7111]^c32[6972]^c32[7241]^c32[7850]^c32[7691]^c32[7694]^c32[8823]^c32[8625]^c32[8635]^c32[9499]^c32[9440]^c32[9329]^c32[9218]^c32[10223]^c32[10066]^c32[10207]^c32[10851]^c32[11106]^c32[10846]^c32[11617]^c32[11885]^c32[11839]^c32[12660]^c32[12524]^c32[12474]^c32[13436]^c32[13069]^c32[13238]^c32[13948]^c32[13946]^c32[13977]^c32[14654]^c32[14805]^c32[14679]^c32[15572]^c32[15639]^c32[15665]^c32[16356]^c32[16506]^c32[16367];
break;
case 36:
d32[13824]=c32[210]^c32[11065]^c32[11817]^c32[13845];
break;
case 37:
d32[14208]=c32[38]^c32[39]^c32[192]^c32[321]^c32[8]^c32[902]^c32[903]^c32[1134]^c32[971]^c32[1037]^c32[1701]^c32[1702]^c32[1728]^c32[1596]^c32[2404]^c32[2405]^c32[2493]^c32[2420]^c32[3092]^c32[3136]^c32[3079]^c32[4136]^c32[4137]^c32[4100]^c32[4212]^c32[4939]^c32[4940]^c32[4885]^c32[4923]^c32[5439]^c32[5759]^c32[5645]^c32[6207]^c32[6420]^c32[6316]^c32[6960]^c32[6961]^c32[7206]^c32[7091]^c32[7699]^c32[7700]^c32[7925]^c32[7928]^c32[8672]^c32[8673]^c32[8475]^c32[8485]^c32[9348]^c32[9349]^c32[9290]^c32[9563]^c32[10072]^c32[10073]^c32[10300]^c32[10057]^c32[10066]^c32[11085]^c32[10956]^c32[11080]^c32[11850]^c32[11851]^c32[11735]^c32[11689]^c32[12509]^c32[12510]^c32[12374]^c32[12324]^c32[13286]^c32[13303]^c32[13088]^c32[14181]^c32[14182]^c32[14180]^c32[13827]^c32[14887]^c32[14888]^c32[14655]^c32[14913]^c32[15421]^c32[15422]^c32[15489]^c32[15515]^c32[16205]^c32[16206]^c32[16356]^c32[16217];
break;
case 38:
d32[14592]=c32[185]^c32[7089]^c32[7969]^c32[9430];
break;
case 39:
d32[14976]=c32[1026]^c32[2397]^c32[5722]^c32[14889];
break;
case 40:
d32[15360]=c32[175]^c32[6181]^c32[13368];
break;
case 41:
d32[15744]=c32[820]^c32[2618]^c32[7051]^c32[14112];
break;
case 42:
d32[16128]=c32[142]^c32[295]^c32[39]^c32[40]^c32[110]^c32[111]^c32[113]^c32[1006]^c32[852]^c32[853]^c32[1073]^c32[1074]^c32[1805]^c32[1831]^c32[1698]^c32[1699]^c32[2508]^c32[2596]^c32[2522]^c32[2523]^c32[3195]^c32[3238]^c32[3239]^c32[3181]^c32[3182]^c32[3086]^c32[3856]^c32[4203]^c32[3930]^c32[3931]^c32[4659]^c32[4987]^c32[4988]^c32[4641]^c32[4642]^c32[5542]^c32[5477]^c32[5478]^c32[5747]^c32[5748]^c32[6310]^c32[6522]^c32[6523]^c32[6418]^c32[6419]^c32[7064]^c32[6925]^c32[7193]^c32[7194]^c32[7803]^c32[8027]^c32[8028]^c32[8030]^c32[8031]^c32[8776]^c32[8578]^c32[8587]^c32[8588]^c32[9452]^c32[9393]^c32[9281]^c32[9282]^c32[10176]^c32[10018]^c32[10019]^c32[10159]^c32[10160]^c32[10804]^c32[11058]^c32[11059]^c32[10798]^c32[10799]^c32[11570]^c32[11838]^c32[11791]^c32[11792]^c32[12613]^c32[12477]^c32[12426]^c32[12427]^c32[13389]^c32[13405]^c32[13406]^c32[13190]^c32[13191]^c32[13901]^c32[13898]^c32[13899]^c32[13929]^c32[13930]^c32[14607]^c32[14758]^c32[14631]^c32[14632]^c32[15525]^c32[15591]^c32[15592]^c32[15617]^c32[15618]^c32[16309]^c32[16459]^c32[16319]^c32[16320];
break;
case 43:
d32[16512]=c32[92]^c32[245]^c32[374]^c32[60]^c32[61]^c32[956]^c32[803]^c32[1023]^c32[1024]^c32[881]^c32[1755]^c32[1781]^c32[1649]^c32[2458]^c32[2546]^c32[2472]^c32[2473]^c32[3145]^c32[3189]^c32[3131]^c32[3132]^c32[4190]^c32[4153]^c32[3881]^c32[4609]^c32[4938]^c32[4975]^c32[4976]^c32[5492]^c32[5428]^c32[5697]^c32[5698]^c32[6260]^c32[6473]^c32[6368]^c32[6369]^c32[7014]^c32[7259]^c32[7144]^c32[7753]^c32[7978]^c32[7980]^c32[7981]^c32[8726]^c32[8528]^c32[8537]^c32[8538]^c32[9402]^c32[9343]^c32[9231]^c32[9232]^c32[10126]^c32[10353]^c32[10109]^c32[10110]^c32[10754]^c32[11009]^c32[11132]^c32[11133]^c32[11520]^c32[11788]^c32[11742]^c32[12563]^c32[12427]^c32[12376]^c32[12377]^c32[12420]^c32[13339]^c32[13356]^c32[13140]^c32[13141]^c32[13851]^c32[13849]^c32[13879]^c32[13880]^c32[13938]^c32[14941]^c32[14708]^c32[14966]^c32[15475]^c32[15542]^c32[15567]^c32[15568]^c32[16259]^c32[16409]^c32[16269]^c32[16270];
break;
case 44:
d32[16896]=c32[197]^c32[350]^c32[95]^c32[166]^c32[80]^c32[1061]^c32[908]^c32[1129]^c32[1860]^c32[1886]^c32[1754]^c32[2563]^c32[2651]^c32[2578]^c32[3250]^c32[3294]^c32[3237]^c32[3911]^c32[3874]^c32[3986]^c32[4714]^c32[4659]^c32[4697]^c32[5597]^c32[5533]^c32[5419]^c32[5454]^c32[6365]^c32[6194]^c32[6474]^c32[7119]^c32[6980]^c32[7249]^c32[7075]^c32[7858]^c32[7699]^c32[7702]^c32[8831]^c32[8633]^c32[8643]^c32[9507]^c32[9448]^c32[9337]^c32[10231]^c32[10074]^c32[10215]^c32[10859]^c32[11114]^c32[10854]^c32[11625]^c32[11893]^c32[11847]^c32[12668]^c32[12532]^c32[12482]^c32[13060]^c32[13077]^c32[13246]^c32[13956]^c32[13954]^c32[13985]^c32[14662]^c32[14813]^c32[14687]^c32[15580]^c32[15647]^c32[15673]^c32[16364]^c32[16130]^c32[16375];
break;
case 45:
d32[17280]=c32[903]^c32[4757]^c32[7695];
break;
}
}
}
extern "C" int ldpc_BG1_Zc384_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs,cudaStream_t *stream,int sidx) {
dim3 numblocks(n_inputs,46);
ldpc_BG1_Zc384_worker<<<numblocks,384,0,stream[sidx]>>>(c,d);
cudaError_t err=cudaPeekAtLastError();
if (err!=cudaSuccess) {
printf("cuda error: %s (c %p, d %p)\n",cudaGetErrorString(err),c,d);
exit(-1);
}
return(0);
}

131
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_deinterleave.cu Normal file
View File

@@ -0,0 +1,131 @@
// deinterleave_u16.cu
#include <cuda_runtime.h>
#include <stdio.h>
#include <stdint.h>
__global__ void deinterleave_i16_2(int16_t* __restrict__ e,
const int16_t* __restrict__ f,
int E1,
int E2,
int r_firstE2)
{
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
int E = (r<r_firstE2) ? E1 : E2;
int EQm = E/2;
if (g >= EQm) return;
int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
const int16_t *in = f +r_off + 2*g;
int16_t* e0 = e + r_off;
int16_t* e1 = e0 + EQm;
e0[g] = in[0];
e1[g] = in[1];
}
__global__ void deinterleave_i16_4(int16_t* __restrict__ e,
const int16_t* __restrict__ f,
int E1,
int E2,
int r_firstE2)
{
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
int E = (r<r_firstE2) ? E1 : E2;
int EQm = E/4;
if (g >= EQm) return;
int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
const int16_t *in = f + r_off + 4*g;
int16_t* e0 = e + r_off;
int16_t* e1 = e0 + EQm;
int16_t* e2 = e1 + EQm;
int16_t* e3 = e2 + EQm;
e0[g] = in[0];
e1[g] = in[1];
e2[g] = in[2];
e3[g] = in[3];
}
__global__ void deinterleave_i16_6(int16_t* __restrict__ e,
const int16_t* __restrict__ f,
int E1,
int E2,
int r_firstE2)
{
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
int E = (r<r_firstE2) ? E1 : E2;
int EQm = E/6;
if (g >= EQm) return;
int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
const int16_t* in = f + r_off + 6*g;
int16_t* e0 = e + r_off;
int16_t* e1 = e0 + EQm;
int16_t* e2 = e1 + EQm;
int16_t* e3 = e2 + EQm;
int16_t* e4 = e3 + EQm;
int16_t* e5 = e4 + EQm;
e0[g] = in[0];
e1[g] = in[1];
e2[g] = in[2];
e3[g] = in[3];
e4[g] = in[4];
e5[g] = in[5];
}
__global__ void deinterleave_i16_8(int16_t* __restrict__ e,
const int16_t* __restrict__ f,
const int E1,
const int E2,
const int r_firstE2)
{
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
int E = (r<r_firstE2) ? E1 : E2;
int EQm = E/8;
if (g >= EQm) return;
int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
const int16_t *in = (f + r_off + 8*g);
int16_t* e0 = e + r_off;
int16_t* e1 = e0 + EQm;
int16_t* e2 = e1 + EQm;
int16_t* e3 = e2 + EQm;
int16_t* e4 = e3 + EQm;
int16_t* e5 = e4 + EQm;
int16_t* e6 = e5 + EQm;
int16_t* e7 = e6 + EQm;
e0[g] = in[0];
e1[g] = in[1];
e2[g] = in[2];
e3[g] = in[3];
e4[g] = in[4];
e5[g] = in[5];
e6[g] = in[6];
e7[g] = in[7];
}
// Host launcher
extern "C" void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx)
{
const int threads = 256;
dim3 blocks(((E2/Qm) + threads - 1) / threads,C);
switch (Qm) {
case 2: deinterleave_i16_2<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
case 4: deinterleave_i16_4<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
case 6: deinterleave_i16_6<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
case 8: deinterleave_i16_8<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
default: /* unsupported */ break;
}
cudaError_t err=cudaPeekAtLastError();
if (err!=cudaSuccess) {
printf("cuda error (deinterleave_i16): %s (e %p, f %p, E1 %d, E2 %d, Qm %d, C %d)\n",cudaGetErrorString(err),e,f,E1,E2,Qm,C);
exit(-1);
}
}

77
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_encode_parity_check_cuda.c Normal file
View File

@@ -0,0 +1,77 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*!\file ldpc_encode_parity_check.c
* \brief Parity check function used by ldpc encoders
* \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
* \email openair_tech@eurecom.fr
* \date 27-03-2018
* \version 1.0
* \note
* \warning
*/
#include <stdlib.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include "assertions.h"
#include "common/utils/LOG/log.h"
#include <cuda_runtime.h>
int ldpc_BG1_Zc384_cuda32(uint32_t **c,uint32_t **d,int n_inputs, cudaStream_t *stream,int sidx);
void encode_parity_check_part_cuda(uint32_t **c, uint32_t **d, short BG,short Zc,short Kb, int ncols, int n_inputs, cudaStream_t *stream,int sidx)
{
if (BG == 1) {
switch (Zc) {
case 176:
case 192:
case 208:
case 224:
case 240:
case 256:
case 288:
case 320:
case 352:
AssertFatal(1==0,"BG %d Zc %d not supported yet for CUDA\n",BG, Zc);
break;
case 384:
ldpc_BG1_Zc384_cuda32(c, d, n_inputs, stream,sidx);
break;
default:
AssertFatal(false, "BG %d Zc %d is not supported yet\n", BG, Zc);
}
} else if (BG == 2) {
switch (Zc) {
case 72:
case 80:
case 88:
case 96:
case 104:
case 112:
case 120:
case 128:
case 144:
case 160:
case 176:
case 192:
case 208:
case 224:
case 240:
case 256:
case 288:
case 320:
case 352:
case 384:
default:
AssertFatal(false , "BG %d Zc %d is not supported yet\n", BG, Zc);
}
} else
AssertFatal(false, "BG %d is not supported\n", BG);
}

194
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_encoder_cuda32.c Normal file
View File

@@ -0,0 +1,194 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file ldpc_encoder_cuda32.c
* \brief Defines the optimized LDPC encoder for NVidia GPUs
* \email openair_tech@eurecom.fr
* \date 11-30-2025
* \version 1.0
* \note
* \warning
*/
#include <stdlib.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include "assertions.h"
#include "common/utils/LOG/log.h"
#include "time_meas.h"
#include "openair1/PHY/CODING/nrLDPC_defs.h"
#include "PHY/sse_intrin.h"
#include "openair1/PHY/CODING/nrLDPC_extern.h"
#include <cuda_runtime.h>
//#define DEBUG_LDPC 1
#include "ldpc_encode_parity_check_cuda.c"
uint32_t *c_dev;
uint32_t **c_host;
uint32_t *c_devh[4];
uint32_t *d_dev;
uint32_t **d_host;
uint32_t *d_devh[4];
uint32_t *input_dev;
uint32_t **input_host;
uint32_t *input_devh[128];
int managed = 0, concurrent = 0, uva = 0, pageable = 0, pageable_uses_host = 0, register_host = 0, integrated = 0;
int cuda_support_set = 0;
extern cudaStream_t encoderStreams[4];
int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *s,int sidx);
void cuda_support_init() {
int dev = 0;
struct cudaDeviceProp prop;
cudaGetDeviceProperties(&prop, dev);
cudaDeviceGetAttribute(&managed, cudaDevAttrManagedMemory, dev);
cudaDeviceGetAttribute(&concurrent, cudaDevAttrConcurrentManagedAccess, dev);
cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, dev);
cudaDeviceGetAttribute(&pageable, cudaDevAttrPageableMemoryAccess, dev);
cudaDeviceGetAttribute(&pageable_uses_host, cudaDevAttrPageableMemoryAccessUsesHostPageTables, dev);
cudaDeviceGetAttribute(&register_host, cudaDevAttrHostRegisterSupported,dev);
cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated,dev);
LOG_I(NR_PHY,"Device: %s (cc %d.%d)\n", prop.name, prop.major, prop.minor);
LOG_I(NR_PHY,"Unified Virtual Addressing (UVA): %s\n", uva ? "YES" : "NO");
LOG_I(NR_PHY,"Managed (Unified) Memory: %s\n", managed ? "YES" : "NO");
LOG_I(NR_PHY,"Concurrent managed access: %s\n", concurrent ? "YES" : "NO");
LOG_I(NR_PHY,"Pageable memory access: %s\n", pageable ? "YES" : "NO");
LOG_I(NR_PHY,"Uses host page tables: %s\n", pageable_uses_host ? "YES" : "NO");
LOG_I(NR_PHY,"Host Register supported: %s\n", register_host ? "YES" : "NO");
LOG_I(NR_PHY,"Integrated (shared) Memory %s\n", integrated ? "YES" : "NO");
if (!pageable && !integrated) {
LOG_I(NR_PHY,"Allocating c,d,cc arrays for GPU \n");
cudaError_t err=cudaMalloc((void **)&c_dev,4*sizeof(uint32_t*));
AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
for (int i=0;i<4;i++) {
err=cudaMalloc((void**)&c_devh[i],2*22*384*sizeof(uint32_t));
AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (chost[%d]): %s\n", i,cudaGetErrorString(err));
}
err = cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
err=cudaMalloc((void**)&d_dev,4*sizeof(uint32_t*));
AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
for (int i=0;i<4;i++) {
err=cudaMalloc((void**)&d_devh[i],68*384*sizeof(uint32_t));
AssertFatal(err == cudaSuccess,"CUDA Error (d_devh[%d]: %s\n", i,cudaGetErrorString(err));
err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
}
err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
err=cudaMalloc((void**)&input_dev,128*sizeof(uint8_t*));
AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
for (int i=0;i<128;i++) {
err=cudaMalloc((void**)&input_devh[i],(8448/8)*sizeof(uint8_t));
AssertFatal(err == cudaSuccess,"CUDA Error (input_devh[%d]: %s\n", i,cudaGetErrorString(err));
err=cudaHostAlloc((void**)&input_host[i],(8448/8)*sizeof(uint8_t),cudaHostAllocDefault);
AssertFatal(err == cudaSuccess,"CUDA Error (input_host[%d]): %s\n", i,cudaGetErrorString(err));
}
err=cudaMemcpy(input_dev,input_devh,128*sizeof(uint8_t*),cudaMemcpyHostToDevice);
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy cc_devh -> d_dev): %s\n", cudaGetErrorString(err));
}
else {
LOG_I(NR_PHY,"Allocating c,d,cc arrays for CPU/GPU shared-memory\n");
cudaError_t err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocMapped|cudaHostAllocPortable);
AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
err = cudaHostGetDevicePointer((void**)&c_dev, c_host, 0);
AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
LOG_I(NR_PHY,"c_host %p, c_dev %p\n",c_host,c_dev);
for (int i=0;i<4;i++) {
err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocMapped);
AssertFatal(err == cudaSuccess,"CUDA Error (c_host[%d]): %s\n", i,cudaGetErrorString(err));
err = cudaHostGetDevicePointer((void**)&c_devh[i], c_host[i], 0);
AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
}
err=cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocMapped);
AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
err=cudaHostGetDevicePointer((void**)&d_dev, d_host, 0);
AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(d_dev): %s\n", cudaGetErrorString(err));
LOG_I(NR_PHY,"d_host %p, d_dev %p\n",d_host,d_dev);
for (int i=0;i<4;i++) {
err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocMapped);
AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
err=cudaHostGetDevicePointer((void**)&d_devh[i], d_host[i], 0);
AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) d_devh[%d]: %s\n", i,cudaGetErrorString(err));
LOG_I(NR_PHY,"d_host[%d] %p, d_devh[%d] %p\n",i,d_host[i],i,d_devh[i]);
}
err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocMapped);
AssertFatal(err == cudaSuccess,"CUDA Error (input_host): %s\n", cudaGetErrorString(err));
err=cudaHostGetDevicePointer((void**)&input_dev, input_host, 0);
AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(cc_host): %s\n", cudaGetErrorString(err));
LOG_I(NR_PHY,"input_host %p, input_dev %p\n",input_host,input_dev);
}
cuda_support_set=1;
}
uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp)
{
//set_log(PHY, 4);
int Zc = impp->Zc;
int Kb = impp->Kb;
short block_length = impp->K;
short BG = impp->BG;
int ncols=22;
int encoder_stream=0;
AssertFatal(BG==1,"BG %d is not supported for CUDA version\n",BG);
AssertFatal(Zc==384,"Zc %d is not supported for CUDA version \n", Zc);
if(impp->tinput != NULL) start_meas(impp->tinput);
#ifdef DEBUG_LDPC
LOG_I(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
LOG_I(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
#endif
int n_inputs = (impp->n_segments/32)+(((impp->n_segments&31) > 0) ? 1: 0);
// uint32_t cc[4][22*Zc]; //padded input, unpacked, max size
if (!pageable&&!integrated) { // this means we are not on shared memory
for (int r=0;r<impp->n_segments;r++) {
cudaMemcpyAsync(input_devh[r],input[r],block_length>>3,cudaMemcpyHostToDevice,encoderStreams[encoder_stream]);
}
}
ldpc_input(pageable||integrated? (uint32_t**)input : (uint32_t**)input_dev,(uint32_t**)c_dev,impp->n_segments,encoderStreams,encoder_stream);
if(impp->tinput != NULL) stop_meas(impp->tinput);
//parity check part
if(impp->tparity != NULL) start_meas(impp->tparity);
encode_parity_check_part_cuda((uint32_t**)c_dev, (uint32_t**)d_dev, BG, Zc, Kb, ncols,n_inputs,encoderStreams,encoder_stream);
if (!pageable&&!integrated) { // this means we are not on shared memory
for (int r=0; r<n_inputs;r++) cudaMemcpyAsync(d_host[r],d_devh[r],68*384*sizeof(uint32_t),cudaMemcpyDeviceToHost,encoderStreams[encoder_stream]);
}
cudaStreamSynchronize(encoderStreams[encoder_stream]);
if(impp->tparity != NULL) stop_meas(impp->tparity);
return d_host;
}

92
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_input.cu Normal file
View File

@@ -0,0 +1,92 @@
#include <stdio.h>
#include <stdint.h>
#include <cuda_runtime.h>
/*
__device__ const uint32_t masks[4] = {0x80,0x8000,0x800000,0x80000000};
__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) {
int block_off = blockIdx.x*blockDim.x<<2;
int i2 = threadIdx.x<<2;
uint32_t *out=cc[blockIdx.y] + block_off + i2;
int nseg0 = (blockIdx.y << 5);
int nseg1;
if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
else nseg1 = nseg0 + (nseg&31);
int bit_offset = i2+block_off;
int uint32_offset = bit_offset>>5;
uint32_t mask = masks[(bit_offset&31)>>3];
uint32_t mask0 = mask>>(bit_offset&7);bit_offset++;
uint32_t mask1 = mask>>(bit_offset&7);bit_offset++;
uint32_t mask2 = mask>>(bit_offset&7);bit_offset++;
uint32_t mask3 = mask>>(bit_offset&7);
uint32_t tmp,jmod;
uint32_t otmp0,otmp1,otmp2,otmp3;
if (bit_offset < block_length) {
tmp=input[nseg0][uint32_offset];
otmp0 = ((tmp&mask0) > 0);
otmp1 = ((tmp&mask1) > 0);
otmp2 = ((tmp&mask2) > 0);
otmp3 = ((tmp&mask3) > 0);
for (int j=nseg0+1;j<nseg1;j++) {
tmp=input[j][uint32_offset];
jmod = j&31;
otmp0 |= (((tmp&mask0) > 0)<<jmod);
otmp1 |= (((tmp&mask1) > 0)<<jmod);
otmp2 |= (((tmp&mask2) > 0)<<jmod);
otmp3 |= (((tmp&mask3) > 0)<<jmod);
}
out[0]=otmp0;
out[1]=otmp1;
out[2]=otmp2;
out[3]=otmp3;
}
}
*/
__device__ uint32_t masks[32] = {
0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1,
0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100,
0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000,
0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000};
__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int nseg) {
// int block_off = blockIdx.y*blockDim.x;
int i1 = blockIdx.y;
int i2 = threadIdx.x;
int nseg0 = (blockIdx.x << 5);
int nseg1;
if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
else nseg1 = nseg0 + (nseg&31);
int bit_offset = i2+(i1*384);
int uint32_offset = bit_offset>>5;
uint32_t mask0 = masks[bit_offset&31];
uint32_t tmp,jmod;
uint32_t otmp0;
if (bit_offset < 8448) {
tmp=input[nseg0][uint32_offset];
otmp0 = ((tmp&mask0) > 0);
for (int j=nseg0+1;j<nseg1;j++) {
tmp=input[j][uint32_offset];
jmod = j&31;
otmp0 |= (((tmp&mask0) > 0)<<jmod);
}
cc[blockIdx.x][(2*i1*384) + i2] = otmp0;
cc[blockIdx.x][(2*i1+1)*384 + i2]=otmp0;
}
}
extern "C" int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *stream,int sidx) {
int ns = nseg>>5;
if ((nseg&31)>0) ns++;
dim3 numblocks(ns,22);
//printf("input %p\n",input);
ldpc_input_worker<<<numblocks,384,0,stream[sidx]>>>(input,cc,nseg);
cudaError_t err=cudaPeekAtLastError();
if (err!=cudaSuccess) {
printf("cuda error: %s (input %p, cc %p, nseg %d, ns %d)\n",cudaGetErrorString(err),input,cc,nseg,ns);
exit(-1);
}
return(0);
}

492
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_input32.c Normal file
View File

@@ -0,0 +1,492 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
#include <stdlib.h>
#include <math.h>
#include <stdio.h>
#include <string.h>
#include "assertions.h"
#include "common/utils/LOG/log.h"
#include "openair1/PHY/CODING/nrLDPC_defs.h"
#include "PHY/sse_intrin.h"
#include "openair1/PHY/CODING/nrLDPC_extern.h"
void ldpc_input32(uint8_t **input, uint32_t **c, int n_inputs, int n_segments) {
#ifndef __aarch64__
simde__m256i andmask = simde_mm256_set_epi32(0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80); // every 8 bits -> 8 bytes, pattern repeats.
simde__m256i zero256 = simde_mm256_setzero_si256();
simde__m256i masks[32];
register simde__m256i c256;
for (int i=0;i<32;i++) {
masks[i] = simde_mm256_set1_epi32(1<<i);
}
for (int b=0;b<8448;b+=384) {
for (int i=0; i < 384; i += 8) {
unsigned int i8 = (2*b + i) >> 3;
for (int j0=0;j0<n_inputs;j0++) {
c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[j0<<5][i8]),andmask),zero256),masks[0]);
int j2 = 32;
if (j0==(n_inputs-1)) j2= n_segments-(j0<<5);
for (int j=1; j < j2; j++) {
c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[(j0<<5)+j][i8]),andmask),zero256),masks[j]),c256);
}
((simde__m256i *)c[j0])[i8] = c256;
((simde__m256i *)c[j0])[i8+48] = c256;
}
}
}
#else
int i2=0;
uint32_t cc[4][8448];
const int32_t ucShifta[32][4] = {
{-7,-6,-5,-4}, // 0
{-6,-5,-4,-3}, // 1
{-5,-4,-3,-2}, // 2
{-4,-3,-2,-1}, // 3
{-3,-2,-1,0}, // 4
{-2,-1,0,1}, // 5
{-1,0,1,2}, // 6
{0,1,2,3}, // 7
{1,2,3,4}, // 8
{2,3,4,5}, // 9
{3,4,5,6}, // 10
{4,5,6,7}, // 11
{5,6,7,8}, // 12
{6,7,8,9}, // 13
{7,8,9,10}, // 14
{8,9,10,11}, // 15
{9,10,11,12}, // 16
{10,11,12,13}, // 17
{11,12,13,14}, // 18
{12,13,14,15}, // 19
{13,14,15,16}, // 20
{14,15,16,17}, // 21
{15,16,17,18}, // 22
{16,17,18,19}, // 23
{17,18,19,20}, // 24
{18,19,20,21}, // 25
{19,20,21,22}, // 26
{20,21,22,23}, // 27
{21,22,23,24}, // 28
{22,23,24,25}, // 29
{23,24,25,26}, // 30
{24,25,26,27}}; // 31
const int32_t ucShiftb[32][4] = {
{-3,-2,-1,0}, // 0
{-2,-1,0,1}, // 1
{-1,0,1,2}, // 2
{0,1,2,3}, // 3
{1,2,3,4}, // 4
{2,3,4,5}, // 5
{3,4,5,6}, // 6
{4,5,6,7}, // 7
{5,6,7,8}, // 8
{6,7,8,9}, // 9
{7,8,9,10}, // 10
{8,9,10,11}, // 11
{9,10,11,12}, // 12
{10,11,12,13}, // 13
{11,12,13,14}, // 14
{12,13,14,15}, // 15
{13,14,15,16}, // 16
{14,15,16,17}, // 17
{15,16,17,18}, // 18
{16,17,18,19}, // 19
{17,18,19,20}, // 20
{18,19,20,21}, // 21
{19,20,21,22}, // 22
{20,21,22,23}, // 23
{21,22,23,24}, // 24
{22,23,24,25}, // 25
{23,24,25,26}, // 26
{24,25,26,27}, // 27
{25,26,27,28}, // 28
{26,27,28,29}, // 29
{27,28,29,30}, // 30
{28,29,30,31}}; // 31
const int32_t ucShiftc[32][4] = {
{-15,-14,-13,-12}, // 0
{-14,-13,-12,-11}, // 1
{-13,-12,-11,-10}, // 2
{-12,-11,-10,-9}, // 3
{-11,-10,-9,-8}, // 4
{-10,-9,-8,-7}, // 5
{-9,-8,-7,-6}, // 6
{-8,-7,-6,-5}, // 7
{-7,-6,-5,-4}, // 8
{-6,-5,-4,-3}, // 9
{-5,-4,-3,-2}, // 10
{-4,-3,-2,-1}, // 11
{-3,-2,-1,0}, // 12
{-2,-1,0,1}, // 13
{-1,0,1,2}, // 14
{0,1,2,3}, // 15
{1,2,3,4}, // 16
{2,3,4,5}, // 17
{3,4,5,6}, // 18
{4,5,6,7}, // 19
{5,6,7,8}, // 20
{6,7,8,9}, // 21
{7,8,9,10}, // 22
{8,9,10,11}, // 23
{9,10,11,12}, // 24
{10,11,12,13}, // 25
{11,12,13,14}, // 26
{12,13,14,15}, // 27
{13,14,15,16}, // 28
{14,15,16,17}, // 29
{15,16,17,18}, // 30
{16,17,18,19}}; // 31
const int32_t ucShiftd[32][4] = {
{-11,-10,-9,-8}, // 0
{-10,-9,-8,-7}, // 1
{-9,-8,-7,-6}, // 2
{-8,-7,-6,-5}, // 3
{-7,-6,-5,-4}, // 4
{-6,-5,-4,-3}, // 5
{-5,-4,-3,-2}, // 6
{-4,-3,-2,-1}, // 7
{-3,-2,-1,0}, // 8
{-2,-1,0,1}, // 9
{-1,0,1,2}, // 10
{0,1,2,3}, // 11
{1,2,3,4}, // 12
{2,3,4,5}, // 13
{3,4,5,6}, // 14
{4,5,6,7}, // 15
{5,6,7,8}, // 16
{6,7,8,9}, // 17
{7,8,9,10}, // 18
{8,9,10,11}, // 19
{9,10,11,12}, // 20
{10,11,12,13}, // 21
{11,12,13,14}, // 22
{12,13,14,15}, // 23
{13,14,15,16}, // 24
{14,15,16,17}, // 25
{15,16,17,18}, // 26
{16,17,18,19}, // 27
{17,18,19,20}, // 28
{18,19,20,21}, // 29
{19,20,21,22}, // 30
{20,21,22,23}};// 31
const int32_t ucShifte[32][4] = {
{-23,-22,-21,-20}, // 0
{-22,-21,-20,-19}, // 1
{-21,-20,-19,-18}, // 2
{-20,-19,-18,-17}, // 3
{-19,-18,-17,-16}, // 4
{-18,-17,-16,-15}, // 5
{-17,-16,-15,-14}, // 6
{-16,-15,-14,-13}, // 7
{-15,-14,-13,-12}, // 8
{-14,-13,-12,-11}, // 9
{-13,-12,-11,-10}, // 10
{-12,-11,-10,-9}, // 11
{-11,-10,-9,-8}, // 12
{-10,-9,-8,-7}, // 13
{-9,-8,-7,-6}, // 14
{-8,-7,-6,-5}, // 15
{-7,-6,-5,-4}, // 16
{-6,-5,-4,-3}, // 17
{-5,-4,-3,-2}, // 18
{-4,-3,-2,-1}, // 19
{-3,-2,-1,0}, // 20
{-2,-1,0,1}, // 21
{-1,0,1,2}, // 22
{0,1,2,3}, // 23
{1,2,3,4}, // 24
{2,3,4,5}, // 25
{3,4,5,6}, // 26
{4,5,6,7}, // 27
{5,6,7,8}, // 28
{6,7,8,9}, // 29
{7,8,9,10}, // 30
{8,9,10,11}}; // 31
const int32_t ucShiftf[32][4] = {
{-19,-18,-17,-16}, // 0
{-18,-17,-16,-15}, // 1
{-17,-16,-15,-14}, // 2
{-16,-15,-14,-13}, // 3
{-15,-14,-13,-12}, // 4
{-14,-13,-12,-11}, // 5
{-13,-12,-11,-10}, // 6
{-12,-11,-10,-9}, // 7
{-11,-10,-9,-8}, // 8
{-10,-9,-8,-7}, // 9
{-9,-8,-7,-6}, // 10
{-8,-7,-6,-5}, // 11
{-7,-6,-5,-4}, // 12
{-6,-5,-4,-3}, // 13
{-5,-4,-3,-2}, // 14
{-4,-3,-2,-1}, // 15
{-3,-2,-1,0}, // 16
{-2,-1,0,1}, // 17
{-1,0,1,2}, // 18
{0,1,2,3}, // 19
{1,2,3,4}, // 20
{2,3,4,5}, // 21
{3,4,5,6}, // 22
{4,5,6,7}, // 23
{5,6,7,8}, // 24
{6,7,8,9}, // 25
{7,8,9,10}, // 26
{8,9,10,11}, // 27
{9,10,11,12}, // 28
{10,11,12,13}, // 29
{11,12,13,14}, // 30
{12,13,14,15}}; // 31
const int32_t ucShiftg[32][4] = {
{-31,-30,-29,-28},
{-30,-29,-28,-27},
{-29,-28,-27,-26},
{-28,-27,-26,-25},
{-27,-26,-25,-24},
{-26,-25,-24,-23},
{-25,-24,-23,-22},
{-24,-23,-22,-21},
{-23,-22,-21,-20}, // 0
{-22,-21,-20,-19}, // 1
{-21,-20,-19,-18}, // 2
{-20,-19,-18,-17}, // 3
{-19,-18,-17,-16}, // 4
{-18,-17,-16,-15}, // 5
{-17,-16,-15,-14}, // 6
{-16,-15,-14,-13}, // 7
{-15,-14,-13,-12}, // 8
{-14,-13,-12,-11}, // 9
{-13,-12,-11,-10}, // 10
{-12,-11,-10,-9}, // 11
{-11,-10,-9,-8}, // 12
{-10,-9,-8,-7}, // 13
{-9,-8,-7,-6}, // 14
{-8,-7,-6,-5}, // 15
{-7,-6,-5,-4}, // 16
{-6,-5,-4,-3}, // 17
{-5,-4,-3,-2}, // 18
{-4,-3,-2,-1}, // 19
{-3,-2,-1,0}, // 20
{-2,-1,0,1}, // 21
{-1,0,1,2}, // 22
{0,1,2,3}}; // 23
const int32_t ucShifth[32][4] = {
{-27,-26,-25,-24}, // 0
{-26,-25,-24,-23}, // 1
{-25,-24,-23,-22}, // 2
{-24,-23,-22,-21}, // 3
{-23,-22,-21,-20}, // 4
{-22,-21,-20,-19}, // 5
{-21,-20,-19,-18}, // 6
{-20,-19,-18,-17}, // 7
{-19,-18,-17,-16}, // 8
{-18,-17,-16,-15}, // 9
{-17,-16,-15,-14}, // 10
{-16,-15,-14,-13}, // 11
{-15,-14,-13,-12}, // 12
{-14,-13,-12,-11}, // 13
{-13,-12,-11,-10}, // 14
{-12,-11,-10,-9}, // 15
{-11,-10,-9,-8}, // 16
{-10,-9,-8,-7}, // 17
{-9,-8,-7,-6}, // 18
{-8,-7,-6,-5}, // 19
{-7,-6,-5,-4}, // 20
{-6,-5,-4,-3}, // 21
{-5,-4,-3,-2}, // 22
{-4,-3,-2,-1}, // 23
{-3,-2,-1,0}, // 24
{-2,-1,0,1}, // 25
{-1,0,1,2}, // 26
{0,1,2,3}, // 27
{1,2,3,4}, // 28
{2,3,4,5}, // 29
{3,4,5,6}, // 30
{4,5,6,7}}; // 31
const uint32_t __attribute__ ((aligned (16))) masksa[4] = {0x80,0x40,0x20,0x10};
const uint32_t __attribute__ ((aligned (16))) masksb[4] = {0x8,0x4,0x2,0x1};
const uint32_t __attribute__ ((aligned (16))) masksc[4] = {0x8000,0x4000,0x2000,0x1000};
const uint32_t __attribute__ ((aligned (16))) masksd[4] = {0x800,0x400,0x200,0x100};
const uint32_t __attribute__ ((aligned (16))) maskse[4] = {0x800000,0x400000,0x200000,0x100000};
const uint32_t __attribute__ ((aligned (16))) masksf[4] = {0x80000,0x40000,0x20000,0x10000};
const uint32_t __attribute__ ((aligned (16))) masksg[4] = {0x80000000,0x40000000,0x20000000,0x10000000};
const uint32_t __attribute__ ((aligned (16))) masksh[4] = {0x8000000,0x4000000,0x2000000,0x1000000};
int32x4_t vshifta[32],vshiftb[32],vshiftc[32],vshiftd[32],vshifte[32],vshiftf[32],vshiftg[32],vshifth[32];
uint32x4_t vmasksa = vld1q_u32(masksa);
uint32x4_t vmasksb = vld1q_u32(masksb);
uint32x4_t vmasksc = vld1q_u32(masksc);
uint32x4_t vmasksd = vld1q_u32(masksd);
uint32x4_t vmaskse = vld1q_u32(maskse);
uint32x4_t vmasksf = vld1q_u32(masksf);
uint32x4_t vmasksg = vld1q_u32(masksg);
uint32x4_t vmasksh = vld1q_u32(masksh);
uint32x4_t in;
for (int n=0;n<32;n++) {
vshifta[n] = vld1q_s32(ucShifta[n]);
vshiftb[n] = vld1q_s32(ucShiftb[n]);
vshiftc[n] = vld1q_s32(ucShiftc[n]);
vshiftd[n] = vld1q_s32(ucShiftd[n]);
vshifte[n] = vld1q_s32(ucShifte[n]);
vshiftf[n] = vld1q_s32(ucShiftf[n]);
vshiftg[n] = vld1q_s32(ucShiftg[n]);
vshifth[n] = vld1q_s32(ucShifth[n]);
}
i2=0;
int j0=0,j1=0,j2=0,j3=0;
if (n_segments <= 32) {
j0=n_segments;
}
else if (n_segments <= 64) {
j0=32;
j1=n_segments-32;
}
else if (n_segments <= 96) {
j0=32; j1=32;
j2=n_segments-64;
}
else if (n_segments <= 128) {
j0=32; j1=32; j2=32;
j3=n_segments-96;
}
uint32x4_t *ccp,cc0,cc1,cc2,cc3,cc4,cc5,cc6,cc7;
for (int i=0; i < 264; i++,i2+=8) {
in = vdupq_n_u32(((uint32_t*)input[0])[i]);
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
for (int j = 1; j < j0; j++) {
in = vdupq_n_u32(((uint32_t*)input[j])[i]);
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
}
ccp=&((uint32x4_t *)cc[0])[i2];
ccp[0] = cc0;
ccp[1] = cc1;
ccp[2] = cc2;
ccp[3] = cc3;
ccp[4] = cc4;
ccp[5] = cc5;
ccp[6] = cc6;
ccp[7] = cc7;
if (j1>0) {
in = vdupq_n_u32(((uint32_t*)input[32])[i]);
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
for (int j = 1; j < j1; j++) {
in = vdupq_n_u32(((uint32_t*)input[32+j])[i]);
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
}
ccp=&((uint32x4_t *)cc[1])[i2];
ccp[0] = cc0;
ccp[1] = cc1;
ccp[2] = cc2;
ccp[3] = cc3;
ccp[4] = cc4;
ccp[5] = cc5;
ccp[6] = cc6;
ccp[7] = cc7;
}
if (j2>0) {
in = vdupq_n_u32(((uint32_t*)input[64])[i]);
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
for (int j = 1; j < j2; j++) {
in = vdupq_n_u32(((uint32_t*)input[64+j])[i]);
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
}
ccp=&((uint32x4_t *)cc[2])[i2];
ccp[0] = cc0;
ccp[1] = cc1;
ccp[2] = cc2;
ccp[3] = cc3;
ccp[4] = cc4;
ccp[5] = cc5;
ccp[6] = cc6;
ccp[7] = cc7;
}
if (j3>0) {
in = vdupq_n_u32(((uint32_t*)input[96])[i]);
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
for (int j = 1; j < j3; j++) {
in = vdupq_n_u32(((uint32_t*)input[96+j])[i]);
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
}
ccp=&((uint32x4_t *)cc[3])[i2];
ccp[0] = cc0;
ccp[1] = cc1;
ccp[2] = cc2;
ccp[3] = cc3;
ccp[4] = cc4;
ccp[5] = cc5;
ccp[6] = cc6;
ccp[7] = cc7;
}
}
for (int s=0;s<n_inputs;s++) {
for (int i1 = 0; i1 < 22; i1++) {
memcpy(&c[s][2 * i1 * 384] , &cc[s][i1 * 384], 384 * sizeof(uint32_t));
memcpy(&c[s][(2 * i1 + 1) * 384], &cc[s][i1 * 384], 384 * sizeof(uint32_t));
}
}
#endif
}

300
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_rate_matching.cu Normal file
View File

@@ -0,0 +1,300 @@
/*
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The OpenAirInterface Software Alliance licenses this file to You under
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
* except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.openairinterface.org/?page_id=698
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*-------------------------------------------------------------------------------
* For more information about the OpenAirInterface (OAI) Software Alliance:
* contact@openairinterface.org
*/
#include <stdio.h>
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h"
__device__ __forceinline__ int clamp_i16_to_i8(int x)
{
// x is int (promoted)
x = (x < -128) ? -128 : x;
x = (x > 127) ? 127 : x;
return x;
}
__device__ __forceinline__ uint32_t packs_4x16_to_4x8(uint32_t a, uint32_t b)
{
int a0 = (int)(int16_t)(a & 0xFFFFu);
int a1 = (int)(int16_t)(a >> 16);
int b0 = (int)(int16_t)(b & 0xFFFFu);
int b1 = (int)(int16_t)(b >> 16);
uint32_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
uint32_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
uint32_t o2 = (uint8_t)(int8_t)clamp_i16_to_i8(b0);
uint32_t o3 = (uint8_t)(int8_t)clamp_i16_to_i8(b1);
return (o0) | (o1 << 8) | (o2 << 16) | (o3 << 24);
}
__device__ __forceinline__ uint16_t packs_2x16_to_2x8(uint32_t a)
{
int a0 = (int)(int16_t)(a & 0xFFFFu);
int a1 = (int)(int16_t)(a >> 16);
uint16_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
uint16_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
return (o0) | (o1 << 8) ;
}
__global__ void rm(int Ncb_4,int ind0_4, int E1, int E2, int r_firstE2, int Foffset_4, int F_4, int clear, int seglen_4, int K_4, int Z_4, uint32_t *d, uint32_t *e, uint32_t *llr_buffer) {
int ind_4 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
uint32_t *dr = d + (2*r*seglen_4);
uint32_t *llr_bufferr = llr_buffer + (r*seglen_4);
#ifdef DEBUG_RM
if (blockIdx.x == 0 && threadIdx.x == 0) printf("r %d, seglen %d\n",r,seglen_4);
#endif
if (ind_4>= Foffset_4 && ind_4 < Foffset_4+F_4) {
llr_bufferr[ind_4-Foffset_4+K_4-F_4] = 0x7f7f7f7f;
#ifdef DEBUG_RM
if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",seglen_4*r + ind_4-Foffset_4+K_4-F_4,ind_4,K_4,Foffset_4,F_4);
#endif
return;
}
if (ind_4 >= Ncb_4) return;
if (clear == 1) { dr[2*ind_4] = 0; dr[(2*ind_4)+1]=0; }
int E_4;
if (r<r_firstE2) E_4=(E1/4); else E_4=(E2/4);
int ind1=ind0_4,ind2;
int k=0;
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
#endif
if (ind1 < Foffset_4) {
int ind2 = ind1 + min(Foffset_4-ind1,E_4);
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0) printf("check 1b: ind %d ind1 %d ind2 %d\n",ind,ind1,ind2);
#endif
if (ind_4 >= ind1 && ind_4 < ind2) {
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(ind_4-ind1)]);
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(ind_4-ind1)+1]);
}
#ifdef DEBUG_RM
if (r<=1 && threadIdx.x == 0 && ind_4 >= ind1 && ind_4 < ind2) printf("write 1. r %d : ind %d, ind1 %d, ind2 %d,k %d/E %d, pos %d, er %d %d\n",r,ind_4,ind1,ind2,ind_4-ind1,E_4,4*(ind_4-ind1),(int16_t)(er[2*(ind_4-ind1)]&0xffff),(int16_t)(er[2*(ind_4-ind1)]>>16));
#endif
k=ind2-ind1;
ind1 = ind2;
}
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
#endif
if (ind1 >= Foffset_4 && ind1 < Foffset_4 + F_4) ind1 = Foffset_4 + F_4;
ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0) printf("check 2b: ind %d ind1 %d ind2 %d\n",ind_4,ind1,ind2);
#endif
if (ind_4 >= ind1 && ind_4 < ind2) {
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+(ind_4-ind1))]);
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+(ind_4-ind1))+1]);
}
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0 && ind >= ind1 && ind < ind2) printf("write 2. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
#endif
k+=(ind2-ind1);
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0) printf("check k %d E %d\n",k,E);
#endif
while (k < E_4) {
ind2 = min(Foffset_4,E_4-k);
if (ind_4 < ind2) {
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+ind_4)]);
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4)+1]);
}
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0 && ind < ind2 && ind >= ind1) printf("3. ind %d, ind2 %d, k %d/E %d\n",ind,ind2,k+ind,E);
#endif
k+=ind2;
ind1=Foffset_4+F_4;
ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
if (ind_4 >= ind1 && ind_4 < ind2 && k < E_4) {
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+ind_4-ind1)]);
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4-ind1)+1]);
}
#ifdef DEBUG_RM
if (r==0 && threadIdx.x == 0 && ind < ind && ind >= ind1) printf("4. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
#endif
k+=(ind2-ind1);
}
// note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in position r*seglen + (Kprime+F = K .. 2Z+(66*Z)=seglen
llr_bufferr[2*Z_4 + ind_4] = packs_4x16_to_4x8(dr[(2*ind_4)], dr[(2*ind_4)+1]);
#ifdef DEBUG_RM
if (r<=1 && threadIdx.x == 0) printf("RM segment %d: writing %x to position %d (ind %d)\n",r,llr_bufferr[2*Z_4 + ind_4],r*seglen_4 + 2*Z_4 + ind_4,ind_4);
#endif
}
__global__ void rm2(int Ncb_2,int ind0_2, int E1, int E2, int r_firstE2, int Foffset_2, int F_2, int clear, int seglen_2, int K_2, int Z_2, uint32_t *d, uint32_t *e, uint16_t *llr_buffer) {
int ind_2 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
int r = (int)blockIdx.y;
uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
uint32_t *dr = d + (r*seglen_2);
uint16_t *llr_bufferr = llr_buffer + (r*seglen_2);
if (ind_2 >= Ncb_2) return;
#ifdef RM_DEBUG
if (r==0) printf("Looking for ind %d...%d, Ncb %d\n",ind_2*2,1+ind_2*2,Ncb_2*2);
#endif
if (ind_2>= Foffset_2 && ind_2 < Foffset_2+F_2) {
llr_bufferr[ind_2-Foffset_2+K_2-F_2] = 0x7f7f;
#ifdef RM_DEBUG
if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",(seglen_2*r + ind_2-Foffset_2+K_2-F_2)*2,ind_2*2,K_2*2,Foffset_2*2,F_2*2);
#endif
return;
}
if (clear == 1) dr[ind_2] = 0;
int E_2;
if (r<r_firstE2) E_2=E1/2; else E_2=E2/2;
int ind1=ind0_2,ind2;
int k=0;
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
if (ind1 < Foffset_2) {
int ind2 = ind1 + min(Foffset_2-ind1,E_2);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM1B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
if (ind_2 >= ind1 && ind_2 < ind2) {
dr[ind_2] = __vaddss2(dr[ind_2],er[ind_2-ind1]);
}
#ifdef RM_DEBUG
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM1A: ind_2 %d, ind1 %d, ind2 %d,k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(ind_2-ind1),2*E_2);
#endif
k=ind2-ind1;
ind1 = ind2;
}
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
if (ind1 >= Foffset_2 && ind1 < Foffset_2 + F_2) ind1 = Foffset_2 + F_2;
ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM2B: ind %d ind1 %d ind2 %d k%d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
if (ind_2 >= ind1 && ind_2 < ind2) {
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+(ind_2-ind1))]);
}
#ifdef RM_DEBUG
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM2A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(k+(ind-ind1)),2*E_2);
#endif
k+=(ind2-ind1);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check k %d E %d\n",k,E);
while (k < E_2) {
ind2 = min(Foffset_2,E_2-k);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM3B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
if (ind_2 < ind2) {
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+ind_2)]);
}
#ifdef RM_DEBUG
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 ) printf("RM3A: ind %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind2,2*(k+ind_2),2*E_2);
#endif
k+=ind2;
ind1=Foffset_2+F_2;
ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM4B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
if (ind_2 >= ind1 && ind_2 < ind2 && k < E_2) {
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+ind_2-ind1)]);
}
#ifdef RM_DEBUG
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 && ind_2 >= ind1) printf("RM4A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2_2,2*(k+ind_2-ind1),2*E_2);
#endif
k+=(ind2-ind1);
}
// note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in potiion r*seglent + (Kprime+F = K .. 2Z+(66*Z)=seglen
llr_bufferr[2*Z_2 + ind_2] = packs_2x16_to_2x8(dr[ind_2]);
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("writing %x to position %d (ind %d)\n",llr_bufferr[2*Z + ind_2],r*seglen + 2*Z + ind_2,ind_2);
}
static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
extern "C" int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
uint8_t BG,
uint16_t Z,
int16_t *d,
int16_t *soft_input,
int8_t *llr_buffer,
uint32_t K,
uint8_t C,
uint8_t rvidx,
uint8_t clear,
uint32_t E1,
uint32_t E2,
uint32_t r_firstE2,
uint32_t F,
uint32_t Foffset,
cudaStream_t *s,
int8_t sidx)
{
if (C == 0 || C>132) {
printf("nr_rate_matching: invalid parameter C %d\n", C);
exit(-1);
}
//Bit selection
uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
uint32_t Ncb;
if (Tbslbrm == 0)
Ncb = N;
else {
uint32_t Nref = (3 * Tbslbrm / (2 * C)); //R_LBRM = 2/3
Ncb = min(N, Nref);
}
uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
int nthreads=Z;
#ifdef RM_DEBUG
printf("\nrm (%d,%d): Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",nblocks.x,nthreads,Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
#endif
if ((E1&3) == 0 && (E2&3) == 0) {
dim3 nblocks(((Ncb>>2) + nthreads-1)/nthreads,C);
rm<<<nblocks, nthreads, 0, s[sidx]>>>(Ncb/4,ind/4,E1,E2,r_firstE2,Foffset/4,F/4,clear,68*384/4,K/4,Z/4,(uint32_t*)d,(uint32_t*)soft_input,(uint32_t*)llr_buffer);
}
else {
dim3 nblocks2(((Ncb>>1) + nthreads-1)/nthreads,C);
rm2<<<nblocks2, nthreads, 0, s[sidx]>>>(Ncb/2,ind/2,E1,E2,r_firstE2,Foffset/2,F/2,clear,68*384/2,K/2,Z/2,(uint32_t*)d,(uint32_t*)soft_input,(uint16_t*)llr_buffer);
}
cudaError_t err=cudaPeekAtLastError();
if (err!=cudaSuccess) {
printf("cuda error (nr_rate_matching_ldpc_rx_cuda): %s Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",cudaGetErrorString(err),Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
exit(-1);
}
return(0);
}

444
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_BnProcKernel_BG1.h Normal file
View File

@@ -0,0 +1,444 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_CUDA_BnProcKernel_BG1.h
* \brief Defines the kernels for bit node processing
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note
* \warning
*/
#include <cuda_runtime.h>
#include <stdint.h>
#include <stdio.h>
#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPCdecoder_defs.h"
__device__ __forceinline__ void unpack_and_sign_extend(uint32_t packed, uint32_t *val_lo, uint32_t *val_hi)
{
uint32_t magic_sub = 0x00800080;
uint32_t lo_zext = __byte_perm(packed, 0, 0x5140);
*val_lo = __vsub2(lo_zext ^ magic_sub, magic_sub);
uint32_t hi_zext = __byte_perm(packed, 0, 0x7362);
*val_hi = __vsub2(hi_zext ^ magic_sub, magic_sub);
}
__device__ __forceinline__ uint32_t saturate_and_pack(uint32_t val_lo, uint32_t val_hi)
{
uint32_t lo_clamped = __vmins2(val_lo, 0x007F007F); // min(v, 127)
lo_clamped = __vmaxs2(lo_clamped, 0xFF80FF80); // max(v, -128)
uint32_t hi_clamped = __vmins2(val_hi, 0x007F007F);
hi_clamped = __vmaxs2(hi_clamped, 0xFF80FF80);
return __byte_perm(lo_clamped, hi_clamped, 0x6420);
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R13(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t BnGrpIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t Bn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)BnGrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < BnGrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
if (BnGrpIdx == 1) {
BricksToBeGet = packed_intrinsic;
uint32_t MsgIdx = Bn2MsgStartIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
} else {
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
BricksToBeGet = __vsubss4(saturated_llr, prev);
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
}
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R23(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t BnGrpIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t Bn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)BnGrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < BnGrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
if (BnGrpIdx == 1) {
BricksToBeGet = packed_intrinsic;
uint32_t MsgIdx = Bn2MsgStartIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
} else {
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
BricksToBeGet = __vsubss4(saturated_llr, prev);
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
}
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R89(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t BnGrpIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t Bn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)BnGrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < BnGrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
if (BnGrpIdx == 1) {
BricksToBeGet = packed_intrinsic;
uint32_t MsgIdx = Bn2MsgStartIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
} else {
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
BricksToBeGet = __vsubss4(saturated_llr, prev);
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
}
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_last(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t BnGrpIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t Bn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)BnGrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < BnGrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
((int32_t *)(d_llrRes))[lane] = saturated_llr;
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t GrpIdx,
uint32_t MsgIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t circShift,
uint32_t Zc)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)GrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < GrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
if (GrpIdx == 1) {
BricksToBeGet = packed_intrinsic;
} else {
uint32_t prevIdxWords = (MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
BricksToBeGet = __vsubss4(saturated_llr, prev);
}
moveBricks_forput_circ(d_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge_last(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane,
uint32_t GrpIdx,
uint32_t MsgIdx,
uint32_t BnIdx,
uint32_t GrpNum,
uint32_t circShift,
uint32_t Zc)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unroll ---
for (; i < (int)GrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < GrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
if (MsgIdx == 0) {
((int32_t *)(d_llrRes))[lane] = saturated_llr;
}
}

566
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_CnProcKernel_BG1.h Normal file
View File

@@ -0,0 +1,566 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_CUDA_CnProcKernel_BG1.h
* \brief Defines the kernels for check node processing
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note
* \warning
*/
#pragma once
#include <cuda_runtime.h>
#include <stdint.h>
#include <stdio.h>
#include "nrLDPC_CUDA_public.h"
#include "nrLDPC_CUDA_lut.h"
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R13_node(const int8_t *__restrict__ d_cnProcBuf,
int8_t *__restrict__ d_bnProcBuf,
uint32_t lane,
uint32_t CnIdx,
uint32_t CnNumInGrp,
uint32_t CnGrpIdxNum,
uint32_t Cn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t min1 = 0x7F7F7F7F;
uint32_t min2 = 0x7F7F7F7F;
uint32_t total_xor = 0;
uint32_t cache_raw[19];
uint32_t cache_abs[19];
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
const int32_t *currPtr = cnProcBufPtr;
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
#pragma unroll
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
uint32_t val = *currPtr;
cache_raw[MsgIdx] = val;
uint32_t v_abs = __vabs4(val);
cache_abs[MsgIdx] = v_abs;
total_xor = __vxor4(total_xor, val);
uint32_t old_min1 = min1;
min1 = __vminu4(old_min1, v_abs);
uint32_t candidate = __vmaxu4(old_min1, v_abs);
min2 = __vminu4(min2, candidate);
currPtr += offset;
}
#pragma unroll
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
uint32_t my_abs = cache_abs[temp_MsgIdx];
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
// final_mag = scale_int8x4(final_mag, 0.75);
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = cn_bn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R23_node(const int8_t *__restrict__ d_cnProcBuf,
int8_t *__restrict__ d_bnProcBuf,
uint32_t lane,
uint32_t CnIdx,
uint32_t CnNumInGrp,
uint32_t CnGrpIdxNum,
uint32_t Cn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t min1 = 0x7F7F7F7F;
uint32_t min2 = 0x7F7F7F7F;
uint32_t total_xor = 0;
uint32_t cache_raw[19];
uint32_t cache_abs[19];
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
const int32_t *currPtr = cnProcBufPtr;
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
#pragma unroll
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
uint32_t val = *currPtr;
cache_raw[MsgIdx] = val;
uint32_t v_abs = __vabs4(val);
cache_abs[MsgIdx] = v_abs;
total_xor = __vxor4(total_xor, val);
uint32_t old_min1 = min1;
min1 = __vminu4(old_min1, v_abs);
uint32_t candidate = __vmaxu4(old_min1, v_abs);
min2 = __vminu4(min2, candidate);
currPtr += offset;
}
#pragma unroll
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
uint32_t my_abs = cache_abs[temp_MsgIdx];
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
// final_mag = scale_int8x4(final_mag, 0.75);
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = cn_bn_map_BG1_Z_R23[MsgIdx][ZcIdx];
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R23[MsgIdx][0]);
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R89_node(const int8_t *__restrict__ d_cnProcBuf,
int8_t *__restrict__ d_bnProcBuf,
uint32_t lane,
uint32_t CnIdx,
uint32_t CnNumInGrp,
uint32_t CnGrpIdxNum,
uint32_t Cn2MsgStartIdx,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t min1 = 0x7F7F7F7F;
uint32_t min2 = 0x7F7F7F7F;
uint32_t total_xor = 0;
uint32_t cache_raw[19];
uint32_t cache_abs[19];
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
const int32_t *currPtr = cnProcBufPtr;
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
#pragma unroll
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
uint32_t val = *currPtr;
cache_raw[MsgIdx] = val;
uint32_t v_abs = __vabs4(val);
cache_abs[MsgIdx] = v_abs;
total_xor = __vxor4(total_xor, val);
uint32_t old_min1 = min1;
min1 = __vminu4(old_min1, v_abs);
uint32_t candidate = __vmaxu4(old_min1, v_abs);
min2 = __vminu4(min2, candidate);
currPtr += offset;
}
#pragma unroll
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
uint32_t my_abs = cache_abs[temp_MsgIdx];
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
// final_mag = scale_int8x4(final_mag, 0.75);
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = cn_bn_map_BG1_Z_R89[MsgIdx][ZcIdx];
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R89[MsgIdx][0]);
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G3(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
// loop starts here
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G4(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G5(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G6(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G7(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][5] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G8(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][5] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][6] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G9(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][5] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][6] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][7] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G10(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][5] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][6] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][7] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][8] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
__device__ __forceinline__ void cnProcKernel_BG1_int8_G19(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][0] * 4);
sgn = __vxor4(ones, ymm0);
min = __vabs4(ymm0);
//-------------------------loop starts here-------------------------------
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][4] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][5] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][6] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][7] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][8] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][9] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][10] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][11] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][12] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][13] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][14] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][15] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][16] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][17] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
//-------------------------------------------------------------------------
uint32_t BricksToBeMoved = __vsign4(min, sgn);
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}

2289
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_lut.h Normal file
View File

File diff suppressed because it is too large Load Diff

186
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_mPassKernel_BG1.h Normal file
View File

@@ -0,0 +1,186 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_CUDA_mPassKernel.h
* \brief Defines the kernels for message passing in CUDA version of LDPC decoder
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note
* \warning
*/
#pragma once
#include <cuda_runtime.h>
#include <stdint.h>
#include <stdio.h>
#include "nrLDPC_CUDA_public.h"
#include "nrLDPC_CUDA_shared_param.h"
__device__ __forceinline__ void llrPreProc_Kernel_BG1_int8_Gn_stream(const int8_t *p_llr,
int8_t *p_llrProcBuf,
int8_t *p_cnProcBuf,
uint32_t MsgIdx,
uint32_t lane,
uint32_t colIdx,
uint32_t idxBn,
uint32_t GrpIdx,
uint32_t circShift,
uint32_t Zc,
uint32_t R)
{
{
uint32_t *p_cnProcBufBit;
uint8_t bricksLocal[4];
uint8_t *BricksToBeMoved = bricksLocal;
p_cnProcBufBit = (uint32_t *)(p_cnProcBuf + d_lut_numCnInCnGroups_BG1_R13[GrpIdx] * NR_LDPC_ZMAX * MsgIdx + lane * 4);
moveBricks_invget_circ((int8_t *)&p_llr[idxBn * Zc], lane * 4, BricksToBeMoved, Zc, circShift);
*p_cnProcBufBit = *(uint32_t *)BricksToBeMoved;
}
if (colIdx >= 68)
return;
const uint8_t numBn2CnG1 =
(R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]); // for R13 is 42
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1; // 26 for BG1
const uint32_t colG1 = startColParity * Zc;
const uint32_t *lut_llr2llrProcBufAddr =
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
const uint32_t *lut_llr2llrProcBufBnPos =
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
if (colIdx < startColParity) {
const uint32_t idxBn = lut_llr2llrProcBufAddr[colIdx] + lut_llr2llrProcBufBnPos[colIdx] * NR_LDPC_ZMAX;
int32_t *dst = (int32_t *)(&p_llrProcBuf[idxBn] + lane * 4);
int32_t *src = (int32_t *)(&p_llr[colIdx * Zc] + lane * 4);
*dst = *src;
} else {
colIdx = colIdx - startColParity;
if (numBn2CnG1 > 0 && colIdx < numBn2CnG1) {
int32_t *dst = (int32_t *)(&p_llrProcBuf[colIdx * NR_LDPC_ZMAX] + lane * 4);
int32_t *src = (int32_t *)(&p_llr[colG1 + colIdx * Zc] + lane * 4);
*dst = *src;
}
}
}
__device__ void llr2bit_Kernel_BG1_int8(uint32_t R,
uint8_t *__restrict__ out,
const int8_t *__restrict__ llrRes,
uint32_t numLLR,
uint32_t Zc)
{
uint32_t lane = threadIdx.x;
uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
if (outColIdx >= num_TotalBlocks_llr_llrRes)
return;
// Constants Setup
const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
const uint32_t *lut_Addr =
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
const uint32_t *lut_Pos =
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
int32_t raw_llrs;
// Load Data: Handle Systematic (Scatter/Gather) vs Parity (Linear) mapping
if (outColIdx < startColParity) {
uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
} else {
uint32_t srcParityIdx = outColIdx - startColParity;
if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
} else {
raw_llrs = 0;
}
}
// Hard Decision: Convert 4 int8 LLRs -> 4 bytes (0 or 1)
int8_t *p_val = (int8_t *)&raw_llrs;
uint32_t my_word = 0;
#pragma unroll
for (int i = 0; i < 4; i++) {
// Hard decision: LLR < 0 implies bit 1
uint32_t byte_val = (p_val[i] < 0) ? 1 : 0;
my_word |= (byte_val << (i * 8));
}
// Store output linearly
*(uint32_t *)(&out[outColIdx * Zc + lane * 4]) = my_word;
}
__device__ void llr2bitPacked_Kernel_BG1_int8(uint32_t R,
uint8_t *__restrict__ out,
const int8_t *__restrict__ llrRes,
uint32_t numLLR,
uint32_t Zc)
{
uint32_t lane = threadIdx.x;
uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
if (outColIdx >= num_TotalBlocks_llr_llrRes)
return;
const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
const uint32_t *lut_Addr =
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
const uint32_t *lut_Pos =
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
int32_t raw_llrs;
// Load Data: Handle Systematic vs Parity mapping
if (outColIdx < startColParity) {
uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
} else {
uint32_t srcParityIdx = outColIdx - startColParity;
if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
} else {
raw_llrs = 0;
}
}
// Extract 4 bits from 4 LLRs (Local Packing)
// Assuming MSB order within nibble: [3][2][1][0]
uint32_t my_4_bits = 0;
int8_t *p_val = (int8_t *)&raw_llrs;
#pragma unroll
for (int i = 0; i < 4; i++) {
if (p_val[i] < 0) {
my_4_bits |= (1 << (3 - i));
}
}
// Thread Cooperation: Pair threads to pack 8 bits (1 byte)
// Even thread (0,2..) takes low nibble, Odd thread (1,3..) takes high nibble.
uint32_t neighbor_bits = __shfl_xor_sync(0xffffffff, my_4_bits, 1);
if ((lane & 1) == 0) {
// Combine: [Odd Thread Bits (High)] | [My Bits (Low)]
uint8_t packed_byte = (neighbor_bits & 0xF) | ((my_4_bits & 0xF) << 4);
// Output addr: lane steps by 4 LLRs, but we output 1 byte per 8 LLRs -> lane >> 1
uint32_t outAddr = outColIdx * (Zc >> 3) + (lane >> 1);
out[outAddr] = packed_byte;
}
}

120
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_public.h Normal file
View File

@@ -0,0 +1,120 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_CUDA_public.h
* \brief Shared functions in CUDA implementation of LDPC decoder
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note
* \warning
*/
#pragma once
#include <cuda_runtime.h>
#include <stdint.h>
#include <stdio.h>
#include <cuda_runtime.h>
#include <stdint.h>
__device__ __forceinline__ void moveBricks_invput_circ(int8_t *__restrict__ dstBuf,
uint32_t dstBuf_Offset,
const uint8_t *__restrict__ Four_Bricks,
uint32_t Z,
uint32_t cshift)
{
uint32_t tmp = cshift + dstBuf_Offset;
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
if (pos <= Z - 4) {
uint32_t val = *(const uint32_t*)Four_Bricks;
memcpy(dstBuf + pos, &val, 4);
}
else {
uint32_t bytes_at_end = Z - pos; // 1, 2, or 3
#pragma unroll
for(int i=0; i<4; i++) {
if (i < bytes_at_end) {
dstBuf[pos + i] = Four_Bricks[i];
} else {
dstBuf[i - bytes_at_end] = Four_Bricks[i];
}
}
}
}
__device__ __forceinline__ void moveBricks_forput_circ(int8_t *__restrict__ dstBuf,
uint32_t dstBuf_Offset,
const uint8_t *__restrict__ Four_Bricks,
uint32_t Z,
uint32_t cshift)
{
uint32_t tmp = dstBuf_Offset + Z - cshift;
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
if (pos <= Z - 4) {
uint32_t val = *(const uint32_t*)Four_Bricks;
memcpy(dstBuf + pos, &val, 4);
}
else {
uint32_t bytes_at_end = Z - pos;
#pragma unroll
for(int i=0; i<4; i++) {
if (i < bytes_at_end) {
dstBuf[pos + i] = Four_Bricks[i];
} else {
dstBuf[i - bytes_at_end] = Four_Bricks[i];
}
}
}
}
__device__ __forceinline__ void moveBricks_invget_circ(const int8_t *__restrict__ dstBuf,
uint32_t dstBuf_Offset,
uint8_t *__restrict__ Four_Bricks,
uint32_t Z,
uint32_t cshift)
{
uint32_t tmp = cshift + dstBuf_Offset;
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
if (pos <= Z - 4) {
uint32_t val;
memcpy(&val, dstBuf + pos, 4);
*(uint32_t*)Four_Bricks = val;
}
else {
uint32_t bytes_at_end = Z - pos;
#pragma unroll
for(int i=0; i<4; i++) {
if (i < bytes_at_end) {
Four_Bricks[i] = dstBuf[pos + i];
} else {
Four_Bricks[i] = dstBuf[i - bytes_at_end];
}
}
}
}
__device__ __forceinline__ uint32_t __vxor4(const uint32_t a, uint32_t b)
{
return a ^ b;
}
__device__ __forceinline__ uint32_t __vsign4(const uint32_t a, uint32_t b)
{
uint32_t mask = __vcmplts4(b, 0);
uint32_t bneg = __vneg4(a);
return (mask & bneg) | (~mask & a);
//uint32_t is_zero_mask = __vcmpeq4(b, 0);
//return result & (~is_zero_mask);
}

84
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_shared_param.h Normal file
View File

@@ -0,0 +1,84 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_CUDA_shared_param.h
* \brief Shared parameters in CUDA implementation of LDPC decoder
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note
* \warning
*/
#ifndef NRLDPC_CUDA_SHARED_PARAM_H_
#define NRLDPC_CUDA_SHARED_PARAM_H_
#include <cuda_runtime.h>
#ifndef MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER
#define MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER 36
#endif
#ifdef __cplusplus
extern "C" {
#endif
#define RowLength 96 //Zc = 384/4 = 96
#define num_TotalBlocks_BG1_R13_Edge 316//based on number of Cn2Bn Msgs
#define num_TotalBlocks_BG1_R23_Edge 144
#define num_TotalBlocks_BG1_R89_Edge 79
#define num_TotalBlocks_llr_llrRes 22 //Only includes systematic bits
#define num_TotalBlocks_cn_BG1_R13_Node 46 //based on number of CNs
#define num_TotalBlocks_bn_BG1_R13_Node 68 //based on number of BNs
#define num_TotalBlocks_cn_BG1_R23_Node 13
#define num_TotalBlocks_bn_BG1_R23_Node 35
#define num_TotalBlocks_cn_BG1_R89_Node 5
#define num_TotalBlocks_bn_BG1_R89_Node 27
#define JETSON_ORIN
//#define GH200
#if defined(GH200)
#define NodeEdge_Switch_Cn_R13 32
#define NodeEdge_Switch_Bn_R13 10
#define NodeEdge_Switch_Cn_R23 32
#define NodeEdge_Switch_Bn_R23 12
#define NodeEdge_Switch_Cn_R89 48
#define NodeEdge_Switch_Bn_R89 24
#elif defined(JETSON_ORIN)
#define NodeEdge_Switch_Cn_R13 3
#define NodeEdge_Switch_Bn_R13 1
#define NodeEdge_Switch_Cn_R23 3
#define NodeEdge_Switch_Bn_R23 1
#define NodeEdge_Switch_Cn_R89 4
#define NodeEdge_Switch_Bn_R89 2
#endif
#ifdef __cplusplus
}
#endif
typedef struct KernelLaunchConfig {
dim3 grid;
dim3 block;
}KernelLaunchConfig;
typedef struct {
int idxBn;
int idxCn;
int preBuf;
int circShift;
int8_t dd;
} DumpEntry;
typedef struct {
int8_t* p_llr_ptr;
int8_t* p_out_ptr;
} ldpc_cuda_bridge_t;
#endif /* NRLDPC_CUDA_SHARED_PARAM_H_ */

189
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_cuda.c Normal file
View File

@@ -0,0 +1,189 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
#include <stdint.h>
#include <dlfcn.h>
#include "log.h"
#include "openair1/PHY/CODING/nrLDPC_defs.h"
#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
/* segment interface */
extern int32_t LDPCinit_cuda();
int32_t LDPCinit()
{
return LDPCinit_cuda();
}
// LDPCshutdown_cuda
extern int32_t LDPCshutdown_cuda();
int32_t LDPCshutdown()
{
return LDPCshutdown_cuda();
}
// LDPCdecoder
extern int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
int8_t* p_llr,
uint8_t* p_out,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab);
int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
int8_t* p_llr,
uint8_t* p_out,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab)
{
return LDPCdecoder_cuda(p_decParams, p_llr, p_out, p_profiler, ab);
}
// LDPCencoder <= (LDPCencoder32)
extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
uint32_t LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
{
uint32_t **output32 = LDPCencoder32(input, impp);
AssertFatal(impp->n_segments < 8, "LDPC CUDA segment interface does not copy more than 8 segs\n");
// the following copied from ldpc_encoder_optim8segmulti.c
int nrows = 46; // assumption BG1
int rate = 3; // assumption BG1
int no_punctured_columns = (int)((nrows-2)*impp->Zc+impp->K-impp->K*rate)/impp->Zc;
int removed_bit = (nrows - no_punctured_columns - 2) * impp->Zc + impp->K - (int)(impp->K * rate);
int len = impp->K + impp->Zc * (nrows - no_punctured_columns) - removed_bit;
// copy to output format
for (int i = 0; i < len; ++i) {
// condition from ldpctest: (channel_input_optim[i] >> j) == (output32[j>>5][i] >> (j&31))
// for more than 8 segments, need to spread output32 into output
uint8_t segs = output32[0][i] & 0xff;
output[i] = segs;
}
return 0;
}
/* slot interface */
// fallback for LDPC sizes not implemented by CUDA LDPC
nrLDPC_coding_interface_t ldpc_cpu;
void *ldpc_lib_handle; // corresponding dlsym() handle, see also below
extern int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters);
int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *slot_params)
{
// this should be the same as previous nrLDPC_coding_encoder() in nrLDPC_coding_segment_encoder.c
nrLDPC_TB_encoding_parameters_t tbCPU[32] = {0};
nrLDPC_slot_encoding_parameters_t cpu = {
.frame = slot_params->frame,
.slot = slot_params->slot,
.threadPool = slot_params->threadPool,
// timing?
.TBs = tbCPU,
};
for (int dlsch_id = 0; dlsch_id < slot_params->nb_TBs; dlsch_id++) {
nrLDPC_TB_encoding_parameters_t *tbp = &slot_params->TBs[dlsch_id];
if (tbp->BG == 1 && tbp->C > 8 && tbp->Z == 384) {
nrLDPC_coding_encoder32(slot_params, tbp);
} else {
// this is not handled by CUDA, handle with CPU
cpu.TBs[cpu.nb_TBs++] = *tbp;
}
}
if (cpu.nb_TBs > 0) {
ldpc_cpu.nrLDPC_coding_encoder(&cpu);
}
return 0;
}
void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *);
int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *slot_params)
{
// this should be the same as previous nrLDPC_coding_decoder() in nrLDPC_coding_segment_decoder.c
nrLDPC_TB_decoding_parameters_t tbCPU[32] = {0};
nrLDPC_slot_decoding_parameters_t cpu = {
.frame = slot_params->frame,
.slot = slot_params->slot,
.threadPool = slot_params->threadPool,
.TBs = tbCPU,
};
for (int pusch_id = 0; pusch_id < slot_params->nb_TBs; pusch_id++) {
nrLDPC_TB_decoding_parameters_t *tbp = &slot_params->TBs[pusch_id];
if (tbp->Z >= 128 && tbp->BG == 1) {
nr_process_decode_segment_cuda(tbp);
} else {
// this is not handled by CUDA, handle with CPU
cpu.TBs[cpu.nb_TBs++] = *tbp;
}
}
if (cpu.nb_TBs > 0) {
ldpc_cpu.nrLDPC_coding_decoder(&cpu);
}
return 0;
}
#define CHECK_DLFCN_ERROR(COND, SYMB) \
if (!(COND)) { \
LOG_E(NR_PHY, "could not open %s: %s\n", SYMB, dlerror()); \
return false; \
}
static bool open_default_ldpc(nrLDPC_coding_interface_t *ldpc, int max_num_pxsch)
{
const char *default_ldpc = "libldpc.so";
if (ldpc_lib_handle) {
LOG_W(NR_PHY, "%s already initialized\n", default_ldpc);
return true;
}
ldpc_lib_handle = dlopen(default_ldpc, RTLD_LAZY|RTLD_NODELETE|RTLD_GLOBAL);
CHECK_DLFCN_ERROR(ldpc_lib_handle, default_ldpc);
ldpc->nrLDPC_coding_init = dlsym(ldpc_lib_handle, "nrLDPC_coding_init");
CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_init, "nrLDPC_coding_init");
ldpc->nrLDPC_coding_shutdown = dlsym(ldpc_lib_handle, "nrLDPC_coding_shutdown");
CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_shutdown, "nrLDPC_coding_shutdown");
ldpc->nrLDPC_coding_decoder = dlsym(ldpc_lib_handle, "nrLDPC_coding_decoder");
CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_decoder, "nrLDPC_coding_decoder");
ldpc->nrLDPC_coding_encoder = dlsym(ldpc_lib_handle, "nrLDPC_coding_encoder");
CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_encoder, "nrLDPC_coding_encoder");
bool success = ldpc->nrLDPC_coding_init(max_num_pxsch) == 0;
if (!success)
LOG_E(NR_PHY, "could not initialize %s\n", default_ldpc);
return success;
}
static void close_default_ldpc(nrLDPC_coding_interface_t *ldpc)
{
if (!ldpc_lib_handle) {
LOG_W(NR_PHY, "cannot stop LDPC, not loaded\n");
return;
}
ldpc->nrLDPC_coding_shutdown();
dlclose(ldpc_lib_handle);
ldpc_lib_handle = NULL;
}
extern int32_t nrLDPC_coding_init_cuda(int max_num_pxsch);
int32_t nrLDPC_coding_init(int max_num_pxsch)
{
// we can't use load_nrLDPC_coding_interface() because:
// 1. it caches the lib under the "ldpc" name, but links to "ldpc_cuda", so
// we recursively load this function again => stack overflow
// 2. the global --loader.ldpc.shlibversion _cuda would be reapplied => we
// cannot actually load ourselves
// hence use dlsym() directly.
bool success = open_default_ldpc(&ldpc_cpu, max_num_pxsch);
AssertFatal(success, "error loading LDPC library\n");
return nrLDPC_coding_init_cuda(max_num_pxsch);
}
extern int32_t nrLDPC_coding_shutdown_cuda(void);
int32_t nrLDPC_coding_shutdown(void)
{
close_default_ldpc(&ldpc_cpu);
return nrLDPC_coding_shutdown_cuda();
}

304
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_segment_decoder_cuda.c Normal file
View File

@@ -0,0 +1,304 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
* \brief Top-level routines for decoding LDPC transport channels
*/
// [from gNB coding]
#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
#include "openair1/PHY/CODING/coding_defs.h"
#include "log.h"
#include "defs.h"
#include <stdalign.h>
#include <stdint.h>
#include <syscall.h>
#include <time.h>
#include <pthread.h>
#include <cuda_runtime.h>
#include "nrLDPC_CUDA_shared_param.h"
// #define gNB_DEBUG_TRACE
#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
// #define DEBUG_CRC
#ifdef DEBUG_CRC
#define PRINT_CRC_CHECK(a) a
#else
#define PRINT_CRC_CHECK(a)
#endif
#define USE_GPU_FOR_RM_DEINTER 1
cudaStream_t decoderStreams[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
cudaStream_t decoderDoneEvents[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
int d_array_size = 0;
void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx);
int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
uint8_t BG,
uint16_t Z,
int16_t *d,
int16_t *soft_input,
int8_t *llr_buffer,
uint32_t K,
uint8_t C,
uint8_t rvidx,
uint8_t clear,
uint32_t E1,
uint32_t E2,
uint32_t r_firstE2,
uint32_t F,
uint32_t Foffset,
cudaStream_t *s,
int8_t sidx);
extern int pageable,integrated;
extern int8_t *p_llr_dev,*p_out_dev;
int16_t **harq_d_array;
int16_t *harq_d_array_dev;
int16_t *harq_e_dev;
int16_t *harq_f_dev;
pthread_mutex_t decoder_mutex=PTHREAD_MUTEX_INITIALIZER;
extern
int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
int8_t* p_llr,
uint8_t* p_out,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab);
void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *segs)
{
// arg points to RDATA array (nrLDPC_decoding_parameters_t *RDATA)
DevAssert(segs != NULL);
// use seg0 as canonical
const int C = segs->C;
const int Z = segs->Z;
const int Kc = segs->BG == 2 ? 52 : 68;
const int K = segs->K;
const int Kprime = K - segs->F;
const int segLen = Kc*Z; // int16 length; after packing we store int8 [segLen]
t_nrLDPC_time_stats procTime = {0};
t_nrLDPC_time_stats *p_procTime = &procTime;
#ifdef USE_GPU_FOR_RM_DEINTER
start_meas(&segs->ts_deinterleave);
int E1 = segs->E;
int E2 = segs->E2;
int r_firstE2 = segs->first_rE2;
LOG_D(NR_PHY,"locking decoder (llr %p)\n",segs->llr);
pthread_mutex_lock(&decoder_mutex);
// for PCIe GPU copy llrs to device memory
if (!pageable&&!integrated) {
LOG_I(NR_PHY,"cudaMemcpyAsynch llr->harq_f_dev\n");
cudaMemcpyAsync(harq_f_dev,
segs->llr,
((r_firstE2*E1) + (C-r_firstE2)*E2)*sizeof(int16_t),
cudaMemcpyHostToDevice,
decoderStreams[0]);
}
#if 0
if (1/*segs->rv_index==2*/)
for (int r=0;r<C;r++) {
int roffset = r < r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
int E = r < r_firstE2 ? E1 : E2;
for (int i=0;i<16;i++) {
printf("f(%d,%d) %d\n",r,i,segs->llr[roffset + i]);
}
}
#endif
LOG_D(NR_PHY,"deinter: e %p llr %p\n",harq_e_dev,pageable || integrated ? segs->llr : harq_f_dev);
launch_deinterleave_i16(segs->Qm,E1,E2,C,r_firstE2,harq_e_dev,pageable||integrated ? segs->llr : harq_f_dev,decoderStreams,0);
stop_meas(&segs->ts_deinterleave);
#if 0
cudaError_t err;
if (1/*segs->rv_index == 2*/) {
int16_t elocal[E2];
for (int r=0;r<C;r++) {
int E = r<r_firstE2 ? E1 : E2;
int r_off = r<r_firstE2 ? (r*E1) : ((r_firstE2*E1) + (r-r_firstE2)*E2);
err = cudaMemcpyAsync(elocal,&harq_e_dev[r_off],sizeof(elocal),cudaMemcpyDeviceToHost,decoderStreams[0]);
AssertFatal(err == cudaSuccess,"cudaMemcpyAsync failed with error %s, r_off %d\n",cudaGetErrorString(err),r_off);
for (int i=0;i<E;i++) {
if (elocal[i] != segs->llr[r_off + ((i<(E/2)) ? 2*i : (-E+1+2*i))]) {
printf("e(%d,%d,%d) %d",r,i,E,elocal[i]);
printf("(%d**)\n",segs->llr[r_off + ((i<(E/2)) ? 2*i : -E+1+2*i)]);
}
}
}
}
#endif
start_meas(&segs->ts_rate_unmatch);
//printf("Running RM with id %d, d_to_be_cleared %d, rv_idx %d, Z %d, C %d, E1 %d, E2 %d, F %d,r_firstE2 %d\n",segs->harq_unique_pid,segs->d_to_be_cleared,segs->rv_index,Z,C,E1,E2,segs->F,r_firstE2);
#if 0
if (segs->d_to_be_cleared == 1) err = cudaMemsetAsync(harq_d_array[segs->harq_unique_pid],0,C*68*Z*sizeof(int16_t),decoderStreams[0]);
AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed with error %s on harq_d_array[%d] %p for %d bytes\n",cudaGetErrorString(err),segs->harq_unique_pid,harq_d_array[segs->harq_unique_pid],C*68*Z*sizeof(int16_t));
cudaMemsetAsync(p_llr_dev,0,C*68*Z*sizeof(int8_t),decoderStreams[0]);
AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed on p_llr_dev %p\n",p_llr_dev);
#endif
AssertFatal(segs->harq_unique_pid < d_array_size,"harq_unique_pid %d > %d\n",segs->harq_unique_pid,d_array_size);
nr_rate_matching_ldpc_rx_cuda(segs->tbslbrm,
segs->BG,
Z,
harq_d_array[segs->harq_unique_pid],
harq_e_dev,
p_llr_dev,
K,
C, // TB segments count
segs->rv_index,
segs->d_to_be_cleared,
E1,
E2,
r_firstE2,
segs->F,
Kprime - 2 * Z,
decoderStreams,
0);
for (int r = 0; r < C; ++r) {
cudaMemsetAsync(p_llr_dev + (size_t)r*segLen,0,sizeof(int8_t)*2*Z,decoderStreams[0]);
#if 0
int8_t llr_local[segLen];
if (r==1 && segs->rv_index==2) {
cudaMemcpyAsync(llr_local,p_llr_dev+(size_t)r*segLen,sizeof(int8_t)*segLen,cudaMemcpyDeviceToHost,decoderStreams[0]);
for (int i=0;i<segLen;i++) printf("llr(%d,%d,%d/%d) %d\n",segs->rv_index,r,i,segLen,llr_local[i]);
}
#endif
}
stop_meas(&segs->ts_rate_unmatch);
#else //USE_GPU_FOR_RM_DEINTER
int16_t *z_local = (int16_t*)alloca(sizeof(int16_t) * segLen); // segLen is safe small
for (int r = 0; r < C; ++r) {
// deinterleave
start_meas(&segs->ts_deinterleave);
int16_t *harq_e = (int16_t*)alloca(sizeof(int16_t) * segs->segments[r].E);
// for (int i=0;i<segs->segments[r].E;i++) printf("llr_in[%d] %d\n",i,segs->segments[r].llr[i]);
nr_deinterleaving_ldpc(segs->segments[r].E, segs->Qm, harq_e, segs->segments[r].llr);
// for (int i=0;i<16;i++) printf("harq_e[%d] %d\n",i,harq_e[i]);
stop_meas(&segs->segments[0].ts_deinterleave);
// rate matching
start_meas(&segs->segments[0].ts_rate_unmatch);
if (nr_rate_matching_ldpc_rx(segs->tbslbrm,
segs->BG,
Z,
segs->segments[r].d,
harq_e,
C, // TB segments count
segs->rv_index,
segs->segments[r].d_to_be_cleared,
segs->segments[r].E,
segs->F,
Kprime - 2 * Z) == -1) {
stop_meas(&segs->segments[0].ts_rate_unmatch);
LOG_E(PHY,"rate matching failed seg %d\n", r);
memset(segs->segments[r].c, 0, K);
//*rdata->decodeSuccess = false;
continue; // skip this segment
}
stop_meas(&segs->segments[0].ts_rate_unmatch);
start_meas(&segs->segments[0].ts_seg_prep);
segs->segments[r].d_to_be_cleared = false;
memset(z_local,0,sizeof(int16_t)*2*Z);
memset(z_local + Kprime,127,sizeof(int16_t)*segs->F);
memcpy(z_local + 2*Z, segs->segments[r].d, (size_t)(Kprime - 2*Z)*sizeof(int16_t));
memcpy(z_local + K, segs->segments[r].d + (K - 2*Z), (size_t)(Kc*Z - K)*sizeof(int16_t));
// pack int16 -> int8 into llrBuffer[r * segLen]
simde__m128i *pv = (simde__m128i*)z_local;
simde__m128i *pl = (simde__m128i*)(llrBuffer + (size_t)r*segLen);
int vecCount = ((Kc * Z) >> 4);
for (int j=0, idx=0; j<vecCount; ++j, idx+=2) {
pl[j] = simde_mm_packs_epi16(pv[idx], pv[idx+1]);
}
stop_meas(&segs->segments[0].ts_seg_prep);
}
#endif
start_meas(&segs->ts_ldpc_decode);
t_nrLDPC_dec_params decParams = {.check_crc = check_crc};
decParams.Z = Z;
decParams.R = segs->R;
decParams.BG = segs->BG;
decParams.crc_type = crcType(C, segs->A);
decParams.Kprime = lenWithCrc(C, segs->A);
decParams.n_segments = C;
decParams.outMode=nrLDPC_outMode_BIT;
decParams.numMaxIter = segs->max_ldpc_iterations;
LOG_D(NR_PHY,"decoder (llr %p): %d segments, Z %d, R %d \n",segs->llr,C,Z,segs->R);
int decodeIterations = LDPCdecoder_cuda(&decParams, p_llr_dev, segs->c, p_procTime, segs->abort_decode);
stop_meas(&segs->ts_ldpc_decode);
if (decodeIterations <= segs->max_ldpc_iterations) {
*segs->processedSegments = C;
for (int r=0; r<C; r++) segs->decodeSuccess[r] = true;
LOG_D(NR_PHY,"Set all segs->decodeSuccess to true\n");
} else {
memset(segs->c, 0, C*(K>>3));
for (int r=0; r<C; r++) segs->decodeSuccess[r] = false;
LOG_D(NR_PHY,"Set all segs->decodeSuccess to false\n");
}
LOG_D(NR_PHY,"unlocking decoder (llr %p)\n",segs->llr);
pthread_mutex_unlock(&decoder_mutex);
}
#define MAXE 4*14*273*12*8 // 4 antennas, 14 symbols, 273 PRBs, 12 RE/prb, 8 bits/RE
void LDPCint_rm_init(int max_num_pxsch) {
LOG_I(NR_PHY,"RM init for %d pxsch\n",max_num_pxsch);
LOG_I(NR_PHY,"Allocating device array for harq_d/harq_e \n");
cudaError_t err=cudaMalloc((void **)&harq_e_dev,MAXE*sizeof(int16_t));
AssertFatal(err == cudaSuccess,"CUDA Error (harq_e_dev): %s\n", cudaGetErrorString(err));
harq_d_array = malloc(sizeof(int16_t*) * max_num_pxsch);
err = cudaMalloc((void *)&harq_d_array_dev,sizeof(int16_t*) * max_num_pxsch);
AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_array_dev): %s\n", cudaGetErrorString(err));
LOG_I(PHY,"Allocated %ld bytes for harq_d_array_dev @ %p\n",sizeof(int16_t*)*max_num_pxsch,harq_d_array_dev);
for (int i=0;i<max_num_pxsch;i++) {
err=cudaMalloc((void **)&harq_d_array[i],MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 *OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t));
LOG_I(PHY,"Allocating %ld bytes for harq_d_array[%d] @ %p\n",MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 *OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t),i,harq_d_array[i]);
AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_dev): %s\n", cudaGetErrorString(err));
}
cudaMemcpy(harq_d_array_dev,harq_d_array,sizeof(int16_t*)*max_num_pxsch,cudaMemcpyHostToDevice);
if (!pageable && !integrated) {
LOG_I(PHY,"Allocating device array for harq_f \n");
err=cudaMalloc((void **)&harq_f_dev,MAXE*sizeof(int16_t));
AssertFatal(err == cudaSuccess,"CUDA Error (harq_f_dev): %s\n", cudaGetErrorString(err));
}
d_array_size = max_num_pxsch;
}
extern void LDPCinit_cuda(void);
extern void cuda_support_init();
int32_t nrLDPC_coding_init_cuda(int max_num_pxsch)
{
cuda_support_init();
LDPCinit_cuda();
LDPCint_rm_init(max_num_pxsch);
return 0;
}
void LDPCshutdown_cuda(void);
int32_t nrLDPC_coding_shutdown_cuda(void)
{
LDPCshutdown_cuda();
return 0;
}

928
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_segment_encoder_cuda.c Normal file
View File

@@ -0,0 +1,928 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
* \brief Top-level routines for implementing LDPC encoding of transport channels
*/
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h"
#include "PHY/sse_intrin.h"
#include "openair1/PHY/CODING/nrLDPC_defs.h"
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
#include "common/utils/LOG/log.h"
#include <syscall.h>
//#define DEBUG_LDPC_ENCODING
//#define DEBUG_LDPC_ENCODING_FREE 1
static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
extern uint32_t **d_host;
static void unpack_output(uint32_t *f,
uint32_t E,
uint32_t *f2,
uint32_t E2,
uint32_t E2_first_segment32,
uint32_t E2_first_segment,
uint32_t nb_segments,
uint8_t *output) {
uint32_t s;
// int s0;
uint32_t *fp;
int foffset;
uint32_t *output_p = (uint32_t *)output;
//printf("unpack: E %d, E2 %d, E2_first_segment %d, E2_first_segment32 %d, nb_segments %d\n",E,E2,E2_first_segment,E2_first_segment32,nb_segments);
uint32_t bit_index = 0;
#ifdef __AVX2__
simde__m256i shift0=simde_mm256_set_epi32(7,6,5,4,3,2,1,0);
simde__m256i shift1=simde_mm256_set_epi32(15,14,13,12,11,10,9,8);
simde__m256i shift2=simde_mm256_set_epi32(23,22,21,20,19,18,17,16);
simde__m256i shift3=simde_mm256_set_epi32(31,30,29,28,27,26,25,24);
simde__m256i vmask0=simde_mm256_set_epi32(0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1);
simde__m256i vmask1=simde_mm256_set_epi32(0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100);
simde__m256i vmask2=simde_mm256_set_epi32(0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000);
simde__m256i vmask3=simde_mm256_set_epi32(0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000);
int s2=0;
for (s = 0; s < E2_first_segment ; s++) {
s2 = s&31;
foffset = (s>>5)*E;
fp = f+foffset;
int i;
if ((bit_index&31) == 0 ) {
for (i = 0; i < (E>>5)<<5; i+=32) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
*(output_p + (bit_index>>5)) = simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7);
bit_index+=32;
}
uint32_t Emod32=E&31;
if (Emod32 != 0) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
*(output_p + (bit_index>>5)) = (simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
bit_index+=Emod32;
}
}
else {
for (i = 0; i < (E>>5)<<5; i+=32) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
uint32_t tmp = simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=32;
}
uint32_t Emod32=E&31;
if (Emod32 != 0) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
uint32_t tmp = (simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=Emod32;
}
}
}
for ( ; s < nb_segments ; s++){
s2 = s&31;
foffset = ((s>>5)-E2_first_segment32)*E2;
fp = f2+foffset;
int i;
if ((bit_index&31) == 0 ) {
for (i = 0; i < (E2>>5)<<5; i+=32) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
*(output_p + (bit_index>>5)) = simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7);
bit_index+=32;
}
uint32_t E2mod32=E2&31;
if (E2mod32 != 0) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
*(output_p + (bit_index>>5)) = (simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
bit_index+=E2mod32;
}
}
else {
for (i = 0; i < (E2>>5)<<5; i+=32) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
uint32_t tmp = simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=32;
}
uint32_t E2mod32=E2&31;
if (E2mod32 != 0) {
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
uint32_t tmp = (simde_mm256_extract_epi32(cshift,0) |
simde_mm256_extract_epi32(cshift,1) |
simde_mm256_extract_epi32(cshift,2) |
simde_mm256_extract_epi32(cshift,3) |
simde_mm256_extract_epi32(cshift,4) |
simde_mm256_extract_epi32(cshift,5) |
simde_mm256_extract_epi32(cshift,6) |
simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=E2mod32;
}
}
}
#elif defined(__aarch64__)
const int32_t ucShift0[32][4] = { {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}, {-28,-27,-26,-25}, {-29,-28,-27,-26}, {-30,-29,-28,-27}, {-31,-30,-29,-28}};
const int32_t ucShift1[32][4] = { {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}};
const int32_t ucShift2[32][4] = { {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19},{-23,-22,-21,-20}};
const int32_t ucShift3[32][4] = { {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}};
const int32_t ucShift4[32][4] = { {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}};
const int32_t ucShift5[32][4] = { {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}};
const int32_t ucShift6[32][4] = { {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}};
const int32_t ucShift7[32][4] = { {28,29,30,31}, {27,28,29,30}, {26,27,28,29}, {25,26,27,28}, {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}};
const uint32_t __attribute__ ((aligned (16))) masks0[4] = {0x1,0x2,0x4,0x8};
const uint32_t __attribute__ ((aligned (16))) masks1[4] = {0x10,0x20,0x40,0x80};
const uint32_t __attribute__ ((aligned (16))) masks2[4] = {0x100,0x200,0x400,0x800};
const uint32_t __attribute__ ((aligned (16))) masks3[4] = {0x1000,0x2000,0x4000,0x8000};
const uint32_t __attribute__ ((aligned (16))) masks4[4] = {0x10000,0x20000,0x40000,0x80000};
const uint32_t __attribute__ ((aligned (16))) masks5[4] = {0x100000,0x200000,0x400000,0x800000};
const uint32_t __attribute__ ((aligned (16))) masks6[4] = {0x1000000,0x2000000,0x4000000,0x8000000};
const uint32_t __attribute__ ((aligned (16))) masks7[4] = {0x10000000,0x20000000,0x40000000,0x80000000};
int32x4_t vshift0[32],vshift1[32],vshift2[32],vshift3[32],vshift4[32],vshift5[32],vshift6[32],vshift7[32];
for (int n=0;n<32;n++) {
vshift0[n] = vld1q_s32(ucShift0[n]);
vshift1[n] = vld1q_s32(ucShift1[n]);
vshift2[n] = vld1q_s32(ucShift2[n]);
vshift3[n] = vld1q_s32(ucShift3[n]);
vshift4[n] = vld1q_s32(ucShift4[n]);
vshift5[n] = vld1q_s32(ucShift5[n]);
vshift6[n] = vld1q_s32(ucShift6[n]);
vshift7[n] = vld1q_s32(ucShift7[n]);
}
uint32x4_t vmask0 = vld1q_u32(masks0);
uint32x4_t vmask1 = vld1q_u32(masks1);
uint32x4_t vmask2 = vld1q_u32(masks2);
uint32x4_t vmask3 = vld1q_u32(masks3);
uint32x4_t vmask4 = vld1q_u32(masks4);
uint32x4_t vmask5 = vld1q_u32(masks5);
uint32x4_t vmask6 = vld1q_u32(masks6);
uint32x4_t vmask7 = vld1q_u32(masks7);
int s2=0;
for (s = 0; s < E2_first_segment ; s++) {
s2 = s&31;
foffset = (s>>5)*E;
fp = f+foffset;
int i;
if ((bit_index&31) == 0 ) {
for (i = 0; i < (E>>5)<<5; i+=32) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift);
bit_index+=32;
}
uint32_t Emod32=E&31;
if (Emod32 != 0) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift)&((1<<Emod32)-1);
bit_index+=Emod32;
}
}
else {
for (i = 0; i < (E>>5)<<5; i+=32) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
/*
uint32_t tmp = vaddvq_u32(cshift);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
*/
uint64_t tmp = (uint64_t)vaddvq_u32(cshift);
*(uint64_t*)(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
bit_index+=32;
}
uint32_t Emod32=E&31;
if (Emod32 != 0) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
uint32_t tmp = vaddvq_u32(cshift);
tmp&=((1<<Emod32)-1);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=Emod32;
}
}
}
// s0 = s;
for ( ; s < nb_segments ; s++){
s2 = s&31;
foffset = ((s>>5)-E2_first_segment32)*E2;
fp = f2+foffset;
int i;
if ((bit_index&31) == 0 ) {
for (i = 0; i < (E2>>5)<<5; i+=32) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift);
bit_index+=32;
}
uint32_t E2mod32=E2&31;
if (E2mod32 != 0) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift)&((1<<E2mod32)-1);
bit_index+=E2mod32;
}
}
else {
for (i = 0; i < (E2>>5)<<5; i+=32) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
uint32_t tmp = vaddvq_u32(cshift);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=32;
}
uint32_t E2mod32=E2&31;
if (E2mod32 != 0) {
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
uint32_t tmp = vaddvq_u32(cshift);
tmp&=((1<<E2mod32)-1);
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
bit_index+=E2mod32;
}
}
}
#else // non SIMD version
unsigned int segpos,s2;
for (s = 0; s < E2_first_segment ; s++) {
foffset = (s>>5)*E;
fp = f+foffset;
s2=s&31;
segpos = (1<<s2);
// printf("E %d s %d: foffset %d, s2 %d, segpos %u\n",E,s,foffset,s2,segpos);
for (int i = 0; i < E; i++) {
output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31);
//printf("bit_index %d, output_p[%d] %x\n",bit_index, bit_index>>5,output_p[bit_index>>5]);
bit_index++;
}
}
for ( ; s < nb_segments ; s++){
foffset = ((s>>5)-E2_first_segment32)*E2;
fp = f2+foffset;
s2=s&31;
segpos = (1<<s2);
// printf("E2 %d s %d: foffset %d, s2 %d, segpos %u\n",E2,s,foffset,s2,segpos);
for (int i = 0; i < E2; i++) {
output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31);
bit_index++;
}
}
#endif
}
static
int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
uint8_t BG,
uint16_t Z,
uint32_t *d,
uint32_t *e,
uint8_t C,
uint32_t F,
uint32_t Foffset,
uint8_t rvidx,
uint32_t E)
{
if (C == 0) {
LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
return -1;
}
//Bit selection
uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
uint32_t Ncb;
if (Tbslbrm == 0)
Ncb = N;
else {
uint32_t Nref = 3 * Tbslbrm / (2 * C); //R_LBRM = 2/3
Ncb = min(N, Nref);
}
uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
#ifdef RM_DEBUG
printf("nr_rate_matching_ldpc: E %u, F %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
E,
F,
Foffset,
ind,
Ncb,
rvidx,
Tbslbrm);
#endif
if (Foffset > E) {
LOG_E(PHY,
"nr_rate_matching: invalid parameters (Foffset %d > E %d) F %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n",
Foffset,
E,
F,
ind,
Ncb,
rvidx,
Tbslbrm);
return -1;
}
if (Foffset > Ncb) {
LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
return -1;
}
if (ind >= Foffset && ind < (F + Foffset))
ind = F + Foffset;
uint32_t k = 0;
if (ind < Foffset) { // case where we have some bits before the filler and the rest after
memcpy((void *)e, (void *)(d + ind), (Foffset - ind)<<2);
if (E + F <= Ncb - ind) { // E+F doesn't contain all coded bits
memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (E - Foffset + ind)<<2);
k = E;
} else {
memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (Ncb - Foffset - F)<<2);
k = Ncb - F - ind;
}
} else {
if (E <= Ncb - ind) { // E+F doesn't contain all coded bits
memcpy((void *)(e), (void *)(d + ind), E<<2);
k = E;
} else {
memcpy((void *)(e), (void *)(d + ind), (Ncb - ind)<<2);
k = Ncb - ind;
}
}
while (k < E) { // case where we do repetitions (low mcs)
for (ind = 0; (ind < Ncb) && (k < E); ind++) {
#ifdef RM_DEBUG
printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
#endif
if (ind == Foffset)
ind = F + Foffset; // skip filler bits
e[k++] = d[ind];
}
}
return 0;
}
static
void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
{
const uint32_t EQm = E / Qm;
memset(f, 0, E * sizeof(uint32_t));
switch(Qm) {
case 2: {
uint32_t *e0 = e;
uint32_t *e1 = e + EQm;
int i = 0;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
}
} break;
case 4: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
int i = 0;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
}
} break;
case 6: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
uint32_t *e4 = e3 + EQm;
uint32_t *e5 = e4 + EQm;
int i = 0;
#if 0
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
simde__m128i *e2_128 = (simde__m128i *)e2;
simde__m128i *e3_128 = (simde__m128i *)e3;
simde__m128i *e4_128 = (simde__m128i *)e4;
simde__m128i *e5_128 = (simde__m128i *)e5;
simde__m128i *f128 = (simde__m128i *)f;
for (; i < (EQm & ~3); i += 4) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
}
e0 = (uint32_t *)e0_128;
e1 = (uint32_t *)e1_128;
e2 = (uint32_t *)e2_128;
e3 = (uint32_t *)e3_128;
e4 = (uint32_t *)e4_128;
e5 = (uint32_t *)e5_128;
f = (uint32_t *)f128;
#endif
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
*f++ = *e4++;
*f++ = *e5++;
}
} break;
case 8: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
uint32_t *e4 = e3 + EQm;
uint32_t *e5 = e4 + EQm;
uint32_t *e6 = e5 + EQm;
uint32_t *e7 = e6 + EQm;
int i = 0;
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
simde__m128i *e2_128 = (simde__m128i *)e2;
simde__m128i *e3_128 = (simde__m128i *)e3;
simde__m128i *e4_128 = (simde__m128i *)e4;
simde__m128i *e5_128 = (simde__m128i *)e5;
simde__m128i *e6_128 = (simde__m128i *)e6;
simde__m128i *e7_128 = (simde__m128i *)e7;
simde__m128i *f128 = (simde__m128i *)f;
for (; i < (EQm & ~3); i += 4) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
tmp3 = simde_mm_unpackhi_epi32(e6j, e7j); // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
}
e0 = (uint32_t *)e0_128;
e1 = (uint32_t *)e1_128;
e2 = (uint32_t *)e2_128;
e3 = (uint32_t *)e3_128;
e4 = (uint32_t *)e4_128;
e5 = (uint32_t *)e5_128;
e6 = (uint32_t *)e6_128;
e7 = (uint32_t *)e7_128;
f = (uint32_t *)f128;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
*f++ = *e4++;
*f++ = *e5++;
*f++ = *e6++;
*f++ = *e7++;
}
}
break;
default:
AssertFatal(false, "Should be here!\n");
}
}
extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
/**
* \typedef ldpc8blocks_args_t
* \struct ldpc8blocks_args_s
* \brief Arguments of an encoding task
* encode up to 8 code blocks
* \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
* \var impp encoder implementation specific parameters for the task
* \var f first interleaver output to be filled by the task
* \var f2 second interleaver output to be filled by the task
* in case of a shift of E in the code blocks group processed by the task
*/
static void ldpcnblocks(nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters, encoder_implemparams_t impp)
{
uint8_t mod_order = nrLDPC_TB_encoding_parameters->Qm;
uint16_t nb_rb = nrLDPC_TB_encoding_parameters->nb_rb;
uint32_t A = nrLDPC_TB_encoding_parameters->A;
unsigned int G = nrLDPC_TB_encoding_parameters->G;
LOG_D(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp.K, G, nb_rb, (int)mod_order);
// nrLDPC_encoder output is in "d"
// let's make this interface happy!
// uint32_t d[4][68*384];
// uint8_t *c[nrLDPC_TB_encoding_parameters->C];
extern uint32_t **input_host;
if (!nrLDPC_TB_encoding_parameters->c_dev)
for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) {
input_host[r] = (uint32_t *)nrLDPC_TB_encoding_parameters->segments[r].c;
}
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
LDPCencoder32(nrLDPC_TB_encoding_parameters->c_dev ? nrLDPC_TB_encoding_parameters->c_dev : (uint8_t **)input_host, &impp);
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
// Compute where to place in output buffer that is concatenation of all segments
#ifdef DEBUG_LDPC_ENCODING
LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
#endif
const uint32_t E = nrLDPC_TB_encoding_parameters->segments[0].E;
uint32_t E2=E;
uint32_t Emax = E;
int n_seg = nrLDPC_TB_encoding_parameters->C>>5;
int n_seg2 = n_seg;
if ((nrLDPC_TB_encoding_parameters->C & 31) > 0) n_seg2++;
int r_shift = n_seg2;
int r_shift2 = nrLDPC_TB_encoding_parameters->C;
for (int s=0;s<nrLDPC_TB_encoding_parameters->C;s++) {
//printf("segment %d E %d\n",s,nrLDPC_TB_encoding_parameters->segments[s].E);
if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
E2=nrLDPC_TB_encoding_parameters->segments[s].E;
if(E2 > Emax)
Emax = E2;
r_shift = s>>5;
r_shift2 = s;
// printf("r_shift %d, r_shift2 %d\n",r_shift,r_shift2);
break;
}
}
LOG_D(NR_PHY,
"Rate Matching, Code segment %d...%d r_shift %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
"%d,nrOfLayer %d)...\n",
0,
impp.n_segments-1,
r_shift,
n_seg2,
G,
E,E2,
impp.F,
impp.K - impp.F - 2 * impp.Zc,
mod_order,
nb_rb,
nrLDPC_TB_encoding_parameters->nb_layers);
/*
printf("Rate Matching, Code segment 0..%d r_shift %d r_shift2 %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
"%d,nrOfLayer %d)...\n",
impp.n_segments-1,
r_shift,
r_shift2,
n_seg2,
G,
E,E2,
impp.F,
impp.K - impp.F - 2 * impp.Zc,
mod_order,
nb_rb,
nrLDPC_TB_encoding_parameters->nb_layers);
*/
uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
uint32_t e[E*(r_shift+1)];
uint32_t e2[E2*(n_seg2-r_shift)];
uint32_t f[E*(r_shift+1)] __attribute__ ((aligned (64)));
uint32_t f2[E2*(n_seg2-r_shift)] __attribute__ ((aligned (64)));
// Interleaver outputs are stored in the output arrays
uint8_t *output = nrLDPC_TB_encoding_parameters->output;
start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
memset(e,0,sizeof(e));
memset(f,0,sizeof(f));
if (1/*r_shift < n_seg2*/) {
memset(e2,0,sizeof(e2));
memset(f2,0,sizeof(f2));
}
for (int r=0;r<n_seg2;r++) {
if (r<=r_shift)
nr_rate_matching_ldpc32(Tbslbrm,
impp.BG,
impp.Zc,
d_host[r],
e+(r*E),
impp.n_segments,
impp.F,
impp.K - impp.F - 2 * impp.Zc,
nrLDPC_TB_encoding_parameters->rv_index,
E);
if (r>=r_shift)
nr_rate_matching_ldpc32(Tbslbrm,
impp.BG,
impp.Zc,
d_host[r],
e2+((r-r_shift)*E2),
impp.n_segments,
impp.F,
impp.K - impp.F - 2 * impp.Zc,
nrLDPC_TB_encoding_parameters->rv_index,
E2);
/*
if (r==(n_seg2-1)) {
for (int i=0;i<16;i++) printf("rm: %x %x\n",d[n_seg2-1][i],e2[((n_seg2-1)*E2)+i]);
}
*/
}
stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
if (impp.K - impp.F - 2 * impp.Zc > E) {
LOG_E(PHY,
"dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n",
A,
impp.K,
G,
nb_rb,
(int)mod_order);
LOG_E(NR_PHY,
"Rate Matching, Code segments 0..%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
"nb_rb %d)...\n",
impp.n_segments,
G,
E,
impp.K,
impp.F,
impp.K - impp.F - 2 * impp.Zc,
mod_order,
nb_rb);
}
//printf("interleaving r_shift %d, n_seg2 %d\n",r_shift,n_seg2);
start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
for (int r=0;r<=r_shift;r++)
nr_interleaving_ldpc32(E,
mod_order,
e+E*r,
f+E*r);
for (int r=r_shift;r<n_seg2;r++)
nr_interleaving_ldpc32(E2,
mod_order,
e2+E2*(r-r_shift),
f2+E2*(r-r_shift));
/*
for (int i=0;i<16;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
printf("-------------------\n");
for (int i=E2-16;i<E2;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
*/
stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
if (impp.tconcat) start_meas(impp.tconcat);
unpack_output(f,E,f2,E2,r_shift,r_shift2,nrLDPC_TB_encoding_parameters->C,output);
if (impp.tconcat) stop_meas(impp.tconcat);
}
int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters)
{
encoder_implemparams_t common_segment_params = {
.n_segments = nrLDPC_TB_encoding_parameters->C,
.tinput = nrLDPC_slot_encoding_parameters->tinput,
.tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
.tprep = nrLDPC_slot_encoding_parameters->tprep,
.tparity = nrLDPC_slot_encoding_parameters->tparity,
.toutput = nrLDPC_slot_encoding_parameters->toutput,
.tconcat = nrLDPC_slot_encoding_parameters->tconcat,
.Kb = nrLDPC_TB_encoding_parameters->Kb,
.Zc = nrLDPC_TB_encoding_parameters->Z,
.BG = nrLDPC_TB_encoding_parameters->BG,
.output = nrLDPC_TB_encoding_parameters->output,
.K = nrLDPC_TB_encoding_parameters->K,
.F = nrLDPC_TB_encoding_parameters->F,
};
ldpcnblocks(nrLDPC_TB_encoding_parameters, common_segment_params);
return 0;
}

1369
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_decoder_BG1_cuda.cu Normal file
View File

File diff suppressed because it is too large Load Diff

657
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_decoder_cuda.c Normal file
View File

@@ -0,0 +1,657 @@
/*
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
*/
/*! \file nrLDPC_decoder_cuda.h
* \brief Defines the CUDA version of nrLDPC decoder, including initialization and driver warmup mechanisms.
* \author Qizhi Pan, Raymond Knopp
* \company EURECOM
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
* \date 2025-12-30
* \version 1.0
* \note Optimized for NVIDIA GH200 (Grace Hopper) architecture using Zero-Copy access.
* \warning
*/
#include <stdint.h>
#include "PHY/sse_intrin.h"
#include "log.h"
#include "openair1/PHY/CODING/coding_defs.h"
#include "openair1/PHY/CODING/nrLDPC_extern.h"
#ifdef NR_LDPC_DEBUG_MODE
#include "nrLDPC_tools/nrLDPC_debug.h"
#endif
// decoder interface
/**
\brief LDPC decoder API type definition
\param p_decParams LDPC decoder parameters
\param p_llr Input LLRs
\param p_llrOut Output vector
\param p_profiler LDPC profiler statistics
*/
//--------------------------CUDA Area---------------------------
#include <cuda_runtime.h>
#include "nrLDPC_CUDA_shared_param.h"
extern cudaStream_t decoderStreams[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
extern cudaEvent_t decoderDoneEvents[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
static bool decoder_streamsCreated = false;
static volatile int cuda_graph_breaker = 0;// 0 by default to enable the CUDA graph
cudaError_t Err;
int8_t* cnProcBuf_dev;
int8_t* bnProcBuf_dev;
int8_t* llrRes_dev;
int8_t* llrProcBuf_dev;
int8_t* p_llr_dev;
int8_t* p_out_dev;
extern int pageable, integrated;
int cuda_support_init_decoder()
{
// use cudaMalloc for all inner buffers
cudaError_t err;
err = cudaMalloc((void**)&cnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_CN_PROC_BUF);
AssertFatal(err == cudaSuccess, "CUDA Error (cnProcBuf_dev): %s\n", cudaGetErrorString(err));
err = cudaMalloc((void**)&bnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_BN_PROC_BUF);
AssertFatal(err == cudaSuccess, "CUDA Error (bnProcBuf_dev): %s\n", cudaGetErrorString(err));
err = cudaMalloc((void**)&llrRes_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
AssertFatal(err == cudaSuccess, "CUDA Error (llrRes_dev): %s\n", cudaGetErrorString(err));
err = cudaMalloc((void**)&llrProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
AssertFatal(err == cudaSuccess, "CUDA Error (llrProcBuf_dev): %s\n", cudaGetErrorString(err));
err = cudaMalloc((void**)&p_llr_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
cudaMemset(p_llr_dev, 0, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
err = cudaMalloc((void**)&p_out_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
printf("[CUDA] Intermediate buffers allocated in Device Memory.\n");
return 0;
}
static ldpc_cuda_bridge_t* stream_bridges[8];
extern cudaError_t nrLDPC_decoder_cuda_GraphRecord(ldpc_cuda_bridge_t* buffer,
uint32_t numLLR,
int8_t* cnProcBuf,
int8_t* bnProcBuf,
int8_t* llrRes,
int8_t* llrProcBuf,
uint32_t Z,
uint32_t K,
uint8_t BG,
uint8_t R,
uint8_t numMaxIter,
uint8_t n_segments,
e_nrLDPC_outMode outMode,
cudaStream_t* streams,
uint8_t CudaStreamIdx,
cudaGraph_t* graphPtr,
cudaGraphExec_t* graphExecPtr,
uint8_t* isCreatedFlag);
extern cudaError_t nrLDPC_decoder_cuda_GraphExecute(cudaGraphExec_t graphExec,
cudaStream_t stream,
cudaEvent_t* doneEvent,
uint8_t CudaStreamIdx);
extern void nrLDPC_decoder_cuda_NormalExecute(ldpc_cuda_bridge_t* buffer,
uint32_t numLLR,
int8_t* cnProcBuf,
int8_t* bnProcBuf,
int8_t* llrRes,
int8_t* llrProcBuf,
uint32_t Z,
uint32_t K,
uint8_t BG,
uint8_t R,
uint8_t numMaxIter,
uint8_t n_segments,
e_nrLDPC_outMode outMode,
cudaStream_t* streams,
uint8_t CudaStreamIdx,
cudaEvent_t* doneEvent);
static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
int8_t* p_out,
int n_segments,
t_nrLDPC_dec_params* p_decParams,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab);
#define MAX_GRAPH_CACHE_SIZE 16
#define PRE_RECORDED_COUNT 6
#define STATIC_SEG_SIZE 9 // n_segments in pre-record graphs, should be determined for real cases
typedef struct {
uint32_t Z;
uint32_t K;
uint32_t numLLR;
uint8_t R;
uint8_t BG;
uint8_t numMaxIter;
uint16_t n_segments;
e_nrLDPC_outMode outMode;
cudaGraph_t graph;
cudaGraphExec_t exec;
ldpc_cuda_bridge_t* bridge_ptr;
bool occupied;
} gpu_graph_node_t;
static gpu_graph_node_t gpu_graph_cache[MAX_GRAPH_CACHE_SIZE];
static int dynamic_cache_idx = PRE_RECORDED_COUNT;
void init_decoder_warmup()
{
// =====================================================================
// CUDA Driver Warm-up
// Purpose: Execute a few representative graphs to trigger CUDA context
// initialization and driver-level JIT/lazy loading.
// Note: These specific Z/R combinations might not match the actual
// run-time traffic, but running them ensures the GPU pipeline is ready.
// =====================================================================
cudaError_t err_warmup = cudaSuccess;
// Sample configurations for warmup
uint32_t Z_list[] = {320, 352, 384};
uint8_t R_list[] = {13, 23};
int node_idx = 0;
int8_t* dummy_input_llr = NULL;
int8_t* dummy_output_bits = NULL;
uint32_t max_z = 384;
uint32_t max_n_segs = STATIC_SEG_SIZE;
size_t input_size_bytes = 68 * max_z * max_n_segs * sizeof(int8_t);
size_t output_size_bytes = 8448 * max_n_segs * sizeof(int8_t);
// Allocate Pinned/Mapped Memory for Zero-Copy access (GH200 friendly)
cudaHostAlloc((void**)&dummy_input_llr, input_size_bytes, cudaHostAllocMapped);
cudaHostAlloc((void**)&dummy_output_bits, output_size_bytes, cudaHostAllocMapped);
memset(dummy_input_llr, 0, input_size_bytes);
memset(dummy_output_bits, 0, output_size_bytes);
printf("[CUDA] Initializing & Warming up Driver Pipeline...\n");
if (cuda_graph_breaker == 0) {
for (int r_idx = 0; r_idx < 2; r_idx++) {
for (int z_idx = 0; z_idx < 3; z_idx++) {
uint32_t Z = Z_list[z_idx];
uint8_t R = R_list[r_idx];
uint8_t BG = 1;
uint32_t K = 22 * Z;
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
uint8_t numMaxIter = 2; // 2 iterations for faster warmup
uint8_t n_segments = STATIC_SEG_SIZE;
// Bind dummy buffers
gpu_graph_cache[node_idx].bridge_ptr->p_llr_ptr = dummy_input_llr;
gpu_graph_cache[node_idx].bridge_ptr->p_out_ptr = dummy_output_bits;
// Record graph
err_warmup = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[node_idx].bridge_ptr,
numLLR,
cnProcBuf_dev,
bnProcBuf_dev,
llrRes_dev,
llrProcBuf_dev,
Z,
K,
BG,
R,
numMaxIter,
n_segments,
nrLDPC_outMode_BIT,
decoderStreams,
0,
&gpu_graph_cache[node_idx].graph,
&gpu_graph_cache[node_idx].exec,
(uint8_t*)&gpu_graph_cache[node_idx].occupied);
if (err_warmup != cudaSuccess) {
cuda_graph_breaker = 1; // Once a graph recording fails, forbidden the graph recording permanently
printf("[CUDA] Warmup Graph Record Failed (err=%s). Circuit breaker triggered\n", cudaGetErrorString(err_warmup));
break;
}
// Save metadata
gpu_graph_cache[node_idx].Z = Z;
gpu_graph_cache[node_idx].R = R;
gpu_graph_cache[node_idx].K = K;
gpu_graph_cache[node_idx].numLLR = numLLR;
gpu_graph_cache[node_idx].BG = BG;
gpu_graph_cache[node_idx].numMaxIter = numMaxIter;
gpu_graph_cache[node_idx].n_segments = n_segments;
gpu_graph_cache[node_idx].outMode = nrLDPC_outMode_BIT;
node_idx++;
}
}
dynamic_cache_idx = node_idx;
// Execute to trigger driver initialization
if (dynamic_cache_idx > 0) {
for (int i = 0; i < dynamic_cache_idx; i++) {
if (gpu_graph_cache[i].occupied) {
nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[i].exec, decoderStreams[0], NULL, 0);
}
}
cudaDeviceSynchronize();
printf("[CUDA] Driver warm-up complete. Executed %d dummy graphs.\n", dynamic_cache_idx);
for (int i = 0; i < dynamic_cache_idx; i++) {
if (gpu_graph_cache[i].occupied) {
if (gpu_graph_cache[i].exec) {
cudaGraphExecDestroy(gpu_graph_cache[i].exec);
gpu_graph_cache[i].exec = NULL;
}
if (gpu_graph_cache[i].graph) {
cudaGraphDestroy(gpu_graph_cache[i].graph);
gpu_graph_cache[i].graph = NULL;
}
if (gpu_graph_cache[i].bridge_ptr) {
gpu_graph_cache[i].bridge_ptr->p_llr_ptr = NULL;
gpu_graph_cache[i].bridge_ptr->p_out_ptr = NULL;
}
gpu_graph_cache[i].occupied = false;
}
}
// Mark slots as free and reset index so real traffic starts from slot 0
for (int i = 0; i < dynamic_cache_idx; i++) {
gpu_graph_cache[i].occupied = false;
}
printf("[CUDA] Cache cleared. Ready for dynamic recording.\n");
}
}
dynamic_cache_idx = 0;
if (cuda_graph_breaker == 1) {
printf("[CUDA] Using Normal Execution for Warmup (Graph Disabled).\n");
for (int r_idx = 0; r_idx < 2; r_idx++) {
for (int z_idx = 0; z_idx < 3; z_idx++) {
uint32_t Z = Z_list[z_idx];
uint8_t R = R_list[r_idx];
uint8_t BG = 1;
uint32_t K = 22 * Z;
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
uint8_t numMaxIter = 2; // 2 iterations for faster warmup
uint8_t n_segments = STATIC_SEG_SIZE;
// Bind dummy buffers
gpu_graph_cache[0].bridge_ptr->p_llr_ptr = dummy_input_llr;
gpu_graph_cache[0].bridge_ptr->p_out_ptr = dummy_output_bits;
// normal execution
nrLDPC_decoder_cuda_NormalExecute(gpu_graph_cache[0].bridge_ptr,
numLLR,
cnProcBuf_dev,
bnProcBuf_dev,
llrRes_dev,
llrProcBuf_dev,
Z,
K,
BG,
R,
numMaxIter,
n_segments,
nrLDPC_outMode_BIT,
decoderStreams,
0,
NULL);
}
}
cudaDeviceSynchronize();
printf("[CUDA] Driver warm-up complete with normal execution\n");
}
cudaFreeHost(dummy_input_llr);
cudaFreeHost(dummy_output_bits);
}
void init_decoder_gpu_structures()
{
printf("[CUDA] Initializing Global GPU Structures...\n");
// Bridge for graphs
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
if (gpu_graph_cache[i].bridge_ptr == NULL) {
cudaHostAlloc((void**)&gpu_graph_cache[i].bridge_ptr, sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
gpu_graph_cache[i].bridge_ptr->p_llr_ptr = NULL;
gpu_graph_cache[i].bridge_ptr->p_out_ptr = NULL;
gpu_graph_cache[i].occupied = false;
}
}
printf("[CUDA] Allocated %d Graph Bridges.\n", MAX_GRAPH_CACHE_SIZE);
// Bridge for normal execute
for (int i = 0; i < 8; i++) {
if (stream_bridges[i] == NULL) {
cudaHostAlloc((void**)&stream_bridges[i], sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
stream_bridges[i]->p_llr_ptr = NULL;
stream_bridges[i]->p_out_ptr = NULL;
}
}
printf("[CUDA] Allocated %d Stream Bridges for Fallback case.\n", 8);
}
void init_decoder_graphs()
{
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
gpu_graph_cache[i].occupied = false;
gpu_graph_cache[i].graph = NULL;
gpu_graph_cache[i].exec = NULL;
gpu_graph_cache[i].bridge_ptr = NULL;
gpu_graph_cache[i].Z = 0;
gpu_graph_cache[i].R = 0;
}
dynamic_cache_idx = 0;
printf("[decoder_graphs] initialized %d dynamic cache slots\n", MAX_GRAPH_CACHE_SIZE);
}
void free_graphs()
{
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
if (gpu_graph_cache[i].occupied) {
if (gpu_graph_cache[i].exec)
cudaGraphExecDestroy(gpu_graph_cache[i].exec);
if (gpu_graph_cache[i].graph)
cudaGraphDestroy(gpu_graph_cache[i].graph);
gpu_graph_cache[i].occupied = false;
}
}
printf("[decoder_graphs] shutdown complete (Dynamic Cache Cleared)\n");
}
extern int cuda_support_set;
bool encoder_streamsCreated = false;
cudaStream_t encoderStreams[4];
void cuda_support_init();
int32_t LDPCinit_cuda()
{
if (cuda_support_set == 0) {
printf("Calling encoder initializations\n");
cuda_support_init();
}
if (!decoder_streamsCreated) {
for (int s = 0; s < 8; ++s) {
cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
cudaEventCreate(&decoderDoneEvents[s]);
}
decoder_streamsCreated = true;
}
if (!encoder_streamsCreated) {
for (int s = 0; s < 4; ++s) {
cudaStreamCreateWithFlags(&encoderStreams[s], cudaStreamNonBlocking);
}
encoder_streamsCreated = true;
}
printf("CUDA LDPC decoder initiating\n");
cuda_support_init_decoder();
init_decoder_graphs();
init_decoder_gpu_structures();
init_decoder_warmup();
return 0;
}
int32_t LDPCshutdown_cuda()
{
if (cnProcBuf_dev) { cudaFree(cnProcBuf_dev); cnProcBuf_dev = NULL; }
if (bnProcBuf_dev) { cudaFree(bnProcBuf_dev); bnProcBuf_dev = NULL; }
if (llrRes_dev) { cudaFree(llrRes_dev); llrRes_dev = NULL; }
if (llrProcBuf_dev) { cudaFree(llrProcBuf_dev); llrProcBuf_dev = NULL; }
if (p_llr_dev) { cudaFree(p_llr_dev); p_llr_dev = NULL; }
if (p_out_dev) { cudaFree(p_out_dev); p_out_dev = NULL; }
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
if (gpu_graph_cache[i].bridge_ptr) {
cudaFreeHost(gpu_graph_cache[i].bridge_ptr);
gpu_graph_cache[i].bridge_ptr = NULL;
}
}
for (int i = 0; i < 8; i++) {
if (stream_bridges[i]) {
cudaFreeHost(stream_bridges[i]);
stream_bridges[i] = NULL;
}
}
for (int s = 0; s < 8; ++s) {
if (decoder_streamsCreated) {
cudaEventDestroy(decoderDoneEvents[s]);
cudaStreamDestroy(decoderStreams[s]);
}
}
for (int s = 0; s < 4; s++) {
if (encoder_streamsCreated) {
cudaStreamDestroy(encoderStreams[s]);
}
}
free_graphs();
decoder_streamsCreated = false;
encoder_streamsCreated = false;
printf("[CUDA] Intermediate buffers and streams destroyed.\n");
return 0;
}
int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
int8_t* p_llr,
uint8_t* p_out,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab)
{
if (!((p_decParams->R == 89 || p_decParams->R == 23 || p_decParams->R == 13) && p_decParams->BG == 1 && p_decParams->Z % 4 == 0
&& p_decParams->Z >= 128 && p_decParams->Z <= 384)) { // format check
printf("Current format: BG = %d, R = %d, Zc = %d\n", p_decParams->BG, p_decParams->R, p_decParams->Z);
AssertFatal(false, "Format cuda not support, only support BG = 1, Zc >= 128 and R = 13, 23, 89 right now\n");
return 0;
}
// Launch LDPC decoder core for all segments
int n_segments = p_decParams->n_segments;
int numIter = nrLDPC_decoder_core_dynamic(p_llr, (int8_t*)p_out, n_segments, p_decParams, p_profiler, ab);
set_abort(ab, false);
return numIter;
}
/**
\brief PerformsnrLDPC decoding of one code block
\param p_llr Input LLRs
\param p_out Output vector
\param numLLR Number of LLRs
\param p_decParamsnrLDPC decoder parameters
\param p_profilernrLDPC profiler statistics
*/
static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
int8_t* p_out,
int n_segments,
t_nrLDPC_dec_params* p_decParams,
t_nrLDPC_time_stats* p_profiler,
decode_abort_t* ab)
{
cudaError_t err_core = cudaSuccess;
bool graph_executed = false;
uint16_t Z = p_decParams->Z;
uint8_t BG = p_decParams->BG;
uint8_t R = p_decParams->R;
uint8_t numMaxIter = p_decParams->numMaxIter;
e_nrLDPC_outMode outMode = p_decParams->outMode;
uint32_t K = Z * 22;
// Calculate LLR size per segment based on Rate
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : ((R == 89) ? NR_LDPC_NCOL_BG1_R89 * Z : NR_LDPC_NCOL_BG1_R23 * Z);
if (p_llr != p_llr_dev)
cudaMemcpyAsync(p_llr_dev, p_llr, n_segments * 68 * 384, cudaMemcpyHostToDevice, decoderStreams[0]);
// Output size safety: assume worst-case unpacked bytes (K * n_segments)
size_t total_output_size = n_segments * K * sizeof(int8_t);
/*
// for debug, remember to remove it---------
cuda_graph_breaker = 1; // skipping all the graph recording
//-----------------------------------------
*/
if (cuda_graph_breaker == 0) {
int found_idx = -1;
// Search in Graph Cache
for (int i = 0; i < dynamic_cache_idx; i++) {
if (gpu_graph_cache[i].occupied && gpu_graph_cache[i].Z == Z && gpu_graph_cache[i].R == R && gpu_graph_cache[i].BG == BG
&& gpu_graph_cache[i].K == K && gpu_graph_cache[i].numLLR == numLLR && gpu_graph_cache[i].numMaxIter == numMaxIter
&& gpu_graph_cache[i].n_segments == n_segments && gpu_graph_cache[i].outMode == outMode) {
found_idx = i;
break;
}
}
if (found_idx >= 0) {
// === Cache HIT: Execute Recorded Graph ===
gpu_graph_cache[found_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
gpu_graph_cache[found_idx].bridge_ptr->p_out_ptr = (pageable || integrated) ? p_out : p_out_dev;
err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[found_idx].exec,
decoderStreams[0],
NULL, // doneEvent
0); // Stream Index
if (err_core == cudaSuccess) {
graph_executed = true;
} else {
cuda_graph_breaker = 1;
}
} else if (dynamic_cache_idx < MAX_GRAPH_CACHE_SIZE) {
// === Cache MISS: Record New Graph and Execute ===
int new_idx = dynamic_cache_idx;
gpu_graph_cache[new_idx].occupied = true;
gpu_graph_cache[new_idx].Z = Z;
gpu_graph_cache[new_idx].R = R;
gpu_graph_cache[new_idx].BG = BG;
gpu_graph_cache[new_idx].K = K;
gpu_graph_cache[new_idx].numLLR = numLLR;
gpu_graph_cache[new_idx].numMaxIter = numMaxIter;
gpu_graph_cache[new_idx].n_segments = n_segments;
gpu_graph_cache[new_idx].outMode = outMode;
// Use the determined pointers (Device ptrs for PCIe, Host ptrs for GH200)
gpu_graph_cache[new_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
gpu_graph_cache[new_idx].bridge_ptr->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
err_core = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[new_idx].bridge_ptr,
numLLR,
cnProcBuf_dev,
bnProcBuf_dev,
llrRes_dev,
llrProcBuf_dev,
Z,
K,
BG,
R,
numMaxIter,
n_segments,
outMode,
decoderStreams,
0, // CudaStreamIdx
&gpu_graph_cache[new_idx].graph,
&gpu_graph_cache[new_idx].exec,
(uint8_t*)&gpu_graph_cache[new_idx].occupied);
if (err_core == cudaSuccess) {
err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[new_idx].exec, decoderStreams[0], NULL, 0);
if (err_core == cudaSuccess) {
graph_executed = true;
dynamic_cache_idx++;
} else {
cuda_graph_breaker = 1; // graph execution fail
}
} else {
cuda_graph_breaker = 1; // graph record fail
}
}
}
if (!graph_executed) {
// === Fallback to Normal Execution ===
// If the cache is full, we cannot record new graphs.
// Or graph operation is not safe in this device or environment.
// Execute kernel directly using standard launch.
/* if (cuda_graph_breaker == 1) {
LOG_W(PHY, "Graph opereations failed, falling back to normal.\n");
}*/
ldpc_cuda_bridge_t* perpack_buffer = stream_bridges[0];
perpack_buffer->p_llr_ptr = p_llr_dev;
perpack_buffer->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
nrLDPC_decoder_cuda_NormalExecute(perpack_buffer,
numLLR,
cnProcBuf_dev,
bnProcBuf_dev,
llrRes_dev,
llrProcBuf_dev,
Z,
K,
BG,
R,
numMaxIter,
n_segments,
outMode,
decoderStreams,
0,
NULL);
}
// Copy back and Cleanup for Discrete GPU
if (!pageable && !integrated) {
// Copy Output from Device to Host
if (outMode == nrLDPC_outMode_BIT) {
cudaMemcpyAsync(p_out, p_out_dev, total_output_size >> 3, cudaMemcpyDeviceToHost, decoderStreams[0]);
}
if (outMode == nrLDPC_outMode_BITINT8) {
cudaMemcpyAsync(p_out, p_out_dev, total_output_size, cudaMemcpyDeviceToHost, decoderStreams[0]);
}
}
cudaStreamSynchronize(decoderStreams[0]);
if (p_decParams->check_crc) {
for (int r = 0; r < n_segments; r++) {
// if (r<=1) for (int i=0;i<(K>>3);i++) printf("byte (%d,%d) %x\n",r,i,((uint8_t*)(p_out+r*(K>>3)))[i]);
if (!p_decParams->check_crc((uint8_t*)(p_out + (r * (K >> 3))), p_decParams->Kprime, p_decParams->crc_type)) {
LOG_D(PHY, "Segment %d/%d CRC NOK\n", r, n_segments);
return (1 + numMaxIter);
}
/*
uint8_t *b=(uint8_t*)(p_out + (r*(K>>3)));
int i=0;
if (b[K-2] == 0 && b[K - 1] == 0) {
while (b[i] == 0 && i < K)
i++;
if (i == K) {
LOG_E(PHY, "received all 0 pdu (K %d, r %d)\n",K,r);
}
}
*/
}
}
return numMaxIter;
}

2
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
View File

@@ -285,7 +285,7 @@ int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_de
return nrLDPC_TB_decoding_parameters->C;
}
int32_t nrLDPC_coding_init(void)
int32_t nrLDPC_coding_init(int max_num_pxsch)
{
return 0;
}

15
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
View File

@@ -380,16 +380,15 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
// Compute number of tasks to encode TB
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
nbTasks += n_seg;
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
nbTasks += n_seg;
// Search for maximum E for sizing encoder output f and f2
for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
Emax = E > Emax ? E : Emax;
}
// Search for maximum E for sizing encoder output f and f2
for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
Emax = E > Emax ? E : Emax;
}
}
// Create f and f2 to old encoding tasks outputs
uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));

213
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c
View File

@@ -429,219 +429,6 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
}
}
void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
{
const uint32_t EQm = E / Qm;
memset(f, 0, E * sizeof(uint32_t));
switch(Qm) {
case 2: {
uint32_t *e0 = e;
uint32_t *e1 = e + EQm;
int i = 0;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
}
} break;
case 4: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
int i = 0;
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
}
} break;
case 6: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
uint32_t *e4 = e3 + EQm;
uint32_t *e5 = e4 + EQm;
int i = 0;
#if 0
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
simde__m128i *e2_128 = (simde__m128i *)e2;
simde__m128i *e3_128 = (simde__m128i *)e3;
simde__m128i *e4_128 = (simde__m128i *)e4;
simde__m128i *e5_128 = (simde__m128i *)e5;
simde__m128i *f128 = (simde__m128i *)f;
for (; i < (EQm & ~3); i += 4) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
}
e0 = (uint32_t *)e0_128;
e1 = (uint32_t *)e1_128;
e2 = (uint32_t *)e2_128;
e3 = (uint32_t *)e3_128;
e4 = (uint32_t *)e4_128;
e5 = (uint32_t *)e5_128;
f = (uint32_t *)f128;
#endif
for (; i < EQm; i++) {
*f++ = *e0++;
*f++ = *e1++;
*f++ = *e2++;
*f++ = *e3++;
*f++ = *e4++;
*f++ = *e5++;
}
} break;
case 8: {
uint32_t *e0 = e;
uint32_t *e1 = e0 + EQm;
uint32_t *e2 = e1 + EQm;
uint32_t *e3 = e2 + EQm;
uint32_t *e4 = e3 + EQm;
uint32_t *e5 = e4 + EQm;
uint32_t *e6 = e5 + EQm;
uint32_t *e7 = e6 + EQm;
int i = 0;
#if 0 /*def __AVX2__, this needs to be fixed, revert to 128-bit below*/
simde__m256i *e0_256 = (simde__m256i *)e0;
simde__m256i *e1_256 = (simde__m256i *)e1;
simde__m256i *e2_256 = (simde__m256i *)e2;
simde__m256i *e3_256 = (simde__m256i *)e3;
simde__m256i *e4_256 = (simde__m256i *)e4;
simde__m256i *e5_256 = (simde__m256i *)e5;
simde__m256i *e6_256 = (simde__m256i *)e6;
simde__m256i *e7_256 = (simde__m256i *)e7;
simde__m256i *f256 = (simde__m256i *)f;
for (; i < (EQm & ~7); i += 8) {
simde__m256i e0j = simde_mm256_loadu_si256(e0_256++);
simde__m256i e1j = simde_mm256_loadu_si256(e1_256++);
simde__m256i e2j = simde_mm256_loadu_si256(e2_256++);
simde__m256i e3j = simde_mm256_loadu_si256(e3_256++);
simde__m256i e4j = simde_mm256_loadu_si256(e4_256++);
simde__m256i e5j = simde_mm256_loadu_si256(e5_256++);
simde__m256i e6j = simde_mm256_loadu_si256(e6_256++);
simde__m256i e7j = simde_mm256_loadu_si256(e7_256++);
simde__m256i tmp0 = simde_mm256_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) e0(i+4) e1(i+4) e0(i+5) e1(i+5)
simde__m256i tmp1 = simde_mm256_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) e2(i+4) e3(i+4) e2(i+5) e3(i+5)
simde__m256i tmp2 = simde_mm256_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) e4(i+4) e5(i+4) e4(i+5) e5(i+5)
simde__m256i tmp3 = simde_mm256_unpacklo_epi32(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) e6(i+4) e7(i+4) e6(i+5) e7(i+5)
//
simde__m256i tmp4 = simde_mm256_unpacklo_epi64(tmp0, tmp1); // e0(i) e1(i) e2(i) e3(i) e0(i+4) e1(i+4) e2(i+4) e3(i+4)
simde__m256i tmp5 = simde_mm256_unpacklo_epi64(tmp2, tmp3); // e4(i) e5(i) e6(i) e7(i) e4(i+4) e5(i+4) e6(i+4) e7(i+4)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x20)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)
simde__m256i tmp6 = simde_mm256_unpackhi_epi64(tmp0, tmp1); // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e0(i+5) e1(i+5) e2(i+5) e3(i+5)
simde__m256i tmp7 = simde_mm256_unpackhi_epi64(tmp2, tmp3); // e4(i+1) e5(i+1) e6(i+1) e7(i+1) e4(i+5) e5(i+5) e6(i+5) e7(i+5)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x20)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e4(i+1) e5(i+1) e6(i+1) e7(i+1)
//
tmp0 = simde_mm256_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3) e0(i+6) e1(i+6) e0(i+7) e1(i+7)
tmp1 = simde_mm256_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3) e2(i+6) e3(i+6) e2(i+7) e3(i+7)
tmp2 = simde_mm256_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3) e4(i+6) e5(i+6) e4(i+7) e5(i+7)
tmp3 = simde_mm256_unpackhi_epi32(e6j, e7j); // e6(i+2) e7(i+2) e6(i+3) e7(i+3) e6(i+6) e7(i+6) e6(i+7) e7(i+7)
//
simde__m256i tmp8 = simde_mm256_unpacklo_epi64(tmp0, tmp1); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e0(i+6) e1(i+6) e2(i+6) e3(i+6)
simde__m256i tmp9 = simde_mm256_unpacklo_epi64(tmp2, tmp3); // e4(i+2) e5(i+2) e6(i+2) e7(i+2) e4(i+6) e5(i+6) e6(i+6) e7(i+6)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x20)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2)
simde__m256i tmp10 = simde_mm256_unpackhi_epi64(tmp0, tmp1); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e0(i+7) e1(i+7) e2(i+7) e3(i+7)
simde__m256i tmp11 = simde_mm256_unpackhi_epi64(tmp2, tmp3); // e4(i+3) e5(i+3) e6(i+3) e7(i+3) e4(i+7) e5(i+7) e6(i+7) e7(i+7)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x20)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x31)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x31)); // e0(i+5) e1(i+5) e2(i+5) e3(i+5) e4(i+5) e5(i+5) e6(i+5) e7(i+5)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x31)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6)
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x31)); // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)
}
e0 = (uint32_t *)e0_256;
e1 = (uint32_t *)e1_256;
e2 = (uint32_t *)e2_256;
e3 = (uint32_t *)e3_256;
e4 = (uint32_t *)e4_256;
e5 = (uint32_t *)e5_256;
e6 = (uint32_t *)e6_256;
e7 = (uint32_t *)e7_256;
f = (uint32_t *)f256;
#endif
simde__m128i *e0_128 = (simde__m128i *)e0;
simde__m128i *e1_128 = (simde__m128i *)e1;
simde__m128i *e2_128 = (simde__m128i *)e2;
simde__m128i *e3_128 = (simde__m128i *)e3;
simde__m128i *e4_128 = (simde__m128i *)e4;
simde__m128i *e5_128 = (simde__m128i *)e5;
simde__m128i *e6_128 = (simde__m128i *)e6;
simde__m128i *e7_128 = (simde__m128i *)e7;
simde__m128i *f128 = (simde__m128i *)f;
for (; i < (EQm & ~3); i += 4) {
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
tmp3 = simde_mm_unpackhi_epi32(e6j, e7j); // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3)
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
}
e0 = (uint32_t *)e0_128;
e1 = (uint32_t *)e1_128;
e2 = (uint32_t *)e2_128;
e3 = (uint32_t *)e3_128;
e4 = (uint32_t *)e4_128;
e5 = (uint32_t *)e5_128;
e6 = (uint32_t *)e6_128;
e7 = (uint32_t *)e7_128;
f = (uint32_t *)f128;
}
break;
default:
AssertFatal(false, "Should be here!\n");
}
}
#if defined(__aarch64__)
static inline uint8x16_t tbl96_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2,
uint8x16_t b3, uint8x16_t b4, uint8x16_t b5,

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/BN30_edge.svg Normal file
View File

@@ -0,0 +1 @@
<svg width="610" height="347" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-585 -57)"><rect x="591" y="125" width="105" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(606.957 152)">Block1</text><rect x="594" y="125" width="5" height="39" fill="#C00000"/><rect x="591" y="205" width="105" height="38" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(606.957 232)">Block</text><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(666.123 232)">2</text><rect x="613" y="205" width="5" height="38" fill="#C00000"/><rect x="589" y="335" width="107" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(599.924 362)">Block30</text><rect x="642" y="336" width="5" height="39" fill="#C00000"/><rect x="760" y="119" width="128" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(771.435 152)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(791.022 88)">Step 1</text><rect x="760" y="198" width="128" height="52" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(771.435 232)">Msg_Sum</text><rect x="759" y="329" width="127" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(769.922 362)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1035.89 87)">Step 2</text><rect x="948" y="118" width="128" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(959.16 151)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1081.49 150)">-</text><rect x="1097" y="126" width="95" height="36" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1114.73 151)">Msg1</text><rect x="948" y="198" width="128" height="52" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(959.16 232)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1081.49 230)">-</text><rect x="1097" y="206" width="95" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1114.73 233)">Msg2</text><rect x="947" y="329" width="127" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(957.648 362)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1079.98 361)">-</text><rect x="1096" y="337" width="95" height="37" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1106.55 363)">Msg30</text><path d="M745.5 126.334C745.5 111.514 757.514 99.5001 772.334 99.5001L879.666 99.5001C894.486 99.5001 906.5 111.514 906.5 126.334L906.5 375.666C906.5 390.486 894.486 402.5 879.666 402.5L772.334 402.5C757.514 402.5 745.5 390.486 745.5 375.666Z" stroke="#C00000" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text fill="#ED7D31" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="48" transform="translate(802.013 305)">…</text></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 4.3 KiB

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/Node_based_explained.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 12 KiB

2925
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/bler_performance_cpu_vs_gpu.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 113 KiB

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/bnProc_behavior.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 6.6 KiB

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/cnProcBuf_layout.svg Normal file
View File

File diff suppressed because one or more lines are too long
Side by Side

After

Width:  |  Height:  |  Size: 8.6 KiB

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/cnProc_behavior.svg Normal file
View File

@@ -0,0 +1 @@
<svg width="601" height="304" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-99 -256)"><rect x="148" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(160.681 286)">Msg1</text><rect x="233" y="259" width="86" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(245.929 286)">Msg2</text><rect x="319" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(331.177 286)">Msg3</text><rect x="404" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(416.425 286)">Msg4</text><text fill="#4472C4" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(649.576 369)">CN</text><text fill="#4472C4" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(651.713 539)">BN</text><path d="M294 531.5C294 520.73 299.596 512 306.5 512 313.404 512 319 520.73 319 531.5 319 542.27 313.404 551 306.5 551 299.596 551 294 542.27 294 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M404 531.5C404 520.73 409.596 512 416.5 512 423.404 512 429 520.73 429 531.5 429 542.27 423.404 551 416.5 551 409.596 551 404 542.27 404 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M493 531.5C493 520.73 498.596 512 505.5 512 512.404 512 518 520.73 518 531.5 518 542.27 512.404 551 505.5 551 498.596 551 493 542.27 493 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M591 531.5C591 520.73 596.373 512 603 512 609.627 512 615 520.73 615 531.5 615 542.27 609.627 551 603 551 596.373 551 591 542.27 591 531.5Z" fill="#4472C4" fill-rule="evenodd"/><rect x="218" y="342" width="31" height="33" fill="#4472C4"/><path d="M0 0 18.023 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 306.5 512.663)"/><path d="M197 531.5C197 520.73 202.373 512 209 512 215.627 512 221 520.73 221 531.5 221 542.27 215.627 551 209 551 202.373 551 197 542.27 197 531.5Z" fill-rule="evenodd"/><path d="M99 531.5C99 520.73 104.596 512 111.5 512 118.404 512 124 520.73 124 531.5 124 542.27 118.404 551 111.5 551 104.596 551 99 542.27 99 531.5Z" fill-rule="evenodd"/><rect x="309" y="342" width="31" height="33"/><rect x="412" y="342" width="32" height="33"/><path d="M306.46 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M416.284 512.803 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M505.348 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M602.884 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 115.633 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 209.5 512.663)"/><path d="M0 0 316.457 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 112.5 512.663)"/><path d="M416.301 513.295 324.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 212.862 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 112.5 512.663)"/><path d="M0 0 219.228 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 209.5 512.663)"/><path d="M603.306 512.663 428.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M505.77 512.663 428.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 121.618 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 306.5 512.663)"/></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 4.4 KiB

1
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/decoder_cuda_pipeline.svg Normal file
View File

@@ -0,0 +1 @@
<svg width="942" height="650" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-83 -42)"><path d="M621 185.334C621 148.147 651.147 118 688.335 118L957.665 118C994.853 118 1025 148.147 1025 185.334L1025 549.666C1025 586.853 994.853 617 957.665 617L688.335 617C651.147 617 621 586.853 621 549.666Z" fill="#ED7D31" fill-rule="evenodd"/><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(171.139 167)">llrPreProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(172.399 294)">cnProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(172.27 422)">bnProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(186.211 565)">output_kernel</text><path d="M152.5 263.001C152.5 238.424 172.424 218.5 197.001 218.5L547.999 218.5C572.576 218.5 592.5 238.424 592.5 263.001L592.5 440.999C592.5 465.576 572.576 485.5 547.999 485.5L197.001 485.5C172.424 485.5 152.5 465.576 152.5 440.999Z" stroke="#2F528F" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(429.568 355)">Iterations</text><path d="M411.5 244.5 411.5 458.349" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9" fill="none" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="64" transform="translate(727.296 378)">Graph</text><path d="M85.5001 201.669C85.5001 155.736 122.736 118.5 168.669 118.5L940.331 118.5C986.264 118.5 1023.5 155.736 1023.5 201.669L1023.5 534.331C1023.5 580.264 986.264 617.5 940.331 617.5L168.669 617.5C122.736 617.5 85.5001 580.264 85.5001 534.331Z" stroke="#ED7D31" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(219.12 82)">Input LLR</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(132.48 664)">Output Decoded bits</text><path d="M294 95.5 294 126.552 291 126.552 291 95.5001ZM297 125.052 292.5 134.052 288 125.052Z"/><path d="M295 597.5 295 628.552 292 628.552 292 597.5ZM298 627.052 293.5 636.052 289 627.052Z"/></g></svg>
Side by Side

After

Width:  |  Height:  |  Size: 2.7 KiB

1816
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/latency_vs_segments.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 50 KiB

2028
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/perf_r13.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 56 KiB

2036
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/perf_r23.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 57 KiB

2058
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/perf_r89.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 57 KiB

1852
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/throughput_vs_segments.svg Normal file
View File

File diff suppressed because it is too large Load Diff
Side by Side

After

Width:  |  Height:  |  Size: 51 KiB

896
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/nrLDPC_cuda.md Normal file
View File

@@ -0,0 +1,896 @@
# OAI nrLDPC GPU Decoder Architecture
For hands-on instructions and execution commands of this GPU implementation, please refer to the [Tutorial Document](nrLDPC_cuda_tutorial.md).
## 0. Scope & Supported Configurations
This document describes the architecture and parallelization strategies of the GPU-accelerated LDPC decoder implemented via CUDA.
> **Note on Algorithm Baseline:** The current CUDA implementation is strictly **bit-exact and algorithm-exact** with the existing OAI [CPU baseline](nrLDPC.tex). It focuses on throughput acceleration, hardware offloading, and architectural refactoring. It does not introduce any algorithmic modifications to the underlying error correction mathematical model. Therefore, it is expected to yield the exact same BLER performance as the [CPU baseline](nrLDPC.tex)
### Currently Supported Parameters
| Parameter | CPU Baseline | GPU Implementation |
| --- | --- | --- |
| **Base Graph (BG)** | 1, 2 | **1** |
| **Lifting Size ($Z_c$)** | All | **$\ge 128$** |
| **Code Rate (R)** | BG1: 1/3, 2/3, 8/9 <br><br> BG2: 1/5, 1/3, 2/3 | **BG1: 1/3, 2/3, 8/9** |
### Related Work & Comparison
| Implementation | Open-Source Framework | 5G NR Spec Aligned | Kernel Source Available | User-Recompilable | Bit-Exact Baseline |
|---|---|---|---|---|---|
| NVIDIA Aerial cuPHY | Yes | Yes | Yes | No | N/A |
| Academic GPU LDPC (e.g. cuLDPC) | Yes | No | Yes | Yes | N/A |
| **This work (OAI GPU decoder)** | **Yes** | **Yes** | **Yes** | **Yes** | **Yes** |
---
[[_TOC_]]
## 1. Overall Architecture
![Pipeline Overview](img/decoder_cuda_pipeline.svg)
The OAI nrLDPC GPU decoder consists of four main kernels: `llrPreProc_kernel`, `cnProc_kernel`, `bnProc_kernel`, and `output_kernel`.
Functionally, the GPU version is strictly aligned with the CPU baseline. Each CUDA kernel is essentially a consolidation of multiple specific functions from the CPU decoder. The exact mapping is as follows:
* **`llrPreProc_kernel`**: Combines `llr2llrProcBuf` and `llr2CnProcBuf`.
* **`cnProc_kernel`**: Combines `cnProc` and `cn2bnProcBuf`.
* **`bnProc_kernel`**: Combines `bnProcPc`, `bnProc`, and `bn2cnProcBuf`.
* **`output_kernel`**: Combines `llrRes2llrOut` and `llr2bit`.
## 2. Message Passing in LDPC Decoding
In this section, we briefly describe the behavior of `cnProc` (Check Node processing) and `bnProc` (Bit Node processing) during decoding. This high-level understanding will help clarify the overall parallelism design discussed later.
In LDPC decoding, the task of a Check Node (CN) in `cnProc` is to generate and send update messages to its connected Bit Nodes (BNs). For example, a degree-4 CN (a CN connected to 4 BNs) needs to generate 4 distinct messages—typically using min-sum operations—and send each message back to the corresponding connected BN. Each of these messages corresponds to one edge in the bipartite graph.
![cnProc message passing](img/cnProc_behavior.svg)
Conversely, the task of a BN in `bnProc` is to gather all the messages from its connected CNs and send updated messages back. This involves accumulation and subtraction operations. For example, a degree-3 BN (a BN connected to 3 CNs) generates 3 messages by first accumulating the 3 incoming messages from the connected CNs along with its original intrinsic LLR, then subtracting each CN's specific value, and sending the newly updated message back to the corresponding CN.
![bnProc message passing](img/bnProc_behavior.svg)
The entire decoding process is executed by iteratively alternating between these two behaviors.
## 3. Memory Management & Buffer Layout
### 3.1 Memory Management
In the current implementation, all relevant buffers reside entirely in GPU memory (Device memory). By keeping the entire receiver processing pipeline's buffers on the GPU, we completely eliminate the overhead of Host-to-Device (H2D) and Device-to-Host (D2H) memory transfers between individual kernels.
The following table summarizes the key data buffers and their read/write access patterns across the kernels:
| Buffer Name | Description | Written By | Read By |
| :--- | :--- | :--- | :--- |
| `p_llr` | Output from the rate matching module (raw input LLRs). | *(Previous module)* | `llrPreProc_kernel` |
| `p_out` | Final hard-decision decoded output bits. | `output_kernel` | *(Next module)* |
| `cnProcBuf_dev` | Stores BN-to-CN messages. Initialized with intrinsic LLRs before the first iteration. | `llrPreProc_kernel` (init), `bnProc_kernel` | `cnProc_kernel` |
| `bnProcBuf_dev` | Stores CN-to-BN messages updated during check node processing. | `cnProc_kernel` | `bnProc_kernel` |
| `llrRes_dev` | Stores the posterior LLRs (a posteriori information) accumulated during the final iteration. | `bnProc_kernel` (last iter) | `output_kernel` |
| `llrProcBuf_dev` | Stores the rearranged intrinsic LLRs to be added back during BN updates, preventing belief drift. | `llrPreProc_kernel` | `bnProc_kernel` |
### 3.2 Buffer Layout
All buffers here follow the exact same layout as the CPU version, especially the two most critical buffers in the decoding process: `cnProcBuf_dev` and `bnProcBuf_dev`.
The OAI decoder implementation groups the Check Nodes (CNs) and Bit Nodes (BNs) based on their degree (the number of connected nodes).
While their layouts are rigorously described using mathematical notation in [`doc/nrLDPC/nrLDPC.tex`](nrLDPC.tex), this section provides a more intuitive description of their structures in memory.
`cnProcBuf_dev` stores the messages that a CN needs to process, which are sent by its connected BNs. For example, a degree-4 CN needs to process 4 incoming messages. However, in the current buffer layout design, these 4 messages belonging to the same CN are **not contiguous** in memory.
To illustrate, consider BG1 (Rate 1/3), which contains 5 CNs with a degree of 4. The actual memory layout groups the same message index across all CNs in that specific group, as depicted below:
![cnProcBuf layout](img/cnProcBuf_layout.svg)
Similarly, `bnProcBuf_dev` follows this exact same interleaved layout strategy.
This specific Structure of Arrays (SoA) interleaving was historically designed to maximize CPU SIMD (e.g., AVX-512) vectorization efficiency. By clustering the same message indices together across multiple CNs, it avoids complex cross-lane shuffles and boundary alignment issues within vector registers.
For the CUDA implementation, we retain this legacy layout. It imposes no performance penalty on the GPU—since global memory coalescing is already naturally guaranteed by the innermost lifting size ($Z_c$) dimension, and it perfectly fulfills the alignment requirements for the SIMD4 operations used in our GPU implementation.
## 4. Flat Message Indexing
The fundamental computational elements in our LDPC decoder are the individual messages passed along the edges of the bipartite graph (either CN-to-BN or BN-to-CN). To specify their behaviors during parallel decoding, each message must be mapped to a unique memory address, a destination buffer address, and a specific circular shift value.
Instead of relying on nested loops and complex multi-dimensional pointers, we introduce a **Flat Message Indexing** strategy. We assign a deterministic, global 1D index to every single message (edge) in the Base Graph.
### 4.1 Total Message Calculation
The total number of messages is calculated based on the degree distribution of the Check Nodes in BG1. By multiplying the number of connected BNs (`numBnInCnGroups`) with the number of corresponding CNs (`numCnInCnGroups`), we obtain the exact number of edges for each Code Rate (R):
* **Degrees (`numBnInCnGroups`)**: {3, 4, 5, 6, 7, 8, 9, 10, 19}
* **R 1/3 CN Counts**: {1, 5, 18, 8, 5, 2, 2, 1, 4} $\rightarrow$ Total Messages = $3(1) + 4(5) + 5(18) + \dots + 19(4) = \mathbf{316}$
* **R 2/3 CN Counts**: {1, 0, 0, 0, 3, 2, 2, 1, 4} $\rightarrow$ Total Messages = $\mathbf{144}$
* **R 8/9 CN Counts**: {1, 0, 0, 0, 0, 0, 0, 0, 4} $\rightarrow$ Total Messages = $\mathbf{79}$
### 4.2 Linear Index Mapping
By flattening the structure, the messages are linearized sequentially. The 1D index effectively acts as a universal coordinate `(CN/BN_Degree, CN/BN_Instance_ID, Local_Message_ID)`.
We use the indexing from CN side as an example, the mappings for the supported code rates are as follows:
```text
[Rate 1/3 Message Index Map - Total 316]
Idx 0 : CN3-0-Msg0
Idx 1 : CN3-0-Msg1
Idx 2 : CN3-0-Msg2
Idx 3 : CN4-0-Msg0
Idx 4 : CN4-0-Msg1
Idx 5 : CN4-0-Msg2
Idx 6 : CN4-0-Msg3
Idx 7 : CN4-1-Msg0
...
Idx 314 : CN19-3-Msg17
Idx 315 : CN19-3-Msg18
[Rate 2/3 Message Index Map - Total 144]
Idx 0 : CN3-0-Msg0
Idx 1 : CN3-0-Msg1
Idx 2 : CN3-0-Msg2
Idx 3 : CN7-0-Msg0
Idx 4 : CN7-0-Msg1
Idx 5 : CN7-0-Msg2
Idx 6 : CN7-0-Msg3
Idx 7 : CN7-0-Msg4
...
Idx 142 : CN19-3-Msg17
Idx 143 : CN19-3-Msg18
[Rate 8/9 Message Index Map - Total 79]
Idx 0 : CN3-0-Msg0
Idx 1 : CN3-0-Msg1
Idx 2 : CN3-0-Msg2
Idx 3 : CN19-0-Msg0
Idx 4 : CN19-0-Msg1
Idx 5 : CN19-0-Msg2
Idx 6 : CN19-0-Msg3
Idx 7 : CN19-0-Msg4
...
Idx 77 : CN19-3-Msg17
Idx 78 : CN19-3-Msg18
```
The indexing from BN side follows the same strategy.
This 1D indexing maps to specific offset LUTs, seamlessly guiding the threads to calculate the exact memory addresses on the interleaved buffer layout described in Section 3.2.
To support other code rates or base graphs in the future (for example, a potential BG3 in 6G), basically, we just need to design new indexings and LUTs accordingly.
## 5. Parallelism Strategies
### 5.1 Edge-Based Parallelism
To extract the maximum concurrency from the GPU hardware, the most intuitive approach is to compute every single message (edge) in the bipartite graph simultaneously. We refer to this approach as **Edge-Based Parallelism**.
#### 5.1.1 Thread Organization
In this strategy, we map the computation of messages directly to the CUDA execution grid. To maximize memory throughput and instruction efficiency, we utilize 32-bit vectorized memory accesses (`simd4`).
The thread organization follows this 2D grid/block topology:
* **X-Dimension (Lifting Size & Messages):** Since we process 4 elements per thread, the block's X-dimension is scaled to $Z_c / 4$ (`Z_c >> 2`). To further increase block-level occupancy, the block's Y-dimension is set to 4, processing 4 messages concurrently within a single block. Consequently, the grid's X-dimension represents the total number of messages divided by 4.
* **Y-Dimension (Segments):** The grid's Y-dimension directly maps to the number of Transport Block segments (`n_segments`), allowing multiple code blocks to be decoded completely in parallel.
The precise kernel launch configurations for the supported code rates in BG1 are formulated as follows:
```cpp
// BG1 Rate 1/3 (num_TotalBlocks_BG1_R13_Edge = 316)
dim3 R13_block(Z_c >> 2, 4, 1);
dim3 R13_grid(num_TotalBlocks_BG1_R13_Edge >> 2, n_segments, 1);
```
```cpp
// BG1 Rate 2/3 (num_TotalBlocks_BG1_R23_Edge = 144)
dim3 R23_block(Z_c >> 2, 4, 1);
dim3 R23_grid(num_TotalBlocks_BG1_R23_Edge >> 2, n_segments, 1);
```
```cpp
// BG1 Rate 8/9 (num_TotalBlocks_BG1_R89_Edge = 79)
// Note: 79 is not strictly divisible by 4; padding (+3) ensures ceiling division.
dim3 R89_block(Z_c >> 2, 4, 1);
dim3 R89_grid((num_TotalBlocks_BG1_R89_Edge + 3) >> 2, n_segments, 1);
```
#### 5.1.2 cnProc_kernel
For the Check Node processing, we implemented specific kernels for each supported code rate with these namings:
* `cnProcKernel_BG1_R13_int8_Edge`
* `cnProcKernel_BG1_R23_int8_Edge`
* `cnProcKernel_BG1_R89_int8_Edge`
We use BG1 R13 as an example. The global kernel acts as a dispatcher. Based on the Check Node degree (`groupIdx`), it routes the thread to a specific unrolled device function:
```cpp
__global__ void cnProcKernel_BG1_R13_int8_Edge(const int8_t *__restrict__ d_cnBufAll,
int8_t *__restrict__ d_bnBufAll,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t lane = threadIdx.x;
// Compute the global 1D flat index (row) for the current message
uint32_t row = (blockIdx.x << 2) + threadIdx.y;
uint32_t segIdx = blockIdx.y;
if (row >= num_TotalBlocks_BG1_R13_Edge)
return;
// LUT Lookups: Fetch node degree, instance ID, and message ID
uint32_t groupIdx = lut_CnGrpIdx_BG1_R13_Edge[row] - 1;
uint32_t CnIdx = lut_CnIdx_BG1_R13_Edge[row] - 1;
uint32_t MsgIdx = lut_CnMsgIdx_BG1_R13_Edge[row] - 1;
// Calculate exact memory offsets
uint32_t InnerOffset = d_lut_startAddrCnGroups_BG1[groupIdx] + NR_LDPC_ZMAX * CnIdx;
uint32_t idxBn = cn_bn_map_BG1_Z_R13[row][0];
uint32_t circShift = cn_bn_map_BG1_Z_R13[row][ZcIdx];
const int8_t *p_cnProcBuf = (const int8_t *)(d_cnBufAll + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + InnerOffset);
int8_t *p_bnProcBuf = (int8_t *)(d_bnBufAll + segIdx * NR_LDPC_SIZE_BN_PROC_BUF);
// Dispatch to the degree-specific device function (Branchless execution inside)
switch (groupIdx) {
case 0: cnProcKernel_BG1_int8_G3(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 1: cnProcKernel_BG1_int8_G4(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 2: cnProcKernel_BG1_int8_G5(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 3: cnProcKernel_BG1_int8_G6(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 4: cnProcKernel_BG1_int8_G7(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 5: cnProcKernel_BG1_int8_G8(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 6: cnProcKernel_BG1_int8_G9(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 7: cnProcKernel_BG1_int8_G10(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
case 8: cnProcKernel_BG1_int8_G19(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc);
break;
}
}
```
Inside the device functions, we heavily utilize `SIMD4` operations and PTX intrinsics to perform the Min-Sum algorithm concurrently on 4 elements per thread. Taking the Degree-5 CN (`G5`) as an example:
```cpp
__device__ __forceinline__ void cnProcKernel_BG1_int8_G5(const int8_t *__restrict__ p_cnProcBuf,
int8_t *__restrict__ p_bnProcBuf,
uint32_t row,
uint32_t lane,
uint32_t idxBn,
uint32_t circShift,
uint32_t Zc)
{
uint32_t ymm0, sgn, min;
const uint32_t ones = 0x01010101;
// Load the first message (4 bytes/int8s at once)
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][0] * 4);
sgn = __vxor4(ones, ymm0); // Extract and track signs
min = __vabs4(ymm0); // Initialize minimum magnitude
// --- Fully Unrolled Min-Sum Loop for remaining 4 edges ---
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][1] * 4);
min = __vminu4(min, __vabs4(ymm0)); // Update minimum
sgn = __vxor4(sgn, ymm0); // Accumulate sign parity
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][2] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][3] * 4);
min = __vminu4(min, __vabs4(ymm0));
sgn = __vxor4(sgn, ymm0);
// ---------------------------------------------------------
// Combine the accumulated sign with the minimum magnitude
uint32_t BricksToBeMoved = __vsign4(min, sgn);
// Apply circular shift and write the generated message back to the BN buffer
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
```
#### 5.1.3 bnProc_kernel
Similarly, the `bnProc` kernel is instantiated for each code rate:
* `bnProcKernel_BG1_R13_int8_Edge`
* `bnProcKernel_BG1_R23_int8_Edge`
* `bnProcKernel_BG1_R89_int8_Edge`
```cpp
__global__ void bnProcKernel_BG1_R13_int8_Edge(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t lane = threadIdx.x;
uint32_t row = (blockIdx.x << 2) + threadIdx.y;
uint32_t segIdx = blockIdx.y;
if (row >= num_TotalBlocks_BG1_R13_Edge)
return;
// Retrieve routing logic and offsets via 1D Flat Index
uint32_t GrpIdx = lut_BnGrpIdx_BG1_R13_Edge[row];
uint32_t MsgIdx = lut_BnMsgIdx_BG1_R13_Edge[row] - 1;
uint32_t BnIdx = lut_BnIdx_BG1_R13_Edge[row];
uint32_t BnToAddrIdx = lut_BnToAddrIdx_BG1_R13[GrpIdx - 1];
uint32_t GrpNum = d_lut_numBnInBnGroups_BG1_R13[GrpIdx - 1];
uint32_t circShift = bn_cn_map_BG1_Z_R13[row][ZcIdx];
const uint32_t baseBn = (BnIdx - 1) * NR_LDPC_ZMAX;
// Calculate base pointers for the specific Variable Node group
const int8_t *p_bnProcBuf_Grp = (const int8_t *)(d_bnProcBuf + baseBn + segIdx * NR_LDPC_SIZE_BN_PROC_BUF + d_lut_startAddrBnGroups_BG1_R13[BnToAddrIdx - 1]);
const int8_t *p_cnProcBuf_Grp = (const int8_t *)(d_cnProcBuf + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + bn_cn_map_BG1_Z_R13[row][0]);
const int8_t *p_llrProcBuf_Grp = (const int8_t *)(d_llrProcBuf + baseBn + segIdx * NR_LDPC_MAX_NUM_LLR + d_lut_startAddrBnGroupsLlr_BG1_R13[BnToAddrIdx - 1]);
const int8_t *p_llrRes_Grp = (const int8_t *)(d_llrRes + baseBn + segIdx * NR_LDPC_MAX_NUM_LLR + d_lut_startAddrBnGroupsLlr_BG1_R13[BnToAddrIdx - 1]);
bnProcKernel_BG1_int8_Gn_Edge(p_bnProcBuf_Grp, (int8_t *)p_cnProcBuf_Grp, p_llrProcBuf_Grp, (int8_t *)p_llrRes_Grp,
lane, GrpIdx, MsgIdx, BnIdx, GrpNum, circShift, Zc);
}
```
To prevent 8-bit integer overflow during Bit Node accumulation, the intrinsic values are unpacked and sign-extended to 16-bit integers (`vaddss2`), accumulated, and then saturated and repacked back to 8-bit.
```cpp
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge(const int8_t *__restrict__ d_bnProcBuf,
int8_t *__restrict__ d_cnProcBuf,
const int8_t *__restrict__ d_llrProcBuf,
int8_t *__restrict__ d_llrRes,
uint32_t lane, uint32_t GrpIdx, uint32_t MsgIdx,
uint32_t BnIdx, uint32_t GrpNum, uint32_t circShift, uint32_t Zc)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
// Load intrinsic LLR (4x int8)
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
// Unpack to 16-bit to prevent overflow during accumulation
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- 2-Way Unrolled Accumulation Loop ---
for (; i < (int)GrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo); // 16-bit SIMD addition
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
// Handle remaining element if GrpIdx is odd
if (i < GrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
// Saturate the 16-bit sums back into a packed 32-bit word (4x int8)
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
if (GrpIdx == 1) {
BricksToBeGet = packed_intrinsic;
} else {
// Subtract the CN's own previous message to prevent belief drift
uint32_t prevIdxWords = (MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
BricksToBeGet = __vsubss4(saturated_llr, prev); // SIMD4 saturated subtraction
}
// Apply circular shift and write to CN buffer
moveBricks_forput_circ(d_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
```
During the final decoding iteration, an alternative `_last` kernel is invoked. Instead of subtracting the previous message and writing back to the `cnProcBuf`, it directly outputs the fully accumulated posterior LLRs (`saturated_llr`) into `d_llrRes` for hard decision extraction.
#### 5.1.4 Complexity Bottleneck Analysis
While Edge-Based Parallelism maximizes concurrent thread execution, it achieves this at the cost of massive computational redundancy. By completely isolating each message computation into an independent thread to prevent memory write collisions and expensive synchonization cost across blocks, the hardware is forced to perform highly repetitive arithmetic operations.
In the `bnProc` , the Bit Node(BN) must first accumulate all incoming messages from its connected Check Nodes (CNs), and subsequently subtract the specific target CN's previous message.
Under the Edge-Based strategy, this logic leads to a severe performance bottleneck for high-degree nodes. For instance, consider BN30 in BG1 R13. This single node spawns 30 independent blocks to compute its 30 outgoing messages simultaneously. Because these blocks do not share intermediate accumulation results, *each* of the 30 blocks must independently perform the costly 30-message summation.
Consequently, a process that mathematically requires only $30$ additions now executes $30 \times 30 = 900$ additions. The computational complexity for processing a node effectively explodes from $O(N)$ to $O(N^2)$, where $N$ represents the node degree.
![BN30_analysis](img/BN30_edge.svg)
As a result, Edge-Based Parallelism performs well for low-throughput scenarios (a small number of segments) due to its high GPU occupancy. However, it hits a compute-bound limit when scaling up to decode a large number of segments. These redundant arithmetic operations easily use up the GPU's ALU resources.
This limitation imposes us to consider **Node-Based Parallelism**, which resolves this redundancy and eases the hardware resource burden.
### 5.2 Node-Based Parallelism
To fundamentally resolve the computational redundancy identified in the Edge-Based approach, we transition to a **Node-Based Parallelism** strategy. Instead of mapping one CUDA block to one edge (message), we map **one block to one node** (either a Check Node or a Bit Node).
#### 5.2.1 Node-Based Parallelism Explained
The core philosophy of Node-Based parallelism is to calculate the shared intermediate result (the aggregate sum or global minimums) exactly *once* per node, cache it in local registers, and then reuse it to compute all outgoing messages.
![Node-Based_explained](img/Node_based_explained.svg)
As illustrated in the diagram above, consider a node with a degree of $N$:
* **Edge-Based (Redundant):** Requires $N$ independent blocks. Each block computes the `Msg_Sum` from scratch ($N$ additions) and then performs 1 subtraction.
* Total Operations: $N \times (N + 1) = N^2 + N \rightarrow \mathbf{O(N^2)}$
* **Node-Based (Optimized):** Uses exactly 1 block. The block computes the `Msg_Sum` once ($N$ additions) in Step 1. In Step 2, it loops through the $N$ edges, performing $N$ subtractions using the cached sum.
* Total Operations: $N + N = 2N \rightarrow \mathbf{O(N)}$
This dramatic reduction in arithmetic complexity from quadratic to linear thoroughly unblocks the GPU ALU bottleneck, especially for high-degree nodes.
#### 5.2.2 Thread Organization
The thread organization follows this updated 2D grid/block topology:
* **X-Dimension (Lifting Size & Nodes):** We continue to utilize 32-bit `simd4` vectorized loads/stores, meaning the block's X-dimension remains scaled to $Z_c / 4$ (`Z >> 2`). However, the Y-dimension of the block is now set to 4 to process **4 nodes** concurrently within a single block. Consequently, the grid's X-dimension represents the total number of Check Nodes or Bit Nodes divided by 4. Ceiling division (`+ 3`) is applied to handle cases where the total number of nodes is not a multiple of 4.
* **Y-Dimension (Segments):** The grid's Y-dimension continues to map directly to the number of Transport Block segments (`n_segments`), decoding multiple code blocks in parallel.
Because Check Nodes (CN) and Bit Nodes (BN) possess different totals depending on the Base Graph and code rate, their execution grids are launched separately. The precise kernel launch configurations for the supported code rates in BG1 are formulated as follows:
```cpp
// Node-Based Block Configuration (Universal across all rates and node types)
// X: Z/4 threads for simd4 access, Y: 4 nodes processed concurrently
dim3 Node_block(Z >> 2, 4, 1);
```
```cpp
// BG1 Rate 1/3 configurations
//num_TotalBlocks_cn_BG1_R13_Node = 46
dim3 cn_R13_grid((num_TotalBlocks_cn_BG1_R13_Node + 3) >> 2, n_segments, 1);
//num_TotalBlocks_bn_BG1_R13_Node = 68
dim3 bn_R13_grid(num_TotalBlocks_bn_BG1_R13_Node >> 2, n_segments, 1);
```
```cpp
// BG1 Rate 2/3 configurations
//num_TotalBlocks_cn_BG1_R23_Node = 13
dim3 cn_R23_grid((num_TotalBlocks_cn_BG1_R23_Node + 3) >> 2, n_segments, 1);
//num_TotalBlocks_bn_BG1_R23_Node = 35
dim3 bn_R23_grid((num_TotalBlocks_bn_BG1_R23_Node + 3) >> 2, n_segments, 1);
```
```cpp
// BG1 Rate 8/9 configurations
//num_TotalBlocks_cn_BG1_R89_Node = 5
dim3 cn_R89_grid((num_TotalBlocks_cn_BG1_R89_Node + 3) >> 2, n_segments, 1);
//num_TotalBlocks_bn_BG1_R89_Node = 27
dim3 bn_R89_grid((num_TotalBlocks_bn_BG1_R89_Node + 3) >> 2, n_segments, 1);
```
#### 5.2.3 cnProc_kernel
For the Check Node processing in Node-Based mode, we implemented specific kernels for each code rate:
* `cnProcKernel_BG1_R13_int8_Node`
* `cnProcKernel_BG1_R23_int8_Node`
* `cnProcKernel_BG1_R89_int8_Node`
Unlike the edge-based dispatcher, the global index `row` now points to a unique **Check Node** rather than a message.
```cpp
__global__ void cnProcKernel_BG1_R13_int8_Node(const int8_t *__restrict__ d_cnBufAll,
int8_t *__restrict__ d_bnBufAll,
uint32_t Zc,
uint32_t ZcIdx)
{
uint32_t lane = threadIdx.x;
uint32_t row = (blockIdx.x << 2) + threadIdx.y;
uint32_t segIdx = blockIdx.y;
if (row >= num_TotalBlocks_cn_BG1_R13_Node)
return;
// 1D Indexing resolves the specific Check Node and its parameters
uint32_t CnGrpIdx = lut_CnGrpIdx_BG1_R13_Node[row] - 1;
uint32_t CnIdx = lut_CnIdx_BG1_R13_Node[row] - 1;
uint32_t InnerOffset = d_lut_startAddrCnGroups_BG1[CnGrpIdx] + NR_LDPC_ZMAX * CnIdx;
uint32_t Cn2MsgStartIdx = lut_CnStartMsgIdx_BG1_R13_Node[row];
uint32_t CnGrpIdxNum = d_lut_numBnInCnGroups_BG1_R13[CnGrpIdx];
uint32_t CnNumInGrp = d_lut_numCnInCnGroups_BG1_R13[CnGrpIdx];
const int8_t *p_cnProcBuf = (const int8_t *)(d_cnBufAll + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + InnerOffset);
int8_t *p_bnProcBuf = (int8_t *)(d_bnBufAll + segIdx * NR_LDPC_SIZE_BN_PROC_BUF);
// Call the consolidated Node-Based device function
cnProcKernel_BG1_int8_Gn_R13_node(p_cnProcBuf, p_bnProcBuf, lane, CnIdx, CnNumInGrp, CnGrpIdxNum, Cn2MsgStartIdx, Zc, ZcIdx);
}
```
The device function executes a classic 2-pass Min-Sum algorithm to achieve $O(N)$ efficiency.
* **Pass 1:** Find the absolute minimum (`min1`), the second minimum (`min2`), and the cumulative XOR sign (`total_xor`) of all connected edges.
* **Pass 2:** Generate the outgoing messages by applying the appropriate minimum value and sign, then write them back sequentially.
```cpp
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R13_node(/* ... parameters ... */)
{
uint32_t min1 = 0x7F7F7F7F;
uint32_t min2 = 0x7F7F7F7F;
uint32_t total_xor = 0;
// Local thread registers to cache values, avoiding redundant global memory access
uint32_t cache_raw[19];
uint32_t cache_abs[19];
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
const int32_t *currPtr = cnProcBufPtr;
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
// --- Pass 1: Global Extrema and Sign Accumulation ---
#pragma unroll
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
uint32_t val = *currPtr;
cache_raw[MsgIdx] = val;
uint32_t v_abs = __vabs4(val);
cache_abs[MsgIdx] = v_abs;
total_xor = __vxor4(total_xor, val);
uint32_t old_min1 = min1;
min1 = __vminu4(old_min1, v_abs);
uint32_t candidate = __vmaxu4(old_min1, v_abs);
min2 = __vminu4(min2, candidate);
currPtr += offset;
}
// --- Pass 2: Message Generation and Write-Back ---
#pragma unroll
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
uint32_t my_abs = cache_abs[temp_MsgIdx];
// Branchless selection: If this edge was min1, use min2; otherwise use min1
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = cn_bn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
}
}
```
#### 5.2.4 bnProc_kernel
The Bit Node update (`bnProc`) mirrors this $O(N)$ logic. We define specific node-based kernels for each code rate:
* `bnProcKernel_BG1_R13_int8_Node`
* `bnProcKernel_BG1_R23_int8_Node`
* `bnProcKernel_BG1_R89_int8_Node`
*(Note: The global dispatcher `bnProcKernel_BG1_R13_int8_Node` follows the exact same 1D node-indexing structure as the `cnProc` equivalent.)*
In the device function, the 2-pass strategy is implemented as:
* **Pass 1:** Accumulate all incoming edge messages and the intrinsic LLR into a single global sum (`MsgSumLo` / `MsgSumHi`).
* **Pass 2:** Iterate through the edges, subtract that specific edge's previous message from the global sum (`__vsubss4`), and scatter the updated messages back.
```cpp
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R13(/* ... parameters ... */)
{
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
uint32_t MsgSumLo, MsgSumHi;
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
const int32_t *currPtr = bnProcBufPtr;
int i = 0;
// --- Pass 1: 2-Way Unrolled Global Accumulation ---
for (; i < (int)BnGrpIdx - 1; i += 2) {
uint32_t val1 = *currPtr;
uint32_t val2 = *(currPtr + off);
uint32_t v1_lo, v1_hi;
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
uint32_t v2_lo, v2_hi;
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
currPtr += (off << 1);
}
if (i < BnGrpIdx) {
uint32_t val = *currPtr;
uint32_t v_lo, v_hi;
unpack_and_sign_extend(val, &v_lo, &v_hi);
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
}
// Pack the total sum back into 8-bit integers
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
uint32_t BricksToBeGet;
// --- Pass 2: Subtraction and Scatter Write-Back ---
if (BnGrpIdx == 1) {
// Degree-1 optimization: No subtraction needed
BricksToBeGet = packed_intrinsic;
uint32_t MsgIdx = Bn2MsgStartIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
} else {
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
uint32_t prev = bnProcBufPtr[prevIdxWords];
// Crucial Step: Subtract the specific edge's previous message
BricksToBeGet = __vsubss4(saturated_llr, prev);
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
}
}
}
```
### 5.3 Hybrid Approach & Switching Policy
#### 5.3.1 Resource vs. Speed Trade-off
The Edge-based approach computes all messages simultaneously, providing optimal latency (e.g., ~50µs) for single-segment workloads. However, its heavy resource consumption quickly saturates GPU Streaming Multiprocessors (SMs) when processing large batches. Conversely, the Node-based approach maps computations to graph nodes rather than edges, shrinking the resource footprint by a factor of the node degree ($N$). This footprint reduction prevents SM contention and delivers superior throughput for larger workloads.
#### 5.3.2 Dynamic Switching Policy
To balance latency and throughput, the decoder dynamically switches between Edge-based and Node-based kernels at runtime. This switching policy is determined by the number of code block segments (`n_segments`) being processed.
For small batch sizes (e.g., `n_segments = 1`), the scheduler executes Edge-based kernels to minimize single-segment latency. For larger batch sizes (e.g., `n_segments = 128`), the scheduler switches to Node-based kernels to maintain overall throughput and optimize SM occupancy.
The following code snippets illustrate this runtime switching mechanism. Empirical threshold parameters, such as `NodeEdge_Switch_Cn_R13`, are defined to trigger the transition:
```cpp
void nrLDPC_cnProc_BG1_R13_cuda_stream_core(int8_t *cnProcBuf,
int8_t *bnProcBuf,
uint32_t n_segments,
uint32_t Z,
uint32_t ZcIdx,
cudaStream_t *streams,
int8_t CudaStreamIdx)
{
if (n_segments > NodeEdge_Switch_Cn_R13) {
cnProcKernel_BG1_R13_int8_Node<<<Kdim_cn_R13_Node[CudaStreamIdx].grid,
Kdim_cn_R13_Node[CudaStreamIdx].block,
0,
streams[CudaStreamIdx]>>>(cnProcBuf, bnProcBuf, Z, ZcIdx);
} else {
cnProcKernel_BG1_R13_int8_Edge<<<Kdim_R13_Edge[CudaStreamIdx].grid,
Kdim_R13_Edge[CudaStreamIdx].block,
0,
streams[CudaStreamIdx]>>>(cnProcBuf, bnProcBuf, Z, ZcIdx);
}
CHECK(cudaGetLastError());
}
void nrLDPC_bnProc_BG1_R13_cuda_stream_core(int8_t *bnProcBuf,
int8_t *cnProcBuf,
int8_t *llrProcBuf,
int8_t *llrRes,
uint32_t n_segments,
uint32_t Z,
uint32_t ZcIdx,
cudaStream_t *streams,
int8_t CudaStreamIdx)
{
if (n_segments > NodeEdge_Switch_Bn_R13) {
bnProcKernel_BG1_R13_int8_Node<<<Kdim_bn_R13_Node[CudaStreamIdx].grid,
Kdim_bn_R13_Node[CudaStreamIdx].block,
0,
streams[CudaStreamIdx]>>>(bnProcBuf, cnProcBuf, llrProcBuf, llrRes, Z, ZcIdx);
} else {
bnProcKernel_BG1_R13_int8_Edge<<<Kdim_R13_Edge[CudaStreamIdx].grid,
Kdim_R13_Edge[CudaStreamIdx].block,
0,
streams[CudaStreamIdx]>>>(bnProcBuf, cnProcBuf, llrProcBuf, llrRes, Z, ZcIdx);
}
CHECK(cudaGetLastError());
}
```
It should be noted that the empirical switching thresholds obtained so far (e.g., NodeEdge_Switch_Cn_R13) were derived from actual performance testing on the NVIDIA GH200(See [Appendix A](#Appendix-A)). For different hardware platforms, these optimal thresholds are expected to vary depending on the available computational resources. A practical heuristic is to scale these thresholds proportionally based on the target hardware's compute resources relative to the GH200 (such as the number of Streaming Multiprocessors). Future work will explore a more precise, analytical method for determining these thresholds across diverse architectures.
## 6. System Integration and Performance Evaluation
#### 6.1 Pipeline Scheduling & Execution Flow
Beyond the arithmetic optimizations at the kernel level, the overall decoder latency is heavily influenced by CPU-side driver overhead, specifically the cost of launching hundreds of iterative CUDA kernels per Transport Block. To mitigate this, the decoder integrates a dynamic pipeline scheduling architecture centered around **CUDA Graphs**.
The execution lifecycle of a batch of incoming code blocks (`ldpc_decoder_cuda.c`) is governed by the following pipeline stages:
**1. Driver Initialization and Warm-up**
To prevent unpredictable latency in processing the first arriving network packets (often caused by lazy loading or JIT compilation within the NVIDIA driver), we perform a warm-up procedure(`init_decoder_warmup`). During init, the decoder allocates dummy buffers and pre-records CUDA graphs for common 5G NR configurations (e.g., Lifting Sizes $Z \in \{320, 352, 384\}$ and Base Graph 1, Rates 1/3 and 2/3). Executing these dummy graphs forces the GPU driver to fully initialize its execution context before real network traffic arrives.
**2. Dynamic Graph Caching and Execution**
The core scheduling logic (`nrLDPC_decoder_core_dynamic`) bypasses standard stream-based kernel launches by utilizing a state-aware graph cache (`gpu_graph_cache`). When a decoding request arrives, the scheduler inspects the decoding parameters (e.g., $Z$, Code Rate, `n_segments`, maximum iterations) and routes the execution through one of three paths:
* **Cache Hit (Fast Path):** If an existing graph matches the current configuration, the CPU overhead is minimized to an $O(1)$ cache lookup. The system updates the input/output buffer pointers via a memory bridge (`ldpc_cuda_bridge_t`) and directly launches the pre-recorded graph (`nrLDPC_decoder_cuda_GraphExecute`).
* **Cache Miss (Graph Record):** If no matching graph exists and the cache is not full, the system records a new CUDA Graph (`nrLDPC_decoder_cuda_GraphRecord`). This operation captures the exact sequence of kernels—including the Edge/Node hybrid switching policy defined in Section 5.3—into a single executable topology. Once recorded, the graph is cached and executed.
* **Normal Execution (Fallback):** In scenarios where graph operations fail, the cache capacity is exhausted, or the runtime environment actively breaks the graph logic (circuit breaker triggered), the scheduler safely falls back to standard, sequential CUDA kernel launches (`nrLDPC_decoder_cuda_NormalExecute`).
**3. Memory Management**
In a fully GPU-accelerated L1 pipeline, the input LLRs from the preceding rate-matching module typically already reside in the GPU device memory. The decoder accepts these device pointers directly, avoiding PCIe transfer overhead.
To maintain API compatibility for partial-offload testing scenarios where inputs might originate from the host, the pipeline checks the memory space of the incoming pointer (`if (p_llr != p_llr_dev)`). It performs an asynchronous PCIe copy (`cudaMemcpyAsync`) only if a host pointer is detected. Additionally, for integrated architectures like the NVIDIA GH200, the implementation supports zero-copy memory access, mapping pageable or integrated host pointers directly without explicit data migration.
### 6.2 Throughput Analysis
#### 6.2.1 Error Correction Performance (BLER Validation)
To guarantee that the architectural transformations—including 8-bit saturation, SIMD4 vectorization, and Node-Based graph restructuring—do not compromise the mathematical integrity of the decoder, we benchmarked the Block Error Rate (BLER) against the standard OAI CPU baseline.
The BLER vs. SNR curves indicate that the CUDA GPU implementation performs consistently with the CPU baseline. Across the evaluated code rates (BG1 Rate 1/3, Rate 2/3, and Rate 8/9) in an AWGN channel, the decoding trajectories of the GPU and CPU implementations overlap. This demonstrates that the proposed GPU offloading strategies do not compromise the error correction performance.
![CPU_GPU_BLER](img/bler_performance_cpu_vs_gpu.svg)
*(Tested with 5 iterations)*
#### 6.2.2 Latency vs. Throughput Scalability
Following the validation of decoding accuracy, the hardware acceleration performance was evaluated on the NVIDIA GH200 architecture. The primary performance metrics include single-segment processing latency and overall decoding throughput, measured across varying workloads (number of code block segments, `n_segments`).
The measurements illustrate the scalability facilitated by the Dynamic Hybrid Switching policy detailed in Section 5.3. By calibrating the `NodeEdge_Switch` thresholds according to the Streaming Multiprocessor (SM) capacity of the GH200, the scheduler enables a transition across different operational regimes.
**Low-Latency Regime (Edge-Based Processing)**
For smaller batch sizes (e.g., `n_segments = 1` to `4`), the scheduler selects the Edge-Based kernels. In this regime, the concurrent thread deployment reduces the absolute decoding time. The implementation records a single-segment decoding latency ranging from approximately **50 µs** to **61 µs** (measured at 5 iterations). This processing time aligns with the latency requirements associated with 5G URLLC (Ultra-Reliable Low-Latency Communication) scenarios.
**High-Throughput Regime (Node-Based Processing)**
As the workload scales to simulate higher-load eMBB (Enhanced Mobile Broadband) base station conditions, the scheduler transitions to the Node-Based kernels. This transition mitigates SM saturation by reducing the algorithmic complexity from $O(N^2)$ to $O(N)$.
With the switching thresholds calibrated to the hardware characteristics, the throughput curve exhibits an initial ascent before plateauing near the hardware limit. Under maximum evaluated parallel segment loading (`n_segments = 128`), the GPU decoder reaches peak sustained throughputs of **~3.5 Gbps** for Rate 1/3, **~5.5 Gbps** for Rate 2/3, and **>8.0 Gbps** for Rate 8/9.
The table below summarizes the measured throughput and latency metrics across different segment batch sizes on the GH200:
<details>
<summary><b>Click to expand: Raw Performance Metrics (GH200)</b></summary>
| `n_segments` | R13 Latency (µs) | R13 Tput (Gbps) | R23 Latency (µs) | R23 Tput (Gbps) | R89 Latency (µs) | R89 Tput (Gbps) |
| --- | --- | --- | --- | --- | --- | --- |
| **1** | 61.66 | 0.137 | 54.04 | 0.156 | 49.79 | 0.170 |
| **2** | 69.98 | 0.241 | 54.62 | 0.309 | 51.45 | 0.328 |
| **4** | 78.24 | 0.432 | 58.65 | 0.576 | 55.17 | 0.612 |
| **8** | 91.52 | 0.738 | 68.89 | 0.981 | 56.89 | 1.188 |
| **16** | 116.76 | 1.157 | 81.95 | 1.649 | 72.64 | 1.861 |
| **32** | 157.88 | 1.712 | 105.75 | 2.556 | 81.60 | 3.313 |
| **64** | 197.88 | 2.732 | 124.73 | 4.335 | 106.36 | 5.083 |
| **128** | 303.47 | 3.563 | 195.77 | 5.523 | 134.42 | 8.044 |
*(Note: Throughput calculations are based on $K=8448$ information bits per segment. Values are measured at 5 decoding iterations.)*
</details>
---
![latency](img/latency_vs_segments.svg)
![throughput](img/throughput_vs_segments.svg)
---
<a id="Appendix-A"></a>
## Appendix A: Node-Edge Performance Profiling on GH200
To validate the bottleneck shift from kernel launch overhead to SM computational saturation, an exhaustive profiling was conducted on the NVIDIA Grace Hopper (GH200) platform.
The architectures evaluated are defined as follows:
* **EE (Baseline)**: Edge-based `cnProc` + Edge-based `bnProc`.
* **EN (Hybrid A)**: Edge-based `cnProc` + Node-based `bnProc`.
* **NE (Hybrid B)**: Node-based `cnProc` + Edge-based `bnProc`.
* **NN (Proposed)**: Node-based `cnProc` + Node-based `bnProc`.
The tables below record the per-segment execution time ($\mu s$) under varying workload scales ($S$).
### A.1 Performance Comparison for Rate 1/3 (BG1)
| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
| --- | --- | --- | --- | --- | --- |
| **1** | 52.055 | 76.164 | 75.641 | 98.361 | **EE** |
| **2** | 30.724 | 39.726 | 40.552 | 50.658 | **EE** |
| **4** | 17.457 | 20.873 | 22.259 | 25.478 | **EE** |
| **8** | 10.815 | 11.664 | 12.725 | 13.481 | **EE** |
| **16** | 8.038 | 7.080 | 8.468 | 7.485 | **EN** |
| **32** | 6.204 | 4.824 | 6.112 | 4.713 | **NN** |
| **64** | 5.251 | 3.461 | 4.884 | 3.100 | **NN** |
| **128** | 4.793 | 2.905 | 4.342 | 2.423 | **NN** |
### A.2 Performance Comparison for Rate 2/3 (BG1)
| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
| --- | --- | --- | --- | --- | --- |
| **1** | 46.088 | 54.576 | 67.666 | 74.247 | **EE** |
| **2** | 23.429 | 27.522 | 35.105 | 38.476 | **EE** |
| **4** | 12.763 | 14.673 | 18.467 | 20.560 | **EE** |
| **8** | 7.611 | 8.030 | 9.977 | 10.521 | **EN** |
| **16** | 4.791 | 4.721 | 5.588 | 5.491 | **EN** |
| **32** | 3.416 | 3.115 | 3.416 | 3.073 | **NN** |
| **64** | 2.670 | 2.267 | 2.375 | 1.953 | **NN** |
| **128** | 2.297 | 1.867 | 2.061 | 1.597 | **NN** |
### A.3 Performance Comparison for Rate 8/9 (BG1)
| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
| --- | --- | --- | --- | --- | --- |
| **1** | 44.686 | 48.255 | 68.518 | 70.156 | **EE** |
| **2** | 21.895 | 24.895 | 35.236 | 36.256 | **EE** |
| **4** | 11.493 | 13.221 | 17.876 | 19.112 | **EE** |
| **8** | 6.551 | 7.100 | 9.245 | 9.867 | **EE** |
| **16** | 3.824 | 4.142 | 5.123 | 5.257 | **EE** |
| **32** | 2.604 | 2.554 | 2.928 | 2.842 | **EN** |
| **64** | 1.896 | 1.751 | 1.759 | 1.646 | **NN** |
| **128** | 1.464 | 1.349 | 1.213 | 1.084 | **NN** |
---
**Analysis summary:** Compared to the baseline (EE), the EN architecture eases the computational burden of `bnProc`, and the NE architecture eases the burden of `cnProc`. As the workload ($S$) scales up, the performance improvement from EN is greater than that of NE. This indicates that the Variable Node processing (`bnProc`) acts as the main parallelization bottleneck in the traditional Edge-centric paradigm. Therefore, adopting the fully Node-centric architecture (NN) addresses both bottlenecks, providing better performance in high-load scenarios.
![R13 Performance](img/perf_r13.svg)
![R23 Performance](img/perf_r23.svg)
![R89 Performance](img/perf_r89.svg)
## Acknowledgments
* **Main Developer:** PAN Qizhi (GPU Architecture & CUDA Implementation & Optimization)
* **System Integration:** Raymond Knopp (E2E Integration & Validation & Optimization)
* **Special Thanks:** To **Jingguo Zhao (BUPT)** for the insightful discussions on node-based baseline architectures which inspired the optimizations in this work.

1186
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/nrLDPC_cuda_tmp.html Normal file
View File

File diff suppressed because it is too large Load Diff

190
openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/nrLDPC_cuda_tutorial.md Normal file
View File

@@ -0,0 +1,190 @@
# OAI nrLDPC GPU Decoder Tutorial
For the detailed mathematical modeling and memory layout of this GPU implementation, please refer to the [Architecture Document](nrLDPC_cuda.md).
## Supported Configurations
| Parameter | CPU Baseline | GPU Implementation |
| --- | --- | --- |
| **Base Graph (BG)** | 1, 2 | **1** |
| **Lifting Size ($Z_c$)** | All | **$\ge 128$** |
| **Code Rate (R)** | BG1: 1/3, 2/3, 8/9 <br><br> BG2: 1/5, 1/3, 2/3 | **BG1: 1/3, 2/3, 8/9** |
*Table of content*
[[_TOC_]]
## Prerequisites
### Tested System Configuration
The GPU decoder has been successfully compiled and verified using the following baseline hardware and software environment:
* **GPU:** NVIDIA GH200 480GB
* **NVIDIA Driver:** 575.57.08
* **CUDA Toolkit:** 12.9
* **CMake:** 3.22.1
* **GCC (C/C++ Compiler):** 12.3.0
**Other Validated Platforms**
Beyond the Grace Hopper architecture, the implementation has also been successfully validated across a range of NVIDIA ecosystems, including:
* NVIDIA L40S
* NVIDIA DGX Spark(GB10)
* NVIDIA Jetson
* NVIDIA A100
### Monitoring & Profiling
To examine runtime details, hardware utilization, and kernel execution efficiency, we recommend using the following official NVIDIA developer tools:
* **`nvidia-smi`**: Use this for real-time monitoring of VRAM usage, overall GPU utilization, and power consumption during the end-to-end tests.
* **`nsys` (Nsight Systems)**: Use this for system-wide performance analysis. It is highly effective for observing the CUDA Graph execution timeline and CPU-GPU pipeline scheduling behaviors.
* **`ncu` (Nsight Compute)**: Use this for deep, kernel-level profiling. It helps verify SM occupancy, shared memory usage, and the execution efficiency of the SIMD4 vectorized PTX instructions.
## Unit Test
### Build
Use the `build_oai` script to compile with CUDA support. If you have previously built a CPU-only version, ensure you clean the build directory or verify the dependencies:
```bash
./build_oai --ninja --phy_simulators --build-lib ldpc_cuda
```
```bash
cd ran_build/build/
```
### ldpctest
`ldpctest` is the foundational tool for evaluating the performance and correctness of the LDPC encoder/decoder.
**Basic Parameters:**
* `-r`: Nominator rate. Supported values: 1, 2, 22. (Default: 1)
* `-d`: Denominator rate. Supported values: 3, 5, 25. (Default: 3)
* `-n`: Number of trials (e.g., `-n300` runs 300 trials).
* `-S`: Number of segments (determines the workload size).
* `-s`: Starting SNR in dBm.
* `-i`: Maximum number of iterations. *(Note: Due to internal index offset implementation, the parameter `-i4` executes 5 decoding iterations.)*
* `-G`: **Enable GPU decoder**. (Omitting this parameter defaults to the CPU baseline).
#### Latency & Baseline Test
To compare the single-segment processing capability between CPU and GPU, set the segments parameter to 1 (`-S1`).
**BG1 Rate 1/3:**
* CPU: `./ldpctest -r1 -n300 -S1 -s4 -i4`
* GPU: `./ldpctest -r1 -n300 -S1 -s4 -i4 -G`
**BG1 Rate 2/3:**
* CPU: `./ldpctest -r2 -n300 -S1 -s4 -i4`
* GPU: `./ldpctest -r2 -n300 -S1 -s4 -i4 -G`
**BG1 Rate 8/9:**
* CPU: `./ldpctest -r22 -d25 -n300 -S1 -s4 -i4`
* GPU: `./ldpctest -r22 -d25 -n300 -S1 -s4 -i4 -G`
#### Maximum Throughput Test
To evaluate the maximum throughput of the GPU, increase the `-S` parameter to add parallel decoding workloads. The current GPU version in `ldpctest` supports a maximum of **128** segments.
Run the following commands to benchmark the maximum GPU throughput:
```bash
# BG1 R13 Max Throughput
./ldpctest -r1 -n30 -S128 -s4 -i4 -G
```
```bash
# BG1 R23 Max Throughput
./ldpctest -r2 -n30 -S128 -s4 -i4 -G
```
```bash
# BG1 R89 Max Throughput
./ldpctest -r22 -d25 -n30 -S128 -s4 -i4 -G
```
### Downlink Simulator (dlsim)
`nr_dlsim` is used to simulate the physical downlink shared channel (PDSCH). To offload LDPC decoding to the GPU, you must specify the CUDA shared library and enable the GPU flag.
#### Mandatory Flags for GPU Acceleration
* `--loader.ldpc.shlibversion _cuda`: Directs the dynamic loader to use the CUDA-accelerated LDPC library.
* `-Q` : Enables the GPU decoding path within the simulator logic.
#### Workload & Segment Calculation
Unlike `ldpctest` where you can manually set the number of segments, in `dlsim`, the number of segments (Code Blocks) is determined dynamically by the Transport Block Size (TBS). The TBS is defined by a combination of the MCS index (`-e`), the number of allocated PRBs (`-R`, `-b`), and the number of MIMO layers (`-x`). To specify the iterations, use `-I`.
Understanding the resulting segment count is crucial for evaluating GPU performance, as higher segment counts better utilize the GPU's parallel processing capabilities.
#### Execution Examples
**1. Target: ~ 9 Segments (Rate 1/3)**
Using MCS 13 with 273 PRBs and a single layer generates a workload of approximately 9 segments per slot.
```bash
./nr_dlsim -n100 -s20 -e13 -R273 -b273 -x1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
```
**2. Target: ~ 18 Segments (Rate 2/3)**
Increasing the MCS to 22 while keeping 273 PRBs and 1 layer doubles the throughput requirement, resulting in about 18 segments.
```bash
./nr_dlsim -n100 -s20 -e22 -R273 -b273 -x1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
```
**3. Target: ~ 50 Segments (Rate 8/9, High Workload)**
Using MCS 27 combined with 2x2 MIMO (`-x2 -y2 -z2`) forces a massive Transport Block Size, pushing the workload to approximately 50 segments. This scenario highly benefits from the Node-Based GPU architecture.
```bash
./nr_dlsim -n100 -s40 -e27 -R273 -b273 -z2 -x2 -y2 -P -Q --loader.ldpc.shlibversion _cuda
```
*(Note: The `-P` flag enables performance printing, allowing you to observe the decoding time per slot)*
### Uplink Simulator (ulsim)
`nr_ulsim` is used to simulate the physical uplink shared channel (PUSCH). The GPU acceleration logic follows the same architectural pattern as `dlsim`.
> **Parameter Mapping Notice:**
> When transitioning from `dlsim` to `ulsim`, the command-line arguments for identical physical layer parameters are different:
> * **MCS Index**: Use `-m` (instead of `-e`).
> * **Allocated PRBs**: Use `-r` (instead of `-b`).
> * **MIMO Layers**: Use `-W` (instead of `-x`).
To properly observe the GPU performance, we use the `-P` flag to print the execution statistics. After running a test, locate the `ULSCH segments decoding time` field in the `gNB RX` section of the output log. This metric isolates the time spent executing the LDPC decoding iterations.
#### Execution Examples
**1. Target: ~ 9 Segments (Rate 1/3)**
Using MCS 13 with 273 PRBs and a single layer.
```bash
./nr_ulsim -n100 -s20 -m13 -R273 -r273 -W1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
```
**2. Target: ~ 18 Segments (Rate 2/3)**
Increasing to MCS 22, maintaining 273 PRBs and a single layer.
```bash
./nr_ulsim -n100 -s20 -m22 -R273 -r273 -W1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
```
**3. Target: ~ 50 Segments (Rate 8/9, Massive Workload)**
Using MCS 27 with 273 PRBs and 2x2 MIMO (`-W2 -y2 -z2`).
```bash
./nr_ulsim -n100 -s40 -m27 -R273 -r273 -W2 -y2 -z2 -P -Q --loader.ldpc.shlibversion _cuda
```
## E2E Test
To be continued :）

1
openair1/PHY/CODING/nrLDPC_defs.h
View File

@@ -60,6 +60,7 @@ typedef struct {
} encoder_implemparams_t;
typedef int32_t(LDPC_initfunc_t)(void);
typedef int32_t(LDPC_shutdownfunc_t)(void);
// decoder interface

1
openair1/PHY/CODING/nrLDPC_load.c
View File

@@ -36,6 +36,7 @@ int load_LDPClib(char *version, ldpc_interface_t *itf)
int ret;
ret = load_module_version_shlib(libname, version, shlib_fdesc, sizeofArray(shlib_fdesc), NULL);
AssertFatal((ret >= 0), "Error loading ldpc decoder");
LOG_I(NR_PHY,"loading LDPC version %s\n",version);
itf->LDPCinit = (LDPC_initfunc_t *)shlib_fdesc[0].fptr;
itf->LDPCshutdown = (LDPC_shutdownfunc_t *)shlib_fdesc[1].fptr;
itf->LDPCdecoder = (LDPC_decoderfunc_t *)shlib_fdesc[2].fptr;

11
openair1/SIMULATION/NR_PHY/dlsim.c
View File

@@ -1056,9 +1056,11 @@ int main(int argc, char **argv)
reset_meas(&gNB->dlsch_encoding_stats);
reset_meas(&gNB->dci_generation_stats);
reset_meas(&gNB->tinput);
reset_meas(&gNB->tinput_memcpy);
reset_meas(&gNB->tprep);
reset_meas(&gNB->tparity);
reset_meas(&gNB->toutput);
reset_meas(&gNB->tconcat);
reset_meas(&gNB->phase_comp_stats);
uint32_t errors_scrambling[16] = {0};
@@ -1081,7 +1083,7 @@ int main(int argc, char **argv)
nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdsch_pdu = &dl_req->dl_tti_pdu_list[1];
nfapi_nr_dl_tti_pdsch_pdu_rel15_t *pdsch_pdu_rel15 = &dl_tti_pdsch_pdu->pdsch_pdu.pdsch_pdu_rel15;
for (trial = 0; trial < n_trials && !stop; trial++) {
for (trial = 0; trial < n_trials /*&& !stop)*/; trial++) {
errors_bit = 0;
//multipath channel
@@ -1101,7 +1103,7 @@ int main(int argc, char **argv)
UE_harq_process->DLround = round;
UE_harq_process->first_rx = 1;
while (round < num_rounds && !UE_harq_process->decodeResult && !stop) {
while (round < num_rounds && !UE_harq_process->decodeResult /*&& !stop*/) {
reset_sched_response(Sched_INFO, frame, slot, 0, 0);
clear_nr_nfapi_information(RC.nrmac[0], 0, frame, slot);
UE_info->UE_sched_ctrl.harq_processes[harq_pid].ndi = !(trial&1);
@@ -1431,11 +1433,14 @@ int main(int argc, char **argv)
fprintf(csv_file,"%.2f,%.4f,%.2f,%u\n", roundStats, effRate, effRate / (8 * pdsch_pdu_rel15->TBSize[0]) * 100, 8 * pdsch_pdu_rel15->TBSize[0]);
}
if (print_perf==1) {
printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, block %d)\n",
printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, Z %d, F %d, K %d, block %d)\n",
1000 >> *scc->ssbSubcarrierSpacing,
g_rbSize,
g_mcsIndex,
UE->dl_harq_processes[0][slot].C,
UE->dl_harq_processes[0][slot].Z,
UE->dl_harq_processes[0][slot].F,
UE->dl_harq_processes[0][slot].K,
8 * pdsch_pdu_rel15->TBSize[0]);
printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");

1
openair1/SIMULATION/NR_PHY/pucchsim.c
View File

@@ -38,7 +38,6 @@ PHY_VARS_NR_UE *UE;
RAN_CONTEXT_t RC;
int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
uint64_t downlink_frequency[MAX_NUM_CCs][4];
double cpuf;
uint8_t const nr_rv_round_map[4] = {0, 2, 3, 1};
// needed for some functions

1
openair1/SIMULATION/NR_PHY/ulsim.c
View File

@@ -1057,7 +1057,6 @@ int main(int argc, char *argv[])
for (i = 1; i < TBS/8; i++) {
ulsch_input_buffer[i] = (uint8_t)rand();
}
uint8_t ptrs_time_density = get_L_ptrs(ptrs_mcs1, ptrs_mcs2, ptrs_mcs3, Imcs, mcs_table);
uint8_t ptrs_freq_density = get_K_ptrs(n_rb0, n_rb1, nb_rb);