mirror of
https://gitlab.eurecom.fr/oai/openairinterface5g.git
synced 2026-07-14 21:20:30 +00:00
Compare commits
26 Commits
pre-commit
...
ldpc-tee-c
| Author | SHA1 | Date | |
|---|---|---|---|
|
|
2335bd69f5 | ||
|
|
fec4b82a42 | ||
|
|
ae410e33be | ||
|
|
199882bce4 | ||
|
|
6859fef39f | ||
|
|
def04a2f91 | ||
|
|
335ad7e93c | ||
|
|
567d4fd8dc | ||
|
|
34d4866c6e | ||
|
|
fbea67658e | ||
|
|
24b496adcb | ||
|
|
0c5cada22a | ||
|
|
a7ece54219 | ||
|
|
7e0568384f | ||
|
|
474afaf6d5 | ||
|
|
0215871d21 | ||
|
|
450831576f | ||
|
|
6459240c58 | ||
|
|
4ea1dc2ba5 | ||
|
|
c0e7da985a | ||
|
|
17c5cf1a13 | ||
|
|
9581cca17b | ||
|
|
5545d9dbae | ||
|
|
f8fa868482 | ||
|
|
49b566ecdd | ||
|
|
7d2afe7e46 |
@@ -5,8 +5,8 @@ project (OpenAirInterface LANGUAGES C CXX)
|
||||
set(CMAKE_CXX_STANDARD 17)
|
||||
set(OAI_VERSION 2.4.0)
|
||||
|
||||
option(CUDA_ENABLE "Enable CUDA accelerated channel simulation" OFF)
|
||||
if(CUDA_ENABLE)
|
||||
option(ENABLE_CHANNEL_SIM_CUDA "Enable CUDA accelerated channel simulation" OFF)
|
||||
if(ENABLE_CHANNEL_SIM_CUDA)
|
||||
find_package(CUDA REQUIRED)
|
||||
find_package(CUDAToolkit REQUIRED)
|
||||
message(STATUS "CUDA explicitly enabled, building with GPU acceleration support.")
|
||||
@@ -34,7 +34,7 @@ if(CUDA_ENABLE)
|
||||
message(STATUS "CUDA Explicit Copy path enabled (default ATS was overridden).")
|
||||
endif()
|
||||
|
||||
add_compile_definitions(ENABLE_CUDA)
|
||||
add_compile_definitions(CHANNEL_SIM_CUDA)
|
||||
endif()
|
||||
|
||||
#########################################################
|
||||
@@ -211,6 +211,10 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang
|
||||
set(commonOpts "${commonOpts} -Wno-unused-command-line-argument")
|
||||
endif()
|
||||
|
||||
if (ENABLE_CUDA)
|
||||
set(commonOpts "${commonOpts} -DENABLE_CUDA")
|
||||
endif()
|
||||
|
||||
set(CMAKE_C_FLAGS
|
||||
"${C_FLAGS_PROCESSOR} ${commonOpts} -std=gnu11 -funroll-loops ${CMAKE_C_FLAGS}")
|
||||
set(CMAKE_CXX_FLAGS
|
||||
@@ -1073,15 +1077,30 @@ target_link_libraries(PHY_UE PRIVATE asn1_lte_rrc_hdrs asn1_nr_rrc_hdrs UTIL)
|
||||
add_library(PHY_NR_COMMON ${PHY_NR_SRC_COMMON})
|
||||
target_link_libraries(PHY_NR_COMMON PUBLIC UTIL)
|
||||
|
||||
add_library(PHY_NR ${PHY_NR_SRC})
|
||||
if (ENABLE_CUDA)
|
||||
find_package(CUDA REQUIRED)
|
||||
if (NOT CUDA_FOUND)
|
||||
message(FATAL_ERROR "no CUDA found")
|
||||
endif()
|
||||
#SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_80 -g -ggdb") # A100
|
||||
#SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_90") # GH200
|
||||
#SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_87") # Jetson
|
||||
#SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_89") # L40S
|
||||
SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_121") # GB10
|
||||
set(CUDA_VERBOS_BUILD_ON)
|
||||
cuda_add_library(PHY_NR ${PHY_NR_SRC} ${PHY_NR_SRC_CU})
|
||||
cuda_add_library(PHY_NR_UE ${PHY_NR_UE_SRC} ${PHY_NR_UE_SRC_CU})
|
||||
else()
|
||||
add_library(PHY_NR ${PHY_NR_SRC})
|
||||
add_library(PHY_NR_UE ${PHY_NR_UE_SRC})
|
||||
endif()
|
||||
target_link_libraries(PHY_NR nr_phy_common nr_common nr_fapi_p5 polar smallblock ds)
|
||||
|
||||
add_library(PHY_NR_NO_AVX_256 ${PHY_NR_SRC})
|
||||
target_link_libraries(PHY_NR_NO_AVX_256 nr_phy_common nr_common)
|
||||
target_compile_definitions(PHY_NR_NO_AVX_256 PUBLIC USE_128BIT)
|
||||
|
||||
add_library(PHY_NR_UE ${PHY_NR_UE_SRC})
|
||||
target_link_libraries(PHY_NR_UE PRIVATE asn1_nr_rrc_hdrs nr_phy_common nr_common UTIL polar smallblock)
|
||||
target_link_libraries(PHY_NR_UE asn1_nr_rrc_hdrs nr_phy_common nr_common UTIL polar smallblock)
|
||||
|
||||
add_library(PHY_RU ${PHY_SRC_RU})
|
||||
target_link_libraries(PHY_RU PRIVATE asn1_lte_rrc_hdrs UTIL)
|
||||
@@ -1984,12 +2003,19 @@ add_executable(ldpctest
|
||||
)
|
||||
target_link_libraries(ldpctest PRIVATE
|
||||
-Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON -Wl,--end-group
|
||||
m pthread dl ${T_LIB}
|
||||
m pthread dl ${T_LIB} ${CUDA_LIBRARIES}
|
||||
# link 'check_crc' to make it resolved in the LDPC coding libraries
|
||||
# 'check_crc' is not used in ldpctest so it is not linked in the executable by default
|
||||
# --whole-archive links 'check_crc' in the executable even though it is note used, see 'man ld'
|
||||
-Wl,--whole-archive crc_byte -Wl,--no-whole-archive
|
||||
)
|
||||
if (ENABLE_CUDA)
|
||||
# ldpctest.c uses cudaHostAlloc(), so include dir/link libr to make available
|
||||
# TODO don't write it directly into ldpctest, but separate compilation unit?
|
||||
find_package(CUDA REQUIRED)
|
||||
target_link_libraries(ldpctest PRIVATE ${CUDA_LIBRARIES})
|
||||
target_include_directories(ldpctest PRIVATE ${CUDA_INCLUDE_DIRS})
|
||||
endif()
|
||||
|
||||
add_library(physim_common OBJECT ${OPENAIR1_DIR}/SIMULATION/NR_PHY/nr_unitary_common.c)
|
||||
target_link_libraries(physim_common PRIVATE UTIL)
|
||||
@@ -2076,14 +2102,12 @@ target_link_libraries(nr_srssim PRIVATE
|
||||
m pthread ITTI dl nr_ue_phy_meas physim_common softmodem_common
|
||||
)
|
||||
|
||||
if(CUDA_ENABLE)
|
||||
if (TARGET oai_cuda_lib)
|
||||
target_link_libraries(nr_dlsim PRIVATE oai_cuda_lib)
|
||||
target_link_libraries(nr_ulsim PRIVATE oai_cuda_lib)
|
||||
if(ENABLE_CHANNEL_SIM_CUDA)
|
||||
target_link_libraries(nr_dlsim PRIVATE oai_cuda_lib)
|
||||
target_link_libraries(nr_ulsim PRIVATE oai_cuda_lib)
|
||||
|
||||
target_compile_definitions(nr_dlsim PRIVATE ENABLE_CUDA)
|
||||
target_compile_definitions(nr_ulsim PRIVATE ENABLE_CUDA)
|
||||
endif()
|
||||
target_compile_definitions(nr_dlsim PRIVATE CHANNEL_SIM_CUDA)
|
||||
target_compile_definitions(nr_ulsim PRIVATE CHANNEL_SIM_CUDA)
|
||||
endif()
|
||||
|
||||
# these simulators do not compile:
|
||||
|
||||
@@ -75,7 +75,7 @@ def AnalyzeBuildLogs(image, lf):
|
||||
committed = committed or lineHasCommit
|
||||
if re.search(r'error:|Errors|ERROR', line):
|
||||
errors.append(f"=> {line.strip()}")
|
||||
status = (committed or tagged) and len(errors) == 0
|
||||
status = (committed or tagged)
|
||||
logging.info(f"Analyzing {image}, file {lf}: {status=}, {len(errors)} errors")
|
||||
for e in errors:
|
||||
logging.info(e)
|
||||
@@ -200,9 +200,20 @@ class Containerize():
|
||||
|
||||
dockerfileprefix = '.ubuntu'
|
||||
|
||||
baseImage = 'ran-base'
|
||||
baseTag = 'develop'
|
||||
buildImage = 'ran-build'
|
||||
forceBaseImageBuild = False
|
||||
imageTag = 'develop'
|
||||
|
||||
result = re.search('native_cuda_armv8', self.imageKind)
|
||||
if result is not None:
|
||||
baseImage = 'ran-base-cuda'
|
||||
buildImage = 'ran-build-cuda'
|
||||
dockerfileprefix = '.cuda.ubuntu'
|
||||
# we always build the ran-build image with all targets
|
||||
# Creating a tupple with the imageName, the DockerFile prefix pattern, targetName and sanitized option
|
||||
imageNames = [('ran-build', 'build', 'ran-build', '')]
|
||||
imageNames = [(buildImage, 'build', f'{buildImage}', '')]
|
||||
result = re.search('eNB', self.imageKind)
|
||||
if result is not None:
|
||||
imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
|
||||
@@ -242,13 +253,13 @@ class Containerize():
|
||||
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
|
||||
imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup', ''))
|
||||
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
|
||||
|
||||
result = re.search('native_cuda_armv8', self.imageKind)
|
||||
if result is not None:
|
||||
imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
|
||||
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
|
||||
|
||||
cmd.cd(lSourcePath)
|
||||
|
||||
baseImage = 'ran-base'
|
||||
baseTag = 'develop'
|
||||
forceBaseImageBuild = False
|
||||
imageTag = 'develop'
|
||||
if (self.ranAllowMerge):
|
||||
imageTag = 'ci-temp'
|
||||
if self.ranTargetBranch == 'develop':
|
||||
@@ -276,17 +287,17 @@ class Containerize():
|
||||
# On when the base image docker file is being modified.
|
||||
if forceBaseImageBuild:
|
||||
cmd.run(f"docker image rm {baseImage}:{baseTag}")
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log'
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/{baseImage}.docker.log'
|
||||
option = f" --build-arg UBUNTU_IMAGE={DEFAULT_REGISTRY}/{ubuntuImage}"
|
||||
cmd.run(f"docker build --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{dockerfileprefix} {option} . &> {logfile}", timeout=1600)
|
||||
t = ("ran-base", archiveArtifact(cmd, ctx, logfile))
|
||||
t = (baseImage, archiveArtifact(cmd, ctx, logfile))
|
||||
log_files.append(t)
|
||||
|
||||
# First verify if the base image was properly created.
|
||||
ret = cmd.run(f"docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
|
||||
|
||||
allImagesSize = {}
|
||||
if ret.returncode != 0:
|
||||
logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
|
||||
logging.error(f'\u001B[1m Could not build properly {baseImage}\u001B[0m')
|
||||
# Recover the name of the failed container?
|
||||
cmd.run(f"docker ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty docker rm -f")
|
||||
cmd.run(f"docker image prune --force")
|
||||
@@ -299,10 +310,10 @@ class Containerize():
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
logging.debug(f'\u001B[1m ran-base size is {imageSizeStr} Mbytes\u001B[0m')
|
||||
allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
|
||||
logging.debug(f'\u001B[1m {baseImage} size is {imageSizeStr} Mbytes\u001B[0m')
|
||||
allImagesSize[baseImage] = f'{imageSizeStr} Mbytes'
|
||||
else:
|
||||
logging.debug('ran-base size is unknown')
|
||||
logging.debug(f'{baseImage} size is unknown')
|
||||
|
||||
# Build the target image(s)
|
||||
status = True
|
||||
@@ -313,11 +324,11 @@ class Containerize():
|
||||
cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
# target images should use the proper ran-build image
|
||||
if image != 'ran-build' and "-asan" in name:
|
||||
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build-asan:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
cmd.run(f'sed -i -e "s#{buildImage}:latest#{buildImage}-asan:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
elif "fhi72" in name or name == "oai-nr-oru":
|
||||
cmd.run(f'sed -i -e "s#ran-build-fhi72:latest#ran-build-fhi72:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
elif image != 'ran-build':
|
||||
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
cmd.run(f'sed -i -e "s#{buildImage}:latest#{buildImage}:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
|
||||
if image == 'oai-gnb-aerial':
|
||||
cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2026.01.07.tar.gz .')
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log'
|
||||
|
||||
@@ -26,7 +26,7 @@ RUN cmake -GNinja -DENABLE_PHYSIM_TESTS=ON -DENABLE_TESTS=ON \
|
||||
-DSANITIZE_UNDEFINED=OFF -DSANITIZE_ADDRESS=OFF \
|
||||
-DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror \
|
||||
-DPHYSIM_CHECK_FILES="ThresholdsCuda.cmake" \
|
||||
-DCUDA_ENABLE=ON \
|
||||
-DENABLE_CHANNEL_SIM_CUDA=ON \
|
||||
-DUSE_UNIFIED_MEMORY=ON \
|
||||
-DUSE_ATS_MEMORY=OFF \
|
||||
-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/oai-ran/cmake_targets/ran_build/build \
|
||||
|
||||
@@ -26,7 +26,7 @@ RUN cmake -GNinja -DENABLE_PHYSIM_TESTS=ON -DENABLE_TESTS=ON \
|
||||
-DSANITIZE_UNDEFINED=OFF -DSANITIZE_ADDRESS=OFF \
|
||||
-DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror \
|
||||
-DPHYSIM_CHECK_FILES="ThresholdsCuda.cmake" \
|
||||
-DCUDA_ENABLE=ON \
|
||||
-DENABLE_CHANNEL_SIM_CUDA=ON \
|
||||
-DUSE_UNIFIED_MEMORY=ON \
|
||||
-DUSE_ATS_MEMORY=OFF \
|
||||
-DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/oai-ran/cmake_targets/ran_build/build \
|
||||
|
||||
@@ -22,4 +22,4 @@ WORKDIR /oai-ran
|
||||
COPY . .
|
||||
|
||||
WORKDIR /oai-ran/build
|
||||
RUN cmake -GNinja -DENABLE_TESTS=ON -DCUDA_ENABLE=ON .. && ninja tests
|
||||
RUN cmake -GNinja -DENABLE_TESTS=ON -DENABLE_CHANNEL_SIM_CUDA=ON .. && ninja tests
|
||||
|
||||
28
ci-scripts/xml_files/container_image_build_cuda_jetson.xml
Normal file
28
ci-scripts/xml_files/container_image_build_cuda_jetson.xml
Normal file
@@ -0,0 +1,28 @@
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CSSL-1.0 -->
|
||||
|
||||
<testCaseList>
|
||||
<htmlTabRef>build-cuda-tab</htmlTabRef>
|
||||
<htmlTabName>Build Container Images for ARM with CUDA</htmlTabName>
|
||||
<htmlTabIcon>wrench</htmlTabIcon>
|
||||
|
||||
<testCase>
|
||||
<class>Create_Workspace</class>
|
||||
<desc>Create new Workspace for server 0</desc>
|
||||
<node>jetson3-oai</node>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Build_Image</class>
|
||||
<desc>Build all Images</desc>
|
||||
<node>jetson3-oai</node>
|
||||
<kind>native_cuda_armv8</kind>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Push_Local_Registry</class>
|
||||
<desc>Push Images to Local Registry</desc>
|
||||
<node>jetson3-oai</node>
|
||||
<tag_prefix>armv8_cuda_</tag_prefix>
|
||||
</testCase>
|
||||
|
||||
</testCaseList>
|
||||
192
ci-scripts/xml_files/container_sa_b200_nrue_cuda_jetson.xml
Normal file
192
ci-scripts/xml_files/container_sa_b200_nrue_cuda_jetson.xml
Normal file
@@ -0,0 +1,192 @@
|
||||
<!-- SPDX-License-Identifier: LicenseRef-CSSL-1.0 -->
|
||||
|
||||
<testCaseList>
|
||||
<htmlTabRef>JETSON-5G-NRUE-CUDA</htmlTabRef>
|
||||
<htmlTabName>30 MHz TDD SA JETSON with CUDA</htmlTabName>
|
||||
<htmlTabIcon>tasks</htmlTabIcon>
|
||||
<testCase>
|
||||
<class>DeployCoreNetwork</class>
|
||||
<desc>Initialize 5G Core</desc>
|
||||
<cn_id>oc-cn5g-00104</cn_id>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Pull_Local_Registry</class>
|
||||
<desc>Pull Images from Local Registry</desc>
|
||||
<node>jetson2-oai</node>
|
||||
<images>oai-nr-ue</images>
|
||||
<tag_prefix>armv8_cuda_</tag_prefix>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Pull_Local_Registry</class>
|
||||
<desc>Pull Images from Local Registry</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<images>oai-gnb-aerial</images>
|
||||
<tag_prefix>arm_</tag_prefix>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Create_Workspace</class>
|
||||
<desc>Create new Workspace</desc>
|
||||
<node>jetson2-oai</node>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Create_Workspace</class>
|
||||
<desc>Create new Workspace</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Custom_Script</class>
|
||||
<desc>Configure 30 MHz bandwidth on WNC RU</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<script>scripts/set-wnc-bandwidth.sh</script>
|
||||
<args>30</args>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Deploy_Object</class>
|
||||
<desc>Deploy PNF/Nvidia CUBB in a container</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
|
||||
<services>nv-cubb</services>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Deploy_Object</class>
|
||||
<desc>Deploy VNF in a container</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
|
||||
<services>oai-gnb-aerial</services>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Deploy_Object</class>
|
||||
<desc>Deploy nrUE (TDD/Band78/40MHz/N310) in a container</desc>
|
||||
<yaml_path>ci-scripts/yaml_files/sa_b200_jetson_cuda_nrue</yaml_path>
|
||||
<node>jetson2-oai</node>
|
||||
<num_attempts>3</num_attempts>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Attach_UE</class>
|
||||
<desc>Attach OAIUE</desc>
|
||||
<id>oai_ue_jetson2</id>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Custom_Command</class>
|
||||
<desc>Configure IP route</desc>
|
||||
<node>jetson2-oai</node>
|
||||
<command>sudo ip route add 172.21.6.105 via 13.1.1.100 dev oaitun_ue1</command>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Ping</class>
|
||||
<desc>Ping: 20pings in 20sec</desc>
|
||||
<id>oai_ue_jetson2</id>
|
||||
<svr_id>oc-cn5g-00104</svr_id>
|
||||
<ping_args>-c 20</ping_args>
|
||||
<ping_packetloss_threshold>1</ping_packetloss_threshold>
|
||||
<ping_rttavg_threshold>25</ping_rttavg_threshold>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Iperf</class>
|
||||
<desc>iperf (UL/10Mbps/UDP)(30 sec)</desc>
|
||||
<iperf_args>-u -b 10M -t 30 -i 1</iperf_args>
|
||||
<id>oai_ue_jetson2</id>
|
||||
<svr_id>oc-cn5g-00104</svr_id>
|
||||
<iperf_packetloss_threshold>25</iperf_packetloss_threshold>
|
||||
<iperf_bitrate_threshold>80</iperf_bitrate_threshold>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Iperf</class>
|
||||
<desc>iperf (DL/10Mbps/UDP)(30 sec)</desc>
|
||||
<iperf_args>-u -b 10M -t 30 -i 1 -R</iperf_args>
|
||||
<id>oai_ue_jetson2</id>
|
||||
<svr_id>oc-cn5g-00104</svr_id>
|
||||
<iperf_packetloss_threshold>25</iperf_packetloss_threshold>
|
||||
<iperf_bitrate_threshold>80</iperf_bitrate_threshold>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Iperf</class>
|
||||
<desc>iperf (UL/30Mbps/UDP)(30 sec)</desc>
|
||||
<may_fail>true</may_fail>
|
||||
<iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
|
||||
<id>oai_ue_jetson2</id>
|
||||
<svr_id>oc-cn5g-00104</svr_id>
|
||||
<iperf_packetloss_threshold>25</iperf_packetloss_threshold>
|
||||
<iperf_bitrate_threshold>80</iperf_bitrate_threshold>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Iperf</class>
|
||||
<desc>iperf (DL/30Mbps/UDP)(30 sec)</desc>
|
||||
<may_fail>true</may_fail>
|
||||
<iperf_args>-u -b 30M -t 30 -i 1 -R</iperf_args>
|
||||
<id>oai_ue_jetson2</id>
|
||||
<svr_id>oc-cn5g-00104</svr_id>
|
||||
<iperf_packetloss_threshold>25</iperf_packetloss_threshold>
|
||||
<iperf_bitrate_threshold>80</iperf_bitrate_threshold>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Undeploy_Object</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Undeploy nr UE</desc>
|
||||
<node>jetson2-oai</node>
|
||||
<yaml_path>ci-scripts/yaml_files/sa_b200_jetson_cuda_nrue</yaml_path>
|
||||
<analysis>
|
||||
<services>oai-nr-ue</services>
|
||||
</analysis>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Undeploy_Object</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Undeploy gNB</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
|
||||
<analysis>
|
||||
<services>oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye</services>
|
||||
</analysis>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Custom_Script</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Configure 100 MHz bandwidth on WNC RU</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<script>scripts/set-wnc-bandwidth.sh</script>
|
||||
<args>100</args>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Clean_Test_Server_Images</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Clean Test Images on Test Server</desc>
|
||||
<node>gracehopper1-oai</node>
|
||||
<images>oai-gnb-aerial</images>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>Clean_Test_Server_Images</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Clean Test Images on Test Server</desc>
|
||||
<node>jetson2-oai</node>
|
||||
<images>oai-nr-ue</images>
|
||||
</testCase>
|
||||
|
||||
<testCase>
|
||||
<class>UndeployCoreNetwork</class>
|
||||
<always_exec>true</always_exec>
|
||||
<desc>Terminate 5G Core</desc>
|
||||
<cn_id>oc-cn5g-00104</cn_id>
|
||||
</testCase>
|
||||
|
||||
</testCaseList>
|
||||
@@ -0,0 +1,40 @@
|
||||
# SPDX-License-Identifier: MIT
|
||||
|
||||
services:
|
||||
oai-nr-ue:
|
||||
image: ${REGISTRY-oaisoftwarealliance/}${NRUE_IMG:-oai-nr-ue}:${TAG:-develop}
|
||||
network_mode: host
|
||||
runtime: nvidia
|
||||
container_name: oai-nr-ue
|
||||
cap_drop:
|
||||
- ALL
|
||||
cap_add:
|
||||
- NET_ADMIN # for interface bringup
|
||||
- NET_RAW # for ping
|
||||
- SYS_NICE # for performance
|
||||
- IPC_LOCK # for memory locking
|
||||
devices:
|
||||
- /dev/net/tun:/dev/net/tun
|
||||
- /dev/bus/usb/:/dev/bus/usb/
|
||||
ulimits:
|
||||
core: -1 # for core dumps
|
||||
environment:
|
||||
TZ: Europe/Paris
|
||||
USE_B2XX: 'yes'
|
||||
USE_ADDITIONAL_OPTIONS: --usrp-args type=b200 -C 3350010000 -r 78 --numerology 1 --band 78 --ssb 345 --ue-fo-compensation --uicc0.imsi 001040000000001 --uicc0.pdu_sessions.[0].dnn oai.ipv4 --thread-pool 5,6,7,8,9 --ue-txgain 15 --ue-rxgain 90 --loader.ldpc.shlibversion _cuda --use_gpu 1 -A 60
|
||||
cpuset: "3-11"
|
||||
volumes:
|
||||
- ../../conf_files/ue.sa.conf:/opt/oai-nr-ue/etc/nr-ue.conf
|
||||
healthcheck:
|
||||
test: /bin/bash -c "pgrep nr-uesoftmodem"
|
||||
interval: 10s
|
||||
timeout: 5s
|
||||
retries: 5
|
||||
deploy:
|
||||
resources:
|
||||
reservations:
|
||||
devices:
|
||||
- driver: nvidia
|
||||
count: all
|
||||
capabilities: [gpu]
|
||||
|
||||
@@ -18,7 +18,7 @@ HWs=""
|
||||
BUILD_DIR=ran_build
|
||||
CMAKE_BUILD_TYPE="RelWithDebInfo"
|
||||
CMAKE_CMD="$CMAKE"
|
||||
OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal ldpc_xdma websrv oai_iqplayer imscope imscope_record"
|
||||
OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal ldpc_cuda ldpc_xdma ldpc_tee websrv oai_iqplayer imscope imscope_record"
|
||||
TARGET_LIST=""
|
||||
BUILD_TOOL_OPT="-j$(nproc)"
|
||||
|
||||
@@ -114,6 +114,8 @@ Options:
|
||||
CC=/usr/bin/clang CXX=/usr/bin/clang++ ./build_oai ... --sanitize-memory
|
||||
--sanitize-thread | -fsanitize=thread
|
||||
Enable the thread sanitizer on all targets
|
||||
--use-cuda
|
||||
Tell build to use NVCC for appropriate .cu files and cuda_runtime functions
|
||||
-h | --help
|
||||
Print this help"
|
||||
}
|
||||
@@ -355,6 +357,9 @@ function main() {
|
||||
--sanitize-thread | -fsanitize=thread)
|
||||
CMAKE_CMD="$CMAKE_CMD -DSANITIZE_THREAD=True"
|
||||
shift;;
|
||||
--use-cuda)
|
||||
CMAKE_CMD="$CMAKE_CMD -DENABLE_CUDA=True"
|
||||
shift;;
|
||||
--trace-asn1c-enc-dec)
|
||||
CMAKE_CMD="$CMAKE_CMD -DTRACE_ASN1C_ENC_DEC=ON"
|
||||
echo_info "Enabling asn1c internal traces via OAI logging system"
|
||||
|
||||
@@ -62,7 +62,7 @@ extern "C" {
|
||||
#endif
|
||||
|
||||
#ifndef malloc16
|
||||
# define malloc16(x) memalign(32,x+32)
|
||||
# define malloc16(x) memalign(64,x+64)
|
||||
#endif
|
||||
#define free16(y,x) free(y)
|
||||
#define openair_free(y,x) free((y))
|
||||
@@ -76,7 +76,7 @@ extern "C" {
|
||||
} while (0)
|
||||
|
||||
static inline void *malloc16_clear( size_t size ) {
|
||||
void *ptr = memalign(32, size+32);
|
||||
void *ptr = memalign(64, size+64);
|
||||
DevAssert(ptr);
|
||||
memset( ptr, 0, size );
|
||||
return ptr;
|
||||
@@ -110,7 +110,7 @@ static inline void *malloc_or_fail(size_t size)
|
||||
# define msg(aRGS...) LOG_D(PHY, ##aRGS)
|
||||
#endif
|
||||
#ifndef malloc16
|
||||
# define malloc16(x) memalign(32,x)
|
||||
# define malloc16(x) memalign(64,x)
|
||||
#endif
|
||||
|
||||
#define free16(y,x) free(y)
|
||||
|
||||
418
doc/NR_SA_Tutorial_OAI_nrUE_GPU_offload.md
Normal file
418
doc/NR_SA_Tutorial_OAI_nrUE_GPU_offload.md
Normal file
@@ -0,0 +1,418 @@
|
||||
# OAI 5G NR SA tutorial with OAI nrUE with GPU Offload
|
||||
|
||||
This tutorial explains how OAI nrUE can use GPU to offload LDPC decoder. In the future we might offload more functions on the GPU.
|
||||
|
||||
**Table of Contents**
|
||||
|
||||
[[_TOC_]]
|
||||
|
||||
## Current Status
|
||||
|
||||
The nrUE GPU offload functionality have been tested on below platforms:
|
||||
|
||||
|Platform |Version | OS/Linux Kernel |
|
||||
|---------------|------------|---------------------------------------|
|
||||
|Jetson AGX Orin|L4T 36.4.7 | U22/5.15.148-tegra (64K page variant) |
|
||||
|DGX Spark |7.4.0 | U24/6.14.0-1013-nvidia-64k |
|
||||
|
||||
The above versions are tested versions, you are free to try on newer versions as well.
|
||||
|
||||
You can use another UE platform with GPU as well but it is preferred to use unified memory GPU variant.
|
||||
|
||||
### KPIs
|
||||
|
||||
The below throughput numbers are similar for TCP/UDP, the gNB was configured for 5 ms TDD pattern (6D3U and Special slot 10D0U) for Nvidia Aerial Testbed.
|
||||
|
||||
The UE was in SISO mode.
|
||||
|
||||
Testbed Architecture:
|
||||
|
||||
UE <--> Over the Air 1.5m to 2m distance <--> WNC RU <--> gNB/DU server
|
||||
|
||||
| Platform | Radio | Bandwidth | DL Throughput | UL Throughput |
|
||||
| ----------- | --------- | --------- | ------------- | ------------- |
|
||||
| Jetson Orin | B210 | 10 MHz | 12 Mbps | 7.5 Mbps |
|
||||
| Jetson Orin | B210 | 20 MHz | 20 Mbps | 9.5 Mbps |
|
||||
| Jetson Orin | B210 | 30 MHz | 61 Mbps | 33 Mbps |
|
||||
| Jetson Orin | B210 | 40 MHz | 69 Mbps | 46 Mbps |
|
||||
| DGX Spark | B210 | 40 MHz | 86 Mbps | 46 Mbps |
|
||||
| DGX Spark | N310/x410 | 100 MHz | 231 Mbps | 118 Mbps |
|
||||
|
||||
Stability or long running test with peak throughput for nrUE was performed for 5hrs.
|
||||
|
||||
All the numbers are with LDPC GPU offload.
|
||||
|
||||
`NOTE`: At the time of writing we only tested the UE in SISO.
|
||||
We are working supporting multi antenna use case.
|
||||
|
||||
## Aerial Testbed Configuration
|
||||
|
||||
If you are using aerial testbed with OAI then depending on the bandwidth you want to use, you will have to adapt the value of `ds_offset_dl` in cuphycontroller configuration file.
|
||||
|
||||
For our testing environment we adapted the `ds_offset_dl` values as below:
|
||||
|
||||
| Bandwidth | ds_offset_dl |
|
||||
| ----------- | -------------|
|
||||
| 10 MHz | 14 |
|
||||
| 20 MHz | 12 |
|
||||
| 30 MHz | 10 |
|
||||
| 40 MHz | 9 |
|
||||
| 100 MHz | 6 |
|
||||
|
||||
On the WNC radio we have the below configuration:
|
||||
|
||||
```
|
||||
transmit-power 24
|
||||
transmit-power-scale 0.0
|
||||
compress tx static bfp iq-bitwidth 9
|
||||
compress rx static bfp iq-bitwidth 9
|
||||
compress prach static bfp iq-bitwidth 9
|
||||
```
|
||||
|
||||
You can adjust the scaling on the RU or you can adjust it in aerial configuration file.
|
||||
|
||||
**NOTE**: You might have to adjust these values for your environment, so that the UE can detect the cell and perform random-access.
|
||||
|
||||
## Configure the nrUE Hardware Platform
|
||||
|
||||
### Configure Jetson AGX Orin
|
||||
|
||||
At the time of writing this tutorial AGX Orin only had Jetpack 6.X release which has U22/5.15.148-tegra, for GPU offload on ARM systems it is preferred to use 64K page size kernel.
|
||||
|
||||
To change the kernel to 64K page size variant you will have to manually build the kernel. Probably for Jetpack 7.X it will be easier. Before building the kernel make sure AGX Orin has the up to date kernel for your current distribution.
|
||||
|
||||
NVIDIA-SMI Version: 540.4.0
|
||||
CUDA Version: 12.6
|
||||
CPU: Cortex-A78AE (12)
|
||||
OS: U22
|
||||
Kernel: 5.15.148-tegra
|
||||
NVCC: Build cuda_12.6.r12.6/compiler.34714021_0
|
||||
UHD: UHD_4.9.0.HEAD-0-g006d7f76 (v4.9.0.0)
|
||||
|
||||
#### Update kernel to 64K page size
|
||||
|
||||
Login to Jetson AGX Orin
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt upgrade
|
||||
# reboot
|
||||
```
|
||||
|
||||
**CAUTION**: While building the kernel if you make a mistake you have to flash again the Jetson. We are just providing these steps to help you build the kernel with 64K page size. If you have questions or issues building the kernel you can check the [official nvidia documentation](https://docs.nvidia.com/jetson/archives/r36.4.4/DeveloperGuide/SD/Kernel/KernelCustomization.html#sd-kernel-kernelcustomization)
|
||||
|
||||
prepare the kernel source code:
|
||||
|
||||
```bash
|
||||
sudo apt install git-core build-essential bc
|
||||
cd ~/
|
||||
wget https://developer.nvidia.com/downloads/embedded/l4t/r36_release_v4.4/sources/public_sources.tbz2
|
||||
# Build Jetson Linux Kernel
|
||||
tar -xf public_sources.tbz2
|
||||
cd ~/Linux_for_Tegra/source
|
||||
tar -xf kernel_src.tbz2
|
||||
tar -xf kernel_oot_modules_src.tbz2
|
||||
tar -xf nvidia_kernel_display_driver_source.tbz2
|
||||
```
|
||||
|
||||
the default kernel configuration file used is `kernel/kernel-jammy-src/arch/arm64/configs/defconfig` it is mentioned in `kernel_src_build_env.sh`.
|
||||
Add `CONFIG_ARM64_64K_PAGES=y` in `kernel/kernel-jammy-src/arch/arm64/configs/defconfig`.
|
||||
|
||||
Build the kernel, out-of-tree modules and dtbs:
|
||||
|
||||
```bash
|
||||
make -C kernel
|
||||
sudo -E make install -C kernel
|
||||
cp kernel/kernel-jammy-src/arch/arm64/boot/Image
|
||||
/boot/Image
|
||||
# Building the NVIDIA Out-of-Tree Modules
|
||||
cd ~/Linux_for_Tegra/source
|
||||
make modules
|
||||
sudo -E make modules_install
|
||||
sudo nv-update-initrd
|
||||
make dtbs
|
||||
cp kernel-devicetree/generic-dts/dtbs/* /boot/dtbs/
|
||||
```
|
||||
|
||||
#### Configure operating system
|
||||
|
||||
```bash
|
||||
jetson_clocks
|
||||
touch /etc/sysctl.d/rt.conf
|
||||
## add below content using vim/vi/nano
|
||||
kernel.sched_rt_runtime_us=-1
|
||||
kernel.timer_migration=0
|
||||
sudo cpufreq-set -c 0 -g performance
|
||||
sudo cpufreq-set -c 5 -g performance
|
||||
sudo cpufreq-set -c 9 -g performance
|
||||
```
|
||||
|
||||
### Configure DGX Spark
|
||||
|
||||
To setup DGX spark you would need to follow the below steps
|
||||
|
||||
NVIDIA-SMI Version: 580.95.05
|
||||
CUDA Version: 13.0
|
||||
OS: Ubuntu 24
|
||||
Kernel: 6.14.0-1013-nvidia-64k (You can use a higher version)
|
||||
NVCC: Build cuda_13.0.r13.0/compiler.36424714_0
|
||||
UHD: UHD_4.9.0.HEAD-0-g006d7f76 (v4.9.0.0)
|
||||
Efficient cores (Cortex-A725): 0,1,2,3,4,10,11,12,13,14
|
||||
Performance cores (Cortex-X925): 5,6,7,8,9,15,16,17,18,19
|
||||
|
||||
You can use `sudo cpupower -c 0-19 frequency-info` to check the above core distribution.
|
||||
|
||||
|
||||
```bash
|
||||
sudo apt update
|
||||
sudo apt upgrade
|
||||
#reboot
|
||||
sudo /etc/apt/apt.conf.d/20auto-upgrades
|
||||
APT::Periodic::Update-Package-Lists "0";
|
||||
APT::Periodic::Unattended-Upgrade "0";
|
||||
# Install 64K kernel corresponding to your running kernel
|
||||
sudo apt install linux-image-6.14.0-1013-nvidia-64k
|
||||
# You can also fix the kernel so it is not changed with apt upgrade
|
||||
sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT="Advanced options for DGX OS
|
||||
GNU\/Linux>DGX OS GNU\/Linux, with Linux 6.14.0-1013-nvidia-64k"/'
|
||||
/etc/default/grub
|
||||
#Configure the command line to isolate the cores
|
||||
cat <<"EOF" | sudo tee /etc/default/grub.d/cmdline.cfg
|
||||
GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX pci=realloc=off pci=pcie_bus_safe
|
||||
default_hugepagesz=512M hugepagesz=512M hugepages=24 tsc=reliable
|
||||
processor.max_cstate=0 audit=0 idle=poll rcu_nocb_poll nosoftlockup
|
||||
irqaffinity=0-3 isolcpus=managed_irq,domain,4-19
|
||||
nohz_full=4-19 rcu_nocbs=4-19 earlycon module_blacklist=nouveau
|
||||
acpi_power_meter.force_cap_on=y numa_balancing=disable init_on_alloc=0
|
||||
preempt=none ras=off iommu=off mitgations=off kpti=off skew_tick=1 nowatchdog
|
||||
mce=ignore_ce transparent_hugepage=never"
|
||||
EOF
|
||||
touch /etc/sysctl.d/rt.conf
|
||||
## add below content using vim/vi/nano
|
||||
kernel.sched_rt_runtime_us=-1
|
||||
kernel.timer_migration=0
|
||||
sudo update-grub
|
||||
sudo reboot
|
||||
# after reboot you can make a script to set the gpu clock speed to maximum
|
||||
nvidia-smi -lgc 2000
|
||||
sudo cpupower frequency-set --governor performance
|
||||
sudo cpupower idle-set -D 0
|
||||
```
|
||||
|
||||
### Configuration for Ethernet based USRP
|
||||
|
||||
If you want to use Ethernet based USRP like N3XX/X3XX/X4XX you would need to run the below commands:
|
||||
|
||||
```bash
|
||||
sudo sysctl -w net.core.wmem_max=62500000
|
||||
sudo sysctl -w net.core.rmem_max=62500000
|
||||
sudo sysctl -w net.core.wmem_default=62500000
|
||||
sudo sysctl -w net.core.rmem_default=62500000
|
||||
sudo ethtool -G <usrp-interface> tx <maximum-value> rx <maximum-value>
|
||||
```
|
||||
|
||||
*NOTE*: Currently we finalizing container images for OAI-NR-UE with GPU offload, we will provide it soon for container based deployments.
|
||||
|
||||
## Using nrUE with B2XX
|
||||
|
||||
### Build OAI-nrUE
|
||||
|
||||
```bash
|
||||
mkdir ~/oai-nr-ue
|
||||
git clone https://gitlab.eurecom.fr/oai/openairinterface5g.git ~/oai-nr-ue
|
||||
cd ~/oai-nr-ue
|
||||
git checkout ldpc_offload_cuda
|
||||
cd ~/oai-nr-ue/openairinterface5g/cmake_targets
|
||||
#Install third-party dependencies
|
||||
./build_oai -I -w USRP
|
||||
# Install nrue with USRP support, default nvcc location is /usr/local/bin/nvcc
|
||||
./build_oai --ninja --build-lib ldpc_cuda --gNB --nrUE -w USRP --use-cuda -C --cmake-opt -DCMAKE_CUDA_COMPILER=<nvcc-location>
|
||||
```
|
||||
|
||||
## Start the nrUE process
|
||||
|
||||
Create a UE configuration file based on the PLMN, slicing information, DNN and DNN type you have configured for the UE in the core network.
|
||||
|
||||
```bash
|
||||
cd ~/oai-nr-ue/
|
||||
# touch nr-ue.conf
|
||||
```
|
||||
|
||||
Sample UE configuration file, you can use the below configuration file and adjust the values as per your testing environment:
|
||||
|
||||
```bash
|
||||
cat nr-ue.conf
|
||||
|
||||
uicc0 = {
|
||||
imsi = "001010000000001";
|
||||
key = "fec86ba6eb707ed08905757b1bb44b8f";
|
||||
opc= "C42449363BBAD02B66D16BC975D77CC1";
|
||||
pdu_sessions = ({ dnn = "oai"; nssai_sst = 1; });
|
||||
}
|
||||
```
|
||||
|
||||
To start the `nr-uesoftmodem` you would need to know the center frequency, bandwidth, band and ssb. This information is printed in the OAI-gNB logs. You can search for the line `[NR_MAC] Command line parameters for OAI UE` for example:
|
||||
|
||||
```bash
|
||||
[NR_MAC] Command line parameters for OAI UE: -C 3350010000 -r 106 --numerology 1 --band 78 --ssb 513
|
||||
```
|
||||
|
||||
Once you have this information you can start the `nr-uesoftmodem` process:
|
||||
|
||||
**Command for Jetson AGX Orin**:
|
||||
|
||||
**MANDATORY**: Before starting the UE process it is important to configure the gNB configuration file with `min_rxtxtime` as 6 or higher integer value.
|
||||
|
||||
```bash
|
||||
cd ~/oai-nr-ue/openairinterface5g/cmake_targets/
|
||||
sudo ./nr-uesoftmodem -O ~/oai-nr-ue/nr-ue.conf -C 3350010000 -r 106 --ssb 513 --numerology 1 --band 78 --thread-pool -1 --num-dl-actors 6 --num-ul-actors 2 -E --loader.ldpc.shlibversion _cuda --usrp-tx-thread-config 1 --use_gpu 1 --ue-fo-compensation
|
||||
```
|
||||
|
||||
**Command for DGX Spark**:
|
||||
|
||||
**MANDATORY**: Before starting the UE process it is important to configure the gNB configuration file with `min_rxtxtime` as 6 or higher integer value.
|
||||
|
||||
```bash
|
||||
cd ~/oai-nr-ue/openairinterface5g/cmake_targets/
|
||||
|
||||
sudo taskset -c "5-9,15-19" ./nr-uesoftmodem -O ~/oai-nr-ue/nr-ue.conf -C 3350010000 -r 106 --ssb 513 --numerology 1 --band 78 --thread-pool -1 -E --loader.ldpc.shlibversion _cuda --use_gpu 1 --ue-fo-compensation
|
||||
```
|
||||
|
||||
In the above command task-set allows running the UE only on the performance cores.
|
||||
|
||||
Understand the command which you just used:
|
||||
|
||||
The main RF/cell settings are:
|
||||
|
||||
- `-O`: load the UE config file.
|
||||
- `-C 3900000000`: set DL carrier to 3.9 GHz.
|
||||
- `-r 106`: use 106 PRBs
|
||||
- `--numerology 1`: numerology 1, meaning 30 kHz subcarrier spacing.
|
||||
- `--band 77`: NR band n77.
|
||||
- `--ssb 516`: expect the SSB to start at sub-carrier 516.
|
||||
- `--ue-fo-compensation`: enable initial frequency-offset compensation, useful OTA when clocks are not perfectly aligned.
|
||||
- `--usrp-tx-thread-config 1`: enable an extra USRP TX processing thread
|
||||
- `-E`: enable threequarter_fs, meaning the UE uses three-quarter sampling rate mode. In OAI this reduces sample rate / transport load where supported.
|
||||
|
||||
The parallelism/offload settings are:
|
||||
|
||||
- `--thread-pool -1`: create a thread-pool worker with no CPU affinity.
|
||||
- `--num-dl-actors 6`: use 6 DL actor threads, parallel RX/PDSCH processing workers.
|
||||
- `--num-ul-actors 2`: use 2 UL actor threads, parallel TX/PUSCH/PUCCH preparation and transmit workers.
|
||||
- `--loader.ldpc.shlibversion_cuda`: load the CUDA-backed LDPC shared library variant.
|
||||
- `--use_gpu 1`: enable GPU offload in the PHY/LDPC path, if you want to disable it and use just CPU then replace 1 with 0.
|
||||
|
||||
When the UE starts it starts with highest gains, but sometimes you would need to adjust the tx/rx gains based on the UEs position from the RU/gNB. To manipulate the gains you can use the below options:
|
||||
|
||||
- `--ue-rxgain x`: set UE RX gain.
|
||||
- `--ue-txgain x`: set UE TX gain.
|
||||
- `-A 50`: apply a command-line sample advance of 50 samples to compensate RF timing offset.
|
||||
|
||||
Once the UE connects you will see the tunnel interface `oaitun_ue1` with ip-address in the subnet which is configured at SMF. You can open another terminal and from there you can do traffic test.
|
||||
|
||||
## Using nrUE with X3XX/N3XX/x4XX
|
||||
|
||||
### Build OAI-nrUE
|
||||
|
||||
For Ethernet based USRPs it is recommended to use DPDK mostly when you want to test for 60-100 MHz bandwidth. If you don't want to use DPDK then you can build the `nr-uesoftmodem` using the previous section.
|
||||
|
||||
```bash
|
||||
mkdir ~/oai-nr-ue
|
||||
git clone https://gitlab.eurecom.fr/oai/openairinterface5g.git ~/oai-nr-ue
|
||||
cd ~/oai-nr-ue
|
||||
git checkout ldpc_offload_cuda
|
||||
cd ~/oai-nr-ue/openairinterface5g/cmake_targets
|
||||
#Install third-party dependencies
|
||||
./build_oai -I
|
||||
#Build and Install DPDK
|
||||
cd ~/
|
||||
wget https://fast.dpdk.org/rel/dpdk-24.11.4.tar.xz
|
||||
sudo apt install wget xz-utils libnuma-dev meson libibverbs-dev rdma-core python3-pyelftools
|
||||
meson setup build -Dplatform=generic
|
||||
ninja -C build
|
||||
sudo ninja install -C build
|
||||
#Build and Install UHD
|
||||
sudo apt install autoconf automake build-essential ccache cmake cpufrequtils doxygen ethtool g++ git inetutils-tools libboost-all-dev libncurses-dev libusb-1.0-0 libusb-1.0-0-dev libusb-dev python3-dev python3-mako python3-numpy python3-requests python3-scipy python3-setuptools python3-ruamel.yaml
|
||||
git clone https://github.com/EttusResearch/uhd ~/uhd && cd uhd
|
||||
git checkout v4.9.0.0
|
||||
cd host && mkdir build && cd build
|
||||
cmake ../ -GNinja
|
||||
ninja
|
||||
sudo ninja install
|
||||
# Install nrue with USRP support, default nvcc location is /usr/local/bin/nvcc
|
||||
./build_oai --ninja --build-lib ldpc_cuda --gNB --nrUE -w USRP --use-cuda -C --cmake-opt -DCMAKE_CUDA_COMPILER=<nvcc-location>
|
||||
```
|
||||
|
||||
Create the DPDK interfaces for the UHD, use the below script and modify it for your environment. We are using VLAN 80 for DPDK in our environment:
|
||||
|
||||
```bash
|
||||
vim setup_sriov_usrp.sh
|
||||
```
|
||||
|
||||
```bash
|
||||
#!/bin/sh
|
||||
set -xeu
|
||||
|
||||
pci_addr()
|
||||
{
|
||||
PF_IF=$1
|
||||
VF_INDEX=$2
|
||||
SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}"
|
||||
|
||||
if [ ! -e "$SYSFS_PATH" ]; then
|
||||
echo "VF $VF_INDEX not found for interface $PF_IF"
|
||||
exit 1
|
||||
fi
|
||||
PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")")
|
||||
echo "$PCI_ADDR"
|
||||
}
|
||||
IF_NAME=enP2p1s0f1np1 ## interface to reach to the USRP on the UHD host
|
||||
HOST_MAC_ADD=00:11:22:33:44:67
|
||||
VLAN=80
|
||||
MTU=9000
|
||||
NUM_VFs=1
|
||||
sudo ethtool -G $IF_NAME rx 8160 tx 8160
|
||||
sudo sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs"
|
||||
sudo sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs"
|
||||
HOST_PCI_ADDR=$(pci_addr $IF_NAME 0)
|
||||
# this next 2 lines is for C/U planes
|
||||
sudo ip link set $IF_NAME vf 0 mac $HOST_MAC_ADD vlan $VLAN spoofchk off mtu $MTU
|
||||
sleep 1
|
||||
sudo dpdk-devbind.py --unbind $HOST_PCI_ADDR
|
||||
sudo modprobe mlx5_core
|
||||
sudo dpdk-devbind.py --bind mlx5_core $HOST_PCI_ADDR
|
||||
echo "Successfully configured interface for USRP:
|
||||
- Mac-address: $HOST_MAC_ADD, VLAN: $VLAN, PCI: $HOST_PCI_ADDR"
|
||||
exit 0
|
||||
```
|
||||
|
||||
Create `/etc/uhd/uhd.conf` configuration file for allowing UHD to use dpdk for reaching to the USRP:
|
||||
|
||||
```txt
|
||||
[use_dpdk=1]
|
||||
dpdk_mtu=9000
|
||||
dpdk_driver=/usr/local/lib/aarch64-linux-gnu/dpdk/pmds-25.0/
|
||||
dpdk_corelist=5,6,7,8,9
|
||||
[dpdk_mac=00:11:22:33:44:67]
|
||||
dpdk_lcore=6
|
||||
dpdk_ipv4=192.168.80.25/24
|
||||
```
|
||||
|
||||
The ip-address 192.168.80.25/24 is the UHD host ip-address. Make sure you adjust the `dpdk_driver` location.
|
||||
|
||||
|
||||
### Start the nrUE process
|
||||
|
||||
Create a UE configuration file based on the PLMN, slicing information, DNN and DNN type you have configured for the UE in the core network. It is similar to previous section.
|
||||
|
||||
To start the `nr-uesoftmodem` process:
|
||||
|
||||
**Command for DGX Spark**:
|
||||
|
||||
```bash
|
||||
sudo devlink dev reload pci/<HOST_PCI_ADDR> && sudo taskset -c "5-19" ./nr-uesoftmodem --ue-fo-compensation --thread-pool 15,16,17,18,19 --loader.ldpc.shlibversion _cuda --use_gpu 1 -O ~/oai-nr-ue/nr-ue.conf --usrp-args "addr=192.168.80.251,use_dpdk=1" -C 3350010000 -r 273 --numerology 1 --band 78 --ssb 1515 --usrp-tx-thread-config 1
|
||||
```
|
||||
|
||||
DPDK deployment have some issues currently and we need to reload the pci interface before every run. The devlink command `sudo devlink dev reload pci/<HOST_PCI_ADDR>` is doing that.
|
||||
|
||||
If you need to adjust the tx/rx gains then you will need to take the parameters from previous section.
|
||||
76
docker/Dockerfile.base.cuda.ubuntu
Normal file
76
docker/Dockerfile.base.cuda.ubuntu
Normal file
@@ -0,0 +1,76 @@
|
||||
# SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
|
||||
#---------------------------------------------------------------------
|
||||
#
|
||||
# Dockerfile for the Open-Air-Interface BUILD service
|
||||
# Valid for Ubuntu 24.04
|
||||
#
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
|
||||
FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 AS ran-base-cuda
|
||||
|
||||
ARG NEEDED_GIT_PROXY
|
||||
ARG TARGETARCH
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=Europe/Paris
|
||||
ENV BUILD_UHD_FROM_SOURCE=True
|
||||
ENV UHD_VERSION=4.8.0.0
|
||||
|
||||
#install developers pkg/repo
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade --yes && \
|
||||
apt-get install --yes \
|
||||
#gcc needed for build_oai
|
||||
build-essential \
|
||||
ccache \
|
||||
psmisc \
|
||||
git \
|
||||
#use gcc-12 to avoid problems of default gcc-11 in FlexRIC
|
||||
gcc-12 \
|
||||
g++-12 \
|
||||
xxd \
|
||||
libpcre2-dev \
|
||||
python3-dev \
|
||||
bison \
|
||||
flex \
|
||||
m4 \
|
||||
# python3-pip for conf template generation
|
||||
python3-pip \
|
||||
wget \
|
||||
flatbuffers-compiler \
|
||||
libflatbuffers-dev \
|
||||
libnanomsg-dev && \
|
||||
update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 --slave /usr/bin/g++ g++ /usr/bin/g++-12 && \
|
||||
pip3 install --ignore-installed pyyaml
|
||||
|
||||
# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
|
||||
# it will be copied into target containers, to print exit numbers and handle signals properly
|
||||
ENV TINI_VERSION=v0.19.0
|
||||
RUN wget https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} -O /tini && chmod +x /tini
|
||||
|
||||
ENV YQ_VERSION=v4.2.0
|
||||
RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} -O /usr/local/bin/yq &&\
|
||||
chmod +x /usr/local/bin/yq
|
||||
|
||||
# In some network environments, GIT proxy is required
|
||||
RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi"
|
||||
|
||||
# Copying only the needed files to create ran-base
|
||||
WORKDIR /oai-ran/cmake_targets/tools
|
||||
COPY cmake_targets/tools/build_helper \
|
||||
cmake_targets/tools/uhd-4.x-tdd-patch.diff \
|
||||
./
|
||||
|
||||
WORKDIR /oai-ran/cmake_targets
|
||||
COPY cmake_targets/build_oai .
|
||||
|
||||
WORKDIR /oai-ran
|
||||
COPY oaienv .
|
||||
|
||||
#run build_oai -I to get the builder image
|
||||
RUN /bin/sh oaienv && \
|
||||
cd cmake_targets && \
|
||||
mkdir -p log && \
|
||||
./build_oai -I -w USRP -w BLADERF --install-optional-packages
|
||||
|
||||
59
docker/Dockerfile.build.cuda.ubuntu
Normal file
59
docker/Dockerfile.build.cuda.ubuntu
Normal file
@@ -0,0 +1,59 @@
|
||||
# SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
|
||||
#---------------------------------------------------------------------
|
||||
#
|
||||
# Dockerfile for the Open-Air-Interface BUILD service
|
||||
# Valid for Ubuntu 24.04
|
||||
#
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
FROM ran-base-cuda:latest AS ran-build-cuda
|
||||
ARG BUILD_OPTION
|
||||
|
||||
ARG E2AP_VERSION=E2AP_V3
|
||||
ARG KPM_VERSION=KPM_V3_00
|
||||
|
||||
RUN rm -Rf /oai-ran
|
||||
WORKDIR /oai-ran
|
||||
COPY . .
|
||||
|
||||
# build all targets so final targets can be created afterwards
|
||||
RUN \
|
||||
# Mount CCache cache direrctory
|
||||
--mount=type=cache,target=/root/.cache/ccache/ \
|
||||
# Mount CPM package cache
|
||||
--mount=type=cache,target=/root/.cache/cpm/ \
|
||||
/bin/sh oaienv && \
|
||||
cd cmake_targets && \
|
||||
mkdir -p log && \
|
||||
./build_oai -c --ninja \
|
||||
--eNB --gNB --RU --UE --nrUE \
|
||||
--build-lib "telnetsrv enbscope uescope nrscope ldpc_cuda" \
|
||||
--use-cuda \
|
||||
-w USRP -t Ethernet -w BLADERF \
|
||||
--build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
|
||||
--cmake-opt -DOAI_VRTSIM_TAPS_CLIENT=ON \
|
||||
--cmake-opt -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12.2 \
|
||||
--noavx512 \
|
||||
--build-tool-opt -k10 \
|
||||
--cmake-opt -DCMAKE_C_FLAGS="-Werror" --cmake-opt -DCMAKE_CXX_FLAGS="-Werror" $BUILD_OPTION && \
|
||||
# Mainly to see if the sanitize option was perfectly executed
|
||||
echo "---- ldd on executables ----" && \
|
||||
ldd ran_build/build/*softmodem* ran_build/build/nr-cuup ran_build/build/oairu && \
|
||||
echo "---- ldd on shared libraries ----" && \
|
||||
ldd ran_build/build/*.so
|
||||
|
||||
## Build FlexRIC for SM Models
|
||||
RUN \
|
||||
# Mount CCache cache direrctory
|
||||
--mount=type=cache,target=/root/.cache/ccache/ \
|
||||
mkdir -p openair2/E2AP/flexric/build && \
|
||||
cd openair2/E2AP/flexric/build && \
|
||||
cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
|
||||
-DKPM_VERSION=$KPM_VERSION \
|
||||
-DCMAKE_C_COMPILER=gcc-12 \
|
||||
-DCMAKE_CXX_COMPILER=g++-12 .. \
|
||||
-DE2AP_VERSION=$E2AP_VERSION .. && \
|
||||
ninja && \
|
||||
ninja install
|
||||
|
||||
115
docker/Dockerfile.gNB.cuda.ubuntu
Normal file
115
docker/Dockerfile.gNB.cuda.ubuntu
Normal file
@@ -0,0 +1,115 @@
|
||||
# SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
|
||||
#---------------------------------------------------------------------
|
||||
#
|
||||
# Dockerfile for the Open-Air-Interface BUILD service
|
||||
# Valid for Ubuntu 24.04
|
||||
#
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
ARG UBUNTU_IMAGE=ubuntu:noble
|
||||
FROM ran-base-cuda:latest AS gnb-base
|
||||
FROM ran-build-cuda:latest AS gnb-build
|
||||
|
||||
|
||||
#start from scratch for target executable
|
||||
FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS oai-gnb
|
||||
ARG BUILD_OPTION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=Europe/Paris
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade --yes && \
|
||||
apt-get install --yes \
|
||||
software-properties-common \
|
||||
procps \
|
||||
libsctp1 \
|
||||
libboost-chrono1.74.0 \
|
||||
libboost-date-time1.74.0 \
|
||||
libboost-filesystem1.74.0 \
|
||||
libboost-regex1.74.0 \
|
||||
libboost-serialization1.74.0 \
|
||||
libboost-thread1.74.0 \
|
||||
libboost-system1.74.0 \
|
||||
libboost-program-options1.74.0 \
|
||||
tzdata \
|
||||
libconfig9 \
|
||||
openssl \
|
||||
net-tools \
|
||||
iperf \
|
||||
iperf3 \
|
||||
iproute2 \
|
||||
iputils-ping \
|
||||
gdb \
|
||||
python3 \
|
||||
python3-six \
|
||||
python3-requests \
|
||||
libusb-1.0-0 \
|
||||
libyaml-cpp-dev \
|
||||
libnanomsg5 && \
|
||||
# if the --sanitize option was used to build, additional packages are required
|
||||
/bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
libasan8 \
|
||||
libubsan1; fi' && \
|
||||
apt-get clean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/oai-gnb/bin
|
||||
COPY --from=gnb-build \
|
||||
/oai-ran/cmake_targets/ran_build/build/nr-softmodem ./
|
||||
COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh
|
||||
COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh
|
||||
|
||||
COPY --from=gnb-build \
|
||||
/oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/librfsimulator.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/liboai_usrpdevif.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libcoding.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libdfts.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libldpc*.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ci.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_bearer.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_rrc.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_o1.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libvrtsim.so \
|
||||
/usr/local/lib/
|
||||
|
||||
# Now we are copying from builder-image the UHD files.
|
||||
COPY --from=gnb-base /usr/local/bin/uhd_find_devices /usr/local/bin
|
||||
COPY --from=gnb-base /usr/local/lib/libuhd.so.4.8.0 /usr/local/lib
|
||||
COPY --from=gnb-base /usr/local/lib/uhd/utils/uhd_images_downloader.py /opt/oai-gnb/bin
|
||||
## Copy E2 SM models
|
||||
COPY --from=gnb-build /usr/local/lib/flexric /usr/local/lib/flexric
|
||||
|
||||
RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai_device.so" && \
|
||||
ldconfig && \
|
||||
echo "---- ldd on nr-softmodem ----" && \
|
||||
ldd /opt/oai-gnb/bin/nr-softmodem && \
|
||||
echo "---- ldd on shared libraries ----" && \
|
||||
ldd /usr/local/lib/liboai_eth_transpro.so \
|
||||
/usr/local/lib/librfsimulator.so \
|
||||
/usr/local/lib/liboai_usrpdevif.so \
|
||||
/usr/local/lib/libcoding.so \
|
||||
/usr/local/lib/libparams_libconfig.so \
|
||||
/usr/local/lib/libdfts.so \
|
||||
/usr/local/lib/libldpc*.so \
|
||||
/usr/local/lib/libtelnetsrv.so \
|
||||
/usr/local/lib/libtelnetsrv_ci.so \
|
||||
/usr/local/lib/libuhd.so.4.8.0
|
||||
|
||||
WORKDIR /opt/oai-gnb
|
||||
#EXPOSE 2152/udp # S1U, GTP/UDP
|
||||
#EXPOSE 22100/tcp # ?
|
||||
#EXPOSE 36412/udp # S1C, SCTP/UDP
|
||||
#EXPOSE 36422/udp # X2C, SCTP/UDP
|
||||
#EXPOSE 50000/udp # IF5 / ORI (control)
|
||||
#EXPOSE 50001/udp # IF5 / ECPRI (data)
|
||||
|
||||
COPY --from=gnb-base /usr/local/bin/yq /usr/local/bin/yq
|
||||
COPY --from=gnb-base /tini /tini
|
||||
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
|
||||
CMD ["/opt/oai-gnb/bin/nr-softmodem"]
|
||||
107
docker/Dockerfile.nrUE.cuda.ubuntu
Normal file
107
docker/Dockerfile.nrUE.cuda.ubuntu
Normal file
@@ -0,0 +1,107 @@
|
||||
# SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
|
||||
#---------------------------------------------------------------------
|
||||
#
|
||||
# Dockerfile for the Open-Air-Interface BUILD service
|
||||
# Valid for Ubuntu 24.04
|
||||
#
|
||||
#---------------------------------------------------------------------
|
||||
|
||||
FROM ran-base-cuda:latest AS nr-ue-base
|
||||
FROM ran-build-cuda:latest AS nr-ue-build
|
||||
|
||||
#start from scratch for target executable
|
||||
FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS oai-nr-ue
|
||||
ARG BUILD_OPTION
|
||||
ENV DEBIAN_FRONTEND=noninteractive
|
||||
ENV TZ=Europe/Paris
|
||||
|
||||
RUN apt-get update && \
|
||||
apt-get upgrade --yes && \
|
||||
apt-get install --yes \
|
||||
arping \
|
||||
software-properties-common \
|
||||
procps \
|
||||
libsctp1 \
|
||||
libboost-chrono1.74.0 \
|
||||
libboost-date-time1.74.0 \
|
||||
libboost-filesystem1.74.0 \
|
||||
libboost-regex1.74.0 \
|
||||
libboost-serialization1.74.0 \
|
||||
libboost-thread1.74.0 \
|
||||
libboost-system1.74.0 \
|
||||
libboost-program-options1.74.0 \
|
||||
tzdata \
|
||||
libconfig9 \
|
||||
openssl \
|
||||
net-tools \
|
||||
gdb \
|
||||
python3 \
|
||||
python3-six \
|
||||
python3-requests \
|
||||
libusb-1.0-0 \
|
||||
iputils-ping \
|
||||
iproute2 \
|
||||
iperf3 \
|
||||
iperf \
|
||||
libyaml-cpp-dev \
|
||||
libnanomsg5 && \
|
||||
# if the --sanitize option was used to build, additional packages are required
|
||||
/bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
|
||||
libasan8 \
|
||||
libubsan1; fi' && \
|
||||
apt-get clean -y && \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /opt/oai-nr-ue/bin
|
||||
COPY --from=nr-ue-build \
|
||||
/oai-ran/cmake_targets/ran_build/build/nr-uesoftmodem \
|
||||
./
|
||||
|
||||
COPY ./docker/scripts/nr_ue_entrypoint.sh ./entrypoint.sh
|
||||
WORKDIR /opt/oai-nr-ue/etc
|
||||
|
||||
COPY --from=nr-ue-build \
|
||||
/oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/librfsimulator.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/liboai_usrpdevif.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libcoding.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libdfts.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libldpc*.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ciUE.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
|
||||
/oai-ran/cmake_targets/ran_build/build/libvrtsim.so \
|
||||
/usr/local/lib/
|
||||
|
||||
# Now we are copying from builder-image the UHD files.
|
||||
COPY --from=nr-ue-base /usr/local/bin/uhd_find_devices /usr/local/bin
|
||||
COPY --from=nr-ue-base /usr/local/lib/libuhd.so.4.8.0 /usr/local/lib
|
||||
COPY --from=nr-ue-base /usr/local/lib/uhd/utils/uhd_images_downloader.py /opt/oai-nr-ue/bin
|
||||
|
||||
RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai_device.so" && \
|
||||
ldconfig && \
|
||||
echo "---- ldd on nr-uesoftmodem ----" && \
|
||||
ldd /opt/oai-nr-ue/bin/nr-uesoftmodem && \
|
||||
echo "---- ldd on shared libraries ----" && \
|
||||
ldd /usr/local/lib/liboai_eth_transpro.so \
|
||||
/usr/local/lib/librfsimulator.so \
|
||||
/usr/local/lib/liboai_usrpdevif.so \
|
||||
/usr/local/lib/libcoding.so \
|
||||
/usr/local/lib/libparams_libconfig.so \
|
||||
/usr/local/lib/libdfts.so \
|
||||
/usr/local/lib/libldpc*.so \
|
||||
/usr/local/lib/libtelnetsrv.so \
|
||||
/usr/local/lib/libtelnetsrv_ciUE.so \
|
||||
/usr/local/lib/libtelnetsrv_5Gue.so \
|
||||
/usr/local/lib/libuhd.so.4.8.0
|
||||
|
||||
WORKDIR /opt/oai-nr-ue
|
||||
|
||||
COPY --from=nr-ue-base /usr/local/bin/yq /usr/local/bin/yq
|
||||
COPY --from=nr-ue-base /tini /tini
|
||||
CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem"]
|
||||
ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]
|
||||
|
||||
@@ -85,6 +85,7 @@ unsigned int mmapped_dma=0;
|
||||
uint64_t downlink_frequency[MAX_NUM_CCs][4];
|
||||
int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
|
||||
char *uecap_file;
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
runmode_t mode = normal_txrx;
|
||||
|
||||
|
||||
@@ -47,6 +47,7 @@ extern uint64_t ulsch_slot_bitmap;
|
||||
extern char *uecap_file;
|
||||
extern uint32_t dlsch_slot_modval;
|
||||
extern uint32_t ulsch_slot_modval;
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
// In nr-gnb.c
|
||||
extern void init_gNB();
|
||||
|
||||
@@ -83,6 +83,8 @@ uint64_t sidelink_frequency[MAX_NUM_CCs][4];
|
||||
// UE and OAI config variables
|
||||
double cpuf;
|
||||
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
int create_tasks_nrue(uint32_t ue_nb) {
|
||||
LOG_D(NR_RRC, "%s(ue_nb:%d)\n", __FUNCTION__, ue_nb);
|
||||
itti_wait_ready(1);
|
||||
@@ -260,7 +262,7 @@ int main(int argc, char **argv)
|
||||
|
||||
init_opt();
|
||||
|
||||
int ret_loader = load_nrLDPC_coding_interface(NULL, &nrLDPC_coding_interface);
|
||||
int ret_loader = load_nrLDPC_coding_interface(NULL, &nrLDPC_coding_interface, 32);
|
||||
AssertFatal(ret_loader == 0, "error loading LDPC library\n");
|
||||
|
||||
// strdup to put the sring in the core file for post mortem identification
|
||||
|
||||
@@ -22,6 +22,8 @@ char *worker_config=NULL;
|
||||
int usrp_tx_thread = 0;
|
||||
uint8_t nfapi_mode=0;
|
||||
|
||||
uint32_t use_gpu=0;
|
||||
|
||||
static struct timespec start;
|
||||
|
||||
static softmodem_params_t softmodem_params;
|
||||
|
||||
@@ -76,7 +76,7 @@ extern "C"
|
||||
#define CONFIG_HLP_TADV \
|
||||
"Set RF board timing_advance to compensate fix delay inside the RF board between Rx and Tx timestamps (RF board internal " \
|
||||
"issues)\n"
|
||||
|
||||
#define CONFIG_HLP_USE_GPU "Use GPU for processing offload"
|
||||
/*-----------------------------------------------------------------------------------------------------------------------------------------------------*/
|
||||
/* command line parameters common to eNodeB and UE */
|
||||
/* optname helpstr paramflags XXXptr defXXXval type numelt */
|
||||
@@ -104,6 +104,7 @@ extern "C"
|
||||
#define DEFAULT_PDU_ID softmodem_params.default_pdu_session_id
|
||||
|
||||
extern int usrp_tx_thread;
|
||||
extern uint32_t use_gpu;
|
||||
// clang-format off
|
||||
#define CMDLINE_PARAMS_DESC { \
|
||||
{"rf-config-file", CONFIG_HLP_RFCFGF, 0, .strptr=&RF_CONFIG_FILE, .defstrval=NULL, TYPE_STRING, 0}, \
|
||||
@@ -141,6 +142,7 @@ extern int usrp_tx_thread;
|
||||
{"E" , CONFIG_HLP_TQFS, PARAMFLAG_BOOL, .iptr=&softmodem_params.threequarter_fs, .defintval=0, TYPE_INT, 0}, \
|
||||
{"imscope" , CONFIG_HLP_IMSCOPE, PARAMFLAG_BOOL, .uptr=&enable_imscope, .defintval=0, TYPE_UINT, 0}, \
|
||||
{"imscope-record" , CONFIG_HLP_IMSCOPE_RECORD,PARAMFLAG_BOOL, .uptr=&enable_imscope_record, .defintval=0, TYPE_UINT, 0}, \
|
||||
{"use_gpu", CONFIG_HLP_USE_GPU, 0, .uptr=&use_gpu, .defintval=0, TYPE_UINT, 0}, \
|
||||
{"default-pdu-id", NULL, 0, .iptr=&DEFAULT_PDU_ID, .defintval=-1, TYPE_INT, 0}, \
|
||||
}
|
||||
// clang-format on
|
||||
@@ -186,6 +188,7 @@ extern int usrp_tx_thread;
|
||||
{ .s5 = { NULL } }, \
|
||||
{ .s5 = { NULL } }, \
|
||||
{ .s5 = { NULL } }, \
|
||||
{ .s5 = { NULL } }, \
|
||||
}
|
||||
// clang-format on
|
||||
|
||||
|
||||
@@ -33,6 +33,64 @@ add_dependencies(nr_ulschsim ldpc ldpc_orig)
|
||||
add_dependencies(nr_dlsim ldpc ldpc_orig)
|
||||
add_dependencies(nr_dlschsim ldpc ldpc_orig)
|
||||
|
||||
##############################################
|
||||
# Base CUDA setting
|
||||
##############################################
|
||||
|
||||
add_boolean_option(ENABLE_LDPC_CUDA OFF "Build support for CUDA" OFF)
|
||||
if (ENABLE_LDPC_CUDA)
|
||||
enable_language(CUDA)
|
||||
|
||||
# it would be possible to auto-detect compute capailities: https://stackoverflow.com/a/68223399
|
||||
#include(FindCUDA/select_compute_arch)
|
||||
#CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
|
||||
|
||||
# Nvidia recommends to build against all targeted archs: https://forums.developer.nvidia.com/t/261162
|
||||
# this is for cmake 3.16 (OAI minimum), starting at cmake 3.18 there is another syntax, see below
|
||||
|
||||
SET(CUDA_VERBOSE_BUILD ON)
|
||||
|
||||
# version using NVLink-C2C in Grace Hopper
|
||||
add_library(ldpc_cuda MODULE
|
||||
# main entry
|
||||
nrLDPC_coding/nrLDPC_coding_cuda.c
|
||||
|
||||
nrLDPC_decoder/nrLDPC_decoder_cuda.c
|
||||
nrLDPC_encoder/ldpc_encoder_cuda32.c
|
||||
nrLDPC_encoder/ldpc_input32.c
|
||||
|
||||
nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c
|
||||
nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder_cuda.c
|
||||
|
||||
# .cu files
|
||||
nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu
|
||||
nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
|
||||
nrLDPC_encoder/ldpc_input.cu
|
||||
nrLDPC_coding/nrLDPC_coding_segment/ldpc_deinterleave.cu
|
||||
nrLDPC_coding/nrLDPC_coding_segment/ldpc_rate_matching.cu
|
||||
)
|
||||
set_target_properties(ldpc_cuda PROPERTIES CUDA_ARCHITECTURES "80;87;90")
|
||||
# we need to find the right include directory, but ti seems that
|
||||
# enable_language(CUDA) does not make this available(?), so also use the CUDA package
|
||||
find_package(CUDA REQUIRED)
|
||||
target_include_directories(ldpc_cuda PRIVATE ${CUDA_INCLUDE_DIRS})
|
||||
|
||||
target_link_libraries(ldpc_cuda ldpc_gen_HEADERS log_headers)
|
||||
set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
|
||||
set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
|
||||
add_dependencies(ldpctest ldpc_cuda)
|
||||
add_dependencies(nr-softmodem ldpc_cuda)
|
||||
add_dependencies(nr-uesoftmodem ldpc_cuda)
|
||||
add_dependencies(nr_ulsim ldpc_cuda)
|
||||
add_dependencies(nr_ulschsim ldpc_cuda)
|
||||
add_dependencies(nr_dlsim ldpc_cuda)
|
||||
add_dependencies(nr_dlschsim ldpc_cuda)
|
||||
|
||||
target_compile_definitions(ldpc_cuda PRIVATE USE_CUDA)
|
||||
|
||||
endif()
|
||||
|
||||
add_library(crc_byte OBJECT crc_byte.c)
|
||||
|
||||
add_subdirectory(nrLDPC_coding)
|
||||
|
||||
@@ -16,6 +16,9 @@
|
||||
#include "openair1/PHY/defs_nr_common.h"
|
||||
#include "coding_unitary_defs.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#ifdef ENABLE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
|
||||
#define MAX_BLOCK_LENGTH 8448
|
||||
|
||||
@@ -25,6 +28,14 @@
|
||||
|
||||
#define NR_LDPC_PROFILER_DETAIL
|
||||
ldpc_interface_t ldpc_orig, ldpc_toCompare;
|
||||
#define MAX_NUM_DLSCH_SEGMENTS_DL_ldpctest 132
|
||||
static double modulated_input[MAX_NUM_DLSCH_SEGMENTS_DL_ldpctest][68 * 384];
|
||||
#define MAX_TRIALS 20000
|
||||
static int8_t Failure_Mask[MAX_TRIALS][(MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER*4)] = {0};
|
||||
|
||||
uint32_t use_gpu=0;
|
||||
|
||||
int PARALLEL_PATH = 0;//control decoder path
|
||||
|
||||
// 4-bit quantizer
|
||||
int8_t quantize4bit(double D, double x)
|
||||
@@ -78,6 +89,9 @@ typedef struct {
|
||||
n_iter_stats_t dec_iter;
|
||||
} one_measurement_t;
|
||||
|
||||
uint8_t *estimated_output_dev,*estimated_output;
|
||||
int8_t *channel_output_fixed_dev,*channel_output_fixed;
|
||||
|
||||
one_measurement_t test_ldpc(short max_iterations,
|
||||
int nom_rate,
|
||||
int denom_rate,
|
||||
@@ -85,14 +99,16 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
unsigned char qbits,
|
||||
short Kprime,
|
||||
unsigned int ntrials,
|
||||
int n_segments)
|
||||
int n_segments,
|
||||
int gen_code)
|
||||
{
|
||||
static int init_done = 0;
|
||||
one_measurement_t ret = {0};
|
||||
reset_meas(&ret.time_optim);
|
||||
reset_meas(&ret.time_decoder);
|
||||
// clock initiate
|
||||
// time_stats_t time,time_optim,tinput,tprep,tparity,toutput, time_decoder;
|
||||
time_stats_t time, tinput, tprep, tparity, toutput;
|
||||
time_stats_t time, tinput, tinput_memcpy, tprep, tparity, toutput;
|
||||
double n_iter_mean = 0;
|
||||
double n_iter_std = 0;
|
||||
int n_iter_max = 0;
|
||||
@@ -100,14 +116,17 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
double sigma;
|
||||
sigma = 1.0 / sqrt(2 * SNR);
|
||||
cpu_meas_enabled = 1;
|
||||
uint8_t *test_input[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * NR_MAX_NB_LAYERS];
|
||||
uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
|
||||
memset(estimated_output, 0, sizeof(estimated_output));
|
||||
uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
|
||||
#ifdef ENABLE_CUDA
|
||||
uint8_t **test_input,*test_input_p;
|
||||
#else
|
||||
uint8_t *test_input[n_segments * NR_MAX_NB_LAYERS];
|
||||
#endif
|
||||
uint8_t *channel_input[n_segments];
|
||||
uint8_t *channel_input_optim;
|
||||
|
||||
// double channel_output[68 * 384];
|
||||
double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
|
||||
int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
|
||||
|
||||
memset(modulated_input,0,sizeof(modulated_input));
|
||||
short BG = 0, nrows = 0; //,ncols;
|
||||
int i1, Kb = 0;
|
||||
int R_ind = 0;
|
||||
@@ -115,7 +134,8 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
int code_rate_vec[8] = {15, 13, 25, 12, 23, 34, 56, 89};
|
||||
// double code_rate_actual_vec[8] = {0.2, 0.33333, 0.4, 0.5, 0.66667, 0.73333, 0.81481, 0.88};
|
||||
|
||||
t_nrLDPC_dec_params decParams[MAX_NUM_DLSCH_SEGMENTS] = {0};
|
||||
t_nrLDPC_dec_params decParams[n_segments];
|
||||
memset(decParams,0,sizeof(decParams));
|
||||
|
||||
t_nrLDPC_time_stats decoder_profiler = {0};
|
||||
|
||||
@@ -123,6 +143,7 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
|
||||
reset_meas(&time);
|
||||
reset_meas(&tinput);
|
||||
reset_meas(&tinput_memcpy);
|
||||
reset_meas(&tprep);
|
||||
reset_meas(&tparity);
|
||||
reset_meas(&toutput);
|
||||
@@ -240,18 +261,26 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
printf("To: %d\n", (Kb + nrows - no_punctured_columns) * Zc - removed_bit);
|
||||
printf("number of undecoded bits: %d\n", (Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
|
||||
|
||||
// ldpc_toCompare.LDPCinit();
|
||||
// generate input block
|
||||
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaHostAlloc((void**)&test_input_p,n_segments*sizeof(uint8_t*),cudaHostAllocMapped);
|
||||
test_input=(uint8_t **)test_input_p;
|
||||
#endif
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaHostAlloc((void**)&test_input[j],((K + 7) & ~7) / 8,cudaHostAllocMapped);
|
||||
#else
|
||||
test_input[j] = malloc16(((K + 7) & ~7) / 8);
|
||||
memset(test_input[j], 0, ((K + 7) & ~7) / 8);
|
||||
#endif
|
||||
channel_input[j] = malloc16(68 * 384);
|
||||
memset(channel_input[j], 0, 68 * 384);
|
||||
}
|
||||
channel_input_optim = malloc16(68 * 384);
|
||||
memset(channel_input_optim, 0, 68 * 384);
|
||||
channel_input_optim = malloc16(68 * 384 * sizeof(uint32_t));
|
||||
|
||||
// Fill input segments with random values
|
||||
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
int i = 0;
|
||||
for (i = 0; i < ((Kprime + 7) & ~7) / 8; i++)
|
||||
test_input[j][i] = (uint8_t)rand();
|
||||
@@ -263,13 +292,21 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
}
|
||||
|
||||
encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K};
|
||||
impp.gen_code = 1;
|
||||
|
||||
impp.gen_code = gen_code;
|
||||
impp.tparity = &tparity;
|
||||
impp.tinput = &tinput;
|
||||
impp.tinput_memcpy = &tinput_memcpy;
|
||||
impp.toutput = &toutput;
|
||||
if (ntrials == 0)
|
||||
ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
|
||||
impp.gen_code = 0;
|
||||
decode_abort_t dec_abort;
|
||||
init_abort(&dec_abort);
|
||||
|
||||
if (init_done == 0) {
|
||||
ldpc_toCompare.LDPCinit();
|
||||
init_done = 1;
|
||||
}
|
||||
for (int trial = 0; trial < ntrials; trial++) {
|
||||
unsigned int segment_bler = 0;
|
||||
//// encoder
|
||||
@@ -280,37 +317,41 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
stop_meas(&time);
|
||||
|
||||
impp.n_segments = n_segments;
|
||||
|
||||
start_meas(&ret.time_optim);
|
||||
impp.first_seg = 0;
|
||||
ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
|
||||
|
||||
stop_meas(&ret.time_optim);
|
||||
|
||||
if (ntrials == 1)
|
||||
for (int j = 0; j < n_segments; j++)
|
||||
for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
|
||||
if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
|
||||
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
|
||||
return ret;
|
||||
}
|
||||
if (((channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)))) {
|
||||
printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (((uint32_t*)channel_input_optim)[i] >> j) & 0x1);
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
int bit=0;
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
for (int i = 2 * Zc; i < (Kb + nrows - no_punctured_columns) * Zc - removed_bit; i++) {
|
||||
#ifdef DEBUG_CODER
|
||||
if ((i & 0xf) == 0)
|
||||
printf("\ne %d..%d: ", i, i + 15);
|
||||
#endif
|
||||
bit = ((channel_input_optim[i - 2 * Zc] >> j) & 0x1);
|
||||
|
||||
if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0)
|
||||
if (bit == 0)
|
||||
modulated_input[j][i] = 1.0; /// sqrt(2); //QPSK
|
||||
else
|
||||
modulated_input[j][i] = -1.0; /// sqrt(2);
|
||||
|
||||
channel_output_fixed[j][i] =
|
||||
channel_output_fixed[j*(384*68) + i] =
|
||||
(int8_t)quantize(sigma / 4.0 / 4.0, modulated_input[j][i] + sigma * gaussdouble(0.0, 1.0), qbits);
|
||||
|
||||
// Uncoded BER
|
||||
uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0;
|
||||
uint8_t channel_output_uncoded = channel_output_fixed[(j*384*68)+i] < 0 ? 1 /* QPSK demod */ : 0;
|
||||
if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1))
|
||||
ret.errors_bit_uncoded++;
|
||||
}
|
||||
@@ -326,25 +367,36 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
decParams[j].numMaxIter = max_iterations;
|
||||
decParams[j].outMode = nrLDPC_outMode_BIT;
|
||||
decParams[j].Kprime = Kprime;
|
||||
ldpc_toCompare.LDPCinit();
|
||||
decParams[j].n_segments = n_segments;
|
||||
//decParams[j].LastTrial = (trial == ntrials - 1) ? 1 : 0;
|
||||
|
||||
|
||||
|
||||
}
|
||||
|
||||
|
||||
//Decoder
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
start_meas(&ret.time_decoder);
|
||||
set_abort(&dec_abort, false);
|
||||
|
||||
n_iter = ldpc_toCompare.LDPCdecoder(&decParams[j],
|
||||
(int8_t *)channel_output_fixed[j],
|
||||
(int8_t *)estimated_output[j],
|
||||
&decoder_profiler,
|
||||
&dec_abort);
|
||||
{
|
||||
n_iter = ldpc_toCompare.LDPCdecoder(&decParams[j],
|
||||
&channel_output_fixed[j*384*68],
|
||||
&estimated_output[j*Kprime],
|
||||
&decoder_profiler,
|
||||
&dec_abort);
|
||||
}
|
||||
stop_meas(&ret.time_decoder);
|
||||
|
||||
// count errors
|
||||
if (memcmp(estimated_output[j], test_input[j], ((Kprime + 7) & ~7) / 8) != 0) {
|
||||
if (memcmp(&estimated_output[j*(Kprime>>3)], test_input[j], ((Kprime + 7) & ~7) / 8) != 0) {
|
||||
segment_bler++;
|
||||
if(trial < 200){
|
||||
Failure_Mask[trial][j] = 1;
|
||||
}
|
||||
}
|
||||
for (int i = 0; i < Kprime; i++) {
|
||||
unsigned char estoutputbit = (estimated_output[j][i / 8] & (1 << (i & 7))) >> (i & 7);
|
||||
unsigned char estoutputbit = (estimated_output[j*(Kprime>>3) + (i / 8)] & (1 << (i & 7))) >> (i & 7);
|
||||
unsigned char inputbit = (test_input[j][i / 8] & (1 << (i & 7))) >> (i & 7); // Further correct for multiple segments
|
||||
if (estoutputbit != inputbit)
|
||||
ret.errors_bit++;
|
||||
@@ -367,17 +419,30 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
sqrt(n_iter_std / (double)ntrials / (double)n_segments - pow(n_iter_mean / (double)ntrials / (double)n_segments - 1, 2));
|
||||
ret.dec_iter.n_iter_max = n_iter_max - 1;
|
||||
|
||||
ret.dec_iter.n_iter_mean = n_iter_mean / (double)ntrials / (double)n_segments - 1;
|
||||
ret.dec_iter.n_iter_std =
|
||||
sqrt(n_iter_std / (double)ntrials / (double)n_segments - pow(n_iter_mean / (double)ntrials / (double)n_segments - 1, 2));
|
||||
ret.dec_iter.n_iter_max = n_iter_max - 1;
|
||||
|
||||
ret.errors_bit_uncoded = ret.errors_bit_uncoded / (double)((Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
|
||||
|
||||
for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
|
||||
for (int j = 0; j < n_segments; j++) {
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaFreeHost(test_input[j]);
|
||||
#else
|
||||
free(test_input[j]);
|
||||
#endif
|
||||
free(channel_input[j]);
|
||||
}
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaFreeHost(test_input);
|
||||
#endif
|
||||
free(channel_input_optim);
|
||||
|
||||
print_meas(&time, "ldpc_encoder", NULL, NULL);
|
||||
print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);
|
||||
print_meas(&tinput, "ldpc_encoder_optim(input)", NULL, NULL);
|
||||
print_meas(&tinput_memcpy, "ldpc_encoder_optim(input memcpy)", NULL, NULL);
|
||||
print_meas(&tprep, "ldpc_encoder_optim(prep)", NULL, NULL);
|
||||
print_meas(&tparity, "ldpc_encoder_optim(parity)", NULL, NULL);
|
||||
print_meas(&toutput, "ldpc_encoder_optim(output)", NULL, NULL);
|
||||
@@ -399,6 +464,9 @@ one_measurement_t test_ldpc(short max_iterations,
|
||||
}
|
||||
|
||||
configmodule_interface_t *uniqCfg = NULL;
|
||||
#ifdef ENABLE_CUDA
|
||||
extern int pageable,integrated;
|
||||
#endif
|
||||
int main(int argc, char *argv[])
|
||||
{
|
||||
short Kprime = 8448;
|
||||
@@ -418,19 +486,18 @@ int main(int argc, char *argv[])
|
||||
|
||||
int n_trials = 1;
|
||||
double SNR_step = 0.1;
|
||||
|
||||
int gen_code = 1;
|
||||
randominit();
|
||||
int test_uncoded = 0;
|
||||
n_iter_stats_t dec_iter[400] = {0};
|
||||
|
||||
short BG = 0, Zc;
|
||||
|
||||
if ((uniqCfg = load_configmodule(argc, argv, CONFIG_ENABLECMDLINEONLY)) == 0) {
|
||||
exit_fun("[LDPCTEST] Error, configuration module init failed\n");
|
||||
}
|
||||
logInit();
|
||||
|
||||
while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:G:n:d:i:t:u:hv:")) != -1) {
|
||||
while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:Gn:d:i:t:u:hv:g:Pn:")) != -1) {
|
||||
/* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
|
||||
/* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
|
||||
if (c == 1 || c == '-' || c == 'O')
|
||||
@@ -454,12 +521,9 @@ int main(int argc, char *argv[])
|
||||
Kprime = atoi(optarg);
|
||||
break;
|
||||
|
||||
case 'G':
|
||||
ldpc_version = "_cuda";
|
||||
break;
|
||||
|
||||
case 'n':
|
||||
n_trials = atoi(optarg);
|
||||
AssertFatal(n_trials < MAX_TRIALS, "max trials %d, please increase MAX_TRIALS\n", MAX_TRIALS);
|
||||
break;
|
||||
|
||||
case 's':
|
||||
@@ -484,6 +548,11 @@ int main(int argc, char *argv[])
|
||||
case 'v':
|
||||
ldpc_version = strdup(optarg);
|
||||
break;
|
||||
case 'g':
|
||||
gen_code = atoi(optarg);
|
||||
AssertFatal(gen_code <= 4, "gen_code %d is not allowed\n", gen_code);
|
||||
n_trials = 0;
|
||||
break;
|
||||
case 'h':
|
||||
default:
|
||||
printf("CURRENTLY SUPPORTED CODE RATES: \n");
|
||||
@@ -494,7 +563,7 @@ int main(int argc, char *argv[])
|
||||
printf("-r Nominator rate, (1, 2, 22), Default: 1\n");
|
||||
printf("-d Denominator rate, (3, 5, 25), Default: 1\n");
|
||||
printf("-l Length of payload bits in a segment (K' in 38.212-5.2.2), [1, 8448], Default: 8448\n");
|
||||
printf("-G give 1 to run cuda for LDPC, Default: 0\n");
|
||||
printf("-P give 1 to run cuda stream parallel for LDPC, Default: 0\n");
|
||||
printf("-n Number of simulation trials, Default: 1\n");
|
||||
// printf("-M MCS2 for TB 2\n");
|
||||
printf("-s SNR per information bit (EbNo) in dB, Default: -2\n");
|
||||
@@ -511,11 +580,26 @@ int main(int argc, char *argv[])
|
||||
printf("n_trials %d: \n", n_trials);
|
||||
printf("SNR0 %f: \n", SNR0);
|
||||
|
||||
printf("loading %s\n",ldpc_version);
|
||||
load_LDPClib(ldpc_version, &ldpc_toCompare);
|
||||
load_LDPClib("_orig", &ldpc_orig);
|
||||
|
||||
// find minimum value in all sets of lifting size
|
||||
Zc = 0;
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaError_t err = cudaHostAlloc((void**)&estimated_output,sizeof(uint8_t)* n_segments * Kprime,cudaHostAllocMapped);
|
||||
AssertFatal(err==cudaSuccess,"estimated_output n_segments %d Kprime %d\n",n_segments,Kprime);
|
||||
err = cudaHostAlloc((void**)&channel_output_fixed,sizeof(int8_t)* n_segments * 68 * 384,cudaHostAllocMapped);
|
||||
AssertFatal(err==cudaSuccess,"channel_output_fixed n_segments %d\n",n_segments);
|
||||
err = cudaHostGetDevicePointer((void**)&estimated_output_dev,estimated_output,0);
|
||||
AssertFatal(err==cudaSuccess,"estimated_output_dev\n");
|
||||
err = cudaHostGetDevicePointer((void**)&channel_output_fixed_dev,channel_output_fixed,0);
|
||||
AssertFatal(err==cudaSuccess,"channel_output_fixed_dev\n");
|
||||
printf("estimated_output_dev %p, channel_output_fixed_dev %p\n",estimated_output_dev,channel_output_fixed_dev);
|
||||
#else
|
||||
estimated_output = malloc(n_segments * Kprime);
|
||||
channel_output_fixed=malloc(n_segments * 68 * 384);
|
||||
#endif
|
||||
|
||||
char fname[200];
|
||||
sprintf(fname, "ldpctest_BG_%d_Zc_%d_rate_%d-%d_Kprime_%d_maxit_%d.txt", BG, Zc, nom_rate, denom_rate, Kprime, max_iterations);
|
||||
@@ -532,7 +616,7 @@ int main(int argc, char *argv[])
|
||||
SNR_lin = pow(10, SNR / 10.0);
|
||||
else
|
||||
SNR_lin = pow(10, SNR / 10.0) * nom_rate / denom_rate;
|
||||
printf("Linear SNR: %f\n", SNR_lin);
|
||||
printf("Linear SNR: %d %f, %d/%d, Kprime %d\n", max_iterations, SNR_lin, nom_rate,denom_rate,Kprime);
|
||||
one_measurement_t res = test_ldpc(max_iterations,
|
||||
nom_rate,
|
||||
denom_rate,
|
||||
@@ -540,13 +624,31 @@ int main(int argc, char *argv[])
|
||||
qbits,
|
||||
Kprime, // block length bytes
|
||||
n_trials,
|
||||
n_segments);
|
||||
n_segments,
|
||||
gen_code);
|
||||
|
||||
decoded_errors[i] = res.errors;
|
||||
dec_iter[i] = res.dec_iter;
|
||||
dec_iter[i].snr = SNR;
|
||||
dec_iter[i].ber = (float)res.errors_bit / (float)n_trials / (float)Kprime / (double)n_segments;
|
||||
dec_iter[i].bler = (float)decoded_errors[i] / (float)n_trials;
|
||||
printf("Failure Mask = ");
|
||||
for(int i=0; i<n_trials;i++){
|
||||
int flag = 0;
|
||||
for(int j = 0; j < n_segments; j++){
|
||||
if(Failure_Mask[i][j] == 1){
|
||||
if(flag == 0){
|
||||
printf(" %d: ", i);
|
||||
flag = 1;
|
||||
}
|
||||
printf(" %d ", j);
|
||||
Failure_Mask[i][j] = 0;
|
||||
}
|
||||
}
|
||||
if(flag == 1) printf(",");
|
||||
}
|
||||
printf("\n");
|
||||
|
||||
printf("SNR %f, BLER %f (%u/%d)\n", SNR, dec_iter[i].bler, decoded_errors[i], n_trials);
|
||||
printf("SNR %f, BER %f (%u/%d)\n", SNR, dec_iter[i].ber, decoded_errors[i], n_trials);
|
||||
printf("SNR %f, Uncoded BER %f (%u/%d)\n",
|
||||
@@ -560,7 +662,9 @@ int main(int argc, char *argv[])
|
||||
printf("\n");
|
||||
|
||||
double cpu_freq = get_cpu_freq_GHz();
|
||||
printf("CPU Freq %f\n",cpu_freq);
|
||||
time_stats_t *t_optim = &res.time_optim;
|
||||
|
||||
printf("Encoding time mean: %15.3f us\n", (double)t_optim->diff / t_optim->trials / 1000.0 / cpu_freq);
|
||||
printf("Encoding time std: %15.3f us\n",
|
||||
sqrt((double)t_optim->diff_square / t_optim->trials / pow(1000, 2) / pow(cpu_freq, 2)
|
||||
@@ -569,12 +673,12 @@ int main(int argc, char *argv[])
|
||||
printf("\n");
|
||||
|
||||
time_stats_t *t_decoder = &res.time_decoder;
|
||||
printf("Decoding time mean: %15.3f us\n", (double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq);
|
||||
printf("Decoding time mean (per segment)");
|
||||
printf(": %15.3f us\n", (double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq);
|
||||
printf("Decoding time std: %15.3f us\n",
|
||||
sqrt((double)t_decoder->diff_square / t_decoder->trials / pow(1000, 2) / pow(cpu_freq, 2)
|
||||
- pow((double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq, 2)));
|
||||
printf("Decoding time max: %15.3f us\n", (double)t_decoder->max / 1000.0 / cpu_freq);
|
||||
|
||||
fprintf(fd,
|
||||
"%f %f %f %f %f %f %f %f %f %f %f %f %d \n",
|
||||
SNR,
|
||||
|
||||
@@ -3,3 +3,4 @@
|
||||
add_subdirectory(nrLDPC_coding_segment)
|
||||
add_subdirectory(nrLDPC_coding_xdma)
|
||||
add_subdirectory(nrLDPC_coding_aal)
|
||||
add_subdirectory(nrLDPC_coding_tee)
|
||||
|
||||
@@ -458,8 +458,10 @@ static int init_op_data_objs_dec(struct rte_bbdev_op_data *bufs,
|
||||
bool large_input = false;
|
||||
int j = 0;
|
||||
for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) {
|
||||
for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) {
|
||||
uint32_t data_len = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E;
|
||||
nrLDPC_TB_decoding_parameters_t *p = &nrLDPC_slot_decoding_parameters->TBs[h];
|
||||
for (int i = 0; i < p->C; ++i) {
|
||||
/* TODO: @romain: better to run the entire TB at once or indiv. segments? */
|
||||
uint32_t data_len = i < p->first_rE2 ? p->E : p->E2;
|
||||
char *data;
|
||||
struct rte_mbuf *m_head = rte_pktmbuf_alloc(mbuf_pool);
|
||||
AssertFatal(m_head != NULL,
|
||||
@@ -602,20 +604,19 @@ static void set_ldpc_dec_op(struct rte_bbdev_dec_op **ops,
|
||||
bool special_case_tb_mode = !active_dev.is_t2 && (nrLDPC_slot_decoding_parameters->nb_TBs == 1)
|
||||
&& (nb_segments_decoding(nrLDPC_slot_decoding_parameters) == 1);
|
||||
for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) {
|
||||
for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) {
|
||||
ops[j]->ldpc_dec.basegraph = nrLDPC_slot_decoding_parameters->TBs[h].BG;
|
||||
ops[j]->ldpc_dec.z_c = nrLDPC_slot_decoding_parameters->TBs[h].Z;
|
||||
ops[j]->ldpc_dec.q_m = nrLDPC_slot_decoding_parameters->TBs[h].Qm;
|
||||
ops[j]->ldpc_dec.n_filler = nrLDPC_slot_decoding_parameters->TBs[h].F;
|
||||
ops[j]->ldpc_dec.n_cb = (nrLDPC_slot_decoding_parameters->TBs[h].BG == 1) ? (66 * nrLDPC_slot_decoding_parameters->TBs[h].Z)
|
||||
: (50 * nrLDPC_slot_decoding_parameters->TBs[h].Z);
|
||||
ops[j]->ldpc_dec.iter_max = nrLDPC_slot_decoding_parameters->TBs[h].max_ldpc_iterations;
|
||||
ops[j]->ldpc_dec.rv_index = nrLDPC_slot_decoding_parameters->TBs[h].rv_index;
|
||||
nrLDPC_TB_decoding_parameters_t *p = &nrLDPC_slot_decoding_parameters->TBs[h];
|
||||
for (int i = 0; i < p->C; ++i) {
|
||||
ops[j]->ldpc_dec.basegraph = p->BG;
|
||||
ops[j]->ldpc_dec.z_c = p->Z;
|
||||
ops[j]->ldpc_dec.q_m = p->Qm;
|
||||
ops[j]->ldpc_dec.n_filler = p->F;
|
||||
ops[j]->ldpc_dec.n_cb = (p->BG == 1) ? (66 * p->Z) : (50 * p->Z);
|
||||
ops[j]->ldpc_dec.iter_max = p->max_ldpc_iterations;
|
||||
ops[j]->ldpc_dec.rv_index = p->rv_index;
|
||||
ops[j]->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE;
|
||||
if (*nrLDPC_slot_decoding_parameters->TBs[h].segments[i].d_to_be_cleared) {
|
||||
*nrLDPC_slot_decoding_parameters->TBs[h].segments[i].d_to_be_cleared = false;
|
||||
if (p->d_to_be_cleared) {
|
||||
if (active_dev.is_t2)
|
||||
*nrLDPC_slot_decoding_parameters->TBs[h].processedSegments = 0;
|
||||
*p->processedSegments = 0;
|
||||
} else {
|
||||
ops[j]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE;
|
||||
if (active_dev.support_internal_harq_memory) {
|
||||
@@ -625,12 +626,12 @@ static void set_ldpc_dec_op(struct rte_bbdev_dec_op **ops,
|
||||
}
|
||||
|
||||
if (!active_dev.is_t2)
|
||||
*nrLDPC_slot_decoding_parameters->TBs[h].processedSegments = 0;
|
||||
*p->processedSegments = 0;
|
||||
|
||||
if (!special_case_tb_mode) {
|
||||
ops[j]->ldpc_dec.code_block_mode = 1;
|
||||
ops[j]->ldpc_dec.cb_params.e = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E;
|
||||
if (nrLDPC_slot_decoding_parameters->TBs[h].C > 1) {
|
||||
ops[j]->ldpc_dec.cb_params.e = i < p->first_rE2 ? p->E : p->E2;
|
||||
if (p->C > 1) {
|
||||
ops[j]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP;
|
||||
ops[j]->ldpc_dec.op_flags |= RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK;
|
||||
}
|
||||
@@ -646,12 +647,12 @@ static void set_ldpc_dec_op(struct rte_bbdev_dec_op **ops,
|
||||
ops[j]->ldpc_dec.tb_params.c = 1;
|
||||
ops[j]->ldpc_dec.tb_params.r = 0;
|
||||
ops[j]->ldpc_dec.tb_params.cab = 1;
|
||||
ops[j]->ldpc_dec.tb_params.ea = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E;
|
||||
ops[j]->ldpc_dec.tb_params.eb = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].E;
|
||||
ops[j]->ldpc_dec.tb_params.ea = i < p->first_rE2 ? p->E : p->E2;
|
||||
ops[j]->ldpc_dec.tb_params.eb = i < p->first_rE2 ? p->E : p->E2;
|
||||
}
|
||||
// Calculate offset in the HARQ combined buffers
|
||||
// Unique segment offset
|
||||
uint32_t segment_offset = (nrLDPC_slot_decoding_parameters->TBs[h].harq_unique_pid * NR_LDPC_MAX_NUM_CB) + i;
|
||||
uint32_t segment_offset = (p->harq_unique_pid * NR_LDPC_MAX_NUM_CB) + i;
|
||||
// Prune to avoid shooting above maximum id
|
||||
uint32_t pruned_segment_offset = segment_offset % active_dev.num_harq_codeblock;
|
||||
// Segment offset to byte offset
|
||||
@@ -673,6 +674,7 @@ static void set_ldpc_dec_op(struct rte_bbdev_dec_op **ops,
|
||||
ops[j]->ldpc_dec.input = inputs[j];
|
||||
++j;
|
||||
}
|
||||
p->d_to_be_cleared = false;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -729,14 +731,16 @@ static int retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, nrLDPC_slot_decod
|
||||
{
|
||||
int j = 0;
|
||||
for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) {
|
||||
for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) {
|
||||
nrLDPC_TB_decoding_parameters_t *p = &nrLDPC_slot_decoding_parameters->TBs[h];
|
||||
size_t data_off = 0;
|
||||
for (int i = 0; i < p->C; ++i) {
|
||||
struct rte_bbdev_op_data *hard_output = &ops[j]->ldpc_dec.hard_output;
|
||||
struct rte_mbuf *m = hard_output->data;
|
||||
uint16_t data_len = rte_pktmbuf_data_len(m) - hard_output->offset;
|
||||
uint8_t *data = rte_pktmbuf_mtod_offset(m, uint8_t *, hard_output->offset);
|
||||
memcpy(nrLDPC_slot_decoding_parameters->TBs[h].segments[i].c, data, data_len);
|
||||
memcpy(p->c + data_off, data, data_len);
|
||||
|
||||
uint32_t segment_offset = (nrLDPC_slot_decoding_parameters->TBs[h].harq_unique_pid * NR_LDPC_MAX_NUM_CB) + i;
|
||||
uint32_t segment_offset = (p->harq_unique_pid * NR_LDPC_MAX_NUM_CB) + i;
|
||||
uint32_t pruned_segment_offset = segment_offset % active_dev.num_harq_codeblock;
|
||||
struct rte_bbdev_op_data *harq_output = &ops[j]->ldpc_dec.harq_combined_output;
|
||||
if (!active_dev.support_internal_harq_memory) {
|
||||
@@ -749,6 +753,7 @@ static int retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, nrLDPC_slot_decod
|
||||
active_dev.harq_buffers[pruned_segment_offset].offset = harq_output->offset;
|
||||
active_dev.harq_buffers[pruned_segment_offset].length = harq_output->length;
|
||||
++j;
|
||||
data_off += p->K >> 3;
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
@@ -823,7 +828,7 @@ static int pmd_lcore_ldpc_dec(void *arg)
|
||||
|
||||
// Start timer
|
||||
// We report timing only once in (0,0) since the timers are merged at the end
|
||||
start_meas(&nrLDPC_slot_decoding_parameters->TBs[0].segments[0].ts_ldpc_decode);
|
||||
start_meas(&nrLDPC_slot_decoding_parameters->TBs[0].ts_ldpc_decode);
|
||||
|
||||
uint16_t enq = 0, deq = 0;
|
||||
while (enq < num_segments) {
|
||||
@@ -840,7 +845,7 @@ static int pmd_lcore_ldpc_dec(void *arg)
|
||||
|
||||
// Stop timer
|
||||
// We report timing only once in (0,0) since the timers are merged at the end
|
||||
stop_meas(&nrLDPC_slot_decoding_parameters->TBs[0].segments[0].ts_ldpc_decode);
|
||||
stop_meas(&nrLDPC_slot_decoding_parameters->TBs[0].ts_ldpc_decode);
|
||||
|
||||
if (deq == enq) {
|
||||
ret = retrieve_ldpc_dec_op(ops_deq, nrLDPC_slot_decoding_parameters);
|
||||
@@ -849,24 +854,24 @@ static int pmd_lcore_ldpc_dec(void *arg)
|
||||
/* get the max of iter_count for all dequeued ops */
|
||||
int j = 0;
|
||||
for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) {
|
||||
for (int i = 0; i < nrLDPC_slot_decoding_parameters->TBs[h].C; ++i) {
|
||||
bool *status = &nrLDPC_slot_decoding_parameters->TBs[h].segments[i].decodeSuccess;
|
||||
nrLDPC_TB_decoding_parameters_t *p = &nrLDPC_slot_decoding_parameters->TBs[h];
|
||||
for (int i = 0; i < p->C; ++i) {
|
||||
bool *status = &p->decodeSuccess[i];
|
||||
tp->iter_count = RTE_MAX(ops_enq[j]->ldpc_dec.iter_count, tp->iter_count);
|
||||
|
||||
// Check if CRC is available otherwise rely on ops_enq[j]->status to detect decoding success
|
||||
// CRC is NOT available if the CRC type is 24_B which is when C is greater than 1
|
||||
if (nrLDPC_slot_decoding_parameters->TBs[h].C > 1) {
|
||||
if (p->C > 1) {
|
||||
*status = (ops_enq[j]->status == 0);
|
||||
} else {
|
||||
uint8_t *decoded_bytes = nrLDPC_slot_decoding_parameters->TBs[h].segments[i].c;
|
||||
uint8_t crc_type = crcType(nrLDPC_slot_decoding_parameters->TBs[h].C, nrLDPC_slot_decoding_parameters->TBs[h].A);
|
||||
uint32_t len_with_crc = lenWithCrc(nrLDPC_slot_decoding_parameters->TBs[h].C, nrLDPC_slot_decoding_parameters->TBs[h].A);
|
||||
uint8_t *decoded_bytes = p->c;
|
||||
uint8_t crc_type = crcType(p->C, p->A);
|
||||
uint32_t len_with_crc = lenWithCrc(p->C, p->A);
|
||||
*status = check_crc(decoded_bytes, len_with_crc, crc_type);
|
||||
}
|
||||
|
||||
if (*status) {
|
||||
*nrLDPC_slot_decoding_parameters->TBs[h].processedSegments =
|
||||
*nrLDPC_slot_decoding_parameters->TBs[h].processedSegments + 1;
|
||||
*p->processedSegments = *p->processedSegments + 1;
|
||||
}
|
||||
++j;
|
||||
}
|
||||
@@ -1057,7 +1062,7 @@ static int normalize_dpdk_dev(const char *input, char *output, size_t out_len)
|
||||
}
|
||||
|
||||
// OAI CODE
|
||||
int32_t nrLDPC_coding_init()
|
||||
int32_t nrLDPC_coding_init(int max_num_pxsch)
|
||||
{
|
||||
pthread_mutex_init(&encode_mutex, NULL);
|
||||
pthread_mutex_init(&decode_mutex, NULL);
|
||||
@@ -1283,28 +1288,30 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
|
||||
int8_t llr_decimal = (active_dev.info.drv.capabilities)[RTE_BBDEV_OP_LDPC_DEC].cap.ldpc_dec.llr_decimals;
|
||||
int offset = 0;
|
||||
for (int h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h) {
|
||||
for (int r = 0; r < nrLDPC_slot_decoding_parameters->TBs[h].C; r++) {
|
||||
nrLDPC_TB_decoding_parameters_t *p = &nrLDPC_slot_decoding_parameters->TBs[h];
|
||||
size_t data_off = 0;
|
||||
for (int r = 0; r < p->C; r++) {
|
||||
uint32_t E = r < p->first_rE2 ? p->E : p->E2;
|
||||
if (active_dev.is_t2) {
|
||||
// For the T2, we simply saturate the LLRs.
|
||||
uint16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16)));
|
||||
memcpy(z_ol,
|
||||
nrLDPC_slot_decoding_parameters->TBs[h].segments[r].llr,
|
||||
nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E * sizeof(uint16_t));
|
||||
memcpy(z_ol, p->llr + data_off, E * sizeof(uint16_t));
|
||||
simde__m128i *pv_ol128 = (simde__m128i *)z_ol;
|
||||
simde__m128i *pl_ol128 = (simde__m128i *)&l_ol[offset];
|
||||
for (int i = 0, j = 0; j < ((nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E + 15) >> 4); i += 2, j++) {
|
||||
for (int i = 0, j = 0; j < ((E + 15) >> 4); i += 2, j++) {
|
||||
pl_ol128[j] = simde_mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]);
|
||||
}
|
||||
} else {
|
||||
llr_scaling(nrLDPC_slot_decoding_parameters->TBs[h].segments[r].llr,
|
||||
nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E,
|
||||
llr_scaling(p->llr + data_off,
|
||||
E,
|
||||
&l_ol[offset],
|
||||
llr_size,
|
||||
llr_decimal,
|
||||
nrLDPC_slot_decoding_parameters->TBs[h].nb_layers,
|
||||
nrLDPC_slot_decoding_parameters->TBs[h].Qm);
|
||||
p->nb_layers,
|
||||
p->Qm);
|
||||
}
|
||||
offset += LDPC_MAX_CB_SIZE;
|
||||
data_off += E;
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
106
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda.c
Normal file
106
openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda.c
Normal file
@@ -0,0 +1,106 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
#include <stdint.h>
|
||||
#include "openair1/PHY/CODING/nrLDPC_defs.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
|
||||
|
||||
|
||||
/* segment interface */
|
||||
extern int32_t LDPCinit_cuda();
|
||||
int32_t LDPCinit()
|
||||
{
|
||||
return LDPCinit_cuda();
|
||||
}
|
||||
|
||||
// LDPCshutdown_cuda
|
||||
extern int32_t LDPCshutdown_cuda();
|
||||
int32_t LDPCshutdown()
|
||||
{
|
||||
return LDPCshutdown_cuda();
|
||||
}
|
||||
|
||||
|
||||
// LDPCdecoder
|
||||
extern int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
|
||||
int8_t* p_llr,
|
||||
uint8_t* p_out,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab);
|
||||
int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
|
||||
int8_t* p_llr,
|
||||
uint8_t* p_out,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab)
|
||||
{
|
||||
return LDPCdecoder_cuda(p_decParams, p_llr, p_out, p_profiler, ab);
|
||||
}
|
||||
|
||||
|
||||
// LDPCencoder <= (LDPCencoder32)
|
||||
extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
|
||||
uint32_t LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
|
||||
{
|
||||
uint32_t **output32 = LDPCencoder32(input, impp);
|
||||
AssertFatal(impp->n_segments < 8, "LDPC CUDA segment interface does not copy more than 8 segs\n");
|
||||
// the following copied from ldpc_encoder_optim8segmulti.c
|
||||
int nrows = 46; // assumption BG1
|
||||
int rate = 3; // assumption BG1
|
||||
int no_punctured_columns = (int)((nrows-2)*impp->Zc+impp->K-impp->K*rate)/impp->Zc;
|
||||
int removed_bit = (nrows - no_punctured_columns - 2) * impp->Zc + impp->K - (int)(impp->K * rate);
|
||||
int len = impp->K + impp->Zc * (nrows - no_punctured_columns) - removed_bit;
|
||||
// copy to output format
|
||||
for (int i = 0; i < len; ++i) {
|
||||
// condition from ldpctest: (channel_input_optim[i] >> j) == (output32[j>>5][i] >> (j&31))
|
||||
// for more than 8 segments, need to spread output32 into output
|
||||
uint8_t segs = output32[0][i] & 0xff;
|
||||
output[i] = segs;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
/* slot interface */
|
||||
extern int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters);
|
||||
int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
|
||||
{
|
||||
// this should be the same as previous nrLDPC_coding_encoder() in nrLDPC_coding_segment_encoder.c
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
nrLDPC_TB_encoding_parameters_t *tbp = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
if (tbp->BG == 1 && tbp->C > 8 && tbp->Z == 384) {
|
||||
nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters, tbp);
|
||||
} else {
|
||||
AssertFatal(false, "unhandled job, use CPU\n");
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *);
|
||||
int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters)
|
||||
{
|
||||
// this should be the same as previous nrLDPC_coding_decoder() in nrLDPC_coding_segment_decoder.c
|
||||
for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
|
||||
nrLDPC_TB_decoding_parameters_t *tbp = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
|
||||
if (tbp->Z >= 128 && tbp->BG == 1) {
|
||||
nr_process_decode_segment_cuda(tbp);
|
||||
} else {
|
||||
AssertFatal(false, "unhandled job, use CPU\n");
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
extern int32_t nrLDPC_coding_init_cuda(int max_num_pxsch);
|
||||
int32_t nrLDPC_coding_init(int max_num_pxsch)
|
||||
{
|
||||
return nrLDPC_coding_init_cuda(max_num_pxsch);
|
||||
}
|
||||
|
||||
extern int32_t nrLDPC_coding_shutdown_cuda(void);
|
||||
int32_t nrLDPC_coding_shutdown(void)
|
||||
{
|
||||
return nrLDPC_coding_shutdown_cuda();
|
||||
}
|
||||
@@ -13,38 +13,6 @@
|
||||
#ifndef __NRLDPC_CODING_INTERFACE__H__
|
||||
#define __NRLDPC_CODING_INTERFACE__H__
|
||||
|
||||
/**
|
||||
* \typedef nrLDPC_segment_decoding_parameters_t
|
||||
* \struct nrLDPC_segment_decoding_parameters_s
|
||||
* \brief decoding parameter of segments
|
||||
* \var E input llr segment size
|
||||
* \var R code rate indication
|
||||
* \var llr segment input llr array
|
||||
* \var d Pointers to code blocks before LDPC decoding (38.212 V15.4.0 section 5.3.2)
|
||||
* \var d_to_be_cleared
|
||||
* pointer to the flag used to clear d properly
|
||||
* when true, clear d after rate dematching
|
||||
* \var c Pointers to code blocks after LDPC decoding (38.212 V15.4.0 section 5.2.2)
|
||||
* \var decodeSuccess
|
||||
* flag indicating that the decoding of the segment was successful
|
||||
* IT MUST BE FILLED BY THE IMPLEMENTATION
|
||||
* \var ts_deinterleave deinterleaving time stats
|
||||
* \var ts_rate_unmatch rate unmatching time stats
|
||||
* \var ts_ldpc_decode decoding time stats
|
||||
*/
|
||||
typedef struct nrLDPC_segment_decoding_parameters_s{
|
||||
int E;
|
||||
uint8_t R;
|
||||
short *llr;
|
||||
int16_t *d;
|
||||
bool *d_to_be_cleared;
|
||||
uint8_t *c;
|
||||
bool decodeSuccess;
|
||||
time_stats_t ts_deinterleave;
|
||||
time_stats_t ts_rate_unmatch;
|
||||
time_stats_t ts_ldpc_decode;
|
||||
} nrLDPC_segment_decoding_parameters_t;
|
||||
|
||||
/**
|
||||
* \typedef nrLDPC_TB_decoding_parameters_t
|
||||
* \struct nrLDPC_TB_decoding_parameters_s
|
||||
@@ -96,7 +64,20 @@ typedef struct nrLDPC_TB_decoding_parameters_s{
|
||||
uint32_t F;
|
||||
|
||||
uint32_t C;
|
||||
nrLDPC_segment_decoding_parameters_t *segments;
|
||||
int E;
|
||||
uint8_t R;
|
||||
int E2;
|
||||
uint8_t R2;
|
||||
int first_rE2;
|
||||
short *llr;
|
||||
uint8_t *c;
|
||||
int16_t *d;
|
||||
bool d_to_be_cleared;
|
||||
bool decodeSuccess[132];
|
||||
time_stats_t ts_deinterleave;
|
||||
time_stats_t ts_rate_unmatch;
|
||||
time_stats_t ts_seg_prep;
|
||||
time_stats_t ts_ldpc_decode;
|
||||
} nrLDPC_TB_decoding_parameters_t;
|
||||
|
||||
/**
|
||||
@@ -137,6 +118,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{
|
||||
time_stats_t ts_interleave;
|
||||
time_stats_t ts_rate_match;
|
||||
time_stats_t ts_ldpc_encode;
|
||||
time_stats_t ts_output;
|
||||
} nrLDPC_segment_encoding_parameters_t;
|
||||
|
||||
/**
|
||||
@@ -186,6 +168,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{
|
||||
uint32_t C;
|
||||
nrLDPC_segment_encoding_parameters_t *segments;
|
||||
unsigned char *output;
|
||||
uint8_t **c_dev;
|
||||
} nrLDPC_TB_encoding_parameters_t;
|
||||
|
||||
/**
|
||||
@@ -211,13 +194,15 @@ typedef struct nrLDPC_slot_encoding_parameters_s{
|
||||
int nb_TBs;
|
||||
tpool_t *threadPool;
|
||||
time_stats_t *tinput;
|
||||
time_stats_t *tinput_memcpy;
|
||||
time_stats_t *tprep;
|
||||
time_stats_t *tparity;
|
||||
time_stats_t *toutput;
|
||||
time_stats_t *tconcat;
|
||||
nrLDPC_TB_encoding_parameters_t *TBs;
|
||||
} nrLDPC_slot_encoding_parameters_t;
|
||||
|
||||
typedef int32_t(nrLDPC_coding_init_t)(void);
|
||||
typedef int32_t(nrLDPC_coding_init_t)(int);
|
||||
typedef int32_t(nrLDPC_coding_shutdown_t)(void);
|
||||
|
||||
/**
|
||||
@@ -239,7 +224,7 @@ typedef struct nrLDPC_coding_interface_s {
|
||||
nrLDPC_coding_encoder_t *nrLDPC_coding_encoder;
|
||||
} nrLDPC_coding_interface_t;
|
||||
|
||||
int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *interface);
|
||||
int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *interface, int);
|
||||
int free_nrLDPC_coding_interface(nrLDPC_coding_interface_t *interface);
|
||||
|
||||
#endif
|
||||
|
||||
@@ -21,7 +21,7 @@
|
||||
/* arg is used to initialize the config module so that the loader works as expected */
|
||||
char *arguments_phy_simulators[64]={"ldpctest",NULL};
|
||||
|
||||
int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *itf)
|
||||
int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *itf,int max_num_pxsch)
|
||||
{
|
||||
char *ptr = (char *)config_get_if();
|
||||
char libname[64] = "ldpc";
|
||||
@@ -46,7 +46,7 @@ int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *itf)
|
||||
itf->nrLDPC_coding_decoder = (nrLDPC_coding_decoder_t *)shlib_fdesc[2].fptr;
|
||||
itf->nrLDPC_coding_encoder = (nrLDPC_coding_encoder_t *)shlib_fdesc[3].fptr;
|
||||
|
||||
AssertFatal(itf->nrLDPC_coding_init() == 0, "error starting LDPC library %s %s\n", libname, version);
|
||||
AssertFatal(itf->nrLDPC_coding_init(max_num_pxsch) == 0, "error starting LDPC library %s %s\n", libname, version);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
// deinterleave_u16.cu
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
|
||||
__global__ void deinterleave_i16_2(int16_t* __restrict__ e,
|
||||
const int16_t* __restrict__ f,
|
||||
int E1,
|
||||
int E2,
|
||||
int r_firstE2)
|
||||
{
|
||||
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
int E = (r<r_firstE2) ? E1 : E2;
|
||||
int EQm = E/2;
|
||||
if (g >= EQm) return;
|
||||
|
||||
int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
|
||||
const int16_t *in = f +r_off + 2*g;
|
||||
|
||||
int16_t* e0 = e + r_off;
|
||||
int16_t* e1 = e0 + EQm;
|
||||
e0[g] = in[0];
|
||||
e1[g] = in[1];
|
||||
}
|
||||
|
||||
__global__ void deinterleave_i16_4(int16_t* __restrict__ e,
|
||||
const int16_t* __restrict__ f,
|
||||
int E1,
|
||||
int E2,
|
||||
int r_firstE2)
|
||||
{
|
||||
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
int E = (r<r_firstE2) ? E1 : E2;
|
||||
int EQm = E/4;
|
||||
if (g >= EQm) return;
|
||||
|
||||
int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
|
||||
const int16_t *in = f + r_off + 4*g;
|
||||
int16_t* e0 = e + r_off;
|
||||
int16_t* e1 = e0 + EQm;
|
||||
int16_t* e2 = e1 + EQm;
|
||||
int16_t* e3 = e2 + EQm;
|
||||
e0[g] = in[0];
|
||||
e1[g] = in[1];
|
||||
e2[g] = in[2];
|
||||
e3[g] = in[3];
|
||||
}
|
||||
|
||||
__global__ void deinterleave_i16_6(int16_t* __restrict__ e,
|
||||
const int16_t* __restrict__ f,
|
||||
int E1,
|
||||
int E2,
|
||||
int r_firstE2)
|
||||
{
|
||||
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
int E = (r<r_firstE2) ? E1 : E2;
|
||||
int EQm = E/6;
|
||||
if (g >= EQm) return;
|
||||
|
||||
int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
|
||||
const int16_t* in = f + r_off + 6*g;
|
||||
int16_t* e0 = e + r_off;
|
||||
int16_t* e1 = e0 + EQm;
|
||||
int16_t* e2 = e1 + EQm;
|
||||
int16_t* e3 = e2 + EQm;
|
||||
int16_t* e4 = e3 + EQm;
|
||||
int16_t* e5 = e4 + EQm;
|
||||
e0[g] = in[0];
|
||||
e1[g] = in[1];
|
||||
e2[g] = in[2];
|
||||
e3[g] = in[3];
|
||||
e4[g] = in[4];
|
||||
e5[g] = in[5];
|
||||
}
|
||||
|
||||
__global__ void deinterleave_i16_8(int16_t* __restrict__ e,
|
||||
const int16_t* __restrict__ f,
|
||||
const int E1,
|
||||
const int E2,
|
||||
const int r_firstE2)
|
||||
{
|
||||
int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
int E = (r<r_firstE2) ? E1 : E2;
|
||||
int EQm = E/8;
|
||||
if (g >= EQm) return;
|
||||
int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
|
||||
|
||||
const int16_t *in = (f + r_off + 8*g);
|
||||
int16_t* e0 = e + r_off;
|
||||
int16_t* e1 = e0 + EQm;
|
||||
int16_t* e2 = e1 + EQm;
|
||||
int16_t* e3 = e2 + EQm;
|
||||
int16_t* e4 = e3 + EQm;
|
||||
int16_t* e5 = e4 + EQm;
|
||||
int16_t* e6 = e5 + EQm;
|
||||
int16_t* e7 = e6 + EQm;
|
||||
e0[g] = in[0];
|
||||
e1[g] = in[1];
|
||||
e2[g] = in[2];
|
||||
e3[g] = in[3];
|
||||
e4[g] = in[4];
|
||||
e5[g] = in[5];
|
||||
e6[g] = in[6];
|
||||
e7[g] = in[7];
|
||||
}
|
||||
|
||||
// Host launcher
|
||||
extern "C" void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx)
|
||||
{
|
||||
const int threads = 256;
|
||||
dim3 blocks(((E2/Qm) + threads - 1) / threads,C);
|
||||
|
||||
switch (Qm) {
|
||||
case 2: deinterleave_i16_2<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
|
||||
case 4: deinterleave_i16_4<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
|
||||
case 6: deinterleave_i16_6<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
|
||||
case 8: deinterleave_i16_8<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
|
||||
default: /* unsupported */ break;
|
||||
}
|
||||
cudaError_t err=cudaPeekAtLastError();
|
||||
|
||||
if (err!=cudaSuccess) {
|
||||
printf("cuda error (deinterleave_i16): %s (e %p, f %p, E1 %d, E2 %d, Qm %d, C %d)\n",cudaGetErrorString(err),e,f,E1,E2,Qm,C);
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -0,0 +1,300 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
#include <stdio.h>
|
||||
#include "nr_rate_matching.h"
|
||||
|
||||
__device__ __forceinline__ int clamp_i16_to_i8(int x)
|
||||
{
|
||||
// x is int (promoted)
|
||||
x = (x < -128) ? -128 : x;
|
||||
x = (x > 127) ? 127 : x;
|
||||
return x;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t packs_4x16_to_4x8(uint32_t a, uint32_t b)
|
||||
{
|
||||
int a0 = (int)(int16_t)(a & 0xFFFFu);
|
||||
int a1 = (int)(int16_t)(a >> 16);
|
||||
int b0 = (int)(int16_t)(b & 0xFFFFu);
|
||||
int b1 = (int)(int16_t)(b >> 16);
|
||||
|
||||
uint32_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
|
||||
uint32_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
|
||||
uint32_t o2 = (uint8_t)(int8_t)clamp_i16_to_i8(b0);
|
||||
uint32_t o3 = (uint8_t)(int8_t)clamp_i16_to_i8(b1);
|
||||
|
||||
return (o0) | (o1 << 8) | (o2 << 16) | (o3 << 24);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint16_t packs_2x16_to_2x8(uint32_t a)
|
||||
{
|
||||
int a0 = (int)(int16_t)(a & 0xFFFFu);
|
||||
int a1 = (int)(int16_t)(a >> 16);
|
||||
|
||||
uint16_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
|
||||
uint16_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
|
||||
|
||||
return (o0) | (o1 << 8) ;
|
||||
}
|
||||
|
||||
__global__ void rm(int Ncb_4,int ind0_4, int E1, int E2, int r_firstE2, int Foffset_4, int F_4, int clear, int seglen_4, int K_4, int Z_4, uint32_t *d, uint32_t *e, uint32_t *llr_buffer) {
|
||||
|
||||
int ind_4 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
|
||||
uint32_t *dr = d + (2*r*seglen_4);
|
||||
uint32_t *llr_bufferr = llr_buffer + (r*seglen_4);
|
||||
#ifdef DEBUG_RM
|
||||
if (blockIdx.x == 0 && threadIdx.x == 0) printf("r %d, seglen %d\n",r,seglen_4);
|
||||
#endif
|
||||
if (ind_4>= Foffset_4 && ind_4 < Foffset_4+F_4) {
|
||||
llr_bufferr[ind_4-Foffset_4+K_4-F_4] = 0x7f7f7f7f;
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",seglen_4*r + ind_4-Foffset_4+K_4-F_4,ind_4,K_4,Foffset_4,F_4);
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (ind_4 >= Ncb_4) return;
|
||||
if (clear == 1) { dr[2*ind_4] = 0; dr[(2*ind_4)+1]=0; }
|
||||
|
||||
int E_4;
|
||||
if (r<r_firstE2) E_4=(E1/4); else E_4=(E2/4);
|
||||
|
||||
int ind1=ind0_4,ind2;
|
||||
int k=0;
|
||||
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
|
||||
#endif
|
||||
if (ind1 < Foffset_4) {
|
||||
int ind2 = ind1 + min(Foffset_4-ind1,E_4);
|
||||
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0) printf("check 1b: ind %d ind1 %d ind2 %d\n",ind,ind1,ind2);
|
||||
#endif
|
||||
if (ind_4 >= ind1 && ind_4 < ind2) {
|
||||
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(ind_4-ind1)]);
|
||||
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(ind_4-ind1)+1]);
|
||||
}
|
||||
#ifdef DEBUG_RM
|
||||
if (r<=1 && threadIdx.x == 0 && ind_4 >= ind1 && ind_4 < ind2) printf("write 1. r %d : ind %d, ind1 %d, ind2 %d,k %d/E %d, pos %d, er %d %d\n",r,ind_4,ind1,ind2,ind_4-ind1,E_4,4*(ind_4-ind1),(int16_t)(er[2*(ind_4-ind1)]&0xffff),(int16_t)(er[2*(ind_4-ind1)]>>16));
|
||||
#endif
|
||||
k=ind2-ind1;
|
||||
ind1 = ind2;
|
||||
}
|
||||
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
|
||||
#endif
|
||||
if (ind1 >= Foffset_4 && ind1 < Foffset_4 + F_4) ind1 = Foffset_4 + F_4;
|
||||
ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
|
||||
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0) printf("check 2b: ind %d ind1 %d ind2 %d\n",ind_4,ind1,ind2);
|
||||
#endif
|
||||
if (ind_4 >= ind1 && ind_4 < ind2) {
|
||||
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+(ind_4-ind1))]);
|
||||
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+(ind_4-ind1))+1]);
|
||||
}
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0 && ind >= ind1 && ind < ind2) printf("write 2. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
|
||||
#endif
|
||||
k+=(ind2-ind1);
|
||||
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0) printf("check k %d E %d\n",k,E);
|
||||
#endif
|
||||
while (k < E_4) {
|
||||
ind2 = min(Foffset_4,E_4-k);
|
||||
if (ind_4 < ind2) {
|
||||
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+ind_4)]);
|
||||
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4)+1]);
|
||||
}
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0 && ind < ind2 && ind >= ind1) printf("3. ind %d, ind2 %d, k %d/E %d\n",ind,ind2,k+ind,E);
|
||||
#endif
|
||||
k+=ind2;
|
||||
|
||||
ind1=Foffset_4+F_4;
|
||||
ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
|
||||
if (ind_4 >= ind1 && ind_4 < ind2 && k < E_4) {
|
||||
dr[2*ind_4] = __vaddss2(dr[2*ind_4],er[2*(k+ind_4-ind1)]);
|
||||
dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4-ind1)+1]);
|
||||
}
|
||||
#ifdef DEBUG_RM
|
||||
if (r==0 && threadIdx.x == 0 && ind < ind && ind >= ind1) printf("4. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
|
||||
#endif
|
||||
k+=(ind2-ind1);
|
||||
}
|
||||
// note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in position r*seglen + (Kprime+F = K .. 2Z+(66*Z)=seglen
|
||||
llr_bufferr[2*Z_4 + ind_4] = packs_4x16_to_4x8(dr[(2*ind_4)], dr[(2*ind_4)+1]);
|
||||
#ifdef DEBUG_RM
|
||||
if (r<=1 && threadIdx.x == 0) printf("RM segment %d: writing %x to position %d (ind %d)\n",r,llr_bufferr[2*Z_4 + ind_4],r*seglen_4 + 2*Z_4 + ind_4,ind_4);
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
__global__ void rm2(int Ncb_2,int ind0_2, int E1, int E2, int r_firstE2, int Foffset_2, int F_2, int clear, int seglen_2, int K_2, int Z_2, uint32_t *d, uint32_t *e, uint16_t *llr_buffer) {
|
||||
|
||||
int ind_2 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
|
||||
int r = (int)blockIdx.y;
|
||||
uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
|
||||
uint32_t *dr = d + (r*seglen_2);
|
||||
uint16_t *llr_bufferr = llr_buffer + (r*seglen_2);
|
||||
|
||||
if (ind_2 >= Ncb_2) return;
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0) printf("Looking for ind %d...%d, Ncb %d\n",ind_2*2,1+ind_2*2,Ncb_2*2);
|
||||
#endif
|
||||
if (ind_2>= Foffset_2 && ind_2 < Foffset_2+F_2) {
|
||||
llr_bufferr[ind_2-Foffset_2+K_2-F_2] = 0x7f7f;
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",(seglen_2*r + ind_2-Foffset_2+K_2-F_2)*2,ind_2*2,K_2*2,Foffset_2*2,F_2*2);
|
||||
#endif
|
||||
return;
|
||||
}
|
||||
|
||||
if (clear == 1) dr[ind_2] = 0;
|
||||
|
||||
int E_2;
|
||||
if (r<r_firstE2) E_2=E1/2; else E_2=E2/2;
|
||||
|
||||
int ind1=ind0_2,ind2;
|
||||
int k=0;
|
||||
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
|
||||
if (ind1 < Foffset_2) {
|
||||
int ind2 = ind1 + min(Foffset_2-ind1,E_2);
|
||||
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM1B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
|
||||
if (ind_2 >= ind1 && ind_2 < ind2) {
|
||||
dr[ind_2] = __vaddss2(dr[ind_2],er[ind_2-ind1]);
|
||||
}
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM1A: ind_2 %d, ind1 %d, ind2 %d,k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(ind_2-ind1),2*E_2);
|
||||
#endif
|
||||
k=ind2-ind1;
|
||||
ind1 = ind2;
|
||||
}
|
||||
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
|
||||
if (ind1 >= Foffset_2 && ind1 < Foffset_2 + F_2) ind1 = Foffset_2 + F_2;
|
||||
ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
|
||||
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM2B: ind %d ind1 %d ind2 %d k%d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
|
||||
if (ind_2 >= ind1 && ind_2 < ind2) {
|
||||
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+(ind_2-ind1))]);
|
||||
}
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM2A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(k+(ind-ind1)),2*E_2);
|
||||
#endif
|
||||
k+=(ind2-ind1);
|
||||
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check k %d E %d\n",k,E);
|
||||
while (k < E_2) {
|
||||
ind2 = min(Foffset_2,E_2-k);
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM3B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
|
||||
if (ind_2 < ind2) {
|
||||
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+ind_2)]);
|
||||
}
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 ) printf("RM3A: ind %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind2,2*(k+ind_2),2*E_2);
|
||||
#endif
|
||||
k+=ind2;
|
||||
|
||||
ind1=Foffset_2+F_2;
|
||||
ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM4B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
|
||||
if (ind_2 >= ind1 && ind_2 < ind2 && k < E_2) {
|
||||
dr[ind_2] = __vaddss2(dr[ind_2],er[(k+ind_2-ind1)]);
|
||||
}
|
||||
#ifdef RM_DEBUG
|
||||
if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 && ind_2 >= ind1) printf("RM4A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2_2,2*(k+ind_2-ind1),2*E_2);
|
||||
#endif
|
||||
k+=(ind2-ind1);
|
||||
|
||||
}
|
||||
// note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in potiion r*seglent + (Kprime+F = K .. 2Z+(66*Z)=seglen
|
||||
llr_bufferr[2*Z_2 + ind_2] = packs_2x16_to_2x8(dr[ind_2]);
|
||||
// if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("writing %x to position %d (ind %d)\n",llr_bufferr[2*Z + ind_2],r*seglen + 2*Z + ind_2,ind_2);
|
||||
}
|
||||
|
||||
|
||||
static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
|
||||
extern "C" int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
|
||||
uint8_t BG,
|
||||
uint16_t Z,
|
||||
int16_t *d,
|
||||
int16_t *soft_input,
|
||||
int8_t *llr_buffer,
|
||||
uint32_t K,
|
||||
uint8_t C,
|
||||
uint8_t rvidx,
|
||||
uint8_t clear,
|
||||
uint32_t E1,
|
||||
uint32_t E2,
|
||||
uint32_t r_firstE2,
|
||||
uint32_t F,
|
||||
uint32_t Foffset,
|
||||
cudaStream_t *s,
|
||||
int8_t sidx)
|
||||
{
|
||||
if (C == 0 || C>132) {
|
||||
printf("nr_rate_matching: invalid parameter C %d\n", C);
|
||||
exit(-1);
|
||||
}
|
||||
|
||||
//Bit selection
|
||||
uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
|
||||
uint32_t Ncb;
|
||||
if (Tbslbrm == 0)
|
||||
Ncb = N;
|
||||
else {
|
||||
uint32_t Nref = (3 * Tbslbrm / (2 * C)); //R_LBRM = 2/3
|
||||
Ncb = min(N, Nref);
|
||||
}
|
||||
|
||||
uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
|
||||
|
||||
|
||||
int nthreads=Z;
|
||||
#ifdef RM_DEBUG
|
||||
printf("\nrm (%d,%d): Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",nblocks.x,nthreads,Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
|
||||
#endif
|
||||
if ((E1&3) == 0 && (E2&3) == 0) {
|
||||
dim3 nblocks(((Ncb>>2) + nthreads-1)/nthreads,C);
|
||||
rm<<<nblocks, nthreads, 0, s[sidx]>>>(Ncb/4,ind/4,E1,E2,r_firstE2,Foffset/4,F/4,clear,68*384/4,K/4,Z/4,(uint32_t*)d,(uint32_t*)soft_input,(uint32_t*)llr_buffer);
|
||||
}
|
||||
else {
|
||||
dim3 nblocks2(((Ncb>>1) + nthreads-1)/nthreads,C);
|
||||
rm2<<<nblocks2, nthreads, 0, s[sidx]>>>(Ncb/2,ind/2,E1,E2,r_firstE2,Foffset/2,F/2,clear,68*384/2,K/2,Z/2,(uint32_t*)d,(uint32_t*)soft_input,(uint16_t*)llr_buffer);
|
||||
}
|
||||
cudaError_t err=cudaPeekAtLastError();
|
||||
|
||||
if (err!=cudaSuccess) {
|
||||
printf("cuda error (nr_rate_matching_ldpc_rx_cuda): %s Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",cudaGetErrorString(err),Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
|
||||
exit(-1);
|
||||
}
|
||||
return(0);
|
||||
}
|
||||
|
||||
|
||||
@@ -19,6 +19,7 @@
|
||||
#include <stdint.h>
|
||||
#include <syscall.h>
|
||||
#include <time.h>
|
||||
#include <stdbool.h>
|
||||
// #define gNB_DEBUG_TRACE
|
||||
|
||||
#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
|
||||
@@ -66,6 +67,7 @@
|
||||
typedef struct nrLDPC_decoding_parameters_s {
|
||||
t_nrLDPC_dec_params decoderParms;
|
||||
|
||||
int r;
|
||||
uint8_t Qm;
|
||||
|
||||
uint8_t Kc;
|
||||
@@ -83,15 +85,16 @@ typedef struct nrLDPC_decoding_parameters_s {
|
||||
int E;
|
||||
short *llr;
|
||||
int16_t *d;
|
||||
bool *d_to_be_cleared;
|
||||
bool d_to_be_cleared;
|
||||
uint8_t *c;
|
||||
bool *decodeSuccess;
|
||||
|
||||
task_ans_t *ans;
|
||||
|
||||
time_stats_t *p_ts_deinterleave;
|
||||
time_stats_t *p_ts_rate_unmatch;
|
||||
time_stats_t *p_ts_ldpc_decode;
|
||||
time_stats_t ts_deinterleave;
|
||||
time_stats_t ts_rate_unmatch;
|
||||
time_stats_t ts_seg_prep;
|
||||
time_stats_t ts_ldpc_decode;
|
||||
} nrLDPC_decoding_parameters_t;
|
||||
|
||||
static void nr_process_decode_segment(void *arg)
|
||||
@@ -117,7 +120,7 @@ static void nr_process_decode_segment(void *arg)
|
||||
|
||||
//////////////////////////// ulsch_llr =====> ulsch_harq->e //////////////////////////////
|
||||
|
||||
start_meas(rdata->p_ts_deinterleave);
|
||||
start_meas(&rdata->ts_deinterleave);
|
||||
|
||||
/// code blocks after bit selection in rate matching for LDPC code (38.212 V15.4.0 section 5.4.2.1)
|
||||
int16_t harq_e[E];
|
||||
@@ -126,9 +129,9 @@ static void nr_process_decode_segment(void *arg)
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
stop_meas(rdata->p_ts_deinterleave);
|
||||
stop_meas(&rdata->ts_deinterleave);
|
||||
|
||||
start_meas(rdata->p_ts_rate_unmatch);
|
||||
start_meas(&rdata->ts_rate_unmatch);
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
//////////////////////////////// nr_rate_matching_ldpc_rx ////////////////////////////////
|
||||
@@ -143,21 +146,20 @@ static void nr_process_decode_segment(void *arg)
|
||||
harq_e,
|
||||
rdata->C,
|
||||
rv_index,
|
||||
*rdata->d_to_be_cleared,
|
||||
rdata->d_to_be_cleared,
|
||||
E,
|
||||
rdata->F,
|
||||
K - rdata->F - 2 * (p_decoderParms->Z))
|
||||
== -1) {
|
||||
stop_meas(rdata->p_ts_rate_unmatch);
|
||||
LOG_E(PHY, "nrLDPC_coding_segment_decoder.c: Problem in rate_matching\n");
|
||||
stop_meas(&rdata->ts_rate_unmatch);
|
||||
LOG_E(PHY, "nrLDPC_coding_segment_decoder.c: Problem in rate_matching BG %d, Z %d, C %d, rv_index %d, E %d, F %d, K%d, K-F-2*Z %d\n",p_decoderParms->BG,p_decoderParms->Z,rdata->C,rv_index, E,rdata->F,K, K-rdata->F - 2*(p_decoderParms->Z));
|
||||
|
||||
// Task completed
|
||||
completed_task_ans(rdata->ans);
|
||||
return;
|
||||
}
|
||||
stop_meas(rdata->p_ts_rate_unmatch);
|
||||
stop_meas(&rdata->ts_rate_unmatch);
|
||||
|
||||
*rdata->d_to_be_cleared = false;
|
||||
|
||||
p_decoderParms->crc_type = crcType(rdata->C, A);
|
||||
p_decoderParms->Kprime = lenWithCrc(rdata->C, A);
|
||||
@@ -166,8 +168,8 @@ static void nr_process_decode_segment(void *arg)
|
||||
|
||||
int16_t z[68 * 384 + 16] __attribute__((aligned(16)));
|
||||
|
||||
start_meas(rdata->p_ts_ldpc_decode);
|
||||
|
||||
start_meas(&rdata->ts_seg_prep);
|
||||
memset(z, 0, 2 * rdata->Z * sizeof(*z));
|
||||
// set Filler bits
|
||||
memset(z + Kprime, 127, rdata->F * sizeof(*z));
|
||||
@@ -182,6 +184,7 @@ static void nr_process_decode_segment(void *arg)
|
||||
for (int i = 0, j = 0; j < ((Kc * rdata->Z) >> 4) + 1; i += 2, j++) {
|
||||
pl[j] = simde_mm_packs_epi16(pv[i], pv[i + 1]);
|
||||
}
|
||||
stop_meas(&rdata->ts_seg_prep);
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
@@ -189,16 +192,18 @@ static void nr_process_decode_segment(void *arg)
|
||||
//////////////////////////////////////////////////////////////////////////////////////////
|
||||
|
||||
////////////////////////////////// pl =====> llrProcBuf //////////////////////////////////
|
||||
int decodeIterations = LDPCdecoder(p_decoderParms, l, llrProcBuf, p_procTime, rdata->abort_decode);
|
||||
|
||||
start_meas(&rdata->ts_ldpc_decode);
|
||||
int decodeIterations = LDPCdecoder(p_decoderParms, l, (uint8_t*)llrProcBuf, p_procTime, rdata->abort_decode);
|
||||
AssertFatal(rdata->c,"rdata->c is null, A %d, K %d\n",rdata->A,rdata->K);
|
||||
if (decodeIterations < p_decoderParms->numMaxIter) {
|
||||
memcpy(rdata->c, llrProcBuf, K >> 3);
|
||||
*rdata->decodeSuccess = true;
|
||||
} else {
|
||||
LOG_D(PHY,"Decoding failed: K %d, Z %d, rv_index %d\n",K,rdata->Z,rdata->rv_index);
|
||||
memset(rdata->c, 0, K >> 3);
|
||||
*rdata->decodeSuccess = false;
|
||||
}
|
||||
stop_meas(rdata->p_ts_ldpc_decode);
|
||||
stop_meas(&rdata->ts_ldpc_decode);
|
||||
|
||||
// Task completed
|
||||
completed_task_ans(rdata->ans);
|
||||
@@ -206,7 +211,8 @@ static void nr_process_decode_segment(void *arg)
|
||||
|
||||
int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters,
|
||||
int pusch_id,
|
||||
thread_info_tm_t *t_info)
|
||||
thread_info_tm_t *t_info
|
||||
)
|
||||
{
|
||||
nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
|
||||
|
||||
@@ -215,45 +221,61 @@ int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_de
|
||||
decParams.BG = nrLDPC_TB_decoding_parameters->BG;
|
||||
decParams.Z = nrLDPC_TB_decoding_parameters->Z;
|
||||
decParams.numMaxIter = nrLDPC_TB_decoding_parameters->max_ldpc_iterations;
|
||||
decParams.outMode = 0;
|
||||
|
||||
decParams.outMode = nrLDPC_outMode_BIT;
|
||||
|
||||
for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
|
||||
nrLDPC_decoding_parameters_t *rdata = &((nrLDPC_decoding_parameters_t *)t_info->buf)[t_info->len];
|
||||
DevAssert(t_info->len < t_info->cap);
|
||||
rdata->ans = t_info->ans;
|
||||
t_info->len += 1;
|
||||
{
|
||||
nrLDPC_decoding_parameters_t *rdata = &((nrLDPC_decoding_parameters_t *)t_info->buf)[t_info->len];
|
||||
DevAssert(t_info->len < t_info->cap);
|
||||
rdata->ans = t_info->ans;
|
||||
t_info->len += 1;
|
||||
int llr_offset;
|
||||
if (r<nrLDPC_TB_decoding_parameters->first_rE2) {
|
||||
decParams.R = nrLDPC_TB_decoding_parameters->R;
|
||||
rdata->E = nrLDPC_TB_decoding_parameters->E;
|
||||
llr_offset=r*rdata->E;
|
||||
}
|
||||
else {
|
||||
decParams.R = nrLDPC_TB_decoding_parameters->R2;
|
||||
rdata->E = nrLDPC_TB_decoding_parameters->E2;
|
||||
llr_offset=nrLDPC_TB_decoding_parameters->first_rE2*nrLDPC_TB_decoding_parameters->E + (r-nrLDPC_TB_decoding_parameters->first_rE2)*rdata->E;
|
||||
}
|
||||
rdata->r = r;
|
||||
rdata->decoderParms = decParams;
|
||||
rdata->Kc = decParams.BG == 2 ? 52 : 68;
|
||||
rdata->C = nrLDPC_TB_decoding_parameters->C;
|
||||
rdata->A = nrLDPC_TB_decoding_parameters->A;
|
||||
rdata->Qm = nrLDPC_TB_decoding_parameters->Qm;
|
||||
rdata->K = nrLDPC_TB_decoding_parameters->K;
|
||||
rdata->Z = nrLDPC_TB_decoding_parameters->Z;
|
||||
rdata->F = nrLDPC_TB_decoding_parameters->F;
|
||||
rdata->rv_index = nrLDPC_TB_decoding_parameters->rv_index;
|
||||
rdata->tbslbrm = nrLDPC_TB_decoding_parameters->tbslbrm;
|
||||
rdata->abort_decode = nrLDPC_TB_decoding_parameters->abort_decode;
|
||||
rdata->d = nrLDPC_TB_decoding_parameters->d + r*rdata->Kc*rdata->Z;
|
||||
rdata->d_to_be_cleared = nrLDPC_TB_decoding_parameters->d_to_be_cleared;
|
||||
rdata->c = nrLDPC_TB_decoding_parameters->c + r*(rdata->K>>3);
|
||||
AssertFatal(rdata->c!=NULL,"rdata->c is null, r %d, K %d, A %d, rv_index %d, TB_decoding_parameters->c %p\n",r,rdata->K,rdata->A,rdata->rv_index,nrLDPC_TB_decoding_parameters->c);
|
||||
rdata->llr = nrLDPC_TB_decoding_parameters->llr + llr_offset; //rdata->Kc*rdata->Z;
|
||||
rdata->decodeSuccess = &nrLDPC_TB_decoding_parameters->decodeSuccess[r];
|
||||
memset(&rdata->ts_deinterleave, 0, sizeof(rdata->ts_deinterleave));
|
||||
memset(&rdata->ts_rate_unmatch, 0, sizeof(rdata->ts_rate_unmatch));
|
||||
memset(&rdata->ts_seg_prep, 0, sizeof(rdata->ts_seg_prep));
|
||||
memset(&rdata->ts_ldpc_decode, 0, sizeof(rdata->ts_ldpc_decode));
|
||||
reset_meas(&rdata->ts_deinterleave);
|
||||
reset_meas(&rdata->ts_rate_unmatch);
|
||||
reset_meas(&rdata->ts_seg_prep);
|
||||
reset_meas(&rdata->ts_ldpc_decode);
|
||||
task_t t = {.func = &nr_process_decode_segment, .args = rdata};
|
||||
pushTpool(nrLDPC_slot_decoding_parameters->threadPool, t);
|
||||
|
||||
decParams.R = nrLDPC_TB_decoding_parameters->segments[r].R;
|
||||
rdata->decoderParms = decParams;
|
||||
rdata->llr = nrLDPC_TB_decoding_parameters->segments[r].llr;
|
||||
rdata->Kc = decParams.BG == 2 ? 52 : 68;
|
||||
rdata->C = nrLDPC_TB_decoding_parameters->C;
|
||||
rdata->E = nrLDPC_TB_decoding_parameters->segments[r].E;
|
||||
rdata->A = nrLDPC_TB_decoding_parameters->A;
|
||||
rdata->Qm = nrLDPC_TB_decoding_parameters->Qm;
|
||||
rdata->K = nrLDPC_TB_decoding_parameters->K;
|
||||
rdata->Z = nrLDPC_TB_decoding_parameters->Z;
|
||||
rdata->F = nrLDPC_TB_decoding_parameters->F;
|
||||
rdata->rv_index = nrLDPC_TB_decoding_parameters->rv_index;
|
||||
rdata->tbslbrm = nrLDPC_TB_decoding_parameters->tbslbrm;
|
||||
rdata->abort_decode = nrLDPC_TB_decoding_parameters->abort_decode;
|
||||
rdata->d = nrLDPC_TB_decoding_parameters->segments[r].d;
|
||||
rdata->d_to_be_cleared = nrLDPC_TB_decoding_parameters->segments[r].d_to_be_cleared;
|
||||
rdata->c = nrLDPC_TB_decoding_parameters->segments[r].c;
|
||||
rdata->decodeSuccess = &nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess;
|
||||
rdata->p_ts_deinterleave = &nrLDPC_TB_decoding_parameters->segments[r].ts_deinterleave;
|
||||
rdata->p_ts_rate_unmatch = &nrLDPC_TB_decoding_parameters->segments[r].ts_rate_unmatch;
|
||||
rdata->p_ts_ldpc_decode = &nrLDPC_TB_decoding_parameters->segments[r].ts_ldpc_decode;
|
||||
|
||||
task_t t = {.func = &nr_process_decode_segment, .args = rdata};
|
||||
pushTpool(nrLDPC_slot_decoding_parameters->threadPool, t);
|
||||
|
||||
LOG_D(PHY, "Added a block to decode, in pipe: %d\n", r);
|
||||
LOG_D(PHY, "Added a block to decode, in pipe: %d, rdata->c %p\n", r,rdata->c);
|
||||
}
|
||||
}
|
||||
return nrLDPC_TB_decoding_parameters->C;
|
||||
}
|
||||
|
||||
int32_t nrLDPC_coding_init(void)
|
||||
int32_t nrLDPC_coding_init(int max_num_pxsch)
|
||||
{
|
||||
return 0;
|
||||
}
|
||||
@@ -280,14 +302,22 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
|
||||
}
|
||||
|
||||
// Execute thread pool tasks
|
||||
join_task_ans(t_info.ans);
|
||||
join_task_ans(t_info.ans);
|
||||
|
||||
size_t r_t_info = 0;
|
||||
for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
|
||||
nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
|
||||
for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
|
||||
if (nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess) {
|
||||
*nrLDPC_TB_decoding_parameters->processedSegments = 0;
|
||||
for (int r=0; r<nrLDPC_TB_decoding_parameters->C;r++) {
|
||||
if (nrLDPC_TB_decoding_parameters->decodeSuccess[r]==true)
|
||||
*nrLDPC_TB_decoding_parameters->processedSegments = *nrLDPC_TB_decoding_parameters->processedSegments + 1;
|
||||
}
|
||||
|
||||
nrLDPC_decoding_parameters_t *rdata = &((nrLDPC_decoding_parameters_t *)t_info.buf)[r_t_info];
|
||||
r_t_info += 1;
|
||||
merge_meas(&nrLDPC_TB_decoding_parameters->ts_deinterleave, &rdata->ts_deinterleave);
|
||||
merge_meas(&nrLDPC_TB_decoding_parameters->ts_rate_unmatch, &rdata->ts_rate_unmatch);
|
||||
merge_meas(&nrLDPC_TB_decoding_parameters->ts_seg_prep, &rdata->ts_seg_prep);
|
||||
merge_meas(&nrLDPC_TB_decoding_parameters->ts_ldpc_decode, &rdata->ts_ldpc_decode);
|
||||
}
|
||||
}
|
||||
return 0;
|
||||
|
||||
@@ -0,0 +1,314 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
|
||||
* \brief Top-level routines for decoding LDPC transport channels
|
||||
*/
|
||||
|
||||
// [from gNB coding]
|
||||
#include "nr_rate_matching.h"
|
||||
#include "PHY/defs_gNB.h"
|
||||
#include "PHY/CODING/coding_extern.h"
|
||||
#include "PHY/CODING/coding_defs.h"
|
||||
#include "PHY/CODING/lte_interleaver_inline.h"
|
||||
#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
|
||||
#include "PHY/CODING/nrLDPC_extern.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_ulsch.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_dlsch.h"
|
||||
#include "SCHED_NR/sched_nr.h"
|
||||
#include "defs.h"
|
||||
#include "common/utils/LOG/vcd_signal_dumper.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
|
||||
#include <stdalign.h>
|
||||
#include <stdint.h>
|
||||
#include <syscall.h>
|
||||
#include <time.h>
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
#include "PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_shared_param.h"
|
||||
|
||||
// #define gNB_DEBUG_TRACE
|
||||
|
||||
#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
|
||||
// #define DEBUG_CRC
|
||||
#ifdef DEBUG_CRC
|
||||
#define PRINT_CRC_CHECK(a) a
|
||||
#else
|
||||
#define PRINT_CRC_CHECK(a)
|
||||
#endif
|
||||
|
||||
#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_interface.h"
|
||||
#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface.h"
|
||||
|
||||
#define USE_GPU_FOR_RM_DEINTER 1
|
||||
|
||||
cudaStream_t decoderStreams[MAX_NUM_DLSCH_SEGMENTS_DL];
|
||||
cudaStream_t decoderDoneEvents[MAX_NUM_DLSCH_SEGMENTS_DL];
|
||||
|
||||
int d_array_size = 0;
|
||||
|
||||
void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx);
|
||||
int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
|
||||
uint8_t BG,
|
||||
uint16_t Z,
|
||||
int16_t *d,
|
||||
int16_t *soft_input,
|
||||
int8_t *llr_buffer,
|
||||
uint32_t K,
|
||||
uint8_t C,
|
||||
uint8_t rvidx,
|
||||
uint8_t clear,
|
||||
uint32_t E1,
|
||||
uint32_t E2,
|
||||
uint32_t r_firstE2,
|
||||
uint32_t F,
|
||||
uint32_t Foffset,
|
||||
cudaStream_t *s,
|
||||
int8_t sidx);
|
||||
|
||||
extern int pageable,integrated;
|
||||
extern int8_t *p_llr_dev,*p_out_dev;
|
||||
int16_t **harq_d_array;
|
||||
int16_t *harq_d_array_dev;
|
||||
int16_t *harq_e_dev;
|
||||
int16_t *harq_f_dev;
|
||||
pthread_mutex_t decoder_mutex=PTHREAD_MUTEX_INITIALIZER;
|
||||
|
||||
extern
|
||||
int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
|
||||
int8_t* p_llr,
|
||||
uint8_t* p_out,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab);
|
||||
|
||||
void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *segs)
|
||||
{
|
||||
// arg points to RDATA array (nrLDPC_decoding_parameters_t *RDATA)
|
||||
DevAssert(segs != NULL);
|
||||
|
||||
// use seg0 as canonical
|
||||
const int C = segs->C;
|
||||
const int Z = segs->Z;
|
||||
const int Kc = segs->BG == 2 ? 52 : 68;
|
||||
const int K = segs->K;
|
||||
const int Kprime = K - segs->F;
|
||||
const int segLen = Kc*Z; // int16 length; after packing we store int8 [segLen]
|
||||
t_nrLDPC_time_stats procTime = {0};
|
||||
t_nrLDPC_time_stats *p_procTime = &procTime;
|
||||
|
||||
#ifdef USE_GPU_FOR_RM_DEINTER
|
||||
start_meas(&segs->ts_deinterleave);
|
||||
int E1 = segs->E;
|
||||
int E2 = segs->E2;
|
||||
int r_firstE2 = segs->first_rE2;
|
||||
|
||||
LOG_D(NR_PHY,"locking decoder (llr %p)\n",segs->llr);
|
||||
pthread_mutex_lock(&decoder_mutex);
|
||||
|
||||
// for PCIe GPU copy llrs to device memory
|
||||
if (!pageable&&!integrated) {
|
||||
LOG_I(NR_PHY,"cudaMemcpyAsynch llr->harq_f_dev\n");
|
||||
cudaMemcpyAsync(harq_f_dev,
|
||||
segs->llr,
|
||||
((r_firstE2*E1) + (C-r_firstE2)*E2)*sizeof(int16_t),
|
||||
cudaMemcpyHostToDevice,
|
||||
decoderStreams[0]);
|
||||
}
|
||||
#if 0
|
||||
if (1/*segs->rv_index==2*/)
|
||||
for (int r=0;r<C;r++) {
|
||||
int roffset = r < r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
|
||||
int E = r < r_firstE2 ? E1 : E2;
|
||||
for (int i=0;i<16;i++) {
|
||||
printf("f(%d,%d) %d\n",r,i,segs->llr[roffset + i]);
|
||||
}
|
||||
}
|
||||
#endif
|
||||
LOG_D(NR_PHY,"deinter: e %p llr %p\n",harq_e_dev,pageable || integrated ? segs->llr : harq_f_dev);
|
||||
launch_deinterleave_i16(segs->Qm,E1,E2,C,r_firstE2,harq_e_dev,pageable||integrated ? segs->llr : harq_f_dev,decoderStreams,0);
|
||||
stop_meas(&segs->ts_deinterleave);
|
||||
#if 0
|
||||
cudaError_t err;
|
||||
if (1/*segs->rv_index == 2*/) {
|
||||
int16_t elocal[E2];
|
||||
for (int r=0;r<C;r++) {
|
||||
int E = r<r_firstE2 ? E1 : E2;
|
||||
int r_off = r<r_firstE2 ? (r*E1) : ((r_firstE2*E1) + (r-r_firstE2)*E2);
|
||||
err = cudaMemcpyAsync(elocal,&harq_e_dev[r_off],sizeof(elocal),cudaMemcpyDeviceToHost,decoderStreams[0]);
|
||||
AssertFatal(err == cudaSuccess,"cudaMemcpyAsync failed with error %s, r_off %d\n",cudaGetErrorString(err),r_off);
|
||||
for (int i=0;i<E;i++) {
|
||||
if (elocal[i] != segs->llr[r_off + ((i<(E/2)) ? 2*i : (-E+1+2*i))]) {
|
||||
printf("e(%d,%d,%d) %d",r,i,E,elocal[i]);
|
||||
printf("(%d**)\n",segs->llr[r_off + ((i<(E/2)) ? 2*i : -E+1+2*i)]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
#endif
|
||||
start_meas(&segs->ts_rate_unmatch);
|
||||
//printf("Running RM with id %d, d_to_be_cleared %d, rv_idx %d, Z %d, C %d, E1 %d, E2 %d, F %d,r_firstE2 %d\n",segs->harq_unique_pid,segs->d_to_be_cleared,segs->rv_index,Z,C,E1,E2,segs->F,r_firstE2);
|
||||
#if 0
|
||||
if (segs->d_to_be_cleared == 1) err = cudaMemsetAsync(harq_d_array[segs->harq_unique_pid],0,C*68*Z*sizeof(int16_t),decoderStreams[0]);
|
||||
AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed with error %s on harq_d_array[%d] %p for %d bytes\n",cudaGetErrorString(err),segs->harq_unique_pid,harq_d_array[segs->harq_unique_pid],C*68*Z*sizeof(int16_t));
|
||||
cudaMemsetAsync(p_llr_dev,0,C*68*Z*sizeof(int8_t),decoderStreams[0]);
|
||||
AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed on p_llr_dev %p\n",p_llr_dev);
|
||||
#endif
|
||||
AssertFatal(segs->harq_unique_pid < d_array_size,"harq_unique_pid %d > %d\n",segs->harq_unique_pid,d_array_size);
|
||||
nr_rate_matching_ldpc_rx_cuda(segs->tbslbrm,
|
||||
segs->BG,
|
||||
Z,
|
||||
harq_d_array[segs->harq_unique_pid],
|
||||
harq_e_dev,
|
||||
p_llr_dev,
|
||||
K,
|
||||
C, // TB segments count
|
||||
segs->rv_index,
|
||||
segs->d_to_be_cleared,
|
||||
E1,
|
||||
E2,
|
||||
r_firstE2,
|
||||
segs->F,
|
||||
Kprime - 2 * Z,
|
||||
decoderStreams,
|
||||
0);
|
||||
for (int r = 0; r < C; ++r) {
|
||||
cudaMemsetAsync(p_llr_dev + (size_t)r*segLen,0,sizeof(int8_t)*2*Z,decoderStreams[0]);
|
||||
#if 0
|
||||
int8_t llr_local[segLen];
|
||||
if (r==1 && segs->rv_index==2) {
|
||||
cudaMemcpyAsync(llr_local,p_llr_dev+(size_t)r*segLen,sizeof(int8_t)*segLen,cudaMemcpyDeviceToHost,decoderStreams[0]);
|
||||
for (int i=0;i<segLen;i++) printf("llr(%d,%d,%d/%d) %d\n",segs->rv_index,r,i,segLen,llr_local[i]);
|
||||
}
|
||||
#endif
|
||||
}
|
||||
stop_meas(&segs->ts_rate_unmatch);
|
||||
|
||||
#else //USE_GPU_FOR_RM_DEINTER
|
||||
int16_t *z_local = (int16_t*)alloca(sizeof(int16_t) * segLen); // segLen is safe small
|
||||
for (int r = 0; r < C; ++r) {
|
||||
// deinterleave
|
||||
start_meas(&segs->ts_deinterleave);
|
||||
int16_t *harq_e = (int16_t*)alloca(sizeof(int16_t) * segs->segments[r].E);
|
||||
// for (int i=0;i<segs->segments[r].E;i++) printf("llr_in[%d] %d\n",i,segs->segments[r].llr[i]);
|
||||
nr_deinterleaving_ldpc(segs->segments[r].E, segs->Qm, harq_e, segs->segments[r].llr);
|
||||
// for (int i=0;i<16;i++) printf("harq_e[%d] %d\n",i,harq_e[i]);
|
||||
stop_meas(&segs->segments[0].ts_deinterleave);
|
||||
// rate matching
|
||||
start_meas(&segs->segments[0].ts_rate_unmatch);
|
||||
if (nr_rate_matching_ldpc_rx(segs->tbslbrm,
|
||||
segs->BG,
|
||||
Z,
|
||||
segs->segments[r].d,
|
||||
harq_e,
|
||||
C, // TB segments count
|
||||
segs->rv_index,
|
||||
segs->segments[r].d_to_be_cleared,
|
||||
segs->segments[r].E,
|
||||
segs->F,
|
||||
Kprime - 2 * Z) == -1) {
|
||||
stop_meas(&segs->segments[0].ts_rate_unmatch);
|
||||
LOG_E(PHY,"rate matching failed seg %d\n", r);
|
||||
memset(segs->segments[r].c, 0, K);
|
||||
//*rdata->decodeSuccess = false;
|
||||
continue; // skip this segment
|
||||
}
|
||||
stop_meas(&segs->segments[0].ts_rate_unmatch);
|
||||
start_meas(&segs->segments[0].ts_seg_prep);
|
||||
segs->segments[r].d_to_be_cleared = false;
|
||||
|
||||
memset(z_local,0,sizeof(int16_t)*2*Z);
|
||||
memset(z_local + Kprime,127,sizeof(int16_t)*segs->F);
|
||||
memcpy(z_local + 2*Z, segs->segments[r].d, (size_t)(Kprime - 2*Z)*sizeof(int16_t));
|
||||
memcpy(z_local + K, segs->segments[r].d + (K - 2*Z), (size_t)(Kc*Z - K)*sizeof(int16_t));
|
||||
|
||||
// pack int16 -> int8 into llrBuffer[r * segLen]
|
||||
simde__m128i *pv = (simde__m128i*)z_local;
|
||||
simde__m128i *pl = (simde__m128i*)(llrBuffer + (size_t)r*segLen);
|
||||
int vecCount = ((Kc * Z) >> 4);
|
||||
for (int j=0, idx=0; j<vecCount; ++j, idx+=2) {
|
||||
pl[j] = simde_mm_packs_epi16(pv[idx], pv[idx+1]);
|
||||
}
|
||||
stop_meas(&segs->segments[0].ts_seg_prep);
|
||||
}
|
||||
#endif
|
||||
|
||||
start_meas(&segs->ts_ldpc_decode);
|
||||
|
||||
t_nrLDPC_dec_params decParams = {.check_crc = check_crc};
|
||||
decParams.Z = Z;
|
||||
decParams.R = segs->R;
|
||||
decParams.BG = segs->BG;
|
||||
decParams.crc_type = crcType(C, segs->A);
|
||||
decParams.Kprime = lenWithCrc(C, segs->A);
|
||||
decParams.n_segments = C;
|
||||
decParams.outMode=nrLDPC_outMode_BIT;
|
||||
decParams.numMaxIter = segs->max_ldpc_iterations;
|
||||
|
||||
LOG_D(NR_PHY,"decoder (llr %p): %d segments, Z %d, R %d \n",segs->llr,C,Z,segs->R);
|
||||
int decodeIterations = LDPCdecoder_cuda(&decParams, p_llr_dev, segs->c, p_procTime, segs->abort_decode);
|
||||
stop_meas(&segs->ts_ldpc_decode);
|
||||
|
||||
if (decodeIterations <= segs->max_ldpc_iterations) {
|
||||
for (int r=0; r<C; r++) segs->decodeSuccess[r] = true;
|
||||
LOG_D(NR_PHY,"Set all segs->decodeSuccess to true\n");
|
||||
} else {
|
||||
memset(segs->c, 0, C*(K>>3));
|
||||
for (int r=0; r<C; r++) segs->decodeSuccess[r] = false;
|
||||
LOG_D(NR_PHY,"Set all segs->decodeSuccess to false\n");
|
||||
}
|
||||
LOG_D(NR_PHY,"unlocking decoder (llr %p)\n",segs->llr);
|
||||
pthread_mutex_unlock(&decoder_mutex);
|
||||
}
|
||||
|
||||
|
||||
#define MAXE 4*14*273*12*8 // 4 antennas, 14 symbols, 273 PRBs, 12 RE/prb, 8 bits/RE
|
||||
|
||||
|
||||
void LDPCint_rm_init(int max_num_pxsch) {
|
||||
|
||||
LOG_I(NR_PHY,"RM init for %d pxsch\n",max_num_pxsch);
|
||||
LOG_I(NR_PHY,"Allocating device array for harq_d/harq_e \n");
|
||||
cudaError_t err=cudaMalloc((void **)&harq_e_dev,MAXE*sizeof(int16_t));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq_e_dev): %s\n", cudaGetErrorString(err));
|
||||
harq_d_array = malloc(sizeof(int16_t*) * max_num_pxsch);
|
||||
err = cudaMalloc((void *)&harq_d_array_dev,sizeof(int16_t*) * max_num_pxsch);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_array_dev): %s\n", cudaGetErrorString(err));
|
||||
LOG_I(PHY,"Allocated %ld bytes for harq_d_array_dev @ %p\n",sizeof(int16_t*)*max_num_pxsch,harq_d_array_dev);
|
||||
for (int i=0;i<max_num_pxsch;i++) {
|
||||
err=cudaMalloc((void **)&harq_d_array[i],MAX_NUM_DLSCH_SEGMENTS_DL*OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t));
|
||||
LOG_I(PHY,"Allocating %ld bytes for harq_d_array[%d] @ %p\n",MAX_NUM_DLSCH_SEGMENTS_DL*OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t),i,harq_d_array[i]);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_dev): %s\n", cudaGetErrorString(err));
|
||||
}
|
||||
cudaMemcpy(harq_d_array_dev,harq_d_array,sizeof(int16_t*)*max_num_pxsch,cudaMemcpyHostToDevice);
|
||||
if (!pageable && !integrated) {
|
||||
LOG_I(PHY,"Allocating device array for harq_f \n");
|
||||
err=cudaMalloc((void **)&harq_f_dev,MAXE*sizeof(int16_t));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq_f_dev): %s\n", cudaGetErrorString(err));
|
||||
}
|
||||
d_array_size = max_num_pxsch;
|
||||
}
|
||||
|
||||
extern void LDPCinit_cuda(void);
|
||||
extern void cuda_support_init();
|
||||
|
||||
int32_t nrLDPC_coding_init_cuda(int max_num_pxsch)
|
||||
{
|
||||
cuda_support_init();
|
||||
|
||||
LDPCinit_cuda();
|
||||
LDPCint_rm_init(max_num_pxsch);
|
||||
return 0;
|
||||
}
|
||||
|
||||
void LDPCshutdown_cuda(void);
|
||||
|
||||
int32_t nrLDPC_coding_shutdown_cuda(void)
|
||||
{
|
||||
LDPCshutdown_cuda();
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -35,6 +35,9 @@
|
||||
* \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits
|
||||
* \param Eoffset offset in number of bits of the first segment of the segment group within output
|
||||
*/
|
||||
|
||||
int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters);
|
||||
|
||||
static void write_task_output(uint8_t *f,
|
||||
uint32_t E,
|
||||
uint8_t *f2,
|
||||
@@ -343,9 +346,11 @@ static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_s
|
||||
encoder_implemparams_t common_segment_params = {
|
||||
.n_segments = nrLDPC_TB_encoding_parameters->C,
|
||||
.tinput = nrLDPC_slot_encoding_parameters->tinput,
|
||||
.tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
|
||||
.tprep = nrLDPC_slot_encoding_parameters->tprep,
|
||||
.tparity = nrLDPC_slot_encoding_parameters->tparity,
|
||||
.toutput = nrLDPC_slot_encoding_parameters->toutput,
|
||||
.tconcat = nrLDPC_slot_encoding_parameters->tconcat,
|
||||
.Kb = nrLDPC_TB_encoding_parameters->Kb,
|
||||
.Zc = nrLDPC_TB_encoding_parameters->Z,
|
||||
.BG = nrLDPC_TB_encoding_parameters->BG,
|
||||
@@ -382,16 +387,15 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
// Compute number of tasks to encode TB
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
|
||||
nbTasks += n_seg;
|
||||
size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
|
||||
nbTasks += n_seg;
|
||||
|
||||
// Search for maximum E for sizing encoder output f and f2
|
||||
for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
|
||||
uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
|
||||
Emax = E > Emax ? E : Emax;
|
||||
}
|
||||
// Search for maximum E for sizing encoder output f and f2
|
||||
for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
|
||||
uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
|
||||
Emax = E > Emax ? E : Emax;
|
||||
}
|
||||
}
|
||||
|
||||
// Create f and f2 to old encoding tasks outputs
|
||||
uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
|
||||
uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
|
||||
@@ -425,16 +429,17 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
|
||||
completed_many_task_ans(&ans, nbTasks - nbEncode);
|
||||
}
|
||||
// Execute thread pool tasks
|
||||
join_task_ans(&ans);
|
||||
if (nbTasks > 0) join_task_ans(&ans);
|
||||
|
||||
// Write output
|
||||
time_stats_t *tconcat = nrLDPC_slot_encoding_parameters->tconcat;
|
||||
if(tconcat != NULL) start_meas(tconcat);
|
||||
nbTasks = 0;
|
||||
for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
|
||||
nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
|
||||
uint32_t C = nrLDPC_TB_encoding_parameters->C;
|
||||
size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1));
|
||||
|
||||
time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput;
|
||||
|
||||
for (int j = 0; j < n_seg; j++) {
|
||||
unsigned int macro_segment = j * 8;
|
||||
@@ -454,7 +459,6 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
|
||||
}
|
||||
}
|
||||
|
||||
if(toutput != NULL) start_meas(toutput);
|
||||
|
||||
uint32_t Eoffset=0;
|
||||
for (int s=0; s<macro_segment; s++)
|
||||
@@ -470,10 +474,10 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
|
||||
nrLDPC_TB_encoding_parameters->output,
|
||||
Eoffset);
|
||||
|
||||
if(toutput != NULL) stop_meas(toutput);
|
||||
}
|
||||
nbTasks += n_seg;
|
||||
}
|
||||
if(tconcat != NULL) stop_meas(tconcat);
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,987 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
|
||||
* \brief Top-level routines for implementing LDPC encoding of transport channels
|
||||
*/
|
||||
|
||||
#include "nr_rate_matching.h"
|
||||
#include "PHY/defs_gNB.h"
|
||||
#include "PHY/CODING/coding_extern.h"
|
||||
#include "PHY/CODING/coding_defs.h"
|
||||
#include "PHY/CODING/lte_interleaver_inline.h"
|
||||
#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
|
||||
#include "PHY/CODING/nrLDPC_extern.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
|
||||
#include "PHY/NR_TRANSPORT/nr_dlsch.h"
|
||||
#include "SCHED_NR/sched_nr.h"
|
||||
#include "common/utils/LOG/vcd_signal_dumper.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include "common/utils/nr/nr_common.h"
|
||||
#include <openair2/UTIL/OPT/opt.h>
|
||||
|
||||
#include <syscall.h>
|
||||
|
||||
//#define DEBUG_LDPC_ENCODING
|
||||
//#define DEBUG_LDPC_ENCODING_FREE 1
|
||||
|
||||
static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
|
||||
|
||||
extern uint32_t **d_host;
|
||||
|
||||
static void unpack_output(uint32_t *f,
|
||||
uint32_t E,
|
||||
uint32_t *f2,
|
||||
uint32_t E2,
|
||||
uint32_t E2_first_segment32,
|
||||
uint32_t E2_first_segment,
|
||||
uint32_t nb_segments,
|
||||
uint8_t *output) {
|
||||
|
||||
uint32_t s;
|
||||
// int s0;
|
||||
uint32_t *fp;
|
||||
int foffset;
|
||||
uint32_t *output_p = (uint32_t *)output;
|
||||
//printf("unpack: E %d, E2 %d, E2_first_segment %d, E2_first_segment32 %d, nb_segments %d\n",E,E2,E2_first_segment,E2_first_segment32,nb_segments);
|
||||
|
||||
uint32_t bit_index = 0;
|
||||
#ifdef __AVX2__
|
||||
simde__m256i shift0=simde_mm256_set_epi32(7,6,5,4,3,2,1,0);
|
||||
simde__m256i shift1=simde_mm256_set_epi32(15,14,13,12,11,10,9,8);
|
||||
simde__m256i shift2=simde_mm256_set_epi32(23,22,21,20,19,18,17,16);
|
||||
simde__m256i shift3=simde_mm256_set_epi32(31,30,29,28,27,26,25,24);
|
||||
simde__m256i vmask0=simde_mm256_set_epi32(0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1);
|
||||
simde__m256i vmask1=simde_mm256_set_epi32(0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100);
|
||||
simde__m256i vmask2=simde_mm256_set_epi32(0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000);
|
||||
simde__m256i vmask3=simde_mm256_set_epi32(0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000);
|
||||
int s2=0;
|
||||
for (s = 0; s < E2_first_segment ; s++) {
|
||||
s2 = s&31;
|
||||
foffset = (s>>5)*E;
|
||||
fp = f+foffset;
|
||||
int i;
|
||||
if ((bit_index&31) == 0 ) {
|
||||
for (i = 0; i < (E>>5)<<5; i+=32) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
*(output_p + (bit_index>>5)) = simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7);
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t Emod32=E&31;
|
||||
if (Emod32 != 0) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
*(output_p + (bit_index>>5)) = (simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
|
||||
bit_index+=Emod32;
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (i = 0; i < (E>>5)<<5; i+=32) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
uint32_t tmp = simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t Emod32=E&31;
|
||||
if (Emod32 != 0) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
uint32_t tmp = (simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=Emod32;
|
||||
}
|
||||
}
|
||||
}
|
||||
for ( ; s < nb_segments ; s++){
|
||||
s2 = s&31;
|
||||
foffset = ((s>>5)-E2_first_segment32)*E2;
|
||||
fp = f2+foffset;
|
||||
int i;
|
||||
if ((bit_index&31) == 0 ) {
|
||||
for (i = 0; i < (E2>>5)<<5; i+=32) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
*(output_p + (bit_index>>5)) = simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7);
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t E2mod32=E2&31;
|
||||
if (E2mod32 != 0) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
*(output_p + (bit_index>>5)) = (simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
|
||||
bit_index+=E2mod32;
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (i = 0; i < (E2>>5)<<5; i+=32) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
uint32_t tmp = simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t E2mod32=E2&31;
|
||||
if (E2mod32 != 0) {
|
||||
simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);
|
||||
simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
|
||||
f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);
|
||||
cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
|
||||
uint32_t tmp = (simde_mm256_extract_epi32(cshift,0) |
|
||||
simde_mm256_extract_epi32(cshift,1) |
|
||||
simde_mm256_extract_epi32(cshift,2) |
|
||||
simde_mm256_extract_epi32(cshift,3) |
|
||||
simde_mm256_extract_epi32(cshift,4) |
|
||||
simde_mm256_extract_epi32(cshift,5) |
|
||||
simde_mm256_extract_epi32(cshift,6) |
|
||||
simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=E2mod32;
|
||||
}
|
||||
}
|
||||
}
|
||||
#elif defined(__aarch64__)
|
||||
const int32_t ucShift0[32][4] = { {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}, {-28,-27,-26,-25}, {-29,-28,-27,-26}, {-30,-29,-28,-27}, {-31,-30,-29,-28}};
|
||||
|
||||
const int32_t ucShift1[32][4] = { {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}};
|
||||
|
||||
const int32_t ucShift2[32][4] = { {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19},{-23,-22,-21,-20}};
|
||||
|
||||
const int32_t ucShift3[32][4] = { {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}};
|
||||
|
||||
const int32_t ucShift4[32][4] = { {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}};
|
||||
|
||||
const int32_t ucShift5[32][4] = { {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}};
|
||||
|
||||
const int32_t ucShift6[32][4] = { {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}};
|
||||
|
||||
const int32_t ucShift7[32][4] = { {28,29,30,31}, {27,28,29,30}, {26,27,28,29}, {25,26,27,28}, {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks0[4] = {0x1,0x2,0x4,0x8};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks1[4] = {0x10,0x20,0x40,0x80};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks2[4] = {0x100,0x200,0x400,0x800};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks3[4] = {0x1000,0x2000,0x4000,0x8000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks4[4] = {0x10000,0x20000,0x40000,0x80000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks5[4] = {0x100000,0x200000,0x400000,0x800000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks6[4] = {0x1000000,0x2000000,0x4000000,0x8000000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masks7[4] = {0x10000000,0x20000000,0x40000000,0x80000000};
|
||||
int32x4_t vshift0[32],vshift1[32],vshift2[32],vshift3[32],vshift4[32],vshift5[32],vshift6[32],vshift7[32];
|
||||
for (int n=0;n<32;n++) {
|
||||
vshift0[n] = vld1q_s32(ucShift0[n]);
|
||||
vshift1[n] = vld1q_s32(ucShift1[n]);
|
||||
vshift2[n] = vld1q_s32(ucShift2[n]);
|
||||
vshift3[n] = vld1q_s32(ucShift3[n]);
|
||||
vshift4[n] = vld1q_s32(ucShift4[n]);
|
||||
vshift5[n] = vld1q_s32(ucShift5[n]);
|
||||
vshift6[n] = vld1q_s32(ucShift6[n]);
|
||||
vshift7[n] = vld1q_s32(ucShift7[n]);
|
||||
}
|
||||
uint32x4_t vmask0 = vld1q_u32(masks0);
|
||||
uint32x4_t vmask1 = vld1q_u32(masks1);
|
||||
uint32x4_t vmask2 = vld1q_u32(masks2);
|
||||
uint32x4_t vmask3 = vld1q_u32(masks3);
|
||||
uint32x4_t vmask4 = vld1q_u32(masks4);
|
||||
uint32x4_t vmask5 = vld1q_u32(masks5);
|
||||
uint32x4_t vmask6 = vld1q_u32(masks6);
|
||||
uint32x4_t vmask7 = vld1q_u32(masks7);
|
||||
int s2=0;
|
||||
for (s = 0; s < E2_first_segment ; s++) {
|
||||
s2 = s&31;
|
||||
foffset = (s>>5)*E;
|
||||
fp = f+foffset;
|
||||
int i;
|
||||
if ((bit_index&31) == 0 ) {
|
||||
for (i = 0; i < (E>>5)<<5; i+=32) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
|
||||
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift);
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t Emod32=E&31;
|
||||
if (Emod32 != 0) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
|
||||
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift)&((1<<Emod32)-1);
|
||||
bit_index+=Emod32;
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (i = 0; i < (E>>5)<<5; i+=32) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
/*
|
||||
uint32_t tmp = vaddvq_u32(cshift);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
*/
|
||||
uint64_t tmp = (uint64_t)vaddvq_u32(cshift);
|
||||
*(uint64_t*)(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t Emod32=E&31;
|
||||
if (Emod32 != 0) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
uint32_t tmp = vaddvq_u32(cshift);
|
||||
tmp&=((1<<Emod32)-1);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=Emod32;
|
||||
}
|
||||
}
|
||||
}
|
||||
// s0 = s;
|
||||
for ( ; s < nb_segments ; s++){
|
||||
s2 = s&31;
|
||||
foffset = ((s>>5)-E2_first_segment32)*E2;
|
||||
fp = f2+foffset;
|
||||
int i;
|
||||
if ((bit_index&31) == 0 ) {
|
||||
for (i = 0; i < (E2>>5)<<5; i+=32) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift);
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t E2mod32=E2&31;
|
||||
if (E2mod32 != 0) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
*(output_p + (bit_index>>5)) = vaddvq_u32(cshift)&((1<<E2mod32)-1);
|
||||
bit_index+=E2mod32;
|
||||
}
|
||||
}
|
||||
else {
|
||||
for (i = 0; i < (E2>>5)<<5; i+=32) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
|
||||
uint32_t tmp = vaddvq_u32(cshift);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=32;
|
||||
}
|
||||
uint32_t E2mod32=E2&31;
|
||||
if (E2mod32 != 0) {
|
||||
uint32x4_t *fp128 = (uint32x4_t*)&fp[i];
|
||||
uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
|
||||
cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
|
||||
uint32_t tmp = vaddvq_u32(cshift);
|
||||
tmp&=((1<<E2mod32)-1);
|
||||
*(output_p + (bit_index>>5)) |= (tmp<<(bit_index&31));
|
||||
*(output_p + (bit_index>>5)+1) |= (tmp>>(32-(bit_index&31)));
|
||||
bit_index+=E2mod32;
|
||||
}
|
||||
}
|
||||
}
|
||||
#else // non SIMD version
|
||||
unsigned int segpos,s2;
|
||||
for (s = 0; s < E2_first_segment ; s++) {
|
||||
foffset = (s>>5)*E;
|
||||
fp = f+foffset;
|
||||
s2=s&31;
|
||||
segpos = (1<<s2);
|
||||
// printf("E %d s %d: foffset %d, s2 %d, segpos %u\n",E,s,foffset,s2,segpos);
|
||||
for (int i = 0; i < E; i++) {
|
||||
output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31);
|
||||
//printf("bit_index %d, output_p[%d] %x\n",bit_index, bit_index>>5,output_p[bit_index>>5]);
|
||||
bit_index++;
|
||||
}
|
||||
}
|
||||
for ( ; s < nb_segments ; s++){
|
||||
foffset = ((s>>5)-E2_first_segment32)*E2;
|
||||
fp = f2+foffset;
|
||||
s2=s&31;
|
||||
segpos = (1<<s2);
|
||||
// printf("E2 %d s %d: foffset %d, s2 %d, segpos %u\n",E2,s,foffset,s2,segpos);
|
||||
for (int i = 0; i < E2; i++) {
|
||||
output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31);
|
||||
bit_index++;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
|
||||
static
|
||||
int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
|
||||
uint8_t BG,
|
||||
uint16_t Z,
|
||||
uint32_t *d,
|
||||
uint32_t *e,
|
||||
uint8_t C,
|
||||
uint32_t F,
|
||||
uint32_t Foffset,
|
||||
uint8_t rvidx,
|
||||
uint32_t E)
|
||||
{
|
||||
if (C == 0) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
|
||||
return -1;
|
||||
}
|
||||
|
||||
//Bit selection
|
||||
uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
|
||||
uint32_t Ncb;
|
||||
if (Tbslbrm == 0)
|
||||
Ncb = N;
|
||||
else {
|
||||
uint32_t Nref = 3 * Tbslbrm / (2 * C); //R_LBRM = 2/3
|
||||
Ncb = min(N, Nref);
|
||||
}
|
||||
|
||||
uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
|
||||
|
||||
#ifdef RM_DEBUG
|
||||
printf("nr_rate_matching_ldpc: E %u, F %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
|
||||
E,
|
||||
F,
|
||||
Foffset,
|
||||
ind,
|
||||
Ncb,
|
||||
rvidx,
|
||||
Tbslbrm);
|
||||
#endif
|
||||
|
||||
if (Foffset > E) {
|
||||
LOG_E(PHY,
|
||||
"nr_rate_matching: invalid parameters (Foffset %d > E %d) F %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n",
|
||||
Foffset,
|
||||
E,
|
||||
F,
|
||||
ind,
|
||||
Ncb,
|
||||
rvidx,
|
||||
Tbslbrm);
|
||||
return -1;
|
||||
}
|
||||
if (Foffset > Ncb) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
|
||||
return -1;
|
||||
}
|
||||
|
||||
if (ind >= Foffset && ind < (F + Foffset))
|
||||
ind = F + Foffset;
|
||||
|
||||
uint32_t k = 0;
|
||||
if (ind < Foffset) { // case where we have some bits before the filler and the rest after
|
||||
memcpy((void *)e, (void *)(d + ind), (Foffset - ind)<<2);
|
||||
|
||||
if (E + F <= Ncb - ind) { // E+F doesn't contain all coded bits
|
||||
memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (E - Foffset + ind)<<2);
|
||||
k = E;
|
||||
} else {
|
||||
memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (Ncb - Foffset - F)<<2);
|
||||
k = Ncb - F - ind;
|
||||
}
|
||||
} else {
|
||||
if (E <= Ncb - ind) { // E+F doesn't contain all coded bits
|
||||
memcpy((void *)(e), (void *)(d + ind), E<<2);
|
||||
k = E;
|
||||
} else {
|
||||
memcpy((void *)(e), (void *)(d + ind), (Ncb - ind)<<2);
|
||||
k = Ncb - ind;
|
||||
}
|
||||
}
|
||||
|
||||
while (k < E) { // case where we do repetitions (low mcs)
|
||||
for (ind = 0; (ind < Ncb) && (k < E); ind++) {
|
||||
#ifdef RM_DEBUG
|
||||
printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
|
||||
#endif
|
||||
|
||||
if (ind == Foffset)
|
||||
ind = F + Foffset; // skip filler bits
|
||||
|
||||
e[k++] = d[ind];
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static
|
||||
void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
|
||||
{
|
||||
const uint32_t EQm = E / Qm;
|
||||
memset(f, 0, E * sizeof(uint32_t));
|
||||
switch(Qm) {
|
||||
case 2: {
|
||||
uint32_t *e0 = e;
|
||||
uint32_t *e1 = e + EQm;
|
||||
int i = 0;
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
}
|
||||
} break;
|
||||
case 4: {
|
||||
uint32_t *e0 = e;
|
||||
uint32_t *e1 = e0 + EQm;
|
||||
uint32_t *e2 = e1 + EQm;
|
||||
uint32_t *e3 = e2 + EQm;
|
||||
int i = 0;
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
*f++ = *e2++;
|
||||
*f++ = *e3++;
|
||||
}
|
||||
} break;
|
||||
case 6: {
|
||||
uint32_t *e0 = e;
|
||||
uint32_t *e1 = e0 + EQm;
|
||||
uint32_t *e2 = e1 + EQm;
|
||||
uint32_t *e3 = e2 + EQm;
|
||||
uint32_t *e4 = e3 + EQm;
|
||||
uint32_t *e5 = e4 + EQm;
|
||||
int i = 0;
|
||||
#if 0
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *e4_128 = (simde__m128i *)e4;
|
||||
simde__m128i *e5_128 = (simde__m128i *)e5;
|
||||
simde__m128i *f128 = (simde__m128i *)f;
|
||||
for (; i < (EQm & ~3); i += 4) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
|
||||
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
|
||||
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
|
||||
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
|
||||
|
||||
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
|
||||
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
|
||||
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
|
||||
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
|
||||
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
|
||||
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
|
||||
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
|
||||
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2)); // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
|
||||
}
|
||||
e0 = (uint32_t *)e0_128;
|
||||
e1 = (uint32_t *)e1_128;
|
||||
e2 = (uint32_t *)e2_128;
|
||||
e3 = (uint32_t *)e3_128;
|
||||
e4 = (uint32_t *)e4_128;
|
||||
e5 = (uint32_t *)e5_128;
|
||||
f = (uint32_t *)f128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*f++ = *e0++;
|
||||
*f++ = *e1++;
|
||||
*f++ = *e2++;
|
||||
*f++ = *e3++;
|
||||
*f++ = *e4++;
|
||||
*f++ = *e5++;
|
||||
}
|
||||
} break;
|
||||
case 8: {
|
||||
uint32_t *e0 = e;
|
||||
uint32_t *e1 = e0 + EQm;
|
||||
uint32_t *e2 = e1 + EQm;
|
||||
uint32_t *e3 = e2 + EQm;
|
||||
uint32_t *e4 = e3 + EQm;
|
||||
uint32_t *e5 = e4 + EQm;
|
||||
uint32_t *e6 = e5 + EQm;
|
||||
uint32_t *e7 = e6 + EQm;
|
||||
|
||||
int i = 0;
|
||||
#if 0 /*def __AVX2__, this needs to be fixed, revert to 128-bit below*/
|
||||
simde__m256i *e0_256 = (simde__m256i *)e0;
|
||||
simde__m256i *e1_256 = (simde__m256i *)e1;
|
||||
simde__m256i *e2_256 = (simde__m256i *)e2;
|
||||
simde__m256i *e3_256 = (simde__m256i *)e3;
|
||||
simde__m256i *e4_256 = (simde__m256i *)e4;
|
||||
simde__m256i *e5_256 = (simde__m256i *)e5;
|
||||
simde__m256i *e6_256 = (simde__m256i *)e6;
|
||||
simde__m256i *e7_256 = (simde__m256i *)e7;
|
||||
simde__m256i *f256 = (simde__m256i *)f;
|
||||
|
||||
for (; i < (EQm & ~7); i += 8) {
|
||||
simde__m256i e0j = simde_mm256_loadu_si256(e0_256++);
|
||||
simde__m256i e1j = simde_mm256_loadu_si256(e1_256++);
|
||||
simde__m256i e2j = simde_mm256_loadu_si256(e2_256++);
|
||||
simde__m256i e3j = simde_mm256_loadu_si256(e3_256++);
|
||||
simde__m256i e4j = simde_mm256_loadu_si256(e4_256++);
|
||||
simde__m256i e5j = simde_mm256_loadu_si256(e5_256++);
|
||||
simde__m256i e6j = simde_mm256_loadu_si256(e6_256++);
|
||||
simde__m256i e7j = simde_mm256_loadu_si256(e7_256++);
|
||||
|
||||
simde__m256i tmp0 = simde_mm256_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) e0(i+4) e1(i+4) e0(i+5) e1(i+5)
|
||||
simde__m256i tmp1 = simde_mm256_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) e2(i+4) e3(i+4) e2(i+5) e3(i+5)
|
||||
simde__m256i tmp2 = simde_mm256_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) e4(i+4) e5(i+4) e4(i+5) e5(i+5)
|
||||
simde__m256i tmp3 = simde_mm256_unpacklo_epi32(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) e6(i+4) e7(i+4) e6(i+5) e7(i+5)
|
||||
//
|
||||
simde__m256i tmp4 = simde_mm256_unpacklo_epi64(tmp0, tmp1); // e0(i) e1(i) e2(i) e3(i) e0(i+4) e1(i+4) e2(i+4) e3(i+4)
|
||||
simde__m256i tmp5 = simde_mm256_unpacklo_epi64(tmp2, tmp3); // e4(i) e5(i) e6(i) e7(i) e4(i+4) e5(i+4) e6(i+4) e7(i+4)
|
||||
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x20)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)
|
||||
|
||||
simde__m256i tmp6 = simde_mm256_unpackhi_epi64(tmp0, tmp1); // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e0(i+5) e1(i+5) e2(i+5) e3(i+5)
|
||||
simde__m256i tmp7 = simde_mm256_unpackhi_epi64(tmp2, tmp3); // e4(i+1) e5(i+1) e6(i+1) e7(i+1) e4(i+5) e5(i+5) e6(i+5) e7(i+5)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x20)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e4(i+1) e5(i+1) e6(i+1) e7(i+1)
|
||||
//
|
||||
tmp0 = simde_mm256_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3) e0(i+6) e1(i+6) e0(i+7) e1(i+7)
|
||||
tmp1 = simde_mm256_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3) e2(i+6) e3(i+6) e2(i+7) e3(i+7)
|
||||
tmp2 = simde_mm256_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3) e4(i+6) e5(i+6) e4(i+7) e5(i+7)
|
||||
tmp3 = simde_mm256_unpackhi_epi32(e6j, e7j); // e6(i+2) e7(i+2) e6(i+3) e7(i+3) e6(i+6) e7(i+6) e6(i+7) e7(i+7)
|
||||
//
|
||||
simde__m256i tmp8 = simde_mm256_unpacklo_epi64(tmp0, tmp1); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e0(i+6) e1(i+6) e2(i+6) e3(i+6)
|
||||
simde__m256i tmp9 = simde_mm256_unpacklo_epi64(tmp2, tmp3); // e4(i+2) e5(i+2) e6(i+2) e7(i+2) e4(i+6) e5(i+6) e6(i+6) e7(i+6)
|
||||
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x20)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2)
|
||||
simde__m256i tmp10 = simde_mm256_unpackhi_epi64(tmp0, tmp1); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e0(i+7) e1(i+7) e2(i+7) e3(i+7)
|
||||
simde__m256i tmp11 = simde_mm256_unpackhi_epi64(tmp2, tmp3); // e4(i+3) e5(i+3) e6(i+3) e7(i+3) e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x20)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x31)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x31)); // e0(i+5) e1(i+5) e2(i+5) e3(i+5) e4(i+5) e5(i+5) e6(i+5) e7(i+5)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x31)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6)
|
||||
simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x31)); // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7)
|
||||
}
|
||||
e0 = (uint32_t *)e0_256;
|
||||
e1 = (uint32_t *)e1_256;
|
||||
e2 = (uint32_t *)e2_256;
|
||||
e3 = (uint32_t *)e3_256;
|
||||
e4 = (uint32_t *)e4_256;
|
||||
e5 = (uint32_t *)e5_256;
|
||||
e6 = (uint32_t *)e6_256;
|
||||
e7 = (uint32_t *)e7_256;
|
||||
f = (uint32_t *)f256;
|
||||
#endif
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *e4_128 = (simde__m128i *)e4;
|
||||
simde__m128i *e5_128 = (simde__m128i *)e5;
|
||||
simde__m128i *e6_128 = (simde__m128i *)e6;
|
||||
simde__m128i *e7_128 = (simde__m128i *)e7;
|
||||
simde__m128i *f128 = (simde__m128i *)f;
|
||||
|
||||
|
||||
for (; i < (EQm & ~3); i += 4) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
|
||||
simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
|
||||
simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
|
||||
simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
|
||||
simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
|
||||
simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
|
||||
|
||||
simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1)
|
||||
simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1)
|
||||
simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1)
|
||||
simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1)
|
||||
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
|
||||
|
||||
tmp0 = simde_mm_unpackhi_epi32(e0j, e1j); // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
|
||||
tmp1 = simde_mm_unpackhi_epi32(e2j, e3j); // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
|
||||
tmp2 = simde_mm_unpackhi_epi32(e4j, e5j); // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
|
||||
tmp3 = simde_mm_unpackhi_epi32(e6j, e7j); // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
|
||||
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3)
|
||||
simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
|
||||
}
|
||||
e0 = (uint32_t *)e0_128;
|
||||
e1 = (uint32_t *)e1_128;
|
||||
e2 = (uint32_t *)e2_128;
|
||||
e3 = (uint32_t *)e3_128;
|
||||
e4 = (uint32_t *)e4_128;
|
||||
e5 = (uint32_t *)e5_128;
|
||||
e6 = (uint32_t *)e6_128;
|
||||
e7 = (uint32_t *)e7_128;
|
||||
f = (uint32_t *)f128;
|
||||
}
|
||||
break;
|
||||
default:
|
||||
AssertFatal(false, "Should be here!\n");
|
||||
}
|
||||
}
|
||||
|
||||
extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
|
||||
|
||||
/**
|
||||
* \typedef ldpc8blocks_args_t
|
||||
* \struct ldpc8blocks_args_s
|
||||
* \brief Arguments of an encoding task
|
||||
* encode up to 8 code blocks
|
||||
* \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
|
||||
* \var impp encoder implementation specific parameters for the task
|
||||
* \var f first interleaver output to be filled by the task
|
||||
* \var f2 second interleaver output to be filled by the task
|
||||
* in case of a shift of E in the code blocks group processed by the task
|
||||
*/
|
||||
|
||||
static void ldpcnblocks(nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters, encoder_implemparams_t impp)
|
||||
{
|
||||
|
||||
uint8_t mod_order = nrLDPC_TB_encoding_parameters->Qm;
|
||||
uint16_t nb_rb = nrLDPC_TB_encoding_parameters->nb_rb;
|
||||
uint32_t A = nrLDPC_TB_encoding_parameters->A;
|
||||
|
||||
unsigned int G = nrLDPC_TB_encoding_parameters->G;
|
||||
LOG_D(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp.K, G, nb_rb, (int)mod_order);
|
||||
|
||||
// nrLDPC_encoder output is in "d"
|
||||
// let's make this interface happy!
|
||||
// uint32_t d[4][68*384];
|
||||
uint8_t *c[nrLDPC_TB_encoding_parameters->C];
|
||||
|
||||
if (!nrLDPC_TB_encoding_parameters->c_dev)
|
||||
for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) {
|
||||
c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
|
||||
}
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
|
||||
LDPCencoder32(nrLDPC_TB_encoding_parameters->c_dev ? nrLDPC_TB_encoding_parameters->c_dev : c, &impp);
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
|
||||
// Compute where to place in output buffer that is concatenation of all segments
|
||||
|
||||
#ifdef DEBUG_LDPC_ENCODING
|
||||
LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
|
||||
#endif
|
||||
const uint32_t E = nrLDPC_TB_encoding_parameters->segments[0].E;
|
||||
uint32_t E2=E;
|
||||
uint32_t Emax = E;
|
||||
int n_seg = nrLDPC_TB_encoding_parameters->C>>5;
|
||||
int n_seg2 = n_seg;
|
||||
if ((nrLDPC_TB_encoding_parameters->C & 31) > 0) n_seg2++;
|
||||
int r_shift = n_seg2;
|
||||
int r_shift2 = nrLDPC_TB_encoding_parameters->C;
|
||||
for (int s=0;s<nrLDPC_TB_encoding_parameters->C;s++) {
|
||||
//printf("segment %d E %d\n",s,nrLDPC_TB_encoding_parameters->segments[s].E);
|
||||
if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
|
||||
E2=nrLDPC_TB_encoding_parameters->segments[s].E;
|
||||
if(E2 > Emax)
|
||||
Emax = E2;
|
||||
r_shift = s>>5;
|
||||
r_shift2 = s;
|
||||
// printf("r_shift %d, r_shift2 %d\n",r_shift,r_shift2);
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
LOG_D(NR_PHY,
|
||||
"Rate Matching, Code segment %d...%d r_shift %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
|
||||
"%d,nrOfLayer %d)...\n",
|
||||
0,
|
||||
impp.n_segments-1,
|
||||
r_shift,
|
||||
n_seg2,
|
||||
G,
|
||||
E,E2,
|
||||
impp.F,
|
||||
impp.K - impp.F - 2 * impp.Zc,
|
||||
mod_order,
|
||||
nb_rb,
|
||||
nrLDPC_TB_encoding_parameters->nb_layers);
|
||||
/*
|
||||
printf("Rate Matching, Code segment 0..%d r_shift %d r_shift2 %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
|
||||
"%d,nrOfLayer %d)...\n",
|
||||
impp.n_segments-1,
|
||||
r_shift,
|
||||
r_shift2,
|
||||
n_seg2,
|
||||
G,
|
||||
E,E2,
|
||||
impp.F,
|
||||
impp.K - impp.F - 2 * impp.Zc,
|
||||
mod_order,
|
||||
nb_rb,
|
||||
nrLDPC_TB_encoding_parameters->nb_layers);
|
||||
*/
|
||||
|
||||
uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
|
||||
|
||||
uint32_t e[E*(r_shift+1)];
|
||||
uint32_t e2[E2*(n_seg2-r_shift)];
|
||||
uint32_t f[E*(r_shift+1)] __attribute__ ((aligned (64)));
|
||||
uint32_t f2[E2*(n_seg2-r_shift)] __attribute__ ((aligned (64)));
|
||||
|
||||
// Interleaver outputs are stored in the output arrays
|
||||
uint8_t *output = nrLDPC_TB_encoding_parameters->output;
|
||||
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
|
||||
memset(e,0,sizeof(e));
|
||||
memset(f,0,sizeof(f));
|
||||
if (1/*r_shift < n_seg2*/) {
|
||||
memset(e2,0,sizeof(e2));
|
||||
memset(f2,0,sizeof(f2));
|
||||
}
|
||||
|
||||
for (int r=0;r<n_seg2;r++) {
|
||||
if (r<=r_shift)
|
||||
nr_rate_matching_ldpc32(Tbslbrm,
|
||||
impp.BG,
|
||||
impp.Zc,
|
||||
d_host[r],
|
||||
e+(r*E),
|
||||
impp.n_segments,
|
||||
impp.F,
|
||||
impp.K - impp.F - 2 * impp.Zc,
|
||||
nrLDPC_TB_encoding_parameters->rv_index,
|
||||
E);
|
||||
if (r>=r_shift)
|
||||
nr_rate_matching_ldpc32(Tbslbrm,
|
||||
impp.BG,
|
||||
impp.Zc,
|
||||
d_host[r],
|
||||
e2+((r-r_shift)*E2),
|
||||
impp.n_segments,
|
||||
impp.F,
|
||||
impp.K - impp.F - 2 * impp.Zc,
|
||||
nrLDPC_TB_encoding_parameters->rv_index,
|
||||
E2);
|
||||
/*
|
||||
if (r==(n_seg2-1)) {
|
||||
for (int i=0;i<16;i++) printf("rm: %x %x\n",d[n_seg2-1][i],e2[((n_seg2-1)*E2)+i]);
|
||||
}
|
||||
*/
|
||||
}
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
|
||||
if (impp.K - impp.F - 2 * impp.Zc > E) {
|
||||
LOG_E(PHY,
|
||||
"dlsch coding A %d Kr %d G %d (nb_rb %d, mod_order %d)\n",
|
||||
A,
|
||||
impp.K,
|
||||
G,
|
||||
nb_rb,
|
||||
(int)mod_order);
|
||||
|
||||
LOG_E(NR_PHY,
|
||||
"Rate Matching, Code segments 0..%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
|
||||
"nb_rb %d)...\n",
|
||||
impp.n_segments,
|
||||
G,
|
||||
E,
|
||||
impp.K,
|
||||
impp.F,
|
||||
impp.K - impp.F - 2 * impp.Zc,
|
||||
mod_order,
|
||||
nb_rb);
|
||||
}
|
||||
|
||||
//printf("interleaving r_shift %d, n_seg2 %d\n",r_shift,n_seg2);
|
||||
start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
|
||||
|
||||
for (int r=0;r<=r_shift;r++)
|
||||
nr_interleaving_ldpc32(E,
|
||||
mod_order,
|
||||
e+E*r,
|
||||
f+E*r);
|
||||
|
||||
for (int r=r_shift;r<n_seg2;r++)
|
||||
nr_interleaving_ldpc32(E2,
|
||||
mod_order,
|
||||
e2+E2*(r-r_shift),
|
||||
f2+E2*(r-r_shift));
|
||||
/*
|
||||
for (int i=0;i<16;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
|
||||
printf("-------------------\n");
|
||||
for (int i=E2-16;i<E2;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
|
||||
*/
|
||||
stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
|
||||
|
||||
if (impp.tconcat) start_meas(impp.tconcat);
|
||||
unpack_output(f,E,f2,E2,r_shift,r_shift2,nrLDPC_TB_encoding_parameters->C,output);
|
||||
if (impp.tconcat) stop_meas(impp.tconcat);
|
||||
|
||||
}
|
||||
|
||||
int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters)
|
||||
{
|
||||
|
||||
|
||||
encoder_implemparams_t common_segment_params = {
|
||||
.n_segments = nrLDPC_TB_encoding_parameters->C,
|
||||
.tinput = nrLDPC_slot_encoding_parameters->tinput,
|
||||
.tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
|
||||
.tprep = nrLDPC_slot_encoding_parameters->tprep,
|
||||
.tparity = nrLDPC_slot_encoding_parameters->tparity,
|
||||
.toutput = nrLDPC_slot_encoding_parameters->toutput,
|
||||
.tconcat = nrLDPC_slot_encoding_parameters->tconcat,
|
||||
.Kb = nrLDPC_TB_encoding_parameters->Kb,
|
||||
.Zc = nrLDPC_TB_encoding_parameters->Z,
|
||||
.BG = nrLDPC_TB_encoding_parameters->BG,
|
||||
.output = nrLDPC_TB_encoding_parameters->output,
|
||||
.K = nrLDPC_TB_encoding_parameters->K,
|
||||
.F = nrLDPC_TB_encoding_parameters->F,
|
||||
};
|
||||
|
||||
ldpcnblocks(nrLDPC_TB_encoding_parameters, common_segment_params);
|
||||
|
||||
|
||||
|
||||
return 0;
|
||||
}
|
||||
@@ -7,7 +7,8 @@
|
||||
#include "common/utils/LOG/log.h"
|
||||
|
||||
// #define RM_DEBUG 1
|
||||
|
||||
#define USE128BIT
|
||||
|
||||
static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
|
||||
|
||||
void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
|
||||
@@ -48,7 +49,7 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
|
||||
simde__m128i *f_128 = (simde__m128i *)f;
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
for (; i < (EQm & ~15); i += 64) {
|
||||
for (; i < (EQm & ~15); i += 16) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
|
||||
@@ -342,14 +343,15 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
|
||||
f = (uint8_t *)f_512;
|
||||
#endif
|
||||
#ifdef USE128BIT
|
||||
e0_128 = (simde__m128i *)e0;
|
||||
e1_128 = (simde__m128i *)e1;
|
||||
e2_128 = (simde__m128i *)e2;
|
||||
e3_128 = (simde__m128i *)e3;
|
||||
e4_128 = (simde__m128i *)e4;
|
||||
e5_128 = (simde__m128i *)e5;
|
||||
e6_128 = (simde__m128i *)e6;
|
||||
e7_128 = (simde__m128i *)e7;
|
||||
simde__m128i *e0_128 = (simde__m128i *)e0;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *e4_128 = (simde__m128i *)e4;
|
||||
simde__m128i *e5_128 = (simde__m128i *)e5;
|
||||
simde__m128i *e6_128 = (simde__m128i *)e6;
|
||||
simde__m128i *e7_128 = (simde__m128i *)e7;
|
||||
simde__m128i *f_128 = (simde__m128i *)f;
|
||||
for (; i < (EQm & ~15); i += 16) {
|
||||
simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
|
||||
simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
|
||||
@@ -427,25 +429,240 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
|
||||
}
|
||||
}
|
||||
|
||||
#if defined(__aarch64__)
|
||||
static inline uint8x16_t tbl96_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2,
|
||||
uint8x16_t b3, uint8x16_t b4, uint8x16_t b5,
|
||||
uint8x16_t idx /* 0..95 */)
|
||||
{
|
||||
// Table 0: bytes 0..63
|
||||
uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
|
||||
uint8x16_t r0 = vqtbl4q_u8(T0, idx);
|
||||
|
||||
// Table 1: bytes 64..95, presented as a 64-byte table:
|
||||
// bytes 0..31 map to original 64..95, bytes 32..63 are dummy (return 0)
|
||||
uint8x16_t z = vdupq_n_u8(0);
|
||||
uint8x16x4_t T1 = {{ b4, b5, z, z }};
|
||||
|
||||
// idx1 = idx - 64 (wrap-safe via unsigned subtract); only valid when idx>=64
|
||||
uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
|
||||
uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
|
||||
|
||||
// Select r1 where idx >= 64, else r0
|
||||
uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64)); // 0xFF where idx>=64
|
||||
return vbslq_u8(sel, r1, r0);
|
||||
}
|
||||
|
||||
static inline uint8x16_t tbl128_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2, uint8x16_t b3,
|
||||
uint8x16_t b4, uint8x16_t b5, uint8x16_t b6, uint8x16_t b7,
|
||||
uint8x16_t idx /* 0..127 */)
|
||||
{
|
||||
// Table low: bytes 0..63
|
||||
uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
|
||||
uint8x16_t r0 = vqtbl4q_u8(T0, idx);
|
||||
|
||||
// Table high: bytes 64..127, mapped to 0..63 by subtracting 64
|
||||
uint8x16x4_t T1 = {{ b4, b5, b6, b7 }};
|
||||
uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
|
||||
uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
|
||||
|
||||
// Select high where idx >= 64 else low
|
||||
uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64));
|
||||
return vbslq_u8(sel, r1, r0);
|
||||
}
|
||||
#elif defined(__AVX512BW__)
|
||||
|
||||
static inline __m512i idx_stride_u16(int ways, int k)
|
||||
{
|
||||
uint16_t idx[32] __attribute__((aligned(64)));
|
||||
for (int n = 0; n < 32; n++) idx[n] = (uint16_t)(k + ways*n);
|
||||
return _mm512_load_si512((const void*)idx);
|
||||
}
|
||||
|
||||
// Build indices for k = 0..5, lanes n=0..15: p = k + 6*n
|
||||
static inline __m512i make_idx_ab(int k) {
|
||||
uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
|
||||
for (int n = 0; n < 16; n++) {
|
||||
idx[n] = (uint16_t)(k + 6*n); // p in [0..95]
|
||||
}
|
||||
return _mm512_load_si512((const void*)idx);
|
||||
}
|
||||
|
||||
static inline __m512i make_idx_bc(int k) {
|
||||
uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
|
||||
|
||||
for (int n = 0; n < 16; n++) {
|
||||
int p = k + 6*n;
|
||||
// For (B,C) pair, global p maps to:
|
||||
// indices 0..31 -> B lanes 0..31 (global 32..63)
|
||||
// indices 32..63 -> C lanes 0..31 (global 64..95)
|
||||
// so use (p - 32) as index into [B|C] when p >= 32.
|
||||
idx[n] = (uint16_t)(p - 32);
|
||||
}
|
||||
return _mm512_load_si512((const void*)idx);
|
||||
}
|
||||
|
||||
static inline __mmask32 make_mask_bc(int k) {
|
||||
// lanes needing BC are those with p >= 64 (since AB covers global 0..63)
|
||||
__mmask32 m = 0;
|
||||
for (int n = 0; n < 16; n++) {
|
||||
int p = k + 6*n;
|
||||
if (p >= 64) m |= (1u << n);
|
||||
}
|
||||
return m;
|
||||
}
|
||||
#endif
|
||||
|
||||
void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
|
||||
{
|
||||
const uint32_t EQm = E / Qm;
|
||||
switch (Qm) {
|
||||
case 2: {
|
||||
AssertFatal(E % 2 == 0, "");
|
||||
int16_t *e1 = e + (E / 2);
|
||||
int16_t *end = f + E - 1;
|
||||
while (f < end) {
|
||||
int16_t *e1 = e + EQm;
|
||||
int i=0;
|
||||
#if defined(__aarch64__)
|
||||
for (; i + 8 <= EQm; i += 8) {
|
||||
int16x8x2_t v = vld2q_s16(f); // 8 groups
|
||||
f += 16;
|
||||
vst1q_s16(e, v.val[0]); e += 8;
|
||||
vst1q_s16(e1, v.val[1]); e1 += 8;
|
||||
}
|
||||
#elif defined(__AVX512BW__)
|
||||
const __m512i idx0 = idx_stride_u16(2, 0);
|
||||
const __m512i idx1 = idx_stride_u16(2, 1);
|
||||
|
||||
for (; i + 32 <= EQm; i += 32) {
|
||||
// 32 groups * 2 u16 = 64 u16 = 2x512b
|
||||
__m512i A = _mm512_loadu_si512((const void*)(f + 0)); // u16[0..31]
|
||||
__m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
|
||||
f += 64;
|
||||
__m512i o0 = _mm512_permutex2var_epi16(A, idx0, B);
|
||||
__m512i o1 = _mm512_permutex2var_epi16(A, idx1, B);
|
||||
_mm512_storeu_si512((void*)e, o0); e += 32;
|
||||
_mm512_storeu_si512((void*)e1, o1); e1 += 32;
|
||||
}
|
||||
#else
|
||||
simde__m128i *e0_128 = (simde__m128i *)e;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *f128 = (simde__m128i *)f;
|
||||
const uint8_t shuf4[16] __attribute__((aligned(16))) = {0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15};
|
||||
|
||||
const simde__m128i *shuf4_128 = (const simde__m128i *)shuf4;
|
||||
for (i=0; i < (EQm & ~7); i += 8) {
|
||||
simde__m128i f0j = simde_mm_loadu_si128(f128++); // f0(i) f0(i+1) f0(i+2) f0(i+3) f0(i+4) f0(i+5) f0(i+6) f0(i+7)
|
||||
simde__m128i f1j = simde_mm_loadu_si128(f128++); // f1(i) f1(i+1) f1(i+2) f1(i+3) f1(i+4) f1(i+5) f1(i+6) f1(i+7)
|
||||
simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf4_128); // f0(i) f0(i+2) f0(i+4) f0(i+6) f0(i+1) f0(i+3) f0(i+5) f0(i+7)
|
||||
simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf4_128); // f1(i) f1(i+2) f1(i+4) f1(i+6) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
|
||||
simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // f0(i) f0(i+2) f0(i+4) f0(i+6) f1(i) f1(i+2) f1(i+4) f1(i+6)
|
||||
simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // f0(i+1) f0(i+3) f0(i+5) f0(i+7) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
|
||||
}
|
||||
e=(int16_t *)e0_128;
|
||||
e1=(int16_t *)e1_128;
|
||||
f=(int16_t *)f128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*e++ = *f++;
|
||||
*e1++ = *f++;
|
||||
}
|
||||
} break;
|
||||
|
||||
case 4: {
|
||||
AssertFatal(E % 4 == 0, "");
|
||||
int16_t *e1 = e + (E / 4);
|
||||
int16_t *e2 = e1 + (E / 4);
|
||||
int16_t *e3 = e2 + (E / 4);
|
||||
int16_t *end = f + E - 3;
|
||||
while (f < end) {
|
||||
int i=0;
|
||||
int16_t *e1 = e + EQm;
|
||||
int16_t *e2 = e1 + EQm;
|
||||
int16_t *e3 = e2 + EQm;
|
||||
|
||||
#if defined(__aarch64__)
|
||||
for (; i + 8 <= EQm; i += 8) {
|
||||
int16x8x4_t v = vld4q_s16(f); // 8 groups
|
||||
f += 32;
|
||||
vst1q_s16(e, v.val[0]); e += 8;
|
||||
vst1q_s16(e1, v.val[1]); e1 += 8;
|
||||
vst1q_s16(e2, v.val[2]); e2 += 8;
|
||||
vst1q_s16(e3, v.val[3]); e3 += 8;
|
||||
}
|
||||
#elif defined(__AVX512BW__)
|
||||
const __m512i idx0 = idx_stride_u16(4, 0);
|
||||
const __m512i idx1 = idx_stride_u16(4, 1);
|
||||
const __m512i idx2 = idx_stride_u16(4, 2);
|
||||
const __m512i idx3 = idx_stride_u16(4, 3);
|
||||
|
||||
for (; i + 32 <= EQm; i += 32) {
|
||||
// 32 groups * 4 u16 = 128 u16 = 4x512b
|
||||
__m512i A = _mm512_loadu_si512((const void*)(f + 0));
|
||||
__m512i B = _mm512_loadu_si512((const void*)(f + 32));
|
||||
__m512i C = _mm512_loadu_si512((const void*)(f + 64));
|
||||
__m512i D = _mm512_loadu_si512((const void*)(f + 96));
|
||||
f += 128;
|
||||
// Build results from (A,B) and (C,D) with masking (boundary at 64 u16)
|
||||
__m512i o0_ab = _mm512_permutex2var_epi16(A, idx0, B);
|
||||
__m512i o1_ab = _mm512_permutex2var_epi16(A, idx1, B);
|
||||
__m512i o2_ab = _mm512_permutex2var_epi16(A, idx2, B);
|
||||
__m512i o3_ab = _mm512_permutex2var_epi16(A, idx3, B);
|
||||
|
||||
__m512i idx0_cd = _mm512_sub_epi16(idx0, _mm512_set1_epi16(64));
|
||||
__m512i idx1_cd = _mm512_sub_epi16(idx1, _mm512_set1_epi16(64));
|
||||
__m512i idx2_cd = _mm512_sub_epi16(idx2, _mm512_set1_epi16(64));
|
||||
__m512i idx3_cd = _mm512_sub_epi16(idx3, _mm512_set1_epi16(64));
|
||||
|
||||
__m512i o0_cd = _mm512_permutex2var_epi16(C, idx0_cd, D);
|
||||
__m512i o1_cd = _mm512_permutex2var_epi16(C, idx1_cd, D);
|
||||
__m512i o2_cd = _mm512_permutex2var_epi16(C, idx2_cd, D);
|
||||
__m512i o3_cd = _mm512_permutex2var_epi16(C, idx3_cd, D);
|
||||
|
||||
// lanes needing CD are those where idx >= 64
|
||||
__mmask32 m0 = _mm512_cmpge_epu16_mask(idx0, _mm512_set1_epi16(64));
|
||||
__mmask32 m1 = _mm512_cmpge_epu16_mask(idx1, _mm512_set1_epi16(64));
|
||||
__mmask32 m2 = _mm512_cmpge_epu16_mask(idx2, _mm512_set1_epi16(64));
|
||||
__mmask32 m3 = _mm512_cmpge_epu16_mask(idx3, _mm512_set1_epi16(64));
|
||||
|
||||
__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_cd);
|
||||
__m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_cd);
|
||||
__m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_cd);
|
||||
__m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_cd);
|
||||
|
||||
_mm512_storeu_si512((void*)e, o0); e += 32;
|
||||
_mm512_storeu_si512((void*)e1, o1); e1 += 32;
|
||||
_mm512_storeu_si512((void*)e2, o2); e2 += 32;
|
||||
_mm512_storeu_si512((void*)e3, o3); e3 += 32;
|
||||
}
|
||||
#else
|
||||
simde__m128i *e0_128 = (simde__m128i *)e;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *f128 = (simde__m128i *)f;
|
||||
|
||||
const uint8_t shuf16[16] __attribute__((aligned(16)))= {0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15};
|
||||
const simde__m128i *shuf16_128 = (const simde__m128i *)shuf16;
|
||||
|
||||
for (i=0; i < (EQm & ~7); i += 8) {
|
||||
simde__m128i f0j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f1j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f2j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f3j = simde_mm_loadu_si128(f128++);
|
||||
|
||||
simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf16_128); // f0(i) f0(i+4) f0(i+1) f0(i+5) f0(i+2) f0(i+6) f0(i+3) f0(i+7)
|
||||
simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf16_128); // f1(i) f1(i+4) f1(i+1) f1(i+5) f1(i+2) f1(i+6) f1(i+3) f1(i+7)
|
||||
simde__m128i tmp2 = simde_mm_shuffle_epi8(f2j,*shuf16_128); // f2(i) f2(i+4) f2(i+1) f2(i+5) f2(i+2) f2(i+6) f2(i+3) f2(i+7)
|
||||
simde__m128i tmp3 = simde_mm_shuffle_epi8(f3j,*shuf16_128); // f3(i) f3(i+2) f3(i+1) f3(i+5) f3(i+2) f3(i+6) f3(i+3) f3(i+7)
|
||||
simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0,tmp1); // f0(i) f0(i+4) f1(i) f1(i+4) f0(i+1) f0(i+5) f1(i+1) f1(i+5)
|
||||
simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2,tmp3); // f2(i) f2(i+4) f3(i) f3(i+4) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
|
||||
simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i) f0(i+4) f1(i) f1(i+4) f2(i) f2(i+4) f3(i) f3(i+4)
|
||||
simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+1) f0(i+5) f1(i+1) f1(i+5) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
|
||||
tmp4 = simde_mm_unpackhi_epi32(tmp0,tmp1); // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f0(i+3) f0(i+7) f1(i+3) f1(i+7)
|
||||
tmp5 = simde_mm_unpackhi_epi32(tmp2,tmp3); // f2(i+2) f2(i+6) f3(i+2) f3(i+6) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
|
||||
simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f2(i+2) f2(i+6) f3(i+2) f3(i+6)
|
||||
simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+3) f0(i+7) f1(i+3) f1(i+7) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
|
||||
}
|
||||
e=(int16_t *)e0_128;
|
||||
e1=(int16_t *)e1_128;
|
||||
e2=(int16_t *)e2_128;
|
||||
e3=(int16_t *)e3_128;
|
||||
f=(int16_t *)f128;
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*e++ = *f++;
|
||||
*e1++ = *f++;
|
||||
*e2++ = *f++;
|
||||
@@ -454,13 +671,108 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
|
||||
} break;
|
||||
case 6: {
|
||||
AssertFatal(E % 6 == 0, "");
|
||||
int16_t *e1 = e + (E / 6);
|
||||
int16_t *e2 = e1 + (E / 6);
|
||||
int16_t *e3 = e2 + (E / 6);
|
||||
int16_t *e4 = e3 + (E / 6);
|
||||
int16_t *e5 = e4 + (E / 6);
|
||||
int16_t *end = f + E - 5;
|
||||
while (f < end) {
|
||||
int16_t *e1 = e + EQm;
|
||||
int16_t *e2 = e1 + EQm;
|
||||
int16_t *e3 = e2 + EQm;
|
||||
int16_t *e4 = e3 + EQm;
|
||||
int16_t *e5 = e4 + EQm;
|
||||
int i=0;
|
||||
#if defined(__aarch64__)
|
||||
|
||||
// Byte indices for extracting each stream
|
||||
// Each s16 occupies 2 bytes → indices are 2*(k + 6*n)
|
||||
const uint8x16_t idx0 = { 0, 1, 12, 13, 24, 25, 36, 37,
|
||||
48, 49, 60, 61, 72, 73, 84, 85 };
|
||||
const uint8x16_t idx1 = { 2, 3, 14, 15, 26, 27, 38, 39,
|
||||
50, 51, 62, 63, 74, 75, 86, 87 };
|
||||
const uint8x16_t idx2 = { 4, 5, 16, 17, 28, 29, 40, 41,
|
||||
52, 53, 64, 65, 76, 77, 88, 89 };
|
||||
const uint8x16_t idx3 = { 6, 7, 18, 19, 30, 31, 42, 43,
|
||||
54, 55, 66, 67, 78, 79, 90, 91 };
|
||||
const uint8x16_t idx4 = { 8, 9, 20, 21, 32, 33, 44, 45,
|
||||
56, 57, 68, 69, 80, 81, 92, 93 };
|
||||
const uint8x16_t idx5 = { 10, 11, 22, 23, 34, 35, 46, 47,
|
||||
58, 59, 70, 71, 82, 83, 94, 95 };
|
||||
|
||||
for (; i + 8 <= EQm; i += 8) {
|
||||
// Load 96 bytes (48 u16)
|
||||
uint8x16_t b0 = vld1q_u8((const uint8_t*)(f + 0)); // bytes 0..15
|
||||
uint8x16_t b1 = vld1q_u8((const uint8_t*)(f + 8)); // bytes 16..31
|
||||
uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
|
||||
uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
|
||||
uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
|
||||
uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
|
||||
f += 48;
|
||||
uint8x16_t o0b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx0);
|
||||
uint8x16_t o1b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx1);
|
||||
uint8x16_t o2b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx2);
|
||||
uint8x16_t o3b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx3);
|
||||
uint8x16_t o4b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx4);
|
||||
uint8x16_t o5b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx5);
|
||||
|
||||
vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
|
||||
vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
|
||||
vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
|
||||
vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
|
||||
vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
|
||||
vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
|
||||
}
|
||||
#elif defined(__AVX512BW__)
|
||||
|
||||
// Precompute permute control once (all compile-time constant patterns)
|
||||
const __m512i idx0_ab = make_idx_ab(0), idx1_ab = make_idx_ab(1);
|
||||
const __m512i idx2_ab = make_idx_ab(2), idx3_ab = make_idx_ab(3);
|
||||
const __m512i idx4_ab = make_idx_ab(4), idx5_ab = make_idx_ab(5);
|
||||
|
||||
const __m512i idx0_bc = make_idx_bc(0), idx1_bc = make_idx_bc(1);
|
||||
const __m512i idx2_bc = make_idx_bc(2), idx3_bc = make_idx_bc(3);
|
||||
const __m512i idx4_bc = make_idx_bc(4), idx5_bc = make_idx_bc(5);
|
||||
|
||||
const __mmask32 m0 = make_mask_bc(0), m1 = make_mask_bc(1), m2 = make_mask_bc(2);
|
||||
const __mmask32 m3 = make_mask_bc(3), m4 = make_mask_bc(4), m5 = make_mask_bc(5);
|
||||
|
||||
const __mmask32 store16 = 0xFFFFu; // store only first 16 lanes (16x u16)
|
||||
for (; i + 16 <= EQm; i += 16) {
|
||||
// 16 groups = 16*(6 u16) = 96 u16 = 192 bytes = 3 * 64B
|
||||
__m512i A = _mm512_loadu_si512((const void*)(f + 0)); // u16[0..31]
|
||||
__m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
|
||||
__m512i C = _mm512_loadu_si512((const void*)(f + 64)); // u16[64..95]
|
||||
f += 96;
|
||||
|
||||
// Gather from AB (global 0..63) and BC (global 32..95)
|
||||
__m512i o0_ab = _mm512_permutex2var_epi16(A, idx0_ab, B);
|
||||
__m512i o1_ab = _mm512_permutex2var_epi16(A, idx1_ab, B);
|
||||
__m512i o2_ab = _mm512_permutex2var_epi16(A, idx2_ab, B);
|
||||
__m512i o3_ab = _mm512_permutex2var_epi16(A, idx3_ab, B);
|
||||
__m512i o4_ab = _mm512_permutex2var_epi16(A, idx4_ab, B);
|
||||
__m512i o5_ab = _mm512_permutex2var_epi16(A, idx5_ab, B);
|
||||
|
||||
__m512i o0_bc = _mm512_permutex2var_epi16(B, idx0_bc, C);
|
||||
__m512i o1_bc = _mm512_permutex2var_epi16(B, idx1_bc, C);
|
||||
__m512i o2_bc = _mm512_permutex2var_epi16(B, idx2_bc, C);
|
||||
__m512i o3_bc = _mm512_permutex2var_epi16(B, idx3_bc, C);
|
||||
__m512i o4_bc = _mm512_permutex2var_epi16(B, idx4_bc, C);
|
||||
__m512i o5_bc = _mm512_permutex2var_epi16(B, idx5_bc, C);
|
||||
|
||||
// Select lanes that crossed the 64-element boundary
|
||||
__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_bc);
|
||||
__m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_bc);
|
||||
__m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_bc);
|
||||
__m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_bc);
|
||||
__m512i o4 = _mm512_mask_mov_epi16(o4_ab, m4, o4_bc);
|
||||
__m512i o5 = _mm512_mask_mov_epi16(o5_ab, m5, o5_bc);
|
||||
|
||||
// Store only the first 16 lanes (16 groups) to each plane
|
||||
_mm512_mask_storeu_epi16((void*)e, store16, o0); e += 16;
|
||||
_mm512_mask_storeu_epi16((void*)e1, store16, o1); e1 += 16;
|
||||
_mm512_mask_storeu_epi16((void*)e2, store16, o2); e2 += 16;
|
||||
_mm512_mask_storeu_epi16((void*)e3, store16, o3); e3 += 16;
|
||||
_mm512_mask_storeu_epi16((void*)e4, store16, o4); e4 += 16;
|
||||
_mm512_mask_storeu_epi16((void*)e5, store16, o5); e5 += 16;
|
||||
}
|
||||
//
|
||||
#endif
|
||||
for (; i < EQm; i++) {
|
||||
*e++ = *f++;
|
||||
*e1++ = *f++;
|
||||
*e2++ = *f++;
|
||||
@@ -471,15 +783,180 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
|
||||
} break;
|
||||
case 8: {
|
||||
AssertFatal(E % 8 == 0, "");
|
||||
int16_t *e1 = e + (E / 8);
|
||||
int16_t *e2 = e1 + (E / 8);
|
||||
int16_t *e3 = e2 + (E / 8);
|
||||
int16_t *e4 = e3 + (E / 8);
|
||||
int16_t *e5 = e4 + (E / 8);
|
||||
int16_t *e6 = e5 + (E / 8);
|
||||
int16_t *e7 = e6 + (E / 8);
|
||||
int16_t *end = f + E - 7;
|
||||
while (f < end) {
|
||||
int16_t *e1 = e + EQm;
|
||||
int16_t *e2 = e1 + EQm;
|
||||
int16_t *e3 = e2 + EQm;
|
||||
int16_t *e4 = e3 + EQm;
|
||||
int16_t *e5 = e4 + EQm;
|
||||
int16_t *e6 = e5 + EQm;
|
||||
int16_t *e7 = e6 + EQm;
|
||||
int i=0;
|
||||
#if 0 //defined(__aarch64__)
|
||||
// SIMDE version below is more efficient than tbl128
|
||||
// For 8 groups: byte indices for stream k are 2*(k + 8*n) for n=0..7.
|
||||
// That is: 2k + 16n (and +1 for the high byte of the u16).
|
||||
const uint8x16_t idx0 = { 0, 1, 16, 17, 32, 33, 48, 49, 64, 65, 80, 81, 96, 97,112,113 };
|
||||
const uint8x16_t idx1 = { 2, 3, 18, 19, 34, 35, 50, 51, 66, 67, 82, 83, 98, 99,114,115 };
|
||||
const uint8x16_t idx2 = { 4, 5, 20, 21, 36, 37, 52, 53, 68, 69, 84, 85,100,101,116,117 };
|
||||
const uint8x16_t idx3 = { 6, 7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87,102,103,118,119 };
|
||||
const uint8x16_t idx4 = { 8, 9, 24, 25, 40, 41, 56, 57, 72, 73, 88, 89,104,105,120,121 };
|
||||
const uint8x16_t idx5 = { 10, 11, 26, 27, 42, 43, 58, 59, 74, 75, 90, 91,106,107,122,123 };
|
||||
const uint8x16_t idx6 = { 12, 13, 28, 29, 44, 45, 60, 61, 76, 77, 92, 93,108,109,124,125 };
|
||||
const uint8x16_t idx7 = { 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95,110,111,126,127 };
|
||||
|
||||
for (; i + 8 <= EQm; i += 8) {
|
||||
// Load 128 bytes = 64 u16 = 8 groups (aligned-friendly, but vld1q_u8 is fine either way)
|
||||
uint8x16_t b0 = vld1q_u8((const uint8_t*)(f + 0)); // bytes 0..15
|
||||
uint8x16_t b1 = vld1q_u8((const uint8_t*)(f + 8)); // bytes 16..31
|
||||
uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
|
||||
uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
|
||||
uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
|
||||
uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
|
||||
uint8x16_t b6 = vld1q_u8((const uint8_t*)(f + 48)); // bytes 96..111
|
||||
uint8x16_t b7 = vld1q_u8((const uint8_t*)(f + 56)); // bytes 112..127
|
||||
f += 64; // consumed 64 u16
|
||||
|
||||
uint8x16_t o0b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx0);
|
||||
uint8x16_t o1b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx1);
|
||||
uint8x16_t o2b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx2);
|
||||
uint8x16_t o3b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx3);
|
||||
uint8x16_t o4b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx4);
|
||||
uint8x16_t o5b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx5);
|
||||
uint8x16_t o6b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx6);
|
||||
uint8x16_t o7b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx7);
|
||||
|
||||
vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
|
||||
vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
|
||||
vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
|
||||
vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
|
||||
vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
|
||||
vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
|
||||
vst1q_s16(e6, vreinterpretq_s16_u8(o6b)); e6 += 8;
|
||||
vst1q_s16(e7, vreinterpretq_s16_u8(o7b)); e7 += 8;
|
||||
}
|
||||
#elif defined(__AVX512BW__)
|
||||
|
||||
// Precompute indices for each stream k=0..7
|
||||
const __m512i idx0 = idx_stride_u16(8,0);
|
||||
const __m512i idx1 = idx_stride_u16(8,1);
|
||||
const __m512i idx2 = idx_stride_u16(8,2);
|
||||
const __m512i idx3 = idx_stride_u16(8,3);
|
||||
const __m512i idx4 = idx_stride_u16(8,4);
|
||||
const __m512i idx5 = idx_stride_u16(8,5);
|
||||
const __m512i idx6 = idx_stride_u16(8,6);
|
||||
const __m512i idx7 = idx_stride_u16(8,7);
|
||||
|
||||
const __m512i c64 = _mm512_set1_epi16(64);
|
||||
const __m512i c128 = _mm512_set1_epi16(128);
|
||||
const __m512i c192 = _mm512_set1_epi16(192);
|
||||
|
||||
for (; i + 32 <= EQm; i += 32) {
|
||||
// 32 groups * 8 u16 = 256 u16 = 8 * 32 u16 = 8 ZMM vectors
|
||||
__m512i A = _mm512_loadu_si512((const void*)(f + 0)); // u16[ 0.. 31]
|
||||
__m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[ 32.. 63]
|
||||
__m512i C = _mm512_loadu_si512((const void*)(f + 64)); // u16[ 64.. 95]
|
||||
__m512i D = _mm512_loadu_si512((const void*)(f + 96)); // u16[ 96..127]
|
||||
__m512i E = _mm512_loadu_si512((const void*)(f + 128)); // u16[128..159]
|
||||
__m512i F = _mm512_loadu_si512((const void*)(f + 160)); // u16[160..191]
|
||||
__m512i G = _mm512_loadu_si512((const void*)(f + 192)); // u16[192..223]
|
||||
__m512i H = _mm512_loadu_si512((const void*)(f + 224)); // u16[224..255]
|
||||
f += 256;
|
||||
// Helper macro: compute one stream (k) by permuting from 4 segments and blending
|
||||
#define DO_STREAM(IDX, OUTPTR) do { \
|
||||
/* segment masks based on global index p = k+8*n */ \
|
||||
__mmask32 m1 = _mm512_cmpge_epu16_mask((IDX), c64); \
|
||||
__mmask32 m2 = _mm512_cmpge_epu16_mask((IDX), c128); \
|
||||
__mmask32 m3 = _mm512_cmpge_epu16_mask((IDX), c192); \
|
||||
/* indices relative to each segment base */ \
|
||||
__m512i i0 = (IDX); \
|
||||
__m512i i1 = _mm512_sub_epi16((IDX), c64); \
|
||||
__m512i i2 = _mm512_sub_epi16((IDX), c128); \
|
||||
__m512i i3 = _mm512_sub_epi16((IDX), c192); \
|
||||
/* permute from each 64-u16 segment (two ZMMs per segment) */ \
|
||||
__m512i r0 = _mm512_permutex2var_epi16(A, i0, B); \
|
||||
__m512i r1 = _mm512_permutex2var_epi16(C, i1, D); \
|
||||
__m512i r2 = _mm512_permutex2var_epi16(E, i2, F); \
|
||||
__m512i r3 = _mm512_permutex2var_epi16(G, i3, H); \
|
||||
/* blend: later segments overwrite earlier where applicable */ \
|
||||
__m512i r = _mm512_mask_mov_epi16(r0, m1, r1); \
|
||||
r = _mm512_mask_mov_epi16(r, m2, r2); \
|
||||
r = _mm512_mask_mov_epi16(r, m3, r3); \
|
||||
_mm512_storeu_si512((void*)(OUTPTR), r); \
|
||||
(OUTPTR) += 32; \
|
||||
} while(0)
|
||||
DO_STREAM(idx0, e);
|
||||
DO_STREAM(idx1, e1);
|
||||
DO_STREAM(idx2, e2);
|
||||
DO_STREAM(idx3, e3);
|
||||
DO_STREAM(idx4, e4);
|
||||
DO_STREAM(idx5, e5);
|
||||
DO_STREAM(idx6, e6);
|
||||
DO_STREAM(idx7, e7);
|
||||
|
||||
#undef DO_STREAM
|
||||
}
|
||||
#else
|
||||
simde__m128i *e0_128 = (simde__m128i *)e;
|
||||
simde__m128i *e1_128 = (simde__m128i *)e1;
|
||||
simde__m128i *e2_128 = (simde__m128i *)e2;
|
||||
simde__m128i *e3_128 = (simde__m128i *)e3;
|
||||
simde__m128i *e4_128 = (simde__m128i *)e4;
|
||||
simde__m128i *e5_128 = (simde__m128i *)e5;
|
||||
simde__m128i *e6_128 = (simde__m128i *)e6;
|
||||
simde__m128i *e7_128 = (simde__m128i *)e7;
|
||||
simde__m128i *f128 = (simde__m128i *)f;
|
||||
|
||||
for (i=0; i < (EQm & ~7); i += 8) {
|
||||
simde__m128i f0j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f1j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f2j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f3j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f4j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f5j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f6j = simde_mm_loadu_si128(f128++);
|
||||
simde__m128i f7j = simde_mm_loadu_si128(f128++);
|
||||
|
||||
simde__m128i tmp0 = simde_mm_unpacklo_epi16(f0j, f1j); // f0(i) f1(i) f0(i+1) f1(i+1) f0(i+2) f1(i+2) f0(i+3) f1(i+3)
|
||||
simde__m128i tmp1 = simde_mm_unpacklo_epi16(f2j, f3j); // f2(i) f3(i) f2(i+1) f3(i+1) f2(i+2) f3(i+2) f2(i+3) f3(i+3)
|
||||
simde__m128i tmp2 = simde_mm_unpacklo_epi16(f4j, f5j); // f4(i) f5(i) f4(i+1) f5(i+1) f4(i+2) f5(i+2) f4(i+3) f5(i+3)
|
||||
simde__m128i tmp3 = simde_mm_unpacklo_epi16(f6j, f7j); // f6(i) f7(i) f6(i+1) f7(i+1) f6(i+2) f7(i+2) f6(i+3) f7(i+3)
|
||||
//
|
||||
simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i) f1(i) f2(i) f3(i) f0(i+1) f1(i+1) f2(i+1) f3(i+1)
|
||||
simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i) f5(i) f6(i) f7(i) f4(i+1) f5(i+1) f6(i+1) f7(i+1)
|
||||
simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i) f1(i) f2(i) f3(i) f4(i) f5(i) f7(i) f7(i)
|
||||
simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+1) f1(i+1) f2(i+1) f3(i+1) f4(i+1) f5(i+1) f6(i+1) f7(i+1)
|
||||
//
|
||||
tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+2) f1(i+2) f2(i+2) f3(i+2) f0(i+3) f1(i+3) f2(i+3) f3(i+3)
|
||||
tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+2) f5(i+2) f6(i+2) f7(i+2) f4(i+3) f5(i+3) f6(i+3) f7(i+3)
|
||||
simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+2) f1(i+2) f2(i+2) f3(i+2) f4(i+2) f5(i+2) f7(i+2) f7(i+2)
|
||||
simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+3) f1(i+3) f2(i+3) f3(i+3) f4(i+3) f5(i+3) f7(i+3) f7(i+3)
|
||||
|
||||
tmp0 = simde_mm_unpackhi_epi16(f0j, f1j); // f0(i+4) f1(i+4) f0(i+5) f1(i+5) f0(i+6) f1(i+6) f0(i+7) f1(i+7)
|
||||
tmp1 = simde_mm_unpackhi_epi16(f2j, f3j); // f2(i+4) f3(i+4) f2(i+5) f3(i+5) f2(i+6) f3(i+6) f2(i+7) f3(i+7)
|
||||
tmp2 = simde_mm_unpackhi_epi16(f4j, f5j); // f4(i+4) f5(i+4) f4(i+5) f5(i+5) f4(i+6) f5(i+6) f4(i+7) f5(i+7)
|
||||
tmp3 = simde_mm_unpackhi_epi16(f6j, f7j); // f6(i+4) f7(i+4) f6(i+5) f7(i+5) f6(i+6) f7(i+6) f6(i+7) f7(i+7)
|
||||
//
|
||||
tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i+4) f1(i+4) f2(i+4) f3(i+4) f0(i+5) f1(i+5) f2(i+5) f3(i+5)
|
||||
tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i+4) f5(i+4) f6(i+4) f7(i+4) f4(i+5) f5(i+5) f6(i+5) f7(i+5)
|
||||
|
||||
simde_mm_storeu_si128(e4_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i+4) f1(i+4) f2(i+4) f3(i+4) f4(i+4) f5(i+4) f7(i+4) f7(i+4)
|
||||
simde_mm_storeu_si128(e5_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+5) f1(i+5) f2(i+5) f3(i+5) f4(i+5) f5(i+5) f6(i+5) f7(i+5)
|
||||
tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+6) f1(i+6) f2(i+6) f3(i+6) f0(i+6) f1(i+6) f2(i+6) f3(i+6)
|
||||
tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+6) f5(i+6) f6(i+6) f7(i+6) f4(i+6) f5(i+6) f6(i+6) f7(i+6)
|
||||
simde_mm_storeu_si128(e6_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+6) f1(i+6) f2(i+6) f3(i+6) f4(i+6) f5(i+6) f7(i+6) f7(i+6)
|
||||
simde_mm_storeu_si128(e7_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+7) f1(i+7) f2(i+7) f3(i+7) f4(i+7) f5(i+7) f7(i+7) f7(i+7)
|
||||
}
|
||||
e = (int16_t *)e0_128;
|
||||
e1 = (int16_t *)e1_128;
|
||||
e2 = (int16_t *)e2_128;
|
||||
e3 = (int16_t *)e3_128;
|
||||
e4 = (int16_t *)e4_128;
|
||||
e5 = (int16_t *)e5_128;
|
||||
e6 = (int16_t *)e6_128;
|
||||
e7 = (int16_t *)e7_128;
|
||||
f = (int16_t *)f128;
|
||||
#endif
|
||||
for(;i < EQm ; i++) {
|
||||
*e++ = *f++;
|
||||
*e1++ = *f++;
|
||||
*e2++ = *f++;
|
||||
@@ -494,6 +971,7 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
|
||||
AssertFatal(1 == 0, "Should not get here : Qm %d\n", Qm);
|
||||
break;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
int nr_rate_matching_ldpc(uint32_t Tbslbrm,
|
||||
@@ -593,6 +1071,82 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
|
||||
return 0;
|
||||
}
|
||||
|
||||
#define RMLOOP for (;ind<(ind2&7);k+=8,ind+=8) \
|
||||
simde_mm_storeu_si128(&d[ind],simde_mm_adds_epi16(simde_mm_loadu_si128(&soft_input[k]),simde_mm_loadu_si128(&d[ind])));\
|
||||
for (; ind<ind2 ; ind++,k++) d[ind] += soft_input[k];
|
||||
|
||||
int nr_rate_matching_ldpc_rx_simd(uint32_t Tbslbrm,
|
||||
uint8_t BG,
|
||||
uint16_t Z,
|
||||
int16_t *d,
|
||||
int16_t *soft_input,
|
||||
uint8_t C,
|
||||
uint8_t rvidx,
|
||||
uint8_t clear,
|
||||
uint32_t E,
|
||||
uint32_t F,
|
||||
uint32_t Foffset)
|
||||
{
|
||||
if (C == 0) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
|
||||
return -1;
|
||||
}
|
||||
|
||||
//Bit selection
|
||||
uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
|
||||
uint32_t Ncb;
|
||||
if (Tbslbrm == 0)
|
||||
Ncb = N;
|
||||
else {
|
||||
uint32_t Nref = (3 * Tbslbrm / (2 * C)); //R_LBRM = 2/3
|
||||
Ncb = min(N, Nref);
|
||||
}
|
||||
|
||||
uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
|
||||
if (Foffset > E) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > E %d)\n", Foffset, E);
|
||||
return -1;
|
||||
}
|
||||
if (Foffset > Ncb) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
|
||||
return -1;
|
||||
}
|
||||
|
||||
#ifdef RM_DEBUG
|
||||
printf("nr_rate_matching_ldpc_rx: Clear %d, E %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
|
||||
clear,
|
||||
E,
|
||||
Foffset,
|
||||
ind,
|
||||
Ncb,
|
||||
rvidx,
|
||||
Tbslbrm);
|
||||
#endif
|
||||
|
||||
if (clear == 1)
|
||||
memset(d, 0, Ncb * sizeof(int16_t));
|
||||
|
||||
uint32_t k = 0;
|
||||
if (ind < Foffset) {
|
||||
int ind2 = ind + min(Foffset-ind,E);
|
||||
RMLOOP;
|
||||
}
|
||||
if (ind >= Foffset && ind < Foffset + F)
|
||||
ind = Foffset + F;
|
||||
int ind2 = ind + min(Ncb-ind,E-k);
|
||||
RMLOOP;
|
||||
|
||||
while (k < E) {
|
||||
ind=0;
|
||||
ind2 = min(Foffset,E-k);
|
||||
RMLOOP;
|
||||
ind = Foffset+F;
|
||||
ind2 = ind + min(Ncb-ind,E-k);
|
||||
RMLOOP;
|
||||
}
|
||||
return 0;
|
||||
}
|
||||
|
||||
int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
|
||||
uint8_t BG,
|
||||
uint16_t Z,
|
||||
@@ -605,6 +1159,21 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
|
||||
uint32_t F,
|
||||
uint32_t Foffset)
|
||||
{
|
||||
if (BG == 1) {
|
||||
nr_rate_matching_ldpc_rx_simd(Tbslbrm,
|
||||
BG,
|
||||
Z,
|
||||
d,
|
||||
soft_input,
|
||||
C,
|
||||
rvidx,
|
||||
clear,
|
||||
E,
|
||||
F,
|
||||
Foffset);
|
||||
return 0;
|
||||
}
|
||||
|
||||
if (C == 0) {
|
||||
LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
|
||||
return -1;
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
##########################################################
|
||||
# LDPC tee library - tee to two libraries
|
||||
##########################################################
|
||||
|
||||
add_boolean_option(ENABLE_LDPC_TEE OFF "Build support for LDPC tee to two LDPC libraries" OFF)
|
||||
|
||||
if (ENABLE_LDPC_TEE)
|
||||
add_library(ldpc_tee MODULE nrLDPC_coding_tee.c)
|
||||
|
||||
set_target_properties(ldpc_tee PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
|
||||
|
||||
add_dependencies(nr-softmodem ldpc_tee)
|
||||
add_dependencies(nr-uesoftmodem ldpc_tee)
|
||||
add_dependencies(nr_ulsim ldpc_tee)
|
||||
add_dependencies(nr_ulschsim ldpc_tee)
|
||||
add_dependencies(nr_dlsim ldpc_tee)
|
||||
add_dependencies(nr_dlschsim ldpc_tee)
|
||||
|
||||
endif()
|
||||
@@ -0,0 +1,182 @@
|
||||
/*
|
||||
* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
* contributor license agreements. See the NOTICE file distributed with
|
||||
* this work for additional information regarding copyright ownership.
|
||||
* The OpenAirInterface Software Alliance licenses this file to You under
|
||||
* the OAI Public License, Version 1.0 (the "License"); you may not use this file
|
||||
* except in compliance with the License.
|
||||
* You may obtain a copy of the License at
|
||||
*
|
||||
* http://www.openairinterface.org/?page_id=698
|
||||
*
|
||||
* Unless required by applicable law or agreed to in writing, software
|
||||
* distributed under the License is distributed on an "AS IS" BASIS,
|
||||
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
* See the License for the specific language governing permissions and
|
||||
* limitations under the License.
|
||||
*-------------------------------------------------------------------------------
|
||||
* For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
* contact@openairinterface.org
|
||||
*/
|
||||
|
||||
/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_tee/nrLDPC_coding_tee.c
|
||||
* \brief LDPC coding library acting as a tee between two LDPC libraries.
|
||||
* It distributes the TBs between the two libraries according to some criterion.
|
||||
*/
|
||||
|
||||
#include "common/config/config_userapi.h"
|
||||
#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
|
||||
|
||||
nrLDPC_coding_interface_t nrLDPC_coding_interface_default, nrLDPC_coding_interface_offload;
|
||||
|
||||
bool choose_offload_decode(nrLDPC_TB_decoding_parameters_t *TB)
|
||||
{
|
||||
return TB->Z >= 128 && TB->BG == 1;
|
||||
}
|
||||
|
||||
int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters)
|
||||
{
|
||||
uint8_t nb_tb = nrLDPC_slot_decoding_parameters->nb_TBs;
|
||||
nrLDPC_TB_decoding_parameters_t *TBs = nrLDPC_slot_decoding_parameters->TBs;
|
||||
|
||||
// split and count TBs
|
||||
nrLDPC_TB_decoding_parameters_t TBs_default[nb_tb];
|
||||
uint8_t tb_id_default = 0;
|
||||
nrLDPC_TB_decoding_parameters_t TBs_offload[nb_tb];
|
||||
uint8_t tb_id_offload = 0;
|
||||
for (uint8_t tb_id = 0; tb_id < nb_tb; tb_id++) {
|
||||
if (choose_offload_decode(&TBs[tb_id])) {
|
||||
TBs_offload[tb_id_offload] = TBs[tb_id];
|
||||
tb_id_offload++;
|
||||
} else {
|
||||
TBs_default[tb_id_default] = TBs[tb_id];
|
||||
tb_id_default++;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO parallelize
|
||||
int ret_decoder = 0;
|
||||
if (tb_id_default > 0) {
|
||||
nrLDPC_slot_decoding_parameters_t slot_parameters_default = {.frame = nrLDPC_slot_decoding_parameters->frame,
|
||||
.slot = nrLDPC_slot_decoding_parameters->slot,
|
||||
.nb_TBs = tb_id_default,
|
||||
.threadPool = nrLDPC_slot_decoding_parameters->threadPool,
|
||||
.TBs = TBs_default};
|
||||
int ret_default = nrLDPC_coding_interface_default.nrLDPC_coding_decoder(&slot_parameters_default);
|
||||
ret_decoder = ret_default == 0 ? ret_decoder : ret_default;
|
||||
}
|
||||
|
||||
if (tb_id_offload > 0) {
|
||||
nrLDPC_slot_decoding_parameters_t slot_parameters_offload = {.frame = nrLDPC_slot_decoding_parameters->frame,
|
||||
.slot = nrLDPC_slot_decoding_parameters->slot,
|
||||
.nb_TBs = tb_id_offload,
|
||||
.threadPool = nrLDPC_slot_decoding_parameters->threadPool,
|
||||
.TBs = TBs_offload};
|
||||
int ret_offload = nrLDPC_coding_interface_offload.nrLDPC_coding_decoder(&slot_parameters_offload);
|
||||
ret_decoder = ret_offload == 0 ? ret_decoder : ret_offload;
|
||||
}
|
||||
return ret_decoder;
|
||||
}
|
||||
|
||||
bool choose_offload_encode(nrLDPC_TB_encoding_parameters_t *TB)
|
||||
{
|
||||
return TB->BG == 1 && TB->C > 8 && TB->Z == 384;
|
||||
}
|
||||
|
||||
int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
|
||||
{
|
||||
uint8_t nb_tb = nrLDPC_slot_encoding_parameters->nb_TBs;
|
||||
nrLDPC_TB_encoding_parameters_t *TBs = nrLDPC_slot_encoding_parameters->TBs;
|
||||
|
||||
// split and count TBs
|
||||
nrLDPC_TB_encoding_parameters_t TBs_default[nb_tb];
|
||||
uint8_t tb_id_default = 0;
|
||||
nrLDPC_TB_encoding_parameters_t TBs_offload[nb_tb];
|
||||
uint8_t tb_id_offload = 0;
|
||||
for (uint8_t tb_id = 0; tb_id < nb_tb; tb_id++) {
|
||||
if (choose_offload_encode(&TBs[tb_id])) {
|
||||
TBs_offload[tb_id_offload] = TBs[tb_id];
|
||||
tb_id_offload++;
|
||||
} else {
|
||||
TBs_default[tb_id_default] = TBs[tb_id];
|
||||
tb_id_default++;
|
||||
}
|
||||
}
|
||||
|
||||
// TODO parallelize
|
||||
int ret_encoder = 0;
|
||||
if (tb_id_default > 0) {
|
||||
nrLDPC_slot_encoding_parameters_t slot_parameters_default = {.frame = nrLDPC_slot_encoding_parameters->frame,
|
||||
.slot = nrLDPC_slot_encoding_parameters->slot,
|
||||
.nb_TBs = tb_id_default,
|
||||
.threadPool = nrLDPC_slot_encoding_parameters->threadPool,
|
||||
.TBs = TBs_default};
|
||||
int ret_default = nrLDPC_coding_interface_default.nrLDPC_coding_encoder(&slot_parameters_default);
|
||||
ret_encoder = ret_default == 0 ? ret_encoder : ret_default;
|
||||
}
|
||||
|
||||
if (tb_id_offload > 0) {
|
||||
nrLDPC_slot_encoding_parameters_t slot_parameters_offload = {.frame = nrLDPC_slot_encoding_parameters->frame,
|
||||
.slot = nrLDPC_slot_encoding_parameters->slot,
|
||||
.nb_TBs = tb_id_offload,
|
||||
.threadPool = nrLDPC_slot_encoding_parameters->threadPool,
|
||||
.TBs = TBs_offload};
|
||||
int ret_offload = nrLDPC_coding_interface_offload.nrLDPC_coding_encoder(&slot_parameters_offload);
|
||||
ret_encoder = ret_offload == 0 ? ret_encoder : ret_offload;
|
||||
}
|
||||
return ret_encoder;
|
||||
}
|
||||
|
||||
int32_t nrLDPC_coding_init(int max_num_pxsch)
|
||||
{
|
||||
int ret_init = 0;
|
||||
|
||||
// load LDPC default library
|
||||
// First query configmodule to know which default library was provided
|
||||
char *shlibversion_default = NULL;
|
||||
// clang-format off
|
||||
paramdef_t LoaderParams_default[] = {
|
||||
{"shlibversion", NULL, 0, .strptr = &shlibversion_default, .defstrval = "", TYPE_STRING, 0, NULL}
|
||||
};
|
||||
// clang-format on
|
||||
char *cfgprefix_default = "nrLDPC_coding_tee.default";
|
||||
int ret_cfgmodule = config_get(config_get_if(), LoaderParams_default, sizeofArray(LoaderParams_default), cfgprefix_default);
|
||||
if (ret_cfgmodule < 0) {
|
||||
fprintf(stderr, "[LOADER] %s %d couldn't retrieve config from section %s\n", __FILE__, __LINE__, cfgprefix_default);
|
||||
}
|
||||
// load
|
||||
int ret_loader = load_nrLDPC_coding_interface(shlibversion_default, &nrLDPC_coding_interface_default, max_num_pxsch);
|
||||
ret_init = ret_loader == 0 ? ret_init : ret_loader;
|
||||
|
||||
// load LDPC offload library
|
||||
// First query configmodule to know which offload library was provided
|
||||
char *shlibversion_offload = NULL;
|
||||
// clang-format off
|
||||
paramdef_t LoaderParams_offload[] = {
|
||||
{"shlibversion", NULL, 0, .strptr = &shlibversion_offload, .defstrval = "", TYPE_STRING, 0, NULL}
|
||||
};
|
||||
// clang-format on
|
||||
char *cfgprefix_offload = "nrLDPC_coding_tee.offload";
|
||||
ret_cfgmodule = config_get(config_get_if(), LoaderParams_offload, sizeofArray(LoaderParams_offload), cfgprefix_offload);
|
||||
if (ret_cfgmodule < 0) {
|
||||
fprintf(stderr, "[LOADER] %s %d couldn't retrieve config from section %s\n", __FILE__, __LINE__, cfgprefix_offload);
|
||||
}
|
||||
// load
|
||||
ret_loader = load_nrLDPC_coding_interface(shlibversion_offload, &nrLDPC_coding_interface_offload, max_num_pxsch);
|
||||
ret_init = ret_loader == 0 ? ret_init : ret_loader;
|
||||
|
||||
return ret_init;
|
||||
}
|
||||
|
||||
int32_t nrLDPC_coding_shutdown(void)
|
||||
{
|
||||
int ret_shutdown = 0;
|
||||
|
||||
int ret_default = nrLDPC_coding_interface_default.nrLDPC_coding_shutdown();
|
||||
ret_shutdown = ret_default == 0 ? ret_shutdown : ret_default;
|
||||
|
||||
int ret_offload = nrLDPC_coding_interface_offload.nrLDPC_coding_shutdown();
|
||||
ret_shutdown = ret_offload == 0 ? ret_shutdown : ret_offload;
|
||||
|
||||
return ret_shutdown;
|
||||
}
|
||||
@@ -60,14 +60,14 @@ typedef struct args_fpga_decode_prepare_s {
|
||||
task_ans_t *ans; /*!< pointer to the answer that is used by thread pool to detect job completion */
|
||||
} args_fpga_decode_prepare_t;
|
||||
|
||||
int32_t nrLDPC_coding_init(void);
|
||||
int32_t nrLDPC_coding_init(int);
|
||||
int32_t nrLDPC_coding_shutdown(void);
|
||||
int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *slot_params, int frame_rx, int slot_rx);
|
||||
// int32_t nrLDPC_coding_encoder(void);
|
||||
int decoder_xdma(nrLDPC_TB_decoding_parameters_t *TB_params, int frame_rx, int slot_rx, tpool_t *ldpc_threadPool);
|
||||
void nr_ulsch_FPGA_decoding_prepare_blocks(void *args);
|
||||
|
||||
int32_t nrLDPC_coding_init(void)
|
||||
int32_t nrLDPC_coding_init(int max_num_pxsch)
|
||||
{
|
||||
paramdef_t LoaderParams[] = {
|
||||
{"num_threads_prepare", NULL, 0, .iptr = &num_threads_prepare_max, .defintval = 0, TYPE_INT, 0, NULL},
|
||||
|
||||
@@ -0,0 +1,444 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
/*! \file nrLDPC_CUDA_BnProcKernel_BG1.h
|
||||
* \brief Defines the kernels for bit node processing
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "nrLDPC_types.h"
|
||||
|
||||
__device__ __forceinline__ void unpack_and_sign_extend(uint32_t packed, uint32_t *val_lo, uint32_t *val_hi)
|
||||
{
|
||||
uint32_t magic_sub = 0x00800080;
|
||||
|
||||
uint32_t lo_zext = __byte_perm(packed, 0, 0x5140);
|
||||
*val_lo = __vsub2(lo_zext ^ magic_sub, magic_sub);
|
||||
|
||||
uint32_t hi_zext = __byte_perm(packed, 0, 0x7362);
|
||||
*val_hi = __vsub2(hi_zext ^ magic_sub, magic_sub);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t saturate_and_pack(uint32_t val_lo, uint32_t val_hi)
|
||||
{
|
||||
uint32_t lo_clamped = __vmins2(val_lo, 0x007F007F); // min(v, 127)
|
||||
lo_clamped = __vmaxs2(lo_clamped, 0xFF80FF80); // max(v, -128)
|
||||
|
||||
uint32_t hi_clamped = __vmins2(val_hi, 0x007F007F);
|
||||
hi_clamped = __vmaxs2(hi_clamped, 0xFF80FF80);
|
||||
|
||||
return __byte_perm(lo_clamped, hi_clamped, 0x6420);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R13(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t BnGrpIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t Bn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)BnGrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < BnGrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
uint32_t BricksToBeGet;
|
||||
if (BnGrpIdx == 1) {
|
||||
BricksToBeGet = packed_intrinsic;
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
} else {
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
|
||||
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
uint32_t prev = bnProcBufPtr[prevIdxWords];
|
||||
BricksToBeGet = __vsubss4(saturated_llr, prev);
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R23(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t BnGrpIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t Bn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)BnGrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < BnGrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
uint32_t BricksToBeGet;
|
||||
if (BnGrpIdx == 1) {
|
||||
BricksToBeGet = packed_intrinsic;
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
} else {
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
|
||||
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
uint32_t prev = bnProcBufPtr[prevIdxWords];
|
||||
BricksToBeGet = __vsubss4(saturated_llr, prev);
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R89(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t BnGrpIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t Bn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)BnGrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < BnGrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
uint32_t BricksToBeGet;
|
||||
if (BnGrpIdx == 1) {
|
||||
BricksToBeGet = packed_intrinsic;
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
} else {
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
|
||||
uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
uint32_t prev = bnProcBufPtr[prevIdxWords];
|
||||
BricksToBeGet = __vsubss4(saturated_llr, prev);
|
||||
uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
|
||||
int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
|
||||
|
||||
moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_last(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t BnGrpIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t Bn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)BnGrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < BnGrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
((int32_t *)(d_llrRes))[lane] = saturated_llr;
|
||||
}
|
||||
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t GrpIdx,
|
||||
uint32_t MsgIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)GrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < GrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
uint32_t BricksToBeGet;
|
||||
if (GrpIdx == 1) {
|
||||
BricksToBeGet = packed_intrinsic;
|
||||
} else {
|
||||
uint32_t prevIdxWords = (MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
uint32_t prev = bnProcBufPtr[prevIdxWords];
|
||||
BricksToBeGet = __vsubss4(saturated_llr, prev);
|
||||
}
|
||||
|
||||
moveBricks_forput_circ(d_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge_last(const int8_t *__restrict__ d_bnProcBuf,
|
||||
int8_t *__restrict__ d_cnProcBuf,
|
||||
const int8_t *__restrict__ d_llrProcBuf,
|
||||
int8_t *__restrict__ d_llrRes,
|
||||
uint32_t lane,
|
||||
uint32_t GrpIdx,
|
||||
uint32_t MsgIdx,
|
||||
uint32_t BnIdx,
|
||||
uint32_t GrpNum,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
|
||||
|
||||
uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
|
||||
|
||||
uint32_t MsgSumLo, MsgSumHi;
|
||||
unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
|
||||
|
||||
uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
|
||||
const int32_t *currPtr = bnProcBufPtr;
|
||||
|
||||
int i = 0;
|
||||
|
||||
// --- 2-Way Unroll ---
|
||||
|
||||
for (; i < (int)GrpIdx - 1; i += 2) {
|
||||
uint32_t val1 = *currPtr;
|
||||
uint32_t val2 = *(currPtr + off);
|
||||
|
||||
uint32_t v1_lo, v1_hi;
|
||||
unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
|
||||
|
||||
uint32_t v2_lo, v2_hi;
|
||||
unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
|
||||
|
||||
currPtr += (off << 1);
|
||||
}
|
||||
|
||||
if (i < GrpIdx) {
|
||||
uint32_t val = *currPtr;
|
||||
uint32_t v_lo, v_hi;
|
||||
unpack_and_sign_extend(val, &v_lo, &v_hi);
|
||||
MsgSumLo = __vaddss2(MsgSumLo, v_lo);
|
||||
MsgSumHi = __vaddss2(MsgSumHi, v_hi);
|
||||
}
|
||||
|
||||
uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
|
||||
|
||||
if (MsgIdx == 0) {
|
||||
((int32_t *)(d_llrRes))[lane] = saturated_llr;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,567 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
/*! \file nrLDPC_CUDA_CnProcKernel_BG1.h
|
||||
* \brief Defines the kernels for check node processing
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "nrLDPC_types.h"
|
||||
#include "nrLDPC_CUDA_public.h"
|
||||
#include "nrLDPC_CUDA_lut.h"
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R13_node(const int8_t *__restrict__ d_cnProcBuf,
|
||||
int8_t *__restrict__ d_bnProcBuf,
|
||||
uint32_t lane,
|
||||
uint32_t CnIdx,
|
||||
uint32_t CnNumInGrp,
|
||||
uint32_t CnGrpIdxNum,
|
||||
uint32_t Cn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
uint32_t min1 = 0x7F7F7F7F;
|
||||
uint32_t min2 = 0x7F7F7F7F;
|
||||
uint32_t total_xor = 0;
|
||||
|
||||
uint32_t cache_raw[19];
|
||||
uint32_t cache_abs[19];
|
||||
|
||||
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
|
||||
const int32_t *currPtr = cnProcBufPtr;
|
||||
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
|
||||
#pragma unroll
|
||||
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
|
||||
uint32_t val = *currPtr;
|
||||
|
||||
cache_raw[MsgIdx] = val;
|
||||
uint32_t v_abs = __vabs4(val);
|
||||
cache_abs[MsgIdx] = v_abs;
|
||||
|
||||
total_xor = __vxor4(total_xor, val);
|
||||
|
||||
uint32_t old_min1 = min1;
|
||||
|
||||
min1 = __vminu4(old_min1, v_abs);
|
||||
|
||||
uint32_t candidate = __vmaxu4(old_min1, v_abs);
|
||||
|
||||
min2 = __vminu4(min2, candidate);
|
||||
currPtr += offset;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
|
||||
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
|
||||
|
||||
uint32_t my_abs = cache_abs[temp_MsgIdx];
|
||||
|
||||
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
|
||||
|
||||
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
|
||||
|
||||
// final_mag = scale_int8x4(final_mag, 0.75);
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
|
||||
|
||||
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = cn_bn_map_BG1_Z_R13[MsgIdx][ZcIdx];
|
||||
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R13[MsgIdx][0]);
|
||||
|
||||
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
}
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R23_node(const int8_t *__restrict__ d_cnProcBuf,
|
||||
int8_t *__restrict__ d_bnProcBuf,
|
||||
uint32_t lane,
|
||||
uint32_t CnIdx,
|
||||
uint32_t CnNumInGrp,
|
||||
uint32_t CnGrpIdxNum,
|
||||
uint32_t Cn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
|
||||
uint32_t min1 = 0x7F7F7F7F;
|
||||
uint32_t min2 = 0x7F7F7F7F;
|
||||
uint32_t total_xor = 0;
|
||||
|
||||
uint32_t cache_raw[19];
|
||||
uint32_t cache_abs[19];
|
||||
|
||||
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
|
||||
const int32_t *currPtr = cnProcBufPtr;
|
||||
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
|
||||
#pragma unroll
|
||||
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
|
||||
uint32_t val = *currPtr;
|
||||
|
||||
cache_raw[MsgIdx] = val;
|
||||
uint32_t v_abs = __vabs4(val);
|
||||
cache_abs[MsgIdx] = v_abs;
|
||||
|
||||
total_xor = __vxor4(total_xor, val);
|
||||
|
||||
uint32_t old_min1 = min1;
|
||||
|
||||
min1 = __vminu4(old_min1, v_abs);
|
||||
|
||||
uint32_t candidate = __vmaxu4(old_min1, v_abs);
|
||||
|
||||
min2 = __vminu4(min2, candidate);
|
||||
currPtr += offset;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
|
||||
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
|
||||
|
||||
uint32_t my_abs = cache_abs[temp_MsgIdx];
|
||||
|
||||
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
|
||||
|
||||
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
|
||||
|
||||
// final_mag = scale_int8x4(final_mag, 0.75);
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
|
||||
|
||||
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = cn_bn_map_BG1_Z_R23[MsgIdx][ZcIdx];
|
||||
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R23[MsgIdx][0]);
|
||||
|
||||
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
}
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R89_node(const int8_t *__restrict__ d_cnProcBuf,
|
||||
int8_t *__restrict__ d_bnProcBuf,
|
||||
uint32_t lane,
|
||||
uint32_t CnIdx,
|
||||
uint32_t CnNumInGrp,
|
||||
uint32_t CnGrpIdxNum,
|
||||
uint32_t Cn2MsgStartIdx,
|
||||
uint32_t Zc,
|
||||
uint32_t ZcIdx)
|
||||
{
|
||||
|
||||
uint32_t min1 = 0x7F7F7F7F;
|
||||
uint32_t min2 = 0x7F7F7F7F;
|
||||
uint32_t total_xor = 0;
|
||||
|
||||
uint32_t cache_raw[19];
|
||||
uint32_t cache_abs[19];
|
||||
|
||||
const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
|
||||
const int32_t *currPtr = cnProcBufPtr;
|
||||
uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
|
||||
#pragma unroll
|
||||
for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
|
||||
uint32_t val = *currPtr;
|
||||
|
||||
cache_raw[MsgIdx] = val;
|
||||
uint32_t v_abs = __vabs4(val);
|
||||
cache_abs[MsgIdx] = v_abs;
|
||||
|
||||
total_xor = __vxor4(total_xor, val);
|
||||
|
||||
uint32_t old_min1 = min1;
|
||||
|
||||
min1 = __vminu4(old_min1, v_abs);
|
||||
|
||||
uint32_t candidate = __vmaxu4(old_min1, v_abs);
|
||||
|
||||
min2 = __vminu4(min2, candidate);
|
||||
currPtr += offset;
|
||||
}
|
||||
|
||||
#pragma unroll
|
||||
for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
|
||||
uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
|
||||
|
||||
uint32_t my_abs = cache_abs[temp_MsgIdx];
|
||||
|
||||
uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
|
||||
|
||||
uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
|
||||
|
||||
// final_mag = scale_int8x4(final_mag, 0.75);
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
|
||||
|
||||
uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
|
||||
uint32_t circShift = cn_bn_map_BG1_Z_R89[MsgIdx][ZcIdx];
|
||||
int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R89[MsgIdx][0]);
|
||||
|
||||
moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
}
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G3(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
// loop starts here
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][1] * 4);
|
||||
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
;
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G4(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][0] * 4);
|
||||
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G5(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G6(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G7(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][5] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G8(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][5] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][6] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G9(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][5] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][6] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][7] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G10(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][0] * 4);
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][5] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][6] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][7] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][8] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void cnProcKernel_BG1_int8_G19(const int8_t *__restrict__ p_cnProcBuf,
|
||||
int8_t *__restrict__ p_bnProcBuf,
|
||||
uint32_t row,
|
||||
uint32_t lane,
|
||||
uint32_t idxBn,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t ymm0, sgn, min;
|
||||
const uint32_t ones = 0x01010101;
|
||||
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][0] * 4);
|
||||
|
||||
sgn = __vxor4(ones, ymm0);
|
||||
min = __vabs4(ymm0);
|
||||
|
||||
//-------------------------loop starts here-------------------------------
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][1] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][2] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][3] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][4] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][5] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][6] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][7] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][8] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][9] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][10] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][11] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][12] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][13] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][14] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][15] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][16] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][17] * 4);
|
||||
min = __vminu4(min, __vabs4(ymm0));
|
||||
sgn = __vxor4(sgn, ymm0);
|
||||
//-------------------------------------------------------------------------
|
||||
uint32_t BricksToBeMoved = __vsign4(min, sgn);
|
||||
|
||||
moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
|
||||
}
|
||||
2287
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_lut.h
Normal file
2287
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_lut.h
Normal file
File diff suppressed because it is too large
Load Diff
187
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_mPassKernel_BG1.h
Normal file
187
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_mPassKernel_BG1.h
Normal file
@@ -0,0 +1,187 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
/*! \file nrLDPC_CUDA_mPassKernel.h
|
||||
* \brief Defines the kernels for message passing in CUDA version of LDPC decoder
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "nrLDPC_types.h"
|
||||
#include "nrLDPC_CUDA_public.h"
|
||||
#include "nrLDPC_CUDA_shared_param.h"
|
||||
|
||||
__device__ __forceinline__ void llrPreProc_Kernel_BG1_int8_Gn_stream(const int8_t *p_llr,
|
||||
int8_t *p_llrProcBuf,
|
||||
int8_t *p_cnProcBuf,
|
||||
uint32_t MsgIdx,
|
||||
uint32_t lane,
|
||||
uint32_t colIdx,
|
||||
uint32_t idxBn,
|
||||
uint32_t GrpIdx,
|
||||
uint32_t circShift,
|
||||
uint32_t Zc,
|
||||
uint32_t R)
|
||||
{
|
||||
{
|
||||
uint32_t *p_cnProcBufBit;
|
||||
|
||||
uint8_t bricksLocal[4];
|
||||
uint8_t *BricksToBeMoved = bricksLocal;
|
||||
|
||||
p_cnProcBufBit = (uint32_t *)(p_cnProcBuf + d_lut_numCnInCnGroups_BG1_R13[GrpIdx] * NR_LDPC_ZMAX * MsgIdx + lane * 4);
|
||||
|
||||
moveBricks_invget_circ((int8_t *)&p_llr[idxBn * Zc], lane * 4, BricksToBeMoved, Zc, circShift);
|
||||
|
||||
*p_cnProcBufBit = *(uint32_t *)BricksToBeMoved;
|
||||
}
|
||||
|
||||
if (colIdx >= 68)
|
||||
return;
|
||||
|
||||
const uint8_t numBn2CnG1 =
|
||||
(R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
|
||||
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]); // for R13 is 42
|
||||
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1; // 26 for BG1
|
||||
const uint32_t colG1 = startColParity * Zc;
|
||||
|
||||
const uint32_t *lut_llr2llrProcBufAddr =
|
||||
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
|
||||
const uint32_t *lut_llr2llrProcBufBnPos =
|
||||
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
|
||||
|
||||
if (colIdx < startColParity) {
|
||||
const uint32_t idxBn = lut_llr2llrProcBufAddr[colIdx] + lut_llr2llrProcBufBnPos[colIdx] * NR_LDPC_ZMAX;
|
||||
int32_t *dst = (int32_t *)(&p_llrProcBuf[idxBn] + lane * 4);
|
||||
int32_t *src = (int32_t *)(&p_llr[colIdx * Zc] + lane * 4);
|
||||
*dst = *src;
|
||||
} else {
|
||||
colIdx = colIdx - startColParity;
|
||||
if (numBn2CnG1 > 0 && colIdx < numBn2CnG1) {
|
||||
int32_t *dst = (int32_t *)(&p_llrProcBuf[colIdx * NR_LDPC_ZMAX] + lane * 4);
|
||||
int32_t *src = (int32_t *)(&p_llr[colG1 + colIdx * Zc] + lane * 4);
|
||||
*dst = *src;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ void llr2bit_Kernel_BG1_int8(uint32_t R,
|
||||
uint8_t *__restrict__ out,
|
||||
const int8_t *__restrict__ llrRes,
|
||||
uint32_t numLLR,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t lane = threadIdx.x;
|
||||
uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
|
||||
|
||||
if (outColIdx >= num_TotalBlocks_llr_llrRes)
|
||||
return;
|
||||
|
||||
// Constants Setup
|
||||
const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
|
||||
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
|
||||
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
|
||||
const uint32_t *lut_Addr =
|
||||
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
|
||||
const uint32_t *lut_Pos =
|
||||
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
|
||||
|
||||
int32_t raw_llrs;
|
||||
|
||||
// Load Data: Handle Systematic (Scatter/Gather) vs Parity (Linear) mapping
|
||||
if (outColIdx < startColParity) {
|
||||
uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
|
||||
raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
|
||||
} else {
|
||||
uint32_t srcParityIdx = outColIdx - startColParity;
|
||||
if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
|
||||
raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
|
||||
} else {
|
||||
raw_llrs = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Hard Decision: Convert 4 int8 LLRs -> 4 bytes (0 or 1)
|
||||
int8_t *p_val = (int8_t *)&raw_llrs;
|
||||
uint32_t my_word = 0;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
// Hard decision: LLR < 0 implies bit 1
|
||||
uint32_t byte_val = (p_val[i] < 0) ? 1 : 0;
|
||||
my_word |= (byte_val << (i * 8));
|
||||
}
|
||||
|
||||
// Store output linearly
|
||||
*(uint32_t *)(&out[outColIdx * Zc + lane * 4]) = my_word;
|
||||
}
|
||||
|
||||
__device__ void llr2bitPacked_Kernel_BG1_int8(uint32_t R,
|
||||
uint8_t *__restrict__ out,
|
||||
const int8_t *__restrict__ llrRes,
|
||||
uint32_t numLLR,
|
||||
uint32_t Zc)
|
||||
{
|
||||
uint32_t lane = threadIdx.x;
|
||||
uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
|
||||
|
||||
if (outColIdx >= num_TotalBlocks_llr_llrRes)
|
||||
return;
|
||||
|
||||
const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
|
||||
: ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
|
||||
const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
|
||||
const uint32_t *lut_Addr =
|
||||
(R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
|
||||
const uint32_t *lut_Pos =
|
||||
(R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
|
||||
|
||||
int32_t raw_llrs;
|
||||
|
||||
// Load Data: Handle Systematic vs Parity mapping
|
||||
if (outColIdx < startColParity) {
|
||||
uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
|
||||
raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
|
||||
} else {
|
||||
uint32_t srcParityIdx = outColIdx - startColParity;
|
||||
if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
|
||||
raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
|
||||
} else {
|
||||
raw_llrs = 0;
|
||||
}
|
||||
}
|
||||
|
||||
// Extract 4 bits from 4 LLRs (Local Packing)
|
||||
// Assuming MSB order within nibble: [3][2][1][0]
|
||||
uint32_t my_4_bits = 0;
|
||||
int8_t *p_val = (int8_t *)&raw_llrs;
|
||||
|
||||
#pragma unroll
|
||||
for (int i = 0; i < 4; i++) {
|
||||
if (p_val[i] < 0) {
|
||||
my_4_bits |= (1 << (3 - i));
|
||||
}
|
||||
}
|
||||
|
||||
// Thread Cooperation: Pair threads to pack 8 bits (1 byte)
|
||||
// Even thread (0,2..) takes low nibble, Odd thread (1,3..) takes high nibble.
|
||||
uint32_t neighbor_bits = __shfl_xor_sync(0xffffffff, my_4_bits, 1);
|
||||
|
||||
if ((lane & 1) == 0) {
|
||||
// Combine: [Odd Thread Bits (High)] | [My Bits (Low)]
|
||||
uint8_t packed_byte = (neighbor_bits & 0xF) | ((my_4_bits & 0xF) << 4);
|
||||
|
||||
// Output addr: lane steps by 4 LLRs, but we output 1 byte per 8 LLRs -> lane >> 1
|
||||
uint32_t outAddr = outColIdx * (Zc >> 3) + (lane >> 1);
|
||||
out[outAddr] = packed_byte;
|
||||
}
|
||||
}
|
||||
121
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_public.h
Normal file
121
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_CUDA_public.h
Normal file
@@ -0,0 +1,121 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
/*! \file nrLDPC_CUDA_public.h
|
||||
* \brief Shared functions in CUDA implementation of LDPC decoder
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
#include <stdio.h>
|
||||
#include "nrLDPC_types.h"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#include <stdint.h>
|
||||
|
||||
|
||||
__device__ __forceinline__ void moveBricks_invput_circ(int8_t *__restrict__ dstBuf,
|
||||
uint32_t dstBuf_Offset,
|
||||
const uint8_t *__restrict__ Four_Bricks,
|
||||
uint32_t Z,
|
||||
uint32_t cshift)
|
||||
{
|
||||
uint32_t tmp = cshift + dstBuf_Offset;
|
||||
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
|
||||
|
||||
if (pos <= Z - 4) {
|
||||
uint32_t val = *(const uint32_t*)Four_Bricks;
|
||||
memcpy(dstBuf + pos, &val, 4);
|
||||
}
|
||||
else {
|
||||
uint32_t bytes_at_end = Z - pos; // 1, 2, or 3
|
||||
|
||||
#pragma unroll
|
||||
for(int i=0; i<4; i++) {
|
||||
if (i < bytes_at_end) {
|
||||
dstBuf[pos + i] = Four_Bricks[i];
|
||||
} else {
|
||||
dstBuf[i - bytes_at_end] = Four_Bricks[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void moveBricks_forput_circ(int8_t *__restrict__ dstBuf,
|
||||
uint32_t dstBuf_Offset,
|
||||
const uint8_t *__restrict__ Four_Bricks,
|
||||
uint32_t Z,
|
||||
uint32_t cshift)
|
||||
{
|
||||
uint32_t tmp = dstBuf_Offset + Z - cshift;
|
||||
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
|
||||
|
||||
if (pos <= Z - 4) {
|
||||
uint32_t val = *(const uint32_t*)Four_Bricks;
|
||||
memcpy(dstBuf + pos, &val, 4);
|
||||
}
|
||||
else {
|
||||
uint32_t bytes_at_end = Z - pos;
|
||||
|
||||
#pragma unroll
|
||||
for(int i=0; i<4; i++) {
|
||||
if (i < bytes_at_end) {
|
||||
dstBuf[pos + i] = Four_Bricks[i];
|
||||
} else {
|
||||
dstBuf[i - bytes_at_end] = Four_Bricks[i];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ void moveBricks_invget_circ(const int8_t *__restrict__ dstBuf,
|
||||
uint32_t dstBuf_Offset,
|
||||
uint8_t *__restrict__ Four_Bricks,
|
||||
uint32_t Z,
|
||||
uint32_t cshift)
|
||||
{
|
||||
uint32_t tmp = cshift + dstBuf_Offset;
|
||||
uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
|
||||
|
||||
if (pos <= Z - 4) {
|
||||
uint32_t val;
|
||||
memcpy(&val, dstBuf + pos, 4);
|
||||
*(uint32_t*)Four_Bricks = val;
|
||||
}
|
||||
else {
|
||||
uint32_t bytes_at_end = Z - pos;
|
||||
|
||||
#pragma unroll
|
||||
for(int i=0; i<4; i++) {
|
||||
if (i < bytes_at_end) {
|
||||
Four_Bricks[i] = dstBuf[pos + i];
|
||||
} else {
|
||||
Four_Bricks[i] = dstBuf[i - bytes_at_end];
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t __vxor4(const uint32_t a, uint32_t b)
|
||||
{
|
||||
return a ^ b;
|
||||
}
|
||||
|
||||
__device__ __forceinline__ uint32_t __vsign4(const uint32_t a, uint32_t b)
|
||||
{
|
||||
uint32_t mask = __vcmplts4(b, 0);
|
||||
uint32_t bneg = __vneg4(a);
|
||||
return (mask & bneg) | (~mask & a);
|
||||
//uint32_t is_zero_mask = __vcmpeq4(b, 0);
|
||||
//return result & (~is_zero_mask);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,79 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
/*! \file nrLDPC_CUDA_shared_param.h
|
||||
* \brief Shared parameters in CUDA implementation of LDPC decoder
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#ifndef NRLDPC_CUDA_SHARED_PARAM_H_
|
||||
#define NRLDPC_CUDA_SHARED_PARAM_H_
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
#define MAX_NUM_DLSCH_SEGMENTS_DL 132
|
||||
#ifdef __cplusplus
|
||||
extern "C" {
|
||||
#endif
|
||||
|
||||
#define RowLength 96 //Zc = 384/4 = 96
|
||||
|
||||
#define num_TotalBlocks_BG1_R13_Edge 316//based on number of Cn2Bn Msgs
|
||||
#define num_TotalBlocks_BG1_R23_Edge 144
|
||||
#define num_TotalBlocks_BG1_R89_Edge 79
|
||||
#define num_TotalBlocks_llr_llrRes 22 //Only includes systematic bits
|
||||
|
||||
#define num_TotalBlocks_cn_BG1_R13_Node 46 //based on number of CNs
|
||||
#define num_TotalBlocks_bn_BG1_R13_Node 68 //based on number of BNs
|
||||
#define num_TotalBlocks_cn_BG1_R23_Node 13
|
||||
#define num_TotalBlocks_bn_BG1_R23_Node 35
|
||||
#define num_TotalBlocks_cn_BG1_R89_Node 5
|
||||
#define num_TotalBlocks_bn_BG1_R89_Node 27
|
||||
|
||||
#define JETSON_ORIN 1
|
||||
|
||||
#if defined(GH200)
|
||||
#define NodeEdge_Switch_Cn_R13 32
|
||||
#define NodeEdge_Switch_Bn_R13 10
|
||||
#define NodeEdge_Switch_Cn_R23 32
|
||||
#define NodeEdge_Switch_Bn_R23 12
|
||||
#define NodeEdge_Switch_Cn_R89 48
|
||||
#define NodeEdge_Switch_Bn_R89 24
|
||||
#elif defined(JETSON_ORIN)
|
||||
#define NodeEdge_Switch_Cn_R13 3
|
||||
#define NodeEdge_Switch_Bn_R13 1
|
||||
#define NodeEdge_Switch_Cn_R23 3
|
||||
#define NodeEdge_Switch_Bn_R23 1
|
||||
#define NodeEdge_Switch_Cn_R89 4
|
||||
#define NodeEdge_Switch_Bn_R89 2
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
typedef struct KernelLaunchConfig {
|
||||
dim3 grid;
|
||||
dim3 block;
|
||||
}KernelLaunchConfig;
|
||||
|
||||
|
||||
typedef struct {
|
||||
int idxBn;
|
||||
int idxCn;
|
||||
int preBuf;
|
||||
int circShift;
|
||||
int8_t dd;
|
||||
} DumpEntry;
|
||||
|
||||
typedef struct {
|
||||
int8_t* p_llr_ptr;
|
||||
int8_t* p_out_ptr;
|
||||
} ldpc_cuda_bridge_t;
|
||||
|
||||
#endif /* NRLDPC_CUDA_SHARED_PARAM_H_ */
|
||||
@@ -1296,7 +1296,7 @@ static inline void nrLDPC_bnProc(t_nrLDPC_lut* p_lut, int8_t* bnProcBuf, int8_t*
|
||||
\param llrOut Pointer to output LLRs
|
||||
\param numLLR Number of LLRs
|
||||
*/
|
||||
static inline void nrLDPC_llr2bit(int8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
static inline void nrLDPC_llr2bit(uint8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
{
|
||||
simde__m256i* p_llrOut = (simde__m256i*) llrOut;
|
||||
simde__m256i* p_out = (simde__m256i*) out;
|
||||
@@ -1313,7 +1313,7 @@ static inline void nrLDPC_llr2bit(int8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
|
||||
// Remaining LLRs that do not fit in multiples of 32 bytes
|
||||
int8_t* p_llrOut8 = (int8_t*)p_llrOut;
|
||||
int8_t* p_out8 = (int8_t*)p_out;
|
||||
uint8_t* p_out8 = (uint8_t*)p_out;
|
||||
|
||||
for (uint32_t i = 0; i < Mr; i++)
|
||||
p_out8[i] = p_llrOut8[i] < 0;
|
||||
@@ -1328,7 +1328,7 @@ static inline void nrLDPC_llr2bit(int8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
\param llrOut Pointer to output LLRs
|
||||
\param numLLR Number of LLRs
|
||||
*/
|
||||
static inline void nrLDPC_llr2bitPacked(int8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
static inline void nrLDPC_llr2bitPacked(uint8_t* out, int8_t* llrOut, uint16_t numLLR)
|
||||
{
|
||||
/** Vector of indices for shuffling input */
|
||||
const uint8_t constShuffle_256_epi8[32] __attribute__ ((aligned(32))) = {7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0,15,14,13,12,11,10,9,8};
|
||||
|
||||
@@ -136,7 +136,7 @@
|
||||
*/
|
||||
|
||||
static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
|
||||
int8_t* p_out,
|
||||
uint8_t* p_out,
|
||||
uint32_t numLLR,
|
||||
t_nrLDPC_lut* p_lut,
|
||||
t_nrLDPC_dec_params* p_decParams,
|
||||
@@ -155,7 +155,7 @@ int32_t LDPCshutdown()
|
||||
|
||||
int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
|
||||
int8_t* p_llr,
|
||||
int8_t* p_out,
|
||||
uint8_t* p_out,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab)
|
||||
{
|
||||
@@ -185,7 +185,7 @@ int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
|
||||
\param p_profilernrLDPC profiler statistics
|
||||
*/
|
||||
static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
|
||||
int8_t* p_out,
|
||||
uint8_t* p_out,
|
||||
uint32_t numLLR,
|
||||
t_nrLDPC_lut* p_lut,
|
||||
t_nrLDPC_dec_params* p_decParams,
|
||||
@@ -828,7 +828,7 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
|
||||
} else {
|
||||
if (numIter > 0) {
|
||||
int8_t llrOut[NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
|
||||
int8_t* p_llrOut = outMode == nrLDPC_outMode_LLRINT8 ? p_out : llrOut;
|
||||
int8_t* p_llrOut = outMode == nrLDPC_outMode_LLRINT8 ? (int8_t*)p_out : llrOut;
|
||||
nrLDPC_llrRes2llrOut(p_lut, p_llrOut, llrRes, Z, BG);
|
||||
if (outMode == nrLDPC_outMode_BIT)
|
||||
nrLDPC_llr2bitPacked(p_out, p_llrOut, numLLR);
|
||||
@@ -838,6 +838,9 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
|
||||
LOG_D(PHY, "Segment CRC OK, exiting LDPC decoder\n");
|
||||
break;
|
||||
}
|
||||
else {
|
||||
LOG_D(PHY, "Segment CRC NOK, Kprime %d, BG %d, Z %d\n",p_decParams->Kprime,BG,Z);
|
||||
}
|
||||
}
|
||||
}
|
||||
// Increase iteration counter
|
||||
@@ -845,7 +848,7 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
|
||||
}
|
||||
if (!p_decParams->check_crc) {
|
||||
int8_t llrOut[NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
|
||||
int8_t* p_llrOut = outMode == nrLDPC_outMode_LLRINT8 ? p_out : llrOut;
|
||||
int8_t* p_llrOut = outMode == nrLDPC_outMode_LLRINT8 ? (int8_t*)p_out : llrOut;
|
||||
// Assign results from processing buffer to output
|
||||
NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llrRes2llrOut));
|
||||
nrLDPC_llrRes2llrOut(p_lut, p_llrOut, llrRes, Z, BG);
|
||||
|
||||
1398
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu
Normal file
1398
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu
Normal file
File diff suppressed because it is too large
Load Diff
622
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_cuda.c
Normal file
622
openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_cuda.c
Normal file
@@ -0,0 +1,622 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
/*! \file nrLDPC_decoder_cuda.h
|
||||
* \brief Defines the CUDA version of nrLDPC decoder, including initialization and driver warmup mechanisms.
|
||||
* \author Qizhi Pan, Raymond Knopp
|
||||
* \company EURECOM
|
||||
* \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
|
||||
* \date 2025-12-30
|
||||
* \version 1.0
|
||||
* \note Optimized for NVIDIA GH200 (Grace Hopper) architecture using Zero-Copy access.
|
||||
* \warning
|
||||
*/
|
||||
#include <stdint.h>
|
||||
#include "PHY/sse_intrin.h"
|
||||
#include "nrLDPCdecoder_defs.h"
|
||||
#include "nrLDPC_types.h"
|
||||
#include "nrLDPC_init.h"
|
||||
#include "nrLDPC_mPass.h"
|
||||
#include "nrLDPC_cnProc.h"
|
||||
#include "nrLDPC_bnProc.h"
|
||||
#include "log.h"
|
||||
#include "openair1/PHY/CODING/coding_defs.h"
|
||||
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
|
||||
#ifdef NR_LDPC_DEBUG_MODE
|
||||
#include "nrLDPC_tools/nrLDPC_debug.h"
|
||||
#endif
|
||||
|
||||
// decoder interface
|
||||
/**
|
||||
\brief LDPC decoder API type definition
|
||||
\param p_decParams LDPC decoder parameters
|
||||
\param p_llr Input LLRs
|
||||
\param p_llrOut Output vector
|
||||
\param p_profiler LDPC profiler statistics
|
||||
*/
|
||||
|
||||
//--------------------------CUDA Area---------------------------
|
||||
#include <cuda_runtime.h>
|
||||
#include "nrLDPC_CUDA_shared_param.h"
|
||||
|
||||
extern cudaStream_t decoderStreams[MAX_NUM_DLSCH_SEGMENTS_DL];
|
||||
extern cudaEvent_t decoderDoneEvents[MAX_NUM_DLSCH_SEGMENTS_DL];
|
||||
static bool decoder_streamsCreated = false;
|
||||
static volatile int cuda_graph_breaker = 0;// 0 by default to enable the CUDA graph
|
||||
cudaError_t Err;
|
||||
|
||||
int8_t* cnProcBuf_dev;
|
||||
int8_t* bnProcBuf_dev;
|
||||
int8_t* llrRes_dev;
|
||||
int8_t* llrProcBuf_dev;
|
||||
|
||||
int8_t* p_llr_dev;
|
||||
int8_t* p_out_dev;
|
||||
|
||||
extern int pageable, integrated;
|
||||
|
||||
int cuda_support_init_decoder()
|
||||
{
|
||||
// use cudaMalloc for all inner buffers
|
||||
cudaError_t err;
|
||||
|
||||
err = cudaMalloc((void**)&cnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_CN_PROC_BUF);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (cnProcBuf_dev): %s\n", cudaGetErrorString(err));
|
||||
|
||||
err = cudaMalloc((void**)&bnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_BN_PROC_BUF);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (bnProcBuf_dev): %s\n", cudaGetErrorString(err));
|
||||
|
||||
err = cudaMalloc((void**)&llrRes_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (llrRes_dev): %s\n", cudaGetErrorString(err));
|
||||
|
||||
err = cudaMalloc((void**)&llrProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (llrProcBuf_dev): %s\n", cudaGetErrorString(err));
|
||||
|
||||
err = cudaMalloc((void**)&p_llr_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
|
||||
cudaMemset(p_llr_dev, 0, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
|
||||
err = cudaMalloc((void**)&p_out_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
|
||||
AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
|
||||
|
||||
printf("[CUDA] Intermediate buffers allocated in Device Memory.\n");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
static ldpc_cuda_bridge_t* stream_bridges[8];
|
||||
|
||||
extern cudaError_t nrLDPC_decoder_cuda_GraphRecord(ldpc_cuda_bridge_t* buffer,
|
||||
uint32_t numLLR,
|
||||
int8_t* cnProcBuf,
|
||||
int8_t* bnProcBuf,
|
||||
int8_t* llrRes,
|
||||
int8_t* llrProcBuf,
|
||||
uint32_t Z,
|
||||
uint32_t K,
|
||||
uint8_t BG,
|
||||
uint8_t R,
|
||||
uint8_t numMaxIter,
|
||||
uint8_t n_segments,
|
||||
e_nrLDPC_outMode outMode,
|
||||
cudaStream_t* streams,
|
||||
uint8_t CudaStreamIdx,
|
||||
cudaGraph_t* graphPtr,
|
||||
cudaGraphExec_t* graphExecPtr,
|
||||
uint8_t* isCreatedFlag);
|
||||
|
||||
extern cudaError_t nrLDPC_decoder_cuda_GraphExecute(cudaGraphExec_t graphExec,
|
||||
cudaStream_t stream,
|
||||
cudaEvent_t* doneEvent,
|
||||
uint8_t CudaStreamIdx);
|
||||
|
||||
extern void nrLDPC_decoder_cuda_NormalExecute(ldpc_cuda_bridge_t* buffer,
|
||||
uint32_t numLLR,
|
||||
int8_t* cnProcBuf,
|
||||
int8_t* bnProcBuf,
|
||||
int8_t* llrRes,
|
||||
int8_t* llrProcBuf,
|
||||
uint32_t Z,
|
||||
uint32_t K,
|
||||
uint8_t BG,
|
||||
uint8_t R,
|
||||
uint8_t numMaxIter,
|
||||
uint8_t n_segments,
|
||||
e_nrLDPC_outMode outMode,
|
||||
cudaStream_t* streams,
|
||||
uint8_t CudaStreamIdx,
|
||||
cudaEvent_t* doneEvent);
|
||||
|
||||
static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
|
||||
int8_t* p_out,
|
||||
int n_segments,
|
||||
t_nrLDPC_dec_params* p_decParams,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab);
|
||||
#define MAX_GRAPH_CACHE_SIZE 16
|
||||
#define PRE_RECORDED_COUNT 6
|
||||
#define STATIC_SEG_SIZE 1 // n_segments in pre-record graphs, should be determined for real cases
|
||||
|
||||
typedef struct {
|
||||
uint32_t Z;
|
||||
uint32_t K;
|
||||
uint32_t numLLR;
|
||||
uint8_t R;
|
||||
uint8_t BG;
|
||||
uint8_t numMaxIter;
|
||||
uint16_t n_segments;
|
||||
e_nrLDPC_outMode outMode;
|
||||
cudaGraph_t graph;
|
||||
cudaGraphExec_t exec;
|
||||
ldpc_cuda_bridge_t* bridge_ptr;
|
||||
bool occupied;
|
||||
} gpu_graph_node_t;
|
||||
|
||||
static gpu_graph_node_t gpu_graph_cache[MAX_GRAPH_CACHE_SIZE];
|
||||
static int dynamic_cache_idx = PRE_RECORDED_COUNT;
|
||||
|
||||
void init_decoder_warmup()
|
||||
{
|
||||
// =====================================================================
|
||||
// CUDA Driver Warm-up
|
||||
// Purpose: Execute a few representative graphs to trigger CUDA context
|
||||
// initialization and driver-level JIT/lazy loading.
|
||||
// Note: These specific Z/R combinations might not match the actual
|
||||
// run-time traffic, but running them ensures the GPU pipeline is ready.
|
||||
// =====================================================================
|
||||
|
||||
cudaError_t err_warmup = cudaSuccess;
|
||||
|
||||
// Sample configurations for warmup
|
||||
uint32_t Z_list[] = {320, 352, 384};
|
||||
uint8_t R_list[] = {13, 23};
|
||||
int node_idx = 0;
|
||||
|
||||
int8_t* dummy_input_llr = NULL;
|
||||
int8_t* dummy_output_bits = NULL;
|
||||
uint32_t max_z = 384;
|
||||
uint32_t max_n_segs = STATIC_SEG_SIZE;
|
||||
|
||||
size_t input_size_bytes = 68 * max_z * max_n_segs * sizeof(int8_t);
|
||||
size_t output_size_bytes = 8448 * max_n_segs * sizeof(int8_t);
|
||||
|
||||
// Allocate Pinned/Mapped Memory for Zero-Copy access (GH200 friendly)
|
||||
cudaHostAlloc((void**)&dummy_input_llr, input_size_bytes, cudaHostAllocMapped);
|
||||
cudaHostAlloc((void**)&dummy_output_bits, output_size_bytes, cudaHostAllocMapped);
|
||||
|
||||
memset(dummy_input_llr, 0, input_size_bytes);
|
||||
memset(dummy_output_bits, 0, output_size_bytes);
|
||||
|
||||
printf("[CUDA] Initializing & Warming up Driver Pipeline...\n");
|
||||
if (cuda_graph_breaker == 0) {
|
||||
for (int r_idx = 0; r_idx < 2; r_idx++) {
|
||||
for (int z_idx = 0; z_idx < 3; z_idx++) {
|
||||
uint32_t Z = Z_list[z_idx];
|
||||
uint8_t R = R_list[r_idx];
|
||||
uint8_t BG = 1;
|
||||
uint32_t K = 22 * Z;
|
||||
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
|
||||
uint8_t numMaxIter = 2; // 2 iterations for faster warmup
|
||||
uint8_t n_segments = STATIC_SEG_SIZE;
|
||||
|
||||
// Bind dummy buffers
|
||||
gpu_graph_cache[node_idx].bridge_ptr->p_llr_ptr = dummy_input_llr;
|
||||
gpu_graph_cache[node_idx].bridge_ptr->p_out_ptr = dummy_output_bits;
|
||||
|
||||
// Record graph
|
||||
err_warmup = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[node_idx].bridge_ptr,
|
||||
numLLR,
|
||||
cnProcBuf_dev,
|
||||
bnProcBuf_dev,
|
||||
llrRes_dev,
|
||||
llrProcBuf_dev,
|
||||
Z,
|
||||
K,
|
||||
BG,
|
||||
R,
|
||||
numMaxIter,
|
||||
n_segments,
|
||||
nrLDPC_outMode_BIT,
|
||||
decoderStreams,
|
||||
0,
|
||||
&gpu_graph_cache[node_idx].graph,
|
||||
&gpu_graph_cache[node_idx].exec,
|
||||
(uint8_t*)&gpu_graph_cache[node_idx].occupied);
|
||||
if (err_warmup != cudaSuccess) {
|
||||
cuda_graph_breaker = 1; // Once a graph recording fails, forbidden the graph recording permanently
|
||||
printf("[CUDA] Warmup Graph Record Failed (err=%s). Circuit breaker triggered\n", cudaGetErrorString(err_warmup));
|
||||
break;
|
||||
}
|
||||
|
||||
// Save metadata
|
||||
gpu_graph_cache[node_idx].Z = Z;
|
||||
gpu_graph_cache[node_idx].R = R;
|
||||
gpu_graph_cache[node_idx].K = K;
|
||||
gpu_graph_cache[node_idx].numLLR = numLLR;
|
||||
gpu_graph_cache[node_idx].BG = BG;
|
||||
gpu_graph_cache[node_idx].numMaxIter = numMaxIter;
|
||||
gpu_graph_cache[node_idx].n_segments = n_segments;
|
||||
gpu_graph_cache[node_idx].outMode = nrLDPC_outMode_BIT;
|
||||
|
||||
node_idx++;
|
||||
}
|
||||
}
|
||||
dynamic_cache_idx = node_idx;
|
||||
|
||||
// Execute to trigger driver initialization
|
||||
if (dynamic_cache_idx > 0) {
|
||||
for (int i = 0; i < dynamic_cache_idx; i++) {
|
||||
if (gpu_graph_cache[i].occupied) {
|
||||
nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[i].exec, decoderStreams[0], NULL, 0);
|
||||
}
|
||||
}
|
||||
cudaDeviceSynchronize();
|
||||
printf("[CUDA] Driver warm-up complete. Executed %d dummy graphs.\n", dynamic_cache_idx);
|
||||
|
||||
// Mark slots as free and reset index so real traffic starts from slot 0
|
||||
for (int i = 0; i < dynamic_cache_idx; i++) {
|
||||
gpu_graph_cache[i].occupied = false;
|
||||
}
|
||||
printf("[CUDA] Cache cleared. Ready for dynamic recording.\n");
|
||||
}
|
||||
}
|
||||
dynamic_cache_idx = 0;
|
||||
|
||||
if (cuda_graph_breaker == 1) {
|
||||
printf("[CUDA] Using Normal Execution for Warmup (Graph Disabled).\n");
|
||||
for (int r_idx = 0; r_idx < 2; r_idx++) {
|
||||
for (int z_idx = 0; z_idx < 3; z_idx++) {
|
||||
uint32_t Z = Z_list[z_idx];
|
||||
uint8_t R = R_list[r_idx];
|
||||
uint8_t BG = 1;
|
||||
uint32_t K = 22 * Z;
|
||||
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
|
||||
uint8_t numMaxIter = 2; // 2 iterations for faster warmup
|
||||
uint8_t n_segments = STATIC_SEG_SIZE;
|
||||
|
||||
// Bind dummy buffers
|
||||
gpu_graph_cache[0].bridge_ptr->p_llr_ptr = dummy_input_llr;
|
||||
gpu_graph_cache[0].bridge_ptr->p_out_ptr = dummy_output_bits;
|
||||
|
||||
// normal execution
|
||||
nrLDPC_decoder_cuda_NormalExecute(gpu_graph_cache[0].bridge_ptr,
|
||||
numLLR,
|
||||
cnProcBuf_dev,
|
||||
bnProcBuf_dev,
|
||||
llrRes_dev,
|
||||
llrProcBuf_dev,
|
||||
Z,
|
||||
K,
|
||||
BG,
|
||||
R,
|
||||
numMaxIter,
|
||||
n_segments,
|
||||
nrLDPC_outMode_BIT,
|
||||
decoderStreams,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
}
|
||||
cudaDeviceSynchronize();
|
||||
printf("[CUDA] Driver warm-up complete with normal execution\n");
|
||||
}
|
||||
cudaFreeHost(dummy_input_llr);
|
||||
cudaFreeHost(dummy_output_bits);
|
||||
}
|
||||
void init_decoder_gpu_structures()
|
||||
{
|
||||
printf("[CUDA] Initializing Global GPU Structures...\n");
|
||||
// Bridge for graphs
|
||||
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
|
||||
if (gpu_graph_cache[i].bridge_ptr == NULL) {
|
||||
cudaHostAlloc((void**)&gpu_graph_cache[i].bridge_ptr, sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
|
||||
|
||||
gpu_graph_cache[i].bridge_ptr->p_llr_ptr = NULL;
|
||||
gpu_graph_cache[i].bridge_ptr->p_out_ptr = NULL;
|
||||
gpu_graph_cache[i].occupied = false;
|
||||
}
|
||||
}
|
||||
printf("[CUDA] Allocated %d Graph Bridges.\n", MAX_GRAPH_CACHE_SIZE);
|
||||
// Bridge for normal execute
|
||||
for (int i = 0; i < 8; i++) {
|
||||
if (stream_bridges[i] == NULL) {
|
||||
cudaHostAlloc((void**)&stream_bridges[i], sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
|
||||
stream_bridges[i]->p_llr_ptr = NULL;
|
||||
stream_bridges[i]->p_out_ptr = NULL;
|
||||
}
|
||||
}
|
||||
printf("[CUDA] Allocated %d Stream Bridges for Fallback case.\n", 8);
|
||||
}
|
||||
|
||||
void init_decoder_graphs()
|
||||
{
|
||||
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
|
||||
gpu_graph_cache[i].occupied = false;
|
||||
gpu_graph_cache[i].graph = NULL;
|
||||
gpu_graph_cache[i].exec = NULL;
|
||||
gpu_graph_cache[i].bridge_ptr = NULL;
|
||||
gpu_graph_cache[i].Z = 0;
|
||||
gpu_graph_cache[i].R = 0;
|
||||
}
|
||||
|
||||
dynamic_cache_idx = 0;
|
||||
|
||||
printf("[decoder_graphs] initialized %d dynamic cache slots\n", MAX_GRAPH_CACHE_SIZE);
|
||||
}
|
||||
|
||||
void free_graphs()
|
||||
{
|
||||
for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
|
||||
if (gpu_graph_cache[i].occupied) {
|
||||
if (gpu_graph_cache[i].exec)
|
||||
cudaGraphExecDestroy(gpu_graph_cache[i].exec);
|
||||
if (gpu_graph_cache[i].graph)
|
||||
cudaGraphDestroy(gpu_graph_cache[i].graph);
|
||||
gpu_graph_cache[i].occupied = false;
|
||||
}
|
||||
}
|
||||
printf("[decoder_graphs] shutdown complete (Dynamic Cache Cleared)\n");
|
||||
}
|
||||
|
||||
extern int cuda_support_set;
|
||||
|
||||
bool encoder_streamsCreated = false;
|
||||
cudaStream_t encoderStreams[4];
|
||||
void cuda_support_init();
|
||||
|
||||
int32_t LDPCinit_cuda()
|
||||
{
|
||||
if (cuda_support_set == 0) {
|
||||
printf("Calling encoder initializations\n");
|
||||
cuda_support_init();
|
||||
}
|
||||
if (!decoder_streamsCreated) {
|
||||
for (int s = 0; s < 8; ++s) {
|
||||
cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
|
||||
cudaEventCreate(&decoderDoneEvents[s]);
|
||||
}
|
||||
decoder_streamsCreated = true;
|
||||
}
|
||||
|
||||
if (!encoder_streamsCreated) {
|
||||
for (int s = 0; s < 4; ++s) {
|
||||
cudaStreamCreateWithFlags(&encoderStreams[s], cudaStreamNonBlocking);
|
||||
}
|
||||
encoder_streamsCreated = true;
|
||||
}
|
||||
printf("CUDA LDPC decoder initiating\n");
|
||||
cuda_support_init_decoder();
|
||||
init_decoder_graphs();
|
||||
init_decoder_gpu_structures();
|
||||
init_decoder_warmup();
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t LDPCshutdown_cuda()
|
||||
{
|
||||
cudaFree(p_llr_dev);
|
||||
cudaFree(p_out_dev);
|
||||
for (int s = 0; s < 8; ++s) {
|
||||
if (decoder_streamsCreated) {
|
||||
cudaEventDestroy(decoderDoneEvents[s]);
|
||||
cudaStreamDestroy(decoderStreams[s]);
|
||||
}
|
||||
}
|
||||
|
||||
for (int s = 0; s < 4; s++) {
|
||||
if (encoder_streamsCreated) {
|
||||
cudaStreamDestroy(encoderStreams[s]);
|
||||
}
|
||||
}
|
||||
free_graphs();
|
||||
|
||||
decoder_streamsCreated = false;
|
||||
encoder_streamsCreated = false;
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
|
||||
int8_t* p_llr,
|
||||
uint8_t* p_out,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab)
|
||||
{
|
||||
if (!((p_decParams->R == 89 || p_decParams->R == 23 || p_decParams->R == 13) && p_decParams->BG == 1 && p_decParams->Z % 4 == 0
|
||||
&& p_decParams->Z >= 128 && p_decParams->Z <= 384)) { // format check
|
||||
printf("Current format: BG = %d, R = %d, Zc = %d\n", p_decParams->BG, p_decParams->R, p_decParams->Z);
|
||||
AssertFatal(false, "Format cuda not support, only support BG = 1, Zc >= 128 and R = 13, 23, 89 right now\n");
|
||||
return 0;
|
||||
}
|
||||
// Launch LDPC decoder core for all segments
|
||||
int n_segments = p_decParams->n_segments;
|
||||
|
||||
int numIter = nrLDPC_decoder_core_dynamic(p_llr, (int8_t*)p_out, n_segments, p_decParams, p_profiler, ab);
|
||||
|
||||
set_abort(ab, false);
|
||||
|
||||
return numIter;
|
||||
}
|
||||
|
||||
/**
|
||||
\brief PerformsnrLDPC decoding of one code block
|
||||
\param p_llr Input LLRs
|
||||
\param p_out Output vector
|
||||
\param numLLR Number of LLRs
|
||||
\param p_decParamsnrLDPC decoder parameters
|
||||
\param p_profilernrLDPC profiler statistics
|
||||
*/
|
||||
|
||||
static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
|
||||
int8_t* p_out,
|
||||
int n_segments,
|
||||
t_nrLDPC_dec_params* p_decParams,
|
||||
t_nrLDPC_time_stats* p_profiler,
|
||||
decode_abort_t* ab)
|
||||
{
|
||||
cudaError_t err_core = cudaSuccess;
|
||||
bool graph_executed = false;
|
||||
|
||||
uint16_t Z = p_decParams->Z;
|
||||
uint8_t BG = p_decParams->BG;
|
||||
uint8_t R = p_decParams->R;
|
||||
uint8_t numMaxIter = p_decParams->numMaxIter;
|
||||
e_nrLDPC_outMode outMode = p_decParams->outMode;
|
||||
uint32_t K = Z * 22;
|
||||
// Calculate LLR size per segment based on Rate
|
||||
uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : ((R == 89) ? NR_LDPC_NCOL_BG1_R89 * Z : NR_LDPC_NCOL_BG1_R23 * Z);
|
||||
if (p_llr != p_llr_dev)
|
||||
cudaMemcpyAsync(p_llr_dev, p_llr, n_segments * 68 * 384, cudaMemcpyHostToDevice, decoderStreams[0]);
|
||||
|
||||
// Output size safety: assume worst-case unpacked bytes (K * n_segments)
|
||||
size_t total_output_size = n_segments * K * sizeof(int8_t);
|
||||
|
||||
/*
|
||||
// for debug, remember to remove it---------
|
||||
cuda_graph_breaker = 1; // skipping all the graph recording
|
||||
//-----------------------------------------
|
||||
*/
|
||||
|
||||
if (cuda_graph_breaker == 0) {
|
||||
int found_idx = -1;
|
||||
// Search in Graph Cache
|
||||
for (int i = 0; i < dynamic_cache_idx; i++) {
|
||||
if (gpu_graph_cache[i].occupied && gpu_graph_cache[i].Z == Z && gpu_graph_cache[i].R == R && gpu_graph_cache[i].BG == BG
|
||||
&& gpu_graph_cache[i].K == K && gpu_graph_cache[i].numLLR == numLLR && gpu_graph_cache[i].numMaxIter == numMaxIter
|
||||
&& gpu_graph_cache[i].n_segments == n_segments && gpu_graph_cache[i].outMode == outMode) {
|
||||
found_idx = i;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (found_idx >= 0) {
|
||||
// === Cache HIT: Execute Recorded Graph ===
|
||||
gpu_graph_cache[found_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
|
||||
gpu_graph_cache[found_idx].bridge_ptr->p_out_ptr = (pageable || integrated) ? p_out : p_out_dev;
|
||||
|
||||
err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[found_idx].exec,
|
||||
decoderStreams[0],
|
||||
NULL, // doneEvent
|
||||
0); // Stream Index
|
||||
if (err_core == cudaSuccess) {
|
||||
graph_executed = true;
|
||||
} else {
|
||||
cuda_graph_breaker = 1;
|
||||
}
|
||||
} else if (dynamic_cache_idx < MAX_GRAPH_CACHE_SIZE) {
|
||||
// === Cache MISS: Record New Graph and Execute ===
|
||||
int new_idx = dynamic_cache_idx;
|
||||
|
||||
gpu_graph_cache[new_idx].occupied = true;
|
||||
gpu_graph_cache[new_idx].Z = Z;
|
||||
gpu_graph_cache[new_idx].R = R;
|
||||
gpu_graph_cache[new_idx].BG = BG;
|
||||
gpu_graph_cache[new_idx].K = K;
|
||||
gpu_graph_cache[new_idx].numLLR = numLLR;
|
||||
gpu_graph_cache[new_idx].numMaxIter = numMaxIter;
|
||||
gpu_graph_cache[new_idx].n_segments = n_segments;
|
||||
gpu_graph_cache[new_idx].outMode = outMode;
|
||||
// Use the determined pointers (Device ptrs for PCIe, Host ptrs for GH200)
|
||||
gpu_graph_cache[new_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
|
||||
gpu_graph_cache[new_idx].bridge_ptr->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
|
||||
|
||||
err_core = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[new_idx].bridge_ptr,
|
||||
numLLR,
|
||||
cnProcBuf_dev,
|
||||
bnProcBuf_dev,
|
||||
llrRes_dev,
|
||||
llrProcBuf_dev,
|
||||
Z,
|
||||
K,
|
||||
BG,
|
||||
R,
|
||||
numMaxIter,
|
||||
n_segments,
|
||||
outMode,
|
||||
decoderStreams,
|
||||
0, // CudaStreamIdx
|
||||
&gpu_graph_cache[new_idx].graph,
|
||||
&gpu_graph_cache[new_idx].exec,
|
||||
(uint8_t*)&gpu_graph_cache[new_idx].occupied);
|
||||
|
||||
if (err_core == cudaSuccess) {
|
||||
err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[new_idx].exec, decoderStreams[0], NULL, 0);
|
||||
|
||||
if (err_core == cudaSuccess) {
|
||||
graph_executed = true;
|
||||
dynamic_cache_idx++;
|
||||
} else {
|
||||
cuda_graph_breaker = 1; // graph execution fail
|
||||
}
|
||||
} else {
|
||||
cuda_graph_breaker = 1; // graph record fail
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!graph_executed) {
|
||||
// === Fallback to Normal Execution ===
|
||||
// If the cache is full, we cannot record new graphs.
|
||||
// Or graph operation is not safe in this device or environment.
|
||||
// Execute kernel directly using standard launch.
|
||||
/* if (cuda_graph_breaker == 1) {
|
||||
LOG_W(PHY, "Graph opereations failed, falling back to normal.\n");
|
||||
}*/
|
||||
ldpc_cuda_bridge_t* perpack_buffer = stream_bridges[0];
|
||||
perpack_buffer->p_llr_ptr = p_llr_dev;
|
||||
perpack_buffer->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
|
||||
|
||||
nrLDPC_decoder_cuda_NormalExecute(perpack_buffer,
|
||||
numLLR,
|
||||
cnProcBuf_dev,
|
||||
bnProcBuf_dev,
|
||||
llrRes_dev,
|
||||
llrProcBuf_dev,
|
||||
Z,
|
||||
K,
|
||||
BG,
|
||||
R,
|
||||
numMaxIter,
|
||||
n_segments,
|
||||
outMode,
|
||||
decoderStreams,
|
||||
0,
|
||||
NULL);
|
||||
}
|
||||
|
||||
// Copy back and Cleanup for Discrete GPU
|
||||
if (!pageable && !integrated) {
|
||||
// Copy Output from Device to Host
|
||||
if (outMode == nrLDPC_outMode_BIT) {
|
||||
cudaMemcpyAsync(p_out, p_out_dev, total_output_size >> 3, cudaMemcpyDeviceToHost, decoderStreams[0]);
|
||||
}
|
||||
if (outMode == nrLDPC_outMode_BITINT8) {
|
||||
cudaMemcpyAsync(p_out, p_out_dev, total_output_size, cudaMemcpyDeviceToHost, decoderStreams[0]);
|
||||
}
|
||||
}
|
||||
|
||||
cudaStreamSynchronize(decoderStreams[0]);
|
||||
if (p_decParams->check_crc) {
|
||||
for (int r = 0; r < n_segments; r++) {
|
||||
// if (r<=1) for (int i=0;i<(K>>3);i++) printf("byte (%d,%d) %x\n",r,i,((uint8_t*)(p_out+r*(K>>3)))[i]);
|
||||
|
||||
if (!p_decParams->check_crc((uint8_t*)(p_out + (r * (K >> 3))), p_decParams->Kprime, p_decParams->crc_type)) {
|
||||
LOG_D(PHY, "Segment %d/%d CRC NOK\n", r, n_segments);
|
||||
return (1 + numMaxIter);
|
||||
}
|
||||
/*
|
||||
uint8_t *b=(uint8_t*)(p_out + (r*(K>>3)));
|
||||
int i=0;
|
||||
if (b[K-2] == 0 && b[K - 1] == 0) {
|
||||
while (b[i] == 0 && i < K)
|
||||
i++;
|
||||
if (i == K) {
|
||||
LOG_E(PHY, "received all 0 pdu (K %d, r %d)\n",K,r);
|
||||
}
|
||||
}
|
||||
*/
|
||||
}
|
||||
}
|
||||
return numMaxIter;
|
||||
}
|
||||
@@ -68,6 +68,8 @@ typedef struct nrLDPC_dec_params {
|
||||
e_nrLDPC_outMode outMode; /**< Output format */
|
||||
int crc_type; /**< Size and type of the parity check bits (16, 24A or 24B) */
|
||||
int (*check_crc)(uint8_t* decoded_bytes, uint32_t n, uint8_t crc_type); /**< Parity check function */
|
||||
int n_segments;/**Infomation for cuda streaming*/
|
||||
int LastTrial;/**Infomation for cuda streaming to create and destroy*/
|
||||
} t_nrLDPC_dec_params;
|
||||
|
||||
/**
|
||||
|
||||
@@ -39,9 +39,11 @@ typedef struct {
|
||||
unsigned int first_seg; // optim8segmulti
|
||||
unsigned char gen_code; // orig
|
||||
time_stats_t *tinput;
|
||||
time_stats_t *tinput_memcpy;
|
||||
time_stats_t *tprep;
|
||||
time_stats_t *tparity;
|
||||
time_stats_t *toutput;
|
||||
time_stats_t *tconcat;
|
||||
/// Size in bits of the code segments
|
||||
uint32_t K;
|
||||
/// Number of lifting sizes to fit the payload
|
||||
@@ -58,6 +60,7 @@ typedef struct {
|
||||
} encoder_implemparams_t;
|
||||
|
||||
typedef int32_t(LDPC_initfunc_t)(void);
|
||||
|
||||
typedef int32_t(LDPC_shutdownfunc_t)(void);
|
||||
|
||||
// decoder interface
|
||||
@@ -71,7 +74,7 @@ typedef int32_t(LDPC_shutdownfunc_t)(void);
|
||||
*/
|
||||
typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
|
||||
int8_t *p_llr,
|
||||
int8_t *p_out,
|
||||
uint8_t *p_out,
|
||||
t_nrLDPC_time_stats *time_stats,
|
||||
decode_abort_t *ab);
|
||||
typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
|
||||
|
||||
218
openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
Normal file
218
openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
Normal file
@@ -0,0 +1,218 @@
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#if USE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
// generated code for Zc=384, byte encoding
|
||||
__global__ void ldpc_BG1_Zc384_worker(uint32_t *c[4],uint32_t *d[4]) {
|
||||
uint32_t *c32=c[blockIdx.x];
|
||||
uint32_t *d32=d[blockIdx.x]+8448-2*384;
|
||||
|
||||
int i2 = threadIdx.x;
|
||||
int i1 = blockIdx.y;
|
||||
// copy 20 c values to d
|
||||
if (i1<20) d[blockIdx.x][(i1*384) + i2] = c32[4*384+(2*384*i1)+i2];
|
||||
|
||||
if (i2 < 384) {
|
||||
c32+=i2;
|
||||
d32+=i2;
|
||||
switch(i1) {
|
||||
case 0:
|
||||
d32[0]=c32[307]^c32[76]^c32[205]^c32[276]^c32[787]^c32[1018]^c32[855]^c32[1586]^c32[1612]^c32[1864]^c32[2673]^c32[2377]^c32[2304]^c32[3360]^c32[3404]^c32[3347]^c32[4021]^c32[3984]^c32[4096]^c32[4824]^c32[4769]^c32[4807]^c32[5707]^c32[5643]^c32[5529]^c32[6475]^c32[6304]^c32[6200]^c32[7229]^c32[7090]^c32[6975]^c32[7968]^c32[7809]^c32[7812]^c32[8557]^c32[8743]^c32[8753]^c32[9233]^c32[9558]^c32[9447]^c32[10341]^c32[10184]^c32[10325]^c32[10969]^c32[10840]^c32[10964]^c32[11735]^c32[11619]^c32[11573]^c32[12394]^c32[12642]^c32[12592]^c32[13170]^c32[13187]^c32[13356]^c32[14066]^c32[14064]^c32[14095]^c32[14772]^c32[14923]^c32[14797]^c32[15690]^c32[15373]^c32[15399]^c32[16474]^c32[16240]^c32[16485];
|
||||
|
||||
break;
|
||||
case 1:
|
||||
d32[384]=c32[307]^c32[308]^c32[77]^c32[206]^c32[277]^c32[787]^c32[788]^c32[1019]^c32[856]^c32[1586]^c32[1587]^c32[1613]^c32[1865]^c32[2673]^c32[2674]^c32[2378]^c32[2305]^c32[3361]^c32[3405]^c32[3348]^c32[4021]^c32[4022]^c32[3985]^c32[4097]^c32[4824]^c32[4825]^c32[4770]^c32[4808]^c32[5708]^c32[5644]^c32[5530]^c32[6476]^c32[6305]^c32[6201]^c32[7229]^c32[7230]^c32[7091]^c32[6976]^c32[7968]^c32[7969]^c32[7810]^c32[7813]^c32[8557]^c32[8558]^c32[8744]^c32[8754]^c32[9233]^c32[9234]^c32[9559]^c32[9448]^c32[10341]^c32[10342]^c32[10185]^c32[10326]^c32[10970]^c32[10841]^c32[10965]^c32[11735]^c32[11736]^c32[11620]^c32[11574]^c32[12394]^c32[12395]^c32[12643]^c32[12593]^c32[13171]^c32[13188]^c32[13357]^c32[14066]^c32[14067]^c32[14065]^c32[14096]^c32[14772]^c32[14773]^c32[14924]^c32[14798]^c32[15690]^c32[15691]^c32[15374]^c32[15400]^c32[16474]^c32[16475]^c32[16241]^c32[16486];
|
||||
|
||||
break;
|
||||
case 2:
|
||||
d32[768]=c32[308]^c32[77]^c32[205]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1018]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1864]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3404]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4096]^c32[4097]^c32[4825]^c32[4769]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5643]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6304]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6975]^c32[6976]^c32[7969]^c32[7809]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10184]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10840]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11573]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13187]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14064]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14797]^c32[14798]^c32[15691]^c32[15373]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
|
||||
|
||||
break;
|
||||
case 3:
|
||||
d32[1152]=c32[308]^c32[77]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4097]^c32[4825]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6976]^c32[7969]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14798]^c32[15691]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
|
||||
|
||||
break;
|
||||
case 4:
|
||||
d32[1536]=c32[332]^c32[949];
|
||||
|
||||
break;
|
||||
case 5:
|
||||
d32[1920]=c32[80]^c32[233]^c32[362]^c32[49]^c32[195]^c32[944]^c32[791]^c32[1012]^c32[782]^c32[1743]^c32[1769]^c32[1637]^c32[2446]^c32[2534]^c32[2461]^c32[2419]^c32[3133]^c32[3177]^c32[3120]^c32[4178]^c32[4141]^c32[3869]^c32[4981]^c32[4926]^c32[4964]^c32[5480]^c32[5416]^c32[5686]^c32[6248]^c32[6461]^c32[6357]^c32[7002]^c32[7247]^c32[7132]^c32[7741]^c32[7966]^c32[7969]^c32[8714]^c32[8516]^c32[8526]^c32[9390]^c32[9331]^c32[9220]^c32[9382]^c32[10114]^c32[10341]^c32[10098]^c32[11126]^c32[10997]^c32[11121]^c32[11892]^c32[11776]^c32[11730]^c32[12551]^c32[12415]^c32[12365]^c32[12529]^c32[13327]^c32[13344]^c32[13129]^c32[13839]^c32[13837]^c32[13868]^c32[14929]^c32[14696]^c32[14954]^c32[15463]^c32[15530]^c32[15556]^c32[16247]^c32[16397]^c32[16258]^c32[16179];
|
||||
|
||||
break;
|
||||
case 6:
|
||||
d32[2304]=c32[278]^c32[4865]^c32[7681]^c32[8799]^c32[10076]^c32[13309]^c32[13842]^c32[15585];
|
||||
|
||||
break;
|
||||
case 7:
|
||||
d32[2688]=c32[9]^c32[830]^c32[3388]^c32[5709]^c32[6434]^c32[10866];
|
||||
|
||||
break;
|
||||
case 8:
|
||||
d32[3072]=c32[374]^c32[94]^c32[143]^c32[247]^c32[272]^c32[375]^c32[376]^c32[343]^c32[62]^c32[63]^c32[307]^c32[854]^c32[958]^c32[1085]^c32[804]^c32[805]^c32[922]^c32[1025]^c32[1026]^c32[947]^c32[1653]^c32[1757]^c32[1679]^c32[1783]^c32[1547]^c32[1650]^c32[1651]^c32[2356]^c32[2460]^c32[2444]^c32[2548]^c32[2371]^c32[2474]^c32[2475]^c32[2469]^c32[3427]^c32[3147]^c32[3087]^c32[3190]^c32[3191]^c32[3414]^c32[3133]^c32[3134]^c32[4088]^c32[4192]^c32[4051]^c32[4155]^c32[4163]^c32[3882]^c32[3883]^c32[4891]^c32[4611]^c32[4836]^c32[4939]^c32[4940]^c32[4874]^c32[4977]^c32[4978]^c32[5390]^c32[5494]^c32[5710]^c32[5429]^c32[5430]^c32[5596]^c32[5699]^c32[5700]^c32[6158]^c32[6262]^c32[6371]^c32[6474]^c32[6475]^c32[6267]^c32[6370]^c32[6371]^c32[6912]^c32[7016]^c32[7157]^c32[7261]^c32[7042]^c32[7145]^c32[7146]^c32[8035]^c32[7755]^c32[7876]^c32[7979]^c32[7980]^c32[7879]^c32[7982]^c32[7983]^c32[8624]^c32[8728]^c32[8810]^c32[8530]^c32[8820]^c32[8539]^c32[8540]^c32[9300]^c32[9404]^c32[9241]^c32[9345]^c32[9514]^c32[9233]^c32[9234]^c32[9234]^c32[10024]^c32[10128]^c32[10251]^c32[10354]^c32[10355]^c32[10008]^c32[10111]^c32[10112]^c32[11036]^c32[10756]^c32[10907]^c32[11010]^c32[11011]^c32[11031]^c32[11134]^c32[11135]^c32[11802]^c32[11522]^c32[11686]^c32[11790]^c32[11640]^c32[11743]^c32[11744]^c32[12461]^c32[12565]^c32[12325]^c32[12429]^c32[12659]^c32[12378]^c32[12379]^c32[12327]^c32[13237]^c32[13341]^c32[13254]^c32[13357]^c32[13358]^c32[13423]^c32[13142]^c32[13143]^c32[14133]^c32[13853]^c32[14131]^c32[13850]^c32[13851]^c32[14162]^c32[13881]^c32[13882]^c32[14839]^c32[14943]^c32[14606]^c32[14710]^c32[14864]^c32[14967]^c32[14968]^c32[14816]^c32[15373]^c32[15477]^c32[15440]^c32[15543]^c32[15544]^c32[15466]^c32[15569]^c32[15570]^c32[16157]^c32[16261]^c32[16307]^c32[16411]^c32[16168]^c32[16271]^c32[16272]^c32[16496];
|
||||
|
||||
break;
|
||||
case 9:
|
||||
d32[3456]=c32[366]^c32[1000]^c32[8001]^c32[8581]^c32[10041]^c32[13359]^c32[13887]^c32[15442];
|
||||
|
||||
break;
|
||||
case 10:
|
||||
d32[3840]=c32[869]^c32[1875]^c32[3346]^c32[5487]^c32[6527]^c32[11106];
|
||||
|
||||
break;
|
||||
case 11:
|
||||
d32[4224]=c32[257]^c32[38]^c32[39]^c32[26]^c32[192]^c32[155]^c32[321]^c32[226]^c32[8]^c32[48]^c32[1121]^c32[902]^c32[903]^c32[968]^c32[1134]^c32[805]^c32[971]^c32[870]^c32[1536]^c32[1701]^c32[1702]^c32[1562]^c32[1728]^c32[1814]^c32[1596]^c32[2623]^c32[2404]^c32[2405]^c32[2327]^c32[2493]^c32[2638]^c32[2420]^c32[3310]^c32[3092]^c32[3354]^c32[3136]^c32[3297]^c32[3079]^c32[3971]^c32[4136]^c32[4137]^c32[3934]^c32[4100]^c32[4046]^c32[4212]^c32[4774]^c32[4939]^c32[4940]^c32[4719]^c32[4885]^c32[4757]^c32[4923]^c32[5657]^c32[5439]^c32[5593]^c32[5759]^c32[5479]^c32[5645]^c32[6425]^c32[6207]^c32[6254]^c32[6420]^c32[6150]^c32[6316]^c32[7179]^c32[6960]^c32[6961]^c32[7040]^c32[7206]^c32[6925]^c32[7091]^c32[7918]^c32[7699]^c32[7700]^c32[7759]^c32[7925]^c32[7762]^c32[7928]^c32[8507]^c32[8672]^c32[8673]^c32[8693]^c32[8475]^c32[8703]^c32[8485]^c32[9567]^c32[9348]^c32[9349]^c32[9508]^c32[9290]^c32[9397]^c32[9563]^c32[9224]^c32[10291]^c32[10072]^c32[10073]^c32[10134]^c32[10300]^c32[10275]^c32[10057]^c32[10919]^c32[11085]^c32[10790]^c32[10956]^c32[10914]^c32[11080]^c32[11685]^c32[11850]^c32[11851]^c32[11569]^c32[11735]^c32[11523]^c32[11689]^c32[12344]^c32[12509]^c32[12510]^c32[12592]^c32[12374]^c32[12542]^c32[12324]^c32[12335]^c32[13120]^c32[13286]^c32[13137]^c32[13303]^c32[13306]^c32[13088]^c32[14016]^c32[14181]^c32[14182]^c32[14014]^c32[14180]^c32[14045]^c32[13827]^c32[14722]^c32[14887]^c32[14888]^c32[14873]^c32[14655]^c32[14747]^c32[14913]^c32[15640]^c32[15421]^c32[15422]^c32[15707]^c32[15489]^c32[15733]^c32[15515]^c32[16424]^c32[16205]^c32[16206]^c32[16190]^c32[16356]^c32[16435]^c32[16217]^c32[16316];
|
||||
|
||||
break;
|
||||
case 12:
|
||||
d32[4608]=c32[77]^c32[954]^c32[7854]^c32[8680]^c32[10034]^c32[13898];
|
||||
|
||||
break;
|
||||
case 13:
|
||||
d32[4992]=c32[293]^c32[294]^c32[63]^c32[192]^c32[263]^c32[313]^c32[773]^c32[774]^c32[1005]^c32[842]^c32[1572]^c32[1573]^c32[1599]^c32[1851]^c32[2659]^c32[2660]^c32[2364]^c32[2675]^c32[2481]^c32[3347]^c32[3391]^c32[3334]^c32[4007]^c32[4008]^c32[3971]^c32[4083]^c32[4810]^c32[4811]^c32[4756]^c32[4794]^c32[5694]^c32[5630]^c32[5516]^c32[5642]^c32[6462]^c32[6291]^c32[6187]^c32[7215]^c32[7216]^c32[7077]^c32[6962]^c32[7954]^c32[7955]^c32[7796]^c32[7799]^c32[8543]^c32[8544]^c32[8730]^c32[8740]^c32[9219]^c32[9220]^c32[9545]^c32[9434]^c32[10327]^c32[10328]^c32[10171]^c32[10312]^c32[10956]^c32[10827]^c32[10951]^c32[11721]^c32[11722]^c32[11606]^c32[11560]^c32[12380]^c32[12381]^c32[12629]^c32[12579]^c32[13157]^c32[13174]^c32[13343]^c32[14052]^c32[14053]^c32[14051]^c32[14082]^c32[14758]^c32[14759]^c32[14910]^c32[14784]^c32[15676]^c32[15677]^c32[15360]^c32[15386]^c32[15475]^c32[16460]^c32[16461]^c32[16227]^c32[16472];
|
||||
|
||||
break;
|
||||
case 14:
|
||||
d32[5376]=c32[142]^c32[9464]^c32[11657]^c32[12377]^c32[13403]^c32[16140];
|
||||
|
||||
break;
|
||||
case 15:
|
||||
d32[5760]=c32[193]^c32[346]^c32[91]^c32[161]^c32[162]^c32[241]^c32[1057]^c32[904]^c32[1124]^c32[1125]^c32[770]^c32[1856]^c32[1882]^c32[1750]^c32[2559]^c32[2647]^c32[2573]^c32[2574]^c32[3246]^c32[3290]^c32[3232]^c32[3233]^c32[3907]^c32[3870]^c32[3982]^c32[4710]^c32[4655]^c32[4692]^c32[4693]^c32[5593]^c32[5529]^c32[5414]^c32[5415]^c32[6361]^c32[6190]^c32[6469]^c32[6470]^c32[7115]^c32[6976]^c32[7245]^c32[7854]^c32[7695]^c32[7697]^c32[7698]^c32[7890]^c32[8827]^c32[8629]^c32[8638]^c32[8639]^c32[9503]^c32[9444]^c32[9332]^c32[9333]^c32[10227]^c32[10070]^c32[10210]^c32[10211]^c32[10302]^c32[10855]^c32[11110]^c32[10849]^c32[10850]^c32[11621]^c32[11889]^c32[11843]^c32[12664]^c32[12528]^c32[12477]^c32[12478]^c32[13056]^c32[13073]^c32[13241]^c32[13242]^c32[13952]^c32[13950]^c32[13980]^c32[13981]^c32[13879]^c32[14658]^c32[14809]^c32[14683]^c32[15576]^c32[15643]^c32[15668]^c32[15669]^c32[16360]^c32[16510]^c32[16370]^c32[16371];
|
||||
|
||||
break;
|
||||
case 16:
|
||||
d32[6144]=c32[364]^c32[133]^c32[262]^c32[333]^c32[844]^c32[1075]^c32[912]^c32[781]^c32[1643]^c32[1669]^c32[1537]^c32[2346]^c32[2434]^c32[2361]^c32[2642]^c32[3417]^c32[3077]^c32[3404]^c32[4078]^c32[4041]^c32[4153]^c32[4881]^c32[4826]^c32[4864]^c32[5380]^c32[5700]^c32[5586]^c32[6148]^c32[6361]^c32[6257]^c32[7286]^c32[7147]^c32[7032]^c32[8025]^c32[7866]^c32[7869]^c32[8614]^c32[8800]^c32[8810]^c32[8505]^c32[9290]^c32[9231]^c32[9504]^c32[10014]^c32[10241]^c32[9998]^c32[11026]^c32[10897]^c32[11021]^c32[11792]^c32[11676]^c32[11630]^c32[12451]^c32[12315]^c32[12649]^c32[13227]^c32[13244]^c32[13413]^c32[14123]^c32[14121]^c32[14152]^c32[14829]^c32[14596]^c32[14854]^c32[15363]^c32[15430]^c32[15456]^c32[15649]^c32[16147]^c32[16297]^c32[16158];
|
||||
|
||||
break;
|
||||
case 17:
|
||||
d32[6528]=c32[260]^c32[11055]^c32[12369]^c32[13414]^c32[16503];
|
||||
|
||||
break;
|
||||
case 18:
|
||||
d32[6912]=c32[898]^c32[9379]^c32[10264]^c32[13956]^c32[14596];
|
||||
|
||||
break;
|
||||
case 19:
|
||||
d32[7296]=c32[145]^c32[981]^c32[5720]^c32[6386]^c32[7877];
|
||||
|
||||
break;
|
||||
case 20:
|
||||
d32[7680]=c32[366]^c32[135]^c32[264]^c32[335]^c32[187]^c32[846]^c32[1077]^c32[914]^c32[1645]^c32[1671]^c32[1539]^c32[2348]^c32[2436]^c32[2363]^c32[2510]^c32[3419]^c32[3079]^c32[3406]^c32[4080]^c32[4043]^c32[4155]^c32[4883]^c32[4828]^c32[4866]^c32[5382]^c32[5702]^c32[5588]^c32[6150]^c32[6363]^c32[6259]^c32[7288]^c32[7149]^c32[7034]^c32[7176]^c32[8027]^c32[7868]^c32[7871]^c32[8616]^c32[8802]^c32[8812]^c32[8789]^c32[9292]^c32[9233]^c32[9506]^c32[10016]^c32[10243]^c32[10000]^c32[11028]^c32[10899]^c32[11023]^c32[11794]^c32[11678]^c32[11632]^c32[12453]^c32[12317]^c32[12651]^c32[13229]^c32[13246]^c32[13415]^c32[14125]^c32[14123]^c32[14154]^c32[14831]^c32[14598]^c32[14856]^c32[15365]^c32[15432]^c32[15458]^c32[16149]^c32[16299]^c32[16160];
|
||||
|
||||
break;
|
||||
case 21:
|
||||
d32[8064]=c32[973]^c32[3942]^c32[12616]^c32[15573]^c32[16225];
|
||||
|
||||
break;
|
||||
case 22:
|
||||
d32[8448]=c32[30]^c32[9227]^c32[10217]^c32[13078];
|
||||
|
||||
break;
|
||||
case 23:
|
||||
d32[8832]=c32[792]^c32[1625]^c32[7741]^c32[13851];
|
||||
|
||||
break;
|
||||
case 24:
|
||||
d32[9216]=c32[157]^c32[310]^c32[55]^c32[126]^c32[298]^c32[1021]^c32[868]^c32[1089]^c32[1820]^c32[1846]^c32[1714]^c32[2523]^c32[2611]^c32[2538]^c32[2462]^c32[3210]^c32[3254]^c32[3197]^c32[3307]^c32[3871]^c32[4218]^c32[3946]^c32[4674]^c32[4619]^c32[4657]^c32[5557]^c32[5493]^c32[5379]^c32[6325]^c32[6154]^c32[6434]^c32[7079]^c32[6940]^c32[7209]^c32[7818]^c32[8043]^c32[8046]^c32[8791]^c32[8593]^c32[8603]^c32[8787]^c32[9467]^c32[9408]^c32[9297]^c32[10191]^c32[10034]^c32[10175]^c32[10819]^c32[11074]^c32[10814]^c32[11585]^c32[11853]^c32[11807]^c32[12628]^c32[12492]^c32[12442]^c32[13404]^c32[13421]^c32[13206]^c32[13916]^c32[13914]^c32[13945]^c32[14622]^c32[14773]^c32[14647]^c32[15540]^c32[15607]^c32[15633]^c32[16324]^c32[16474]^c32[16335];
|
||||
|
||||
break;
|
||||
case 25:
|
||||
d32[9600]=c32[840]^c32[4625]^c32[5759]^c32[11064];
|
||||
|
||||
break;
|
||||
case 26:
|
||||
d32[9984]=c32[71]^c32[1617]^c32[3148]^c32[11656];
|
||||
|
||||
break;
|
||||
case 27:
|
||||
d32[10368]=c32[962]^c32[4802]^c32[6245];
|
||||
|
||||
break;
|
||||
case 28:
|
||||
d32[10752]=c32[222]^c32[3091]^c32[14836]^c32[16402];
|
||||
|
||||
break;
|
||||
case 29:
|
||||
d32[11136]=c32[2]^c32[155]^c32[284]^c32[354]^c32[355]^c32[866]^c32[1097]^c32[933]^c32[934]^c32[1020]^c32[1665]^c32[1691]^c32[1559]^c32[2368]^c32[2456]^c32[2382]^c32[2383]^c32[3439]^c32[3099]^c32[3425]^c32[3426]^c32[4100]^c32[4063]^c32[4175]^c32[4903]^c32[4848]^c32[4885]^c32[4886]^c32[5402]^c32[5722]^c32[5607]^c32[5608]^c32[6170]^c32[6383]^c32[6278]^c32[6279]^c32[6924]^c32[7169]^c32[7054]^c32[8047]^c32[7888]^c32[7890]^c32[7891]^c32[8636]^c32[8822]^c32[8831]^c32[8448]^c32[9312]^c32[9253]^c32[9525]^c32[9526]^c32[10036]^c32[10263]^c32[10019]^c32[10020]^c32[11048]^c32[10919]^c32[11042]^c32[11043]^c32[10757]^c32[11814]^c32[11698]^c32[11652]^c32[12473]^c32[12337]^c32[12670]^c32[12671]^c32[13249]^c32[13266]^c32[13434]^c32[13435]^c32[14145]^c32[14143]^c32[14173]^c32[14174]^c32[13971]^c32[14851]^c32[14618]^c32[14876]^c32[15385]^c32[15452]^c32[15477]^c32[15478]^c32[16169]^c32[16319]^c32[16179]^c32[16180];
|
||||
|
||||
break;
|
||||
case 30:
|
||||
d32[11520]=c32[14]^c32[167]^c32[295]^c32[296]^c32[366]^c32[367]^c32[159]^c32[878]^c32[1108]^c32[1109]^c32[945]^c32[946]^c32[1677]^c32[1703]^c32[1570]^c32[1571]^c32[2380]^c32[2468]^c32[2394]^c32[2395]^c32[3451]^c32[3110]^c32[3111]^c32[3437]^c32[3438]^c32[4112]^c32[4075]^c32[4186]^c32[4187]^c32[4915]^c32[4859]^c32[4860]^c32[4897]^c32[4898]^c32[5414]^c32[5733]^c32[5734]^c32[5619]^c32[5620]^c32[6182]^c32[6394]^c32[6395]^c32[6290]^c32[6291]^c32[6936]^c32[7181]^c32[7065]^c32[7066]^c32[8059]^c32[7899]^c32[7900]^c32[7902]^c32[7903]^c32[7909]^c32[8648]^c32[8450]^c32[8459]^c32[8460]^c32[9324]^c32[9265]^c32[9537]^c32[9538]^c32[10048]^c32[10274]^c32[10275]^c32[10031]^c32[10032]^c32[10244]^c32[11060]^c32[10930]^c32[10931]^c32[11054]^c32[11055]^c32[11826]^c32[11710]^c32[11663]^c32[11664]^c32[12485]^c32[12349]^c32[12298]^c32[12299]^c32[13261]^c32[13277]^c32[13278]^c32[13062]^c32[13063]^c32[14157]^c32[14154]^c32[14155]^c32[14185]^c32[14186]^c32[14863]^c32[14630]^c32[14887]^c32[14888]^c32[15397]^c32[15463]^c32[15464]^c32[15489]^c32[15490]^c32[16181]^c32[16331]^c32[16191]^c32[16192];
|
||||
|
||||
break;
|
||||
case 31:
|
||||
d32[11904]=c32[181]^c32[180]^c32[334]^c32[333]^c32[79]^c32[78]^c32[150]^c32[148]^c32[149]^c32[1045]^c32[1044]^c32[892]^c32[891]^c32[1113]^c32[1111]^c32[1112]^c32[868]^c32[1844]^c32[1843]^c32[1870]^c32[1869]^c32[1738]^c32[1737]^c32[2547]^c32[2546]^c32[2635]^c32[2634]^c32[2562]^c32[2560]^c32[2561]^c32[3234]^c32[3233]^c32[3278]^c32[3277]^c32[3221]^c32[3219]^c32[3220]^c32[3895]^c32[3894]^c32[3858]^c32[3857]^c32[3970]^c32[3969]^c32[4698]^c32[4697]^c32[4643]^c32[4642]^c32[4681]^c32[4679]^c32[4680]^c32[5581]^c32[5580]^c32[5517]^c32[5516]^c32[5403]^c32[5401]^c32[5402]^c32[5591]^c32[6349]^c32[6348]^c32[6178]^c32[6177]^c32[6458]^c32[6456]^c32[6457]^c32[7103]^c32[7102]^c32[6964]^c32[6963]^c32[7233]^c32[7232]^c32[7842]^c32[7841]^c32[7683]^c32[7682]^c32[7686]^c32[7684]^c32[7685]^c32[8815]^c32[8814]^c32[8617]^c32[8616]^c32[8627]^c32[8625]^c32[8626]^c32[9491]^c32[9490]^c32[9432]^c32[9431]^c32[9321]^c32[9319]^c32[9320]^c32[10215]^c32[10214]^c32[10058]^c32[10057]^c32[10199]^c32[10197]^c32[10198]^c32[10843]^c32[10842]^c32[11098]^c32[11097]^c32[10838]^c32[10836]^c32[10837]^c32[11609]^c32[11608]^c32[11877]^c32[11876]^c32[11831]^c32[11830]^c32[12652]^c32[12651]^c32[12516]^c32[12515]^c32[12466]^c32[12464]^c32[12465]^c32[13428]^c32[13427]^c32[13061]^c32[13060]^c32[13230]^c32[13228]^c32[13229]^c32[13940]^c32[13939]^c32[13938]^c32[13937]^c32[13969]^c32[13967]^c32[13968]^c32[14646]^c32[14645]^c32[14797]^c32[14796]^c32[14671]^c32[14670]^c32[15564]^c32[15563]^c32[15631]^c32[15630]^c32[15657]^c32[15655]^c32[15656]^c32[16348]^c32[16347]^c32[16498]^c32[16497]^c32[16359]^c32[16357]^c32[16358];
|
||||
|
||||
break;
|
||||
case 32:
|
||||
d32[12288]=c32[211]^c32[364]^c32[108]^c32[109]^c32[179]^c32[180]^c32[102]^c32[1075]^c32[921]^c32[922]^c32[1142]^c32[1143]^c32[1874]^c32[1900]^c32[1767]^c32[1768]^c32[2577]^c32[2665]^c32[2591]^c32[2592]^c32[3264]^c32[3307]^c32[3308]^c32[3250]^c32[3251]^c32[3925]^c32[3888]^c32[3999]^c32[4000]^c32[4728]^c32[4672]^c32[4673]^c32[4710]^c32[4711]^c32[5611]^c32[5546]^c32[5547]^c32[5432]^c32[5433]^c32[6379]^c32[6207]^c32[6208]^c32[6487]^c32[6488]^c32[7133]^c32[6994]^c32[7262]^c32[7263]^c32[7872]^c32[7712]^c32[7713]^c32[7715]^c32[7716]^c32[8461]^c32[8647]^c32[8656]^c32[8657]^c32[9521]^c32[9462]^c32[9350]^c32[9351]^c32[9417]^c32[10245]^c32[10087]^c32[10088]^c32[10228]^c32[10229]^c32[10873]^c32[11127]^c32[11128]^c32[10867]^c32[10868]^c32[10927]^c32[11639]^c32[11523]^c32[11860]^c32[11861]^c32[12298]^c32[12546]^c32[12495]^c32[12496]^c32[13074]^c32[13090]^c32[13091]^c32[13259]^c32[13260]^c32[13970]^c32[13967]^c32[13968]^c32[13998]^c32[13999]^c32[14676]^c32[14827]^c32[14700]^c32[14701]^c32[15594]^c32[15660]^c32[15661]^c32[15686]^c32[15687]^c32[16378]^c32[16144]^c32[16388]^c32[16389];
|
||||
|
||||
break;
|
||||
case 33:
|
||||
d32[12672]=c32[1091]^c32[1544]^c32[8809]^c32[16233];
|
||||
|
||||
break;
|
||||
case 34:
|
||||
d32[13056]=c32[230]^c32[5524]^c32[11722]^c32[13368];
|
||||
|
||||
break;
|
||||
case 35:
|
||||
d32[13440]=c32[189]^c32[342]^c32[87]^c32[158]^c32[1053]^c32[900]^c32[1121]^c32[1088]^c32[1852]^c32[1878]^c32[1746]^c32[2555]^c32[2643]^c32[2570]^c32[3242]^c32[3286]^c32[3229]^c32[3903]^c32[3866]^c32[3978]^c32[4706]^c32[4651]^c32[4689]^c32[4943]^c32[5589]^c32[5525]^c32[5411]^c32[6357]^c32[6186]^c32[6466]^c32[7111]^c32[6972]^c32[7241]^c32[7850]^c32[7691]^c32[7694]^c32[8823]^c32[8625]^c32[8635]^c32[9499]^c32[9440]^c32[9329]^c32[9218]^c32[10223]^c32[10066]^c32[10207]^c32[10851]^c32[11106]^c32[10846]^c32[11617]^c32[11885]^c32[11839]^c32[12660]^c32[12524]^c32[12474]^c32[13436]^c32[13069]^c32[13238]^c32[13948]^c32[13946]^c32[13977]^c32[14654]^c32[14805]^c32[14679]^c32[15572]^c32[15639]^c32[15665]^c32[16356]^c32[16506]^c32[16367];
|
||||
|
||||
break;
|
||||
case 36:
|
||||
d32[13824]=c32[210]^c32[11065]^c32[11817]^c32[13845];
|
||||
|
||||
break;
|
||||
case 37:
|
||||
d32[14208]=c32[38]^c32[39]^c32[192]^c32[321]^c32[8]^c32[902]^c32[903]^c32[1134]^c32[971]^c32[1037]^c32[1701]^c32[1702]^c32[1728]^c32[1596]^c32[2404]^c32[2405]^c32[2493]^c32[2420]^c32[3092]^c32[3136]^c32[3079]^c32[4136]^c32[4137]^c32[4100]^c32[4212]^c32[4939]^c32[4940]^c32[4885]^c32[4923]^c32[5439]^c32[5759]^c32[5645]^c32[6207]^c32[6420]^c32[6316]^c32[6960]^c32[6961]^c32[7206]^c32[7091]^c32[7699]^c32[7700]^c32[7925]^c32[7928]^c32[8672]^c32[8673]^c32[8475]^c32[8485]^c32[9348]^c32[9349]^c32[9290]^c32[9563]^c32[10072]^c32[10073]^c32[10300]^c32[10057]^c32[10066]^c32[11085]^c32[10956]^c32[11080]^c32[11850]^c32[11851]^c32[11735]^c32[11689]^c32[12509]^c32[12510]^c32[12374]^c32[12324]^c32[13286]^c32[13303]^c32[13088]^c32[14181]^c32[14182]^c32[14180]^c32[13827]^c32[14887]^c32[14888]^c32[14655]^c32[14913]^c32[15421]^c32[15422]^c32[15489]^c32[15515]^c32[16205]^c32[16206]^c32[16356]^c32[16217];
|
||||
|
||||
break;
|
||||
case 38:
|
||||
d32[14592]=c32[185]^c32[7089]^c32[7969]^c32[9430];
|
||||
|
||||
break;
|
||||
case 39:
|
||||
d32[14976]=c32[1026]^c32[2397]^c32[5722]^c32[14889];
|
||||
|
||||
break;
|
||||
case 40:
|
||||
d32[15360]=c32[175]^c32[6181]^c32[13368];
|
||||
|
||||
break;
|
||||
case 41:
|
||||
d32[15744]=c32[820]^c32[2618]^c32[7051]^c32[14112];
|
||||
|
||||
break;
|
||||
case 42:
|
||||
d32[16128]=c32[142]^c32[295]^c32[39]^c32[40]^c32[110]^c32[111]^c32[113]^c32[1006]^c32[852]^c32[853]^c32[1073]^c32[1074]^c32[1805]^c32[1831]^c32[1698]^c32[1699]^c32[2508]^c32[2596]^c32[2522]^c32[2523]^c32[3195]^c32[3238]^c32[3239]^c32[3181]^c32[3182]^c32[3086]^c32[3856]^c32[4203]^c32[3930]^c32[3931]^c32[4659]^c32[4987]^c32[4988]^c32[4641]^c32[4642]^c32[5542]^c32[5477]^c32[5478]^c32[5747]^c32[5748]^c32[6310]^c32[6522]^c32[6523]^c32[6418]^c32[6419]^c32[7064]^c32[6925]^c32[7193]^c32[7194]^c32[7803]^c32[8027]^c32[8028]^c32[8030]^c32[8031]^c32[8776]^c32[8578]^c32[8587]^c32[8588]^c32[9452]^c32[9393]^c32[9281]^c32[9282]^c32[10176]^c32[10018]^c32[10019]^c32[10159]^c32[10160]^c32[10804]^c32[11058]^c32[11059]^c32[10798]^c32[10799]^c32[11570]^c32[11838]^c32[11791]^c32[11792]^c32[12613]^c32[12477]^c32[12426]^c32[12427]^c32[13389]^c32[13405]^c32[13406]^c32[13190]^c32[13191]^c32[13901]^c32[13898]^c32[13899]^c32[13929]^c32[13930]^c32[14607]^c32[14758]^c32[14631]^c32[14632]^c32[15525]^c32[15591]^c32[15592]^c32[15617]^c32[15618]^c32[16309]^c32[16459]^c32[16319]^c32[16320];
|
||||
|
||||
break;
|
||||
case 43:
|
||||
d32[16512]=c32[92]^c32[245]^c32[374]^c32[60]^c32[61]^c32[956]^c32[803]^c32[1023]^c32[1024]^c32[881]^c32[1755]^c32[1781]^c32[1649]^c32[2458]^c32[2546]^c32[2472]^c32[2473]^c32[3145]^c32[3189]^c32[3131]^c32[3132]^c32[4190]^c32[4153]^c32[3881]^c32[4609]^c32[4938]^c32[4975]^c32[4976]^c32[5492]^c32[5428]^c32[5697]^c32[5698]^c32[6260]^c32[6473]^c32[6368]^c32[6369]^c32[7014]^c32[7259]^c32[7144]^c32[7753]^c32[7978]^c32[7980]^c32[7981]^c32[8726]^c32[8528]^c32[8537]^c32[8538]^c32[9402]^c32[9343]^c32[9231]^c32[9232]^c32[10126]^c32[10353]^c32[10109]^c32[10110]^c32[10754]^c32[11009]^c32[11132]^c32[11133]^c32[11520]^c32[11788]^c32[11742]^c32[12563]^c32[12427]^c32[12376]^c32[12377]^c32[12420]^c32[13339]^c32[13356]^c32[13140]^c32[13141]^c32[13851]^c32[13849]^c32[13879]^c32[13880]^c32[13938]^c32[14941]^c32[14708]^c32[14966]^c32[15475]^c32[15542]^c32[15567]^c32[15568]^c32[16259]^c32[16409]^c32[16269]^c32[16270];
|
||||
|
||||
break;
|
||||
case 44:
|
||||
d32[16896]=c32[197]^c32[350]^c32[95]^c32[166]^c32[80]^c32[1061]^c32[908]^c32[1129]^c32[1860]^c32[1886]^c32[1754]^c32[2563]^c32[2651]^c32[2578]^c32[3250]^c32[3294]^c32[3237]^c32[3911]^c32[3874]^c32[3986]^c32[4714]^c32[4659]^c32[4697]^c32[5597]^c32[5533]^c32[5419]^c32[5454]^c32[6365]^c32[6194]^c32[6474]^c32[7119]^c32[6980]^c32[7249]^c32[7075]^c32[7858]^c32[7699]^c32[7702]^c32[8831]^c32[8633]^c32[8643]^c32[9507]^c32[9448]^c32[9337]^c32[10231]^c32[10074]^c32[10215]^c32[10859]^c32[11114]^c32[10854]^c32[11625]^c32[11893]^c32[11847]^c32[12668]^c32[12532]^c32[12482]^c32[13060]^c32[13077]^c32[13246]^c32[13956]^c32[13954]^c32[13985]^c32[14662]^c32[14813]^c32[14687]^c32[15580]^c32[15647]^c32[15673]^c32[16364]^c32[16130]^c32[16375];
|
||||
|
||||
break;
|
||||
case 45:
|
||||
d32[17280]=c32[903]^c32[4757]^c32[7695];
|
||||
|
||||
break;
|
||||
|
||||
}
|
||||
}
|
||||
}
|
||||
extern "C" int ldpc_BG1_Zc384_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs,cudaStream_t *stream,int sidx) {
|
||||
|
||||
dim3 numblocks(n_inputs,46);
|
||||
ldpc_BG1_Zc384_worker<<<numblocks,384,0,stream[sidx]>>>(c,d);
|
||||
cudaError_t err=cudaPeekAtLastError();
|
||||
if (err!=cudaSuccess) {
|
||||
printf("cuda error: %s (c %p, d %p)\n",cudaGetErrorString(err),c,d);
|
||||
exit(-1);
|
||||
}
|
||||
return(0);
|
||||
}
|
||||
@@ -80,9 +80,11 @@
|
||||
#include "ldpc_BG2_Zc80_byte.c"
|
||||
#include "ldpc_BG2_Zc72_byte.c"
|
||||
|
||||
static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols)
|
||||
|
||||
static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols,time_stats_t *tinput_memcpy)
|
||||
{
|
||||
unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c
|
||||
if (tinput_memcpy) start_meas(tinput_memcpy);
|
||||
for (int i1 = 0; i1 < ncols; i1++) {
|
||||
memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
|
||||
memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
|
||||
@@ -95,6 +97,7 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho
|
||||
memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1);
|
||||
}
|
||||
}
|
||||
if (tinput_memcpy) stop_meas(tinput_memcpy);
|
||||
if (BG == 1) {
|
||||
switch (Zc) {
|
||||
case 176:
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
/*!\file ldpc_encode_parity_check.c
|
||||
* \brief Parity check function used by ldpc encoders
|
||||
* \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
|
||||
* \email openair_tech@eurecom.fr
|
||||
* \date 27-03-2018
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "assertions.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
|
||||
|
||||
int ldpc_BG1_Zc384_cuda32(uint32_t **c,uint32_t **d,int n_inputs, cudaStream_t *stream,int sidx);
|
||||
|
||||
|
||||
void encode_parity_check_part_cuda(uint32_t **c, uint32_t **d, short BG,short Zc,short Kb, int ncols, int n_inputs, cudaStream_t *stream,int sidx)
|
||||
{
|
||||
|
||||
if (BG == 1) {
|
||||
switch (Zc) {
|
||||
case 176:
|
||||
case 192:
|
||||
case 208:
|
||||
case 224:
|
||||
case 240:
|
||||
case 256:
|
||||
case 288:
|
||||
case 320:
|
||||
case 352:
|
||||
AssertFatal(1==0,"BG %d Zc %d not supported yet for CUDA\n",BG, Zc);
|
||||
break;
|
||||
case 384:
|
||||
ldpc_BG1_Zc384_cuda32(c, d, n_inputs, stream,sidx);
|
||||
break;
|
||||
default:
|
||||
AssertFatal(false, "BG %d Zc %d is not supported yet\n", BG, Zc);
|
||||
}
|
||||
} else if (BG == 2) {
|
||||
switch (Zc) {
|
||||
case 72:
|
||||
case 80:
|
||||
case 88:
|
||||
case 96:
|
||||
case 104:
|
||||
case 112:
|
||||
case 120:
|
||||
case 128:
|
||||
case 144:
|
||||
case 160:
|
||||
case 176:
|
||||
case 192:
|
||||
case 208:
|
||||
case 224:
|
||||
case 240:
|
||||
case 256:
|
||||
case 288:
|
||||
case 320:
|
||||
case 352:
|
||||
case 384:
|
||||
default:
|
||||
AssertFatal(false , "BG %d Zc %d is not supported yet\n", BG, Zc);
|
||||
}
|
||||
} else
|
||||
AssertFatal(false, "BG %d is not supported\n", BG);
|
||||
}
|
||||
@@ -35,11 +35,10 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
|
||||
int no_punctured_columns, removed_bit;
|
||||
int nind=0;
|
||||
int indlist[1000];
|
||||
int indlist2[1000];
|
||||
|
||||
const short *Gen_shift_values = choose_generator_matrix(BG, Zc);
|
||||
if (Gen_shift_values==NULL) {
|
||||
printf("ldpc_encoder_orig: could not find generator matrix\n");
|
||||
printf("ldpc_encoder_orig: could not find generator matrix BG %d Zc %d\n",BG,Zc);
|
||||
return(-1);
|
||||
}
|
||||
|
||||
@@ -70,32 +69,34 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
|
||||
|
||||
// parity check part
|
||||
|
||||
if (gen_code>=1)
|
||||
if (gen_code==1 || gen_code==2 || gen_code==3)
|
||||
{
|
||||
char fname[100];
|
||||
sprintf(fname,"ldpc_BG%d_Zc%d_byte.c",BG,Zc);
|
||||
FILE *fd=fopen(fname,"w");
|
||||
AssertFatal(fd!=NULL,"cannot open %s\n",fname);
|
||||
sprintf(fname,"ldpc_BG%d_Zc%d_16bit.c",BG,Zc);
|
||||
FILE *fd2=fopen(fname,"w");
|
||||
AssertFatal(fd2!=NULL,"cannot open %s\n",fname);
|
||||
|
||||
|
||||
int shift;
|
||||
char data_type[100];
|
||||
char xor_command[100];
|
||||
int mask;
|
||||
|
||||
|
||||
char permutex_command[100];
|
||||
int use_permutex=0;
|
||||
char alignr_command[100];
|
||||
int use_alignr=0;
|
||||
|
||||
|
||||
fprintf(fd,"#include \"PHY/sse_intrin.h\"\n");
|
||||
fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n");
|
||||
|
||||
if (gen_code == 1 && (Zc&63)==0) {
|
||||
shift=6;
|
||||
mask=63;
|
||||
strcpy(data_type,"__m512i");
|
||||
strcpy(xor_command,"_mm512_xor_si512");
|
||||
strcpy(permutex_command,"_mm512_permutex2var_epi8");
|
||||
strcpy(alignr_command,"_mm512_alignr_epi8");
|
||||
use_alignr=1;
|
||||
}
|
||||
else if (gen_code == 1 && (Zc&31)==0) {
|
||||
shift=5; // AVX2 - 256-bit SIMD
|
||||
@@ -103,14 +104,15 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
|
||||
strcpy(data_type,"simde__m256i");
|
||||
strcpy(xor_command,"simde_mm256_xor_si256");
|
||||
}
|
||||
else if ((Zc&15)==0) {
|
||||
else if ((gen_code <=2) && (Zc&15)==0) {
|
||||
shift=4; // SSE4 - 128-bit SIMD
|
||||
mask=15;
|
||||
strcpy(data_type,"simde__m128i");
|
||||
strcpy(xor_command,"simde_mm_xor_si128");
|
||||
|
||||
strcpy(alignr_command,"simde_mm_alignr_epi8");
|
||||
use_alignr=1;
|
||||
}
|
||||
else if ((Zc&7)==0) {
|
||||
else if ((gen_code <=2) && (Zc&7)==0) {
|
||||
shift=3; // MMX - 64-bit SIMD
|
||||
mask=7;
|
||||
strcpy(data_type,"simde__m64");
|
||||
@@ -122,21 +124,30 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
|
||||
strcpy(data_type,"uint8_t");
|
||||
strcpy(xor_command,"scalar_xor");
|
||||
fprintf(fd,"#define scalar_xor(a,b) ((a)^(b))\n");
|
||||
fprintf(fd2,"#define scalar_xor(a,b) ((a)^(b))\n");
|
||||
|
||||
}
|
||||
fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
|
||||
fprintf(fd2,"// generated code for Zc=%d, 16bit encoding\n",Zc);
|
||||
if (use_permutex==1) {
|
||||
fprintf(fd," const uint8_t abshift_bytes[%d][%d] __attribute__((aligned(64)))= {\n",1+mask,1+mask);
|
||||
for (int i=0;i<=mask;i++) {
|
||||
fprintf(fd,"{");
|
||||
for (int j=0;j<=mask;j++) {
|
||||
fprintf(fd,"%d",i+j);
|
||||
if (j<mask) fprintf(fd,",");
|
||||
else fprintf(fd,"}");
|
||||
}
|
||||
if (i<mask) fprintf(fd,",\n");
|
||||
else fprintf(fd,"};\n");
|
||||
}
|
||||
fprintf(fd,"static inline __m512i abshift_get(int r)\n {\n");
|
||||
fprintf(fd," return * (const __m512i *) abshift_bytes[r];\n}");
|
||||
}
|
||||
fprintf(fd,"static inline void ldpc_BG%d_Zc%d_byte(uint8_t *c,uint8_t *d) {\n",BG,Zc);
|
||||
fprintf(fd2,"static inline void ldpc_BG%d_Zc%d_16bit(uint16_t *c,uint16_t *d) {\n",BG,Zc);
|
||||
fprintf(fd," %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
|
||||
fprintf(fd2," %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
|
||||
fprintf(fd," %s *c2,*d2;\n\n",data_type);
|
||||
fprintf(fd2," %s *c2,*d2;\n\n",data_type);
|
||||
fprintf(fd," int i2;\n");
|
||||
fprintf(fd2," int i2;\n");
|
||||
fprintf(fd," for (i2=0; i2<%d; i2++) {\n",Zc>>shift);
|
||||
if (shift > 0)
|
||||
fprintf(fd2," for (i2=0; i2<%d; i2++) {\n",Zc>>(shift-1));
|
||||
|
||||
for (i2=0; i2 < 1; i2++)
|
||||
{
|
||||
//t=Kb*Zc+i2;
|
||||
@@ -146,53 +157,122 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
|
||||
|
||||
fprintf(fd," c2=&csimd[i2];\n");
|
||||
fprintf(fd," d2=&dsimd[i2];\n");
|
||||
fprintf(fd2," c2=&csimd[i2];\n");
|
||||
fprintf(fd2," d2=&dsimd[i2];\n");
|
||||
|
||||
for (i1 = 0; i1 < nrows; i1++)
|
||||
|
||||
{
|
||||
fprintf(fd,"\n//row: %d\n",i1);
|
||||
fprintf(fd2,"\n//row: %d\n",i1);
|
||||
AssertFatal(shift > 0 , "The result of the right shift is undefined because the right operand is negative\n");
|
||||
AssertFatal(shift >= 0 , "The result of the right shift is undefined because the right operand is negative\n");
|
||||
fprintf(fd," d2[%d]=",(Zc*i1)>>shift);
|
||||
fprintf(fd2," d2[%d]=",(Zc*i1)>>(shift-1));
|
||||
|
||||
nind=0;
|
||||
|
||||
for (i3=0; i3 < ncols; i3++)
|
||||
{
|
||||
temp_prime=i1 * ncols + i3;
|
||||
|
||||
|
||||
for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
|
||||
{
|
||||
|
||||
{
|
||||
var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
|
||||
int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
|
||||
printf("var %d, i3 %d, i4 %d, index %d, shift %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d\n",var,i3,i4,index,shift,Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]]);
|
||||
indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
|
||||
printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
|
||||
indlist2[nind++] = ((index&(mask>>1))*((2*Zc*ncols)>>(shift-1))*Kb)+(index>>(shift-1));
|
||||
}
|
||||
|
||||
|
||||
}
|
||||
printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
|
||||
if (use_permutex==1 || use_alignr==1) {
|
||||
indlist[nind++] = index;
|
||||
}
|
||||
else {
|
||||
indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
|
||||
}
|
||||
}//i4
|
||||
}//i3
|
||||
for (i4=0;i4<nind-1;i4++) {
|
||||
fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
|
||||
fprintf(fd2,"%s(c2[%d],",xor_command,indlist2[i4]);
|
||||
if (use_permutex==0 && use_alignr==0) {
|
||||
fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
|
||||
}
|
||||
else {
|
||||
if ((indlist[i4]&mask) == 0) {
|
||||
fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]>>shift);
|
||||
}
|
||||
else if (use_permutex==1) {
|
||||
fprintf(fd,"%s(%s(c2[%d],abshift_get(%d),c2[%d]),",xor_command,permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
|
||||
}
|
||||
else if (use_alignr==1) {
|
||||
fprintf(fd,"%s(%s(c2[%d],c2[%d],%d),",xor_command,alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
|
||||
}
|
||||
}
|
||||
} //i4
|
||||
if (use_permutex==0 && use_alignr==0)
|
||||
fprintf(fd,"c2[%d]",indlist[i4]);
|
||||
else {
|
||||
if ((indlist[i4]&mask) == 0) {
|
||||
fprintf(fd,"c2[%d]",indlist[i4]>>shift);
|
||||
}
|
||||
else if (use_permutex==1){
|
||||
fprintf(fd,"%s(c2[%d],abshift_get(%d),c2[%d])",permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
|
||||
}
|
||||
else if (use_alignr==1) {
|
||||
fprintf(fd,"%s(c2[%d],c2[%d],%d)",alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
|
||||
|
||||
}
|
||||
}
|
||||
fprintf(fd,"c2[%d]",indlist[i4]);
|
||||
fprintf(fd2,"c2[%d]",indlist2[i4]);
|
||||
for (i4=0;i4<nind-1;i4++) { fprintf(fd,")"); fprintf(fd2,")"); }
|
||||
fprintf(fd,";\n");
|
||||
fprintf(fd2, ";\n");
|
||||
}
|
||||
fprintf(fd," }\n}\n");
|
||||
fprintf(fd2," }\n}\n");
|
||||
}
|
||||
for (i4=0;i4<nind-1;i4++) fprintf(fd,")");
|
||||
fprintf(fd,";\n");
|
||||
} // i1
|
||||
fprintf(fd," }\n}\n");
|
||||
} // i2
|
||||
fclose(fd);
|
||||
}
|
||||
else if (gen_code == 4) { // CUDA
|
||||
char fname[100];
|
||||
sprintf(fname,"ldpc_BG%d_Zc%d_32bit.cu",BG,Zc);
|
||||
FILE *fd=fopen(fname,"w");
|
||||
AssertFatal(fd!=NULL,"cannot open %s\n",fname);
|
||||
printf("Writing to %s\n",fname);
|
||||
fprintf(fd,"#include <stdio.h>\n#include <stdint.h>\n#include <cuda_runtime.h>\n");
|
||||
|
||||
fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
|
||||
fprintf(fd,"__global__ void ldpc_BG%d_Zc%d_worker(uint32_t *c[4],uint32_t *d[4]) {\n",BG,Zc);
|
||||
fprintf(fd," uint32_t *c32=c[blockIdx.x];\n uint32_t *d32=d[blockIdx.x];\n\n");
|
||||
fprintf(fd," int i2 = threadIdx.x;\n");
|
||||
fprintf(fd," int i1 = blockIdx.y;\n");
|
||||
fprintf(fd," if (i2 < %d) {\n",Zc);
|
||||
fprintf(fd," c32+=i2;\n");
|
||||
fprintf(fd," d32+=i2;\n");
|
||||
fprintf(fd," switch(i1) {\n");
|
||||
|
||||
for (int i1=0;i1<nrows;i1++) {
|
||||
nind = 0;
|
||||
fprintf(fd," case %d:\n",i1);
|
||||
fprintf(fd," d32[%d]=",(Zc*i1));
|
||||
for (i3=0; i3 < ncols; i3++)
|
||||
{
|
||||
temp_prime=i1 * ncols + i3;
|
||||
for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
|
||||
{
|
||||
var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
|
||||
int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
|
||||
printf("var %d, i3 %d, i4 %d, index %d (index mod 2Zc) %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d offset %d\n",var,i3,i4,index,index%(2*Zc),Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]],(i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc);
|
||||
if (index%(2*Zc) >= Zc) printf("***********************\n");
|
||||
indlist[nind] = index;
|
||||
printf("indlist[%d] %d, index %d\n",nind,indlist[nind],index);
|
||||
nind++;
|
||||
} //i4
|
||||
} // i3
|
||||
for (i4=0;i4<nind-1;i4++) {
|
||||
fprintf(fd,"c32[%d]^",indlist[i4]);
|
||||
} //i4
|
||||
fprintf(fd,"c32[%d];\n\n",indlist[i4]);
|
||||
fprintf(fd," break;\n");
|
||||
}// i1
|
||||
fprintf(fd," }\n");
|
||||
fprintf(fd," }\n");
|
||||
fprintf(fd,"}\n");
|
||||
|
||||
fprintf(fd,"extern \"C\" int ldpc_BG%d_Zc%d_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs) { \n",BG,Zc);
|
||||
fprintf(fd,"dim3 numblocks(n_inputs,%d);\n",nrows);
|
||||
fprintf(fd,"ldpc_BG%d_Zc%d_worker<<<numblocks,%d>>>(c,d);\n",BG,Zc,Zc);
|
||||
fprintf(fd," cudaDeviceSynchronize();\n");
|
||||
fprintf(fd," return(0);\n");
|
||||
fprintf(fd,"}\n");
|
||||
fclose(fd);
|
||||
fclose(fd2);
|
||||
}
|
||||
else if(gen_code==0)
|
||||
{
|
||||
|
||||
195
openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_cuda32.c
Normal file
195
openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_cuda32.c
Normal file
@@ -0,0 +1,195 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
/*! \file ldpc_encoder_cuda32.c
|
||||
* \brief Defines the optimized LDPC encoder for NVidia GPUs
|
||||
* \email openair_tech@eurecom.fr
|
||||
* \date 11-30-2025
|
||||
* \version 1.0
|
||||
* \note
|
||||
* \warning
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "assertions.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include "time_meas.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_defs.h"
|
||||
#include "PHY/sse_intrin.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
|
||||
#include "ldpc_generate_coefficient.c"
|
||||
|
||||
#include <cuda_runtime.h>
|
||||
|
||||
//#define DEBUG_LDPC 1
|
||||
|
||||
#include "ldpc_encode_parity_check_cuda.c"
|
||||
uint32_t *c_dev;
|
||||
uint32_t **c_host;
|
||||
uint32_t *c_devh[4];
|
||||
uint32_t *d_dev;
|
||||
uint32_t **d_host;
|
||||
uint32_t *d_devh[4];
|
||||
uint32_t *input_dev;
|
||||
uint32_t **input_host;
|
||||
uint32_t *input_devh[128];
|
||||
int managed = 0, concurrent = 0, uva = 0, pageable = 0, pageable_uses_host = 0, register_host = 0, integrated = 0;
|
||||
|
||||
int cuda_support_set = 0;
|
||||
|
||||
extern cudaStream_t encoderStreams[4];
|
||||
|
||||
int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *s,int sidx);
|
||||
|
||||
void cuda_support_init() {
|
||||
|
||||
int dev = 0;
|
||||
struct cudaDeviceProp prop;
|
||||
cudaGetDeviceProperties(&prop, dev);
|
||||
|
||||
|
||||
cudaDeviceGetAttribute(&managed, cudaDevAttrManagedMemory, dev);
|
||||
cudaDeviceGetAttribute(&concurrent, cudaDevAttrConcurrentManagedAccess, dev);
|
||||
cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, dev);
|
||||
cudaDeviceGetAttribute(&pageable, cudaDevAttrPageableMemoryAccess, dev);
|
||||
cudaDeviceGetAttribute(&pageable_uses_host, cudaDevAttrPageableMemoryAccessUsesHostPageTables, dev);
|
||||
cudaDeviceGetAttribute(®ister_host, cudaDevAttrHostRegisterSupported,dev);
|
||||
cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated,dev);
|
||||
|
||||
LOG_I(NR_PHY,"Device: %s (cc %d.%d)\n", prop.name, prop.major, prop.minor);
|
||||
LOG_I(NR_PHY,"Unified Virtual Addressing (UVA): %s\n", uva ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Managed (Unified) Memory: %s\n", managed ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Concurrent managed access: %s\n", concurrent ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Pageable memory access: %s\n", pageable ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Uses host page tables: %s\n", pageable_uses_host ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Host Register supported: %s\n", register_host ? "YES" : "NO");
|
||||
LOG_I(NR_PHY,"Integrated (shared) Memory %s\n", integrated ? "YES" : "NO");
|
||||
|
||||
if (!pageable && !integrated) {
|
||||
LOG_I(NR_PHY,"Allocating c,d,cc arrays for GPU \n");
|
||||
cudaError_t err=cudaMalloc((void **)&c_dev,4*sizeof(uint32_t*));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
|
||||
for (int i=0;i<4;i++) {
|
||||
err=cudaMalloc((void**)&c_devh[i],2*22*384*sizeof(uint32_t));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (chost[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
}
|
||||
err = cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
|
||||
err=cudaMalloc((void**)&d_dev,4*sizeof(uint32_t*));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
|
||||
for (int i=0;i<4;i++) {
|
||||
err=cudaMalloc((void**)&d_devh[i],68*384*sizeof(uint32_t));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (d_devh[%d]: %s\n", i,cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
}
|
||||
err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
|
||||
err=cudaMalloc((void**)&input_dev,128*sizeof(uint8_t*));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
|
||||
for (int i=0;i<128;i++) {
|
||||
err=cudaMalloc((void**)&input_devh[i],(8448/8)*sizeof(uint8_t));
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (input_devh[%d]: %s\n", i,cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void**)&input_host[i],(8448/8)*sizeof(uint8_t),cudaHostAllocDefault);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (input_host[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
}
|
||||
err=cudaMemcpy(input_dev,input_devh,128*sizeof(uint8_t*),cudaMemcpyHostToDevice);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy cc_devh -> d_dev): %s\n", cudaGetErrorString(err));
|
||||
}
|
||||
else {
|
||||
LOG_I(NR_PHY,"Allocating c,d,cc arrays for CPU/GPU shared-memory\n");
|
||||
cudaError_t err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocMapped|cudaHostAllocPortable);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
|
||||
err = cudaHostGetDevicePointer((void**)&c_dev, c_host, 0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
|
||||
LOG_I(NR_PHY,"c_host %p, c_dev %p\n",c_host,c_dev);
|
||||
for (int i=0;i<4;i++) {
|
||||
err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_host[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
err = cudaHostGetDevicePointer((void**)&c_devh[i], c_host[i], 0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
}
|
||||
err=cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&d_dev, d_host, 0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(d_dev): %s\n", cudaGetErrorString(err));
|
||||
LOG_I(NR_PHY,"d_host %p, d_dev %p\n",d_host,d_dev);
|
||||
for (int i=0;i<4;i++) {
|
||||
err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&d_devh[i], d_host[i], 0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) d_devh[%d]: %s\n", i,cudaGetErrorString(err));
|
||||
LOG_I(NR_PHY,"d_host[%d] %p, d_devh[%d] %p\n",i,d_host[i],i,d_devh[i]);
|
||||
}
|
||||
err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (input_host): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&input_dev, input_host, 0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(cc_host): %s\n", cudaGetErrorString(err));
|
||||
LOG_I(NR_PHY,"input_host %p, input_dev %p\n",input_host,input_dev);
|
||||
}
|
||||
|
||||
|
||||
cuda_support_set=1;
|
||||
}
|
||||
|
||||
uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp)
|
||||
{
|
||||
//set_log(PHY, 4);
|
||||
|
||||
int Zc = impp->Zc;
|
||||
int Kb = impp->Kb;
|
||||
short block_length = impp->K;
|
||||
short BG = impp->BG;
|
||||
int ncols=22;
|
||||
|
||||
int encoder_stream=0;
|
||||
|
||||
AssertFatal(BG==1,"BG %d is not supported for CUDA version\n",BG);
|
||||
AssertFatal(Zc==384,"Zc %d is not supported for CUDA version \n", Zc);
|
||||
|
||||
if(impp->tinput != NULL) start_meas(impp->tinput);
|
||||
|
||||
#ifdef DEBUG_LDPC
|
||||
LOG_I(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
|
||||
LOG_I(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
|
||||
#endif
|
||||
|
||||
int n_inputs = (impp->n_segments/32)+(((impp->n_segments&31) > 0) ? 1: 0);
|
||||
// uint32_t cc[4][22*Zc]; //padded input, unpacked, max size
|
||||
|
||||
if (!pageable&&!integrated) { // this means we are not on shared memory
|
||||
for (int r=0;r<impp->n_segments;r++) {
|
||||
cudaMemcpyAsync(input_devh[r],input[r],block_length>>3,cudaMemcpyHostToDevice,encoderStreams[encoder_stream]);
|
||||
}
|
||||
}
|
||||
ldpc_input(pageable||integrated? (uint32_t**)input : (uint32_t**)input_dev,(uint32_t**)c_dev,impp->n_segments,encoderStreams,encoder_stream);
|
||||
if(impp->tinput != NULL) stop_meas(impp->tinput);
|
||||
//parity check part
|
||||
if(impp->tparity != NULL) start_meas(impp->tparity);
|
||||
encode_parity_check_part_cuda((uint32_t**)c_dev, (uint32_t**)d_dev, BG, Zc, Kb, ncols,n_inputs,encoderStreams,encoder_stream);
|
||||
if (!pageable&&!integrated) { // this means we are not on shared memory
|
||||
for (int r=0; r<n_inputs;r++) cudaMemcpyAsync(d_host[r],d_devh[r],68*384*sizeof(uint32_t),cudaMemcpyDeviceToHost,encoderStreams[encoder_stream]);
|
||||
}
|
||||
cudaStreamSynchronize(encoderStreams[encoder_stream]);
|
||||
if(impp->tparity != NULL) stop_meas(impp->tparity);
|
||||
|
||||
return d_host;
|
||||
}
|
||||
|
||||
@@ -170,12 +170,9 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
|
||||
if(impp->tinput != NULL) stop_meas(impp->tinput);
|
||||
|
||||
if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
|
||||
// extend matrix
|
||||
if(impp->tprep != NULL) start_meas(impp->tprep);
|
||||
if(impp->tprep != NULL) stop_meas(impp->tprep);
|
||||
//parity check part
|
||||
if(impp->tparity != NULL) start_meas(impp->tparity);
|
||||
encode_parity_check_part_optim(cc, dd, BG, Zc, Kb, simd_size, ncols);
|
||||
encode_parity_check_part_optim(cc, dd, BG, Zc, Kb, simd_size, ncols,impp->tinput_memcpy);
|
||||
if(impp->tparity != NULL) stop_meas(impp->tparity);
|
||||
}
|
||||
else {
|
||||
@@ -184,8 +181,10 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
|
||||
return(-1);
|
||||
}
|
||||
}
|
||||
if(impp->toutput != NULL) start_meas(impp->toutput);
|
||||
memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
|
||||
memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
|
||||
if(impp->toutput != NULL) stop_meas(impp->toutput);
|
||||
return 0;
|
||||
}
|
||||
|
||||
|
||||
92
openair1/PHY/CODING/nrLDPC_encoder/ldpc_input.cu
Normal file
92
openair1/PHY/CODING/nrLDPC_encoder/ldpc_input.cu
Normal file
@@ -0,0 +1,92 @@
|
||||
#include <stdio.h>
|
||||
#include <stdint.h>
|
||||
#include <cuda_runtime.h>
|
||||
/*
|
||||
__device__ const uint32_t masks[4] = {0x80,0x8000,0x800000,0x80000000};
|
||||
__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) {
|
||||
|
||||
int block_off = blockIdx.x*blockDim.x<<2;
|
||||
int i2 = threadIdx.x<<2;
|
||||
uint32_t *out=cc[blockIdx.y] + block_off + i2;
|
||||
int nseg0 = (blockIdx.y << 5);
|
||||
int nseg1;
|
||||
if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
|
||||
else nseg1 = nseg0 + (nseg&31);
|
||||
int bit_offset = i2+block_off;
|
||||
int uint32_offset = bit_offset>>5;
|
||||
uint32_t mask = masks[(bit_offset&31)>>3];
|
||||
uint32_t mask0 = mask>>(bit_offset&7);bit_offset++;
|
||||
uint32_t mask1 = mask>>(bit_offset&7);bit_offset++;
|
||||
uint32_t mask2 = mask>>(bit_offset&7);bit_offset++;
|
||||
uint32_t mask3 = mask>>(bit_offset&7);
|
||||
uint32_t tmp,jmod;
|
||||
uint32_t otmp0,otmp1,otmp2,otmp3;
|
||||
if (bit_offset < block_length) {
|
||||
tmp=input[nseg0][uint32_offset];
|
||||
otmp0 = ((tmp&mask0) > 0);
|
||||
otmp1 = ((tmp&mask1) > 0);
|
||||
otmp2 = ((tmp&mask2) > 0);
|
||||
otmp3 = ((tmp&mask3) > 0);
|
||||
for (int j=nseg0+1;j<nseg1;j++) {
|
||||
tmp=input[j][uint32_offset];
|
||||
jmod = j&31;
|
||||
otmp0 |= (((tmp&mask0) > 0)<<jmod);
|
||||
otmp1 |= (((tmp&mask1) > 0)<<jmod);
|
||||
otmp2 |= (((tmp&mask2) > 0)<<jmod);
|
||||
otmp3 |= (((tmp&mask3) > 0)<<jmod);
|
||||
}
|
||||
out[0]=otmp0;
|
||||
out[1]=otmp1;
|
||||
out[2]=otmp2;
|
||||
out[3]=otmp3;
|
||||
}
|
||||
}
|
||||
*/
|
||||
|
||||
__device__ uint32_t masks[32] = {
|
||||
0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1,
|
||||
0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100,
|
||||
0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000,
|
||||
0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000};
|
||||
__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int nseg) {
|
||||
|
||||
// int block_off = blockIdx.y*blockDim.x;
|
||||
int i1 = blockIdx.y;
|
||||
int i2 = threadIdx.x;
|
||||
int nseg0 = (blockIdx.x << 5);
|
||||
int nseg1;
|
||||
if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
|
||||
else nseg1 = nseg0 + (nseg&31);
|
||||
int bit_offset = i2+(i1*384);
|
||||
int uint32_offset = bit_offset>>5;
|
||||
uint32_t mask0 = masks[bit_offset&31];
|
||||
uint32_t tmp,jmod;
|
||||
uint32_t otmp0;
|
||||
if (bit_offset < 8448) {
|
||||
tmp=input[nseg0][uint32_offset];
|
||||
otmp0 = ((tmp&mask0) > 0);
|
||||
for (int j=nseg0+1;j<nseg1;j++) {
|
||||
tmp=input[j][uint32_offset];
|
||||
jmod = j&31;
|
||||
otmp0 |= (((tmp&mask0) > 0)<<jmod);
|
||||
}
|
||||
|
||||
cc[blockIdx.x][(2*i1*384) + i2] = otmp0;
|
||||
cc[blockIdx.x][(2*i1+1)*384 + i2]=otmp0;
|
||||
}
|
||||
}
|
||||
extern "C" int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *stream,int sidx) {
|
||||
|
||||
int ns = nseg>>5;
|
||||
if ((nseg&31)>0) ns++;
|
||||
|
||||
dim3 numblocks(ns,22);
|
||||
//printf("input %p\n",input);
|
||||
ldpc_input_worker<<<numblocks,384,0,stream[sidx]>>>(input,cc,nseg);
|
||||
cudaError_t err=cudaPeekAtLastError();
|
||||
if (err!=cudaSuccess) {
|
||||
printf("cuda error: %s (input %p, cc %p, nseg %d, ns %d)\n",cudaGetErrorString(err),input,cc,nseg,ns);
|
||||
exit(-1);
|
||||
}
|
||||
return(0);
|
||||
}
|
||||
492
openair1/PHY/CODING/nrLDPC_encoder/ldpc_input32.c
Normal file
492
openair1/PHY/CODING/nrLDPC_encoder/ldpc_input32.c
Normal file
@@ -0,0 +1,492 @@
|
||||
/*
|
||||
* SPDX-License-Identifier: LicenseRef-CSSL-1.0
|
||||
*/
|
||||
|
||||
#include <stdlib.h>
|
||||
#include <math.h>
|
||||
#include <stdio.h>
|
||||
#include <string.h>
|
||||
#include "assertions.h"
|
||||
#include "common/utils/LOG/log.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_defs.h"
|
||||
#include "PHY/sse_intrin.h"
|
||||
#include "openair1/PHY/CODING/nrLDPC_extern.h"
|
||||
|
||||
|
||||
void ldpc_input32(uint8_t **input, uint32_t **c, int n_inputs, int n_segments) {
|
||||
|
||||
#ifndef __aarch64__
|
||||
simde__m256i andmask = simde_mm256_set_epi32(0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80); // every 8 bits -> 8 bytes, pattern repeats.
|
||||
simde__m256i zero256 = simde_mm256_setzero_si256();
|
||||
simde__m256i masks[32];
|
||||
register simde__m256i c256;
|
||||
for (int i=0;i<32;i++) {
|
||||
masks[i] = simde_mm256_set1_epi32(1<<i);
|
||||
}
|
||||
|
||||
for (int b=0;b<8448;b+=384) {
|
||||
for (int i=0; i < 384; i += 8) {
|
||||
unsigned int i8 = (2*b + i) >> 3;
|
||||
for (int j0=0;j0<n_inputs;j0++) {
|
||||
c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[j0<<5][i8]),andmask),zero256),masks[0]);
|
||||
int j2 = 32;
|
||||
if (j0==(n_inputs-1)) j2= n_segments-(j0<<5);
|
||||
for (int j=1; j < j2; j++) {
|
||||
c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[(j0<<5)+j][i8]),andmask),zero256),masks[j]),c256);
|
||||
}
|
||||
((simde__m256i *)c[j0])[i8] = c256;
|
||||
((simde__m256i *)c[j0])[i8+48] = c256;
|
||||
}
|
||||
}
|
||||
}
|
||||
#else
|
||||
int i2=0;
|
||||
uint32_t cc[4][8448];
|
||||
const int32_t ucShifta[32][4] = {
|
||||
{-7,-6,-5,-4}, // 0
|
||||
{-6,-5,-4,-3}, // 1
|
||||
{-5,-4,-3,-2}, // 2
|
||||
{-4,-3,-2,-1}, // 3
|
||||
{-3,-2,-1,0}, // 4
|
||||
{-2,-1,0,1}, // 5
|
||||
{-1,0,1,2}, // 6
|
||||
{0,1,2,3}, // 7
|
||||
{1,2,3,4}, // 8
|
||||
{2,3,4,5}, // 9
|
||||
{3,4,5,6}, // 10
|
||||
{4,5,6,7}, // 11
|
||||
{5,6,7,8}, // 12
|
||||
{6,7,8,9}, // 13
|
||||
{7,8,9,10}, // 14
|
||||
{8,9,10,11}, // 15
|
||||
{9,10,11,12}, // 16
|
||||
{10,11,12,13}, // 17
|
||||
{11,12,13,14}, // 18
|
||||
{12,13,14,15}, // 19
|
||||
{13,14,15,16}, // 20
|
||||
{14,15,16,17}, // 21
|
||||
{15,16,17,18}, // 22
|
||||
{16,17,18,19}, // 23
|
||||
{17,18,19,20}, // 24
|
||||
{18,19,20,21}, // 25
|
||||
{19,20,21,22}, // 26
|
||||
{20,21,22,23}, // 27
|
||||
{21,22,23,24}, // 28
|
||||
{22,23,24,25}, // 29
|
||||
{23,24,25,26}, // 30
|
||||
{24,25,26,27}}; // 31
|
||||
const int32_t ucShiftb[32][4] = {
|
||||
{-3,-2,-1,0}, // 0
|
||||
{-2,-1,0,1}, // 1
|
||||
{-1,0,1,2}, // 2
|
||||
{0,1,2,3}, // 3
|
||||
{1,2,3,4}, // 4
|
||||
{2,3,4,5}, // 5
|
||||
{3,4,5,6}, // 6
|
||||
{4,5,6,7}, // 7
|
||||
{5,6,7,8}, // 8
|
||||
{6,7,8,9}, // 9
|
||||
{7,8,9,10}, // 10
|
||||
{8,9,10,11}, // 11
|
||||
{9,10,11,12}, // 12
|
||||
{10,11,12,13}, // 13
|
||||
{11,12,13,14}, // 14
|
||||
{12,13,14,15}, // 15
|
||||
{13,14,15,16}, // 16
|
||||
{14,15,16,17}, // 17
|
||||
{15,16,17,18}, // 18
|
||||
{16,17,18,19}, // 19
|
||||
{17,18,19,20}, // 20
|
||||
{18,19,20,21}, // 21
|
||||
{19,20,21,22}, // 22
|
||||
{20,21,22,23}, // 23
|
||||
{21,22,23,24}, // 24
|
||||
{22,23,24,25}, // 25
|
||||
{23,24,25,26}, // 26
|
||||
{24,25,26,27}, // 27
|
||||
{25,26,27,28}, // 28
|
||||
{26,27,28,29}, // 29
|
||||
{27,28,29,30}, // 30
|
||||
{28,29,30,31}}; // 31
|
||||
|
||||
const int32_t ucShiftc[32][4] = {
|
||||
{-15,-14,-13,-12}, // 0
|
||||
{-14,-13,-12,-11}, // 1
|
||||
{-13,-12,-11,-10}, // 2
|
||||
{-12,-11,-10,-9}, // 3
|
||||
{-11,-10,-9,-8}, // 4
|
||||
{-10,-9,-8,-7}, // 5
|
||||
{-9,-8,-7,-6}, // 6
|
||||
{-8,-7,-6,-5}, // 7
|
||||
{-7,-6,-5,-4}, // 8
|
||||
{-6,-5,-4,-3}, // 9
|
||||
{-5,-4,-3,-2}, // 10
|
||||
{-4,-3,-2,-1}, // 11
|
||||
{-3,-2,-1,0}, // 12
|
||||
{-2,-1,0,1}, // 13
|
||||
{-1,0,1,2}, // 14
|
||||
{0,1,2,3}, // 15
|
||||
{1,2,3,4}, // 16
|
||||
{2,3,4,5}, // 17
|
||||
{3,4,5,6}, // 18
|
||||
{4,5,6,7}, // 19
|
||||
{5,6,7,8}, // 20
|
||||
{6,7,8,9}, // 21
|
||||
{7,8,9,10}, // 22
|
||||
{8,9,10,11}, // 23
|
||||
{9,10,11,12}, // 24
|
||||
{10,11,12,13}, // 25
|
||||
{11,12,13,14}, // 26
|
||||
{12,13,14,15}, // 27
|
||||
{13,14,15,16}, // 28
|
||||
{14,15,16,17}, // 29
|
||||
{15,16,17,18}, // 30
|
||||
{16,17,18,19}}; // 31
|
||||
const int32_t ucShiftd[32][4] = {
|
||||
{-11,-10,-9,-8}, // 0
|
||||
{-10,-9,-8,-7}, // 1
|
||||
{-9,-8,-7,-6}, // 2
|
||||
{-8,-7,-6,-5}, // 3
|
||||
{-7,-6,-5,-4}, // 4
|
||||
{-6,-5,-4,-3}, // 5
|
||||
{-5,-4,-3,-2}, // 6
|
||||
{-4,-3,-2,-1}, // 7
|
||||
{-3,-2,-1,0}, // 8
|
||||
{-2,-1,0,1}, // 9
|
||||
{-1,0,1,2}, // 10
|
||||
{0,1,2,3}, // 11
|
||||
{1,2,3,4}, // 12
|
||||
{2,3,4,5}, // 13
|
||||
{3,4,5,6}, // 14
|
||||
{4,5,6,7}, // 15
|
||||
{5,6,7,8}, // 16
|
||||
{6,7,8,9}, // 17
|
||||
{7,8,9,10}, // 18
|
||||
{8,9,10,11}, // 19
|
||||
{9,10,11,12}, // 20
|
||||
{10,11,12,13}, // 21
|
||||
{11,12,13,14}, // 22
|
||||
{12,13,14,15}, // 23
|
||||
{13,14,15,16}, // 24
|
||||
{14,15,16,17}, // 25
|
||||
{15,16,17,18}, // 26
|
||||
{16,17,18,19}, // 27
|
||||
{17,18,19,20}, // 28
|
||||
{18,19,20,21}, // 29
|
||||
{19,20,21,22}, // 30
|
||||
{20,21,22,23}};// 31
|
||||
|
||||
const int32_t ucShifte[32][4] = {
|
||||
{-23,-22,-21,-20}, // 0
|
||||
{-22,-21,-20,-19}, // 1
|
||||
{-21,-20,-19,-18}, // 2
|
||||
{-20,-19,-18,-17}, // 3
|
||||
{-19,-18,-17,-16}, // 4
|
||||
{-18,-17,-16,-15}, // 5
|
||||
{-17,-16,-15,-14}, // 6
|
||||
{-16,-15,-14,-13}, // 7
|
||||
{-15,-14,-13,-12}, // 8
|
||||
{-14,-13,-12,-11}, // 9
|
||||
{-13,-12,-11,-10}, // 10
|
||||
{-12,-11,-10,-9}, // 11
|
||||
{-11,-10,-9,-8}, // 12
|
||||
{-10,-9,-8,-7}, // 13
|
||||
{-9,-8,-7,-6}, // 14
|
||||
{-8,-7,-6,-5}, // 15
|
||||
{-7,-6,-5,-4}, // 16
|
||||
{-6,-5,-4,-3}, // 17
|
||||
{-5,-4,-3,-2}, // 18
|
||||
{-4,-3,-2,-1}, // 19
|
||||
{-3,-2,-1,0}, // 20
|
||||
{-2,-1,0,1}, // 21
|
||||
{-1,0,1,2}, // 22
|
||||
{0,1,2,3}, // 23
|
||||
{1,2,3,4}, // 24
|
||||
{2,3,4,5}, // 25
|
||||
{3,4,5,6}, // 26
|
||||
{4,5,6,7}, // 27
|
||||
{5,6,7,8}, // 28
|
||||
{6,7,8,9}, // 29
|
||||
{7,8,9,10}, // 30
|
||||
{8,9,10,11}}; // 31
|
||||
|
||||
const int32_t ucShiftf[32][4] = {
|
||||
{-19,-18,-17,-16}, // 0
|
||||
{-18,-17,-16,-15}, // 1
|
||||
{-17,-16,-15,-14}, // 2
|
||||
{-16,-15,-14,-13}, // 3
|
||||
{-15,-14,-13,-12}, // 4
|
||||
{-14,-13,-12,-11}, // 5
|
||||
{-13,-12,-11,-10}, // 6
|
||||
{-12,-11,-10,-9}, // 7
|
||||
{-11,-10,-9,-8}, // 8
|
||||
{-10,-9,-8,-7}, // 9
|
||||
{-9,-8,-7,-6}, // 10
|
||||
{-8,-7,-6,-5}, // 11
|
||||
{-7,-6,-5,-4}, // 12
|
||||
{-6,-5,-4,-3}, // 13
|
||||
{-5,-4,-3,-2}, // 14
|
||||
{-4,-3,-2,-1}, // 15
|
||||
{-3,-2,-1,0}, // 16
|
||||
{-2,-1,0,1}, // 17
|
||||
{-1,0,1,2}, // 18
|
||||
{0,1,2,3}, // 19
|
||||
{1,2,3,4}, // 20
|
||||
{2,3,4,5}, // 21
|
||||
{3,4,5,6}, // 22
|
||||
{4,5,6,7}, // 23
|
||||
{5,6,7,8}, // 24
|
||||
{6,7,8,9}, // 25
|
||||
{7,8,9,10}, // 26
|
||||
{8,9,10,11}, // 27
|
||||
{9,10,11,12}, // 28
|
||||
{10,11,12,13}, // 29
|
||||
{11,12,13,14}, // 30
|
||||
{12,13,14,15}}; // 31
|
||||
|
||||
const int32_t ucShiftg[32][4] = {
|
||||
{-31,-30,-29,-28},
|
||||
{-30,-29,-28,-27},
|
||||
{-29,-28,-27,-26},
|
||||
{-28,-27,-26,-25},
|
||||
{-27,-26,-25,-24},
|
||||
{-26,-25,-24,-23},
|
||||
{-25,-24,-23,-22},
|
||||
{-24,-23,-22,-21},
|
||||
{-23,-22,-21,-20}, // 0
|
||||
{-22,-21,-20,-19}, // 1
|
||||
{-21,-20,-19,-18}, // 2
|
||||
{-20,-19,-18,-17}, // 3
|
||||
{-19,-18,-17,-16}, // 4
|
||||
{-18,-17,-16,-15}, // 5
|
||||
{-17,-16,-15,-14}, // 6
|
||||
{-16,-15,-14,-13}, // 7
|
||||
{-15,-14,-13,-12}, // 8
|
||||
{-14,-13,-12,-11}, // 9
|
||||
{-13,-12,-11,-10}, // 10
|
||||
{-12,-11,-10,-9}, // 11
|
||||
{-11,-10,-9,-8}, // 12
|
||||
{-10,-9,-8,-7}, // 13
|
||||
{-9,-8,-7,-6}, // 14
|
||||
{-8,-7,-6,-5}, // 15
|
||||
{-7,-6,-5,-4}, // 16
|
||||
{-6,-5,-4,-3}, // 17
|
||||
{-5,-4,-3,-2}, // 18
|
||||
{-4,-3,-2,-1}, // 19
|
||||
{-3,-2,-1,0}, // 20
|
||||
{-2,-1,0,1}, // 21
|
||||
{-1,0,1,2}, // 22
|
||||
{0,1,2,3}}; // 23
|
||||
|
||||
const int32_t ucShifth[32][4] = {
|
||||
{-27,-26,-25,-24}, // 0
|
||||
{-26,-25,-24,-23}, // 1
|
||||
{-25,-24,-23,-22}, // 2
|
||||
{-24,-23,-22,-21}, // 3
|
||||
{-23,-22,-21,-20}, // 4
|
||||
{-22,-21,-20,-19}, // 5
|
||||
{-21,-20,-19,-18}, // 6
|
||||
{-20,-19,-18,-17}, // 7
|
||||
{-19,-18,-17,-16}, // 8
|
||||
{-18,-17,-16,-15}, // 9
|
||||
{-17,-16,-15,-14}, // 10
|
||||
{-16,-15,-14,-13}, // 11
|
||||
{-15,-14,-13,-12}, // 12
|
||||
{-14,-13,-12,-11}, // 13
|
||||
{-13,-12,-11,-10}, // 14
|
||||
{-12,-11,-10,-9}, // 15
|
||||
{-11,-10,-9,-8}, // 16
|
||||
{-10,-9,-8,-7}, // 17
|
||||
{-9,-8,-7,-6}, // 18
|
||||
{-8,-7,-6,-5}, // 19
|
||||
{-7,-6,-5,-4}, // 20
|
||||
{-6,-5,-4,-3}, // 21
|
||||
{-5,-4,-3,-2}, // 22
|
||||
{-4,-3,-2,-1}, // 23
|
||||
{-3,-2,-1,0}, // 24
|
||||
{-2,-1,0,1}, // 25
|
||||
{-1,0,1,2}, // 26
|
||||
{0,1,2,3}, // 27
|
||||
{1,2,3,4}, // 28
|
||||
{2,3,4,5}, // 29
|
||||
{3,4,5,6}, // 30
|
||||
{4,5,6,7}}; // 31
|
||||
const uint32_t __attribute__ ((aligned (16))) masksa[4] = {0x80,0x40,0x20,0x10};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksb[4] = {0x8,0x4,0x2,0x1};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksc[4] = {0x8000,0x4000,0x2000,0x1000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksd[4] = {0x800,0x400,0x200,0x100};
|
||||
const uint32_t __attribute__ ((aligned (16))) maskse[4] = {0x800000,0x400000,0x200000,0x100000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksf[4] = {0x80000,0x40000,0x20000,0x10000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksg[4] = {0x80000000,0x40000000,0x20000000,0x10000000};
|
||||
const uint32_t __attribute__ ((aligned (16))) masksh[4] = {0x8000000,0x4000000,0x2000000,0x1000000};
|
||||
int32x4_t vshifta[32],vshiftb[32],vshiftc[32],vshiftd[32],vshifte[32],vshiftf[32],vshiftg[32],vshifth[32];
|
||||
uint32x4_t vmasksa = vld1q_u32(masksa);
|
||||
uint32x4_t vmasksb = vld1q_u32(masksb);
|
||||
uint32x4_t vmasksc = vld1q_u32(masksc);
|
||||
uint32x4_t vmasksd = vld1q_u32(masksd);
|
||||
uint32x4_t vmaskse = vld1q_u32(maskse);
|
||||
uint32x4_t vmasksf = vld1q_u32(masksf);
|
||||
uint32x4_t vmasksg = vld1q_u32(masksg);
|
||||
uint32x4_t vmasksh = vld1q_u32(masksh);
|
||||
uint32x4_t in;
|
||||
|
||||
for (int n=0;n<32;n++) {
|
||||
vshifta[n] = vld1q_s32(ucShifta[n]);
|
||||
vshiftb[n] = vld1q_s32(ucShiftb[n]);
|
||||
vshiftc[n] = vld1q_s32(ucShiftc[n]);
|
||||
vshiftd[n] = vld1q_s32(ucShiftd[n]);
|
||||
vshifte[n] = vld1q_s32(ucShifte[n]);
|
||||
vshiftf[n] = vld1q_s32(ucShiftf[n]);
|
||||
vshiftg[n] = vld1q_s32(ucShiftg[n]);
|
||||
vshifth[n] = vld1q_s32(ucShifth[n]);
|
||||
}
|
||||
i2=0;
|
||||
int j0=0,j1=0,j2=0,j3=0;
|
||||
if (n_segments <= 32) {
|
||||
j0=n_segments;
|
||||
}
|
||||
else if (n_segments <= 64) {
|
||||
j0=32;
|
||||
j1=n_segments-32;
|
||||
}
|
||||
else if (n_segments <= 96) {
|
||||
j0=32; j1=32;
|
||||
j2=n_segments-64;
|
||||
}
|
||||
else if (n_segments <= 128) {
|
||||
j0=32; j1=32; j2=32;
|
||||
j3=n_segments-96;
|
||||
}
|
||||
uint32x4_t *ccp,cc0,cc1,cc2,cc3,cc4,cc5,cc6,cc7;
|
||||
for (int i=0; i < 264; i++,i2+=8) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[0])[i]);
|
||||
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
|
||||
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
|
||||
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
|
||||
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
|
||||
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
|
||||
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
|
||||
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
|
||||
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
|
||||
for (int j = 1; j < j0; j++) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[j])[i]);
|
||||
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
|
||||
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
|
||||
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
|
||||
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
|
||||
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
|
||||
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
|
||||
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
|
||||
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
|
||||
}
|
||||
ccp=&((uint32x4_t *)cc[0])[i2];
|
||||
ccp[0] = cc0;
|
||||
ccp[1] = cc1;
|
||||
ccp[2] = cc2;
|
||||
ccp[3] = cc3;
|
||||
ccp[4] = cc4;
|
||||
ccp[5] = cc5;
|
||||
ccp[6] = cc6;
|
||||
ccp[7] = cc7;
|
||||
if (j1>0) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[32])[i]);
|
||||
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
|
||||
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
|
||||
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
|
||||
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
|
||||
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
|
||||
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
|
||||
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
|
||||
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
|
||||
for (int j = 1; j < j1; j++) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[32+j])[i]);
|
||||
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
|
||||
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
|
||||
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
|
||||
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
|
||||
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
|
||||
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
|
||||
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
|
||||
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
|
||||
}
|
||||
ccp=&((uint32x4_t *)cc[1])[i2];
|
||||
ccp[0] = cc0;
|
||||
ccp[1] = cc1;
|
||||
ccp[2] = cc2;
|
||||
ccp[3] = cc3;
|
||||
ccp[4] = cc4;
|
||||
ccp[5] = cc5;
|
||||
ccp[6] = cc6;
|
||||
ccp[7] = cc7;
|
||||
}
|
||||
if (j2>0) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[64])[i]);
|
||||
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
|
||||
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
|
||||
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
|
||||
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
|
||||
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
|
||||
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
|
||||
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
|
||||
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
|
||||
for (int j = 1; j < j2; j++) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[64+j])[i]);
|
||||
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
|
||||
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
|
||||
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
|
||||
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
|
||||
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
|
||||
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
|
||||
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
|
||||
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
|
||||
}
|
||||
ccp=&((uint32x4_t *)cc[2])[i2];
|
||||
ccp[0] = cc0;
|
||||
ccp[1] = cc1;
|
||||
ccp[2] = cc2;
|
||||
ccp[3] = cc3;
|
||||
ccp[4] = cc4;
|
||||
ccp[5] = cc5;
|
||||
ccp[6] = cc6;
|
||||
ccp[7] = cc7;
|
||||
}
|
||||
if (j3>0) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[96])[i]);
|
||||
cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
|
||||
cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
|
||||
cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
|
||||
cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
|
||||
cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
|
||||
cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
|
||||
cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
|
||||
cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
|
||||
for (int j = 1; j < j3; j++) {
|
||||
in = vdupq_n_u32(((uint32_t*)input[96+j])[i]);
|
||||
cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
|
||||
cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
|
||||
cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
|
||||
cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
|
||||
cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
|
||||
cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
|
||||
cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
|
||||
cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
|
||||
}
|
||||
ccp=&((uint32x4_t *)cc[3])[i2];
|
||||
ccp[0] = cc0;
|
||||
ccp[1] = cc1;
|
||||
ccp[2] = cc2;
|
||||
ccp[3] = cc3;
|
||||
ccp[4] = cc4;
|
||||
ccp[5] = cc5;
|
||||
ccp[6] = cc6;
|
||||
ccp[7] = cc7;
|
||||
}
|
||||
}
|
||||
for (int s=0;s<n_inputs;s++) {
|
||||
for (int i1 = 0; i1 < 22; i1++) {
|
||||
memcpy(&c[s][2 * i1 * 384] , &cc[s][i1 * 384], 384 * sizeof(uint32_t));
|
||||
memcpy(&c[s][(2 * i1 + 1) * 384], &cc[s][i1 * 384], 384 * sizeof(uint32_t));
|
||||
}
|
||||
}
|
||||
#endif
|
||||
}
|
||||
@@ -36,6 +36,7 @@ int load_LDPClib(char *version, ldpc_interface_t *itf)
|
||||
int ret;
|
||||
ret = load_module_version_shlib(libname, version, shlib_fdesc, sizeofArray(shlib_fdesc), NULL);
|
||||
AssertFatal((ret >= 0), "Error loading ldpc decoder");
|
||||
LOG_I(NR_PHY,"loading LDPC version %s\n",version);
|
||||
itf->LDPCinit = (LDPC_initfunc_t *)shlib_fdesc[0].fptr;
|
||||
itf->LDPCshutdown = (LDPC_shutdownfunc_t *)shlib_fdesc[1].fptr;
|
||||
itf->LDPCdecoder = (LDPC_decoderfunc_t *)shlib_fdesc[2].fptr;
|
||||
|
||||
@@ -25,6 +25,10 @@
|
||||
#include "PHY/NR_REFSIG/ul_ref_seq_nr.h"
|
||||
#include <string.h>
|
||||
#include "nfapi/open-nFAPI/fapi/inc/nr_fapi_p5_utils.h"
|
||||
#ifdef ENABLE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
static void init_DLSCH_struct(PHY_VARS_gNB *gNB);
|
||||
static void destroy_DLSCH_struct(const PHY_VARS_gNB *gNB);
|
||||
@@ -92,7 +96,8 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
|
||||
nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
|
||||
NR_gNB_COMMON *const common_vars = &gNB->common_vars;
|
||||
common_vars->analog_bf = cfg->analog_beamforming_ve.analog_bf_vendor_ext.value;
|
||||
LOG_I(PHY, "L1 configured with%s analog beamforming\n", common_vars->analog_bf ? "" : "out");
|
||||
gNB->use_gpu = use_gpu;
|
||||
LOG_I(PHY, "L1 configured with%s analog beamforming, use_gpu = %d\n", common_vars->analog_bf ? "" : "out",use_gpu);
|
||||
if (common_vars->analog_bf) {
|
||||
// True only if nrmac->beam_info.beam_mode == FAPI_ANALOG_BEAM, thus analog_beamforming=2
|
||||
common_vars->num_beams_period = cfg->analog_beamforming_ve.num_beams_period_vendor_ext.value;
|
||||
@@ -119,8 +124,6 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
|
||||
|
||||
nr_init_fde(); // Init array for frequency equalization of transform precoding of PUSCH
|
||||
|
||||
int ret_loader = load_nrLDPC_coding_interface(NULL, &gNB->nrLDPC_coding_interface);
|
||||
AssertFatal(ret_loader == 0, "error loading LDPC library\n");
|
||||
|
||||
gNB->max_nb_pdsch = MAX_MOBILES_PER_GNB;
|
||||
init_delay_table(fp->ofdm_symbol_size, MAX_DELAY_COMP, NR_MAX_OFDM_SYMBOL_SIZE, fp->delay_table);
|
||||
@@ -141,6 +144,9 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
|
||||
/// Transport init necessary for NR synchro
|
||||
init_nr_transport(gNB);
|
||||
|
||||
int ret_loader = load_nrLDPC_coding_interface(NULL, &gNB->nrLDPC_coding_interface,16*gNB->max_nb_pusch);
|
||||
AssertFatal(ret_loader == 0, "error loading LDPC library\n");
|
||||
|
||||
init_DLSCH_struct(gNB);
|
||||
|
||||
/* Do NOT allocate per-antenna rxdataF: the gNB gets a pointer to the
|
||||
@@ -188,8 +194,14 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
|
||||
|
||||
for (int i = 0; i < max_ul_mimo_layers; i++) {
|
||||
}
|
||||
pusch->llr = (int16_t *)malloc16_clear((8 * ((3 * 8 * 6144) + 12))
|
||||
* sizeof(int16_t)); // [hna] 6144 is LTE and (8*((3*8*6144)+12)) is not clear
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaError_t err = cudaHostAlloc((void**)&pusch->llr,(132 * 3 * 8448 )*sizeof(int16_t),cudaHostAllocMapped); // 132 segments 8448*3 coded bits per segment
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr): %s\n",cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&pusch->llr_dev,pusch->llr,0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq_f_dev): %s\n",cudaGetErrorString(err));
|
||||
#else
|
||||
pusch->llr = (int16_t *)malloc16_clear((132 * 3 * 8448) * sizeof(int16_t)); //132 segments 3*8448 coded bits per segment
|
||||
#endif
|
||||
pusch->ul_valid_re_per_slot = (int16_t *)malloc16_clear(sizeof(int16_t) * fp->symbols_per_slot);
|
||||
} // ulsch_id
|
||||
}
|
||||
@@ -244,7 +256,11 @@ void phy_free_nr_gNB(PHY_VARS_gNB *gNB)
|
||||
free_and_zero(pusch_vars->ul_valid_re_per_slot);
|
||||
free_and_zero(pusch_vars->rxdataF_comp);
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaFreeHost(pusch_vars->llr_dev);
|
||||
#else
|
||||
free_and_zero(pusch_vars->llr);
|
||||
#endif
|
||||
} // ULSCH_id
|
||||
free(gNB->pusch_vars);
|
||||
|
||||
@@ -376,7 +392,7 @@ static void init_DLSCH_struct(PHY_VARS_gNB *gNB)
|
||||
gNB->dlsch = calloc(gNB->max_nb_pdsch, sizeof(*gNB->dlsch));
|
||||
for (int i = 0; i < gNB->max_nb_pdsch; i++) {
|
||||
LOG_D(PHY, "Allocating Transport Channel Buffers for DLSCH %d/%d\n", i, gNB->max_nb_pdsch);
|
||||
gNB->dlsch[i] = new_gNB_dlsch(fp, grid_size);
|
||||
gNB->dlsch[i] = new_gNB_dlsch(fp, grid_size,gNB->use_gpu);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -386,7 +402,7 @@ static void destroy_DLSCH_struct(const PHY_VARS_gNB *gNB)
|
||||
const nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
|
||||
const uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
|
||||
for (int i = 0; i < gNB->max_nb_pdsch; i++) {
|
||||
free_gNB_dlsch(&gNB->dlsch[i], grid_size, fp);
|
||||
free_gNB_dlsch(&gNB->dlsch[i], grid_size, fp,gNB->use_gpu);
|
||||
}
|
||||
free(gNB->dlsch);
|
||||
}
|
||||
|
||||
@@ -18,6 +18,10 @@
|
||||
#include "SCHED_NR_UE/harq_nr.h"
|
||||
#include "nr-uesoftmodem.h"
|
||||
#include "common/config/config_userapi.h"
|
||||
#ifdef ENABLE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
void RCconfig_nrUE_prs(void *cfg)
|
||||
{
|
||||
@@ -261,6 +265,18 @@ int init_nr_ue_signal(PHY_VARS_NR_UE *ue, int nb_connected_gNB)
|
||||
if (IS_SA_MODE(get_softmodem_params()))
|
||||
ue->received_config_request = false;
|
||||
|
||||
ue->use_gpu = use_gpu;
|
||||
#ifdef ENABLE_CUDA
|
||||
for (int j=0;j<10;j++) {
|
||||
for (int i=0;i<2;i++) {
|
||||
cudaError_t err = cudaHostAlloc((void**)&ue->llr[j][i],(66 * 3 * 8448) * sizeof(int16_t),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr): %s\n",cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&ue->llr_dev[j][i],ue->llr[j][i],0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr_dev): %s\n",cudaGetErrorString(err));
|
||||
|
||||
}
|
||||
}
|
||||
#endif
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -316,6 +332,14 @@ void term_nr_ue_signal(PHY_VARS_NR_UE *ue)
|
||||
free_and_zero(ue->prs_vars[idx]);
|
||||
}
|
||||
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
for (int j=0;j<10;j++) {
|
||||
cudaFreeHost(ue->llr[j][0]);
|
||||
cudaFreeHost(ue->llr[j][1]);
|
||||
}
|
||||
#endif
|
||||
|
||||
sl_ue_free(ue);
|
||||
}
|
||||
|
||||
@@ -330,13 +354,15 @@ void free_nr_ue_dl_harq(NR_DL_UE_HARQ_t harq_list[2][NR_MAX_HARQ_PROCESSES], int
|
||||
for (int j=0; j < 2; j++) {
|
||||
for (int i=0; i<number_of_processes; i++) {
|
||||
|
||||
for (int r=0; r<a_segments; r++) {
|
||||
free_and_zero(harq_list[j][i].c[r]);
|
||||
free_and_zero(harq_list[j][i].d[r]);
|
||||
}
|
||||
free_and_zero(harq_list[j][i].b);
|
||||
free_and_zero(harq_list[j][i].c);
|
||||
free_and_zero(harq_list[j][i].d);
|
||||
free_and_zero(harq_list[j][i].b);
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaFreeHost(harq_list[j][i].c);
|
||||
#else
|
||||
free_and_zero(harq_list[j][i].c);
|
||||
#endif
|
||||
free_and_zero(harq_list[j][i].d);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -385,15 +411,18 @@ void nr_init_dl_harq_processes(NR_DL_UE_HARQ_t harq_list[2][NR_MAX_HARQ_PROCESSE
|
||||
memset(harq_list[j] + i, 0, sizeof(NR_DL_UE_HARQ_t));
|
||||
init_downlink_harq_status(harq_list[j] + i);
|
||||
|
||||
const int sz=3*8448*sizeof(int16_t);
|
||||
harq_list[j][i].b = malloc16_clear(a_segments * 1056);
|
||||
harq_list[j][i].c = malloc16(a_segments*sizeof(uint8_t *));
|
||||
harq_list[j][i].d = malloc16(a_segments*sizeof(int16_t *));
|
||||
const int sz=5*8448*sizeof(int16_t);
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaError_t err = cudaHostAlloc((void**)&harq_list[j][i].c,a_segments*sizeof(uint8_t *)*1056,cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq.c): %s\n",cudaGetErrorString(err));
|
||||
err = cudaHostGetDevicePointer((void**)&harq_list[j][i].cdev,(void*)harq_list[j][i].c,0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (harq.cdev): %s\n",cudaGetErrorString(err));
|
||||
#else
|
||||
harq_list[j][i].c = malloc16(a_segments*sizeof(uint8_t *)*1056);
|
||||
#endif
|
||||
harq_list[j][i].d = malloc16(a_segments*sizeof(int16_t *)*sz);
|
||||
init_abort(&harq_list[j][i].abort_decode);
|
||||
for (int r=0; r<a_segments; r++) {
|
||||
harq_list[j][i].c[r] = malloc16_clear(1056);
|
||||
harq_list[j][i].d[r] = malloc16_clear(sz);
|
||||
}
|
||||
harq_list[j][i].status = 0;
|
||||
harq_list[j][i].DLround = 0;
|
||||
}
|
||||
|
||||
@@ -741,9 +741,11 @@ void nr_generate_pdsch(PHY_VARS_gNB *gNB, int n_dlsch, NR_gNB_DLSCH_t *dlsch_arr
|
||||
{
|
||||
time_stats_t *dlsch_encoding_stats = &gNB->dlsch_encoding_stats;
|
||||
time_stats_t *tinput = &gNB->tinput;
|
||||
time_stats_t *tinput_memcpy = &gNB->tinput_memcpy;
|
||||
time_stats_t *tprep = &gNB->tprep;
|
||||
time_stats_t *tparity = &gNB->tparity;
|
||||
time_stats_t *toutput = &gNB->toutput;
|
||||
time_stats_t *tconcat = &gNB->tconcat;
|
||||
time_stats_t *dlsch_rate_matching_stats = &gNB->dlsch_rate_matching_stats;
|
||||
time_stats_t *dlsch_interleaving_stats = &gNB->dlsch_interleaving_stats;
|
||||
time_stats_t *dlsch_segmentation_stats = &gNB->dlsch_segmentation_stats;
|
||||
@@ -799,9 +801,11 @@ void nr_generate_pdsch(PHY_VARS_gNB *gNB, int n_dlsch, NR_gNB_DLSCH_t *dlsch_arr
|
||||
slot,
|
||||
output,
|
||||
tinput,
|
||||
tinput_memcpy,
|
||||
tprep,
|
||||
tparity,
|
||||
toutput,
|
||||
tconcat,
|
||||
dlsch_rate_matching_stats,
|
||||
dlsch_interleaving_stats,
|
||||
dlsch_segmentation_stats)
|
||||
|
||||
@@ -20,9 +20,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
|
||||
uint8_t slot,
|
||||
unsigned char *output,
|
||||
time_stats_t *tinput,
|
||||
time_stats_t *tinput_memcpy,
|
||||
time_stats_t *tprep,
|
||||
time_stats_t *tparity,
|
||||
time_stats_t *toutput,
|
||||
time_stats_t *tconcat,
|
||||
time_stats_t *dlsch_rate_matching_stats,
|
||||
time_stats_t *dlsch_interleaving_stats,
|
||||
time_stats_t *dlsch_segmentation_stats);
|
||||
|
||||
@@ -20,10 +20,13 @@
|
||||
#include <syscall.h>
|
||||
#include <openair2/UTIL/OPT/opt.h>
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
// #define DEBUG_DLSCH_CODING
|
||||
// #define DEBUG_DLSCH_FREE 1
|
||||
|
||||
void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms)
|
||||
void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms, int use_gpumem)
|
||||
{
|
||||
int max_layers = (frame_parms->nb_antennas_tx < NR_MAX_NB_LAYERS) ? frame_parms->nb_antennas_tx : NR_MAX_NB_LAYERS;
|
||||
uint16_t a_segments = MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * max_layers;
|
||||
@@ -34,7 +37,12 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM
|
||||
}
|
||||
|
||||
if (dlsch->b) {
|
||||
free16(dlsch->b, a_segments * 1056);
|
||||
#ifdef ENABLE_CUDA
|
||||
if (use_gpumem)
|
||||
cudaFreeHost(dlsch->b);
|
||||
else
|
||||
#endif
|
||||
free16(dlsch->b, a_segments * 1056);
|
||||
dlsch->b = NULL;
|
||||
}
|
||||
if (dlsch->f) {
|
||||
@@ -48,7 +56,7 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM
|
||||
free(dlsch->c);
|
||||
}
|
||||
|
||||
NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
|
||||
NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB, int use_gpumem)
|
||||
{
|
||||
int max_layers = (frame_parms->nb_antennas_tx < NR_MAX_NB_LAYERS) ? frame_parms->nb_antennas_tx : NR_MAX_NB_LAYERS;
|
||||
uint16_t a_segments = MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * max_layers; // number of segments to be allocated
|
||||
@@ -58,7 +66,7 @@ NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
|
||||
a_segments = a_segments / 273 + 1;
|
||||
}
|
||||
|
||||
LOG_D(PHY, "Allocating %d segments (MAX %d, N_PRB %d)\n", a_segments, MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER, N_RB);
|
||||
LOG_D(PHY, "Allocating %d segments (MAX %d, N_PRB %d) use_gpumem %d\n", a_segments, MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER, N_RB, use_gpumem);
|
||||
uint32_t dlsch_bytes = a_segments * 1056; // allocated bytes per segment
|
||||
NR_gNB_DLSCH_t dlsch = {0};
|
||||
|
||||
@@ -67,14 +75,31 @@ NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
|
||||
bzero(dlsch.b, dlsch_bytes);
|
||||
|
||||
dlsch.c = (uint8_t **)malloc16(a_segments * sizeof(uint8_t *));
|
||||
#ifdef ENABLE_CUDA
|
||||
cudaError_t err=cudaHostAlloc((void**)&dlsch.c_devh,a_segments*sizeof(uint8_t*),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c_devh): %s\n", cudaGetErrorString(err));
|
||||
err=cudaHostGetDevicePointer((void**)&dlsch.c_dev,(void*)dlsch.c_devh,0);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c_dev): %s\n", cudaGetErrorString(err));
|
||||
#endif
|
||||
for (int r = 0; r < a_segments; r++) {
|
||||
// account for filler in first segment and CRCs for multiple segment case
|
||||
// [hna] 8448 is the maximum CB size in NR
|
||||
// 68*348 = 68*(maximum size of Zc)
|
||||
// In section 5.3.2 in 38.212, the for loop is up to N + 2*Zc (maximum size of N is 66*Zc, therefore 68*Zc)
|
||||
dlsch.c[r] = malloc16(8448);
|
||||
AssertFatal(dlsch.c[r], "cannot allocate dlsch.c[%d]\n", r);
|
||||
bzero(dlsch.c[r], 8448);
|
||||
#ifdef ENABLE_CUDA
|
||||
if (use_gpumem) {
|
||||
err=cudaHostAlloc((void**)&dlsch.c[r],(8448/8)*sizeof(uint8_t),cudaHostAllocMapped);
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c[%d]): %s\n", r,cudaGetErrorString(err));
|
||||
uint8_t *tmpcr;
|
||||
err=cudaHostGetDevicePointer((void**)&tmpcr, (void*)dlsch.c[r], 0);
|
||||
((uint8_t**)dlsch.c_devh)[r]=tmpcr;
|
||||
AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) dlsch->c_devh[%d]: %s\n", r,cudaGetErrorString(err));
|
||||
}
|
||||
else
|
||||
#endif
|
||||
dlsch.c[r] = malloc16(8448/8);
|
||||
AssertFatal(dlsch.c[r], "cannot allocate dlsch->c[%d]\n", r);
|
||||
bzero(dlsch.c[r], 8448/8);
|
||||
}
|
||||
|
||||
dlsch.f = malloc16(N_RB * frame_parms->symbols_per_slot * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
|
||||
@@ -91,9 +116,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
|
||||
uint8_t slot,
|
||||
unsigned char *output,
|
||||
time_stats_t *tinput,
|
||||
time_stats_t *tinput_memcpy,
|
||||
time_stats_t *tprep,
|
||||
time_stats_t *tparity,
|
||||
time_stats_t *toutput,
|
||||
time_stats_t *tconcat,
|
||||
time_stats_t *dlsch_rate_matching_stats,
|
||||
time_stats_t *dlsch_interleaving_stats,
|
||||
time_stats_t *dlsch_segmentation_stats)
|
||||
@@ -215,6 +242,9 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
|
||||
TB_parameters->nb_layers = rel15->nrOfLayers;
|
||||
TB_parameters->rv_index = rel15->rvIndex[0];
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
if (gNB->use_gpu) TB_parameters->c_dev = (uint8_t**)dlsch->c_dev;
|
||||
#endif
|
||||
int nb_re_dmrs =
|
||||
(rel15->dmrsConfigType == NFAPI_NR_DMRS_TYPE1) ? (6 * rel15->numDmrsCdmGrpsNoData) : (4 * rel15->numDmrsCdmGrpsNoData);
|
||||
TB_parameters->G = nr_get_G(rel15->rbSize,
|
||||
|
||||
@@ -74,9 +74,9 @@ void nr_generate_pbch(PHY_VARS_gNB *gNB,
|
||||
void nr_init_pbch_interleaver(uint8_t *interleaver);
|
||||
uint32_t nr_pbch_extra_byte_generation(int sfn, int n_hf, int ssb_index, int ssb_sc_offset, int Lmax);
|
||||
|
||||
NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB);
|
||||
NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB,int use_gpu);
|
||||
|
||||
void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms);
|
||||
void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms,int use_gpu);
|
||||
|
||||
/** \brief This function is the top-level entry point to PUSCH demodulation, after frequency-domain transformation and channel estimation. It performs
|
||||
- RB extraction (signal and channel estimates)
|
||||
|
||||
@@ -46,10 +46,6 @@ void free_gNB_ulsch(NR_gNB_ULSCH_t *ulsch, uint16_t N_RB_UL)
|
||||
free_and_zero(ulsch->harq_process->b);
|
||||
ulsch->harq_process->b = NULL;
|
||||
}
|
||||
for (int r = 0; r < a_segments; r++) {
|
||||
free_and_zero(ulsch->harq_process->c[r]);
|
||||
free_and_zero(ulsch->harq_process->d[r]);
|
||||
}
|
||||
free_and_zero(ulsch->harq_process->c);
|
||||
free_and_zero(ulsch->harq_process->d);
|
||||
free_and_zero(ulsch->harq_process->d_to_be_cleared);
|
||||
@@ -78,12 +74,9 @@ NR_gNB_ULSCH_t new_gNB_ulsch(uint8_t max_ldpc_iterations, uint16_t N_RB_UL)
|
||||
init_abort(&harq->abort_decode);
|
||||
ulsch.harq_process = harq;
|
||||
harq->b = malloc16_clear(ulsch_bytes * sizeof(*harq->b));
|
||||
harq->c = malloc16_clear(a_segments * sizeof(*harq->c));
|
||||
harq->d = malloc16_clear(a_segments * sizeof(*harq->d));
|
||||
for (int r = 0; r < a_segments; r++) {
|
||||
harq->c[r] = malloc16_clear(8448 * sizeof(*harq->c[r]));
|
||||
harq->d[r] = malloc16_clear(68 * 384 * sizeof(*harq->d[r]));
|
||||
}
|
||||
// Allocate one contiguous buffer fr all c/d arrays to simplify addressing for GPU LDPC offload
|
||||
harq->c = malloc16_clear(a_segments * 8448 * sizeof(*harq->c));
|
||||
harq->d = malloc16_clear(a_segments * 64 * 384 * sizeof(*harq->d));
|
||||
harq->d_to_be_cleared = calloc(a_segments, sizeof(bool));
|
||||
AssertFatal(harq->d_to_be_cleared != NULL, "out of memory\n");
|
||||
return (ulsch);
|
||||
@@ -112,6 +105,7 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
|
||||
NR_gNB_PUSCH *pusch = &phy_vars_gNB->pusch_vars[ULSCH_id];
|
||||
NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
|
||||
const nfapi_nr_pusch_pdu_t *pusch_pdu = &harq_process->ulsch_pdu;
|
||||
uint8_t harq_pid = ulsch->harq_pid;
|
||||
|
||||
nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pusch_id];
|
||||
|
||||
@@ -135,7 +129,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
|
||||
|
||||
|
||||
// The harq_pid is not unique among the active HARQ processes in the instance so we use ULSCH_id instead
|
||||
TB_parameters->harq_unique_pid = ULSCH_id;
|
||||
TB_parameters->harq_unique_pid = (phy_vars_gNB->max_nb_pusch * harq_pid) + ULSCH_id;
|
||||
AssertFatal(TB_parameters->harq_unique_pid < (16*phy_vars_gNB->max_nb_pusch),"harq_unique_pid >= %d, harq_pid %d, ULSCH_id %d\n",16*phy_vars_gNB->max_nb_pusch,harq_pid,ULSCH_id);
|
||||
|
||||
// ------------------------------------------------------------------
|
||||
TB_parameters->nb_rb = pusch_pdu->rb_size;
|
||||
@@ -163,7 +158,6 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
|
||||
}
|
||||
}
|
||||
|
||||
uint8_t harq_pid = ulsch->harq_pid;
|
||||
LOG_D(PHY,
|
||||
"ULSCH Decoding, harq_pid %d rnti %x TBS %d G %d mcs %d Nl %d nb_rb %d, Qm %d, Coderate %f RV %d round %d new RX %d\n",
|
||||
harq_pid,
|
||||
@@ -222,14 +216,15 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
|
||||
set_abort(&harq_process->abort_decode, false);
|
||||
}
|
||||
|
||||
nrLDPC_segment_decoding_parameters_t segments[nb_pusch][max_num_segments];
|
||||
memset(segments, 0, sizeof(segments));
|
||||
|
||||
for (uint8_t pusch_id = 0; pusch_id < nb_pusch; pusch_id++) {
|
||||
uint8_t ULSCH_id = ULSCH_ids[pusch_id];
|
||||
NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id];
|
||||
NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
|
||||
short *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr;
|
||||
#ifdef ENABLE_CUDA
|
||||
int16_t *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr_dev;
|
||||
#else
|
||||
int16_t *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr;
|
||||
#endif
|
||||
|
||||
if (!ulsch_llr) {
|
||||
LOG_E(PHY, "ulsch_decoding.c: NULL ulsch_llr pointer\n");
|
||||
@@ -237,66 +232,78 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
|
||||
}
|
||||
|
||||
nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pusch_id];
|
||||
TB_parameters->segments = segments[pusch_id];
|
||||
|
||||
uint32_t r_offset = 0;
|
||||
TB_parameters->llr = ulsch_llr;
|
||||
TB_parameters->d = harq_process->d;
|
||||
TB_parameters->c = harq_process->c;
|
||||
TB_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, TB_parameters->nb_layers, 0);
|
||||
TB_parameters->first_rE2 = TB_parameters->C;
|
||||
TB_parameters->E2 = TB_parameters->E;
|
||||
TB_parameters->R = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
TB_parameters->E,
|
||||
TB_parameters->BG,
|
||||
TB_parameters->Z,
|
||||
&harq_process->llrLen,
|
||||
harq_process->round);
|
||||
for (int r = 0; r < TB_parameters->C; r++) TB_parameters->decodeSuccess[r] = false;
|
||||
TB_parameters->d_to_be_cleared = harq_process->harq_to_be_cleared;
|
||||
reset_meas(&TB_parameters->ts_deinterleave);
|
||||
reset_meas(&TB_parameters->ts_rate_unmatch);
|
||||
reset_meas(&TB_parameters->ts_seg_prep);
|
||||
reset_meas(&TB_parameters->ts_ldpc_decode);
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
nrLDPC_segment_decoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
|
||||
segment_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, TB_parameters->nb_layers, r);
|
||||
segment_parameters->R = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
segment_parameters->E,
|
||||
TB_parameters->BG,
|
||||
TB_parameters->Z,
|
||||
&harq_process->llrLen,
|
||||
harq_process->round);
|
||||
segment_parameters->llr = ulsch_llr + r_offset;
|
||||
segment_parameters->d = harq_process->d[r];
|
||||
segment_parameters->d_to_be_cleared = &harq_process->d_to_be_cleared[r];
|
||||
segment_parameters->c = harq_process->c[r];
|
||||
segment_parameters->decodeSuccess = false;
|
||||
|
||||
reset_meas(&segment_parameters->ts_deinterleave);
|
||||
reset_meas(&segment_parameters->ts_rate_unmatch);
|
||||
reset_meas(&segment_parameters->ts_ldpc_decode);
|
||||
|
||||
r_offset += segment_parameters->E;
|
||||
}
|
||||
if (harq_process->harq_to_be_cleared) {
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
harq_process->d_to_be_cleared[r] = true;
|
||||
int Etmp = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, TB_parameters->nb_layers, r);
|
||||
if (TB_parameters->E != Etmp) {
|
||||
TB_parameters->E2 = Etmp;
|
||||
TB_parameters->R2 = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
TB_parameters->E2,
|
||||
TB_parameters->BG,
|
||||
TB_parameters->Z,
|
||||
&harq_process->llrLen,
|
||||
harq_process->round);
|
||||
TB_parameters->first_rE2 = r;
|
||||
break;
|
||||
}
|
||||
harq_process->harq_to_be_cleared = false;
|
||||
}
|
||||
}
|
||||
|
||||
int ret_decoder = phy_vars_gNB->nrLDPC_coding_interface.nrLDPC_coding_decoder(&slot_parameters);
|
||||
|
||||
// post decode
|
||||
for (uint8_t pusch_id = 0; pusch_id < nb_pusch; pusch_id++) {
|
||||
uint8_t ULSCH_id = ULSCH_ids[pusch_id];
|
||||
NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id];
|
||||
NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
|
||||
|
||||
nrLDPC_TB_decoding_parameters_t TB_parameters = TBs[pusch_id];
|
||||
nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pusch_id];
|
||||
|
||||
uint32_t offset = 0;
|
||||
for (int r = 0; r < TB_parameters.C; r++) {
|
||||
nrLDPC_segment_decoding_parameters_t nrLDPC_segment_decoding_parameters = TB_parameters.segments[r];
|
||||
// Copy c to b in case of decoding success
|
||||
if (nrLDPC_segment_decoding_parameters.decodeSuccess) {
|
||||
memcpy(harq_process->b + offset,
|
||||
harq_process->c[r],
|
||||
(harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
} else {
|
||||
LOG_D(PHY, "uplink segment error %d/%d\n", r, harq_process->C);
|
||||
LOG_D(PHY, "ULSCH %d in error\n", ULSCH_id);
|
||||
}
|
||||
offset += ((harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
|
||||
merge_meas(&phy_vars_gNB->ts_deinterleave, &nrLDPC_segment_decoding_parameters.ts_deinterleave);
|
||||
merge_meas(&phy_vars_gNB->ts_rate_unmatch, &nrLDPC_segment_decoding_parameters.ts_rate_unmatch);
|
||||
merge_meas(&phy_vars_gNB->ts_ldpc_decode, &nrLDPC_segment_decoding_parameters.ts_ldpc_decode);
|
||||
uint32_t offset = 0,r_offset=0;
|
||||
bool crcok=true;
|
||||
LOG_D(PHY,"C = %d\n",TB_parameters->C);
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
LOG_D(PHY,"Segment %d %d\n",r,TB_parameters->decodeSuccess[r]);
|
||||
if (TB_parameters->decodeSuccess[r]==false) {
|
||||
LOG_D(PHY,"Segment %d/%d in error\n",r,TB_parameters->C);
|
||||
crcok=false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (crcok) {
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
// Copy c to b in case of decoding success
|
||||
memcpy(harq_process->b + offset,
|
||||
harq_process->c + r_offset,
|
||||
(harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
offset += ((harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
r_offset += (harq_process->K >> 3);
|
||||
}
|
||||
} else {
|
||||
LOG_D(PHY, "ULSCH %d in error\n", ULSCH_id);
|
||||
}
|
||||
merge_meas(&phy_vars_gNB->ts_deinterleave, &TB_parameters->ts_deinterleave);
|
||||
merge_meas(&phy_vars_gNB->ts_rate_unmatch, &TB_parameters->ts_rate_unmatch);
|
||||
merge_meas(&phy_vars_gNB->ts_seg_prep, &TB_parameters->ts_seg_prep);
|
||||
merge_meas(&phy_vars_gNB->ts_ldpc_decode, &TB_parameters->ts_ldpc_decode);
|
||||
harq_process->harq_to_be_cleared = false;
|
||||
}
|
||||
|
||||
return ret_decoder;
|
||||
|
||||
@@ -19,6 +19,10 @@
|
||||
#include "PHY/log_tools.h"
|
||||
|
||||
|
||||
#ifndef __AVX2__
|
||||
#define USE128BIT
|
||||
#endif
|
||||
|
||||
#if T_TRACER
|
||||
static void copy_c16_data_to_slot_memory(c16_t *src, c16_t *dst_slot, int nb_re_pusch, int symbol)
|
||||
{
|
||||
@@ -176,6 +180,41 @@ static int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, const nfapi_nr_pusch
|
||||
} else return(rel15_ul->rb_size * NR_NB_SC_PER_RB);
|
||||
}
|
||||
|
||||
#ifdef __AVX512BW__
|
||||
#define s16 __m512i
|
||||
#define setzero _mm512_setzero_si512
|
||||
#define set1_s16 _mm512_set1_epi16
|
||||
#define cpx_mult_conj_s16 oai_mm512_cpx_mult_conj
|
||||
#define adds_s16 _mm512_adds_epi16
|
||||
#define smadd_s16 oai_mm512_smadd
|
||||
#define packs_s16 _mm512_packs_epi32
|
||||
#define unpacklo_s16 _mm512_unpacklo_epi16
|
||||
#define mulhrs_s16 _mm512_mulhrs_epi16
|
||||
#define COMPSHIFT 4
|
||||
#elif defined(__AVX2__)
|
||||
#define s16 simde__m256i
|
||||
#define setzero simde_mm256_setzero_si256
|
||||
#define set1_s16 simde_mm256_set1_epi16
|
||||
#define cpx_mult_conj_s16 oai_mm256_cpx_mult_conj
|
||||
#define adds_s16 simde_mm256_adds_epi16
|
||||
#define smadd_s16 oai_mm256_smadd
|
||||
#define packs_s16 simde_mm256_packs_epi32
|
||||
#define unpacklo_s16 simde_mm256_unpacklo_epi16
|
||||
#define mulhrs_s16 simde_mm256_mulhrs_epi16
|
||||
#define COMPSHIFT 3
|
||||
#else
|
||||
#define s16 simde__m128i
|
||||
#define setzero simde_mm_setzero_si128
|
||||
#define set1_s16 simde_mm_set1_epi16
|
||||
#define cpx_mult_conj_s16 oai_mm_cpx_mult_conj
|
||||
#define adds_s16 simde_mm_adds_epi16
|
||||
#define smadd_s16 oai_mm_smadd
|
||||
#define packs_s16 simde_mm_packs_epi32
|
||||
#define unpacklo_s16 simde_mm_unpacklo_epi16
|
||||
#define mulhrs_s16 simde_mm_mulhrs_epi16
|
||||
#define COMPSHIFT 2
|
||||
#endif
|
||||
#ifndef __aarch64__
|
||||
static void nr_ulsch_channel_compensation(uint32_t buffer_length,
|
||||
int nb_rx_ant,
|
||||
c16_t rxFext[][buffer_length],
|
||||
@@ -193,63 +232,60 @@ static void nr_ulsch_channel_compensation(uint32_t buffer_length,
|
||||
int mod_order = rel15_ul->qam_mod_order;
|
||||
int nrOfLayers = rel15_ul->nrOfLayers;
|
||||
|
||||
simde__m256i QAM_ampa_256 = simde_mm256_setzero_si256();
|
||||
simde__m256i QAM_ampb_256 = simde_mm256_setzero_si256();
|
||||
simde__m256i QAM_ampc_256 = simde_mm256_setzero_si256();
|
||||
s16 QAM_ampa = setzero();
|
||||
s16 QAM_ampb = setzero();
|
||||
s16 QAM_ampc = setzero();
|
||||
|
||||
if (mod_order == 4) {
|
||||
QAM_ampa_256 = simde_mm256_set1_epi16(QAM16_n1);
|
||||
QAM_ampb_256 = simde_mm256_setzero_si256();
|
||||
QAM_ampc_256 = simde_mm256_setzero_si256();
|
||||
QAM_ampa = set1_s16(QAM16_n1);
|
||||
}
|
||||
else if (mod_order == 6) {
|
||||
QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1);
|
||||
QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2);
|
||||
QAM_ampc_256 = simde_mm256_setzero_si256();
|
||||
QAM_ampa = set1_s16(QAM64_n1);
|
||||
QAM_ampb = set1_s16(QAM64_n2);
|
||||
}
|
||||
else if (mod_order == 8) {
|
||||
QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1);
|
||||
QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2);
|
||||
QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3);
|
||||
QAM_ampa = set1_s16(QAM256_n1);
|
||||
QAM_ampb = set1_s16(QAM256_n2);
|
||||
QAM_ampc = set1_s16(QAM256_n3);
|
||||
}
|
||||
|
||||
for (int aatx = 0; aatx < nrOfLayers; aatx++) {
|
||||
simde__m256i *rxComp_256 = (simde__m256i *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];
|
||||
simde__m256i *rxF_ch_maga_256 = (simde__m256i *)ul_ch_maga[aatx];
|
||||
simde__m256i *rxF_ch_magb_256 = (simde__m256i *)ul_ch_magb[aatx];
|
||||
simde__m256i *rxF_ch_magc_256 = (simde__m256i *)ul_ch_magc[aatx];
|
||||
s16 *rxComp_s16 = (s16 *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];
|
||||
s16 *rxF_ch_maga = (s16 *)ul_ch_maga[aatx];
|
||||
s16 *rxF_ch_magb = (s16 *)ul_ch_magb[aatx];
|
||||
s16 *rxF_ch_magc = (s16 *)ul_ch_magc[aatx];
|
||||
for (int aarx = 0; aarx < nb_rx_ant; aarx++) {
|
||||
simde__m256i *rxF_256 = (simde__m256i *)rxFext[aarx];
|
||||
simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx];
|
||||
s16 *rxF = (s16 *)rxFext[aarx];
|
||||
s16 *chF = (s16 *)chFext[aatx][aarx];
|
||||
|
||||
for (int i = 0; i < buffer_length >> 3; i++)
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++)
|
||||
{
|
||||
// MRC
|
||||
simde__m256i comp = oai_mm256_cpx_mult_conj(chF_256[i], rxF_256[i], output_shift);
|
||||
rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], comp);
|
||||
s16 comp = cpx_mult_conj_s16(chF[i], rxF[i], output_shift);
|
||||
rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);
|
||||
|
||||
if (mod_order > 2) {
|
||||
simde__m256i mag = oai_mm256_smadd(chF_256[i], chF_256[i], output_shift); // |h|^2
|
||||
s16 mag = smadd_s16(chF[i], chF[i], output_shift); // |h|^2
|
||||
// pack and duplicate
|
||||
mag = simde_mm256_packs_epi32(mag, mag);
|
||||
mag = simde_mm256_unpacklo_epi16(mag, mag);
|
||||
mag = packs_s16(mag, mag);
|
||||
mag = unpacklo_s16(mag, mag);
|
||||
|
||||
rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampa_256));
|
||||
rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));
|
||||
|
||||
if (mod_order > 4)
|
||||
rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampb_256));
|
||||
rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));
|
||||
|
||||
if (mod_order > 6)
|
||||
rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampc_256));
|
||||
rxF_ch_magc[i] = adds_s16(rxF_ch_magc[i], mulhrs_s16(mag, QAM_ampc));
|
||||
}
|
||||
}
|
||||
if (nb_layers > 1) {
|
||||
for (int atx = 0; atx < nrOfLayers; atx++) {
|
||||
simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx];
|
||||
simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx];
|
||||
simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx];
|
||||
for (int i = 0; i < buffer_length >> 3; i++) {
|
||||
rho_256[i] = simde_mm256_adds_epi16(rho_256[i], oai_mm256_cpx_mult_conj(chF_256[i], chF2_256[i], output_shift));
|
||||
s16 *rho_s16 = (s16 *)rho[aatx][atx];
|
||||
s16 *chF = (s16 *)chFext[aatx][aarx];
|
||||
s16 *chF2 = (s16 *)chFext[atx][aarx];
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) {
|
||||
rho_s16[i] = adds_s16(rho_s16[i], cpx_mult_conj_s16(chF[i], chF2[i], output_shift));
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -257,7 +293,159 @@ static void nr_ulsch_channel_compensation(uint32_t buffer_length,
|
||||
}
|
||||
|
||||
}
|
||||
#else
|
||||
#define COMP_SHIFT(SHIFT) \
|
||||
static void nr_ulsch_channel_compensation##SHIFT(uint32_t buffer_length,\
|
||||
int nb_rx_ant,\
|
||||
c16_t rxFext[][buffer_length],\
|
||||
c16_t chFext[][nb_rx_ant][buffer_length],\
|
||||
c16_t ul_ch_maga[][buffer_length],\
|
||||
c16_t ul_ch_magb[][buffer_length],\
|
||||
c16_t ul_ch_magc[][buffer_length],\
|
||||
c16_t **rxComp,\
|
||||
int nb_layers,\
|
||||
c16_t rho[][nb_layers][buffer_length],\
|
||||
const nfapi_nr_pusch_pdu_t *rel15_ul,\
|
||||
uint32_t symbol)\
|
||||
{\
|
||||
int mod_order = rel15_ul->qam_mod_order;\
|
||||
int nrOfLayers = rel15_ul->nrOfLayers;\
|
||||
\
|
||||
s16 QAM_ampa = setzero();\
|
||||
s16 QAM_ampb = setzero();\
|
||||
s16 QAM_ampc = setzero();\
|
||||
\
|
||||
if (mod_order == 4) {\
|
||||
QAM_ampa = set1_s16(QAM16_n1);\
|
||||
}\
|
||||
else if (mod_order == 6) {\
|
||||
QAM_ampa = set1_s16(QAM64_n1);\
|
||||
QAM_ampb = set1_s16(QAM64_n2);\
|
||||
}\
|
||||
else if (mod_order == 8) {\
|
||||
QAM_ampa = set1_s16(QAM256_n1);\
|
||||
QAM_ampb = set1_s16(QAM256_n2);\
|
||||
QAM_ampc = set1_s16(QAM256_n3);\
|
||||
}\
|
||||
\
|
||||
for (int aatx = 0; aatx < nrOfLayers; aatx++) {\
|
||||
s16 *rxComp_s16 = (s16 *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];\
|
||||
s16 *rxF_ch_maga = (s16 *)ul_ch_maga[aatx];\
|
||||
s16 *rxF_ch_magb = (s16 *)ul_ch_magb[aatx];\
|
||||
s16 *rxF_ch_magc = (s16 *)ul_ch_magc[aatx];\
|
||||
for (int aarx = 0; aarx < nb_rx_ant; aarx++) {\
|
||||
s16 *rxF = (s16 *)rxFext[aarx];\
|
||||
s16 *chF = (s16 *)chFext[aatx][aarx];\
|
||||
\
|
||||
if (mod_order==8) {\
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
|
||||
{\
|
||||
s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
|
||||
rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
|
||||
s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
|
||||
mag = packs_s16(mag, mag);\
|
||||
mag = unpacklo_s16(mag, mag);\
|
||||
rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
|
||||
rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));\
|
||||
rxF_ch_magc[i] = adds_s16(rxF_ch_magc[i], mulhrs_s16(mag, QAM_ampc));\
|
||||
}\
|
||||
}\
|
||||
else if (mod_order==6){\
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
|
||||
{\
|
||||
s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
|
||||
rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
|
||||
s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
|
||||
mag = packs_s16(mag, mag);\
|
||||
mag = unpacklo_s16(mag, mag);\
|
||||
rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
|
||||
rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));\
|
||||
}\
|
||||
}\
|
||||
else if (mod_order==4){\
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
|
||||
{\
|
||||
s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
|
||||
rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
|
||||
s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
|
||||
mag = packs_s16(mag, mag);\
|
||||
mag = unpacklo_s16(mag, mag);\
|
||||
rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
|
||||
}\
|
||||
}\
|
||||
else if (mod_order==2){\
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
|
||||
{\
|
||||
s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
|
||||
rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
|
||||
}\
|
||||
}\
|
||||
if (nb_layers > 1) {\
|
||||
for (int atx = 0; atx < nrOfLayers; atx++) {\
|
||||
s16 *rho_s16 = (s16 *)rho[aatx][atx];\
|
||||
s16 *chF = (s16 *)chFext[aatx][aarx];\
|
||||
s16 *chF2 = (s16 *)chFext[atx][aarx];\
|
||||
for (int i = 0; i < buffer_length >> COMPSHIFT; i++) {\
|
||||
rho_s16[i] = adds_s16(rho_s16[i], cpx_mult_conj_s16(chF[i], chF2[i], SHIFT));\
|
||||
}\
|
||||
}\
|
||||
}\
|
||||
}\
|
||||
}\
|
||||
\
|
||||
}
|
||||
COMP_SHIFT(1)
|
||||
COMP_SHIFT(2)
|
||||
COMP_SHIFT(3)
|
||||
COMP_SHIFT(4)
|
||||
COMP_SHIFT(5)
|
||||
COMP_SHIFT(6)
|
||||
COMP_SHIFT(7)
|
||||
COMP_SHIFT(8)
|
||||
COMP_SHIFT(9)
|
||||
COMP_SHIFT(10)
|
||||
COMP_SHIFT(11)
|
||||
COMP_SHIFT(12)
|
||||
COMP_SHIFT(13)
|
||||
COMP_SHIFT(14)
|
||||
COMP_SHIFT(15)
|
||||
|
||||
static void nr_ulsch_channel_compensation(uint32_t buffer_length,
|
||||
int nb_rx_ant,
|
||||
c16_t rxFext[][buffer_length],
|
||||
c16_t chFext[][nb_rx_ant][buffer_length],
|
||||
c16_t ul_ch_maga[][buffer_length],
|
||||
c16_t ul_ch_magb[][buffer_length],
|
||||
c16_t ul_ch_magc[][buffer_length],
|
||||
c16_t **rxComp,
|
||||
int nb_layers,
|
||||
c16_t rho[][nb_layers][buffer_length],
|
||||
const nfapi_nr_pusch_pdu_t *rel15_ul,
|
||||
uint32_t symbol,
|
||||
uint32_t output_shift)
|
||||
{
|
||||
switch(output_shift) {
|
||||
// case 0: nr_ulsch_channel_compensation0(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol);
|
||||
case 1: nr_ulsch_channel_compensation1(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 2: nr_ulsch_channel_compensation2(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 3: nr_ulsch_channel_compensation3(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 4: nr_ulsch_channel_compensation4(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 5: nr_ulsch_channel_compensation5(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 6: nr_ulsch_channel_compensation6(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 7: nr_ulsch_channel_compensation7(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 8: nr_ulsch_channel_compensation8(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 9: nr_ulsch_channel_compensation9(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 10: nr_ulsch_channel_compensation10(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 11: nr_ulsch_channel_compensation11(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 12: nr_ulsch_channel_compensation12(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 13: nr_ulsch_channel_compensation13(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 14: nr_ulsch_channel_compensation14(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
case 15: nr_ulsch_channel_compensation15(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
|
||||
default: LOG_E(NR_PHY,"Illegal shift %d\n",output_shift);
|
||||
break;
|
||||
}
|
||||
}
|
||||
#endif
|
||||
// Zero Forcing Rx function: nr_det_HhH()
|
||||
static void nr_ulsch_det_HhH(c16_t *after_mf_00, // a
|
||||
c16_t *after_mf_01, // b
|
||||
@@ -874,8 +1062,8 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
int nb_rx_ant = frame_parms->nb_antennas_rx;
|
||||
int dmrs_symbol_flag = (rel15_ul->ul_dmrs_symb_pos >> symbol) & 0x01;
|
||||
int buffer_length = ceil_mod(rel15_ul->rb_size * NR_NB_SC_PER_RB, 16);
|
||||
c16_t rxFext[nb_rx_ant][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t chFext[nb_layer][nb_rx_ant][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t rxFext[nb_rx_ant][buffer_length] __attribute__((aligned(64)));
|
||||
c16_t chFext[nb_layer][nb_rx_ant][buffer_length] __attribute__((aligned(64)));
|
||||
|
||||
memset(rxFext, 0, sizeof(rxFext));
|
||||
memset(chFext, 0, sizeof(chFext));
|
||||
@@ -889,6 +1077,7 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
|
||||
for (int aarx = 0; aarx < nb_rx_ant; aarx++) {
|
||||
for (int aatx = 0; aatx < nb_layer; aatx++) {
|
||||
start_meas(&gNB->pusch_extraction_stats);
|
||||
nr_ulsch_extract_rbs(rxF[aarx],
|
||||
(c16_t *)pusch_vars->ul_ch_estimates[aatx * nb_rx_ant + aarx],
|
||||
rxFext[aarx],
|
||||
@@ -898,6 +1087,7 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
dmrs_symbol_flag,
|
||||
rel15_ul,
|
||||
frame_parms);
|
||||
stop_meas(&gNB->pusch_extraction_stats);
|
||||
#if T_TRACER
|
||||
int nb_re_pusch = NR_NB_SC_PER_RB * rel15_ul->rb_size;
|
||||
// Assume assume Tx and Rx = 1
|
||||
@@ -910,10 +1100,11 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
#endif
|
||||
}
|
||||
}
|
||||
c16_t rho[nb_layer][nb_layer][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t rxF_ch_maga [nb_layer][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t rxF_ch_magb [nb_layer][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t rxF_ch_magc [nb_layer][buffer_length] __attribute__((aligned(32)));
|
||||
start_meas(&gNB->pusch_channel_compensation_stats);
|
||||
c16_t rho[nb_layer][nb_layer][buffer_length] __attribute__((aligned(64)));
|
||||
c16_t rxF_ch_maga [nb_layer][buffer_length] __attribute__((aligned(64)));
|
||||
c16_t rxF_ch_magb [nb_layer][buffer_length] __attribute__((aligned(64)));
|
||||
c16_t rxF_ch_magc [nb_layer][buffer_length] __attribute__((aligned(64)));
|
||||
|
||||
memset(rho, 0, sizeof(rho));
|
||||
memset(rxF_ch_maga, 0, sizeof(rxF_ch_maga));
|
||||
@@ -935,6 +1126,7 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
rel15_ul,
|
||||
symbol,
|
||||
output_shift);
|
||||
stop_meas(&gNB->pusch_channel_compensation_stats);
|
||||
|
||||
if (nb_layer == 1 && rel15_ul->transform_precoding == transformPrecoder_enabled && rel15_ul->qam_mod_order <= 6) {
|
||||
if (rel15_ul->qam_mod_order > 2)
|
||||
@@ -957,7 +1149,7 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
buffer_length);
|
||||
pusch_vars->ul_valid_re_per_slot[symbol] -= pusch_vars->ptrs_re_per_slot;
|
||||
}
|
||||
|
||||
start_meas(&gNB->ulsch_llr_stats);
|
||||
if (nb_layer == 2) {
|
||||
if (rel15_ul->qam_mod_order <= 6) {
|
||||
nr_ulsch_compute_ML_llr((c16_t *)&pusch_vars->rxdataF_comp[0][symbol * buffer_length],
|
||||
@@ -997,6 +1189,7 @@ static void inner_rx(PHY_VARS_gNB *gNB,
|
||||
pusch_vars->ul_valid_re_per_slot[symbol],
|
||||
symbol,
|
||||
rel15_ul->qam_mod_order);
|
||||
stop_meas(&gNB->ulsch_llr_stats);
|
||||
}
|
||||
|
||||
typedef struct puschSymbolProc_s {
|
||||
@@ -1051,6 +1244,7 @@ static void nr_pusch_symbol_processing(void *arg)
|
||||
|
||||
int nb_re_pusch = pusch_vars->ul_valid_re_per_slot[symbol];
|
||||
// layer de-mapping
|
||||
start_meas(&gNB->ulsch_layer_demapping_stats);
|
||||
int16_t *llr_ptr = llrs[0];
|
||||
if (rel15_ul->nrOfLayers != 1) {
|
||||
llr_ptr = &rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers];
|
||||
@@ -1060,12 +1254,21 @@ static void nr_pusch_symbol_processing(void *arg)
|
||||
llr_ptr[i * rel15_ul->nrOfLayers * rel15_ul->qam_mod_order + l * rel15_ul->qam_mod_order + m] =
|
||||
llrss[l][i * rel15_ul->qam_mod_order + m];
|
||||
}
|
||||
stop_meas(&gNB->ulsch_layer_demapping_stats);
|
||||
// unscrambling
|
||||
start_meas(&gNB->ulsch_unscrambling_stats);
|
||||
int16_t *llr16 = (int16_t*)&rdata->llr[pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers];
|
||||
int16_t *s = rdata->scramblingSequence + pusch_vars->llr_offset[symbol] * rel15_ul->nrOfLayers;
|
||||
const int end = nb_re_pusch * rel15_ul->qam_mod_order * rel15_ul->nrOfLayers;
|
||||
for (int i = 0; i < end; i++)
|
||||
llr16[i] = llr_ptr[i] * s[i];
|
||||
int i=0;
|
||||
for (; (i+8) <= end; i+=8) {
|
||||
simde__m128i llr128 = simde_mm_loadu_si128((simde__m128i *)&llr_ptr[i]);
|
||||
simde__m128i s128 = simde_mm_loadu_si128((simde__m128i *)&s[i]);
|
||||
simde_mm_storeu_si128(llr16+i, simde_mm_mullo_epi16(llr128, s128));
|
||||
}
|
||||
for (;i<end;i++)
|
||||
llr16[i]=llr_ptr[i] * s[i];
|
||||
stop_meas(&gNB->ulsch_unscrambling_stats);
|
||||
}
|
||||
|
||||
// Task running in // completed
|
||||
@@ -1115,13 +1318,13 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
|
||||
int nb_layer = rel15_ul->nrOfLayers;
|
||||
|
||||
// Initialize memory for DMRS signals
|
||||
c16_t pusch_dmrs_slot_mem[nb_layer * buffer_length_slot] __attribute__((aligned(32)));
|
||||
c16_t pusch_dmrs_slot_mem[nb_layer * buffer_length_slot] __attribute__((aligned(64)));
|
||||
// Initialize memory for channel estimates based on DMRS positions
|
||||
c16_t pusch_ch_est_dmrs_pos_slot_mem[buffer_length_slot * nb_layer * nb_rx_ant] __attribute__((aligned(32)));
|
||||
c16_t pusch_ch_est_dmrs_pos_slot_mem[buffer_length_slot * nb_layer * nb_rx_ant] __attribute__((aligned(64)));
|
||||
// memory to store slot grid with channel coefficients based on DMRS positions after interpolation
|
||||
c16_t pusch_ch_est_dmrs_interpl_slot_mem[buffer_length_slot * nb_layer * nb_rx_ant] __attribute__((aligned(32)));
|
||||
c16_t pusch_ch_est_dmrs_interpl_slot_mem[buffer_length_slot * nb_layer * nb_rx_ant] __attribute__((aligned(64)));
|
||||
// memory to store extracted data including PUSCH + DMRS
|
||||
c16_t rxFext_slot_mem[nb_rx_ant * buffer_length_slot] __attribute__((aligned(32)));
|
||||
c16_t rxFext_slot_mem[nb_rx_ant * buffer_length_slot] __attribute__((aligned(64)));
|
||||
|
||||
#if T_TRACER
|
||||
// Initialize memory for DMRS signals
|
||||
@@ -1265,7 +1468,7 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
|
||||
*ret_unav_res = unav_res;
|
||||
|
||||
// initialize scrambling sequence //
|
||||
int16_t scramblingSequence[G + 96] __attribute__((aligned(32)));
|
||||
int16_t scramblingSequence[G + 96] __attribute__((aligned(64)));
|
||||
|
||||
nr_codeword_unscrambling_init(scramblingSequence, G, 0, rel15_ul->data_scrambling_id, rel15_ul->rnti);
|
||||
|
||||
@@ -1292,12 +1495,13 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
|
||||
else // average of channel estimates stored in first symbol
|
||||
dmrs_symbol = get_next_dmrs_symbol_in_slot(rel15_ul->ul_dmrs_symb_pos, rel15_ul->start_symbol_index, end_symbol);
|
||||
int size_est = nb_re_pusch * frame_parms->symbols_per_slot;
|
||||
__attribute__((aligned(32))) int ul_ch_estimates_ext[rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx][size_est];
|
||||
__attribute__((aligned(64))) int ul_ch_estimates_ext[rel15_ul->nrOfLayers * frame_parms->nb_antennas_rx][size_est];
|
||||
memset(ul_ch_estimates_ext, 0, sizeof(ul_ch_estimates_ext));
|
||||
int buffer_length = rel15_ul->rb_size * NR_NB_SC_PER_RB;
|
||||
c16_t temp_rxFext[frame_parms->nb_antennas_rx][buffer_length] __attribute__((aligned(32)));
|
||||
c16_t temp_rxFext[frame_parms->nb_antennas_rx][buffer_length] __attribute__((aligned(64)));
|
||||
for (int aarx = 0; aarx < frame_parms->nb_antennas_rx; aarx++)
|
||||
for (int nl = 0; nl < rel15_ul->nrOfLayers; nl++)
|
||||
for (int nl = 0; nl < rel15_ul->nrOfLayers; nl++) {
|
||||
start_meas(&gNB->pusch_extraction_stats);
|
||||
nr_ulsch_extract_rbs(gNB->common_vars.rxdataF[beam_nb][aarx],
|
||||
(c16_t *)pusch_vars->ul_ch_estimates[nl * frame_parms->nb_antennas_rx + aarx],
|
||||
temp_rxFext[aarx],
|
||||
@@ -1307,6 +1511,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
|
||||
(rel15_ul->ul_dmrs_symb_pos >> meas_symbol) & 0x01,
|
||||
rel15_ul,
|
||||
frame_parms);
|
||||
stop_meas(&gNB->pusch_extraction_stats);
|
||||
}
|
||||
|
||||
uint8_t shift_ch_ext = rel15_ul->nrOfLayers > 1 ? log2_approx(max_ch >> 11) : 0;
|
||||
|
||||
@@ -1342,9 +1548,10 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
|
||||
else
|
||||
pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 1 + log2_approx(frame_parms->nb_antennas_rx >> 1);
|
||||
|
||||
if (pusch_vars->log2_maxh < 0)
|
||||
pusch_vars->log2_maxh = 0;
|
||||
|
||||
if (pusch_vars->log2_maxh < 1)
|
||||
pusch_vars->log2_maxh = 1;
|
||||
else if (pusch_vars->log2_maxh > 14)
|
||||
pusch_vars->log2_maxh = 14;
|
||||
stop_meas(&gNB->rx_pusch_init_stats);
|
||||
|
||||
start_meas(&gNB->rx_pusch_symbol_processing_stats);
|
||||
|
||||
@@ -71,7 +71,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
AssertFatal(dmrs_Type == 0 || dmrs_Type == 1, "Illegal dmrs_type %d\n", dmrs_Type);
|
||||
uint8_t nb_re_dmrs;
|
||||
|
||||
LOG_D(PHY, "Round %d RV idx %d\n", harq_process->DLround, dlsch->dlsch_config.rv);
|
||||
LOG_D(PHY, "Round %d RV idx %d\n", harq_process->DLround, dlsch_config->rv);
|
||||
|
||||
if (dmrs_Type == NFAPI_NR_DMRS_TYPE1)
|
||||
nb_re_dmrs = 6 * dlsch_config->n_dmrs_cdm_groups;
|
||||
@@ -105,7 +105,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
|
||||
TB_parameters->processedSegments = &harq_process->processedSegments;
|
||||
|
||||
float Coderate = (float)dlsch->dlsch_config.targetCodeRate / 10240.0f;
|
||||
float Coderate = (float)dlsch_config->targetCodeRate / 10240.0f;
|
||||
|
||||
LOG_D(
|
||||
PHY,
|
||||
@@ -149,8 +149,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
if (LOG_DEBUGFLAG(DEBUG_DLSCH_DECOD) && (!slot_parameters.frame % 100))
|
||||
LOG_I(PHY, "K %d C %d Z %d nl %d \n", harq_process->K, harq_process->C, harq_process->Z, TB_parameters->nb_layers);
|
||||
// clear HARQ buffer
|
||||
for (int i = 0; i < harq_process->C; i++)
|
||||
memset(harq_process->d[i], 0, 5 * 8448 * sizeof(int16_t));
|
||||
memset(harq_process->d, 0, 3 * harq_process->K * harq_process->C * sizeof(int16_t));
|
||||
} else {
|
||||
// This is not a new packet, so retrieve previously computed quantities regarding segmentation
|
||||
TB_parameters->C = harq_process->C;
|
||||
@@ -170,10 +169,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
set_abort(&harq_process->abort_decode, false);
|
||||
}
|
||||
|
||||
nrLDPC_segment_decoding_parameters_t segments[nb_dlsch][max_num_segments];
|
||||
memset(segments, 0, sizeof(segments));
|
||||
bool d_to_be_cleared[nb_dlsch][max_num_segments];
|
||||
memset(d_to_be_cleared, 0, sizeof(d_to_be_cleared));
|
||||
bool d_to_be_cleared[nb_dlsch];
|
||||
|
||||
for (uint8_t pdsch_id = 0; pdsch_id < nb_dlsch; pdsch_id++) {
|
||||
uint8_t DLSCH_id = DLSCH_ids[pdsch_id];
|
||||
@@ -182,37 +178,65 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
NR_DL_UE_HARQ_t *harq_process = &phy_vars_ue->dl_harq_processes[DLSCH_id][harq_pid];
|
||||
|
||||
nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pdsch_id];
|
||||
TB_parameters->segments = segments[pdsch_id];
|
||||
|
||||
uint32_t r_offset = 0;
|
||||
TB_parameters->llr = dlsch_llr[DLSCH_id];
|
||||
LOG_D(NR_PHY,"Decoding pdsch %d, C %d, Qm %d, TB_parameters->llr %p, DLSCH_id %d\n",pdsch_id,TB_parameters->C, TB_parameters->Qm,TB_parameters->llr,DLSCH_id);
|
||||
TB_parameters->c = harq_process->c;
|
||||
TB_parameters->d = harq_process->d;
|
||||
TB_parameters->E = nr_get_E(TB_parameters->G,
|
||||
TB_parameters->C,
|
||||
TB_parameters->Qm,
|
||||
TB_parameters->nb_layers,
|
||||
0);
|
||||
TB_parameters->E2 = TB_parameters->E;
|
||||
TB_parameters->first_rE2 = TB_parameters->C;
|
||||
for (int r=1;r<TB_parameters->C;r++) {
|
||||
int Er = nr_get_E(TB_parameters->G,
|
||||
TB_parameters->C,
|
||||
TB_parameters->Qm,
|
||||
TB_parameters->nb_layers,
|
||||
r);
|
||||
if (Er != TB_parameters->E) {
|
||||
TB_parameters->E2=Er;
|
||||
TB_parameters->first_rE2 = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
TB_parameters->R = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
TB_parameters->E,
|
||||
TB_parameters->BG,
|
||||
TB_parameters->Z,
|
||||
&harq_process->llrLen,
|
||||
harq_process->DLround);
|
||||
|
||||
if (harq_process->first_rx == 1)
|
||||
d_to_be_cleared[pdsch_id] = true;
|
||||
else
|
||||
d_to_be_cleared[pdsch_id] = false;
|
||||
TB_parameters->d_to_be_cleared = d_to_be_cleared[pdsch_id];
|
||||
for (int r = 0; r<TB_parameters->C; r++) TB_parameters->decodeSuccess[r] = false;
|
||||
reset_meas(&TB_parameters->ts_deinterleave);
|
||||
reset_meas(&TB_parameters->ts_rate_unmatch);
|
||||
reset_meas(&TB_parameters->ts_seg_prep);
|
||||
reset_meas(&TB_parameters->ts_ldpc_decode);
|
||||
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
if (harq_process->first_rx == 1)
|
||||
d_to_be_cleared[pdsch_id][r] = true;
|
||||
else
|
||||
d_to_be_cleared[pdsch_id][r] = false;
|
||||
nrLDPC_segment_decoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
|
||||
segment_parameters->E = nr_get_E(TB_parameters->G,
|
||||
TB_parameters->C,
|
||||
TB_parameters->Qm,
|
||||
TB_parameters->nb_layers,
|
||||
r);
|
||||
segment_parameters->R = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
segment_parameters->E,
|
||||
int Etmp = nr_get_E(TB_parameters->G,
|
||||
TB_parameters->C,
|
||||
TB_parameters->Qm,
|
||||
TB_parameters->nb_layers,
|
||||
r);
|
||||
if (Etmp != TB_parameters->E) {
|
||||
TB_parameters->E2 = Etmp;
|
||||
TB_parameters->R2 = nr_get_R_ldpc_decoder(TB_parameters->rv_index,
|
||||
TB_parameters->E2,
|
||||
TB_parameters->BG,
|
||||
TB_parameters->Z,
|
||||
&harq_process->llrLen,
|
||||
harq_process->DLround);
|
||||
segment_parameters->llr = dlsch_llr[DLSCH_id] + r_offset;
|
||||
segment_parameters->d = harq_process->d[r];
|
||||
segment_parameters->d_to_be_cleared = &d_to_be_cleared[pdsch_id][r];
|
||||
segment_parameters->c = harq_process->c[r];
|
||||
segment_parameters->decodeSuccess = false;
|
||||
|
||||
reset_meas(&segment_parameters->ts_deinterleave);
|
||||
reset_meas(&segment_parameters->ts_rate_unmatch);
|
||||
reset_meas(&segment_parameters->ts_ldpc_decode);
|
||||
|
||||
r_offset += segment_parameters->E;
|
||||
TB_parameters->first_rE2 = r;
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -232,26 +256,27 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
|
||||
nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pdsch_id];
|
||||
|
||||
uint32_t offset = 0;
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
nrLDPC_segment_decoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
|
||||
if (segment_parameters->decodeSuccess) {
|
||||
memcpy(b[DLSCH_id] + offset,
|
||||
harq_process->c[r],
|
||||
(harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
} else {
|
||||
fapi_nr_dl_config_dlsch_pdu_rel15_t *dlsch_config = &dlsch[DLSCH_id].dlsch_config;
|
||||
LOG_D(PHY, "frame=%d, slot=%d, first_rx=%d, rv_index=%d\n", proc->frame_rx, proc->nr_slot_rx, harq_process->first_rx, dlsch_config->rv);
|
||||
LOG_D(PHY, "downlink segment error %d/%d\n", r, harq_process->C);
|
||||
LOG_D(PHY, "DLSCH %d in error\n", DLSCH_id);
|
||||
uint32_t offset = 0,r_offset = 0;
|
||||
bool crcok=true;
|
||||
for (int r = 0; r < TB_parameters->C ; r++) if (TB_parameters->decodeSuccess[r] == false) {crcok=false; break;}
|
||||
if (crcok) {
|
||||
for (int r = 0; r < TB_parameters->C; r++) {
|
||||
memcpy(b[DLSCH_id] + offset,
|
||||
harq_process->c + r_offset,
|
||||
(harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
|
||||
offset += (harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0);
|
||||
r_offset += (harq_process->K>>3);
|
||||
}
|
||||
offset += (harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0);
|
||||
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_DEINTERLEAVING_STATS], &segment_parameters->ts_deinterleave);
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_RATE_UNMATCHING_STATS], &segment_parameters->ts_rate_unmatch);
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_LDPC_DECODING_STATS], &segment_parameters->ts_ldpc_decode);
|
||||
|
||||
} else {
|
||||
fapi_nr_dl_config_dlsch_pdu_rel15_t *dlsch_config = &dlsch[DLSCH_id].dlsch_config;
|
||||
LOG_D(PHY, "frame=%d, slot=%d, harq_pid %d, first_rx=%d, rv_index=%d, A %d, K %d, F %d, C %d,E %d,E2 %d,Qm %d\n", proc->frame_rx, proc->nr_slot_rx, harq_pid, harq_process->first_rx, dlsch_config->rv,TB_parameters->A, harq_process->K,harq_process->F,harq_process->C,TB_parameters->E,TB_parameters->E2,TB_parameters->Qm);
|
||||
LOG_D(PHY, "DLSCH %d in error\n", DLSCH_id);
|
||||
}
|
||||
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_DEINTERLEAVING_STATS], &TB_parameters->ts_deinterleave);
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_RATE_UNMATCHING_STATS], &TB_parameters->ts_rate_unmatch);
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_LDPC_DECODING_STATS], &TB_parameters->ts_ldpc_decode);
|
||||
merge_meas(&phy_vars_ue->phy_cpu_stats.cpu_time_stats[DLSCH_SEG_PREP_STATS], &TB_parameters->ts_seg_prep);
|
||||
|
||||
kpiStructure.nb_total++;
|
||||
kpiStructure.blockSize = dlsch_config->TBS;
|
||||
@@ -262,10 +287,10 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
|
||||
if (harq_process->decodeResult && harq_process->C > 1) {
|
||||
/* check global CRC */
|
||||
int A = dlsch->dlsch_config.TBS;
|
||||
int A = dlsch_config->TBS;
|
||||
// we have regrouped the transport block
|
||||
if (!check_crc(b[DLSCH_id], lenWithCrc(1, A), crcType(1, A))) {
|
||||
LOG_E(PHY,
|
||||
LOG_D(PHY,
|
||||
" Frame %d.%d LDPC global CRC fails, but individual LDPC CRC succeeded. %d segs\n",
|
||||
proc->frame_rx,
|
||||
proc->nr_slot_rx,
|
||||
@@ -284,19 +309,19 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
i++;
|
||||
if (i == sz) {
|
||||
LOG_E(PHY,
|
||||
"received all 0 pdu, consider it false reception, even if the TS 38.212 7.2.1 says only we should attach the "
|
||||
"corresponding CRC, and nothing prevents to have a all 0 packet\n");
|
||||
"received all 0 pdu (TBS %d, mcs %d, C %d, nb_rb %d, decodedSegments %d) consider it false reception, even if the TS 38.212 7.2.1 says only we should attach the "
|
||||
"corresponding CRC, and nothing prevents to have a all 0 packet\n",dlsch_config->TBS, dlsch_config->mcs, harq_process->C, dlsch->dlsch_config.number_rbs,harq_process->processedSegments);
|
||||
harq_process->decodeResult = false;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (harq_process->decodeResult) {
|
||||
LOG_D(PHY, "DLSCH received ok \n");
|
||||
LOG_D(PHY, "%d.%d DLSCH received ok \n",proc->frame_rx,proc->nr_slot_rx);
|
||||
harq_process->status = NR_SCH_IDLE;
|
||||
dlsch->last_iteration_cnt = dlsch->max_ldpc_iterations - 1;
|
||||
} else {
|
||||
LOG_D(PHY, "DLSCH received nok \n");
|
||||
LOG_D(PHY, "%d.%d DLSCH received nok \n",proc->frame_rx,proc->nr_slot_rx);
|
||||
kpiStructure.nb_nack++;
|
||||
dlsch->last_iteration_cnt = dlsch->max_ldpc_iterations;
|
||||
UEdumpScopeData(phy_vars_ue, proc->nr_slot_rx, proc->frame_rx, "DLSCH_NACK");
|
||||
@@ -309,7 +334,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
else
|
||||
nb_re_dmrs = 4 * dlsch_config->n_dmrs_cdm_groups;
|
||||
uint16_t dmrs_length = get_num_dmrs(dlsch_config->dlDmrsSymbPos);
|
||||
float Coderate = (float)dlsch->dlsch_config.targetCodeRate / 10240.0f;
|
||||
float Coderate = (float)dlsch_config->targetCodeRate / 10240.0f;
|
||||
LOG_D(PHY,
|
||||
"%d.%d DLSCH Decoded, harq_pid %d, round %d, result: %d TBS %d (%d) G %d nb_re_dmrs %d length dmrs %d mcs %d Nl %d "
|
||||
"nb_symb_sch %d "
|
||||
@@ -319,12 +344,12 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
|
||||
harq_pid,
|
||||
harq_process->DLround,
|
||||
harq_process->decodeResult,
|
||||
dlsch->dlsch_config.TBS,
|
||||
dlsch->dlsch_config.TBS / 8,
|
||||
dlsch_config->TBS,
|
||||
dlsch_config->TBS / 8,
|
||||
G[DLSCH_id],
|
||||
nb_re_dmrs,
|
||||
dmrs_length,
|
||||
dlsch->dlsch_config.mcs,
|
||||
dlsch_config->mcs,
|
||||
dlsch->Nl,
|
||||
dlsch_config->number_symbols,
|
||||
dlsch_config->number_rbs,
|
||||
|
||||
@@ -78,11 +78,15 @@ typedef struct {
|
||||
NR_SCH_status_t status;
|
||||
/// Pointer to the payload (38.212 V15.4.0 section 5.1)
|
||||
uint8_t *b;
|
||||
/// Pointers to transport block segments
|
||||
uint8_t **c;
|
||||
/// Pointer to transport block segments
|
||||
uint8_t *c;
|
||||
#ifdef ENABLE_CUDA
|
||||
/// Pointer to transport block segments (GPU pointer for c when using shared memory, should be the same on Jetson
|
||||
uint8_t *cdev;
|
||||
#endif
|
||||
/// soft bits for each received segment ("d"-sequence)(for definition see 36-212 V8.6 2009-03, p.15)
|
||||
/// Accumulates the soft bits for each round to increase decoding success (HARQ)
|
||||
int16_t **d;
|
||||
int16_t *d;
|
||||
/// Index of current HARQ round for this DLSCH
|
||||
uint8_t DLround;
|
||||
/// Number of code segments
|
||||
|
||||
@@ -48,49 +48,41 @@ target_link_libraries(benchmark_channel_pipeline PRIVATE UTIL SIMU PHY_COMMON LO
|
||||
add_executable(test_channel_pipeline test_channel_pipeline.cpp test_channel_pipeline_tools.c)
|
||||
target_link_libraries(test_channel_pipeline PRIVATE UTIL SIMU PHY_COMMON LOG CONFIG_LIB shlib_loader m channel_pipeline GTest::gtest thread-pool)
|
||||
|
||||
if (CUDA_ENABLE)
|
||||
target_compile_definitions(test_channel_pipeline PRIVATE CUDA_ENABLE)
|
||||
if (ENABLE_CHANNEL_SIM_CUDA)
|
||||
target_compile_definitions(test_channel_pipeline PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests test_channel_pipeline)
|
||||
add_test(NAME test_channel_pipeline COMMAND test_channel_pipeline)
|
||||
set_tests_properties(test_channel_pipeline PROPERTIES LABELS "cuda")
|
||||
|
||||
target_compile_definitions(benchmark_channel_pipeline PRIVATE CUDA_ENABLE)
|
||||
target_compile_definitions(benchmark_channel_pipeline PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests benchmark_channel_pipeline)
|
||||
add_test(NAME benchmark_channel_pipeline COMMAND benchmark_channel_pipeline)
|
||||
set_tests_properties(benchmark_channel_pipeline PROPERTIES LABELS "cuda")
|
||||
endif()
|
||||
|
||||
if(CUDA_ENABLE)
|
||||
add_executable(test_multipath test_multipath.c)
|
||||
target_link_libraries(test_multipath PRIVATE UTIL SIMU LOG CONFIG_LIB shlib_loader m oai_cuda_lib)
|
||||
|
||||
target_compile_definitions(test_multipath PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests test_multipath)
|
||||
add_test(NAME test_multipath COMMAND test_multipath)
|
||||
set_tests_properties(test_multipath PROPERTIES LABELS "cuda")
|
||||
|
||||
add_executable(test_noise test_noise.c)
|
||||
target_link_libraries(test_noise PRIVATE UTIL SIMU PHY_COMMON LOG CONFIG_LIB shlib_loader m oai_cuda_lib)
|
||||
|
||||
target_compile_definitions(test_noise PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests test_noise)
|
||||
add_test(NAME test_noise COMMAND test_noise)
|
||||
set_tests_properties(test_noise PROPERTIES LABELS "cuda")
|
||||
|
||||
add_executable(test_channel_simulation test_channel_simulation.c)
|
||||
target_link_libraries(test_channel_simulation PRIVATE UTIL SIMU PHY_COMMON LOG CONFIG_LIB shlib_loader m oai_cuda_lib)
|
||||
|
||||
target_compile_definitions(test_channel_simulation PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests test_channel_simulation)
|
||||
add_test(NAME test_channel_simulation COMMAND test_channel_simulation)
|
||||
|
||||
add_executable(test_channel_scalability test_channel_scalability.c)
|
||||
target_link_libraries(test_channel_scalability PRIVATE UTIL SIMU PHY_COMMON LOG CONFIG_LIB shlib_loader m oai_cuda_lib)
|
||||
|
||||
target_compile_definitions(test_channel_scalability PRIVATE CHANNEL_SIM_CUDA)
|
||||
add_dependencies(tests test_channel_scalability)
|
||||
add_test(NAME test_channel_scalability COMMAND test_channel_scalability)
|
||||
|
||||
if (TARGET oai_cuda_lib)
|
||||
target_compile_definitions(test_multipath PRIVATE ENABLE_CUDA)
|
||||
target_compile_definitions(test_noise PRIVATE ENABLE_CUDA)
|
||||
target_compile_definitions(test_channel_simulation PRIVATE ENABLE_CUDA)
|
||||
target_compile_definitions(test_channel_scalability PRIVATE ENABLE_CUDA)
|
||||
endif()
|
||||
endif()
|
||||
|
||||
@@ -25,7 +25,7 @@ extern "C" void exit_function(const char *file, const char *function, const int
|
||||
exit(EXIT_FAILURE);
|
||||
}
|
||||
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
static void BM_channel_convolution_gpu(benchmark::State &state)
|
||||
{
|
||||
int nb_rx = state.range(0);
|
||||
@@ -203,7 +203,7 @@ static void BM_channel_convolution_tpool(benchmark::State &state)
|
||||
}
|
||||
}
|
||||
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
BENCHMARK(BM_channel_convolution_gpu)
|
||||
->ArgsProduct({
|
||||
{1, 2, 4, 16, 64}, // nb_rx
|
||||
|
||||
@@ -26,7 +26,7 @@ class ChannelConvolutionTest : public ::testing::TestWithParam<std::tuple<int, i
|
||||
protected:
|
||||
void SetUp() override
|
||||
{
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
gpu_context = cuda_channel_pipeline_init(614400 * 4);
|
||||
#endif
|
||||
tpool = init_tpool(8);
|
||||
@@ -35,7 +35,7 @@ class ChannelConvolutionTest : public ::testing::TestWithParam<std::tuple<int, i
|
||||
|
||||
void TearDown() override
|
||||
{
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
cuda_channel_pipeline_shutdown(gpu_context);
|
||||
#endif
|
||||
destroy_tpool(tpool);
|
||||
@@ -46,7 +46,7 @@ class ChannelConvolutionTest : public ::testing::TestWithParam<std::tuple<int, i
|
||||
void *tpool = nullptr;
|
||||
};
|
||||
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
TEST_P(ChannelConvolutionTest, CompareCpuGpu)
|
||||
{
|
||||
int nb_rx = std::get<0>(GetParam());
|
||||
|
||||
@@ -107,6 +107,12 @@ typedef struct {
|
||||
uint8_t *b;
|
||||
/// Pointers to transport block segments
|
||||
uint8_t **c;
|
||||
#ifdef ENABLE_CUDA
|
||||
/// Pointers to transport block segments (contains pointers in c above)
|
||||
uint8_t *c_devh;
|
||||
/// Pointers to transport block segments (GPU mapping)
|
||||
uint8_t *c_dev;
|
||||
#endif
|
||||
/// Interleaver outputs
|
||||
uint8_t *f;
|
||||
/// REs unavailable for DLSCH (overlapping with PTRS, CSIRS etc.)
|
||||
@@ -130,16 +136,16 @@ typedef struct {
|
||||
bool harq_to_be_cleared;
|
||||
/// Pointer to the payload (38.212 V15.4.0 section 5.1)
|
||||
uint8_t *b;
|
||||
/// Pointers to code blocks after code block segmentation and CRC attachment (38.212 V15.4.0 section 5.2.2)
|
||||
uint8_t **c;
|
||||
/// Pointer to aggregated code blocks after code block segmentation and CRC attachment (38.212 V15.4.0 section 5.2.2)
|
||||
uint8_t *c;
|
||||
/// Number of bits in each code block (38.212 V15.4.0 section 5.2.2)
|
||||
uint32_t K;
|
||||
/// Number of "Filler" bits added in the code block segmentation (38.212 V15.4.0 section 5.2.2)
|
||||
uint32_t F;
|
||||
/// Number of code blocks after code block segmentation (38.212 V15.4.0 section 5.2.2)
|
||||
uint32_t C;
|
||||
/// Pointers to code blocks after LDPC coding (38.212 V15.4.0 section 5.3.2)
|
||||
int16_t **d;
|
||||
/// Pointers to aggregated code blocks after LDPC coding (38.212 V15.4.0 section 5.3.2)
|
||||
int16_t *d;
|
||||
/// flag used to clear d properly (together with harq_to_be_cleared above)
|
||||
/// set to true in nr_ulsch_decoding() when harq_to_be_cleared is true
|
||||
/// when true, clear d in the next call to function nr_rate_matching_ldpc_rx()
|
||||
@@ -253,6 +259,10 @@ typedef struct {
|
||||
/// \brief llr values.
|
||||
/// - first index: ? [0..1179743] (hard coded)
|
||||
int16_t *llr;
|
||||
#ifdef ENABLE_CUDA
|
||||
/// \brief llr values link to device memory
|
||||
int16_t* llr_dev;
|
||||
#endif
|
||||
// PTRS symbol index, to be updated every PTRS symbol within a slot.
|
||||
uint8_t ptrs_symbol_index;
|
||||
/// bit mask of PT-RS ofdm symbol indicies
|
||||
@@ -442,9 +452,11 @@ typedef struct PHY_VARS_gNB_s {
|
||||
time_stats_t dlsch_resource_mapping_stats;
|
||||
time_stats_t dlsch_precoding_stats;
|
||||
time_stats_t tinput;
|
||||
time_stats_t tinput_memcpy;
|
||||
time_stats_t tprep;
|
||||
time_stats_t tparity;
|
||||
time_stats_t toutput;
|
||||
time_stats_t tconcat;
|
||||
|
||||
time_stats_t dlsch_rate_matching_stats;
|
||||
time_stats_t dlsch_interleaving_stats;
|
||||
@@ -460,11 +472,16 @@ typedef struct PHY_VARS_gNB_s {
|
||||
time_stats_t ulsch_decoding_stats;
|
||||
time_stats_t ts_deinterleave;
|
||||
time_stats_t ts_rate_unmatch;
|
||||
time_stats_t ts_seg_prep;
|
||||
time_stats_t ts_ldpc_decode;
|
||||
time_stats_t ulsch_deinterleaving_stats;
|
||||
time_stats_t ulsch_channel_estimation_stats;
|
||||
time_stats_t pusch_channel_estimation_antenna_processing_stats;
|
||||
time_stats_t ulsch_llr_stats;
|
||||
time_stats_t ulsch_layer_demapping_stats;
|
||||
time_stats_t ulsch_unscrambling_stats;
|
||||
time_stats_t pusch_extraction_stats;
|
||||
time_stats_t pusch_channel_compensation_stats;
|
||||
time_stats_t rx_srs_stats;
|
||||
time_stats_t generate_srs_stats;
|
||||
time_stats_t get_srs_signal_stats;
|
||||
@@ -486,6 +503,7 @@ typedef struct PHY_VARS_gNB_s {
|
||||
pthread_t L1_tx_thread;
|
||||
int L1_tx_thread_core;
|
||||
void *scopeData;
|
||||
uint32_t use_gpu;
|
||||
} PHY_VARS_gNB;
|
||||
|
||||
struct puschSymbolReqId {
|
||||
@@ -515,7 +533,7 @@ typedef struct LDPCDecode_s {
|
||||
NR_UL_gNB_HARQ_t *ulsch_harq;
|
||||
t_nrLDPC_dec_params decoderParms;
|
||||
NR_gNB_ULSCH_t *ulsch;
|
||||
short* ulsch_llr;
|
||||
int16_t* ulsch_llr;
|
||||
int ulsch_id;
|
||||
int harq_pid;
|
||||
int rv_index;
|
||||
|
||||
@@ -436,6 +436,13 @@ typedef struct PHY_VARS_NR_UE_s {
|
||||
Actor_t *ul_actors;
|
||||
pthread_t main_thread;
|
||||
pthread_t stat_thread;
|
||||
uint32_t use_gpu;
|
||||
#ifdef ENABLE_CUDA
|
||||
// page locked memory (cudaHostAlloc) for llr input to LDPC decoder for GPU offload
|
||||
int16_t *llr[10][2];
|
||||
// gpu mapped version (cudaDeviceGetHostPointer), typically the same for Jetson/GH/GB
|
||||
int16_t *llr_dev[10][2];
|
||||
#endif
|
||||
} PHY_VARS_NR_UE;
|
||||
|
||||
typedef struct {
|
||||
|
||||
@@ -16,6 +16,7 @@
|
||||
FN(DLSCH_CHANNEL_ESTIMATION_STATS),\
|
||||
FN(DLSCH_DECODING_STATS),\
|
||||
FN(DLSCH_RATE_UNMATCHING_STATS),\
|
||||
FN(DLSCH_SEG_PREP_STATS),\
|
||||
FN(DLSCH_LDPC_DECODING_STATS),\
|
||||
FN(DLSCH_DEINTERLEAVING_STATS),\
|
||||
FN(DLSCH_EXTRACT_RBS_STATS),\
|
||||
|
||||
@@ -150,8 +150,14 @@ __attribute__((always_inline)) static inline int32_t simde_mm_average(simde__m12
|
||||
*/
|
||||
__attribute__((always_inline)) static inline simde__m128i oai_mm_conj(simde__m128i a)
|
||||
{
|
||||
#ifdef __aarch64__
|
||||
const oai128_t neg_imag = {.i16 = {0, -1, 0, -1, 0, -1, 0, -1}};
|
||||
int16x8_t aneg = vnegq_s16((int16x8_t)a);
|
||||
return (simde__m128i)vbslq_s16((uint16x8_t)neg_imag.v, aneg, (int16x8_t)a);
|
||||
#else
|
||||
const oai128_t neg_imag = {.i16 = {1, -1, 1, -1, 1, -1, 1, -1}};
|
||||
return simde_mm_sign_epi16(a, neg_imag.v);
|
||||
#endif
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -167,9 +173,13 @@ __attribute__((always_inline)) static inline simde__m128i oai_mm_conj(simde__m12
|
||||
__attribute__((always_inline)) static inline
|
||||
simde__m128i oai_mm_swap(simde__m128i a)
|
||||
{
|
||||
#ifdef __aarch64__
|
||||
return (simde__m128i)vrev32q_s16((int16x8_t)a);
|
||||
#else
|
||||
// Shuffle mask to swap bytes for IQ swapping
|
||||
const oai128_t shuffle_mask_swap = {.i8 = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}};
|
||||
return simde_mm_shuffle_epi8(a, shuffle_mask_swap.v);
|
||||
#endif
|
||||
}
|
||||
|
||||
__attribute__((always_inline)) static inline
|
||||
@@ -216,7 +226,7 @@ simde__m128i oai_mm_cpx_mult(simde__m128i z1, simde__m128i z2, int shift)
|
||||
* Input: z1 = (a + bi) [ a0, b0, ..., a3, b3]
|
||||
* Input: z2 = (c + di) [ c0, d0, ..., c3, d3]
|
||||
* Output: z3 = (e + fi) [ e0, f0, ..., e3, f3]
|
||||
* z3 = z1 * conj(z2) = + (ac+bd) + i(bc-ad)
|
||||
* z3 = conj(z1) * z2 = + (ac+bd) + i(ad-bc)
|
||||
*
|
||||
* @param 128-bit SIMD vector of four complex 16-bit integers.
|
||||
* @return a 128-bit SIMD vector.
|
||||
@@ -229,6 +239,43 @@ simde__m128i oai_mm_cpx_mult_conj(simde__m128i a, simde__m128i b, int shift)
|
||||
return oai_mm_pack(re, im);
|
||||
}
|
||||
|
||||
#ifdef __aarch64__
|
||||
#define CPX_MUL_CONJ(SHIFT) \
|
||||
__attribute__((always_inline)) static inline simde__m128i oai_mm_cpx_mult_conj##SHIFT(simde__m128i a, simde__m128i b) \
|
||||
{ \
|
||||
const oai128_t neg_imag = {.i16 = {0, -1, 0, -1, 0, -1, 0, -1}};\
|
||||
\
|
||||
int16x8_t aneg = vnegq_s16((int16x8_t)a);\
|
||||
int16x8_t arevn = vbslq_s16((uint16x8_t)neg_imag.v, aneg, (int16x8_t)a);\
|
||||
arevn = vrev32q_s16(arevn);\
|
||||
int32x4_t abl = vmull_s16(vget_low_s16((int16x8_t)a),vget_low_s16((int16x8_t)b));\
|
||||
int32x4_t abh = vmull_high_s16((int16x8_t)a,(int16x8_t)b);\
|
||||
int32x4_t arevnbl = vmull_s16(vget_low_s16(arevn),vget_low_s16((int16x8_t)b));\
|
||||
int32x4_t arevnbh = vmull_high_s16(arevn,(int16x8_t)b);\
|
||||
int32x4_t re = vpaddq_s32(abl,abh);\
|
||||
int32x4_t im = vpaddq_s32(arevnbl,arevnbh);\
|
||||
int32x4_t outl = vzip1q_s32(re,im);\
|
||||
int32x4_t outh = vzip2q_s32(re,im);\
|
||||
int16x4_t r = vqrshrn_n_s32(outl, (const int)SHIFT);\
|
||||
return (simde__m128i)vqrshrn_high_n_s32(r, outh, (const int)SHIFT); \
|
||||
}
|
||||
//CPX_MUL_CONJ(0)
|
||||
CPX_MUL_CONJ(1)
|
||||
CPX_MUL_CONJ(2)
|
||||
CPX_MUL_CONJ(3)
|
||||
CPX_MUL_CONJ(4)
|
||||
CPX_MUL_CONJ(5)
|
||||
CPX_MUL_CONJ(6)
|
||||
CPX_MUL_CONJ(7)
|
||||
CPX_MUL_CONJ(8)
|
||||
CPX_MUL_CONJ(9)
|
||||
CPX_MUL_CONJ(10)
|
||||
CPX_MUL_CONJ(11)
|
||||
CPX_MUL_CONJ(12)
|
||||
CPX_MUL_CONJ(13)
|
||||
CPX_MUL_CONJ(14)
|
||||
CPX_MUL_CONJ(15)
|
||||
#endif
|
||||
/*
|
||||
* OAI specific AVX2 section
|
||||
*/
|
||||
@@ -313,7 +360,7 @@ simde__m256i oai_mm256_cpx_mult(simde__m256i z1, simde__m256i z2, int shift)
|
||||
* Input: z1 = (a + bi) [ a0, b0, ..., a3, b3]
|
||||
* Input: z2 = (c + di) [ c0, d0, ..., c3, d3]
|
||||
* Output: z3 = (e + fi) [ e0, f0, ..., e3, f3]
|
||||
* z3 = z1 * conj(z2) = (ac+bd) + i(bc-ad)
|
||||
* z3 = conj(z1) * z2 = (ac+bd) + i(ad-bc)
|
||||
*
|
||||
* @param 256-bit SIMD vector of eight complex 16-bit integers.
|
||||
* @return a 256-bit SIMD vector.
|
||||
@@ -326,6 +373,50 @@ simde__m256i oai_mm256_cpx_mult_conj(simde__m256i a, simde__m256i b, int shift)
|
||||
return oai_mm256_pack(re, im);
|
||||
}
|
||||
|
||||
#ifdef __AVX512BW__
|
||||
__attribute__((always_inline)) static inline __m512i oai_mm512_conj(__m512i a)
|
||||
{
|
||||
const __mmask32 odd = 0xAAAAAAAAu;
|
||||
// For odd lanes: 0 - b (two's complement negate)
|
||||
return(_mm512_mask_sub_epi16(a, odd, _mm512_setzero_si512(), a));
|
||||
}
|
||||
|
||||
__attribute__((always_inline)) static inline __m512i oai_mm512_swap(__m512i a)
|
||||
|
||||
{
|
||||
// Shuffle mask to swap bytes for IQ swapping
|
||||
const oai512_t shuffle_mask_swap = {.i8 = {
|
||||
2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13,
|
||||
18, 19, 16, 17, 22, 23, 20, 21, 26, 27, 24, 25, 30, 31, 28, 29,
|
||||
34, 35, 32, 33, 38, 39, 36, 37, 42, 43, 40, 41, 46, 47, 44, 45,
|
||||
50, 51, 48, 49, 54, 55, 52, 53, 58, 59, 56, 57, 62, 63, 60, 61
|
||||
}};
|
||||
return _mm512_shuffle_epi8(a, shuffle_mask_swap.v);
|
||||
}
|
||||
|
||||
__attribute__((always_inline)) static inline
|
||||
__m512i oai_mm512_smadd(__m512i z1, __m512i z2, int shift)
|
||||
{
|
||||
return _mm512_srai_epi32(_mm512_madd_epi16(z1, z2), shift);
|
||||
}
|
||||
|
||||
__attribute__((always_inline)) static inline
|
||||
__m512i oai_mm512_pack(__m512i a, __m512i b)
|
||||
{
|
||||
return _mm512_packs_epi32(
|
||||
_mm512_unpacklo_epi32(a, b), // real
|
||||
_mm512_unpackhi_epi32(a, b) // imag
|
||||
);
|
||||
}
|
||||
__attribute__((always_inline)) static inline
|
||||
__m512i oai_mm512_cpx_mult_conj(__m512i a, __m512i b, int shift)
|
||||
{
|
||||
__m512i re = oai_mm512_smadd(a, b, shift);
|
||||
__m512i im = oai_mm512_smadd(oai_mm512_swap(oai_mm512_conj(a)), b, shift);
|
||||
return oai_mm512_pack(re, im);
|
||||
}
|
||||
#endif
|
||||
|
||||
#ifdef __cplusplus
|
||||
}
|
||||
#endif
|
||||
|
||||
@@ -51,6 +51,10 @@ fifo_dump_emos_UE emos_dump_UE;
|
||||
#include "T.h"
|
||||
#include "instrumentation.h"
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#endif
|
||||
|
||||
static const unsigned int gain_table[31] = {100, 112, 126, 141, 158, 178, 200, 224, 251, 282, 316,
|
||||
359, 398, 447, 501, 562, 631, 708, 794, 891, 1000, 1122,
|
||||
1258, 1412, 1585, 1778, 1995, 2239, 2512, 2818, 3162};
|
||||
@@ -1291,8 +1295,16 @@ void pdsch_processing(PHY_VARS_NR_UE *ue, const UE_nr_rxtx_proc_t *proc, nr_phy_
|
||||
const uint32_t rx_llr_buf_sz = ALIGNARRAYSIZE(G, 32); // each LLR is 2 bytes hence 64 byte aligned
|
||||
const uint32_t nb_codewords = NR_MAX_NB_LAYERS > 4 ? 2 : 1;
|
||||
int16_t* llr[2];
|
||||
#ifdef ENABLE_CUDA
|
||||
for (int i = 0; i < nb_codewords; i++) {
|
||||
llr[i]=ue->llr_dev[nr_slot_rx%10][i];
|
||||
LOG_D(NR_PHY,"nr_slot_rx %d, llr[%d] %p\n",nr_slot_rx,i,llr[i]);
|
||||
// memset(llr[i],0,rx_llr_buf_sz * sizeof(int16_t));
|
||||
}
|
||||
#else
|
||||
for (int i = 0; i < nb_codewords; i++)
|
||||
llr[i] = (int16_t *)malloc16_clear(rx_llr_buf_sz * sizeof(int16_t));
|
||||
#endif
|
||||
|
||||
// it returns -1 in case of internal failure, or 0 in case of normal result
|
||||
int ret_pdsch = nr_ue_pdsch_procedures(ue, proc, dlsch, llr, rxdataF, G);
|
||||
@@ -1328,8 +1340,10 @@ void pdsch_processing(PHY_VARS_NR_UE *ue, const UE_nr_rxtx_proc_t *proc, nr_phy_
|
||||
if (ue->phy_sim_pdsch_llr)
|
||||
memcpy(ue->phy_sim_pdsch_llr, llr[0], sizeof(int16_t) * rx_llr_buf_sz);
|
||||
|
||||
#ifndef ENABLE_CUDA
|
||||
for (int i=0; i<nb_codewords; i++)
|
||||
free(llr[i]);
|
||||
#endif
|
||||
}
|
||||
|
||||
if (nr_slot_rx==9) {
|
||||
|
||||
@@ -43,6 +43,7 @@ int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
|
||||
uint64_t downlink_frequency[MAX_NUM_CCs][4];
|
||||
|
||||
double cpuf;
|
||||
uint32_t use_gpu=0;
|
||||
|
||||
uint8_t const nr_rv_round_map[4] = {0, 2, 3, 1};
|
||||
// needed for some functions
|
||||
@@ -489,7 +490,7 @@ int main(int argc, char **argv)
|
||||
unsigned char output[nb_rb * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64)));
|
||||
bzero(output, sizeof(output));
|
||||
if (input_fd == NULL) {
|
||||
nr_dlsch_encoding(gNB, 1, dlsch, frame, slot, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
|
||||
nr_dlsch_encoding(gNB, 1, dlsch, frame, slot, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
|
||||
}
|
||||
|
||||
for (SNR = snr0; SNR < snr1 && !stop; SNR += snr_step) {
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
#define inMicroS(a) (((double)(a))/(get_cpu_freq_GHz()*1000.0))
|
||||
#include "SIMULATION/LTE_PHY/common_sim.h"
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#include "SIMULATION/TOOLS/oai_cuda.h"
|
||||
#endif
|
||||
@@ -295,8 +295,8 @@ void validate_input_pmi(nfapi_nr_config_request_scf_t *gNB_config,
|
||||
num_antenna_ports, pmi_pdu->num_ant_ports, pmi);
|
||||
}
|
||||
|
||||
|
||||
configmodule_interface_t *uniqCfg = NULL;
|
||||
extern uint32_t use_gpu;
|
||||
int main(int argc, char **argv)
|
||||
{
|
||||
stop = false;
|
||||
@@ -396,14 +396,14 @@ int main(int argc, char **argv)
|
||||
|
||||
void *h_tx_sig_pinned = NULL;
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
void *d_tx_sig = NULL, *d_intermediate_sig = NULL, *d_final_output = NULL;
|
||||
void *d_curand_states = NULL;
|
||||
void *h_final_output_pinned = NULL;
|
||||
void *d_channel_coeffs_gpu = NULL;
|
||||
#endif
|
||||
|
||||
while ((c = getopt(argc, argv, "--:O:f:hA:p:f:g:i:n:s:S:t:v:x:y:z:o:H:M:N:F:GR:d:PI:L:a:b:e:m:w:T:U:q:X:Y:Z:")) != -1) {
|
||||
while ((c = getopt(argc, argv, "--:O:f:hA:p:f:g:i:n:s:S:t:v:x:y:z:o:H:M:N:F:GR:d:PQI:L:a:b:e:m:w:T:U:q:X:Y:Z:")) != -1) {
|
||||
/* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
|
||||
/* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
|
||||
if (c == 1 || c == '-' || c == 'O')
|
||||
@@ -412,7 +412,7 @@ int main(int argc, char **argv)
|
||||
printf("handling optarg %c\n",c);
|
||||
switch (c) {
|
||||
case 'f':
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
if (strcmp(optarg, "cuda") == 0) {
|
||||
use_cuda = 1;
|
||||
} else
|
||||
@@ -525,7 +525,11 @@ int main(int argc, char **argv)
|
||||
print_perf=1;
|
||||
cpu_meas_enabled = 1;
|
||||
break;
|
||||
|
||||
|
||||
case 'Q':
|
||||
use_gpu = 1;
|
||||
break;
|
||||
|
||||
case 'I':
|
||||
max_ldpc_iterations = atoi(optarg);
|
||||
break;
|
||||
@@ -633,7 +637,7 @@ int main(int argc, char **argv)
|
||||
printf("-d number of dlsch threads, 0: no dlsch parallelization\n");
|
||||
printf("-e MSC index\n");
|
||||
printf("-f <flag> Enable optional feature flag. Available flags:\n");
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
printf(" cuda Enable CUDA channel simulation\n");
|
||||
#else
|
||||
printf(" (none) No optional features were compiled into this executable\n");
|
||||
@@ -669,13 +673,13 @@ int main(int argc, char **argv)
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
printf("%d\n", slot);
|
||||
logInit();
|
||||
set_glog(loglvl);
|
||||
/* initialize the sin table */
|
||||
InitSinLUT();
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
init_cuda_chsim_buffers(use_cuda,
|
||||
n_tx,
|
||||
n_rx,
|
||||
@@ -688,7 +692,7 @@ int main(int argc, char **argv)
|
||||
&d_channel_coeffs_gpu);
|
||||
#endif
|
||||
|
||||
#if !defined(ENABLE_CUDA) || !use_cuda
|
||||
#if !defined(CHANNEL_SIM_CUDA) || !use_cuda
|
||||
printf("Pre-allocating padded host memory for the CPU channel pipeline...\n");
|
||||
int num_samples_alloc = 153600;
|
||||
const int max_padding_alloc = 256 - 1;
|
||||
@@ -866,7 +870,7 @@ int main(int argc, char **argv)
|
||||
0,
|
||||
0);
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
float *h_channel_coeffs = NULL;
|
||||
if (use_cuda) {
|
||||
int num_links = n_tx * n_rx;
|
||||
@@ -1034,9 +1038,11 @@ int main(int argc, char **argv)
|
||||
reset_meas(&gNB->dlsch_encoding_stats);
|
||||
reset_meas(&gNB->dci_generation_stats);
|
||||
reset_meas(&gNB->tinput);
|
||||
reset_meas(&gNB->tinput_memcpy);
|
||||
reset_meas(&gNB->tprep);
|
||||
reset_meas(&gNB->tparity);
|
||||
reset_meas(&gNB->toutput);
|
||||
reset_meas(&gNB->tconcat);
|
||||
reset_meas(&gNB->phase_comp_stats);
|
||||
|
||||
uint32_t errors_scrambling[16] = {0};
|
||||
@@ -1059,7 +1065,7 @@ int main(int argc, char **argv)
|
||||
nfapi_nr_dl_tti_request_pdu_t *dl_tti_pdsch_pdu = &dl_req->dl_tti_pdu_list[1];
|
||||
nfapi_nr_dl_tti_pdsch_pdu_rel15_t *pdsch_pdu_rel15 = &dl_tti_pdsch_pdu->pdsch_pdu.pdsch_pdu_rel15;
|
||||
|
||||
for (trial = 0; trial < n_trials && !stop; trial++) {
|
||||
for (trial = 0; trial < n_trials /*&& !stop)*/; trial++) {
|
||||
|
||||
errors_bit = 0;
|
||||
//multipath channel
|
||||
@@ -1079,7 +1085,7 @@ int main(int argc, char **argv)
|
||||
UE_harq_process->DLround = round;
|
||||
UE_harq_process->first_rx = 1;
|
||||
|
||||
while (round < num_rounds && !UE_harq_process->decodeResult && !stop) {
|
||||
while (round < num_rounds && !UE_harq_process->decodeResult /*&& !stop*/) {
|
||||
reset_sched_response(Sched_INFO, frame, slot, 0, 0);
|
||||
clear_nr_nfapi_information(RC.nrmac[0], 0, frame, slot);
|
||||
UE_info->UE_sched_ctrl.harq_processes[harq_pid].ndi = !(trial&1);
|
||||
@@ -1213,7 +1219,7 @@ int main(int argc, char **argv)
|
||||
}
|
||||
|
||||
// Apply MIMO Channel
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
if (use_cuda) {
|
||||
#if defined(USE_UNIFIED_MEMORY)
|
||||
int deviceId;
|
||||
@@ -1400,11 +1406,14 @@ int main(int argc, char **argv)
|
||||
fprintf(csv_file,"%.2f,%.4f,%.2f,%u\n", roundStats, effRate, effRate / (8 * pdsch_pdu_rel15->TBSize[0]) * 100, 8 * pdsch_pdu_rel15->TBSize[0]);
|
||||
}
|
||||
if (print_perf==1) {
|
||||
printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, block %d)\n",
|
||||
printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, Z %d, F %d, K %d, block %d)\n",
|
||||
1000 >> *scc->ssbSubcarrierSpacing,
|
||||
g_rbSize,
|
||||
g_mcsIndex,
|
||||
UE->dl_harq_processes[0][slot].C,
|
||||
UE->dl_harq_processes[0][slot].Z,
|
||||
UE->dl_harq_processes[0][slot].F,
|
||||
UE->dl_harq_processes[0][slot].K,
|
||||
8 * pdsch_pdu_rel15->TBSize[0]);
|
||||
printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
|
||||
printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");
|
||||
@@ -1512,7 +1521,7 @@ int main(int argc, char **argv)
|
||||
free(r_im[i]);
|
||||
}
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
free_cuda_chsim_buffers(use_cuda,
|
||||
&d_tx_sig,
|
||||
&d_intermediate_sig,
|
||||
|
||||
@@ -15,6 +15,7 @@
|
||||
|
||||
// Define signal handler to attempt graceful termination
|
||||
bool stop = false;
|
||||
extern uint32_t use_gpu;
|
||||
|
||||
void sigint_handler(int arg)
|
||||
{
|
||||
|
||||
@@ -47,6 +47,7 @@ openair0_config_t openair0_cfg[MAX_CARDS];
|
||||
|
||||
uint8_t const nr_rv_round_map[4] = {0, 2, 3, 1};
|
||||
|
||||
uint32_t use_gpu;
|
||||
void inc_ref_sched_response(int _)
|
||||
{
|
||||
LOG_E(PHY, "fatal\n");
|
||||
|
||||
@@ -64,6 +64,7 @@ double cpuf;
|
||||
uint64_t downlink_frequency[MAX_NUM_CCs][4];
|
||||
int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
|
||||
uint32_t N_RB_DL = 106;
|
||||
uint32_t use_gpu=0;
|
||||
|
||||
NR_IF_Module_t *NR_IF_Module_init(int Mod_id) { return (NULL); }
|
||||
nfapi_mode_t nfapi_getmode(void) { return NFAPI_MODE_UNKNOWN; }
|
||||
|
||||
@@ -38,7 +38,7 @@ PHY_VARS_NR_UE *UE;
|
||||
RAN_CONTEXT_t RC;
|
||||
int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
|
||||
uint64_t downlink_frequency[MAX_NUM_CCs][4];
|
||||
|
||||
uint32_t use_gpu=0;
|
||||
double cpuf;
|
||||
uint8_t const nr_rv_round_map[4] = {0, 2, 3, 1};
|
||||
// needed for some functions
|
||||
|
||||
@@ -40,6 +40,7 @@ PHY_VARS_NR_UE *UE;
|
||||
RAN_CONTEXT_t RC;
|
||||
int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
|
||||
uint64_t downlink_frequency[MAX_NUM_CCs][4];
|
||||
uint32_t use_gpu=0;
|
||||
|
||||
static softmodem_params_t softmodem_params;
|
||||
softmodem_params_t *get_softmodem_params(void) {
|
||||
|
||||
@@ -73,7 +73,7 @@
|
||||
#include "time_meas.h"
|
||||
#include "utils.h"
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
#include <cuda_runtime.h>
|
||||
#include "SIMULATION/TOOLS/oai_cuda.h"
|
||||
#endif
|
||||
@@ -104,6 +104,7 @@ instance_t CUuniqInstance=0;
|
||||
// NTN cellSpecificKoffset-r17, but in slots for DL SCS
|
||||
unsigned int NTN_UE_Koffset = 0;
|
||||
|
||||
extern uint32_t use_gpu;
|
||||
void nr_derive_key_ng_ran_star(uint16_t pci, uint64_t nr_arfcn_dl, const uint8_t key[32], uint8_t *key_ng_ran_star)
|
||||
{
|
||||
}
|
||||
@@ -355,7 +356,7 @@ int main(int argc, char *argv[])
|
||||
|
||||
void *h_tx_sig_pinned = NULL;
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
void *d_tx_sig = NULL, *d_intermediate_sig = NULL, *d_final_output = NULL;
|
||||
void *d_curand_states = NULL;
|
||||
void *h_final_output_pinned = NULL;
|
||||
@@ -363,7 +364,7 @@ int main(int argc, char *argv[])
|
||||
void *d_channel_coeffs_gpu = NULL;
|
||||
#endif
|
||||
|
||||
while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:o::p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:Y:"))
|
||||
while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:o::p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PQR:S:T:U:L:ZW:E:X:Y:"))
|
||||
!= -1) {
|
||||
/* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
|
||||
/* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
|
||||
@@ -396,7 +397,7 @@ int main(int argc, char *argv[])
|
||||
break;
|
||||
|
||||
case 'f':
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
if (strcmp(optarg, "cuda") == 0) {
|
||||
use_cuda = 1;
|
||||
} else
|
||||
@@ -559,6 +560,7 @@ int main(int argc, char *argv[])
|
||||
printf("Problem with filename %s\n", optarg);
|
||||
exit(-1);
|
||||
}
|
||||
params_from_file=1;
|
||||
break;
|
||||
|
||||
case 'G':
|
||||
@@ -593,6 +595,9 @@ int main(int argc, char *argv[])
|
||||
cpu_meas_enabled = 1;
|
||||
break;
|
||||
|
||||
case 'Q':
|
||||
use_gpu=1;
|
||||
break;
|
||||
case 'L':
|
||||
loglvl = atoi(optarg);
|
||||
break;
|
||||
@@ -615,10 +620,6 @@ int main(int argc, char *argv[])
|
||||
} while (optarg[i-1] == ',');
|
||||
break;
|
||||
|
||||
case 'Q':
|
||||
params_from_file = 1;
|
||||
break;
|
||||
|
||||
case 'X' :
|
||||
filename_csv = strdup(optarg);
|
||||
AssertFatal(filename_csv != NULL, "strdup() error: errno %d\n", errno);
|
||||
@@ -659,7 +660,7 @@ int main(int argc, char *argv[])
|
||||
printf("-d Introduce delay in terms of number of samples\n");
|
||||
printf("-e To simulate MSG3 configuration\n");
|
||||
printf("-f <flag> Enable optional feature flag. Available flags:\n");
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
printf(" cuda Enable CUDA channel simulation\n");
|
||||
#else
|
||||
printf(" (none) No optional features were compiled into this executable\n");
|
||||
@@ -876,7 +877,7 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
|
||||
const int num_samples_alloc = 153600;
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
init_cuda_chsim_buffers(use_cuda,
|
||||
n_tx,
|
||||
n_rx,
|
||||
@@ -893,7 +894,7 @@ int main(int argc, char *argv[])
|
||||
}
|
||||
#endif
|
||||
|
||||
#if !defined(ENABLE_CUDA) || !use_cuda
|
||||
#if !defined(CHANNEL_SIM_CUDA) || !use_cuda
|
||||
printf("Pre-allocating padded host memory for the CPU channel pipeline...\n");
|
||||
const int max_padding_alloc = 256 - 1;
|
||||
size_t padded_tx_alloc_bytes = n_tx * (num_samples_alloc + max_padding_alloc) * 2 * sizeof(float);
|
||||
@@ -1037,9 +1038,8 @@ int main(int argc, char *argv[])
|
||||
|
||||
ulsch_input_buffer[0] = 0x31;
|
||||
for (i = 1; i < TBS/8; i++) {
|
||||
ulsch_input_buffer[i] = (uint8_t)rand();
|
||||
ulsch_input_buffer[i] = (uint8_t)(256*uniformrandom());
|
||||
}
|
||||
|
||||
uint8_t ptrs_time_density = get_L_ptrs(ptrs_mcs1, ptrs_mcs2, ptrs_mcs3, Imcs, mcs_table);
|
||||
uint8_t ptrs_freq_density = get_K_ptrs(n_rb0, n_rb1, nb_rb);
|
||||
|
||||
@@ -1173,6 +1173,11 @@ int main(int argc, char *argv[])
|
||||
reset_meas(&gNB->rx_pusch_stats);
|
||||
reset_meas(&gNB->rx_pusch_init_stats);
|
||||
reset_meas(&gNB->rx_pusch_symbol_processing_stats);
|
||||
reset_meas(&gNB->pusch_extraction_stats);
|
||||
reset_meas(&gNB->pusch_channel_compensation_stats);
|
||||
reset_meas(&gNB->ulsch_llr_stats);
|
||||
reset_meas(&gNB->ulsch_layer_demapping_stats);
|
||||
reset_meas(&gNB->ulsch_unscrambling_stats);
|
||||
reset_meas(&gNB->ulsch_decoding_stats);
|
||||
reset_meas(&gNB->ts_deinterleave);
|
||||
reset_meas(&gNB->ts_rate_unmatch);
|
||||
@@ -1464,7 +1469,7 @@ int main(int argc, char *argv[])
|
||||
memcpy(data_start_ptr, s_interleaved[j], slot_length * 2 * sizeof(float));
|
||||
}
|
||||
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
if (use_cuda) {
|
||||
#if defined(USE_UNIFIED_MEMORY)
|
||||
int deviceId;
|
||||
@@ -1833,9 +1838,22 @@ int main(int argc, char *argv[])
|
||||
printStatIndent3(&gNB->pusch_channel_estimation_antenna_processing_stats, "Antenna Processing time");
|
||||
printStatIndent2(&gNB->rx_pusch_init_stats, "RX PUSCH Initialization time");
|
||||
printStatIndent2(&gNB->rx_pusch_symbol_processing_stats, "RX PUSCH Symbol Processing time");
|
||||
gNB->pusch_extraction_stats.trials=gNB->rx_pusch_symbol_processing_stats.trials;
|
||||
/*if (threadCnt!=0)*/ printStatIndent3(&gNB->pusch_extraction_stats, "RX PUSCH extraction");
|
||||
gNB->pusch_channel_compensation_stats.trials=gNB->rx_pusch_symbol_processing_stats.trials;
|
||||
/*if (threadCnt!=0)*/ printStatIndent3(&gNB->pusch_channel_compensation_stats, "RX PUSCH channel compensation");
|
||||
gNB->ulsch_llr_stats.trials=gNB->rx_pusch_symbol_processing_stats.trials;
|
||||
/*if (threadCnt!=0)*/ printStatIndent3(&gNB->ulsch_llr_stats, "RX PUSCH LLR");
|
||||
gNB->ulsch_layer_demapping_stats.trials=gNB->rx_pusch_symbol_processing_stats.trials;
|
||||
/*if (threadCnt!=0)*/ printStatIndent3(&gNB->ulsch_layer_demapping_stats, "RX PUSCH layer demapping");
|
||||
gNB->ulsch_unscrambling_stats.trials=gNB->rx_pusch_symbol_processing_stats.trials;
|
||||
/*if (threadCnt!=0)*/ printStatIndent3(&gNB->ulsch_unscrambling_stats, "RX PUSCH unscrambling");
|
||||
printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time");
|
||||
gNB->ts_deinterleave.trials=n_trials;
|
||||
printStatIndent2(&gNB->ts_deinterleave, "ULSCH segment deinterleaving time");
|
||||
gNB->ts_rate_unmatch.trials=n_trials;
|
||||
printStatIndent2(&gNB->ts_rate_unmatch, "ULSCH segment rate matching time");
|
||||
gNB->ts_ldpc_decode.trials=n_trials;
|
||||
printStatIndent2(&gNB->ts_ldpc_decode, "ULSCH segments decoding time");
|
||||
printStatIndent(&gNB->rx_srs_stats,"RX SRS time");
|
||||
printStatIndent2(&gNB->generate_srs_stats,"Generate SRS sequence time");
|
||||
@@ -1904,7 +1922,7 @@ int main(int argc, char *argv[])
|
||||
fclose(uci_ulsch_matlab_vec);
|
||||
|
||||
free_and_zero(UE->phy_sim_test_buf);
|
||||
#ifdef ENABLE_CUDA
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
free_cuda_chsim_buffers(use_cuda,
|
||||
&d_tx_sig,
|
||||
&d_intermediate_sig,
|
||||
|
||||
@@ -3,17 +3,16 @@
|
||||
add_library(noise_device noise_device.c)
|
||||
target_link_libraries(noise_device PRIVATE log_headers SIMU)
|
||||
target_include_directories(noise_device PUBLIC ./)
|
||||
set(CHANNEL_PIPELINE_SOURCES channel_pipeline.c)
|
||||
if (CUDA_ENABLE)
|
||||
list(APPEND CHANNEL_PIPELINE_SOURCES channel_pipeline_v2.cu)
|
||||
add_library(channel_pipeline channel_pipeline.c)
|
||||
if (ENABLE_CHANNEL_SIM_CUDA)
|
||||
target_sources(channel_pipeline PRIVATE channel_pipeline_v2.cu)
|
||||
endif()
|
||||
add_library(channel_pipeline ${CHANNEL_PIPELINE_SOURCES})
|
||||
target_link_libraries(channel_pipeline PUBLIC thread-pool noise_device)
|
||||
if (CUDA_ENABLE)
|
||||
if (ENABLE_CHANNEL_SIM_CUDA)
|
||||
target_link_libraries(channel_pipeline PUBLIC CUDA::toolkit)
|
||||
endif()
|
||||
|
||||
if(CUDA_ENABLE)
|
||||
if(ENABLE_CHANNEL_SIM_CUDA)
|
||||
add_library(oai_cuda_lib STATIC
|
||||
multipath_channel.cu
|
||||
phase_noise.cu
|
||||
|
||||
@@ -132,9 +132,9 @@ This is the traditional CUDA programming model. The host and device have separat
|
||||
|
||||
## Project Integration
|
||||
|
||||
To ensure the feature is modular, the integration into the OAI project was handled at both the build system and source code levels. All of the new CUDA source files (`channel_pipeline.cu`, `multipath_channel.cu`, `phase_noise.cu`) are compiled into a single static library named `oai_cuda_lib`. This is defined in the main `CMakeLists.txt` file and is controlled by the `CUDA_ENABLE` CMake option, which is disabled by default. When this option is activated (`-DCUDA_ENABLE=ON`), the build system compiles the CUDA library and also defines a global `ENABLE_CUDA` preprocessor macro that is visible to the rest of the project.
|
||||
To ensure the feature is modular, the integration into the OAI project was handled at both the build system and source code levels. All of the new CUDA source files (`channel_pipeline.cu`, `multipath_channel.cu`, `phase_noise.cu`) are compiled into a single static library named `oai_cuda_lib`. This is defined in the main `CMakeLists.txt` file and is controlled by the `ENABLE_CHANNEL_SIM_CUDA` CMake option, which is disabled by default. When this option is activated (`-DENABLE_CHANNEL_SIM_CUDA=ON`), the build system compiles the CUDA library and also defines a global `CHANNEL_SIM_CUDA` preprocessor macro that is visible to the rest of the project.
|
||||
|
||||
This `ENABLE_CUDA` macro is then used within the simulator files, such as `dlsim.c` and `ulsim.c`, to conditionally compile all CUDA-related code. This approach allows for the inclusion of the `oai_cuda.h` header, and the pre-allocation of GPU memory at startup, all without affecting the standard CPU-only build.
|
||||
This `CHANNEL_SIM_CUDA` macro is then used within the simulator files, such as `dlsim.c` and `ulsim.c`, to conditionally compile all CUDA-related code. This approach allows for the inclusion of the `oai_cuda.h` header, and the pre-allocation of GPU memory at startup, all without affecting the standard CPU-only build.
|
||||
|
||||
Inside the main processing loop, an `if (use_cuda)` statement acts as the primary runtime switch. To enable a direct comparison, timing measurement calls have been added to both the GPU and CPU execution paths. Also, both paths now source their data from the same common, pre-padded host buffer. The GPU path receives a pointer to the entire padded buffer and leverages its optimized kernel, while the CPU path uses a temporary array of offset pointers to work on the same data without being aware of the padding. This ensures both computations operate on the exact same source data, isolating the performance measurement to the channel processing itself. To execute the simulation on the GPU, the `-f cuda` flag must be provided at runtime; otherwise, the program defaults to this refined CPU implementation.
|
||||
|
||||
|
||||
@@ -10,7 +10,7 @@ extern "C" {
|
||||
#endif
|
||||
#include "common/platform_types.h"
|
||||
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
void *cuda_channel_pipeline_init(int max_samples);
|
||||
void cuda_channel_pipeline_shutdown(void *context_handle);
|
||||
void cuda_channel_pipeline(void *context_handle,
|
||||
|
||||
@@ -39,7 +39,7 @@ function(add_physim_test test_name test_description test_exec)
|
||||
# pass test description also through environment variable: for cmake < 3.30,
|
||||
# in JSON export, we cannot recover the description otherwise
|
||||
# see also https://gitlab.kitware.com/cmake/cmake/-/issues/21490
|
||||
ENVIRONMENT "LD_LIBRARY_PATH=.;TEST_DESCRIPTION=${test_description}"
|
||||
ENVIRONMENT "LD_LIBRARY_PATH=${CMAKE_BINARY_DIR};TEST_DESCRIPTION=${test_description}"
|
||||
)
|
||||
set_tests_properties(${test_name} PROPERTIES CHECK_COUNT 0)
|
||||
add_dependencies(tests ${test_exec})
|
||||
@@ -439,7 +439,7 @@ endif()
|
||||
###### cuda-specific tests ######
|
||||
####################################################################################
|
||||
|
||||
if (CUDA_ENABLE)
|
||||
if (ENABLE_CHANNEL_SIM_CUDA)
|
||||
|
||||
# --- Test Suite 1: 61440 samples, 2/4/8/32 TX - 2/4/8/32 RX ---
|
||||
add_physim_test(cuda.5g.channelsim.test_channel_scalability.test1 "16 channels 61440 samples 2T2R Batch" test_channel_scalability -c 16 -t 2 -r 2 -s 61440 -l 32 -m batch -n 12)
|
||||
|
||||
@@ -22,9 +22,8 @@ target_link_libraries(vrtsim
|
||||
channel_pipeline
|
||||
thread-pool
|
||||
)
|
||||
if (CUDA_ENABLE)
|
||||
target_compile_definitions(vrtsim PRIVATE CUDA_ENABLE)
|
||||
target_compile_definitions(vrtsim PRIVATE CUDA_ENABLE)
|
||||
if (ENABLE_CHANNEL_SIM_CUDA)
|
||||
target_compile_definitions(vrtsim PRIVATE CHANNEL_SIM_CUDA)
|
||||
endif()
|
||||
|
||||
# Optional taps client
|
||||
|
||||
@@ -666,7 +666,7 @@ static int vrtsim_write_with_chanmod(vrtsim_state_t *vrtsim_state,
|
||||
}
|
||||
size_t saved_samples_input_len = channel_length - 1;
|
||||
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
cuda_channel_pipeline(vrtsim_state->channel_pipeline_context,
|
||||
(const cf_t **)channel_impulse_response_p,
|
||||
(const c16_t **)saved_samples_ptr,
|
||||
@@ -873,7 +873,7 @@ static void vrtsim_end(openair0_device_t *device)
|
||||
|
||||
tx_timing_t *tx_timing = &vrtsim_state->tx_timing;
|
||||
if (vrtsim_state->chanmod || vrtsim_state->taps_socket || vrtsim_state->use_cirdb) {
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
cuda_channel_pipeline_shutdown(vrtsim_state->channel_pipeline_context);
|
||||
#else
|
||||
abortTpool(&vrtsim_state->tpool);
|
||||
@@ -976,7 +976,7 @@ __attribute__((__visibility__("default"))) int device_init(openair0_device_t *de
|
||||
int noise_power_dBFS = get_noise_power_dBFS();
|
||||
int16_t noise_power = noise_power_dBFS == INVALID_DBFS_VALUE ? 0 : (int16_t)(32767.0 / powf(10.0, .05 * -noise_power_dBFS));
|
||||
LOG_A(HW, "VRTSIM: Noise power %d sample value\n", noise_power);
|
||||
#ifdef CUDA_ENABLE
|
||||
#ifdef CHANNEL_SIM_CUDA
|
||||
size_t samples_in_one_ms = openair0_cfg->sample_rate / 1000 / 1000;
|
||||
vrtsim_state->channel_pipeline_context = cuda_channel_pipeline_init(openair0_cfg->tx_num_channels * samples_in_one_ms);
|
||||
#else
|
||||
|
||||
Reference in New Issue
Block a user