channel_output

This reverts commit 3cf853e129.

CMakeLists.txt

@@ -233,6 +233,11 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang
   set(commonOpts "${commonOpts} -Wno-unused-command-line-argument -Wno-default-const-init-field-unsafe")
 endif()
 
+if (ENABLE_LDPC_CUDA)
+  # TODO only on specific targets!!
+  set(commonOpts "${commonOpts} -DLDPC_CUDA")
+endif()
+
 set(CMAKE_C_FLAGS
   "${C_FLAGS_PROCESSOR} ${commonOpts} -std=gnu11 -funroll-loops ${CMAKE_C_FLAGS}")
 set(CMAKE_CXX_FLAGS
@@ -1094,15 +1099,30 @@ target_link_libraries(PHY_UE PRIVATE asn1_lte_rrc_hdrs asn1_nr_rrc_hdrs UTIL)
 add_library(PHY_NR_COMMON ${PHY_NR_SRC_COMMON})
 target_link_libraries(PHY_NR_COMMON PUBLIC UTIL)
 
-add_library(PHY_NR ${PHY_NR_SRC})
+if (ENABLE_LDPC_CUDA)
+  find_package(CUDA REQUIRED)
+  if (NOT CUDA_FOUND)
+    message(FATAL_ERROR "no CUDA found")
+  endif()
+  #SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_80 -g -ggdb") # A100
+  #SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_90") # GH200
+  #SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_87") # Jetson
+  #SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_89") # L40S
+  SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_121") # GB10
+  set(CUDA_VERBOS_BUILD_ON)
+  cuda_add_library(PHY_NR ${PHY_NR_SRC} ${PHY_NR_SRC_CU})
+  cuda_add_library(PHY_NR_UE ${PHY_NR_UE_SRC} ${PHY_NR_UE_SRC_CU})
+else()
+  add_library(PHY_NR ${PHY_NR_SRC})
+  add_library(PHY_NR_UE ${PHY_NR_UE_SRC})
+endif()
 target_link_libraries(PHY_NR nr_phy_common nr_common nr_fapi_p5 polar smallblock ds)
 
 add_library(PHY_NR_NO_AVX_256 ${PHY_NR_SRC})
 target_link_libraries(PHY_NR_NO_AVX_256 nr_phy_common nr_common)
 target_compile_definitions(PHY_NR_NO_AVX_256 PUBLIC USE_128BIT)
 
-add_library(PHY_NR_UE ${PHY_NR_UE_SRC})
-target_link_libraries(PHY_NR_UE PRIVATE asn1_nr_rrc_hdrs nr_phy_common nr_common UTIL polar smallblock)
+target_link_libraries(PHY_NR_UE asn1_nr_rrc_hdrs nr_phy_common nr_common UTIL polar smallblock)
 
 add_library(PHY_RU ${PHY_SRC_RU})
 target_link_libraries(PHY_RU PRIVATE asn1_lte_rrc_hdrs UTIL)
@@ -2016,12 +2036,19 @@ add_executable(ldpctest
 )
 target_link_libraries(ldpctest PRIVATE
   -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON -Wl,--end-group
-  m pthread dl ${T_LIB}
+  m pthread dl ${T_LIB} ${CUDA_LIBRARIES}
   # link 'check_crc' to make it resolved in the LDPC coding libraries
   # 'check_crc' is not used in ldpctest so it is not linked in the executable by default
   # --whole-archive links 'check_crc' in the executable even though it is note used, see 'man ld'
   -Wl,--whole-archive crc_byte -Wl,--no-whole-archive
 )
+if (ENABLE_LDPC_CUDA)
+  # ldpctest.c uses cudaHostAlloc(), so include dir/link libr to make available
+  # TODO don't write it directly into ldpctest, but separate compilation unit?
+  find_package(CUDA REQUIRED)
+  target_link_libraries(ldpctest PRIVATE ${CUDA_LIBRARIES})
+  target_include_directories(ldpctest PRIVATE ${CUDA_INCLUDE_DIRS})
+endif()
 
 add_library(physim_common OBJECT ${OPENAIR1_DIR}/SIMULATION/NR_PHY/nr_unitary_common.c)
 target_link_libraries(physim_common PRIVATE UTIL)

ci-scripts/cls_containerize.py

@@ -55,7 +55,7 @@ def AnalyzeBuildLogs(image, lf):
 			committed = committed or lineHasCommit
 			if re.search(r'error:|Errors|ERROR', line):
 				errors.append(f"=> {line.strip()}")
-	status = (committed or tagged) and len(errors) == 0
+	status = (committed or tagged)
 	logging.info(f"Analyzing {image}, file {lf}: {status=}, {len(errors)} errors")
 	for e in errors:
 		logging.info(e)
@@ -178,9 +178,20 @@ class Containerize():
 	
 		dockerfileprefix = '.ubuntu'
 
+		baseImage = 'ran-base'
+		baseTag = 'develop'
+		buildImage = 'ran-build'
+		forceBaseImageBuild = False
+		imageTag = 'develop'
+
+		result = re.search('native_cuda_armv8', self.imageKind)
+		if result is not None:
+			baseImage = 'ran-base-cuda'
+			buildImage = 'ran-build-cuda'
+			dockerfileprefix = '.cuda.ubuntu'
 		# we always build the ran-build image with all targets
 		# Creating a tupple with the imageName, the DockerFile prefix pattern, targetName and sanitized option
-		imageNames = [('ran-build', 'build', 'ran-build', '')]
+		imageNames = [(buildImage, 'build', f'{buildImage}', '')]
 		result = re.search('eNB', self.imageKind)
 		if result is not None:
 			imageNames.append(('oai-enb', 'eNB', 'oai-enb', ''))
@@ -220,13 +231,13 @@ class Containerize():
 			imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
 			imageNames.append(('oai-nr-cuup', 'nr-cuup', 'oai-nr-cuup', ''))
 			imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
-		
+		result = re.search('native_cuda_armv8', self.imageKind)
+		if result is not None:
+			imageNames.append(('oai-gnb', 'gNB', 'oai-gnb', ''))
+			imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
+
 		cmd.cd(lSourcePath)
 
-		baseImage = 'ran-base'
-		baseTag = 'develop'
-		forceBaseImageBuild = False
-		imageTag = 'develop'
 		if (self.merge):
 			imageTag = 'ci-temp'
 			if self.targetBranch == 'develop':
@@ -254,17 +265,17 @@ class Containerize():
 		# On when the base image docker file is being modified.
 		if forceBaseImageBuild:
 			cmd.run(f"docker image rm {baseImage}:{baseTag}")
-			logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log'
+			logfile = f'{lSourcePath}/cmake_targets/log/{baseImage}.docker.log'
 			option = f" --build-arg UBUNTU_IMAGE={DEFAULT_REGISTRY}/{ubuntuImage}"
 			cmd.run(f"docker build --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{dockerfileprefix} {option} . &> {logfile}", timeout=1600)
-			t = ("ran-base", archiveArtifact(cmd, ctx, logfile))
+			t = (baseImage, archiveArtifact(cmd, ctx, logfile))
 			log_files.append(t)
 
-		# First verify if the base image was properly created.
 		ret = cmd.run(f"docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
+
 		allImagesSize = {}
 		if ret.returncode != 0:
-			logging.error('\u001B[1m Could not build properly ran-base\u001B[0m')
+			logging.error(f'\u001B[1m Could not build properly {baseImage}\u001B[0m')
 			# Recover the name of the failed container?
 			cmd.run(f"docker ps --quiet --filter \"status=exited\" -n1 | xargs --no-run-if-empty docker rm -f")
 			cmd.run(f"docker image prune --force")
@@ -277,10 +288,10 @@ class Containerize():
 			if result is not None:
 				size = float(result.group("size")) / 1000000
 				imageSizeStr = f'{size:.1f}'
-				logging.debug(f'\u001B[1m   ran-base size is {imageSizeStr} Mbytes\u001B[0m')
-				allImagesSize['ran-base'] = f'{imageSizeStr} Mbytes'
+				logging.debug(f'\u001B[1m {baseImage} size is {imageSizeStr} Mbytes\u001B[0m')
+				allImagesSize[baseImage] = f'{imageSizeStr} Mbytes'
 			else:
-				logging.debug('ran-base size is unknown')
+				logging.debug(f'{baseImage} size is unknown')
 
 		# Build the target image(s)
 		status = True
@@ -291,11 +302,11 @@ class Containerize():
 			cmd.run(f'sed -i -e "s#{baseImage}:latest#{baseImage}:{baseTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
 			# target images should use the proper ran-build image
 			if image != 'ran-build' and "-asan" in name:
-				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build-asan:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
+				cmd.run(f'sed -i -e "s#{buildImage}:latest#{buildImage}-asan:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
 			elif "fhi72" in name or name == "oai-nr-oru":
 				cmd.run(f'sed -i -e "s#ran-build-fhi72:latest#ran-build-fhi72:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
 			elif image != 'ran-build':
-				cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
+				cmd.run(f'sed -i -e "s#{buildImage}:latest#{buildImage}:{imageTag}#" docker/Dockerfile.{pattern}{dockerfileprefix}')
 			if image == 'oai-gnb-aerial':
 				cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2026.01.07.tar.gz .')
 			logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log'

ci-scripts/xml_files/container_image_build_cuda_jetson.xml

@@ -0,0 +1,28 @@
+<!-- SPDX-License-Identifier: LicenseRef-CSSL-1.0 -->
+
+<testCaseList>
+  <htmlTabRef>build-cuda-tab</htmlTabRef>
+  <htmlTabName>Build Container Images for ARM with CUDA</htmlTabName>
+  <htmlTabIcon>wrench</htmlTabIcon>
+
+  <testCase>
+    <class>Create_Workspace</class>
+    <desc>Create new Workspace for server 0</desc>
+    <node>jetson3-oai</node>
+  </testCase>
+
+  <testCase>
+    <class>Build_Image</class>
+    <desc>Build all Images</desc>
+    <node>jetson3-oai</node>
+    <kind>native_cuda_armv8</kind>
+  </testCase>
+
+  <testCase>
+    <class>Push_Local_Registry</class>
+    <desc>Push Images to Local Registry</desc>
+    <node>jetson3-oai</node>
+    <tag_prefix>armv8_cuda_</tag_prefix>
+  </testCase>
+
+</testCaseList>

ci-scripts/xml_files/container_sa_b200_nrue_cuda_jetson.xml

@@ -0,0 +1,192 @@
+<!-- SPDX-License-Identifier: LicenseRef-CSSL-1.0 -->
+
+<testCaseList>
+  <htmlTabRef>JETSON-5G-NRUE-CUDA</htmlTabRef>
+  <htmlTabName>30 MHz TDD SA JETSON with CUDA</htmlTabName>
+  <htmlTabIcon>tasks</htmlTabIcon>
+  <testCase>
+    <class>DeployCoreNetwork</class>
+    <desc>Initialize 5G Core</desc>
+    <cn_id>oc-cn5g-00104</cn_id>
+  </testCase>
+
+  <testCase>
+    <class>Pull_Local_Registry</class>
+    <desc>Pull Images from Local Registry</desc>
+    <node>jetson2-oai</node>
+    <images>oai-nr-ue</images>
+    <tag_prefix>armv8_cuda_</tag_prefix>
+  </testCase>
+
+  <testCase>
+    <class>Pull_Local_Registry</class>
+    <desc>Pull Images from Local Registry</desc>
+    <node>gracehopper1-oai</node>
+    <images>oai-gnb-aerial</images>
+    <tag_prefix>arm_</tag_prefix>
+  </testCase>
+
+  <testCase>
+    <class>Create_Workspace</class>
+    <desc>Create new Workspace</desc>
+    <node>jetson2-oai</node>
+  </testCase>
+
+  <testCase>
+    <class>Create_Workspace</class>
+    <desc>Create new Workspace</desc>
+    <node>gracehopper1-oai</node>
+  </testCase>
+
+  <testCase>
+    <class>Custom_Script</class>
+    <desc>Configure 30 MHz bandwidth on WNC RU</desc>
+    <node>gracehopper1-oai</node>
+    <script>scripts/set-wnc-bandwidth.sh</script>
+    <args>30</args>
+  </testCase>
+
+  <testCase>
+    <class>Deploy_Object</class>
+    <desc>Deploy PNF/Nvidia CUBB in a container</desc>
+    <node>gracehopper1-oai</node>
+    <yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
+    <services>nv-cubb</services>
+  </testCase>
+
+  <testCase>
+    <class>Deploy_Object</class>
+    <desc>Deploy VNF in a container</desc>
+    <node>gracehopper1-oai</node>
+    <yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
+    <services>oai-gnb-aerial</services>
+  </testCase>
+
+  <testCase>
+    <class>Deploy_Object</class>
+    <desc>Deploy nrUE (TDD/Band78/40MHz/N310) in a container</desc>
+    <yaml_path>ci-scripts/yaml_files/sa_b200_jetson_cuda_nrue</yaml_path>
+    <node>jetson2-oai</node>
+    <num_attempts>3</num_attempts>
+  </testCase>
+
+  <testCase>
+    <class>Attach_UE</class>
+    <desc>Attach OAIUE</desc>
+    <id>oai_ue_jetson2</id>
+  </testCase>
+
+  <testCase>
+    <class>Custom_Command</class>
+    <desc>Configure IP route</desc>
+    <node>jetson2-oai</node>
+    <command>sudo ip route add 172.21.6.105 via 13.1.1.100 dev oaitun_ue1</command>
+  </testCase>
+
+  <testCase>
+    <class>Ping</class>
+    <desc>Ping: 20pings in 20sec</desc>
+    <id>oai_ue_jetson2</id>
+    <svr_id>oc-cn5g-00104</svr_id>
+    <ping_args>-c 20</ping_args>
+    <ping_packetloss_threshold>1</ping_packetloss_threshold>
+    <ping_rttavg_threshold>25</ping_rttavg_threshold>
+  </testCase>
+
+  <testCase>
+    <class>Iperf</class>
+    <desc>iperf (UL/10Mbps/UDP)(30 sec)</desc>
+    <iperf_args>-u -b 10M -t 30 -i 1</iperf_args>
+    <id>oai_ue_jetson2</id>
+    <svr_id>oc-cn5g-00104</svr_id>
+    <iperf_packetloss_threshold>25</iperf_packetloss_threshold>
+    <iperf_bitrate_threshold>80</iperf_bitrate_threshold>
+  </testCase>
+
+  <testCase>
+    <class>Iperf</class>
+    <desc>iperf (DL/10Mbps/UDP)(30 sec)</desc>
+    <iperf_args>-u -b 10M -t 30 -i 1 -R</iperf_args>
+    <id>oai_ue_jetson2</id>
+    <svr_id>oc-cn5g-00104</svr_id>
+    <iperf_packetloss_threshold>25</iperf_packetloss_threshold>
+    <iperf_bitrate_threshold>80</iperf_bitrate_threshold>
+  </testCase>
+
+  <testCase>
+    <class>Iperf</class>
+    <desc>iperf (UL/30Mbps/UDP)(30 sec)</desc>
+    <may_fail>true</may_fail>
+    <iperf_args>-u -b 30M -t 30 -i 1</iperf_args>
+    <id>oai_ue_jetson2</id>
+    <svr_id>oc-cn5g-00104</svr_id>
+    <iperf_packetloss_threshold>25</iperf_packetloss_threshold>
+    <iperf_bitrate_threshold>80</iperf_bitrate_threshold>
+  </testCase>
+
+  <testCase>
+    <class>Iperf</class>
+    <desc>iperf (DL/30Mbps/UDP)(30 sec)</desc>
+    <may_fail>true</may_fail>
+    <iperf_args>-u -b 30M -t 30 -i 1 -R</iperf_args>
+    <id>oai_ue_jetson2</id>
+    <svr_id>oc-cn5g-00104</svr_id>
+    <iperf_packetloss_threshold>25</iperf_packetloss_threshold>
+    <iperf_bitrate_threshold>80</iperf_bitrate_threshold>
+  </testCase>
+
+  <testCase>
+    <class>Undeploy_Object</class>
+    <always_exec>true</always_exec>
+    <desc>Undeploy nr UE</desc>
+    <node>jetson2-oai</node>
+    <yaml_path>ci-scripts/yaml_files/sa_b200_jetson_cuda_nrue</yaml_path>
+    <analysis>
+      <services>oai-nr-ue</services>
+    </analysis>
+  </testCase>
+
+  <testCase>
+    <class>Undeploy_Object</class>
+    <always_exec>true</always_exec>
+    <desc>Undeploy gNB</desc>
+    <node>gracehopper1-oai</node>
+    <yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_30MHz</yaml_path>
+    <analysis>
+      <services>oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye</services>
+    </analysis>
+  </testCase>
+
+  <testCase>
+    <class>Custom_Script</class>
+    <always_exec>true</always_exec>
+    <desc>Configure 100 MHz bandwidth on WNC RU</desc>
+    <node>gracehopper1-oai</node>
+    <script>scripts/set-wnc-bandwidth.sh</script>
+    <args>100</args>
+  </testCase>
+
+  <testCase>
+    <class>Clean_Test_Server_Images</class>
+    <always_exec>true</always_exec>
+    <desc>Clean Test Images on Test Server</desc>
+    <node>gracehopper1-oai</node>
+    <images>oai-gnb-aerial</images>
+  </testCase>
+
+  <testCase>
+    <class>Clean_Test_Server_Images</class>
+    <always_exec>true</always_exec>
+    <desc>Clean Test Images on Test Server</desc>
+    <node>jetson2-oai</node>
+    <images>oai-nr-ue</images>
+  </testCase>
+
+  <testCase>
+    <class>UndeployCoreNetwork</class>
+    <always_exec>true</always_exec>
+    <desc>Terminate 5G Core</desc>
+    <cn_id>oc-cn5g-00104</cn_id>
+  </testCase>
+
+</testCaseList>

ci-scripts/yaml_files/sa_b200_jetson_cuda_nrue/docker-compose.yml

@@ -0,0 +1,40 @@
+# SPDX-License-Identifier: MIT
+
+services:
+    oai-nr-ue:
+        image: ${REGISTRY-oaisoftwarealliance/}${NRUE_IMG:-oai-nr-ue}:${TAG:-develop}
+        network_mode: host
+        runtime: nvidia
+        container_name: oai-nr-ue
+        cap_drop:
+            - ALL
+        cap_add:
+            - NET_ADMIN  # for interface bringup
+            - NET_RAW    # for ping
+            - SYS_NICE   # for performance
+            - IPC_LOCK   # for memory locking
+        devices:
+            - /dev/net/tun:/dev/net/tun
+            - /dev/bus/usb/:/dev/bus/usb/
+        ulimits:
+          core: -1 # for core dumps
+        environment:
+            TZ: Europe/Paris
+            USE_B2XX: 'yes'
+            USE_ADDITIONAL_OPTIONS:  --usrp-args type=b200 -C 3350010000 -r 78 --numerology 1 --band 78 --ssb 345 --ue-fo-compensation --uicc0.imsi 001040000000001 --uicc0.pdu_sessions.[0].dnn oai.ipv4 --thread-pool 5,6,7,8,9 --ue-txgain 15 --ue-rxgain 90 --loader.ldpc.shlibversion _cuda -A 60
+        cpuset: "3-11"
+        volumes:
+            - ../../conf_files/ue.sa.conf:/opt/oai-nr-ue/etc/nr-ue.conf
+        healthcheck:
+            test: /bin/bash -c "pgrep nr-uesoftmodem"
+            interval: 10s
+            timeout: 5s
+            retries: 5
+        deploy:
+          resources:
+            reservations:
+              devices:
+                - driver: nvidia
+                  count: all
+                  capabilities: [gpu]
+

cmake_targets/build_oai

@@ -18,7 +18,7 @@ HWs=""
 BUILD_DIR=ran_build
 CMAKE_BUILD_TYPE="RelWithDebInfo"
 CMAKE_CMD="$CMAKE"
-OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal websrv oai_iqplayer imscope imscope_record"
+OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_aal ldpc_cuda websrv oai_iqplayer imscope imscope_record"
 TARGET_LIST=""
 BUILD_TOOL_OPT="-j$(nproc)"
 

doc/NR_SA_Tutorial_OAI_nrUE_GPU_offload.md

@@ -0,0 +1,417 @@
+# OAI 5G NR SA tutorial with OAI nrUE with GPU Offload
+
+This tutorial explains how OAI nrUE can use GPU to offload LDPC decoder. In the future we might offload more functions on the GPU. 
+
+**Table of Contents**
+
+[[_TOC_]]
+
+## Current Status
+
+The nrUE GPU offload functionality have been tested on below platforms:
+
+|Platform       |Version     | OS/Linux Kernel                       |
+|---------------|------------|---------------------------------------|
+|Jetson AGX Orin|L4T 36.4.7  | U22/5.15.148-tegra (64K page variant) |
+|DGX Spark      |7.4.0       | U24/6.14.0-1013-nvidia-64k            |
+
+The above versions are tested versions, you are free to try on newer versions as well. 
+
+You can use another UE platform with GPU as well but it is preferred to use unified memory GPU variant.
+
+### KPIs
+
+The below throughput numbers are similar for TCP/UDP, the gNB was configured for 5 ms TDD pattern (6D3U and Special slot 10D0U) for Nvidia Aerial Testbed. 
+
+The UE was in SISO mode.
+
+Testbed Architecture: 
+
+UE <--> Over the Air 1.5m to 2m distance <--> WNC RU <--> gNB/DU server
+
+| Platform    | Radio     | Bandwidth | DL Throughput | UL Throughput |
+| ----------- | --------- | --------- | ------------- | ------------- |
+| Jetson Orin | B210      | 10 MHz    | 12 Mbps       | 7.5 Mbps      |
+| Jetson Orin | B210      | 20 MHz    | 20 Mbps       | 9.5 Mbps      |
+| Jetson Orin | B210      | 30 MHz    | 61 Mbps       | 33 Mbps       |
+| Jetson Orin | B210      | 40 MHz    | 69 Mbps       | 46 Mbps       |
+| DGX Spark   | B210      | 40 MHz    | 86 Mbps       | 46 Mbps       |
+| DGX Spark   | N310/x410 | 100 MHz   | 231 Mbps      | 118 Mbps      |
+
+Stability or long running test with peak throughput for nrUE was performed for 5hrs.
+
+All the numbers are with LDPC GPU offload.
+
+`NOTE`: At the time of writing we only tested the UE in SISO. 
+We are working supporting multi antenna use case.
+
+## Aerial Testbed Configuration
+
+If you are using aerial testbed with OAI then depending on the bandwidth you want to use, you will have to adapt the value of `ds_offset_dl` in cuphycontroller configuration file. 
+
+For our testing environment we adapted the `ds_offset_dl` values as below:
+
+| Bandwidth   | ds_offset_dl |
+| ----------- | -------------| 
+| 10 MHz      | 14           | 
+| 20 MHz      | 12           | 
+| 30 MHz      | 10           | 
+| 40 MHz      | 9            | 
+| 100 MHz     | 6            |
+
+On the WNC radio we have the below configuration: 
+
+```
+transmit-power 24
+transmit-power-scale 0.0
+compress tx static bfp iq-bitwidth 9
+compress rx static bfp iq-bitwidth 9
+compress prach static bfp iq-bitwidth 9
+```
+
+You can adjust the scaling on the RU or you can adjust it in aerial configuration file. 
+
+**NOTE**: You might have to adjust these values for your environment, so that the UE can detect the cell and perform random-access.
+
+## Configure the nrUE Hardware Platform
+
+### Configure Jetson AGX Orin
+
+At the time of writing this tutorial AGX Orin only had Jetpack 6.X release which has U22/5.15.148-tegra, for GPU offload on ARM systems it is preferred to use 64K page size kernel. 
+
+To change the kernel to 64K page size variant you will have to manually build the kernel. Probably for Jetpack 7.X it will be easier. Before building the kernel make sure AGX Orin has the up to date kernel for your current distribution. 
+
+NVIDIA-SMI Version: 540.4.0
+CUDA Version: 12.6
+CPU: Cortex-A78AE (12)
+OS: U22
+Kernel: 5.15.148-tegra
+NVCC: Build cuda_12.6.r12.6/compiler.34714021_0
+UHD: UHD_4.9.0.HEAD-0-g006d7f76 (v4.9.0.0)
+
+#### Update kernel to 64K page size
+
+Login to Jetson AGX Orin
+
+```bash
+sudo apt update
+sudo apt upgrade
+# reboot
+```
+
+**CAUTION**: While building the kernel if you make a mistake you have to flash again the Jetson. We are just providing these steps to help you build the kernel with 64K page size. If you have questions or issues building the kernel you can check the [official nvidia documentation](https://docs.nvidia.com/jetson/archives/r36.4.4/DeveloperGuide/SD/Kernel/KernelCustomization.html#sd-kernel-kernelcustomization)
+
+prepare the kernel source code:
+
+```bash
+sudo apt install git-core build-essential bc
+cd ~/
+wget https://developer.nvidia.com/downloads/embedded/l4t/r36_release_v4.4/sources/public_sources.tbz2
+# Build Jetson Linux Kernel
+tar -xf public_sources.tbz2
+cd ~/Linux_for_Tegra/source
+tar -xf kernel_src.tbz2
+tar -xf kernel_oot_modules_src.tbz2
+tar -xf nvidia_kernel_display_driver_source.tbz2
+```
+
+the default kernel configuration file used is `kernel/kernel-jammy-src/arch/arm64/configs/defconfig` it is mentioned in `kernel_src_build_env.sh`. 
+Add `CONFIG_ARM64_64K_PAGES=y` in `kernel/kernel-jammy-src/arch/arm64/configs/defconfig`.
+
+Build the kernel, out-of-tree modules and dtbs:
+
+```bash
+make -C kernel
+sudo -E make install -C kernel
+cp kernel/kernel-jammy-src/arch/arm64/boot/Image 
+  /boot/Image
+# Building the NVIDIA Out-of-Tree Modules
+cd ~/Linux_for_Tegra/source
+make modules
+sudo -E make modules_install
+sudo nv-update-initrd
+make dtbs
+cp kernel-devicetree/generic-dts/dtbs/* /boot/dtbs/
+```
+
+#### Configure operating system
+
+```bash
+jetson_clocks
+touch /etc/sysctl.d/rt.conf
+## add below content using vim/vi/nano
+kernel.sched_rt_runtime_us=-1
+kernel.timer_migration=0
+sudo cpufreq-set -c 0 -g performance
+sudo cpufreq-set -c 5 -g performance
+sudo cpufreq-set -c 9 -g performance
+```
+
+### Configure DGX Spark
+
+To setup DGX spark you would need to follow the below steps
+
+NVIDIA-SMI Version: 580.95.05
+CUDA Version: 13.0
+OS: Ubuntu 24
+Kernel: 6.14.0-1013-nvidia-64k (You can use a higher version)
+NVCC: Build cuda_13.0.r13.0/compiler.36424714_0
+UHD: UHD_4.9.0.HEAD-0-g006d7f76 (v4.9.0.0)
+Efficient cores (Cortex-A725): 0,1,2,3,4,10,11,12,13,14
+Performance cores (Cortex-X925): 5,6,7,8,9,15,16,17,18,19
+
+You can use `sudo cpupower -c 0-19 frequency-info` to check the above core distribution. 
+
+
+```bash
+sudo apt update
+sudo apt upgrade
+#reboot
+sudo /etc/apt/apt.conf.d/20auto-upgrades
+APT::Periodic::Update-Package-Lists "0";
+APT::Periodic::Unattended-Upgrade "0";
+# Install 64K kernel corresponding to your running kernel
+sudo apt install linux-image-6.14.0-1013-nvidia-64k
+# You can also fix the kernel so it is not changed with apt upgrade
+sudo sed -i 's/^GRUB_DEFAULT=.*/GRUB_DEFAULT="Advanced options for DGX OS
+GNU\/Linux>DGX OS GNU\/Linux, with Linux 6.14.0-1013-nvidia-64k"/'
+/etc/default/grub
+#Configure the command line to isolate the cores
+cat <<"EOF" | sudo tee /etc/default/grub.d/cmdline.cfg
+GRUB_CMDLINE_LINUX="$GRUB_CMDLINE_LINUX pci=realloc=off pci=pcie_bus_safe
+default_hugepagesz=512M hugepagesz=512M hugepages=24 tsc=reliable
+processor.max_cstate=0 audit=0 idle=poll rcu_nocb_poll nosoftlockup
+irqaffinity=0-3 isolcpus=managed_irq,domain,4-19
+nohz_full=4-19 rcu_nocbs=4-19 earlycon module_blacklist=nouveau
+acpi_power_meter.force_cap_on=y numa_balancing=disable init_on_alloc=0
+preempt=none ras=off iommu=off mitgations=off kpti=off skew_tick=1 nowatchdog
+mce=ignore_ce transparent_hugepage=never"
+EOF
+touch /etc/sysctl.d/rt.conf
+## add below content using vim/vi/nano
+kernel.sched_rt_runtime_us=-1
+kernel.timer_migration=0
+sudo update-grub
+sudo reboot
+# after reboot you can make a script to set the gpu clock speed to maximum  
+nvidia-smi -lgc 2000
+sudo cpupower frequency-set --governor performance
+sudo cpupower idle-set -D 0
+```
+
+### Configuration for Ethernet based USRP
+
+If you want to use Ethernet based USRP like N3XX/X3XX/X4XX you would need to run the below commands: 
+
+```bash
+sudo sysctl -w net.core.wmem_max=62500000
+sudo sysctl -w net.core.rmem_max=62500000
+sudo sysctl -w net.core.wmem_default=62500000
+sudo sysctl -w net.core.rmem_default=62500000
+sudo ethtool -G <usrp-interface> tx <maximum-value> rx <maximum-value>
+```
+
+*NOTE*: Currently we finalizing container images for OAI-NR-UE with GPU offload, we will provide it soon for container based deployments.
+
+## Using nrUE with B2XX
+
+### Build OAI-nrUE
+
+```bash
+mkdir ~/oai-nr-ue
+git clone https://gitlab.eurecom.fr/oai/openairinterface5g.git ~/oai-nr-ue
+cd ~/oai-nr-ue
+git checkout ldpc_offload_cuda
+cd ~/oai-nr-ue/openairinterface5g/cmake_targets
+#Install third-party dependencies
+./build_oai -I -w USRP
+# Install nrue with USRP support, default nvcc location is /usr/local/bin/nvcc
+./build_oai --ninja --build-lib ldpc_cuda --gNB --nrUE -w USRP -C --cmake-opt -DCMAKE_CUDA_COMPILER=<nvcc-location>
+``` 
+
+## Start the nrUE process
+
+Create a UE configuration file based on the PLMN, slicing information, DNN and DNN type you have configured for the UE in the core network.
+
+```bash
+cd ~/oai-nr-ue/
+# touch nr-ue.conf
+```
+
+Sample UE configuration file, you can use the below configuration file and adjust the values as per your testing environment: 
+
+```bash
+cat nr-ue.conf
+
+uicc0 = {
+imsi = "001010000000001";
+key = "fec86ba6eb707ed08905757b1bb44b8f";
+opc= "C42449363BBAD02B66D16BC975D77CC1";
+pdu_sessions = ({ dnn = "oai"; nssai_sst = 1; });
+}
+```
+
+To start the `nr-uesoftmodem` you would need to know the center frequency, bandwidth, band and ssb. This information is printed in the OAI-gNB logs. You can search for the line `[NR_MAC] Command line parameters for OAI UE` for example:
+
+```bash
+[NR_MAC] Command line parameters for OAI UE: -C 3350010000 -r 106 --numerology 1 --band 78 --ssb 513
+```
+
+Once you have this information you can start the `nr-uesoftmodem` process: 
+
+**Command for Jetson AGX Orin**: 
+
+**MANDATORY**: Before starting the UE process it is important to configure the gNB configuration file with `min_rxtxtime` as 6 or higher integer value.
+
+```bash
+cd ~/oai-nr-ue/openairinterface5g/cmake_targets/
+sudo ./nr-uesoftmodem -O ~/oai-nr-ue/nr-ue.conf -C 3350010000 -r 106 --ssb 513 --numerology 1 --band 78 --thread-pool -1 --num-dl-actors 6 --num-ul-actors 2 -E --loader.ldpc.shlibversion _cuda --usrp-tx-thread-config 1 --ue-fo-compensation
+```
+
+**Command for DGX Spark**: 
+
+**MANDATORY**: Before starting the UE process it is important to configure the gNB configuration file with `min_rxtxtime` as 6 or higher integer value.
+
+```bash
+cd ~/oai-nr-ue/openairinterface5g/cmake_targets/
+
+sudo taskset -c "5-9,15-19" ./nr-uesoftmodem -O ~/oai-nr-ue/nr-ue.conf -C 3350010000 -r 106 --ssb 513 --numerology 1 --band 78 --thread-pool -1 -E  --loader.ldpc.shlibversion _cuda --ue-fo-compensation
+```
+
+In the above command task-set allows running the UE only on the performance cores. 
+
+Understand the command which you just used: 
+
+The main RF/cell settings are:
+
+- `-O`: load the UE config file.
+- `-C 3900000000`: set DL carrier to 3.9 GHz.
+- `-r 106`: use 106 PRBs
+- `--numerology 1`: numerology 1, meaning 30 kHz subcarrier spacing.
+- `--band 77`: NR band n77.
+- `--ssb 516`: expect the SSB to start at sub-carrier 516.
+- `--ue-fo-compensation`: enable initial frequency-offset compensation, useful OTA when clocks are not perfectly aligned.
+- `--usrp-tx-thread-config 1`: enable an extra USRP TX processing thread
+- `-E`: enable threequarter_fs, meaning the UE uses three-quarter sampling rate mode. In OAI this reduces sample rate / transport load where supported.
+
+The parallelism/offload settings are:
+
+- `--thread-pool -1`: create a thread-pool worker with no CPU affinity.
+- `--num-dl-actors 6`: use 6 DL actor threads, parallel RX/PDSCH processing workers.
+- `--num-ul-actors 2`: use 2 UL actor threads, parallel TX/PUSCH/PUCCH preparation and transmit workers.
+- `--loader.ldpc.shlibversion_cuda`: load the CUDA-backed LDPC shared library variant.
+
+When the UE starts it starts with highest gains, but sometimes you would need to adjust the tx/rx gains based on the UEs position from the RU/gNB. To manipulate the gains you can use the below options: 
+
+- `--ue-rxgain x`: set UE RX gain.
+- `--ue-txgain x`: set UE TX gain.
+- `-A 50`: apply a command-line sample advance of 50 samples to compensate RF timing offset.
+
+Once the UE connects you will see the tunnel interface `oaitun_ue1` with ip-address in the subnet which is configured at SMF. You can open another terminal and from there you can do traffic test. 
+
+## Using nrUE with X3XX/N3XX/x4XX
+
+### Build OAI-nrUE
+
+For Ethernet based USRPs it is recommended to use DPDK mostly when you want to test for 60-100 MHz bandwidth. If you don't want to use DPDK then you can build the `nr-uesoftmodem` using the previous section. 
+
+```bash
+mkdir ~/oai-nr-ue
+git clone https://gitlab.eurecom.fr/oai/openairinterface5g.git ~/oai-nr-ue
+cd ~/oai-nr-ue
+git checkout ldpc_offload_cuda
+cd ~/oai-nr-ue/openairinterface5g/cmake_targets
+#Install third-party dependencies
+./build_oai -I
+#Build and Install DPDK
+cd ~/
+wget https://fast.dpdk.org/rel/dpdk-24.11.4.tar.xz
+sudo apt install wget xz-utils libnuma-dev meson libibverbs-dev rdma-core python3-pyelftools
+meson setup build -Dplatform=generic
+ninja -C build
+sudo ninja install -C build
+#Build and Install UHD
+sudo apt install autoconf automake build-essential ccache cmake cpufrequtils doxygen ethtool g++ git inetutils-tools libboost-all-dev libncurses-dev libusb-1.0-0 libusb-1.0-0-dev libusb-dev python3-dev python3-mako python3-numpy python3-requests python3-scipy python3-setuptools python3-ruamel.yaml
+git clone https://github.com/EttusResearch/uhd ~/uhd && cd uhd
+git checkout v4.9.0.0
+cd host && mkdir build && cd build
+cmake ../ -GNinja
+ninja
+sudo ninja install
+# Install nrue with USRP support, default nvcc location is /usr/local/bin/nvcc
+./build_oai --ninja --build-lib ldpc_cuda --gNB --nrUE -w USRP -C --cmake-opt -DCMAKE_CUDA_COMPILER=<nvcc-location>
+```
+
+Create the DPDK interfaces for the UHD, use the below script and modify it for your environment. We are using VLAN 80 for DPDK in our environment:  
+
+```bash
+vim setup_sriov_usrp.sh
+```
+
+```bash
+#!/bin/sh
+set -xeu
+
+pci_addr()
+{
+    PF_IF=$1
+    VF_INDEX=$2
+    SYSFS_PATH="/sys/class/net/${PF_IF}/device/virtfn${VF_INDEX}"
+
+    if [ ! -e "$SYSFS_PATH" ]; then
+            echo "VF $VF_INDEX not found for interface $PF_IF"
+                exit 1
+    fi
+    PCI_ADDR=$(basename "$(readlink "$SYSFS_PATH")")
+    echo "$PCI_ADDR"
+}
+IF_NAME=enP2p1s0f1np1 ## interface to reach to the USRP on the UHD host
+HOST_MAC_ADD=00:11:22:33:44:67
+VLAN=80
+MTU=9000
+NUM_VFs=1
+sudo ethtool -G $IF_NAME rx 8160 tx 8160
+sudo sh -c "echo 0 > /sys/class/net/$IF_NAME/device/sriov_numvfs"
+sudo sh -c "echo $NUM_VFs > /sys/class/net/$IF_NAME/device/sriov_numvfs"
+HOST_PCI_ADDR=$(pci_addr $IF_NAME 0)
+# this next 2 lines is for C/U planes
+sudo ip link set $IF_NAME vf 0 mac $HOST_MAC_ADD vlan $VLAN spoofchk off mtu $MTU
+sleep 1
+sudo dpdk-devbind.py --unbind $HOST_PCI_ADDR
+sudo modprobe mlx5_core
+sudo dpdk-devbind.py --bind mlx5_core $HOST_PCI_ADDR
+echo "Successfully configured interface for USRP:
+  - Mac-address: $HOST_MAC_ADD, VLAN: $VLAN, PCI: $HOST_PCI_ADDR"
+exit 0
+```
+
+Create `/etc/uhd/uhd.conf` configuration file for allowing UHD to use dpdk for reaching to the USRP:
+
+```txt
+[use_dpdk=1]
+dpdk_mtu=9000
+dpdk_driver=/usr/local/lib/aarch64-linux-gnu/dpdk/pmds-25.0/
+dpdk_corelist=5,6,7,8,9
+[dpdk_mac=00:11:22:33:44:67]
+dpdk_lcore=6
+dpdk_ipv4=192.168.80.25/24
+```
+
+The ip-address 192.168.80.25/24 is the UHD host ip-address. Make sure you adjust the `dpdk_driver` location. 
+
+
+### Start the nrUE process
+
+Create a UE configuration file based on the PLMN, slicing information, DNN and DNN type you have configured for the UE in the core network. It is similar to previous section. 
+
+To start the `nr-uesoftmodem` process: 
+
+**Command for DGX Spark**: 
+
+```bash
+sudo devlink dev reload pci/<HOST_PCI_ADDR> && sudo taskset -c "5-19" ./nr-uesoftmodem --ue-fo-compensation --thread-pool 15,16,17,18,19 --loader.ldpc.shlibversion _cuda -O ~/oai-nr-ue/nr-ue.conf --usrp-args "addr=192.168.80.251,use_dpdk=1" -C 3350010000 -r 273 --numerology 1 --band 78 --ssb 1515 --usrp-tx-thread-config 1
+```
+
+DPDK deployment have some issues currently and we need to reload the pci interface before every run. The devlink command `sudo devlink dev reload pci/<HOST_PCI_ADDR>` is doing that. 
+
+If you need to adjust the tx/rx gains then you will need to take the parameters from previous section. 

docker/Dockerfile.base.cuda.ubuntu

@@ -0,0 +1,76 @@
+# SPDX-License-Identifier: LicenseRef-CSSL-1.0
+
+#---------------------------------------------------------------------
+#
+# Dockerfile for the Open-Air-Interface BUILD service
+#   Valid for Ubuntu 24.04
+#
+#---------------------------------------------------------------------
+
+
+FROM nvidia/cuda:12.2.0-devel-ubuntu22.04 AS ran-base-cuda
+
+ARG NEEDED_GIT_PROXY
+ARG TARGETARCH
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Paris
+ENV BUILD_UHD_FROM_SOURCE=True
+ENV UHD_VERSION=4.8.0.0
+
+#install developers pkg/repo
+RUN apt-get update && \
+    apt-get upgrade --yes && \
+    apt-get install --yes \
+       #gcc needed for build_oai
+       build-essential \
+       ccache \
+       psmisc \
+       git \
+       #use gcc-12 to avoid problems of default gcc-11 in FlexRIC
+       gcc-12 \
+       g++-12 \
+       xxd \
+       libpcre2-dev \
+       python3-dev \
+       bison \
+       flex \
+       m4 \
+       # python3-pip for conf template generation
+       python3-pip \
+       wget \
+       flatbuffers-compiler \
+       libflatbuffers-dev \
+       libnanomsg-dev && \
+    update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-12 100 --slave /usr/bin/g++ g++ /usr/bin/g++-12 && \
+    pip3 install --ignore-installed pyyaml
+
+# Add "Tini - A tiny but valid init for containers", https://github.com/krallin/tini
+# it will be copied into target containers, to print exit numbers and handle signals properly
+ENV TINI_VERSION=v0.19.0
+RUN wget https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${TARGETARCH} -O /tini && chmod +x /tini
+
+ENV YQ_VERSION=v4.2.0
+RUN wget https://github.com/mikefarah/yq/releases/download/${YQ_VERSION}/yq_linux_${TARGETARCH} -O /usr/local/bin/yq &&\
+    chmod +x /usr/local/bin/yq
+
+# In some network environments, GIT proxy is required
+RUN /bin/bash -c "if [[ -v NEEDED_GIT_PROXY ]]; then git config --global http.proxy $NEEDED_GIT_PROXY; fi"
+
+# Copying only the needed files to create ran-base
+WORKDIR /oai-ran/cmake_targets/tools
+COPY cmake_targets/tools/build_helper \
+     cmake_targets/tools/uhd-4.x-tdd-patch.diff \
+     ./
+
+WORKDIR /oai-ran/cmake_targets
+COPY cmake_targets/build_oai .
+
+WORKDIR /oai-ran
+COPY oaienv .
+
+#run build_oai -I to get the builder image
+RUN /bin/sh oaienv && \ 
+    cd cmake_targets && \
+    mkdir -p log && \
+    ./build_oai -I -w USRP -w BLADERF --install-optional-packages
+

docker/Dockerfile.build.cuda.ubuntu

@@ -0,0 +1,58 @@
+# SPDX-License-Identifier: LicenseRef-CSSL-1.0
+
+#---------------------------------------------------------------------
+#
+# Dockerfile for the Open-Air-Interface BUILD service
+#   Valid for Ubuntu 24.04
+#
+#---------------------------------------------------------------------
+
+FROM ran-base-cuda:latest AS ran-build-cuda
+ARG BUILD_OPTION
+
+ARG E2AP_VERSION=E2AP_V3
+ARG KPM_VERSION=KPM_V3_00
+
+RUN rm -Rf /oai-ran
+WORKDIR /oai-ran
+COPY . .
+
+# build all targets so final targets can be created afterwards
+RUN \
+   # Mount CCache cache direrctory
+   --mount=type=cache,target=/root/.cache/ccache/ \
+   # Mount CPM package cache
+   --mount=type=cache,target=/root/.cache/cpm/ \
+    /bin/sh oaienv && \
+    cd cmake_targets && \
+    mkdir -p log && \
+    ./build_oai -c --ninja \
+      --eNB --gNB --RU --UE --nrUE \
+      --build-lib "telnetsrv enbscope uescope nrscope ldpc_cuda" \
+      -w USRP -t Ethernet -w BLADERF \
+      --build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
+      --cmake-opt -DOAI_VRTSIM_TAPS_CLIENT=ON \
+      --cmake-opt -DCUDA_TOOLKIT_ROOT_DIR=/usr/local/cuda-12.2 \
+      --noavx512 \
+      --build-tool-opt -k10 \
+      --cmake-opt -DCMAKE_C_FLAGS="-Werror" --cmake-opt -DCMAKE_CXX_FLAGS="-Werror" $BUILD_OPTION && \
+    # Mainly to see if the sanitize option was perfectly executed
+    echo "---- ldd on executables ----" && \
+    ldd ran_build/build/*softmodem* ran_build/build/nr-cuup ran_build/build/oairu && \
+    echo "---- ldd on shared libraries ----" && \
+    ldd ran_build/build/*.so
+
+## Build FlexRIC for SM Models
+RUN \
+    # Mount CCache cache direrctory
+    --mount=type=cache,target=/root/.cache/ccache/ \
+    mkdir -p openair2/E2AP/flexric/build && \
+    cd openair2/E2AP/flexric/build && \
+    cmake -GNinja -DCMAKE_BUILD_TYPE=Release \
+          -DKPM_VERSION=$KPM_VERSION \
+          -DCMAKE_C_COMPILER=gcc-12 \
+          -DCMAKE_CXX_COMPILER=g++-12 .. \
+          -DE2AP_VERSION=$E2AP_VERSION .. && \
+    ninja && \
+    ninja install
+

docker/Dockerfile.gNB.cuda.ubuntu

@@ -0,0 +1,115 @@
+# SPDX-License-Identifier: LicenseRef-CSSL-1.0
+
+#---------------------------------------------------------------------
+#
+# Dockerfile for the Open-Air-Interface BUILD service
+#   Valid for Ubuntu 24.04
+#
+#---------------------------------------------------------------------
+
+ARG UBUNTU_IMAGE=ubuntu:noble
+FROM ran-base-cuda:latest AS gnb-base
+FROM ran-build-cuda:latest AS gnb-build
+
+
+#start from scratch for target executable
+FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS oai-gnb
+ARG BUILD_OPTION
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Paris
+
+RUN apt-get update && \
+    apt-get upgrade --yes && \
+    apt-get install --yes \
+        software-properties-common \
+        procps \
+        libsctp1 \
+        libboost-chrono1.74.0 \
+        libboost-date-time1.74.0 \
+        libboost-filesystem1.74.0 \
+        libboost-regex1.74.0 \
+        libboost-serialization1.74.0 \
+        libboost-thread1.74.0 \
+        libboost-system1.74.0 \
+        libboost-program-options1.74.0 \
+        tzdata \
+        libconfig9 \
+        openssl \
+        net-tools \
+        iperf \
+        iperf3 \
+        iproute2 \
+        iputils-ping \
+        gdb \
+        python3 \
+        python3-six \
+        python3-requests \
+        libusb-1.0-0 \
+        libyaml-cpp-dev \
+        libnanomsg5 && \
+# if the --sanitize option was used to build, additional packages are required
+    /bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+        libasan8 \ 
+        libubsan1; fi' && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/oai-gnb/bin
+COPY --from=gnb-build \
+    /oai-ran/cmake_targets/ran_build/build/nr-softmodem ./
+COPY ./docker/scripts/gnb_entrypoint.sh ./entrypoint.sh
+COPY ./docker/scripts/check-prach-io.sh ./check-prach-io.sh
+
+COPY --from=gnb-build \
+    /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \
+    /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \
+    /oai-ran/cmake_targets/ran_build/build/liboai_usrpdevif.so \
+    /oai-ran/cmake_targets/ran_build/build/libcoding.so \
+    /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
+    /oai-ran/cmake_targets/ran_build/build/libdfts.so \
+    /oai-ran/cmake_targets/ran_build/build/libldpc*.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ci.so \
+    /oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_bearer.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_rrc.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_o1.so \
+    /oai-ran/cmake_targets/ran_build/build/libvrtsim.so \
+    /usr/local/lib/
+
+# Now we are copying from builder-image the UHD files.
+COPY --from=gnb-base /usr/local/bin/uhd_find_devices /usr/local/bin
+COPY --from=gnb-base /usr/local/lib/libuhd.so.4.8.0 /usr/local/lib
+COPY --from=gnb-base /usr/local/lib/uhd/utils/uhd_images_downloader.py /opt/oai-gnb/bin
+## Copy E2 SM models
+COPY --from=gnb-build /usr/local/lib/flexric /usr/local/lib/flexric
+
+RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai_device.so" && \
+    ldconfig && \
+    echo "---- ldd on nr-softmodem ----" && \
+    ldd /opt/oai-gnb/bin/nr-softmodem && \
+    echo "---- ldd on shared libraries ----" && \
+    ldd /usr/local/lib/liboai_eth_transpro.so \
+        /usr/local/lib/librfsimulator.so \
+        /usr/local/lib/liboai_usrpdevif.so \
+        /usr/local/lib/libcoding.so \
+        /usr/local/lib/libparams_libconfig.so \
+        /usr/local/lib/libdfts.so \
+        /usr/local/lib/libldpc*.so \
+        /usr/local/lib/libtelnetsrv.so \
+        /usr/local/lib/libtelnetsrv_ci.so \
+        /usr/local/lib/libuhd.so.4.8.0
+
+WORKDIR /opt/oai-gnb
+#EXPOSE 2152/udp  # S1U, GTP/UDP
+#EXPOSE 22100/tcp # ?
+#EXPOSE 36412/udp # S1C, SCTP/UDP
+#EXPOSE 36422/udp # X2C, SCTP/UDP
+#EXPOSE 50000/udp # IF5 / ORI (control)
+#EXPOSE 50001/udp # IF5 / ECPRI (data)
+
+COPY --from=gnb-base /usr/local/bin/yq /usr/local/bin/yq
+COPY --from=gnb-base /tini /tini
+ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-gnb/bin/entrypoint.sh"]
+CMD ["/opt/oai-gnb/bin/nr-softmodem"]

docker/Dockerfile.nrUE.cuda.ubuntu

@@ -0,0 +1,107 @@
+# SPDX-License-Identifier: LicenseRef-CSSL-1.0
+
+#---------------------------------------------------------------------
+#
+# Dockerfile for the Open-Air-Interface BUILD service
+#   Valid for Ubuntu 24.04
+#
+#---------------------------------------------------------------------
+
+FROM ran-base-cuda:latest AS nr-ue-base
+FROM ran-build-cuda:latest AS nr-ue-build
+
+#start from scratch for target executable
+FROM nvidia/cuda:12.2.0-runtime-ubuntu22.04 AS oai-nr-ue
+ARG BUILD_OPTION
+ENV DEBIAN_FRONTEND=noninteractive
+ENV TZ=Europe/Paris
+
+RUN apt-get update && \
+    apt-get upgrade --yes && \
+    apt-get install --yes \
+        arping \
+        software-properties-common \
+        procps \
+        libsctp1 \
+        libboost-chrono1.74.0 \
+        libboost-date-time1.74.0 \
+        libboost-filesystem1.74.0 \
+        libboost-regex1.74.0 \
+        libboost-serialization1.74.0 \
+        libboost-thread1.74.0 \
+        libboost-system1.74.0 \
+        libboost-program-options1.74.0 \
+        tzdata \
+        libconfig9 \
+        openssl \
+        net-tools \
+        gdb \
+        python3 \
+        python3-six \
+        python3-requests \
+        libusb-1.0-0 \
+        iputils-ping \
+        iproute2 \
+        iperf3 \
+        iperf \
+        libyaml-cpp-dev \
+        libnanomsg5 && \
+# if the --sanitize option was used to build, additional packages are required
+    /bin/bash -c 'if [[ "$BUILD_OPTION" = "--sanitize" ]]; then DEBIAN_FRONTEND=noninteractive apt-get install --yes \
+        libasan8 \
+        libubsan1; fi' && \
+    apt-get clean -y && \
+    rm -rf /var/lib/apt/lists/*
+
+WORKDIR /opt/oai-nr-ue/bin
+COPY --from=nr-ue-build \
+    /oai-ran/cmake_targets/ran_build/build/nr-uesoftmodem \
+    ./
+
+COPY ./docker/scripts/nr_ue_entrypoint.sh ./entrypoint.sh
+WORKDIR /opt/oai-nr-ue/etc
+
+COPY --from=nr-ue-build \
+    /oai-ran/cmake_targets/ran_build/build/liboai_eth_transpro.so \
+    /oai-ran/cmake_targets/ran_build/build/librfsimulator.so \
+    /oai-ran/cmake_targets/ran_build/build/liboai_usrpdevif.so \
+    /oai-ran/cmake_targets/ran_build/build/libcoding.so \
+    /oai-ran/cmake_targets/ran_build/build/libparams_libconfig.so \
+    /oai-ran/cmake_targets/ran_build/build/libdfts.so \
+    /oai-ran/cmake_targets/ran_build/build/libldpc*.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_ciUE.so \
+    /oai-ran/cmake_targets/ran_build/build/libtelnetsrv_5Gue.so \
+    /oai-ran/cmake_targets/ran_build/build/libparams_yaml.so \
+    /oai-ran/cmake_targets/ran_build/build/libvrtsim.so \
+    /usr/local/lib/
+
+# Now we are copying from builder-image the UHD files.
+COPY --from=nr-ue-base /usr/local/bin/uhd_find_devices /usr/local/bin
+COPY --from=nr-ue-base /usr/local/lib/libuhd.so.4.8.0 /usr/local/lib
+COPY --from=nr-ue-base /usr/local/lib/uhd/utils/uhd_images_downloader.py /opt/oai-nr-ue/bin
+
+RUN /bin/bash -c "ln -s /usr/local/lib/liboai_usrpdevif.so /usr/local/lib/liboai_device.so" && \
+    ldconfig && \
+    echo "---- ldd on nr-uesoftmodem ----" && \
+    ldd /opt/oai-nr-ue/bin/nr-uesoftmodem && \
+    echo "---- ldd on shared libraries ----" && \
+    ldd /usr/local/lib/liboai_eth_transpro.so \
+        /usr/local/lib/librfsimulator.so \
+        /usr/local/lib/liboai_usrpdevif.so \
+        /usr/local/lib/libcoding.so \
+        /usr/local/lib/libparams_libconfig.so \
+        /usr/local/lib/libdfts.so \
+        /usr/local/lib/libldpc*.so \
+        /usr/local/lib/libtelnetsrv.so \
+        /usr/local/lib/libtelnetsrv_ciUE.so \
+        /usr/local/lib/libtelnetsrv_5Gue.so \
+        /usr/local/lib/libuhd.so.4.8.0
+
+WORKDIR /opt/oai-nr-ue
+
+COPY --from=nr-ue-base /usr/local/bin/yq /usr/local/bin/yq
+COPY --from=nr-ue-base /tini /tini
+CMD ["/opt/oai-nr-ue/bin/nr-uesoftmodem"]
+ENTRYPOINT ["/tini", "-v", "--", "/opt/oai-nr-ue/bin/entrypoint.sh"]
+

openair1/PHY/CODING/TESTBENCH/ldpctest.c

@@ -24,7 +24,9 @@
 #endif
 
 #define NR_LDPC_PROFILER_DETAIL
+
 ldpc_interface_t ldpc_orig, ldpc_toCompare;
+static double modulated_input[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4][68 * 384];
 
 // 4-bit quantizer
 int8_t quantize4bit(double D, double x)
@@ -78,6 +80,9 @@ typedef struct {
   n_iter_stats_t dec_iter;
 } one_measurement_t;
 
+uint8_t *estimated_output;
+int8_t *channel_output_fixed;
+
 one_measurement_t test_ldpc(short max_iterations,
                             int nom_rate,
                             int denom_rate,
@@ -85,14 +90,15 @@ one_measurement_t test_ldpc(short max_iterations,
                             unsigned char qbits,
                             short Kprime,
                             unsigned int ntrials,
-                            int n_segments)
+                            int n_segments,
+                            int gen_code)
 {
   one_measurement_t ret = {0};
   reset_meas(&ret.time_optim);
   reset_meas(&ret.time_decoder);
   // clock initiate
   // time_stats_t time,time_optim,tinput,tprep,tparity,toutput, time_decoder;
-  time_stats_t time, tinput, tprep, tparity, toutput;
+  time_stats_t time, tinput, tinput_memcpy, tprep, tparity, toutput;
   double n_iter_mean = 0;
   double n_iter_std = 0;
   int n_iter_max = 0;
@@ -100,14 +106,13 @@ one_measurement_t test_ldpc(short max_iterations,
   double sigma;
   sigma = 1.0 / sqrt(2 * SNR);
   cpu_meas_enabled = 1;
-  uint8_t *test_input[MAX_NUM_NR_DLSCH_SEGMENTS];
-  uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
-  memset(estimated_output, 0, sizeof(estimated_output));
-  uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
+  uint8_t *test_input[n_segments * NR_MAX_NB_LAYERS];
+  uint8_t *channel_input[n_segments];
   uint8_t *channel_input_optim;
+
   // double channel_output[68 * 384];
-  double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
-  int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
+  memset(modulated_input,0,sizeof(modulated_input));
+
   short BG = 0, nrows = 0; //,ncols;
   int i1, Kb = 0;
   int R_ind = 0;
@@ -115,7 +120,8 @@ one_measurement_t test_ldpc(short max_iterations,
   int code_rate_vec[8] = {15, 13, 25, 12, 23, 34, 56, 89};
   // double code_rate_actual_vec[8] = {0.2, 0.33333, 0.4, 0.5, 0.66667, 0.73333, 0.81481, 0.88};
 
-  t_nrLDPC_dec_params decParams[MAX_NUM_DLSCH_SEGMENTS] = {0};
+  t_nrLDPC_dec_params decParams[n_segments];
+  memset(decParams,0,sizeof(decParams));
 
   t_nrLDPC_time_stats decoder_profiler = {0};
 
@@ -123,6 +129,7 @@ one_measurement_t test_ldpc(short max_iterations,
 
   reset_meas(&time);
   reset_meas(&tinput);
+  reset_meas(&tinput_memcpy);
   reset_meas(&tprep);
   reset_meas(&tparity);
   reset_meas(&toutput);
@@ -241,17 +248,16 @@ one_measurement_t test_ldpc(short max_iterations,
   printf("number of undecoded bits: %d\n", (Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
 
   // generate input block
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+  for (int j = 0; j < n_segments; j++) {
     test_input[j] = malloc16(((K + 7) & ~7) / 8);
     memset(test_input[j], 0, ((K + 7) & ~7) / 8);
     channel_input[j] = malloc16(68 * 384);
     memset(channel_input[j], 0, 68 * 384);
   }
-  channel_input_optim = malloc16(68 * 384);
-  memset(channel_input_optim, 0, 68 * 384);
+  channel_input_optim = malloc16(68 * 384 * sizeof(uint32_t));
 
   // Fill input segments with random values
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+  for (int j = 0; j < n_segments; j++) {
     int i = 0;
     for (i = 0; i < ((Kprime + 7) & ~7) / 8; i++)
       test_input[j][i] = (uint8_t)rand();
@@ -263,7 +269,11 @@ one_measurement_t test_ldpc(short max_iterations,
   }
 
   encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K};
-  impp.gen_code = 1;
+  impp.gen_code = gen_code;
+  impp.tparity = &tparity;
+  impp.tinput = &tinput;
+  impp.tinput_memcpy = &tinput_memcpy;
+  impp.toutput = &toutput;
 
   if (ntrials == 0)
     ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
@@ -289,7 +299,7 @@ one_measurement_t test_ldpc(short max_iterations,
       for (int j = 0; j < n_segments; j++)
         for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
           if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
-            printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
+            printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (((uint32_t*)channel_input_optim)[i] >> j) & 0x1);
             return ret;
           }
         }
@@ -306,11 +316,11 @@ one_measurement_t test_ldpc(short max_iterations,
         else
           modulated_input[j][i] = -1.0; /// sqrt(2);
 
-        channel_output_fixed[j][i] =
+        channel_output_fixed[j*(384*68) + i] =
             (int8_t)quantize(sigma / 4.0 / 4.0, modulated_input[j][i] + sigma * gaussdouble(0.0, 1.0), qbits);
 
         // Uncoded BER
-        uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0;
+        uint8_t channel_output_uncoded = channel_output_fixed[(j*384*68)+i] < 0 ? 1 /* QPSK demod */ : 0;
         if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1))
           ret.errors_bit_uncoded++;
       }
@@ -326,25 +336,26 @@ one_measurement_t test_ldpc(short max_iterations,
       decParams[j].numMaxIter = max_iterations;
       decParams[j].outMode = nrLDPC_outMode_BIT;
       decParams[j].Kprime = Kprime;
-      ldpc_toCompare.LDPCinit();
+      decParams[j].n_segments = n_segments;
     }
+
+    //Decoder
     for (int j = 0; j < n_segments; j++) {
       start_meas(&ret.time_decoder);
       set_abort(&dec_abort, false);
-
       n_iter = ldpc_toCompare.LDPCdecoder(&decParams[j],
-                                          (int8_t *)channel_output_fixed[j],
-                                          estimated_output[j],
+                                          &channel_output_fixed[j*384*68],
+                                          &estimated_output[j*Kprime],
                                           &decoder_profiler,
                                           &dec_abort);
       stop_meas(&ret.time_decoder);
 
       // count errors
-      if (memcmp(estimated_output[j], test_input[j], ((Kprime + 7) & ~7) / 8) != 0) {
+      if (memcmp(&estimated_output[j*Kprime], test_input[j], ((Kprime + 7) & ~7) / 8) != 0) {
         segment_bler++;
       }
       for (int i = 0; i < Kprime; i++) {
-        unsigned char estoutputbit = (estimated_output[j][i / 8] & (1 << (i & 7))) >> (i & 7);
+        unsigned char estoutputbit = (estimated_output[j*Kprime + (i / 8)] & (1 << (i & 7))) >> (i & 7);
         unsigned char inputbit = (test_input[j][i / 8] & (1 << (i & 7))) >> (i & 7); // Further correct for multiple segments
         if (estoutputbit != inputbit)
           ret.errors_bit++;
@@ -369,7 +380,7 @@ one_measurement_t test_ldpc(short max_iterations,
 
   ret.errors_bit_uncoded = ret.errors_bit_uncoded / (double)((Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
 
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+  for (int j = 0; j < n_segments; j++) {
     free(test_input[j]);
     free(channel_input[j]);
   }
@@ -378,6 +389,7 @@ one_measurement_t test_ldpc(short max_iterations,
   print_meas(&time, "ldpc_encoder", NULL, NULL);
   print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);
   print_meas(&tinput, "ldpc_encoder_optim(input)", NULL, NULL);
+  print_meas(&tinput_memcpy, "ldpc_encoder_optim(input memcpy)", NULL, NULL);
   print_meas(&tprep, "ldpc_encoder_optim(prep)", NULL, NULL);
   print_meas(&tparity, "ldpc_encoder_optim(parity)", NULL, NULL);
   print_meas(&toutput, "ldpc_encoder_optim(output)", NULL, NULL);
@@ -419,6 +431,7 @@ int main(int argc, char *argv[])
   int n_trials = 1;
   double SNR_step = 0.1;
 
+  int gen_code = 1;
   randominit();
   int test_uncoded = 0;
   n_iter_stats_t dec_iter[400] = {0};
@@ -430,7 +443,7 @@ int main(int argc, char *argv[])
   }
   logInit();
 
-  while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:G:n:d:i:t:u:hv:")) != -1) {
+  while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:Gn:d:i:t:u:hv:g:Pn:")) != -1) {
     /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
     /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
     if (c == 1 || c == '-' || c == 'O')
@@ -454,10 +467,6 @@ int main(int argc, char *argv[])
         Kprime = atoi(optarg);
         break;
 
-      case 'G':
-        ldpc_version = "_cuda";
-        break;
-
       case 'n':
         n_trials = atoi(optarg);
         break;
@@ -484,6 +493,11 @@ int main(int argc, char *argv[])
       case 'v':
         ldpc_version = strdup(optarg);
         break;
+      case 'g':
+        gen_code = atoi(optarg);
+        AssertFatal(gen_code <= 4, "gen_code %d is not allowed\n", gen_code);
+        n_trials = 0;
+        break;
       case 'h':
       default:
         printf("CURRENTLY SUPPORTED CODE RATES: \n");
@@ -494,7 +508,7 @@ int main(int argc, char *argv[])
         printf("-r Nominator rate, (1, 2, 22), Default: 1\n");
         printf("-d Denominator rate, (3, 5, 25), Default: 1\n");
         printf("-l Length of payload bits in a segment (K' in 38.212-5.2.2), [1, 8448], Default: 8448\n");
-        printf("-G give 1 to run cuda for LDPC, Default: 0\n");
+        printf("-P give 1 to run cuda stream parallel for LDPC, Default: 0\n");
         printf("-n Number of simulation trials, Default: 1\n");
         // printf("-M MCS2 for TB 2\n");
         printf("-s SNR per information bit (EbNo) in dB, Default: -2\n");
@@ -511,11 +525,16 @@ int main(int argc, char *argv[])
   printf("n_trials %d: \n", n_trials);
   printf("SNR0 %f: \n", SNR0);
 
+  printf("loading %s\n",ldpc_version);
   load_LDPClib(ldpc_version, &ldpc_toCompare);
+  ldpc_toCompare.LDPCinit();
   load_LDPClib("_orig", &ldpc_orig);
+  ldpc_orig.LDPCinit();
 
   // find minimum value in all sets of lifting size
   Zc = 0;
+  estimated_output = malloc(n_segments * Kprime);
+  channel_output_fixed=malloc(n_segments * 68 * 384);
 
   char fname[200];
   sprintf(fname, "ldpctest_BG_%d_Zc_%d_rate_%d-%d_Kprime_%d_maxit_%d.txt", BG, Zc, nom_rate, denom_rate, Kprime, max_iterations);
@@ -532,7 +551,7 @@ int main(int argc, char *argv[])
       SNR_lin = pow(10, SNR / 10.0);
     else
       SNR_lin = pow(10, SNR / 10.0) * nom_rate / denom_rate;
-    printf("Linear SNR: %f\n", SNR_lin);
+    printf("Linear SNR: %d %f, %d/%d, Kprime %d\n", max_iterations, SNR_lin, nom_rate,denom_rate,Kprime);
     one_measurement_t res = test_ldpc(max_iterations,
                                       nom_rate,
                                       denom_rate,
@@ -540,7 +559,8 @@ int main(int argc, char *argv[])
                                       qbits,
                                       Kprime, // block length bytes
                                       n_trials,
-                                      n_segments);
+                                      n_segments,
+                                      gen_code);
 
     decoded_errors[i] = res.errors;
     dec_iter[i] = res.dec_iter;
@@ -566,10 +586,8 @@ int main(int argc, char *argv[])
            sqrt((double)t_optim->diff_square / t_optim->trials / pow(1000, 2) / pow(cpu_freq, 2)
                 - pow((double)t_optim->diff / t_optim->trials / 1000.0 / cpu_freq, 2)));
     printf("Encoding time max: %15.3f us\n", (double)t_optim->max / 1000.0 / cpu_freq);
-    printf("\n");
-
     time_stats_t *t_decoder = &res.time_decoder;
-    printf("Decoding time mean: %15.3f us\n", (double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq);
+    printf("Decoding time mean (per segment): %15.3f us\n", (double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq);
     printf("Decoding time std: %15.3f us\n",
            sqrt((double)t_decoder->diff_square / t_decoder->trials / pow(1000, 2) / pow(cpu_freq, 2)
                 - pow((double)t_decoder->diff / t_decoder->trials / 1000.0 / cpu_freq, 2)));

openair1/PHY/CODING/nrLDPC_coding/CMakeLists.txt

@@ -2,3 +2,7 @@
 
 add_subdirectory(nrLDPC_coding_segment)
 add_subdirectory(nrLDPC_coding_aal)
+add_boolean_option(ENABLE_LDPC_CUDA OFF "Build support for CUDA" OFF)
+if (ENABLE_LDPC_CUDA)
+  add_subdirectory(nrLDPC_coding_cuda)
+endif()

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/CMakeLists.txt

@@ -0,0 +1,71 @@
+enable_language(CUDA)
+
+# it would be possible to auto-detect compute capailities: https://stackoverflow.com/a/68223399
+#include(FindCUDA/select_compute_arch)
+#CUDA_DETECT_INSTALLED_GPUS(INSTALLED_GPU_CCS_1)
+
+# Nvidia recommends to build against all targeted archs: https://forums.developer.nvidia.com/t/261162
+# this is for cmake 3.16 (OAI minimum), starting at cmake 3.18 there is another syntax, see below
+
+SET(CUDA_VERBOSE_BUILD ON)
+
+# version using NVLink-C2C in Grace Hopper
+add_library(ldpc_cuda MODULE
+            # main entry
+            nrLDPC_coding_cuda.c
+
+            nrLDPC_decoder_cuda.c
+            ldpc_encoder_cuda32.c
+            ldpc_input32.c
+
+            nrLDPC_coding_segment_encoder_cuda.c
+            nrLDPC_coding_segment_decoder_cuda.c
+
+            # .cu files
+            nrLDPC_decoder_BG1_cuda.cu
+            ldpc_BG1_Zc384_32bit.cu
+            ldpc_input.cu
+            ldpc_deinterleave.cu
+            ldpc_rate_matching.cu
+)
+set_target_properties(ldpc_cuda PROPERTIES CUDA_ARCHITECTURES "80;87;90")
+# we need to find the right include directory, but ti seems that
+# enable_language(CUDA) does not make this available(?), so also use the CUDA package
+find_package(CUDA REQUIRED)
+target_include_directories(ldpc_cuda PRIVATE ${CUDA_INCLUDE_DIRS})
+
+target_link_libraries(ldpc_cuda ldpc_gen_HEADERS log_headers)
+set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+
+add_dependencies(ldpctest ldpc_cuda)
+add_dependencies(nr-softmodem ldpc_cuda)
+add_dependencies(nr-uesoftmodem ldpc_cuda)
+add_dependencies(nr_ulsim ldpc_cuda)
+add_dependencies(nr_ulschsim ldpc_cuda)
+add_dependencies(nr_dlsim ldpc_cuda)
+add_dependencies(nr_dlschsim ldpc_cuda)
+
+if(ENABLE_PHYSIM_TESTS)
+
+  # this is a list of physical simulators designed to test the functioning of
+  # ldpc_cuda. Of course, one could also modify
+  # openair1/SIMULATION/tests/CMakeLists.txt to run them with ldpc_cuda
+
+  add_physim_test(ldpctest.ldpc_cuda.simple "ldpctest: simple CUDA load check" ldpctest -v _cuda -l8448 -s10 -n300)
+  add_physim_test(ldpctest.ldpc_cuda.r13 "ldpctest: base graph 1, rate 1/3" ldpctest -v _cuda -r1 -n300 -S64 -s4 -i4)
+  add_physim_test(ldpctest.ldpc_cuda.r23 "ldpctest: base graph 1, rate 2/3" ldpctest -v _cuda -r2 -n300 -S64 -s4 -i4)
+  add_physim_test(ldpctest.ldpc_cuda.r89 "ldpctest: base graph 1, rate 8/9" ldpctest -v _cuda -r22 -d25 -n300 -S64 -s4 -i4)
+
+  add_physim_test(nr_dlsim.ldpc_cuda.9seg_r13 "nr_dlsim: 9 segments, rate 1/3" nr_dlsim -n100 -s20 -e13 -R273 -b273 -x1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_dlsim.ldpc_cuda.18seg_r23 "nr_dlsim: 18 segments, rate 2/3" nr_dlsim -n100 -s20 -e22 -R273 -b273 -x1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_dlsim.ldpc_cuda.50seg_r89 "nr_dlsim: 50 segments, rate 8/9" nr_dlsim -n100 -s40 -e27 -R273 -b273 -z2 -x2 -y2 -P --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_dlsim.ldpc_cuda.2t2r_mcs25_64qam "nr_dlsim: 2T2R/2 layers, MCS 25, 64QAM, 100 MHz 30 kHz SCS)" nr_dlsim -n100 -R273 -b273 -P -e 25 -s30 -x2 -y2 -z2 --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_dlsim.ldpc_cuda.1t1r_mcs25_256qam "nr_dlsim: 1T1R/1 layer, MCS 25, 256QAM, 100 MHz 30 kHz SCS)" nr_dlsim -n100 -R273 -b273 -P -e 25 -s30 -q 1 --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_dlsim.ldpc_cuda.1t1r_mcs25_256qam_bw "nr_dlsim: 1T1R/1 layer, MCS 25 256QAM, 40 MHz 30 kHz SCS" nr_dlsim -n100 -R106 -b106 -P -e 25 -s30 -q 1 --loader.ldpc.shlibversion _cuda)
+
+  add_physim_test(nr_ulsim.ldpc_cuda.9seg_r13 "nr_ulsim: 9 segments, rate 1/3" nr_ulsim -n100 -s20 -m13 -R273 -r273 -W1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_ulsim.ldpc_cuda.18seg_r23 "nr_ulsim: 18 segments, rate 2/3" nr_ulsim -n100 -s20 -m22 -R273 -r273 -W1 -y1 -z1 -P --loader.ldpc.shlibversion _cuda)
+  add_physim_test(nr_ulsim.ldpc_cuda.50seg_r89 "nr_ulsim: 50 segments, rate 8/9" nr_ulsim -n100 -s40 -m27 -R273 -r273 -W2 -I9 -y2 -z2 -P --loader.ldpc.shlibversion _cuda)
+
+endif()

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_BG1_Zc384_32bit.cu

@@ -0,0 +1,216 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <cuda_runtime.h>
+// generated code for Zc=384, byte encoding
+__global__ void ldpc_BG1_Zc384_worker(uint32_t *c[4],uint32_t *d[4]) {
+  uint32_t *c32=c[blockIdx.x];
+  uint32_t *d32=d[blockIdx.x]+8448-2*384;
+
+  int i2 = threadIdx.x;
+  int i1 = blockIdx.y;
+  // copy 20 c values to d
+  if (i1<20) d[blockIdx.x][(i1*384) + i2] = c32[4*384+(2*384*i1)+i2]; 
+
+  if (i2 < 384) {
+    c32+=i2;
+    d32+=i2;
+    switch(i1) {
+    case 0:
+      d32[0]=c32[307]^c32[76]^c32[205]^c32[276]^c32[787]^c32[1018]^c32[855]^c32[1586]^c32[1612]^c32[1864]^c32[2673]^c32[2377]^c32[2304]^c32[3360]^c32[3404]^c32[3347]^c32[4021]^c32[3984]^c32[4096]^c32[4824]^c32[4769]^c32[4807]^c32[5707]^c32[5643]^c32[5529]^c32[6475]^c32[6304]^c32[6200]^c32[7229]^c32[7090]^c32[6975]^c32[7968]^c32[7809]^c32[7812]^c32[8557]^c32[8743]^c32[8753]^c32[9233]^c32[9558]^c32[9447]^c32[10341]^c32[10184]^c32[10325]^c32[10969]^c32[10840]^c32[10964]^c32[11735]^c32[11619]^c32[11573]^c32[12394]^c32[12642]^c32[12592]^c32[13170]^c32[13187]^c32[13356]^c32[14066]^c32[14064]^c32[14095]^c32[14772]^c32[14923]^c32[14797]^c32[15690]^c32[15373]^c32[15399]^c32[16474]^c32[16240]^c32[16485];
+
+       break;
+    case 1:
+      d32[384]=c32[307]^c32[308]^c32[77]^c32[206]^c32[277]^c32[787]^c32[788]^c32[1019]^c32[856]^c32[1586]^c32[1587]^c32[1613]^c32[1865]^c32[2673]^c32[2674]^c32[2378]^c32[2305]^c32[3361]^c32[3405]^c32[3348]^c32[4021]^c32[4022]^c32[3985]^c32[4097]^c32[4824]^c32[4825]^c32[4770]^c32[4808]^c32[5708]^c32[5644]^c32[5530]^c32[6476]^c32[6305]^c32[6201]^c32[7229]^c32[7230]^c32[7091]^c32[6976]^c32[7968]^c32[7969]^c32[7810]^c32[7813]^c32[8557]^c32[8558]^c32[8744]^c32[8754]^c32[9233]^c32[9234]^c32[9559]^c32[9448]^c32[10341]^c32[10342]^c32[10185]^c32[10326]^c32[10970]^c32[10841]^c32[10965]^c32[11735]^c32[11736]^c32[11620]^c32[11574]^c32[12394]^c32[12395]^c32[12643]^c32[12593]^c32[13171]^c32[13188]^c32[13357]^c32[14066]^c32[14067]^c32[14065]^c32[14096]^c32[14772]^c32[14773]^c32[14924]^c32[14798]^c32[15690]^c32[15691]^c32[15374]^c32[15400]^c32[16474]^c32[16475]^c32[16241]^c32[16486];
+
+       break;
+    case 2:
+      d32[768]=c32[308]^c32[77]^c32[205]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1018]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1864]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3404]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4096]^c32[4097]^c32[4825]^c32[4769]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5643]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6304]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6975]^c32[6976]^c32[7969]^c32[7809]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10184]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10840]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11573]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13187]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14064]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14797]^c32[14798]^c32[15691]^c32[15373]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
+
+       break;
+    case 3:
+      d32[1152]=c32[308]^c32[77]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4097]^c32[4825]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6976]^c32[7969]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14798]^c32[15691]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
+
+       break;
+    case 4:
+      d32[1536]=c32[332]^c32[949];
+
+       break;
+    case 5:
+      d32[1920]=c32[80]^c32[233]^c32[362]^c32[49]^c32[195]^c32[944]^c32[791]^c32[1012]^c32[782]^c32[1743]^c32[1769]^c32[1637]^c32[2446]^c32[2534]^c32[2461]^c32[2419]^c32[3133]^c32[3177]^c32[3120]^c32[4178]^c32[4141]^c32[3869]^c32[4981]^c32[4926]^c32[4964]^c32[5480]^c32[5416]^c32[5686]^c32[6248]^c32[6461]^c32[6357]^c32[7002]^c32[7247]^c32[7132]^c32[7741]^c32[7966]^c32[7969]^c32[8714]^c32[8516]^c32[8526]^c32[9390]^c32[9331]^c32[9220]^c32[9382]^c32[10114]^c32[10341]^c32[10098]^c32[11126]^c32[10997]^c32[11121]^c32[11892]^c32[11776]^c32[11730]^c32[12551]^c32[12415]^c32[12365]^c32[12529]^c32[13327]^c32[13344]^c32[13129]^c32[13839]^c32[13837]^c32[13868]^c32[14929]^c32[14696]^c32[14954]^c32[15463]^c32[15530]^c32[15556]^c32[16247]^c32[16397]^c32[16258]^c32[16179];
+
+       break;
+    case 6:
+      d32[2304]=c32[278]^c32[4865]^c32[7681]^c32[8799]^c32[10076]^c32[13309]^c32[13842]^c32[15585];
+
+       break;
+    case 7:
+      d32[2688]=c32[9]^c32[830]^c32[3388]^c32[5709]^c32[6434]^c32[10866];
+
+       break;
+    case 8:
+      d32[3072]=c32[374]^c32[94]^c32[143]^c32[247]^c32[272]^c32[375]^c32[376]^c32[343]^c32[62]^c32[63]^c32[307]^c32[854]^c32[958]^c32[1085]^c32[804]^c32[805]^c32[922]^c32[1025]^c32[1026]^c32[947]^c32[1653]^c32[1757]^c32[1679]^c32[1783]^c32[1547]^c32[1650]^c32[1651]^c32[2356]^c32[2460]^c32[2444]^c32[2548]^c32[2371]^c32[2474]^c32[2475]^c32[2469]^c32[3427]^c32[3147]^c32[3087]^c32[3190]^c32[3191]^c32[3414]^c32[3133]^c32[3134]^c32[4088]^c32[4192]^c32[4051]^c32[4155]^c32[4163]^c32[3882]^c32[3883]^c32[4891]^c32[4611]^c32[4836]^c32[4939]^c32[4940]^c32[4874]^c32[4977]^c32[4978]^c32[5390]^c32[5494]^c32[5710]^c32[5429]^c32[5430]^c32[5596]^c32[5699]^c32[5700]^c32[6158]^c32[6262]^c32[6371]^c32[6474]^c32[6475]^c32[6267]^c32[6370]^c32[6371]^c32[6912]^c32[7016]^c32[7157]^c32[7261]^c32[7042]^c32[7145]^c32[7146]^c32[8035]^c32[7755]^c32[7876]^c32[7979]^c32[7980]^c32[7879]^c32[7982]^c32[7983]^c32[8624]^c32[8728]^c32[8810]^c32[8530]^c32[8820]^c32[8539]^c32[8540]^c32[9300]^c32[9404]^c32[9241]^c32[9345]^c32[9514]^c32[9233]^c32[9234]^c32[9234]^c32[10024]^c32[10128]^c32[10251]^c32[10354]^c32[10355]^c32[10008]^c32[10111]^c32[10112]^c32[11036]^c32[10756]^c32[10907]^c32[11010]^c32[11011]^c32[11031]^c32[11134]^c32[11135]^c32[11802]^c32[11522]^c32[11686]^c32[11790]^c32[11640]^c32[11743]^c32[11744]^c32[12461]^c32[12565]^c32[12325]^c32[12429]^c32[12659]^c32[12378]^c32[12379]^c32[12327]^c32[13237]^c32[13341]^c32[13254]^c32[13357]^c32[13358]^c32[13423]^c32[13142]^c32[13143]^c32[14133]^c32[13853]^c32[14131]^c32[13850]^c32[13851]^c32[14162]^c32[13881]^c32[13882]^c32[14839]^c32[14943]^c32[14606]^c32[14710]^c32[14864]^c32[14967]^c32[14968]^c32[14816]^c32[15373]^c32[15477]^c32[15440]^c32[15543]^c32[15544]^c32[15466]^c32[15569]^c32[15570]^c32[16157]^c32[16261]^c32[16307]^c32[16411]^c32[16168]^c32[16271]^c32[16272]^c32[16496];
+
+       break;
+    case 9:
+      d32[3456]=c32[366]^c32[1000]^c32[8001]^c32[8581]^c32[10041]^c32[13359]^c32[13887]^c32[15442];
+
+       break;
+    case 10:
+      d32[3840]=c32[869]^c32[1875]^c32[3346]^c32[5487]^c32[6527]^c32[11106];
+
+       break;
+    case 11:
+      d32[4224]=c32[257]^c32[38]^c32[39]^c32[26]^c32[192]^c32[155]^c32[321]^c32[226]^c32[8]^c32[48]^c32[1121]^c32[902]^c32[903]^c32[968]^c32[1134]^c32[805]^c32[971]^c32[870]^c32[1536]^c32[1701]^c32[1702]^c32[1562]^c32[1728]^c32[1814]^c32[1596]^c32[2623]^c32[2404]^c32[2405]^c32[2327]^c32[2493]^c32[2638]^c32[2420]^c32[3310]^c32[3092]^c32[3354]^c32[3136]^c32[3297]^c32[3079]^c32[3971]^c32[4136]^c32[4137]^c32[3934]^c32[4100]^c32[4046]^c32[4212]^c32[4774]^c32[4939]^c32[4940]^c32[4719]^c32[4885]^c32[4757]^c32[4923]^c32[5657]^c32[5439]^c32[5593]^c32[5759]^c32[5479]^c32[5645]^c32[6425]^c32[6207]^c32[6254]^c32[6420]^c32[6150]^c32[6316]^c32[7179]^c32[6960]^c32[6961]^c32[7040]^c32[7206]^c32[6925]^c32[7091]^c32[7918]^c32[7699]^c32[7700]^c32[7759]^c32[7925]^c32[7762]^c32[7928]^c32[8507]^c32[8672]^c32[8673]^c32[8693]^c32[8475]^c32[8703]^c32[8485]^c32[9567]^c32[9348]^c32[9349]^c32[9508]^c32[9290]^c32[9397]^c32[9563]^c32[9224]^c32[10291]^c32[10072]^c32[10073]^c32[10134]^c32[10300]^c32[10275]^c32[10057]^c32[10919]^c32[11085]^c32[10790]^c32[10956]^c32[10914]^c32[11080]^c32[11685]^c32[11850]^c32[11851]^c32[11569]^c32[11735]^c32[11523]^c32[11689]^c32[12344]^c32[12509]^c32[12510]^c32[12592]^c32[12374]^c32[12542]^c32[12324]^c32[12335]^c32[13120]^c32[13286]^c32[13137]^c32[13303]^c32[13306]^c32[13088]^c32[14016]^c32[14181]^c32[14182]^c32[14014]^c32[14180]^c32[14045]^c32[13827]^c32[14722]^c32[14887]^c32[14888]^c32[14873]^c32[14655]^c32[14747]^c32[14913]^c32[15640]^c32[15421]^c32[15422]^c32[15707]^c32[15489]^c32[15733]^c32[15515]^c32[16424]^c32[16205]^c32[16206]^c32[16190]^c32[16356]^c32[16435]^c32[16217]^c32[16316];
+
+       break;
+    case 12:
+      d32[4608]=c32[77]^c32[954]^c32[7854]^c32[8680]^c32[10034]^c32[13898];
+
+       break;
+    case 13:
+      d32[4992]=c32[293]^c32[294]^c32[63]^c32[192]^c32[263]^c32[313]^c32[773]^c32[774]^c32[1005]^c32[842]^c32[1572]^c32[1573]^c32[1599]^c32[1851]^c32[2659]^c32[2660]^c32[2364]^c32[2675]^c32[2481]^c32[3347]^c32[3391]^c32[3334]^c32[4007]^c32[4008]^c32[3971]^c32[4083]^c32[4810]^c32[4811]^c32[4756]^c32[4794]^c32[5694]^c32[5630]^c32[5516]^c32[5642]^c32[6462]^c32[6291]^c32[6187]^c32[7215]^c32[7216]^c32[7077]^c32[6962]^c32[7954]^c32[7955]^c32[7796]^c32[7799]^c32[8543]^c32[8544]^c32[8730]^c32[8740]^c32[9219]^c32[9220]^c32[9545]^c32[9434]^c32[10327]^c32[10328]^c32[10171]^c32[10312]^c32[10956]^c32[10827]^c32[10951]^c32[11721]^c32[11722]^c32[11606]^c32[11560]^c32[12380]^c32[12381]^c32[12629]^c32[12579]^c32[13157]^c32[13174]^c32[13343]^c32[14052]^c32[14053]^c32[14051]^c32[14082]^c32[14758]^c32[14759]^c32[14910]^c32[14784]^c32[15676]^c32[15677]^c32[15360]^c32[15386]^c32[15475]^c32[16460]^c32[16461]^c32[16227]^c32[16472];
+
+       break;
+    case 14:
+      d32[5376]=c32[142]^c32[9464]^c32[11657]^c32[12377]^c32[13403]^c32[16140];
+
+       break;
+    case 15:
+      d32[5760]=c32[193]^c32[346]^c32[91]^c32[161]^c32[162]^c32[241]^c32[1057]^c32[904]^c32[1124]^c32[1125]^c32[770]^c32[1856]^c32[1882]^c32[1750]^c32[2559]^c32[2647]^c32[2573]^c32[2574]^c32[3246]^c32[3290]^c32[3232]^c32[3233]^c32[3907]^c32[3870]^c32[3982]^c32[4710]^c32[4655]^c32[4692]^c32[4693]^c32[5593]^c32[5529]^c32[5414]^c32[5415]^c32[6361]^c32[6190]^c32[6469]^c32[6470]^c32[7115]^c32[6976]^c32[7245]^c32[7854]^c32[7695]^c32[7697]^c32[7698]^c32[7890]^c32[8827]^c32[8629]^c32[8638]^c32[8639]^c32[9503]^c32[9444]^c32[9332]^c32[9333]^c32[10227]^c32[10070]^c32[10210]^c32[10211]^c32[10302]^c32[10855]^c32[11110]^c32[10849]^c32[10850]^c32[11621]^c32[11889]^c32[11843]^c32[12664]^c32[12528]^c32[12477]^c32[12478]^c32[13056]^c32[13073]^c32[13241]^c32[13242]^c32[13952]^c32[13950]^c32[13980]^c32[13981]^c32[13879]^c32[14658]^c32[14809]^c32[14683]^c32[15576]^c32[15643]^c32[15668]^c32[15669]^c32[16360]^c32[16510]^c32[16370]^c32[16371];
+
+       break;
+    case 16:
+      d32[6144]=c32[364]^c32[133]^c32[262]^c32[333]^c32[844]^c32[1075]^c32[912]^c32[781]^c32[1643]^c32[1669]^c32[1537]^c32[2346]^c32[2434]^c32[2361]^c32[2642]^c32[3417]^c32[3077]^c32[3404]^c32[4078]^c32[4041]^c32[4153]^c32[4881]^c32[4826]^c32[4864]^c32[5380]^c32[5700]^c32[5586]^c32[6148]^c32[6361]^c32[6257]^c32[7286]^c32[7147]^c32[7032]^c32[8025]^c32[7866]^c32[7869]^c32[8614]^c32[8800]^c32[8810]^c32[8505]^c32[9290]^c32[9231]^c32[9504]^c32[10014]^c32[10241]^c32[9998]^c32[11026]^c32[10897]^c32[11021]^c32[11792]^c32[11676]^c32[11630]^c32[12451]^c32[12315]^c32[12649]^c32[13227]^c32[13244]^c32[13413]^c32[14123]^c32[14121]^c32[14152]^c32[14829]^c32[14596]^c32[14854]^c32[15363]^c32[15430]^c32[15456]^c32[15649]^c32[16147]^c32[16297]^c32[16158];
+
+       break;
+    case 17:
+      d32[6528]=c32[260]^c32[11055]^c32[12369]^c32[13414]^c32[16503];
+
+       break;
+    case 18:
+      d32[6912]=c32[898]^c32[9379]^c32[10264]^c32[13956]^c32[14596];
+
+       break;
+    case 19:
+      d32[7296]=c32[145]^c32[981]^c32[5720]^c32[6386]^c32[7877];
+
+       break;
+    case 20:
+      d32[7680]=c32[366]^c32[135]^c32[264]^c32[335]^c32[187]^c32[846]^c32[1077]^c32[914]^c32[1645]^c32[1671]^c32[1539]^c32[2348]^c32[2436]^c32[2363]^c32[2510]^c32[3419]^c32[3079]^c32[3406]^c32[4080]^c32[4043]^c32[4155]^c32[4883]^c32[4828]^c32[4866]^c32[5382]^c32[5702]^c32[5588]^c32[6150]^c32[6363]^c32[6259]^c32[7288]^c32[7149]^c32[7034]^c32[7176]^c32[8027]^c32[7868]^c32[7871]^c32[8616]^c32[8802]^c32[8812]^c32[8789]^c32[9292]^c32[9233]^c32[9506]^c32[10016]^c32[10243]^c32[10000]^c32[11028]^c32[10899]^c32[11023]^c32[11794]^c32[11678]^c32[11632]^c32[12453]^c32[12317]^c32[12651]^c32[13229]^c32[13246]^c32[13415]^c32[14125]^c32[14123]^c32[14154]^c32[14831]^c32[14598]^c32[14856]^c32[15365]^c32[15432]^c32[15458]^c32[16149]^c32[16299]^c32[16160];
+
+       break;
+    case 21:
+      d32[8064]=c32[973]^c32[3942]^c32[12616]^c32[15573]^c32[16225];
+
+       break;
+    case 22:
+      d32[8448]=c32[30]^c32[9227]^c32[10217]^c32[13078];
+
+       break;
+    case 23:
+      d32[8832]=c32[792]^c32[1625]^c32[7741]^c32[13851];
+
+       break;
+    case 24:
+      d32[9216]=c32[157]^c32[310]^c32[55]^c32[126]^c32[298]^c32[1021]^c32[868]^c32[1089]^c32[1820]^c32[1846]^c32[1714]^c32[2523]^c32[2611]^c32[2538]^c32[2462]^c32[3210]^c32[3254]^c32[3197]^c32[3307]^c32[3871]^c32[4218]^c32[3946]^c32[4674]^c32[4619]^c32[4657]^c32[5557]^c32[5493]^c32[5379]^c32[6325]^c32[6154]^c32[6434]^c32[7079]^c32[6940]^c32[7209]^c32[7818]^c32[8043]^c32[8046]^c32[8791]^c32[8593]^c32[8603]^c32[8787]^c32[9467]^c32[9408]^c32[9297]^c32[10191]^c32[10034]^c32[10175]^c32[10819]^c32[11074]^c32[10814]^c32[11585]^c32[11853]^c32[11807]^c32[12628]^c32[12492]^c32[12442]^c32[13404]^c32[13421]^c32[13206]^c32[13916]^c32[13914]^c32[13945]^c32[14622]^c32[14773]^c32[14647]^c32[15540]^c32[15607]^c32[15633]^c32[16324]^c32[16474]^c32[16335];
+
+       break;
+    case 25:
+      d32[9600]=c32[840]^c32[4625]^c32[5759]^c32[11064];
+
+       break;
+    case 26:
+      d32[9984]=c32[71]^c32[1617]^c32[3148]^c32[11656];
+
+       break;
+    case 27:
+      d32[10368]=c32[962]^c32[4802]^c32[6245];
+
+       break;
+    case 28:
+      d32[10752]=c32[222]^c32[3091]^c32[14836]^c32[16402];
+
+       break;
+    case 29:
+      d32[11136]=c32[2]^c32[155]^c32[284]^c32[354]^c32[355]^c32[866]^c32[1097]^c32[933]^c32[934]^c32[1020]^c32[1665]^c32[1691]^c32[1559]^c32[2368]^c32[2456]^c32[2382]^c32[2383]^c32[3439]^c32[3099]^c32[3425]^c32[3426]^c32[4100]^c32[4063]^c32[4175]^c32[4903]^c32[4848]^c32[4885]^c32[4886]^c32[5402]^c32[5722]^c32[5607]^c32[5608]^c32[6170]^c32[6383]^c32[6278]^c32[6279]^c32[6924]^c32[7169]^c32[7054]^c32[8047]^c32[7888]^c32[7890]^c32[7891]^c32[8636]^c32[8822]^c32[8831]^c32[8448]^c32[9312]^c32[9253]^c32[9525]^c32[9526]^c32[10036]^c32[10263]^c32[10019]^c32[10020]^c32[11048]^c32[10919]^c32[11042]^c32[11043]^c32[10757]^c32[11814]^c32[11698]^c32[11652]^c32[12473]^c32[12337]^c32[12670]^c32[12671]^c32[13249]^c32[13266]^c32[13434]^c32[13435]^c32[14145]^c32[14143]^c32[14173]^c32[14174]^c32[13971]^c32[14851]^c32[14618]^c32[14876]^c32[15385]^c32[15452]^c32[15477]^c32[15478]^c32[16169]^c32[16319]^c32[16179]^c32[16180];
+
+       break;
+    case 30:
+      d32[11520]=c32[14]^c32[167]^c32[295]^c32[296]^c32[366]^c32[367]^c32[159]^c32[878]^c32[1108]^c32[1109]^c32[945]^c32[946]^c32[1677]^c32[1703]^c32[1570]^c32[1571]^c32[2380]^c32[2468]^c32[2394]^c32[2395]^c32[3451]^c32[3110]^c32[3111]^c32[3437]^c32[3438]^c32[4112]^c32[4075]^c32[4186]^c32[4187]^c32[4915]^c32[4859]^c32[4860]^c32[4897]^c32[4898]^c32[5414]^c32[5733]^c32[5734]^c32[5619]^c32[5620]^c32[6182]^c32[6394]^c32[6395]^c32[6290]^c32[6291]^c32[6936]^c32[7181]^c32[7065]^c32[7066]^c32[8059]^c32[7899]^c32[7900]^c32[7902]^c32[7903]^c32[7909]^c32[8648]^c32[8450]^c32[8459]^c32[8460]^c32[9324]^c32[9265]^c32[9537]^c32[9538]^c32[10048]^c32[10274]^c32[10275]^c32[10031]^c32[10032]^c32[10244]^c32[11060]^c32[10930]^c32[10931]^c32[11054]^c32[11055]^c32[11826]^c32[11710]^c32[11663]^c32[11664]^c32[12485]^c32[12349]^c32[12298]^c32[12299]^c32[13261]^c32[13277]^c32[13278]^c32[13062]^c32[13063]^c32[14157]^c32[14154]^c32[14155]^c32[14185]^c32[14186]^c32[14863]^c32[14630]^c32[14887]^c32[14888]^c32[15397]^c32[15463]^c32[15464]^c32[15489]^c32[15490]^c32[16181]^c32[16331]^c32[16191]^c32[16192];
+
+       break;
+    case 31:
+      d32[11904]=c32[181]^c32[180]^c32[334]^c32[333]^c32[79]^c32[78]^c32[150]^c32[148]^c32[149]^c32[1045]^c32[1044]^c32[892]^c32[891]^c32[1113]^c32[1111]^c32[1112]^c32[868]^c32[1844]^c32[1843]^c32[1870]^c32[1869]^c32[1738]^c32[1737]^c32[2547]^c32[2546]^c32[2635]^c32[2634]^c32[2562]^c32[2560]^c32[2561]^c32[3234]^c32[3233]^c32[3278]^c32[3277]^c32[3221]^c32[3219]^c32[3220]^c32[3895]^c32[3894]^c32[3858]^c32[3857]^c32[3970]^c32[3969]^c32[4698]^c32[4697]^c32[4643]^c32[4642]^c32[4681]^c32[4679]^c32[4680]^c32[5581]^c32[5580]^c32[5517]^c32[5516]^c32[5403]^c32[5401]^c32[5402]^c32[5591]^c32[6349]^c32[6348]^c32[6178]^c32[6177]^c32[6458]^c32[6456]^c32[6457]^c32[7103]^c32[7102]^c32[6964]^c32[6963]^c32[7233]^c32[7232]^c32[7842]^c32[7841]^c32[7683]^c32[7682]^c32[7686]^c32[7684]^c32[7685]^c32[8815]^c32[8814]^c32[8617]^c32[8616]^c32[8627]^c32[8625]^c32[8626]^c32[9491]^c32[9490]^c32[9432]^c32[9431]^c32[9321]^c32[9319]^c32[9320]^c32[10215]^c32[10214]^c32[10058]^c32[10057]^c32[10199]^c32[10197]^c32[10198]^c32[10843]^c32[10842]^c32[11098]^c32[11097]^c32[10838]^c32[10836]^c32[10837]^c32[11609]^c32[11608]^c32[11877]^c32[11876]^c32[11831]^c32[11830]^c32[12652]^c32[12651]^c32[12516]^c32[12515]^c32[12466]^c32[12464]^c32[12465]^c32[13428]^c32[13427]^c32[13061]^c32[13060]^c32[13230]^c32[13228]^c32[13229]^c32[13940]^c32[13939]^c32[13938]^c32[13937]^c32[13969]^c32[13967]^c32[13968]^c32[14646]^c32[14645]^c32[14797]^c32[14796]^c32[14671]^c32[14670]^c32[15564]^c32[15563]^c32[15631]^c32[15630]^c32[15657]^c32[15655]^c32[15656]^c32[16348]^c32[16347]^c32[16498]^c32[16497]^c32[16359]^c32[16357]^c32[16358];
+
+       break;
+    case 32:
+      d32[12288]=c32[211]^c32[364]^c32[108]^c32[109]^c32[179]^c32[180]^c32[102]^c32[1075]^c32[921]^c32[922]^c32[1142]^c32[1143]^c32[1874]^c32[1900]^c32[1767]^c32[1768]^c32[2577]^c32[2665]^c32[2591]^c32[2592]^c32[3264]^c32[3307]^c32[3308]^c32[3250]^c32[3251]^c32[3925]^c32[3888]^c32[3999]^c32[4000]^c32[4728]^c32[4672]^c32[4673]^c32[4710]^c32[4711]^c32[5611]^c32[5546]^c32[5547]^c32[5432]^c32[5433]^c32[6379]^c32[6207]^c32[6208]^c32[6487]^c32[6488]^c32[7133]^c32[6994]^c32[7262]^c32[7263]^c32[7872]^c32[7712]^c32[7713]^c32[7715]^c32[7716]^c32[8461]^c32[8647]^c32[8656]^c32[8657]^c32[9521]^c32[9462]^c32[9350]^c32[9351]^c32[9417]^c32[10245]^c32[10087]^c32[10088]^c32[10228]^c32[10229]^c32[10873]^c32[11127]^c32[11128]^c32[10867]^c32[10868]^c32[10927]^c32[11639]^c32[11523]^c32[11860]^c32[11861]^c32[12298]^c32[12546]^c32[12495]^c32[12496]^c32[13074]^c32[13090]^c32[13091]^c32[13259]^c32[13260]^c32[13970]^c32[13967]^c32[13968]^c32[13998]^c32[13999]^c32[14676]^c32[14827]^c32[14700]^c32[14701]^c32[15594]^c32[15660]^c32[15661]^c32[15686]^c32[15687]^c32[16378]^c32[16144]^c32[16388]^c32[16389];
+
+       break;
+    case 33:
+      d32[12672]=c32[1091]^c32[1544]^c32[8809]^c32[16233];
+
+       break;
+    case 34:
+      d32[13056]=c32[230]^c32[5524]^c32[11722]^c32[13368];
+
+       break;
+    case 35:
+      d32[13440]=c32[189]^c32[342]^c32[87]^c32[158]^c32[1053]^c32[900]^c32[1121]^c32[1088]^c32[1852]^c32[1878]^c32[1746]^c32[2555]^c32[2643]^c32[2570]^c32[3242]^c32[3286]^c32[3229]^c32[3903]^c32[3866]^c32[3978]^c32[4706]^c32[4651]^c32[4689]^c32[4943]^c32[5589]^c32[5525]^c32[5411]^c32[6357]^c32[6186]^c32[6466]^c32[7111]^c32[6972]^c32[7241]^c32[7850]^c32[7691]^c32[7694]^c32[8823]^c32[8625]^c32[8635]^c32[9499]^c32[9440]^c32[9329]^c32[9218]^c32[10223]^c32[10066]^c32[10207]^c32[10851]^c32[11106]^c32[10846]^c32[11617]^c32[11885]^c32[11839]^c32[12660]^c32[12524]^c32[12474]^c32[13436]^c32[13069]^c32[13238]^c32[13948]^c32[13946]^c32[13977]^c32[14654]^c32[14805]^c32[14679]^c32[15572]^c32[15639]^c32[15665]^c32[16356]^c32[16506]^c32[16367];
+
+       break;
+    case 36:
+      d32[13824]=c32[210]^c32[11065]^c32[11817]^c32[13845];
+
+       break;
+    case 37:
+      d32[14208]=c32[38]^c32[39]^c32[192]^c32[321]^c32[8]^c32[902]^c32[903]^c32[1134]^c32[971]^c32[1037]^c32[1701]^c32[1702]^c32[1728]^c32[1596]^c32[2404]^c32[2405]^c32[2493]^c32[2420]^c32[3092]^c32[3136]^c32[3079]^c32[4136]^c32[4137]^c32[4100]^c32[4212]^c32[4939]^c32[4940]^c32[4885]^c32[4923]^c32[5439]^c32[5759]^c32[5645]^c32[6207]^c32[6420]^c32[6316]^c32[6960]^c32[6961]^c32[7206]^c32[7091]^c32[7699]^c32[7700]^c32[7925]^c32[7928]^c32[8672]^c32[8673]^c32[8475]^c32[8485]^c32[9348]^c32[9349]^c32[9290]^c32[9563]^c32[10072]^c32[10073]^c32[10300]^c32[10057]^c32[10066]^c32[11085]^c32[10956]^c32[11080]^c32[11850]^c32[11851]^c32[11735]^c32[11689]^c32[12509]^c32[12510]^c32[12374]^c32[12324]^c32[13286]^c32[13303]^c32[13088]^c32[14181]^c32[14182]^c32[14180]^c32[13827]^c32[14887]^c32[14888]^c32[14655]^c32[14913]^c32[15421]^c32[15422]^c32[15489]^c32[15515]^c32[16205]^c32[16206]^c32[16356]^c32[16217];
+
+       break;
+    case 38:
+      d32[14592]=c32[185]^c32[7089]^c32[7969]^c32[9430];
+
+       break;
+    case 39:
+      d32[14976]=c32[1026]^c32[2397]^c32[5722]^c32[14889];
+
+       break;
+    case 40:
+      d32[15360]=c32[175]^c32[6181]^c32[13368];
+
+       break;
+    case 41:
+      d32[15744]=c32[820]^c32[2618]^c32[7051]^c32[14112];
+
+       break;
+    case 42:
+      d32[16128]=c32[142]^c32[295]^c32[39]^c32[40]^c32[110]^c32[111]^c32[113]^c32[1006]^c32[852]^c32[853]^c32[1073]^c32[1074]^c32[1805]^c32[1831]^c32[1698]^c32[1699]^c32[2508]^c32[2596]^c32[2522]^c32[2523]^c32[3195]^c32[3238]^c32[3239]^c32[3181]^c32[3182]^c32[3086]^c32[3856]^c32[4203]^c32[3930]^c32[3931]^c32[4659]^c32[4987]^c32[4988]^c32[4641]^c32[4642]^c32[5542]^c32[5477]^c32[5478]^c32[5747]^c32[5748]^c32[6310]^c32[6522]^c32[6523]^c32[6418]^c32[6419]^c32[7064]^c32[6925]^c32[7193]^c32[7194]^c32[7803]^c32[8027]^c32[8028]^c32[8030]^c32[8031]^c32[8776]^c32[8578]^c32[8587]^c32[8588]^c32[9452]^c32[9393]^c32[9281]^c32[9282]^c32[10176]^c32[10018]^c32[10019]^c32[10159]^c32[10160]^c32[10804]^c32[11058]^c32[11059]^c32[10798]^c32[10799]^c32[11570]^c32[11838]^c32[11791]^c32[11792]^c32[12613]^c32[12477]^c32[12426]^c32[12427]^c32[13389]^c32[13405]^c32[13406]^c32[13190]^c32[13191]^c32[13901]^c32[13898]^c32[13899]^c32[13929]^c32[13930]^c32[14607]^c32[14758]^c32[14631]^c32[14632]^c32[15525]^c32[15591]^c32[15592]^c32[15617]^c32[15618]^c32[16309]^c32[16459]^c32[16319]^c32[16320];
+
+       break;
+    case 43:
+      d32[16512]=c32[92]^c32[245]^c32[374]^c32[60]^c32[61]^c32[956]^c32[803]^c32[1023]^c32[1024]^c32[881]^c32[1755]^c32[1781]^c32[1649]^c32[2458]^c32[2546]^c32[2472]^c32[2473]^c32[3145]^c32[3189]^c32[3131]^c32[3132]^c32[4190]^c32[4153]^c32[3881]^c32[4609]^c32[4938]^c32[4975]^c32[4976]^c32[5492]^c32[5428]^c32[5697]^c32[5698]^c32[6260]^c32[6473]^c32[6368]^c32[6369]^c32[7014]^c32[7259]^c32[7144]^c32[7753]^c32[7978]^c32[7980]^c32[7981]^c32[8726]^c32[8528]^c32[8537]^c32[8538]^c32[9402]^c32[9343]^c32[9231]^c32[9232]^c32[10126]^c32[10353]^c32[10109]^c32[10110]^c32[10754]^c32[11009]^c32[11132]^c32[11133]^c32[11520]^c32[11788]^c32[11742]^c32[12563]^c32[12427]^c32[12376]^c32[12377]^c32[12420]^c32[13339]^c32[13356]^c32[13140]^c32[13141]^c32[13851]^c32[13849]^c32[13879]^c32[13880]^c32[13938]^c32[14941]^c32[14708]^c32[14966]^c32[15475]^c32[15542]^c32[15567]^c32[15568]^c32[16259]^c32[16409]^c32[16269]^c32[16270];
+
+       break;
+    case 44:
+      d32[16896]=c32[197]^c32[350]^c32[95]^c32[166]^c32[80]^c32[1061]^c32[908]^c32[1129]^c32[1860]^c32[1886]^c32[1754]^c32[2563]^c32[2651]^c32[2578]^c32[3250]^c32[3294]^c32[3237]^c32[3911]^c32[3874]^c32[3986]^c32[4714]^c32[4659]^c32[4697]^c32[5597]^c32[5533]^c32[5419]^c32[5454]^c32[6365]^c32[6194]^c32[6474]^c32[7119]^c32[6980]^c32[7249]^c32[7075]^c32[7858]^c32[7699]^c32[7702]^c32[8831]^c32[8633]^c32[8643]^c32[9507]^c32[9448]^c32[9337]^c32[10231]^c32[10074]^c32[10215]^c32[10859]^c32[11114]^c32[10854]^c32[11625]^c32[11893]^c32[11847]^c32[12668]^c32[12532]^c32[12482]^c32[13060]^c32[13077]^c32[13246]^c32[13956]^c32[13954]^c32[13985]^c32[14662]^c32[14813]^c32[14687]^c32[15580]^c32[15647]^c32[15673]^c32[16364]^c32[16130]^c32[16375];
+
+       break;
+    case 45:
+      d32[17280]=c32[903]^c32[4757]^c32[7695];
+
+       break;
+    
+    }
+  }
+}
+extern "C" int ldpc_BG1_Zc384_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs,cudaStream_t *stream,int sidx) { 
+
+ dim3 numblocks(n_inputs,46);
+ ldpc_BG1_Zc384_worker<<<numblocks,384,0,stream[sidx]>>>(c,d);
+ cudaError_t err=cudaPeekAtLastError();
+ if (err!=cudaSuccess) {
+    printf("cuda error: %s (c %p, d %p)\n",cudaGetErrorString(err),c,d);
+    exit(-1);
+ }
+ return(0);
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_deinterleave.cu

@@ -0,0 +1,131 @@
+// deinterleave_u16.cu
+#include <cuda_runtime.h>
+#include <stdio.h>
+#include <stdint.h>
+
+__global__ void deinterleave_i16_2(int16_t* __restrict__ e,
+                                   const int16_t* __restrict__ f,
+                                   int E1,
+				   int E2,
+				   int r_firstE2)
+{
+    int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+    int r = (int)blockIdx.y;
+    int E = (r<r_firstE2) ? E1 : E2;
+    int EQm = E/2;
+    if (g >= EQm) return;
+
+    int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
+    const int16_t *in = f +r_off + 2*g;
+
+    int16_t* e0 = e + r_off;
+    int16_t* e1 = e0 + EQm;
+    e0[g] = in[0];
+    e1[g] = in[1];
+}
+
+__global__ void deinterleave_i16_4(int16_t* __restrict__ e,
+                                   const int16_t* __restrict__ f,
+                                   int E1,
+                       		   int E2,
+				   int r_firstE2)
+{
+    int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+    int r = (int)blockIdx.y;
+    int E = (r<r_firstE2) ? E1 : E2;
+    int EQm = E/4;
+    if (g >= EQm) return;
+
+    int r_off = r<r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
+    const int16_t *in = f + r_off + 4*g;
+    int16_t* e0 = e + r_off;
+    int16_t* e1 = e0 + EQm;
+    int16_t* e2 = e1 + EQm;
+    int16_t* e3 = e2 + EQm;
+    e0[g] = in[0];
+    e1[g] = in[1];
+    e2[g] = in[2];
+    e3[g] = in[3];
+}
+
+__global__ void deinterleave_i16_6(int16_t* __restrict__ e,
+                                   const int16_t* __restrict__ f,
+                                   int E1,
+				   int E2,
+				   int r_firstE2)
+{
+    int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+    int r = (int)blockIdx.y;
+    int E = (r<r_firstE2) ? E1 : E2;
+    int EQm = E/6;
+    if (g >= EQm) return;
+
+    int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
+    const int16_t* in = f + r_off + 6*g;
+    int16_t* e0 = e + r_off;
+    int16_t* e1 = e0 + EQm;
+    int16_t* e2 = e1 + EQm;
+    int16_t* e3 = e2 + EQm;
+    int16_t* e4 = e3 + EQm;
+    int16_t* e5 = e4 + EQm;
+    e0[g] = in[0];
+    e1[g] = in[1];
+    e2[g] = in[2];
+    e3[g] = in[3];
+    e4[g] = in[4];
+    e5[g] = in[5];
+}
+
+__global__ void deinterleave_i16_8(int16_t* __restrict__ e,
+                                   const int16_t* __restrict__ f,
+                                   const int E1,
+				   const int E2,
+				   const int r_firstE2)
+{
+    int g = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+    int r = (int)blockIdx.y;
+    int E = (r<r_firstE2) ? E1 : E2;
+    int EQm = E/8;
+    if (g >= EQm) return;
+    int r_off = r<r_firstE2 ? r*E1 : (r_firstE2*E1)+(r-r_firstE2)*E2;
+
+    const int16_t *in = (f + r_off + 8*g);
+    int16_t* e0 = e + r_off;
+    int16_t* e1 = e0 + EQm;
+    int16_t* e2 = e1 + EQm;
+    int16_t* e3 = e2 + EQm;
+    int16_t* e4 = e3 + EQm;
+    int16_t* e5 = e4 + EQm;
+    int16_t* e6 = e5 + EQm;
+    int16_t* e7 = e6 + EQm;
+    e0[g] = in[0];
+    e1[g] = in[1];
+    e2[g] = in[2];
+    e3[g] = in[3];
+    e4[g] = in[4];
+    e5[g] = in[5];
+    e6[g] = in[6];
+    e7[g] = in[7];
+}
+
+// Host launcher
+extern "C" void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx)
+{
+    const int threads = 256;
+    dim3 blocks(((E2/Qm)  + threads - 1) / threads,C);
+
+    switch (Qm) {
+        case 2: deinterleave_i16_2<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
+        case 4: deinterleave_i16_4<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
+        case 6: deinterleave_i16_6<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
+        case 8: deinterleave_i16_8<<<blocks, threads, 0, s[sidx]>>>(e, f, E1,E2,r_firstE2); break;
+        default: /* unsupported */ break;
+    }
+    cudaError_t err=cudaPeekAtLastError();
+  
+    if (err!=cudaSuccess) {
+      printf("cuda error (deinterleave_i16): %s (e %p, f %p, E1 %d, E2 %d, Qm %d, C %d)\n",cudaGetErrorString(err),e,f,E1,E2,Qm,C);
+      exit(-1);
+    }
+
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_encode_parity_check_cuda.c

@@ -0,0 +1,77 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+/*!\file ldpc_encode_parity_check.c
+ * \brief Parity check function used by ldpc encoders
+ * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
+ * \email openair_tech@eurecom.fr
+ * \date 27-03-2018
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "assertions.h"
+#include "common/utils/LOG/log.h"
+#include <cuda_runtime.h>
+
+
+
+int ldpc_BG1_Zc384_cuda32(uint32_t **c,uint32_t **d,int n_inputs, cudaStream_t *stream,int sidx);
+
+
+void encode_parity_check_part_cuda(uint32_t **c, uint32_t **d, short BG,short Zc,short Kb, int ncols, int n_inputs, cudaStream_t *stream,int sidx)
+{
+  
+  if (BG == 1) {
+    switch (Zc) {
+      case 176:
+      case 192:
+      case 208:
+      case 224:
+      case 240:
+      case 256:
+      case 288:
+      case 320:
+      case 352:
+	AssertFatal(1==0,"BG %d Zc %d not supported yet for CUDA\n",BG, Zc);
+        break;
+      case 384:
+	ldpc_BG1_Zc384_cuda32(c, d, n_inputs, stream,sidx);
+        break;
+      default:
+        AssertFatal(false, "BG %d Zc %d is not supported yet\n", BG, Zc);
+    }
+  } else if (BG == 2) {
+    switch (Zc) {
+      case 72:
+      case 80:
+      case 88:
+      case 96:
+      case 104:
+      case 112:
+      case 120:
+      case 128:
+      case 144:
+      case 160:
+      case 176:
+      case 192:
+      case 208:
+      case 224:
+      case 240:
+      case 256:
+      case 288:
+      case 320:
+      case 352:
+      case 384:
+      default:
+        AssertFatal(false , "BG %d Zc %d is not supported yet\n", BG, Zc);
+    }
+  } else
+    AssertFatal(false, "BG %d is not supported\n", BG);
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_encoder_cuda32.c

@@ -0,0 +1,194 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+/*! \file ldpc_encoder_cuda32.c
+ * \brief Defines the optimized LDPC encoder for NVidia GPUs
+ * \email openair_tech@eurecom.fr
+ * \date 11-30-2025
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "assertions.h"
+#include "common/utils/LOG/log.h"
+#include "time_meas.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "PHY/sse_intrin.h"
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+#include <cuda_runtime.h>
+
+//#define DEBUG_LDPC 1
+
+#include "ldpc_encode_parity_check_cuda.c"
+
+uint32_t *c_dev;
+uint32_t **c_host;
+uint32_t *c_devh[4];
+uint32_t *d_dev;
+uint32_t **d_host;
+uint32_t *d_devh[4];
+uint32_t *input_dev;
+uint32_t **input_host;
+uint32_t *input_devh[128];
+int managed = 0, concurrent = 0, uva = 0, pageable = 0, pageable_uses_host = 0, register_host = 0, integrated = 0;
+
+int cuda_support_set = 0;
+
+extern cudaStream_t encoderStreams[4];
+
+int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *s,int sidx);
+
+void cuda_support_init() {
+
+    int dev = 0;
+    struct cudaDeviceProp prop;
+    cudaGetDeviceProperties(&prop, dev);
+
+
+    cudaDeviceGetAttribute(&managed, cudaDevAttrManagedMemory, dev);
+    cudaDeviceGetAttribute(&concurrent, cudaDevAttrConcurrentManagedAccess, dev);
+    cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, dev);
+    cudaDeviceGetAttribute(&pageable, cudaDevAttrPageableMemoryAccess, dev);
+    cudaDeviceGetAttribute(&pageable_uses_host, cudaDevAttrPageableMemoryAccessUsesHostPageTables, dev);
+    cudaDeviceGetAttribute(&register_host, cudaDevAttrHostRegisterSupported,dev);
+    cudaDeviceGetAttribute(&integrated, cudaDevAttrIntegrated,dev);
+
+    LOG_I(NR_PHY,"Device: %s (cc %d.%d)\n", prop.name, prop.major, prop.minor);
+    LOG_I(NR_PHY,"Unified Virtual Addressing (UVA): %s\n", uva ? "YES" : "NO");
+    LOG_I(NR_PHY,"Managed (Unified) Memory:        %s\n", managed ? "YES" : "NO");
+    LOG_I(NR_PHY,"Concurrent managed access:       %s\n", concurrent ? "YES" : "NO");
+    LOG_I(NR_PHY,"Pageable memory access:          %s\n", pageable ? "YES" : "NO");
+    LOG_I(NR_PHY,"Uses host page tables:           %s\n", pageable_uses_host ? "YES" : "NO");
+    LOG_I(NR_PHY,"Host Register supported:         %s\n", register_host ? "YES" : "NO");
+    LOG_I(NR_PHY,"Integrated (shared) Memory       %s\n", integrated ? "YES" : "NO");
+
+  if (!pageable && !integrated) {
+    LOG_I(NR_PHY,"Allocating c,d,cc arrays for GPU \n");
+    cudaError_t err=cudaMalloc((void **)&c_dev,4*sizeof(uint32_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<4;i++) {
+      err=cudaMalloc((void**)&c_devh[i],2*22*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (chost[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err = cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaMalloc((void**)&d_dev,4*sizeof(uint32_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<4;i++) {
+      err=cudaMalloc((void**)&d_devh[i],68*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaMalloc((void**)&input_dev,128*sizeof(uint8_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<128;i++) {
+      err=cudaMalloc((void**)&input_devh[i],(8448/8)*sizeof(uint8_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (input_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&input_host[i],(8448/8)*sizeof(uint8_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (input_host[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(input_dev,input_devh,128*sizeof(uint8_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy cc_devh -> d_dev): %s\n", cudaGetErrorString(err));
+  }
+  else {
+    LOG_I(NR_PHY,"Allocating c,d,cc arrays for CPU/GPU shared-memory\n");
+    cudaError_t err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocMapped|cudaHostAllocPortable);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
+    err = cudaHostGetDevicePointer((void**)&c_dev, c_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"c_host %p, c_dev %p\n",c_host,c_dev);
+    for (int i=0;i<4;i++) {
+      err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err = cudaHostGetDevicePointer((void**)&c_devh[i], c_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&d_dev, d_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(d_dev): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"d_host %p, d_dev %p\n",d_host,d_dev);
+    for (int i=0;i<4;i++) {
+      err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&d_devh[i], d_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) d_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      LOG_I(NR_PHY,"d_host[%d] %p, d_devh[%d] %p\n",i,d_host[i],i,d_devh[i]);
+    }
+    err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (input_host): %s\n", cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&input_dev, input_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(cc_host): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"input_host %p, input_dev %p\n",input_host,input_dev);
+  }
+
+
+  cuda_support_set=1;
+}
+
+uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp)
+{
+  //set_log(PHY, 4);
+
+  int Zc = impp->Zc;
+  int Kb = impp->Kb;
+  short block_length = impp->K;
+  short BG = impp->BG;
+  int ncols=22;
+
+  int encoder_stream=0;
+
+  AssertFatal(BG==1,"BG %d is not supported for CUDA version\n",BG);
+  AssertFatal(Zc==384,"Zc %d is not supported for CUDA version \n", Zc);
+ 
+  if(impp->tinput != NULL) start_meas(impp->tinput);
+
+#ifdef DEBUG_LDPC
+  LOG_I(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
+  LOG_I(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
+#endif
+
+  int n_inputs = (impp->n_segments/32)+(((impp->n_segments&31) > 0) ? 1: 0);
+//  uint32_t  cc[4][22*Zc]; //padded input, unpacked, max size
+
+  if (!pageable&&!integrated) { // this means we are not on shared memory
+    for (int r=0;r<impp->n_segments;r++) {
+        cudaMemcpyAsync(input_devh[r],input[r],block_length>>3,cudaMemcpyHostToDevice,encoderStreams[encoder_stream]);
+    }
+  }
+  ldpc_input(pageable||integrated? (uint32_t**)input : (uint32_t**)input_dev,(uint32_t**)c_dev,impp->n_segments,encoderStreams,encoder_stream);
+  if(impp->tinput != NULL) stop_meas(impp->tinput);
+  //parity check part
+  if(impp->tparity != NULL) start_meas(impp->tparity);
+  encode_parity_check_part_cuda((uint32_t**)c_dev, (uint32_t**)d_dev, BG, Zc, Kb, ncols,n_inputs,encoderStreams,encoder_stream);
+  if (!pageable&&!integrated) { // this means we are not on shared memory
+     for (int r=0; r<n_inputs;r++) cudaMemcpyAsync(d_host[r],d_devh[r],68*384*sizeof(uint32_t),cudaMemcpyDeviceToHost,encoderStreams[encoder_stream]);  
+  }
+  cudaStreamSynchronize(encoderStreams[encoder_stream]);
+  if(impp->tparity != NULL) stop_meas(impp->tparity);
+  
+  return d_host;
+}
+

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_input.cu

@@ -0,0 +1,92 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <cuda_runtime.h>
+/*
+__device__ const uint32_t masks[4] = {0x80,0x8000,0x800000,0x80000000};
+__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) {
+
+  int block_off = blockIdx.x*blockDim.x<<2;	
+  int i2 = threadIdx.x<<2;
+  uint32_t *out=cc[blockIdx.y] + block_off + i2;
+  int nseg0 = (blockIdx.y << 5);
+  int nseg1;
+  if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
+  else nseg1 = nseg0 + (nseg&31);
+  int bit_offset = i2+block_off;
+  int uint32_offset = bit_offset>>5;  
+  uint32_t mask = masks[(bit_offset&31)>>3]; 
+  uint32_t mask0 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask1 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask2 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask3 = mask>>(bit_offset&7);
+  uint32_t tmp,jmod;
+  uint32_t otmp0,otmp1,otmp2,otmp3;
+  if (bit_offset  < block_length) {
+      tmp=input[nseg0][uint32_offset];
+      otmp0 = ((tmp&mask0) > 0); 
+      otmp1 = ((tmp&mask1) > 0); 
+      otmp2 = ((tmp&mask2) > 0); 
+      otmp3 = ((tmp&mask3) > 0); 
+      for (int j=nseg0+1;j<nseg1;j++) {
+	 tmp=input[j][uint32_offset];
+	 jmod = j&31;
+         otmp0 |= (((tmp&mask0) > 0)<<jmod); 
+         otmp1 |= (((tmp&mask1) > 0)<<jmod); 
+         otmp2 |= (((tmp&mask2) > 0)<<jmod); 
+         otmp3 |= (((tmp&mask3) > 0)<<jmod); 
+      }	      
+      out[0]=otmp0;
+      out[1]=otmp1;
+      out[2]=otmp2;
+      out[3]=otmp3;
+  } 
+}
+*/
+
+__device__ uint32_t masks[32] = {
+	0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1,
+	0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100,
+	0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000,
+	0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000};
+__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int nseg) {
+
+//  int block_off = blockIdx.y*blockDim.x;	
+  int i1 = blockIdx.y;	
+  int i2 = threadIdx.x;
+  int nseg0 = (blockIdx.x << 5);
+  int nseg1;
+  if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
+  else nseg1 = nseg0 + (nseg&31);
+  int bit_offset = i2+(i1*384);
+  int uint32_offset = bit_offset>>5;  
+  uint32_t mask0 = masks[bit_offset&31];
+  uint32_t tmp,jmod;
+  uint32_t otmp0;
+  if (bit_offset  < 8448) {
+      tmp=input[nseg0][uint32_offset];
+      otmp0 = ((tmp&mask0) > 0); 
+      for (int j=nseg0+1;j<nseg1;j++) {
+	 tmp=input[j][uint32_offset];
+	 jmod = j&31;
+         otmp0 |= (((tmp&mask0) > 0)<<jmod); 
+      }	      
+      
+      cc[blockIdx.x][(2*i1*384) + i2] = otmp0;
+      cc[blockIdx.x][(2*i1+1)*384 + i2]=otmp0;
+  } 
+}
+extern "C" int ldpc_input(uint32_t **input,uint32_t *cc[4],int nseg,cudaStream_t *stream,int sidx) { 
+
+ int ns = nseg>>5;
+ if ((nseg&31)>0) ns++;
+
+ dim3 numblocks(ns,22);
+ //printf("input %p\n",input);
+ ldpc_input_worker<<<numblocks,384,0,stream[sidx]>>>(input,cc,nseg);
+ cudaError_t err=cudaPeekAtLastError();
+ if (err!=cudaSuccess) {
+    printf("cuda error: %s (input %p, cc %p, nseg %d, ns %d)\n",cudaGetErrorString(err),input,cc,nseg,ns);
+    exit(-1);
+ }
+ return(0);
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_input32.c

@@ -0,0 +1,492 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "assertions.h"
+#include "common/utils/LOG/log.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "PHY/sse_intrin.h"
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+
+void ldpc_input32(uint8_t **input, uint32_t **c, int n_inputs, int n_segments) {
+
+#ifndef __aarch64__
+  simde__m256i andmask = simde_mm256_set_epi32(0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80);  // every 8 bits -> 8 bytes, pattern repeats.
+  simde__m256i zero256 = simde_mm256_setzero_si256();
+  simde__m256i masks[32];
+  register simde__m256i c256;
+  for (int i=0;i<32;i++) {
+    masks[i] = simde_mm256_set1_epi32(1<<i);
+  }
+
+  for (int b=0;b<8448;b+=384) {
+    for (int i=0; i < 384; i += 8) {
+      unsigned int i8 = (2*b + i) >> 3;
+      for (int j0=0;j0<n_inputs;j0++) {
+        c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[j0<<5][i8]),andmask),zero256),masks[0]);
+        int j2 = 32;
+        if (j0==(n_inputs-1)) j2= n_segments-(j0<<5);
+        for (int j=1; j < j2; j++) {
+          c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[(j0<<5)+j][i8]),andmask),zero256),masks[j]),c256);
+        }
+        ((simde__m256i *)c[j0])[i8]    = c256;
+        ((simde__m256i *)c[j0])[i8+48] = c256;
+      }
+    }
+  }
+#else
+  int i2=0;
+  uint32_t cc[4][8448];
+  const int32_t ucShifta[32][4] = { 
+	  {-7,-6,-5,-4}, // 0  
+	  {-6,-5,-4,-3}, // 1
+	  {-5,-4,-3,-2}, // 2 
+	  {-4,-3,-2,-1}, // 3 
+	  {-3,-2,-1,0}, // 4 
+	  {-2,-1,0,1}, // 5
+	  {-1,0,1,2}, // 6
+	  {0,1,2,3}, // 7
+	  {1,2,3,4}, // 8
+	  {2,3,4,5}, // 9
+	  {3,4,5,6}, // 10
+	  {4,5,6,7}, // 11
+	  {5,6,7,8}, // 12
+	  {6,7,8,9}, // 13
+	  {7,8,9,10}, // 14
+	  {8,9,10,11}, // 15
+	  {9,10,11,12}, // 16
+	  {10,11,12,13}, // 17
+	  {11,12,13,14}, // 18
+	  {12,13,14,15}, // 19
+	  {13,14,15,16}, // 20
+	  {14,15,16,17}, // 21
+	  {15,16,17,18}, // 22
+	  {16,17,18,19}, // 23
+	  {17,18,19,20}, // 24
+	  {18,19,20,21}, // 25
+	  {19,20,21,22}, // 26
+	  {20,21,22,23}, // 27
+	  {21,22,23,24}, // 28
+	  {22,23,24,25}, // 29
+	  {23,24,25,26}, // 30
+	  {24,25,26,27}}; // 31
+  const int32_t ucShiftb[32][4] = { 
+	  {-3,-2,-1,0}, // 0 
+	  {-2,-1,0,1}, // 1
+	  {-1,0,1,2}, // 2
+	  {0,1,2,3}, // 3
+	  {1,2,3,4}, // 4
+	  {2,3,4,5}, // 5
+	  {3,4,5,6}, // 6 
+	  {4,5,6,7}, // 7
+	  {5,6,7,8}, // 8
+	  {6,7,8,9}, // 9
+	  {7,8,9,10}, // 10
+	  {8,9,10,11}, // 11
+	  {9,10,11,12}, // 12
+	  {10,11,12,13}, // 13
+	  {11,12,13,14}, // 14
+	  {12,13,14,15}, // 15
+	  {13,14,15,16}, // 16
+	  {14,15,16,17}, // 17
+	  {15,16,17,18}, // 18
+	  {16,17,18,19}, // 19
+	  {17,18,19,20}, // 20
+	  {18,19,20,21}, // 21
+	  {19,20,21,22}, // 22
+	  {20,21,22,23}, // 23
+	  {21,22,23,24}, // 24
+	  {22,23,24,25}, // 25
+	  {23,24,25,26}, // 26
+	  {24,25,26,27}, // 27
+	  {25,26,27,28}, // 28
+	  {26,27,28,29}, // 29
+	  {27,28,29,30}, // 30
+	  {28,29,30,31}}; // 31
+			      
+  const int32_t ucShiftc[32][4] = { 
+	  {-15,-14,-13,-12}, // 0
+	  {-14,-13,-12,-11}, // 1
+	  {-13,-12,-11,-10}, // 2
+          {-12,-11,-10,-9}, // 3
+	  {-11,-10,-9,-8}, // 4
+	  {-10,-9,-8,-7}, // 5
+	  {-9,-8,-7,-6}, // 6
+	  {-8,-7,-6,-5}, // 7
+	  {-7,-6,-5,-4}, // 8  
+	  {-6,-5,-4,-3}, // 9
+	  {-5,-4,-3,-2}, // 10 
+	  {-4,-3,-2,-1}, // 11 
+	  {-3,-2,-1,0}, // 12 
+	  {-2,-1,0,1}, // 13
+	  {-1,0,1,2}, // 14
+	  {0,1,2,3}, // 15
+	  {1,2,3,4}, // 16
+	  {2,3,4,5}, // 17
+	  {3,4,5,6}, // 18
+	  {4,5,6,7}, // 19
+	  {5,6,7,8}, // 20
+	  {6,7,8,9}, // 21
+	  {7,8,9,10}, // 22
+	  {8,9,10,11}, // 23
+	  {9,10,11,12}, // 24
+	  {10,11,12,13}, // 25
+	  {11,12,13,14}, // 26
+	  {12,13,14,15}, // 27
+	  {13,14,15,16}, // 28
+	  {14,15,16,17}, // 29
+	  {15,16,17,18}, // 30
+	  {16,17,18,19}}; // 31
+  const int32_t ucShiftd[32][4] = { 
+	  {-11,-10,-9,-8}, // 0
+	  {-10,-9,-8,-7}, // 1
+	  {-9,-8,-7,-6}, // 2
+	  {-8,-7,-6,-5}, // 3
+	  {-7,-6,-5,-4}, // 4  
+	  {-6,-5,-4,-3}, // 5
+	  {-5,-4,-3,-2}, // 6
+	  {-4,-3,-2,-1}, // 7 
+	  {-3,-2,-1,0}, // 8
+	  {-2,-1,0,1}, // 9
+	  {-1,0,1,2}, // 10
+	  {0,1,2,3}, // 11 
+	  {1,2,3,4}, // 12
+	  {2,3,4,5}, // 13
+	  {3,4,5,6}, // 14 
+	  {4,5,6,7}, // 15
+	  {5,6,7,8}, // 16
+	  {6,7,8,9}, // 17
+	  {7,8,9,10}, // 18
+	  {8,9,10,11}, // 19
+	  {9,10,11,12}, // 20
+	  {10,11,12,13}, // 21
+	  {11,12,13,14}, // 22
+	  {12,13,14,15}, // 23
+	  {13,14,15,16}, // 24
+	  {14,15,16,17}, // 25
+	  {15,16,17,18}, // 26
+	  {16,17,18,19}, // 27
+	  {17,18,19,20}, // 28
+	  {18,19,20,21}, // 29
+	  {19,20,21,22}, // 30
+	  {20,21,22,23}};// 31
+			  
+  const int32_t ucShifte[32][4] = { 
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}, // 23
+	  {1,2,3,4}, // 24
+	  {2,3,4,5}, // 25
+	  {3,4,5,6}, // 26
+	  {4,5,6,7}, // 27
+	  {5,6,7,8}, // 28
+	  {6,7,8,9}, // 29
+	  {7,8,9,10}, // 30
+	  {8,9,10,11}}; // 31 
+			  
+  const int32_t ucShiftf[32][4] = { 
+	  {-19,-18,-17,-16}, // 0 
+	  {-18,-17,-16,-15}, // 1
+	  {-17,-16,-15,-14}, // 2
+	  {-16,-15,-14,-13}, // 3
+	  {-15,-14,-13,-12}, // 4
+	  {-14,-13,-12,-11}, // 5
+	  {-13,-12,-11,-10}, // 6
+          {-12,-11,-10,-9}, // 7
+	  {-11,-10,-9,-8}, // 8
+	  {-10,-9,-8,-7}, // 9  
+	  {-9,-8,-7,-6}, // 10 
+	  {-8,-7,-6,-5}, // 11
+	  {-7,-6,-5,-4}, // 12 
+	  {-6,-5,-4,-3}, // 13
+	  {-5,-4,-3,-2}, // 14
+	  {-4,-3,-2,-1}, // 15 
+	  {-3,-2,-1,0}, // 16
+	  {-2,-1,0,1}, // 17
+	  {-1,0,1,2}, // 18
+	  {0,1,2,3}, // 19
+	  {1,2,3,4}, // 20
+	  {2,3,4,5}, // 21
+	  {3,4,5,6}, // 22 
+	  {4,5,6,7}, // 23
+	  {5,6,7,8}, // 24
+	  {6,7,8,9}, // 25
+	  {7,8,9,10}, // 26
+	  {8,9,10,11}, // 27
+	  {9,10,11,12}, // 28
+	  {10,11,12,13}, // 29
+	  {11,12,13,14}, // 30
+	  {12,13,14,15}}; // 31
+			  
+  const int32_t ucShiftg[32][4] = { 
+	  {-31,-30,-29,-28},
+	  {-30,-29,-28,-27},
+	  {-29,-28,-27,-26},
+	  {-28,-27,-26,-25},
+	  {-27,-26,-25,-24},
+	  {-26,-25,-24,-23},
+	  {-25,-24,-23,-22},
+	  {-24,-23,-22,-21},
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}}; // 23
+			  
+  const int32_t ucShifth[32][4] = { 
+	  {-27,-26,-25,-24}, // 0
+	  {-26,-25,-24,-23}, // 1
+	  {-25,-24,-23,-22}, // 2
+	  {-24,-23,-22,-21}, // 3
+	  {-23,-22,-21,-20}, // 4 
+	  {-22,-21,-20,-19}, // 5
+	  {-21,-20,-19,-18}, // 6
+	  {-20,-19,-18,-17}, // 7
+	  {-19,-18,-17,-16}, // 8 
+	  {-18,-17,-16,-15}, // 9
+	  {-17,-16,-15,-14}, // 10
+	  {-16,-15,-14,-13}, // 11
+	  {-15,-14,-13,-12}, // 12
+	  {-14,-13,-12,-11}, // 13
+	  {-13,-12,-11,-10}, // 14
+          {-12,-11,-10,-9}, // 15
+	  {-11,-10,-9,-8}, // 16
+	  {-10,-9,-8,-7}, // 17 
+	  {-9,-8,-7,-6}, // 18 
+	  {-8,-7,-6,-5}, // 19
+	  {-7,-6,-5,-4}, // 20 
+	  {-6,-5,-4,-3}, // 21
+	  {-5,-4,-3,-2}, // 22
+	  {-4,-3,-2,-1}, // 23 
+	  {-3,-2,-1,0}, // 24 
+	  {-2,-1,0,1}, // 25
+	  {-1,0,1,2}, // 26
+	  {0,1,2,3}, // 27
+	  {1,2,3,4}, // 28
+	  {2,3,4,5}, // 29
+	  {3,4,5,6}, // 30
+          {4,5,6,7}}; // 31 
+  const uint32_t __attribute__ ((aligned (16))) masksa[4] = {0x80,0x40,0x20,0x10};
+  const uint32_t __attribute__ ((aligned (16))) masksb[4] = {0x8,0x4,0x2,0x1};
+  const uint32_t __attribute__ ((aligned (16))) masksc[4] = {0x8000,0x4000,0x2000,0x1000};
+  const uint32_t __attribute__ ((aligned (16))) masksd[4] = {0x800,0x400,0x200,0x100};
+  const uint32_t __attribute__ ((aligned (16))) maskse[4] = {0x800000,0x400000,0x200000,0x100000};
+  const uint32_t __attribute__ ((aligned (16))) masksf[4] = {0x80000,0x40000,0x20000,0x10000};
+  const uint32_t __attribute__ ((aligned (16))) masksg[4] = {0x80000000,0x40000000,0x20000000,0x10000000};
+  const uint32_t __attribute__ ((aligned (16))) masksh[4] = {0x8000000,0x4000000,0x2000000,0x1000000};
+  int32x4_t vshifta[32],vshiftb[32],vshiftc[32],vshiftd[32],vshifte[32],vshiftf[32],vshiftg[32],vshifth[32];
+  uint32x4_t vmasksa  = vld1q_u32(masksa);
+  uint32x4_t vmasksb  = vld1q_u32(masksb);
+  uint32x4_t vmasksc  = vld1q_u32(masksc);
+  uint32x4_t vmasksd  = vld1q_u32(masksd);
+  uint32x4_t vmaskse  = vld1q_u32(maskse);
+  uint32x4_t vmasksf  = vld1q_u32(masksf);
+  uint32x4_t vmasksg  = vld1q_u32(masksg);
+  uint32x4_t vmasksh  = vld1q_u32(masksh);
+  uint32x4_t in;
+
+  for (int n=0;n<32;n++) { 
+    vshifta[n] = vld1q_s32(ucShifta[n]);
+    vshiftb[n] = vld1q_s32(ucShiftb[n]);
+    vshiftc[n] = vld1q_s32(ucShiftc[n]);
+    vshiftd[n] = vld1q_s32(ucShiftd[n]);
+    vshifte[n] = vld1q_s32(ucShifte[n]);
+    vshiftf[n] = vld1q_s32(ucShiftf[n]);
+    vshiftg[n] = vld1q_s32(ucShiftg[n]);
+    vshifth[n] = vld1q_s32(ucShifth[n]);
+  }
+  i2=0;
+  int j0=0,j1=0,j2=0,j3=0;
+  if (n_segments <= 32) {
+	  j0=n_segments;
+  }
+  else if (n_segments <= 64) {
+     j0=32;
+     j1=n_segments-32;
+  }
+  else if (n_segments <= 96) {
+     j0=32; j1=32;
+     j2=n_segments-64;
+  }
+  else if (n_segments <= 128) {
+     j0=32; j1=32; j2=32;
+     j3=n_segments-96;
+  }
+  uint32x4_t *ccp,cc0,cc1,cc2,cc3,cc4,cc5,cc6,cc7;
+  for (int i=0; i < 264; i++,i2+=8) {
+    in = vdupq_n_u32(((uint32_t*)input[0])[i]);
+    cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+    cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+    cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+    cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+    cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+    cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+    cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+    cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+    for (int j = 1; j < j0; j++) {
+      in = vdupq_n_u32(((uint32_t*)input[j])[i]);
+      cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+      cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+      cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+      cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+      cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+      cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+      cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+      cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+    }
+    ccp=&((uint32x4_t *)cc[0])[i2];
+    ccp[0] = cc0;
+    ccp[1] = cc1;
+    ccp[2] = cc2;
+    ccp[3] = cc3;
+    ccp[4] = cc4;
+    ccp[5] = cc5;
+    ccp[6] = cc6;
+    ccp[7] = cc7;
+    if (j1>0) {
+      in = vdupq_n_u32(((uint32_t*)input[32])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j1; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[32+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[1])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j2>0) {
+      in = vdupq_n_u32(((uint32_t*)input[64])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j2; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[64+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[2])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j3>0) {
+      in = vdupq_n_u32(((uint32_t*)input[96])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j3; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[96+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[3])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+  }
+  for (int s=0;s<n_inputs;s++)  {
+    for (int i1 = 0; i1 < 22; i1++)   {
+      memcpy(&c[s][2 * i1 * 384]      , &cc[s][i1 * 384], 384 * sizeof(uint32_t));
+      memcpy(&c[s][(2 * i1 + 1) * 384], &cc[s][i1 * 384], 384 * sizeof(uint32_t));
+    }
+  }
+#endif
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/ldpc_rate_matching.cu

@@ -0,0 +1,300 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+#include <stdio.h>
+#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h"
+
+__device__ __forceinline__ int clamp_i16_to_i8(int x)
+{
+    // x is int (promoted)
+    x = (x < -128) ? -128 : x;
+    x = (x >  127) ?  127 : x;
+    return x;
+}
+
+__device__ __forceinline__ uint32_t packs_4x16_to_4x8(uint32_t a, uint32_t b)
+{
+    int a0 = (int)(int16_t)(a & 0xFFFFu);
+    int a1 = (int)(int16_t)(a >> 16);
+    int b0 = (int)(int16_t)(b & 0xFFFFu);
+    int b1 = (int)(int16_t)(b >> 16);
+
+    uint32_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
+    uint32_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
+    uint32_t o2 = (uint8_t)(int8_t)clamp_i16_to_i8(b0);
+    uint32_t o3 = (uint8_t)(int8_t)clamp_i16_to_i8(b1);
+
+    return (o0) | (o1 << 8) | (o2 << 16) | (o3 << 24);
+}
+
+__device__ __forceinline__ uint16_t packs_2x16_to_2x8(uint32_t a)
+{
+    int a0 = (int)(int16_t)(a & 0xFFFFu);
+    int a1 = (int)(int16_t)(a >> 16);
+
+    uint16_t o0 = (uint8_t)(int8_t)clamp_i16_to_i8(a0);
+    uint16_t o1 = (uint8_t)(int8_t)clamp_i16_to_i8(a1);
+
+    return (o0) | (o1 << 8) ;
+}
+
+__global__ void rm(int Ncb_4,int ind0_4, int E1, int E2, int r_firstE2, int Foffset_4, int F_4, int clear, int seglen_4, int K_4, int Z_4, uint32_t *d, uint32_t *e, uint32_t *llr_buffer) {
+
+     int ind_4 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+     int r = (int)blockIdx.y;
+     uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
+     uint32_t *dr = d + (2*r*seglen_4);
+     uint32_t *llr_bufferr = llr_buffer + (r*seglen_4);
+#ifdef DEBUG_RM
+     if (blockIdx.x == 0 && threadIdx.x == 0) printf("r %d, seglen %d\n",r,seglen_4);
+#endif
+     if (ind_4>= Foffset_4 && ind_4 < Foffset_4+F_4) {
+	     llr_bufferr[ind_4-Foffset_4+K_4-F_4] = 0x7f7f7f7f;
+#ifdef DEBUG_RM
+	     if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",seglen_4*r + ind_4-Foffset_4+K_4-F_4,ind_4,K_4,Foffset_4,F_4);
+#endif
+	     return;
+     }
+
+     if (ind_4 >= Ncb_4) return;
+     if (clear == 1) { dr[2*ind_4] = 0; dr[(2*ind_4)+1]=0; }
+
+     int E_4;
+     if (r<r_firstE2) E_4=(E1/4); else E_4=(E2/4);
+
+     int ind1=ind0_4,ind2;
+     int k=0;
+
+#ifdef DEBUG_RM
+     if (r==0 && threadIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
+#endif
+     if (ind1 < Foffset_4) {
+       int ind2 = ind1 + min(Foffset_4-ind1,E_4);
+
+#ifdef DEBUG_RM
+       if (r==0 && threadIdx.x == 0) printf("check 1b: ind %d ind1 %d ind2 %d\n",ind,ind1,ind2);
+#endif
+       if (ind_4 >= ind1 && ind_4 < ind2) {
+	       dr[2*ind_4]     = __vaddss2(dr[2*ind_4],er[2*(ind_4-ind1)]);
+	       dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(ind_4-ind1)+1]);
+       }
+#ifdef DEBUG_RM
+       if (r<=1 && threadIdx.x == 0 && ind_4 >= ind1 && ind_4 < ind2) printf("write 1. r %d : ind %d, ind1 %d, ind2 %d,k %d/E %d, pos %d, er %d %d\n",r,ind_4,ind1,ind2,ind_4-ind1,E_4,4*(ind_4-ind1),(int16_t)(er[2*(ind_4-ind1)]&0xffff),(int16_t)(er[2*(ind_4-ind1)]>>16));
+#endif
+       k=ind2-ind1;
+       ind1 = ind2;
+     }
+
+#ifdef DEBUG_RM
+     if (r==0 && threadIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
+#endif
+     if (ind1 >= Foffset_4 && ind1 < Foffset_4 + F_4) ind1 = Foffset_4 + F_4;
+     ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
+
+#ifdef DEBUG_RM
+     if (r==0 && threadIdx.x == 0) printf("check 2b: ind %d ind1 %d ind2 %d\n",ind_4,ind1,ind2);
+#endif
+     if (ind_4 >= ind1 && ind_4 < ind2)  { 
+	     dr[2*ind_4]     = __vaddss2(dr[2*ind_4],er[2*(k+(ind_4-ind1))]);
+	     dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+(ind_4-ind1))+1]);
+     }
+#ifdef DEBUG_RM
+     if (r==0 && threadIdx.x == 0 && ind >= ind1 && ind < ind2) printf("write 2. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
+#endif
+     k+=(ind2-ind1);
+
+#ifdef DEBUG_RM
+     if (r==0 && threadIdx.x == 0) printf("check k %d E %d\n",k,E);
+#endif
+     while (k < E_4) {
+	ind2 = min(Foffset_4,E_4-k);
+	if (ind_4 < ind2) {
+	 	dr[2*ind_4]     = __vaddss2(dr[2*ind_4],er[2*(k+ind_4)]);
+	 	dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4)+1]);
+	}
+#ifdef DEBUG_RM
+        if (r==0 && threadIdx.x == 0 && ind < ind2 && ind >= ind1) printf("3. ind %d, ind2 %d, k %d/E %d\n",ind,ind2,k+ind,E);
+#endif
+	k+=ind2;
+
+	ind1=Foffset_4+F_4;
+	ind2 = ind1 + min(Ncb_4-ind1,E_4-k);
+	if (ind_4 >= ind1 && ind_4 < ind2 && k < E_4) {
+		dr[2*ind_4]     = __vaddss2(dr[2*ind_4],er[2*(k+ind_4-ind1)]);
+		dr[(2*ind_4)+1] = __vaddss2(dr[(2*ind_4)+1],er[2*(k+ind_4-ind1)+1]);
+	}
+#ifdef DEBUG_RM
+        if (r==0 && threadIdx.x == 0 && ind < ind && ind >= ind1) printf("4. ind %d, ind1 %d, ind2 %d, k %d/E %d\n",ind,ind1,ind2,k+ind-ind1,E);
+#endif
+	k+=(ind2-ind1);
+     }
+       // note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in position r*seglen + (Kprime+F = K .. 2Z+(66*Z)=seglen  
+     llr_bufferr[2*Z_4 + ind_4] = packs_4x16_to_4x8(dr[(2*ind_4)], dr[(2*ind_4)+1]);
+#ifdef DEBUG_RM
+     if (r<=1 && threadIdx.x == 0) printf("RM segment %d: writing %x to position %d (ind %d)\n",r,llr_bufferr[2*Z_4 + ind_4],r*seglen_4 + 2*Z_4 + ind_4,ind_4);
+#endif
+     
+}	
+
+__global__ void rm2(int Ncb_2,int ind0_2, int E1, int E2, int r_firstE2, int Foffset_2, int F_2, int clear, int seglen_2, int K_2, int Z_2, uint32_t *d, uint32_t *e, uint16_t *llr_buffer) {
+
+     int ind_2 = (int)(blockIdx.x * blockDim.x + threadIdx.x);
+     int r = (int)blockIdx.y;
+     uint32_t *er = e + ((r<r_firstE2) ? r*(E1>>1) : ((r_firstE2*E1)+(r-r_firstE2)*E2)>>1);
+     uint32_t *dr = d + (r*seglen_2);
+     uint16_t *llr_bufferr = llr_buffer + (r*seglen_2);
+
+     if (ind_2 >= Ncb_2) return;
+#ifdef RM_DEBUG
+     if (r==0) printf("Looking for ind %d...%d, Ncb %d\n",ind_2*2,1+ind_2*2,Ncb_2*2);
+#endif
+     if (ind_2>= Foffset_2 && ind_2 < Foffset_2+F_2) {
+             llr_bufferr[ind_2-Foffset_2+K_2-F_2] = 0x7f7f;
+#ifdef RM_DEBUG
+             if (r==0) printf("writing 0x7f7f7f7f to position %d (ind %d, K %d, Foffset %d, F %d)\n",(seglen_2*r + ind_2-Foffset_2+K_2-F_2)*2,ind_2*2,K_2*2,Foffset_2*2,F_2*2);
+#endif
+             return;
+     }
+
+     if (clear == 1) dr[ind_2] = 0;
+
+     int E_2;
+     if (r<r_firstE2) E_2=E1/2; else E_2=E2/2;
+
+     int ind1=ind0_2,ind2;
+     int k=0;
+
+//     if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 1a: ind %d ind1 %d Foffset %d\n",ind,ind1,Foffset);
+     if (ind1 < Foffset_2) {
+       int ind2 = ind1 + min(Foffset_2-ind1,E_2);
+
+//       if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM1B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
+       if (ind_2 >= ind1 && ind_2 < ind2) {
+               dr[ind_2] = __vaddss2(dr[ind_2],er[ind_2-ind1]);
+       }
+#ifdef RM_DEBUG
+       if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM1A: ind_2 %d, ind1 %d, ind2 %d,k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(ind_2-ind1),2*E_2);
+#endif
+       k=ind2-ind1;
+       ind1 = ind2;
+     }
+
+//     if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check 2a: ind %d ind1 %d Foffset %d Foffset+F %d\n",ind,ind1,Foffset,Foffset+F);
+     if (ind1 >= Foffset_2 && ind1 < Foffset_2 + F_2) ind1 = Foffset_2 + F_2;
+     ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
+
+//     if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM2B: ind %d ind1 %d ind2 %d k%d/E %d\n",4*ind,4*ind1,4*ind2,4*k,4*E);
+     if (ind_2 >= ind1 && ind_2 < ind2)  {
+             dr[ind_2]     = __vaddss2(dr[ind_2],er[(k+(ind_2-ind1))]);
+     }
+#ifdef RM_DEBUG
+     if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 >= ind1 && ind_2 < ind2) printf("RM2A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2,2*(k+(ind-ind1)),2*E_2);
+#endif
+     k+=(ind2-ind1);
+
+//     if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check k %d E %d\n",k,E);
+     while (k < E_2) {
+        ind2 = min(Foffset_2,E_2-k);
+//        if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM3B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
+        if (ind_2 < ind2) {
+                dr[ind_2]     = __vaddss2(dr[ind_2],er[(k+ind_2)]);
+        }
+#ifdef RM_DEBUG
+        if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 ) printf("RM3A: ind %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind2,2*(k+ind_2),2*E_2);
+#endif
+        k+=ind2;
+
+        ind1=Foffset_2+F_2;
+        ind2 = ind1 + min(Ncb_2-ind1,E_2-k);
+//        if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("check RM4B: ind %d ind1 %d ind2 %d k %d/E %d\n",4*ind,0,4*ind2,4*k,4*E);
+        if (ind_2 >= ind1 && ind_2 < ind2 && k < E_2) {
+                dr[ind_2]     = __vaddss2(dr[ind_2],er[(k+ind_2-ind1)]);
+        }
+#ifdef RM_DEBUG
+        if (r==0 /*&& threadIdx.x == 0 && blockIdx.x == 0*/ && ind_2 < ind2 && ind_2 >= ind1) printf("RM4A: ind %d, ind1 %d, ind2 %d, k %d/E %d\n",2*ind_2,2*ind1,2*ind2_2,2*(k+ind_2-ind1),2*E_2);
+#endif
+        k+=(ind2-ind1);
+
+     }
+       // note the offset here is such that when ind < Foffset = Kprime - 2Z, the output is put in position r*seglen + (2Z ... Kprime) and when ind > Foffset+F, it is in potiion r*seglent + (Kprime+F = K .. 2Z+(66*Z)=seglen
+     llr_bufferr[2*Z_2 + ind_2] = packs_2x16_to_2x8(dr[ind_2]);
+//     if (r==0 && threadIdx.x == 0 && blockIdx.x == 0) printf("writing %x to position %d (ind %d)\n",llr_bufferr[2*Z + ind_2],r*seglen + 2*Z + ind_2,ind_2);
+}
+
+
+static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
+extern "C" int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
+                                             uint8_t BG,
+                                             uint16_t Z,
+                                             int16_t *d,
+                                             int16_t *soft_input,
+					     int8_t *llr_buffer,
+					     uint32_t K,
+                                             uint8_t C,
+                                             uint8_t rvidx,
+                                             uint8_t clear,
+                                             uint32_t E1,
+				             uint32_t E2,
+				             uint32_t r_firstE2,
+                                             uint32_t F,
+                                             uint32_t Foffset,
+					     cudaStream_t *s,
+					     int8_t sidx)
+{
+  if (C == 0 || C>132) {
+    printf("nr_rate_matching: invalid parameter C %d\n", C);
+    exit(-1);
+  }
+
+  //Bit selection
+  uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
+  uint32_t Ncb;
+  if (Tbslbrm == 0)
+    Ncb = N;
+  else {
+    uint32_t Nref = (3 * Tbslbrm / (2 * C)); //R_LBRM = 2/3
+    Ncb = min(N, Nref);
+  }
+
+  uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
+
+
+  int nthreads=Z;
+#ifdef RM_DEBUG
+  printf("\nrm (%d,%d): Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",nblocks.x,nthreads,Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
+#endif
+  if ((E1&3) == 0 && (E2&3) == 0) {
+    dim3 nblocks(((Ncb>>2) + nthreads-1)/nthreads,C);
+    rm<<<nblocks, nthreads, 0, s[sidx]>>>(Ncb/4,ind/4,E1,E2,r_firstE2,Foffset/4,F/4,clear,68*384/4,K/4,Z/4,(uint32_t*)d,(uint32_t*)soft_input,(uint32_t*)llr_buffer);
+  }
+  else {
+    dim3 nblocks2(((Ncb>>1) + nthreads-1)/nthreads,C);
+    rm2<<<nblocks2, nthreads, 0, s[sidx]>>>(Ncb/2,ind/2,E1,E2,r_firstE2,Foffset/2,F/2,clear,68*384/2,K/2,Z/2,(uint32_t*)d,(uint32_t*)soft_input,(uint16_t*)llr_buffer);
+  }
+  cudaError_t err=cudaPeekAtLastError();
+  
+  if (err!=cudaSuccess) {
+    printf("cuda error (nr_rate_matching_ldpc_rx_cuda): %s  Ncb %d, ind %d, rvidx %d, E1 %d, E2 %d, Foffset %d, F %d, K %d, Z %d, clear %d\n",cudaGetErrorString(err),Ncb, ind, rvidx, E1, E2, Foffset, F,K,Z,clear);
+    exit(-1);
+  }
+  return(0);
+} 
+
+

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_BnProcKernel_BG1.h

@@ -0,0 +1,444 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+/*! \file nrLDPC_CUDA_BnProcKernel_BG1.h
+ * \brief Defines the kernels for bit node processing
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note
+ * \warning
+ */
+#include <cuda_runtime.h>
+#include <stdint.h>
+#include <stdio.h>
+#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPCdecoder_defs.h"
+
+__device__ __forceinline__ void unpack_and_sign_extend(uint32_t packed, uint32_t *val_lo, uint32_t *val_hi)
+{
+  uint32_t magic_sub = 0x00800080;
+
+  uint32_t lo_zext = __byte_perm(packed, 0, 0x5140);
+  *val_lo = __vsub2(lo_zext ^ magic_sub, magic_sub);
+
+  uint32_t hi_zext = __byte_perm(packed, 0, 0x7362);
+  *val_hi = __vsub2(hi_zext ^ magic_sub, magic_sub);
+}
+
+__device__ __forceinline__ uint32_t saturate_and_pack(uint32_t val_lo, uint32_t val_hi)
+{
+  uint32_t lo_clamped = __vmins2(val_lo, 0x007F007F); // min(v, 127)
+  lo_clamped = __vmaxs2(lo_clamped, 0xFF80FF80); // max(v, -128)
+
+  uint32_t hi_clamped = __vmins2(val_hi, 0x007F007F);
+  hi_clamped = __vmaxs2(hi_clamped, 0xFF80FF80);
+
+  return __byte_perm(lo_clamped, hi_clamped, 0x6420);
+}
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R13(const int8_t *__restrict__ d_bnProcBuf,
+                                                             int8_t *__restrict__ d_cnProcBuf,
+                                                             const int8_t *__restrict__ d_llrProcBuf,
+                                                             int8_t *__restrict__ d_llrRes,
+                                                             uint32_t lane,
+                                                             uint32_t BnGrpIdx,
+                                                             uint32_t BnIdx,
+                                                             uint32_t GrpNum,
+                                                             uint32_t Bn2MsgStartIdx,
+                                                             uint32_t Zc,
+                                                             uint32_t ZcIdx)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)BnGrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < BnGrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  uint32_t BricksToBeGet;
+  if (BnGrpIdx == 1) {
+    BricksToBeGet = packed_intrinsic;
+    uint32_t MsgIdx = Bn2MsgStartIdx;
+    uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+    int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
+
+    moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+  } else {
+    for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
+      uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+      uint32_t prev = bnProcBufPtr[prevIdxWords];
+      BricksToBeGet = __vsubss4(saturated_llr, prev);
+      uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
+      uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+      int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
+
+      moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+    }
+  }
+}
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R23(const int8_t *__restrict__ d_bnProcBuf,
+                                                             int8_t *__restrict__ d_cnProcBuf,
+                                                             const int8_t *__restrict__ d_llrProcBuf,
+                                                             int8_t *__restrict__ d_llrRes,
+                                                             uint32_t lane,
+                                                             uint32_t BnGrpIdx,
+                                                             uint32_t BnIdx,
+                                                             uint32_t GrpNum,
+                                                             uint32_t Bn2MsgStartIdx,
+                                                             uint32_t Zc,
+                                                             uint32_t ZcIdx)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)BnGrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < BnGrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  uint32_t BricksToBeGet;
+  if (BnGrpIdx == 1) {
+    BricksToBeGet = packed_intrinsic;
+    uint32_t MsgIdx = Bn2MsgStartIdx;
+    uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
+    int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
+
+    moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+  } else {
+    for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
+      uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+      uint32_t prev = bnProcBufPtr[prevIdxWords];
+      BricksToBeGet = __vsubss4(saturated_llr, prev);
+      uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
+      uint32_t circShift = bn_cn_map_BG1_Z_R23[MsgIdx][ZcIdx];
+      int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R23[MsgIdx][0]);
+
+      moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+    }
+  }
+}
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R89(const int8_t *__restrict__ d_bnProcBuf,
+                                                             int8_t *__restrict__ d_cnProcBuf,
+                                                             const int8_t *__restrict__ d_llrProcBuf,
+                                                             int8_t *__restrict__ d_llrRes,
+                                                             uint32_t lane,
+                                                             uint32_t BnGrpIdx,
+                                                             uint32_t BnIdx,
+                                                             uint32_t GrpNum,
+                                                             uint32_t Bn2MsgStartIdx,
+                                                             uint32_t Zc,
+                                                             uint32_t ZcIdx)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)BnGrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < BnGrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  uint32_t BricksToBeGet;
+  if (BnGrpIdx == 1) {
+    BricksToBeGet = packed_intrinsic;
+    uint32_t MsgIdx = Bn2MsgStartIdx;
+    uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
+    int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
+
+    moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+  } else {
+    for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
+      uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+      uint32_t prev = bnProcBufPtr[prevIdxWords];
+      BricksToBeGet = __vsubss4(saturated_llr, prev);
+      uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
+      uint32_t circShift = bn_cn_map_BG1_Z_R89[MsgIdx][ZcIdx];
+      int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R89[MsgIdx][0]);
+
+      moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+    }
+  }
+}
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_last(const int8_t *__restrict__ d_bnProcBuf,
+                                                              int8_t *__restrict__ d_cnProcBuf,
+                                                              const int8_t *__restrict__ d_llrProcBuf,
+                                                              int8_t *__restrict__ d_llrRes,
+                                                              uint32_t lane,
+                                                              uint32_t BnGrpIdx,
+                                                              uint32_t BnIdx,
+                                                              uint32_t GrpNum,
+                                                              uint32_t Bn2MsgStartIdx,
+                                                              uint32_t Zc,
+                                                              uint32_t ZcIdx)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)BnGrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < BnGrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  ((int32_t *)(d_llrRes))[lane] = saturated_llr;
+}
+
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge(const int8_t *__restrict__ d_bnProcBuf,
+                                                              int8_t *__restrict__ d_cnProcBuf,
+                                                              const int8_t *__restrict__ d_llrProcBuf,
+                                                              int8_t *__restrict__ d_llrRes,
+                                                              uint32_t lane,
+                                                              uint32_t GrpIdx,
+                                                              uint32_t MsgIdx,
+                                                              uint32_t BnIdx,
+                                                              uint32_t GrpNum,
+                                                              uint32_t circShift,
+                                                              uint32_t Zc)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)GrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < GrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  uint32_t BricksToBeGet;
+  if (GrpIdx == 1) {
+    BricksToBeGet = packed_intrinsic;
+  } else {
+    uint32_t prevIdxWords = (MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+    uint32_t prev = bnProcBufPtr[prevIdxWords];
+    BricksToBeGet = __vsubss4(saturated_llr, prev);
+  }
+
+  moveBricks_forput_circ(d_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+}
+
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge_last(const int8_t *__restrict__ d_bnProcBuf,
+                                                                   int8_t *__restrict__ d_cnProcBuf,
+                                                                   const int8_t *__restrict__ d_llrProcBuf,
+                                                                   int8_t *__restrict__ d_llrRes,
+                                                                   uint32_t lane,
+                                                                   uint32_t GrpIdx,
+                                                                   uint32_t MsgIdx,
+                                                                   uint32_t BnIdx,
+                                                                   uint32_t GrpNum,
+                                                                   uint32_t circShift,
+                                                                   uint32_t Zc)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // ---  2-Way Unroll ---
+
+  for (; i < (int)GrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < GrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  if (MsgIdx == 0) {
+    ((int32_t *)(d_llrRes))[lane] = saturated_llr;
+  }
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_CnProcKernel_BG1.h

@@ -0,0 +1,566 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+/*! \file nrLDPC_CUDA_CnProcKernel_BG1.h
+ * \brief Defines the kernels for check node processing
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note
+ * \warning
+ */
+#pragma once
+
+#include <cuda_runtime.h>
+#include <stdint.h>
+#include <stdio.h>
+#include "nrLDPC_CUDA_public.h"
+#include "nrLDPC_CUDA_lut.h"
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R13_node(const int8_t *__restrict__ d_cnProcBuf,
+                                                                  int8_t *__restrict__ d_bnProcBuf,
+                                                                  uint32_t lane,
+                                                                  uint32_t CnIdx,
+                                                                  uint32_t CnNumInGrp,
+                                                                  uint32_t CnGrpIdxNum,
+                                                                  uint32_t Cn2MsgStartIdx,
+                                                                  uint32_t Zc,
+                                                                  uint32_t ZcIdx)
+{
+  uint32_t min1 = 0x7F7F7F7F;
+  uint32_t min2 = 0x7F7F7F7F;
+  uint32_t total_xor = 0;
+
+  uint32_t cache_raw[19];
+  uint32_t cache_abs[19];
+
+  const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
+  const int32_t *currPtr = cnProcBufPtr;
+  uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
+#pragma unroll
+  for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
+    uint32_t val = *currPtr;
+
+    cache_raw[MsgIdx] = val;
+    uint32_t v_abs = __vabs4(val);
+    cache_abs[MsgIdx] = v_abs;
+
+    total_xor = __vxor4(total_xor, val);
+
+    uint32_t old_min1 = min1;
+
+    min1 = __vminu4(old_min1, v_abs);
+
+    uint32_t candidate = __vmaxu4(old_min1, v_abs);
+
+    min2 = __vminu4(min2, candidate);
+    currPtr += offset;
+  }
+
+#pragma unroll
+  for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
+    uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
+
+    uint32_t my_abs = cache_abs[temp_MsgIdx];
+
+    uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
+
+    uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
+
+    // final_mag = scale_int8x4(final_mag, 0.75);
+
+    uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
+
+    uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
+    uint32_t circShift = cn_bn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+    int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R13[MsgIdx][0]);
+
+    moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+  }
+}
+__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R23_node(const int8_t *__restrict__ d_cnProcBuf,
+                                                                  int8_t *__restrict__ d_bnProcBuf,
+                                                                  uint32_t lane,
+                                                                  uint32_t CnIdx,
+                                                                  uint32_t CnNumInGrp,
+                                                                  uint32_t CnGrpIdxNum,
+                                                                  uint32_t Cn2MsgStartIdx,
+                                                                  uint32_t Zc,
+                                                                  uint32_t ZcIdx)
+{
+
+  uint32_t min1 = 0x7F7F7F7F;
+  uint32_t min2 = 0x7F7F7F7F;
+  uint32_t total_xor = 0;
+
+  uint32_t cache_raw[19];
+  uint32_t cache_abs[19];
+
+  const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
+  const int32_t *currPtr = cnProcBufPtr;
+  uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
+#pragma unroll
+  for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
+    uint32_t val = *currPtr;
+
+    cache_raw[MsgIdx] = val;
+    uint32_t v_abs = __vabs4(val);
+    cache_abs[MsgIdx] = v_abs;
+
+    total_xor = __vxor4(total_xor, val);
+
+    uint32_t old_min1 = min1;
+
+    min1 = __vminu4(old_min1, v_abs);
+
+    uint32_t candidate = __vmaxu4(old_min1, v_abs);
+
+    min2 = __vminu4(min2, candidate);
+    currPtr += offset;
+  }
+
+#pragma unroll
+  for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
+    uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
+
+    uint32_t my_abs = cache_abs[temp_MsgIdx];
+
+    uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
+
+    uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
+
+    // final_mag = scale_int8x4(final_mag, 0.75);
+
+    uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
+
+    uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
+    uint32_t circShift = cn_bn_map_BG1_Z_R23[MsgIdx][ZcIdx];
+    int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R23[MsgIdx][0]);
+
+    moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+  }
+}
+__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R89_node(const int8_t *__restrict__ d_cnProcBuf,
+                                                                  int8_t *__restrict__ d_bnProcBuf,
+                                                                  uint32_t lane,
+                                                                  uint32_t CnIdx,
+                                                                  uint32_t CnNumInGrp,
+                                                                  uint32_t CnGrpIdxNum,
+                                                                  uint32_t Cn2MsgStartIdx,
+                                                                  uint32_t Zc,
+                                                                  uint32_t ZcIdx)
+{
+
+  uint32_t min1 = 0x7F7F7F7F;
+  uint32_t min2 = 0x7F7F7F7F;
+  uint32_t total_xor = 0;
+
+  uint32_t cache_raw[19];
+  uint32_t cache_abs[19];
+
+  const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
+  const int32_t *currPtr = cnProcBufPtr;
+  uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
+#pragma unroll
+  for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
+    uint32_t val = *currPtr;
+
+    cache_raw[MsgIdx] = val;
+    uint32_t v_abs = __vabs4(val);
+    cache_abs[MsgIdx] = v_abs;
+
+    total_xor = __vxor4(total_xor, val);
+
+    uint32_t old_min1 = min1;
+
+    min1 = __vminu4(old_min1, v_abs);
+
+    uint32_t candidate = __vmaxu4(old_min1, v_abs);
+
+    min2 = __vminu4(min2, candidate);
+    currPtr += offset;
+  }
+
+#pragma unroll
+  for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
+    uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
+
+    uint32_t my_abs = cache_abs[temp_MsgIdx];
+
+    uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
+
+    uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
+
+    // final_mag = scale_int8x4(final_mag, 0.75);
+
+    uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
+
+    uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
+    uint32_t circShift = cn_bn_map_BG1_Z_R89[MsgIdx][ZcIdx];
+    int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R89[MsgIdx][0]);
+
+    moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+  }
+}
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G3(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  // loop starts here
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG3[row][1] * 4);
+
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G4(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][0] * 4);
+
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG4[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G5(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G6(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG6[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G7(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG7[row][5] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G8(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][5] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG8[row][6] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G9(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][5] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][6] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG9[row][7] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G10(const int8_t *__restrict__ p_cnProcBuf,
+                                                          int8_t *__restrict__ p_bnProcBuf,
+                                                          uint32_t row,
+                                                          uint32_t lane,
+                                                          uint32_t idxBn,
+                                                          uint32_t circShift,
+                                                          uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][0] * 4);
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][5] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][6] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][7] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG10[row][8] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G19(const int8_t *__restrict__ p_cnProcBuf,
+                                                          int8_t *__restrict__ p_bnProcBuf,
+                                                          uint32_t row,
+                                                          uint32_t lane,
+                                                          uint32_t idxBn,
+                                                          uint32_t circShift,
+                                                          uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][0] * 4);
+
+  sgn = __vxor4(ones, ymm0);
+  min = __vabs4(ymm0);
+
+  //-------------------------loop starts here-------------------------------
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][4] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][5] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][6] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][7] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][8] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][9] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][10] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][11] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][12] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][13] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][14] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][15] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][16] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG19[row][17] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  //-------------------------------------------------------------------------
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_mPassKernel_BG1.h

@@ -0,0 +1,186 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+/*! \file nrLDPC_CUDA_mPassKernel.h
+ * \brief Defines the kernels for message passing in CUDA version of LDPC decoder
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note
+ * \warning
+ */
+#pragma once
+
+#include <cuda_runtime.h>
+#include <stdint.h>
+#include <stdio.h>
+#include "nrLDPC_CUDA_public.h"
+#include "nrLDPC_CUDA_shared_param.h"
+
+__device__ __forceinline__ void llrPreProc_Kernel_BG1_int8_Gn_stream(const int8_t *p_llr,
+                                                                     int8_t *p_llrProcBuf,
+                                                                     int8_t *p_cnProcBuf,
+                                                                     uint32_t MsgIdx,
+                                                                     uint32_t lane,
+                                                                     uint32_t colIdx,
+                                                                     uint32_t idxBn,
+                                                                     uint32_t GrpIdx,
+                                                                     uint32_t circShift,
+                                                                     uint32_t Zc,
+                                                                     uint32_t R)
+{
+  {
+    uint32_t *p_cnProcBufBit;
+
+    uint8_t bricksLocal[4];
+    uint8_t *BricksToBeMoved = bricksLocal;
+
+    p_cnProcBufBit = (uint32_t *)(p_cnProcBuf + d_lut_numCnInCnGroups_BG1_R13[GrpIdx] * NR_LDPC_ZMAX * MsgIdx + lane * 4);
+
+    moveBricks_invget_circ((int8_t *)&p_llr[idxBn * Zc], lane * 4, BricksToBeMoved, Zc, circShift);
+
+    *p_cnProcBufBit = *(uint32_t *)BricksToBeMoved;
+  }
+
+  if (colIdx >= 68)
+    return;
+
+  const uint8_t numBn2CnG1 =
+      (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
+                : ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]); // for R13 is 42
+  const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1; // 26 for BG1
+  const uint32_t colG1 = startColParity * Zc;
+
+  const uint32_t *lut_llr2llrProcBufAddr =
+      (R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
+  const uint32_t *lut_llr2llrProcBufBnPos =
+      (R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
+
+  if (colIdx < startColParity) {
+    const uint32_t idxBn = lut_llr2llrProcBufAddr[colIdx] + lut_llr2llrProcBufBnPos[colIdx] * NR_LDPC_ZMAX;
+    int32_t *dst = (int32_t *)(&p_llrProcBuf[idxBn] + lane * 4);
+    int32_t *src = (int32_t *)(&p_llr[colIdx * Zc] + lane * 4);
+    *dst = *src;
+  } else {
+    colIdx = colIdx - startColParity;
+    if (numBn2CnG1 > 0 && colIdx < numBn2CnG1) {
+      int32_t *dst = (int32_t *)(&p_llrProcBuf[colIdx * NR_LDPC_ZMAX] + lane * 4);
+      int32_t *src = (int32_t *)(&p_llr[colG1 + colIdx * Zc] + lane * 4);
+      *dst = *src;
+    }
+  }
+}
+
+__device__ void llr2bit_Kernel_BG1_int8(uint32_t R,
+                                        uint8_t *__restrict__ out,
+                                        const int8_t *__restrict__ llrRes,
+                                        uint32_t numLLR,
+                                        uint32_t Zc)
+{
+  uint32_t lane = threadIdx.x;
+  uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
+
+  if (outColIdx >= num_TotalBlocks_llr_llrRes)
+    return;
+
+  // Constants Setup
+  const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
+                                       : ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
+  const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
+  const uint32_t *lut_Addr =
+      (R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
+  const uint32_t *lut_Pos =
+      (R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
+
+  int32_t raw_llrs;
+
+  // Load Data: Handle Systematic (Scatter/Gather) vs Parity (Linear) mapping
+  if (outColIdx < startColParity) {
+    uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
+    raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
+  } else {
+    uint32_t srcParityIdx = outColIdx - startColParity;
+    if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
+      raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
+    } else {
+      raw_llrs = 0;
+    }
+  }
+
+  // Hard Decision: Convert 4 int8 LLRs -> 4 bytes (0 or 1)
+  int8_t *p_val = (int8_t *)&raw_llrs;
+  uint32_t my_word = 0;
+
+#pragma unroll
+  for (int i = 0; i < 4; i++) {
+    // Hard decision: LLR < 0 implies bit 1
+    uint32_t byte_val = (p_val[i] < 0) ? 1 : 0;
+    my_word |= (byte_val << (i * 8));
+  }
+
+  // Store output linearly
+  *(uint32_t *)(&out[outColIdx * Zc + lane * 4]) = my_word;
+}
+
+__device__ void llr2bitPacked_Kernel_BG1_int8(uint32_t R,
+                                              uint8_t *__restrict__ out,
+                                              const int8_t *__restrict__ llrRes,
+                                              uint32_t numLLR,
+                                              uint32_t Zc)
+{
+  uint32_t lane = threadIdx.x;
+  uint32_t outColIdx = (blockIdx.x << 2) + threadIdx.y;
+
+  if (outColIdx >= num_TotalBlocks_llr_llrRes)
+    return;
+
+  const uint8_t numBn2CnG1 = (R == 13) ? d_lut_numBnInBnGroups_BG1_R13[0]
+                                       : ((R == 89) ? d_lut_numBnInBnGroups_BG1_R89[0] : d_lut_numBnInBnGroups_BG1_R23[0]);
+  const uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;
+  const uint32_t *lut_Addr =
+      (R == 13) ? d_llr2llrProcBufAddr_BG1_R13 : ((R == 89) ? d_llr2llrProcBufAddr_BG1_R89 : d_llr2llrProcBufAddr_BG1_R23);
+  const uint32_t *lut_Pos =
+      (R == 13) ? d_llr2llrProcBufBnPos_BG1_R13 : ((R == 89) ? d_llr2llrProcBufBnPos_BG1_R89 : d_llr2llrProcBufBnPos_BG1_R23);
+
+  int32_t raw_llrs;
+
+  // Load Data: Handle Systematic vs Parity mapping
+  if (outColIdx < startColParity) {
+    uint32_t idxBn = lut_Addr[outColIdx] + lut_Pos[outColIdx] * NR_LDPC_ZMAX;
+    raw_llrs = *(const int32_t *)(&llrRes[idxBn] + lane * 4);
+  } else {
+    uint32_t srcParityIdx = outColIdx - startColParity;
+    if (numBn2CnG1 > 0 && outColIdx < numBn2CnG1) {
+      raw_llrs = *(const int32_t *)(llrRes + srcParityIdx * NR_LDPC_ZMAX + lane * 4);
+    } else {
+      raw_llrs = 0;
+    }
+  }
+
+  // Extract 4 bits from 4 LLRs (Local Packing)
+  // Assuming MSB order within nibble: [3][2][1][0]
+  uint32_t my_4_bits = 0;
+  int8_t *p_val = (int8_t *)&raw_llrs;
+
+#pragma unroll
+  for (int i = 0; i < 4; i++) {
+    if (p_val[i] < 0) {
+      my_4_bits |= (1 << (3 - i));
+    }
+  }
+
+  // Thread Cooperation: Pair threads to pack 8 bits (1 byte)
+  // Even thread (0,2..) takes low nibble, Odd thread (1,3..) takes high nibble.
+  uint32_t neighbor_bits = __shfl_xor_sync(0xffffffff, my_4_bits, 1);
+
+  if ((lane & 1) == 0) {
+    // Combine: [Odd Thread Bits (High)] | [My Bits (Low)]
+    uint8_t packed_byte = (neighbor_bits & 0xF) | ((my_4_bits & 0xF) << 4);
+
+    // Output addr: lane steps by 4 LLRs, but we output 1 byte per 8 LLRs -> lane >> 1
+    uint32_t outAddr = outColIdx * (Zc >> 3) + (lane >> 1);
+    out[outAddr] = packed_byte;
+  }
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_public.h

@@ -0,0 +1,120 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+/*! \file nrLDPC_CUDA_public.h
+ * \brief Shared functions in CUDA implementation of LDPC decoder 
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note 
+ * \warning
+ */
+
+#pragma once
+
+#include <cuda_runtime.h>
+#include <stdint.h>
+#include <stdio.h>
+
+#include <cuda_runtime.h>
+#include <stdint.h>
+
+
+__device__ __forceinline__ void moveBricks_invput_circ(int8_t *__restrict__ dstBuf,
+                                                       uint32_t dstBuf_Offset,
+                                                       const uint8_t *__restrict__ Four_Bricks,
+                                                       uint32_t Z,
+                                                       uint32_t cshift)
+{
+    uint32_t tmp = cshift + dstBuf_Offset;
+    uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
+
+    if (pos <= Z - 4) {
+        uint32_t val = *(const uint32_t*)Four_Bricks;
+        memcpy(dstBuf + pos, &val, 4);
+    } 
+    else {
+        uint32_t bytes_at_end = Z - pos; // 1, 2, or 3
+        
+        #pragma unroll
+        for(int i=0; i<4; i++) {
+            if (i < bytes_at_end) {
+                dstBuf[pos + i] = Four_Bricks[i];
+            } else {
+                dstBuf[i - bytes_at_end] = Four_Bricks[i];
+            }
+        }
+    }
+}
+
+__device__ __forceinline__ void moveBricks_forput_circ(int8_t *__restrict__ dstBuf,
+                                                       uint32_t dstBuf_Offset,
+                                                       const uint8_t *__restrict__ Four_Bricks,
+                                                       uint32_t Z,
+                                                       uint32_t cshift)
+{
+    uint32_t tmp = dstBuf_Offset + Z - cshift;
+    uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
+
+    if (pos <= Z - 4) {
+        uint32_t val = *(const uint32_t*)Four_Bricks;
+        memcpy(dstBuf + pos, &val, 4);
+    } 
+    else {
+        uint32_t bytes_at_end = Z - pos;
+        
+        #pragma unroll
+        for(int i=0; i<4; i++) {
+            if (i < bytes_at_end) {
+                dstBuf[pos + i] = Four_Bricks[i];
+            } else {
+                dstBuf[i - bytes_at_end] = Four_Bricks[i];
+            }
+        }
+    }
+}
+
+__device__ __forceinline__ void moveBricks_invget_circ(const int8_t *__restrict__ dstBuf,
+                                                       uint32_t dstBuf_Offset,
+                                                       uint8_t *__restrict__ Four_Bricks,
+                                                       uint32_t Z,
+                                                       uint32_t cshift)
+{
+    uint32_t tmp = cshift + dstBuf_Offset;
+    uint32_t pos = (tmp >= Z) ? tmp - Z : tmp;
+
+    if (pos <= Z - 4) {
+        uint32_t val;
+        memcpy(&val, dstBuf + pos, 4);
+        *(uint32_t*)Four_Bricks = val;
+    } 
+    else {
+        uint32_t bytes_at_end = Z - pos;
+        
+        #pragma unroll
+        for(int i=0; i<4; i++) {
+            if (i < bytes_at_end) {
+                Four_Bricks[i] = dstBuf[pos + i];
+            } else {
+                Four_Bricks[i] = dstBuf[i - bytes_at_end];
+            }
+        }
+    }
+}
+
+__device__ __forceinline__ uint32_t __vxor4(const uint32_t a, uint32_t b)
+{
+  return a ^ b; 
+}
+
+__device__ __forceinline__ uint32_t __vsign4(const uint32_t a, uint32_t b)
+{
+    uint32_t mask = __vcmplts4(b, 0); 
+    uint32_t bneg = __vneg4(a); 
+    return (mask & bneg) | (~mask & a);
+    //uint32_t is_zero_mask = __vcmpeq4(b, 0);
+    //return result & (~is_zero_mask);
+}
+

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_CUDA_shared_param.h

@@ -0,0 +1,84 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+/*! \file nrLDPC_CUDA_shared_param.h
+ * \brief Shared parameters in CUDA implementation of LDPC decoder 
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note 
+ * \warning
+ */
+
+#ifndef NRLDPC_CUDA_SHARED_PARAM_H_
+#define NRLDPC_CUDA_SHARED_PARAM_H_
+
+#include <cuda_runtime.h>
+
+#ifndef MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER
+#define MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER 36
+#endif
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#define RowLength 96 //Zc = 384/4 = 96
+
+#define num_TotalBlocks_BG1_R13_Edge 316//based on number of Cn2Bn Msgs
+#define num_TotalBlocks_BG1_R23_Edge 144
+#define num_TotalBlocks_BG1_R89_Edge 79
+#define num_TotalBlocks_llr_llrRes 22 //Only includes systematic bits
+
+#define num_TotalBlocks_cn_BG1_R13_Node 46 //based on number of CNs
+#define num_TotalBlocks_bn_BG1_R13_Node 68 //based on number of BNs
+#define num_TotalBlocks_cn_BG1_R23_Node 13
+#define num_TotalBlocks_bn_BG1_R23_Node 35
+#define num_TotalBlocks_cn_BG1_R89_Node 5
+#define num_TotalBlocks_bn_BG1_R89_Node 27
+
+#define JETSON_ORIN 
+//#define GH200
+
+#if defined(GH200)
+#define NodeEdge_Switch_Cn_R13 32
+#define NodeEdge_Switch_Bn_R13 10
+#define NodeEdge_Switch_Cn_R23 32
+#define NodeEdge_Switch_Bn_R23 12
+#define NodeEdge_Switch_Cn_R89 48
+#define NodeEdge_Switch_Bn_R89 24
+#elif defined(JETSON_ORIN)
+#define NodeEdge_Switch_Cn_R13 3
+#define NodeEdge_Switch_Bn_R13 1
+#define NodeEdge_Switch_Cn_R23 3
+#define NodeEdge_Switch_Bn_R23 1
+#define NodeEdge_Switch_Cn_R89 4
+#define NodeEdge_Switch_Bn_R89 2
+#endif
+
+#ifdef __cplusplus
+}
+#endif
+
+typedef struct KernelLaunchConfig {
+    dim3 grid;
+    dim3 block;
+}KernelLaunchConfig;
+
+
+typedef struct {
+    int idxBn;
+    int idxCn;
+    int preBuf;
+    int circShift;
+    int8_t dd;
+} DumpEntry;
+
+typedef struct {
+    int8_t* p_llr_ptr;     
+    int8_t* p_out_ptr;      
+} ldpc_cuda_bridge_t;
+
+#endif /* NRLDPC_CUDA_SHARED_PARAM_H_ */

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_cuda.c

@@ -0,0 +1,164 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+#include <stdint.h>
+#include <dlfcn.h>
+
+#include "log.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
+#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+
+
+/* segment interface */
+extern int32_t LDPCinit_cuda();
+int32_t LDPCinit()
+{
+  return LDPCinit_cuda();
+}
+
+// LDPCshutdown_cuda
+extern int32_t LDPCshutdown_cuda();
+int32_t LDPCshutdown()
+{
+  return LDPCshutdown_cuda();
+}
+
+
+// LDPCdecoder
+extern int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
+                         int8_t* p_llr,
+                         uint8_t* p_out,
+                         t_nrLDPC_time_stats* p_profiler,
+                         decode_abort_t* ab);
+
+
+// LDPCencoder <= (LDPCencoder32)
+extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
+
+
+/* slot interface */
+
+// fallback for LDPC sizes not implemented by CUDA LDPC
+nrLDPC_coding_interface_t ldpc_cpu;
+void *ldpc_lib_handle; // corresponding dlsym() handle, see also below
+
+extern int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters);
+int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *slot_params)
+{
+  // this should be the same as previous nrLDPC_coding_encoder() in nrLDPC_coding_segment_encoder.c
+  nrLDPC_TB_encoding_parameters_t tbCPU[32] = {0};
+  nrLDPC_slot_encoding_parameters_t cpu = {
+    .frame = slot_params->frame,
+    .slot = slot_params->slot,
+    .threadPool = slot_params->threadPool,
+    // timing?
+    .TBs = tbCPU,
+  };
+
+  for (int dlsch_id = 0; dlsch_id < slot_params->nb_TBs; dlsch_id++) {
+    nrLDPC_TB_encoding_parameters_t *tbp = &slot_params->TBs[dlsch_id];
+    if (tbp->BG == 1 && tbp->C > 8 && tbp->Z == 384) {
+      nrLDPC_coding_encoder32(slot_params, tbp);
+    } else {
+      // this is not handled by CUDA, handle with CPU
+      cpu.TBs[cpu.nb_TBs++] = *tbp;
+    }
+  }
+  if (cpu.nb_TBs > 0) {
+    ldpc_cpu.nrLDPC_coding_encoder(&cpu);
+    slot_params->TBs[0] = cpu.TBs[0];
+  }
+  return 0;
+}
+
+void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *);
+int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *slot_params)
+{
+  // this should be the same as previous nrLDPC_coding_decoder() in nrLDPC_coding_segment_decoder.c
+  nrLDPC_TB_decoding_parameters_t tbCPU[32] = {0};
+  nrLDPC_slot_decoding_parameters_t cpu = {
+    .frame = slot_params->frame,
+    .slot = slot_params->slot,
+    .threadPool = slot_params->threadPool,
+    .TBs = tbCPU,
+  };
+  for (int pusch_id = 0; pusch_id < slot_params->nb_TBs; pusch_id++) {
+    nrLDPC_TB_decoding_parameters_t *tbp = &slot_params->TBs[pusch_id];
+    if (tbp->Z >= 128 && tbp->BG == 1) {
+      nr_process_decode_segment_cuda(tbp);
+    } else {
+      // this is not handled by CUDA, handle with CPU
+      cpu.TBs[cpu.nb_TBs++] = *tbp;
+    }
+  }
+  if (cpu.nb_TBs > 0) {
+    ldpc_cpu.nrLDPC_coding_decoder(&cpu);
+    slot_params->TBs[0] = cpu.TBs[0];
+  }
+  return 0;
+}
+
+#define CHECK_DLFCN_ERROR(COND, SYMB) \
+  if (!(COND)) { \
+    LOG_E(NR_PHY, "could not open %s: %s\n", SYMB, dlerror()); \
+    return false; \
+  }
+static bool open_default_ldpc(nrLDPC_coding_interface_t *ldpc, int max_num_pxsch)
+{
+  const char *default_ldpc = "libldpc.so";
+  if (ldpc_lib_handle) {
+    LOG_W(NR_PHY, "%s already initialized\n", default_ldpc);
+    return true;
+  }
+  ldpc_lib_handle = dlopen(default_ldpc, RTLD_LAZY|RTLD_NODELETE|RTLD_GLOBAL);
+  CHECK_DLFCN_ERROR(ldpc_lib_handle, default_ldpc);
+
+  ldpc->nrLDPC_coding_init = dlsym(ldpc_lib_handle, "nrLDPC_coding_init");
+  CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_init, "nrLDPC_coding_init");
+  ldpc->nrLDPC_coding_shutdown = dlsym(ldpc_lib_handle, "nrLDPC_coding_shutdown");
+  CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_shutdown, "nrLDPC_coding_shutdown");
+  ldpc->nrLDPC_coding_decoder = dlsym(ldpc_lib_handle, "nrLDPC_coding_decoder");
+  CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_decoder, "nrLDPC_coding_decoder");
+  ldpc->nrLDPC_coding_encoder = dlsym(ldpc_lib_handle, "nrLDPC_coding_encoder");
+  CHECK_DLFCN_ERROR(ldpc->nrLDPC_coding_encoder, "nrLDPC_coding_encoder");
+
+  bool success = ldpc->nrLDPC_coding_init(max_num_pxsch) == 0;
+  if (!success)
+    LOG_E(NR_PHY, "could not initialize %s\n", default_ldpc);
+  return success;
+}
+
+static void close_default_ldpc(nrLDPC_coding_interface_t *ldpc)
+{
+  if (!ldpc_lib_handle) {
+    LOG_W(NR_PHY, "cannot stop LDPC, not loaded\n");
+    return;
+  }
+
+  ldpc->nrLDPC_coding_shutdown();
+  dlclose(ldpc_lib_handle);
+  ldpc_lib_handle = NULL;
+}
+
+extern int32_t nrLDPC_coding_init_cuda(int max_num_pxsch);
+int32_t nrLDPC_coding_init(int max_num_pxsch)
+{
+  // we can't use load_nrLDPC_coding_interface() because:
+  // 1. it caches the lib under the "ldpc" name, but links to "ldpc_cuda", so
+  //    we recursively load this function again => stack overflow
+  // 2. the global --loader.ldpc.shlibversion _cuda would be reapplied => we
+  //    cannot actually load ourselves
+  // hence use dlsym() directly.
+  bool success = open_default_ldpc(&ldpc_cpu, max_num_pxsch);
+  AssertFatal(success, "error loading LDPC library\n");
+  return nrLDPC_coding_init_cuda(max_num_pxsch);
+}
+
+extern int32_t nrLDPC_coding_shutdown_cuda(void);
+int32_t nrLDPC_coding_shutdown(void)
+{
+  close_default_ldpc(&ldpc_cpu);
+  return nrLDPC_coding_shutdown_cuda();
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_segment_decoder_cuda.c

@@ -0,0 +1,304 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+ * \brief Top-level routines for decoding LDPC transport channels
+ */
+
+// [from gNB coding]
+#include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
+#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "openair1/PHY/CODING/coding_defs.h"
+#include "log.h"
+#include "defs.h"
+
+#include <stdalign.h>
+#include <stdint.h>
+#include <syscall.h>
+#include <time.h>
+#include <pthread.h>
+#include <cuda_runtime.h>
+
+#include "nrLDPC_CUDA_shared_param.h"
+
+// #define gNB_DEBUG_TRACE
+
+#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
+// #define DEBUG_CRC
+#ifdef DEBUG_CRC
+#define PRINT_CRC_CHECK(a) a
+#else
+#define PRINT_CRC_CHECK(a)
+#endif
+
+#define USE_GPU_FOR_RM_DEINTER 1
+
+cudaStream_t decoderStreams[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
+cudaStream_t decoderDoneEvents[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
+
+int d_array_size = 0;
+
+void launch_deinterleave_i16(int Qm, int E1, int E2, int C, int r_firstE2,int16_t* e, const int16_t* f,cudaStream_t *s,int8_t sidx);
+int nr_rate_matching_ldpc_rx_cuda(uint32_t Tbslbrm,
+                                             uint8_t BG,
+                                             uint16_t Z,
+                                             int16_t *d,
+                                             int16_t *soft_input,
+                                             int8_t *llr_buffer,
+                                             uint32_t K,
+                                             uint8_t C,
+                                             uint8_t rvidx,
+                                             uint8_t clear,
+                                             uint32_t E1,
+                                             uint32_t E2,
+                                             uint32_t r_firstE2,
+                                             uint32_t F,
+                                             uint32_t Foffset,
+                                             cudaStream_t *s,
+                                             int8_t sidx);
+
+extern int pageable,integrated;
+extern int8_t *p_llr_dev,*p_out_dev;
+int16_t **harq_d_array;
+int16_t *harq_d_array_dev;
+int16_t *harq_e_dev;
+int16_t *harq_f_dev;
+pthread_mutex_t decoder_mutex=PTHREAD_MUTEX_INITIALIZER;
+
+extern
+int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
+                         int8_t* p_llr,
+                         uint8_t* p_out,
+                         t_nrLDPC_time_stats* p_profiler,
+                         decode_abort_t* ab);
+
+void nr_process_decode_segment_cuda(nrLDPC_TB_decoding_parameters_t *segs)
+{
+  // arg points to RDATA array (nrLDPC_decoding_parameters_t *RDATA)
+  DevAssert(segs != NULL);
+
+  // use seg0 as canonical
+  const int C = segs->C;
+  const int Z = segs->Z;
+  const int Kc = segs->BG == 2 ? 52 : 68;
+  const int K  = segs->K;
+  const int Kprime = K - segs->F;
+  const int segLen = Kc*Z; // int16 length; after packing we store int8 [segLen]
+  t_nrLDPC_time_stats procTime = {0};
+  t_nrLDPC_time_stats *p_procTime = &procTime;
+
+#ifdef USE_GPU_FOR_RM_DEINTER								 
+  start_meas(&segs->ts_deinterleave);
+  int E1 = segs->E;
+  int E2 = segs->E2;
+  int r_firstE2 = segs->first_rE2;
+  
+  LOG_D(NR_PHY,"locking decoder (llr %p)\n",segs->llr);
+  pthread_mutex_lock(&decoder_mutex);
+
+  // for PCIe GPU copy llrs to device memory
+  if (!pageable&&!integrated) {
+	                   LOG_I(NR_PHY,"cudaMemcpyAsynch llr->harq_f_dev\n");
+	                   cudaMemcpyAsync(harq_f_dev,
+		                           segs->llr,
+		                           ((r_firstE2*E1) + (C-r_firstE2)*E2)*sizeof(int16_t),
+					   cudaMemcpyHostToDevice,
+					   decoderStreams[0]);
+  }                           
+#if 0
+  if (1/*segs->rv_index==2*/)
+    for (int r=0;r<C;r++) {
+      int roffset = r < r_firstE2 ? r*E1 : ((r_firstE2*E1)+(r-r_firstE2)*E2);
+      int E = r < r_firstE2 ? E1 : E2;
+      for (int i=0;i<16;i++) {
+         printf("f(%d,%d) %d\n",r,i,segs->llr[roffset + i]);
+      }
+    }
+#endif
+  LOG_D(NR_PHY,"deinter: e %p llr %p\n",harq_e_dev,pageable || integrated ? segs->llr : harq_f_dev);
+  launch_deinterleave_i16(segs->Qm,E1,E2,C,r_firstE2,harq_e_dev,pageable||integrated ? segs->llr : harq_f_dev,decoderStreams,0);
+  stop_meas(&segs->ts_deinterleave);
+#if 0
+  cudaError_t err;
+  if (1/*segs->rv_index == 2*/) {
+    int16_t elocal[E2];
+    for (int r=0;r<C;r++) {
+      int E = r<r_firstE2 ? E1 : E2;
+      int r_off = r<r_firstE2 ? (r*E1) : ((r_firstE2*E1) + (r-r_firstE2)*E2);
+      err = cudaMemcpyAsync(elocal,&harq_e_dev[r_off],sizeof(elocal),cudaMemcpyDeviceToHost,decoderStreams[0]);
+      AssertFatal(err == cudaSuccess,"cudaMemcpyAsync failed with error %s, r_off %d\n",cudaGetErrorString(err),r_off);
+      for (int i=0;i<E;i++) {
+	 if (elocal[i] != segs->llr[r_off + ((i<(E/2)) ? 2*i : (-E+1+2*i))])  { 
+            printf("e(%d,%d,%d) %d",r,i,E,elocal[i]);
+            printf("(%d**)\n",segs->llr[r_off + ((i<(E/2)) ? 2*i : -E+1+2*i)]); 
+	 }
+      }
+    }
+  }
+#endif
+  start_meas(&segs->ts_rate_unmatch);
+  //printf("Running RM with id %d, d_to_be_cleared %d, rv_idx %d, Z %d, C %d, E1 %d, E2 %d, F %d,r_firstE2 %d\n",segs->harq_unique_pid,segs->d_to_be_cleared,segs->rv_index,Z,C,E1,E2,segs->F,r_firstE2); 
+#if 0
+  if (segs->d_to_be_cleared == 1) err = cudaMemsetAsync(harq_d_array[segs->harq_unique_pid],0,C*68*Z*sizeof(int16_t),decoderStreams[0]);
+  AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed with error %s on harq_d_array[%d] %p for %d bytes\n",cudaGetErrorString(err),segs->harq_unique_pid,harq_d_array[segs->harq_unique_pid],C*68*Z*sizeof(int16_t));
+  cudaMemsetAsync(p_llr_dev,0,C*68*Z*sizeof(int8_t),decoderStreams[0]);
+  AssertFatal(err==cudaSuccess,"cudaMemsetAsync failed on p_llr_dev %p\n",p_llr_dev);
+#endif
+  AssertFatal(segs->harq_unique_pid < d_array_size,"harq_unique_pid %d > %d\n",segs->harq_unique_pid,d_array_size);
+  nr_rate_matching_ldpc_rx_cuda(segs->tbslbrm,
+                                segs->BG,
+                                Z,
+                                harq_d_array[segs->harq_unique_pid],
+                                harq_e_dev,
+   		                p_llr_dev,
+			        K,
+                                C,             // TB segments count
+                                segs->rv_index,
+                                segs->d_to_be_cleared,
+                                E1,
+				E2,
+				r_firstE2,
+                                segs->F,
+                                Kprime - 2 * Z,
+				decoderStreams,
+				0); 
+  for (int r = 0; r < C; ++r) {
+     cudaMemsetAsync(p_llr_dev + (size_t)r*segLen,0,sizeof(int8_t)*2*Z,decoderStreams[0]);
+#if 0
+     int8_t llr_local[segLen];
+     if (r==1 && segs->rv_index==2) {
+       cudaMemcpyAsync(llr_local,p_llr_dev+(size_t)r*segLen,sizeof(int8_t)*segLen,cudaMemcpyDeviceToHost,decoderStreams[0]);
+       for (int i=0;i<segLen;i++) printf("llr(%d,%d,%d/%d) %d\n",segs->rv_index,r,i,segLen,llr_local[i]);
+     }
+#endif
+  }
+  stop_meas(&segs->ts_rate_unmatch);
+  
+#else //USE_GPU_FOR_RM_DEINTER
+  int16_t *z_local = (int16_t*)alloca(sizeof(int16_t) * segLen); // segLen is safe small
+  for (int r = 0; r < C; ++r) {
+    // deinterleave
+    start_meas(&segs->ts_deinterleave);
+    int16_t *harq_e = (int16_t*)alloca(sizeof(int16_t) * segs->segments[r].E);
+//    for (int i=0;i<segs->segments[r].E;i++) printf("llr_in[%d] %d\n",i,segs->segments[r].llr[i]);
+    nr_deinterleaving_ldpc(segs->segments[r].E, segs->Qm, harq_e, segs->segments[r].llr);
+//    for (int i=0;i<16;i++) printf("harq_e[%d] %d\n",i,harq_e[i]);
+    stop_meas(&segs->segments[0].ts_deinterleave);
+    // rate matching
+    start_meas(&segs->segments[0].ts_rate_unmatch);
+    if (nr_rate_matching_ldpc_rx(segs->tbslbrm,
+                                 segs->BG,
+                                 Z,
+                                 segs->segments[r].d,
+				 harq_e,
+                                 C,             // TB segments count
+                                 segs->rv_index,
+                                 segs->segments[r].d_to_be_cleared,
+                                 segs->segments[r].E,
+                                 segs->F,
+                                 Kprime - 2 * Z) == -1) {
+      stop_meas(&segs->segments[0].ts_rate_unmatch);
+      LOG_E(PHY,"rate matching failed seg %d\n", r);
+      memset(segs->segments[r].c, 0, K);
+      //*rdata->decodeSuccess = false;
+      continue; // skip this segment
+    }
+    stop_meas(&segs->segments[0].ts_rate_unmatch);
+    start_meas(&segs->segments[0].ts_seg_prep);
+    segs->segments[r].d_to_be_cleared = false;
+
+    memset(z_local,0,sizeof(int16_t)*2*Z);
+    memset(z_local + Kprime,127,sizeof(int16_t)*segs->F);
+    memcpy(z_local + 2*Z, segs->segments[r].d, (size_t)(Kprime - 2*Z)*sizeof(int16_t));
+    memcpy(z_local + K, segs->segments[r].d + (K - 2*Z), (size_t)(Kc*Z - K)*sizeof(int16_t));
+
+    // pack int16 -> int8 into llrBuffer[r * segLen]
+    simde__m128i *pv = (simde__m128i*)z_local;
+    simde__m128i *pl = (simde__m128i*)(llrBuffer + (size_t)r*segLen);
+    int vecCount = ((Kc * Z) >> 4);
+    for (int j=0, idx=0; j<vecCount; ++j, idx+=2) {
+      pl[j] = simde_mm_packs_epi16(pv[idx], pv[idx+1]);
+    }
+    stop_meas(&segs->segments[0].ts_seg_prep);
+  }
+#endif
+
+  start_meas(&segs->ts_ldpc_decode);
+  
+  t_nrLDPC_dec_params decParams = {.check_crc = check_crc};
+  decParams.Z = Z;
+  decParams.R = segs->R;
+  decParams.BG = segs->BG;
+  decParams.crc_type = crcType(C, segs->A);
+  decParams.Kprime = lenWithCrc(C, segs->A);
+  decParams.n_segments = C;
+  decParams.outMode=nrLDPC_outMode_BIT;
+  decParams.numMaxIter = segs->max_ldpc_iterations;
+
+  LOG_D(NR_PHY,"decoder (llr %p): %d segments, Z %d, R %d \n",segs->llr,C,Z,segs->R);
+
+  int decodeIterations = LDPCdecoder_cuda(&decParams, p_llr_dev, segs->c, p_procTime, segs->abort_decode);
+  stop_meas(&segs->ts_ldpc_decode);
+  
+  if (decodeIterations <= segs->max_ldpc_iterations) {
+    *segs->processedSegments = C;
+    for (int r=0; r<C; r++) segs->decodeSuccess[r] = true;
+    LOG_D(NR_PHY,"Set all segs->decodeSuccess to true\n");
+  } else {
+    memset(segs->c, 0, C*(K>>3));
+    for (int r=0; r<C; r++) segs->decodeSuccess[r] = false;
+    LOG_D(NR_PHY,"Set all segs->decodeSuccess to false\n");
+  }
+  LOG_D(NR_PHY,"unlocking decoder (llr %p)\n",segs->llr);
+  pthread_mutex_unlock(&decoder_mutex);
+}
+
+
+#define MAXE 4*14*273*12*8 // 4 antennas, 14 symbols, 273 PRBs, 12 RE/prb, 8 bits/RE
+
+
+void LDPCint_rm_init(int max_num_pxsch) {
+
+  LOG_I(NR_PHY,"RM init for %d pxsch\n",max_num_pxsch);
+  LOG_I(NR_PHY,"Allocating device array for harq_d/harq_e \n");
+  cudaError_t err=cudaMalloc((void **)&harq_e_dev,MAXE*sizeof(int16_t));
+  AssertFatal(err == cudaSuccess,"CUDA Error (harq_e_dev): %s\n", cudaGetErrorString(err));
+  harq_d_array = malloc(sizeof(int16_t*) * max_num_pxsch);
+  err = cudaMalloc((void *)&harq_d_array_dev,sizeof(int16_t*) * max_num_pxsch);
+  AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_array_dev): %s\n", cudaGetErrorString(err));
+  LOG_I(PHY,"Allocated %ld bytes for harq_d_array_dev @ %p\n",sizeof(int16_t*)*max_num_pxsch,harq_d_array_dev);
+  for (int i=0;i<max_num_pxsch;i++) {
+    err=cudaMalloc((void **)&harq_d_array[i],MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 *OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t));
+    LOG_I(PHY,"Allocating %ld bytes for harq_d_array[%d] @ %p\n",MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 *OAI_LDPC_DECODER_MAX_NUM_LLR*sizeof(int16_t),i,harq_d_array[i]);
+    AssertFatal(err == cudaSuccess,"CUDA Error (harq_d_dev): %s\n", cudaGetErrorString(err));
+  }
+  cudaMemcpy(harq_d_array_dev,harq_d_array,sizeof(int16_t*)*max_num_pxsch,cudaMemcpyHostToDevice);
+  if (!pageable && !integrated) {
+    LOG_I(PHY,"Allocating device array for harq_f \n");
+    err=cudaMalloc((void **)&harq_f_dev,MAXE*sizeof(int16_t));
+    AssertFatal(err == cudaSuccess,"CUDA Error (harq_f_dev): %s\n", cudaGetErrorString(err));
+  }
+  d_array_size = max_num_pxsch;
+}
+
+extern void LDPCinit_cuda(void);
+extern void cuda_support_init();
+
+int32_t nrLDPC_coding_init_cuda(int max_num_pxsch)
+{
+  cuda_support_init();
+
+  LDPCinit_cuda();
+  LDPCint_rm_init(max_num_pxsch);
+  return 0;
+}
+
+void LDPCshutdown_cuda(void);
+
+int32_t nrLDPC_coding_shutdown_cuda(void)
+{
+  LDPCshutdown_cuda();
+  return 0;
+}
+

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_coding_segment_encoder_cuda.c

@@ -0,0 +1,928 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
+ * \brief Top-level routines for implementing LDPC encoding of transport channels
+ */
+
+#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h"
+#include "PHY/sse_intrin.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "common/utils/LOG/log.h"
+
+#include <syscall.h>
+
+//#define DEBUG_LDPC_ENCODING
+//#define DEBUG_LDPC_ENCODING_FREE 1
+
+static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
+
+extern uint32_t **d_host;
+
+static void unpack_output(uint32_t *f,
+                         uint32_t E,
+                         uint32_t *f2,
+                         uint32_t E2,
+			 uint32_t E2_first_segment32,
+                         uint32_t E2_first_segment,
+                         uint32_t nb_segments,
+                         uint8_t *output) {
+
+  uint32_t s;
+ // int s0;
+  uint32_t *fp;
+  int foffset;
+  uint32_t *output_p = (uint32_t *)output;
+  //printf("unpack: E %d, E2 %d, E2_first_segment %d, E2_first_segment32 %d, nb_segments %d\n",E,E2,E2_first_segment,E2_first_segment32,nb_segments);
+
+  uint32_t bit_index = 0;
+#ifdef __AVX2__
+  simde__m256i shift0=simde_mm256_set_epi32(7,6,5,4,3,2,1,0);
+  simde__m256i shift1=simde_mm256_set_epi32(15,14,13,12,11,10,9,8);
+  simde__m256i shift2=simde_mm256_set_epi32(23,22,21,20,19,18,17,16);
+  simde__m256i shift3=simde_mm256_set_epi32(31,30,29,28,27,26,25,24);
+  simde__m256i vmask0=simde_mm256_set_epi32(0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1);
+  simde__m256i vmask1=simde_mm256_set_epi32(0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100);
+  simde__m256i vmask2=simde_mm256_set_epi32(0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000);
+  simde__m256i vmask3=simde_mm256_set_epi32(0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000);
+  int s2=0;
+  for (s = 0; s < E2_first_segment ; s++) { 
+    s2 = s&31;	  
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7);
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = (simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
+        bit_index+=Emod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp    = simde_mm256_extract_epi32(cshift,0) | 
+	                  simde_mm256_extract_epi32(cshift,1) | 
+	                  simde_mm256_extract_epi32(cshift,2) | 
+	                  simde_mm256_extract_epi32(cshift,3) | 
+	                  simde_mm256_extract_epi32(cshift,4) | 
+	                  simde_mm256_extract_epi32(cshift,5) | 
+	                  simde_mm256_extract_epi32(cshift,6) | 
+	                  simde_mm256_extract_epi32(cshift,7);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));					   
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp = 	(simde_mm256_extract_epi32(cshift,0) | 
+	                 simde_mm256_extract_epi32(cshift,1) | 
+	                 simde_mm256_extract_epi32(cshift,2) | 
+	                 simde_mm256_extract_epi32(cshift,3) | 
+	                 simde_mm256_extract_epi32(cshift,4) | 
+	                 simde_mm256_extract_epi32(cshift,5) | 
+	                 simde_mm256_extract_epi32(cshift,6) | 
+	                 simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=Emod32;
+      }
+    }
+  }
+  for ( ; s < nb_segments ; s++){
+    s2 = s&31;	  
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7);
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = (simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
+        bit_index+=E2mod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp    = simde_mm256_extract_epi32(cshift,0) | 
+	                  simde_mm256_extract_epi32(cshift,1) | 
+	                  simde_mm256_extract_epi32(cshift,2) | 
+	                  simde_mm256_extract_epi32(cshift,3) | 
+	                  simde_mm256_extract_epi32(cshift,4) | 
+	                  simde_mm256_extract_epi32(cshift,5) | 
+	                  simde_mm256_extract_epi32(cshift,6) | 
+	                  simde_mm256_extract_epi32(cshift,7);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));					   
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp = 	(simde_mm256_extract_epi32(cshift,0) | 
+	                 simde_mm256_extract_epi32(cshift,1) | 
+	                 simde_mm256_extract_epi32(cshift,2) | 
+	                 simde_mm256_extract_epi32(cshift,3) | 
+	                 simde_mm256_extract_epi32(cshift,4) | 
+	                 simde_mm256_extract_epi32(cshift,5) | 
+	                 simde_mm256_extract_epi32(cshift,6) | 
+	                 simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=E2mod32;
+      }
+    }
+  }
+#elif defined(__aarch64__)
+  const int32_t ucShift0[32][4] = { {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}, {-28,-27,-26,-25}, {-29,-28,-27,-26}, {-30,-29,-28,-27}, {-31,-30,-29,-28}}; 
+
+  const int32_t ucShift1[32][4] = { {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}}; 
+
+  const int32_t ucShift2[32][4] = { {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19},{-23,-22,-21,-20}}; 
+
+  const int32_t ucShift3[32][4] = { {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}}; 
+
+  const int32_t ucShift4[32][4] = { {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}}; 
+
+  const int32_t ucShift5[32][4] = { {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}}; 
+
+  const int32_t ucShift6[32][4] = { {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}}; 
+
+  const int32_t ucShift7[32][4] = { {28,29,30,31}, {27,28,29,30}, {26,27,28,29}, {25,26,27,28}, {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}}; 
+  const uint32_t __attribute__ ((aligned (16))) masks0[4] = {0x1,0x2,0x4,0x8};
+  const uint32_t __attribute__ ((aligned (16))) masks1[4] = {0x10,0x20,0x40,0x80};
+  const uint32_t __attribute__ ((aligned (16))) masks2[4] = {0x100,0x200,0x400,0x800};
+  const uint32_t __attribute__ ((aligned (16))) masks3[4] = {0x1000,0x2000,0x4000,0x8000};
+  const uint32_t __attribute__ ((aligned (16))) masks4[4] = {0x10000,0x20000,0x40000,0x80000};
+  const uint32_t __attribute__ ((aligned (16))) masks5[4] = {0x100000,0x200000,0x400000,0x800000};
+  const uint32_t __attribute__ ((aligned (16))) masks6[4] = {0x1000000,0x2000000,0x4000000,0x8000000};
+  const uint32_t __attribute__ ((aligned (16))) masks7[4] = {0x10000000,0x20000000,0x40000000,0x80000000};
+  int32x4_t vshift0[32],vshift1[32],vshift2[32],vshift3[32],vshift4[32],vshift5[32],vshift6[32],vshift7[32];
+  for (int n=0;n<32;n++) {
+	  vshift0[n] = vld1q_s32(ucShift0[n]);
+	  vshift1[n] = vld1q_s32(ucShift1[n]);
+	  vshift2[n] = vld1q_s32(ucShift2[n]);
+	  vshift3[n] = vld1q_s32(ucShift3[n]);
+	  vshift4[n] = vld1q_s32(ucShift4[n]);
+	  vshift5[n] = vld1q_s32(ucShift5[n]);
+	  vshift6[n] = vld1q_s32(ucShift6[n]);
+	  vshift7[n] = vld1q_s32(ucShift7[n]);
+  }
+  uint32x4_t vmask0  = vld1q_u32(masks0);
+  uint32x4_t vmask1  = vld1q_u32(masks1);
+  uint32x4_t vmask2  = vld1q_u32(masks2);
+  uint32x4_t vmask3  = vld1q_u32(masks3);
+  uint32x4_t vmask4  = vld1q_u32(masks4);
+  uint32x4_t vmask5  = vld1q_u32(masks5);
+  uint32x4_t vmask6  = vld1q_u32(masks6);
+  uint32x4_t vmask7  = vld1q_u32(masks7);
+  int s2=0;
+  for (s = 0; s < E2_first_segment ; s++) {
+    s2 = s&31;	  
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<Emod32)-1);
+        bit_index+=Emod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+	/*
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	*/
+        uint64_t tmp = (uint64_t)vaddvq_u32(cshift);
+	*(uint64_t*)(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<Emod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=Emod32;
+      }
+    }
+  }
+//  s0 = s;
+  for ( ; s < nb_segments ; s++){
+    s2 = s&31;	  
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        *(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<E2mod32)-1);
+        bit_index+=E2mod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<E2mod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=E2mod32;
+      }
+    }
+  }
+#else // non SIMD version
+  unsigned int segpos,s2;	
+  for (s = 0; s < E2_first_segment ; s++) {
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    s2=s&31;
+    segpos = (1<<s2);
+  //  printf("E %d s %d: foffset %d, s2 %d, segpos %u\n",E,s,foffset,s2,segpos);
+    for (int i = 0; i < E; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      //printf("bit_index %d, output_p[%d] %x\n",bit_index, bit_index>>5,output_p[bit_index>>5]);
+      bit_index++;
+    }
+  }
+  for ( ; s < nb_segments ; s++){
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    s2=s&31;
+    segpos = (1<<s2);
+//    printf("E2 %d s %d: foffset %d, s2 %d, segpos %u\n",E2,s,foffset,s2,segpos);
+    for (int i = 0; i < E2; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      bit_index++;
+    }
+  }
+#endif  
+}
+
+static
+int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
+                            uint8_t BG,
+                            uint16_t Z,
+                            uint32_t *d,
+                            uint32_t *e,
+                            uint8_t C,
+                            uint32_t F,
+                            uint32_t Foffset,
+                            uint8_t rvidx,
+                            uint32_t E)
+{
+  if (C == 0) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
+    return -1;
+  }
+
+  //Bit selection
+  uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
+  uint32_t Ncb;
+  if (Tbslbrm == 0)
+    Ncb = N;
+  else {
+    uint32_t Nref = 3 * Tbslbrm / (2 * C); //R_LBRM = 2/3
+    Ncb = min(N, Nref);
+  }
+
+  uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
+
+#ifdef RM_DEBUG
+  printf("nr_rate_matching_ldpc: E %u, F %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
+         E,
+         F,
+         Foffset,
+         ind,
+         Ncb,
+         rvidx,
+         Tbslbrm);
+#endif
+
+  if (Foffset > E) {
+    LOG_E(PHY,
+          "nr_rate_matching: invalid parameters (Foffset %d > E %d) F %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n",
+          Foffset,
+          E,
+          F,
+          ind,
+          Ncb,
+          rvidx,
+          Tbslbrm);
+    return -1;
+  }
+  if (Foffset > Ncb) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
+    return -1;
+  }
+
+  if (ind >= Foffset && ind < (F + Foffset))
+    ind = F + Foffset;
+
+  uint32_t k = 0;
+  if (ind < Foffset) { // case where we have some bits before the filler and the rest after
+    memcpy((void *)e, (void *)(d + ind), (Foffset - ind)<<2);
+
+    if (E + F <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (E - Foffset + ind)<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (Ncb - Foffset - F)<<2);
+      k = Ncb - F - ind;
+    }
+  } else {
+    if (E <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e), (void *)(d + ind), E<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e), (void *)(d + ind), (Ncb - ind)<<2);
+      k = Ncb - ind;
+    }
+  }
+
+  while (k < E) { // case where we do repetitions (low mcs)
+    for (ind = 0; (ind < Ncb) && (k < E); ind++) {
+#ifdef RM_DEBUG
+      printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
+#endif
+
+      if (ind == Foffset)
+        ind = F + Foffset; // skip filler bits
+
+      e[k++] = d[ind];
+
+    }
+  }
+
+  return 0;
+}
+
+static
+void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
+{
+  const uint32_t EQm = E / Qm;
+  memset(f, 0, E * sizeof(uint32_t));
+  switch(Qm) {
+    case 2: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e + EQm;
+      int i = 0;
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+      }
+    } break;
+    case 4: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      int i = 0;
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+      }
+    } break;
+    case 6: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      int i = 0;
+#if 0
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *f128 = (simde__m128i *)f;
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i) e1(i) e2(i) e3(i) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3) 
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3) 
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i+2) e1(i+2) e2(i+2) e3(i+2) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      f  = (uint32_t *)f128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+      }
+    } break;
+    case 8: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      uint32_t *e6 = e5 + EQm;
+      uint32_t *e7 = e6 + EQm;
+      
+      int i = 0;
+
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f128   = (simde__m128i *)f;
+
+
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+        simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
+        simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1)
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1)
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1)
+        simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j);   // e6(i) e7(i) e6(i+1) e7(i+1)
+
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1));  // e0(i+1) e1(i+1) e2(i+1) e3(i+1)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
+
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
+        tmp3 = simde_mm_unpackhi_epi32(e6j, e7j);   // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
+
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      e6 = (uint32_t *)e6_128;
+      e7 = (uint32_t *)e7_128;
+       f = (uint32_t *)f128;
+      
+      for (; i < EQm; i++) { 
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+        *f++ = *e6++;
+        *f++ = *e7++;
+      }
+    } 
+    break;
+    default:
+      AssertFatal(false, "Should be here!\n");
+  }
+}
+
+extern uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp);
+
+/**
+ * \typedef ldpc8blocks_args_t
+ * \struct ldpc8blocks_args_s
+ * \brief Arguments of an encoding task
+ * encode up to 8 code blocks
+ * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
+ * \var impp encoder implementation specific parameters for the task
+ * \var f first interleaver output to be filled by the task
+ * \var f2 second interleaver output to be filled by the task
+ * in case of a shift of E in the code blocks group processed by the task
+ */
+
+static void ldpcnblocks(nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters, encoder_implemparams_t impp)
+{
+ 
+  uint8_t mod_order = nrLDPC_TB_encoding_parameters->Qm;
+  uint16_t nb_rb = nrLDPC_TB_encoding_parameters->nb_rb;
+  uint32_t A = nrLDPC_TB_encoding_parameters->A;
+
+  unsigned int G = nrLDPC_TB_encoding_parameters->G;
+  LOG_D(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp.K, G, nb_rb, (int)mod_order);
+
+  // nrLDPC_encoder output is in "d"
+  // let's make this interface happy!
+//  uint32_t d[4][68*384];
+ // uint8_t *c[nrLDPC_TB_encoding_parameters->C];
+  extern uint32_t **input_host;
+
+  if (!nrLDPC_TB_encoding_parameters->c_dev)  
+    for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++) {
+      input_host[r] = (uint32_t *)nrLDPC_TB_encoding_parameters->segments[r].c;
+    }
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  LDPCencoder32(nrLDPC_TB_encoding_parameters->c_dev ? nrLDPC_TB_encoding_parameters->c_dev : (uint8_t **)input_host, &impp);
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  // Compute where to place in output buffer that is concatenation of all segments
+
+#ifdef DEBUG_LDPC_ENCODING
+  LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
+#endif
+  const uint32_t E = nrLDPC_TB_encoding_parameters->segments[0].E;
+  uint32_t E2=E;
+  uint32_t Emax = E;
+  int n_seg   = nrLDPC_TB_encoding_parameters->C>>5;
+  int n_seg2  = n_seg;
+  if ((nrLDPC_TB_encoding_parameters->C & 31) > 0) n_seg2++;
+  int r_shift = n_seg2; 
+  int r_shift2 = nrLDPC_TB_encoding_parameters->C;
+  for (int s=0;s<nrLDPC_TB_encoding_parameters->C;s++) {
+      //printf("segment %d E %d\n",s,nrLDPC_TB_encoding_parameters->segments[s].E);	  
+      if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
+	 E2=nrLDPC_TB_encoding_parameters->segments[s].E;
+         if(E2 > Emax)
+           Emax = E2;
+	 r_shift = s>>5;
+	 r_shift2 = s;
+	// printf("r_shift %d, r_shift2 %d\n",r_shift,r_shift2);
+         break;
+      }	 
+  }    
+
+  LOG_D(NR_PHY,
+        "Rate Matching, Code segment %d...%d r_shift %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        0,
+        impp.n_segments-1,
+	r_shift,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+/*
+  printf("Rate Matching, Code segment 0..%d r_shift %d r_shift2 %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        impp.n_segments-1,
+	r_shift,
+	r_shift2,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+*/
+
+  uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
+
+  uint32_t e[E*(r_shift+1)];
+  uint32_t e2[E2*(n_seg2-r_shift)];
+  uint32_t f[E*(r_shift+1)] __attribute__ ((aligned (64)));
+  uint32_t f2[E2*(n_seg2-r_shift)] __attribute__ ((aligned (64)));
+
+  // Interleaver outputs are stored in the output arrays
+  uint8_t *output = nrLDPC_TB_encoding_parameters->output;
+
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  memset(e,0,sizeof(e));
+  memset(f,0,sizeof(f));
+  if (1/*r_shift < n_seg2*/) { 
+    memset(e2,0,sizeof(e2));
+    memset(f2,0,sizeof(f2));
+  }
+
+  for (int r=0;r<n_seg2;r++) {
+    if (r<=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d_host[r],
+                              e+(r*E),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E);
+    if (r>=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d_host[r],
+                              e2+((r-r_shift)*E2),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E2);
+   /* 
+    if (r==(n_seg2-1)) {
+	    for (int i=0;i<16;i++) printf("rm: %x %x\n",d[n_seg2-1][i],e2[((n_seg2-1)*E2)+i]);
+    }
+    */
+  }
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  if (impp.K - impp.F - 2 * impp.Zc > E) {
+    LOG_E(PHY,
+          "dlsch coding A %d  Kr %d G %d (nb_rb %d, mod_order %d)\n",
+          A,
+          impp.K,
+          G,
+          nb_rb,
+          (int)mod_order);
+
+    LOG_E(NR_PHY,
+          "Rate Matching, Code segments 0..%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
+          "nb_rb %d)...\n",
+          impp.n_segments,
+          G,
+          E,
+          impp.K,
+          impp.F,
+          impp.K - impp.F - 2 * impp.Zc,
+          mod_order,
+          nb_rb);
+  }
+  
+  //printf("interleaving r_shift %d, n_seg2 %d\n",r_shift,n_seg2);
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+  
+  for (int r=0;r<=r_shift;r++)
+    nr_interleaving_ldpc32(E,
+                           mod_order,
+                           e+E*r,
+                           f+E*r);
+
+  for (int r=r_shift;r<n_seg2;r++)
+    nr_interleaving_ldpc32(E2,
+                           mod_order,
+                           e2+E2*(r-r_shift),
+                           f2+E2*(r-r_shift));
+/*
+  for (int i=0;i<16;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  printf("-------------------\n");
+  for (int i=E2-16;i<E2;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  */
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+
+  if (impp.tconcat) start_meas(impp.tconcat);
+  unpack_output(f,E,f2,E2,r_shift,r_shift2,nrLDPC_TB_encoding_parameters->C,output);
+  if (impp.tconcat) stop_meas(impp.tconcat);
+
+}
+
+int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters)
+{
+
+
+    encoder_implemparams_t common_segment_params = {
+      .n_segments = nrLDPC_TB_encoding_parameters->C,
+      .tinput = nrLDPC_slot_encoding_parameters->tinput,
+      .tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
+      .tprep = nrLDPC_slot_encoding_parameters->tprep,
+      .tparity = nrLDPC_slot_encoding_parameters->tparity,
+      .toutput = nrLDPC_slot_encoding_parameters->toutput,
+      .tconcat = nrLDPC_slot_encoding_parameters->tconcat,
+      .Kb = nrLDPC_TB_encoding_parameters->Kb,
+      .Zc = nrLDPC_TB_encoding_parameters->Z,
+      .BG = nrLDPC_TB_encoding_parameters->BG,
+      .output = nrLDPC_TB_encoding_parameters->output, 
+      .K = nrLDPC_TB_encoding_parameters->K,
+      .F = nrLDPC_TB_encoding_parameters->F,
+    };
+
+    ldpcnblocks(nrLDPC_TB_encoding_parameters, common_segment_params);
+
+
+
+  return 0;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_cuda/nrLDPC_decoder_cuda.c

@@ -0,0 +1,657 @@
+/*
+ * SPDX-License-Identifier: LicenseRef-CSSL-1.0
+ */
+
+/*! \file nrLDPC_decoder_cuda.h
+ * \brief Defines the CUDA version of nrLDPC decoder, including initialization and driver warmup mechanisms.
+ * \author Qizhi Pan, Raymond Knopp
+ * \company EURECOM
+ * \email: qizhi.pan@eurecom.fr, raymond.knopp@eurecom.fr
+ * \date 2025-12-30
+ * \version 1.0
+ * \note Optimized for NVIDIA GH200 (Grace Hopper) architecture using Zero-Copy access.
+ * \warning
+ */
+#include <stdint.h>
+#include "PHY/sse_intrin.h"
+#include "log.h"
+#include "openair1/PHY/CODING/coding_defs.h"
+
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+#ifdef NR_LDPC_DEBUG_MODE
+#include "nrLDPC_tools/nrLDPC_debug.h"
+#endif
+
+// decoder interface
+/**
+   \brief LDPC decoder API type definition
+   \param p_decParams LDPC decoder parameters
+   \param p_llr Input LLRs
+   \param p_llrOut Output vector
+   \param p_profiler LDPC profiler statistics
+*/
+
+//--------------------------CUDA Area---------------------------
+#include <cuda_runtime.h>
+#include "nrLDPC_CUDA_shared_param.h"
+
+extern cudaStream_t decoderStreams[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
+extern cudaEvent_t decoderDoneEvents[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4];
+static bool decoder_streamsCreated = false;
+static volatile int cuda_graph_breaker = 0;// 0 by default to enable the CUDA graph
+cudaError_t Err;
+
+int8_t* cnProcBuf_dev;
+int8_t* bnProcBuf_dev;
+int8_t* llrRes_dev;
+int8_t* llrProcBuf_dev;
+
+int8_t* p_llr_dev;
+int8_t* p_out_dev;
+
+extern int pageable, integrated;
+
+int cuda_support_init_decoder()
+{
+  // use cudaMalloc for all inner buffers
+  cudaError_t err;
+
+  err = cudaMalloc((void**)&cnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_CN_PROC_BUF);
+  AssertFatal(err == cudaSuccess, "CUDA Error (cnProcBuf_dev): %s\n", cudaGetErrorString(err));
+
+  err = cudaMalloc((void**)&bnProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_SIZE_BN_PROC_BUF);
+  AssertFatal(err == cudaSuccess, "CUDA Error (bnProcBuf_dev): %s\n", cudaGetErrorString(err));
+
+  err = cudaMalloc((void**)&llrRes_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
+  AssertFatal(err == cudaSuccess, "CUDA Error (llrRes_dev): %s\n", cudaGetErrorString(err));
+
+  err = cudaMalloc((void**)&llrProcBuf_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
+  AssertFatal(err == cudaSuccess, "CUDA Error (llrProcBuf_dev): %s\n", cudaGetErrorString(err));
+
+  err = cudaMalloc((void**)&p_llr_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
+  AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
+  cudaMemset(p_llr_dev, 0, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
+  err = cudaMalloc((void**)&p_out_dev, sizeof(int8_t) * MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * 4 * NR_LDPC_MAX_NUM_LLR);
+  AssertFatal(err == cudaSuccess, "CUDA Error (p_llr_dev): %s\n", cudaGetErrorString(err));
+
+  printf("[CUDA] Intermediate buffers allocated in Device Memory.\n");
+
+  return 0;
+}
+
+static ldpc_cuda_bridge_t* stream_bridges[8];
+
+extern cudaError_t nrLDPC_decoder_cuda_GraphRecord(ldpc_cuda_bridge_t* buffer,
+                                                   uint32_t numLLR,
+                                                   int8_t* cnProcBuf,
+                                                   int8_t* bnProcBuf,
+                                                   int8_t* llrRes,
+                                                   int8_t* llrProcBuf,
+                                                   uint32_t Z,
+                                                   uint32_t K,
+                                                   uint8_t BG,
+                                                   uint8_t R,
+                                                   uint8_t numMaxIter,
+                                                   uint8_t n_segments,
+                                                   e_nrLDPC_outMode outMode,
+                                                   cudaStream_t* streams,
+                                                   uint8_t CudaStreamIdx,
+                                                   cudaGraph_t* graphPtr,
+                                                   cudaGraphExec_t* graphExecPtr,
+                                                   uint8_t* isCreatedFlag);
+
+extern cudaError_t nrLDPC_decoder_cuda_GraphExecute(cudaGraphExec_t graphExec,
+                                                    cudaStream_t stream,
+                                                    cudaEvent_t* doneEvent,
+                                                    uint8_t CudaStreamIdx);
+
+extern void nrLDPC_decoder_cuda_NormalExecute(ldpc_cuda_bridge_t* buffer,
+                                              uint32_t numLLR,
+                                              int8_t* cnProcBuf,
+                                              int8_t* bnProcBuf,
+                                              int8_t* llrRes,
+                                              int8_t* llrProcBuf,
+                                              uint32_t Z,
+                                              uint32_t K,
+                                              uint8_t BG,
+                                              uint8_t R,
+                                              uint8_t numMaxIter,
+                                              uint8_t n_segments,
+                                              e_nrLDPC_outMode outMode,
+                                              cudaStream_t* streams,
+                                              uint8_t CudaStreamIdx,
+                                              cudaEvent_t* doneEvent);
+
+static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
+                                                   int8_t* p_out,
+                                                   int n_segments,
+                                                   t_nrLDPC_dec_params* p_decParams,
+                                                   t_nrLDPC_time_stats* p_profiler,
+                                                   decode_abort_t* ab);
+#define MAX_GRAPH_CACHE_SIZE 16
+#define PRE_RECORDED_COUNT 6
+#define STATIC_SEG_SIZE 9 // n_segments in pre-record graphs, should be determined for real cases
+
+typedef struct {
+  uint32_t Z;
+  uint32_t K;
+  uint32_t numLLR;
+  uint8_t R;
+  uint8_t BG;
+  uint8_t numMaxIter;
+  uint16_t n_segments;
+  e_nrLDPC_outMode outMode;
+  cudaGraph_t graph;
+  cudaGraphExec_t exec;
+  ldpc_cuda_bridge_t* bridge_ptr;
+  bool occupied;
+} gpu_graph_node_t;
+
+static gpu_graph_node_t gpu_graph_cache[MAX_GRAPH_CACHE_SIZE];
+static int dynamic_cache_idx = PRE_RECORDED_COUNT;
+
+void init_decoder_warmup()
+{
+  // =====================================================================
+  // CUDA Driver Warm-up
+  // Purpose: Execute a few representative graphs to trigger CUDA context
+  // initialization and driver-level JIT/lazy loading.
+  // Note: These specific Z/R combinations might not match the actual
+  // run-time traffic, but running them ensures the GPU pipeline is ready.
+  // =====================================================================
+
+  cudaError_t err_warmup = cudaSuccess;
+
+  // Sample configurations for warmup
+  uint32_t Z_list[] = {320, 352, 384};
+  uint8_t R_list[] = {13, 23};
+  int node_idx = 0;
+
+  int8_t* dummy_input_llr = NULL;
+  int8_t* dummy_output_bits = NULL;
+  uint32_t max_z = 384;
+  uint32_t max_n_segs = STATIC_SEG_SIZE;
+
+  size_t input_size_bytes = 68 * max_z * max_n_segs * sizeof(int8_t);
+  size_t output_size_bytes = 8448 * max_n_segs * sizeof(int8_t);
+
+  // Allocate Pinned/Mapped Memory for Zero-Copy access (GH200 friendly)
+  cudaHostAlloc((void**)&dummy_input_llr, input_size_bytes, cudaHostAllocMapped);
+  cudaHostAlloc((void**)&dummy_output_bits, output_size_bytes, cudaHostAllocMapped);
+
+  memset(dummy_input_llr, 0, input_size_bytes);
+  memset(dummy_output_bits, 0, output_size_bytes);
+
+  printf("[CUDA] Initializing & Warming up Driver Pipeline...\n");
+  if (cuda_graph_breaker == 0) {
+    for (int r_idx = 0; r_idx < 2; r_idx++) {
+      for (int z_idx = 0; z_idx < 3; z_idx++) {
+        uint32_t Z = Z_list[z_idx];
+        uint8_t R = R_list[r_idx];
+        uint8_t BG = 1;
+        uint32_t K = 22 * Z;
+        uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
+        uint8_t numMaxIter = 2; // 2 iterations for faster warmup
+        uint8_t n_segments = STATIC_SEG_SIZE;
+
+        // Bind dummy buffers
+        gpu_graph_cache[node_idx].bridge_ptr->p_llr_ptr = dummy_input_llr;
+        gpu_graph_cache[node_idx].bridge_ptr->p_out_ptr = dummy_output_bits;
+
+        // Record graph
+        err_warmup = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[node_idx].bridge_ptr,
+                                                     numLLR,
+                                                     cnProcBuf_dev,
+                                                     bnProcBuf_dev,
+                                                     llrRes_dev,
+                                                     llrProcBuf_dev,
+                                                     Z,
+                                                     K,
+                                                     BG,
+                                                     R,
+                                                     numMaxIter,
+                                                     n_segments,
+                                                     nrLDPC_outMode_BIT,
+                                                     decoderStreams,
+                                                     0,
+                                                     &gpu_graph_cache[node_idx].graph,
+                                                     &gpu_graph_cache[node_idx].exec,
+                                                     (uint8_t*)&gpu_graph_cache[node_idx].occupied);
+        if (err_warmup != cudaSuccess) {
+          cuda_graph_breaker = 1; // Once a graph recording fails, forbidden the graph recording permanently
+          printf("[CUDA] Warmup Graph Record Failed (err=%s). Circuit breaker triggered\n", cudaGetErrorString(err_warmup));
+          break;
+        }
+
+        // Save metadata
+        gpu_graph_cache[node_idx].Z = Z;
+        gpu_graph_cache[node_idx].R = R;
+        gpu_graph_cache[node_idx].K = K;
+        gpu_graph_cache[node_idx].numLLR = numLLR;
+        gpu_graph_cache[node_idx].BG = BG;
+        gpu_graph_cache[node_idx].numMaxIter = numMaxIter;
+        gpu_graph_cache[node_idx].n_segments = n_segments;
+        gpu_graph_cache[node_idx].outMode = nrLDPC_outMode_BIT;
+
+        node_idx++;
+      }
+    }
+    dynamic_cache_idx = node_idx;
+
+    // Execute to trigger driver initialization
+    if (dynamic_cache_idx > 0) {
+      for (int i = 0; i < dynamic_cache_idx; i++) {
+        if (gpu_graph_cache[i].occupied) {
+          nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[i].exec, decoderStreams[0], NULL, 0);
+        }
+      }
+      cudaDeviceSynchronize();
+      printf("[CUDA] Driver warm-up complete. Executed %d dummy graphs.\n", dynamic_cache_idx);
+
+    for (int i = 0; i < dynamic_cache_idx; i++) {
+        if (gpu_graph_cache[i].occupied) {
+            if (gpu_graph_cache[i].exec) {
+                cudaGraphExecDestroy(gpu_graph_cache[i].exec);
+                gpu_graph_cache[i].exec = NULL;
+            }
+            if (gpu_graph_cache[i].graph) {
+                cudaGraphDestroy(gpu_graph_cache[i].graph);
+                gpu_graph_cache[i].graph = NULL;
+            }
+            
+            if (gpu_graph_cache[i].bridge_ptr) {
+                gpu_graph_cache[i].bridge_ptr->p_llr_ptr = NULL;
+                gpu_graph_cache[i].bridge_ptr->p_out_ptr = NULL;
+            }
+            
+            gpu_graph_cache[i].occupied = false;
+        }
+    }
+
+      // Mark slots as free and reset index so real traffic starts from slot 0
+      for (int i = 0; i < dynamic_cache_idx; i++) {
+        gpu_graph_cache[i].occupied = false;
+      }
+      printf("[CUDA] Cache cleared. Ready for dynamic recording.\n");
+    }
+  }
+  dynamic_cache_idx = 0;
+
+  if (cuda_graph_breaker == 1) {
+    printf("[CUDA] Using Normal Execution for Warmup (Graph Disabled).\n");
+    for (int r_idx = 0; r_idx < 2; r_idx++) {
+      for (int z_idx = 0; z_idx < 3; z_idx++) {
+        uint32_t Z = Z_list[z_idx];
+        uint8_t R = R_list[r_idx];
+        uint8_t BG = 1;
+        uint32_t K = 22 * Z;
+        uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : NR_LDPC_NCOL_BG1_R23 * Z;
+        uint8_t numMaxIter = 2; // 2 iterations for faster warmup
+        uint8_t n_segments = STATIC_SEG_SIZE;
+
+        // Bind dummy buffers
+        gpu_graph_cache[0].bridge_ptr->p_llr_ptr = dummy_input_llr;
+        gpu_graph_cache[0].bridge_ptr->p_out_ptr = dummy_output_bits;
+
+        // normal execution
+        nrLDPC_decoder_cuda_NormalExecute(gpu_graph_cache[0].bridge_ptr,
+                                          numLLR,
+                                          cnProcBuf_dev,
+                                          bnProcBuf_dev,
+                                          llrRes_dev,
+                                          llrProcBuf_dev,
+                                          Z,
+                                          K,
+                                          BG,
+                                          R,
+                                          numMaxIter,
+                                          n_segments,
+                                          nrLDPC_outMode_BIT,
+                                          decoderStreams,
+                                          0,
+                                          NULL);
+      }
+    }
+    cudaDeviceSynchronize();
+    printf("[CUDA] Driver warm-up complete with normal execution\n");
+  }
+  cudaFreeHost(dummy_input_llr);
+  cudaFreeHost(dummy_output_bits);
+}
+void init_decoder_gpu_structures()
+{
+  printf("[CUDA] Initializing Global GPU Structures...\n");
+  // Bridge for graphs
+  for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
+    if (gpu_graph_cache[i].bridge_ptr == NULL) {
+      cudaHostAlloc((void**)&gpu_graph_cache[i].bridge_ptr, sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
+
+      gpu_graph_cache[i].bridge_ptr->p_llr_ptr = NULL;
+      gpu_graph_cache[i].bridge_ptr->p_out_ptr = NULL;
+      gpu_graph_cache[i].occupied = false;
+    }
+  }
+  printf("[CUDA] Allocated %d Graph Bridges.\n", MAX_GRAPH_CACHE_SIZE);
+  // Bridge for normal execute
+  for (int i = 0; i < 8; i++) {
+    if (stream_bridges[i] == NULL) {
+      cudaHostAlloc((void**)&stream_bridges[i], sizeof(ldpc_cuda_bridge_t), cudaHostAllocMapped);
+      stream_bridges[i]->p_llr_ptr = NULL;
+      stream_bridges[i]->p_out_ptr = NULL;
+    }
+  }
+  printf("[CUDA] Allocated %d Stream Bridges for Fallback case.\n", 8);
+}
+
+void init_decoder_graphs()
+{
+  for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
+    gpu_graph_cache[i].occupied = false;
+    gpu_graph_cache[i].graph = NULL;
+    gpu_graph_cache[i].exec = NULL;
+    gpu_graph_cache[i].bridge_ptr = NULL;
+    gpu_graph_cache[i].Z = 0;
+    gpu_graph_cache[i].R = 0;
+  }
+
+  dynamic_cache_idx = 0;
+
+  printf("[decoder_graphs] initialized %d dynamic cache slots\n", MAX_GRAPH_CACHE_SIZE);
+}
+
+void free_graphs()
+{
+  for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
+    if (gpu_graph_cache[i].occupied) {
+      if (gpu_graph_cache[i].exec)
+        cudaGraphExecDestroy(gpu_graph_cache[i].exec);
+      if (gpu_graph_cache[i].graph)
+        cudaGraphDestroy(gpu_graph_cache[i].graph);
+      gpu_graph_cache[i].occupied = false;
+    }
+  }
+  printf("[decoder_graphs] shutdown complete (Dynamic Cache Cleared)\n");
+}
+
+extern int cuda_support_set;
+
+bool encoder_streamsCreated = false;
+cudaStream_t encoderStreams[4];
+void cuda_support_init();
+
+int32_t LDPCinit_cuda()
+{
+  if (cuda_support_set == 0) {
+    printf("Calling encoder initializations\n");
+    cuda_support_init();
+  }
+  if (!decoder_streamsCreated) {
+    for (int s = 0; s < 8; ++s) {
+      cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
+      cudaEventCreate(&decoderDoneEvents[s]);
+    }
+    decoder_streamsCreated = true;
+  }
+
+  if (!encoder_streamsCreated) {
+    for (int s = 0; s < 4; ++s) {
+      cudaStreamCreateWithFlags(&encoderStreams[s], cudaStreamNonBlocking);
+    }
+    encoder_streamsCreated = true;
+  }
+  printf("CUDA LDPC decoder initiating\n");
+  cuda_support_init_decoder();
+  init_decoder_graphs();
+  init_decoder_gpu_structures();
+  init_decoder_warmup();
+  return 0;
+}
+
+int32_t LDPCshutdown_cuda()
+{
+  if (cnProcBuf_dev) { cudaFree(cnProcBuf_dev); cnProcBuf_dev = NULL; }
+  if (bnProcBuf_dev) { cudaFree(bnProcBuf_dev); bnProcBuf_dev = NULL; }
+  if (llrRes_dev)   { cudaFree(llrRes_dev);   llrRes_dev = NULL; }
+  if (llrProcBuf_dev) { cudaFree(llrProcBuf_dev); llrProcBuf_dev = NULL; }
+
+  if (p_llr_dev)   { cudaFree(p_llr_dev);   p_llr_dev = NULL; }
+  if (p_out_dev)   { cudaFree(p_out_dev);   p_out_dev = NULL; }
+
+  for (int i = 0; i < MAX_GRAPH_CACHE_SIZE; i++) {
+    if (gpu_graph_cache[i].bridge_ptr) {
+        cudaFreeHost(gpu_graph_cache[i].bridge_ptr);
+        gpu_graph_cache[i].bridge_ptr = NULL;
+    }
+}
+  for (int i = 0; i < 8; i++) {
+    if (stream_bridges[i]) {
+        cudaFreeHost(stream_bridges[i]);
+        stream_bridges[i] = NULL;
+    }
+}
+
+  for (int s = 0; s < 8; ++s) {
+    if (decoder_streamsCreated) {
+      cudaEventDestroy(decoderDoneEvents[s]);
+      cudaStreamDestroy(decoderStreams[s]);
+    }
+  }
+
+  for (int s = 0; s < 4; s++) {
+    if (encoder_streamsCreated) {
+      cudaStreamDestroy(encoderStreams[s]);
+    }
+  }
+
+  free_graphs();
+
+  decoder_streamsCreated = false;
+  encoder_streamsCreated = false;
+
+  printf("[CUDA] Intermediate buffers and streams destroyed.\n");
+
+  return 0;
+}
+
+int32_t LDPCdecoder_cuda(t_nrLDPC_dec_params* p_decParams,
+                         int8_t* p_llr,
+                         uint8_t* p_out,
+                         t_nrLDPC_time_stats* p_profiler,
+                         decode_abort_t* ab)
+{
+  if (!((p_decParams->R == 89 || p_decParams->R == 23 || p_decParams->R == 13) && p_decParams->BG == 1 && p_decParams->Z % 4 == 0
+        && p_decParams->Z >= 128 && p_decParams->Z <= 384)) { // format check
+    printf("Current format: BG = %d, R = %d, Zc = %d\n", p_decParams->BG, p_decParams->R, p_decParams->Z);
+    AssertFatal(false, "Format cuda not support, only support BG = 1, Zc >= 128 and R = 13, 23, 89 right now\n");
+    return 0;
+  }
+  // Launch LDPC decoder core for all segments
+  int n_segments = p_decParams->n_segments;
+
+  int numIter = nrLDPC_decoder_core_dynamic(p_llr, (int8_t*)p_out, n_segments, p_decParams, p_profiler, ab);
+
+  set_abort(ab, false);
+
+  return numIter;
+}
+
+/**
+   \brief PerformsnrLDPC decoding of one code block
+   \param p_llr Input LLRs
+   \param p_out Output vector
+   \param numLLR Number of LLRs
+   \param p_decParamsnrLDPC decoder parameters
+   \param p_profilernrLDPC profiler statistics
+*/
+
+static inline uint32_t nrLDPC_decoder_core_dynamic(int8_t* p_llr,
+                                                   int8_t* p_out,
+                                                   int n_segments,
+                                                   t_nrLDPC_dec_params* p_decParams,
+                                                   t_nrLDPC_time_stats* p_profiler,
+                                                   decode_abort_t* ab)
+{
+  cudaError_t err_core = cudaSuccess;
+  bool graph_executed = false;
+  uint16_t Z = p_decParams->Z;
+  uint8_t BG = p_decParams->BG;
+  uint8_t R = p_decParams->R;
+  uint8_t numMaxIter = p_decParams->numMaxIter;
+  e_nrLDPC_outMode outMode = p_decParams->outMode;
+  uint32_t K = Z * 22;
+  // Calculate LLR size per segment based on Rate
+  uint32_t numLLR = (R == 13) ? NR_LDPC_NCOL_BG1_R13 * Z : ((R == 89) ? NR_LDPC_NCOL_BG1_R89 * Z : NR_LDPC_NCOL_BG1_R23 * Z);
+  if (p_llr != p_llr_dev)
+    cudaMemcpyAsync(p_llr_dev, p_llr, n_segments * 68 * 384, cudaMemcpyHostToDevice, decoderStreams[0]);
+
+  // Output size safety: assume worst-case unpacked bytes (K * n_segments)
+  size_t total_output_size = n_segments * K * sizeof(int8_t);
+
+/*
+  // for debug, remember to remove it---------
+  cuda_graph_breaker = 1; // skipping all the graph recording
+  //-----------------------------------------
+  */
+
+  if (cuda_graph_breaker == 0) {
+    int found_idx = -1;
+    // Search in Graph Cache
+    for (int i = 0; i < dynamic_cache_idx; i++) {
+      if (gpu_graph_cache[i].occupied && gpu_graph_cache[i].Z == Z && gpu_graph_cache[i].R == R && gpu_graph_cache[i].BG == BG
+          && gpu_graph_cache[i].K == K && gpu_graph_cache[i].numLLR == numLLR && gpu_graph_cache[i].numMaxIter == numMaxIter
+          && gpu_graph_cache[i].n_segments == n_segments && gpu_graph_cache[i].outMode == outMode) {
+        found_idx = i;
+        break;
+      }
+    }
+    if (found_idx >= 0) {
+      // === Cache HIT: Execute Recorded Graph ===
+      gpu_graph_cache[found_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
+      gpu_graph_cache[found_idx].bridge_ptr->p_out_ptr = (pageable || integrated) ? p_out : p_out_dev;
+
+      err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[found_idx].exec,
+                                                  decoderStreams[0],
+                                                  NULL, // doneEvent
+                                                  0); // Stream Index
+      if (err_core == cudaSuccess) {
+        graph_executed = true;
+      } else {
+        cuda_graph_breaker = 1;
+      }
+    } else if (dynamic_cache_idx < MAX_GRAPH_CACHE_SIZE) {
+      // === Cache MISS: Record New Graph and Execute ===
+      int new_idx = dynamic_cache_idx;
+
+      gpu_graph_cache[new_idx].occupied = true;
+      gpu_graph_cache[new_idx].Z = Z;
+      gpu_graph_cache[new_idx].R = R;
+      gpu_graph_cache[new_idx].BG = BG;
+      gpu_graph_cache[new_idx].K = K;
+      gpu_graph_cache[new_idx].numLLR = numLLR;
+      gpu_graph_cache[new_idx].numMaxIter = numMaxIter;
+      gpu_graph_cache[new_idx].n_segments = n_segments;
+      gpu_graph_cache[new_idx].outMode = outMode;
+      // Use the determined pointers (Device ptrs for PCIe, Host ptrs for GH200)
+      gpu_graph_cache[new_idx].bridge_ptr->p_llr_ptr = p_llr_dev;
+      gpu_graph_cache[new_idx].bridge_ptr->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
+
+      err_core = nrLDPC_decoder_cuda_GraphRecord(gpu_graph_cache[new_idx].bridge_ptr,
+                                                 numLLR,
+                                                 cnProcBuf_dev,
+                                                 bnProcBuf_dev,
+                                                 llrRes_dev,
+                                                 llrProcBuf_dev,
+                                                 Z,
+                                                 K,
+                                                 BG,
+                                                 R,
+                                                 numMaxIter,
+                                                 n_segments,
+                                                 outMode,
+                                                 decoderStreams,
+                                                 0, // CudaStreamIdx
+                                                 &gpu_graph_cache[new_idx].graph,
+                                                 &gpu_graph_cache[new_idx].exec,
+                                                 (uint8_t*)&gpu_graph_cache[new_idx].occupied);
+
+      if (err_core == cudaSuccess) {
+        err_core = nrLDPC_decoder_cuda_GraphExecute(gpu_graph_cache[new_idx].exec, decoderStreams[0], NULL, 0);
+
+        if (err_core == cudaSuccess) {
+          graph_executed = true;
+          dynamic_cache_idx++;
+        } else {
+          cuda_graph_breaker = 1; // graph execution fail
+        }
+      } else {
+        cuda_graph_breaker = 1; // graph record fail
+      }
+    }
+  }
+
+  if (!graph_executed) {
+    // === Fallback to Normal Execution ===
+    // If the cache is full, we cannot record new graphs.
+    // Or graph operation is not safe in this device or environment.
+    // Execute kernel directly using standard launch.
+/*    if (cuda_graph_breaker == 1) {
+      LOG_W(PHY, "Graph opereations failed, falling back to normal.\n");
+    }*/
+    ldpc_cuda_bridge_t* perpack_buffer = stream_bridges[0];
+    perpack_buffer->p_llr_ptr = p_llr_dev;
+    perpack_buffer->p_out_ptr = pageable || integrated ? p_out : p_out_dev;
+
+    nrLDPC_decoder_cuda_NormalExecute(perpack_buffer,
+                                      numLLR,
+                                      cnProcBuf_dev,
+                                      bnProcBuf_dev,
+                                      llrRes_dev,
+                                      llrProcBuf_dev,
+                                      Z,
+                                      K,
+                                      BG,
+                                      R,
+                                      numMaxIter,
+                                      n_segments,
+                                      outMode,
+                                      decoderStreams,
+                                      0,
+                                      NULL);
+  }
+
+  // Copy back and Cleanup for Discrete GPU
+  if (!pageable && !integrated) {
+    // Copy Output from Device to Host
+    if (outMode == nrLDPC_outMode_BIT) {
+      cudaMemcpyAsync(p_out, p_out_dev, total_output_size >> 3, cudaMemcpyDeviceToHost, decoderStreams[0]);
+    }
+    if (outMode == nrLDPC_outMode_BITINT8) {
+      cudaMemcpyAsync(p_out, p_out_dev, total_output_size, cudaMemcpyDeviceToHost, decoderStreams[0]);
+    }
+  }
+
+  cudaStreamSynchronize(decoderStreams[0]);
+  if (p_decParams->check_crc) {
+    for (int r = 0; r < n_segments; r++) {
+      //      if (r<=1) for (int i=0;i<(K>>3);i++) printf("byte (%d,%d) %x\n",r,i,((uint8_t*)(p_out+r*(K>>3)))[i]);
+
+      if (!p_decParams->check_crc((uint8_t*)(p_out + (r * (K >> 3))), p_decParams->Kprime, p_decParams->crc_type)) {
+        LOG_D(PHY, "Segment %d/%d CRC NOK\n", r, n_segments);
+        return (1 + numMaxIter);
+      }
+      /*
+      uint8_t *b=(uint8_t*)(p_out + (r*(K>>3)));
+      int i=0;
+      if (b[K-2] == 0 && b[K - 1] == 0) {
+            while (b[i] == 0 && i < K)
+           i++;
+            if (i == K) {
+              LOG_E(PHY, "received all 0 pdu (K %d, r %d)\n",K,r);
+            }
+      }
+      */
+    }
+  }
+  return numMaxIter;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h

@@ -169,6 +169,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{
   uint32_t C;
   nrLDPC_segment_encoding_parameters_t *segments;
   unsigned char *output;
+  uint8_t **c_dev;
 } nrLDPC_TB_encoding_parameters_t;
 
 /**

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c

@@ -285,7 +285,7 @@ int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_de
   return nrLDPC_TB_decoding_parameters->C;
 }
 
-int32_t nrLDPC_coding_init(void)
+int32_t nrLDPC_coding_init(int max_num_pxsch)
 {
   return 0;
 }

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c

@@ -380,16 +380,15 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
     // Compute number of tasks to encode TB
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
-    size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
-    nbTasks += n_seg;
+      size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
+      nbTasks += n_seg;
 
-    // Search for maximum E for sizing encoder output f and f2
-    for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
-      uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
-      Emax = E > Emax ? E : Emax;
-    }
+      // Search for maximum E for sizing encoder output f and f2
+      for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
+        uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
+        Emax = E > Emax ? E : Emax;
+      }
   }
-
   // Create f and f2 to old encoding tasks outputs
   uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
   uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c

@@ -7,7 +7,8 @@
 #include "common/utils/LOG/log.h"
 
 // #define RM_DEBUG 1
-
+#define USE128BIT
+ 
 static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
 
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
@@ -48,7 +49,7 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
       simde__m128i *f_128 = (simde__m128i *)f;
       simde__m128i *e0_128 = (simde__m128i *)e0;
       simde__m128i *e1_128 = (simde__m128i *)e1;
-      for (; i < (EQm & ~15); i += 64) {
+      for (; i < (EQm & ~15); i += 16) {
         simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
         simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
         simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
@@ -342,14 +343,15 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
       f = (uint8_t *)f_512;
 #endif
 #ifdef USE128BIT
-      e0_128 = (simde__m128i *)e0;
-      e1_128 = (simde__m128i *)e1;
-      e2_128 = (simde__m128i *)e2;
-      e3_128 = (simde__m128i *)e3;
-      e4_128 = (simde__m128i *)e4;
-      e5_128 = (simde__m128i *)e5;
-      e6_128 = (simde__m128i *)e6;
-      e7_128 = (simde__m128i *)e7;
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f_128  = (simde__m128i *)f;
       for (; i < (EQm & ~15); i += 16) {
         simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
         simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
@@ -427,25 +429,240 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
   }
 }
 
+#if defined(__aarch64__)
+static inline uint8x16_t tbl96_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2,
+                                  uint8x16_t b3, uint8x16_t b4, uint8x16_t b5,
+                                  uint8x16_t idx /* 0..95 */)
+{
+   // Table 0: bytes 0..63
+   uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
+   uint8x16_t r0 = vqtbl4q_u8(T0, idx);
+
+   // Table 1: bytes 64..95, presented as a 64-byte table:
+   // bytes 0..31 map to original 64..95, bytes 32..63 are dummy (return 0)
+   uint8x16_t z = vdupq_n_u8(0);
+   uint8x16x4_t T1 = {{ b4, b5, z, z }};
+
+   // idx1 = idx - 64 (wrap-safe via unsigned subtract); only valid when idx>=64
+   uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
+   uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
+
+   // Select r1 where idx >= 64, else r0
+   uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64));  // 0xFF where idx>=64
+   return vbslq_u8(sel, r1, r0);
+}
+
+static inline uint8x16_t tbl128_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2, uint8x16_t b3,
+	                           uint8x16_t b4, uint8x16_t b5, uint8x16_t b6, uint8x16_t b7,
+	                           uint8x16_t idx /* 0..127 */)
+{
+  // Table low: bytes 0..63
+     uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
+     uint8x16_t r0 = vqtbl4q_u8(T0, idx);
+
+ // Table high: bytes 64..127, mapped to 0..63 by subtracting 64
+     uint8x16x4_t T1 = {{ b4, b5, b6, b7 }};
+     uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
+     uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
+
+ // Select high where idx >= 64 else low
+     uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64));
+     return vbslq_u8(sel, r1, r0);
+}
+#elif defined(__AVX512BW__)
+
+static inline __m512i idx_stride_u16(int ways, int k)
+{
+    uint16_t idx[32] __attribute__((aligned(64)));
+    for (int n = 0; n < 32; n++) idx[n] = (uint16_t)(k + ways*n);
+      return _mm512_load_si512((const void*)idx);
+}
+
+// Build indices for k = 0..5, lanes n=0..15: p = k + 6*n
+static inline __m512i make_idx_ab(int k) {
+    uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
+    for (int n = 0; n < 16; n++) {
+        idx[n] = (uint16_t)(k + 6*n); // p in [0..95]
+    }
+    return _mm512_load_si512((const void*)idx);
+}
+
+static inline __m512i make_idx_bc(int k) {
+    uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
+
+    for (int n = 0; n < 16; n++) {
+        int p = k + 6*n;
+   // For (B,C) pair, global p maps to:
+   //   indices 0..31  -> B lanes 0..31 (global 32..63)
+   //   indices 32..63 -> C lanes 0..31 (global 64..95)
+   // so use (p - 32) as index into [B|C] when p >= 32.
+        idx[n] = (uint16_t)(p - 32);
+    }
+    return _mm512_load_si512((const void*)idx);
+}
+
+static inline __mmask32 make_mask_bc(int k) {
+   // lanes needing BC are those with p >= 64 (since AB covers global 0..63)
+   __mmask32 m = 0;
+   for (int n = 0; n < 16; n++) {
+       int p = k + 6*n;
+       if (p >= 64) m |= (1u << n);
+   }
+   return m;
+}
+#endif
+
 void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
 {
+  const uint32_t EQm = E / Qm;
   switch (Qm) {
     case 2: {
       AssertFatal(E % 2 == 0, "");
-      int16_t *e1 = e + (E / 2);
-      int16_t *end = f + E - 1;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int i=0;	  
+#if defined(__aarch64__)
+      for (; i + 8 <= EQm; i += 8) {
+        int16x8x2_t v = vld2q_s16(f);  // 8 groups
+        f += 16;
+        vst1q_s16(e, v.val[0]); e += 8;
+        vst1q_s16(e1, v.val[1]); e1 += 8;
+      }
+#elif defined(__AVX512BW__)
+    const __m512i idx0 = idx_stride_u16(2, 0);
+    const __m512i idx1 = idx_stride_u16(2, 1);
+
+    for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 2 u16 = 64 u16 = 2x512b
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0)); // u16[0..31]
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
+        f += 64;
+        __m512i o0 = _mm512_permutex2var_epi16(A, idx0, B);
+        __m512i o1 = _mm512_permutex2var_epi16(A, idx1, B);
+        _mm512_storeu_si512((void*)e, o0); e += 32;
+        _mm512_storeu_si512((void*)e1, o1); e1 += 32;
+    }
+#else 
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *f128   = (simde__m128i *)f;	  
+      const uint8_t shuf4[16]  __attribute__((aligned(16))) = {0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15};
+      
+      const simde__m128i *shuf4_128 = (const simde__m128i *)shuf4;
+      for (i=0; i < (EQm & ~7); i += 8) {      
+            simde__m128i f0j = simde_mm_loadu_si128(f128++); // f0(i) f0(i+1) f0(i+2) f0(i+3) f0(i+4) f0(i+5) f0(i+6) f0(i+7)
+            simde__m128i f1j = simde_mm_loadu_si128(f128++); // f1(i) f1(i+1) f1(i+2) f1(i+3) f1(i+4) f1(i+5) f1(i+6) f1(i+7)
+    	    simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf4_128); // f0(i) f0(i+2) f0(i+4) f0(i+6) f0(i+1) f0(i+3) f0(i+5) f0(i+7)
+     	    simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf4_128); // f1(i) f1(i+2) f1(i+4) f1(i+6) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
+            simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp0, tmp1));   // f0(i) f0(i+2) f0(i+4) f0(i+6) f1(i) f1(i+2) f1(i+4) f1(i+6)
+            simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp0, tmp1));   // f0(i+1) f0(i+3) f0(i+5) f0(i+7) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
+      }
+      e=(int16_t *)e0_128;
+      e1=(int16_t *)e1_128;
+      f=(int16_t *)f128;      
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
       }
     } break;
+
     case 4: {
       AssertFatal(E % 4 == 0, "");
-      int16_t *e1 = e + (E / 4);
-      int16_t *e2 = e1 + (E / 4);
-      int16_t *e3 = e2 + (E / 4);
-      int16_t *end = f + E - 3;
-      while (f < end) {
+      int i=0;
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+
+#if defined(__aarch64__)
+      for (; i + 8 <= EQm; i += 8) {
+        int16x8x4_t v = vld4q_s16(f);  // 8 groups
+        f += 32;
+        vst1q_s16(e, v.val[0]); e += 8;
+        vst1q_s16(e1, v.val[1]); e1 += 8;
+        vst1q_s16(e2, v.val[2]); e2 += 8;
+        vst1q_s16(e3, v.val[3]); e3 += 8;
+      }
+#elif defined(__AVX512BW__)
+      const __m512i idx0 = idx_stride_u16(4, 0);
+      const __m512i idx1 = idx_stride_u16(4, 1);
+      const __m512i idx2 = idx_stride_u16(4, 2);
+      const __m512i idx3 = idx_stride_u16(4, 3);
+
+      for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 4 u16 = 128 u16 = 4x512b
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0));
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32));
+        __m512i C = _mm512_loadu_si512((const void*)(f + 64));
+        __m512i D = _mm512_loadu_si512((const void*)(f + 96));
+        f += 128;
+       // Build results from (A,B) and (C,D) with masking (boundary at 64 u16)
+        __m512i o0_ab = _mm512_permutex2var_epi16(A, idx0, B);
+        __m512i o1_ab = _mm512_permutex2var_epi16(A, idx1, B);
+        __m512i o2_ab = _mm512_permutex2var_epi16(A, idx2, B);
+        __m512i o3_ab = _mm512_permutex2var_epi16(A, idx3, B);
+
+        __m512i idx0_cd = _mm512_sub_epi16(idx0, _mm512_set1_epi16(64));
+        __m512i idx1_cd = _mm512_sub_epi16(idx1, _mm512_set1_epi16(64));
+        __m512i idx2_cd = _mm512_sub_epi16(idx2, _mm512_set1_epi16(64));
+        __m512i idx3_cd = _mm512_sub_epi16(idx3, _mm512_set1_epi16(64));
+
+        __m512i o0_cd = _mm512_permutex2var_epi16(C, idx0_cd, D);
+        __m512i o1_cd = _mm512_permutex2var_epi16(C, idx1_cd, D);
+        __m512i o2_cd = _mm512_permutex2var_epi16(C, idx2_cd, D);
+        __m512i o3_cd = _mm512_permutex2var_epi16(C, idx3_cd, D);
+
+        // lanes needing CD are those where idx >= 64
+	__mmask32 m0 = _mm512_cmpge_epu16_mask(idx0, _mm512_set1_epi16(64));
+	__mmask32 m1 = _mm512_cmpge_epu16_mask(idx1, _mm512_set1_epi16(64));
+	__mmask32 m2 = _mm512_cmpge_epu16_mask(idx2, _mm512_set1_epi16(64));
+	__mmask32 m3 = _mm512_cmpge_epu16_mask(idx3, _mm512_set1_epi16(64));
+	
+	__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_cd);
+	__m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_cd);
+	__m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_cd);
+	__m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_cd);
+	
+	_mm512_storeu_si512((void*)e, o0); e += 32;
+	_mm512_storeu_si512((void*)e1, o1); e1 += 32;
+	_mm512_storeu_si512((void*)e2, o2); e2 += 32;
+	_mm512_storeu_si512((void*)e3, o3); e3 += 32;
+      }
+#else 
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *f128   = (simde__m128i *)f;	  
+     
+      const uint8_t shuf16[16] __attribute__((aligned(16)))= {0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15};
+      const simde__m128i *shuf16_128 = (const simde__m128i *)shuf16;
+
+      for (i=0; i < (EQm & ~7); i += 8) {      
+        simde__m128i f0j = simde_mm_loadu_si128(f128++);
+        simde__m128i f1j = simde_mm_loadu_si128(f128++);
+        simde__m128i f2j = simde_mm_loadu_si128(f128++);
+        simde__m128i f3j = simde_mm_loadu_si128(f128++);
+
+        simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf16_128); // f0(i) f0(i+4) f0(i+1) f0(i+5) f0(i+2) f0(i+6) f0(i+3) f0(i+7)
+        simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf16_128); // f1(i) f1(i+4) f1(i+1) f1(i+5) f1(i+2) f1(i+6) f1(i+3) f1(i+7)
+        simde__m128i tmp2 = simde_mm_shuffle_epi8(f2j,*shuf16_128); // f2(i) f2(i+4) f2(i+1) f2(i+5) f2(i+2) f2(i+6) f2(i+3) f2(i+7)
+	simde__m128i tmp3 = simde_mm_shuffle_epi8(f3j,*shuf16_128); // f3(i) f3(i+2) f3(i+1) f3(i+5) f3(i+2) f3(i+6) f3(i+3) f3(i+7)
+        simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0,tmp1); // f0(i) f0(i+4) f1(i) f1(i+4) f0(i+1) f0(i+5) f1(i+1) f1(i+5)
+        simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2,tmp3); // f2(i) f2(i+4) f3(i) f3(i+4) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
+       	simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));   // f0(i)   f0(i+4) f1(i)   f1(i+4) f2(i)   f2(i+4) f3(i)   f3(i+4)
+        simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));   // f0(i+1) f0(i+5) f1(i+1) f1(i+5) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
+        tmp4 = simde_mm_unpackhi_epi32(tmp0,tmp1); // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f0(i+3) f0(i+7) f1(i+3) f1(i+7)
+        tmp5 = simde_mm_unpackhi_epi32(tmp2,tmp3); // f2(i+2) f2(i+6) f3(i+2) f3(i+6) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
+        simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));   // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f2(i+2) f2(i+6) f3(i+2) f3(i+6)
+  	simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));   // f0(i+3) f0(i+7) f1(i+3) f1(i+7) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
+      } 
+      e=(int16_t *)e0_128;                                   
+      e1=(int16_t *)e1_128;                                 
+      e2=(int16_t *)e2_128;                                
+      e3=(int16_t *)e3_128;                                 
+      f=(int16_t *)f128;                                      
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -454,13 +671,108 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
     } break;
     case 6: {
       AssertFatal(E % 6 == 0, "");
-      int16_t *e1 = e + (E / 6);
-      int16_t *e2 = e1 + (E / 6);
-      int16_t *e3 = e2 + (E / 6);
-      int16_t *e4 = e3 + (E / 6);
-      int16_t *e5 = e4 + (E / 6);
-      int16_t *end = f + E - 5;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+      int16_t *e4 = e3 + EQm;
+      int16_t *e5 = e4 + EQm;
+      int i=0;
+#if defined(__aarch64__)
+
+          // Byte indices for extracting each stream
+	  // Each s16 occupies 2 bytes → indices are 2*(k + 6*n)
+      const uint8x16_t idx0 = {  0,  1, 12, 13, 24, 25, 36, 37,
+                                48, 49, 60, 61, 72, 73, 84, 85 };
+      const uint8x16_t idx1 = {  2,  3, 14, 15, 26, 27, 38, 39,
+                                50, 51, 62, 63, 74, 75, 86, 87 };
+      const uint8x16_t idx2 = {  4,  5, 16, 17, 28, 29, 40, 41,
+                                52, 53, 64, 65, 76, 77, 88, 89 };
+      const uint8x16_t idx3 = {  6,  7, 18, 19, 30, 31, 42, 43,
+                                54, 55, 66, 67, 78, 79, 90, 91 };
+      const uint8x16_t idx4 = {  8,  9, 20, 21, 32, 33, 44, 45,
+                                56, 57, 68, 69, 80, 81, 92, 93 };
+      const uint8x16_t idx5 = { 10, 11, 22, 23, 34, 35, 46, 47,
+                                58, 59, 70, 71, 82, 83, 94, 95 };
+
+      for (; i + 8 <= EQm; i += 8) {
+	// Load 96 bytes (48 u16)
+	uint8x16_t b0 = vld1q_u8((const uint8_t*)(f +  0)); // bytes  0..15
+	uint8x16_t b1 = vld1q_u8((const uint8_t*)(f +  8)); // bytes 16..31
+	uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
+	uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
+	uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
+	uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
+	f += 48;
+        uint8x16_t o0b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx0);
+        uint8x16_t o1b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx1);
+        uint8x16_t o2b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx2);
+        uint8x16_t o3b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx3);
+        uint8x16_t o4b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx4);
+        uint8x16_t o5b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx5);
+
+        vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
+        vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
+        vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
+        vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
+        vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
+        vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
+      }	
+#elif defined(__AVX512BW__)
+
+      // Precompute permute control once (all compile-time constant patterns)
+      const __m512i idx0_ab = make_idx_ab(0), idx1_ab = make_idx_ab(1);
+      const __m512i idx2_ab = make_idx_ab(2), idx3_ab = make_idx_ab(3);
+      const __m512i idx4_ab = make_idx_ab(4), idx5_ab = make_idx_ab(5);
+
+      const __m512i idx0_bc = make_idx_bc(0), idx1_bc = make_idx_bc(1);
+      const __m512i idx2_bc = make_idx_bc(2), idx3_bc = make_idx_bc(3);
+      const __m512i idx4_bc = make_idx_bc(4), idx5_bc = make_idx_bc(5);
+
+      const __mmask32 m0 = make_mask_bc(0), m1 = make_mask_bc(1), m2 = make_mask_bc(2);
+      const __mmask32 m3 = make_mask_bc(3), m4 = make_mask_bc(4), m5 = make_mask_bc(5);
+
+      const __mmask32 store16 = 0xFFFFu; // store only first 16 lanes (16x u16)
+      for (; i + 16 <= EQm; i += 16) {
+        // 16 groups = 16*(6 u16) = 96 u16 = 192 bytes = 3 * 64B
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0)); // u16[0..31]
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
+        __m512i C = _mm512_loadu_si512((const void*)(f + 64)); // u16[64..95]
+        f += 96;
+
+        // Gather from AB (global 0..63) and BC (global 32..95)
+        __m512i o0_ab = _mm512_permutex2var_epi16(A, idx0_ab, B);
+        __m512i o1_ab = _mm512_permutex2var_epi16(A, idx1_ab, B);
+        __m512i o2_ab = _mm512_permutex2var_epi16(A, idx2_ab, B);
+        __m512i o3_ab = _mm512_permutex2var_epi16(A, idx3_ab, B);
+        __m512i o4_ab = _mm512_permutex2var_epi16(A, idx4_ab, B);
+        __m512i o5_ab = _mm512_permutex2var_epi16(A, idx5_ab, B);
+
+        __m512i o0_bc = _mm512_permutex2var_epi16(B, idx0_bc, C);
+        __m512i o1_bc = _mm512_permutex2var_epi16(B, idx1_bc, C);
+        __m512i o2_bc = _mm512_permutex2var_epi16(B, idx2_bc, C);
+        __m512i o3_bc = _mm512_permutex2var_epi16(B, idx3_bc, C);
+        __m512i o4_bc = _mm512_permutex2var_epi16(B, idx4_bc, C);
+        __m512i o5_bc = _mm512_permutex2var_epi16(B, idx5_bc, C);
+
+        // Select lanes that crossed the 64-element boundary
+   	__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_bc);
+        __m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_bc);
+        __m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_bc);
+        __m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_bc);
+        __m512i o4 = _mm512_mask_mov_epi16(o4_ab, m4, o4_bc);
+        __m512i o5 = _mm512_mask_mov_epi16(o5_ab, m5, o5_bc);
+
+        // Store only the first 16 lanes (16 groups) to each plane
+        _mm512_mask_storeu_epi16((void*)e, store16, o0); e += 16;
+        _mm512_mask_storeu_epi16((void*)e1, store16, o1); e1 += 16;
+        _mm512_mask_storeu_epi16((void*)e2, store16, o2); e2 += 16;
+        _mm512_mask_storeu_epi16((void*)e3, store16, o3); e3 += 16;
+        _mm512_mask_storeu_epi16((void*)e4, store16, o4); e4 += 16;
+        _mm512_mask_storeu_epi16((void*)e5, store16, o5); e5 += 16;
+     }
+			//
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -471,15 +783,180 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
     } break;
     case 8: {
       AssertFatal(E % 8 == 0, "");
-      int16_t *e1 = e + (E / 8);
-      int16_t *e2 = e1 + (E / 8);
-      int16_t *e3 = e2 + (E / 8);
-      int16_t *e4 = e3 + (E / 8);
-      int16_t *e5 = e4 + (E / 8);
-      int16_t *e6 = e5 + (E / 8);
-      int16_t *e7 = e6 + (E / 8);
-      int16_t *end = f + E - 7;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+      int16_t *e4 = e3 + EQm;
+      int16_t *e5 = e4 + EQm;
+      int16_t *e6 = e5 + EQm;
+      int16_t *e7 = e6 + EQm;
+      int i=0;
+#if 0 //defined(__aarch64__)
+      // SIMDE version below is more efficient than tbl128
+          // For 8 groups: byte indices for stream k are 2*(k + 8*n) for n=0..7.
+	  // That is: 2k + 16n (and +1 for the high byte of the u16).
+      const uint8x16_t idx0 = {  0,  1, 16, 17, 32, 33, 48, 49, 64, 65, 80, 81, 96, 97,112,113 };
+      const uint8x16_t idx1 = {  2,  3, 18, 19, 34, 35, 50, 51, 66, 67, 82, 83, 98, 99,114,115 };
+      const uint8x16_t idx2 = {  4,  5, 20, 21, 36, 37, 52, 53, 68, 69, 84, 85,100,101,116,117 };
+      const uint8x16_t idx3 = {  6,  7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87,102,103,118,119 };
+      const uint8x16_t idx4 = {  8,  9, 24, 25, 40, 41, 56, 57, 72, 73, 88, 89,104,105,120,121 };
+      const uint8x16_t idx5 = { 10, 11, 26, 27, 42, 43, 58, 59, 74, 75, 90, 91,106,107,122,123 };
+      const uint8x16_t idx6 = { 12, 13, 28, 29, 44, 45, 60, 61, 76, 77, 92, 93,108,109,124,125 };
+      const uint8x16_t idx7 = { 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95,110,111,126,127 };
+
+      for (; i + 8 <= EQm; i += 8) {
+        // Load 128 bytes = 64 u16 = 8 groups (aligned-friendly, but vld1q_u8 is fine either way)
+        uint8x16_t b0 = vld1q_u8((const uint8_t*)(f +  0)); // bytes  0..15
+        uint8x16_t b1 = vld1q_u8((const uint8_t*)(f +  8)); // bytes 16..31
+        uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
+        uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
+        uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
+        uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
+        uint8x16_t b6 = vld1q_u8((const uint8_t*)(f + 48)); // bytes 96..111
+        uint8x16_t b7 = vld1q_u8((const uint8_t*)(f + 56)); // bytes 112..127
+        f += 64; // consumed 64 u16
+	       
+        uint8x16_t o0b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx0);
+        uint8x16_t o1b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx1);
+        uint8x16_t o2b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx2);
+        uint8x16_t o3b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx3);
+        uint8x16_t o4b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx4);
+        uint8x16_t o5b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx5);
+        uint8x16_t o6b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx6);
+        uint8x16_t o7b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx7);
+
+        vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
+        vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
+        vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
+        vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
+        vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
+        vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
+        vst1q_s16(e6, vreinterpretq_s16_u8(o6b)); e6 += 8;
+        vst1q_s16(e7, vreinterpretq_s16_u8(o7b)); e7 += 8;	
+      }
+#elif defined(__AVX512BW__)
+
+    // Precompute indices for each stream k=0..7
+    const __m512i idx0 = idx_stride_u16(8,0);
+    const __m512i idx1 = idx_stride_u16(8,1);
+    const __m512i idx2 = idx_stride_u16(8,2);
+    const __m512i idx3 = idx_stride_u16(8,3);
+    const __m512i idx4 = idx_stride_u16(8,4);
+    const __m512i idx5 = idx_stride_u16(8,5);
+    const __m512i idx6 = idx_stride_u16(8,6);
+    const __m512i idx7 = idx_stride_u16(8,7);
+
+    const __m512i c64  = _mm512_set1_epi16(64);
+    const __m512i c128 = _mm512_set1_epi16(128);
+    const __m512i c192 = _mm512_set1_epi16(192);
+    
+    for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 8 u16 = 256 u16 = 8 * 32 u16 = 8 ZMM vectors
+	__m512i A = _mm512_loadu_si512((const void*)(f +  0));   // u16[  0.. 31]
+	__m512i B = _mm512_loadu_si512((const void*)(f + 32));   // u16[ 32.. 63]
+	__m512i C = _mm512_loadu_si512((const void*)(f + 64));   // u16[ 64.. 95]
+	__m512i D = _mm512_loadu_si512((const void*)(f + 96));   // u16[ 96..127]
+	__m512i E = _mm512_loadu_si512((const void*)(f + 128));  // u16[128..159]
+	__m512i F = _mm512_loadu_si512((const void*)(f + 160));  // u16[160..191]
+	__m512i G = _mm512_loadu_si512((const void*)(f + 192));  // u16[192..223]
+	__m512i H = _mm512_loadu_si512((const void*)(f + 224));  // u16[224..255]
+	f += 256;
+        // Helper macro: compute one stream (k) by permuting from 4 segments and blending
+	#define DO_STREAM(IDX, OUTPTR) do { \
+	   /* segment masks based on global index p = k+8*n */ \
+	   __mmask32 m1 = _mm512_cmpge_epu16_mask((IDX), c64); \
+	   __mmask32 m2 = _mm512_cmpge_epu16_mask((IDX), c128); \
+	   __mmask32 m3 = _mm512_cmpge_epu16_mask((IDX), c192); \
+	   /* indices relative to each segment base */ \
+	   __m512i i0 = (IDX); \
+	   __m512i i1 = _mm512_sub_epi16((IDX), c64); \
+	   __m512i i2 = _mm512_sub_epi16((IDX), c128); \
+	   __m512i i3 = _mm512_sub_epi16((IDX), c192); \
+	   /* permute from each 64-u16 segment (two ZMMs per segment) */ \
+	   __m512i r0 = _mm512_permutex2var_epi16(A, i0, B); \
+	   __m512i r1 = _mm512_permutex2var_epi16(C, i1, D); \
+	   __m512i r2 = _mm512_permutex2var_epi16(E, i2, F); \
+	   __m512i r3 = _mm512_permutex2var_epi16(G, i3, H); \
+           /* blend: later segments overwrite earlier where applicable */ \
+	   __m512i r = _mm512_mask_mov_epi16(r0, m1, r1); \
+	   r = _mm512_mask_mov_epi16(r,  m2, r2); \
+	   r = _mm512_mask_mov_epi16(r,  m3, r3); \
+	   _mm512_storeu_si512((void*)(OUTPTR), r); \
+	   (OUTPTR) += 32; \
+	} while(0)
+        DO_STREAM(idx0, e);
+	DO_STREAM(idx1, e1);
+	DO_STREAM(idx2, e2);
+	DO_STREAM(idx3, e3);
+	DO_STREAM(idx4, e4);
+	DO_STREAM(idx5, e5);
+	DO_STREAM(idx6, e6);
+	DO_STREAM(idx7, e7);
+
+        #undef DO_STREAM
+    }
+#else
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f128   = (simde__m128i *)f;
+
+      for (i=0; i < (EQm & ~7); i += 8) {
+        simde__m128i f0j = simde_mm_loadu_si128(f128++);
+        simde__m128i f1j = simde_mm_loadu_si128(f128++);
+        simde__m128i f2j = simde_mm_loadu_si128(f128++);
+        simde__m128i f3j = simde_mm_loadu_si128(f128++);
+        simde__m128i f4j = simde_mm_loadu_si128(f128++);
+        simde__m128i f5j = simde_mm_loadu_si128(f128++);
+        simde__m128i f6j = simde_mm_loadu_si128(f128++);
+        simde__m128i f7j = simde_mm_loadu_si128(f128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi16(f0j, f1j);   // f0(i) f1(i) f0(i+1) f1(i+1) f0(i+2) f1(i+2) f0(i+3) f1(i+3) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi16(f2j, f3j);   // f2(i) f3(i) f2(i+1) f3(i+1) f2(i+2) f3(i+2) f2(i+3) f3(i+3) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi16(f4j, f5j);   // f4(i) f5(i) f4(i+1) f5(i+1) f4(i+2) f5(i+2) f4(i+3) f5(i+3)
+        simde__m128i tmp3 = simde_mm_unpacklo_epi16(f6j, f7j);   // f6(i) f7(i) f6(i+1) f7(i+1) f6(i+2) f7(i+2) f6(i+3) f7(i+3)
+								 //
+        simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i) f1(i) f2(i)   f3(i)   f0(i+1) f1(i+1) f2(i+1) f3(i+1) 
+        simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i) f5(i) f6(i)   f7(i)   f4(i+1) f5(i+1) f6(i+1) f7(i+1) 
+        simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i) f1(i) f2(i) f3(i) f4(i) f5(i) f7(i) f7(i)
+        simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+1) f1(i+1) f2(i+1) f3(i+1) f4(i+1) f5(i+1) f6(i+1) f7(i+1)
+									    //
+        tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+2) f1(i+2) f2(i+2) f3(i+2) f0(i+3) f1(i+3) f2(i+3) f3(i+3) 
+        tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+2) f5(i+2) f6(i+2) f7(i+2) f4(i+3) f5(i+3) f6(i+3) f7(i+3) 
+	simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+2) f1(i+2) f2(i+2) f3(i+2) f4(i+2) f5(i+2) f7(i+2) f7(i+2) 
+        simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+3) f1(i+3) f2(i+3) f3(i+3) f4(i+3) f5(i+3) f7(i+3) f7(i+3)
+									     
+        tmp0 = simde_mm_unpackhi_epi16(f0j, f1j);   // f0(i+4) f1(i+4) f0(i+5) f1(i+5) f0(i+6) f1(i+6) f0(i+7) f1(i+7) 
+        tmp1 = simde_mm_unpackhi_epi16(f2j, f3j);   // f2(i+4) f3(i+4) f2(i+5) f3(i+5) f2(i+6) f3(i+6) f2(i+7) f3(i+7) 
+        tmp2 = simde_mm_unpackhi_epi16(f4j, f5j);   // f4(i+4) f5(i+4) f4(i+5) f5(i+5) f4(i+6) f5(i+6) f4(i+7) f5(i+7)
+        tmp3 = simde_mm_unpackhi_epi16(f6j, f7j);   // f6(i+4) f7(i+4) f6(i+5) f7(i+5) f6(i+6) f7(i+6) f6(i+7) f7(i+7)
+								 //
+        tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i+4) f1(i+4) f2(i+4)   f3(i+4)   f0(i+5) f1(i+5) f2(i+5) f3(i+5) 
+        tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i+4) f5(i+4) f6(i+4)   f7(i+4)   f4(i+5) f5(i+5) f6(i+5) f7(i+5) 
+								 
+        simde_mm_storeu_si128(e4_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i+4) f1(i+4) f2(i+4) f3(i+4) f4(i+4) f5(i+4) f7(i+4) f7(i+4)
+        simde_mm_storeu_si128(e5_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+5) f1(i+5) f2(i+5) f3(i+5) f4(i+5) f5(i+5) f6(i+5) f7(i+5)
+        tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+6) f1(i+6) f2(i+6) f3(i+6) f0(i+6) f1(i+6) f2(i+6) f3(i+6) 
+        tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+6) f5(i+6) f6(i+6) f7(i+6) f4(i+6) f5(i+6) f6(i+6) f7(i+6) 
+	simde_mm_storeu_si128(e6_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+6) f1(i+6) f2(i+6) f3(i+6) f4(i+6) f5(i+6) f7(i+6) f7(i+6) 
+        simde_mm_storeu_si128(e7_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+7) f1(i+7) f2(i+7) f3(i+7) f4(i+7) f5(i+7) f7(i+7) f7(i+7)
+      }
+      e  = (int16_t *)e0_128;
+      e1 = (int16_t *)e1_128;
+      e2 = (int16_t *)e2_128;
+      e3 = (int16_t *)e3_128;
+      e4 = (int16_t *)e4_128;
+      e5 = (int16_t *)e5_128;
+      e6 = (int16_t *)e6_128;
+      e7 = (int16_t *)e7_128;
+      f  = (int16_t *)f128;
+#endif
+      for(;i < EQm ; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -494,6 +971,7 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
       AssertFatal(1 == 0, "Should not get here : Qm %d\n", Qm);
       break;
   }
+
 }
 
 int nr_rate_matching_ldpc(uint32_t Tbslbrm,
@@ -597,6 +1075,82 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
   return 0;
 }
 
+#define RMLOOP for (;ind<(ind2&7);k+=8,ind+=8) \
+      simde_mm_storeu_si128(&d[ind],simde_mm_adds_epi16(simde_mm_loadu_si128(&soft_input[k]),simde_mm_loadu_si128(&d[ind])));\
+   for (; ind<ind2 ; ind++,k++) d[ind] += soft_input[k];  
+
+int nr_rate_matching_ldpc_rx_simd(uint32_t Tbslbrm,
+                                  uint8_t BG,
+                                  uint16_t Z,
+                                  int16_t *d,
+                                  int16_t *soft_input,
+                                  uint8_t C,
+                                  uint8_t rvidx,
+                                  uint8_t clear,
+                                  uint32_t E,
+                                  uint32_t F,
+                                  uint32_t Foffset)
+{
+  if (C == 0) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
+    return -1;
+  }
+
+  //Bit selection
+  uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
+  uint32_t Ncb;
+  if (Tbslbrm == 0)
+    Ncb = N;
+  else {
+    uint32_t Nref = (3 * Tbslbrm / (2 * C)); //R_LBRM = 2/3
+    Ncb = min(N, Nref);
+  }
+
+  uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
+  if (Foffset > E) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > E %d)\n", Foffset, E);
+    return -1;
+  }
+  if (Foffset > Ncb) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
+    return -1;
+  }
+
+#ifdef RM_DEBUG
+  printf("nr_rate_matching_ldpc_rx: Clear %d, E %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
+         clear,
+         E,
+         Foffset,
+         ind,
+         Ncb,
+         rvidx,
+         Tbslbrm);
+#endif
+
+  if (clear == 1)
+    memset(d, 0, Ncb * sizeof(int16_t));
+
+  uint32_t k = 0;
+  if (ind < Foffset) {
+   int ind2 = ind + min(Foffset-ind,E);
+   RMLOOP;
+  }
+  if (ind >= Foffset && ind < Foffset + F)
+    ind = Foffset + F;
+  int ind2 = ind + min(Ncb-ind,E-k);
+  RMLOOP;
+
+  while (k < E) {
+   ind=0;
+   ind2 = min(Foffset,E-k);
+   RMLOOP;
+   ind = Foffset+F;
+   ind2 = ind + min(Ncb-ind,E-k);
+   RMLOOP;
+  }
+  return 0;
+}
+
 int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
                              uint8_t BG,
                              uint16_t Z,
@@ -609,6 +1163,21 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
                              uint32_t F,
                              uint32_t Foffset)
 {
+  if (BG == 1) {
+     nr_rate_matching_ldpc_rx_simd(Tbslbrm,
+                                   BG,
+                                   Z,
+                                   d,
+                                   soft_input,
+                                   C,
+                                   rvidx,
+                                   clear,
+                                   E,
+                                   F,
+                                   Foffset);
+     return 0;
+  }
+
   if (C == 0) {
     LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
     return -1;

openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/BN30_edge.svg

@@ -0,0 +1 @@
+<svg width="610" height="347" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-585 -57)"><rect x="591" y="125" width="105" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(606.957 152)">Block1</text><rect x="594" y="125" width="5" height="39" fill="#C00000"/><rect x="591" y="205" width="105" height="38" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(606.957 232)">Block</text><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(666.123 232)">2</text><rect x="613" y="205" width="5" height="38" fill="#C00000"/><rect x="589" y="335" width="107" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(599.924 362)">Block30</text><rect x="642" y="336" width="5" height="39" fill="#C00000"/><rect x="760" y="119" width="128" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(771.435 152)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(791.022 88)">Step 1</text><rect x="760" y="198" width="128" height="52" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(771.435 232)">Msg_Sum</text><rect x="759" y="329" width="127" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(769.922 362)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1035.89 87)">Step 2</text><rect x="948" y="118" width="128" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(959.16 151)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1081.49 150)">-</text><rect x="1097" y="126" width="95" height="36" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1114.73 151)">Msg1</text><rect x="948" y="198" width="128" height="52" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(959.16 232)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1081.49 230)">-</text><rect x="1097" y="206" width="95" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1114.73 233)">Msg2</text><rect x="947" y="329" width="127" height="51" fill="#ED7D31"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(957.648 362)">Msg_Sum</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="400" font-size="24" transform="translate(1079.98 361)">-</text><rect x="1096" y="337" width="95" height="37" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(1106.55 363)">Msg30</text><path d="M745.5 126.334C745.5 111.514 757.514 99.5001 772.334 99.5001L879.666 99.5001C894.486 99.5001 906.5 111.514 906.5 126.334L906.5 375.666C906.5 390.486 894.486 402.5 879.666 402.5L772.334 402.5C757.514 402.5 745.5 390.486 745.5 375.666Z" stroke="#C00000" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text fill="#ED7D31" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="48" transform="translate(802.013 305)">…</text></g></svg>

openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/cnProc_behavior.svg

@@ -0,0 +1 @@
+<svg width="601" height="304" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-99 -256)"><rect x="148" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(160.681 286)">Msg1</text><rect x="233" y="259" width="86" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(245.929 286)">Msg2</text><rect x="319" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(331.177 286)">Msg3</text><rect x="404" y="259" width="85" height="39" fill="#4472C4"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(416.425 286)">Msg4</text><text fill="#4472C4" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(649.576 369)">CN</text><text fill="#4472C4" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="24" transform="translate(651.713 539)">BN</text><path d="M294 531.5C294 520.73 299.596 512 306.5 512 313.404 512 319 520.73 319 531.5 319 542.27 313.404 551 306.5 551 299.596 551 294 542.27 294 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M404 531.5C404 520.73 409.596 512 416.5 512 423.404 512 429 520.73 429 531.5 429 542.27 423.404 551 416.5 551 409.596 551 404 542.27 404 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M493 531.5C493 520.73 498.596 512 505.5 512 512.404 512 518 520.73 518 531.5 518 542.27 512.404 551 505.5 551 498.596 551 493 542.27 493 531.5Z" fill="#4472C4" fill-rule="evenodd"/><path d="M591 531.5C591 520.73 596.373 512 603 512 609.627 512 615 520.73 615 531.5 615 542.27 609.627 551 603 551 596.373 551 591 542.27 591 531.5Z" fill="#4472C4" fill-rule="evenodd"/><rect x="218" y="342" width="31" height="33" fill="#4472C4"/><path d="M0 0 18.023 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 306.5 512.663)"/><path d="M197 531.5C197 520.73 202.373 512 209 512 215.627 512 221 520.73 221 531.5 221 542.27 215.627 551 209 551 202.373 551 197 542.27 197 531.5Z" fill-rule="evenodd"/><path d="M99 531.5C99 520.73 104.596 512 111.5 512 118.404 512 124 520.73 124 531.5 124 542.27 118.404 551 111.5 551 104.596 551 99 542.27 99 531.5Z" fill-rule="evenodd"/><rect x="309" y="342" width="31" height="33"/><rect x="412" y="342" width="32" height="33"/><path d="M306.46 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M416.284 512.803 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M505.348 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M602.884 512.172 233.5 375.5" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 115.633 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 209.5 512.663)"/><path d="M0 0 316.457 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 112.5 512.663)"/><path d="M416.301 513.295 324.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 212.862 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 112.5 512.663)"/><path d="M0 0 219.228 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 209.5 512.663)"/><path d="M603.306 512.663 428.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M505.77 512.663 428.5 375.5" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd"/><path d="M0 0 121.618 137.163" stroke="#000000" stroke-width="3" stroke-miterlimit="8" fill="none" fill-rule="evenodd" transform="matrix(1 0 0 -1 306.5 512.663)"/></g></svg>

openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/img/decoder_cuda_pipeline.svg

@@ -0,0 +1 @@
+<svg width="942" height="650" xmlns="http://www.w3.org/2000/svg" xmlns:xlink="http://www.w3.org/1999/xlink" overflow="hidden"><g transform="translate(-83 -42)"><path d="M621 185.334C621 148.147 651.147 118 688.335 118L957.665 118C994.853 118 1025 148.147 1025 185.334L1025 549.666C1025 586.853 994.853 617 957.665 617L688.335 617C651.147 617 621 586.853 621 549.666Z" fill="#ED7D31" fill-rule="evenodd"/><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(171.139 167)">llrPreProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(172.399 294)">cnProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(172.27 422)">bnProc_kernel</text><text fill="#0070C0" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(186.211 565)">output_kernel</text><path d="M152.5 263.001C152.5 238.424 172.424 218.5 197.001 218.5L547.999 218.5C572.576 218.5 592.5 238.424 592.5 263.001L592.5 440.999C592.5 465.576 572.576 485.5 547.999 485.5L197.001 485.5C172.424 485.5 152.5 465.576 152.5 440.999Z" stroke="#2F528F" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(429.568 355)">Iterations</text><path d="M411.5 244.5 411.5 458.349" stroke="#4472C4" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9" fill="none" fill-rule="evenodd"/><text fill="#FFFFFF" font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="64" transform="translate(727.296 378)">Graph</text><path d="M85.5001 201.669C85.5001 155.736 122.736 118.5 168.669 118.5L940.331 118.5C986.264 118.5 1023.5 155.736 1023.5 201.669L1023.5 534.331C1023.5 580.264 986.264 617.5 940.331 617.5L168.669 617.5C122.736 617.5 85.5001 580.264 85.5001 534.331Z" stroke="#ED7D31" stroke-width="3" stroke-miterlimit="8" stroke-dasharray="12 9 3 9" fill="none" fill-rule="evenodd"/><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(219.12 82)">Input LLR</text><text font-family="DengXian,DengXian_MSFontService,sans-serif" font-weight="700" font-size="32" transform="translate(132.48 664)">Output Decoded bits</text><path d="M294 95.5 294 126.552 291 126.552 291 95.5001ZM297 125.052 292.5 134.052 288 125.052Z"/><path d="M295 597.5 295 628.552 292 628.552 292 597.5ZM298 627.052 293.5 636.052 289 627.052Z"/></g></svg>

openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/nrLDPC_cuda.md

@@ -0,0 +1,896 @@
+
+# OAI nrLDPC GPU Decoder Architecture
+
+For hands-on instructions and execution commands of this GPU implementation, please refer to the [Tutorial Document](nrLDPC_cuda_tutorial.md).
+
+
+## 0. Scope & Supported Configurations
+
+This document describes the architecture and parallelization strategies of the GPU-accelerated LDPC decoder implemented via CUDA. 
+
+> **Note on Algorithm Baseline:** The current CUDA implementation is strictly **bit-exact and algorithm-exact** with the existing OAI [CPU baseline](nrLDPC.tex). It focuses on throughput acceleration, hardware offloading, and architectural refactoring. It does not introduce any algorithmic modifications to the underlying error correction mathematical model. Therefore, it is expected to yield the exact same BLER performance as the [CPU baseline](nrLDPC.tex)
+
+### Currently Supported Parameters
+
+| Parameter | CPU Baseline  | GPU Implementation |
+| --- | --- | --- |
+| **Base Graph (BG)** | 1, 2 | **1** |
+| **Lifting Size ($Z_c$)** | All  | **$\ge 128$** |
+| **Code Rate (R)** | BG1: 1/3, 2/3, 8/9 <br><br> BG2: 1/5, 1/3, 2/3 | **BG1: 1/3, 2/3, 8/9** |
+
+
+### Related Work & Comparison
+
+| Implementation | Open-Source Framework | 5G NR Spec Aligned | Kernel Source Available | User-Recompilable | Bit-Exact Baseline |
+|---|---|---|---|---|---|
+| NVIDIA Aerial cuPHY | Yes | Yes | Yes | No | N/A |
+| Academic GPU LDPC (e.g. cuLDPC) | Yes | No | Yes | Yes | N/A |
+| **This work (OAI GPU decoder)** | **Yes** | **Yes** | **Yes** | **Yes** | **Yes** |
+
+---
+
+
+
+[[_TOC_]]
+
+## 1. Overall Architecture
+
+![Pipeline Overview](img/decoder_cuda_pipeline.svg)
+
+The OAI nrLDPC GPU decoder consists of four main kernels: `llrPreProc_kernel`, `cnProc_kernel`, `bnProc_kernel`, and `output_kernel`. 
+
+Functionally, the GPU version is strictly aligned with the CPU baseline. Each CUDA kernel is essentially a consolidation of multiple specific functions from the CPU decoder. The exact mapping is as follows:
+
+* **`llrPreProc_kernel`**: Combines `llr2llrProcBuf` and `llr2CnProcBuf`.
+* **`cnProc_kernel`**: Combines `cnProc` and `cn2bnProcBuf`.
+* **`bnProc_kernel`**: Combines `bnProcPc`, `bnProc`, and `bn2cnProcBuf`.
+* **`output_kernel`**: Combines `llrRes2llrOut` and `llr2bit`.
+
+## 2. Message Passing in LDPC Decoding
+
+In this section, we briefly describe the behavior of `cnProc` (Check Node processing) and `bnProc` (Bit Node processing) during decoding. This high-level understanding will help clarify the overall parallelism design discussed later.
+
+In LDPC decoding, the task of a Check Node (CN) in `cnProc` is to generate and send update messages to its connected Bit Nodes (BNs). For example, a degree-4 CN (a CN connected to 4 BNs) needs to generate 4 distinct messages—typically using min-sum operations—and send each message back to the corresponding connected BN. Each of these messages corresponds to one edge in the bipartite graph.
+
+![cnProc message passing](img/cnProc_behavior.svg)
+
+Conversely, the task of a BN in `bnProc` is to gather all the messages from its connected CNs and send updated messages back. This involves accumulation and subtraction operations. For example, a degree-3 BN (a BN connected to 3 CNs) generates 3 messages by first accumulating the 3 incoming messages from the connected CNs along with its original intrinsic LLR, then subtracting each CN's specific value, and sending the newly updated message back to the corresponding CN.
+
+![bnProc message passing](img/bnProc_behavior.svg)
+
+The entire decoding process is executed by iteratively alternating between these two behaviors. 
+## 3. Memory Management & Buffer Layout
+### 3.1 Memory Management
+In the current implementation, all relevant buffers reside entirely in GPU memory (Device memory). By keeping the entire receiver processing pipeline's buffers on the GPU, we completely eliminate the overhead of Host-to-Device (H2D) and Device-to-Host (D2H) memory transfers between individual kernels.
+
+The following table summarizes the key data buffers and their read/write access patterns across the kernels:
+
+| Buffer Name | Description | Written By | Read By |
+| :--- | :--- | :--- | :--- |
+| `p_llr` | Output from the rate matching module (raw input LLRs). | *(Previous module)* | `llrPreProc_kernel` |
+| `p_out` | Final hard-decision decoded output bits. | `output_kernel` | *(Next module)* |
+| `cnProcBuf_dev` | Stores BN-to-CN messages. Initialized with intrinsic LLRs before the first iteration. | `llrPreProc_kernel` (init), `bnProc_kernel` | `cnProc_kernel` |
+| `bnProcBuf_dev` | Stores CN-to-BN messages updated during check node processing. | `cnProc_kernel` | `bnProc_kernel` |
+| `llrRes_dev` | Stores the posterior LLRs (a posteriori information) accumulated during the final iteration. | `bnProc_kernel` (last iter) | `output_kernel` |
+| `llrProcBuf_dev` | Stores the rearranged intrinsic LLRs to be added back during BN updates, preventing belief drift. | `llrPreProc_kernel` | `bnProc_kernel` |
+
+### 3.2 Buffer Layout
+
+All buffers here follow the exact same layout as the CPU version, especially the two most critical buffers in the decoding process: `cnProcBuf_dev` and `bnProcBuf_dev`. 
+
+The OAI decoder implementation groups the Check Nodes (CNs) and Bit Nodes (BNs) based on their degree (the number of connected nodes). 
+While their layouts are rigorously described using mathematical notation in [`doc/nrLDPC/nrLDPC.tex`](nrLDPC.tex), this section provides a more intuitive description of their structures in memory.
+
+`cnProcBuf_dev` stores the messages that a CN needs to process, which are sent by its connected BNs. For example, a degree-4 CN needs to process 4 incoming messages. However, in the current buffer layout design, these 4 messages belonging to the same CN are **not contiguous** in memory. 
+
+To illustrate, consider BG1 (Rate 1/3), which contains 5 CNs with a degree of 4. The actual memory layout groups the same message index across all CNs in that specific group, as depicted below:
+
+![cnProcBuf layout](img/cnProcBuf_layout.svg)
+
+Similarly, `bnProcBuf_dev` follows this exact same interleaved layout strategy.
+
+This specific Structure of Arrays (SoA) interleaving was historically designed to maximize CPU SIMD (e.g., AVX-512) vectorization efficiency. By clustering the same message indices together across multiple CNs, it avoids complex cross-lane shuffles and boundary alignment issues within vector registers. 
+
+For the CUDA implementation, we retain this legacy layout. It imposes no performance penalty on the GPU—since global memory coalescing is already naturally guaranteed by the innermost lifting size ($Z_c$) dimension, and it perfectly fulfills the alignment requirements for the SIMD4 operations used in our GPU implementation.
+## 4. Flat Message Indexing
+
+The fundamental computational elements in our LDPC decoder are the individual messages passed along the edges of the bipartite graph (either CN-to-BN or BN-to-CN). To specify their behaviors during parallel decoding, each message must be mapped to a unique memory address, a destination buffer address, and a specific circular shift value.
+
+Instead of relying on nested loops and complex multi-dimensional pointers, we introduce a **Flat Message Indexing** strategy. We assign a deterministic, global 1D index to every single message (edge) in the Base Graph.
+
+### 4.1 Total Message Calculation
+
+The total number of messages is calculated based on the degree distribution of the Check Nodes in BG1. By multiplying the number of connected BNs (`numBnInCnGroups`) with the number of corresponding CNs (`numCnInCnGroups`), we obtain the exact number of edges for each Code Rate (R):
+
+* **Degrees (`numBnInCnGroups`)**: {3, 4, 5, 6, 7, 8, 9, 10, 19}
+* **R 1/3 CN Counts**: {1, 5, 18, 8, 5, 2, 2, 1, 4} $\rightarrow$ Total Messages = $3(1) + 4(5) + 5(18) + \dots + 19(4) = \mathbf{316}$
+* **R 2/3 CN Counts**: {1, 0, 0, 0, 3, 2, 2, 1, 4} $\rightarrow$ Total Messages = $\mathbf{144}$
+* **R 8/9 CN Counts**: {1, 0, 0, 0, 0, 0, 0, 0, 4} $\rightarrow$ Total Messages = $\mathbf{79}$
+
+### 4.2 Linear Index Mapping
+
+By flattening the structure, the messages are linearized sequentially. The 1D index effectively acts as a universal coordinate `(CN/BN_Degree, CN/BN_Instance_ID, Local_Message_ID)`. 
+
+We use the indexing from CN side as an example, the mappings for the supported code rates are as follows:
+
+```text
+[Rate 1/3 Message Index Map - Total 316]
+Idx 0   : CN3-0-Msg0
+Idx 1   : CN3-0-Msg1
+Idx 2   : CN3-0-Msg2
+Idx 3   : CN4-0-Msg0
+Idx 4   : CN4-0-Msg1
+Idx 5   : CN4-0-Msg2
+Idx 6   : CN4-0-Msg3
+Idx 7   : CN4-1-Msg0
+...
+Idx 314 : CN19-3-Msg17
+Idx 315 : CN19-3-Msg18
+
+[Rate 2/3 Message Index Map - Total 144]
+Idx 0   : CN3-0-Msg0
+Idx 1   : CN3-0-Msg1
+Idx 2   : CN3-0-Msg2
+Idx 3   : CN7-0-Msg0
+Idx 4   : CN7-0-Msg1
+Idx 5   : CN7-0-Msg2
+Idx 6   : CN7-0-Msg3
+Idx 7   : CN7-0-Msg4
+...
+Idx 142 : CN19-3-Msg17
+Idx 143 : CN19-3-Msg18
+
+[Rate 8/9 Message Index Map - Total 79]
+Idx 0   : CN3-0-Msg0
+Idx 1   : CN3-0-Msg1
+Idx 2   : CN3-0-Msg2
+Idx 3   : CN19-0-Msg0
+Idx 4   : CN19-0-Msg1
+Idx 5   : CN19-0-Msg2
+Idx 6   : CN19-0-Msg3
+Idx 7   : CN19-0-Msg4
+...
+Idx 77  : CN19-3-Msg17
+Idx 78  : CN19-3-Msg18
+```
+The indexing from BN side follows the same strategy.
+
+This 1D indexing maps to specific offset LUTs, seamlessly guiding the threads to calculate the exact memory addresses on the interleaved buffer layout described in Section 3.2. 
+
+To support other code rates or base graphs in the future (for example, a potential BG3 in 6G), basically, we just need to design new indexings and LUTs accordingly.
+
+## 5. Parallelism Strategies
+### 5.1 Edge-Based Parallelism
+
+To extract the maximum concurrency from the GPU hardware, the most intuitive approach is to compute every single message (edge) in the bipartite graph simultaneously. We refer to this approach as **Edge-Based Parallelism**.
+
+#### 5.1.1 Thread Organization
+
+In this strategy, we map the computation of messages directly to the CUDA execution grid. To maximize memory throughput and instruction efficiency, we utilize 32-bit vectorized memory accesses (`simd4`). 
+
+The thread organization follows this 2D grid/block topology:
+* **X-Dimension (Lifting Size & Messages):** Since we process 4 elements per thread, the block's X-dimension is scaled to $Z_c / 4$ (`Z_c >> 2`). To further increase block-level occupancy, the block's Y-dimension is set to 4, processing 4 messages concurrently within a single block. Consequently, the grid's X-dimension represents the total number of messages divided by 4.
+* **Y-Dimension (Segments):** The grid's Y-dimension directly maps to the number of Transport Block segments (`n_segments`), allowing multiple code blocks to be decoded completely in parallel.
+
+The precise kernel launch configurations for the supported code rates in BG1 are formulated as follows:
+
+```cpp
+// BG1 Rate 1/3 (num_TotalBlocks_BG1_R13_Edge = 316)
+dim3 R13_block(Z_c >> 2, 4, 1);
+dim3 R13_grid(num_TotalBlocks_BG1_R13_Edge >> 2, n_segments, 1);
+```
+```cpp
+// BG1 Rate 2/3 (num_TotalBlocks_BG1_R23_Edge = 144)
+dim3 R23_block(Z_c >> 2, 4, 1);
+dim3 R23_grid(num_TotalBlocks_BG1_R23_Edge >> 2, n_segments, 1);
+```
+```cpp
+// BG1 Rate 8/9 (num_TotalBlocks_BG1_R89_Edge = 79)
+// Note: 79 is not strictly divisible by 4; padding (+3) ensures ceiling division.
+dim3 R89_block(Z_c >> 2, 4, 1);
+dim3 R89_grid((num_TotalBlocks_BG1_R89_Edge + 3) >> 2, n_segments, 1);
+```
+
+#### 5.1.2 cnProc_kernel
+
+For the Check Node processing, we implemented specific kernels for each supported code rate with these namings:
+* `cnProcKernel_BG1_R13_int8_Edge`
+* `cnProcKernel_BG1_R23_int8_Edge`
+* `cnProcKernel_BG1_R89_int8_Edge`
+
+We use BG1 R13 as an example. The global kernel acts as a dispatcher. Based on the Check Node degree (`groupIdx`), it routes the thread to a specific unrolled device function:
+
+```cpp
+__global__ void cnProcKernel_BG1_R13_int8_Edge(const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               uint32_t Zc,
+                                               uint32_t ZcIdx)
+{
+  uint32_t lane = threadIdx.x;
+  // Compute the global 1D flat index (row) for the current message
+  uint32_t row = (blockIdx.x << 2) + threadIdx.y;
+  uint32_t segIdx = blockIdx.y;
+
+  if (row >= num_TotalBlocks_BG1_R13_Edge)
+    return;
+
+  // LUT Lookups: Fetch node degree, instance ID, and message ID
+  uint32_t groupIdx = lut_CnGrpIdx_BG1_R13_Edge[row] - 1;
+  uint32_t CnIdx = lut_CnIdx_BG1_R13_Edge[row] - 1;
+  uint32_t MsgIdx = lut_CnMsgIdx_BG1_R13_Edge[row] - 1;
+  
+  // Calculate exact memory offsets
+  uint32_t InnerOffset = d_lut_startAddrCnGroups_BG1[groupIdx] + NR_LDPC_ZMAX * CnIdx;
+  uint32_t idxBn = cn_bn_map_BG1_Z_R13[row][0];
+  uint32_t circShift = cn_bn_map_BG1_Z_R13[row][ZcIdx];
+
+  const int8_t *p_cnProcBuf = (const int8_t *)(d_cnBufAll + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + InnerOffset);
+  int8_t *p_bnProcBuf = (int8_t *)(d_bnBufAll + segIdx * NR_LDPC_SIZE_BN_PROC_BUF);
+
+  // Dispatch to the degree-specific device function (Branchless execution inside)
+  switch (groupIdx) {
+    case 0: cnProcKernel_BG1_int8_G3(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 1: cnProcKernel_BG1_int8_G4(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 2: cnProcKernel_BG1_int8_G5(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 3: cnProcKernel_BG1_int8_G6(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 4: cnProcKernel_BG1_int8_G7(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 5: cnProcKernel_BG1_int8_G8(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 6: cnProcKernel_BG1_int8_G9(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 7: cnProcKernel_BG1_int8_G10(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+    case 8: cnProcKernel_BG1_int8_G19(p_cnProcBuf, p_bnProcBuf, MsgIdx, lane, idxBn, circShift, Zc); 
+        break;
+  }
+}
+
+```
+
+Inside the device functions, we heavily utilize `SIMD4` operations and PTX intrinsics to perform the Min-Sum algorithm concurrently on 4 elements per thread. Taking the Degree-5 CN (`G5`) as an example:
+
+```cpp
+__device__ __forceinline__ void cnProcKernel_BG1_int8_G5(const int8_t *__restrict__ p_cnProcBuf,
+                                                         int8_t *__restrict__ p_bnProcBuf,
+                                                         uint32_t row,
+                                                         uint32_t lane,
+                                                         uint32_t idxBn,
+                                                         uint32_t circShift,
+                                                         uint32_t Zc)
+{
+  uint32_t ymm0, sgn, min;
+  const uint32_t ones = 0x01010101;
+
+  // Load the first message (4 bytes/int8s at once)
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][0] * 4);
+  sgn = __vxor4(ones, ymm0); // Extract and track signs
+  min = __vabs4(ymm0);       // Initialize minimum magnitude
+
+  // --- Fully Unrolled Min-Sum Loop for remaining 4 edges ---
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][1] * 4);
+  min = __vminu4(min, __vabs4(ymm0)); // Update minimum
+  sgn = __vxor4(sgn, ymm0);           // Accumulate sign parity
+  
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][2] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  
+  ymm0 = *(const uint32_t *)(p_cnProcBuf + lane * 4 + c_lut_idxG5[row][3] * 4);
+  min = __vminu4(min, __vabs4(ymm0));
+  sgn = __vxor4(sgn, ymm0);
+  // ---------------------------------------------------------
+  
+  // Combine the accumulated sign with the minimum magnitude
+  uint32_t BricksToBeMoved = __vsign4(min, sgn);
+
+  // Apply circular shift and write the generated message back to the BN buffer
+  moveBricks_invput_circ((int8_t *)&p_bnProcBuf[idxBn], lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+}
+
+```
+
+#### 5.1.3 bnProc_kernel
+
+Similarly, the `bnProc` kernel is instantiated for each code rate:
+
+* `bnProcKernel_BG1_R13_int8_Edge`
+* `bnProcKernel_BG1_R23_int8_Edge`
+* `bnProcKernel_BG1_R89_int8_Edge`
+
+```cpp
+__global__ void bnProcKernel_BG1_R13_int8_Edge(const int8_t *__restrict__ d_bnProcBuf,
+                                               int8_t *__restrict__ d_cnProcBuf,
+                                               int8_t *__restrict__ d_llrProcBuf,
+                                               int8_t *__restrict__ d_llrRes,
+                                               uint32_t Zc,
+                                               uint32_t ZcIdx)
+{
+  uint32_t lane = threadIdx.x;
+  uint32_t row = (blockIdx.x << 2) + threadIdx.y;
+  uint32_t segIdx = blockIdx.y;
+
+  if (row >= num_TotalBlocks_BG1_R13_Edge)
+    return;
+
+  // Retrieve routing logic and offsets via 1D Flat Index
+  uint32_t GrpIdx = lut_BnGrpIdx_BG1_R13_Edge[row];
+  uint32_t MsgIdx = lut_BnMsgIdx_BG1_R13_Edge[row] - 1;
+  uint32_t BnIdx = lut_BnIdx_BG1_R13_Edge[row];
+  uint32_t BnToAddrIdx = lut_BnToAddrIdx_BG1_R13[GrpIdx - 1];
+  uint32_t GrpNum = d_lut_numBnInBnGroups_BG1_R13[GrpIdx - 1];
+  uint32_t circShift = bn_cn_map_BG1_Z_R13[row][ZcIdx];
+  const uint32_t baseBn = (BnIdx - 1) * NR_LDPC_ZMAX;
+
+  // Calculate base pointers for the specific Variable Node group
+  const int8_t *p_bnProcBuf_Grp = (const int8_t *)(d_bnProcBuf + baseBn + segIdx * NR_LDPC_SIZE_BN_PROC_BUF + d_lut_startAddrBnGroups_BG1_R13[BnToAddrIdx - 1]);
+  const int8_t *p_cnProcBuf_Grp = (const int8_t *)(d_cnProcBuf + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + bn_cn_map_BG1_Z_R13[row][0]);
+  const int8_t *p_llrProcBuf_Grp = (const int8_t *)(d_llrProcBuf + baseBn + segIdx * NR_LDPC_MAX_NUM_LLR + d_lut_startAddrBnGroupsLlr_BG1_R13[BnToAddrIdx - 1]);
+  const int8_t *p_llrRes_Grp = (const int8_t *)(d_llrRes + baseBn + segIdx * NR_LDPC_MAX_NUM_LLR + d_lut_startAddrBnGroupsLlr_BG1_R13[BnToAddrIdx - 1]);
+
+  bnProcKernel_BG1_int8_Gn_Edge(p_bnProcBuf_Grp, (int8_t *)p_cnProcBuf_Grp, p_llrProcBuf_Grp, (int8_t *)p_llrRes_Grp,
+                                lane, GrpIdx, MsgIdx, BnIdx, GrpNum, circShift, Zc);
+}
+
+```
+
+To prevent 8-bit integer overflow during Bit Node accumulation, the intrinsic values are unpacked and sign-extended to 16-bit integers (`vaddss2`), accumulated, and then saturated and repacked back to 8-bit.
+
+```cpp
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Edge(const int8_t *__restrict__ d_bnProcBuf,
+                                                              int8_t *__restrict__ d_cnProcBuf,
+                                                              const int8_t *__restrict__ d_llrProcBuf,
+                                                              int8_t *__restrict__ d_llrRes,
+                                                              uint32_t lane, uint32_t GrpIdx, uint32_t MsgIdx,
+                                                              uint32_t BnIdx, uint32_t GrpNum, uint32_t circShift, uint32_t Zc)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+  
+  // Load intrinsic LLR (4x int8)
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  // Unpack to 16-bit to prevent overflow during accumulation
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+
+  int i = 0;
+
+  // --- 2-Way Unrolled Accumulation Loop ---
+  for (; i < (int)GrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo); // 16-bit SIMD addition
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  // Handle remaining element if GrpIdx is odd
+  if (i < GrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  // Saturate the 16-bit sums back into a packed 32-bit word (4x int8)
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+
+  uint32_t BricksToBeGet;
+  if (GrpIdx == 1) {
+    BricksToBeGet = packed_intrinsic;
+  } else {
+    // Subtract the CN's own previous message to prevent belief drift
+    uint32_t prevIdxWords = (MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+    uint32_t prev = bnProcBufPtr[prevIdxWords];
+    BricksToBeGet = __vsubss4(saturated_llr, prev); // SIMD4 saturated subtraction
+  }
+
+  // Apply circular shift and write to CN buffer
+  moveBricks_forput_circ(d_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+}
+
+```
+
+During the final decoding iteration, an alternative `_last` kernel is invoked. Instead of subtracting the previous message and writing back to the `cnProcBuf`, it directly outputs the fully accumulated posterior LLRs (`saturated_llr`) into `d_llrRes` for hard decision extraction.
+
+#### 5.1.4 Complexity Bottleneck Analysis
+
+While Edge-Based Parallelism maximizes concurrent thread execution, it achieves this at the cost of massive computational redundancy. By completely isolating each message computation into an independent thread to prevent memory write collisions and expensive synchonization cost across blocks, the hardware is forced to perform highly repetitive arithmetic operations.
+
+In the `bnProc` , the Bit Node(BN) must first accumulate all incoming messages from its connected Check Nodes (CNs), and subsequently subtract the specific target CN's previous message.
+
+Under the Edge-Based strategy, this logic leads to a severe performance bottleneck for high-degree nodes. For instance, consider BN30 in BG1 R13. This single node spawns 30 independent blocks to compute its 30 outgoing messages simultaneously. Because these blocks do not share intermediate accumulation results, *each* of the 30 blocks must independently perform the costly 30-message summation. 
+
+Consequently, a process that mathematically requires only $30$ additions now executes $30 \times 30 = 900$ additions. The computational complexity for processing a node effectively explodes from $O(N)$ to $O(N^2)$, where $N$ represents the node degree.
+
+![BN30_analysis](img/BN30_edge.svg)
+
+As a result, Edge-Based Parallelism performs well for low-throughput scenarios (a small number of segments) due to its high GPU occupancy. However, it hits a compute-bound limit when scaling up to decode a large number of segments. These redundant arithmetic operations easily use up the GPU's ALU resources. 
+
+This limitation imposes us to consider **Node-Based Parallelism**, which  resolves this redundancy and eases the hardware resource burden.
+
+
+### 5.2 Node-Based Parallelism
+
+To fundamentally resolve the computational redundancy identified in the Edge-Based approach, we transition to a **Node-Based Parallelism** strategy. Instead of mapping one CUDA block to one edge (message), we map **one block to one node** (either a Check Node or a Bit Node).
+
+#### 5.2.1 Node-Based Parallelism Explained
+
+The core philosophy of Node-Based parallelism is to calculate the shared intermediate result (the aggregate sum or global minimums) exactly *once* per node, cache it in local registers, and then reuse it to compute all outgoing messages.
+
+![Node-Based_explained](img/Node_based_explained.svg)
+
+As illustrated in the diagram above, consider a node with a degree of $N$:
+* **Edge-Based (Redundant):** Requires $N$ independent blocks. Each block computes the `Msg_Sum` from scratch ($N$ additions) and then performs 1 subtraction. 
+  * Total Operations: $N \times (N + 1) = N^2 + N \rightarrow \mathbf{O(N^2)}$
+* **Node-Based (Optimized):** Uses exactly 1 block. The block computes the `Msg_Sum` once ($N$ additions) in Step 1. In Step 2, it loops through the $N$ edges, performing $N$ subtractions using the cached sum. 
+  * Total Operations: $N + N = 2N \rightarrow \mathbf{O(N)}$
+
+This dramatic reduction in arithmetic complexity from quadratic to linear thoroughly unblocks the GPU ALU bottleneck, especially for high-degree nodes.
+
+#### 5.2.2 Thread Organization
+
+The thread organization follows this updated 2D grid/block topology:
+
+* **X-Dimension (Lifting Size & Nodes):** We continue to utilize 32-bit `simd4` vectorized loads/stores, meaning the block's X-dimension remains scaled to $Z_c / 4$ (`Z >> 2`). However, the Y-dimension of the block is now set to 4 to process **4 nodes** concurrently within a single block. Consequently, the grid's X-dimension represents the total number of Check Nodes or Bit Nodes divided by 4. Ceiling division (`+ 3`) is applied to handle cases where the total number of nodes is not a multiple of 4.
+* **Y-Dimension (Segments):** The grid's Y-dimension continues to map directly to the number of Transport Block segments (`n_segments`), decoding multiple code blocks in parallel.
+
+Because Check Nodes (CN) and Bit Nodes (BN) possess different totals depending on the Base Graph and code rate, their execution grids are launched separately. The precise kernel launch configurations for the supported code rates in BG1 are formulated as follows:
+
+```cpp
+// Node-Based Block Configuration (Universal across all rates and node types)
+// X: Z/4 threads for simd4 access, Y: 4 nodes processed concurrently
+dim3 Node_block(Z >> 2, 4, 1);
+
+```
+
+```cpp
+// BG1 Rate 1/3 configurations
+//num_TotalBlocks_cn_BG1_R13_Node = 46
+dim3 cn_R13_grid((num_TotalBlocks_cn_BG1_R13_Node + 3) >> 2, n_segments, 1);
+//num_TotalBlocks_bn_BG1_R13_Node = 68
+dim3 bn_R13_grid(num_TotalBlocks_bn_BG1_R13_Node >> 2, n_segments, 1); 
+
+```
+
+```cpp
+// BG1 Rate 2/3 configurations
+//num_TotalBlocks_cn_BG1_R23_Node = 13
+dim3 cn_R23_grid((num_TotalBlocks_cn_BG1_R23_Node + 3) >> 2, n_segments, 1);
+//num_TotalBlocks_bn_BG1_R23_Node = 35
+dim3 bn_R23_grid((num_TotalBlocks_bn_BG1_R23_Node + 3) >> 2, n_segments, 1); 
+
+```
+
+```cpp
+// BG1 Rate 8/9 configurations
+//num_TotalBlocks_cn_BG1_R89_Node = 5
+dim3 cn_R89_grid((num_TotalBlocks_cn_BG1_R89_Node + 3) >> 2, n_segments, 1);
+//num_TotalBlocks_bn_BG1_R89_Node = 27
+dim3 bn_R89_grid((num_TotalBlocks_bn_BG1_R89_Node + 3) >> 2, n_segments, 1); 
+
+```
+
+
+#### 5.2.3 cnProc_kernel
+
+For the Check Node processing in Node-Based mode, we implemented specific kernels for each code rate:
+* `cnProcKernel_BG1_R13_int8_Node`
+* `cnProcKernel_BG1_R23_int8_Node`
+* `cnProcKernel_BG1_R89_int8_Node`
+
+Unlike the edge-based dispatcher, the global index `row` now points to a unique **Check Node** rather than a message.
+
+```cpp
+__global__ void cnProcKernel_BG1_R13_int8_Node(const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               uint32_t Zc,
+                                               uint32_t ZcIdx)
+{
+  uint32_t lane = threadIdx.x;
+  uint32_t row = (blockIdx.x << 2) + threadIdx.y;
+  uint32_t segIdx = blockIdx.y;
+
+  if (row >= num_TotalBlocks_cn_BG1_R13_Node)
+    return;
+
+  // 1D Indexing resolves the specific Check Node and its parameters
+  uint32_t CnGrpIdx = lut_CnGrpIdx_BG1_R13_Node[row] - 1;
+  uint32_t CnIdx = lut_CnIdx_BG1_R13_Node[row] - 1;
+  uint32_t InnerOffset = d_lut_startAddrCnGroups_BG1[CnGrpIdx] + NR_LDPC_ZMAX * CnIdx;
+  uint32_t Cn2MsgStartIdx = lut_CnStartMsgIdx_BG1_R13_Node[row];
+  uint32_t CnGrpIdxNum = d_lut_numBnInCnGroups_BG1_R13[CnGrpIdx];
+  uint32_t CnNumInGrp = d_lut_numCnInCnGroups_BG1_R13[CnGrpIdx]; 
+
+  const int8_t *p_cnProcBuf = (const int8_t *)(d_cnBufAll + segIdx * NR_LDPC_SIZE_CN_PROC_BUF + InnerOffset);
+  int8_t *p_bnProcBuf = (int8_t *)(d_bnBufAll + segIdx * NR_LDPC_SIZE_BN_PROC_BUF);
+
+  // Call the consolidated Node-Based device function
+  cnProcKernel_BG1_int8_Gn_R13_node(p_cnProcBuf, p_bnProcBuf, lane, CnIdx, CnNumInGrp, CnGrpIdxNum, Cn2MsgStartIdx, Zc, ZcIdx);
+}
+
+```
+
+The device function executes a classic 2-pass Min-Sum algorithm to achieve $O(N)$ efficiency.
+
+* **Pass 1:** Find the absolute minimum (`min1`), the second minimum (`min2`), and the cumulative XOR sign (`total_xor`) of all connected edges.
+* **Pass 2:** Generate the outgoing messages by applying the appropriate minimum value and sign, then write them back sequentially.
+
+```cpp
+__device__ __forceinline__ void cnProcKernel_BG1_int8_Gn_R13_node(/* ... parameters ... */)
+{
+  uint32_t min1 = 0x7F7F7F7F;
+  uint32_t min2 = 0x7F7F7F7F;
+  uint32_t total_xor = 0;
+
+  // Local thread registers to cache values, avoiding redundant global memory access
+  uint32_t cache_raw[19];
+  uint32_t cache_abs[19];
+
+  const int32_t *cnProcBufPtr = (const int32_t *)(d_cnProcBuf) + lane;
+  const int32_t *currPtr = cnProcBufPtr;
+  uint32_t offset = (CnNumInGrp * NR_LDPC_ZMAX) >> 2;
+
+  // --- Pass 1: Global Extrema and Sign Accumulation ---
+  #pragma unroll
+  for (int MsgIdx = 0; MsgIdx < CnGrpIdxNum; MsgIdx++) {
+    uint32_t val = *currPtr;
+
+    cache_raw[MsgIdx] = val;
+    uint32_t v_abs = __vabs4(val);
+    cache_abs[MsgIdx] = v_abs;
+
+    total_xor = __vxor4(total_xor, val);
+
+    uint32_t old_min1 = min1;
+    min1 = __vminu4(old_min1, v_abs);
+    uint32_t candidate = __vmaxu4(old_min1, v_abs);
+    min2 = __vminu4(min2, candidate);
+    
+    currPtr += offset;
+  }
+
+  // --- Pass 2: Message Generation and Write-Back ---
+  #pragma unroll
+  for (int temp_MsgIdx = 0; temp_MsgIdx < CnGrpIdxNum; temp_MsgIdx++) {
+    uint32_t target_sign = __vxor4(total_xor, cache_raw[temp_MsgIdx]);
+    uint32_t my_abs = cache_abs[temp_MsgIdx];
+
+    // Branchless selection: If this edge was min1, use min2; otherwise use min1
+    uint32_t is_min_mask = __vcmpeq4(my_abs, min1);
+    uint32_t final_mag = (min2 & is_min_mask) | (min1 & ~is_min_mask);
+
+    uint32_t BricksToBeMoved = __vsign4(final_mag, target_sign);
+
+    uint32_t MsgIdx = Cn2MsgStartIdx + temp_MsgIdx;
+    uint32_t circShift = cn_bn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+    int8_t *p_bnProcBuf = (int8_t *)(d_bnProcBuf + cn_bn_map_BG1_Z_R13[MsgIdx][0]);
+
+    moveBricks_invput_circ(p_bnProcBuf, lane * 4, (uint8_t *)&BricksToBeMoved, Zc, circShift);
+  }
+}
+
+```
+
+#### 5.2.4 bnProc_kernel
+
+The Bit Node update (`bnProc`) mirrors this $O(N)$ logic. We define specific node-based kernels for each code rate:
+
+* `bnProcKernel_BG1_R13_int8_Node`
+* `bnProcKernel_BG1_R23_int8_Node`
+* `bnProcKernel_BG1_R89_int8_Node`
+
+*(Note: The global dispatcher `bnProcKernel_BG1_R13_int8_Node` follows the exact same 1D node-indexing structure as the `cnProc` equivalent.)*
+
+In the device function, the 2-pass strategy is implemented as:
+
+* **Pass 1:** Accumulate all incoming edge messages and the intrinsic LLR into a single global sum (`MsgSumLo` / `MsgSumHi`).
+* **Pass 2:** Iterate through the edges, subtract that specific edge's previous message from the global sum (`__vsubss4`), and scatter the updated messages back.
+
+```cpp
+__device__ __forceinline__ void bnProcKernel_BG1_int8_Gn_Node_R13(/* ... parameters ... */)
+{
+  const int32_t *bnProcBufPtr = (const int32_t *)(d_bnProcBuf) + lane;
+  uint32_t packed_intrinsic = ((const int32_t *)(d_llrProcBuf))[lane];
+
+  uint32_t MsgSumLo, MsgSumHi;
+  unpack_and_sign_extend(packed_intrinsic, &MsgSumLo, &MsgSumHi);
+
+  uint32_t off = (GrpNum * NR_LDPC_ZMAX) >> 2;
+  const int32_t *currPtr = bnProcBufPtr;
+  int i = 0;
+
+  // --- Pass 1: 2-Way Unrolled Global Accumulation ---
+  for (; i < (int)BnGrpIdx - 1; i += 2) {
+    uint32_t val1 = *currPtr;
+    uint32_t val2 = *(currPtr + off);
+
+    uint32_t v1_lo, v1_hi;
+    unpack_and_sign_extend(val1, &v1_lo, &v1_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v1_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v1_hi);
+
+    uint32_t v2_lo, v2_hi;
+    unpack_and_sign_extend(val2, &v2_lo, &v2_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v2_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v2_hi);
+
+    currPtr += (off << 1);
+  }
+
+  if (i < BnGrpIdx) {
+    uint32_t val = *currPtr;
+    uint32_t v_lo, v_hi;
+    unpack_and_sign_extend(val, &v_lo, &v_hi);
+    MsgSumLo = __vaddss2(MsgSumLo, v_lo);
+    MsgSumHi = __vaddss2(MsgSumHi, v_hi);
+  }
+
+  // Pack the total sum back into 8-bit integers
+  uint32_t saturated_llr = saturate_and_pack(MsgSumLo, MsgSumHi);
+  uint32_t BricksToBeGet;
+
+  // --- Pass 2: Subtraction and Scatter Write-Back ---
+  if (BnGrpIdx == 1) {
+    // Degree-1 optimization: No subtraction needed
+    BricksToBeGet = packed_intrinsic;
+    uint32_t MsgIdx = Bn2MsgStartIdx;
+    uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+    int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
+
+    moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+  } else {
+    for (int temp_MsgIdx = 0; temp_MsgIdx < BnGrpIdx; temp_MsgIdx++) {
+      uint32_t prevIdxWords = (temp_MsgIdx * GrpNum * NR_LDPC_ZMAX) >> 2;
+      uint32_t prev = bnProcBufPtr[prevIdxWords];
+      
+      // Crucial Step: Subtract the specific edge's previous message
+      BricksToBeGet = __vsubss4(saturated_llr, prev); 
+      
+      uint32_t MsgIdx = Bn2MsgStartIdx + temp_MsgIdx;
+      uint32_t circShift = bn_cn_map_BG1_Z_R13[MsgIdx][ZcIdx];
+      int8_t *p_cnProcBuf = (int8_t *)(d_cnProcBuf + bn_cn_map_BG1_Z_R13[MsgIdx][0]);
+
+      moveBricks_forput_circ(p_cnProcBuf, lane * 4, (uint8_t *)&BricksToBeGet, Zc, circShift);
+    }
+  }
+}
+
+```
+ ### 5.3 Hybrid Approach & Switching Policy
+
+
+#### 5.3.1 Resource vs. Speed Trade-off
+
+The Edge-based approach computes all messages simultaneously, providing optimal latency (e.g., ~50µs) for single-segment workloads. However, its heavy resource consumption quickly saturates GPU Streaming Multiprocessors (SMs) when processing large batches. Conversely, the Node-based approach maps computations to graph nodes rather than edges, shrinking the resource footprint by a factor of the node degree ($N$). This footprint reduction prevents SM contention and delivers superior throughput for larger workloads.
+
+
+#### 5.3.2 Dynamic Switching Policy
+
+To balance latency and throughput, the decoder dynamically switches between Edge-based and Node-based kernels at runtime. This switching policy is determined by the number of code block segments (`n_segments`) being processed.
+
+For small batch sizes (e.g., `n_segments = 1`), the scheduler executes Edge-based kernels to minimize single-segment latency. For larger batch sizes (e.g., `n_segments = 128`), the scheduler switches to Node-based kernels to maintain overall throughput and optimize SM occupancy.
+
+The following code snippets illustrate this runtime switching mechanism. Empirical threshold parameters, such as `NodeEdge_Switch_Cn_R13`, are defined to trigger the transition:
+
+```cpp
+void nrLDPC_cnProc_BG1_R13_cuda_stream_core(int8_t *cnProcBuf,
+                                            int8_t *bnProcBuf,
+                                            uint32_t n_segments,
+                                            uint32_t Z,
+                                            uint32_t ZcIdx,
+                                            cudaStream_t *streams,
+                                            int8_t CudaStreamIdx)
+{
+  if (n_segments > NodeEdge_Switch_Cn_R13) {
+    cnProcKernel_BG1_R13_int8_Node<<<Kdim_cn_R13_Node[CudaStreamIdx].grid,
+                                     Kdim_cn_R13_Node[CudaStreamIdx].block,
+                                     0,
+                                     streams[CudaStreamIdx]>>>(cnProcBuf, bnProcBuf, Z, ZcIdx);
+  } else {
+    cnProcKernel_BG1_R13_int8_Edge<<<Kdim_R13_Edge[CudaStreamIdx].grid,
+                                     Kdim_R13_Edge[CudaStreamIdx].block,
+                                     0,
+                                     streams[CudaStreamIdx]>>>(cnProcBuf, bnProcBuf, Z, ZcIdx);
+  }
+
+  CHECK(cudaGetLastError());
+}
+
+void nrLDPC_bnProc_BG1_R13_cuda_stream_core(int8_t *bnProcBuf,
+                                            int8_t *cnProcBuf,
+                                            int8_t *llrProcBuf,
+                                            int8_t *llrRes,
+                                            uint32_t n_segments,
+                                            uint32_t Z,
+                                            uint32_t ZcIdx,
+                                            cudaStream_t *streams,
+                                            int8_t CudaStreamIdx)
+{
+  if (n_segments > NodeEdge_Switch_Bn_R13) {
+    bnProcKernel_BG1_R13_int8_Node<<<Kdim_bn_R13_Node[CudaStreamIdx].grid,
+                                     Kdim_bn_R13_Node[CudaStreamIdx].block,
+                                     0,
+                                     streams[CudaStreamIdx]>>>(bnProcBuf, cnProcBuf, llrProcBuf, llrRes, Z, ZcIdx);
+  } else {
+    bnProcKernel_BG1_R13_int8_Edge<<<Kdim_R13_Edge[CudaStreamIdx].grid,
+                                     Kdim_R13_Edge[CudaStreamIdx].block,
+                                     0,
+                                     streams[CudaStreamIdx]>>>(bnProcBuf, cnProcBuf, llrProcBuf, llrRes, Z, ZcIdx);
+  }
+  
+  CHECK(cudaGetLastError());
+}
+
+```
+It should be noted that the empirical switching thresholds obtained so far (e.g., NodeEdge_Switch_Cn_R13) were derived from actual performance testing on the NVIDIA GH200(See [Appendix A](#Appendix-A)). For different hardware platforms, these optimal thresholds are expected to vary depending on the available computational resources. A practical heuristic is to scale these thresholds proportionally based on the target hardware's compute resources relative to the GH200 (such as the number of Streaming Multiprocessors). Future work will explore a more precise, analytical method for determining these thresholds across diverse architectures.
+
+## 6. System Integration and Performance Evaluation
+
+#### 6.1 Pipeline Scheduling & Execution Flow
+
+Beyond the arithmetic optimizations at the kernel level, the overall decoder latency is heavily influenced by CPU-side driver overhead, specifically the cost of launching hundreds of iterative CUDA kernels per Transport Block. To mitigate this, the decoder integrates a dynamic pipeline scheduling architecture centered around **CUDA Graphs**.
+
+The execution lifecycle of a batch of incoming code blocks (`ldpc_decoder_cuda.c`) is governed by the following pipeline stages:
+
+**1. Driver Initialization and Warm-up**
+To prevent unpredictable latency in processing the first arriving network packets (often caused by lazy loading or JIT compilation within the NVIDIA driver), we perform a warm-up procedure(`init_decoder_warmup`). During init, the decoder allocates dummy buffers and pre-records CUDA graphs for common 5G NR configurations (e.g., Lifting Sizes $Z \in \{320, 352, 384\}$ and Base Graph 1, Rates 1/3 and 2/3). Executing these dummy graphs forces the GPU driver to fully initialize its execution context before real network traffic arrives.
+
+**2. Dynamic Graph Caching and Execution**
+The core scheduling logic (`nrLDPC_decoder_core_dynamic`) bypasses standard stream-based kernel launches by utilizing a state-aware graph cache (`gpu_graph_cache`). When a decoding request arrives, the scheduler inspects the decoding parameters (e.g., $Z$, Code Rate, `n_segments`, maximum iterations) and routes the execution through one of three paths:
+
+* **Cache Hit (Fast Path):** If an existing graph matches the current configuration, the CPU overhead is minimized to an $O(1)$ cache lookup. The system updates the input/output buffer pointers via a memory bridge (`ldpc_cuda_bridge_t`) and directly launches the pre-recorded graph (`nrLDPC_decoder_cuda_GraphExecute`).
+* **Cache Miss (Graph Record):** If no matching graph exists and the cache is not full, the system records a new CUDA Graph (`nrLDPC_decoder_cuda_GraphRecord`). This operation captures the exact sequence of kernels—including the Edge/Node hybrid switching policy defined in Section 5.3—into a single executable topology. Once recorded, the graph is cached and executed.
+* **Normal Execution (Fallback):** In scenarios where graph operations fail, the cache capacity is exhausted, or the runtime environment actively breaks the graph logic (circuit breaker triggered), the scheduler safely falls back to standard, sequential CUDA kernel launches (`nrLDPC_decoder_cuda_NormalExecute`).
+
+**3. Memory Management** 
+
+In a fully GPU-accelerated L1 pipeline, the input LLRs from the preceding rate-matching module typically already reside in the GPU device memory. The decoder accepts these device pointers directly, avoiding PCIe transfer overhead.
+
+To maintain API compatibility for partial-offload testing scenarios where inputs might originate from the host, the pipeline checks the memory space of the incoming pointer (`if (p_llr != p_llr_dev)`). It performs an asynchronous PCIe copy (`cudaMemcpyAsync`) only if a host pointer is detected. Additionally, for integrated architectures like the NVIDIA GH200, the implementation supports zero-copy memory access, mapping pageable or integrated host pointers directly without explicit data migration.
+
+
+
+
+### 6.2 Throughput Analysis
+
+#### 6.2.1 Error Correction Performance (BLER Validation)
+
+To guarantee that the architectural transformations—including 8-bit saturation, SIMD4 vectorization, and Node-Based graph restructuring—do not compromise the mathematical integrity of the decoder, we benchmarked the Block Error Rate (BLER) against the standard OAI CPU baseline.
+
+The BLER vs. SNR curves indicate that the CUDA GPU implementation performs consistently with the CPU baseline. Across the evaluated code rates (BG1 Rate 1/3, Rate 2/3, and Rate 8/9) in an AWGN channel, the decoding trajectories of the GPU and CPU implementations overlap. This demonstrates that the proposed GPU offloading strategies do not compromise the error correction performance.
+
+![CPU_GPU_BLER](img/bler_performance_cpu_vs_gpu.svg)
+
+*(Tested with 5 iterations)*
+
+#### 6.2.2 Latency vs. Throughput Scalability
+
+
+Following the validation of decoding accuracy, the hardware acceleration performance was evaluated on the NVIDIA GH200 architecture. The primary performance metrics include single-segment processing latency and overall decoding throughput, measured across varying workloads (number of code block segments, `n_segments`).
+
+The measurements illustrate the scalability facilitated by the Dynamic Hybrid Switching policy detailed in Section 5.3. By calibrating the `NodeEdge_Switch` thresholds according to the Streaming Multiprocessor (SM) capacity of the GH200, the scheduler enables a transition across different operational regimes.
+
+**Low-Latency Regime (Edge-Based Processing)**
+For smaller batch sizes (e.g., `n_segments = 1` to `4`), the scheduler selects the Edge-Based kernels. In this regime, the concurrent thread deployment reduces the absolute decoding time. The implementation records a single-segment decoding latency ranging from approximately **50 µs** to **61 µs** (measured at 5 iterations). This processing time aligns with the latency requirements associated with 5G URLLC (Ultra-Reliable Low-Latency Communication) scenarios.
+
+**High-Throughput Regime (Node-Based Processing)**
+As the workload scales to simulate higher-load eMBB (Enhanced Mobile Broadband) base station conditions, the scheduler transitions to the Node-Based kernels. This transition mitigates SM saturation by reducing the algorithmic complexity from $O(N^2)$ to $O(N)$.
+
+With the switching thresholds calibrated to the hardware characteristics, the throughput curve exhibits an initial ascent before plateauing near the hardware limit. Under maximum evaluated parallel segment loading (`n_segments = 128`), the GPU decoder reaches peak sustained throughputs of **~3.5 Gbps** for Rate 1/3, **~5.5 Gbps** for Rate 2/3, and **>8.0 Gbps** for Rate 8/9.
+
+
+The table below summarizes the measured throughput and latency metrics across different segment batch sizes on the GH200:
+
+<details>
+<summary><b>Click to expand: Raw Performance Metrics (GH200)</b></summary>
+
+| `n_segments` | R13 Latency (µs) | R13 Tput (Gbps) | R23 Latency (µs) | R23 Tput (Gbps) | R89 Latency (µs) | R89 Tput (Gbps) |
+| --- | --- | --- | --- | --- | --- | --- |
+| **1** | 61.66 | 0.137 | 54.04 | 0.156 | 49.79 | 0.170 |
+| **2** | 69.98 | 0.241 | 54.62 | 0.309 | 51.45 | 0.328 |
+| **4** | 78.24 | 0.432 | 58.65 | 0.576 | 55.17 | 0.612 |
+| **8** | 91.52 | 0.738 | 68.89 | 0.981 | 56.89 | 1.188 |
+| **16** | 116.76 | 1.157 | 81.95 | 1.649 | 72.64 | 1.861 |
+| **32** | 157.88 | 1.712 | 105.75 | 2.556 | 81.60 | 3.313 |
+| **64** | 197.88 | 2.732 | 124.73 | 4.335 | 106.36 | 5.083 |
+| **128** | 303.47 | 3.563 | 195.77 | 5.523 | 134.42 | 8.044 |
+
+*(Note: Throughput calculations are based on $K=8448$ information bits per segment. Values are measured at 5 decoding iterations.)*
+
+</details>
+
+---
+
+![latency](img/latency_vs_segments.svg)
+
+![throughput](img/throughput_vs_segments.svg)
+
+
+---
+<a id="Appendix-A"></a>
+## Appendix A: Node-Edge Performance Profiling on GH200
+
+To validate the bottleneck shift from kernel launch overhead to SM computational saturation, an exhaustive profiling was conducted on the NVIDIA Grace Hopper (GH200) platform.
+
+The architectures evaluated are defined as follows:
+
+* **EE (Baseline)**: Edge-based `cnProc` + Edge-based `bnProc`.
+* **EN (Hybrid A)**: Edge-based `cnProc` + Node-based `bnProc`. 
+* **NE (Hybrid B)**: Node-based `cnProc` + Edge-based `bnProc`. 
+* **NN (Proposed)**: Node-based `cnProc` + Node-based `bnProc`.
+
+The tables below record the per-segment execution time ($\mu s$) under varying workload scales ($S$).
+
+### A.1 Performance Comparison for Rate 1/3 (BG1)
+
+| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
+| --- | --- | --- | --- | --- | --- |
+| **1** | 52.055 | 76.164 | 75.641 | 98.361 | **EE** |
+| **2** | 30.724 | 39.726 | 40.552 | 50.658 | **EE** |
+| **4** | 17.457 | 20.873 | 22.259 | 25.478 | **EE** |
+| **8** | 10.815 | 11.664 | 12.725 | 13.481 | **EE** |
+| **16** | 8.038 | 7.080 | 8.468 | 7.485 | **EN** |
+| **32** | 6.204 | 4.824 | 6.112 | 4.713 | **NN** |
+| **64** | 5.251 | 3.461 | 4.884 | 3.100 | **NN** |
+| **128** | 4.793 | 2.905 | 4.342 | 2.423 | **NN** |
+
+### A.2 Performance Comparison for Rate 2/3 (BG1)
+
+| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
+| --- | --- | --- | --- | --- | --- |
+| **1** | 46.088 | 54.576 | 67.666 | 74.247 | **EE** |
+| **2** | 23.429 | 27.522 | 35.105 | 38.476 | **EE** |
+| **4** | 12.763 | 14.673 | 18.467 | 20.560 | **EE** |
+| **8** | 7.611 | 8.030 | 9.977 | 10.521 | **EN** |
+| **16** | 4.791 | 4.721 | 5.588 | 5.491 | **EN** |
+| **32** | 3.416 | 3.115 | 3.416 | 3.073 | **NN** |
+| **64** | 2.670 | 2.267 | 2.375 | 1.953 | **NN** |
+| **128** | 2.297 | 1.867 | 2.061 | 1.597 | **NN** |
+
+### A.3 Performance Comparison for Rate 8/9 (BG1)
+
+| Segments ($S$) | EE (Baseline) | EN (Hybrid A) | NE (Hybrid B) | NN (All Node) | Best Arch. |
+| --- | --- | --- | --- | --- | --- |
+| **1** | 44.686 | 48.255 | 68.518 | 70.156 | **EE** |
+| **2** | 21.895 | 24.895 | 35.236 | 36.256 | **EE** |
+| **4** | 11.493 | 13.221 | 17.876 | 19.112 | **EE** |
+| **8** | 6.551 | 7.100 | 9.245 | 9.867 | **EE** |
+| **16** | 3.824 | 4.142 | 5.123 | 5.257 | **EE** |
+| **32** | 2.604 | 2.554 | 2.928 | 2.842 | **EN** |
+| **64** | 1.896 | 1.751 | 1.759 | 1.646 | **NN** |
+| **128** | 1.464 | 1.349 | 1.213 | 1.084 | **NN** |
+
+---
+**Analysis summary:** Compared to the baseline (EE), the EN architecture eases the computational burden of `bnProc`, and the NE architecture eases the burden of `cnProc`. As the workload ($S$) scales up, the performance improvement from EN is greater than that of NE. This indicates that the Variable Node processing (`bnProc`) acts as the main parallelization bottleneck in the traditional Edge-centric paradigm. Therefore, adopting the fully Node-centric architecture (NN) addresses both bottlenecks, providing better performance in high-load scenarios.
+
+
+![R13 Performance](img/perf_r13.svg)
+
+![R23 Performance](img/perf_r23.svg)
+
+![R89 Performance](img/perf_r89.svg)
+
+
+## Acknowledgments
+
+* **Main Developer:** PAN Qizhi (GPU Architecture & CUDA Implementation & Optimization)
+* **System Integration:** Raymond Knopp (E2E Integration & Validation & Optimization)
+* **Special Thanks:** To **Jingguo Zhao (BUPT)** for the insightful discussions on node-based baseline architectures which inspired the optimizations in this work.

openair1/PHY/CODING/nrLDPC_decoder/doc/nrLDPC/nrLDPC_cuda_tutorial.md

@@ -0,0 +1,190 @@
+# OAI nrLDPC GPU Decoder Tutorial
+For the detailed mathematical modeling and memory layout of this GPU implementation, please refer to the [Architecture Document](nrLDPC_cuda.md).
+
+## Supported Configurations
+
+| Parameter | CPU Baseline  | GPU Implementation |
+| --- | --- | --- |
+| **Base Graph (BG)** | 1, 2 | **1** |
+| **Lifting Size ($Z_c$)** | All  | **$\ge 128$** |
+| **Code Rate (R)** | BG1: 1/3, 2/3, 8/9 <br><br> BG2: 1/5, 1/3, 2/3 | **BG1: 1/3, 2/3, 8/9** |
+
+
+*Table of content*
+
+[[_TOC_]]
+
+## Prerequisites
+
+### Tested System Configuration
+
+The GPU decoder has been successfully compiled and verified using the following baseline hardware and software environment:
+
+* **GPU:** NVIDIA GH200 480GB
+* **NVIDIA Driver:** 575.57.08
+* **CUDA Toolkit:** 12.9
+* **CMake:** 3.22.1
+* **GCC (C/C++ Compiler):** 12.3.0
+
+**Other Validated Platforms**
+Beyond the Grace Hopper architecture, the implementation has also been successfully validated across a range of NVIDIA ecosystems, including:
+
+* NVIDIA L40S 
+* NVIDIA DGX Spark(GB10)
+* NVIDIA Jetson
+* NVIDIA A100
+
+### Monitoring & Profiling
+
+To examine runtime details, hardware utilization, and kernel execution efficiency, we recommend using the following official NVIDIA developer tools:
+
+* **`nvidia-smi`**: Use this for real-time monitoring of VRAM usage, overall GPU utilization, and power consumption during the end-to-end tests.
+* **`nsys` (Nsight Systems)**: Use this for system-wide performance analysis. It is highly effective for observing the CUDA Graph execution timeline and CPU-GPU pipeline scheduling behaviors.
+* **`ncu` (Nsight Compute)**: Use this for deep, kernel-level profiling. It helps verify SM occupancy, shared memory usage, and the execution efficiency of the SIMD4 vectorized PTX instructions.
+
+
+
+## Unit Test
+
+### Build
+
+Use the `build_oai` script to compile with CUDA support. If you have previously built a CPU-only version, ensure you clean the build directory or verify the dependencies:
+
+```bash
+./build_oai --ninja --phy_simulators --build-lib ldpc_cuda
+```
+```bash
+cd ran_build/build/
+```
+
+### ldpctest
+
+`ldpctest` is the foundational tool for evaluating the performance and correctness of the LDPC encoder/decoder.
+
+**Basic Parameters:**
+* `-r`: Nominator rate. Supported values: 1, 2, 22. (Default: 1)
+* `-d`: Denominator rate. Supported values: 3, 5, 25. (Default: 3)
+* `-n`: Number of trials (e.g., `-n300` runs 300 trials).
+* `-S`: Number of segments (determines the workload size).
+* `-s`: Starting SNR in dBm.
+* `-i`: Maximum number of iterations. *(Note: Due to internal index offset implementation, the parameter `-i4` executes 5 decoding iterations.)*
+* `-G`: **Enable GPU decoder**. (Omitting this parameter defaults to the CPU baseline).
+
+#### Latency & Baseline Test
+
+To compare the single-segment processing capability between CPU and GPU, set the segments parameter to 1 (`-S1`).
+
+**BG1 Rate 1/3:**
+
+* CPU: `./ldpctest -r1 -n300 -S1 -s4 -i4`
+* GPU: `./ldpctest -r1 -n300 -S1 -s4 -i4 -G`
+
+**BG1 Rate 2/3:**
+
+* CPU: `./ldpctest -r2 -n300 -S1 -s4 -i4`
+* GPU: `./ldpctest -r2 -n300 -S1 -s4 -i4 -G`
+
+**BG1 Rate 8/9:**
+
+* CPU: `./ldpctest -r22 -d25 -n300 -S1 -s4 -i4`
+* GPU: `./ldpctest -r22 -d25 -n300 -S1 -s4 -i4 -G`
+
+#### Maximum Throughput Test
+
+To evaluate the maximum throughput of the GPU, increase the `-S` parameter to add parallel decoding workloads. The current GPU version in `ldpctest` supports a maximum of **128** segments.
+
+Run the following commands to benchmark the maximum GPU throughput:
+
+```bash
+# BG1 R13 Max Throughput
+./ldpctest -r1 -n30 -S128 -s4 -i4 -G
+```
+```bash
+# BG1 R23 Max Throughput
+./ldpctest -r2 -n30 -S128 -s4 -i4 -G
+```
+```bash
+# BG1 R89 Max Throughput
+./ldpctest -r22 -d25 -n30 -S128 -s4 -i4 -G
+```
+
+### Downlink Simulator (dlsim)
+
+`nr_dlsim` is used to simulate the physical downlink shared channel (PDSCH). To offload LDPC decoding to the GPU, you must specify the CUDA shared library and enable the GPU flag.
+
+#### Mandatory Flags for GPU Acceleration
+
+* `--loader.ldpc.shlibversion _cuda`: Directs the dynamic loader to use the CUDA-accelerated LDPC library.
+* `-Q` : Enables the GPU decoding path within the simulator logic.
+
+#### Workload & Segment Calculation
+
+Unlike `ldpctest` where you can manually set the number of segments, in `dlsim`, the number of segments (Code Blocks) is determined dynamically by the Transport Block Size (TBS). The TBS is defined by a combination of the MCS index (`-e`), the number of allocated PRBs (`-R`, `-b`), and the number of MIMO layers (`-x`). To specify the iterations, use `-I`.
+
+Understanding the resulting segment count is crucial for evaluating GPU performance, as higher segment counts better utilize the GPU's parallel processing capabilities.
+
+#### Execution Examples
+
+**1. Target: ~ 9 Segments (Rate 1/3)**
+Using MCS 13 with 273 PRBs and a single layer generates a workload of approximately 9 segments per slot.
+
+```bash
+./nr_dlsim -n100 -s20 -e13 -R273 -b273 -x1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+**2. Target: ~ 18 Segments (Rate 2/3)**
+Increasing the MCS to 22 while keeping 273 PRBs and 1 layer doubles the throughput requirement, resulting in about 18 segments.
+
+```bash
+./nr_dlsim -n100 -s20 -e22 -R273 -b273 -x1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+**3. Target: ~ 50 Segments (Rate 8/9, High Workload)**
+Using MCS 27 combined with 2x2 MIMO (`-x2 -y2 -z2`) forces a massive Transport Block Size, pushing the workload to approximately 50 segments. This scenario highly benefits from the Node-Based GPU architecture.
+
+```bash
+./nr_dlsim -n100 -s40 -e27 -R273 -b273 -z2 -x2 -y2 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+*(Note: The `-P` flag enables performance printing, allowing you to observe the decoding time per slot)*
+
+
+
+### Uplink Simulator (ulsim)
+
+`nr_ulsim` is used to simulate the physical uplink shared channel (PUSCH). The GPU acceleration logic follows the same architectural pattern as `dlsim`.
+
+> **Parameter Mapping Notice:**
+> When transitioning from `dlsim` to `ulsim`, the command-line arguments for identical physical layer parameters are different:
+> * **MCS Index**: Use `-m` (instead of `-e`).
+> * **Allocated PRBs**: Use `-r` (instead of `-b`).
+> * **MIMO Layers**: Use `-W` (instead of `-x`).
+
+To properly observe the GPU performance, we use the `-P` flag to print the execution statistics. After running a test, locate the `ULSCH segments decoding time` field in the `gNB RX` section of the output log. This metric isolates the time spent executing the LDPC decoding iterations.
+
+#### Execution Examples
+
+**1. Target: ~ 9 Segments (Rate 1/3)**
+Using MCS 13 with 273 PRBs and a single layer.
+
+```bash
+./nr_ulsim -n100 -s20 -m13 -R273 -r273 -W1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+**2. Target: ~ 18 Segments (Rate 2/3)**
+Increasing to MCS 22, maintaining 273 PRBs and a single layer.
+
+```bash
+./nr_ulsim -n100 -s20 -m22 -R273 -r273 -W1 -y1 -z1 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+**3. Target: ~ 50 Segments (Rate 8/9, Massive Workload)**
+Using MCS 27 with 273 PRBs and 2x2 MIMO (`-W2 -y2 -z2`).
+```bash
+./nr_ulsim -n100 -s40 -m27 -R273 -r273 -W2 -y2 -z2 -P -Q --loader.ldpc.shlibversion _cuda
+```
+
+## E2E Test
+
+To be continued :）
+

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h

@@ -68,6 +68,8 @@ typedef struct nrLDPC_dec_params {
     e_nrLDPC_outMode outMode; /**< Output format */
     int crc_type; /**< Size and type of the parity check bits (16, 24A or 24B) */
     int (*check_crc)(uint8_t* decoded_bytes, uint32_t n, uint8_t crc_type); /**< Parity check function */
+    int n_segments;/**Infomation for cuda streaming*/
+    int LastTrial;/**Infomation for cuda streaming to create and destroy*/
 } t_nrLDPC_dec_params;
 
 /**

openair1/PHY/CODING/nrLDPC_defs.h

@@ -60,6 +60,7 @@ typedef struct {
 } encoder_implemparams_t;
 
 typedef int32_t(LDPC_initfunc_t)(void);
+
 typedef int32_t(LDPC_shutdownfunc_t)(void);
 
 // decoder interface

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c

@@ -35,11 +35,10 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
   int no_punctured_columns, removed_bit;
   int nind=0;
   int indlist[1000];
-  int indlist2[1000];
 
   const short *Gen_shift_values = choose_generator_matrix(BG, Zc);
   if (Gen_shift_values==NULL) {
-    printf("ldpc_encoder_orig: could not find generator matrix\n");
+    printf("ldpc_encoder_orig: could not find generator matrix BG %d Zc %d\n",BG,Zc);
     return(-1);
   }
 
@@ -70,32 +69,34 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
 
   // parity check part
 
-  if (gen_code>=1)
+  if (gen_code==1 || gen_code==2 || gen_code==3)
   {
     char fname[100];
     sprintf(fname,"ldpc_BG%d_Zc%d_byte.c",BG,Zc);
     FILE *fd=fopen(fname,"w");
     AssertFatal(fd!=NULL,"cannot open %s\n",fname);
-    sprintf(fname,"ldpc_BG%d_Zc%d_16bit.c",BG,Zc);
-    FILE *fd2=fopen(fname,"w");
-    AssertFatal(fd2!=NULL,"cannot open %s\n",fname);
+
 
     int shift;
     char data_type[100];
     char xor_command[100];
     int mask;
-
-
+    char permutex_command[100];
+    int use_permutex=0;
+    char alignr_command[100];
+    int use_alignr=0;
 
 
     fprintf(fd,"#include \"PHY/sse_intrin.h\"\n");
-    fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n");
 
     if (gen_code == 1 && (Zc&63)==0) {
       shift=6;
       mask=63;
       strcpy(data_type,"__m512i");
       strcpy(xor_command,"_mm512_xor_si512");
+      strcpy(permutex_command,"_mm512_permutex2var_epi8");
+      strcpy(alignr_command,"_mm512_alignr_epi8");
+      use_alignr=1;
     }
     else if (gen_code == 1 && (Zc&31)==0) {
       shift=5; // AVX2 - 256-bit SIMD
@@ -103,14 +104,15 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
       strcpy(data_type,"simde__m256i");
       strcpy(xor_command,"simde_mm256_xor_si256");
     }
-    else if ((Zc&15)==0) {
+    else if ((gen_code <=2) && (Zc&15)==0) {
       shift=4; // SSE4 - 128-bit SIMD
       mask=15;
       strcpy(data_type,"simde__m128i");
       strcpy(xor_command,"simde_mm_xor_si128");
-
+      strcpy(alignr_command,"simde_mm_alignr_epi8");
+      use_alignr=1;
     }
-    else if ((Zc&7)==0) {
+    else if ((gen_code <=2) && (Zc&7)==0) {
       shift=3; // MMX  - 64-bit SIMD
       mask=7;
       strcpy(data_type,"simde__m64");
@@ -122,21 +124,30 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
       strcpy(data_type,"uint8_t");
       strcpy(xor_command,"scalar_xor");
       fprintf(fd,"#define scalar_xor(a,b) ((a)^(b))\n");
-      fprintf(fd2,"#define scalar_xor(a,b) ((a)^(b))\n");
+
     }
     fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
-    fprintf(fd2,"// generated code for Zc=%d, 16bit encoding\n",Zc);
+    if (use_permutex==1) {
+	 fprintf(fd,"  const uint8_t abshift_bytes[%d][%d] __attribute__((aligned(64)))= {\n",1+mask,1+mask); 
+         for (int i=0;i<=mask;i++) {
+	    fprintf(fd,"{");
+	    for (int j=0;j<=mask;j++) {
+              fprintf(fd,"%d",i+j);
+	      if (j<mask) fprintf(fd,",");
+	      else fprintf(fd,"}");
+	    }
+	    if (i<mask) fprintf(fd,",\n");
+	    else fprintf(fd,"};\n");
+      	}
+	fprintf(fd,"static inline __m512i abshift_get(int r)\n {\n");
+        fprintf(fd,"   return * (const __m512i *) abshift_bytes[r];\n}");
+    }
     fprintf(fd,"static inline void ldpc_BG%d_Zc%d_byte(uint8_t *c,uint8_t *d) {\n",BG,Zc);
-    fprintf(fd2,"static inline void ldpc_BG%d_Zc%d_16bit(uint16_t *c,uint16_t *d) {\n",BG,Zc);
     fprintf(fd,"  %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
-    fprintf(fd2,"  %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
     fprintf(fd,"  %s *c2,*d2;\n\n",data_type);
-    fprintf(fd2,"  %s *c2,*d2;\n\n",data_type);
     fprintf(fd,"  int i2;\n");
-    fprintf(fd2,"  int i2;\n");
     fprintf(fd,"  for (i2=0; i2<%d; i2++) {\n",Zc>>shift);
-    if (shift > 0)
-      fprintf(fd2,"  for (i2=0; i2<%d; i2++) {\n",Zc>>(shift-1));
+
     for (i2=0; i2 < 1; i2++)
     {
       //t=Kb*Zc+i2;
@@ -146,53 +157,122 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
 
       fprintf(fd,"     c2=&csimd[i2];\n");
       fprintf(fd,"     d2=&dsimd[i2];\n");
-      fprintf(fd2,"     c2=&csimd[i2];\n");
-      fprintf(fd2,"     d2=&dsimd[i2];\n");
 
       for (i1 = 0; i1 < nrows; i1++)
-
       {
         fprintf(fd,"\n//row: %d\n",i1);
-        fprintf(fd2,"\n//row: %d\n",i1);
-        AssertFatal(shift > 0 , "The result of the right shift is undefined because the right operand is negative\n");
+        AssertFatal(shift >= 0 , "The result of the right shift is undefined because the right operand is negative\n");
 	fprintf(fd,"     d2[%d]=",(Zc*i1)>>shift);
-	fprintf(fd2,"     d2[%d]=",(Zc*i1)>>(shift-1));
 
         nind=0;
 
         for (i3=0; i3 < ncols; i3++)
         {
           temp_prime=i1 * ncols + i3;
-
-
 	  for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
-	    {
-	          
+  	  {
 	      var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
 	      int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
 	      printf("var %d, i3 %d, i4 %d, index %d, shift %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d\n",var,i3,i4,index,shift,Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]]);
-	      indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
-	      printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
-	      indlist2[nind++] = ((index&(mask>>1))*((2*Zc*ncols)>>(shift-1))*Kb)+(index>>(shift-1));
-	    }
-	  
-
-        }
+   	      printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
+	      if (use_permutex==1 || use_alignr==1) {
+                indlist[nind++] = index;
+	      }
+	      else {
+	        indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
+	      }
+	  }//i4
+        }//i3
 	for (i4=0;i4<nind-1;i4++) {
-	  fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
-	  fprintf(fd2,"%s(c2[%d],",xor_command,indlist2[i4]);
+           if (use_permutex==0 && use_alignr==0) {
+  	      fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
+	   }
+	   else {
+              if ((indlist[i4]&mask) == 0) {
+  	        fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]>>shift);
+	      }
+	      else if (use_permutex==1) {
+                fprintf(fd,"%s(%s(c2[%d],abshift_get(%d),c2[%d]),",xor_command,permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
+	      }
+	      else if (use_alignr==1) {
+                fprintf(fd,"%s(%s(c2[%d],c2[%d],%d),",xor_command,alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
+	      }
+	   }
+	} //i4
+	if (use_permutex==0 && use_alignr==0)
+	  fprintf(fd,"c2[%d]",indlist[i4]);
+	else {
+          if ((indlist[i4]&mask) == 0) {
+  	     fprintf(fd,"c2[%d]",indlist[i4]>>shift);
+	  }
+	  else if (use_permutex==1){
+             fprintf(fd,"%s(c2[%d],abshift_get(%d),c2[%d])",permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
+	  }
+	  else if (use_alignr==1) {
+             fprintf(fd,"%s(c2[%d],c2[%d],%d)",alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
+
+	  }
 	}
-	fprintf(fd,"c2[%d]",indlist[i4]);
-	fprintf(fd2,"c2[%d]",indlist2[i4]);
-	for (i4=0;i4<nind-1;i4++) { fprintf(fd,")"); fprintf(fd2,")"); }
-	fprintf(fd,";\n");
-  fprintf(fd2, ";\n");
-      }
-      fprintf(fd,"  }\n}\n");
-      fprintf(fd2,"  }\n}\n");
-    }
+	for (i4=0;i4<nind-1;i4++) fprintf(fd,")"); 
+  	fprintf(fd,";\n");
+       } // i1 
+       fprintf(fd,"  }\n}\n");
+    } // i2
+    fclose(fd);
+  }
+  else if (gen_code == 4) { // CUDA
+    char fname[100];
+    sprintf(fname,"ldpc_BG%d_Zc%d_32bit.cu",BG,Zc);
+    FILE *fd=fopen(fname,"w");
+    AssertFatal(fd!=NULL,"cannot open %s\n",fname);
+    printf("Writing to %s\n",fname);
+    fprintf(fd,"#include <stdio.h>\n#include <stdint.h>\n#include <cuda_runtime.h>\n");
+
+    fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
+    fprintf(fd,"__global__ void ldpc_BG%d_Zc%d_worker(uint32_t *c[4],uint32_t *d[4]) {\n",BG,Zc);
+    fprintf(fd,"  uint32_t *c32=c[blockIdx.x];\n  uint32_t *d32=d[blockIdx.x];\n\n");
+    fprintf(fd,"  int i2 = threadIdx.x;\n");
+    fprintf(fd,"  int i1 = blockIdx.y;\n");
+    fprintf(fd,"  if (i2 < %d) {\n",Zc);
+    fprintf(fd,"    c32+=i2;\n");
+    fprintf(fd,"    d32+=i2;\n");
+    fprintf(fd,"    switch(i1) {\n");
+
+    for (int i1=0;i1<nrows;i1++) {
+      nind = 0;
+      fprintf(fd,"    case %d:\n",i1);
+      fprintf(fd,"      d32[%d]=",(Zc*i1));
+      for (i3=0; i3 < ncols; i3++)
+      {
+          temp_prime=i1 * ncols + i3;
+	  for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
+  	  {
+	     var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
+  	     int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
+	     printf("var %d, i3 %d, i4 %d, index %d (index mod 2Zc) %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d offset %d\n",var,i3,i4,index,index%(2*Zc),Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]],(i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc);
+	     if (index%(2*Zc) >= Zc) printf("***********************\n");
+   	     indlist[nind] = index;
+	     printf("indlist[%d] %d, index %d\n",nind,indlist[nind],index);
+	     nind++;
+   	  } //i4
+      } // i3
+      for (i4=0;i4<nind-1;i4++) {
+         fprintf(fd,"c32[%d]^",indlist[i4]);
+      } //i4
+      fprintf(fd,"c32[%d];\n\n",indlist[i4]);
+      fprintf(fd,"       break;\n");
+    }// i1
+    fprintf(fd,"     }\n");
+    fprintf(fd,"  }\n");
+    fprintf(fd,"}\n");
+
+    fprintf(fd,"extern \"C\" int ldpc_BG%d_Zc%d_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs) { \n",BG,Zc);
+    fprintf(fd,"dim3 numblocks(n_inputs,%d);\n",nrows);
+    fprintf(fd,"ldpc_BG%d_Zc%d_worker<<<numblocks,%d>>>(c,d);\n",BG,Zc,Zc);
+    fprintf(fd," cudaDeviceSynchronize();\n");
+    fprintf(fd,"  return(0);\n");
+    fprintf(fd,"}\n");
     fclose(fd);
-    fclose(fd2);
   }
   else if(gen_code==0)
   {

openair1/PHY/CODING/nrLDPC_load.c

@@ -36,6 +36,7 @@ int load_LDPClib(char *version, ldpc_interface_t *itf)
   int ret;
   ret = load_module_version_shlib(libname, version, shlib_fdesc, sizeofArray(shlib_fdesc), NULL);
   AssertFatal((ret >= 0), "Error loading ldpc decoder");
+  LOG_I(NR_PHY,"loading LDPC version %s\n",version);
   itf->LDPCinit = (LDPC_initfunc_t *)shlib_fdesc[0].fptr;
   itf->LDPCshutdown = (LDPC_shutdownfunc_t *)shlib_fdesc[1].fptr;
   itf->LDPCdecoder = (LDPC_decoderfunc_t *)shlib_fdesc[2].fptr;

openair1/PHY/INIT/nr_init.c

@@ -26,6 +26,10 @@
 #include <string.h>
 #include "nfapi/open-nFAPI/fapi/inc/nr_fapi_p5_utils.h"
 
+#ifdef LDPC_CUDA
+#include <cuda_runtime.h>
+#endif
+
 static void init_DLSCH_struct(PHY_VARS_gNB *gNB);
 static void destroy_DLSCH_struct(const PHY_VARS_gNB *gNB);
 
@@ -189,8 +193,14 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
 
     for (int i = 0; i < max_ul_mimo_layers; i++) {
     }
-    pusch->llr = (int16_t *)malloc16_clear((8 * ((3 * 8 * 6144) + 12))
-                                           * sizeof(int16_t)); // [hna] 6144 is LTE and (8*((3*8*6144)+12)) is not clear
+#ifdef LDPC_CUDA
+    cudaError_t err = cudaHostAlloc((void**)&pusch->llr,(144 * 3 * 8448 )*sizeof(int16_t),cudaHostAllocMapped); // 144 segments 8448*3 coded bits per segment 
+    AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr): %s\n",cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&pusch->llr_dev,pusch->llr,0);
+    AssertFatal(err == cudaSuccess,"CUDA Error (harq_f_dev): %s\n",cudaGetErrorString(err));
+#else
+    pusch->llr = (int16_t *)malloc16_clear((144 * 3 * 8448) * sizeof(int16_t)); //144 segments 3*8448 coded bits per segment 
+#endif
     pusch->ul_valid_re_per_slot = (int16_t *)malloc16_clear(sizeof(int16_t) * fp->symbols_per_slot);
   } // ulsch_id
 }
@@ -245,7 +255,11 @@ void phy_free_nr_gNB(PHY_VARS_gNB *gNB)
     free_and_zero(pusch_vars->ul_valid_re_per_slot);
     free_and_zero(pusch_vars->rxdataF_comp);
 
+#ifdef LDPC_CUDA
+    cudaFreeHost(pusch_vars->llr_dev);
+#else
     free_and_zero(pusch_vars->llr);
+#endif
   } // ULSCH_id
   free(gNB->pusch_vars);
 

openair1/PHY/INIT/nr_init_ue.c

@@ -18,6 +18,9 @@
 #include "SCHED_NR_UE/harq_nr.h"
 #include "nr-uesoftmodem.h"
 #include "common/config/config_userapi.h"
+#ifdef LDPC_CUDA
+#include <cuda_runtime.h>
+#endif
 
 void RCconfig_nrUE_prs(void *cfg)
 {
@@ -261,6 +264,17 @@ int init_nr_ue_signal(PHY_VARS_NR_UE *ue, int nb_connected_gNB)
   if (IS_SA_MODE(get_softmodem_params()))
     ue->received_config_request = false;
 
+#ifdef LDPC_CUDA
+  for (int j=0;j<10;j++) {
+    for (int i=0;i<2;i++) {
+      cudaError_t err = cudaHostAlloc((void**)&ue->llr[j][i],(66 * 3 * 8448) * sizeof(int16_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr): %s\n",cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&ue->llr_dev[j][i],ue->llr[j][i],0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (pusch_llr_dev): %s\n",cudaGetErrorString(err));
+
+    }
+  }
+#endif
   return 0;
 }
 
@@ -316,6 +330,14 @@ void term_nr_ue_signal(PHY_VARS_NR_UE *ue)
     free_and_zero(ue->prs_vars[idx]);
   }
 
+  
+#ifdef LDPC_CUDA
+  for (int j=0;j<10;j++) {
+   cudaFreeHost(ue->llr[j][0]);
+   cudaFreeHost(ue->llr[j][1]);
+  }
+#endif
+ 
   sl_ue_free(ue);
 }
 
@@ -330,7 +352,11 @@ void free_nr_ue_dl_harq(NR_DL_UE_HARQ_t harq_list[2][NR_MAX_HARQ_PROCESSES], int
   for (int j=0; j < 2; j++) {
     for (int i=0; i<number_of_processes; i++) {
       free_and_zero(harq_list[j][i].b);
+#ifdef LDPC_CUDA
+      cudaFreeHost(harq_list[j][i].c);
+#else
       free_and_zero(harq_list[j][i].c);
+#endif
       free_and_zero(harq_list[j][i].d);
     }
   }
@@ -381,7 +407,14 @@ void nr_init_dl_harq_processes(NR_DL_UE_HARQ_t harq_list[2][NR_MAX_HARQ_PROCESSE
       init_downlink_harq_status(harq_list[j] + i);
 
       harq_list[j][i].b = malloc16_clear(a_segments * 1056);
+#ifdef LDPC_CUDA
+      cudaError_t err = cudaHostAlloc((void**)&harq_list[j][i].c,a_segments*sizeof(uint8_t *)*1056,cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (harq.c): %s\n",cudaGetErrorString(err));
+      err = cudaHostGetDevicePointer((void**)&harq_list[j][i].cdev,(void*)harq_list[j][i].c,0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (harq.cdev): %s\n",cudaGetErrorString(err));
+#else
       harq_list[j][i].c = malloc16(a_segments * sizeof(*harq_list[j][i].c) * 1056);
+#endif
       harq_list[j][i].d = malloc16(a_segments * sizeof(*harq_list[j][i].d) * 3 * 8448);
       init_abort(&harq_list[j][i].abort_decode);
       harq_list[j][i].status  = 0;
@@ -396,40 +429,48 @@ void nr_init_ul_harq_processes(NR_UL_UE_HARQ_t harq_list[NR_MAX_HARQ_PROCESSES],
   uint16_t a_segments = MAX_NUM_NR_ULSCH_SEGMENTS_PER_LAYER*max_layers;  //number of segments to be allocated
 
   if (num_rb != 273) {
-    a_segments = a_segments*num_rb;
-    a_segments = a_segments/273 +1;
+    a_segments = a_segments * num_rb;
+    a_segments = a_segments / 273 + 1;
   }
 
-  uint32_t ulsch_bytes = a_segments*1056;  // allocated bytes per segment
+  uint32_t ulsch_bytes = a_segments * 1056;
 
   for (int i = 0; i < number_of_processes; i++) {
+    memset(&harq_list[i], 0, sizeof(NR_UL_UE_HARQ_t));
+    
+    harq_list[i].payload_AB = malloc16_clear(ulsch_bytes);
 
-    memset(harq_list + i, 0, sizeof(NR_UL_UE_HARQ_t));
+#ifdef LDPC_CUDA
+    uint8_t *tmp_c, *tmp_d;
+    size_t total_c_size = a_segments * 8448;
+    size_t total_d_size = a_segments * 68 * 384 * sizeof(uint32_t);
 
-    harq_list[i].payload_AB = malloc16(ulsch_bytes);
-    DevAssert(harq_list[i].payload_AB);
-    bzero(harq_list[i].payload_AB, ulsch_bytes);
+    AssertFatal(cudaHostAlloc((void**)&tmp_c, total_c_size, cudaHostAllocMapped) == cudaSuccess, "CUDA Data Alloc c failed\n");
+    AssertFatal(cudaHostAlloc((void**)&tmp_d, total_d_size, cudaHostAllocMapped) == cudaSuccess, "CUDA Data Alloc d failed\n");
+    memset(tmp_c, 0, total_c_size);
+    memset(tmp_d, 0, total_d_size);
+
+    AssertFatal(cudaHostAlloc((void**)&harq_list[i].c, a_segments * sizeof(uint8_t *), cudaHostAllocMapped) == cudaSuccess, "CUDA Pointer Alloc c failed\n");
+    AssertFatal(cudaHostAlloc((void**)&harq_list[i].d, a_segments * sizeof(uint8_t *), cudaHostAllocMapped) == cudaSuccess, "CUDA Pointer Alloc d failed\n");
 
-    harq_list[i].c = malloc16(a_segments*sizeof(uint8_t *));
-    harq_list[i].d = malloc16(a_segments*sizeof(uint16_t *));
     for (int r = 0; r < a_segments; r++) {
-      harq_list[i].c[r] = malloc16(8448);
-      DevAssert(harq_list[i].c[r]);
-      bzero(harq_list[i].c[r],8448);
-
-      harq_list[i].d[r] = malloc16(68*384); //max size for coded output
-      DevAssert(harq_list[i].d[r]);
-      bzero(harq_list[i].d[r],(68*384));
+      harq_list[i].c[r] = tmp_c + (r * 8448);
+      harq_list[i].d[r] = (uint8_t*)(tmp_d + (r * 68 * 384 * sizeof(uint32_t)));
     }
 
-    harq_list[i].e = malloc16(14*num_rb*12*16);
-    DevAssert(harq_list[i].e);
-    bzero(harq_list[i].e,14*num_rb*12*16);
+#else
+    harq_list[i].c = malloc16(a_segments * sizeof(uint8_t *));
+    harq_list[i].d = malloc16(a_segments * sizeof(uint8_t *));
 
-    harq_list[i].f = malloc16(14*num_rb*12*16);
-    DevAssert(harq_list[i].f);
-    bzero(harq_list[i].f,14*num_rb*12*16);
+    for (int r = 0; r < a_segments; r++) {
+      harq_list[i].c[r] = malloc16_clear(8448);
+      harq_list[i].d[r] = malloc16_clear(68 * 384);
+    }
+#endif
 
+    uint32_t ef_size = 14 * num_rb * 12 * 16;
+    harq_list[i].e = malloc16_clear(ef_size);
+    harq_list[i].f = malloc16_clear(ef_size);
     harq_list[i].round = 0;
   }
 }

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -20,6 +20,9 @@
 #include <syscall.h>
 #include <openair2/UTIL/OPT/opt.h>
 
+#ifdef LDPC_CUDA
+#include <cuda_runtime.h>
+#endif
 // #define DEBUG_DLSCH_CODING
 // #define DEBUG_DLSCH_FREE 1
 
@@ -34,7 +37,11 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM
   }
 
   if (dlsch->b) {
+#ifdef LDPC_CUDA
+    cudaFreeHost(dlsch->b);
+#else
     free16(dlsch->b, a_segments * 1056);
+#endif  
     dlsch->b = NULL;
   }
   if (dlsch->f) {
@@ -67,14 +74,29 @@ NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
   bzero(dlsch.b, dlsch_bytes);
 
   dlsch.c = (uint8_t **)malloc16(a_segments * sizeof(uint8_t *));
+#ifdef LDPC_CUDA
+  cudaError_t err=cudaHostAlloc((void**)&dlsch.c_devh,a_segments*sizeof(uint8_t*),cudaHostAllocMapped);
+  AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c_devh): %s\n", cudaGetErrorString(err));
+  err=cudaHostGetDevicePointer((void**)&dlsch.c_dev,(void*)dlsch.c_devh,0);
+  AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c_dev): %s\n", cudaGetErrorString(err));
+#endif
   for (int r = 0; r < a_segments; r++) {
     // account for filler in first segment and CRCs for multiple segment case
     // [hna] 8448 is the maximum CB size in NR
     //       68*348 = 68*(maximum size of Zc)
     //       In section 5.3.2 in 38.212, the for loop is up to N + 2*Zc (maximum size of N is 66*Zc, therefore 68*Zc)
-    dlsch.c[r] = malloc16(8448);
-    AssertFatal(dlsch.c[r], "cannot allocate dlsch.c[%d]\n", r);
-    bzero(dlsch.c[r], 8448);
+#ifdef LDPC_CUDA
+    err=cudaHostAlloc((void**)&dlsch.c[r],(8448/8)*sizeof(uint8_t),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (dlsch->c[%d]): %s\n", r,cudaGetErrorString(err));
+    uint8_t *tmpcr;
+    err=cudaHostGetDevicePointer((void**)&tmpcr, (void*)dlsch.c[r], 0);
+    ((uint8_t**)dlsch.c_devh)[r]=tmpcr;
+    AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) dlsch->c_devh[%d]: %s\n", r,cudaGetErrorString(err));
+#else
+    dlsch.c[r] = malloc16(8448/8);
+#endif
+    AssertFatal(dlsch.c[r], "cannot allocate dlsch->c[%d]\n", r);
+    bzero(dlsch.c[r], 8448/8);
   }
 
   dlsch.f = malloc16(N_RB * frame_parms->symbols_per_slot * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
@@ -218,6 +240,9 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
     TB_parameters->nb_layers = rel15->nrOfLayers;
     TB_parameters->rv_index = rel15->rvIndex[0];
 
+#ifdef LDPC_CUDA
+    TB_parameters->c_dev = (uint8_t**)dlsch->c_dev;
+#endif  
     int nb_re_dmrs =
         (rel15->dmrsConfigType == NFAPI_NR_DMRS_TYPE1) ? (6 * rel15->numDmrsCdmGrpsNoData) : (4 * rel15->numDmrsCdmGrpsNoData);
     TB_parameters->G = nr_get_G(rbsize,

openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c

@@ -33,6 +33,10 @@
 #define PRINT_CRC_CHECK(a)
 #endif
 
+#ifdef LDPC_CUDA
+#include <cuda_runtime.h>
+#endif
+
 void free_gNB_ulsch(NR_gNB_ULSCH_t *ulsch, uint16_t N_RB_UL)
 {
   uint16_t a_segments = MAX_NUM_NR_ULSCH_SEGMENTS; // number of segments to be allocated
@@ -47,8 +51,12 @@ void free_gNB_ulsch(NR_gNB_ULSCH_t *ulsch, uint16_t N_RB_UL)
       free_and_zero(ulsch->harq_process->b);
       ulsch->harq_process->b = NULL;
     }
+#ifdef LDPC_CUDA
+    // TODO
+#else
     free_and_zero(ulsch->harq_process->c);
     free_and_zero(ulsch->harq_process->d);
+#endif
     free_and_zero(ulsch->harq_process);
     ulsch->harq_process = NULL;
   }
@@ -75,8 +83,16 @@ NR_gNB_ULSCH_t new_gNB_ulsch(uint8_t max_ldpc_iterations, uint16_t N_RB_UL)
   ulsch.harq_process = harq;
   harq->b = malloc16_clear(ulsch_bytes * sizeof(*harq->b));
   // Allocate one contiguous buffer fr all c/d arrays to simplify addressing for GPU LDPC offload
+#ifdef LDPC_CUDA
+    cudaError_t err = cudaHostAlloc((void**)&harq->c, a_segments * 8448 * sizeof(*harq->c), cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess, "CUDA cudaHostAlloc failed for harq->c: %s\n", cudaGetErrorString(err));
+    
+    err = cudaHostAlloc((void**)&harq->d, a_segments * 64 * 384 * sizeof(*harq->d), cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess, "CUDA cudaHostAlloc failed for harq->d: %s\n", cudaGetErrorString(err));
+#else
   harq->c = malloc16_clear(a_segments * 8448 * sizeof(*harq->c));
   harq->d = malloc16_clear(a_segments * 68 * 384 * sizeof(*harq->d));
+#endif
   return (ulsch);
 }
 
@@ -103,6 +119,7 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
     NR_gNB_PUSCH *pusch = &phy_vars_gNB->pusch_vars[ULSCH_id];
     NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
     const nfapi_nr_pusch_pdu_t *pusch_pdu = &harq_process->ulsch_pdu;
+    uint8_t harq_pid = ulsch->harq_pid;
 
     nrLDPC_TB_decoding_parameters_t *TB_parameters = &TBs[pusch_id];
 
@@ -126,7 +143,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
 
 
     // The harq_pid is not unique among the active HARQ processes in the instance so we use ULSCH_id instead
-    TB_parameters->harq_unique_pid = ULSCH_id;
+    TB_parameters->harq_unique_pid = (phy_vars_gNB->max_nb_pusch * harq_pid) + ULSCH_id;
+    AssertFatal(TB_parameters->harq_unique_pid < 0xffffffff,"harq_unique_pid >= %d, harq_pid %d, ULSCH_id %d\n",16*phy_vars_gNB->max_nb_pusch,harq_pid,ULSCH_id);
 
     // ------------------------------------------------------------------
     TB_parameters->nb_rb = pusch_pdu->rb_size;
@@ -154,7 +172,6 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
       }
     }
 
-    uint8_t harq_pid = ulsch->harq_pid;
     LOG_D(PHY,
           "ULSCH Decoding, harq_pid %d rnti %x TBS %d G %d mcs %d Nl %d nb_rb %d, Qm %d, Coderate %f RV %d round %d new RX %d\n",
           harq_pid,
@@ -217,7 +234,11 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
     uint8_t ULSCH_id = ULSCH_ids[pusch_id];
     NR_gNB_ULSCH_t *ulsch = &phy_vars_gNB->ulsch[ULSCH_id];
     NR_UL_gNB_HARQ_t *harq_process = ulsch->harq_process;
-    short *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr;
+#ifdef LDPC_CUDA
+    int16_t *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr_dev;
+#else
+    int16_t *ulsch_llr = phy_vars_gNB->pusch_vars[ULSCH_id].llr;
+#endif
 
     if (!ulsch_llr) {
       LOG_E(PHY, "ulsch_decoding.c: NULL ulsch_llr pointer\n");

openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c

@@ -19,6 +19,10 @@
 #include <sys/time.h>
 
 
+#ifndef __AVX2__
+#define USE128BIT
+#endif
+
 #if T_TRACER
 static void copy_c16_data_to_slot_memory(c16_t *src, c16_t *dst_slot, int nb_re_pusch, int symbol)
 {
@@ -174,6 +178,41 @@ static int get_nb_re_pusch (NR_DL_FRAME_PARMS *frame_parms, const nfapi_nr_pusch
     return (rel15_ul->rb_size * NR_NB_SC_PER_RB);
 }
 
+#ifdef __AVX512BW__
+#define s16 __m512i
+#define setzero _mm512_setzero_si512
+#define set1_s16 _mm512_set1_epi16
+#define cpx_mult_conj_s16 oai_mm512_cpx_mult_conj
+#define adds_s16 _mm512_adds_epi16
+#define smadd_s16 oai_mm512_smadd
+#define packs_s16 _mm512_packs_epi32
+#define unpacklo_s16 _mm512_unpacklo_epi16
+#define mulhrs_s16 _mm512_mulhrs_epi16
+#define COMPSHIFT 4
+#elif defined(__AVX2__)
+#define s16 simde__m256i
+#define setzero simde_mm256_setzero_si256
+#define set1_s16 simde_mm256_set1_epi16
+#define cpx_mult_conj_s16 oai_mm256_cpx_mult_conj
+#define adds_s16 simde_mm256_adds_epi16
+#define smadd_s16 oai_mm256_smadd
+#define packs_s16 simde_mm256_packs_epi32
+#define unpacklo_s16 simde_mm256_unpacklo_epi16
+#define mulhrs_s16 simde_mm256_mulhrs_epi16
+#define COMPSHIFT 3
+#else
+#define s16 simde__m128i
+#define setzero simde_mm_setzero_si128
+#define set1_s16 simde_mm_set1_epi16
+#define cpx_mult_conj_s16 oai_mm_cpx_mult_conj
+#define adds_s16 simde_mm_adds_epi16
+#define smadd_s16 oai_mm_smadd
+#define packs_s16 simde_mm_packs_epi32
+#define unpacklo_s16 simde_mm_unpacklo_epi16
+#define mulhrs_s16 simde_mm_mulhrs_epi16
+#define COMPSHIFT 2
+#endif
+#ifndef __aarch64__
 static void nr_ulsch_channel_compensation(uint32_t buffer_length,
                                           int nb_rx_ant,
                                           c16_t rxFext[][buffer_length],
@@ -191,63 +230,60 @@ static void nr_ulsch_channel_compensation(uint32_t buffer_length,
   int mod_order  = rel15_ul->qam_mod_order;
   int nrOfLayers = rel15_ul->nrOfLayers;
 
-  simde__m256i QAM_ampa_256 = simde_mm256_setzero_si256();
-  simde__m256i QAM_ampb_256 = simde_mm256_setzero_si256();
-  simde__m256i QAM_ampc_256 = simde_mm256_setzero_si256();
+  s16 QAM_ampa = setzero();
+  s16 QAM_ampb = setzero();
+  s16 QAM_ampc = setzero();
 
   if (mod_order == 4) {
-    QAM_ampa_256 = simde_mm256_set1_epi16(QAM16_n1);
-    QAM_ampb_256 = simde_mm256_setzero_si256();
-    QAM_ampc_256 = simde_mm256_setzero_si256();
+    QAM_ampa = set1_s16(QAM16_n1);
   }
   else if (mod_order == 6) {
-    QAM_ampa_256 = simde_mm256_set1_epi16(QAM64_n1);
-    QAM_ampb_256 = simde_mm256_set1_epi16(QAM64_n2);
-    QAM_ampc_256 = simde_mm256_setzero_si256();
+    QAM_ampa = set1_s16(QAM64_n1);
+    QAM_ampb = set1_s16(QAM64_n2);
   }
   else if (mod_order == 8) {
-    QAM_ampa_256 = simde_mm256_set1_epi16(QAM256_n1);
-    QAM_ampb_256 = simde_mm256_set1_epi16(QAM256_n2);
-    QAM_ampc_256 = simde_mm256_set1_epi16(QAM256_n3);
+    QAM_ampa = set1_s16(QAM256_n1);
+    QAM_ampb = set1_s16(QAM256_n2);
+    QAM_ampc = set1_s16(QAM256_n3);
   }
 
   for (int aatx = 0; aatx < nrOfLayers; aatx++) {
-    simde__m256i *rxComp_256 = (simde__m256i *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];
-    simde__m256i *rxF_ch_maga_256 = (simde__m256i *)ul_ch_maga[aatx];
-    simde__m256i *rxF_ch_magb_256 = (simde__m256i *)ul_ch_magb[aatx];
-    simde__m256i *rxF_ch_magc_256 = (simde__m256i *)ul_ch_magc[aatx];
+    s16 *rxComp_s16 = (s16 *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];
+    s16 *rxF_ch_maga = (s16 *)ul_ch_maga[aatx];
+    s16 *rxF_ch_magb = (s16 *)ul_ch_magb[aatx];
+    s16 *rxF_ch_magc = (s16 *)ul_ch_magc[aatx];
     for (int aarx = 0; aarx < nb_rx_ant; aarx++) {
-      simde__m256i *rxF_256 = (simde__m256i *)rxFext[aarx];
-      simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx];
+      s16 *rxF = (s16 *)rxFext[aarx];
+      s16 *chF = (s16 *)chFext[aatx][aarx];
 
-      for (int i = 0; i < buffer_length >> 3; i++) 
+      for (int i = 0; i < buffer_length >> COMPSHIFT; i++) 
       {
         // MRC        
-        simde__m256i comp = oai_mm256_cpx_mult_conj(chF_256[i], rxF_256[i], output_shift);
-        rxComp_256[i] = simde_mm256_add_epi16(rxComp_256[i], comp); 
+        s16 comp = cpx_mult_conj_s16(chF[i], rxF[i], output_shift);
+        rxComp_s16[i] = adds_s16(rxComp_s16[i], comp); 
 
         if (mod_order > 2) {
-          simde__m256i mag = oai_mm256_smadd(chF_256[i], chF_256[i], output_shift); // |h|^2
+          s16 mag = smadd_s16(chF[i], chF[i], output_shift); // |h|^2
           // pack and duplicate
-          mag = simde_mm256_packs_epi32(mag, mag);
-          mag = simde_mm256_unpacklo_epi16(mag, mag);
+          mag = packs_s16(mag, mag);
+          mag = unpacklo_s16(mag, mag);
 
-          rxF_ch_maga_256[i] = simde_mm256_add_epi16(rxF_ch_maga_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampa_256));
+          rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));
 
           if (mod_order > 4)
-            rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampb_256));
+            rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));
 
           if (mod_order > 6)
-            rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], simde_mm256_mulhrs_epi16(mag, QAM_ampc_256));
+            rxF_ch_magc[i] = adds_s16(rxF_ch_magc[i], mulhrs_s16(mag, QAM_ampc));
         }        
       }
       if (nb_layers > 1) {
         for (int atx = 0; atx < nrOfLayers; atx++) {
-          simde__m256i *rho_256 = (simde__m256i *)rho[aatx][atx];
-          simde__m256i *chF_256 = (simde__m256i *)chFext[aatx][aarx];
-          simde__m256i *chF2_256 = (simde__m256i *)chFext[atx][aarx];
-          for (int i = 0; i < buffer_length >> 3; i++) {
-            rho_256[i] = simde_mm256_adds_epi16(rho_256[i], oai_mm256_cpx_mult_conj(chF_256[i], chF2_256[i], output_shift));
+          s16 *rho_s16 = (s16 *)rho[aatx][atx];
+          s16 *chF = (s16 *)chFext[aatx][aarx];
+          s16 *chF2 = (s16 *)chFext[atx][aarx];
+          for (int i = 0; i < buffer_length >> COMPSHIFT; i++) {
+            rho_s16[i] = adds_s16(rho_s16[i], cpx_mult_conj_s16(chF[i], chF2[i], output_shift));
           }
         }
       }
@@ -255,7 +291,159 @@ static void nr_ulsch_channel_compensation(uint32_t buffer_length,
   }
 
 }
+#else
+#define COMP_SHIFT(SHIFT) \
+static void nr_ulsch_channel_compensation##SHIFT(uint32_t buffer_length,\
+                                          int nb_rx_ant,\
+                                          c16_t rxFext[][buffer_length],\
+                                          c16_t chFext[][nb_rx_ant][buffer_length],\
+                                          c16_t ul_ch_maga[][buffer_length],\
+                                          c16_t ul_ch_magb[][buffer_length],\
+                                          c16_t ul_ch_magc[][buffer_length],\
+                                          c16_t **rxComp,\
+                                          int nb_layers,\
+                                          c16_t rho[][nb_layers][buffer_length],\
+                                          const nfapi_nr_pusch_pdu_t *rel15_ul,\
+                                          uint32_t symbol)\
+{\
+  int mod_order  = rel15_ul->qam_mod_order;\
+  int nrOfLayers = rel15_ul->nrOfLayers;\
+\
+  s16 QAM_ampa = setzero();\
+  s16 QAM_ampb = setzero();\
+  s16 QAM_ampc = setzero();\
+\
+  if (mod_order == 4) {\
+    QAM_ampa = set1_s16(QAM16_n1);\
+  }\
+  else if (mod_order == 6) {\
+    QAM_ampa = set1_s16(QAM64_n1);\
+    QAM_ampb = set1_s16(QAM64_n2);\
+  }\
+  else if (mod_order == 8) {\
+    QAM_ampa = set1_s16(QAM256_n1);\
+    QAM_ampb = set1_s16(QAM256_n2);\
+    QAM_ampc = set1_s16(QAM256_n3);\
+  }\
+\
+  for (int aatx = 0; aatx < nrOfLayers; aatx++) {\
+    s16 *rxComp_s16 = (s16 *)&rxComp[aatx * nb_rx_ant][symbol * buffer_length];\
+    s16 *rxF_ch_maga = (s16 *)ul_ch_maga[aatx];\
+    s16 *rxF_ch_magb = (s16 *)ul_ch_magb[aatx];\
+    s16 *rxF_ch_magc = (s16 *)ul_ch_magc[aatx];\
+    for (int aarx = 0; aarx < nb_rx_ant; aarx++) {\
+      s16 *rxF = (s16 *)rxFext[aarx];\
+      s16 *chF = (s16 *)chFext[aatx][aarx];\
+\
+      if (mod_order==8) {\
+        for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
+        {\
+          s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
+          rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
+          s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
+          mag = packs_s16(mag, mag);\
+          mag = unpacklo_s16(mag, mag);\
+          rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
+          rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));\
+          rxF_ch_magc[i] = adds_s16(rxF_ch_magc[i], mulhrs_s16(mag, QAM_ampc));\
+        }\
+      }\
+      else if (mod_order==6){\
+        for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
+        {\
+          s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
+          rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
+          s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
+          mag = packs_s16(mag, mag);\
+          mag = unpacklo_s16(mag, mag);\
+          rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
+          rxF_ch_magb[i] = adds_s16(rxF_ch_magb[i], mulhrs_s16(mag, QAM_ampb));\
+        }\
+      }\
+      else if (mod_order==4){\
+        for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
+        {\
+          s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
+          rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
+          s16 mag = smadd_s16(chF[i], chF[i], SHIFT);\
+          mag = packs_s16(mag, mag);\
+          mag = unpacklo_s16(mag, mag);\
+          rxF_ch_maga[i] = adds_s16(rxF_ch_maga[i], mulhrs_s16(mag, QAM_ampa));\
+        }\
+      }\
+      else if (mod_order==2){\
+        for (int i = 0; i < buffer_length >> COMPSHIFT; i++) \
+        {\
+          s16 comp = oai_mm_cpx_mult_conj##SHIFT(chF[i], rxF[i]);\
+          rxComp_s16[i] = adds_s16(rxComp_s16[i], comp);\
+        }\
+      }\
+      if (nb_layers > 1) {\
+        for (int atx = 0; atx < nrOfLayers; atx++) {\
+          s16 *rho_s16 = (s16 *)rho[aatx][atx];\
+          s16 *chF = (s16 *)chFext[aatx][aarx];\
+          s16 *chF2 = (s16 *)chFext[atx][aarx];\
+          for (int i = 0; i < buffer_length >> COMPSHIFT; i++) {\
+            rho_s16[i] = adds_s16(rho_s16[i], cpx_mult_conj_s16(chF[i], chF2[i], SHIFT));\
+          }\
+        }\
+      }\
+    }\
+  }\
+\
+}
+COMP_SHIFT(1)
+COMP_SHIFT(2)
+COMP_SHIFT(3)
+COMP_SHIFT(4)
+COMP_SHIFT(5)
+COMP_SHIFT(6)
+COMP_SHIFT(7)
+COMP_SHIFT(8)
+COMP_SHIFT(9)
+COMP_SHIFT(10)
+COMP_SHIFT(11)
+COMP_SHIFT(12)
+COMP_SHIFT(13)
+COMP_SHIFT(14)
+COMP_SHIFT(15)
 
+static void nr_ulsch_channel_compensation(uint32_t buffer_length,
+                                          int nb_rx_ant,
+                                          c16_t rxFext[][buffer_length],
+                                          c16_t chFext[][nb_rx_ant][buffer_length],
+                                          c16_t ul_ch_maga[][buffer_length],
+                                          c16_t ul_ch_magb[][buffer_length],
+                                          c16_t ul_ch_magc[][buffer_length],
+                                          c16_t **rxComp,
+                                          int nb_layers,
+                                          c16_t rho[][nb_layers][buffer_length],
+                                          const nfapi_nr_pusch_pdu_t *rel15_ul,
+                                          uint32_t symbol,
+                                          uint32_t output_shift)
+{
+	switch(output_shift) {
+//        case 0: nr_ulsch_channel_compensation0(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol);
+        case 1: nr_ulsch_channel_compensation1(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 2: nr_ulsch_channel_compensation2(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 3: nr_ulsch_channel_compensation3(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 4: nr_ulsch_channel_compensation4(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 5: nr_ulsch_channel_compensation5(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 6: nr_ulsch_channel_compensation6(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 7: nr_ulsch_channel_compensation7(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 8: nr_ulsch_channel_compensation8(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 9: nr_ulsch_channel_compensation9(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 10: nr_ulsch_channel_compensation10(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 11: nr_ulsch_channel_compensation11(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 12: nr_ulsch_channel_compensation12(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 13: nr_ulsch_channel_compensation13(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 14: nr_ulsch_channel_compensation14(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+        case 15: nr_ulsch_channel_compensation15(buffer_length,nb_rx_ant,rxFext,chFext,ul_ch_maga,ul_ch_magb,ul_ch_magc,rxComp,nb_layers,rho,rel15_ul,symbol); break;
+	default: LOG_E(NR_PHY,"Illegal shift %d\n",output_shift);
+	break;
+	}
+}
+#endif
 // Zero Forcing Rx function: nr_det_HhH()
 static void nr_ulsch_det_HhH(c16_t *after_mf_00, // a
                              c16_t *after_mf_01, // b
@@ -1378,9 +1566,10 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
   else 
     pusch_vars->log2_maxh = (log2_approx(avgs) >> 1) + 1 + log2_approx(frame_parms->nb_antennas_rx >> 1);
 
-  if (pusch_vars->log2_maxh < 0)
-    pusch_vars->log2_maxh = 0;
-
+  if (pusch_vars->log2_maxh < 1)
+    pusch_vars->log2_maxh = 1;
+  else if (pusch_vars->log2_maxh > 14)
+    pusch_vars->log2_maxh = 14;
   stop_meas(&gNB->rx_pusch_init_stats);
 
   start_meas(&gNB->rx_pusch_symbol_processing_stats);

openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_decoding.c

@@ -56,7 +56,7 @@ void nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
     .slot = proc->nr_slot_rx,
     .nb_TBs = 1,
     .threadPool = &get_nrUE_params()->Tpool,
-    .TBs = &TB_parameters
+    .TBs = &TB_parameters,
   };
 
   uint8_t dmrs_Type = dlsch_config->dmrsConfigType;

openair1/PHY/NR_UE_TRANSPORT/nr_transport_ue.h

@@ -71,6 +71,10 @@ typedef struct {
   uint8_t *b;
   /// Pointer to transport block segments
   uint8_t *c;
+#ifdef LDPC_CUDA
+  /// Pointer to transport block segments (GPU pointer for c when using shared memory, should be the same on Jetson
+  uint8_t *cdev;
+#endif
   /// soft bits for each received segment ("d"-sequence)(for definition see 36-212 V8.6 2009-03, p.15)
   /// Accumulates the soft bits for each round to increase decoding success (HARQ)
   int16_t *d;

openair1/PHY/defs_gNB.h

@@ -110,6 +110,12 @@ typedef struct {
   uint8_t *b;
   /// Pointers to transport block segments
   uint8_t **c;
+#ifdef LDPC_CUDA
+  /// Pointers to transport block segments (contains pointers in c above)
+  uint8_t *c_devh;
+  /// Pointers to transport block segments (GPU mapping)
+  uint8_t *c_dev;
+#endif
   /// Interleaver outputs
   uint8_t *f;
   /// REs unavailable for DLSCH (overlapping with PTRS, CSIRS etc.)
@@ -252,6 +258,10 @@ typedef struct {
   /// \brief llr values.
   /// - first index: ? [0..1179743] (hard coded)
   int16_t *llr;
+#ifdef LDPC_CUDA
+  /// \brief llr values link to device memory
+  int16_t* llr_dev; 
+#endif
   // PTRS symbol index, to be updated every PTRS symbol within a slot.
   uint8_t ptrs_symbol_index;
   /// bit mask of PT-RS ofdm symbol indicies

openair1/PHY/defs_nr_UE.h

@@ -436,6 +436,12 @@ typedef struct PHY_VARS_NR_UE_s {
   Actor_t *ul_actors;
   pthread_t main_thread;
   pthread_t stat_thread;
+#ifdef LDPC_CUDA
+  // page locked memory (cudaHostAlloc) for llr input to LDPC decoder for GPU offload
+  int16_t *llr[10][2];
+  // gpu mapped version (cudaDeviceGetHostPointer), typically the same for Jetson/GH/GB
+  int16_t *llr_dev[10][2];
+#endif
 } PHY_VARS_NR_UE;
 
 typedef struct {

openair1/PHY/sse_intrin.h

@@ -150,8 +150,14 @@ __attribute__((always_inline)) static inline int32_t simde_mm_average(simde__m12
  */
 __attribute__((always_inline)) static inline simde__m128i oai_mm_conj(simde__m128i a)
 {
+#ifdef __aarch64__
+  const oai128_t neg_imag = {.i16 = {0, -1, 0, -1, 0, -1, 0, -1}};
+  int16x8_t aneg = vnegq_s16((int16x8_t)a);
+  return (simde__m128i)vbslq_s16((uint16x8_t)neg_imag.v, aneg, (int16x8_t)a);
+#else  
   const oai128_t neg_imag = {.i16 = {1, -1, 1, -1, 1, -1, 1, -1}};
   return simde_mm_sign_epi16(a, neg_imag.v);
+#endif
 }
 
 /**
@@ -167,9 +173,13 @@ __attribute__((always_inline)) static inline simde__m128i oai_mm_conj(simde__m12
 __attribute__((always_inline)) static inline
 simde__m128i oai_mm_swap(simde__m128i a)
 {
+#ifdef __aarch64__
+  return (simde__m128i)vrev32q_s16((int16x8_t)a);
+#else
   // Shuffle mask to swap bytes for IQ swapping
   const oai128_t shuffle_mask_swap = {.i8 = {2, 3, 0, 1, 6, 7, 4, 5, 10, 11, 8, 9, 14, 15, 12, 13}};
   return simde_mm_shuffle_epi8(a, shuffle_mask_swap.v);
+#endif
 }
 
 __attribute__((always_inline)) static inline
@@ -216,7 +226,7 @@ simde__m128i oai_mm_cpx_mult(simde__m128i z1, simde__m128i z2, int shift)
  * Input:  z1 = (a + bi) [ a0,  b0,  ...,  a3,  b3]
  * Input:  z2 = (c + di) [ c0,  d0,  ...,  c3,  d3]
  * Output: z3 = (e + fi) [ e0,  f0,  ...,  e3,  f3]
- * z3 = z1 * conj(z2) = + (ac+bd) + i(bc-ad)
+ * z3 = conj(z1) * z2 = + (ac+bd) + i(ad-bc)
  *
  * @param 128-bit SIMD vector of four complex 16-bit integers.
  * @return a 128-bit SIMD vector.
@@ -229,6 +239,43 @@ simde__m128i oai_mm_cpx_mult_conj(simde__m128i a, simde__m128i b, int shift)
   return oai_mm_pack(re, im);
 }
 
+#ifdef __aarch64__
+#define CPX_MUL_CONJ(SHIFT) \
+__attribute__((always_inline)) static inline simde__m128i oai_mm_cpx_mult_conj##SHIFT(simde__m128i a, simde__m128i b) \
+{ \
+    const oai128_t neg_imag = {.i16 = {0, -1, 0, -1, 0, -1, 0, -1}};\
+\
+    int16x8_t aneg    = vnegq_s16((int16x8_t)a);\
+    int16x8_t arevn   = vbslq_s16((uint16x8_t)neg_imag.v, aneg, (int16x8_t)a);\
+              arevn   = vrev32q_s16(arevn);\
+    int32x4_t abl     = vmull_s16(vget_low_s16((int16x8_t)a),vget_low_s16((int16x8_t)b));\
+    int32x4_t abh     = vmull_high_s16((int16x8_t)a,(int16x8_t)b);\
+    int32x4_t arevnbl = vmull_s16(vget_low_s16(arevn),vget_low_s16((int16x8_t)b));\
+    int32x4_t arevnbh = vmull_high_s16(arevn,(int16x8_t)b);\
+    int32x4_t re      = vpaddq_s32(abl,abh);\
+    int32x4_t im      = vpaddq_s32(arevnbl,arevnbh);\
+    int32x4_t outl    = vzip1q_s32(re,im);\
+    int32x4_t outh    = vzip2q_s32(re,im);\
+    int16x4_t r       = vqrshrn_n_s32(outl, (const int)SHIFT);\
+    return (simde__m128i)vqrshrn_high_n_s32(r, outh, (const int)SHIFT);	\
+}
+//CPX_MUL_CONJ(0)
+CPX_MUL_CONJ(1)
+CPX_MUL_CONJ(2)
+CPX_MUL_CONJ(3)
+CPX_MUL_CONJ(4)
+CPX_MUL_CONJ(5)
+CPX_MUL_CONJ(6)
+CPX_MUL_CONJ(7)
+CPX_MUL_CONJ(8)
+CPX_MUL_CONJ(9)
+CPX_MUL_CONJ(10)
+CPX_MUL_CONJ(11)
+CPX_MUL_CONJ(12)
+CPX_MUL_CONJ(13)
+CPX_MUL_CONJ(14)
+CPX_MUL_CONJ(15)
+#endif
 /*
  * OAI specific AVX2 section
  */
@@ -313,7 +360,7 @@ simde__m256i oai_mm256_cpx_mult(simde__m256i z1, simde__m256i z2, int shift)
  * Input:  z1 = (a + bi) [ a0,  b0,  ...,  a3,  b3]
  * Input:  z2 = (c + di) [ c0,  d0,  ...,  c3,  d3]
  * Output: z3 = (e + fi) [ e0,  f0,  ...,  e3,  f3]
- * z3 =  z1 * conj(z2) =  (ac+bd) + i(bc-ad)
+ * z3 =  conj(z1) * z2 =  (ac+bd) + i(ad-bc)
  *
  * @param 256-bit SIMD vector of eight complex 16-bit integers.
  * @return a 256-bit SIMD vector.
@@ -326,6 +373,50 @@ simde__m256i oai_mm256_cpx_mult_conj(simde__m256i a, simde__m256i b, int shift)
   return oai_mm256_pack(re, im);
 }
 
+#ifdef __AVX512BW__
+__attribute__((always_inline)) static inline __m512i oai_mm512_conj(__m512i a)
+{
+  const __mmask32 odd = 0xAAAAAAAAu;
+    // For odd lanes: 0 - b (two's complement negate)
+  return(_mm512_mask_sub_epi16(a, odd, _mm512_setzero_si512(), a));
+}
+
+__attribute__((always_inline)) static inline __m512i oai_mm512_swap(__m512i a)
+
+{
+  // Shuffle mask to swap bytes for IQ swapping
+  const oai512_t shuffle_mask_swap = {.i8 = {
+                                          2,  3,  0,  1,  6,  7,  4,  5,  10, 11, 8,  9,  14, 15, 12, 13,
+                                          18, 19, 16, 17, 22, 23, 20, 21, 26, 27, 24, 25, 30, 31, 28, 29,
+                                          34, 35, 32, 33, 38, 39, 36, 37, 42, 43, 40, 41, 46, 47, 44, 45,
+                                          50, 51, 48, 49, 54, 55, 52, 53, 58, 59, 56, 57, 62, 63, 60, 61
+                                      }};
+  return _mm512_shuffle_epi8(a, shuffle_mask_swap.v);
+}
+
+__attribute__((always_inline)) static inline
+__m512i oai_mm512_smadd(__m512i z1, __m512i z2, int shift)
+{
+  return _mm512_srai_epi32(_mm512_madd_epi16(z1, z2), shift);
+}
+
+__attribute__((always_inline)) static inline
+__m512i oai_mm512_pack(__m512i a, __m512i b)
+{
+  return _mm512_packs_epi32(
+    _mm512_unpacklo_epi32(a, b), // real
+    _mm512_unpackhi_epi32(a, b)  // imag
+  );
+}
+__attribute__((always_inline)) static inline
+__m512i oai_mm512_cpx_mult_conj(__m512i a, __m512i b, int shift)
+{
+  __m512i re = oai_mm512_smadd(a, b, shift);
+  __m512i im = oai_mm512_smadd(oai_mm512_swap(oai_mm512_conj(a)), b, shift);
+  return oai_mm512_pack(re, im);
+}
+#endif
+
 #ifdef __cplusplus
 }
 #endif

openair1/SCHED_NR_UE/phy_procedures_nr_ue.c

@@ -44,6 +44,10 @@
 #include "T.h"
 #include "instrumentation.h"
 
+#ifdef LDPC_CUDA
+#include <cuda_runtime.h>
+#endif
+
 static const unsigned int gain_table[31] = {100,  112,  126,  141,  158,  178,  200,  224,  251, 282,  316,
                                             359,  398,  447,  501,  562,  631,  708,  794,  891, 1000, 1122,
                                             1258, 1412, 1585, 1778, 1995, 2239, 2512, 2818, 3162};
@@ -1249,7 +1253,13 @@ void pdsch_processing(PHY_VARS_NR_UE *ue, const UE_nr_rxtx_proc_t *proc, nr_phy_
                      dlsch->cw_info.qamModOrder,
                      dlsch->cw_info.Nl);
     const uint32_t rx_llr_buf_sz = ALIGNARRAYSIZE(G, 32); // each LLR is 2 bytes hence 64 byte aligned
+#ifdef LDPC_CUDA
+    llr[c]=ue->llr_dev[nr_slot_rx%10][c];
+    LOG_D(NR_PHY,"nr_slot_rx %d, llr[%d] %p\n",nr_slot_rx,c,llr[c]);
+//      memset(llr[i],0,rx_llr_buf_sz * sizeof(int16_t));
+#else
     llr[c] = (int16_t *)malloc16_clear(rx_llr_buf_sz * sizeof(int16_t));
+#endif
 
     // dlsch_harq contains the previous transmissions data for this harq pid
     NR_DL_UE_HARQ_t *harq = &ue->dl_harq_processes[c][dlsch_config->harq_process_nbr];
@@ -1286,7 +1296,9 @@ void pdsch_processing(PHY_VARS_NR_UE *ue, const UE_nr_rxtx_proc_t *proc, nr_phy_
     if (ue->phy_sim_pdsch_llr)
       memcpy(ue->phy_sim_pdsch_llr, llr[c], sizeof(int16_t) * rx_llr_buf_sz);
 
+#ifndef LDPC_CUDA
     free(llr[c]);
+#endif
   }
 
   if (nr_slot_rx==9) {

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -1056,9 +1056,11 @@ int main(int argc, char **argv)
     reset_meas(&gNB->dlsch_encoding_stats);
     reset_meas(&gNB->dci_generation_stats);
     reset_meas(&gNB->tinput);
+    reset_meas(&gNB->tinput_memcpy);
     reset_meas(&gNB->tprep);
     reset_meas(&gNB->tparity);
     reset_meas(&gNB->toutput);
+    reset_meas(&gNB->tconcat);
     reset_meas(&gNB->phase_comp_stats);
 
     uint32_t errors_scrambling[16] = {0};
@@ -1431,11 +1433,14 @@ int main(int argc, char **argv)
       fprintf(csv_file,"%.2f,%.4f,%.2f,%u\n", roundStats, effRate, effRate / (8 * pdsch_pdu_rel15->TBSize[0]) * 100, 8 * pdsch_pdu_rel15->TBSize[0]);
     }
     if (print_perf==1) {
-      printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, block %d)\n",
+      printf("\ngNB TX function statistics (per %d us slot, NPRB %d, mcs %d, C %d, Z %d, F %d, K %d, block %d)\n",
              1000 >> *scc->ssbSubcarrierSpacing,
              g_rbSize,
              g_mcsIndex,
              UE->dl_harq_processes[0][slot].C,
+             UE->dl_harq_processes[0][slot].Z,
+             UE->dl_harq_processes[0][slot].F,
+             UE->dl_harq_processes[0][slot].K,
              8 * pdsch_pdu_rel15->TBSize[0]);
       printDistribution(&gNB->phy_proc_tx,table_tx,"PHY proc tx");
       printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");

openair1/SIMULATION/NR_PHY/pucchsim.c

@@ -38,7 +38,6 @@ PHY_VARS_NR_UE *UE;
 RAN_CONTEXT_t RC;
 int64_t uplink_frequency_offset[MAX_NUM_CCs][4];
 uint64_t downlink_frequency[MAX_NUM_CCs][4];
-
 double cpuf;
 uint8_t const nr_rv_round_map[4] = {0, 2, 3, 1};
 // needed for some functions

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -1057,7 +1057,6 @@ int main(int argc, char *argv[])
   for (i = 1; i < TBS/8; i++) {
     ulsch_input_buffer[i] = (uint8_t)rand();
   }
-
   uint8_t ptrs_time_density = get_L_ptrs(ptrs_mcs1, ptrs_mcs2, ptrs_mcs3, Imcs, mcs_table);
   uint8_t ptrs_freq_density = get_K_ptrs(n_rb0, n_rb1, nb_rb);