try to merge the encoder branch, but bug exist

openair1/PHY/CODING/CMakeLists.txt

@@ -13,6 +13,7 @@ target_link_libraries(coding PRIVATE log_headers)
 add_library(ldpc_orig MODULE
             nrLDPC_decoder/nrLDPC_decoder.c
             nrLDPC_encoder/ldpc_encoder.c
+            nrLDPC_encoder/ldpc_encoder32.c
 )
 set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
@@ -20,6 +21,7 @@ target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
 add_library(ldpc MODULE
             nrLDPC_decoder/nrLDPC_decoder.c
             nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
+            nrLDPC_encoder/ldpc_encoder32.c
 )
 set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS)
@@ -49,18 +51,22 @@ add_dependencies(ldpc_cl nrLDPC_decoder_kernels_CL)
 # Base CUDA setting
 ##############################################
 
-add_boolean_option(ENABLE_LDPC_CUDA OFF "Build support for CUDA" OFF ENABLE_PARALLEL_STREAM OFF "Enable CUDA stream parallel version" OFF)
+add_boolean_option(ENABLE_LDPC_CUDA OFF "Build support for CUDA" OFF)
 
 if (ENABLE_LDPC_CUDA)
   find_package(CUDA REQUIRED)
   if (NOT CUDA_FOUND)
     message(FATAL_ERROR "no CUDA found")
   endif()
-  SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=native;")
+  SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_80;")
   SET(CUDA_VERBOSE_BUILD ON)
-  cuda_add_library(ldpc_cuda MODULE
+
+  # version using NVLink-C2C in Grace Hopper
+  cuda_add_library(ldpc_cuda_GH MODULE
 	  #                   nrLDPC_decoder_LYC/nrLDPC_decoder_LYC.cu
-                   nrLDPC_decoder/nrLDPC_decoder.c
+    nrLDPC_decoder/kernel_test.cu  
+                   nrLDPC_decoder/nrLDPC_decoder_cuda_GH.c
+		   nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
 		   nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu
 		   nrLDPC_encoder/ldpc_encoder_cuda32.c
 		   nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
@@ -68,34 +74,51 @@ if (ENABLE_LDPC_CUDA)
                    # target_link_libraries like above
                    # because of cuda_add_library
                    # which already uses target_link_libraries
-                   nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+                   #nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+                   nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c
+                   nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder_cuda_GH.c
                    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
                    nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c
   )
   
-  target_link_libraries(ldpc_cuda ldpc_gen_HEADERS ${T_LIB})
-  set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-  set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+  target_link_libraries(ldpc_cuda_GH ldpc_gen_HEADERS ${T_LIB})
+  set_target_properties(ldpc_cuda_GH PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+  set_target_properties(ldpc_cuda_GH PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 
-  target_compile_definitions(ldpc_cuda PRIVATE USE_CUDA)
-  if (ENABLE_PARALLEL_STREAM)
-    cuda_add_library(ldpc_cuda_stream MODULE
-    nrLDPC_decoder/nrLDPC_decoder.c
+  add_dependencies(ldpctest ldpc_cuda_GH)
+  add_dependencies(nr-softmodem ldpc_cuda_GH)
+  add_dependencies(nr-uesoftmodem ldpc_cuda_GH)
+  add_dependencies(nr_ulsim ldpc_cuda_GH)
+  add_dependencies(nr_ulschsim ldpc_cuda_GH)
+  add_dependencies(nr_dlsim ldpc_cuda_GH)
+  add_dependencies(nr_dlschsim ldpc_cuda_GH)
+
+  target_compile_definitions(ldpc_cuda_GH PRIVATE USE_CUDA)
+
+  #version using HBM in general GPU
+  cuda_add_library(ldpc_cuda MODULE
+    nrLDPC_decoder/nrLDPC_decoder_cuda.c
     nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu  
+    nrLDPC_decoder/kernel_test.cu  
     nrLDPC_encoder/ldpc_encoder_cuda32.c
     nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
-    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+                   # The slot coding layer cannot be linked with
+                   # target_link_libraries like above
+                   # because of cuda_add_library
+                   # which already uses target_link_libraries
+    #nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder_cuda.c
+    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c
     nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
     nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c
-    )
-    set_target_properties(ldpc_cuda_stream PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
-    set_target_properties(ldpc_cuda_stream PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-    target_compile_definitions(ldpc_cuda_stream PRIVATE USE_CUDA PARALLEL_STREAM)
-    target_link_libraries(ldpc_cuda_stream ldpc_gen_HEADERS ${T_LIB})
-    target_compile_definitions(ldpctest PRIVATE PARALLEL_STREAM)
+  )
+  set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
+  set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
+  target_compile_definitions(ldpc_cuda PRIVATE USE_CUDA PARALLEL_STREAM)
+  target_link_libraries(ldpc_cuda ldpc_gen_HEADERS ${T_LIB})
+  target_compile_definitions(ldpctest PRIVATE PARALLEL_STREAM)
   
-    add_dependencies(ldpctest ldpc_cuda_stream)
-  endif()
+  add_dependencies(ldpctest ldpc_cuda)
   add_dependencies(ldpctest ldpc_cuda)
   add_dependencies(nr-softmodem ldpc_cuda)
   add_dependencies(nr-uesoftmodem ldpc_cuda)
@@ -103,6 +126,7 @@ if (ENABLE_LDPC_CUDA)
   add_dependencies(nr_ulschsim ldpc_cuda)
   add_dependencies(nr_dlsim ldpc_cuda)
   add_dependencies(nr_dlschsim ldpc_cuda)
+
 endif()
 
 add_subdirectory(nrLDPC_coding)

openair1/PHY/CODING/TESTBENCH/ldpctest.c

@@ -145,10 +145,12 @@ one_measurement_t test_ldpc(short max_iterations,
   cpu_meas_enabled = 1;
   uint8_t *test_input[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * NR_MAX_NB_LAYERS];
   uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
-  printf("Output Address: %p\n", estimated_output);
+  //printf("Output Address: %p\n", estimated_output);
   memset(estimated_output, 0, sizeof(estimated_output));
   uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
   uint8_t *channel_input_optim;
+  uint32_t channel_input_optim32[4][68*384];
+
   // double channel_output[68 * 384];
   double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
   int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
@@ -292,7 +294,8 @@ one_measurement_t test_ldpc(short max_iterations,
     memset(channel_input[j], 0, 68 * 384);
   }
   channel_input_optim = malloc16(68 * 384 * sizeof(uint32_t));
-  memset(channel_input_optim, 0, 68 * 384 * sizeof(uint32_t));
+  if (use32bit ==0) memset(channel_input_optim, 0, 68 * 384 * sizeof(uint32_t));
+  else memset(channel_input_optim32[0],0,68*384*sizeof(uint32_t));
 
   // Fill input segments with random values
   for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
@@ -329,21 +332,17 @@ one_measurement_t test_ldpc(short max_iterations,
     impp.n_segments = n_segments;
     start_meas(&ret.time_optim);
     impp.first_seg = 0;
-    ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
+    if (use32bit==0) ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
+    else ldpc_toCompare.LDPCencoder32(test_input, channel_input_optim32, &impp);
     stop_meas(&ret.time_optim);
 
     if (ntrials == 1)
       for (int j = 0; j < n_segments; j++)
         for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
-          if (((use32bit == 0) && (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)))
-              || ((use32bit == 1) && (channel_input[j][i] != ((((uint32_t *)channel_input_optim)[i] >> j) & 0x1)))) {
-            printf("differ in seg %d pos %d (%u,%u)\n",
-                   j,
-                   i,
-                   channel_input[j][i],
-                   (((uint32_t *)channel_input_optim)[i] >> j) & 0x1);
-            return ret;
-          }
+          if (((use32bit == 0) && (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1))) ||                 ((use32bit == 1) && (channel_input[j][i] != (((channel_input_optim32[0][i] >> j) & 0x1))))) {
+               printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (((uint32_t*)channel_input_optim)[i] >> j) & 0x1);
+               return ret;
+            }
         }
     int bit;
     for (int j = 0; j < n_segments; j++) {
@@ -352,8 +351,7 @@ one_measurement_t test_ldpc(short max_iterations,
         if ((i & 0xf) == 0)
           printf("\ne %d..%d:    ", i, i + 15);
 #endif
-        bit = (use32bit == 0) ? ((channel_input_optim[i - 2 * Zc] >> j) & 0x1)
-                              : ((((uint32_t *)channel_input_optim)[i - 2 * Zc] >> j) & 0x1);
+        bit = (use32bit==0) ? ((channel_input_optim[i - 2 * Zc] >> j) & 0x1) : ((channel_input_optim32[0][i - 2 * Zc] >> j) & 0x1);
         if (bit == 0)
           modulated_input[j][i] = 1.0; /// sqrt(2);  //QPSK
         else
@@ -392,6 +390,7 @@ if(PARALLEL_PATH == 1){
     set_abort(&dec_abort, false);
     //printf("Are you here?\n");
     // printf("n_segments = %d in test\n",n_segments);
+    //dumpASS(channel_output_fixed, "ldpctest_ChannelOutput_stream.txt");
     n_iter = ldpc_toCompare.LDPCdecoder(&decParams,
                                         0,
                                         0,
@@ -403,7 +402,7 @@ if(PARALLEL_PATH == 1){
     stop_meas(&ret.time_decoder);
     // printf("7:It works here\n");
     dumpASS(estimated_output, "ldpctest_estimateOutput_stream.txt");
-    //dumpASS(test_input, "ldpctest_TestInput_stream.txt");
+
     for (int j = 0; j < n_segments; j++) {
       //printf("estimated_output[%d] = %p\n", j, &estimated_output[j]);
       if (memcmp(estimated_output[j], test_input[j], ((Kprime + 7) & ~7) / 8) != 0) {
@@ -436,7 +435,7 @@ else{
     for (int j = 0; j < n_segments; j++) {
       start_meas(&ret.time_decoder);
       set_abort(&dec_abort, false);
-
+//dumpASS(channel_output_fixed, "ldpctest_ChannelOutput_128.txt");
       n_iter = ldpc_toCompare.LDPCdecoder(&decParams[j],
                                           0,
                                           0,
@@ -468,7 +467,7 @@ else{
         n_iter_max = n_iter;
 
     } // end segments
-        dumpASS(estimated_output, "ldpctest_estimateOutput_cuda128.txt");
+    dumpASS(estimated_output, "ldpctest_estimateOutput_cuda128.txt");
     //dumpASS(test_input, "ldpctest_TestInput_cuda128.txt");
     if (segment_bler != 0)
       ret.errors++;
@@ -570,12 +569,12 @@ int main(int argc, char *argv[])
         break;
 
       case 'G':
-        ldpc_version = "_cuda";
+        ldpc_version = "_cuda_GH";
         use32bit = 1;
         break;
 
       case 'P': // stands for "Parallel"
-        ldpc_version = "_cuda_stream";
+        ldpc_version = "_cuda";
         PARALLEL_PATH = 1;
         use32bit = 1;
         break;

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h

@@ -152,6 +152,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{
   time_stats_t ts_interleave;
   time_stats_t ts_rate_match;
   time_stats_t ts_ldpc_encode;
+  time_stats_t ts_output;
 } nrLDPC_segment_encoding_parameters_t;
 
 /**
@@ -229,6 +230,7 @@ typedef struct nrLDPC_slot_encoding_parameters_s{
   time_stats_t *tprep;
   time_stats_t *tparity;
   time_stats_t *toutput;
+  time_stats_t *tconcat;
   nrLDPC_TB_encoding_parameters_t *TBs;
 } nrLDPC_slot_encoding_parameters_t;
 

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c

@@ -281,6 +281,7 @@ int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_de
 
 int32_t nrLDPC_coding_init(void)
 {
+  cuda_support_init();	
   return 0;
 }
 

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder_cuda.c

@@ -0,0 +1,321 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.0  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+ * \brief Top-level routines for decoding LDPC transport channels
+ */
+
+// [from gNB coding]
+#include "nr_rate_matching.h"
+#include "PHY/defs_gNB.h"
+#include "PHY/CODING/coding_extern.h"
+#include "PHY/CODING/coding_defs.h"
+#include "PHY/CODING/lte_interleaver_inline.h"
+#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "PHY/CODING/nrLDPC_extern.h"
+#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
+#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#include "PHY/NR_TRANSPORT/nr_ulsch.h"
+#include "PHY/NR_TRANSPORT/nr_dlsch.h"
+#include "SCHED_NR/sched_nr.h"
+#include "SCHED_NR/fapi_nr_l1.h"
+#include "defs.h"
+#include "common/utils/LOG/vcd_signal_dumper.h"
+#include "common/utils/LOG/log.h"
+
+#include <stdalign.h>
+#include <stdint.h>
+#include <syscall.h>
+#include <time.h>
+// #define gNB_DEBUG_TRACE
+
+#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
+// #define DEBUG_CRC
+#ifdef DEBUG_CRC
+#define PRINT_CRC_CHECK(a) a
+#else
+#define PRINT_CRC_CHECK(a)
+#endif
+
+#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_interface.h"
+#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface.h"
+
+/**
+ * \typedef nrLDPC_decoding_parameters_t
+ * \struct nrLDPC_decoding_parameters_s
+ * \brief decoding parameter of transport blocks
+ * \var decoderParms decoder parameters
+ * \var Qm modulation order
+ * \var Kc ratio between the number of columns in the parity check matrix and the lifting size
+ * it is fixed for a given base graph while the lifting size is chosen to have a sufficient number of columns
+ * \var rv_index
+ * \var max_number_iterations maximum number of LDPC iterations
+ * \var abort_decode pointer to decode abort flag
+ * \var tbslbrm transport block size LBRM in bytes
+ * \var A Transport block size (This is A from 38.212 V15.4.0 section 5.1)
+ * \var K Code block size at decoder output
+ * \var Z lifting size
+ * \var F filler bits size
+ * \var r segment index in TB
+ * \var E input llr segment size
+ * \var C number of segments
+ * \var llr input llr segment array
+ * \var d Pointers to code blocks before LDPC decoding (38.212 V15.4.0 section 5.3.2)
+ * \var d_to_be_cleared
+ * pointer to the flag used to clear d properly
+ * when true, clear d after rate dematching
+ * \var c Pointers to code blocks after LDPC decoding (38.212 V15.4.0 section 5.2.2)
+ * \var decodeSuccess pointer to the flag indicating that the decoding of the segment was successful
+ * \var ans pointer to task answer used by the thread pool to detect task completion
+ * \var p_ts_deinterleave pointer to deinterleaving time stats
+ * \var p_ts_rate_unmatch pointer to rate unmatching time stats
+ * \var p_ts_ldpc_decode pointer to decoding time stats
+ */
+typedef struct nrLDPC_decoding_parameters_s {
+  t_nrLDPC_dec_params decoderParms;
+
+  uint8_t Qm;
+
+  uint8_t Kc;
+  uint8_t rv_index;
+  decode_abort_t *abort_decode;
+
+  uint32_t tbslbrm;
+  uint32_t A;
+  uint32_t K;
+  uint32_t Z;
+  uint32_t F;
+
+  uint32_t C;
+
+  int E;
+  short *llr;
+  int16_t *d;
+  bool *d_to_be_cleared;
+  uint8_t *c;
+  bool *decodeSuccess;
+
+  task_ans_t *ans;
+
+  time_stats_t *p_ts_deinterleave;
+  time_stats_t *p_ts_rate_unmatch;
+  time_stats_t *p_ts_ldpc_decode;
+} nrLDPC_decoding_parameters_t;
+
+static void nr_process_decode_segment(void *arg)
+{
+  nrLDPC_decoding_parameters_t *rdata = (nrLDPC_decoding_parameters_t *)arg;
+  t_nrLDPC_dec_params *p_decoderParms = &rdata->decoderParms;
+  const int K = rdata->K;
+  const int Kprime = K - rdata->F;
+  const int A = rdata->A;
+  const int E = rdata->E;
+  const int Qm = rdata->Qm;
+  const int rv_index = rdata->rv_index;
+  const uint8_t Kc = rdata->Kc;
+  short *ulsch_llr = rdata->llr;
+  int8_t llrProcBuf[OAI_LDPC_DECODER_MAX_NUM_LLR] __attribute__((aligned(32)));
+
+  t_nrLDPC_time_stats procTime = {0};
+  t_nrLDPC_time_stats *p_procTime = &procTime;
+
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////// nr_deinterleaving_ldpc ///////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  //////////////////////////// ulsch_llr =====> ulsch_harq->e //////////////////////////////
+
+  start_meas(rdata->p_ts_deinterleave);
+
+  /// code blocks after bit selection in rate matching for LDPC code (38.212 V15.4.0 section 5.4.2.1)
+  int16_t harq_e[E];
+
+  nr_deinterleaving_ldpc(E, Qm, harq_e, ulsch_llr);
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  stop_meas(rdata->p_ts_deinterleave);
+
+  start_meas(rdata->p_ts_rate_unmatch);
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+  //////////////////////////////// nr_rate_matching_ldpc_rx ////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  ///////////////////////// ulsch_harq->e =====> ulsch_harq->d /////////////////////////
+
+  if (nr_rate_matching_ldpc_rx(rdata->tbslbrm,
+                               p_decoderParms->BG,
+                               p_decoderParms->Z,
+                               rdata->d,
+                               harq_e,
+                               rdata->C,
+                               rv_index,
+                               *rdata->d_to_be_cleared,
+                               E,
+                               rdata->F,
+                               K - rdata->F - 2 * (p_decoderParms->Z))
+      == -1) {
+    stop_meas(rdata->p_ts_rate_unmatch);
+    LOG_E(PHY, "nrLDPC_coding_segment_decoder.c: Problem in rate_matching\n");
+
+    // Task completed
+    completed_task_ans(rdata->ans);
+    return;
+  }
+  stop_meas(rdata->p_ts_rate_unmatch);
+
+  *rdata->d_to_be_cleared = false;
+
+  p_decoderParms->crc_type = crcType(rdata->C, A);
+  p_decoderParms->Kprime = lenWithCrc(rdata->C, A);
+
+  // set first 2*Z_c bits to zeros
+
+  int16_t z[68 * 384 + 16] __attribute__((aligned(16)));
+
+  start_meas(rdata->p_ts_ldpc_decode);
+
+  memset(z, 0, 2 * rdata->Z * sizeof(*z));
+  // set Filler bits
+  memset(z + Kprime, 127, rdata->F * sizeof(*z));
+  // Move coded bits before filler bits
+  memcpy(z + 2 * rdata->Z, rdata->d, (Kprime - 2 * rdata->Z) * sizeof(*z));
+  // skip filler bits
+  memcpy(z + K, rdata->d + (K - 2 * rdata->Z), (Kc * rdata->Z - K) * sizeof(*z));
+  // Saturate coded bits before decoding into 8 bits values
+  simde__m128i *pv = (simde__m128i *)&z;
+  int8_t l[68 * 384 + 16] __attribute__((aligned(16)));
+  simde__m128i *pl = (simde__m128i *)&l;
+  for (int i = 0, j = 0; j < ((Kc * rdata->Z) >> 4) + 1; i += 2, j++) {
+    pl[j] = simde_mm_packs_epi16(pv[i], pv[i + 1]);
+  }
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////// nrLDPC_decoder /////////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  ////////////////////////////////// pl =====> llrProcBuf //////////////////////////////////
+  int decodeIterations =  
+    LDPCdecoder(p_decoderParms, 0, 0, 0, l, llrProcBuf, p_procTime, rdata->abort_decode);
+  if (decodeIterations < p_decoderParms->numMaxIter) {
+    memcpy(rdata->c, llrProcBuf, K >> 3);
+    *rdata->decodeSuccess = true;
+  } else {
+    memset(rdata->c, 0, K >> 3);
+    *rdata->decodeSuccess = false;
+  }
+  stop_meas(rdata->p_ts_ldpc_decode);
+
+  // Task completed
+  completed_task_ans(rdata->ans);
+}
+
+int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters,
+                               int pusch_id,
+                               thread_info_tm_t *t_info)
+{
+  nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+
+  *nrLDPC_TB_decoding_parameters->processedSegments = 0;
+  t_nrLDPC_dec_params decParams = {.check_crc = check_crc};
+  decParams.BG = nrLDPC_TB_decoding_parameters->BG;
+  decParams.Z = nrLDPC_TB_decoding_parameters->Z;
+  decParams.numMaxIter = nrLDPC_TB_decoding_parameters->max_ldpc_iterations;
+  decParams.outMode = 0;
+
+  for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
+    nrLDPC_decoding_parameters_t *rdata = &((nrLDPC_decoding_parameters_t *)t_info->buf)[t_info->len];
+    DevAssert(t_info->len < t_info->cap);
+    rdata->ans = t_info->ans;
+    t_info->len += 1;
+
+    decParams.R = nrLDPC_TB_decoding_parameters->segments[r].R;
+    rdata->decoderParms = decParams;
+    rdata->llr = nrLDPC_TB_decoding_parameters->segments[r].llr;
+    rdata->Kc = decParams.BG == 2 ? 52 : 68;
+    rdata->C = nrLDPC_TB_decoding_parameters->C;
+    rdata->E = nrLDPC_TB_decoding_parameters->segments[r].E;
+    rdata->A = nrLDPC_TB_decoding_parameters->A;
+    rdata->Qm = nrLDPC_TB_decoding_parameters->Qm;
+    rdata->K = nrLDPC_TB_decoding_parameters->K;
+    rdata->Z = nrLDPC_TB_decoding_parameters->Z;
+    rdata->F = nrLDPC_TB_decoding_parameters->F;
+    rdata->rv_index = nrLDPC_TB_decoding_parameters->rv_index;
+    rdata->tbslbrm = nrLDPC_TB_decoding_parameters->tbslbrm;
+    rdata->abort_decode = nrLDPC_TB_decoding_parameters->abort_decode;
+    rdata->d = nrLDPC_TB_decoding_parameters->segments[r].d;
+    rdata->d_to_be_cleared = nrLDPC_TB_decoding_parameters->segments[r].d_to_be_cleared;
+    rdata->c = nrLDPC_TB_decoding_parameters->segments[r].c;
+    rdata->decodeSuccess = &nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess;
+    rdata->p_ts_deinterleave = &nrLDPC_TB_decoding_parameters->segments[r].ts_deinterleave;
+    rdata->p_ts_rate_unmatch = &nrLDPC_TB_decoding_parameters->segments[r].ts_rate_unmatch;
+    rdata->p_ts_ldpc_decode = &nrLDPC_TB_decoding_parameters->segments[r].ts_ldpc_decode;
+
+    task_t t = {.func = &nr_process_decode_segment, .args = rdata};
+    pushTpool(nrLDPC_slot_decoding_parameters->threadPool, t);
+
+    LOG_D(PHY, "Added a block to decode, in pipe: %d\n", r);
+  }
+  return nrLDPC_TB_decoding_parameters->C;
+}
+
+int32_t nrLDPC_coding_init(void)
+{
+  cuda_support_init();
+  return 0;
+}
+
+int32_t nrLDPC_coding_shutdown(void)
+{
+  return 0;
+}
+
+int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters)
+{
+  int nbSegments = 0;
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+    nbSegments += nrLDPC_TB_decoding_parameters->C;
+  }
+  nrLDPC_decoding_parameters_t arr[nbSegments];
+  task_ans_t ans;
+  init_task_ans(&ans, nbSegments);
+  thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbSegments, .ans = &ans};
+
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    (void)nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters, pusch_id, &t_info);
+  }
+
+  // Execute thread pool tasks
+  join_task_ans(t_info.ans);
+
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+    for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
+      if (nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess) {
+        *nrLDPC_TB_decoding_parameters->processedSegments = *nrLDPC_TB_decoding_parameters->processedSegments + 1;
+      }
+    }
+  }
+  return 0;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder_cuda_GH.c

@@ -0,0 +1,322 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.0  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+ * \brief Top-level routines for decoding LDPC transport channels
+ */
+
+// [from gNB coding]
+#include "nr_rate_matching.h"
+#include "PHY/defs_gNB.h"
+#include "PHY/CODING/coding_extern.h"
+#include "PHY/CODING/coding_defs.h"
+#include "PHY/CODING/lte_interleaver_inline.h"
+#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "PHY/CODING/nrLDPC_extern.h"
+#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
+#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#include "PHY/NR_TRANSPORT/nr_ulsch.h"
+#include "PHY/NR_TRANSPORT/nr_dlsch.h"
+#include "SCHED_NR/sched_nr.h"
+#include "SCHED_NR/fapi_nr_l1.h"
+#include "defs.h"
+#include "common/utils/LOG/vcd_signal_dumper.h"
+#include "common/utils/LOG/log.h"
+
+#include <stdalign.h>
+#include <stdint.h>
+#include <syscall.h>
+#include <time.h>
+// #define gNB_DEBUG_TRACE
+
+#define OAI_LDPC_DECODER_MAX_NUM_LLR 27000 // 26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
+// #define DEBUG_CRC
+#ifdef DEBUG_CRC
+#define PRINT_CRC_CHECK(a) a
+#else
+#define PRINT_CRC_CHECK(a)
+#endif
+
+#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_interface.h"
+#include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface.h"
+
+/**
+ * \typedef nrLDPC_decoding_parameters_t
+ * \struct nrLDPC_decoding_parameters_s
+ * \brief decoding parameter of transport blocks
+ * \var decoderParms decoder parameters
+ * \var Qm modulation order
+ * \var Kc ratio between the number of columns in the parity check matrix and the lifting size
+ * it is fixed for a given base graph while the lifting size is chosen to have a sufficient number of columns
+ * \var rv_index
+ * \var max_number_iterations maximum number of LDPC iterations
+ * \var abort_decode pointer to decode abort flag
+ * \var tbslbrm transport block size LBRM in bytes
+ * \var A Transport block size (This is A from 38.212 V15.4.0 section 5.1)
+ * \var K Code block size at decoder output
+ * \var Z lifting size
+ * \var F filler bits size
+ * \var r segment index in TB
+ * \var E input llr segment size
+ * \var C number of segments
+ * \var llr input llr segment array
+ * \var d Pointers to code blocks before LDPC decoding (38.212 V15.4.0 section 5.3.2)
+ * \var d_to_be_cleared
+ * pointer to the flag used to clear d properly
+ * when true, clear d after rate dematching
+ * \var c Pointers to code blocks after LDPC decoding (38.212 V15.4.0 section 5.2.2)
+ * \var decodeSuccess pointer to the flag indicating that the decoding of the segment was successful
+ * \var ans pointer to task answer used by the thread pool to detect task completion
+ * \var p_ts_deinterleave pointer to deinterleaving time stats
+ * \var p_ts_rate_unmatch pointer to rate unmatching time stats
+ * \var p_ts_ldpc_decode pointer to decoding time stats
+ */
+typedef struct nrLDPC_decoding_parameters_s {
+  t_nrLDPC_dec_params decoderParms;
+
+  uint8_t Qm;
+
+  uint8_t Kc;
+  uint8_t rv_index;
+  decode_abort_t *abort_decode;
+
+  uint32_t tbslbrm;
+  uint32_t A;
+  uint32_t K;
+  uint32_t Z;
+  uint32_t F;
+
+  uint32_t C;
+
+  int E;
+  short *llr;
+  int16_t *d;
+  bool *d_to_be_cleared;
+  uint8_t *c;
+  bool *decodeSuccess;
+
+  task_ans_t *ans;
+
+  time_stats_t *p_ts_deinterleave;
+  time_stats_t *p_ts_rate_unmatch;
+  time_stats_t *p_ts_ldpc_decode;
+} nrLDPC_decoding_parameters_t;
+
+static void nr_process_decode_segment(void *arg)
+{
+  nrLDPC_decoding_parameters_t *rdata = (nrLDPC_decoding_parameters_t *)arg;
+  t_nrLDPC_dec_params *p_decoderParms = &rdata->decoderParms;
+  const int K = rdata->K;
+  const int Kprime = K - rdata->F;
+  const int A = rdata->A;
+  const int E = rdata->E;
+  const int Qm = rdata->Qm;
+  const int rv_index = rdata->rv_index;
+  const uint8_t Kc = rdata->Kc;
+  short *ulsch_llr = rdata->llr;
+  int8_t llrProcBuf[OAI_LDPC_DECODER_MAX_NUM_LLR] __attribute__((aligned(32)));
+
+  t_nrLDPC_time_stats procTime = {0};
+  t_nrLDPC_time_stats *p_procTime = &procTime;
+
+  ////////////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////// nr_deinterleaving_ldpc ///////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  //////////////////////////// ulsch_llr =====> ulsch_harq->e //////////////////////////////
+
+  start_meas(rdata->p_ts_deinterleave);
+
+  /// code blocks after bit selection in rate matching for LDPC code (38.212 V15.4.0 section 5.4.2.1)
+  int16_t harq_e[E];
+
+  nr_deinterleaving_ldpc(E, Qm, harq_e, ulsch_llr);
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  stop_meas(rdata->p_ts_deinterleave);
+
+  start_meas(rdata->p_ts_rate_unmatch);
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+  //////////////////////////////// nr_rate_matching_ldpc_rx ////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  ///////////////////////// ulsch_harq->e =====> ulsch_harq->d /////////////////////////
+
+  if (nr_rate_matching_ldpc_rx(rdata->tbslbrm,
+                               p_decoderParms->BG,
+                               p_decoderParms->Z,
+                               rdata->d,
+                               harq_e,
+                               rdata->C,
+                               rv_index,
+                               *rdata->d_to_be_cleared,
+                               E,
+                               rdata->F,
+                               K - rdata->F - 2 * (p_decoderParms->Z))
+      == -1) {
+    stop_meas(rdata->p_ts_rate_unmatch);
+    LOG_E(PHY, "nrLDPC_coding_segment_decoder.c: Problem in rate_matching\n");
+
+    // Task completed
+    completed_task_ans(rdata->ans);
+    return;
+  }
+  stop_meas(rdata->p_ts_rate_unmatch);
+
+  *rdata->d_to_be_cleared = false;
+
+  p_decoderParms->crc_type = crcType(rdata->C, A);
+  p_decoderParms->Kprime = lenWithCrc(rdata->C, A);
+  p_decoderParms->n_segments = rdata->C; //parameter for cuda stream
+
+  // set first 2*Z_c bits to zeros
+
+  int16_t z[68 * 384 + 16] __attribute__((aligned(16)));
+
+  start_meas(rdata->p_ts_ldpc_decode);
+
+  memset(z, 0, 2 * rdata->Z * sizeof(*z));
+  // set Filler bits
+  memset(z + Kprime, 127, rdata->F * sizeof(*z));
+  // Move coded bits before filler bits
+  memcpy(z + 2 * rdata->Z, rdata->d, (Kprime - 2 * rdata->Z) * sizeof(*z));
+  // skip filler bits
+  memcpy(z + K, rdata->d + (K - 2 * rdata->Z), (Kc * rdata->Z - K) * sizeof(*z));
+  // Saturate coded bits before decoding into 8 bits values
+  simde__m128i *pv = (simde__m128i *)&z;
+  int8_t l[68 * 384 + 16] __attribute__((aligned(16)));
+  simde__m128i *pl = (simde__m128i *)&l;
+  for (int i = 0, j = 0; j < ((Kc * rdata->Z) >> 4) + 1; i += 2, j++) {
+    pl[j] = simde_mm_packs_epi16(pv[i], pv[i + 1]);
+  }
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  //////////////////////////////////////////////////////////////////////////////////////////
+  ///////////////////////////////////// nrLDPC_decoder /////////////////////////////////////
+  //////////////////////////////////////////////////////////////////////////////////////////
+
+  ////////////////////////////////// pl =====> llrProcBuf //////////////////////////////////
+  int decodeIterations =  
+    LDPCdecoder(p_decoderParms, 0, 0, 0, l, llrProcBuf, p_procTime, rdata->abort_decode);
+  if (decodeIterations < p_decoderParms->numMaxIter) {
+    memcpy(rdata->c, llrProcBuf, K >> 3);
+    *rdata->decodeSuccess = true;
+  } else {
+    memset(rdata->c, 0, K >> 3);
+    *rdata->decodeSuccess = false;
+  }
+  stop_meas(rdata->p_ts_ldpc_decode);
+
+  // Task completed
+  completed_task_ans(rdata->ans);
+}
+
+int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters,
+                               int pusch_id,
+                               thread_info_tm_t *t_info)
+{
+  nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+
+  *nrLDPC_TB_decoding_parameters->processedSegments = 0;
+  t_nrLDPC_dec_params decParams = {.check_crc = check_crc};
+  decParams.BG = nrLDPC_TB_decoding_parameters->BG;
+  decParams.Z = nrLDPC_TB_decoding_parameters->Z;
+  decParams.numMaxIter = nrLDPC_TB_decoding_parameters->max_ldpc_iterations;
+  decParams.outMode = 0;
+
+  for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
+    nrLDPC_decoding_parameters_t *rdata = &((nrLDPC_decoding_parameters_t *)t_info->buf)[t_info->len];
+    DevAssert(t_info->len < t_info->cap);
+    rdata->ans = t_info->ans;
+    t_info->len += 1;
+
+    decParams.R = nrLDPC_TB_decoding_parameters->segments[r].R;
+    rdata->decoderParms = decParams;
+    rdata->llr = nrLDPC_TB_decoding_parameters->segments[r].llr;
+    rdata->Kc = decParams.BG == 2 ? 52 : 68;
+    rdata->C = nrLDPC_TB_decoding_parameters->C;
+    rdata->E = nrLDPC_TB_decoding_parameters->segments[r].E;
+    rdata->A = nrLDPC_TB_decoding_parameters->A;
+    rdata->Qm = nrLDPC_TB_decoding_parameters->Qm;
+    rdata->K = nrLDPC_TB_decoding_parameters->K;
+    rdata->Z = nrLDPC_TB_decoding_parameters->Z;
+    rdata->F = nrLDPC_TB_decoding_parameters->F;
+    rdata->rv_index = nrLDPC_TB_decoding_parameters->rv_index;
+    rdata->tbslbrm = nrLDPC_TB_decoding_parameters->tbslbrm;
+    rdata->abort_decode = nrLDPC_TB_decoding_parameters->abort_decode;
+    rdata->d = nrLDPC_TB_decoding_parameters->segments[r].d;
+    rdata->d_to_be_cleared = nrLDPC_TB_decoding_parameters->segments[r].d_to_be_cleared;
+    rdata->c = nrLDPC_TB_decoding_parameters->segments[r].c;
+    rdata->decodeSuccess = &nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess;
+    rdata->p_ts_deinterleave = &nrLDPC_TB_decoding_parameters->segments[r].ts_deinterleave;
+    rdata->p_ts_rate_unmatch = &nrLDPC_TB_decoding_parameters->segments[r].ts_rate_unmatch;
+    rdata->p_ts_ldpc_decode = &nrLDPC_TB_decoding_parameters->segments[r].ts_ldpc_decode;
+
+    task_t t = {.func = &nr_process_decode_segment, .args = rdata};
+    pushTpool(nrLDPC_slot_decoding_parameters->threadPool, t);
+
+    LOG_D(PHY, "Added a block to decode, in pipe: %d\n", r);
+  }
+  return nrLDPC_TB_decoding_parameters->C;
+}
+
+int32_t nrLDPC_coding_init(void)
+{
+  cuda_support_init();
+  return 0;
+}
+
+int32_t nrLDPC_coding_shutdown(void)
+{
+  return 0;
+}
+
+int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_decoding_parameters)
+{
+  int nbSegments = 0;
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+    nbSegments += nrLDPC_TB_decoding_parameters->C;
+  }
+  nrLDPC_decoding_parameters_t arr[nbSegments];
+  task_ans_t ans;
+  init_task_ans(&ans, nbSegments);
+  thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbSegments, .ans = &ans};
+
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    (void)nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters, pusch_id, &t_info);
+  }
+
+  // Execute thread pool tasks
+  join_task_ans(t_info.ans);
+
+  for (int pusch_id = 0; pusch_id < nrLDPC_slot_decoding_parameters->nb_TBs; pusch_id++) {
+    nrLDPC_TB_decoding_parameters_t *nrLDPC_TB_decoding_parameters = &nrLDPC_slot_decoding_parameters->TBs[pusch_id];
+    for (int r = 0; r < nrLDPC_TB_decoding_parameters->C; r++) {
+      if (nrLDPC_TB_decoding_parameters->segments[r].decodeSuccess) {
+        *nrLDPC_TB_decoding_parameters->processedSegments = *nrLDPC_TB_decoding_parameters->processedSegments + 1;
+      }
+    }
+  }
+  return 0;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c

@@ -59,6 +59,9 @@
  * \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits
  * \param Eoffset offset in number of bits of the first segment of the segment group within output
  */
+
+ int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters);
+
 static void write_task_output(uint8_t *f,
                               uint32_t E,
                               uint8_t *f2,
@@ -370,6 +373,7 @@ static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_s
     .tprep = nrLDPC_slot_encoding_parameters->tprep,
     .tparity = nrLDPC_slot_encoding_parameters->tparity,
     .toutput = nrLDPC_slot_encoding_parameters->toutput,
+    .tconcat = nrLDPC_slot_encoding_parameters->tconcat,
     .Kb = nrLDPC_TB_encoding_parameters->Kb,
     .Zc = nrLDPC_TB_encoding_parameters->Z,
     .BG = nrLDPC_TB_encoding_parameters->BG,
@@ -406,16 +410,20 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
     // Compute number of tasks to encode TB
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
-    size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
-    nbTasks += n_seg;
+    if (nrLDPC_TB_encoding_parameters->C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) {
+        nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters,nrLDPC_TB_encoding_parameters);
+    }
+    else {
+      size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
+      nbTasks += n_seg;
 
-    // Search for maximum E for sizing encoder output f and f2
-    for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
-      uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
-      Emax = E > Emax ? E : Emax;
+      // Search for maximum E for sizing encoder output f and f2
+      for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
+        uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
+        Emax = E > Emax ? E : Emax;
+      }
     }
   }
-
   // Create f and f2 to old encoding tasks outputs
   uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
   uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
@@ -432,6 +440,10 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
     // For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays
     // Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j]
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
+
+    if (nrLDPC_TB_encoding_parameters->C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) 
+      continue;
+
     size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
     uint8_t *f_2d[n_seg];
     uint8_t *f2_2d[n_seg];
@@ -449,16 +461,19 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
     completed_many_task_ans(&ans, nbTasks - nbEncode);
   }
   // Execute thread pool tasks
-  join_task_ans(&ans);
+  if (nbTasks > 0)  join_task_ans(&ans);
 
   // Write output
+  time_stats_t *tconcat = nrLDPC_slot_encoding_parameters->tconcat;
+  if(tconcat != NULL) start_meas(tconcat);
   nbTasks = 0;
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
     uint32_t C = nrLDPC_TB_encoding_parameters->C;
+    if (C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) 
+      continue;
     size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1));
 
-    time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput;
 
     for (int j = 0; j < n_seg; j++) {
       unsigned int macro_segment = j * 8;
@@ -478,7 +493,6 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
         }
       }
 
-      if(toutput != NULL) start_meas(toutput);
  
       uint32_t Eoffset=0;
       for (int s=0; s<macro_segment; s++)
@@ -494,10 +508,10 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
                         nrLDPC_TB_encoding_parameters->output,
                         Eoffset);
 
-      if(toutput != NULL) stop_meas(toutput);
     }
     nbTasks += n_seg;
   }
+  if(tconcat != NULL) stop_meas(tconcat);
 
   return 0;
 }

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c

@@ -0,0 +1,630 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.0  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
+ * \brief Top-level routines for implementing LDPC encoding of transport channels
+ */
+
+#include "nr_rate_matching.h"
+#include "PHY/defs_gNB.h"
+#include "PHY/CODING/coding_extern.h"
+#include "PHY/CODING/coding_defs.h"
+#include "PHY/CODING/lte_interleaver_inline.h"
+#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "PHY/CODING/nrLDPC_extern.h"
+#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
+#include "PHY/NR_TRANSPORT/nr_dlsch.h"
+#include "SCHED_NR/sched_nr.h"
+#include "common/utils/LOG/vcd_signal_dumper.h"
+#include "common/utils/LOG/log.h"
+#include "common/utils/nr/nr_common.h"
+#include <openair2/UTIL/OPT/opt.h>
+
+#include <syscall.h>
+
+#define DEBUG_LDPC_ENCODING
+//#define DEBUG_LDPC_ENCODING_FREE 1
+/*
+static void write_task_output(uint8_t *f,
+                              uint32_t E,
+                              uint8_t *f2,
+                              uint32_t E2,
+                              bool Eshift,
+                              uint32_t E2_first_segment,
+                              uint32_t nb_segments,
+                              uint8_t *output,
+                              uint32_t Eoffset)
+{
+
+#if defined(__AVX512VBMI__)
+  uint64_t *output_p = (uint64_t*)output;
+  __m512i inc = _mm512_set1_epi8(0x1);
+
+  for (int i=0;i<E2;i+=64) {
+    uint32_t Eoffset2 = Eoffset;
+    __m512i bitperm = _mm512_set1_epi64(0x3830282018100800);
+    if (i<E) {
+      for (int j=0; j < E2_first_segment; j++) {
+        // Note: Here and below for AVX2, we are using the 64-bit SIMD instruction
+        // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
+        // and >> operations are undefined and in fact don't give "0" which is
+        // what we want here. The SIMD version do give 0 when the shift is 64
+        uint32_t Eoffset2_byte = Eoffset2 >> 6;
+        uint32_t Eoffset2_bit = Eoffset2 & 63;
+        __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i >> 6],bitperm);
+        *(__m64*)(output_p + Eoffset2_byte)   = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
+        *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
+        Eoffset2 += E;
+        bitperm = _mm512_add_epi8(bitperm ,inc);
+      }
+    } else {
+      for (int j=0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+        bitperm = _mm512_add_epi8(bitperm ,inc);
+      }
+    }
+    for (int j=E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 6;
+      uint32_t Eoffset2_bit = Eoffset2 & 63;
+      __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i >> 6],bitperm);
+      *(__m64*)(output_p + Eoffset2_byte)   = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
+      *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
+      Eoffset2 += E2;
+      bitperm = _mm512_add_epi8(bitperm ,inc);
+    }
+    output_p++;
+  }
+
+#elif defined(__aarch64__)
+  uint16_t *output_p = (uint16_t*)output;
+  const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = {
+    {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7},     // segment 0
+    {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6},   // segment 1
+    {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2
+    {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3
+    {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4
+    {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5
+    {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6
+    {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7
+  const uint8_t __attribute__ ((aligned (16))) masks[16] = 
+      {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80};
+  int8x16_t vshift[8];
+  for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]);
+  uint8x16_t vmask  = vld1q_u8(masks);
+
+  for (int i=0;i<E2;i+=16) {
+    uint32_t Eoffset2 = Eoffset;
+    if (i<E) {	
+      for (int j=0; j < E2_first_segment; j++) {
+        uint32_t Eoffset2_byte = Eoffset2 >> 4;
+        uint32_t Eoffset2_bit = Eoffset2 & 15;
+        uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[i >> 4],vshift[j]),vmask);
+        int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
+        tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
+        *(output_p + Eoffset2_byte)   |= (uint16_t)(tmp<<Eoffset2_bit);
+        *(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
+        Eoffset2 += E;
+      }
+    } else {
+      for (int j=0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+      }
+    }
+    for (int j=E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 4;
+      uint32_t Eoffset2_bit = Eoffset2 & 15;
+      uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i >> 4],vshift[j]),vmask);
+      int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
+      tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
+      *(output_p + Eoffset2_byte)   |= (uint16_t)(tmp<<Eoffset2_bit);
+      *(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
+      Eoffset2 += E2;
+    }
+    output_p++;
+  }
+       
+#else
+  uint32_t *output_p = (uint32_t*)output;
+
+  for (int i=0; i < E2; i += 32) {
+    uint32_t Eoffset2 = Eoffset;
+    if (i < E) {
+      for (int j = 0; j < E2_first_segment; j++) {
+        // Note: Here and below, we are using the 64-bit SIMD instruction
+        // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
+        // and >> operations are undefined and in fact don't give "0" which is
+        // what we want here. The SIMD version do give 0 when the shift is 64
+        uint32_t Eoffset2_byte = Eoffset2 >> 5;
+        uint32_t Eoffset2_bit = Eoffset2 & 31;
+        int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i >> 5], 7 - j));
+        __m64 tmp64 = _mm_set1_pi32(tmp);
+        __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
+        __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
+        __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
+        *(output_p + Eoffset2_byte) = _m_to_int(tmp64b);
+        *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
+        Eoffset2 += E;
+      }
+    } else {
+      for (int j = 0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+      }
+    } 
+    for (int j = E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 5;
+      uint32_t Eoffset2_bit = Eoffset2 & 31;
+      int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i >> 5], 7 - j));
+      __m64 tmp64 = _mm_set1_pi32(tmp);
+      __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
+      __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
+      __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
+      *(output_p + Eoffset2_byte)  = _m_to_int(tmp64b);
+      *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
+      Eoffset2 += E2;
+    }
+    output_p++;
+  }
+
+  
+#endif
+}
+*/
+
+
+static void unpack_output(uint32_t *f,
+                         uint32_t E,
+                         uint32_t *f2,
+                         uint32_t E2,
+			 uint32_t E2_first_segment32,
+                         uint32_t E2_first_segment,
+                         uint32_t nb_segments,
+                         uint8_t *output) {
+
+
+  int s;
+ // int s0;
+  uint32_t *fp;
+  int foffset;
+  uint32_t *output_p = (uint32_t *)output;
+//  printf("E %d, E2 %d, E2_first_segment %d, E2_first_segment32 %d, nb_segments %d\n",E,E2,E2_first_segment,E2_first_segment32,nb_segments);
+  uint32_t bit_index = 0;
+#if 0
+  const int32_t ucShift0[32][4] = { {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}, {-28,-27,-26,-25}, {-29,-28,-27,-26}, {-30,-29,-28,-27}, {-31,-30,-29,-28}}; 
+
+  const int32_t ucShift1[32][4] = { {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}}; 
+
+  const int32_t ucShift2[32][4] = { {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19},{-23,-22,-21,-20}}; 
+
+  const int32_t ucShift3[32][4] = { {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}}; 
+
+  const int32_t ucShift4[32][4] = { {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}}; 
+
+  const int32_t ucShift5[32][4] = { {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}}; 
+
+  const int32_t ucShift6[32][4] = { {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}}; 
+
+  const int32_t ucShift7[32][4] = { {28,29,30,31}, {27,28,29,30}, {26,27,28,29}, {25,26,27,28}, {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}}; 
+  const uint32_t __attribute__ ((aligned (16))) masks0[4] = {0x1,0x2,0x4,0x8};
+  const uint32_t __attribute__ ((aligned (16))) masks1[4] = {0x10,0x20,0x40,0x80};
+  const uint32_t __attribute__ ((aligned (16))) masks2[4] = {0x100,0x200,0x400,0x800};
+  const uint32_t __attribute__ ((aligned (16))) masks3[4] = {0x1000,0x2000,0x4000,0x8000};
+  const uint32_t __attribute__ ((aligned (16))) masks4[4] = {0x10000,0x20000,0x40000,0x80000};
+  const uint32_t __attribute__ ((aligned (16))) masks5[4] = {0x100000,0x200000,0x400000,0x800000};
+  const uint32_t __attribute__ ((aligned (16))) masks6[4] = {0x1000000,0x2000000,0x4000000,0x8000000};
+  const uint32_t __attribute__ ((aligned (16))) masks7[4] = {0x10000000,0x20000000,0x40000000,0x80000000};
+  int32x4_t vshift0[32],vshift1[32],vshift2[32],vshift3[32],vshift4[32],vshift5[32],vshift6[32],vshift7[32];
+  for (int n=0;n<32;n++) {
+	  vshift0[n] = vld1q_s32(ucShift0[n]);
+	  vshift1[n] = vld1q_s32(ucShift1[n]);
+	  vshift2[n] = vld1q_s32(ucShift2[n]);
+	  vshift3[n] = vld1q_s32(ucShift3[n]);
+	  vshift4[n] = vld1q_s32(ucShift4[n]);
+	  vshift5[n] = vld1q_s32(ucShift5[n]);
+	  vshift6[n] = vld1q_s32(ucShift6[n]);
+	  vshift7[n] = vld1q_s32(ucShift7[n]);
+  }
+  uint32x4_t vmask0  = vld1q_u32(masks0);
+  uint32x4_t vmask1  = vld1q_u32(masks1);
+  uint32x4_t vmask2  = vld1q_u32(masks2);
+  uint32x4_t vmask3  = vld1q_u32(masks3);
+  uint32x4_t vmask4  = vld1q_u32(masks4);
+  uint32x4_t vmask5  = vld1q_u32(masks5);
+  uint32x4_t vmask6  = vld1q_u32(masks6);
+  uint32x4_t vmask7  = vld1q_u32(masks7);
+  uint32_t output_tmp=0;
+  int s2=0;
+  for (s = 0; s < E2_first_segment ; s++) {
+    s2 = s&31;	  
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<Emod32)-1);
+        bit_index+=Emod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<Emod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=Emod32;
+      }
+    }
+  }
+//  s0 = s;
+  for ( ; s < nb_segments ; s++){
+    s2 = s&31;	  
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        *(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<E2mod32)-1);
+        bit_index+=E2mod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<E2mod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=E2mod32;
+      }
+    }
+  }
+#else // non SIMD version
+int s0;
+  int segpos;	
+  for (s = 0; s < E2_first_segment ; s++) {
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    segpos = (1<<s);
+    for (int i = 0; i < E; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      bit_index++;
+    }
+  }
+  s0 = s;
+  for ( ; s < nb_segments ; s++){
+    foffset = ((s-s0)>>5)*E2;
+    fp = f2+foffset;
+    segpos = (1<<s);
+    for (int i = 0; i < E2; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      bit_index++;
+    }
+  }
+#endif  
+}
+/**
+ * \typedef ldpc8blocks_args_t
+ * \struct ldpc8blocks_args_s
+ * \brief Arguments of an encoding task
+ * encode up to 8 code blocks
+ * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
+ * \var impp encoder implementation specific parameters for the task
+ * \var f first interleaver output to be filled by the task
+ * \var f2 second interleaver output to be filled by the task
+ * in case of a shift of E in the code blocks group processed by the task
+ */
+
+static void ldpcnblocks(nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters, encoder_implemparams_t impp)
+{
+ 
+  uint8_t mod_order = nrLDPC_TB_encoding_parameters->Qm;
+  uint16_t nb_rb = nrLDPC_TB_encoding_parameters->nb_rb;
+  uint32_t A = nrLDPC_TB_encoding_parameters->A;
+
+  unsigned int G = nrLDPC_TB_encoding_parameters->G;
+  LOG_D(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp.K, G, nb_rb, (int)mod_order);
+
+  // nrLDPC_encoder output is in "d"
+  // let's make this interface happy!
+  uint32_t d[4][68*384];
+  uint8_t *c[nrLDPC_TB_encoding_parameters->C];
+
+  
+  for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++)
+    c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  LDPCencoder32(c, d, &impp);
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  // Compute where to place in output buffer that is concatenation of all segments
+
+#ifdef DEBUG_LDPC_ENCODING
+  LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
+#endif
+  const uint32_t E = nrLDPC_TB_encoding_parameters->segments[0].E;
+  uint32_t E2=E;
+  uint32_t Emax = E;
+  int n_seg   = nrLDPC_TB_encoding_parameters->C>>5;
+  int n_seg2  = n_seg;
+  if ((nrLDPC_TB_encoding_parameters->C & 31) > 0) n_seg2++;
+  int r_shift = n_seg2; 
+  int r_shift2 = nrLDPC_TB_encoding_parameters->C;
+  for (int s=0;s<nrLDPC_TB_encoding_parameters->C;s++) {
+      //printf("segment %d E %d\n",s,nrLDPC_TB_encoding_parameters->segments[s].E);	  
+      if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
+	 E2=nrLDPC_TB_encoding_parameters->segments[s].E;
+         if(E2 > Emax)
+           Emax = E2;
+	 r_shift = s>>5;
+	 r_shift2 = s;
+	// printf("r_shift %d, r_shift2 %d\n",r_shift,r_shift2);
+         break;
+      }	 
+  }    
+
+  LOG_D(NR_PHY,
+        "Rate Matching, Code segment %d...%d r_shift %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        0,
+        impp.n_segments-1,
+	r_shift,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+/*
+  printf("Rate Matching, Code segment 0..%d r_shift %d r_shift2 %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        impp.n_segments-1,
+	r_shift,
+	r_shift2,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+*/
+
+  uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
+
+  uint32_t e[E*(r_shift+1)];
+  uint32_t e2[E2*(n_seg2-r_shift)];
+  uint32_t f[E*(r_shift+1)];
+  uint32_t f2[E2*(n_seg2-r_shift)];
+
+  // Interleaver outputs are stored in the output arrays
+  uint8_t *output = nrLDPC_TB_encoding_parameters->output;
+
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  memset(e,0,sizeof(e));
+  memset(f,0,sizeof(f));
+  if (1/*r_shift < n_seg2*/) { 
+    memset(e2,0,sizeof(e2));
+    memset(f2,0,sizeof(f2));
+  }
+
+  for (int r=0;r<n_seg2;r++) {
+    if (r<=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d[r],
+                              e+(r*E),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E);
+    if (r>=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d[r],
+                              e2+((r-r_shift)*E2),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E2);
+   /* 
+    if (r==(n_seg2-1)) {
+	    for (int i=0;i<16;i++) printf("rm: %x %x\n",d[n_seg2-1][i],e2[((n_seg2-1)*E2)+i]);
+    }
+    */
+  }
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  if (impp.K - impp.F - 2 * impp.Zc > E) {
+    LOG_E(PHY,
+          "dlsch coding A %d  Kr %d G %d (nb_rb %d, mod_order %d)\n",
+          A,
+          impp.K,
+          G,
+          nb_rb,
+          (int)mod_order);
+
+    LOG_E(NR_PHY,
+          "Rate Matching, Code segments 0..%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
+          "nb_rb %d)...\n",
+          impp.n_segments,
+          G,
+          E,
+          impp.K,
+          impp.F,
+          impp.K - impp.F - 2 * impp.Zc,
+          mod_order,
+          nb_rb);
+  }
+  
+  //printf("interleaving r_shift %d, n_seg2 %d\n",r_shift,n_seg2);
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+  
+  for (int r=0;r<=r_shift;r++)
+    nr_interleaving_ldpc32(E,
+                           mod_order,
+                           e+E*r,
+                           f+E*r);
+
+  for (int r=r_shift;r<n_seg2;r++)
+    nr_interleaving_ldpc32(E2,
+                           mod_order,
+                           e2+E2*(r-r_shift),
+                           f2+E2*(r-r_shift));
+/*
+  for (int i=0;i<16;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  printf("-------------------\n");
+  for (int i=E2-16;i<E2;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  */
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+
+  if (impp.tconcat) start_meas(impp.tconcat);
+  unpack_output(f,E,f2,E2,r_shift,r_shift2,nrLDPC_TB_encoding_parameters->C,output);
+  if (impp.tconcat) stop_meas(impp.tconcat);
+
+}
+
+int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters)
+{
+
+
+    encoder_implemparams_t common_segment_params = {
+      .n_segments = nrLDPC_TB_encoding_parameters->C,
+      .tinput = nrLDPC_slot_encoding_parameters->tinput,
+      .tprep = nrLDPC_slot_encoding_parameters->tprep,
+      .tparity = nrLDPC_slot_encoding_parameters->tparity,
+      .toutput = nrLDPC_slot_encoding_parameters->toutput,
+      .tconcat = nrLDPC_slot_encoding_parameters->tconcat,
+      .Kb = nrLDPC_TB_encoding_parameters->Kb,
+      .Zc = nrLDPC_TB_encoding_parameters->Z,
+      .BG = nrLDPC_TB_encoding_parameters->BG,
+      .output = nrLDPC_TB_encoding_parameters->output, 
+      .K = nrLDPC_TB_encoding_parameters->K,
+      .F = nrLDPC_TB_encoding_parameters->F,
+    };
+
+    LOG_D(NR_PHY,"Calling ldpcnblocks (C %d, Z %d, K %d)\n",common_segment_params.n_segments,common_segment_params.Zc,common_segment_params.K);
+    ldpcnblocks(nrLDPC_TB_encoding_parameters, common_segment_params);
+
+
+
+  return 0;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c

@@ -24,7 +24,8 @@
 #include "common/utils/LOG/log.h"
 
 // #define RM_DEBUG 1
-
+#define USE128BIT
+ 
 static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
 
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
@@ -359,14 +360,15 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
       f = (uint8_t *)f_512;
 #endif
 #ifdef USE128BIT
-      e0_128 = (simde__m128i *)e0;
-      e1_128 = (simde__m128i *)e1;
-      e2_128 = (simde__m128i *)e2;
-      e3_128 = (simde__m128i *)e3;
-      e4_128 = (simde__m128i *)e4;
-      e5_128 = (simde__m128i *)e5;
-      e6_128 = (simde__m128i *)e6;
-      e7_128 = (simde__m128i *)e7;
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f_128  = (simde__m128i *)f;
       for (; i < (EQm & ~15); i += 16) {
         simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
         simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
@@ -444,6 +446,452 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
   }
 }
 
+void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
+{
+  const uint32_t EQm = E / Qm;
+  memset(f, 0, E * sizeof(uint32_t));
+  switch(Qm) {
+    case 2: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      int i = 0;
+      /*
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24,
+                                          87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16,
+                                          79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8,
+                                          71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120,
+                                          56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112,
+                                          48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104,
+                                          40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      // clang-format on
+      __m512i *f_512 = (__m512i *)f;
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      f = (uint8_t *)f_512;
+#endif
+#ifdef USE128BIT
+      simde__m128i *f_128 = (simde__m128i *)f;
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      for (; i < (EQm & ~15); i += 64) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j));
+      }
+      e0 = (uint8_t *)e0_128;
+      e1 = (uint8_t *)e1_128;
+      f = (uint8_t *)f_128;
+#endif
+*/
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+      }
+    } break;
+    case 4: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      int i = 0;
+      /*
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
+                                          22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
+                                          76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
+                                          55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47,
+                                          110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102,
+                                          38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
+                                            37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
+                                            22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
+      }
+      
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      f = (uint8_t *)f_512;
+#endif
+#ifdef USE128BIT
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *f_128 = (simde__m128i *)f;
+      for (; i < (EQm & ~15); i += 16) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
+        simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
+        simde_mm_storeu_si128(f_128++,
+                              simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        simde_mm_storeu_si128(
+            f_128++,
+            simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
+        tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1));
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1));
+      }
+      e0 = (uint8_t *)e0_128;
+      e1 = (uint8_t *)e1_128;
+      e2 = (uint8_t *)e2_128;
+      e3 = (uint8_t *)e3_128;
+      f = (uint8_t *)f_128;
+#endif
+      */
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+      }
+    } break;
+    case 6: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      int i = 0;
+      /*
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
+                                          86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
+                                          77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
+                                          55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110,
+                                          46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+      const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0);
+      const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42);
+      const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0,
+                                             15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *e4_512 = (__m512i *)e4;
+      __m512i *e5_512 = (__m512i *)e5;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i e4j = _mm512_loadu_si512(e4_512++);
+        __m512i e5j = _mm512_loadu_si512(e5_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31)
+        __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        __m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15)
+        // e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
+        __m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16)  e0(i+20)
+        // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
+        // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
+
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63)
+        tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63)
+        tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
+        tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16)  e0(i+20)
+        // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
+        // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      e4 = (uint8_t *)e4_512;
+      e5 = (uint8_t *)e5_512;
+      f = (uint8_t *)f_512;
+#endif
+      */
+#ifdef USE128BIT
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *f128 = (simde__m128i *)f;
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i) e1(i) e2(i) e3(i) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3) 
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3) 
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i+2) e1(i+2) e2(i+2) e3(i+2) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      f  = (uint32_t *)f128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+      }
+    } break;
+    case 8: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      uint32_t *e6 = e5 + EQm;
+      uint32_t *e7 = e6 + EQm;
+      
+      int i = 0;
+      /*
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
+                                          86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
+                                          76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55,
+                                          118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46,
+                                          109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+
+      const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+      const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *e4_512 = (__m512i *)e4;
+      __m512i *e5_512 = (__m512i *)e5;
+      __m512i *e6_512 = (__m512i *)e6;
+      __m512i *e7_512 = (__m512i *)e7;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i e4j = _mm512_loadu_si512(e4_512++);
+        __m512i e5j = _mm512_loadu_si512(e5_512++);
+        __m512i e6j = _mm512_loadu_si512(e6_512++);
+        __m512i e7j = _mm512_loadu_si512(e7_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
+        __m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
+        __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3)
+        __m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
+        // e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)...
+        // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+        // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
+        // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11)
+        // e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15)
+        // e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
+        tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
+        tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
+        // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
+        // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19)
+        // e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23)
+        // e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+        // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
+        // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27)
+        // e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31)
+        // e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      e4 = (uint8_t *)e4_512;
+      e5 = (uint8_t *)e5_512;
+      e6 = (uint8_t *)e6_512;
+      e7 = (uint8_t *)e7_512;
+      f = (uint8_t *)f_512;
+#endif
+      */
+#ifdef USE128BIT
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f128   = (simde__m128i *)f;
+
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+        simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
+        simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1) 
+        simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j);   // e6(i) e7(i) e6(i+1) e7(i+1)
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1));  // e0(i+1) e1(i+1) e2(i+1) e3(i+1) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
+									   //
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3) 
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3) 
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3) 
+        tmp3 = simde_mm_unpackhi_epi32(e6j, e7j);   // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      e6 = (uint32_t *)e6_128;
+      e7 = (uint32_t *)e7_128;
+      f = (uint32_t *)f128;
+#endif
+      
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+        *f++ = *e6++;
+        *f++ = *e7++;
+      }
+    } break;
+    default:
+      AssertFatal(false, "Should be here!\n");
+  }
+}
 void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
 {
   switch (Qm) {
@@ -610,6 +1058,103 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
   return 0;
 }
 
+int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
+                            uint8_t BG,
+                            uint16_t Z,
+                            uint32_t *d,
+                            uint32_t *e,
+                            uint8_t C,
+                            uint32_t F,
+                            uint32_t Foffset,
+                            uint8_t rvidx,
+                            uint32_t E)
+{
+  if (C == 0) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
+    return -1;
+  }
+
+  //Bit selection
+  uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
+  uint32_t Ncb;
+  if (Tbslbrm == 0)
+    Ncb = N;
+  else {
+    uint32_t Nref = 3 * Tbslbrm / (2 * C); //R_LBRM = 2/3
+    Ncb = min(N, Nref);
+  }
+
+  uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
+
+#ifdef RM_DEBUG
+  printf("nr_rate_matching_ldpc: E %u, F %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
+         E,
+         F,
+         Foffset,
+         ind,
+         Ncb,
+         rvidx,
+         Tbslbrm);
+#endif
+
+  if (Foffset > E) {
+    LOG_E(PHY,
+          "nr_rate_matching: invalid parameters (Foffset %d > E %d) F %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n",
+          Foffset,
+          E,
+          F,
+          ind,
+          Ncb,
+          rvidx,
+          Tbslbrm);
+    return -1;
+  }
+  if (Foffset > Ncb) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
+    return -1;
+  }
+
+  if (ind >= Foffset && ind < (F + Foffset))
+    ind = F + Foffset;
+
+  uint32_t k = 0;
+  if (ind < Foffset) { // case where we have some bits before the filler and the rest after
+    memcpy((void *)e, (void *)(d + ind), (Foffset - ind)<<2);
+
+    if (E + F <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (E - Foffset + ind)<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (Ncb - Foffset - F)<<2);
+      k = Ncb - F - ind;
+    }
+  } else {
+    if (E <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e), (void *)(d + ind), E<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e), (void *)(d + ind), (Ncb - ind)<<2);
+      k = Ncb - ind;
+    }
+  }
+
+  while (k < E) { // case where we do repetitions (low mcs)
+    for (ind = 0; (ind < Ncb) && (k < E); ind++) {
+#ifdef RM_DEBUG
+      printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
+#endif
+
+      if (ind == Foffset)
+        ind = F + Foffset; // skip filler bits
+
+      e[k++] = d[ind];
+
+    }
+  }
+
+  return 0;
+}
+
 int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
                              uint8_t BG,
                              uint16_t Z,

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h

@@ -33,6 +33,15 @@
  */
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f);
 
+/**
+ * \brief interleave a code segment after encoding and rate matching (32bit)
+ * \param E size of the code segment in bits
+ * \param Qm modulation order
+ * \param e input rate matched segment
+ * \param f output interleaved segment
+ */
+
+void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f);
 /**
  * \brief deinterleave a code segment before RX rate matching and decoding
  * \param E size of the code segment in bits
@@ -66,6 +75,30 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
                           uint8_t rvidx,
                           uint32_t E);
 
+/**
+ * \brief rate match a code segment after encoding (32bit)
+ * \Tbslbrm Transport Block size LBRM
+ * \param BG LDPC base graph number
+ * \param Z segment lifting size
+ * \param d input encoded segment
+ * \param e output rate matched segment
+ * \param C number of segments in the Transport Block
+ * \param F number of filler bits in the segment
+ * \param Foffset offset of the filler bits in the segment
+ * \param rvidx redundancy version index
+ * \param E size of the code segment in bits
+ */
+int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
+                            uint8_t BG,
+                            uint16_t Z,
+                            uint32_t *d,
+                            uint32_t *e,
+                            uint8_t C,
+                            uint32_t F,
+                            uint32_t Foffset,
+                            uint8_t rvidx,
+                            uint32_t E);
+
 /**
  * \brief rate match a code segment before decoding
  * \Tbslbrm Transport Block size LBRM

openair1/PHY/CODING/nrLDPC_decoder/cuda_test.cu

@@ -0,0 +1,38 @@
+#include <stdio.h>
+#include <cuda_runtime.h>
+
+__global__ void hello_kernel(int id) {
+    if (threadIdx.x == 0 && blockIdx.x == 0) {
+        printf("Hello from GPU kernel, id=%d\n", id);
+    }
+}
+
+int main() {
+    // 查询 GPU 属性
+    int device;
+    cudaGetDevice(&device);
+    cudaDeviceProp prop;
+    cudaGetDeviceProperties(&prop, device);
+    printf("Using device %d: %s (compute capability %d.%d)\n",
+           device, prop.name, prop.major, prop.minor);
+
+    // 启动 kernel
+    hello_kernel<<<1,1>>>(42);
+
+    // 检查 launch 是否成功
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        printf("Kernel launch error: %s\n", cudaGetErrorString(err));
+        return -1;
+    }
+
+    // 同步，等待 GPU 执行完
+    err = cudaDeviceSynchronize();
+    if (err != cudaSuccess) {
+        printf("Kernel execution error: %s\n", cudaGetErrorString(err));
+        return -1;
+    }
+
+    printf("Kernel finished successfully!\n");
+    return 0;
+}

openair1/PHY/CODING/nrLDPC_decoder/decoder_graphs.h

@@ -0,0 +1,14 @@
+#pragma once
+#include <cuda_runtime.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern cudaGraph_t decoderGraphs[MAX_NUM_DLSCH_SEGMENTS];
+extern cudaGraphExec_t decoderGraphExec[MAX_NUM_DLSCH_SEGMENTS];
+extern bool graphCreated[MAX_NUM_DLSCH_SEGMENTS];
+
+#ifdef __cplusplus
+}
+#endif

openair1/PHY/CODING/nrLDPC_decoder/kernel_test.cu

@@ -0,0 +1,18 @@
+#if PARALLEL_STREAM
+#include <cstdio>
+
+__global__ void test_kernel() {
+    if (threadIdx.x == 0 && blockIdx.x == 0) {
+        printf("Hello from test kernel!\n");
+    }
+}
+
+extern "C" void run_test_kernel() {
+    test_kernel<<<1,1>>>();
+    cudaError_t err = cudaGetLastError();
+    if (err != cudaSuccess) {
+        printf("Kernel launch failed: %s\n", cudaGetErrorString(err));
+    }
+    cudaDeviceSynchronize();
+}
+#endif

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_BnToCnPC_Kernel_BG1_cuda.h

@@ -103,7 +103,7 @@ __device__ void CnToBnPC_Kernel_int8_G3(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G3, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G3, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -171,7 +171,7 @@ __device__ void CnToBnPC_Kernel_int8_G4(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G4, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G4, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -239,7 +239,7 @@ __device__ void CnToBnPC_Kernel_int8_G5(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G5, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G5, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -306,7 +306,7 @@ __device__ void CnToBnPC_Kernel_int8_G6(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G6, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G6, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -373,7 +373,7 @@ __device__ void CnToBnPC_Kernel_int8_G7(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G7, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G7, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -440,7 +440,7 @@ __device__ void CnToBnPC_Kernel_int8_G8(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G8, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G8, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -507,7 +507,7 @@ __device__ void CnToBnPC_Kernel_int8_G9(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G9, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G9, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -574,7 +574,7 @@ __device__ void CnToBnPC_Kernel_int8_G10(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G10, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G10, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -672,7 +672,7 @@ __device__ void CnToBnPC_Kernel_int8_G19(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (Tid % warpSize == 0) {
-        //printf("It's wrong here G19, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G19, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -680,15 +680,15 @@ __device__ void CnToBnPC_Kernel_int8_G19(const t_nrLDPC_lut *p_lut,
 }
 //------------------------------Stream Version------------------------
 __device__ void CnToBnPC_Kernel_int8_G3_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 3; // Gn = 3
   if (tid >= NUM * Zc / 4)
@@ -738,15 +738,15 @@ __device__ void CnToBnPC_Kernel_int8_G3_Stream(const t_nrLDPC_lut *p_lut,
 }
 
 __device__ void CnToBnPC_Kernel_int8_G4_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 4; // Gn = 4
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -784,7 +784,7 @@ __device__ void CnToBnPC_Kernel_int8_G4_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
-uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 4; i++) {
@@ -806,15 +806,15 @@ uint32_t pcRes = 0;
 }
 
 __device__ void CnToBnPC_Kernel_int8_G5_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 5; // Gn = 5
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -852,7 +852,7 @@ __device__ void CnToBnPC_Kernel_int8_G5_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 5; i++) {
@@ -866,22 +866,22 @@ __device__ void CnToBnPC_Kernel_int8_G5_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G5, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G5, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G6_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 6; // Gn = 6
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -919,7 +919,7 @@ __device__ void CnToBnPC_Kernel_int8_G6_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 6; i++) {
@@ -933,22 +933,22 @@ __device__ void CnToBnPC_Kernel_int8_G6_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G6, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G6, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G7_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 7; // Gn = 7
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -986,7 +986,7 @@ __device__ void CnToBnPC_Kernel_int8_G7_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 7; i++) {
@@ -1000,22 +1000,22 @@ __device__ void CnToBnPC_Kernel_int8_G7_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G7, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G7, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G8_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 8; // Gn = 8
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -1053,7 +1053,7 @@ __device__ void CnToBnPC_Kernel_int8_G8_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 8; i++) {
@@ -1067,22 +1067,22 @@ __device__ void CnToBnPC_Kernel_int8_G8_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G8, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G8, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G9_Stream(const t_nrLDPC_lut *p_lut,
-                                        int8_t *__restrict__ d_bnOutAll,
-                                        const int8_t *__restrict__ d_cnBufAll,
-                                        int8_t *__restrict__ d_cnOutAll,
-                                        int8_t *__restrict__ d_bnBufAll,
-                                        int tid,
-                                        uint8_t groupId,
-                                        uint8_t CnIdx,
-                                        int Zc,
-                                        int *PC_Flag)
+                                               int8_t *__restrict__ d_bnOutAll,
+                                               const int8_t *__restrict__ d_cnBufAll,
+                                               int8_t *__restrict__ d_cnOutAll,
+                                               int8_t *__restrict__ d_bnBufAll,
+                                               int tid,
+                                               uint8_t groupId,
+                                               uint8_t CnIdx,
+                                               int Zc,
+                                               int *PC_Flag)
 {
   const uint8_t NUM = 9; // Gn = 9
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -1120,7 +1120,7 @@ __device__ void CnToBnPC_Kernel_int8_G9_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 9; i++) {
@@ -1134,22 +1134,22 @@ __device__ void CnToBnPC_Kernel_int8_G9_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G9, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G9, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G10_Stream(const t_nrLDPC_lut *p_lut,
-                                         int8_t *__restrict__ d_bnOutAll,
-                                         const int8_t *__restrict__ d_cnBufAll,
-                                         int8_t *__restrict__ d_cnOutAll,
-                                         int8_t *__restrict__ d_bnBufAll,
-                                         int tid,
-                                         uint8_t groupId,
-                                         uint8_t CnIdx,
-                                         int Zc,
-                                         int *PC_Flag)
+                                                int8_t *__restrict__ d_bnOutAll,
+                                                const int8_t *__restrict__ d_cnBufAll,
+                                                int8_t *__restrict__ d_cnOutAll,
+                                                int8_t *__restrict__ d_bnBufAll,
+                                                int tid,
+                                                uint8_t groupId,
+                                                uint8_t CnIdx,
+                                                int Zc,
+                                                int *PC_Flag)
 {
   const uint8_t NUM = 10; // Gn = 10
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -1187,7 +1187,7 @@ __device__ void CnToBnPC_Kernel_int8_G10_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (tid < 96) {
     for (int i = 0; i < 10; i++) {
@@ -1201,22 +1201,22 @@ __device__ void CnToBnPC_Kernel_int8_G10_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (tid % warpSize == 0) {
-        //printf("It's wrong here G10, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G10, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
   }
 }
 __device__ void CnToBnPC_Kernel_int8_G19_Stream(const t_nrLDPC_lut *p_lut,
-                                         int8_t *__restrict__ d_bnOutAll,
-                                         const int8_t *__restrict__ d_cnBufAll,
-                                         int8_t *__restrict__ d_cnOutAll,
-                                         int8_t *__restrict__ d_bnBufAll,
-                                         int Tid,
-                                         uint8_t groupId,
-                                         uint8_t CnIdx,
-                                         int Zc,
-                                         int *PC_Flag)
+                                                int8_t *__restrict__ d_bnOutAll,
+                                                const int8_t *__restrict__ d_cnBufAll,
+                                                int8_t *__restrict__ d_cnOutAll,
+                                                int8_t *__restrict__ d_bnBufAll,
+                                                int Tid,
+                                                uint8_t groupId,
+                                                uint8_t CnIdx,
+                                                int Zc,
+                                                int *PC_Flag)
 {
   const uint8_t NUM = 19; // Gn = 19
   const int8_t *p_bnProcBufRes = (const int8_t *)d_bnOutAll;
@@ -1285,7 +1285,7 @@ __device__ void CnToBnPC_Kernel_int8_G19_Stream(const t_nrLDPC_lut *p_lut,
 
   //-------------------------------------Second half DONE----------------------------------------
   __syncthreads();
- uint32_t pcRes = 0;
+  uint32_t pcRes = 0;
   uint32_t ymm0, ymm1;
   if (Tid < 96) {
     for (int i = 0; i < 19; i++) {
@@ -1299,7 +1299,7 @@ __device__ void CnToBnPC_Kernel_int8_G19_Stream(const t_nrLDPC_lut *p_lut,
 
     if (__any_sync(0xffffffff, pcRes != 0)) {
       if (Tid % warpSize == 0) {
-        //printf("It's wrong here G19, pcRes = %d\n", pcRes);
+        // printf("It's wrong here G19, pcRes = %d\n", pcRes);
         *PC_Flag = 1; // atomicOr(PC_Flag, 1);
       }
     }
@@ -1309,71 +1309,69 @@ __device__ void CnToBnPC_Kernel_int8_G19_Stream(const t_nrLDPC_lut *p_lut,
 __device__ void llrRes2llrOut_Kernel_int8_BG1(const t_nrLDPC_lut *p_lut, int8_t *llrOut, int8_t *llrRes, int Zc)
 {
   int colIdx = blockIdx.x; //
-  int tid = threadIdx.x; // 
+  int tid = threadIdx.x; //
 
   if (tid >= (Zc / 4))
     return;
-    
+
   const uint8_t numBn2CnG1 = p_lut->numBnInBnGroups[0];
-  uint32_t startColParity = NR_LDPC_START_COL_PARITY_BG1;//(BG == 1) ? (NR_LDPC_START_COL_PARITY_BG1) : (NR_LDPC_START_COL_PARITY_BG2);
+  uint32_t startColParity =
+      NR_LDPC_START_COL_PARITY_BG1; //(BG == 1) ? (NR_LDPC_START_COL_PARITY_BG1) : (NR_LDPC_START_COL_PARITY_BG2);
 
   uint32_t colG1 = startColParity * Zc;
 
-  const uint16_t* lut_llr2llrProcBufAddr = p_lut->llr2llrProcBufAddr;
-  const uint8_t* lut_llr2llrProcBufBnPos = p_lut->llr2llrProcBufBnPos;
+  const uint16_t *lut_llr2llrProcBufAddr = p_lut->llr2llrProcBufAddr;
+  const uint8_t *lut_llr2llrProcBufBnPos = p_lut->llr2llrProcBufBnPos;
 
-  int8_t* p_llrOut = &llrOut[0];
- if(colIdx < startColParity){
+  int8_t *p_llrOut = &llrOut[0];
+  if (colIdx < startColParity) {
     const int idxBn = lut_llr2llrProcBufAddr[colIdx] + lut_llr2llrProcBufBnPos[colIdx] * Zc;
-    int32_t* dst_ptr2 = (int32_t*)(p_llrOut + colIdx * Zc + tid * 4);
-    int32_t* src_ptr2 = (int32_t*)(&llrRes[idxBn] + tid * 4);
-    *dst_ptr2 = *src_ptr2;//0x01010101*colIdx;//
+    int32_t *dst_ptr2 = (int32_t *)(p_llrOut + colIdx * Zc + tid * 4);
+    int32_t *src_ptr2 = (int32_t *)(&llrRes[idxBn] + tid * 4);
+    *dst_ptr2 = *src_ptr2; // 0x01010101*colIdx;//
   }
 
-//  __syncthreads();
+  //  __syncthreads();
   if (numBn2CnG1 > 0) {
-    if(colIdx<numBn2CnG1){
-      int32_t* dst_ptr1 = (int32_t*)(&llrOut[colG1] + colIdx * Zc + tid * 4);
-      int32_t* src_ptr1 = (int32_t*)(llrRes + colIdx * Zc + tid * 4);
-      *dst_ptr1 = *src_ptr1;//0x10101010*colIdx;//
+    if (colIdx < numBn2CnG1) {
+      int32_t *dst_ptr1 = (int32_t *)(&llrOut[colG1] + colIdx * Zc + tid * 4);
+      int32_t *src_ptr1 = (int32_t *)(llrRes + colIdx * Zc + tid * 4);
+      *dst_ptr1 = *src_ptr1; // 0x10101010*colIdx;//
     }
   }
-
- 
 }
 
-__device__ void llr2bitPacked_Kernel_int8_BG1(uint8_t* out, int8_t* llrOut, uint32_t numLLR) {
-    int tid = blockIdx.x * blockDim.x + threadIdx.x;
-    int totalGroups = numLLR >> 3;  // every 8  LLR as a group
+__device__ void llr2bitPacked_Kernel_int8_BG1(uint8_t *out, int8_t *llrOut, uint32_t numLLR)
+{
+  int tid = blockIdx.x * blockDim.x + threadIdx.x;
+  int totalGroups = numLLR >> 3; // every 8  LLR as a group
 
-    if (tid >= totalGroups)
-        return;
+  if (tid >= totalGroups)
+    return;
 
-    int8_t* p_llr = llrOut + tid * 8;
-    uint8_t result = 0;
+  int8_t *p_llr = llrOut + tid * 8;
+  uint8_t result = 0;
 
-    //  shuffling
-    for (int i = 0; i < 8; i++) {
-        result |= (p_llr[7 - i] < 0) << i;
-    }
+  //  shuffling
+  for (int i = 0; i < 8; i++) {
+    result |= (p_llr[7 - i] < 0) << i;
+  }
 
-    out[tid] = result;
+  out[tid] = result;
 }
 
-__device__ void llr2bit_Kernel_int8_BG1(uint8_t* out, int8_t* llrOut, uint32_t numLLR) {
-    int tid = blockIdx.x * blockDim.x + threadIdx.x;
-    int totalGroups = numLLR >> 3;  // every 8 LLR as a group
+__device__ void llr2bit_Kernel_int8_BG1(uint8_t *out, int8_t *llrOut, uint32_t numLLR)
+{
+  int tid = blockIdx.x * blockDim.x + threadIdx.x;
+  int totalGroups = numLLR >> 3; // every 8 LLR as a group
 
-    if (tid >= totalGroups)
-        return;
+  if (tid >= totalGroups)
+    return;
 
-    int8_t* p_llr = llrOut + tid * 8;
-    uint8_t result = 0;
+  int8_t *p_llr = llrOut + tid * 8;
+  uint8_t result = 0;
 
-    // don't need shuffle 
-    for (int i = 0; i < 8; i++) {
-        result |= (p_llr[i] < 0) << i;
-    }
-
-    out[tid] = result;
+  // don't need shuffle
+  for (int i = 0; i < 8; i++)
+    out[tid * 8 + i] = (p_llr[i] < 0);
 }

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c

@@ -152,22 +152,18 @@
 */
 
 //--------------------------CUDA Area---------------------------
-#include <cuda_runtime.h>
 
-#ifdef PARALLEL_STREAM
-static cudaStream_t decoderStreams[MAX_NUM_DLSCH_SEGMENTS];
-static cudaEvent_t decoderDoneEvents[MAX_NUM_DLSCH_SEGMENTS];
-static bool streamsCreated = false;
-static int currentStreamCount = 0;
-static int8_t iter_ptr_array[MAX_NUM_DLSCH_SEGMENTS];
-static  int PC_Flag_array[MAX_NUM_DLSCH_SEGMENTS];
-static  int8_t cnProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_CN_PROC_BUF] __attribute__((aligned(64))) = {0};
-static  int8_t cnProcBufRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_CN_PROC_BUF] __attribute__((aligned(64))) = {0};
-static  int8_t bnProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_BN_PROC_BUF] __attribute__((aligned(64))) = {0};
-static  int8_t bnProcBufRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_BN_PROC_BUF] __attribute__((aligned(64))) = {0};
-static  int8_t llrRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
-static  int8_t llrProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
-static  int8_t pp_llrOut[NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
+#define STATIC_LUT 1
+
+#if STATIC_LUT
+static bool p_lutCreated = false;
+static uint32_t numLLR;
+static t_nrLDPC_lut lut;
+static t_nrLDPC_lut* p_lut = &lut;
+#endif
+
+#if USE_CUDA
+#include <cuda_runtime.h>
 #endif
 
 
@@ -191,29 +187,7 @@ extern void nrLDPC_BnToCnPC_BG1_cuda(const t_nrLDPC_lut* p_lut,
                                      int8_t* bnProcBuf,
                                      uint16_t Z,
                                      int* PC_Flag);
-#ifdef PARALLEL_STREAM
-extern void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut* p_lut,
-                                                   int8_t* p_out,
-                                                   uint32_t numLLR,
-                                                   int8_t* cnProcBuf,
-                                                   int8_t* cnProcBufRes,
-                                                   int8_t* bnProcBuf,
-                                                   int8_t* bnProcBufRes,
-                                                   int8_t* llrRes,
-                                                   int8_t* llrProcBuf,
-                                                   int8_t* llrOut,
-                                                   int8_t* p_llrOut,
-                                                   int Z,
-                                                   uint8_t BG,
-                                                   uint8_t R,
-                                                   uint8_t numMaxIter,
-                                                   e_nrLDPC_outMode outMode,
-                                                   cudaStream_t* streams,
-                                                   uint8_t CudaStreamIdx,
-                                                   cudaEvent_t* doneEvent,
-                                                   int8_t* iter_ptr,
-                                                   int* PC_Flag);
-#endif
+
 
 //--------------------------------------------------------------
 
@@ -236,28 +210,10 @@ void dump_cnProcBufRes_to_file(const int8_t* cnProcBufRes, const char* filename)
   fclose(fp);
 }
 
-void dumpASS(int8_t* cnProcBufRes, const char* filename)
-{
-  FILE* fp = fopen(filename, "w");
-  if (fp == NULL) {
-    perror("Failed to open dump file");
-    exit(EXIT_FAILURE);
-  }
-  // printf("\nNR_LDPC_SIZE_CN_PROC_BUF: %d\n", NR_LDPC_SIZE_CN_PROC_BUF);
 
-  for (int i = 0; i < MAX_NUM_DLSCH_SEGMENTS * 68 * 384; i++) {
-    fprintf(fp, "%02x ", (uint8_t)cnProcBufRes[i]);
-    if ((i + 1) % 16 == 0)
-      fprintf(fp, "\n");
-  }
 
-  fclose(fp);
-}
-//--------------------------------------------------------------
-#ifdef PARALLEL_STREAM
 static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
                                            int8_t* p_out,
-                                           int n_segments,
                                            uint32_t numLLR,
                                            t_nrLDPC_lut* p_lut,
                                            t_nrLDPC_dec_params* p_decParams,
@@ -283,198 +239,18 @@ int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
                     t_nrLDPC_time_stats* p_profiler,
                     decode_abort_t* ab)
 {
-  uint32_t numLLR;
-  t_nrLDPC_lut lut;
-  t_nrLDPC_lut* p_lut = &lut;
-
-  // Initialize decoder core(s) with correct LUTs
-  numLLR = nrLDPC_init(p_decParams, p_lut);
-
-  // Launch LDPC decoder core for one segment
-  int n_segments = p_decParams->n_segments;
-  int numIter = nrLDPC_decoder_core(p_llr, p_out, n_segments, numLLR, p_lut, p_decParams, p_profiler, ab);
-  // printf("6.1: It works here\n");
-  if (numIter >= p_decParams->numMaxIter) {
-    LOG_D(PHY, "set abort: %d, %d\n", numIter, p_decParams->numMaxIter);
-    set_abort(ab, true);
+#if STATIC_LUT
+  if (!p_lutCreated) {
+    numLLR = nrLDPC_init(p_decParams, p_lut);
+    printf("I'm here everytime\n");
+    p_lutCreated = true;
   }
-  // printf("6.2: It works here\n");
-  return numIter;
-}
-
-/**
-   \brief PerformsnrLDPC decoding of one code block
-   \param p_llr Input LLRs
-   \param p_out Output vector
-   \param numLLR Number of LLRs
-   \param p_lut Pointer to decoder LUTs
-   \param p_decParamsnrLDPC decoder parameters
-   \param p_profilernrLDPC profiler statistics
-*/
-
-static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
-                                           int8_t* p_out,
-                                           int n_segments,
-                                           uint32_t numLLR,
-                                           t_nrLDPC_lut* p_lut,
-                                           t_nrLDPC_dec_params* p_decParams,
-                                           t_nrLDPC_time_stats* p_profiler,
-                                           decode_abort_t* ab)
-{
-  // printf("n_segments = %d\n", n_segments);
-  uint16_t Z = p_decParams->Z;
-  uint8_t BG = p_decParams->BG;
-  uint8_t R = p_decParams->R; // Decoding rate: Format 15,13,... for code rates 1/5, 1/3,... */
-  uint8_t numMaxIter = p_decParams->numMaxIter;
-  e_nrLDPC_outMode outMode = p_decParams->outMode;
-  int Kprime = p_decParams->Kprime;
-  int LastTrial = p_decParams->LastTrial;
-  // printf("Kprime = %d\n", Kprime);
-  //  int8_t* cnProcBuf=  cnProcBuf;
-  //  int8_t* cnProcBufRes= cnProcBufRes;
-  // printf("1: It works here\n");
-
-
-  // printf("2: It works here\n");
-  //  Minimum number of iterations is 1
-  //  0 iterations means hard-decision on input LLRs
-  //  Initialize with parity check fail != 0
-  // printf("3: It works here\n");
-  //  Initialization
-  //cudaStream_t streams[MAX_NUM_DLSCH_SEGMENTS];
-  //cudaEvent_t done[MAX_NUM_DLSCH_SEGMENTS]; // MAX_NUM_SEGMENTS = stream数量
-
-  for (int s = 0; s < MAX_NUM_DLSCH_SEGMENTS; s++) {
-    iter_ptr_array[s] = 0;
-    PC_Flag_array[s] = 1;
-  }
-  // printf("3.1: It works here\n");
-  if (!streamsCreated) {
-  for (int s = 0; s < n_segments; ++s) {
-    cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
-    cudaEventCreate(&decoderDoneEvents[s]);
-  }
-  streamsCreated = true;
-  currentStreamCount = n_segments;
-}
-
-  // printf("3.2: It works here\n");
-  for (int CudaStreamIdx = 0; CudaStreamIdx < n_segments; CudaStreamIdx++) {
-    int8_t* pp_llr = p_llr + CudaStreamIdx * 68 * 384;
-    int8_t* pp_out = p_out + CudaStreamIdx * Kprime;
-    // printf("Stream %d: pp_out = %p\n", CudaStreamIdx, pp_out);
-    int8_t* pp_cnProcBuf = cnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
-    int8_t* pp_cnProcBufRes = cnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
-    int8_t* pp_bnProcBuf = bnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
-    int8_t* pp_bnProcBufRes = bnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
-    int8_t* pp_llrRes = llrRes + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
-    int8_t* pp_llrProcBuf = llrProcBuf + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
-    // printf("4: It works here\n");
-    //  LLR preprocessing
-    //NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2llrProcBuf));
-    nrLDPC_llr2llrProcBuf(p_lut, pp_llr, pp_llrProcBuf, Z, BG);
-    //NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2llrProcBuf));
-    //NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2CnProcBuf));
-    if (BG == 1)
-      nrLDPC_llr2CnProcBuf_BG1(p_lut, pp_llr, pp_cnProcBuf, Z);
-    else
-      nrLDPC_llr2CnProcBuf_BG2(p_lut, pp_llr, pp_cnProcBuf, Z);
-    //NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2CnProcBuf));
-    // Call scheduler for this segment and stream
-    int8_t* pp_p_llrOut = (outMode == nrLDPC_outMode_LLRINT8) ? pp_out : pp_llrOut;
-    // printf("5: It works here\n");
-    //  Launch decoder on stream s
-    
-    //cudaEventCreate(&decoderDoneEvents[CudaStreamIdx]);
-    //printf("Launching segment %d \n",CudaStreamIdx);
-    nrLDPC_decoder_scheduler_BG1_cuda_core(p_lut,
-                                           pp_out,
-                                           numLLR,
-                                           pp_cnProcBuf,
-                                           pp_cnProcBufRes,
-                                           pp_bnProcBuf,
-                                           pp_bnProcBufRes,
-                                           pp_llrRes,
-                                           pp_llrProcBuf,
-                                           pp_llrOut,
-                                           pp_p_llrOut,
-                                           Z,
-                                           BG,
-                                           R,
-                                           numMaxIter,
-                                           outMode,
-                                           decoderStreams,
-                                           CudaStreamIdx,
-                                           decoderDoneEvents,
-                                           &iter_ptr_array[CudaStreamIdx],
-                                           &PC_Flag_array[CudaStreamIdx]); // stream index passed in
-  
-    
-    
-                                           //cudaEventRecord(done[CudaStreamIdx], streams[CudaStreamIdx]);
-    // printf("5: It works here\n");
-  }
-for (int s = 0; s < n_segments; ++s) {
- // printf("Synchronizing segment %d \n",s);
-    cudaEventSynchronize(decoderDoneEvents[s]);  // 阻塞直到该 segment 解码完成
-}
-cudaDeviceSynchronize();
-//cudaDeviceSynchronize();
-  // Wait for all streams
- // for (int s = 0; s < n_segments; s++) {
- //   cudaEventSynchronize(done[s]); // 等待stream[i]完成
- //   // 可安全访问对应的解码输出结果 p_llrOut[i]
-//}
-if(LastTrial == 1){
-  //printf("Now is the last trial\n");
-  for (int s = 0; s < n_segments; s++) {
-    cudaEventDestroy(decoderDoneEvents[s]);
-    cudaStreamSynchronize(decoderStreams[s]);
-    cudaStreamDestroy(decoderStreams[s]);
-    streamsCreated = false;
-  }
-}
-//cudaDeviceSynchronize();
-  // dumpASS(p_out, "Dump_Output_Stream.txt");
-  // printf("6: It works here\n");
-
-  return numMaxIter;
-}
-
 #else
-static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
-                                           int8_t* p_out,
-                                           uint32_t numLLR,
-                                           t_nrLDPC_lut* p_lut,
-                                           t_nrLDPC_dec_params* p_decParams,
-                                           t_nrLDPC_time_stats* p_profiler,
-                                           decode_abort_t* ab);
-
-int32_t LDPCinit()
-{
-  return 0;
-}
-
-int32_t LDPCshutdown()
-{
-  return 0;
-}
-
-int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
-                    uint8_t harq_pid,
-                    uint8_t ulsch_id,
-                    uint8_t C,
-                    int8_t* p_llr,
-                    int8_t* p_out,
-                    t_nrLDPC_time_stats* p_profiler,
-                    decode_abort_t* ab)
-{
   uint32_t numLLR;
   t_nrLDPC_lut lut;
   t_nrLDPC_lut* p_lut = &lut;
-
+#endif
   // Initialize decoder core(s) with correct LUTs
-  numLLR = nrLDPC_init(p_decParams, p_lut);
 
   // Launch LDPC decoder core for one segment
   int numIter = nrLDPC_decoder_core(p_llr, p_out, numLLR, p_lut, p_decParams, p_profiler, ab);
@@ -519,8 +295,8 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
   // Minimum number of iterations is 1
   // 0 iterations means hard-decision on input LLRs
   // Initialize with parity check fail != 0
-//printf("cnProcBuf address: %p\n",cnProcBuf);
-  // Initialization
+  // printf("cnProcBuf address: %p\n",cnProcBuf);
+  //  Initialization
   NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2llrProcBuf));
   nrLDPC_llr2llrProcBuf(p_lut, p_llr, llrProcBuf, Z, BG);
   NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2llrProcBuf));
@@ -1280,4 +1056,3 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
   // #endif
   return numIter;
 }
-#endif

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_BG1_cuda.cu

@@ -18,11 +18,147 @@
 
 #define BIG_KERNEL 1
 
-static cudaGraph_t decoderGraphs[MAX_NUM_DLSCH_SEGMENTS] = {nullptr};
-static cudaGraphExec_t decoderGraphExec[MAX_NUM_DLSCH_SEGMENTS] = {nullptr};
-static bool graphCreated[MAX_NUM_DLSCH_SEGMENTS] = {false};
+// decoder_graphs.cu
+#include "decoder_graphs.h"
+
+cudaGraph_t decoderGraphs[MAX_NUM_DLSCH_SEGMENTS] = {nullptr};
+cudaGraphExec_t decoderGraphExec[MAX_NUM_DLSCH_SEGMENTS] = {nullptr};
+bool graphCreated[MAX_NUM_DLSCH_SEGMENTS] = {false};
 
 
+// 适配 CUDA 11+/12+
+static const char* ptrTypeName(cudaMemoryType type) {
+  switch (type) {
+    case cudaMemoryTypeUnregistered: return "Unregistered/Unknown";
+    case cudaMemoryTypeHost:         return "Host (pinned)";
+    case cudaMemoryTypeDevice:       return "Device";
+    case cudaMemoryTypeManaged:      return "Managed";
+    default:                         return "Unknown";
+  }
+}
+// 简单的错误检查宏
+#define CHECK_CUDA(call) do {                                      \
+  cudaError_t _e = (call);                                         \
+  if (_e != cudaSuccess) {                                         \
+    fprintf(stderr, "CUDA error %s:%d: %s\n",                      \
+            __FILE__, __LINE__, cudaGetErrorString(_e));           \
+    return;                                                        \
+  }                                                                \
+} while(0)
+
+// 你已有的：打印指针属性（别去解引用）
+extern "C" void check_ptr_host(const void *p, const char *name) {
+  cudaPointerAttributes attr;
+  cudaError_t e = cudaPointerGetAttributes(&attr, p);
+  if (e != cudaSuccess) {
+    printf("Ptr %-24s = %p  <cudaPointerGetAttributes failed: %s>\n",
+           name, p, cudaGetErrorString(e));
+    return;
+  }
+  const char *type = "Unregistered/Unknown";
+  if (attr.type == cudaMemoryTypeHost)    type = "Host";
+  if (attr.type == cudaMemoryTypeDevice)  type = "Device";
+  if (attr.type == cudaMemoryTypeManaged) type = "Managed";
+  printf("Ptr %-24s = %p  type=%s  device=%d  devicePointer=%p  hostPointer=%p\n",
+         name, p, type, attr.device, attr.devicePointer, attr.hostPointer);
+}
+
+static void dump_arr8_host(const arr8_t *a, const char *name, int idx) {
+  char tag[64];
+  snprintf(tag, sizeof(tag), "%s[%d].d", name, idx);
+  printf("%s[%d]: dim1=%d dim2=%d\n", name, idx, a->dim1, a->dim2);
+  check_ptr_host(a->d, tag);
+}
+static void dump_arr16_host(const arr16_t *a, const char *name, int idx) {
+  char tag[64];
+  snprintf(tag, sizeof(tag), "%s[%d].d", name, idx);
+  printf("%s[%d]: dim1=%d dim2=%d\n", name, idx, a->dim1, a->dim2);
+  check_ptr_host(a->d, tag);
+}
+static void dump_arr32_host(const arr32_t *a, const char *name, int idx) {
+  char tag[64];
+  snprintf(tag, sizeof(tag), "%s[%d].d", name, idx);
+  printf("%s[%d]: dim1=%d dim2=%d\n", name, idx, a->dim1, a->dim2);
+  check_ptr_host(a->d, tag);
+}
+
+// 用设备指针调用这个函数
+void inspect_lut(const t_nrLDPC_lut *p_lut_dev) {
+  printf("==== Inspect t_nrLDPC_lut(dev) @ %p ====\n", (void*)p_lut_dev);
+  check_ptr_host(p_lut_dev, "p_lut_dev");
+
+  // 1) 先把“头”拷回主机（浅拷贝）
+  t_nrLDPC_lut h = {0};
+  CHECK_CUDA(cudaMemcpy(&h, p_lut_dev, sizeof(h), cudaMemcpyDeviceToHost));
+
+  // 2) 现在用这份主机副本里的“设备指针值”做属性查询即可
+  check_ptr_host(h.startAddrCnGroups,    "startAddrCnGroups");
+  check_ptr_host(h.numCnInCnGroups,      "numCnInCnGroups");
+  check_ptr_host(h.numBnInBnGroups,      "numBnInBnGroups");
+  check_ptr_host(h.startAddrBnGroups,    "startAddrBnGroups");
+  check_ptr_host(h.startAddrBnGroupsLlr, "startAddrBnGroupsLlr");
+  check_ptr_host(h.llr2llrProcBufAddr,   "llr2llrProcBufAddr");
+  check_ptr_host(h.llr2llrProcBufBnPos,  "llr2llrProcBufBnPos");
+
+  for (int i = 0; i < NR_LDPC_NUM_CN_GROUPS_BG1; ++i) {
+    dump_arr16_host(&h.circShift[i],          "circShift",          i);
+    dump_arr32_host(&h.startAddrBnProcBuf[i], "startAddrBnProcBuf", i);
+    dump_arr8_host (&h.bnPosBnProcBuf[i],     "bnPosBnProcBuf",     i);
+    dump_arr8_host (&h.posBnInCnProcBuf[i],   "posBnInCnProcBuf",   i);
+  }
+  printf("========================================\n");
+}
+
+
+ __global__ void check_ptr_kernel(const void* ptr, int id) {
+  if (threadIdx.x == 0 && blockIdx.x == 0) {
+    printf("check_ptr id=%d ptr=%p\n", id, ptr);
+  }
+}
+
+__global__ void check_ptr_kernel_easy(int id) {
+    printf("hello!\n");
+}
+
+  __device__ __constant__ uint8_t h_block_group_ids_cnProc[50] = {0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
+                                                3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8};
+
+  __device__ __constant__ uint8_t h_block_CN_idx_cnProc[50] = {0, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0,
+                                             1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 0, 1, 0, 1, 0,  0,  0,  1,  1,  2,  2,  3,  3};
+
+  __device__ __constant__ uint16_t h_block_thread_counts_cnProc[50] = {
+      288, 384, 384, 384, 384, 384, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 576,
+      576, 576, 576, 576, 576, 576, 576, 672, 672, 672, 672, 672, 768, 768, 864, 864, 960, 912, 912, 912, 912, 912, 912, 912, 912};
+
+  __device__ __constant__ uint32_t h_block_input_offsets_cnProc[50] = {
+      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288, 12672,
+      13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080, 61824, 62208,
+      62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92160, 92544, 92544, 92928, 92928, 93312, 93312};
+
+  __device__ __constant__ uint32_t h_block_output_offsets_cnProc[50] = {
+      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288, 12672,
+      13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080, 61824, 62208,
+      62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92160, 92544, 92544, 92928, 92928, 93312, 93312};
+
+__device__ __constant__ uint8_t h_block_group_ids_BnToCnPC[46] = {0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
+                                                2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 8, 8, 8, 8};
+
+  __device__ __constant__ uint8_t h_block_CN_idx_BnToCnPC[46] = {0,  0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
+                                             17, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 0, 1, 0,  1,  0,  0,  1,  2,  3};
+
+  __device__ __constant__ uint16_t h_block_thread_counts_BnToCnPC[46] = {288, 384, 384, 384, 384, 384, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480,
+                                                     480, 480, 480, 480, 480, 480, 480, 480, 576, 576, 576, 576, 576, 576, 576, 576,
+                                                     672, 672, 672, 672, 672, 768, 768, 864, 864, 960, 912, 912, 912, 912};
+
+  __device__ __constant__ uint32_t h_block_input_offsets_BnToCnPC[46] = {
+      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288,
+      12672, 13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080,
+      61824, 62208, 62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92544, 92928, 93312};
+
+  __device__ __constant__ uint32_t h_block_output_offsets_BnToCnPC[46] = {
+      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288,
+      12672, 13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080,
+      61824, 62208, 62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92544, 92928, 93312};
 
 
 
@@ -1021,11 +1157,6 @@ __global__ void cnProcKernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
                                              const int8_t *__restrict__ d_cnBufAll,
                                              int8_t *__restrict__ d_cnOutAll,
                                              int8_t *__restrict__ d_bnBufAll,
-                                             const uint8_t *__restrict__ block_group_ids,
-                                             const uint8_t *__restrict__ block_CN_idx,
-                                             const uint16_t *__restrict__ block_thread_counts,
-                                             const uint32_t *__restrict__ block_input_offsets,
-                                             const uint32_t *__restrict__ block_output_offsets,
                                              int Zc,
                                              int8_t *iter_ptr,
                                              int8_t numMaxIter,
@@ -1037,15 +1168,15 @@ __global__ void cnProcKernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
   if (*iter_ptr > numMaxIter || *PC_Flag == 0) {
     return;
   }
-
+//printf("I'm inside cnProc_kernel\n");
   int blk = blockIdx.x;
   int tid = threadIdx.x;
 
-  uint8_t groupId = block_group_ids[blk];
-  uint8_t CnIdx = block_CN_idx[blk];
-  uint16_t blockSize = block_thread_counts[blk];
-  uint32_t inOffset = block_input_offsets[blk];
-  uint32_t outOffset = block_output_offsets[blk];
+  uint8_t groupId = h_block_group_ids_cnProc[blk];
+  uint8_t CnIdx = h_block_CN_idx_cnProc[blk];
+  uint16_t blockSize = h_block_thread_counts_cnProc[blk];
+  uint32_t inOffset = h_block_input_offsets_cnProc[blk];
+  uint32_t outOffset = h_block_output_offsets_cnProc[blk];
 
   if (tid >= blockSize)
     return;
@@ -1053,20 +1184,6 @@ __global__ void cnProcKernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
   const int8_t *p_cnProcBuf = (const int8_t *)(d_cnBufAll + inOffset);
   int8_t *p_cnProcBufRes = (int8_t *)(d_cnOutAll + outOffset);
   int8_t *p_bnProcBuf = (int8_t *)d_bnBufAll;
-  // if(blk == 45 && tid == 64){
-  // printf("d_cnBufAll = %p, d_cnOutAll = %p, p_cnProcBuf = %p, p_cnProcBufRes = %p, inOffset = %d, outOffset = %d \n", d_cnBufAll,
-  // d_cnOutAll, p_cnProcBuf, p_cnProcBufRes, inOffset, outOffset);
-  //}
-  /*
-if(tid == 65 && blk == 24){
-    printf("=== cnProcKernel_int8_G3 INPUTS ===\n");
-    printf("p_lut = %p\n", p_lut);
-    printf("d_cnBufAll = %p\n", d_cnBufAll);
-    printf("d_cnOutAll = %p\n", d_cnOutAll);
-    printf("d_bnBufAll = %p\n", d_bnBufAll);
-    printf("tid = %d\n", tid);
-    printf("Zc = %d\n", Zc);
-}*/
 
   switch (groupId) {
     case 0:
@@ -1110,58 +1227,21 @@ void nrLDPC_cnProc_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
                                         cudaStream_t *streams,
                                         int8_t CudaStreamIdx)
 {
-  // const uint8_t h_lut_numBnInCnGroups_BG1_R13[] = {3, 4, 5, 6, 7, 8, 9, 10, 19};
-  // const int h_lut_numThreadsEachCnGroupsNeed_BG1_R13[] = {288, 384, 480, 576, 672, 768, 864, 960, 1824};
-  // const uint8_t h_lut_numCnInCnGroups_BG1_R13[] = {1, 5, 18, 8, 5, 2, 2, 1, 4};
-
-  // const uint8_t *lut_numCnInCnGroups = (const uint8_t *)p_lut->numCnInCnGroups;
-  const uint32_t *lut_startAddrCnGroups = lut_startAddrCnGroups_BG1;
-
-  const int numGroups = 9;
-
 #if BIG_KERNEL
-
-  static const uint8_t h_block_group_ids[50] = {0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 3,
-                                                3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 8, 8, 8, 8, 8, 8, 8, 8};
-
-  static const uint8_t h_block_CN_idx[50] = {0, 0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 0,
-                                             1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 0, 1, 0, 1, 0,  0,  0,  1,  1,  2,  2,  3,  3};
-
-  static const uint16_t h_block_thread_counts[50] = {
-      288, 384, 384, 384, 384, 384, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480, 576,
-      576, 576, 576, 576, 576, 576, 576, 672, 672, 672, 672, 672, 768, 768, 864, 864, 960, 912, 912, 912, 912, 912, 912, 912, 912};
-
-  static const uint32_t h_block_input_offsets[50] = {
-      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288, 12672,
-      13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080, 61824, 62208,
-      62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92160, 92544, 92544, 92928, 92928, 93312, 93312};
-
-  static const uint32_t h_block_output_offsets[50] = {
-      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288, 12672,
-      13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080, 61824, 62208,
-      62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92160, 92544, 92544, 92928, 92928, 93312, 93312};
-
-  // printf("\nInitial addr : cnProcBuf = %p, cnProcBufRes = %p\n", cnProcBuf, cnProcBufRes);
-
+ // printf("\nInitial addr : cnProcBuf = %p, cnProcBufRes = %p\n", cnProcBuf, cnProcBufRes);
   int maxBlockSize = 960; // Maximun threads are 960
   dim3 gridDim(50);
   dim3 blockDim(maxBlockSize);
-  // printf("bnProcBuf =  %p\n", bnProcBuf);
-  //printf("In stream %d C: Iter = %d, PC_Flag = %d\n", CudaStreamIdx, *iter_ptr, *PC_Flag);
+
   cnProcKernel_int8_BIG_stream<<<gridDim, blockDim, 0, streams[CudaStreamIdx]>>>(p_lut,
                                                                                  cnProcBuf,
                                                                                  cnProcBufRes,
                                                                                  bnProcBuf,
-                                                                                 h_block_group_ids,
-                                                                                 h_block_CN_idx,
-                                                                                 h_block_thread_counts,
-                                                                                 h_block_input_offsets,
-                                                                                 h_block_output_offsets,
                                                                                  Z,
                                                                                  iter_ptr,
                                                                                  numMaxIter,
                                                                                  PC_Flag);
-  // printf("Check point 1001: ");
+   //printf("Check point 1001: ");
    //CHECK(cudaGetLastError());
 
 #else
@@ -1319,9 +1399,7 @@ __global__ void bnProcKernel_int8_BIG_stream(const int8_t *__restrict__ d_bnProc
                        BnIdx,
                        GrpNum,
                        Zc);
-  // grid);
 
-  // t1:
 }
 
 void nrLDPC_bnProc_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
@@ -1340,23 +1418,15 @@ void nrLDPC_bnProc_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
   const uint32_t *lut_startAddrBnGroups;
   const uint16_t *lut_startAddrBnGroupsLlr;
 
-  lut_numBnInBnGroups = lut_numBnInBnGroups_BG1_R13;
-  lut_startAddrBnGroups = lut_startAddrBnGroups_BG1_R13;
-  lut_startAddrBnGroupsLlr = lut_startAddrBnGroupsLlr_BG1_R13;
+  lut_numBnInBnGroups = p_lut->numBnInBnGroups;
+  lut_startAddrBnGroups = p_lut->startAddrBnGroups;
+  lut_startAddrBnGroupsLlr = p_lut->startAddrBnGroupsLlr;
 
   int8_t *p_bnProcBuf = (int8_t *)bnProcBuf;
   int8_t *p_bnProcBufRes = (int8_t *)bnProcBufRes;
   int8_t *p_llrProcBuf = (int8_t *)llrProcBuf;
   int8_t *p_llrRes = (int8_t *)llrRes;
-/*
-  // --- compute totalBlocks ---
-  int totalBlocks = 0;
-  for (int k = 1; k <= 30; k++) {
-      int numBn = lut_numBnInBnGroups[k - 1];
-      totalBlocks += numBn * k;
-  }
-  printf("Total blocks required = %d\n", totalBlocks);
-*/
+
 #if BIG_KERNEL
   int maxBlockSize = 1024; // Z;
   int totalBlocks = 30;
@@ -1364,17 +1434,7 @@ void nrLDPC_bnProc_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
   dim3 gridDim(totalBlocks);
   dim3 blockDim(maxBlockSize);
 
-  /*
-    bnProcKernel_int8_BIG_United<<<gridDim, blockDim>>>(p_bnProcBuf,
-                                                 p_bnProcBufRes,
-                                                 p_llrProcBuf,
-                                                 p_llrRes,
-                                                 lut_numBnInBnGroups,
-                                                 lut_startAddrBnGroups,
-                                                 lut_startAddrBnGroupsLlr,
-                                                 Z);
-  */
-  //printf("In stream %d B_PC: Iter = %d, PC_Flag = %d\n", CudaStreamIdx, *iter_ptr, *PC_Flag);
+
   bnProcPcKernel_int8_BIG_stream<<<gridDim, blockDim, 0, streams[CudaStreamIdx]>>>(p_bnProcBuf,
                                                                                    p_bnProcBufRes,
                                                                                    p_llrProcBuf,
@@ -1430,11 +1490,11 @@ __global__ void BnToCnPC_Kernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
   int blk = blockIdx.x;
   int tid = threadIdx.x;
 
-  uint8_t groupId = block_group_ids[blk];
-  uint8_t CnIdx = block_CN_idx[blk];
-  uint16_t blockSize = block_thread_counts[blk];
-  uint32_t inOffset = block_input_offsets[blk];
-  uint32_t outOffset = block_output_offsets[blk];
+  uint8_t groupId = h_block_group_ids_BnToCnPC[blk];
+  uint8_t CnIdx = h_block_CN_idx_BnToCnPC[blk];
+  uint16_t blockSize = h_block_thread_counts_BnToCnPC[blk];
+  uint32_t inOffset = h_block_input_offsets_BnToCnPC[blk];
+  uint32_t outOffset = h_block_output_offsets_BnToCnPC[blk];
 
   if (tid >= blockSize)
     return;
@@ -1572,14 +1632,16 @@ __global__ void BnToCnPC_Kernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
       break;
   }
 }
-  if (*iter_ptr == numMaxIter) { // output
+
+if (*iter_ptr == numMaxIter) { // output
     llrRes2llrOut_Kernel_int8_BG1(p_lut, p_llrOut, p_llrRes, Zc);
-  } else {
+}   else {
     if (tid == 0 && blk == 0) {
       //printf("Why you guys not here when iter_ptr = %d???\n",*iter_ptr);
       (*iter_ptr)++;
     }
   }
+
 }
 
 __global__ void OutPut_Kernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
@@ -1596,9 +1658,6 @@ __global__ void OutPut_Kernel_int8_BIG_stream(const t_nrLDPC_lut *p_lut,
   // only activate in the last iteration
 
   if (*iter_ptr == numMaxIter) {
-    int blk = blockIdx.x;
-    int tid = threadIdx.x;
-
     if (outMode == nrLDPC_outMode_BIT)
       llr2bitPacked_Kernel_int8_BG1((uint8_t *)p_out, p_llrOut, numLLR);
 
@@ -1626,37 +1685,11 @@ void nrLDPC_BnToCnPC_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
                                           cudaStream_t *streams,
                                           int8_t CudaStreamIdx)
 {
-  // const uint8_t h_lut_numBnInCnGroups_BG1_R13[] = {3, 4, 5, 6, 7, 8, 9, 10, 19};
-  // const int h_lut_numThreadsEachCnGroupsNeed_BG1_R13[] = {288, 384, 480, 576, 672, 768, 864, 960, 1824};
-  // const uint8_t h_lut_numCnInCnGroups_BG1_R13[] = {1, 5, 18, 8, 5, 2, 2, 1, 4};
-
-  // const uint8_t *lut_numCnInCnGroups = (const uint8_t *)p_lut->numCnInCnGroups;
-  const uint32_t *lut_startAddrCnGroups = lut_startAddrCnGroups_BG1;
+  const uint32_t *lut_startAddrCnGroups = p_lut->startAddrCnGroups;
 
   const int numGroups = 9;
 
 #if BIG_KERNEL
-
-  static const uint8_t h_block_group_ids[46] = {0, 1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2,
-                                                2, 3, 3, 3, 3, 3, 3, 3, 3, 4, 4, 4, 4, 4, 5, 5, 6, 6, 7, 8, 8, 8, 8};
-
-  static const uint8_t h_block_CN_idx[46] = {0,  0, 1, 2, 3, 4, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16,
-                                             17, 0, 1, 2, 3, 4, 5, 6, 7, 0, 1, 2, 3, 4, 0, 1, 0,  1,  0,  0,  1,  2,  3};
-
-  static const uint16_t h_block_thread_counts[46] = {288, 384, 384, 384, 384, 384, 480, 480, 480, 480, 480, 480, 480, 480, 480, 480,
-                                                     480, 480, 480, 480, 480, 480, 480, 480, 576, 576, 576, 576, 576, 576, 576, 576,
-                                                     672, 672, 672, 672, 672, 768, 768, 864, 864, 960, 912, 912, 912, 912};
-
-  static const uint32_t h_block_input_offsets[46] = {
-      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288,
-      12672, 13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080,
-      61824, 62208, 62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92544, 92928, 93312};
-
-  static const uint32_t h_block_output_offsets[46] = {
-      0,     1152,  1536,  1920,  2304,  2688,  8832,  9216,  9600,  9984,  10368, 10752, 11136, 11520, 11904, 12288,
-      12672, 13056, 13440, 13824, 14208, 14592, 14976, 15360, 43392, 43776, 44160, 44544, 44928, 45312, 45696, 46080,
-      61824, 62208, 62592, 62976, 63360, 75264, 75648, 81408, 81792, 88320, 92160, 92544, 92928, 93312};
-
   // printf("\nInitial addr : cnProcBuf = %p, cnProcBufRes = %p\n", cnProcBuf, cnProcBufRes);
 
   int maxBlockSize = 960; // Maximun threads are 960
@@ -1670,11 +1703,11 @@ void nrLDPC_BnToCnPC_BG1_cuda_stream_core(const t_nrLDPC_lut *p_lut,
                                                                                     cnProcBufRes,
                                                                                     bnProcBuf,
                                                                                     llrRes,
-                                                                                    h_block_group_ids,
-                                                                                    h_block_CN_idx,
-                                                                                    h_block_thread_counts,
-                                                                                    h_block_input_offsets,
-                                                                                    h_block_output_offsets,
+                                                                                    h_block_group_ids_BnToCnPC,
+                                                                                    h_block_CN_idx_BnToCnPC,
+                                                                                    h_block_thread_counts_BnToCnPC,
+                                                                                    h_block_input_offsets_BnToCnPC,
+                                                                                    h_block_output_offsets_BnToCnPC,
                                                                                     Z,
                                                                                     iter_ptr,
                                                                                     numMaxIter,
@@ -1727,7 +1760,8 @@ extern "C" void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut *p_lut
                                                        int8_t* iter_ptr,
                                                        int* PC_Flag)
 { 
-#if CPU_ADDRESSING
+#if 1//CPU_ADDRESSING
+
   cudaStream_t stream = streams[CudaStreamIdx];
   //cudaEvent_t captureDoneEvent[MAX_NUM_DLSCH_SEGMENTS];
   //cudaEvent_t captureDoneEvent[MAX_NUM_DLSCH_SEGMENTS];
@@ -1737,9 +1771,29 @@ extern "C" void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut *p_lut
     if(CudaStreamIdx != 0){
       cudaEventSynchronize(doneEvent[CudaStreamIdx - 1]);
     }
+    //CHECK(cudaGetLastError());
+/*
+    // print all the address to see if they are isolated
+    printf("Stream %d parameter addresses:\n", CudaStreamIdx);
+    printf("  p_lut       = %p\n", (void*)p_lut);
+    printf("  p_out       = %p\n", (void*)p_out);
+    printf("  cnProcBuf   = %p\n", (void*)cnProcBuf);
+    printf("  cnProcBufRes= %p\n", (void*)cnProcBufRes);
+    printf("  bnProcBuf   = %p\n", (void*)bnProcBuf);
+    printf("  bnProcBufRes= %p\n", (void*)bnProcBufRes);
+    printf("  llrRes      = %p\n", (void*)llrRes);
+    printf("  llrProcBuf  = %p\n", (void*)llrProcBuf);
+    printf("  llrOut      = %p\n", (void*)llrOut);
+    printf("  p_llrOut    = %p\n", (void*)p_llrOut);
+    printf("  iter_ptr    = %p\n", (void*)iter_ptr);
+    printf("  PC_Flag     = %p\n", (void*)PC_Flag);
+    fflush(stdout);
+*/
     // Start graph recording
-    cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal);
-
+  cudaStreamBeginCapture(stream, cudaStreamCaptureModeGlobal);
+//check_ptr_kernel_easy<<<1,10>>>(2);
+//cudaDeviceSynchronize();
+//CHECK(cudaGetLastError());
   for (int i = 0; i <= numMaxIter; i++) {
     // printf("I'm inside the loop i = %d\n", i);
     nrLDPC_cnProc_BG1_cuda_stream_core(p_lut,
@@ -1769,7 +1823,7 @@ extern "C" void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut *p_lut
      //cudaDeviceSynchronize();
 
      //printf("In stream %d 2: Iter = %d, PC_Flag = %d\n", CudaStreamIdx, *iter_ptr, *PC_Flag);
-     //CHECK(cudaGetLastError());
+    //CHECK(cudaGetLastError());
     //cudaDeviceSynchronize();
     nrLDPC_BnToCnPC_BG1_cuda_stream_core(p_lut,
                                          bnProcBufRes,
@@ -1794,8 +1848,9 @@ extern "C" void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut *p_lut
     
      }
 
-    // stop recording
+      // stop recording
     cudaStreamEndCapture(stream, &decoderGraphs[CudaStreamIdx]);
+    //printf("5\n");
     cudaGraphInstantiate(&decoderGraphExec[CudaStreamIdx], decoderGraphs[CudaStreamIdx], NULL, NULL, 0);
     graphCreated[CudaStreamIdx] = true;
 
@@ -1807,14 +1862,30 @@ extern "C" void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut *p_lut
     //printf("Are you here???\n");
     // reuse the graph after
     if(CudaStreamIdx != 0){
-      //cudaEventSynchronize(doneEvent[CudaStreamIdx - 1]); //uncomment it if you want streams works in sequence
+      //uncomment below if you want streams works in sequence
+      //cudaStreamWaitEvent(streams[CudaStreamIdx], doneEvent[CudaStreamIdx-1], 0);//cudaEventSynchronize(doneEvent[CudaStreamIdx - 1]); 
     }
     cudaGraphLaunch(decoderGraphExec[CudaStreamIdx], stream);
     cudaEventRecord(doneEvent[CudaStreamIdx], stream);
     //
   }
-  CHECK(cudaGetLastError());
+  //CHECK(cudaGetLastError());
 #else
   printf("To be continued ^ ^\n");
 #endif
+}
+
+extern "C" bool is_device_pointer(const void* p) {
+    if (p == NULL) return false;
+    cudaPointerAttributes attrs;
+    cudaError_t err = cudaPointerGetAttributes(&attrs, p);
+    if (err != cudaSuccess) {
+        // cudaPointerGetAttributes return value might vary in different runtime version
+        return false;
+    }
+#if CUDART_VERSION >= 10000
+    return (attrs.type == cudaMemoryTypeDevice);
+#else
+    return (attrs.memoryType == cudaMemoryTypeDevice);
+#endif
 }

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_cuda.c

@@ -0,0 +1,785 @@
+
+
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*!\file nrLDPC_decoder.c
+ * \brief Defines thenrLDPC decoder
+ */
+
+#include <stdint.h>
+#include "PHY/sse_intrin.h"
+#include "nrLDPCdecoder_defs.h"
+#include "nrLDPC_types.h"
+#include "nrLDPC_init.h"
+#include "nrLDPC_mPass.h"
+#include "nrLDPC_cnProc.h"
+#include "nrLDPC_bnProc.h"
+#include "openair1/PHY/CODING/coding_defs.h"
+#define UNROLL_CN_PROC 1
+#define UNROLL_BN_PROC 1
+#define UNROLL_BN_PROC_PC 1
+#define UNROLL_BN2CN_PROC 1
+/*----------------------------------------------------------------------
+|                  cn processing files -->AVX512
+/----------------------------------------------------------------------*/
+
+// BG1-------------------------------------------------------------------
+#if defined(__AVX512BW__)
+
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R13_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R23_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R89_AVX512.h"
+// BG2-------------------------------------------------------------------
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R15_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R13_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R23_AVX512.h"
+
+#elif defined(__AVX2__)
+
+/*----------------------------------------------------------------------
+|                  cn Processing files -->AVX2
+/----------------------------------------------------------------------*/
+
+// BG1------------------------------------------------------------------
+#include "cnProc/nrLDPC_cnProc_BG1_R13_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG1_R23_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "cnProc/nrLDPC_cnProc_BG2_R15_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG2_R13_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG2_R23_AVX2.h"
+
+#else
+
+// BG1------------------------------------------------------------------
+#include "cnProc128/nrLDPC_cnProc_BG1_R13_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG1_R23_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG1_R89_128.h"
+// BG2 --------------------------------------------------------------------
+#include "cnProc128/nrLDPC_cnProc_BG2_R15_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG2_R13_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG2_R23_128.h"
+#endif
+
+/*----------------------------------------------------------------------
+|                 bn Processing files -->AVX2
+/----------------------------------------------------------------------*/
+
+// bnProcPc-------------------------------------------------------------
+#ifdef __AVX2__
+// BG1------------------------------------------------------------------
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R13_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R23_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R15_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R13_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R23_AVX2.h"
+#else
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R13_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R23_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R89_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R15_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R13_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R23_128.h"
+#endif
+
+// bnProc----------------------------------------------------------------
+
+#if defined(__AVX512BW__)
+// BG1-------------------------------------------------------------------
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R13_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R23_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R89_AVX512.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R15_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R13_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R23_AVX512.h"
+
+#elif defined(__AVX2__)
+#include "bnProc/nrLDPC_bnProc_BG1_R13_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG1_R23_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc/nrLDPC_bnProc_BG2_R15_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG2_R13_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG2_R23_AVX2.h"
+#else
+#include "bnProc128/nrLDPC_bnProc_BG1_R13_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG1_R23_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG1_R89_128.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc128/nrLDPC_bnProc_BG2_R15_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG2_R13_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG2_R23_128.h"
+#endif
+
+// #define NR_LDPC_PROFILER_DETAIL(a) a
+#define NR_LDPC_PROFILER_DETAIL(a)
+
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+#ifdef NR_LDPC_DEBUG_MODE
+#include "nrLDPC_tools/nrLDPC_debug.h"
+#endif
+
+// decoder interface
+/**
+   \brief LDPC decoder API type definition
+   \param p_decParams LDPC decoder parameters
+   \param p_llr Input LLRs
+   \param p_llrOut Output vector
+   \param p_profiler LDPC profiler statistics
+*/
+
+//--------------------------CUDA Area---------------------------
+
+#define STATIC_LUT 1
+
+#if STATIC_LUT
+static bool p_lutCreated = false;
+static uint32_t numLLR;
+static t_nrLDPC_lut lut;
+static t_nrLDPC_lut* p_lut = &lut;
+#endif
+
+#if USE_CUDA
+#include <cuda_runtime.h>
+#endif
+
+#if PARALLEL_STREAM
+#define COPY_ARR_MEMBER(member, type, groups) do { \
+    for (int i = 0; i < (groups); i++) { \
+        type* tmp_dev; \
+        if (h_lut->member[i].d != NULL && h_lut->member[i].dim1 > 0 && h_lut->member[i].dim2 > 0) { \
+            size_t sz = h_lut->member[i].dim1 * h_lut->member[i].dim2 * sizeof(type); \
+            err = cudaMalloc((void**)&tmp_dev, sz); \
+            if (err != cudaSuccess) { \
+                fprintf(stderr, "cudaMalloc failed for " #member "[%d]: %s\n", i, cudaGetErrorString(err)); \
+                exit(EXIT_FAILURE); \
+            } \
+            cudaMemcpy(tmp_dev, h_lut->member[i].d, sz, cudaMemcpyHostToDevice); \
+            /* updtae d_lut->member[i].d pointer */ \
+            cudaMemcpy(&(d_lut->member[i].d), &tmp_dev, sizeof(type*), cudaMemcpyHostToDevice); \
+            /* copy dim1 and dim2 */ \
+            cudaMemcpy(&(d_lut->member[i].dim1), &(h_lut->member[i].dim1), sizeof(int), cudaMemcpyHostToDevice); \
+            cudaMemcpy(&(d_lut->member[i].dim2), &(h_lut->member[i].dim2), sizeof(int), cudaMemcpyHostToDevice); \
+        } \
+    } \
+} while(0)
+
+#define COPY_POINTER_MEMBER(member, type, count) do { \
+    type* tmp_dev; \
+    printf("tmp_dev = %p\n", (void*)tmp_dev);\
+    err = cudaMalloc((void**)&tmp_dev, (count) * sizeof(type)); \
+    printf("malloc tmp_dev = %p\n", (void*)tmp_dev);\
+    if (err != cudaSuccess) { \
+        fprintf(stderr, "cudaMalloc failed for " #member ": %s\n", cudaGetErrorString(err)); \
+        exit(EXIT_FAILURE); \
+    } \
+    printf("h_lut->member = %p\n", (void*)h_lut->member);\
+    cudaMemcpy(tmp_dev, h_lut->member, (count) * sizeof(type), cudaMemcpyHostToDevice); \
+    printf("d_lut->member");\
+    printf(" = %p\n", (void*)d_lut->member);\
+    cudaMemcpy(&(d_lut->member), &tmp_dev, sizeof(type*), cudaMemcpyHostToDevice); \
+} while(0)
+
+#include "decoder_graphs.h"
+static cudaStream_t decoderStreams[MAX_NUM_DLSCH_SEGMENTS];
+static cudaEvent_t decoderDoneEvents[MAX_NUM_DLSCH_SEGMENTS];
+static bool streamsCreated = false;
+static bool d_mem_exist = false;
+static int currentStreamCount = 0;
+static int8_t* iter_ptr_array;//size of [MAX_NUM_DLSCH_SEGMENTS];
+static int* PC_Flag_array;// size of[MAX_NUM_DLSCH_SEGMENTS];
+ t_nrLDPC_lut* p_lut_dev = NULL;
+static t_nrLDPC_lut* P_lut = NULL;
+
+// device buffers (allocated in LDPCinit)
+static int8_t* d_cnProcBuf = NULL;
+static int8_t* d_cnProcBufRes = NULL;
+static int8_t* d_bnProcBuf = NULL;
+static int8_t* d_bnProcBufRes = NULL;
+static int8_t* d_llrRes = NULL;
+static int8_t* d_llrProcBuf = NULL;
+static int8_t* d_llrOut = NULL;
+static int8_t* d_out = NULL; // optional if needed per-seg
+int gpuDeviceId;
+#endif
+
+extern void nrLDPC_cnProc_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                   int8_t* cnProcBuf,
+                                   int8_t* cnProcBufRes,
+                                   int8_t* bnProcBuf,
+                                   uint16_t Z);
+
+extern void nrLDPC_bnProc_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                   int8_t* bnProcBuf,
+                                   int8_t* bnProcBufRes,
+                                   int8_t* llrProcBuf,
+                                   int8_t* llrRes,
+                                   uint16_t Z);
+
+extern void nrLDPC_BnToCnPC_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                     int8_t* bnProcBufRes,
+                                     int8_t* cnProcBuf,
+                                     int8_t* cnProcBufRes,
+                                     int8_t* bnProcBuf,
+                                     uint16_t Z,
+                                     int* PC_Flag);
+#ifdef PARALLEL_STREAM
+extern void run_test_kernel();
+
+extern void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut* p_lut,
+                                                   int8_t* p_out,
+                                                   uint32_t numLLR,
+                                                   int8_t* cnProcBuf,
+                                                   int8_t* cnProcBufRes,
+                                                   int8_t* bnProcBuf,
+                                                   int8_t* bnProcBufRes,
+                                                   int8_t* llrRes,
+                                                   int8_t* llrProcBuf,
+                                                   int8_t* llrOut,
+                                                   int8_t* p_llrOut,
+                                                   int Z,
+                                                   uint8_t BG,
+                                                   uint8_t R,
+                                                   uint8_t numMaxIter,
+                                                   e_nrLDPC_outMode outMode,
+                                                   cudaStream_t* streams,
+                                                   uint8_t CudaStreamIdx,
+                                                   cudaEvent_t* doneEvent,
+                                                   int8_t* iter_ptr,
+                                                   int* PC_Flag);
+#endif
+
+//--------------------------------------------------------------
+
+//-------------------------Debug Function-----------------------
+void dump_cnProcBufRes_to_file(const int8_t* cnProcBufRes, const char* filename)
+{
+  FILE* fp = fopen(filename, "w");
+  if (fp == NULL) {
+    perror("Failed to open dump file");
+    exit(EXIT_FAILURE);
+  }
+  // printf("\nNR_LDPC_SIZE_CN_PROC_BUF: %d\n", NR_LDPC_SIZE_CN_PROC_BUF);
+
+  for (int i = 0; i < NR_LDPC_SIZE_CN_PROC_BUF; i++) {
+    fprintf(fp, "%02x ", (uint8_t)cnProcBufRes[i]);
+    if ((i + 1) % 16 == 0)
+      fprintf(fp, "\n");
+  }
+
+  fclose(fp);
+}
+
+void check_lut_pointers(const t_nrLDPC_lut* lut) {
+    if (!lut) {
+        printf("check_lut_pointers: lut is NULL\n");
+        return;
+    }
+
+    printf("Checking LUT pointers:\n");
+    printf("startAddrCnGroups       = %p\n", (void*)lut->startAddrCnGroups);
+    printf("numCnInCnGroups         = %p\n", (void*)lut->numCnInCnGroups);
+    printf("numBnInBnGroups         = %p\n", (void*)lut->numBnInBnGroups);
+    printf("startAddrBnGroups       = %p\n", (void*)lut->startAddrBnGroups);
+    printf("startAddrBnGroupsLlr    = %p\n", (void*)lut->startAddrBnGroupsLlr);
+    printf("llr2llrProcBufAddr      = %p\n", (void*)lut->llr2llrProcBufAddr);
+    printf("llr2llrProcBufBnPos     = %p\n", (void*)lut->llr2llrProcBufBnPos);
+
+    printf("circShift               = %p\n", (void*)lut->circShift);
+    printf("startAddrBnProcBuf       = %p\n", (void*)lut->startAddrBnProcBuf);
+    printf("bnPosBnProcBuf           = %p\n", (void*)lut->bnPosBnProcBuf);
+    printf("posBnInCnProcBuf         = %p\n", (void*)lut->posBnInCnProcBuf);
+}
+
+
+void dumpASS(int8_t* cnProcBufRes, const char* filename)
+{
+  FILE* fp = fopen(filename, "w");
+  if (fp == NULL) {
+    perror("Failed to open dump file");
+    exit(EXIT_FAILURE);
+  }
+  // printf("\nNR_LDPC_SIZE_CN_PROC_BUF: %d\n", NR_LDPC_SIZE_CN_PROC_BUF);
+
+  for (int i = 0; i < MAX_NUM_DLSCH_SEGMENTS * 68 * 384; i++) {
+    fprintf(fp, "%02x ", (uint8_t)cnProcBufRes[i]);
+    if ((i + 1) % 16 == 0)
+      fprintf(fp, "\n");
+  }
+
+  fclose(fp);
+}
+//--------------------------------------------------------------
+
+t_nrLDPC_lut* copy_lut_to_device(const t_nrLDPC_lut* h_lut) {
+    cudaError_t err;
+    t_nrLDPC_lut* d_lut;
+//printf("Inside copy 1\n");
+    // malloc device end struct
+    err = cudaMallocManaged((void**)&d_lut, sizeof(t_nrLDPC_lut), cudaMemAttachGlobal);
+    if (err != cudaSuccess) {
+        fprintf(stderr, "cudaMalloc failed for d_lut: %s\n", cudaGetErrorString(err));
+        exit(EXIT_FAILURE);
+    }
+//printf("Inside copy 2\n");
+    // ---------------------------
+    // copy all the member pointers
+    // ---------------------------
+
+    COPY_POINTER_MEMBER(startAddrCnGroups, uint32_t, 9);
+    printf("Inside copy 3\n");
+    COPY_POINTER_MEMBER(numCnInCnGroups, uint8_t, 9);
+    printf("Inside copy 4\n");
+    printf("host ptr = %p\n", (void*)d_lut->numBnInBnGroups);
+    COPY_POINTER_MEMBER(numBnInBnGroups, uint8_t, 30);
+    printf("Inside copy 5\n");
+    printf("host ptr = %p\n", (void*)d_lut->startAddrBnGroups);
+    printf("Inside copy 5.1\n");
+    COPY_POINTER_MEMBER(startAddrBnGroups, uint32_t, 30);
+    printf("Inside copy 6\n");
+    COPY_POINTER_MEMBER(startAddrBnGroupsLlr, uint16_t, 30);
+    printf("Inside copy 7\n");
+    COPY_POINTER_MEMBER(llr2llrProcBufAddr, uint16_t, 26);
+    printf("Inside copy 8\n");
+    COPY_POINTER_MEMBER(llr2llrProcBufBnPos, uint8_t, 26);
+    printf("Inside copy 9\n");
+    //  COPY_POINTER_MEMBER
+    // COPY_POINTER_MEMBER(numCnInCnGroups,  uint8_t,  X);
+    // COPY_POINTER_MEMBER(numBnInBnGroups,  uint8_t,  Y);
+    // ...
+
+    // ---------------------------
+    // cope with arr8_t/16_t/32_t
+    // ---------------------------
+
+
+    COPY_ARR_MEMBER(circShift,uint16_t, 9);
+    COPY_ARR_MEMBER(startAddrBnProcBuf,uint32_t, 9);
+    COPY_ARR_MEMBER(bnPosBnProcBuf,uint8_t, 9);
+    COPY_ARR_MEMBER(posBnInCnProcBuf,uint8_t, 9);
+
+    return d_lut;
+}
+
+
+extern void check_ptr_host(const void* p, const char* name);
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+bool is_device_pointer(const void* p);
+
+#ifdef __cplusplus
+}
+#endif
+
+static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
+                                           int8_t* p_out,
+                                           int n_segments,
+                                           uint32_t numLLR,
+                                           t_nrLDPC_lut* p_lut,
+                                           t_nrLDPC_dec_params* p_decParams,
+                                           t_nrLDPC_time_stats* p_profiler,
+                                           decode_abort_t* ab);
+
+void free_graphs()
+{
+  for (int i = 0; i < MAX_NUM_DLSCH_SEGMENTS; i++) {
+    if (graphCreated[i]) {
+      cudaGraphExecDestroy(decoderGraphExec[i]);
+      cudaGraphDestroy(decoderGraphs[i]);
+      graphCreated[i] = false;
+    }
+  }
+}
+
+bool check_kernel_args_for_graph(const void* p_lut, // device
+                                 const void* p_out, // may be host or device 
+                                 const void* cnProcBuf, // device expected
+                                 const void* cnProcBufRes, // device expected
+                                 const void* bnProcBuf, // device expected
+                                 const void* bnProcBufRes, // device expected
+                                 const void* llrRes, // device expected
+                                 const void* llrProcBuf, // device expected
+                                 const void* llrOut, // device expected         // may be host or device
+                                 const void* iter_ptr_array, // device expected (kernel iteration state)
+                                 const void* iter_ptr_array2, // device expected 
+                                 bool strict)
+{
+  bool ok = true;
+
+  // check p_lut: should be dvice pointer
+  if (is_device_pointer(p_lut)) {
+    fprintf(stderr, "check_kernel_args_for_graph: p_lut should be HOST pointer: %p\n", p_lut);
+    if (strict)
+      return false;
+    ok = false;
+  }
+
+  // check all the other buffer
+  const void* device_ptrs[] =
+      {cnProcBuf, cnProcBufRes, bnProcBuf, bnProcBufRes, llrRes, llrProcBuf, llrOut, iter_ptr_array, iter_ptr_array2};
+  const char* device_names[] = {"cnProcBuf",
+                                "cnProcBufRes",
+                                "bnProcBuf",
+                                "bnProcBufRes",
+                                "llrRes",
+                                "llrProcBuf",
+                                "llrOut",
+                                "iter_ptr_array",
+                                "iter_ptr_array2"};
+  for (int i = 0; i < (int)(sizeof(device_ptrs) / sizeof(device_ptrs[0])); i++) {
+    if (!is_device_pointer(device_ptrs[i])) {
+      fprintf(stderr, "check_kernel_args_for_graph: %s is NOT device pointer: %p\n", device_names[i], device_ptrs[i]);
+      if (strict)
+        return false;
+      ok = false;
+    }
+  }
+
+  if (!is_device_pointer(p_out)) {
+    fprintf(stderr, "check_kernel_args_for_graph: p_out is NOT device pointer: %p\n", p_out);
+    if (strict)
+      return false;
+    ok = false;
+  }
+  if (!is_device_pointer(d_llrOut)) {
+    fprintf(stderr, "check_kernel_args_for_graph: p_llrOut is NOT device pointer: %p\n", d_llrOut);
+    if (strict)
+      return false;
+    ok = false;
+  }
+
+  return ok;
+}
+
+int32_t LDPCinit()
+{
+  return 0;
+}
+int32_t LDPCinit_cuda()
+{
+  size_t cn_bytes = MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_CN_PROC_BUF * sizeof(int8_t);
+  size_t bn_bytes = MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_BN_PROC_BUF * sizeof(int8_t);
+  size_t llr_bytes = MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR * sizeof(int8_t);
+  size_t llrOut_bytes = NR_LDPC_MAX_NUM_LLR * sizeof(int8_t);
+
+  cudaGetDevice(&gpuDeviceId); //get device id
+
+  cudaError_t err;
+  err = cudaMallocManaged((void**)&d_cnProcBuf, cn_bytes, cudaMemAttachGlobal);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_cnProcBuf failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  err = cudaMalloc((void**)&d_cnProcBufRes, cn_bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_cnProcBufRes failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  err = cudaMalloc((void**)&d_bnProcBuf, bn_bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_bnProcBuf failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  err = cudaMalloc((void**)&d_bnProcBufRes, bn_bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_bnProcBufRes failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  err = cudaMalloc((void**)&d_llrRes, llr_bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_llrRes failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+ err = cudaMallocManaged((void**)&d_llrProcBuf, llr_bytes, cudaMemAttachGlobal);
+if (err != cudaSuccess) {
+  fprintf(stderr, "cudaMallocManaged d_llrProcBuf failed: %s\n", cudaGetErrorString(err));
+  return -1;
+}
+ err = cudaMallocManaged((void**)&iter_ptr_array, MAX_NUM_DLSCH_SEGMENTS*sizeof(int8_t), cudaMemAttachGlobal);
+if (err != cudaSuccess) {
+  fprintf(stderr, "cudaMallocManaged iter_ptr_array failed: %s\n", cudaGetErrorString(err));
+  return -1;
+}
+
+ err = cudaMallocManaged((void**)&PC_Flag_array, MAX_NUM_DLSCH_SEGMENTS*sizeof(int), cudaMemAttachGlobal);
+if (err != cudaSuccess) {
+  fprintf(stderr, "cudaMallocManaged PC_Flag_array failed: %s\n", cudaGetErrorString(err));
+  return -1;
+}
+  err = cudaMalloc((void**)&d_llrOut, MAX_NUM_DLSCH_SEGMENTS * llrOut_bytes);
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_pp_llrOut failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  err = cudaMalloc((void**)&d_out, 13*8448*sizeof(uint8_t));
+  if (err != cudaSuccess) {
+    fprintf(stderr, "cudaMalloc d_out failed: %s\n", cudaGetErrorString(err));
+    return -1;
+  }
+  if (!streamsCreated) {
+    for (int s = 0; s < MAX_NUM_DLSCH_SEGMENTS; ++s) {
+      cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
+      cudaEventCreate(&decoderDoneEvents[s]);
+    }
+    streamsCreated = true;
+  }
+  return 0;
+}
+
+int32_t LDPCshutdown()
+{
+  return 0;
+}
+int32_t LDPCshutdown_cuda()
+{
+  if (d_cnProcBuf)
+    cudaFree(d_cnProcBuf);
+  if (d_cnProcBufRes)
+    cudaFree(d_cnProcBufRes);
+  if (d_bnProcBuf)
+    cudaFree(d_bnProcBuf);
+  if (d_bnProcBufRes)
+    cudaFree(d_bnProcBufRes);
+  if (d_llrRes)
+    cudaFree(d_llrRes);
+  if (d_llrProcBuf)
+    cudaFree(d_llrProcBuf);
+  if (d_llrOut)
+    cudaFree(d_llrOut);
+  if (d_out)
+    cudaFree(d_out);
+
+  for (int s = 0; s < MAX_NUM_DLSCH_SEGMENTS; ++s) {
+    if (streamsCreated) {
+      cudaEventDestroy(decoderDoneEvents[s]);
+      cudaStreamDestroy(decoderStreams[s]);
+    }
+  }
+
+  free_graphs();
+
+  streamsCreated = false;
+  d_mem_exist = false;
+
+  return 0;
+}
+
+int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
+                    uint8_t harq_pid,
+                    uint8_t ulsch_id,
+                    uint8_t C,
+                    int8_t* p_llr,
+                    int8_t* p_out,
+                    t_nrLDPC_time_stats* p_profiler,
+                    decode_abort_t* ab)
+{ // Initialize decoder core(s) with correct LUTs
+#if STATIC_LUT
+  if (!p_lutCreated) {
+    //P_lut = p_lut;
+    printf("Start to create p_lut\n");
+    numLLR = nrLDPC_init(p_decParams, p_lut);
+    printf("p_lut Created\n");
+    //check p_lut
+    check_lut_pointers(p_lut);
+
+    #ifdef PARALLEL_STREAM
+    printf("Start to create p_lut_dev\n");
+    p_lut_dev = copy_lut_to_device(p_lut);
+    printf("p_lut_dev Created\n");
+    #endif
+    p_lutCreated = true;
+  }
+#else
+  uint32_t numLLR;
+  t_nrLDPC_lut lut;
+  t_nrLDPC_lut* p_lut = &lut;
+#endif
+
+  // Launch LDPC decoder core for one segment
+  int n_segments = p_decParams->n_segments;
+  int numIter = nrLDPC_decoder_core(p_llr, p_out, n_segments, numLLR, p_lut, p_decParams, p_profiler, ab);
+  // printf("6.1: It works here\n");
+  if (numIter >= p_decParams->numMaxIter) {
+    LOG_D(PHY, "set abort: %d, %d\n", numIter, p_decParams->numMaxIter);
+    set_abort(ab, true);
+  }
+  // printf("6.2: It works here\n");
+  return numIter;
+}
+
+/**
+   \brief PerformsnrLDPC decoding of one code block
+   \param p_llr Input LLRs
+   \param p_out Output vector
+   \param numLLR Number of LLRs
+   \param p_lut Pointer to decoder LUTs
+   \param p_decParamsnrLDPC decoder parameters
+   \param p_profilernrLDPC profiler statistics
+*/
+
+static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
+                                           int8_t* p_out,
+                                           int n_segments,
+                                           uint32_t numLLR,
+                                           t_nrLDPC_lut* p_lut,
+                                           t_nrLDPC_dec_params* p_decParams,
+                                           t_nrLDPC_time_stats* p_profiler,
+                                           decode_abort_t* ab)
+{
+  //run_test_kernel();//just for testing
+
+  // printf("n_segments = %d\n", n_segments);
+  uint16_t Z = p_decParams->Z;
+  uint8_t BG = p_decParams->BG;
+  uint8_t R = p_decParams->R; // Decoding rate: Format 15,13,... for code rates 1/5, 1/3,... */
+  uint8_t numMaxIter = p_decParams->numMaxIter;
+  e_nrLDPC_outMode outMode = p_decParams->outMode;
+  int Kprime = p_decParams->Kprime;
+  int LastTrial = p_decParams->LastTrial;
+
+  if (d_mem_exist == false) {
+    P_lut = p_lut_dev;
+    //printf("2.1\n");
+    //printf("Check P_lut = %d\n", P_lut->posBnInCnProcBuf[0]);
+    //printf("2.2\n");
+    LDPCinit_cuda(); // allocate device memory for the first time
+    //printf("2.3\n");
+    d_mem_exist = true;
+  }
+
+  //check_ptr_host(iter_ptr_array, "iter_ptr_array");
+  //check_ptr_host(PC_Flag_array, "PC_Flag_array");
+
+    for (int s = 0; s < MAX_NUM_DLSCH_SEGMENTS; s++) {
+    iter_ptr_array[s] = 0;
+    PC_Flag_array[s] = 1;
+  }
+    cudaMemPrefetchAsync(p_lut_dev, sizeof(p_lut_dev), gpuDeviceId,0);
+    cudaMemPrefetchAsync(iter_ptr_array, MAX_NUM_DLSCH_SEGMENTS*sizeof(int8_t), gpuDeviceId,0);
+    cudaMemPrefetchAsync(PC_Flag_array, MAX_NUM_DLSCH_SEGMENTS*sizeof(int), gpuDeviceId,0);
+
+  // printf("3.2: It works here\n");
+  for (int CudaStreamIdx = 0; CudaStreamIdx < n_segments; CudaStreamIdx++) {
+    int8_t* pp_llr = p_llr + CudaStreamIdx * 68 * 384; // no need put it into device
+    int8_t* pp_out = d_out + CudaStreamIdx * Kprime;
+    //printf("2.4\n");
+    // printf("Stream %d: pp_out = %p\n", CudaStreamIdx, pp_out);
+    int8_t* pp_cnProcBuf = d_cnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
+    int8_t* pp_cnProcBufRes = d_cnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
+    int8_t* pp_bnProcBuf = d_bnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
+    int8_t* pp_bnProcBufRes = d_bnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
+    int8_t* pp_llrRes = d_llrRes + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    int8_t* pp_llrProcBuf = d_llrProcBuf + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    int8_t* pp_llrOut = d_llrOut + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    // printf("4: It works here\n");
+    //  LLR preprocessing
+    // NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2llrProcBuf));
+    //printf("2.5\n");
+    nrLDPC_llr2llrProcBuf(p_lut, pp_llr, pp_llrProcBuf, Z, BG);
+    //printf("2.51\n");
+    // NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2llrProcBuf));
+    // NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2CnProcBuf));
+    if (BG == 1)
+      nrLDPC_llr2CnProcBuf_BG1(p_lut, pp_llr, pp_cnProcBuf, Z);
+      //printf("2.6\n");}
+    else
+      nrLDPC_llr2CnProcBuf_BG2(p_lut, pp_llr, pp_cnProcBuf, Z);
+    // NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2CnProcBuf));
+    //  Call scheduler for this segment and stream
+    //printf("3\n");
+    int8_t* PP_llrOut = (outMode == nrLDPC_outMode_LLRINT8) ? pp_out : pp_llrOut;
+    // printf("5: It works here\n");
+    //  Launch decoder on stream s
+
+    // cudaEventCreate(&decoderDoneEvents[CudaStreamIdx]);
+    // printf("Launching segment %d \n",CudaStreamIdx);
+
+    //-------------------------check device pointer-----------------------
+    /*check_kernel_args_for_graph(p_lut_dev,
+                                pp_out, // pointer that will be passed to kernel (可能 host)
+                                pp_cnProcBuf, // device expected
+                                pp_cnProcBufRes,
+                                pp_bnProcBuf,
+                                pp_bnProcBufRes,
+                                pp_llrRes,
+                                pp_llrProcBuf,
+                                pp_llrOut,
+                                iter_ptr_array,
+                                PC_Flag_array,
+                                false);*/
+    //------------------------check area end-------------------------------
+//printf("4\n");
+    cudaMemPrefetchAsync(d_cnProcBuf, NR_LDPC_SIZE_CN_PROC_BUF, gpuDeviceId, decoderStreams[CudaStreamIdx]);//fetch cn_proc_buf to GPU
+    cudaMemPrefetchAsync(pp_llrProcBuf, NR_LDPC_MAX_NUM_LLR*sizeof(int8_t), gpuDeviceId, decoderStreams[CudaStreamIdx]);
+
+    nrLDPC_decoder_scheduler_BG1_cuda_core(p_lut_dev,
+                                           pp_out,
+                                           numLLR,
+                                           pp_cnProcBuf,
+                                           pp_cnProcBufRes,
+                                           pp_bnProcBuf,
+                                           pp_bnProcBufRes,
+                                           pp_llrRes,
+                                           pp_llrProcBuf,
+                                           pp_llrOut,
+                                           PP_llrOut,
+                                           Z,
+                                           BG,
+                                           R,
+                                           numMaxIter,
+                                           outMode,
+                                           decoderStreams,
+                                           CudaStreamIdx,
+                                           decoderDoneEvents,
+                                           &iter_ptr_array[CudaStreamIdx],
+                                           &PC_Flag_array[CudaStreamIdx]); // stream index passed in
+
+    // cudaEventRecord(done[CudaStreamIdx], streams[CudaStreamIdx]);
+    // printf("5: It works here\n");
+  }
+  for (int s = 0; s < n_segments; ++s) {
+    // printf("Synchronizing segment %d \n",s);
+    cudaEventSynchronize(decoderDoneEvents[s]); // stop it until segment finish
+    }
+  cudaDeviceSynchronize();
+  cudaMemcpy(p_out, d_out, 13 * Kprime * sizeof(uint8_t), cudaMemcpyDeviceToHost);
+  cudaDeviceSynchronize();
+  // cudaDeviceSynchronize();
+  //  Wait for all streams
+
+  if (LastTrial == 1) {
+    // printf("Now is the last trial\n");
+    LDPCshutdown_cuda();
+  }
+  // cudaDeviceSynchronize();
+  //  dumpASS(p_out, "Dump_Output_Stream.txt");
+  //  printf("6: It works here\n");
+
+  return numMaxIter;
+}

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_cuda_GH.c

@@ -0,0 +1,434 @@
+
+
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*!\file nrLDPC_decoder.c
+ * \brief Defines thenrLDPC decoder
+ */
+
+#include <stdint.h>
+#include "PHY/sse_intrin.h"
+#include "nrLDPCdecoder_defs.h"
+#include "nrLDPC_types.h"
+#include "nrLDPC_init.h"
+#include "nrLDPC_mPass.h"
+#include "nrLDPC_cnProc.h"
+#include "nrLDPC_bnProc.h"
+#include "openair1/PHY/CODING/coding_defs.h"
+#define UNROLL_CN_PROC 1
+#define UNROLL_BN_PROC 1
+#define UNROLL_BN_PROC_PC 1
+#define UNROLL_BN2CN_PROC 1
+/*----------------------------------------------------------------------
+|                  cn processing files -->AVX512
+/----------------------------------------------------------------------*/
+
+// BG1-------------------------------------------------------------------
+#if defined(__AVX512BW__)
+
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R13_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R23_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG1_R89_AVX512.h"
+// BG2-------------------------------------------------------------------
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R15_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R13_AVX512.h"
+#include "cnProc_avx512/nrLDPC_cnProc_BG2_R23_AVX512.h"
+
+#elif defined(__AVX2__)
+
+/*----------------------------------------------------------------------
+|                  cn Processing files -->AVX2
+/----------------------------------------------------------------------*/
+
+// BG1------------------------------------------------------------------
+#include "cnProc/nrLDPC_cnProc_BG1_R13_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG1_R23_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "cnProc/nrLDPC_cnProc_BG2_R15_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG2_R13_AVX2.h"
+#include "cnProc/nrLDPC_cnProc_BG2_R23_AVX2.h"
+
+#else
+
+// BG1------------------------------------------------------------------
+#include "cnProc128/nrLDPC_cnProc_BG1_R13_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG1_R23_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG1_R89_128.h"
+// BG2 --------------------------------------------------------------------
+#include "cnProc128/nrLDPC_cnProc_BG2_R15_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG2_R13_128.h"
+#include "cnProc128/nrLDPC_cnProc_BG2_R23_128.h"
+#endif
+
+/*----------------------------------------------------------------------
+|                 bn Processing files -->AVX2
+/----------------------------------------------------------------------*/
+
+// bnProcPc-------------------------------------------------------------
+#ifdef __AVX2__
+// BG1------------------------------------------------------------------
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R13_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R23_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R15_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R13_AVX2.h"
+#include "bnProcPc/nrLDPC_bnProcPc_BG2_R23_AVX2.h"
+#else
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R13_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R23_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG1_R89_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R15_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R13_128.h"
+#include "bnProcPc128/nrLDPC_bnProcPc_BG2_R23_128.h"
+#endif
+
+// bnProc----------------------------------------------------------------
+
+#if defined(__AVX512BW__)
+// BG1-------------------------------------------------------------------
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R13_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R23_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG1_R89_AVX512.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R15_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R13_AVX512.h"
+#include "bnProc_avx512/nrLDPC_bnProc_BG2_R23_AVX512.h"
+
+#elif defined(__AVX2__)
+#include "bnProc/nrLDPC_bnProc_BG1_R13_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG1_R23_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG1_R89_AVX2.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc/nrLDPC_bnProc_BG2_R15_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG2_R13_AVX2.h"
+#include "bnProc/nrLDPC_bnProc_BG2_R23_AVX2.h"
+#else
+#include "bnProc128/nrLDPC_bnProc_BG1_R13_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG1_R23_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG1_R89_128.h"
+// BG2 --------------------------------------------------------------------
+#include "bnProc128/nrLDPC_bnProc_BG2_R15_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG2_R13_128.h"
+#include "bnProc128/nrLDPC_bnProc_BG2_R23_128.h"
+#endif
+
+// #define NR_LDPC_PROFILER_DETAIL(a) a
+#define NR_LDPC_PROFILER_DETAIL(a)
+
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+#ifdef NR_LDPC_DEBUG_MODE
+#include "nrLDPC_tools/nrLDPC_debug.h"
+#endif
+
+// decoder interface
+/**
+   \brief LDPC decoder API type definition
+   \param p_decParams LDPC decoder parameters
+   \param p_llr Input LLRs
+   \param p_llrOut Output vector
+   \param p_profiler LDPC profiler statistics
+*/
+
+//--------------------------CUDA Area---------------------------
+#include <cuda_runtime.h>
+
+
+static cudaStream_t decoderStreams[MAX_NUM_DLSCH_SEGMENTS];
+static cudaEvent_t decoderDoneEvents[MAX_NUM_DLSCH_SEGMENTS];
+static bool streamsCreated = false;
+static int currentStreamCount = 0;
+static int8_t iter_ptr_array[MAX_NUM_DLSCH_SEGMENTS];
+static  int PC_Flag_array[MAX_NUM_DLSCH_SEGMENTS];
+static  int8_t cnProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_CN_PROC_BUF] __attribute__((aligned(64))) = {0};
+static  int8_t cnProcBufRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_CN_PROC_BUF] __attribute__((aligned(64))) = {0};
+static  int8_t bnProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_BN_PROC_BUF] __attribute__((aligned(64))) = {0};
+static  int8_t bnProcBufRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_SIZE_BN_PROC_BUF] __attribute__((aligned(64))) = {0};
+static  int8_t llrRes[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
+static  int8_t llrProcBuf[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
+static  int8_t llrOut[MAX_NUM_DLSCH_SEGMENTS * NR_LDPC_MAX_NUM_LLR] __attribute__((aligned(64))) = {0};
+
+
+
+extern void nrLDPC_cnProc_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                   int8_t* cnProcBuf,
+                                   int8_t* cnProcBufRes,
+                                   int8_t* bnProcBuf,
+                                   uint16_t Z);
+
+extern void nrLDPC_bnProc_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                   int8_t* bnProcBuf,
+                                   int8_t* bnProcBufRes,
+                                   int8_t* llrProcBuf,
+                                   int8_t* llrRes,
+                                   uint16_t Z);
+
+extern void nrLDPC_BnToCnPC_BG1_cuda(const t_nrLDPC_lut* p_lut,
+                                     int8_t* bnProcBufRes,
+                                     int8_t* cnProcBuf,
+                                     int8_t* cnProcBufRes,
+                                     int8_t* bnProcBuf,
+                                     uint16_t Z,
+                                     int* PC_Flag);
+
+extern void nrLDPC_decoder_scheduler_BG1_cuda_core(const t_nrLDPC_lut* p_lut,
+                                                   int8_t* p_out,
+                                                   uint32_t numLLR,
+                                                   int8_t* cnProcBuf,
+                                                   int8_t* cnProcBufRes,
+                                                   int8_t* bnProcBuf,
+                                                   int8_t* bnProcBufRes,
+                                                   int8_t* llrRes,
+                                                   int8_t* llrProcBuf,
+                                                   int8_t* llrOut,
+                                                   int8_t* p_llrOut,
+                                                   int Z,
+                                                   uint8_t BG,
+                                                   uint8_t R,
+                                                   uint8_t numMaxIter,
+                                                   e_nrLDPC_outMode outMode,
+                                                   cudaStream_t* streams,
+                                                   uint8_t CudaStreamIdx,
+                                                   cudaEvent_t* doneEvent,
+                                                   int8_t* iter_ptr,
+                                                   int* PC_Flag);
+
+
+//--------------------------------------------------------------
+
+//-------------------------Debug Function-----------------------
+void dump_cnProcBufRes_to_file(const int8_t* cnProcBufRes, const char* filename)
+{
+  FILE* fp = fopen(filename, "w");
+  if (fp == NULL) {
+    perror("Failed to open dump file");
+    exit(EXIT_FAILURE);
+  }
+  // printf("\nNR_LDPC_SIZE_CN_PROC_BUF: %d\n", NR_LDPC_SIZE_CN_PROC_BUF);
+
+  for (int i = 0; i < NR_LDPC_SIZE_CN_PROC_BUF; i++) {
+    fprintf(fp, "%02x ", (uint8_t)cnProcBufRes[i]);
+    if ((i + 1) % 16 == 0)
+      fprintf(fp, "\n");
+  }
+
+  fclose(fp);
+}
+
+void dumpASS(int8_t* cnProcBufRes, const char* filename)
+{
+  FILE* fp = fopen(filename, "w");
+  if (fp == NULL) {
+    perror("Failed to open dump file");
+    exit(EXIT_FAILURE);
+  }
+  // printf("\nNR_LDPC_SIZE_CN_PROC_BUF: %d\n", NR_LDPC_SIZE_CN_PROC_BUF);
+
+  for (int i = 0; i < MAX_NUM_DLSCH_SEGMENTS * 68 * 384; i++) {
+    fprintf(fp, "%02x ", (uint8_t)cnProcBufRes[i]);
+    if ((i + 1) % 16 == 0)
+      fprintf(fp, "\n");
+  }
+
+  fclose(fp);
+}
+//--------------------------------------------------------------
+
+static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
+                                           int8_t* p_out,
+                                           int n_segments,
+                                           uint32_t numLLR,
+                                           t_nrLDPC_lut* p_lut,
+                                           t_nrLDPC_dec_params* p_decParams,
+                                           t_nrLDPC_time_stats* p_profiler,
+                                           decode_abort_t* ab);
+
+int32_t LDPCinit()
+{
+  return 0;
+}
+
+int32_t LDPCshutdown()
+{
+  return 0;
+}
+
+int32_t LDPCdecoder(t_nrLDPC_dec_params* p_decParams,
+                    uint8_t harq_pid,
+                    uint8_t ulsch_id,
+                    uint8_t C,
+                    int8_t* p_llr,
+                    int8_t* p_out,
+                    t_nrLDPC_time_stats* p_profiler,
+                    decode_abort_t* ab)
+{
+  uint32_t numLLR;
+  t_nrLDPC_lut lut;
+  t_nrLDPC_lut* p_lut = &lut;
+
+  // Initialize decoder core(s) with correct LUTs
+  numLLR = nrLDPC_init(p_decParams, p_lut);
+
+  // Launch LDPC decoder core for one segment
+  int n_segments = p_decParams->n_segments;
+  int numIter = nrLDPC_decoder_core(p_llr, p_out, n_segments, numLLR, p_lut, p_decParams, p_profiler, ab);
+  // printf("6.1: It works here\n");
+  if (numIter >= p_decParams->numMaxIter) {
+    LOG_D(PHY, "set abort: %d, %d\n", numIter, p_decParams->numMaxIter);
+    set_abort(ab, true);
+  }
+  // printf("6.2: It works here\n");
+  return numIter;
+}
+
+/**
+   \brief PerformsnrLDPC decoding of one code block
+   \param p_llr Input LLRs
+   \param p_out Output vector
+   \param numLLR Number of LLRs
+   \param p_lut Pointer to decoder LUTs
+   \param p_decParamsnrLDPC decoder parameters
+   \param p_profilernrLDPC profiler statistics
+*/
+
+static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
+                                           int8_t* p_out,
+                                           int n_segments,
+                                           uint32_t numLLR,
+                                           t_nrLDPC_lut* p_lut,
+                                           t_nrLDPC_dec_params* p_decParams,
+                                           t_nrLDPC_time_stats* p_profiler,
+                                           decode_abort_t* ab)
+{
+  // printf("n_segments = %d\n", n_segments);
+  uint16_t Z = p_decParams->Z;
+  uint8_t BG = p_decParams->BG;
+  uint8_t R = p_decParams->R; // Decoding rate: Format 15,13,... for code rates 1/5, 1/3,... */
+  uint8_t numMaxIter = p_decParams->numMaxIter;
+  e_nrLDPC_outMode outMode = p_decParams->outMode;
+  int Kprime = p_decParams->Kprime;
+  int LastTrial = p_decParams->LastTrial;
+  // printf("Kprime = %d\n", Kprime);
+  //  int8_t* cnProcBuf=  cnProcBuf;
+  //  int8_t* cnProcBufRes= cnProcBufRes;
+  // printf("1: It works here\n");
+
+
+  // printf("2: It works here\n");
+  //  Minimum number of iterations is 1
+  //  0 iterations means hard-decision on input LLRs
+  //  Initialize with parity check fail != 0
+  // printf("3: It works here\n");
+  //  Initialization
+  //cudaStream_t streams[MAX_NUM_DLSCH_SEGMENTS];
+  //cudaEvent_t done[MAX_NUM_DLSCH_SEGMENTS]; // MAX_NUM_SEGMENTS = stream num
+
+  for (int s = 0; s < MAX_NUM_DLSCH_SEGMENTS; s++) {
+    iter_ptr_array[s] = 0;
+    PC_Flag_array[s] = 1;
+  }
+  // printf("3.1: It works here\n");
+  if (!streamsCreated) {
+  for (int s = 0; s < n_segments; ++s) {
+    cudaStreamCreateWithFlags(&decoderStreams[s], cudaStreamNonBlocking);
+    cudaEventCreate(&decoderDoneEvents[s]);
+  }
+  streamsCreated = true;
+  currentStreamCount = n_segments;
+}
+
+  // printf("3.2: It works here\n");
+  for (int CudaStreamIdx = 0; CudaStreamIdx < n_segments; CudaStreamIdx++) {
+    int8_t* pp_llr = p_llr + CudaStreamIdx * 68 * 384;
+    int8_t* pp_out = p_out + CudaStreamIdx * Kprime;
+    // printf("Stream %d: pp_out = %p\n", CudaStreamIdx, pp_out);
+    int8_t* pp_cnProcBuf = cnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
+    int8_t* pp_cnProcBufRes = cnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_CN_PROC_BUF;
+    int8_t* pp_bnProcBuf = bnProcBuf + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
+    int8_t* pp_bnProcBufRes = bnProcBufRes + CudaStreamIdx * NR_LDPC_SIZE_BN_PROC_BUF;
+    int8_t* pp_llrRes = llrRes + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    int8_t* pp_llrProcBuf = llrProcBuf + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    int8_t* pp_llrOut = llrOut + CudaStreamIdx * NR_LDPC_MAX_NUM_LLR;
+    // printf("4: It works here\n");
+    //  LLR preprocessing
+    //NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2llrProcBuf));
+    nrLDPC_llr2llrProcBuf(p_lut, pp_llr, pp_llrProcBuf, Z, BG);
+    //NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2llrProcBuf));
+    //NR_LDPC_PROFILER_DETAIL(start_meas(&p_profiler->llr2CnProcBuf));
+    if (BG == 1)
+      nrLDPC_llr2CnProcBuf_BG1(p_lut, pp_llr, pp_cnProcBuf, Z);
+    else
+      nrLDPC_llr2CnProcBuf_BG2(p_lut, pp_llr, pp_cnProcBuf, Z);
+    //NR_LDPC_PROFILER_DETAIL(stop_meas(&p_profiler->llr2CnProcBuf));
+    // Call scheduler for this segment and stream
+    int8_t* pp_p_llrOut = (outMode == nrLDPC_outMode_LLRINT8) ? pp_out : pp_llrOut;
+    // printf("5: It works here\n");
+    //  Launch decoder on stream s
+    
+    //cudaEventCreate(&decoderDoneEvents[CudaStreamIdx]);
+    //printf("Launching segment %d \n",CudaStreamIdx);
+    nrLDPC_decoder_scheduler_BG1_cuda_core(p_lut,
+                                           pp_out,
+                                           numLLR,
+                                           pp_cnProcBuf,
+                                           pp_cnProcBufRes,
+                                           pp_bnProcBuf,
+                                           pp_bnProcBufRes,
+                                           pp_llrRes,
+                                           pp_llrProcBuf,
+                                           pp_llrOut,
+                                           pp_p_llrOut,
+                                           Z,
+                                           BG,
+                                           R,
+                                           numMaxIter,
+                                           outMode,
+                                           decoderStreams,
+                                           CudaStreamIdx,
+                                           decoderDoneEvents,
+                                           &iter_ptr_array[CudaStreamIdx],
+                                           &PC_Flag_array[CudaStreamIdx]); // stream index passed in
+  
+  }
+for (int s = 0; s < n_segments; ++s) {
+ // printf("Synchronizing segment %d \n",s);
+    cudaEventSynchronize(decoderDoneEvents[s]);  // stop until segment decode
+}
+cudaDeviceSynchronize();
+
+if(LastTrial == 1){
+  //printf("Now is the last trial\n");
+  for (int s = 0; s < n_segments; s++) {
+    cudaEventDestroy(decoderDoneEvents[s]);
+    cudaStreamSynchronize(decoderStreams[s]);
+    cudaStreamDestroy(decoderStreams[s]);
+    streamsCreated = false;
+  }
+}
+//cudaDeviceSynchronize();
+  // dumpASS(p_out, "Dump_Output_Stream.txt");
+  // printf("6: It works here\n");
+
+  return numMaxIter;
+}

openair1/PHY/CODING/nrLDPC_defs.h

@@ -58,6 +58,7 @@ typedef struct {
   time_stats_t *tprep;
   time_stats_t *tparity;
   time_stats_t *toutput;
+  time_stats_t *tconcat;
   /// Size in bits of the code segments
   uint32_t K;
   /// Number of lifting sizes to fit the payload
@@ -93,5 +94,7 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
                                     t_nrLDPC_time_stats *,
                                     decode_abort_t *ab);
 typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
+typedef int32_t(LDPC_encoderfunc32_t)(uint8_t **, uint32_t [4][68*384], encoder_implemparams_t *);
+
 #endif
 

openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu

@@ -1,13 +1,15 @@
 #include <stdio.h>
 #include <stdint.h>
+#if USE_CUDA
 #include <cuda_runtime.h>
+#endif
 // generated code for Zc=384, byte encoding
-__global__ void ldpc_BG1_Zc384_worker(uint8_t *c,uint8_t *d) {
-  uint32_t *c32=(uint32_t *)c;
-  uint32_t *d32=(uint32_t *)d;
+__global__ void ldpc_BG1_Zc384_worker(uint32_t *c[4],uint32_t *d[4]) {
+  uint32_t *c32=c[blockIdx.x];
+  uint32_t *d32=d[blockIdx.x];
 
   int i2 = threadIdx.x;
-  int i1 = blockIdx.x;
+  int i1 = blockIdx.y;
   if (i2 < 384) {
     c32+=i2;
     d32+=i2;
@@ -200,8 +202,26 @@ __global__ void ldpc_BG1_Zc384_worker(uint8_t *c,uint8_t *d) {
     }
   }
 }
-extern "C" int ldpc_BG1_Zc384_cuda32(uint8_t *c,uint8_t *d) { 
-ldpc_BG1_Zc384_worker<<<46,384>>>(c,d);
+extern "C" int ldpc_BG1_Zc384_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs) { 
+for (int i=0;i<n_inputs;i++){
+  printf("host c[%d] = %p, d[%d] = %p\n", i, (void*)c[i], i, (void*)d[i]);
+  cudaPointerAttributes attr;
+  cudaPointerAttributes attr1;
+  cudaError_t aerr = cudaPointerGetAttributes(&attr, c[i]);
+  cudaError_t aerr1 = cudaPointerGetAttributes(&attr1, d[i]);
+  if (aerr == cudaSuccess)
+    printf("  attr.memoryType=%d device=%d\n", attr.type, attr.device);
+  else
+    printf("  cudaPointerGetAttributes failed: %s\n", cudaGetErrorString(aerr));
+  if (aerr1 == cudaSuccess)
+    printf("  attr1.memoryType=%d device=%d\n", attr1.type, attr1.device);
+  else
+    printf("  cudaPointerGetAttributes failed: %s\n", cudaGetErrorString(aerr1));
+}
+
+printf("n_input = %d\n", n_inputs);
+dim3 numblocks(n_inputs,46);
+ldpc_BG1_Zc384_worker<<<numblocks,384>>>(c,d);//<<<numblocks,384>>>(c,d);
  cudaDeviceSynchronize();
   return(0);
 }

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check_cuda.c

@@ -37,26 +37,37 @@
 #include "common/utils/LOG/log.h"
 #include <cuda_runtime.h>
 
-#define USE_UMEM 1 
 
-static void encode_parity_check_part_cuda(uint32_t *cc, uint32_t *d, short BG,short Zc,short Kb, int ncols)
+uint32_t *cc0[4];
+
+int ldpc_BG1_Zc384_cuda32(uint32_t **c,uint32_t **d,int n_inputs);
+
+extern int managed;
+
+void encode_parity_check_part_cuda(uint32_t **cc, uint32_t **d, short BG,short Zc,short Kb, int ncols, int n_inputs)
 {
-#ifdef USE_UMEM
-  uint32_t c[2 * 22 * Zc] ; //double size matrix of c
-  for (int i1 = 0; i1 < ncols; i1++)   {
-    memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(uint32_t));
-    memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(uint32_t));
+  
+  uint32_t c[n_inputs][2 * 22 * Zc] ; //double size matrix of c
+				      printf("managed = %d\n",managed);
+  for (int s=0;s<n_inputs;s++)				      
+    for (int i1 = 0; i1 < ncols; i1++)   {
+      if (managed) {	    
+        memcpy(&c[s][2 * i1 * Zc], &cc[s][i1 * Zc], Zc * sizeof(uint32_t));
+        memcpy(&c[s][(2 * i1 + 1) * Zc], &cc[s][i1 * Zc], Zc * sizeof(uint32_t));
+      }
+      else {
+        cudaMemcpy(&cc0[s][2 * i1 * Zc], &cc[s][i1 * Zc], Zc * sizeof(uint32_t),1);
+        cudaMemcpy(&cc0[s][(2 * i1 + 1) * Zc], &cc[s][i1 * Zc], Zc * sizeof(uint32_t),1);
+      }
+    }
+  uint32_t *cp[n_inputs];
+  for (int s=0; s<n_inputs;s++) {
+    if (managed)	  
+      cp[s]=c[s];
+    else
+      cp[s]=cc0[s];
   }
-#else
-  cudaError_t err;
-  uint32_t *c=NULL;
-  err = cudaMalloc((void**)&c,2 * 22 * Zc * sizeof(uint32_t));
-  if (err != cudaSuccess) printf("CUDA Error: %s\n", cudaGetErrorString(err)); 							
-  for (int i1 = 0; i1 < ncols; i1++)   {
-    cudaMemcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(uint32_t),1);
-    cudaMemcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(uint32_t),1);
-  }
-#endif
+
   if (BG == 1) {
     switch (Zc) {
       case 176:
@@ -71,7 +82,7 @@ static void encode_parity_check_part_cuda(uint32_t *cc, uint32_t *d, short BG,sh
 	AssertFatal(1==0,"BG %d Zc %d not supported yet for CUDA\n",BG, Zc);
         break;
       case 384:
-	ldpc_BG1_Zc384_cuda32(c, d);
+	ldpc_BG1_Zc384_cuda32(cp, d, n_inputs);
         break;
       default:
         AssertFatal(false, "BG %d Zc %d is not supported yet\n", BG, Zc);
@@ -103,7 +114,6 @@ static void encode_parity_check_part_cuda(uint32_t *cc, uint32_t *d, short BG,sh
     }
   } else
     AssertFatal(false, "BG %d is not supported\n", BG);
-  cudaFree(c);
 }
 
 

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder32.c

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*!\file ldpc_encoder.c
+ * \brief Defines the LDPC encoder
+ * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
+ * \email openair_tech@eurecom.fr
+ * \date 27-03-2018
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "defs.h"
+#include "assertions.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+#include "ldpc_generate_coefficient.c"
+
+
+void cuda_support_init() {
+   return;
+}
+int LDPCencoder32(uint8_t **input, uint32_t output[4][68*384], encoder_implemparams_t *impp)
+{
+	AssertFatal(1==0,"Should not be getting here\n");
+}
+

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_cuda32.c

@@ -42,10 +42,50 @@
 
 #include "ldpc_encode_parity_check_cuda.c"
 #include "ldpc_generate_coefficient.c"
-#define USE_UMEM 0
+
+#include <cuda_runtime.h>
+
+uint32_t *cc0[4];
+uint32_t *dd0[4];
+
+int managed = 0, concurrent = 0, uva = 0, pageable = 0, pageable_uses_host = 0;
+
+void cuda_support_init() {
+printf("We are doing the init here!\n");
+
+    int dev = 0;
+    struct cudaDeviceProp prop;
+    cudaGetDeviceProperties(&prop, dev);
 
 
-int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
+    //cudaDeviceGetAttribute(&managed, cudaDevAttrManagedMemory, dev);
+    cudaDeviceGetAttribute(&concurrent, cudaDevAttrConcurrentManagedAccess, dev);
+    cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, dev);
+    cudaDeviceGetAttribute(&pageable, cudaDevAttrPageableMemoryAccess, dev);
+    cudaDeviceGetAttribute(&pageable_uses_host, cudaDevAttrPageableMemoryAccessUsesHostPageTables, dev);
+
+    LOG_I(NR_PHY,"Device: %s (cc %d.%d)\n", prop.name, prop.major, prop.minor);
+    LOG_I(NR_PHY,"Unified Virtual Addressing (UVA): %s\n", uva ? "YES" : "NO");
+    //LOG_I(NR_PHY,"Managed (Unified) Memory:        %s\n", managed ? "YES" : "NO");
+    LOG_I(NR_PHY,"Concurrent managed access:       %s\n", concurrent ? "YES" : "NO");
+    LOG_I(NR_PHY,"Pageable memory access:          %s\n", pageable ? "YES" : "NO");
+    LOG_I(NR_PHY,"Uses host page tables:           %s\n", pageable_uses_host ? "YES" : "NO");
+
+  // initialize input and output memory
+  if (!managed) {
+    printf("How you doing?\n");
+    for (int i=0;i<4;i++) {
+      cudaError_t err=cudaMalloc((void**)&cc0[i],22*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    }
+  }
+  for (int i=0;i<4;i++) {
+      cudaError_t err=cudaMalloc((void**)&dd0[i],46*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+  }
+}
+
+int LDPCencoder32(uint8_t **input, uint32_t output[4][68*384], encoder_implemparams_t *impp)
 {
   //set_log(PHY, 4);
 
@@ -53,22 +93,11 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
   int Kb = impp->Kb;
   short block_length = impp->K;
   short BG = impp->BG;
-  uint32_t *out32=(uint32_t*)output;
   int nrows=0,ncols=0;
   int rate=3;
   int no_punctured_columns,removed_bit;
-  //Table of possible lifting sizes
-  uint8_t temp;
-  int simd_size;
-  unsigned int macro_segment, macro_segment_end;
-
-//  printf("input %p output %p\n",input[0],output);
-
-  macro_segment = impp->first_seg;
-  macro_segment_end = (impp->n_segments > impp->first_seg + 32) ? impp->first_seg + 32 : impp->n_segments;
-  ///printf("macro_segment: %d\n", macro_segment);
-  ///printf("macro_segment_end: %d\n", macro_segment_end );
 
+  if(impp->tinput != NULL) start_meas(impp->tinput);
   //determine number of bits in codeword
   if (BG==1)
     {
@@ -84,157 +113,504 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
     }
 
 #ifdef DEBUG_LDPC
-  LOG_D(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
-  LOG_D(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
+  LOG_I(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
+  LOG_I(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
 #endif
 
   AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length);
 
-  uint32_t  cc[22*Zc]; //padded input, unpacked, max size
-		       //
-#ifndef USE_UMEM 
-  uint32_t *dd;
+  int n_inputs = (impp->n_segments/32)+(((impp->n_segments&31) > 0) ? 1: 0);
+  uint32_t  cc[4][22*Zc]; //padded input, unpacked, max size
 
-  cudaError_t err=cudaMalloc((void**)&dd,46*Zc*sizeof(uint32_t));
-  if (err != cudaSuccess) printf("CUDA Error: %s\n", cudaGetErrorString(err)); 							
-#else
- uint32_t dd[46*Zc];
-#endif
   // calculate number of punctured bits
   no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
   removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate);
-  //printf("%d\n",no_punctured_columns);
-  //printf("%d\n",removed_bit);
-  // unpack input
-  memset(cc,0,sizeof(cc));
-#ifndef USE_UMEM 
-  err = cudaMemset(dd,0,46*Zc*sizeof(uint32_t));
-  if (err != cudaSuccess) printf("CUDA Error: %s\n", cudaGetErrorString(err)); 							
-#else
-  memset(dd,0,sizeof(dd));
-#endif
-  if(impp->tinput != NULL) start_meas(impp->tinput);
-
-  //interleave up to 32 transport-block segements at a time
-
-  unsigned int i_dword = 0;
-
-#if 0 //defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
-  const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2),
-                              _mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8),
-                              _mm512_set1_epi8(0x10), _mm512_set1_epi8(0x20),
-                              _mm512_set1_epi8(0x40), _mm512_set1_epi8(0x80)};
-  const __m512i zero512 = _mm512_setzero_si512();
-  const uint8_t perm[64]__attribute__((aligned(64))) = {7,6,5,4,3,2,1,0,         15,14,13,12,11,10,9,8,
-                                                        23,22,21,20,19,18,17,16, 31,30,29,28,27,26,25,24,
-                                                        39,38,37,36,35,34,33,32, 47,46,45,44,43,42,41,40,
-                                                        55,54,53,52,51,50,49,48, 63,62,61,60,59,58,57,56};
-  register __m512i c512;
-
-  for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) {
-    unsigned int i = i_byte >> 6;
-    c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]);
-    for (int j = macro_segment + 1; j < macro_segment_end; j++) {
-      c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment]));
-    }
-    c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512);
-    ((__m512i *)cc)[i] = c512;
+  // clear input
+  for (int i=0;i<n_inputs;i++) {
+    memset(cc[i],0,22*Zc*sizeof(uint32_t));
   }
-#endif
 
-#if 0//ndef __aarch64__
-  simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
-  simde__m256i andmask  = simde_mm256_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
-  simde__m256i zero256   = simde_mm256_setzero_si256();
-  simde__m256i masks[8];
-  register simde__m256i c256;
-  masks[0] = simde_mm256_set1_epi8(0x1);
-  masks[1] = simde_mm256_set1_epi8(0x2);
-  masks[2] = simde_mm256_set1_epi8(0x4);
-  masks[3] = simde_mm256_set1_epi8(0x8);
-  masks[4] = simde_mm256_set1_epi8(0x10);
-  masks[5] = simde_mm256_set1_epi8(0x20);
-  masks[6] = simde_mm256_set1_epi8(0x40);
-  masks[7] = simde_mm256_set1_epi8(0x80);
-
-  for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) {
-    unsigned int i = i_byte >> 5;
-    c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]);
-    for (int j=macro_segment+1; j < macro_segment_end; j++) {    
-      c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[j])[i]), shufmask),andmask),zero256),masks[j-macro_segment]),c256);
-    }
-    ((simde__m256i *)cc)[i] = c256;
-  }
-#endif
-
-#if 0//def __aarch64__
-  // s0_0 s1_0 s2_0 ... s31_0 s0_1 ... s31_1 ... s0_3 ... s31_3
-  // s0_4 s1_4 s2_4 ....s31_4 s0_5 ... s31_5 ... s0_7 ... s31_7
-  simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
-  simde__m128i andmask  = simde_mm_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
-  simde__m128i zero128   = simde_mm_setzero_si128();
-  simde__m128i masks[8];
-  register simde__m128i c128;
-  masks[0] = simde_mm_set1_epi8(0x1);
-  masks[1] = simde_mm_set1_epi8(0x2);
-  masks[2] = simde_mm_set1_epi8(0x4);
-  masks[3] = simde_mm_set1_epi8(0x8);
-  masks[4] = simde_mm_set1_epi8(0x10);
-  masks[5] = simde_mm_set1_epi8(0x20);
-  masks[6] = simde_mm_set1_epi8(0x40);
-  masks[7] = simde_mm_set1_epi8(0x80);
-
-  for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) {
-    unsigned int i = i_byte >> 4;
-    c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]);
-    for (int j=macro_segment+1; j < macro_segment_end; j++) {    
-      c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128);
-    }
-    ((simde__m128i *)cc)[i] = c128;
-  }
-#endif
 
+#if 0
+  // unoptimized version of input processing
   for (; i_dword < block_length; i_dword++) {
     unsigned int i = i_dword;
-    for (int j = macro_segment; j < macro_segment_end; j++) {
+    for (int j = 0; j < impp->n_segments; j++) {
 
-      temp = ((input[j][i/8]&((1<<7))>>(i&7)))>>(7-(i&7));
-      cc[i] |= (temp << (j-macro_segment));
+      temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
+      cc[j>>5][i] |= (temp << (j&31));
     }
   }
+#else
+#ifdef __AVX2__
 
+#else
+  int i2=0;
+  const int32_t ucShifta[32][4] = { 
+	  {-7,-6,-5,-4}, // 0  
+	  {-6,-5,-4,-3}, // 1
+	  {-5,-4,-3,-2}, // 2 
+	  {-4,-3,-2,-1}, // 3 
+	  {-3,-2,-1,0}, // 4 
+	  {-2,-1,0,1}, // 5
+	  {-1,0,1,2}, // 6
+	  {0,1,2,3}, // 7
+	  {1,2,3,4}, // 8
+	  {2,3,4,5}, // 9
+	  {3,4,5,6}, // 10
+	  {4,5,6,7}, // 11
+	  {5,6,7,8}, // 12
+	  {6,7,8,9}, // 13
+	  {7,8,9,10}, // 14
+	  {8,9,10,11}, // 15
+	  {9,10,11,12}, // 16
+	  {10,11,12,13}, // 17
+	  {11,12,13,14}, // 18
+	  {12,13,14,15}, // 19
+	  {13,14,15,16}, // 20
+	  {14,15,16,17}, // 21
+	  {15,16,17,18}, // 22
+	  {16,17,18,19}, // 23
+	  {17,18,19,20}, // 24
+	  {18,19,20,21}, // 25
+	  {19,20,21,22}, // 26
+	  {20,21,22,23}, // 27
+	  {21,22,23,24}, // 28
+	  {22,23,24,25}, // 29
+	  {23,24,25,26}, // 30
+	  {24,25,26,27}}; // 31
+  const int32_t ucShiftb[32][4] = { 
+	  {-3,-2,-1,0}, // 0 
+	  {-2,-1,0,1}, // 1
+	  {-1,0,1,2}, // 2
+	  {0,1,2,3}, // 3
+	  {1,2,3,4}, // 4
+	  {2,3,4,5}, // 5
+	  {3,4,5,6}, // 6 
+	  {4,5,6,7}, // 7
+	  {5,6,7,8}, // 8
+	  {6,7,8,9}, // 9
+	  {7,8,9,10}, // 10
+	  {8,9,10,11}, // 11
+	  {9,10,11,12}, // 12
+	  {10,11,12,13}, // 13
+	  {11,12,13,14}, // 14
+	  {12,13,14,15}, // 15
+	  {13,14,15,16}, // 16
+	  {14,15,16,17}, // 17
+	  {15,16,17,18}, // 18
+	  {16,17,18,19}, // 19
+	  {17,18,19,20}, // 20
+	  {18,19,20,21}, // 21
+	  {19,20,21,22}, // 22
+	  {20,21,22,23}, // 23
+	  {21,22,23,24}, // 24
+	  {22,23,24,25}, // 25
+	  {23,24,25,26}, // 26
+	  {24,25,26,27}, // 27
+	  {25,26,27,28}, // 28
+	  {26,27,28,29}, // 29
+	  {27,28,29,30}, // 30
+	  {28,29,30,31}}; // 31
+			      
+  const int32_t ucShiftc[32][4] = { 
+	  {-15,-14,-13,-12}, // 0
+	  {-14,-13,-12,-11}, // 1
+	  {-13,-12,-11,-10}, // 2
+          {-12,-11,-10,-9}, // 3
+	  {-11,-10,-9,-8}, // 4
+	  {-10,-9,-8,-7}, // 5
+	  {-9,-8,-7,-6}, // 6
+	  {-8,-7,-6,-5}, // 7
+	  {-7,-6,-5,-4}, // 8  
+	  {-6,-5,-4,-3}, // 9
+	  {-5,-4,-3,-2}, // 10 
+	  {-4,-3,-2,-1}, // 11 
+	  {-3,-2,-1,0}, // 12 
+	  {-2,-1,0,1}, // 13
+	  {-1,0,1,2}, // 14
+	  {0,1,2,3}, // 15
+	  {1,2,3,4}, // 16
+	  {2,3,4,5}, // 17
+	  {3,4,5,6}, // 18
+	  {4,5,6,7}, // 19
+	  {5,6,7,8}, // 20
+	  {6,7,8,9}, // 21
+	  {7,8,9,10}, // 22
+	  {8,9,10,11}, // 23
+	  {9,10,11,12}, // 24
+	  {10,11,12,13}, // 25
+	  {11,12,13,14}, // 26
+	  {12,13,14,15}, // 27
+	  {13,14,15,16}, // 28
+	  {14,15,16,17}, // 29
+	  {15,16,17,18}, // 30
+	  {16,17,18,19}}; // 31
+  const int32_t ucShiftd[32][4] = { 
+	  {-11,-10,-9,-8}, // 0
+	  {-10,-9,-8,-7}, // 1
+	  {-9,-8,-7,-6}, // 2
+	  {-8,-7,-6,-5}, // 3
+	  {-7,-6,-5,-4}, // 4  
+	  {-6,-5,-4,-3}, // 5
+	  {-5,-4,-3,-2}, // 6
+	  {-4,-3,-2,-1}, // 7 
+	  {-3,-2,-1,0}, // 8
+	  {-2,-1,0,1}, // 9
+	  {-1,0,1,2}, // 10
+	  {0,1,2,3}, // 11 
+	  {1,2,3,4}, // 12
+	  {2,3,4,5}, // 13
+	  {3,4,5,6}, // 14 
+	  {4,5,6,7}, // 15
+	  {5,6,7,8}, // 16
+	  {6,7,8,9}, // 17
+	  {7,8,9,10}, // 18
+	  {8,9,10,11}, // 19
+	  {9,10,11,12}, // 20
+	  {10,11,12,13}, // 21
+	  {11,12,13,14}, // 22
+	  {12,13,14,15}, // 23
+	  {13,14,15,16}, // 24
+	  {14,15,16,17}, // 25
+	  {15,16,17,18}, // 26
+	  {16,17,18,19}, // 27
+	  {17,18,19,20}, // 28
+	  {18,19,20,21}, // 29
+	  {19,20,21,22}, // 30
+	  {20,21,22,23}};// 31
+			  
+  const int32_t ucShifte[32][4] = { 
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}, // 23
+	  {1,2,3,4}, // 24
+	  {2,3,4,5}, // 25
+	  {3,4,5,6}, // 26
+	  {4,5,6,7}, // 27
+	  {5,6,7,8}, // 28
+	  {6,7,8,9}, // 29
+	  {7,8,9,10}, // 30
+	  {8,9,10,11}}; // 31 
+			  
+  const int32_t ucShiftf[32][4] = { 
+	  {-19,-18,-17,-16}, // 0 
+	  {-18,-17,-16,-15}, // 1
+	  {-17,-16,-15,-14}, // 2
+	  {-16,-15,-14,-13}, // 3
+	  {-15,-14,-13,-12}, // 4
+	  {-14,-13,-12,-11}, // 5
+	  {-13,-12,-11,-10}, // 6
+          {-12,-11,-10,-9}, // 7
+	  {-11,-10,-9,-8}, // 8
+	  {-10,-9,-8,-7}, // 9  
+	  {-9,-8,-7,-6}, // 10 
+	  {-8,-7,-6,-5}, // 11
+	  {-7,-6,-5,-4}, // 12 
+	  {-6,-5,-4,-3}, // 13
+	  {-5,-4,-3,-2}, // 14
+	  {-4,-3,-2,-1}, // 15 
+	  {-3,-2,-1,0}, // 16
+	  {-2,-1,0,1}, // 17
+	  {-1,0,1,2}, // 18
+	  {0,1,2,3}, // 19
+	  {1,2,3,4}, // 20
+	  {2,3,4,5}, // 21
+	  {3,4,5,6}, // 22 
+	  {4,5,6,7}, // 23
+	  {5,6,7,8}, // 24
+	  {6,7,8,9}, // 25
+	  {7,8,9,10}, // 26
+	  {8,9,10,11}, // 27
+	  {9,10,11,12}, // 28
+	  {10,11,12,13}, // 29
+	  {11,12,13,14}, // 30
+	  {12,13,14,15}}; // 31
+			  
+  const int32_t ucShiftg[32][4] = { 
+	  {-31,-30,-29,-28},
+	  {-30,-29,-28,-27},
+	  {-29,-28,-27,-26},
+	  {-28,-27,-26,-25},
+	  {-27,-26,-25,-24},
+	  {-26,-25,-24,-23},
+	  {-25,-24,-23,-22},
+	  {-24,-23,-22,-21},
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}}; // 23
+			  
+  const int32_t ucShifth[32][4] = { 
+	  {-27,-26,-25,-24}, // 0
+	  {-26,-25,-24,-23}, // 1
+	  {-25,-24,-23,-22}, // 2
+	  {-24,-23,-22,-21}, // 3
+	  {-23,-22,-21,-20}, // 4 
+	  {-22,-21,-20,-19}, // 5
+	  {-21,-20,-19,-18}, // 6
+	  {-20,-19,-18,-17}, // 7
+	  {-19,-18,-17,-16}, // 8 
+	  {-18,-17,-16,-15}, // 9
+	  {-17,-16,-15,-14}, // 10
+	  {-16,-15,-14,-13}, // 11
+	  {-15,-14,-13,-12}, // 12
+	  {-14,-13,-12,-11}, // 13
+	  {-13,-12,-11,-10}, // 14
+          {-12,-11,-10,-9}, // 15
+	  {-11,-10,-9,-8}, // 16
+	  {-10,-9,-8,-7}, // 17 
+	  {-9,-8,-7,-6}, // 18 
+	  {-8,-7,-6,-5}, // 19
+	  {-7,-6,-5,-4}, // 20 
+	  {-6,-5,-4,-3}, // 21
+	  {-5,-4,-3,-2}, // 22
+	  {-4,-3,-2,-1}, // 23 
+	  {-3,-2,-1,0}, // 24 
+	  {-2,-1,0,1}, // 25
+	  {-1,0,1,2}, // 26
+	  {0,1,2,3}, // 27
+	  {1,2,3,4}, // 28
+	  {2,3,4,5}, // 29
+	  {3,4,5,6}, // 30
+          {4,5,6,7}}; // 31 
+  const uint32_t __attribute__ ((aligned (16))) masksa[4] = {0x80,0x40,0x20,0x10};
+  const uint32_t __attribute__ ((aligned (16))) masksb[4] = {0x8,0x4,0x2,0x1};
+  const uint32_t __attribute__ ((aligned (16))) masksc[4] = {0x8000,0x4000,0x2000,0x1000};
+  const uint32_t __attribute__ ((aligned (16))) masksd[4] = {0x800,0x400,0x200,0x100};
+  const uint32_t __attribute__ ((aligned (16))) maskse[4] = {0x800000,0x400000,0x200000,0x100000};
+  const uint32_t __attribute__ ((aligned (16))) masksf[4] = {0x80000,0x40000,0x20000,0x10000};
+  const uint32_t __attribute__ ((aligned (16))) masksg[4] = {0x80000000,0x40000000,0x20000000,0x10000000};
+  const uint32_t __attribute__ ((aligned (16))) masksh[4] = {0x8000000,0x4000000,0x2000000,0x1000000};
+  int32x4_t vshifta[32],vshiftb[32],vshiftc[32],vshiftd[32],vshifte[32],vshiftf[32],vshiftg[32],vshifth[32];
+  uint32x4_t vmasksa  = vld1q_u32(masksa);
+  uint32x4_t vmasksb  = vld1q_u32(masksb);
+  uint32x4_t vmasksc  = vld1q_u32(masksc);
+  uint32x4_t vmasksd  = vld1q_u32(masksd);
+  uint32x4_t vmaskse  = vld1q_u32(maskse);
+  uint32x4_t vmasksf  = vld1q_u32(masksf);
+  uint32x4_t vmasksg  = vld1q_u32(masksg);
+  uint32x4_t vmasksh  = vld1q_u32(masksh);
+  uint32x4_t in;
+
+  for (int n=0;n<32;n++) { 
+    vshifta[n] = vld1q_s32(ucShifta[n]);
+    vshiftb[n] = vld1q_s32(ucShiftb[n]);
+    vshiftc[n] = vld1q_s32(ucShiftc[n]);
+    vshiftd[n] = vld1q_s32(ucShiftd[n]);
+    vshifte[n] = vld1q_s32(ucShifte[n]);
+    vshiftf[n] = vld1q_s32(ucShiftf[n]);
+    vshiftg[n] = vld1q_s32(ucShiftg[n]);
+    vshifth[n] = vld1q_s32(ucShifth[n]);
+  }
+  i2=0;
+  int j0=0,j1=0,j2=0,j3=0;
+  if (impp->n_segments <= 32) {
+	  j0=impp->n_segments;
+  }
+  else if (impp->n_segments <= 64) {
+     j0=32;
+     j1=impp->n_segments-32;
+  }
+  else if (impp->n_segments <= 96) {
+     j0=32; j1=32;
+     j2=impp->n_segments-64;
+  }
+  else if (impp->n_segments <= 128) {
+     j0=32; j1=32; j2=32;
+     j3=impp->n_segments-96;
+  }
+  uint32x4_t *ccp,cc0,cc1,cc2,cc3,cc4,cc5,cc6,cc7;
+  for (int i=0; i < (block_length>>5); i++,i2+=8) {
+    in = vdupq_n_u32(((uint32_t*)input[0])[i]);
+    cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+    cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+    cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+    cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+    cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+    cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+    cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+    cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+    for (int j = 1; j < j0; j++) {
+      in = vdupq_n_u32(((uint32_t*)input[j])[i]);
+      cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+      cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+      cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+      cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+      cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+      cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+      cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+      cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+    }
+    ccp=&((uint32x4_t *)cc[0])[i2];
+    ccp[0] = cc0;
+    ccp[1] = cc1;
+    ccp[2] = cc2;
+    ccp[3] = cc3;
+    ccp[4] = cc4;
+    ccp[5] = cc5;
+    ccp[6] = cc6;
+    ccp[7] = cc7;
+    if (j1>0) {
+      in = vdupq_n_u32(((uint32_t*)input[32])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j1; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[32+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[1])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j2>0) {
+      in = vdupq_n_u32(((uint32_t*)input[64])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j2; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[64+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[2])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j3>0) {
+      in = vdupq_n_u32(((uint32_t*)input[96])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j3; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[96+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc[3])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+  }
+#endif
+#endif
   if(impp->tinput != NULL) stop_meas(impp->tinput);
 
   if (BG==1 && Zc==384)  {
-    // extend matrix
-    if(impp->tprep != NULL) start_meas(impp->tprep);
-    if(impp->tprep != NULL) stop_meas(impp->tprep);
     //parity check part
-
-//    printf("calling encode_parity_check_part_cuda cc %p dd %p\n",cc,dd);
     if(impp->tparity != NULL) start_meas(impp->tparity);
-    encode_parity_check_part_cuda(cc, dd, BG, Zc, Kb, ncols);
+    uint32_t *ccp[n_inputs];
+    for (int s=0;s<n_inputs;s++) {
+      ccp[s]=cc[s];
+    }
+    encode_parity_check_part_cuda(ccp, dd0, BG, Zc, Kb, ncols,n_inputs);
     if(impp->tparity != NULL) stop_meas(impp->tparity);
   }
   else {
 	  AssertFatal(1==0,"Only BG1 Zc=384 for now\n");
   }
 
-if(impp->toutput != NULL) start_meas(impp->toutput);
-  memcpy(out32,&cc[2*Zc],sizeof(uint32_t)*(block_length-(2*Zc)));
-//  printf("cudaMemcpy: dst %p, src %p, length %d, block_length %d, nrows %d, no_punctured_columns\n",
-//         &out32[block_length-(2*Zc)],dd,sizeof(uint32_t)*((nrows-no_punctured_columns) * Zc-removed_bit),block_length,nrows,no_punctured_columns);
- // uint32_t dummy[((nrows-no_punctured_columns) * Zc-removed_bit)];
-#ifdef USE_UMEM
-  memcpy(&out32[block_length-(2*Zc)],dd,sizeof(uint32_t)*((nrows-no_punctured_columns) * Zc-removed_bit));
-#else
-  err = cudaMemcpy(&out32[block_length-(2*Zc)],dd,sizeof(uint32_t)*((nrows-no_punctured_columns) * Zc-removed_bit),2);
-  if (err != cudaSuccess) printf("CUDA Error: %s\n", cudaGetErrorString(err)); 							
-#endif
-#ifdef USE_UMEM
-#else
-  cudaFree(dd);
-#endif
+
+  if(impp->toutput != NULL) start_meas(impp->toutput);
+  for (int s=0;s<n_inputs;s++) {
+    memcpy(output[s],&cc[s][2*Zc],sizeof(uint32_t)*(block_length-(2*Zc)));
+    cudaError_t err = cudaMemcpy(&output[s][block_length-(2*Zc)],dd0[s],sizeof(uint32_t)*((nrows-no_punctured_columns) * Zc-removed_bit),2);
+    AssertFatal(err == cudaSuccess, "dd0[%d] %p CUDA Error: %s\n", s, dd0[s],cudaGetErrorString(err)); 							
+  }
   if(impp->toutput != NULL) stop_meas(impp->toutput);
   return 0;
 }

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c

@@ -43,6 +43,10 @@
 #include "ldpc_encode_parity_check.c"
 #include "ldpc_generate_coefficient.c"
 
+void ini_input32_luts() {
+	return;
+}
+
 int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
 {
   //set_log(PHY, 4);
@@ -193,9 +197,6 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
   if(impp->tinput != NULL) stop_meas(impp->tinput);
 
   if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
-    // extend matrix
-    if(impp->tprep != NULL) start_meas(impp->tprep);
-    if(impp->tprep != NULL) stop_meas(impp->tprep);
     //parity check part
     if(impp->tparity != NULL) start_meas(impp->tparity);
     encode_parity_check_part_optim(cc, dd, BG, Zc, Kb, simd_size, ncols);
@@ -207,8 +208,10 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
       return(-1);
     }
   }
+  if(impp->toutput != NULL) start_meas(impp->toutput);
   memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
   memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
+  if(impp->toutput != NULL) stop_meas(impp->toutput);
   return 0;
 }
 

openair1/PHY/CODING/nrLDPC_extern.h

@@ -29,6 +29,7 @@ typedef struct ldpc_interface_s {
   LDPC_shutdownfunc_t *LDPCshutdown;
   LDPC_decoderfunc_t *LDPCdecoder;
   LDPC_encoderfunc_t *LDPCencoder;
+  LDPC_encoderfunc32_t *LDPCencoder32;
 } ldpc_interface_t;
 
 // Global var to limit the rework of the dirty legacy code
@@ -42,6 +43,7 @@ LDPC_initfunc_t LDPCinit;
 LDPC_shutdownfunc_t LDPCshutdown;
 LDPC_decoderfunc_t LDPCdecoder;
 LDPC_encoderfunc_t LDPCencoder;
+LDPC_encoderfunc32_t LDPCencoder32;
 
 // inline functions:
 #endif

openair1/PHY/CODING/nrLDPC_load.c

@@ -56,14 +56,17 @@ int load_LDPClib(char *version, ldpc_interface_t *itf)
   loader_shlibfunc_t shlib_fdesc[] = {{.fname = "LDPCinit"},
                                       {.fname = "LDPCshutdown"},
                                       {.fname = "LDPCdecoder"},
-                                      {.fname = "LDPCencoder"}};
+                                      {.fname = "LDPCencoder"},
+                                      {.fname = "LDPCencoder32"}};
   int ret;
   ret = load_module_version_shlib(libname, version, shlib_fdesc, sizeofArray(shlib_fdesc), NULL);
   AssertFatal((ret >= 0), "Error loading ldpc decoder");
+  LOG_I(NR_PHY,"loading LDPC version %s\n",version);
   itf->LDPCinit = (LDPC_initfunc_t *)shlib_fdesc[0].fptr;
   itf->LDPCshutdown = (LDPC_shutdownfunc_t *)shlib_fdesc[1].fptr;
   itf->LDPCdecoder = (LDPC_decoderfunc_t *)shlib_fdesc[2].fptr;
   itf->LDPCencoder = (LDPC_encoderfunc_t *)shlib_fdesc[3].fptr;
+  itf->LDPCencoder32 = (LDPC_encoderfunc32_t *)shlib_fdesc[4].fptr;
 
   AssertFatal(itf->LDPCinit() == 0, "error starting LDPC library %s %s\n", libname, version);
 

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -779,6 +779,7 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
   time_stats_t *tprep = &gNB->tprep;
   time_stats_t *tparity = &gNB->tparity;
   time_stats_t *toutput = &gNB->toutput;
+  time_stats_t *tconcat = &gNB->tconcat;
   time_stats_t *dlsch_rate_matching_stats = &gNB->dlsch_rate_matching_stats;
   time_stats_t *dlsch_interleaving_stats = &gNB->dlsch_interleaving_stats;
   time_stats_t *dlsch_segmentation_stats = &gNB->dlsch_segmentation_stats;
@@ -826,7 +827,6 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
 
   unsigned char output[size_output >> 3] __attribute__((aligned(64)));
   bzero(output, sizeof(output));
-
   start_meas(dlsch_encoding_stats);
   if (nr_dlsch_encoding(gNB,
                         msgTx,
@@ -838,12 +838,14 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
                         tprep,
                         tparity,
                         toutput,
+                        tconcat,
                         dlsch_rate_matching_stats,
                         dlsch_interleaving_stats,
                         dlsch_segmentation_stats)
       == -1) {
     return;
   }
+
   stop_meas(dlsch_encoding_stats);
 
   unsigned char *output_ptr = output;

openair1/PHY/NR_TRANSPORT/nr_dlsch.h

@@ -54,7 +54,8 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
                       time_stats_t *toutput,
                       time_stats_t *dlsch_rate_matching_stats,
                       time_stats_t *dlsch_interleaving_stats,
-                      time_stats_t *dlsch_segmentation_stats);
+                      time_stats_t *dlsch_segmentation_stats,
+                      time_stats_t *dlsch_output_stats);
 
 void dump_pdsch_stats(FILE *fd,PHY_VARS_gNB *gNB);
 

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -116,6 +116,7 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
                       time_stats_t *tprep,
                       time_stats_t *tparity,
                       time_stats_t *toutput,
+		      time_stats_t *tconcat,
                       time_stats_t *dlsch_rate_matching_stats,
                       time_stats_t *dlsch_interleaving_stats,
                       time_stats_t *dlsch_segmentation_stats)
@@ -132,6 +133,7 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
                                                        .tprep = tprep,
                                                        .tparity = tparity,
                                                        .toutput = toutput,
+						       .tconcat = tconcat,
                                                        .TBs = TBs};
 
   int num_segments = 0;

openair1/PHY/defs_gNB.h

@@ -506,6 +506,7 @@ typedef struct PHY_VARS_gNB_s {
   time_stats_t tprep;
   time_stats_t tparity;
   time_stats_t toutput;
+  time_stats_t tconcat;
   
   time_stats_t dlsch_rate_matching_stats;
   time_stats_t dlsch_interleaving_stats;

openair1/SIMULATION/NR_PHY/dlschsim.c

@@ -516,7 +516,7 @@ int main(int argc, char **argv)
         bzero(output, sizeof(output));
 	if (input_fd == NULL) {
     msgDataTx.num_pdsch_slot = 1;
-	  nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+	  nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 	}
 
 	for (SNR = snr0; SNR < snr1 && !stop; SNR += snr_step) {

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -1015,6 +1015,7 @@ printf("%d\n", slot);
     reset_meas(&gNB->tprep);
     reset_meas(&gNB->tparity);
     reset_meas(&gNB->toutput);
+    reset_meas(&gNB->tconcat);
     reset_meas(&gNB->phase_comp_stats);
 
     uint32_t errors_scrambling[16] = {0};
@@ -1314,12 +1315,17 @@ printf("%d\n", slot);
       printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");
       printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time");
       printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time");
+      gNB->tinput.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time");
-      printStatIndent3(&gNB->tprep,"DLSCH LDPC input preparation time");
+      gNB->tparity.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->tparity,"DLSCH LDPC parity generation time");
+      gNB->toutput.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->toutput,"DLSCH LDPC output generation time");
-      printStatIndent3(&gNB->dlsch_rate_matching_stats,"DLSCH Rate Mataching time");
+      gNB->dlsch_rate_matching_stats.trials = gNB->dlsch_encoding_stats.trials;
+      printStatIndent3(&gNB->dlsch_rate_matching_stats,"DLSCH Rate Matching time");
+      gNB->dlsch_interleaving_stats.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->dlsch_interleaving_stats,  "DLSCH Interleaving time");
+      printStatIndent3(&gNB->tconcat,  "DLSCH Segment Concatenation time");
       printStatIndent2(&gNB->dlsch_modulation_stats,"DLSCH modulation time");
       printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time");
       printStatIndent2(&gNB->dlsch_precoding_stats,"DLSCH Mapping/Precoding time");