initial commit of fp16 procedures (tested only on TX for AVX512_FP16)

simplify SIMD version of precoder

.gitmodules

@@ -1,4 +1,3 @@
 [submodule "openair2/E2AP/flexric"]
 	path = openair2/E2AP/flexric
-	url = https://gitlab.eurecom.fr/mosaic5g/flexric.git
-	branch = remotes/origin/service-models-integration
+	url = https://gitlab.eurecom.fr/mosaic5g/flexric

CMakeLists.txt

@@ -113,6 +113,9 @@ else()
   message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables")
 endif()
 
+eval_boolean(AUTODETECT_AVX512_FP16 DEFINED CPUFLAGS AND CPUFLAGS MATCHES "avx512_fp16")
+add_boolean_option(AVX512_FP16 ${AUTODETECT_AVX512_FP16} "Whether AVX512_FP16 intrinsics are available on the host processor" ON)
+
 eval_boolean(AUTODETECT_AVX512 DEFINED CPUFLAGS AND CPUFLAGS MATCHES "avx512")
 add_boolean_option(AVX512 ${AUTODETECT_AVX512} "Whether AVX512 intrinsics is available on the host processor" ON)
 
@@ -127,8 +130,11 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
   # The following intrinsics are assumed to be available on any x86 system
   # (avx, f16c, fma, gnfi, mmx, pclmul, sse, sse2, sse3, xop)
   set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_AVX_NATIVE -DSIMDE_X86_F16C_NATIVE -DSIMDE_X86_FMA_NATIVE -DSIMDE_X86_GFNI_NATIVE -DSIMDE_X86_MMX_NATIVE -DSIMDE_X86_PCLMUL_NATIVE -DSIMDE_X86_SSE2_NATIVE -DSIMDE_X86_SSE3_NATIVE -DSIMDE_X86_SSE_NATIVE -DSIMDE_X86_XOP_HAVE_COM_ -DSIMDE_X86_XOP_NATIVE")
+  message(STATUS "AVX512_FP16 intrinsics are ${AVX512_FP16}")
   message(STATUS "AVX512 intrinsics are ${AVX512}")
-  if(${AVX512})
+  if (${AVX512_FP16})
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX512BW_NATIVE -DSIMDE_X86_AVX512F_NATIVE -DSIMDE_X86_AVX512VL_NATIVE -mavx512bw -march=sapphirerapids -mtune=sapphirerapids")
+  elseif(${AVX512})
     set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_AVX512BW_NATIVE -DSIMDE_X86_AVX512F_NATIVE -DSIMDE_X86_AVX512VL_NATIVE -mavx512bw -march=skylake-avx512 -mtune=skylake-avx512")
   else()
     set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -mno-avx512f -march=native")
@@ -183,6 +189,10 @@ if(CMAKE_C_COMPILER_ID STREQUAL "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "Clang
   set(commonOpts "${commonOpts} -Wno-unused-command-line-argument")
 endif()
 
+if (ENABLE_CUDA)
+  set(commonOpts "${commonOpts} -DENABLE_CUDA")
+endif()
+
 set(CMAKE_C_FLAGS
   "${C_FLAGS_PROCESSOR} ${commonOpts} -std=gnu11 -funroll-loops ${CMAKE_C_FLAGS}")
 set(CMAKE_CXX_FLAGS
@@ -970,6 +980,8 @@ set(PHY_SRC_UE
   ${OPENAIR1_DIR}/PHY/NR_TRANSPORT/srs_rx.c
   ${OPENAIR1_DIR}/PHY/NR_TRANSPORT/nr_uci_tools_common.c
   )
+  set(PHY_NR_SRC_CU
+     )
   set(PHY_NR_UE_SRC
   ${OPENAIR1_DIR}/PHY/INIT/nr_parms.c
   ${OPENAIR1_DIR}/PHY/MODULATION/nr_modulation.c
@@ -1047,7 +1059,17 @@ target_include_directories(PHY_UE PRIVATE ${blas_INCLUDE_DIRS} ${lapacke_INCLUDE
 add_library(PHY_NR_COMMON ${PHY_NR_SRC_COMMON})
 target_link_libraries(PHY_NR_COMMON PUBLIC UTIL)
 
-add_library(PHY_NR ${PHY_NR_SRC})
+if (ENABLE_CUDA)
+  find_package(CUDA REQUIRED)
+  if (NOT CUDA_FOUND)
+    message(FATAL_ERROR "no CUDA found")
+  endif()
+  set(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_87;")
+  set(CUDA_VERBOS_BUILD_ON)
+  cuda_add_library(PHY_NR ${PHY_NR_SRC} ${PHY_NR_SRC_CU})
+else()
+  add_library(PHY_NR ${PHY_NR_SRC})
+endif()
 target_link_libraries(PHY_NR nr_phy_common nr_common nr_fapi_p5)
 
 add_library(PHY_NR_NO_AVX_256 ${PHY_NR_SRC})
@@ -1943,13 +1965,21 @@ target_link_libraries(smallblocktest PRIVATE
 )
 
 
+#if (ENABLE_CUDA)
+#  find_package(CUDA REQUIRED)
+#  if (NOT CUDA_FOUND)
+#    message(FATAL_ERROR "no CUDA found")
+#  endif()
+#  cuda_add_library(ldpctest)
+#else()
+#endif()
 add_executable(ldpctest
   ${OPENAIR1_DIR}/PHY/CODING/TESTBENCH/ldpctest.c
   ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_load.c
 )
 target_link_libraries(ldpctest PRIVATE
   -Wl,--start-group UTIL SIMU PHY_COMMON PHY_NR_COMMON -Wl,--end-group
-  m pthread dl shlib_loader ${T_LIB}
+  m pthread dl shlib_loader ${T_LIB} ${CUDA_LIBRARIES}
   # link 'check_crc' to make it resolved in the LDPC coding libraries
   # 'check_crc' is not used in ldpctest so it is not linked in the executable by default
   # --whole-archive links 'check_crc' in the executable even though it is note used, see 'man ld'

cmake_targets/build_oai

@@ -135,6 +135,8 @@ Options:
   CC=/usr/bin/clang CXX=/usr/bin/clang++ ./build_oai ... --sanitize-memory
 --sanitize-thread | -fsanitize=thread
   Enable the thread sanitizer on all targets
+--use-cuda
+  Tell build to use NVCC for appropriate .cu files and cuda_runtime functions  
 -h | --help
    Print this help"
 }
@@ -382,6 +384,9 @@ function main() {
         --sanitize-thread | -fsanitize=thread)
             CMAKE_CMD="$CMAKE_CMD -DSANITIZE_THREAD=True"
             shift;;
+	--use-cuda)
+	    CMAKE_CMD="$CMAKE_CMD -DENABLE_CUDA=True"
+	    shift;;
        --trace-asn1c-enc-dec)
             CMAKE_CMD="$CMAKE_CMD -DTRACE_ASN1C_ENC_DEC=ON"
             echo_info "Enabling asn1c internal traces via OAI logging system"

common/platform_types.h

@@ -32,7 +32,11 @@
 
 #include <stdint.h>
 #include <stdbool.h>
-
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#include <float.h>
+#ifdef __aarch64__
+#include <arm_fp16.h>
+#endif
 #define ALIGNARRAYSIZE(a, b) (((a + b - 1) / b) * b)
 #define ALNARS_16_4(a) ALIGNARRAYSIZE(a, 4)
 
@@ -46,6 +50,18 @@ typedef struct complexf {
   float i;
 } cf_t;
 
+#ifdef FLT16_MAX
+typedef struct complexf16 {
+#ifdef __aarch64__
+  float16_t r;
+  float16_t i;
+#else
+  _Float16 r;
+  _Float16 i;
+#endif
+} cf16_t;
+#endif
+
 typedef struct complex8 {
   int8_t r;
   int8_t i;

common/utils/time_meas.h

@@ -105,7 +105,7 @@ static inline uint64_t rdtsc_oai(void) __attribute__((always_inline));
 static inline uint64_t rdtsc_oai(void)
 {
 	  uint64_t r = 0;
-	    asm volatile("mrs %0, cntvct_el0" : "=r"(r));
+	    __asm__ volatile("mrs %0, cntvct_el0" : "=r"(r));
 	      return r;
 }
 
@@ -113,7 +113,7 @@ static inline uint64_t rdtsc_oai(void)
 static inline uint32_t rdtsc_oai(void) __attribute__((always_inline));
 static inline uint32_t rdtsc_oai(void) {
   uint32_t r = 0;
-  asm volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(r) );
+  __asm__ volatile("mrc p15, 0, %0, c9, c13, 0" : "=r"(r) );
   return r;
 }
 #endif

executables/nr-softmodem-common.h

@@ -96,7 +96,8 @@
 #define CONFIG_HLP_TELN          "Start embedded telnet server \n"
 #define CONFIG_HLP_SNR           "Set average SNR in dB (for --siml1 option)\n"
 #define CONFIG_HLP_NOS1          "Disable s1 interface\n"
-
+#define CONFIG_HLP_USEGPU        "Use GPU for processing offload"
+#define CONFIG_HLP_USEFP16       "Use FP16 arithmetic when possible"
 /*--------------------------------------------------------------------------------------------------------------------------------*/
 /*                                            command line parameters for LOG utility                                             */
 /*   optname         helpstr          paramflags          XXXptr                     defXXXval            type           numelt   */

executables/nr-softmodem.c

@@ -103,6 +103,7 @@ unsigned int mmapped_dma=0;
 uint64_t downlink_frequency[MAX_NUM_CCs][4];
 int32_t uplink_frequency_offset[MAX_NUM_CCs][4];
 char *uecap_file;
+uint32_t use_gpu;
 
 runmode_t mode = normal_txrx;
 

executables/nr-softmodem.h

@@ -27,6 +27,8 @@
   {"U" ,                    CONFIG_HLP_ULBM_PHYTEST,   0,                .u64ptr=&ulsch_slot_bitmap,          .defintval=0,                     TYPE_UINT64, 0},        \
   {"usrp-tx-thread-config", CONFIG_HLP_USRP_THREAD,    0,                .iptr=&usrp_tx_thread,               .defstrval=0,                     TYPE_INT,    0},        \
   {"uecap_file",            CONFIG_HLP_UECAP_FILE,     0,                .strptr=&uecap_file,                 .defstrval="./uecap_ports1.xml",  TYPE_STRING, 0},        \
+  {"use_gpu",            CONFIG_HLP_USE_GPU,     0,                .uptr=&use_gpu,                 .defintval=0,  TYPE_UINT, 0},        \
+  {"use_fp16",            CONFIG_HLP_USE_FP16,     0,                .uptr=&use_fp16,                 .defintval=0,  TYPE_UINT, 0},        \
 }
 // clang-format on
 
@@ -39,7 +41,8 @@ extern uint32_t target_ul_bw;
 extern uint64_t dlsch_slot_bitmap;
 extern uint64_t ulsch_slot_bitmap;
 extern char *uecap_file;
-
+extern uint32_t use_gpu;
+extern uint32_t use_fp16;
 // In nr-gnb.c
 extern void init_gNB();
 extern void stop_gNB(int);

openair1/PHY/CODING/CMakeLists.txt

@@ -12,6 +12,7 @@ target_link_libraries(coding PRIVATE log_headers)
 add_library(ldpc_orig MODULE
             nrLDPC_decoder/nrLDPC_decoder.c
             nrLDPC_encoder/ldpc_encoder.c
+            nrLDPC_encoder/ldpc_encoder32.c
 )
 set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
@@ -19,6 +20,7 @@ target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
 add_library(ldpc MODULE
             nrLDPC_decoder/nrLDPC_decoder.c
             nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
+            nrLDPC_encoder/ldpc_encoder32.c
 )
 set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS)
@@ -41,22 +43,28 @@ if (ENABLE_LDPC_CUDA)
   if (NOT CUDA_FOUND)
     message(FATAL_ERROR "no CUDA found")
   endif()
-  SET(CUDA_NVCC_FLAG "${CUDA_NVCC_FLAGS};-arch=sm_60;")
+  SET(CUDA_NVCC_FLAGS "${CUDA_NVCC_FLAGS};-arch=sm_90;")
   SET(CUDA_VERBOSE_BUILD ON)
   cuda_add_library(ldpc_cuda MODULE
-                   nrLDPC_decoder_LYC/nrLDPC_decoder_LYC.cu
-                   nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
+	  #                   nrLDPC_decoder_LYC/nrLDPC_decoder_LYC.cu
+                   nrLDPC_decoder/nrLDPC_decoder.c
+		   nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
+		   nrLDPC_encoder/ldpc_encoder_cuda32.c
+		   nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu
+		   nrLDPC_encoder/ldpc_input.cu
                    # The slot coding layer cannot be linked with
                    # target_link_libraries like above
                    # because of cuda_add_library
                    # which already uses target_link_libraries
                    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c
+                   nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c
                    nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
                    nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c
   )
+  target_link_libraries(ldpc_cuda ldpc_gen_HEADERS ${T_LIB})
   set_target_properties(ldpc_cuda PROPERTIES CUDA_SEPARABLE_COMPILATION ON)
   set_target_properties(ldpc_cuda PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-
+  target_compile_definitions(ldpc_cuda PRIVATE ENABLE_CUDA)
   add_dependencies(ldpctest ldpc_cuda)
   add_dependencies(nr-softmodem ldpc_cuda)
   add_dependencies(nr-uesoftmodem ldpc_cuda)

openair1/PHY/CODING/TESTBENCH/ldpctest.c

@@ -34,6 +34,9 @@
 #include "openair1/PHY/defs_nr_common.h"
 #include "coding_unitary_defs.h"
 #include "common/utils/LOG/log.h"
+#ifdef ENABLE_CUDA
+#include <cuda_runtime.h>
+#endif
 
 #define MAX_BLOCK_LENGTH 8448
 
@@ -104,14 +107,16 @@ one_measurement_t test_ldpc(short max_iterations,
                             unsigned char qbits,
                             short Kprime,
                             unsigned int ntrials,
-                            int n_segments)
+                            int n_segments,
+	                    int gen_code,
+			    int use32bit)
 {
   one_measurement_t ret = {0};
   reset_meas(&ret.time_optim);
   reset_meas(&ret.time_decoder);
   // clock initiate
   // time_stats_t time,time_optim,tinput,tprep,tparity,toutput, time_decoder;
-  time_stats_t time, tinput, tprep, tparity, toutput;
+  time_stats_t time, tinput, tinput_memcpy, tprep, tparity, toutput;
   double n_iter_mean = 0;
   double n_iter_std = 0;
   int n_iter_max = 0;
@@ -119,14 +124,22 @@ one_measurement_t test_ldpc(short max_iterations,
   double sigma;
   sigma = 1.0 / sqrt(2 * SNR);
   cpu_meas_enabled = 1;
-  uint8_t *test_input[MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * NR_MAX_NB_LAYERS];
-  uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
+#ifdef ENABLE_CUDA
+  uint8_t **test_input,*test_input_p;
+#else
+  uint8_t *test_input[n_segments * NR_MAX_NB_LAYERS];
+#endif
+  uint8_t *channel_input[n_segments];
+  uint8_t estimated_output[n_segments][Kprime];
   memset(estimated_output, 0, sizeof(estimated_output));
-  uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
   uint8_t *channel_input_optim;
+
   // double channel_output[68 * 384];
-  double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
-  int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
+  double modulated_input[n_segments][68 * 384];
+  memset(modulated_input,0,sizeof(modulated_input));
+  int8_t channel_output_fixed[n_segments][68 * 384];
+  memset(channel_output_fixed,0,sizeof(channel_output_fixed));
+
   short BG = 0, nrows = 0; //,ncols;
   int i1, Kb = 0;
   int R_ind = 0;
@@ -134,7 +147,7 @@ one_measurement_t test_ldpc(short max_iterations,
   int code_rate_vec[8] = {15, 13, 25, 12, 23, 34, 56, 89};
   // double code_rate_actual_vec[8] = {0.2, 0.33333, 0.4, 0.5, 0.66667, 0.73333, 0.81481, 0.88};
 
-  t_nrLDPC_dec_params decParams[MAX_NUM_DLSCH_SEGMENTS] = {0};
+  t_nrLDPC_dec_params decParams[n_segments];
 
   t_nrLDPC_time_stats decoder_profiler = {0};
 
@@ -142,6 +155,7 @@ one_measurement_t test_ldpc(short max_iterations,
 
   reset_meas(&time);
   reset_meas(&tinput);
+  reset_meas(&tinput_memcpy);
   reset_meas(&tprep);
   reset_meas(&tparity);
   reset_meas(&toutput);
@@ -259,18 +273,29 @@ one_measurement_t test_ldpc(short max_iterations,
   printf("To: %d\n", (Kb + nrows - no_punctured_columns) * Zc - removed_bit);
   printf("number of undecoded bits: %d\n", (Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
 
+//  ldpc_toCompare.LDPCinit();
   // generate input block
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+#ifdef ENABLE_CUDA
+  cudaHostAlloc((void**)&test_input_p,n_segments*sizeof(uint8_t*),cudaHostAllocMapped);
+  test_input=(uint8_t **)test_input_p;
+  printf("test input %p\n",test_input);
+#endif
+  for (int j = 0; j < n_segments; j++) {
+#ifdef ENABLE_CUDA
+    cudaHostAlloc((void**)&test_input[j],((K + 7) & ~7) / 8,cudaHostAllocMapped);
+    printf("test input[%d] %p\n",j,test_input[j]);
+#else
     test_input[j] = malloc16(((K + 7) & ~7) / 8);
     memset(test_input[j], 0, ((K + 7) & ~7) / 8);
+#endif
     channel_input[j] = malloc16(68 * 384);
     memset(channel_input[j], 0, 68 * 384);
   }
-  channel_input_optim = malloc16(68 * 384);
-  memset(channel_input_optim, 0, 68 * 384);
+  channel_input_optim = malloc16(68 * 384 * sizeof(uint32_t));
+  if (use32bit ==0) memset(channel_input_optim, 0, 68 * 384 * sizeof(uint32_t));
 
   // Fill input segments with random values
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+  for (int j = 0; j < n_segments; j++) {
     int i = 0;
     for (i = 0; i < ((Kprime + 7) & ~7) / 8; i++)
       test_input[j][i] = (uint8_t)rand();
@@ -282,13 +307,17 @@ one_measurement_t test_ldpc(short max_iterations,
   }
 
   encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K};
-  impp.gen_code = 1;
-
+  impp.gen_code = gen_code;
+  impp.tparity = &tparity;
+  impp.tinput = &tinput;
+  impp.tinput_memcpy = &tinput_memcpy;
+  impp.toutput = &toutput;
   if (ntrials == 0)
     ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
   impp.gen_code = 0;
   decode_abort_t dec_abort;
   init_abort(&dec_abort);
+  uint32_t **output32;
   for (int trial = 0; trial < ntrials; trial++) {
     unsigned int segment_bler = 0;
     //// encoder
@@ -301,26 +330,27 @@ one_measurement_t test_ldpc(short max_iterations,
     impp.n_segments = n_segments;
     start_meas(&ret.time_optim);
     impp.first_seg = 0;
-    ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
+    if (use32bit==0) ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
+    else output32=ldpc_toCompare.LDPCencoder32(test_input, &impp);
     stop_meas(&ret.time_optim);
 
     if (ntrials == 1)
       for (int j = 0; j < n_segments; j++)
         for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
-          if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
-            printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
-            return ret;
-          }
+          if (((use32bit == 0) && (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1))) ||                 ((use32bit == 1) && (channel_input[j][i] != (((output32[0][i] >> j) & 0x1))))) {
+               printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (((uint32_t*)channel_input_optim)[i] >> j) & 0x1);
+               return ret;
+            }
         }
-
+    int bit;
     for (int j = 0; j < n_segments; j++) {
       for (int i = 2 * Zc; i < (Kb + nrows - no_punctured_columns) * Zc - removed_bit; i++) {
 #ifdef DEBUG_CODER
         if ((i & 0xf) == 0)
           printf("\ne %d..%d:    ", i, i + 15);
 #endif
-
-        if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0)
+        bit = (use32bit==0) ? ((channel_input_optim[i - 2 * Zc] >> j) & 0x1) : ((output32[0][i - 2 * Zc] >> j) & 0x1);
+        if (bit == 0)
           modulated_input[j][i] = 1.0; /// sqrt(2);  //QPSK
         else
           modulated_input[j][i] = -1.0; /// sqrt(2);
@@ -345,7 +375,6 @@ one_measurement_t test_ldpc(short max_iterations,
       decParams[j].numMaxIter = max_iterations;
       decParams[j].outMode = nrLDPC_outMode_BIT;
       decParams[j].Kprime = Kprime;
-      ldpc_toCompare.LDPCinit();
     }
     for (int j = 0; j < n_segments; j++) {
       start_meas(&ret.time_decoder);
@@ -388,8 +417,12 @@ one_measurement_t test_ldpc(short max_iterations,
 
   ret.errors_bit_uncoded = ret.errors_bit_uncoded / (double)((Kb + nrows - no_punctured_columns - 2) * Zc - removed_bit);
 
-  for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
+  for (int j = 0; j < n_segments; j++) {
+#ifdef ENABLE_CUDA
+    cudaFree(test_input[j]);	  
+#else
     free(test_input[j]);
+#endif
     free(channel_input[j]);
   }
   free(channel_input_optim);
@@ -397,6 +430,7 @@ one_measurement_t test_ldpc(short max_iterations,
   print_meas(&time, "ldpc_encoder", NULL, NULL);
   print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);
   print_meas(&tinput, "ldpc_encoder_optim(input)", NULL, NULL);
+  print_meas(&tinput_memcpy, "ldpc_encoder_optim(input memcpy)", NULL, NULL);
   print_meas(&tprep, "ldpc_encoder_optim(prep)", NULL, NULL);
   print_meas(&tparity, "ldpc_encoder_optim(parity)", NULL, NULL);
   print_meas(&toutput, "ldpc_encoder_optim(output)", NULL, NULL);
@@ -437,19 +471,20 @@ int main(int argc, char *argv[])
 
   int n_trials = 1;
   double SNR_step = 0.1;
-
-  randominit();
+  int gen_code = 1;
+  randominit(0);
   int test_uncoded = 0;
   n_iter_stats_t dec_iter[400] = {0};
 
   short BG = 0, Zc;
+  int use32bit=0;
 
   if ((uniqCfg = load_configmodule(argc, argv, CONFIG_ENABLECMDLINEONLY)) == 0) {
     exit_fun("[LDPCTEST] Error, configuration module init failed\n");
   }
   logInit();
 
-  while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:G:n:d:i:t:u:hv:")) != -1) {
+  while ((c = getopt(argc, argv, "--:O:q:r:s:S:l:Gn:d:i:t:u:hv:g:")) != -1) {
     /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
     /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
     if (c == 1 || c == '-' || c == 'O')
@@ -475,6 +510,7 @@ int main(int argc, char *argv[])
 
       case 'G':
         ldpc_version = "_cuda";
+	use32bit=1;
         break;
 
       case 'n':
@@ -503,6 +539,11 @@ int main(int argc, char *argv[])
       case 'v':
         ldpc_version = strdup(optarg);
         break;
+      case 'g':
+	gen_code = atoi(optarg);
+	AssertFatal(gen_code <= 4, "gen_code %d is not allowed\n",gen_code);
+        n_trials=0;	
+	break;
       case 'h':
       default:
         printf("CURRENTLY SUPPORTED CODE RATES: \n");
@@ -559,7 +600,9 @@ int main(int argc, char *argv[])
                                       qbits,
                                       Kprime, // block length bytes
                                       n_trials,
-                                      n_segments);
+                                      n_segments,
+				      gen_code,
+				      use32bit);
 
     decoded_errors[i] = res.errors;
     dec_iter[i] = res.dec_iter;

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h

@@ -152,6 +152,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{
   time_stats_t ts_interleave;
   time_stats_t ts_rate_match;
   time_stats_t ts_ldpc_encode;
+  time_stats_t ts_output;
 } nrLDPC_segment_encoding_parameters_t;
 
 /**
@@ -201,6 +202,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{
   uint32_t C;
   nrLDPC_segment_encoding_parameters_t *segments;
   unsigned char *output;
+  uint8_t **c_dev;
 } nrLDPC_TB_encoding_parameters_t;
 
 /**
@@ -226,10 +228,13 @@ typedef struct nrLDPC_slot_encoding_parameters_s{
   int nb_TBs;
   tpool_t *threadPool;
   time_stats_t *tinput;
+  time_stats_t *tinput_memcpy;
   time_stats_t *tprep;
   time_stats_t *tparity;
   time_stats_t *toutput;
+  time_stats_t *tconcat;
   nrLDPC_TB_encoding_parameters_t *TBs;
+  uint32_t use_gpu;
 } nrLDPC_slot_encoding_parameters_t;
 
 typedef int32_t(nrLDPC_coding_init_t)(void);

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_decoder.c

@@ -280,6 +280,11 @@ int nrLDPC_prepare_TB_decoding(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_de
 
 int32_t nrLDPC_coding_init(void)
 {
+  LOG_I(NR_PHY,"Initializing coding library\n");
+#ifdef ENABLE_CUDA
+  LOG_I(NR_PHY,"Calling cuda_support_init()\n");
+#endif
+  cuda_support_init();	
   return 0;
 }
 

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c

@@ -367,9 +367,11 @@ static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_s
   encoder_implemparams_t common_segment_params = {
     .n_segments = nrLDPC_TB_encoding_parameters->C,
     .tinput = nrLDPC_slot_encoding_parameters->tinput,
+    .tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
     .tprep = nrLDPC_slot_encoding_parameters->tprep,
     .tparity = nrLDPC_slot_encoding_parameters->tparity,
     .toutput = nrLDPC_slot_encoding_parameters->toutput,
+    .tconcat = nrLDPC_slot_encoding_parameters->tconcat,
     .Kb = nrLDPC_TB_encoding_parameters->Kb,
     .Zc = nrLDPC_TB_encoding_parameters->Z,
     .BG = nrLDPC_TB_encoding_parameters->BG,
@@ -406,16 +408,24 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
     // Compute number of tasks to encode TB
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
-    size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
-    nbTasks += n_seg;
-
-    // Search for maximum E for sizing encoder output f and f2
-    for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
-      uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
-      Emax = E > Emax ? E : Emax;
+#if defined(ENABLE_CUDA)     
+    if (nrLDPC_slot_encoding_parameters->use_gpu && nrLDPC_TB_encoding_parameters->C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) {
+        nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters,nrLDPC_TB_encoding_parameters);
     }
-  }
+    else {
+#endif
+      size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
+      nbTasks += n_seg;
 
+      // Search for maximum E for sizing encoder output f and f2
+      for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
+        uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
+        Emax = E > Emax ? E : Emax;
+      }
+#if defined(ENABLE_CUDA) 
+    }
+#endif
+  }
   // Create f and f2 to old encoding tasks outputs
   uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
   uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
@@ -432,6 +442,10 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
     // For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays
     // Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j]
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
+#if defined(ENABLE_CUDA) 
+    if (nrLDPC_slot_encoding_parameters->use_gpu && nrLDPC_TB_encoding_parameters->C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) 
+      continue;
+#endif
     size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
     uint8_t *f_2d[n_seg];
     uint8_t *f2_2d[n_seg];
@@ -449,16 +463,21 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
     completed_many_task_ans(&ans, nbTasks - nbEncode);
   }
   // Execute thread pool tasks
-  join_task_ans(&ans);
+  if (nbTasks > 0)  join_task_ans(&ans);
 
   // Write output
+  time_stats_t *tconcat = nrLDPC_slot_encoding_parameters->tconcat;
+  if(tconcat != NULL) start_meas(tconcat);
   nbTasks = 0;
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
     uint32_t C = nrLDPC_TB_encoding_parameters->C;
+#if defined(ENABLE_CUDA)
+    if (nrLDPC_slot_encoding_parameters->use_gpu && C > 8 && nrLDPC_TB_encoding_parameters->Z == 384) 
+      continue;
+#endif
     size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1));
 
-    time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput;
 
     for (int j = 0; j < n_seg; j++) {
       unsigned int macro_segment = j * 8;
@@ -478,7 +497,6 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
         }
       }
 
-      if(toutput != NULL) start_meas(toutput);
  
       uint32_t Eoffset=0;
       for (int s=0; s<macro_segment; s++)
@@ -494,10 +512,10 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
                         nrLDPC_TB_encoding_parameters->output,
                         Eoffset);
 
-      if(toutput != NULL) stop_meas(toutput);
     }
     nbTasks += n_seg;
   }
+  if(tconcat != NULL) stop_meas(tconcat);
 
   return 0;
 }

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder_cuda.c

@@ -0,0 +1,688 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.0  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c
+ * \brief Top-level routines for implementing LDPC encoding of transport channels
+ */
+
+#include "nr_rate_matching.h"
+#include "PHY/defs_gNB.h"
+#include "PHY/CODING/coding_extern.h"
+#include "PHY/CODING/coding_defs.h"
+#include "PHY/CODING/lte_interleaver_inline.h"
+#include "PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h"
+#include "PHY/CODING/nrLDPC_extern.h"
+#include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#include "PHY/NR_TRANSPORT/nr_transport_common_proto.h"
+#include "PHY/NR_TRANSPORT/nr_dlsch.h"
+#include "SCHED_NR/sched_nr.h"
+#include "common/utils/LOG/vcd_signal_dumper.h"
+#include "common/utils/LOG/log.h"
+#include "common/utils/nr/nr_common.h"
+#include <openair2/UTIL/OPT/opt.h>
+
+#include <syscall.h>
+
+//#define DEBUG_LDPC_ENCODING
+//#define DEBUG_LDPC_ENCODING_FREE 1
+
+extern uint32_t **d_host;
+
+static void unpack_output(uint32_t *f,
+                         uint32_t E,
+                         uint32_t *f2,
+                         uint32_t E2,
+			 uint32_t E2_first_segment32,
+                         uint32_t E2_first_segment,
+                         uint32_t nb_segments,
+                         uint8_t *output) {
+
+  uint32_t s;
+ // int s0;
+  uint32_t *fp;
+  int foffset;
+  uint32_t *output_p = (uint32_t *)output;
+  //printf("unpack: E %d, E2 %d, E2_first_segment %d, E2_first_segment32 %d, nb_segments %d\n",E,E2,E2_first_segment,E2_first_segment32,nb_segments);
+
+  uint32_t bit_index = 0;
+#ifdef __AVX2__
+  simde__m256i shift0=simde_mm256_set_epi32(7,6,5,4,3,2,1,0);
+  simde__m256i shift1=simde_mm256_set_epi32(15,14,13,12,11,10,9,8);
+  simde__m256i shift2=simde_mm256_set_epi32(23,22,21,20,19,18,17,16);
+  simde__m256i shift3=simde_mm256_set_epi32(31,30,29,28,27,26,25,24);
+  simde__m256i vmask0=simde_mm256_set_epi32(0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1);
+  simde__m256i vmask1=simde_mm256_set_epi32(0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100);
+  simde__m256i vmask2=simde_mm256_set_epi32(0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000);
+  simde__m256i vmask3=simde_mm256_set_epi32(0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000);
+  int s2=0;
+  for (s = 0; s < E2_first_segment ; s++) { 
+    s2 = s&31;	  
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7);
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = (simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
+        bit_index+=Emod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp    = simde_mm256_extract_epi32(cshift,0) | 
+	                  simde_mm256_extract_epi32(cshift,1) | 
+	                  simde_mm256_extract_epi32(cshift,2) | 
+	                  simde_mm256_extract_epi32(cshift,3) | 
+	                  simde_mm256_extract_epi32(cshift,4) | 
+	                  simde_mm256_extract_epi32(cshift,5) | 
+	                  simde_mm256_extract_epi32(cshift,6) | 
+	                  simde_mm256_extract_epi32(cshift,7);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));					   
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp = 	(simde_mm256_extract_epi32(cshift,0) | 
+	                 simde_mm256_extract_epi32(cshift,1) | 
+	                 simde_mm256_extract_epi32(cshift,2) | 
+	                 simde_mm256_extract_epi32(cshift,3) | 
+	                 simde_mm256_extract_epi32(cshift,4) | 
+	                 simde_mm256_extract_epi32(cshift,5) | 
+	                 simde_mm256_extract_epi32(cshift,6) | 
+	                 simde_mm256_extract_epi32(cshift,7))&((1<<Emod32)-1);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=Emod32;
+      }
+    }
+  }
+  for ( ; s < nb_segments ; s++){
+    s2 = s&31;	  
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7);
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	*(output_p + (bit_index>>5))     = (simde_mm256_extract_epi32(cshift,0) | 
+	                                   simde_mm256_extract_epi32(cshift,1) | 
+	                                   simde_mm256_extract_epi32(cshift,2) | 
+	                                   simde_mm256_extract_epi32(cshift,3) | 
+	                                   simde_mm256_extract_epi32(cshift,4) | 
+	                                   simde_mm256_extract_epi32(cshift,5) | 
+	                                   simde_mm256_extract_epi32(cshift,6) | 
+	                                   simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
+        bit_index+=E2mod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp    = simde_mm256_extract_epi32(cshift,0) | 
+	                  simde_mm256_extract_epi32(cshift,1) | 
+	                  simde_mm256_extract_epi32(cshift,2) | 
+	                  simde_mm256_extract_epi32(cshift,3) | 
+	                  simde_mm256_extract_epi32(cshift,4) | 
+	                  simde_mm256_extract_epi32(cshift,5) | 
+	                  simde_mm256_extract_epi32(cshift,6) | 
+	                  simde_mm256_extract_epi32(cshift,7);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));					   
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+	simde__m256i f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i],s2);    
+  	simde__m256i cshift = simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift0),vmask0);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+8],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift1),vmask1),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+16],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift2),vmask2),cshift);
+	f256 = simde_mm256_srli_epi32(*(simde__m256i*)&fp[i+24],s2);    
+  	cshift = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_sllv_epi32(f256,shift3),vmask3),cshift);
+	uint32_t tmp = 	(simde_mm256_extract_epi32(cshift,0) | 
+	                 simde_mm256_extract_epi32(cshift,1) | 
+	                 simde_mm256_extract_epi32(cshift,2) | 
+	                 simde_mm256_extract_epi32(cshift,3) | 
+	                 simde_mm256_extract_epi32(cshift,4) | 
+	                 simde_mm256_extract_epi32(cshift,5) | 
+	                 simde_mm256_extract_epi32(cshift,6) | 
+	                 simde_mm256_extract_epi32(cshift,7))&((1<<E2mod32)-1);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+        *(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=E2mod32;
+      }
+    }
+  }
+#elif defined(__aarch64__)
+  const int32_t ucShift0[32][4] = { {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}, {-28,-27,-26,-25}, {-29,-28,-27,-26}, {-30,-29,-28,-27}, {-31,-30,-29,-28}}; 
+
+  const int32_t ucShift1[32][4] = { {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19}, {-23,-22,-21,-20}, {-24,-23,-22,-21}, {-25,-24,-23,-22}, {-26,-25,-24,-23}, {-27,-26,-25,-24}}; 
+
+  const int32_t ucShift2[32][4] = { {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}, {-20,-19,-18,-17}, {-21,-20,-19,-18}, {-22,-21,-20,-19},{-23,-22,-21,-20}}; 
+
+  const int32_t ucShift3[32][4] = { {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}, {-16,-15,-14,-13}, {-17,-16,-15,-14}, {-18,-17,-16,-15}, {-19,-18,-17,-16}}; 
+
+  const int32_t ucShift4[32][4] = { {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}, {-12,-11,-10,-9}, {-13,-12,-11,-10}, {-14,-13,-12,-11}, {-15,-14,-13,-12}}; 
+
+  const int32_t ucShift5[32][4] = { {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}, {-8,-7,-6,-5}, {-9,-8,-7,-6}, {-10,-9,-8,-7}, {-11,-10,-9,-8}}; 
+
+  const int32_t ucShift6[32][4] = { {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}, {-4,-3,-2,-1}, {-5,-4,-3,-2}, {-6,-5,-4,-3}, {-7,-6,-5,-4}}; 
+
+  const int32_t ucShift7[32][4] = { {28,29,30,31}, {27,28,29,30}, {26,27,28,29}, {25,26,27,28}, {24,25,26,27}, {23,24,25,26}, {22,23,24,25}, {21,22,23,24}, {20,21,22,23}, {19,20,21,22}, {18,19,20,21}, {17,18,19,20}, {16,17,18,19}, {15,16,17,18}, {14,15,16,17}, {13,14,15,16}, {12,13,14,15}, {11,12,13,14}, {10,11,12,13}, {9,10,11,12}, {8,9,10,11},{7,8,9,10}, {6,7,8,9}, {5,6,7,8}, {4,5,6,7}, {3,4,5,6}, {2,3,4,5}, {1,2,3,4}, {0,1,2,3}, {-1,0,1,2},{-2,-1,0,1}, {-3,-2,-1,0}}; 
+  const uint32_t __attribute__ ((aligned (16))) masks0[4] = {0x1,0x2,0x4,0x8};
+  const uint32_t __attribute__ ((aligned (16))) masks1[4] = {0x10,0x20,0x40,0x80};
+  const uint32_t __attribute__ ((aligned (16))) masks2[4] = {0x100,0x200,0x400,0x800};
+  const uint32_t __attribute__ ((aligned (16))) masks3[4] = {0x1000,0x2000,0x4000,0x8000};
+  const uint32_t __attribute__ ((aligned (16))) masks4[4] = {0x10000,0x20000,0x40000,0x80000};
+  const uint32_t __attribute__ ((aligned (16))) masks5[4] = {0x100000,0x200000,0x400000,0x800000};
+  const uint32_t __attribute__ ((aligned (16))) masks6[4] = {0x1000000,0x2000000,0x4000000,0x8000000};
+  const uint32_t __attribute__ ((aligned (16))) masks7[4] = {0x10000000,0x20000000,0x40000000,0x80000000};
+  int32x4_t vshift0[32],vshift1[32],vshift2[32],vshift3[32],vshift4[32],vshift5[32],vshift6[32],vshift7[32];
+  for (int n=0;n<32;n++) {
+	  vshift0[n] = vld1q_s32(ucShift0[n]);
+	  vshift1[n] = vld1q_s32(ucShift1[n]);
+	  vshift2[n] = vld1q_s32(ucShift2[n]);
+	  vshift3[n] = vld1q_s32(ucShift3[n]);
+	  vshift4[n] = vld1q_s32(ucShift4[n]);
+	  vshift5[n] = vld1q_s32(ucShift5[n]);
+	  vshift6[n] = vld1q_s32(ucShift6[n]);
+	  vshift7[n] = vld1q_s32(ucShift7[n]);
+  }
+  uint32x4_t vmask0  = vld1q_u32(masks0);
+  uint32x4_t vmask1  = vld1q_u32(masks1);
+  uint32x4_t vmask2  = vld1q_u32(masks2);
+  uint32x4_t vmask3  = vld1q_u32(masks3);
+  uint32x4_t vmask4  = vld1q_u32(masks4);
+  uint32x4_t vmask5  = vld1q_u32(masks5);
+  uint32x4_t vmask6  = vld1q_u32(masks6);
+  uint32x4_t vmask7  = vld1q_u32(masks7);
+  uint32_t output_tmp=0;
+  int s2=0;
+  for (s = 0; s < E2_first_segment ; s++) {
+    s2 = s&31;	  
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<Emod32)-1);
+        bit_index+=Emod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+	/*
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	*/
+        uint64_t tmp = (uint64_t)vaddvq_u32(cshift);
+	*(uint64_t*)(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	bit_index+=32;
+      }
+      uint32_t Emod32=E&31;
+      if (Emod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<Emod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=Emod32;
+      }
+    }
+  }
+//  s0 = s;
+  for ( ; s < nb_segments ; s++){
+    s2 = s&31;	  
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    int i;
+    if ((bit_index&31) == 0 ) {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+  	uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+	*(output_p + (bit_index>>5))     = vaddvq_u32(cshift);
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        *(output_p + (bit_index>>5))     = vaddvq_u32(cshift)&((1<<E2mod32)-1);
+        bit_index+=E2mod32;
+      }
+    }
+    else {
+      for (i = 0; i < (E2>>5)<<5; i+=32) {
+	uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[7],vshift7[s2]),vmask7));
+        uint32_t tmp = vaddvq_u32(cshift);
+	*(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+	bit_index+=32;
+      }
+      uint32_t E2mod32=E2&31;
+      if (E2mod32 != 0) {
+        uint32x4_t *fp128 = (uint32x4_t*)&fp[i];    
+        uint32x4_t cshift = vandq_u32(vshlq_u32(fp128[0],vshift0[s2]),vmask0);
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[1],vshift1[s2]),vmask1));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[2],vshift2[s2]),vmask2));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[3],vshift3[s2]),vmask3));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[4],vshift4[s2]),vmask4));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[5],vshift5[s2]),vmask5));
+  	cshift = vorrq_u32(cshift,vandq_u32(vshlq_u32(fp128[6],vshift6[s2]),vmask6));
+        uint32_t tmp = vaddvq_u32(cshift);
+        tmp&=((1<<E2mod32)-1);
+        *(output_p + (bit_index>>5))     |= (tmp<<(bit_index&31));
+	*(output_p + (bit_index>>5)+1)   |= (tmp>>(32-(bit_index&31)));
+        bit_index+=E2mod32;
+      }
+    }
+  }
+#else // non SIMD version
+  unsigned int segpos,s2;	
+  for (s = 0; s < E2_first_segment ; s++) {
+    foffset = (s>>5)*E;
+    fp = f+foffset;
+    s2=s&31;
+    segpos = (1<<s2);
+  //  printf("E %d s %d: foffset %d, s2 %d, segpos %u\n",E,s,foffset,s2,segpos);
+    for (int i = 0; i < E; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      //printf("bit_index %d, output_p[%d] %x\n",bit_index, bit_index>>5,output_p[bit_index>>5]);
+      bit_index++;
+    }
+  }
+  for ( ; s < nb_segments ; s++){
+    foffset = ((s>>5)-E2_first_segment32)*E2;
+    fp = f2+foffset;
+    s2=s&31;
+    segpos = (1<<s2);
+//    printf("E2 %d s %d: foffset %d, s2 %d, segpos %u\n",E2,s,foffset,s2,segpos);
+    for (int i = 0; i < E2; i++) {
+      output_p[bit_index>>5]|=((fp[i] & segpos)!=0)<<(bit_index&31); 
+      bit_index++;
+    }
+  }
+#endif  
+}
+/**
+ * \typedef ldpc8blocks_args_t
+ * \struct ldpc8blocks_args_s
+ * \brief Arguments of an encoding task
+ * encode up to 8 code blocks
+ * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
+ * \var impp encoder implementation specific parameters for the task
+ * \var f first interleaver output to be filled by the task
+ * \var f2 second interleaver output to be filled by the task
+ * in case of a shift of E in the code blocks group processed by the task
+ */
+
+static void ldpcnblocks(nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters, encoder_implemparams_t impp)
+{
+ 
+  uint8_t mod_order = nrLDPC_TB_encoding_parameters->Qm;
+  uint16_t nb_rb = nrLDPC_TB_encoding_parameters->nb_rb;
+  uint32_t A = nrLDPC_TB_encoding_parameters->A;
+
+  unsigned int G = nrLDPC_TB_encoding_parameters->G;
+  LOG_D(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp.K, G, nb_rb, (int)mod_order);
+
+  // nrLDPC_encoder output is in "d"
+  // let's make this interface happy!
+//  uint32_t d[4][68*384];
+  uint8_t *c[nrLDPC_TB_encoding_parameters->C];
+
+  if (!nrLDPC_TB_encoding_parameters->c_dev)  
+    for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++)
+      c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  LDPCencoder32(nrLDPC_TB_encoding_parameters->c_dev ? nrLDPC_TB_encoding_parameters->c_dev : c, &impp);
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp.first_seg].ts_ldpc_encode);
+  // Compute where to place in output buffer that is concatenation of all segments
+
+#ifdef DEBUG_LDPC_ENCODING
+  LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
+#endif
+  const uint32_t E = nrLDPC_TB_encoding_parameters->segments[0].E;
+  uint32_t E2=E;
+  uint32_t Emax = E;
+  int n_seg   = nrLDPC_TB_encoding_parameters->C>>5;
+  int n_seg2  = n_seg;
+  if ((nrLDPC_TB_encoding_parameters->C & 31) > 0) n_seg2++;
+  int r_shift = n_seg2; 
+  int r_shift2 = nrLDPC_TB_encoding_parameters->C;
+  for (int s=0;s<nrLDPC_TB_encoding_parameters->C;s++) {
+      //printf("segment %d E %d\n",s,nrLDPC_TB_encoding_parameters->segments[s].E);	  
+      if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
+	 E2=nrLDPC_TB_encoding_parameters->segments[s].E;
+         if(E2 > Emax)
+           Emax = E2;
+	 r_shift = s>>5;
+	 r_shift2 = s;
+	// printf("r_shift %d, r_shift2 %d\n",r_shift,r_shift2);
+         break;
+      }	 
+  }    
+
+  LOG_D(NR_PHY,
+        "Rate Matching, Code segment %d...%d r_shift %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        0,
+        impp.n_segments-1,
+	r_shift,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+/*
+  printf("Rate Matching, Code segment 0..%d r_shift %d r_shift2 %d n_seg2 %d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        impp.n_segments-1,
+	r_shift,
+	r_shift2,
+	n_seg2,
+        G,
+        E,E2,
+        impp.F,
+        impp.K - impp.F - 2 * impp.Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+*/
+
+  uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
+
+  uint32_t e[E*(r_shift+1)];
+  uint32_t e2[E2*(n_seg2-r_shift)];
+  uint32_t f[E*(r_shift+1)] __attribute__ ((aligned (64)));
+  uint32_t f2[E2*(n_seg2-r_shift)] __attribute__ ((aligned (64)));
+
+  // Interleaver outputs are stored in the output arrays
+  uint8_t *output = nrLDPC_TB_encoding_parameters->output;
+
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  memset(e,0,sizeof(e));
+  memset(f,0,sizeof(f));
+  if (1/*r_shift < n_seg2*/) { 
+    memset(e2,0,sizeof(e2));
+    memset(f2,0,sizeof(f2));
+  }
+
+  for (int r=0;r<n_seg2;r++) {
+    if (r<=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d_host[r],
+                              e+(r*E),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E);
+    if (r>=r_shift)	  
+      nr_rate_matching_ldpc32(Tbslbrm,
+                              impp.BG,
+                              impp.Zc,
+                              d_host[r],
+                              e2+((r-r_shift)*E2),
+                              impp.n_segments,
+                              impp.F,
+                              impp.K - impp.F - 2 * impp.Zc,
+                              nrLDPC_TB_encoding_parameters->rv_index,
+                              E2);
+   /* 
+    if (r==(n_seg2-1)) {
+	    for (int i=0;i<16;i++) printf("rm: %x %x\n",d[n_seg2-1][i],e2[((n_seg2-1)*E2)+i]);
+    }
+    */
+  }
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_rate_match);
+  if (impp.K - impp.F - 2 * impp.Zc > E) {
+    LOG_E(PHY,
+          "dlsch coding A %d  Kr %d G %d (nb_rb %d, mod_order %d)\n",
+          A,
+          impp.K,
+          G,
+          nb_rb,
+          (int)mod_order);
+
+    LOG_E(NR_PHY,
+          "Rate Matching, Code segments 0..%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
+          "nb_rb %d)...\n",
+          impp.n_segments,
+          G,
+          E,
+          impp.K,
+          impp.F,
+          impp.K - impp.F - 2 * impp.Zc,
+          mod_order,
+          nb_rb);
+  }
+  
+  //printf("interleaving r_shift %d, n_seg2 %d\n",r_shift,n_seg2);
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+  
+  for (int r=0;r<=r_shift;r++)
+    nr_interleaving_ldpc32(E,
+                           mod_order,
+                           e+E*r,
+                           f+E*r);
+
+  for (int r=r_shift;r<n_seg2;r++)
+    nr_interleaving_ldpc32(E2,
+                           mod_order,
+                           e2+E2*(r-r_shift),
+                           f2+E2*(r-r_shift));
+/*
+  for (int i=0;i<16;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  printf("-------------------\n");
+  for (int i=E2-16;i<E2;i++) printf("intl (f offset %d): %x %x\n",(n_seg2-1)*E2,e2[((n_seg2-1)*E2)+i],f2[((n_seg2-1)*E2)+i]);
+  */
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[0].ts_interleave);
+
+  if (impp.tconcat) start_meas(impp.tconcat);
+  unpack_output(f,E,f2,E2,r_shift,r_shift2,nrLDPC_TB_encoding_parameters->C,output);
+  if (impp.tconcat) stop_meas(impp.tconcat);
+
+}
+
+int nrLDPC_coding_encoder32(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters, nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters)
+{
+
+
+    encoder_implemparams_t common_segment_params = {
+      .n_segments = nrLDPC_TB_encoding_parameters->C,
+      .tinput = nrLDPC_slot_encoding_parameters->tinput,
+      .tinput_memcpy = nrLDPC_slot_encoding_parameters->tinput_memcpy,
+      .tprep = nrLDPC_slot_encoding_parameters->tprep,
+      .tparity = nrLDPC_slot_encoding_parameters->tparity,
+      .toutput = nrLDPC_slot_encoding_parameters->toutput,
+      .tconcat = nrLDPC_slot_encoding_parameters->tconcat,
+      .Kb = nrLDPC_TB_encoding_parameters->Kb,
+      .Zc = nrLDPC_TB_encoding_parameters->Z,
+      .BG = nrLDPC_TB_encoding_parameters->BG,
+      .output = nrLDPC_TB_encoding_parameters->output, 
+      .K = nrLDPC_TB_encoding_parameters->K,
+      .F = nrLDPC_TB_encoding_parameters->F,
+    };
+
+    LOG_D(NR_PHY,"Calling ldpcnblocks (C %d, Z %d, K %d)\n",common_segment_params.n_segments,common_segment_params.Zc,common_segment_params.K);
+    ldpcnblocks(nrLDPC_TB_encoding_parameters, common_segment_params);
+
+
+
+  return 0;
+}

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c

@@ -24,7 +24,8 @@
 #include "common/utils/LOG/log.h"
 
 // #define RM_DEBUG 1
-
+#define USE128BIT
+ 
 static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
 
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
@@ -65,7 +66,7 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
       simde__m128i *f_128 = (simde__m128i *)f;
       simde__m128i *e0_128 = (simde__m128i *)e0;
       simde__m128i *e1_128 = (simde__m128i *)e1;
-      for (; i < (EQm & ~15); i += 64) {
+      for (; i < (EQm & ~15); i += 16) {
         simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
         simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
         simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
@@ -359,14 +360,15 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
       f = (uint8_t *)f_512;
 #endif
 #ifdef USE128BIT
-      e0_128 = (simde__m128i *)e0;
-      e1_128 = (simde__m128i *)e1;
-      e2_128 = (simde__m128i *)e2;
-      e3_128 = (simde__m128i *)e3;
-      e4_128 = (simde__m128i *)e4;
-      e5_128 = (simde__m128i *)e5;
-      e6_128 = (simde__m128i *)e6;
-      e7_128 = (simde__m128i *)e7;
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f_128  = (simde__m128i *)f;
       for (; i < (EQm & ~15); i += 16) {
         simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
         simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
@@ -444,25 +446,461 @@ void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
   }
 }
 
+void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f)
+{
+  const uint32_t EQm = E / Qm;
+  memset(f, 0, E * sizeof(uint32_t));
+  switch(Qm) {
+    case 2: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e + EQm;
+      int i = 0;
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+      }
+    } break;
+    case 4: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      int i = 0;
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+      }
+    } break;
+    case 6: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      int i = 0;
+#if 0
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *f128 = (simde__m128i *)f;
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i) e1(i) e2(i) e3(i) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i) e5(i) e0(i+1) e1(i+1) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+1) e3(i+1) e4(i+1) e5(i+1)
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3) 
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3) 
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3) 
+								 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1));                        // e0(i+2) e1(i+2) e2(i+2) e3(i+2) 
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2,simde_mm_unpackhi_epi64(tmp0,tmp0))); // e4(i+2) e5(i+2) e0(i+3) e1(i+3) 
+	simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp1,tmp2));                         // e2(i+3) e3(i+3) e4(i+3) e5(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      f  = (uint32_t *)f128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+      }
+    } break;
+    case 8: {
+      uint32_t *e0 = e;
+      uint32_t *e1 = e0 + EQm;
+      uint32_t *e2 = e1 + EQm;
+      uint32_t *e3 = e2 + EQm;
+      uint32_t *e4 = e3 + EQm;
+      uint32_t *e5 = e4 + EQm;
+      uint32_t *e6 = e5 + EQm;
+      uint32_t *e7 = e6 + EQm;
+      
+      int i = 0;
+#if 0 /*def __AVX2__, this needs to be fixed, revert to 128-bit below*/
+      simde__m256i *e0_256 = (simde__m256i *)e0;
+      simde__m256i *e1_256 = (simde__m256i *)e1;
+      simde__m256i *e2_256 = (simde__m256i *)e2;
+      simde__m256i *e3_256 = (simde__m256i *)e3;
+      simde__m256i *e4_256 = (simde__m256i *)e4;
+      simde__m256i *e5_256 = (simde__m256i *)e5;
+      simde__m256i *e6_256 = (simde__m256i *)e6;
+      simde__m256i *e7_256 = (simde__m256i *)e7;
+      simde__m256i *f256   = (simde__m256i *)f;
+
+      for (; i < (EQm & ~7); i += 8) {
+        simde__m256i e0j = simde_mm256_loadu_si256(e0_256++);
+        simde__m256i e1j = simde_mm256_loadu_si256(e1_256++);
+        simde__m256i e2j = simde_mm256_loadu_si256(e2_256++);
+        simde__m256i e3j = simde_mm256_loadu_si256(e3_256++);
+        simde__m256i e4j = simde_mm256_loadu_si256(e4_256++);
+        simde__m256i e5j = simde_mm256_loadu_si256(e5_256++);
+        simde__m256i e6j = simde_mm256_loadu_si256(e6_256++);
+        simde__m256i e7j = simde_mm256_loadu_si256(e7_256++);
+
+        simde__m256i tmp0 = simde_mm256_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1) e0(i+4) e1(i+4) e0(i+5) e1(i+5) 
+        simde__m256i tmp1 = simde_mm256_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1) e2(i+4) e3(i+4) e2(i+5) e3(i+5)
+        simde__m256i tmp2 = simde_mm256_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1) e4(i+4) e5(i+4) e4(i+5) e5(i+5) 
+        simde__m256i tmp3 = simde_mm256_unpacklo_epi32(e6j, e7j);   // e6(i) e7(i) e6(i+1) e7(i+1) e6(i+4) e7(i+4) e6(i+5) e7(i+5)
+								    //
+        simde__m256i tmp4 = simde_mm256_unpacklo_epi64(tmp0, tmp1);   // e0(i) e1(i) e2(i) e3(i) e0(i+4) e1(i+4) e2(i+4) e3(i+4) 
+        simde__m256i tmp5 = simde_mm256_unpacklo_epi64(tmp2, tmp3);   // e4(i) e5(i) e6(i) e7(i) e4(i+4) e5(i+4) e6(i+4) e7(i+4) 
+								 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x20)); // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) 
+										    
+        simde__m256i tmp6 = simde_mm256_unpackhi_epi64(tmp0, tmp1);   // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e0(i+5) e1(i+5) e2(i+5) e3(i+5) 
+        simde__m256i tmp7 = simde_mm256_unpackhi_epi64(tmp2, tmp3);   // e4(i+1) e5(i+1) e6(i+1) e7(i+1) e4(i+5) e5(i+5) e6(i+5) e7(i+5) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x20)); // e0(i+1) e1(i+1) e2(i+1) e3(i+1) e4(i+1) e5(i+1) e6(i+1) e7(i+1) 
+										    //
+        tmp0 = simde_mm256_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3) e0(i+6) e1(i+6) e0(i+7) e1(i+7) 
+        tmp1 = simde_mm256_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3) e2(i+6) e3(i+6) e2(i+7) e3(i+7)
+        tmp2 = simde_mm256_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3) e4(i+6) e5(i+6) e4(i+7) e5(i+7) 
+        tmp3 = simde_mm256_unpackhi_epi32(e6j, e7j);   // e6(i+2) e7(i+2) e6(i+3) e7(i+3) e6(i+6) e7(i+6) e6(i+7) e7(i+7)
+								    //
+        simde__m256i tmp8 = simde_mm256_unpacklo_epi64(tmp0, tmp1);   // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e0(i+6) e1(i+6) e2(i+6) e3(i+6) 
+        simde__m256i tmp9 = simde_mm256_unpacklo_epi64(tmp2, tmp3);   // e4(i+2) e5(i+2) e6(i+2) e7(i+2) e4(i+6) e5(i+6) e6(i+6) e7(i+6) 
+								 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x20)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) 
+        simde__m256i tmp10 = simde_mm256_unpackhi_epi64(tmp0, tmp1);   // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e0(i+7) e1(i+7) e2(i+7) e3(i+7) 
+        simde__m256i tmp11 = simde_mm256_unpackhi_epi64(tmp2, tmp3);   // e4(i+3) e5(i+3) e6(i+3) e7(i+3) e4(i+7) e5(i+7) e6(i+7) e7(i+7) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x20)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp4,tmp5,0x31)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp6,tmp7,0x31)); // e0(i+5) e1(i+5) e2(i+5) e3(i+5) e4(i+5) e5(i+5) e6(i+5) e7(i+5) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp8,tmp9,0x31)); // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) 
+        simde_mm256_storeu_si256(f256++,_mm256_permute2x128_si256(tmp10,tmp11,0x31)); // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7) 
+      }
+      e0 = (uint32_t *)e0_256;
+      e1 = (uint32_t *)e1_256;
+      e2 = (uint32_t *)e2_256;
+      e3 = (uint32_t *)e3_256;
+      e4 = (uint32_t *)e4_256;
+      e5 = (uint32_t *)e5_256;
+      e6 = (uint32_t *)e6_256;
+      e7 = (uint32_t *)e7_256;
+      f = (uint32_t *)f256;
+#endif
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f128   = (simde__m128i *)f;
+
+      for (; i < (EQm & ~3); i += 4) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+        simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
+        simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi32(e0j, e1j);   // e0(i) e1(i) e0(i+1) e1(i+1)
+        simde__m128i tmp1 = simde_mm_unpacklo_epi32(e2j, e3j);   // e2(i) e3(i) e2(i+1) e3(i+1)
+        simde__m128i tmp2 = simde_mm_unpacklo_epi32(e4j, e5j);   // e4(i) e5(i) e4(i+1) e5(i+1)
+        simde__m128i tmp3 = simde_mm_unpacklo_epi32(e6j, e7j);   // e6(i) e7(i) e6(i+1) e7(i+1)
+
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i)
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i) e5(i) e6(i) e7(i)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1));  // e0(i+1) e1(i+1) e2(i+1) e3(i+1)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); // e4(i+1) e5(i+1) e6(i+1) e7(i+1)
+
+        tmp0 = simde_mm_unpackhi_epi32(e0j, e1j);   // e0(i+2) e1(i+2) e0(i+3) e1(i+3)
+        tmp1 = simde_mm_unpackhi_epi32(e2j, e3j);   // e2(i+2) e3(i+2) e2(i+3) e3(i+3)
+        tmp2 = simde_mm_unpackhi_epi32(e4j, e5j);   // e4(i+2) e5(i+2) e4(i+3) e5(i+3)
+        tmp3 = simde_mm_unpackhi_epi32(e6j, e7j);   // e6(i+2) e7(i+2) e6(i+3) e7(i+3)
+
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp0, tmp1)); // e0(i+2) e1(i+2) e2(i+2) e3(i+2)
+        simde_mm_storeu_si128(f128++,simde_mm_unpacklo_epi64(tmp2, tmp3)); // e4(i+2) e5(i+2) e6(i+2) e7(i+2)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp0, tmp1)); // e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        simde_mm_storeu_si128(f128++,simde_mm_unpackhi_epi64(tmp2, tmp3)); //e4(i+3) e5(i+3) e6(i+3) e7(i+3)
+      }
+      e0 = (uint32_t *)e0_128;
+      e1 = (uint32_t *)e1_128;
+      e2 = (uint32_t *)e2_128;
+      e3 = (uint32_t *)e3_128;
+      e4 = (uint32_t *)e4_128;
+      e5 = (uint32_t *)e5_128;
+      e6 = (uint32_t *)e6_128;
+      e7 = (uint32_t *)e7_128;
+       f = (uint32_t *)f128;
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+        *f++ = *e6++;
+        *f++ = *e7++;
+      }
+    } 
+    break;
+    default:
+      AssertFatal(false, "Should be here!\n");
+  }
+}
+
+#if defined(__aarch64__)
+static inline uint8x16_t tbl96_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2,
+                                  uint8x16_t b3, uint8x16_t b4, uint8x16_t b5,
+                                  uint8x16_t idx /* 0..95 */)
+{
+   // Table 0: bytes 0..63
+   uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
+   uint8x16_t r0 = vqtbl4q_u8(T0, idx);
+
+   // Table 1: bytes 64..95, presented as a 64-byte table:
+   // bytes 0..31 map to original 64..95, bytes 32..63 are dummy (return 0)
+   uint8x16_t z = vdupq_n_u8(0);
+   uint8x16x4_t T1 = {{ b4, b5, z, z }};
+
+   // idx1 = idx - 64 (wrap-safe via unsigned subtract); only valid when idx>=64
+   uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
+   uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
+
+   // Select r1 where idx >= 64, else r0
+   uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64));  // 0xFF where idx>=64
+   return vbslq_u8(sel, r1, r0);
+}
+
+static inline uint8x16_t tbl128_u8(uint8x16_t b0, uint8x16_t b1, uint8x16_t b2, uint8x16_t b3,
+	                           uint8x16_t b4, uint8x16_t b5, uint8x16_t b6, uint8x16_t b7,
+	                           uint8x16_t idx /* 0..127 */)
+{
+  // Table low: bytes 0..63
+     uint8x16x4_t T0 = {{ b0, b1, b2, b3 }};
+     uint8x16_t r0 = vqtbl4q_u8(T0, idx);
+
+ // Table high: bytes 64..127, mapped to 0..63 by subtracting 64
+     uint8x16x4_t T1 = {{ b4, b5, b6, b7 }};
+     uint8x16_t idx1 = vsubq_u8(idx, vdupq_n_u8(64));
+     uint8x16_t r1 = vqtbl4q_u8(T1, idx1);
+
+ // Select high where idx >= 64 else low
+     uint8x16_t sel = vcgeq_u8(idx, vdupq_n_u8(64));
+     return vbslq_u8(sel, r1, r0);
+}
+#elif defined(__AVX512BW__)
+
+static inline __m512i idx_stride_u16(int ways, int k)
+{
+    uint16_t idx[32] __attribute__((aligned(64)));
+    for (int n = 0; n < 32; n++) idx[n] = (uint16_t)(k + ways*n);
+      return _mm512_load_si512((const void*)idx);
+}
+
+// Build indices for k = 0..5, lanes n=0..15: p = k + 6*n
+static inline __m512i make_idx_ab(int k) {
+    uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
+    for (int n = 0; n < 16; n++) {
+        idx[n] = (uint16_t)(k + 6*n); // p in [0..95]
+    }
+    return _mm512_load_si512((const void*)idx);
+}
+
+static inline __m512i make_idx_bc(int k) {
+    uint16_t idx[32] __attribute__((aligned(64))) = { 0 };
+
+    for (int n = 0; n < 16; n++) {
+        int p = k + 6*n;
+   // For (B,C) pair, global p maps to:
+   //   indices 0..31  -> B lanes 0..31 (global 32..63)
+   //   indices 32..63 -> C lanes 0..31 (global 64..95)
+   // so use (p - 32) as index into [B|C] when p >= 32.
+        idx[n] = (uint16_t)(p - 32);
+    }
+    return _mm512_load_si512((const void*)idx);
+}
+
+static inline __mmask32 make_mask_bc(int k) {
+   // lanes needing BC are those with p >= 64 (since AB covers global 0..63)
+   __mmask32 m = 0;
+   for (int n = 0; n < 16; n++) {
+       int p = k + 6*n;
+       if (p >= 64) m |= (1u << n);
+   }
+   return m;
+}
+#endif
+
 void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
 {
+  const uint32_t EQm = E / Qm;
   switch (Qm) {
     case 2: {
       AssertFatal(E % 2 == 0, "");
-      int16_t *e1 = e + (E / 2);
-      int16_t *end = f + E - 1;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int i=0;	  
+#if defined(__aarch64__)
+      for (; i + 8 <= EQm; i += 8) {
+        int16x8x2_t v = vld2q_s16(f);  // 8 groups
+        f += 16;
+        vst1q_s16(e, v.val[0]); e += 8;
+        vst1q_s16(e1, v.val[1]); e1 += 8;
+      }
+#elif defined(__AVX512BW__)
+    const __m512i idx0 = idx_stride_u16(2, 0);
+    const __m512i idx1 = idx_stride_u16(2, 1);
+
+    for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 2 u16 = 64 u16 = 2x512b
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0)); // u16[0..31]
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
+        f += 64;
+        __m512i o0 = _mm512_permutex2var_epi16(A, idx0, B);
+        __m512i o1 = _mm512_permutex2var_epi16(A, idx1, B);
+        _mm512_storeu_si512((void*)e, o0); e += 32;
+        _mm512_storeu_si512((void*)e1, o1); e1 += 32;
+    }
+#else 
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *f128   = (simde__m128i *)f;	  
+      const uint8_t shuf4[16]  __attribute__((aligned(16))) = {0,1,4,5,8,9,12,13,2,3,6,7,10,11,14,15};
+      const simde__m128i *shuf4_128 = (const simde__m128i *)shuf4;
+      for (; i < (EQm & ~7); i += 8) {      
+            simde__m128i f0j = simde_mm_loadu_si128(f128++); // f0(i) f0(i+1) f0(i+2) f0(i+3) f0(i+4) f0(i+5) f0(i+6) f0(i+7)
+            simde__m128i f1j = simde_mm_loadu_si128(f128++); // f1(i) f1(i+1) f1(i+2) f1(i+3) f1(i+4) f1(i+5) f1(i+6) f1(i+7)
+    	    simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf4_128); // f0(i) f0(i+2) f0(i+4) f0(i+6) f0(i+1) f0(i+3) f0(i+5) f0(i+7)
+     	    simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf4_128); // f1(i) f1(i+2) f1(i+4) f1(i+6) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
+            simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp0, tmp1));   // f0(i) f0(i+2) f0(i+4) f0(i+6) f1(i) f1(i+2) f1(i+4) f1(i+6)
+            simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp0, tmp1));   // f0(i+1) f0(i+3) f0(i+5) f0(i+7) f1(i+1) f1(i+3) f1(i+5) f1(i+7)
+      }
+      e=(int16_t *)e0_128;
+      e1=(int16_t *)e1_128;
+      f=(int16_t *)f128;
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
       }
     } break;
+
     case 4: {
       AssertFatal(E % 4 == 0, "");
-      int16_t *e1 = e + (E / 4);
-      int16_t *e2 = e1 + (E / 4);
-      int16_t *e3 = e2 + (E / 4);
-      int16_t *end = f + E - 3;
-      while (f < end) {
+      int i=0;
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+
+#if defined(__aarch64__)
+      for (; i + 8 <= EQm; i += 8) {
+        int16x8x4_t v = vld4q_s16(f);  // 8 groups
+        f += 32;
+        vst1q_s16(e, v.val[0]); e += 8;
+        vst1q_s16(e1, v.val[1]); e1 += 8;
+        vst1q_s16(e2, v.val[2]); e2 += 8;
+        vst1q_s16(e3, v.val[3]); e3 += 8;
+      }
+#elif defined(__AVX512BW__)
+      const __m512i idx0 = idx_stride_u16(4, 0);
+      const __m512i idx1 = idx_stride_u16(4, 1);
+      const __m512i idx2 = idx_stride_u16(4, 2);
+      const __m512i idx3 = idx_stride_u16(4, 3);
+
+      for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 4 u16 = 128 u16 = 4x512b
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0));
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32));
+        __m512i C = _mm512_loadu_si512((const void*)(f + 64));
+        __m512i D = _mm512_loadu_si512((const void*)(f + 96));
+        f += 128;
+       // Build results from (A,B) and (C,D) with masking (boundary at 64 u16)
+        __m512i o0_ab = _mm512_permutex2var_epi16(A, idx0, B);
+        __m512i o1_ab = _mm512_permutex2var_epi16(A, idx1, B);
+        __m512i o2_ab = _mm512_permutex2var_epi16(A, idx2, B);
+        __m512i o3_ab = _mm512_permutex2var_epi16(A, idx3, B);
+
+        __m512i idx0_cd = _mm512_sub_epi16(idx0, _mm512_set1_epi16(64));
+        __m512i idx1_cd = _mm512_sub_epi16(idx1, _mm512_set1_epi16(64));
+        __m512i idx2_cd = _mm512_sub_epi16(idx2, _mm512_set1_epi16(64));
+        __m512i idx3_cd = _mm512_sub_epi16(idx3, _mm512_set1_epi16(64));
+
+        __m512i o0_cd = _mm512_permutex2var_epi16(C, idx0_cd, D);
+        __m512i o1_cd = _mm512_permutex2var_epi16(C, idx1_cd, D);
+        __m512i o2_cd = _mm512_permutex2var_epi16(C, idx2_cd, D);
+        __m512i o3_cd = _mm512_permutex2var_epi16(C, idx3_cd, D);
+
+        // lanes needing CD are those where idx >= 64
+	__mmask32 m0 = _mm512_cmpge_epu16_mask(idx0, _mm512_set1_epi16(64));
+	__mmask32 m1 = _mm512_cmpge_epu16_mask(idx1, _mm512_set1_epi16(64));
+	__mmask32 m2 = _mm512_cmpge_epu16_mask(idx2, _mm512_set1_epi16(64));
+	__mmask32 m3 = _mm512_cmpge_epu16_mask(idx3, _mm512_set1_epi16(64));
+	
+	__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_cd);
+	__m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_cd);
+	__m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_cd);
+	__m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_cd);
+	
+	_mm512_storeu_si512((void*)e, o0); e += 32;
+	_mm512_storeu_si512((void*)e1, o1); e1 += 32;
+	_mm512_storeu_si512((void*)e2, o2); e2 += 32;
+	_mm512_storeu_si512((void*)e3, o3); e3 += 32;
+      }
+#else 
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *f128   = (simde__m128i *)f;	  
+     
+      const uint8_t shuf16[16]  __attribute__((aligned(16))) = {0,1,8,9,2,3,10,11,4,5,12,13,6,7,14,15};
+      const simde__m128i *shuf16_128 = (const simde__m128i *)shuf16;
+
+      for (i=0; i < (EQm & ~7); i += 8) {      
+        simde__m128i f0j = simde_mm_loadu_si128(f128++);
+        simde__m128i f1j = simde_mm_loadu_si128(f128++);
+        simde__m128i f2j = simde_mm_loadu_si128(f128++);
+        simde__m128i f3j = simde_mm_loadu_si128(f128++);
+
+        simde__m128i tmp0 = simde_mm_shuffle_epi8(f0j,*shuf16_128); // f0(i) f0(i+4) f0(i+1) f0(i+5) f0(i+2) f0(i+6) f0(i+3) f0(i+7)
+        simde__m128i tmp1 = simde_mm_shuffle_epi8(f1j,*shuf16_128); // f1(i) f1(i+4) f1(i+1) f1(i+5) f1(i+2) f1(i+6) f1(i+3) f1(i+7)
+        simde__m128i tmp2 = simde_mm_shuffle_epi8(f2j,*shuf16_128); // f2(i) f2(i+4) f2(i+1) f2(i+5) f2(i+2) f2(i+6) f2(i+3) f2(i+7)
+	simde__m128i tmp3 = simde_mm_shuffle_epi8(f3j,*shuf16_128); // f3(i) f3(i+2) f3(i+1) f3(i+5) f3(i+2) f3(i+6) f3(i+3) f3(i+7)
+        simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0,tmp1); // f0(i) f0(i+4) f1(i) f1(i+4) f0(i+1) f0(i+5) f1(i+1) f1(i+5)
+        simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2,tmp3); // f2(i) f2(i+4) f3(i) f3(i+4) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
+       	simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));   // f0(i)   f0(i+4) f1(i)   f1(i+4) f2(i)   f2(i+4) f3(i)   f3(i+4)
+        simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));   // f0(i+1) f0(i+5) f1(i+1) f1(i+5) f2(i+1) f2(i+5) f3(i+1) f3(i+5)
+        tmp4 = simde_mm_unpackhi_epi32(tmp0,tmp1); // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f0(i+3) f0(i+7) f1(i+3) f1(i+7)
+        tmp5 = simde_mm_unpackhi_epi32(tmp2,tmp3); // f2(i+2) f2(i+6) f3(i+2) f3(i+6) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
+        simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));   // f0(i+2) f0(i+6) f1(i+2) f1(i+6) f2(i+2) f2(i+6) f3(i+2) f3(i+6)
+  	simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));   // f0(i+3) f0(i+7) f1(i+3) f1(i+7) f2(i+3) f2(i+7) f3(i+3) f3(i+7)
+      } 
+      e=(int16_t *)e0_128;
+      e1=(int16_t *)e1_128;
+      e2=(int16_t *)e2_128;
+      e3=(int16_t *)e3_128;
+      f=(int16_t *)f128;
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -471,13 +909,108 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
     } break;
     case 6: {
       AssertFatal(E % 6 == 0, "");
-      int16_t *e1 = e + (E / 6);
-      int16_t *e2 = e1 + (E / 6);
-      int16_t *e3 = e2 + (E / 6);
-      int16_t *e4 = e3 + (E / 6);
-      int16_t *e5 = e4 + (E / 6);
-      int16_t *end = f + E - 5;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+      int16_t *e4 = e3 + EQm;
+      int16_t *e5 = e4 + EQm;
+      int i=0;
+#if defined(__aarch64__)
+
+          // Byte indices for extracting each stream
+	  // Each s16 occupies 2 bytes → indices are 2*(k + 6*n)
+      const uint8x16_t idx0 = {  0,  1, 12, 13, 24, 25, 36, 37,
+                                48, 49, 60, 61, 72, 73, 84, 85 };
+      const uint8x16_t idx1 = {  2,  3, 14, 15, 26, 27, 38, 39,
+                                50, 51, 62, 63, 74, 75, 86, 87 };
+      const uint8x16_t idx2 = {  4,  5, 16, 17, 28, 29, 40, 41,
+                                52, 53, 64, 65, 76, 77, 88, 89 };
+      const uint8x16_t idx3 = {  6,  7, 18, 19, 30, 31, 42, 43,
+                                54, 55, 66, 67, 78, 79, 90, 91 };
+      const uint8x16_t idx4 = {  8,  9, 20, 21, 32, 33, 44, 45,
+                                56, 57, 68, 69, 80, 81, 92, 93 };
+      const uint8x16_t idx5 = { 10, 11, 22, 23, 34, 35, 46, 47,
+                                58, 59, 70, 71, 82, 83, 94, 95 };
+
+      for (; i + 8 < EQm; i += 8) {
+	// Load 96 bytes (48 u16)
+	uint8x16_t b0 = vld1q_u8((const uint8_t*)(f +  0)); // bytes  0..15
+	uint8x16_t b1 = vld1q_u8((const uint8_t*)(f +  8)); // bytes 16..31
+	uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
+	uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
+	uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
+	uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
+	f += 48;
+        uint8x16_t o0b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx0);
+        uint8x16_t o1b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx1);
+        uint8x16_t o2b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx2);
+        uint8x16_t o3b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx3);
+        uint8x16_t o4b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx4);
+        uint8x16_t o5b = tbl96_u8(b0,b1,b2,b3,b4,b5, idx5);
+
+        vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
+        vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
+        vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
+        vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
+        vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
+        vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
+      }	
+#elif defined(__AVX512BW__)
+
+      // Precompute permute control once (all compile-time constant patterns)
+      const __m512i idx0_ab = make_idx_ab(0), idx1_ab = make_idx_ab(1);
+      const __m512i idx2_ab = make_idx_ab(2), idx3_ab = make_idx_ab(3);
+      const __m512i idx4_ab = make_idx_ab(4), idx5_ab = make_idx_ab(5);
+
+      const __m512i idx0_bc = make_idx_bc(0), idx1_bc = make_idx_bc(1);
+      const __m512i idx2_bc = make_idx_bc(2), idx3_bc = make_idx_bc(3);
+      const __m512i idx4_bc = make_idx_bc(4), idx5_bc = make_idx_bc(5);
+
+      const __mmask32 m0 = make_mask_bc(0), m1 = make_mask_bc(1), m2 = make_mask_bc(2);
+      const __mmask32 m3 = make_mask_bc(3), m4 = make_mask_bc(4), m5 = make_mask_bc(5);
+
+      const __mmask32 store16 = 0xFFFFu; // store only first 16 lanes (16x u16)
+      for (; i + 16 <= EQm; i += 16) {
+        // 16 groups = 16*(6 u16) = 96 u16 = 192 bytes = 3 * 64B
+        __m512i A = _mm512_loadu_si512((const void*)(f +  0)); // u16[0..31]
+        __m512i B = _mm512_loadu_si512((const void*)(f + 32)); // u16[32..63]
+        __m512i C = _mm512_loadu_si512((const void*)(f + 64)); // u16[64..95]
+        f += 96;
+
+        // Gather from AB (global 0..63) and BC (global 32..95)
+        __m512i o0_ab = _mm512_permutex2var_epi16(A, idx0_ab, B);
+        __m512i o1_ab = _mm512_permutex2var_epi16(A, idx1_ab, B);
+        __m512i o2_ab = _mm512_permutex2var_epi16(A, idx2_ab, B);
+        __m512i o3_ab = _mm512_permutex2var_epi16(A, idx3_ab, B);
+        __m512i o4_ab = _mm512_permutex2var_epi16(A, idx4_ab, B);
+        __m512i o5_ab = _mm512_permutex2var_epi16(A, idx5_ab, B);
+
+        __m512i o0_bc = _mm512_permutex2var_epi16(B, idx0_bc, C);
+        __m512i o1_bc = _mm512_permutex2var_epi16(B, idx1_bc, C);
+        __m512i o2_bc = _mm512_permutex2var_epi16(B, idx2_bc, C);
+        __m512i o3_bc = _mm512_permutex2var_epi16(B, idx3_bc, C);
+        __m512i o4_bc = _mm512_permutex2var_epi16(B, idx4_bc, C);
+        __m512i o5_bc = _mm512_permutex2var_epi16(B, idx5_bc, C);
+
+        // Select lanes that crossed the 64-element boundary
+   	__m512i o0 = _mm512_mask_mov_epi16(o0_ab, m0, o0_bc);
+        __m512i o1 = _mm512_mask_mov_epi16(o1_ab, m1, o1_bc);
+        __m512i o2 = _mm512_mask_mov_epi16(o2_ab, m2, o2_bc);
+        __m512i o3 = _mm512_mask_mov_epi16(o3_ab, m3, o3_bc);
+        __m512i o4 = _mm512_mask_mov_epi16(o4_ab, m4, o4_bc);
+        __m512i o5 = _mm512_mask_mov_epi16(o5_ab, m5, o5_bc);
+
+        // Store only the first 16 lanes (16 groups) to each plane
+        _mm512_mask_storeu_epi16((void*)e, store16, o0); e += 16;
+        _mm512_mask_storeu_epi16((void*)e1, store16, o1); e1 += 16;
+        _mm512_mask_storeu_epi16((void*)e2, store16, o2); e2 += 16;
+        _mm512_mask_storeu_epi16((void*)e3, store16, o3); e3 += 16;
+        _mm512_mask_storeu_epi16((void*)e4, store16, o4); e4 += 16;
+        _mm512_mask_storeu_epi16((void*)e5, store16, o5); e5 += 16;
+     }
+			//
+#endif
+      for (; i < EQm; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -488,15 +1021,181 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
     } break;
     case 8: {
       AssertFatal(E % 8 == 0, "");
-      int16_t *e1 = e + (E / 8);
-      int16_t *e2 = e1 + (E / 8);
-      int16_t *e3 = e2 + (E / 8);
-      int16_t *e4 = e3 + (E / 8);
-      int16_t *e5 = e4 + (E / 8);
-      int16_t *e6 = e5 + (E / 8);
-      int16_t *e7 = e6 + (E / 8);
-      int16_t *end = f + E - 7;
-      while (f < end) {
+      int16_t *e1 = e + EQm;
+      int16_t *e2 = e1 + EQm;
+      int16_t *e3 = e2 + EQm;
+      int16_t *e4 = e3 + EQm;
+      int16_t *e5 = e4 + EQm;
+      int16_t *e6 = e5 + EQm;
+      int16_t *e7 = e6 + EQm;
+      int i=0;
+#if 0 //defined(__aarch64__)
+      // SIMDE version below is more efficient than tbl128
+          // For 8 groups: byte indices for stream k are 2*(k + 8*n) for n=0..7.
+	  // That is: 2k + 16n (and +1 for the high byte of the u16).
+      const uint8x16_t idx0 = {  0,  1, 16, 17, 32, 33, 48, 49, 64, 65, 80, 81, 96, 97,112,113 };
+      const uint8x16_t idx1 = {  2,  3, 18, 19, 34, 35, 50, 51, 66, 67, 82, 83, 98, 99,114,115 };
+      const uint8x16_t idx2 = {  4,  5, 20, 21, 36, 37, 52, 53, 68, 69, 84, 85,100,101,116,117 };
+      const uint8x16_t idx3 = {  6,  7, 22, 23, 38, 39, 54, 55, 70, 71, 86, 87,102,103,118,119 };
+      const uint8x16_t idx4 = {  8,  9, 24, 25, 40, 41, 56, 57, 72, 73, 88, 89,104,105,120,121 };
+      const uint8x16_t idx5 = { 10, 11, 26, 27, 42, 43, 58, 59, 74, 75, 90, 91,106,107,122,123 };
+      const uint8x16_t idx6 = { 12, 13, 28, 29, 44, 45, 60, 61, 76, 77, 92, 93,108,109,124,125 };
+      const uint8x16_t idx7 = { 14, 15, 30, 31, 46, 47, 62, 63, 78, 79, 94, 95,110,111,126,127 };
+
+      for (; i + 8 <= EQm; i += 8) {
+        // Load 128 bytes = 64 u16 = 8 groups (aligned-friendly, but vld1q_u8 is fine either way)
+        uint8x16_t b0 = vld1q_u8((const uint8_t*)(f +  0)); // bytes  0..15
+        uint8x16_t b1 = vld1q_u8((const uint8_t*)(f +  8)); // bytes 16..31
+        uint8x16_t b2 = vld1q_u8((const uint8_t*)(f + 16)); // bytes 32..47
+        uint8x16_t b3 = vld1q_u8((const uint8_t*)(f + 24)); // bytes 48..63
+        uint8x16_t b4 = vld1q_u8((const uint8_t*)(f + 32)); // bytes 64..79
+        uint8x16_t b5 = vld1q_u8((const uint8_t*)(f + 40)); // bytes 80..95
+        uint8x16_t b6 = vld1q_u8((const uint8_t*)(f + 48)); // bytes 96..111
+        uint8x16_t b7 = vld1q_u8((const uint8_t*)(f + 56)); // bytes 112..127
+        f += 64; // consumed 64 u16
+	       
+        uint8x16_t o0b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx0);
+        uint8x16_t o1b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx1);
+        uint8x16_t o2b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx2);
+        uint8x16_t o3b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx3);
+        uint8x16_t o4b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx4);
+        uint8x16_t o5b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx5);
+        uint8x16_t o6b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx6);
+        uint8x16_t o7b = tbl128_u8(b0,b1,b2,b3,b4,b5,b6,b7, idx7);
+
+        vst1q_s16(e, vreinterpretq_s16_u8(o0b)); e += 8;
+        vst1q_s16(e1, vreinterpretq_s16_u8(o1b)); e1 += 8;
+        vst1q_s16(e2, vreinterpretq_s16_u8(o2b)); e2 += 8;
+        vst1q_s16(e3, vreinterpretq_s16_u8(o3b)); e3 += 8;
+        vst1q_s16(e4, vreinterpretq_s16_u8(o4b)); e4 += 8;
+        vst1q_s16(e5, vreinterpretq_s16_u8(o5b)); e5 += 8;
+        vst1q_s16(e6, vreinterpretq_s16_u8(o6b)); e6 += 8;
+        vst1q_s16(e7, vreinterpretq_s16_u8(o7b)); e7 += 8;	
+      }
+#elif defined(__AVX512BW__)
+
+    // Precompute indices for each stream k=0..7
+    const __m512i idx0 = idx_stride_u16(8,0);
+    const __m512i idx1 = idx_stride_u16(8,1);
+    const __m512i idx2 = idx_stride_u16(8,2);
+    const __m512i idx3 = idx_stride_u16(8,3);
+    const __m512i idx4 = idx_stride_u16(8,4);
+    const __m512i idx5 = idx_stride_u16(8,5);
+    const __m512i idx6 = idx_stride_u16(8,6);
+    const __m512i idx7 = idx_stride_u16(8,7);
+
+    const __m512i c64  = _mm512_set1_epi16(64);
+    const __m512i c128 = _mm512_set1_epi16(128);
+    const __m512i c192 = _mm512_set1_epi16(192);
+    
+    for (; i + 32 <= EQm; i += 32) {
+        // 32 groups * 8 u16 = 256 u16 = 8 * 32 u16 = 8 ZMM vectors
+	__m512i A = _mm512_loadu_si512((const void*)(f +  0));   // u16[  0.. 31]
+	__m512i B = _mm512_loadu_si512((const void*)(f + 32));   // u16[ 32.. 63]
+	__m512i C = _mm512_loadu_si512((const void*)(f + 64));   // u16[ 64.. 95]
+	__m512i D = _mm512_loadu_si512((const void*)(f + 96));   // u16[ 96..127]
+	__m512i E = _mm512_loadu_si512((const void*)(f + 128));  // u16[128..159]
+	__m512i F = _mm512_loadu_si512((const void*)(f + 160));  // u16[160..191]
+	__m512i G = _mm512_loadu_si512((const void*)(f + 192));  // u16[192..223]
+	__m512i H = _mm512_loadu_si512((const void*)(f + 224));  // u16[224..255]
+	f += 256;
+        // Helper macro: compute one stream (k) by permuting from 4 segments and blending
+	#define DO_STREAM(IDX, OUTPTR) do { \
+	   /* segment masks based on global index p = k+8*n */ \
+	   __mmask32 m1 = _mm512_cmpge_epu16_mask((IDX), c64); \
+	   __mmask32 m2 = _mm512_cmpge_epu16_mask((IDX), c128); \
+	   __mmask32 m3 = _mm512_cmpge_epu16_mask((IDX), c192); \
+	   /* indices relative to each segment base */ \
+	   __m512i i0 = (IDX); \
+	   __m512i i1 = _mm512_sub_epi16((IDX), c64); \
+	   __m512i i2 = _mm512_sub_epi16((IDX), c128); \
+	   __m512i i3 = _mm512_sub_epi16((IDX), c192); \
+	   /* permute from each 64-u16 segment (two ZMMs per segment) */ \
+	   __m512i r0 = _mm512_permutex2var_epi16(A, i0, B); \
+	   __m512i r1 = _mm512_permutex2var_epi16(C, i1, D); \
+	   __m512i r2 = _mm512_permutex2var_epi16(E, i2, F); \
+	   __m512i r3 = _mm512_permutex2var_epi16(G, i3, H); \
+           /* blend: later segments overwrite earlier where applicable */ \
+	   __m512i r = _mm512_mask_mov_epi16(r0, m1, r1); \
+	   r = _mm512_mask_mov_epi16(r,  m2, r2); \
+	   r = _mm512_mask_mov_epi16(r,  m3, r3); \
+	   _mm512_storeu_si512((void*)(OUTPTR), r); \
+	   (OUTPTR) += 32; \
+	} while(0)
+        DO_STREAM(idx0, e);
+	DO_STREAM(idx1, e1);
+	DO_STREAM(idx2, e2);
+	DO_STREAM(idx3, e3);
+	DO_STREAM(idx4, e4);
+	DO_STREAM(idx5, e5);
+	DO_STREAM(idx6, e6);
+	DO_STREAM(idx7, e7);
+
+        #undef DO_STREAM
+    }
+
+#else
+      simde__m128i *e0_128 = (simde__m128i *)e;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *e4_128 = (simde__m128i *)e4;
+      simde__m128i *e5_128 = (simde__m128i *)e5;
+      simde__m128i *e6_128 = (simde__m128i *)e6;
+      simde__m128i *e7_128 = (simde__m128i *)e7;
+      simde__m128i *f128   = (simde__m128i *)f;
+
+      for (i=0; i < (EQm & ~7); i += 8) {
+        simde__m128i f0j = simde_mm_loadu_si128(f128++);
+        simde__m128i f1j = simde_mm_loadu_si128(f128++);
+        simde__m128i f2j = simde_mm_loadu_si128(f128++);
+        simde__m128i f3j = simde_mm_loadu_si128(f128++);
+        simde__m128i f4j = simde_mm_loadu_si128(f128++);
+        simde__m128i f5j = simde_mm_loadu_si128(f128++);
+        simde__m128i f6j = simde_mm_loadu_si128(f128++);
+        simde__m128i f7j = simde_mm_loadu_si128(f128++);
+
+        simde__m128i tmp0 = simde_mm_unpacklo_epi16(f0j, f1j);   // f0(i) f1(i) f0(i+1) f1(i+1) f0(i+2) f1(i+2) f0(i+3) f1(i+3) 
+        simde__m128i tmp1 = simde_mm_unpacklo_epi16(f2j, f3j);   // f2(i) f3(i) f2(i+1) f3(i+1) f2(i+2) f3(i+2) f2(i+3) f3(i+3) 
+        simde__m128i tmp2 = simde_mm_unpacklo_epi16(f4j, f5j);   // f4(i) f5(i) f4(i+1) f5(i+1) f4(i+2) f5(i+2) f4(i+3) f5(i+3)
+        simde__m128i tmp3 = simde_mm_unpacklo_epi16(f6j, f7j);   // f6(i) f7(i) f6(i+1) f7(i+1) f6(i+2) f7(i+2) f6(i+3) f7(i+3)
+								 //
+        simde__m128i tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i) f1(i) f2(i)   f3(i)   f0(i+1) f1(i+1) f2(i+1) f3(i+1) 
+        simde__m128i tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i) f5(i) f6(i)   f7(i)   f4(i+1) f5(i+1) f6(i+1) f7(i+1) 
+        simde_mm_storeu_si128(e0_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i) f1(i) f2(i) f3(i) f4(i) f5(i) f7(i) f7(i)
+        simde_mm_storeu_si128(e1_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+1) f1(i+1) f2(i+1) f3(i+1) f4(i+1) f5(i+1) f6(i+1) f7(i+1)
+									    //
+        tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+2) f1(i+2) f2(i+2) f3(i+2) f0(i+3) f1(i+3) f2(i+3) f3(i+3) 
+        tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+2) f5(i+2) f6(i+2) f7(i+2) f4(i+3) f5(i+3) f6(i+3) f7(i+3) 
+	simde_mm_storeu_si128(e2_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+2) f1(i+2) f2(i+2) f3(i+2) f4(i+2) f5(i+2) f7(i+2) f7(i+2) 
+        simde_mm_storeu_si128(e3_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+3) f1(i+3) f2(i+3) f3(i+3) f4(i+3) f5(i+3) f7(i+3) f7(i+3)
+									     
+        tmp0 = simde_mm_unpackhi_epi16(f0j, f1j);   // f0(i+4) f1(i+4) f0(i+5) f1(i+5) f0(i+6) f1(i+6) f0(i+7) f1(i+7) 
+        tmp1 = simde_mm_unpackhi_epi16(f2j, f3j);   // f2(i+4) f3(i+4) f2(i+5) f3(i+5) f2(i+6) f3(i+6) f2(i+7) f3(i+7) 
+        tmp2 = simde_mm_unpackhi_epi16(f4j, f5j);   // f4(i+4) f5(i+4) f4(i+5) f5(i+5) f4(i+6) f5(i+6) f4(i+7) f5(i+7)
+        tmp3 = simde_mm_unpackhi_epi16(f6j, f7j);   // f6(i+4) f7(i+4) f6(i+5) f7(i+5) f6(i+6) f7(i+6) f6(i+7) f7(i+7)
+								 //
+        tmp4 = simde_mm_unpacklo_epi32(tmp0, tmp1); // f0(i+4) f1(i+4) f2(i+4)   f3(i+4)   f0(i+5) f1(i+5) f2(i+5) f3(i+5) 
+        tmp5 = simde_mm_unpacklo_epi32(tmp2, tmp3); // f4(i+4) f5(i+4) f6(i+4)   f7(i+4)   f4(i+5) f5(i+5) f6(i+5) f7(i+5) 
+								 
+        simde_mm_storeu_si128(e4_128++,simde_mm_unpacklo_epi64(tmp4, tmp5)); // f0(i+4) f1(i+4) f2(i+4) f3(i+4) f4(i+4) f5(i+4) f7(i+4) f7(i+4)
+        simde_mm_storeu_si128(e5_128++,simde_mm_unpackhi_epi64(tmp4, tmp5));// f0(i+5) f1(i+5) f2(i+5) f3(i+5) f4(i+5) f5(i+5) f6(i+5) f7(i+5)
+        tmp4 = simde_mm_unpackhi_epi32(tmp0, tmp1); // f0(i+6) f1(i+6) f2(i+6) f3(i+6) f0(i+6) f1(i+6) f2(i+6) f3(i+6) 
+        tmp5 = simde_mm_unpackhi_epi32(tmp2, tmp3); // f4(i+6) f5(i+6) f6(i+6) f7(i+6) f4(i+6) f5(i+6) f6(i+6) f7(i+6) 
+	simde_mm_storeu_si128(e6_128++,simde_mm_unpacklo_epi64(tmp4, tmp5));// f0(i+6) f1(i+6) f2(i+6) f3(i+6) f4(i+6) f5(i+6) f7(i+6) f7(i+6) 
+        simde_mm_storeu_si128(e7_128++,simde_mm_unpackhi_epi64(tmp4, tmp5)); // f0(i+7) f1(i+7) f2(i+7) f3(i+7) f4(i+7) f5(i+7) f7(i+7) f7(i+7)
+      }
+      e  = (int16_t *)e0_128;
+      e1 = (int16_t *)e1_128;
+      e2 = (int16_t *)e2_128;
+      e3 = (int16_t *)e3_128;
+      e4 = (int16_t *)e4_128;
+      e5 = (int16_t *)e5_128;
+      e6 = (int16_t *)e6_128;
+      e7 = (int16_t *)e7_128;
+      f  = (int16_t *)f128;
+#endif
+      for(;i < EQm ; i++) {
         *e++ = *f++;
         *e1++ = *f++;
         *e2++ = *f++;
@@ -511,6 +1210,7 @@ void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
       AssertFatal(1 == 0, "Should not get here : Qm %d\n", Qm);
       break;
   }
+
 }
 
 int nr_rate_matching_ldpc(uint32_t Tbslbrm,
@@ -610,6 +1310,108 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
   return 0;
 }
 
+int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
+                            uint8_t BG,
+                            uint16_t Z,
+                            uint32_t *d,
+                            uint32_t *e,
+                            uint8_t C,
+                            uint32_t F,
+                            uint32_t Foffset,
+                            uint8_t rvidx,
+                            uint32_t E)
+{
+  if (C == 0) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameter C %d\n", C);
+    return -1;
+  }
+
+  //Bit selection
+  uint32_t N = (BG == 1) ? (66 * Z) : (50 * Z);
+  uint32_t Ncb;
+  if (Tbslbrm == 0)
+    Ncb = N;
+  else {
+    uint32_t Nref = 3 * Tbslbrm / (2 * C); //R_LBRM = 2/3
+    Ncb = min(N, Nref);
+  }
+
+  uint32_t ind = (index_k0[BG - 1][rvidx] * Ncb / N) * Z;
+
+#ifdef RM_DEBUG
+  printf("nr_rate_matching_ldpc: E %u, F %u, Foffset %u, k0 %u, Ncb %u, rvidx %d, Tbslbrm %u\n",
+         E,
+         F,
+         Foffset,
+         ind,
+         Ncb,
+         rvidx,
+         Tbslbrm);
+#endif
+
+  if (Foffset > E) {
+    LOG_E(PHY,
+          "nr_rate_matching: invalid parameters (Foffset %d > E %d) F %d, k0 %d, Ncb %d, rvidx %d, Tbslbrm %d\n",
+          Foffset,
+          E,
+          F,
+          ind,
+          Ncb,
+          rvidx,
+          Tbslbrm);
+    return -1;
+  }
+  if (Foffset > Ncb) {
+    LOG_E(PHY, "nr_rate_matching: invalid parameters (Foffset %d > Ncb %d)\n", Foffset, Ncb);
+    return -1;
+  }
+
+  if (ind >= Foffset && ind < (F + Foffset))
+    ind = F + Foffset;
+
+  uint32_t k = 0;
+  if (ind < Foffset) { // case where we have some bits before the filler and the rest after
+    memcpy((void *)e, (void *)(d + ind), (Foffset - ind)<<2);
+
+    if (E + F <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (E - Foffset + ind)<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e + Foffset - ind), (void *)(d + Foffset + F), (Ncb - Foffset - F)<<2);
+      k = Ncb - F - ind;
+    }
+  } else {
+    if (E <= Ncb - ind) { // E+F doesn't contain all coded bits
+      memcpy((void *)(e), (void *)(d + ind), E<<2);
+      k = E;
+    } else {
+      memcpy((void *)(e), (void *)(d + ind), (Ncb - ind)<<2);
+      k = Ncb - ind;
+    }
+  }
+
+  while (k < E) { // case where we do repetitions (low mcs)
+    for (ind = 0; (ind < Ncb) && (k < E); ind++) {
+#ifdef RM_DEBUG
+      printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
+#endif
+
+      if (ind == Foffset)
+        ind = F + Foffset; // skip filler bits
+
+      e[k++] = d[ind];
+
+    }
+  }
+
+  return 0;
+}
+
+//#define USE_SCALAR 1
+#define RMLOOP for (;ind<(ind2&7);k+=8,ind+=8) \
+      simde_mm_storeu_si128(&d[ind],simde_mm_adds_epi16(simde_mm_loadu_si128(&soft_input[k]),simde_mm_loadu_si128(&d[ind])));\
+   for (; ind<ind2 ; ind++,k++) d[ind] += soft_input[k];  
+  
 int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
                              uint8_t BG,
                              uint16_t Z,
@@ -662,7 +1464,8 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
     memset(d, 0, Ncb * sizeof(int16_t));
 
   uint32_t k = 0;
-  if (ind < Foffset)
+  if (ind < Foffset) {
+#ifdef USE_SCALAR 
     for (; (ind < Foffset) && (k < E); ind++) {
 #ifdef RM_DEBUG
       printf("RM_RX k%u Ind %u(before filler): %d (%d)=>", k, ind, d[ind], soft_input[k]);
@@ -672,9 +1475,14 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
       printf("%d\n", d[ind]);
 #endif
     }
+#else
+   int ind2 = ind + min(Foffset-ind,E);
+   RMLOOP;
+#endif
+  }
   if (ind >= Foffset && ind < Foffset + F)
     ind = Foffset + F;
-
+#ifdef USE_SCALAR
   for (; (ind < Ncb) && (k < E); ind++) {
 #ifdef RM_DEBUG
     printf("RM_RX k%u Ind %u(after filler) %d (%d)=>", k, ind, d[ind], soft_input[k]);
@@ -684,8 +1492,13 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
     printf("%d\n", d[ind]);
 #endif
   }
+#else
+   int ind2 = ind + min(Ncb-ind,E-k);
+   RMLOOP;
+#endif
 
   while (k < E) {
+#ifdef USE_SCALAR
     for (ind = 0; (ind < Foffset) && (k < E); ind++) {
 #ifdef RM_DEBUG
       printf("RM_RX k%u Ind %u(before filler) %d(%d)=>", k, ind, d[ind], soft_input[k]);
@@ -704,6 +1517,14 @@ int nr_rate_matching_ldpc_rx(uint32_t Tbslbrm,
       printf("%d\n", d[ind]);
 #endif
     }
+#else
+   ind=0;
+   ind2 = min(Foffset,E-k);
+   RMLOOP;
+   ind = Foffset+F;
+   ind2 = ind + min(Ncb-ind,E-k);
+   RMLOOP;
+#endif
   }
   return 0;
 }

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h

@@ -33,6 +33,15 @@
  */
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f);
 
+/**
+ * \brief interleave a code segment after encoding and rate matching (32bit)
+ * \param E size of the code segment in bits
+ * \param Qm modulation order
+ * \param e input rate matched segment
+ * \param f output interleaved segment
+ */
+
+void nr_interleaving_ldpc32(uint32_t E, uint8_t Qm, uint32_t *e, uint32_t *f);
 /**
  * \brief deinterleave a code segment before RX rate matching and decoding
  * \param E size of the code segment in bits
@@ -66,6 +75,30 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
                           uint8_t rvidx,
                           uint32_t E);
 
+/**
+ * \brief rate match a code segment after encoding (32bit)
+ * \Tbslbrm Transport Block size LBRM
+ * \param BG LDPC base graph number
+ * \param Z segment lifting size
+ * \param d input encoded segment
+ * \param e output rate matched segment
+ * \param C number of segments in the Transport Block
+ * \param F number of filler bits in the segment
+ * \param Foffset offset of the filler bits in the segment
+ * \param rvidx redundancy version index
+ * \param E size of the code segment in bits
+ */
+int nr_rate_matching_ldpc32(uint32_t Tbslbrm,
+                            uint8_t BG,
+                            uint16_t Z,
+                            uint32_t *d,
+                            uint32_t *e,
+                            uint8_t C,
+                            uint32_t F,
+                            uint32_t Foffset,
+                            uint8_t rvidx,
+                            uint32_t E);
+
 /**
  * \brief rate match a code segment before decoding
  * \Tbslbrm Transport Block size LBRM

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c

@@ -161,6 +161,7 @@ static inline uint32_t nrLDPC_decoder_core(int8_t* p_llr,
 
 int32_t LDPCinit()
 {
+  cuda_support_init();
   return 0;
 }
 

openair1/PHY/CODING/nrLDPC_defs.h

@@ -55,9 +55,11 @@ typedef struct {
   unsigned int first_seg; // optim8segmulti
   unsigned char gen_code; // orig
   time_stats_t *tinput;
+  time_stats_t *tinput_memcpy;
   time_stats_t *tprep;
   time_stats_t *tparity;
   time_stats_t *toutput;
+  time_stats_t *tconcat;
   /// Size in bits of the code segments
   uint32_t K;
   /// Number of lifting sizes to fit the payload
@@ -91,5 +93,7 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
                                     t_nrLDPC_time_stats *time_stats,
                                     decode_abort_t *ab);
 typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
+typedef uint32_t**(LDPC_encoderfunc32_t)(uint8_t **, encoder_implemparams_t *);
 
+void cuda_support_init(void);
 #endif

openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG1_Zc384_32bit.cu

@@ -0,0 +1,214 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <cuda_runtime.h>
+// generated code for Zc=384, byte encoding
+__global__ void ldpc_BG1_Zc384_worker(uint32_t *c[4],uint32_t *d[4]) {
+  uint32_t *c32=c[blockIdx.x];
+  uint32_t *d32=d[blockIdx.x]+8448-2*384;
+
+  int i2 = threadIdx.x;
+  int i1 = blockIdx.y;
+  if (i2 < 384) {
+    c32+=i2;
+    d32+=i2;
+    switch(i1) {
+    case 0:
+      d32[0]=c32[307]^c32[76]^c32[205]^c32[276]^c32[787]^c32[1018]^c32[855]^c32[1586]^c32[1612]^c32[1864]^c32[2673]^c32[2377]^c32[2304]^c32[3360]^c32[3404]^c32[3347]^c32[4021]^c32[3984]^c32[4096]^c32[4824]^c32[4769]^c32[4807]^c32[5707]^c32[5643]^c32[5529]^c32[6475]^c32[6304]^c32[6200]^c32[7229]^c32[7090]^c32[6975]^c32[7968]^c32[7809]^c32[7812]^c32[8557]^c32[8743]^c32[8753]^c32[9233]^c32[9558]^c32[9447]^c32[10341]^c32[10184]^c32[10325]^c32[10969]^c32[10840]^c32[10964]^c32[11735]^c32[11619]^c32[11573]^c32[12394]^c32[12642]^c32[12592]^c32[13170]^c32[13187]^c32[13356]^c32[14066]^c32[14064]^c32[14095]^c32[14772]^c32[14923]^c32[14797]^c32[15690]^c32[15373]^c32[15399]^c32[16474]^c32[16240]^c32[16485];
+
+       break;
+    case 1:
+      d32[384]=c32[307]^c32[308]^c32[77]^c32[206]^c32[277]^c32[787]^c32[788]^c32[1019]^c32[856]^c32[1586]^c32[1587]^c32[1613]^c32[1865]^c32[2673]^c32[2674]^c32[2378]^c32[2305]^c32[3361]^c32[3405]^c32[3348]^c32[4021]^c32[4022]^c32[3985]^c32[4097]^c32[4824]^c32[4825]^c32[4770]^c32[4808]^c32[5708]^c32[5644]^c32[5530]^c32[6476]^c32[6305]^c32[6201]^c32[7229]^c32[7230]^c32[7091]^c32[6976]^c32[7968]^c32[7969]^c32[7810]^c32[7813]^c32[8557]^c32[8558]^c32[8744]^c32[8754]^c32[9233]^c32[9234]^c32[9559]^c32[9448]^c32[10341]^c32[10342]^c32[10185]^c32[10326]^c32[10970]^c32[10841]^c32[10965]^c32[11735]^c32[11736]^c32[11620]^c32[11574]^c32[12394]^c32[12395]^c32[12643]^c32[12593]^c32[13171]^c32[13188]^c32[13357]^c32[14066]^c32[14067]^c32[14065]^c32[14096]^c32[14772]^c32[14773]^c32[14924]^c32[14798]^c32[15690]^c32[15691]^c32[15374]^c32[15400]^c32[16474]^c32[16475]^c32[16241]^c32[16486];
+
+       break;
+    case 2:
+      d32[768]=c32[308]^c32[77]^c32[205]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1018]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1864]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3404]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4096]^c32[4097]^c32[4825]^c32[4769]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5643]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6304]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6975]^c32[6976]^c32[7969]^c32[7809]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10184]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10840]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11573]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13187]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14064]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14797]^c32[14798]^c32[15691]^c32[15373]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
+
+       break;
+    case 3:
+      d32[1152]=c32[308]^c32[77]^c32[206]^c32[276]^c32[277]^c32[788]^c32[1019]^c32[855]^c32[856]^c32[1587]^c32[1613]^c32[1865]^c32[2674]^c32[2378]^c32[2304]^c32[2305]^c32[3361]^c32[3405]^c32[3347]^c32[3348]^c32[4022]^c32[3985]^c32[4097]^c32[4825]^c32[4770]^c32[4807]^c32[4808]^c32[5708]^c32[5644]^c32[5529]^c32[5530]^c32[6476]^c32[6305]^c32[6200]^c32[6201]^c32[7230]^c32[7091]^c32[6976]^c32[7969]^c32[7810]^c32[7812]^c32[7813]^c32[8558]^c32[8744]^c32[8753]^c32[8754]^c32[9234]^c32[9559]^c32[9447]^c32[9448]^c32[10342]^c32[10185]^c32[10325]^c32[10326]^c32[10970]^c32[10841]^c32[10964]^c32[10965]^c32[11736]^c32[11620]^c32[11574]^c32[12395]^c32[12643]^c32[12592]^c32[12593]^c32[13171]^c32[13188]^c32[13356]^c32[13357]^c32[14067]^c32[14065]^c32[14095]^c32[14096]^c32[14773]^c32[14924]^c32[14798]^c32[15691]^c32[15374]^c32[15399]^c32[15400]^c32[16475]^c32[16241]^c32[16485]^c32[16486];
+
+       break;
+    case 4:
+      d32[1536]=c32[332]^c32[949];
+
+       break;
+    case 5:
+      d32[1920]=c32[80]^c32[233]^c32[362]^c32[49]^c32[195]^c32[944]^c32[791]^c32[1012]^c32[782]^c32[1743]^c32[1769]^c32[1637]^c32[2446]^c32[2534]^c32[2461]^c32[2419]^c32[3133]^c32[3177]^c32[3120]^c32[4178]^c32[4141]^c32[3869]^c32[4981]^c32[4926]^c32[4964]^c32[5480]^c32[5416]^c32[5686]^c32[6248]^c32[6461]^c32[6357]^c32[7002]^c32[7247]^c32[7132]^c32[7741]^c32[7966]^c32[7969]^c32[8714]^c32[8516]^c32[8526]^c32[9390]^c32[9331]^c32[9220]^c32[9382]^c32[10114]^c32[10341]^c32[10098]^c32[11126]^c32[10997]^c32[11121]^c32[11892]^c32[11776]^c32[11730]^c32[12551]^c32[12415]^c32[12365]^c32[12529]^c32[13327]^c32[13344]^c32[13129]^c32[13839]^c32[13837]^c32[13868]^c32[14929]^c32[14696]^c32[14954]^c32[15463]^c32[15530]^c32[15556]^c32[16247]^c32[16397]^c32[16258]^c32[16179];
+
+       break;
+    case 6:
+      d32[2304]=c32[278]^c32[4865]^c32[7681]^c32[8799]^c32[10076]^c32[13309]^c32[13842]^c32[15585];
+
+       break;
+    case 7:
+      d32[2688]=c32[9]^c32[830]^c32[3388]^c32[5709]^c32[6434]^c32[10866];
+
+       break;
+    case 8:
+      d32[3072]=c32[374]^c32[94]^c32[143]^c32[247]^c32[272]^c32[375]^c32[376]^c32[343]^c32[62]^c32[63]^c32[307]^c32[854]^c32[958]^c32[1085]^c32[804]^c32[805]^c32[922]^c32[1025]^c32[1026]^c32[947]^c32[1653]^c32[1757]^c32[1679]^c32[1783]^c32[1547]^c32[1650]^c32[1651]^c32[2356]^c32[2460]^c32[2444]^c32[2548]^c32[2371]^c32[2474]^c32[2475]^c32[2469]^c32[3427]^c32[3147]^c32[3087]^c32[3190]^c32[3191]^c32[3414]^c32[3133]^c32[3134]^c32[4088]^c32[4192]^c32[4051]^c32[4155]^c32[4163]^c32[3882]^c32[3883]^c32[4891]^c32[4611]^c32[4836]^c32[4939]^c32[4940]^c32[4874]^c32[4977]^c32[4978]^c32[5390]^c32[5494]^c32[5710]^c32[5429]^c32[5430]^c32[5596]^c32[5699]^c32[5700]^c32[6158]^c32[6262]^c32[6371]^c32[6474]^c32[6475]^c32[6267]^c32[6370]^c32[6371]^c32[6912]^c32[7016]^c32[7157]^c32[7261]^c32[7042]^c32[7145]^c32[7146]^c32[8035]^c32[7755]^c32[7876]^c32[7979]^c32[7980]^c32[7879]^c32[7982]^c32[7983]^c32[8624]^c32[8728]^c32[8810]^c32[8530]^c32[8820]^c32[8539]^c32[8540]^c32[9300]^c32[9404]^c32[9241]^c32[9345]^c32[9514]^c32[9233]^c32[9234]^c32[9234]^c32[10024]^c32[10128]^c32[10251]^c32[10354]^c32[10355]^c32[10008]^c32[10111]^c32[10112]^c32[11036]^c32[10756]^c32[10907]^c32[11010]^c32[11011]^c32[11031]^c32[11134]^c32[11135]^c32[11802]^c32[11522]^c32[11686]^c32[11790]^c32[11640]^c32[11743]^c32[11744]^c32[12461]^c32[12565]^c32[12325]^c32[12429]^c32[12659]^c32[12378]^c32[12379]^c32[12327]^c32[13237]^c32[13341]^c32[13254]^c32[13357]^c32[13358]^c32[13423]^c32[13142]^c32[13143]^c32[14133]^c32[13853]^c32[14131]^c32[13850]^c32[13851]^c32[14162]^c32[13881]^c32[13882]^c32[14839]^c32[14943]^c32[14606]^c32[14710]^c32[14864]^c32[14967]^c32[14968]^c32[14816]^c32[15373]^c32[15477]^c32[15440]^c32[15543]^c32[15544]^c32[15466]^c32[15569]^c32[15570]^c32[16157]^c32[16261]^c32[16307]^c32[16411]^c32[16168]^c32[16271]^c32[16272]^c32[16496];
+
+       break;
+    case 9:
+      d32[3456]=c32[366]^c32[1000]^c32[8001]^c32[8581]^c32[10041]^c32[13359]^c32[13887]^c32[15442];
+
+       break;
+    case 10:
+      d32[3840]=c32[869]^c32[1875]^c32[3346]^c32[5487]^c32[6527]^c32[11106];
+
+       break;
+    case 11:
+      d32[4224]=c32[257]^c32[38]^c32[39]^c32[26]^c32[192]^c32[155]^c32[321]^c32[226]^c32[8]^c32[48]^c32[1121]^c32[902]^c32[903]^c32[968]^c32[1134]^c32[805]^c32[971]^c32[870]^c32[1536]^c32[1701]^c32[1702]^c32[1562]^c32[1728]^c32[1814]^c32[1596]^c32[2623]^c32[2404]^c32[2405]^c32[2327]^c32[2493]^c32[2638]^c32[2420]^c32[3310]^c32[3092]^c32[3354]^c32[3136]^c32[3297]^c32[3079]^c32[3971]^c32[4136]^c32[4137]^c32[3934]^c32[4100]^c32[4046]^c32[4212]^c32[4774]^c32[4939]^c32[4940]^c32[4719]^c32[4885]^c32[4757]^c32[4923]^c32[5657]^c32[5439]^c32[5593]^c32[5759]^c32[5479]^c32[5645]^c32[6425]^c32[6207]^c32[6254]^c32[6420]^c32[6150]^c32[6316]^c32[7179]^c32[6960]^c32[6961]^c32[7040]^c32[7206]^c32[6925]^c32[7091]^c32[7918]^c32[7699]^c32[7700]^c32[7759]^c32[7925]^c32[7762]^c32[7928]^c32[8507]^c32[8672]^c32[8673]^c32[8693]^c32[8475]^c32[8703]^c32[8485]^c32[9567]^c32[9348]^c32[9349]^c32[9508]^c32[9290]^c32[9397]^c32[9563]^c32[9224]^c32[10291]^c32[10072]^c32[10073]^c32[10134]^c32[10300]^c32[10275]^c32[10057]^c32[10919]^c32[11085]^c32[10790]^c32[10956]^c32[10914]^c32[11080]^c32[11685]^c32[11850]^c32[11851]^c32[11569]^c32[11735]^c32[11523]^c32[11689]^c32[12344]^c32[12509]^c32[12510]^c32[12592]^c32[12374]^c32[12542]^c32[12324]^c32[12335]^c32[13120]^c32[13286]^c32[13137]^c32[13303]^c32[13306]^c32[13088]^c32[14016]^c32[14181]^c32[14182]^c32[14014]^c32[14180]^c32[14045]^c32[13827]^c32[14722]^c32[14887]^c32[14888]^c32[14873]^c32[14655]^c32[14747]^c32[14913]^c32[15640]^c32[15421]^c32[15422]^c32[15707]^c32[15489]^c32[15733]^c32[15515]^c32[16424]^c32[16205]^c32[16206]^c32[16190]^c32[16356]^c32[16435]^c32[16217]^c32[16316];
+
+       break;
+    case 12:
+      d32[4608]=c32[77]^c32[954]^c32[7854]^c32[8680]^c32[10034]^c32[13898];
+
+       break;
+    case 13:
+      d32[4992]=c32[293]^c32[294]^c32[63]^c32[192]^c32[263]^c32[313]^c32[773]^c32[774]^c32[1005]^c32[842]^c32[1572]^c32[1573]^c32[1599]^c32[1851]^c32[2659]^c32[2660]^c32[2364]^c32[2675]^c32[2481]^c32[3347]^c32[3391]^c32[3334]^c32[4007]^c32[4008]^c32[3971]^c32[4083]^c32[4810]^c32[4811]^c32[4756]^c32[4794]^c32[5694]^c32[5630]^c32[5516]^c32[5642]^c32[6462]^c32[6291]^c32[6187]^c32[7215]^c32[7216]^c32[7077]^c32[6962]^c32[7954]^c32[7955]^c32[7796]^c32[7799]^c32[8543]^c32[8544]^c32[8730]^c32[8740]^c32[9219]^c32[9220]^c32[9545]^c32[9434]^c32[10327]^c32[10328]^c32[10171]^c32[10312]^c32[10956]^c32[10827]^c32[10951]^c32[11721]^c32[11722]^c32[11606]^c32[11560]^c32[12380]^c32[12381]^c32[12629]^c32[12579]^c32[13157]^c32[13174]^c32[13343]^c32[14052]^c32[14053]^c32[14051]^c32[14082]^c32[14758]^c32[14759]^c32[14910]^c32[14784]^c32[15676]^c32[15677]^c32[15360]^c32[15386]^c32[15475]^c32[16460]^c32[16461]^c32[16227]^c32[16472];
+
+       break;
+    case 14:
+      d32[5376]=c32[142]^c32[9464]^c32[11657]^c32[12377]^c32[13403]^c32[16140];
+
+       break;
+    case 15:
+      d32[5760]=c32[193]^c32[346]^c32[91]^c32[161]^c32[162]^c32[241]^c32[1057]^c32[904]^c32[1124]^c32[1125]^c32[770]^c32[1856]^c32[1882]^c32[1750]^c32[2559]^c32[2647]^c32[2573]^c32[2574]^c32[3246]^c32[3290]^c32[3232]^c32[3233]^c32[3907]^c32[3870]^c32[3982]^c32[4710]^c32[4655]^c32[4692]^c32[4693]^c32[5593]^c32[5529]^c32[5414]^c32[5415]^c32[6361]^c32[6190]^c32[6469]^c32[6470]^c32[7115]^c32[6976]^c32[7245]^c32[7854]^c32[7695]^c32[7697]^c32[7698]^c32[7890]^c32[8827]^c32[8629]^c32[8638]^c32[8639]^c32[9503]^c32[9444]^c32[9332]^c32[9333]^c32[10227]^c32[10070]^c32[10210]^c32[10211]^c32[10302]^c32[10855]^c32[11110]^c32[10849]^c32[10850]^c32[11621]^c32[11889]^c32[11843]^c32[12664]^c32[12528]^c32[12477]^c32[12478]^c32[13056]^c32[13073]^c32[13241]^c32[13242]^c32[13952]^c32[13950]^c32[13980]^c32[13981]^c32[13879]^c32[14658]^c32[14809]^c32[14683]^c32[15576]^c32[15643]^c32[15668]^c32[15669]^c32[16360]^c32[16510]^c32[16370]^c32[16371];
+
+       break;
+    case 16:
+      d32[6144]=c32[364]^c32[133]^c32[262]^c32[333]^c32[844]^c32[1075]^c32[912]^c32[781]^c32[1643]^c32[1669]^c32[1537]^c32[2346]^c32[2434]^c32[2361]^c32[2642]^c32[3417]^c32[3077]^c32[3404]^c32[4078]^c32[4041]^c32[4153]^c32[4881]^c32[4826]^c32[4864]^c32[5380]^c32[5700]^c32[5586]^c32[6148]^c32[6361]^c32[6257]^c32[7286]^c32[7147]^c32[7032]^c32[8025]^c32[7866]^c32[7869]^c32[8614]^c32[8800]^c32[8810]^c32[8505]^c32[9290]^c32[9231]^c32[9504]^c32[10014]^c32[10241]^c32[9998]^c32[11026]^c32[10897]^c32[11021]^c32[11792]^c32[11676]^c32[11630]^c32[12451]^c32[12315]^c32[12649]^c32[13227]^c32[13244]^c32[13413]^c32[14123]^c32[14121]^c32[14152]^c32[14829]^c32[14596]^c32[14854]^c32[15363]^c32[15430]^c32[15456]^c32[15649]^c32[16147]^c32[16297]^c32[16158];
+
+       break;
+    case 17:
+      d32[6528]=c32[260]^c32[11055]^c32[12369]^c32[13414]^c32[16503];
+
+       break;
+    case 18:
+      d32[6912]=c32[898]^c32[9379]^c32[10264]^c32[13956]^c32[14596];
+
+       break;
+    case 19:
+      d32[7296]=c32[145]^c32[981]^c32[5720]^c32[6386]^c32[7877];
+
+       break;
+    case 20:
+      d32[7680]=c32[366]^c32[135]^c32[264]^c32[335]^c32[187]^c32[846]^c32[1077]^c32[914]^c32[1645]^c32[1671]^c32[1539]^c32[2348]^c32[2436]^c32[2363]^c32[2510]^c32[3419]^c32[3079]^c32[3406]^c32[4080]^c32[4043]^c32[4155]^c32[4883]^c32[4828]^c32[4866]^c32[5382]^c32[5702]^c32[5588]^c32[6150]^c32[6363]^c32[6259]^c32[7288]^c32[7149]^c32[7034]^c32[7176]^c32[8027]^c32[7868]^c32[7871]^c32[8616]^c32[8802]^c32[8812]^c32[8789]^c32[9292]^c32[9233]^c32[9506]^c32[10016]^c32[10243]^c32[10000]^c32[11028]^c32[10899]^c32[11023]^c32[11794]^c32[11678]^c32[11632]^c32[12453]^c32[12317]^c32[12651]^c32[13229]^c32[13246]^c32[13415]^c32[14125]^c32[14123]^c32[14154]^c32[14831]^c32[14598]^c32[14856]^c32[15365]^c32[15432]^c32[15458]^c32[16149]^c32[16299]^c32[16160];
+
+       break;
+    case 21:
+      d32[8064]=c32[973]^c32[3942]^c32[12616]^c32[15573]^c32[16225];
+
+       break;
+    case 22:
+      d32[8448]=c32[30]^c32[9227]^c32[10217]^c32[13078];
+
+       break;
+    case 23:
+      d32[8832]=c32[792]^c32[1625]^c32[7741]^c32[13851];
+
+       break;
+    case 24:
+      d32[9216]=c32[157]^c32[310]^c32[55]^c32[126]^c32[298]^c32[1021]^c32[868]^c32[1089]^c32[1820]^c32[1846]^c32[1714]^c32[2523]^c32[2611]^c32[2538]^c32[2462]^c32[3210]^c32[3254]^c32[3197]^c32[3307]^c32[3871]^c32[4218]^c32[3946]^c32[4674]^c32[4619]^c32[4657]^c32[5557]^c32[5493]^c32[5379]^c32[6325]^c32[6154]^c32[6434]^c32[7079]^c32[6940]^c32[7209]^c32[7818]^c32[8043]^c32[8046]^c32[8791]^c32[8593]^c32[8603]^c32[8787]^c32[9467]^c32[9408]^c32[9297]^c32[10191]^c32[10034]^c32[10175]^c32[10819]^c32[11074]^c32[10814]^c32[11585]^c32[11853]^c32[11807]^c32[12628]^c32[12492]^c32[12442]^c32[13404]^c32[13421]^c32[13206]^c32[13916]^c32[13914]^c32[13945]^c32[14622]^c32[14773]^c32[14647]^c32[15540]^c32[15607]^c32[15633]^c32[16324]^c32[16474]^c32[16335];
+
+       break;
+    case 25:
+      d32[9600]=c32[840]^c32[4625]^c32[5759]^c32[11064];
+
+       break;
+    case 26:
+      d32[9984]=c32[71]^c32[1617]^c32[3148]^c32[11656];
+
+       break;
+    case 27:
+      d32[10368]=c32[962]^c32[4802]^c32[6245];
+
+       break;
+    case 28:
+      d32[10752]=c32[222]^c32[3091]^c32[14836]^c32[16402];
+
+       break;
+    case 29:
+      d32[11136]=c32[2]^c32[155]^c32[284]^c32[354]^c32[355]^c32[866]^c32[1097]^c32[933]^c32[934]^c32[1020]^c32[1665]^c32[1691]^c32[1559]^c32[2368]^c32[2456]^c32[2382]^c32[2383]^c32[3439]^c32[3099]^c32[3425]^c32[3426]^c32[4100]^c32[4063]^c32[4175]^c32[4903]^c32[4848]^c32[4885]^c32[4886]^c32[5402]^c32[5722]^c32[5607]^c32[5608]^c32[6170]^c32[6383]^c32[6278]^c32[6279]^c32[6924]^c32[7169]^c32[7054]^c32[8047]^c32[7888]^c32[7890]^c32[7891]^c32[8636]^c32[8822]^c32[8831]^c32[8448]^c32[9312]^c32[9253]^c32[9525]^c32[9526]^c32[10036]^c32[10263]^c32[10019]^c32[10020]^c32[11048]^c32[10919]^c32[11042]^c32[11043]^c32[10757]^c32[11814]^c32[11698]^c32[11652]^c32[12473]^c32[12337]^c32[12670]^c32[12671]^c32[13249]^c32[13266]^c32[13434]^c32[13435]^c32[14145]^c32[14143]^c32[14173]^c32[14174]^c32[13971]^c32[14851]^c32[14618]^c32[14876]^c32[15385]^c32[15452]^c32[15477]^c32[15478]^c32[16169]^c32[16319]^c32[16179]^c32[16180];
+
+       break;
+    case 30:
+      d32[11520]=c32[14]^c32[167]^c32[295]^c32[296]^c32[366]^c32[367]^c32[159]^c32[878]^c32[1108]^c32[1109]^c32[945]^c32[946]^c32[1677]^c32[1703]^c32[1570]^c32[1571]^c32[2380]^c32[2468]^c32[2394]^c32[2395]^c32[3451]^c32[3110]^c32[3111]^c32[3437]^c32[3438]^c32[4112]^c32[4075]^c32[4186]^c32[4187]^c32[4915]^c32[4859]^c32[4860]^c32[4897]^c32[4898]^c32[5414]^c32[5733]^c32[5734]^c32[5619]^c32[5620]^c32[6182]^c32[6394]^c32[6395]^c32[6290]^c32[6291]^c32[6936]^c32[7181]^c32[7065]^c32[7066]^c32[8059]^c32[7899]^c32[7900]^c32[7902]^c32[7903]^c32[7909]^c32[8648]^c32[8450]^c32[8459]^c32[8460]^c32[9324]^c32[9265]^c32[9537]^c32[9538]^c32[10048]^c32[10274]^c32[10275]^c32[10031]^c32[10032]^c32[10244]^c32[11060]^c32[10930]^c32[10931]^c32[11054]^c32[11055]^c32[11826]^c32[11710]^c32[11663]^c32[11664]^c32[12485]^c32[12349]^c32[12298]^c32[12299]^c32[13261]^c32[13277]^c32[13278]^c32[13062]^c32[13063]^c32[14157]^c32[14154]^c32[14155]^c32[14185]^c32[14186]^c32[14863]^c32[14630]^c32[14887]^c32[14888]^c32[15397]^c32[15463]^c32[15464]^c32[15489]^c32[15490]^c32[16181]^c32[16331]^c32[16191]^c32[16192];
+
+       break;
+    case 31:
+      d32[11904]=c32[181]^c32[180]^c32[334]^c32[333]^c32[79]^c32[78]^c32[150]^c32[148]^c32[149]^c32[1045]^c32[1044]^c32[892]^c32[891]^c32[1113]^c32[1111]^c32[1112]^c32[868]^c32[1844]^c32[1843]^c32[1870]^c32[1869]^c32[1738]^c32[1737]^c32[2547]^c32[2546]^c32[2635]^c32[2634]^c32[2562]^c32[2560]^c32[2561]^c32[3234]^c32[3233]^c32[3278]^c32[3277]^c32[3221]^c32[3219]^c32[3220]^c32[3895]^c32[3894]^c32[3858]^c32[3857]^c32[3970]^c32[3969]^c32[4698]^c32[4697]^c32[4643]^c32[4642]^c32[4681]^c32[4679]^c32[4680]^c32[5581]^c32[5580]^c32[5517]^c32[5516]^c32[5403]^c32[5401]^c32[5402]^c32[5591]^c32[6349]^c32[6348]^c32[6178]^c32[6177]^c32[6458]^c32[6456]^c32[6457]^c32[7103]^c32[7102]^c32[6964]^c32[6963]^c32[7233]^c32[7232]^c32[7842]^c32[7841]^c32[7683]^c32[7682]^c32[7686]^c32[7684]^c32[7685]^c32[8815]^c32[8814]^c32[8617]^c32[8616]^c32[8627]^c32[8625]^c32[8626]^c32[9491]^c32[9490]^c32[9432]^c32[9431]^c32[9321]^c32[9319]^c32[9320]^c32[10215]^c32[10214]^c32[10058]^c32[10057]^c32[10199]^c32[10197]^c32[10198]^c32[10843]^c32[10842]^c32[11098]^c32[11097]^c32[10838]^c32[10836]^c32[10837]^c32[11609]^c32[11608]^c32[11877]^c32[11876]^c32[11831]^c32[11830]^c32[12652]^c32[12651]^c32[12516]^c32[12515]^c32[12466]^c32[12464]^c32[12465]^c32[13428]^c32[13427]^c32[13061]^c32[13060]^c32[13230]^c32[13228]^c32[13229]^c32[13940]^c32[13939]^c32[13938]^c32[13937]^c32[13969]^c32[13967]^c32[13968]^c32[14646]^c32[14645]^c32[14797]^c32[14796]^c32[14671]^c32[14670]^c32[15564]^c32[15563]^c32[15631]^c32[15630]^c32[15657]^c32[15655]^c32[15656]^c32[16348]^c32[16347]^c32[16498]^c32[16497]^c32[16359]^c32[16357]^c32[16358];
+
+       break;
+    case 32:
+      d32[12288]=c32[211]^c32[364]^c32[108]^c32[109]^c32[179]^c32[180]^c32[102]^c32[1075]^c32[921]^c32[922]^c32[1142]^c32[1143]^c32[1874]^c32[1900]^c32[1767]^c32[1768]^c32[2577]^c32[2665]^c32[2591]^c32[2592]^c32[3264]^c32[3307]^c32[3308]^c32[3250]^c32[3251]^c32[3925]^c32[3888]^c32[3999]^c32[4000]^c32[4728]^c32[4672]^c32[4673]^c32[4710]^c32[4711]^c32[5611]^c32[5546]^c32[5547]^c32[5432]^c32[5433]^c32[6379]^c32[6207]^c32[6208]^c32[6487]^c32[6488]^c32[7133]^c32[6994]^c32[7262]^c32[7263]^c32[7872]^c32[7712]^c32[7713]^c32[7715]^c32[7716]^c32[8461]^c32[8647]^c32[8656]^c32[8657]^c32[9521]^c32[9462]^c32[9350]^c32[9351]^c32[9417]^c32[10245]^c32[10087]^c32[10088]^c32[10228]^c32[10229]^c32[10873]^c32[11127]^c32[11128]^c32[10867]^c32[10868]^c32[10927]^c32[11639]^c32[11523]^c32[11860]^c32[11861]^c32[12298]^c32[12546]^c32[12495]^c32[12496]^c32[13074]^c32[13090]^c32[13091]^c32[13259]^c32[13260]^c32[13970]^c32[13967]^c32[13968]^c32[13998]^c32[13999]^c32[14676]^c32[14827]^c32[14700]^c32[14701]^c32[15594]^c32[15660]^c32[15661]^c32[15686]^c32[15687]^c32[16378]^c32[16144]^c32[16388]^c32[16389];
+
+       break;
+    case 33:
+      d32[12672]=c32[1091]^c32[1544]^c32[8809]^c32[16233];
+
+       break;
+    case 34:
+      d32[13056]=c32[230]^c32[5524]^c32[11722]^c32[13368];
+
+       break;
+    case 35:
+      d32[13440]=c32[189]^c32[342]^c32[87]^c32[158]^c32[1053]^c32[900]^c32[1121]^c32[1088]^c32[1852]^c32[1878]^c32[1746]^c32[2555]^c32[2643]^c32[2570]^c32[3242]^c32[3286]^c32[3229]^c32[3903]^c32[3866]^c32[3978]^c32[4706]^c32[4651]^c32[4689]^c32[4943]^c32[5589]^c32[5525]^c32[5411]^c32[6357]^c32[6186]^c32[6466]^c32[7111]^c32[6972]^c32[7241]^c32[7850]^c32[7691]^c32[7694]^c32[8823]^c32[8625]^c32[8635]^c32[9499]^c32[9440]^c32[9329]^c32[9218]^c32[10223]^c32[10066]^c32[10207]^c32[10851]^c32[11106]^c32[10846]^c32[11617]^c32[11885]^c32[11839]^c32[12660]^c32[12524]^c32[12474]^c32[13436]^c32[13069]^c32[13238]^c32[13948]^c32[13946]^c32[13977]^c32[14654]^c32[14805]^c32[14679]^c32[15572]^c32[15639]^c32[15665]^c32[16356]^c32[16506]^c32[16367];
+
+       break;
+    case 36:
+      d32[13824]=c32[210]^c32[11065]^c32[11817]^c32[13845];
+
+       break;
+    case 37:
+      d32[14208]=c32[38]^c32[39]^c32[192]^c32[321]^c32[8]^c32[902]^c32[903]^c32[1134]^c32[971]^c32[1037]^c32[1701]^c32[1702]^c32[1728]^c32[1596]^c32[2404]^c32[2405]^c32[2493]^c32[2420]^c32[3092]^c32[3136]^c32[3079]^c32[4136]^c32[4137]^c32[4100]^c32[4212]^c32[4939]^c32[4940]^c32[4885]^c32[4923]^c32[5439]^c32[5759]^c32[5645]^c32[6207]^c32[6420]^c32[6316]^c32[6960]^c32[6961]^c32[7206]^c32[7091]^c32[7699]^c32[7700]^c32[7925]^c32[7928]^c32[8672]^c32[8673]^c32[8475]^c32[8485]^c32[9348]^c32[9349]^c32[9290]^c32[9563]^c32[10072]^c32[10073]^c32[10300]^c32[10057]^c32[10066]^c32[11085]^c32[10956]^c32[11080]^c32[11850]^c32[11851]^c32[11735]^c32[11689]^c32[12509]^c32[12510]^c32[12374]^c32[12324]^c32[13286]^c32[13303]^c32[13088]^c32[14181]^c32[14182]^c32[14180]^c32[13827]^c32[14887]^c32[14888]^c32[14655]^c32[14913]^c32[15421]^c32[15422]^c32[15489]^c32[15515]^c32[16205]^c32[16206]^c32[16356]^c32[16217];
+
+       break;
+    case 38:
+      d32[14592]=c32[185]^c32[7089]^c32[7969]^c32[9430];
+
+       break;
+    case 39:
+      d32[14976]=c32[1026]^c32[2397]^c32[5722]^c32[14889];
+
+       break;
+    case 40:
+      d32[15360]=c32[175]^c32[6181]^c32[13368];
+
+       break;
+    case 41:
+      d32[15744]=c32[820]^c32[2618]^c32[7051]^c32[14112];
+
+       break;
+    case 42:
+      d32[16128]=c32[142]^c32[295]^c32[39]^c32[40]^c32[110]^c32[111]^c32[113]^c32[1006]^c32[852]^c32[853]^c32[1073]^c32[1074]^c32[1805]^c32[1831]^c32[1698]^c32[1699]^c32[2508]^c32[2596]^c32[2522]^c32[2523]^c32[3195]^c32[3238]^c32[3239]^c32[3181]^c32[3182]^c32[3086]^c32[3856]^c32[4203]^c32[3930]^c32[3931]^c32[4659]^c32[4987]^c32[4988]^c32[4641]^c32[4642]^c32[5542]^c32[5477]^c32[5478]^c32[5747]^c32[5748]^c32[6310]^c32[6522]^c32[6523]^c32[6418]^c32[6419]^c32[7064]^c32[6925]^c32[7193]^c32[7194]^c32[7803]^c32[8027]^c32[8028]^c32[8030]^c32[8031]^c32[8776]^c32[8578]^c32[8587]^c32[8588]^c32[9452]^c32[9393]^c32[9281]^c32[9282]^c32[10176]^c32[10018]^c32[10019]^c32[10159]^c32[10160]^c32[10804]^c32[11058]^c32[11059]^c32[10798]^c32[10799]^c32[11570]^c32[11838]^c32[11791]^c32[11792]^c32[12613]^c32[12477]^c32[12426]^c32[12427]^c32[13389]^c32[13405]^c32[13406]^c32[13190]^c32[13191]^c32[13901]^c32[13898]^c32[13899]^c32[13929]^c32[13930]^c32[14607]^c32[14758]^c32[14631]^c32[14632]^c32[15525]^c32[15591]^c32[15592]^c32[15617]^c32[15618]^c32[16309]^c32[16459]^c32[16319]^c32[16320];
+
+       break;
+    case 43:
+      d32[16512]=c32[92]^c32[245]^c32[374]^c32[60]^c32[61]^c32[956]^c32[803]^c32[1023]^c32[1024]^c32[881]^c32[1755]^c32[1781]^c32[1649]^c32[2458]^c32[2546]^c32[2472]^c32[2473]^c32[3145]^c32[3189]^c32[3131]^c32[3132]^c32[4190]^c32[4153]^c32[3881]^c32[4609]^c32[4938]^c32[4975]^c32[4976]^c32[5492]^c32[5428]^c32[5697]^c32[5698]^c32[6260]^c32[6473]^c32[6368]^c32[6369]^c32[7014]^c32[7259]^c32[7144]^c32[7753]^c32[7978]^c32[7980]^c32[7981]^c32[8726]^c32[8528]^c32[8537]^c32[8538]^c32[9402]^c32[9343]^c32[9231]^c32[9232]^c32[10126]^c32[10353]^c32[10109]^c32[10110]^c32[10754]^c32[11009]^c32[11132]^c32[11133]^c32[11520]^c32[11788]^c32[11742]^c32[12563]^c32[12427]^c32[12376]^c32[12377]^c32[12420]^c32[13339]^c32[13356]^c32[13140]^c32[13141]^c32[13851]^c32[13849]^c32[13879]^c32[13880]^c32[13938]^c32[14941]^c32[14708]^c32[14966]^c32[15475]^c32[15542]^c32[15567]^c32[15568]^c32[16259]^c32[16409]^c32[16269]^c32[16270];
+
+       break;
+    case 44:
+      d32[16896]=c32[197]^c32[350]^c32[95]^c32[166]^c32[80]^c32[1061]^c32[908]^c32[1129]^c32[1860]^c32[1886]^c32[1754]^c32[2563]^c32[2651]^c32[2578]^c32[3250]^c32[3294]^c32[3237]^c32[3911]^c32[3874]^c32[3986]^c32[4714]^c32[4659]^c32[4697]^c32[5597]^c32[5533]^c32[5419]^c32[5454]^c32[6365]^c32[6194]^c32[6474]^c32[7119]^c32[6980]^c32[7249]^c32[7075]^c32[7858]^c32[7699]^c32[7702]^c32[8831]^c32[8633]^c32[8643]^c32[9507]^c32[9448]^c32[9337]^c32[10231]^c32[10074]^c32[10215]^c32[10859]^c32[11114]^c32[10854]^c32[11625]^c32[11893]^c32[11847]^c32[12668]^c32[12532]^c32[12482]^c32[13060]^c32[13077]^c32[13246]^c32[13956]^c32[13954]^c32[13985]^c32[14662]^c32[14813]^c32[14687]^c32[15580]^c32[15647]^c32[15673]^c32[16364]^c32[16130]^c32[16375];
+
+       break;
+    case 45:
+      d32[17280]=c32[903]^c32[4757]^c32[7695];
+
+       break;
+    
+    }
+  }
+}
+extern "C" int ldpc_BG1_Zc384_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs) { 
+
+ dim3 numblocks(n_inputs,46);
+ ldpc_BG1_Zc384_worker<<<numblocks,384>>>(c,d);
+ cudaError_t err=cudaPeekAtLastError();
+ if (err!=cudaSuccess) {
+    printf("cuda error: %s (c %p, d %p)\n",cudaGetErrorString(err),c,d);
+    exit(-1);
+ }
+ cudaDeviceSynchronize();
+ return(0);
+}

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c

@@ -103,9 +103,11 @@
 #include "ldpc_BG2_Zc80_byte.c"
 #include "ldpc_BG2_Zc72_byte.c"
 
-static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols)
+
+static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols,time_stats_t *tinput_memcpy)
 {
   unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c
+  if (tinput_memcpy) start_meas(tinput_memcpy);
   for (int i1 = 0; i1 < ncols; i1++)   {
     memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
     memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
@@ -118,6 +120,7 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho
       memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1);
     }
   }
+  if (tinput_memcpy) stop_meas(tinput_memcpy);
   if (BG == 1) {
     switch (Zc) {
       case 176:

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check_cuda.c

@@ -0,0 +1,94 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*!\file ldpc_encode_parity_check.c
+ * \brief Parity check function used by ldpc encoders
+ * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
+ * \email openair_tech@eurecom.fr
+ * \date 27-03-2018
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "assertions.h"
+#include "common/utils/LOG/log.h"
+#include <cuda_runtime.h>
+
+
+
+int ldpc_BG1_Zc384_cuda32(uint32_t **c,uint32_t **d,int n_inputs);
+
+
+void encode_parity_check_part_cuda(uint32_t **c, uint32_t **d, short BG,short Zc,short Kb, int ncols, int n_inputs)
+{
+  
+  if (BG == 1) {
+    switch (Zc) {
+      case 176:
+      case 192:
+      case 208:
+      case 224:
+      case 240:
+      case 256:
+      case 288:
+      case 320:
+      case 352:
+	AssertFatal(1==0,"BG %d Zc %d not supported yet for CUDA\n",BG, Zc);
+        break;
+      case 384:
+	ldpc_BG1_Zc384_cuda32(c, d, n_inputs);
+        break;
+      default:
+        AssertFatal(false, "BG %d Zc %d is not supported yet\n", BG, Zc);
+    }
+  } else if (BG == 2) {
+    switch (Zc) {
+      case 72:
+      case 80:
+      case 88:
+      case 96:
+      case 104:
+      case 112:
+      case 120:
+      case 128:
+      case 144:
+      case 160:
+      case 176:
+      case 192:
+      case 208:
+      case 224:
+      case 240:
+      case 256:
+      case 288:
+      case 320:
+      case 352:
+      case 384:
+      default:
+        AssertFatal(false , "BG %d Zc %d is not supported yet\n", BG, Zc);
+    }
+  } else
+    AssertFatal(false, "BG %d is not supported\n", BG);
+}

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c

@@ -58,11 +58,10 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
   int no_punctured_columns, removed_bit;
   int nind=0;
   int indlist[1000];
-  int indlist2[1000];
 
   const short *Gen_shift_values = choose_generator_matrix(BG, Zc);
   if (Gen_shift_values==NULL) {
-    printf("ldpc_encoder_orig: could not find generator matrix\n");
+    printf("ldpc_encoder_orig: could not find generator matrix BG %d Zc %d\n",BG,Zc);
     return(-1);
   }
 
@@ -93,32 +92,35 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
 
   // parity check part
 
-  if (gen_code>=1)
+  if (gen_code==1 || gen_code==2 || gen_code==3)
   {
     char fname[100];
     sprintf(fname,"ldpc_BG%d_Zc%d_byte.c",BG,Zc);
     FILE *fd=fopen(fname,"w");
     AssertFatal(fd!=NULL,"cannot open %s\n",fname);
-    sprintf(fname,"ldpc_BG%d_Zc%d_16bit.c",BG,Zc);
-    FILE *fd2=fopen(fname,"w");
-    AssertFatal(fd2!=NULL,"cannot open %s\n",fname);
+
 
     int shift;
     char data_type[100];
     char xor_command[100];
     int mask;
-
-
+    char permutex_command[100];
+    int use_permutex=0;
+    char alignr_command[100];
+    int use_alignr=0;
 
 
     fprintf(fd,"#include \"PHY/sse_intrin.h\"\n");
-    fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n");
+
 
     if (gen_code == 1 && (Zc&63)==0) {
       shift=6;
       mask=63;
       strcpy(data_type,"__m512i");
       strcpy(xor_command,"_mm512_xor_si512");
+      strcpy(permutex_command,"_mm512_permutex2var_epi8");
+      strcpy(alignr_command,"_mm512_alignr_epi8");
+      use_alignr=1;
     }
     else if (gen_code == 1 && (Zc&31)==0) {
       shift=5; // AVX2 - 256-bit SIMD
@@ -126,14 +128,15 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
       strcpy(data_type,"simde__m256i");
       strcpy(xor_command,"simde_mm256_xor_si256");
     }
-    else if ((Zc&15)==0) {
+    else if ((gen_code <=2) && (Zc&15)==0) {
       shift=4; // SSE4 - 128-bit SIMD
       mask=15;
       strcpy(data_type,"simde__m128i");
       strcpy(xor_command,"simde_mm_xor_si128");
-
+      strcpy(alignr_command,"simde_mm_alignr_epi8");
+      use_alignr=1;
     }
-    else if ((Zc&7)==0) {
+    else if ((gen_code <=2) && (Zc&7)==0) {
       shift=3; // MMX  - 64-bit SIMD
       mask=7;
       strcpy(data_type,"simde__m64");
@@ -145,21 +148,30 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
       strcpy(data_type,"uint8_t");
       strcpy(xor_command,"scalar_xor");
       fprintf(fd,"#define scalar_xor(a,b) ((a)^(b))\n");
-      fprintf(fd2,"#define scalar_xor(a,b) ((a)^(b))\n");
+
     }
     fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
-    fprintf(fd2,"// generated code for Zc=%d, 16bit encoding\n",Zc);
+    if (use_permutex==1) {
+	 fprintf(fd,"  const uint8_t abshift_bytes[%d][%d] __attribute__((aligned(64)))= {\n",1+mask,1+mask); 
+         for (int i=0;i<=mask;i++) {
+	    fprintf(fd,"{");
+	    for (int j=0;j<=mask;j++) {
+              fprintf(fd,"%d",i+j);
+	      if (j<mask) fprintf(fd,",");
+	      else fprintf(fd,"}");
+	    }
+	    if (i<mask) fprintf(fd,",\n");
+	    else fprintf(fd,"};\n");
+      	}
+	fprintf(fd,"static inline __m512i abshift_get(int r)\n {\n");
+        fprintf(fd,"   return * (const __m512i *) abshift_bytes[r];\n}");
+    }
     fprintf(fd,"static inline void ldpc_BG%d_Zc%d_byte(uint8_t *c,uint8_t *d) {\n",BG,Zc);
-    fprintf(fd2,"static inline void ldpc_BG%d_Zc%d_16bit(uint16_t *c,uint16_t *d) {\n",BG,Zc);
     fprintf(fd,"  %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
-    fprintf(fd2,"  %s *csimd=(%s *)c,*dsimd=(%s *)d;\n\n",data_type,data_type,data_type);
     fprintf(fd,"  %s *c2,*d2;\n\n",data_type);
-    fprintf(fd2,"  %s *c2,*d2;\n\n",data_type);
     fprintf(fd,"  int i2;\n");
-    fprintf(fd2,"  int i2;\n");
     fprintf(fd,"  for (i2=0; i2<%d; i2++) {\n",Zc>>shift);
-    if (shift > 0)
-      fprintf(fd2,"  for (i2=0; i2<%d; i2++) {\n",Zc>>(shift-1));
+
     for (i2=0; i2 < 1; i2++)
     {
       //t=Kb*Zc+i2;
@@ -169,53 +181,122 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
 
       fprintf(fd,"     c2=&csimd[i2];\n");
       fprintf(fd,"     d2=&dsimd[i2];\n");
-      fprintf(fd2,"     c2=&csimd[i2];\n");
-      fprintf(fd2,"     d2=&dsimd[i2];\n");
 
       for (i1 = 0; i1 < nrows; i1++)
-
       {
         fprintf(fd,"\n//row: %d\n",i1);
-        fprintf(fd2,"\n//row: %d\n",i1);
-        AssertFatal(shift > 0 , "The result of the right shift is undefined because the right operand is negative\n");
+        AssertFatal(shift >= 0 , "The result of the right shift is undefined because the right operand is negative\n");
 	fprintf(fd,"     d2[%d]=",(Zc*i1)>>shift);
-	fprintf(fd2,"     d2[%d]=",(Zc*i1)>>(shift-1));
 
         nind=0;
 
         for (i3=0; i3 < ncols; i3++)
         {
           temp_prime=i1 * ncols + i3;
-
-
 	  for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
-	    {
-	          
+  	  {
 	      var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
 	      int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
 	      printf("var %d, i3 %d, i4 %d, index %d, shift %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d\n",var,i3,i4,index,shift,Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]]);
-	      indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
-	      printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
-	      indlist2[nind++] = ((index&(mask>>1))*((2*Zc*ncols)>>(shift-1))*Kb)+(index>>(shift-1));
-	    }
-	  
-
-        }
+   	      printf("indlist[%d] %d, index&mask %d, index>>shift %d\n",nind,indlist[nind],index&mask,index>>shift);
+	      if (use_permutex==1 || use_alignr==1) {
+                indlist[nind++] = index;
+	      }
+	      else {
+	        indlist[nind] = ((index&mask)*((2*Zc*ncols)>>shift)/* *Kb */)+(index>>shift);
+	      }
+	  }//i4
+        }//i3
 	for (i4=0;i4<nind-1;i4++) {
-	  fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
-	  fprintf(fd2,"%s(c2[%d],",xor_command,indlist2[i4]);
+           if (use_permutex==0 && use_alignr==0) {
+  	      fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]);
+	   }
+	   else {
+              if ((indlist[i4]&mask) == 0) {
+  	        fprintf(fd,"%s(c2[%d],",xor_command,indlist[i4]>>shift);
+	      }
+	      else if (use_permutex==1) {
+                fprintf(fd,"%s(%s(c2[%d],abshift_get(%d),c2[%d]),",xor_command,permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
+	      }
+	      else if (use_alignr==1) {
+                fprintf(fd,"%s(%s(c2[%d],c2[%d],%d),",xor_command,alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
+	      }
+	   }
+	} //i4
+	if (use_permutex==0 && use_alignr==0)
+	  fprintf(fd,"c2[%d]",indlist[i4]);
+	else {
+          if ((indlist[i4]&mask) == 0) {
+  	     fprintf(fd,"c2[%d]",indlist[i4]>>shift);
+	  }
+	  else if (use_permutex==1){
+             fprintf(fd,"%s(c2[%d],abshift_get(%d),c2[%d])",permutex_command,(indlist[i4]>>shift),indlist[i4]&mask,(indlist[i4]>>shift)+1);
+	  }
+	  else if (use_alignr==1) {
+             fprintf(fd,"%s(c2[%d],c2[%d],%d)",alignr_command,(indlist[i4]>>shift)+1,(indlist[i4]>>shift),indlist[i4]&mask);
+
+	  }
 	}
-	fprintf(fd,"c2[%d]",indlist[i4]);
-	fprintf(fd2,"c2[%d]",indlist2[i4]);
-	for (i4=0;i4<nind-1;i4++) { fprintf(fd,")"); fprintf(fd2,")"); }
-	fprintf(fd,";\n");
-  fprintf(fd2, ";\n");
-      }
-      fprintf(fd,"  }\n}\n");
-      fprintf(fd2,"  }\n}\n");
-    }
+	for (i4=0;i4<nind-1;i4++) fprintf(fd,")"); 
+  	fprintf(fd,";\n");
+       } // i1 
+       fprintf(fd,"  }\n}\n");
+    } // i2
+    fclose(fd);
+  }
+  else if (gen_code == 4) { // CUDA
+    char fname[100];
+    sprintf(fname,"ldpc_BG%d_Zc%d_32bit.cu",BG,Zc);
+    FILE *fd=fopen(fname,"w");
+    AssertFatal(fd!=NULL,"cannot open %s\n",fname);
+    printf("Writing to %s\n",fname);
+    fprintf(fd,"#include <stdio.h>\n#include <stdint.h>\n#include <cuda_runtime.h>\n");
+
+    fprintf(fd,"// generated code for Zc=%d, byte encoding\n",Zc);
+    fprintf(fd,"__global__ void ldpc_BG%d_Zc%d_worker(uint32_t *c[4],uint32_t *d[4]) {\n",BG,Zc);
+    fprintf(fd,"  uint32_t *c32=c[blockIdx.x];\n  uint32_t *d32=d[blockIdx.x];\n\n");
+    fprintf(fd,"  int i2 = threadIdx.x;\n");
+    fprintf(fd,"  int i1 = blockIdx.y;\n");
+    fprintf(fd,"  if (i2 < %d) {\n",Zc);
+    fprintf(fd,"    c32+=i2;\n");
+    fprintf(fd,"    d32+=i2;\n");
+    fprintf(fd,"    switch(i1) {\n");
+
+    for (int i1=0;i1<nrows;i1++) {
+      nind = 0;
+      fprintf(fd,"    case %d:\n",i1);
+      fprintf(fd,"      d32[%d]=",(Zc*i1));
+      for (i3=0; i3 < ncols; i3++)
+      {
+          temp_prime=i1 * ncols + i3;
+	  for (i4=0; i4 < no_shift_values[temp_prime]; i4++)
+  	  {
+	     var=(int)((i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc)/Zc);
+  	     int index =var*2*Zc + (i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc;
+	     printf("var %d, i3 %d, i4 %d, index %d (index mod 2Zc) %d, Zc %d, pointer_shift_values[%d] %d gen_shift_value %d offset %d\n",var,i3,i4,index,index%(2*Zc),Zc,temp_prime,pointer_shift_values[temp_prime],Gen_shift_values[pointer_shift_values[temp_prime]],(i3*Zc + (Gen_shift_values[ pointer_shift_values[temp_prime]+i4 ]+1)%Zc) % Zc);
+	     if (index%(2*Zc) >= Zc) printf("***********************\n");
+   	     indlist[nind] = index;
+	     printf("indlist[%d] %d, index %d\n",nind,indlist[nind],index);
+	     nind++;
+   	  } //i4
+      } // i3
+      for (i4=0;i4<nind-1;i4++) {
+         fprintf(fd,"c32[%d]^",indlist[i4]);
+      } //i4
+      fprintf(fd,"c32[%d];\n\n",indlist[i4]);
+      fprintf(fd,"       break;\n");
+    }// i1
+    fprintf(fd,"     }\n");
+    fprintf(fd,"  }\n");
+    fprintf(fd,"}\n");
+
+    fprintf(fd,"extern \"C\" int ldpc_BG%d_Zc%d_cuda32(uint32_t *c[4],uint32_t *d[4],int n_inputs) { \n",BG,Zc);
+    fprintf(fd,"dim3 numblocks(n_inputs,%d);\n",nrows);
+    fprintf(fd,"ldpc_BG%d_Zc%d_worker<<<numblocks,%d>>>(c,d);\n",BG,Zc,Zc);
+    fprintf(fd," cudaDeviceSynchronize();\n");
+    fprintf(fd,"  return(0);\n");
+    fprintf(fd,"}\n");
     fclose(fd);
-    fclose(fd2);
   }
   else if(gen_code==0)
   {
@@ -241,10 +322,12 @@ int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_
 
           for (i4 = 0; i4 < no_shift_values[temp_prime]; i4++) {
             channel_temp = channel_temp ^ c[i3 * Zc + Gen_shift_values[pointer_shift_values[temp_prime] + i4]];
+	    //if (i1==0) printf("index %d\n",i3 * Zc + Gen_shift_values[pointer_shift_values[temp_prime] + i4]);
           }
         }
 
         d[i2+i1*Zc]=channel_temp;
+	//if (i1==0) printf("reference: d[%d] %d c[%d] %d\n",i2,d[i2],i2,c[i2]);
         // output[t+i1*Zc]=channel_temp;
       }
     }

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder32.c

@@ -0,0 +1,52 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*!\file ldpc_encoder.c
+ * \brief Defines the LDPC encoder
+ * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
+ * \email openair_tech@eurecom.fr
+ * \date 27-03-2018
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "defs.h"
+#include "assertions.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+#include "ldpc_generate_coefficient.c"
+
+
+void cuda_support_init() {
+   return;
+}
+uint32_t **LDPCencoder32(uint8_t **input,encoder_implemparams_t *impp)
+{
+	AssertFatal(1==0,"Should not be getting here\n");
+}
+

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_cuda32.c

@@ -0,0 +1,791 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+/*! \file ldpc_encoder_optim8segmulti.c
+ * \brief Defines the optimized LDPC encoder
+ * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
+ * \email openair_tech@eurecom.fr
+ * \date 27-03-2018
+ * \version 1.0
+ * \note
+ * \warning
+ */
+
+#include <stdlib.h>
+#include <math.h>
+#include <stdio.h>
+#include <string.h>
+#include "assertions.h"
+#include "common/utils/LOG/log.h"
+#include "time_meas.h"
+#include "openair1/PHY/CODING/nrLDPC_defs.h"
+#include "PHY/sse_intrin.h"
+#include "openair1/PHY/CODING/nrLDPC_extern.h"
+
+#include "ldpc_generate_coefficient.c"
+
+#include <cuda_runtime.h>
+
+//#define DEBUG_LDPC 1 
+
+#include "ldpc_encode_parity_check_cuda.c"
+uint32_t *c_dev;
+uint32_t **c_host;
+uint32_t *c_devh[4];
+uint32_t *d_dev;
+uint32_t **d_host;
+uint32_t *d_devh[4];
+uint32_t *cc_dev;
+uint32_t **cc_host;
+uint32_t *cc_devh[4];
+uint32_t *input_dev;
+uint32_t **input_host;
+uint32_t *input_devh[128];
+int managed = 0, concurrent = 0, uva = 0, pageable = 0, pageable_uses_host = 0, register_host = 0;
+
+//#define USE_GPU_CIRCCOPY 1
+//#define USE_GPU_FOR_INPUT 1
+
+void cuda_support_init() {
+
+
+    int dev = 0;
+    struct cudaDeviceProp prop;
+    cudaGetDeviceProperties(&prop, dev);
+
+
+    cudaDeviceGetAttribute(&managed, cudaDevAttrManagedMemory, dev);
+    cudaDeviceGetAttribute(&concurrent, cudaDevAttrConcurrentManagedAccess, dev);
+    cudaDeviceGetAttribute(&uva, cudaDevAttrUnifiedAddressing, dev);
+    cudaDeviceGetAttribute(&pageable, cudaDevAttrPageableMemoryAccess, dev);
+    cudaDeviceGetAttribute(&pageable_uses_host, cudaDevAttrPageableMemoryAccessUsesHostPageTables, dev);
+    cudaDeviceGetAttribute(&register_host, cudaDevAttrHostRegisterSupported,dev);
+
+    LOG_I(NR_PHY,"Device: %s (cc %d.%d)\n", prop.name, prop.major, prop.minor);
+    LOG_I(NR_PHY,"Unified Virtual Addressing (UVA): %s\n", uva ? "YES" : "NO");
+    LOG_I(NR_PHY,"Managed (Unified) Memory:        %s\n", managed ? "YES" : "NO");
+    LOG_I(NR_PHY,"Concurrent managed access:       %s\n", concurrent ? "YES" : "NO");
+    LOG_I(NR_PHY,"Pageable memory access:          %s\n", pageable ? "YES" : "NO");
+    LOG_I(NR_PHY,"Uses host page tables:           %s\n", pageable_uses_host ? "YES" : "NO");
+    LOG_I(NR_PHY,"Host Register supported:         %s\n", register_host ? "YES" : "NO");
+
+  // initialize input and output memory
+  if (!pageable && !register_host) {
+    cudaError_t err=cudaMalloc((void **)&c_dev,4*sizeof(uint32_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<4;i++) {
+      err=cudaMalloc((void**)&c_devh[i],2*22*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (chost[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err = cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaMalloc((void**)&d_dev,4*sizeof(uint32_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<4;i++) {
+      err=cudaMalloc((void**)&d_devh[i],68*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaMalloc((void**)&cc_dev,4*sizeof(uint32_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&cc_host,4*sizeof(uint32_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<4;i++) {
+      err=cudaMalloc((void**)&cc_devh[i],68*384*sizeof(uint32_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (cc_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&cc_host[i],68*384*sizeof(uint32_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cc_host[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(cc_dev,cc_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy cc_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaMalloc((void**)&input_dev,128*sizeof(uint8_t*));
+    AssertFatal(err == cudaSuccess,"CUDA Error: %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocDefault);
+    AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
+    for (int i=0;i<128;i++) {
+      err=cudaMalloc((void**)&input_devh[i],(8448/8)*sizeof(uint8_t));
+      AssertFatal(err == cudaSuccess,"CUDA Error (input_devh[%d]: %s\n", i,cudaGetErrorString(err));
+      err=cudaHostAlloc((void**)&input_host[i],(8448/8)*sizeof(uint8_t),cudaHostAllocDefault);
+      AssertFatal(err == cudaSuccess,"CUDA Error (input_host[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(input_dev,input_devh,128*sizeof(uint8_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy cc_devh -> d_dev): %s\n", cudaGetErrorString(err));
+  }
+  else {
+    LOG_I(NR_PHY,"Allocating c,d,cc arrays for CPU/GPU shared-memory\n");
+    cudaError_t err=cudaHostAlloc((void **)&c_host,4*sizeof(uint32_t*),cudaHostAllocMapped|cudaHostAllocPortable);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_host): %s\n", cudaGetErrorString(err));
+    err = cudaHostGetDevicePointer((void**)&c_dev, c_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error (c_dev): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"c_host %p, c_dev %p\n",c_host,c_dev);
+    for (int i=0;i<4;i++) {
+      err=cudaHostAlloc((void**)&c_host[i],2*22*384*sizeof(uint32_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err = cudaHostGetDevicePointer((void**)&c_devh[i], c_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (c_devh[%d]): %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(c_dev,c_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> c_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&d_host,4*sizeof(uint32_t*),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (d_host): %s\n", cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&d_dev, d_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(d_dev): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"d_host %p, d_dev %p\n",d_host,d_dev);
+    for (int i=0;i<4;i++) {
+      err=cudaHostAlloc((void**)&d_host[i],68*384*sizeof(uint32_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (d_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&d_devh[i], d_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) d_devh[%d]: %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(d_dev,d_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&cc_host,4*sizeof(uint32_t*),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (cc_host): %s\n", cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&cc_dev, cc_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(cc_dev): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"cc_host %p, cc_dev %p\n",cc_host,cc_dev);
+    for (int i=0;i<4;i++) {
+      err=cudaHostAlloc((void**)&cc_host[i],68*384*sizeof(uint32_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cc_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&cc_devh[i], cc_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) cc_devh[%d]: %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(cc_dev,cc_devh,4*sizeof(uint32_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy d_devh -> d_dev): %s\n", cudaGetErrorString(err));
+    err=cudaHostAlloc((void **)&input_host,128*sizeof(uint8_t*),cudaHostAllocMapped);
+    AssertFatal(err == cudaSuccess,"CUDA Error (input_host): %s\n", cudaGetErrorString(err));
+    err=cudaHostGetDevicePointer((void**)&input_dev, input_host, 0);
+    AssertFatal(err == cudaSuccess,"CUDA Error cudaHostGetDevicePointer(cc_host): %s\n", cudaGetErrorString(err));
+    LOG_I(NR_PHY,"input_host %p, input_dev %p\n",input_host,input_dev);
+    /*
+    for (int i=0;i<128;i++) {
+      err=cudaHostAlloc((void**)&input_host[i],(8448/8)*sizeof(uint8_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (input_host[%d]): %s\n", i,cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&input_devh[i], input_host[i], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) input_devh[%d]: %s\n", i,cudaGetErrorString(err));
+    }
+    err=cudaMemcpy(input_dev,input_devh,128*sizeof(uint8_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy input_devh -> input_dev): %s\n", cudaGetErrorString(err));
+    */
+  }
+}
+
+uint32_t **LDPCencoder32(uint8_t **input, encoder_implemparams_t *impp)
+{
+  //set_log(PHY, 4);
+
+  int Zc = impp->Zc;
+  int Kb = impp->Kb;
+  short block_length = impp->K;
+  short BG = impp->BG;
+  int nrows=0,ncols=0;
+  int rate=3;
+  int no_punctured_columns,removed_bit;
+
+  if(impp->tinput != NULL) start_meas(impp->tinput);
+  //determine number of bits in codeword
+  if (BG==1)
+    {
+      nrows=46; //parity check bits
+      ncols=22; //info bits
+      rate=3;
+    }
+    else if (BG==2)
+    {
+      nrows=42; //parity check bits
+      ncols=10; // info bits
+      rate=5;
+    }
+
+#ifdef DEBUG_LDPC
+  LOG_I(PHY,"ldpc_encoder_cuda32: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,impp->n_segments);
+  LOG_I(PHY,"ldpc_encoder_cuda32: PDU (seg 0) %x %x %x %x\n",input[0][0],input[0][1],input[0][2],input[0][3]);
+#endif
+
+  AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length);
+
+  int n_inputs = (impp->n_segments/32)+(((impp->n_segments&31) > 0) ? 1: 0);
+//  uint32_t  cc[4][22*Zc]; //padded input, unpacked, max size
+
+  // calculate number of punctured bits
+  no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
+  removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate);
+  // clear input
+  /*
+  for (int i=0;i<n_inputs;i++) { 
+    if (!pageable && !register_host)
+      cudaMemset(cc_devh[i],0,22*Zc*sizeof(uint32_t));
+    else
+      memset(cc_host[i],0,22*Zc*sizeof(uint32_t));
+  }*/
+
+
+#if 0
+  // unoptimized version of input processing
+  unsigned int temp;
+  for (int i_dword=0; i_dword < block_length; i_dword++) {
+    unsigned int i = i_dword;
+    for (int j = 0; j < impp->n_segments; j++) {
+
+      temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
+      cc[j>>5][i] |= (temp << (j&31));
+    }
+  }
+#else
+//  uint32_t *ccp[4];
+//  for (int s=0;s<n_inputs;s++) ccp[s]=cc[s];  
+#ifdef USE_GPU_FOR_INPUT
+  if (!pageable && !register_host) {
+    for (int r=0;r<impp->n_segments;r++) {
+        cudaMemcpy(input_devh[r],input[r],block_length>>3,cudaMemcpyHostToDevice);
+    }
+  }
+  ldpc_input(pageable||register_host? input : input_dev,(uint32_t**)cc_dev,block_length,impp->n_segments);
+//  for (int i=0;i<16;i++) printf("i %d: cc[0] %x\n",i,cc[0][i]);
+#else  
+#ifndef __aarch64__
+  simde__m256i andmask = simde_mm256_set_epi32(0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80);  // every 8 bits -> 8 bytes, pattern repeats.
+  simde__m256i zero256 = simde_mm256_setzero_si256();
+  simde__m256i masks[32];
+  register simde__m256i c256;
+  for (int i=0;i<32;i++) {
+    masks[i] = simde_mm256_set1_epi32(1<<i);
+  }
+
+  int i;
+  for (i=0; i < block_length; i += 8) {
+    unsigned int i8 = i >> 3;
+    for (int j0=0;j0<n_inputs;j0++) {
+      c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[j0<<5][i8]),andmask),zero256),masks[0]);
+      int j2 = 32;
+      if (j0==(n_inputs-1)) j2= impp->n_segments-(j0<<5);
+      for (int j=1; j < j2; j++) {
+        c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi32(simde_mm256_andnot_si256(simde_mm256_set1_epi32(input[(j0<<5)+j][i8]),andmask),zero256),masks[j]),c256);
+      }
+      ((simde__m256i *)cc_host[j0])[i8] = c256;
+    }
+  }
+#else
+  int i2=0;
+  const int32_t ucShifta[32][4] = { 
+	  {-7,-6,-5,-4}, // 0  
+	  {-6,-5,-4,-3}, // 1
+	  {-5,-4,-3,-2}, // 2 
+	  {-4,-3,-2,-1}, // 3 
+	  {-3,-2,-1,0}, // 4 
+	  {-2,-1,0,1}, // 5
+	  {-1,0,1,2}, // 6
+	  {0,1,2,3}, // 7
+	  {1,2,3,4}, // 8
+	  {2,3,4,5}, // 9
+	  {3,4,5,6}, // 10
+	  {4,5,6,7}, // 11
+	  {5,6,7,8}, // 12
+	  {6,7,8,9}, // 13
+	  {7,8,9,10}, // 14
+	  {8,9,10,11}, // 15
+	  {9,10,11,12}, // 16
+	  {10,11,12,13}, // 17
+	  {11,12,13,14}, // 18
+	  {12,13,14,15}, // 19
+	  {13,14,15,16}, // 20
+	  {14,15,16,17}, // 21
+	  {15,16,17,18}, // 22
+	  {16,17,18,19}, // 23
+	  {17,18,19,20}, // 24
+	  {18,19,20,21}, // 25
+	  {19,20,21,22}, // 26
+	  {20,21,22,23}, // 27
+	  {21,22,23,24}, // 28
+	  {22,23,24,25}, // 29
+	  {23,24,25,26}, // 30
+	  {24,25,26,27}}; // 31
+  const int32_t ucShiftb[32][4] = { 
+	  {-3,-2,-1,0}, // 0 
+	  {-2,-1,0,1}, // 1
+	  {-1,0,1,2}, // 2
+	  {0,1,2,3}, // 3
+	  {1,2,3,4}, // 4
+	  {2,3,4,5}, // 5
+	  {3,4,5,6}, // 6 
+	  {4,5,6,7}, // 7
+	  {5,6,7,8}, // 8
+	  {6,7,8,9}, // 9
+	  {7,8,9,10}, // 10
+	  {8,9,10,11}, // 11
+	  {9,10,11,12}, // 12
+	  {10,11,12,13}, // 13
+	  {11,12,13,14}, // 14
+	  {12,13,14,15}, // 15
+	  {13,14,15,16}, // 16
+	  {14,15,16,17}, // 17
+	  {15,16,17,18}, // 18
+	  {16,17,18,19}, // 19
+	  {17,18,19,20}, // 20
+	  {18,19,20,21}, // 21
+	  {19,20,21,22}, // 22
+	  {20,21,22,23}, // 23
+	  {21,22,23,24}, // 24
+	  {22,23,24,25}, // 25
+	  {23,24,25,26}, // 26
+	  {24,25,26,27}, // 27
+	  {25,26,27,28}, // 28
+	  {26,27,28,29}, // 29
+	  {27,28,29,30}, // 30
+	  {28,29,30,31}}; // 31
+			      
+  const int32_t ucShiftc[32][4] = { 
+	  {-15,-14,-13,-12}, // 0
+	  {-14,-13,-12,-11}, // 1
+	  {-13,-12,-11,-10}, // 2
+          {-12,-11,-10,-9}, // 3
+	  {-11,-10,-9,-8}, // 4
+	  {-10,-9,-8,-7}, // 5
+	  {-9,-8,-7,-6}, // 6
+	  {-8,-7,-6,-5}, // 7
+	  {-7,-6,-5,-4}, // 8  
+	  {-6,-5,-4,-3}, // 9
+	  {-5,-4,-3,-2}, // 10 
+	  {-4,-3,-2,-1}, // 11 
+	  {-3,-2,-1,0}, // 12 
+	  {-2,-1,0,1}, // 13
+	  {-1,0,1,2}, // 14
+	  {0,1,2,3}, // 15
+	  {1,2,3,4}, // 16
+	  {2,3,4,5}, // 17
+	  {3,4,5,6}, // 18
+	  {4,5,6,7}, // 19
+	  {5,6,7,8}, // 20
+	  {6,7,8,9}, // 21
+	  {7,8,9,10}, // 22
+	  {8,9,10,11}, // 23
+	  {9,10,11,12}, // 24
+	  {10,11,12,13}, // 25
+	  {11,12,13,14}, // 26
+	  {12,13,14,15}, // 27
+	  {13,14,15,16}, // 28
+	  {14,15,16,17}, // 29
+	  {15,16,17,18}, // 30
+	  {16,17,18,19}}; // 31
+  const int32_t ucShiftd[32][4] = { 
+	  {-11,-10,-9,-8}, // 0
+	  {-10,-9,-8,-7}, // 1
+	  {-9,-8,-7,-6}, // 2
+	  {-8,-7,-6,-5}, // 3
+	  {-7,-6,-5,-4}, // 4  
+	  {-6,-5,-4,-3}, // 5
+	  {-5,-4,-3,-2}, // 6
+	  {-4,-3,-2,-1}, // 7 
+	  {-3,-2,-1,0}, // 8
+	  {-2,-1,0,1}, // 9
+	  {-1,0,1,2}, // 10
+	  {0,1,2,3}, // 11 
+	  {1,2,3,4}, // 12
+	  {2,3,4,5}, // 13
+	  {3,4,5,6}, // 14 
+	  {4,5,6,7}, // 15
+	  {5,6,7,8}, // 16
+	  {6,7,8,9}, // 17
+	  {7,8,9,10}, // 18
+	  {8,9,10,11}, // 19
+	  {9,10,11,12}, // 20
+	  {10,11,12,13}, // 21
+	  {11,12,13,14}, // 22
+	  {12,13,14,15}, // 23
+	  {13,14,15,16}, // 24
+	  {14,15,16,17}, // 25
+	  {15,16,17,18}, // 26
+	  {16,17,18,19}, // 27
+	  {17,18,19,20}, // 28
+	  {18,19,20,21}, // 29
+	  {19,20,21,22}, // 30
+	  {20,21,22,23}};// 31
+			  
+  const int32_t ucShifte[32][4] = { 
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}, // 23
+	  {1,2,3,4}, // 24
+	  {2,3,4,5}, // 25
+	  {3,4,5,6}, // 26
+	  {4,5,6,7}, // 27
+	  {5,6,7,8}, // 28
+	  {6,7,8,9}, // 29
+	  {7,8,9,10}, // 30
+	  {8,9,10,11}}; // 31 
+			  
+  const int32_t ucShiftf[32][4] = { 
+	  {-19,-18,-17,-16}, // 0 
+	  {-18,-17,-16,-15}, // 1
+	  {-17,-16,-15,-14}, // 2
+	  {-16,-15,-14,-13}, // 3
+	  {-15,-14,-13,-12}, // 4
+	  {-14,-13,-12,-11}, // 5
+	  {-13,-12,-11,-10}, // 6
+          {-12,-11,-10,-9}, // 7
+	  {-11,-10,-9,-8}, // 8
+	  {-10,-9,-8,-7}, // 9  
+	  {-9,-8,-7,-6}, // 10 
+	  {-8,-7,-6,-5}, // 11
+	  {-7,-6,-5,-4}, // 12 
+	  {-6,-5,-4,-3}, // 13
+	  {-5,-4,-3,-2}, // 14
+	  {-4,-3,-2,-1}, // 15 
+	  {-3,-2,-1,0}, // 16
+	  {-2,-1,0,1}, // 17
+	  {-1,0,1,2}, // 18
+	  {0,1,2,3}, // 19
+	  {1,2,3,4}, // 20
+	  {2,3,4,5}, // 21
+	  {3,4,5,6}, // 22 
+	  {4,5,6,7}, // 23
+	  {5,6,7,8}, // 24
+	  {6,7,8,9}, // 25
+	  {7,8,9,10}, // 26
+	  {8,9,10,11}, // 27
+	  {9,10,11,12}, // 28
+	  {10,11,12,13}, // 29
+	  {11,12,13,14}, // 30
+	  {12,13,14,15}}; // 31
+			  
+  const int32_t ucShiftg[32][4] = { 
+	  {-31,-30,-29,-28},
+	  {-30,-29,-28,-27},
+	  {-29,-28,-27,-26},
+	  {-28,-27,-26,-25},
+	  {-27,-26,-25,-24},
+	  {-26,-25,-24,-23},
+	  {-25,-24,-23,-22},
+	  {-24,-23,-22,-21},
+	  {-23,-22,-21,-20}, // 0 
+	  {-22,-21,-20,-19}, // 1
+	  {-21,-20,-19,-18}, // 2
+	  {-20,-19,-18,-17}, // 3
+	  {-19,-18,-17,-16}, // 4 
+	  {-18,-17,-16,-15}, // 5
+	  {-17,-16,-15,-14}, // 6
+	  {-16,-15,-14,-13}, // 7
+	  {-15,-14,-13,-12}, // 8
+	  {-14,-13,-12,-11}, // 9
+	  {-13,-12,-11,-10}, // 10
+          {-12,-11,-10,-9}, // 11
+	  {-11,-10,-9,-8}, // 12
+	  {-10,-9,-8,-7}, // 13
+	  {-9,-8,-7,-6}, // 14 
+	  {-8,-7,-6,-5}, // 15
+	  {-7,-6,-5,-4}, // 16 
+	  {-6,-5,-4,-3}, // 17
+	  {-5,-4,-3,-2}, // 18 
+	  {-4,-3,-2,-1}, // 19 
+	  {-3,-2,-1,0}, // 20 
+	  {-2,-1,0,1}, // 21
+	  {-1,0,1,2}, // 22
+	  {0,1,2,3}}; // 23
+			  
+  const int32_t ucShifth[32][4] = { 
+	  {-27,-26,-25,-24}, // 0
+	  {-26,-25,-24,-23}, // 1
+	  {-25,-24,-23,-22}, // 2
+	  {-24,-23,-22,-21}, // 3
+	  {-23,-22,-21,-20}, // 4 
+	  {-22,-21,-20,-19}, // 5
+	  {-21,-20,-19,-18}, // 6
+	  {-20,-19,-18,-17}, // 7
+	  {-19,-18,-17,-16}, // 8 
+	  {-18,-17,-16,-15}, // 9
+	  {-17,-16,-15,-14}, // 10
+	  {-16,-15,-14,-13}, // 11
+	  {-15,-14,-13,-12}, // 12
+	  {-14,-13,-12,-11}, // 13
+	  {-13,-12,-11,-10}, // 14
+          {-12,-11,-10,-9}, // 15
+	  {-11,-10,-9,-8}, // 16
+	  {-10,-9,-8,-7}, // 17 
+	  {-9,-8,-7,-6}, // 18 
+	  {-8,-7,-6,-5}, // 19
+	  {-7,-6,-5,-4}, // 20 
+	  {-6,-5,-4,-3}, // 21
+	  {-5,-4,-3,-2}, // 22
+	  {-4,-3,-2,-1}, // 23 
+	  {-3,-2,-1,0}, // 24 
+	  {-2,-1,0,1}, // 25
+	  {-1,0,1,2}, // 26
+	  {0,1,2,3}, // 27
+	  {1,2,3,4}, // 28
+	  {2,3,4,5}, // 29
+	  {3,4,5,6}, // 30
+          {4,5,6,7}}; // 31 
+  const uint32_t __attribute__ ((aligned (16))) masksa[4] = {0x80,0x40,0x20,0x10};
+  const uint32_t __attribute__ ((aligned (16))) masksb[4] = {0x8,0x4,0x2,0x1};
+  const uint32_t __attribute__ ((aligned (16))) masksc[4] = {0x8000,0x4000,0x2000,0x1000};
+  const uint32_t __attribute__ ((aligned (16))) masksd[4] = {0x800,0x400,0x200,0x100};
+  const uint32_t __attribute__ ((aligned (16))) maskse[4] = {0x800000,0x400000,0x200000,0x100000};
+  const uint32_t __attribute__ ((aligned (16))) masksf[4] = {0x80000,0x40000,0x20000,0x10000};
+  const uint32_t __attribute__ ((aligned (16))) masksg[4] = {0x80000000,0x40000000,0x20000000,0x10000000};
+  const uint32_t __attribute__ ((aligned (16))) masksh[4] = {0x8000000,0x4000000,0x2000000,0x1000000};
+  int32x4_t vshifta[32],vshiftb[32],vshiftc[32],vshiftd[32],vshifte[32],vshiftf[32],vshiftg[32],vshifth[32];
+  uint32x4_t vmasksa  = vld1q_u32(masksa);
+  uint32x4_t vmasksb  = vld1q_u32(masksb);
+  uint32x4_t vmasksc  = vld1q_u32(masksc);
+  uint32x4_t vmasksd  = vld1q_u32(masksd);
+  uint32x4_t vmaskse  = vld1q_u32(maskse);
+  uint32x4_t vmasksf  = vld1q_u32(masksf);
+  uint32x4_t vmasksg  = vld1q_u32(masksg);
+  uint32x4_t vmasksh  = vld1q_u32(masksh);
+  uint32x4_t in;
+
+  for (int n=0;n<32;n++) { 
+    vshifta[n] = vld1q_s32(ucShifta[n]);
+    vshiftb[n] = vld1q_s32(ucShiftb[n]);
+    vshiftc[n] = vld1q_s32(ucShiftc[n]);
+    vshiftd[n] = vld1q_s32(ucShiftd[n]);
+    vshifte[n] = vld1q_s32(ucShifte[n]);
+    vshiftf[n] = vld1q_s32(ucShiftf[n]);
+    vshiftg[n] = vld1q_s32(ucShiftg[n]);
+    vshifth[n] = vld1q_s32(ucShifth[n]);
+  }
+  i2=0;
+  int j0=0,j1=0,j2=0,j3=0;
+  if (impp->n_segments <= 32) {
+	  j0=impp->n_segments;
+  }
+  else if (impp->n_segments <= 64) {
+     j0=32;
+     j1=impp->n_segments-32;
+  }
+  else if (impp->n_segments <= 96) {
+     j0=32; j1=32;
+     j2=impp->n_segments-64;
+  }
+  else if (impp->n_segments <= 128) {
+     j0=32; j1=32; j2=32;
+     j3=impp->n_segments-96;
+  }
+  uint32x4_t *ccp,cc0,cc1,cc2,cc3,cc4,cc5,cc6,cc7;
+  for (int i=0; i < (block_length>>5); i++,i2+=8) {
+    in = vdupq_n_u32(((uint32_t*)input[0])[i]);
+    cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+    cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+    cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+    cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+    cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+    cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+    cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+    cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+    for (int j = 1; j < j0; j++) {
+      in = vdupq_n_u32(((uint32_t*)input[j])[i]);
+      cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+      cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+      cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+      cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+      cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+      cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+      cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+      cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+    }
+    ccp=&((uint32x4_t *)cc_host[0])[i2];
+    ccp[0] = cc0;
+    ccp[1] = cc1;
+    ccp[2] = cc2;
+    ccp[3] = cc3;
+    ccp[4] = cc4;
+    ccp[5] = cc5;
+    ccp[6] = cc6;
+    ccp[7] = cc7;
+    if (j1>0) {
+      in = vdupq_n_u32(((uint32_t*)input[32])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j1; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[32+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc_host[1])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j2>0) {
+      in = vdupq_n_u32(((uint32_t*)input[64])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j2; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[64+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc_host[2])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+    if (j3>0) {
+      in = vdupq_n_u32(((uint32_t*)input[96])[i]);
+      cc0 = vshlq_u32(vandq_u32(in,vmasksa),vshifta[0]);
+      cc1 = vshlq_u32(vandq_u32(in,vmasksb),vshiftb[0]);
+      cc2 = vshlq_u32(vandq_u32(in,vmasksc),vshiftc[0]);
+      cc3 = vshlq_u32(vandq_u32(in,vmasksd),vshiftd[0]);
+      cc4 = vshlq_u32(vandq_u32(in,vmaskse),vshifte[0]);
+      cc5 = vshlq_u32(vandq_u32(in,vmasksf),vshiftf[0]);
+      cc6 = vshlq_u32(vandq_u32(in,vmasksg),vshiftg[0]);
+      cc7 = vshlq_u32(vandq_u32(in,vmasksh),vshifth[0]);
+      for (int j = 1; j < j3; j++) {
+        in = vdupq_n_u32(((uint32_t*)input[96+j])[i]);
+        cc0 = vorrq_u32(cc0,vshlq_u32(vandq_u32(in,vmasksa),vshifta[j]));
+        cc1 = vorrq_u32(cc1,vshlq_u32(vandq_u32(in,vmasksb),vshiftb[j]));
+        cc2 = vorrq_u32(cc2,vshlq_u32(vandq_u32(in,vmasksc),vshiftc[j]));
+        cc3 = vorrq_u32(cc3,vshlq_u32(vandq_u32(in,vmasksd),vshiftd[j]));
+        cc4 = vorrq_u32(cc4,vshlq_u32(vandq_u32(in,vmaskse),vshifte[j]));
+        cc5 = vorrq_u32(cc5,vshlq_u32(vandq_u32(in,vmasksf),vshiftf[j]));
+        cc6 = vorrq_u32(cc6,vshlq_u32(vandq_u32(in,vmasksg),vshiftg[j]));
+        cc7 = vorrq_u32(cc7,vshlq_u32(vandq_u32(in,vmasksh),vshifth[j]));
+      }
+      ccp=&((uint32x4_t *)cc_host[3])[i2];
+      ccp[0] = cc0;
+      ccp[1] = cc1;
+      ccp[2] = cc2;
+      ccp[3] = cc3;
+      ccp[4] = cc4;
+      ccp[5] = cc5;
+      ccp[6] = cc6;
+      ccp[7] = cc7;
+    }
+  }
+#endif
+#endif
+#endif
+  if(impp->tinput != NULL) stop_meas(impp->tinput);
+  if(impp->tinput_memcpy != NULL) start_meas(impp->tinput_memcpy);
+#ifdef USE_GPU_CIRCCOPY
+  circcopy_c(cc_dev,c_dev,n_inputs);
+#else
+#ifdef USE_CUDA_MEMCPY 
+  for (int s=0;s<n_inputs;s++)  {
+    for (int i1 = 0; i1 < ncols; i1++)   {
+      cudaError_t err = cudaMemcpy(&c_devh[s][2 * i1 * Zc], &cc_devh[s][i1 * Zc], Zc * sizeof(uint32_t),cudaMemcpyDeviceToDevice);
+      AssertFatal(err == cudaSuccess, "1st memcpy cc_devh[%d] -> c_devh[%d] %p CUDA Error: %s\n", s, s, c_devh[s],cudaGetErrorString(err)); 			
+      err = cudaMemcpy(&c_devh[s][(2 * i1 + 1) * Zc], &cc_devh[s][i1 * Zc], Zc * sizeof(uint32_t),cudaMemcpyDeviceToDevice);
+      AssertFatal(err == cudaSuccess, "2nd memcpy cc_devh[%d] -> c_devh[%d] %p CUDA Error: %s\n", s, s, c_devh[s],cudaGetErrorString(err)); 			
+    }
+#else
+  for (int s=0;s<n_inputs;s++)  {
+    for (int i1 = 0; i1 < ncols; i1++)   {
+      memcpy(&c_host[s][2 * i1 * Zc], &cc_host[s][i1 * Zc], Zc * sizeof(uint32_t));
+      memcpy(&c_host[s][(2 * i1 + 1) * Zc], &cc_host[s][i1 * Zc], Zc * sizeof(uint32_t));
+    }
+    if (!pageable && !register_host) {
+      cudaError_t err = cudaMemcpy(c_devh[s],c_host[s],sizeof(uint32_t)*2*22*Zc,cudaMemcpyHostToDevice);
+      AssertFatal(err == cudaSuccess, "memcpy c_host[%d] -> c_devh[%d] %p CUDA Error: %s\n", s, s, c_devh[s],cudaGetErrorString(err)); 			
+    }
+  }
+#endif 
+#endif
+  if(impp->tinput != NULL) stop_meas(impp->tinput_memcpy);
+  if (BG==1 && Zc==384)  {
+    //parity check part
+    if(impp->tparity != NULL) start_meas(impp->tparity);
+    encode_parity_check_part_cuda((uint32_t**)c_dev, (uint32_t**)d_dev, BG, Zc, Kb, ncols,n_inputs);
+    if(impp->tparity != NULL) stop_meas(impp->tparity);
+  }
+  else {
+    AssertFatal(1==0,"Only BG1 Zc=384 for now\n");
+  }
+  if(impp->toutput != NULL) start_meas(impp->toutput);
+  for (int s=0;s<n_inputs;s++) {
+	  
+    if (!pageable && !register_host) {
+      cudaMemcpy(d_host[s],&cc_devh[s][2*Zc],sizeof(uint32_t)*(block_length-(2*Zc)),cudaMemcpyDeviceToHost);
+      cudaError_t err = cudaMemcpy(&d_host[s][block_length-(2*Zc)],d_devh[s]+block_length-(2*Zc),sizeof(uint32_t)*((nrows-no_punctured_columns) * Zc-removed_bit),cudaMemcpyDeviceToHost);
+      AssertFatal(err == cudaSuccess, "d_dev[%d] %p CUDA Error: %s\n", s, d_devh[s],cudaGetErrorString(err)); 			
+    }
+    else {
+       memcpy(d_host[s],&cc_host[s][2*Zc],sizeof(uint32_t)*(block_length-(2*Zc)));
+    }
+  }
+  if(impp->toutput != NULL) stop_meas(impp->toutput);
+  return d_host;
+}
+

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c

@@ -43,6 +43,11 @@
 #include "ldpc_encode_parity_check.c"
 #include "ldpc_generate_coefficient.c"
 
+
+void ini_input32_luts() {
+	return;
+}
+
 int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
 {
   //set_log(PHY, 4);
@@ -107,7 +112,6 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
   //interleave up to 8 transport-block segements at a time
 
   unsigned int i_byte = 0;
-
 #if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
   const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2),
                               _mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8),
@@ -193,12 +197,9 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
   if(impp->tinput != NULL) stop_meas(impp->tinput);
 
   if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
-    // extend matrix
-    if(impp->tprep != NULL) start_meas(impp->tprep);
-    if(impp->tprep != NULL) stop_meas(impp->tprep);
     //parity check part
     if(impp->tparity != NULL) start_meas(impp->tparity);
-    encode_parity_check_part_optim(cc, dd, BG, Zc, Kb, simd_size, ncols);
+    encode_parity_check_part_optim(cc, dd, BG, Zc, Kb, simd_size, ncols,impp->tinput_memcpy);
     if(impp->tparity != NULL) stop_meas(impp->tparity);
   }
   else {
@@ -207,8 +208,10 @@ int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
       return(-1);
     }
   }
+  if(impp->toutput != NULL) start_meas(impp->toutput);
   memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
   memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
+  if(impp->toutput != NULL) stop_meas(impp->toutput);
   return 0;
 }
 

openair1/PHY/CODING/nrLDPC_encoder/ldpc_input.cu

@@ -0,0 +1,175 @@
+#include <stdio.h>
+#include <stdint.h>
+#include <cuda_runtime.h>
+/*
+__device__ const uint32_t masks[4] = {0x80,0x8000,0x800000,0x80000000};
+__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) {
+
+  int block_off = blockIdx.x*blockDim.x<<2;	
+  int i2 = threadIdx.x<<2;
+  uint32_t *out=cc[blockIdx.y] + block_off + i2;
+  int nseg0 = (blockIdx.y << 5);
+  int nseg1;
+  if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
+  else nseg1 = nseg0 + (nseg&31);
+  int bit_offset = i2+block_off;
+  int uint32_offset = bit_offset>>5;  
+  uint32_t mask = masks[(bit_offset&31)>>3]; 
+  uint32_t mask0 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask1 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask2 = mask>>(bit_offset&7);bit_offset++;
+  uint32_t mask3 = mask>>(bit_offset&7);
+  uint32_t tmp,jmod;
+  uint32_t otmp0,otmp1,otmp2,otmp3;
+  if (bit_offset  < block_length) {
+      tmp=input[nseg0][uint32_offset];
+      otmp0 = ((tmp&mask0) > 0); 
+      otmp1 = ((tmp&mask1) > 0); 
+      otmp2 = ((tmp&mask2) > 0); 
+      otmp3 = ((tmp&mask3) > 0); 
+      for (int j=nseg0+1;j<nseg1;j++) {
+	 tmp=input[j][uint32_offset];
+	 jmod = j&31;
+         otmp0 |= (((tmp&mask0) > 0)<<jmod); 
+         otmp1 |= (((tmp&mask1) > 0)<<jmod); 
+         otmp2 |= (((tmp&mask2) > 0)<<jmod); 
+         otmp3 |= (((tmp&mask3) > 0)<<jmod); 
+      }	      
+      out[0]=otmp0;
+      out[1]=otmp1;
+      out[2]=otmp2;
+      out[3]=otmp3;
+  } 
+}
+*/
+
+#define ITERATIONS 1
+#define NTHREADS 384 
+__device__ uint32_t masks[32] = {
+	0x80,0x40,0x20,0x10,0x8,0x4,0x2,0x1,
+	0x8000,0x4000,0x2000,0x1000,0x800,0x400,0x200,0x100,
+	0x800000,0x400000,0x200000,0x100000,0x80000,0x40000,0x20000,0x10000,
+	0x80000000,0x40000000,0x20000000,0x10000000,0x8000000,0x4000000,0x2000000,0x1000000};
+__global__ void ldpc_input_worker(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) {
+
+  int block_off = blockIdx.x*blockDim.x*ITERATIONS;	
+  int i2 = threadIdx.x*ITERATIONS;
+  uint32_t *out=cc[blockIdx.y] + block_off + i2;
+  int nseg0 = (blockIdx.y << 5);
+  int nseg1;
+  if ((nseg0 + 32) <= nseg) nseg1 = nseg0+32;
+  else nseg1 = nseg0 + (nseg&31);
+  int bit_offset = i2+block_off;
+  int uint32_offset = bit_offset>>5;  
+#if ITERATIONS==1
+  uint32_t mask0 = masks[bit_offset&31];
+#else
+  uint32_t *mask = &masks[bit_offset&31]; 
+  uint32_t mask0 = mask[0];
+#endif
+#if ITERATIONS==4 || ITERATIONS==8
+  uint32_t mask1 = mask[1];
+  uint32_t mask2 = mask[2];
+  uint32_t mask3 = mask[3];
+#endif
+#if ITERATIONS==8
+  uint32_t mask4 = mask[4];
+  uint32_t mask5 = mask[5];
+  uint32_t mask6 = mask[6];
+  uint32_t mask7 = mask[7];
+#endif
+  uint32_t tmp,jmod;
+  uint32_t otmp0;
+#if ITERATIONS==4 || ITERATIONS==8
+  uint32_t otmp1,otmp2,otmp3;
+#endif
+#if ITERATIONS==8
+  uint32_t otmp4,otmp5,otmp6,otmp7;
+#endif
+  if (bit_offset  < block_length) {
+      tmp=input[nseg0][uint32_offset];
+      otmp0 = ((tmp&mask0) > 0); 
+#if ITERATIONS==4 || ITERATIONS==8
+      otmp1 = ((tmp&mask1) > 0); 
+      otmp2 = ((tmp&mask2) > 0); 
+      otmp3 = ((tmp&mask3) > 0); 
+#endif
+#if ITERATIONS==8
+      otmp4 = ((tmp&mask4) > 0); 
+      otmp5 = ((tmp&mask5) > 0); 
+      otmp6 = ((tmp&mask6) > 0); 
+      otmp7 = ((tmp&mask7) > 0);
+#endif 
+      for (int j=nseg0+1;j<nseg1;j++) {
+	 tmp=input[j][uint32_offset];
+	 jmod = j&31;
+         otmp0 |= (((tmp&mask0) > 0)<<jmod); 
+#if ITERATIONS==4 || ITERATIONS==8
+         otmp1 |= (((tmp&mask1) > 0)<<jmod); 
+         otmp2 |= (((tmp&mask2) > 0)<<jmod); 
+         otmp3 |= (((tmp&mask3) > 0)<<jmod);
+#endif 
+#if ITERATIONS==8
+         otmp4 |= (((tmp&mask4) > 0)<<jmod); 
+         otmp5 |= (((tmp&mask5) > 0)<<jmod); 
+         otmp6 |= (((tmp&mask6) > 0)<<jmod); 
+         otmp7 |= (((tmp&mask7) > 0)<<jmod); 
+#endif
+      }	      
+      out[0]=otmp0;
+#if ITERATIONS==4 || ITERATIONS==8
+      out[1]=otmp1;
+      out[2]=otmp2;
+      out[3]=otmp3;
+#endif
+#if ITERATIONS==8
+      out[4]=otmp4;
+      out[5]=otmp5;
+      out[6]=otmp6;
+      out[7]=otmp7;
+#endif
+  } 
+}
+__global__ void circcopy_c_worker(uint32_t **cc,uint32_t **c) {
+
+  int s = blockIdx.x;	
+  int i1 = blockIdx.y;
+  int i  = threadIdx.x;
+  uint32_t tmp;
+  if (i<384) {
+    tmp = cc[s][(i1*384) + i];	  
+    c[s][2*i1*384 + i] = tmp;
+    c[s][(2*i1+1)*384 + i] = tmp; 
+  }
+}
+
+extern "C" int ldpc_input(uint32_t **input,uint32_t *cc[4],int block_length,int nseg) { 
+
+ int numb = block_length/(NTHREADS*ITERATIONS);
+ if ((block_length%(NTHREADS*ITERATIONS)) > 0) numb++;
+ int ns = nseg>>5;
+ if ((nseg&31)>0) ns++;
+
+ dim3 numblocks(numb,ns);
+ ldpc_input_worker<<<numblocks,NTHREADS>>>(input,cc,block_length,nseg);
+ cudaError_t err=cudaPeekAtLastError();
+ if (err!=cudaSuccess) {
+    printf("cuda error: %s (input %p, cc %p, block_length %d, nseg %d, numb %d, ns %d)\n",cudaGetErrorString(err),input,cc,block_length,nseg,numb,ns);
+    exit(-1);
+ }
+ cudaDeviceSynchronize();
+ return(0);
+}
+
+extern "C" int circcopy_c(uint32_t **cc,uint32_t **c,int n_inputs) { 
+
+ dim3 numblocks(n_inputs,22);
+ circcopy_c_worker<<<numblocks,384>>>(cc,c);
+ cudaError_t err=cudaPeekAtLastError();
+ if (err!=cudaSuccess) {
+    printf("cuda error: %s (cc %p, c %p, n_inputs %d)\n",cudaGetErrorString(err),cc,c,n_inputs);
+    exit(-1);
+ }
+ cudaDeviceSynchronize();
+ return(0);
+}

openair1/PHY/CODING/nrLDPC_extern.h

@@ -29,6 +29,7 @@ typedef struct ldpc_interface_s {
   LDPC_shutdownfunc_t *LDPCshutdown;
   LDPC_decoderfunc_t *LDPCdecoder;
   LDPC_encoderfunc_t *LDPCencoder;
+  LDPC_encoderfunc32_t *LDPCencoder32;
 } ldpc_interface_t;
 
 /* functions to load the LDPC shared lib, implemented in openair1/PHY/CODING/nrLDPC_load.c */
@@ -37,6 +38,7 @@ int free_LDPClib(ldpc_interface_t *ldpc_interface);
 
 LDPC_decoderfunc_t LDPCdecoder;
 LDPC_encoderfunc_t LDPCencoder;
+LDPC_encoderfunc32_t LDPCencoder32;
 
 // inline functions:
 #endif

openair1/PHY/CODING/nrLDPC_load.c

@@ -56,14 +56,17 @@ int load_LDPClib(char *version, ldpc_interface_t *itf)
   loader_shlibfunc_t shlib_fdesc[] = {{.fname = "LDPCinit"},
                                       {.fname = "LDPCshutdown"},
                                       {.fname = "LDPCdecoder"},
-                                      {.fname = "LDPCencoder"}};
+                                      {.fname = "LDPCencoder"},
+                                      {.fname = "LDPCencoder32"}};
   int ret;
   ret = load_module_version_shlib(libname, version, shlib_fdesc, sizeofArray(shlib_fdesc), NULL);
   AssertFatal((ret >= 0), "Error loading ldpc decoder");
+  LOG_I(NR_PHY,"loading LDPC version %s\n",version);
   itf->LDPCinit = (LDPC_initfunc_t *)shlib_fdesc[0].fptr;
   itf->LDPCshutdown = (LDPC_shutdownfunc_t *)shlib_fdesc[1].fptr;
   itf->LDPCdecoder = (LDPC_decoderfunc_t *)shlib_fdesc[2].fptr;
   itf->LDPCencoder = (LDPC_encoderfunc_t *)shlib_fdesc[3].fptr;
+  itf->LDPCencoder32 = (LDPC_encoderfunc32_t *)shlib_fdesc[4].fptr;
 
   AssertFatal(itf->LDPCinit() == 0, "error starting LDPC library %s %s\n", libname, version);
 

openair1/PHY/INIT/nr_init.c

@@ -21,6 +21,7 @@
 
 #include "executables/softmodem-common.h"
 #include "executables/nr-softmodem-common.h"
+#include "executables/nr-softmodem.h"
 #include "common/utils/nr/nr_common.h"
 #include "common/ran_context.h"
 #include "PHY/defs_gNB.h"
@@ -105,6 +106,10 @@ void phy_init_nr_gNB(PHY_VARS_gNB *gNB)
   // shortcuts
   NR_DL_FRAME_PARMS *const fp       = &gNB->frame_parms;
   nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
+
+  gNB->use_gpu = use_gpu;
+  gNB->use_fp16 = use_fp16;
+
   NR_gNB_COMMON *const common_vars = &gNB->common_vars;
   common_vars->analog_bf = cfg->analog_beamforming_ve.analog_bf_vendor_ext.value;
   LOG_I(PHY, "L1 configured with%s analog beamforming\n", common_vars->analog_bf ? "" : "out");
@@ -342,7 +347,12 @@ void nr_phy_config_request_sim(PHY_VARS_gNB *gNB,
   nr_init_frame_parms(gNB_config, fp);
 
   fp->ofdm_offset_divisor = UINT_MAX;
-  init_symbol_rotation(fp);
+  printf("Init symbol rotation, fp16 %s\n",gNB->use_fp16?"yes":"no");
+  init_symbol_rotation(fp
+#ifdef FLT16_MAX
+		  ,gNB->use_fp16
+#endif
+		  );
   init_timeshift_rotation(fp);
 
   gNB->configured = 1;
@@ -395,7 +405,12 @@ void nr_phy_config_request(NR_PHY_Config_t *phy_config)
   RC.gNB[Mod_id]->configured     = 1;
 
   fp->ofdm_offset_divisor = RC.gNB[Mod_id]->ofdm_offset_divisor;
-  init_symbol_rotation(fp);
+  printf("Init symbol rotation, fp16 %s\n",RC.gNB[Mod_id]->use_fp16?"yes":"no");
+  init_symbol_rotation(fp
+#ifdef FLT16_MAX
+		  ,RC.gNB[Mod_id]->use_fp16
+#endif
+		  );
   init_timeshift_rotation(fp);
 }
 
@@ -407,7 +422,7 @@ static void init_DLSCH_struct(PHY_VARS_gNB *gNB)
   gNB->dlsch = calloc(gNB->max_nb_pdsch, sizeof(*gNB->dlsch));
   for (int i = 0; i < gNB->max_nb_pdsch; i++) {
     LOG_D(PHY, "Allocating Transport Channel Buffers for DLSCH %d/%d\n", i, gNB->max_nb_pdsch);
-    gNB->dlsch[i] = new_gNB_dlsch(fp, grid_size);
+    gNB->dlsch[i] = new_gNB_dlsch(fp, grid_size, gNB->use_gpu);
   }
 }
 
@@ -417,7 +432,7 @@ static void destroy_DLSCH_struct(const PHY_VARS_gNB *gNB)
   const nfapi_nr_config_request_scf_t *cfg = &gNB->gNB_config;
   const uint16_t grid_size = cfg->carrier_config.dl_grid_size[fp->numerology_index].value;
   for (int i = 0; i < gNB->max_nb_pdsch; i++) {
-    free_gNB_dlsch(&gNB->dlsch[i], grid_size, fp);
+    free_gNB_dlsch(&gNB->dlsch[i], grid_size, fp, gNB->use_gpu);
   }
   free(gNB->dlsch);
 }

openair1/PHY/INIT/nr_init_ue.c

@@ -273,7 +273,11 @@ int init_nr_ue_signal(PHY_VARS_NR_UE *ue, int nb_connected_gNB)
   }
 
   ue->init_averaging = 1;
-  init_symbol_rotation(fp);
+  init_symbol_rotation(fp
+#ifdef FLT16_MAX
+		       ,ue->use_fp16 
+#endif
+		  );
   init_timeshift_rotation(fp);
 
   // initialize to false only for SA since in do-ra and phy-test it is already set to true before getting here

openair1/PHY/INIT/nr_parms.c

@@ -656,7 +656,11 @@ int nr_init_frame_parms_ue_sl(NR_DL_FRAME_PARMS *fp,
   // ssb_offset_pointa points to the first RE where Sidelink-PSBCH starts
   fp->ssb_start_subcarrier = config->sl_bwp_config.sl_ssb_offset_point_a;
 
-  perform_symbol_rotation(fp, fp->sl_CarrierFreq, fp->symbol_rotation[link_type_sl]);
+  perform_symbol_rotation(fp, fp->sl_CarrierFreq, fp->symbol_rotation[link_type_sl]
+#ifdef FLT16_MAX
+		  ,0//use_fp16
+#endif
+		  );
   init_timeshift_rotation(fp);
 
   // Not used for Sidelink

openair1/PHY/MODULATION/nr_modulation.c

@@ -18,11 +18,11 @@
  * For more information about the OpenAirInterface (OAI) Software Alliance:
  *      contact@openairinterface.org
  */
-
 #include "nr_modulation.h"
-#include "PHY/NR_REFSIG/nr_mod_table.h"
 #include "executables/softmodem-common.h"
 #include <simde/x86/avx512.h>
+#include "PHY/NR_REFSIG/nr_mod_table.h"
+#include <float.h>
 // Lacking declaration in present simde external package, will be detected as compilation error when they will add it
 #define simde_mm512_extracti64x2_epi64(a...) _mm512_extracti64x2_epi64(a)
 
@@ -117,112 +117,179 @@ const char nr_W_4l_4p[5][4][4] = {
     {{'1', '1', '1', '1'}, {'1', 'n', '1', 'n'}, {'j', 'j', 'o', 'o'}, {'j', 'o', 'o', 'j'}} // pmi 4
 };
 
+#ifdef FLT16_MAX
+void nr_modulation(const uint32_t *in, uint32_t length, uint16_t mod_order, int16_t *out, _Float16 *out_fp16)
+#else
 void nr_modulation(const uint32_t *in, uint32_t length, uint16_t mod_order, int16_t *out)
+#endif
 {
-  const uint16_t mask = ((1 << mod_order) - 1);
+#ifdef FLT16_MAX	
+  AssertFatal((!out && out_fp16) || (out && !out_fp16), "out %p, out_fp16 %p, one needs to be null\n",out,out_fp16);
+#else
+  AssertFatal(out == NULL, "out is NULL\n");
+#endif  
+  uint16_t mask = ((1 << mod_order) - 1);
+  int32_t *nr_mod_table32;
+  int64_t *nr_mod_table64;
+
+#ifdef FLT16_MAX
+  int32_t *out32 = out?(int32_t*)out:(int32_t*)out_fp16;
+#else
   int32_t *out32 = (int32_t *)out;
+#endif
+  int64_t *out64 = (int64_t *)out32;
   const uint8_t *in_bytes = (const uint8_t *)in;
   const uint64_t *in64 = (const uint64_t *)in;
-  int64_t *out64 = (int64_t *)out;
   uint32_t i = 0;
 
-  LOG_D(PHY, "nr_modulation: length %d, mod_order %d\n", length, mod_order);
+#if defined(__SSE2__) || defined(__aarch64__)
+  simde__m128i *nr_mod_table128;
+  simde__m128i *out128 = (simde__m128i *)out32;
+#endif
+
+  LOG_D(PHY, "nr_modulation: length %d, mod_order %d (fp16 %s)\n", length, mod_order,out_fp16?"yes":"no");
 
   switch (mod_order) {
-    case 2: {
-      simde__m128i *nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table;
-      simde__m128i *out128 = (simde__m128i *)out;
+#if defined(__SSE2__) || defined(__aarch64__)
+    case 2:
+#ifdef FLT16_MAX
+      if (out_fp16) 
+	nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table_fp16;
+      else 
+#endif
+	nr_mod_table128 = (simde__m128i *)nr_qpsk_byte_mod_table;
+
       for (i = 0; i < length / 8; i++)
         out128[i] = nr_mod_table128[in_bytes[i]];
       // the bits that are left out
       i = i * 8 / 2;
-      int32_t *nr_mod_table32 = (int32_t *)nr_qpsk_mod_table;
+#ifdef FLT16_MAX
+      if (out_fp16) 
+       nr_mod_table32 = (int32_t *)nr_qpsk_mod_table_fp16; 
+      else 
+#endif
+        nr_mod_table32 = (int32_t *)nr_qpsk_mod_table;
       while (i < length / 2) {
         const int idx = ((in_bytes[(i * 2) / 8] >> ((i * 2) & 0x7)) & mask);
         out32[i] = nr_mod_table32[idx];
         i++;
       }
-    }
       return;
+#else
+    case 2:
+#ifdef FLT16_MAX
+      if (out_fp16) 
+       nr_mod_table32 = (int32_t *)nr_qpsk_mod_table_fp16; 
+      else 
+#endif
+       nr_mod_table32 = (int32_t *)nr_qpsk_mod_table;
+      for (i = 0; i < length / mod_order; i++) {
+        const int idx = ((in[i * 2 / 32] >> ((i * 2) & 0x1f)) & mask);
+        out32[i] = nr_mod_table32[idx];
+      }
+      return;
+#endif
 
     case 4:
+#ifdef FLT16_MAX
+      if (out_fp16) {
+       nr_mod_table64 = (int64_t *)nr_16qam_byte_mod_table_fp16; 
+       nr_mod_table32 = (int32_t *)nr_16qam_mod_table_fp16;
+      }
+      else 
+#endif
+      {	      
+       nr_mod_table64 = (int64_t *)nr_16qam_byte_mod_table; 
+       nr_mod_table32 = (int32_t *)nr_16qam_mod_table;
+      }
+
       for (i = 0; i < length / 8; i++)
-        out64[i] = nr_16qam_byte_mod_table[in_bytes[i]];
+        out64[i] = nr_mod_table64[in_bytes[i]];
       // the bits that are left out
       i = i * 8 / 4;
       while (i < length / 4) {
         const int idx = ((in_bytes[(i * 4) / 8] >> ((i * 4) & 0x7)) & mask);
-        out32[i] = nr_16qam_mod_table[idx];
+        out32[i] = nr_mod_table32[idx];
         i++;
       }
       return;
 
     case 6:
+#ifdef FLT16_MAX
+      if (out_fp16) 
+       nr_mod_table64 = (int64_t *)nr_64qam_mod_table_fp16; 
+      else  
+#endif
+       nr_mod_table64 = (int64_t *)nr_64qam_mod_table; 
       if (length > (3 * 64))
         for (i = 0; i < length - 3 * 64; i += 3 * 64) {
           uint64_t x = *in64++;
           uint64_t x1 = x & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x >> 12) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x >> 24) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x >> 36) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x >> 48) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           uint64_t x2 = (x >> 60);
           x = *in64++;
           x2 |= x << 4;
           x1 = x2 & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 12) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 24) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 36) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 48) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x2 = ((x >> 56) & 0xf0) | (x2 >> 60);
           x = *in64++;
           x2 |= x << 8;
           x1 = x2 & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 12) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 24) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 36) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x1 = (x2 >> 48) & 0xfff;
-          *out64++ = nr_64qam_mod_table[x1];
+          *out64++ = nr_mod_table64[x1];
           x2 = ((x >> 52) & 0xff0) | (x2 >> 60);
-          *out64++ = nr_64qam_mod_table[x2];
+          *out64++ = nr_mod_table64[x2];
         }
 
       while (i + 24 <= length) {
         uint32_t xx = 0;
         memcpy(&xx, in_bytes + i / 8, 3);
         uint64_t x1 = xx & 0xfff;
-        *out64++ = nr_64qam_mod_table[x1];
+        *out64++ = nr_mod_table64[x1];
         x1 = (xx >> 12) & 0xfff;
-        *out64++ = nr_64qam_mod_table[x1];
+        *out64++ = nr_mod_table64[x1];
         i += 24;
       }
       if (i != length) {
         uint32_t xx = 0;
         memcpy(&xx, in_bytes + i / 8, 2);
         uint64_t x1 = xx & 0xfff;
-        *out64++ = nr_64qam_mod_table[x1];
+        *out64++ = nr_mod_table64[x1];
       }
       return;
 
-    case 8: {
-      int32_t *nr_mod_table32 = (int32_t *)nr_256qam_mod_table;
+    case 8:
+#ifdef FLT16_MAX
+      if (out_fp16) 
+       nr_mod_table32 = (int32_t *)nr_256qam_mod_table_fp16; 
+      else 
+#endif
+       nr_mod_table32 = (int32_t *)nr_256qam_mod_table;
       for (i = 0; i < length / 8; i++)
         out32[i] = nr_mod_table32[in_bytes[i]];
-    }
       return;
 
     default:
@@ -589,7 +656,11 @@ void nr_dft(c16_t *z, c16_t *d, uint32_t Msc_PUSCH)
   }
 }
 
-void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation)
+void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation
+#ifdef FLT16_MAX
+			  ,int use_fp16
+#endif
+		)
 {
   const int nsymb = fp->symbols_per_slot * fp->slots_per_frame / 10;
   const double Tc = (1 / 480e3 / 4096);
@@ -615,23 +686,35 @@ void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rot
     poff = 2 * M_PI * (tl + (Ncp * Tc)) * f0;
     exp_re = cos(poff);
     exp_im = sin(-poff);
-    symbol_rotation[l].r = (int16_t)floor(exp_re * 32767);
-    symbol_rotation[l].i = (int16_t)floor(exp_im * 32767);
-
-    LOG_D(PHY,
-          "Symbol rotation %d/%d => tl %f (%d,%d) (%f)\n",
-          l,
-          nsymb,
-          tl,
-          symbol_rotation[l].r,
-          symbol_rotation[l].i,
-          (poff / 2 / M_PI) - floor(poff / 2 / M_PI));
-
+ #ifdef FLT16_MAX
+    if (use_fp16) {
+      (*(cf16_t*)&symbol_rotation[l]).r = (_Float16)exp_re;
+      (*(cf16_t*)&symbol_rotation[l]).i = (_Float16)exp_im;
+//      printf("initialize fp16 rotation  %d: %f %f\n", l, (double) (*(cf16_t*)&symbol_rotation[l]).r, (double) (*(cf16_t*)&symbol_rotation[l]).i);
+    }
+    else
+#endif
+    {
+      symbol_rotation[l].r = (int16_t)floor(exp_re * 32767);
+      symbol_rotation[l].i = (int16_t)floor(exp_im * 32767);
+      LOG_D(PHY,
+            "Symbol rotation %d/%d => tl %f (%d,%d) (%f)\n",
+            l,
+            nsymb,
+            tl,
+            symbol_rotation[l].r,
+            symbol_rotation[l].i,
+            (poff / 2 / M_PI) - floor(poff / 2 / M_PI));
+    }
     tl += (Nu + Ncp) * Tc;
   }
 }
 
-void init_symbol_rotation(NR_DL_FRAME_PARMS *fp)
+void init_symbol_rotation(NR_DL_FRAME_PARMS *fp
+#ifdef FLT16_MAX
+			  ,int use_fp16
+#endif
+		)
 {
   double f[2] = {(double)fp->dl_CarrierFreq, (double)fp->ul_CarrierFreq};
 
@@ -640,8 +723,16 @@ void init_symbol_rotation(NR_DL_FRAME_PARMS *fp)
     if (f0 == 0)
       continue;
     c16_t *rot = fp->symbol_rotation[ll];
-
-    perform_symbol_rotation(fp, f0, rot);
+#ifdef FLT16_MAX
+    printf("Initializing rotation %d fp16 %s\n",ll,use_fp16?"yes":"no");
+#else
+    printf("Initializing rotation %d\n",ll);
+#endif
+    perform_symbol_rotation(fp, f0, rot
+#ifdef FLT16_MAX
+			  ,use_fp16
+#endif
+		    );
   }
 }
 
@@ -706,7 +797,11 @@ c16_t nr_layer_precoder_cm(int n_layers,
                            c16_t datatx_F_precoding[n_layers][symSz],
                            int ap,
                            nfapi_nr_pm_pdu_t *pmi_pdu,
-                           int offset)
+                           int offset
+#ifdef FLT16_MAX
+			   ,int use_fp16
+#endif
+	 	          )
 {
   c16_t precodatatx_F = {0};
   for (int al = 0; al < n_layers; al++) {
@@ -808,6 +903,9 @@ static inline __attribute__((always_inline)) __m128i cmac_prec128(__m128i y, __m
   const Type w_s##Rank = Instruct(c16toI32(c16swap(pmi_pdu->weights[Rank][ant]))); \
   const Type *in##Rank = (Type *)(txdataF_res_mapped[Rank] + sc_offset + (out-beginning));
 
+#define load_consts_fp16(Type, Instruct, Rank)                                          \
+  const Type w##Rank = (Type)Instruct(*(uint32_t *)&(weights_fp16[Rank])); \
+  const Type *in##Rank = (Type*)(txdataF_res_mapped[Rank] + sc_offset + (out-beginning));
 void nr_layer_precoder_simd(const int n_layers,
                             const int symSz,
                             const c16_t txdataF_res_mapped[n_layers][symSz],
@@ -815,7 +913,11 @@ void nr_layer_precoder_simd(const int n_layers,
                             const nfapi_nr_pm_pdu_t *pmi_pdu,
                             const int sc_offset,
                             const int re_cnt,
-                            c16_t *txdataF_precoded)
+                            c16_t *txdataF_precoded
+#ifdef FLT16_MAX
+			    ,int use_fp16
+#endif
+			    )
 {
   // For x86, use 256 SIMD for every 8 RE and 128 SIMD for last 4 RE
   // For aarch64, use 128 SIMD for every 4 RE
@@ -824,7 +926,63 @@ void nr_layer_precoder_simd(const int n_layers,
   // 512/256 SIMD: Do 16/8 RE in one iteration, 3 iterations for 2 RB
   c16_t *beginning = txdataF_precoded + sc_offset;
   c16_t *out=beginning;
-#if defined(__AVX512F__) && defined(__AVX512BW__)
+  printf("nr_layer_precoder_simd, use_fp16 %d\n",use_fp16);
+
+// 512-bit SIMD section
+#if defined(__AVX512BW__)
+  c16_t *end = out + (re_cnt & ~15);
+#if defined(FLT16_MAX) && defined(__AVX512FP16__)
+  cf16_t weights_fp16[n_layers];
+  if (use_fp16) {
+    for (int l=0;l<n_layers;l++) {
+      weights_fp16[l].r = ((_Float16)pmi_pdu->weights[l][ant].r)/32768.0;
+      weights_fp16[l].i = ((_Float16)pmi_pdu->weights[l][ant].i)/32768.0;
+    }
+    load_consts_fp16(__m512h, _mm512_set1_epi32,0);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(__m512i) / sizeof(*out)) {
+        const __m512h x0 = (__m512h)_mm512_loadu_si512(in0++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m512h y = _mm512_cmul_pch(x0, w0);
+        _mm512_storeu_si512(out, (__m512i)y);
+      }
+    } else if (n_layers == 2) {
+        load_consts_fp16(__m512h, _mm512_set1_epi32,1);
+        const __m512h x0 = (__m512h)_mm512_loadu_si512(in0++);
+        const __m512h x1 = (__m512h)_mm512_loadu_si512(in1++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m512h y = _mm512_cmul_pch(x0, w0);
+	y=_mm512_fmadd_pch(x1,w1,y);
+        _mm512_storeu_si512(out, (__m512i)y);
+    } else if (n_layers == 3) {
+        load_consts_fp16(__m512h, _mm512_set1_epi32,1);
+        load_consts_fp16(__m512h, _mm512_set1_epi32,2);
+        const __m512h x0 = (__m512h)_mm512_loadu_si512(in0++);
+        const __m512h x1 = (__m512h)_mm512_loadu_si512(in1++);
+        const __m512h x2 = (__m512h)_mm512_loadu_si512(in2++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m512h y = _mm512_cmul_pch(x0, w0);
+	y=_mm512_fmadd_pch(x1,w1,y);
+	y=_mm512_fmadd_pch(x2,w2,y);
+        _mm512_storeu_si512(out, (__m512i)y);
+    } else if (n_layers == 4) {
+        load_consts_fp16(__m512h, _mm512_set1_epi32,1);
+        load_consts_fp16(__m512h, _mm512_set1_epi32,2);
+        load_consts_fp16(__m512h, _mm512_set1_epi32,3);
+        const __m512h x0 = (__m512h)_mm512_loadu_si512(in0++);
+        const __m512h x1 = (__m512h)_mm512_loadu_si512(in1++);
+        const __m512h x2 = (__m512h)_mm512_loadu_si512(in2++);
+        const __m512h x3 = (__m512h)_mm512_loadu_si512(in3++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m512h y = _mm512_cmul_pch(x0, w0);
+	y=_mm512_fmadd_pch(x1,w1,y);
+	y=_mm512_fmadd_pch(x2,w2,y);
+	y=_mm512_fmadd_pch(x3,w3,y);
+        _mm512_storeu_si512(out, (__m512i)y);
+    }
+  }
+  else
+#endif
   {
     c16_t *end = out + (re_cnt & ~15);
     load_consts(__m512i, _mm512_set1_epi32, 0);
@@ -877,7 +1035,55 @@ void nr_layer_precoder_simd(const int n_layers,
     }
   }
 #endif
+  // 256-bit SIMD section
 #ifdef __AVX2__
+#if defined(FLT16_MAX) && defined(__AVX512FP16__)
+  if (use_fp16) {
+    load_consts_fp16(__m256h, _mm256_set1_epi32,0);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(__m256i) / sizeof(*out)) {
+        const __m256h x0 = (__m256h)_mm256_loadu_si256((__m256i*)in0++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m256h y = _mm256_cmul_pch(x0, w0);
+        _mm256_storeu_si256((__m256i*)out, (__m256i)y);
+      }
+    } else if (n_layers == 2) {
+        load_consts_fp16(__m256h, _mm256_set1_epi32,1);
+        const __m256h x0 = (__m256h)_mm256_loadu_si256((__m256i*)in0++);
+        const __m256h x1 = (__m256h)_mm256_loadu_si256((__m256i*)in1++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m256h y = _mm256_cmul_pch(x0, w0);
+	y=_mm256_fmadd_pch(x1,w1,y);
+        _mm256_storeu_si256((__m256i*)out, (__m256i)y);
+    } else if (n_layers == 3) {
+        load_consts_fp16(__m256h, _mm256_set1_epi32,1);
+        load_consts_fp16(__m256h, _mm256_set1_epi32,2);
+        const __m256h x0 = (__m256h)_mm256_loadu_si256((__m256i*)in0++);
+        const __m256h x1 = (__m256h)_mm256_loadu_si256((__m256i*)in1++);
+        const __m256h x2 = (__m256h)_mm256_loadu_si256((__m256i*)in2++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m256h y = _mm256_cmul_pch(x0, w0);
+	y=_mm256_fmadd_pch(x1,w1,y);
+	y=_mm256_fmadd_pch(x2,w2,y);
+        _mm256_storeu_si256((__m256i*)out, (__m256i)y);
+    } else if (n_layers == 4) {
+        load_consts_fp16(__m256h, _mm256_set1_epi32,1);
+        load_consts_fp16(__m256h, _mm256_set1_epi32,2);
+        load_consts_fp16(__m256h, _mm256_set1_epi32,3);
+        const __m256h x0 = (__m256h)_mm256_loadu_si256((__m256i*)in0++);
+        const __m256h x1 = (__m256h)_mm256_loadu_si256((__m256i*)in1++);
+        const __m256h x2 = (__m256h)_mm256_loadu_si256((__m256i*)in2++);
+        const __m256h x3 = (__m256h)_mm256_loadu_si256((__m256i*)in3++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m256h y = _mm256_cmul_pch(x0, w0);
+	y=_mm256_fmadd_pch(x1,w1,y);
+	y=_mm256_fmadd_pch(x2,w2,y);
+	y=_mm256_fmadd_pch(x3,w3,y);
+        _mm256_storeu_si256((__m256i*)out, (__m256i)y);
+    }
+  }
+  else
+#endif
   {
     c16_t *end = beginning + (re_cnt & ~7);
     load_consts(simde__m256i, simde_mm256_set1_epi32, 0);
@@ -932,7 +1138,6 @@ void nr_layer_precoder_simd(const int n_layers,
     }
   }
 #endif
-  c16_t *end = beginning + (re_cnt & ~3);
 #ifdef DEBUG_DLSCH_PRECODING_PRINT_WITH_TRIVIAL // Get result with trivial solution, TODO: To be removed
   // 128 SIMD: Do 4 RE in one iteration, 3 iterations for 1 RB
   for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
@@ -943,112 +1148,244 @@ void nr_layer_precoder_simd(const int n_layers,
   }
 #endif
 #ifdef __aarch64__
-  load_consts(int16x8_t, vdupq_n_s16, 0);
-  if (n_layers == 1) {
-    for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
-      const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
-      // Accumulate the product
-      int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
-      // Store the result to txdataF
-      *(int16x8_t *)out = y;
+  c16_t *end = beginning + (re_cnt & ~3);
+#ifdef FLT16_MAX
+  if (use_fp16) {
+    printf("Running precoding for aarch64 fp16\n");
+    cf16_t weights_fp16[n_layers];
+    for (int l=0;l<n_layers;l++) {
+      weights_fp16[l].r = ((float16_t)pmi_pdu->weights[l][ant].r)/32768.0;
+      weights_fp16[l].i = ((float16_t)pmi_pdu->weights[l][ant].i)/32768.0;
+    }
+    load_consts_fp16(float16x8_t, vdupq_n_f16, 0);
+    float16x8_t zero=vdupq_n_f16(0.0f);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const float16x8_t x0 = vld1q_f16((float16_t*)(in0++));
+        // Accumulate the product
+        float16x8_t y = vcmlaq_f16(zero,x0,w0);
+        y = vcmlaq_rot90_f16(y,x0,w0);
+        // Store the result to txdataF
+        *(float16x8_t *)out = y;
+	printf("re %ld %f.%f\n",out-beginning, ((cf16_t*)out)[0].r, ((cf16_t*)out)[0].i);
+	printf("re %ld %f.%f\n",out-beginning+1, ((cf16_t*)out)[1].r, ((cf16_t*)out)[1].i);
+	printf("re %ld %f.%f\n",out-beginning+2, ((cf16_t*)out)[2].r, ((cf16_t*)out)[2].i);
+	printf("re %ld %f.%f\n",out-beginning+3, ((cf16_t*)out)[3].r, ((cf16_t*)out)[3].i);
+      }
+    }
+    else if (n_layers == 2) {
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 1);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const float16x8_t x0 = vld1q_f16((float16_t*)(in0++));
+        const float16x8_t x1 = vld1q_f16((float16_t*)(in1++));
+        // Accumulate the product
+        float16x8_t y = vcmlaq_f16(zero,x0,w0);
+        y = vcmlaq_rot90_f16(y,x0,w0);
+        y = vcmlaq_f16(y,x1,w1);
+        y = vcmlaq_rot90_f16(y,x1,w1);
+        // Store the result to txdataF
+        *(float16x8_t *)out = y;
+      }
+    }
+    else if (n_layers == 3) {
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 1);
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 2);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const float16x8_t x0 = vld1q_f16((float16_t*)(in0++));
+        const float16x8_t x1 = vld1q_f16((float16_t*)(in1++));
+        const float16x8_t x2 = vld1q_f16((float16_t*)(in2++));
+        // Accumulate the product
+        float16x8_t y = vcmlaq_f16(zero,x0,w0);
+        y = vcmlaq_rot90_f16(y,x0,w0);
+        y = vcmlaq_f16(y,x1,w1);
+        y = vcmlaq_rot90_f16(y,x1,w1);
+        y = vcmlaq_f16(y,x2,w2);
+        y = vcmlaq_rot90_f16(y,x2,w2);
+        // Store the result to txdataF
+        *(float16x8_t *)out = y;
+      }
+    }
+    else if (n_layers == 4) {
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 1);
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 2);
+      load_consts_fp16(float16x8_t, vdupq_n_f16, 3);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const float16x8_t x0 = vld1q_f16((float16_t*)(in0++));
+        const float16x8_t x1 = vld1q_f16((float16_t*)(in1++));
+        const float16x8_t x2 = vld1q_f16((float16_t*)(in2++));
+        const float16x8_t x3 = vld1q_f16((float16_t*)(in3++));
+        // Accumulate the product
+        float16x8_t y = vcmlaq_f16(zero,x0,w0);
+        y = vcmlaq_rot90_f16(y,x0,w0);
+        y = vcmlaq_f16(y,x1,w1);
+        y = vcmlaq_rot90_f16(y,x1,w1);
+        y = vcmlaq_f16(y,x2,w2);
+        y = vcmlaq_rot90_f16(y,x2,w2);
+        y = vcmlaq_f16(y,x3,w3);
+        y = vcmlaq_rot90_f16(y,x3,w3);
+        // Store the result to txdataF
+        *(float16x8_t *)out = y;
+      }
     }
   }
-  if (n_layers == 2) {
-    load_consts(int16x8_t, vdupq_n_s16, 1);
-    for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
-      const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
-      const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
-      // Accumulate the product
-      int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      // Store the result to txdataF
-      *(int16x8_t *)out = y;
+  else
+#endif
+  {	  
+    load_consts(int16x8_t, vdupq_n_s16, 0);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
+        // Accumulate the product
+        int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
+        // Store the result to txdataF
+        *(int16x8_t *)out = y;
+      }
     }
-  }
-  if (n_layers == 3) {
-    load_consts(int16x8_t, vdupq_n_s16, 1);
-    load_consts(int16x8_t, vdupq_n_s16, 2);
-    for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
-      const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
-      const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
-      const int16x8_t x2 = vld1q_s16((const int16_t *)in2++);
-      // Accumulate the product
-      int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
-      ;
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      y = cmac_prec128(y, x2, w_c2, w_s2);
-      // Store the result to txdataF
-      *(int16x8_t *)out = y;
+    if (n_layers == 2) {
+      load_consts(int16x8_t, vdupq_n_s16, 1);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
+        const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
+        // Accumulate the product
+        int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        // Store the result to txdataF
+        *(int16x8_t *)out = y;
+      }
     }
-  }
-  if (n_layers == 4) {
-    load_consts(int16x8_t, vdupq_n_s16, 1);
-    load_consts(int16x8_t, vdupq_n_s16, 2);
-    load_consts(int16x8_t, vdupq_n_s16, 3);
-    for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
-      const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
-      const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
-      const int16x8_t x2 = vld1q_s16((const int16_t *)in2++);
-      const int16x8_t x3 = vld1q_s16((const int16_t *)in3++);
-      // Accumulate the product
-      int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
-      ;
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      y = cmac_prec128(y, x2, w_c2, w_s2);
-      y = cmac_prec128(y, x3, w_c3, w_s3);
-      // Store the result to txdataF
-      *(int16x8_t *)out = y;
+    if (n_layers == 3) {
+      load_consts(int16x8_t, vdupq_n_s16, 1);
+      load_consts(int16x8_t, vdupq_n_s16, 2);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
+        const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
+        const int16x8_t x2 = vld1q_s16((const int16_t *)in2++);
+        // Accumulate the product
+        int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        y = cmac_prec128(y, x2, w_c2, w_s2);
+        // Store the result to txdataF
+        *(int16x8_t *)out = y;
+      }
+    }
+    if (n_layers == 4) {
+      load_consts(int16x8_t, vdupq_n_s16, 1);
+      load_consts(int16x8_t, vdupq_n_s16, 2);
+      load_consts(int16x8_t, vdupq_n_s16, 3);
+      for (; out < end; out += sizeof(int16x8_t) / sizeof(*out)) {
+        const int16x8_t x0 = vld1q_s16((const int16_t *)in0++);
+        const int16x8_t x1 = vld1q_s16((const int16_t *)in1++);
+        const int16x8_t x2 = vld1q_s16((const int16_t *)in2++);
+        const int16x8_t x3 = vld1q_s16((const int16_t *)in3++);
+        // Accumulate the product
+        int16x8_t y = cmac0_prec128(x0, w_c0, w_s0);
+        ;
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        y = cmac_prec128(y, x2, w_c2, w_s2);
+        y = cmac_prec128(y, x3, w_c3, w_s3);
+        // Store the result to txdataF
+        *(int16x8_t *)out = y;
+      }
     }
   }
 #else
-  load_consts(simde__m128i, simde_mm_set1_epi32, 0);
-  if (n_layers == 1) {
-    for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
-      const simde__m128i x0 = simde_mm_loadu_si128(in0++);
-      // Accumulate the product
-      simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
-      // Store the result to txdataF
-      simde_mm_storeu_si128(out, y);
+#if defined(FLT16_MAX) && defined(__AVX512FP16__)
+  if (use_fp16) {
+    load_consts_fp16(__m128h, _mm_set1_epi32,0);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(__m128i) / sizeof(*out)) {
+        const __m128h x0 = (__m128h)_mm_loadu_si128((__m128i*)in0++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m128h y = _mm_cmul_pch(x0, w0);
+        _mm_storeu_si128((__m128i*)out, (__m128i)y);
+      }
+    } else if (n_layers == 2) {
+        load_consts_fp16(__m128h, _mm_set1_epi32,1);
+        const __m128h x0 = (__m128h)_mm_loadu_si128((__m128i*)in0++);
+        const __m128h x1 = (__m128h)_mm_loadu_si128((__m128i*)in1++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m128h y = _mm_cmul_pch(x0, w0);
+	y=_mm_fmadd_pch(x1,w1,y);
+        _mm_storeu_si128((__m128i*)out, (__m128i)y);
+    } else if (n_layers == 3) {
+        load_consts_fp16(__m128h, _mm_set1_epi32,1);
+        load_consts_fp16(__m128h, _mm_set1_epi32,2);
+        const __m128h x0 = (__m128h)_mm_loadu_si128((__m128i*)in0++);
+        const __m128h x1 = (__m128h)_mm_loadu_si128((__m128i*)in1++);
+        const __m128h x2 = (__m128h)_mm_loadu_si128((__m128i*)in2++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m128h y = _mm_cmul_pch(x0, w0);
+	y=_mm_fmadd_pch(x1,w1,y);
+	y=_mm_fmadd_pch(x2,w2,y);
+        _mm_storeu_si128((__m128i*)out, (__m128i)y);
+    } else if (n_layers == 4) {
+        load_consts_fp16(__m128h, _mm_set1_epi32,1);
+        load_consts_fp16(__m128h, _mm_set1_epi32,2);
+        load_consts_fp16(__m128h, _mm_set1_epi32,3);
+        const __m128h x0 = (__m128h)_mm_loadu_si128((__m128i*)in0++);
+        const __m128h x1 = (__m128h)_mm_loadu_si128((__m128i*)in1++);
+        const __m128h x2 = (__m128h)_mm_loadu_si128((__m128i*)in2++);
+        const __m128h x3 = (__m128h)_mm_loadu_si128((__m128i*)in3++);
+        // Matrix multiplication for 4 elements of the result (sizeof(simde__m256i) / sizeof(*prec_matrix) = 8)
+        __m128h y = _mm_cmul_pch(x0, w0);
+	y=_mm_fmadd_pch(x1,w1,y);
+	y=_mm_fmadd_pch(x2,w2,y);
+	y=_mm_fmadd_pch(x3,w3,y);
+        _mm_storeu_si128((__m128i*)out, (__m128i)y);
     }
-  } else if (n_layers == 2) {
-    load_consts(simde__m128i, simde_mm_set1_epi32, 1);
-    for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
-      const simde__m128i x0 = simde_mm_loadu_si128(in0++);
-      const simde__m128i x1 = simde_mm_loadu_si128(in1++);
-      // Accumulate the product
-      simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      // Store the result to txdataF
-      simde_mm_storeu_si128(out, y);
-    }
-  } else if (n_layers == 3) {
-    load_consts(simde__m128i, simde_mm_set1_epi32, 1);
-    load_consts(simde__m128i, simde_mm_set1_epi32, 2);
-    for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
-      const simde__m128i x0 = simde_mm_loadu_si128(in0++);
-      const simde__m128i x1 = simde_mm_loadu_si128(in1++);
-      const simde__m128i x2 = simde_mm_loadu_si128(in2++);
-      simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      y = cmac_prec128(y, x2, w_c2, w_s2);
-      // Store the result to txdataF
-      simde_mm_storeu_si128(out, y);
-    }
-  } else if (n_layers == 4) {
-    load_consts(simde__m128i, simde_mm_set1_epi32, 1);
-    load_consts(simde__m128i, simde_mm_set1_epi32, 2);
-    load_consts(simde__m128i, simde_mm_set1_epi32, 3);
-    for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
-      const simde__m128i x0 = simde_mm_loadu_si128(in0++);
-      const simde__m128i x1 = simde_mm_loadu_si128(in1++);
-      const simde__m128i x2 = simde_mm_loadu_si128(in2++);
-      const simde__m128i x3 = simde_mm_loadu_si128(in3++);
-      simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
-      y = cmac_prec128(y, x1, w_c1, w_s1);
-      y = cmac_prec128(y, x2, w_c2, w_s2);
-      y = cmac_prec128(y, x3, w_c3, w_s3);
-      // Store the result to txdataF
-      simde_mm_storeu_si128(out, y);
+  }
+  else
+#endif
+  {
+    load_consts(simde__m128i, simde_mm_set1_epi32, 0);
+    if (n_layers == 1) {
+      for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
+        const simde__m128i x0 = simde_mm_loadu_si128(in0++);
+        // Accumulate the product
+        simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
+        // Store the result to txdataF
+        simde_mm_storeu_si128(out, y);
+      }
+    } else if (n_layers == 2) {
+      load_consts(simde__m128i, simde_mm_set1_epi32, 1);
+      for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
+        const simde__m128i x0 = simde_mm_loadu_si128(in0++);
+        const simde__m128i x1 = simde_mm_loadu_si128(in1++);
+        // Accumulate the product
+        simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        // Store the result to txdataF
+        simde_mm_storeu_si128(out, y);
+      }
+    } else if (n_layers == 3) {
+      load_consts(simde__m128i, simde_mm_set1_epi32, 1);
+      load_consts(simde__m128i, simde_mm_set1_epi32, 2);
+      for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
+        const simde__m128i x0 = simde_mm_loadu_si128(in0++);
+        const simde__m128i x1 = simde_mm_loadu_si128(in1++);
+        const simde__m128i x2 = simde_mm_loadu_si128(in2++);
+        simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        y = cmac_prec128(y, x2, w_c2, w_s2);
+        // Store the result to txdataF
+        simde_mm_storeu_si128(out, y);
+      }
+    } else if (n_layers == 4) {
+      load_consts(simde__m128i, simde_mm_set1_epi32, 1);
+      load_consts(simde__m128i, simde_mm_set1_epi32, 2);
+      load_consts(simde__m128i, simde_mm_set1_epi32, 3);
+      for (; out < end; out += sizeof(simde__m128i) / sizeof(*out)) {
+        const simde__m128i x0 = simde_mm_loadu_si128(in0++);
+        const simde__m128i x1 = simde_mm_loadu_si128(in1++);
+        const simde__m128i x2 = simde_mm_loadu_si128(in2++);
+        const simde__m128i x3 = simde_mm_loadu_si128(in3++);
+        simde__m128i y = cmac0_prec128(x0, w_c0, w_s0);
+        y = cmac_prec128(y, x1, w_c1, w_s1);
+        y = cmac_prec128(y, x2, w_c2, w_s2);
+        y = cmac_prec128(y, x3, w_c3, w_s3);
+        // Store the result to txdataF
+        simde_mm_storeu_si128(out, y);
+      }
     }
   }
 #endif

openair1/PHY/MODULATION/nr_modulation.h

@@ -44,7 +44,11 @@ extern const char nr_W_4l_4p[5][4][4];
 void nr_modulation(const uint32_t *in,
                    uint32_t length,
                    uint16_t mod_order,
-                   int16_t *out);
+                   int16_t *out
+#ifdef FLT16_MAX
+		   ,_Float16 *out_fp16
+#endif
+  		   );
 
 /*! \brief Perform NR layer mapping. TS 38.211 V15.4.0 subclause 7.3.1.3
   @param[in] mod_symbs, double Pointer to modulated symbols for each codeword
@@ -109,11 +113,23 @@ void apply_nr_rotation_TX(const NR_DL_FRAME_PARMS *fp,
                           int slot,
                           int nb_rb,
                           int first_symbol,
-                          int nsymb);
+                          int nsymb
+#ifdef FLT16_MAX
+			  ,int use_fp16,int tx_amp
+#endif
+			  );
 
-void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation);
+void perform_symbol_rotation(NR_DL_FRAME_PARMS *fp, double f0, c16_t *symbol_rotation
+#ifdef FLT16_MAX
+  		,int use_fp16
+#endif
+		);
 
-void init_symbol_rotation(NR_DL_FRAME_PARMS *fp);
+void init_symbol_rotation(NR_DL_FRAME_PARMS *fp
+#ifdef FLT16_MAX
+		,int use_fp16
+#endif
+		);
 
 void init_timeshift_rotation(NR_DL_FRAME_PARMS *fp);
 
@@ -136,7 +152,11 @@ c16_t nr_layer_precoder_cm(int n_layers,
                            c16_t datatx_F_precoding[n_layers][symSz],
                            int ap,
                            nfapi_nr_pm_pdu_t *pmi_pdu,
-                           int offset);
+                           int offset
+#ifdef FLT16_MAX
+			   ,int use_fp16
+#endif
+			   );
 
 /*! \brief Precoding with SIMDe, txdataF_precoded[] = prec_matrix[] * txdataF_res_mapped[]
   @param[in]  txdataF_res_mapped Tx data after resource mapping, before precoding.
@@ -151,5 +171,9 @@ void nr_layer_precoder_simd(const int n_layers,
                             const nfapi_nr_pm_pdu_t *pmi_pdu,
                             const int sc_offset,
                             const int re_cnt,
-                            c16_t *txdataF_precoded);
+                            c16_t *txdataF_precoded
+#ifdef FLT16_MAX
+			   ,int use_fp16
+#endif
+		           );
 #endif

openair1/PHY/MODULATION/ofdm_mod.c

@@ -38,6 +38,8 @@ This section deals with basic functions for OFDM Modulation.
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include "modulation_common.h"
 #include "PHY/LTE_TRANSPORT/transport_common_proto.h"
+#include <math.h>
+#include <float.h>
 //#define DEBUG_OFDM_MOD
 
 // Use 64-byte alignment for IDFT output buffer to ensure no
@@ -292,7 +294,11 @@ void apply_nr_rotation_TX(const NR_DL_FRAME_PARMS *fp,
                           int slot,
                           int nb_rb,
                           int first_symbol,
-                          int nsymb)
+                          int nsymb
+#ifdef FLT16_MAX
+			  ,int use_fp16,int tx_amp
+#endif
+			  )
 {
   int symb_offset = (slot % fp->slots_per_subframe) * fp->symbols_per_slot;
 
@@ -310,19 +316,54 @@ void apply_nr_rotation_TX(const NR_DL_FRAME_PARMS *fp,
       this_rotation->i);
 
     if (nb_rb & 1) {
-      rotate_cpx_vector(this_symbol, this_rotation, this_symbol,
-                        (nb_rb + 1) * 6, 15);
-      rotate_cpx_vector(this_symbol + fp->first_carrier_offset - 6,
-                        this_rotation,
-                        this_symbol + fp->first_carrier_offset - 6,
-                        (nb_rb + 1) * 6, 15);
+#ifdef FLT16_MAX
+      if (use_fp16) {
+        rotate_cpx_vector_fp16((cf16_t*)this_symbol, (cf16_t*)this_rotation, (cf16_t*)this_symbol,
+                          (nb_rb + 1) * 6);
+	fp16_to_q15((cf16_t*)this_symbol,this_symbol,(nb_rb + 1) * 6,tx_amp);
+        rotate_cpx_vector_fp16((cf16_t*)this_symbol + fp->first_carrier_offset - 6,
+                          (cf16_t*)this_rotation,
+                          (cf16_t*)this_symbol + fp->first_carrier_offset - 6,
+                          (nb_rb + 1) * 6);
+	fp16_to_q15((cf16_t*)this_symbol + fp->first_carrier_offset - 6,
+	            this_symbol + fp->first_carrier_offset - 6,
+		    (nb_rb + 1) * 6,tx_amp);
+      }
+      else
+#endif
+      {	      
+        rotate_cpx_vector(this_symbol, this_rotation, this_symbol,
+                          (nb_rb + 1) * 6, 15);
+        rotate_cpx_vector(this_symbol + fp->first_carrier_offset - 6,
+                          this_rotation,
+                          this_symbol + fp->first_carrier_offset - 6,
+                          (nb_rb + 1) * 6, 15);
+      }
     } else {
-      rotate_cpx_vector(this_symbol, this_rotation, this_symbol,
-                        nb_rb * 6, 15);
-      rotate_cpx_vector(this_symbol + fp->first_carrier_offset,
-                        this_rotation,
-                        this_symbol + fp->first_carrier_offset,
-                        nb_rb * 6, 15);
+#ifdef FLT16_MAX
+      if (use_fp16) {
+        rotate_cpx_vector_fp16in_q15out((cf16_t*)this_symbol, (cf16_t*)this_rotation, tx_amp, (cf16_t*)this_symbol,
+                          nb_rb * 6);
+	//fp16_to_q15((cf16_t*)this_symbol,this_symbol,nb_rb  * 6,tx_amp);
+        rotate_cpx_vector_fp16in_q15out((cf16_t*)this_symbol + fp->first_carrier_offset,
+                          (cf16_t*)this_rotation, tx_amp,
+                          (cf16_t*)this_symbol + fp->first_carrier_offset,
+                          nb_rb * 6);
+	//fp16_to_q15((cf16_t*)this_symbol + fp->first_carrier_offset,
+	//	    this_symbol + fp->first_carrier_offset,
+	//	    nb_rb  * 6,
+	//	    tx_amp);
+      }
+      else
+#endif
+      {
+        rotate_cpx_vector(this_symbol, this_rotation, this_symbol,
+                          nb_rb * 6, 15);
+        rotate_cpx_vector(this_symbol + fp->first_carrier_offset,
+                          this_rotation,
+                          this_symbol + fp->first_carrier_offset,
+                          nb_rb * 6, 15);
+      }
     }
   }
 }

openair1/PHY/MODULATION/slot_fep_nr.c

@@ -47,7 +47,7 @@ void nr_symbol_fep(const NR_DL_FRAME_PARMS *frame_parms,
     if (dft_stats) start_meas(dft_stats);
     dft(dftsize, (int16_t *)rxdata[aa], (int16_t *)rxdataF[aa], 1);
     if (dft_stats) stop_meas(dft_stats);
-
+    //if (symbol==11) for (int i =0 ; i<72 ; i++) printf("rxdataF[%d] %d + (%dj)\n",i,(rxdataF[aa]+frame_parms->first_carrier_offset+i)->r,(rxdataF[aa]+frame_parms->first_carrier_offset + i)->i);
     const bool is_sl = (link_type == link_type_sl);
     apply_nr_rotation_symbol_RX(frame_parms,
                                 rxdataF[aa],

openair1/PHY/NR_REFSIG/nr_gen_mod_table.c

@@ -18,61 +18,102 @@
  * For more information about the OpenAirInterface (OAI) Software Alliance:
  *      contact@openairinterface.org
  */
-
 #include "nr_refsig.h"
 #include "nr_mod_table.h"
 c16_t nr_qpsk_mod_table[4];
 simde__m128i nr_qpsk_byte_mod_table[256];
 int32_t nr_16qam_mod_table[16];
-int64_t nr_16qam_byte_mod_table[256];
+
+
+int64_t nr_16qam_byte_mod_table[1024];
+
 int64_t nr_64qam_mod_table[4096];
 int32_t nr_256qam_mod_table[256];
 
+#ifdef FLT16_MAX
+cf16_t nr_qpsk_mod_table_fp16[4];
+int32_t nr_16qam_mod_table_fp16[16];
+simde__m128i nr_qpsk_byte_mod_table_fp16[256];
+int64_t nr_16qam_byte_mod_table_fp16[1024];
+int64_t nr_64qam_mod_table_fp16[4096];
+int32_t nr_256qam_mod_table_fp16[256];
+#endif
+
 void nr_generate_modulation_table() {
   float sqrt2 = 0.70711;
   float sqrt10 = 0.31623;
   float sqrt42 = 0.15430;
   float sqrt170 = 0.076696;
   float val = 32768.0;
-  uint32_t i,j;
-  short* table;
-
+  int i,j;
+  int16_t* table;
+#ifdef FLT16_MAX
+  _Float16* table2;
+#endif
   // QPSK
   for (i=0; i<4; i++) {
     nr_qpsk_mod_table[i].r = (short)(1 - 2 * (i & 1)) * val * sqrt2 * sqrt2;
     nr_qpsk_mod_table[i].i = (short)(1 - 2 * ((i >> 1) & 1)) * val * sqrt2 * sqrt2;
+#ifdef FLT16_MAX
+    nr_qpsk_mod_table_fp16[i].r = (_Float16)((1 - 2 * (i & 1)) *  sqrt2 * sqrt2);
+    nr_qpsk_mod_table_fp16[i].i = (_Float16)((1 - 2 * ((i >> 1) & 1)) * sqrt2 * sqrt2);
+#endif
     //printf("%d j%d\n",nr_qpsk_mod_table[i*2],nr_qpsk_mod_table[i*2+1]);
   }
 
   //QPSK m128
-  table = (short*) nr_qpsk_byte_mod_table;
+  table = (int16_t*) nr_qpsk_byte_mod_table;
+#ifdef FLT16_MAX
+  table2 = (_Float16*) nr_qpsk_byte_mod_table_fp16;
+#endif
   for (i=0; i<256; i++) {
     for (j=0; j<4; j++) {
-      *table++ = (short)(1 - 2 * ((i >> (j * 2)) & 1)) * val * sqrt2 * sqrt2;
-      *table++ = (short)(1 - 2 * ((i >> (j * 2 + 1)) & 1)) * val * sqrt2 * sqrt2;
+      *table++ = (int16_t)(1 - 2 * ((i >> (j * 2)) & 1)) * val * sqrt2 * sqrt2;
+      *table++ = (int16_t)(1 - 2 * ((i >> (j * 2 + 1)) & 1)) * val * sqrt2 * sqrt2;
+#ifdef FLT16_MAX
+      *table2++ = (_Float16)((1 - 2 * ((i >> (j * 2)) & 1)) * sqrt2 * sqrt2);
+      *table2++ = (_Float16)((1 - 2 * ((i >> (j * 2 + 1)) & 1)) * sqrt2 * sqrt2);
+#endif
       //printf("%d j%d\n",nr_qpsk_byte_mod_table[i*8+(j*2)],nr_qpsk_byte_mod_table[i*8+(j*2)+1]);
     }
   }
 
   //16QAM
-  table = (short*) nr_16qam_byte_mod_table;
+  table = (int16_t*) nr_16qam_byte_mod_table;
+#ifdef FLT16_MAX
+  table2 = (_Float16*) nr_16qam_byte_mod_table_fp16;
+#endif
   for (i=0; i<256; i++) {
     for (j=0; j<2; j++) {
-      *table++ = (short)((1 - 2 * ((i >> (j * 4)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 2)) & 1)))) * val * sqrt10 * sqrt2;
-      *table++ = (short)((1 - 2 * ((i >> (j * 4 + 1)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 3)) & 1)))) * val * sqrt10 * sqrt2;
+      *table++ = (int16_t)((1 - 2 * ((i >> (j * 4)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 2)) & 1)))) * val * sqrt10 * sqrt2;
+      *table++ = (int16_t)((1 - 2 * ((i >> (j * 4 + 1)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 3)) & 1)))) * val * sqrt10 * sqrt2;
+#ifdef FLT16_MAX
+      *table2++ = (_Float16)(((1 - 2 * ((i >> (j * 4)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 2)) & 1)))) * sqrt10 * sqrt2);
+      *table2++ = (_Float16)(((1 - 2 * ((i >> (j * 4 + 1)) & 1)) * (2 - (1 - 2 * ((i >> (j * 4 + 3)) & 1)))) * sqrt10 * sqrt2);
+#endif
       // printf("%d j%d\n",nr_16qam_byte_mod_table[i*4+(j*2)],nr_16qam_byte_mod_table[i*4+(j*2)+1]);
     }
   }
 
-  table = (short*) nr_16qam_mod_table;
+  table = (int16_t*) nr_16qam_mod_table;
+#ifdef FLT16_MAX
+  table2 = (_Float16*) nr_16qam_mod_table_fp16;
+#endif
   for (i=0; i<16; i++) {
-    *table++ = (short)((1 - 2 * (i & 1)) * (2 - (1 - 2 * ((i >> 2) & 1)))) * val * sqrt10 * sqrt2;
-    *table++ = (short)((1 - 2 * ((i >> 1) & 1)) * (2 - (1 - 2 * ((i >> 3) & 1)))) * val * sqrt10 * sqrt2;
-    //printf("%d j%d\n",table[i*2],table[i*2+1]);
+    *table++ = (int16_t)((1 - 2 * (i & 1)) * (2 - (1 - 2 * ((i >> 2) & 1)))) * val * sqrt10 * sqrt2;
+    *table++ = (int16_t)((1 - 2 * ((i >> 1) & 1)) * (2 - (1 - 2 * ((i >> 3) & 1)))) * val * sqrt10 * sqrt2;
+#ifdef FLT16_MAX
+    *table2++ = (_Float16)(((1 - 2 * (i & 1)) * (2 - (1 - 2 * ((i >> 2) & 1)))) * sqrt10 * sqrt2);
+    *table2++ = (_Float16)(((1 - 2 * ((i >> 1) & 1)) * (2 - (1 - 2 * ((i >> 3) & 1)))) * sqrt10 * sqrt2);
+#endif
+      //printf("%d j%d\n",table[i*2],table[i*2+1]);
   }
 
   //64QAM
   table = (short*) nr_64qam_mod_table;
+#ifdef FLT16_MAX
+  table2 = (_Float16*) nr_64qam_mod_table_fp16;
+#endif
   for (i=0; i<4096; i++) {
     for (j=0; j<2; j++) {
       *table++ = (short)((1 - 2 * ((i >> (j * 6)) & 1))
@@ -81,19 +122,48 @@ void nr_generate_modulation_table() {
       *table++ = (short)((1 - 2 * ((i >> (j * 6 + 1)) & 1))
                          * (4 - (1 - 2 * ((i >> (j * 6 + 3)) & 1)) * (2 - (1 - 2 * ((i >> (j * 6 + 5)) & 1)))))
                  * val * sqrt42 * sqrt2;
+#ifdef FLT16_MAX
+      *table2++ = (_Float16)((1 - 2 * ((i >> (j * 6)) & 1))
+                         * (4 - (1 - 2 * ((i >> (j * 6 + 2)) & 1)) * (2 - (1 - 2 * ((i >> (j * 6 + 4)) & 1)))))
+                 * sqrt42 * sqrt2;
+      *table2++ = (_Float16)((1 - 2 * ((i >> (j * 6 + 1)) & 1))
+                         * (4 - (1 - 2 * ((i >> (j * 6 + 3)) & 1)) * (2 - (1 - 2 * ((i >> (j * 6 + 5)) & 1)))))
+                 * sqrt42 * sqrt2;
+#endif
       //printf("%d j%d\n",table[i*4+(j*2)],table[i*4+(j*2)+1]);
     }
   }
 
   //256QAM
   table = (short*) nr_256qam_mod_table;
+#ifdef FLT16_MAX
+  table2 = (_Float16*) nr_256qam_mod_table_fp16;
+#endif
   for (i=0; i<256; i++) {
-    *table++ = (short)((1 - 2 * (i & 1))
+    table[2*i] = (short)((1 - 2 * (i & 1))
                        * (8 - (1 - 2 * ((i >> 2) & 1)) * (4 - (1 - 2 * ((i >> 4) & 1)) * (2 - (1 - 2 * ((i >> 6) & 1))))))
                * val * sqrt170 * sqrt2;
-    *table++ = (short)((1 - 2 * ((i >> 1) & 1))
+    table[1+2*i] = (short)((1 - 2 * ((i >> 1) & 1))
                        * (8 - (1 - 2 * ((i >> 3) & 1)) * (4 - (1 - 2 * ((i >> 5) & 1)) * (2 - (1 - 2 * ((i >> 7) & 1))))))
                * val * sqrt170 * sqrt2;
+#ifdef FLT16_MAX
+    table2[2*i] = (_Float16)(((1 - 2 * (i & 1))
+                       * (8 - (1 - 2 * ((i >> 2) & 1)) * (4 - (1 - 2 * ((i >> 4) & 1)) * (2 - (1 - 2 * ((i >> 6) & 1))))))
+               * sqrt170 * sqrt2);
+    table2[1+2*i] = (_Float16)(((1 - 2 * ((i >> 1) & 1))
+                       * (8 - (1 - 2 * ((i >> 3) & 1)) * (4 - (1 - 2 * ((i >> 5) & 1)) * (2 - (1 - 2 * ((i >> 7) & 1))))))
+               * sqrt170 * sqrt2);
+/*   printf("256QAM %d : %f,%f, (%f,%f) (%f,%f)\n",i,(float)table2[2*i],(float)table2[1+2*i],
+		  (float)((1 - 2 * (i & 1))
+		                         * (8 - (1 - 2 * ((i >> 2) & 1)) * (4 - (1 - 2 * ((i >> 4) & 1)) * (2 - (1 - 2 * ((i >> 6) & 1))))))
+		  * sqrt170 * sqrt2,
+		  (float)((1 - 2 * ((i >> 1) & 1))
+		                        * (8 - (1 - 2 * ((i >> 3) & 1)) * (4 - (1 - 2 * ((i >> 5) & 1)) * (2 - (1 - 2 * ((i >> 7) & 1))))))
+		  * sqrt170 * sqrt2, ((double)table[2*i])/32768.0,((double)table[1+2*i])/32768  
+		   
+		   );
+*/
+#endif
   }
 }
 

openair1/PHY/NR_REFSIG/nr_mod_table.h

@@ -22,6 +22,10 @@
 #ifndef __PHY_NR_REFSIG_NR_MOD_TABLE__H__
 #define __PHY_NR_REFSIG_NR_MOD_TABLE__H__
 
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 #define NR_MOD_TABLE_SIZE_SHORT 686
 #define NR_MOD_TABLE_BPSK_OFFSET 1
 #define NR_MOD_TABLE_QAM16_OFFSET 7
@@ -29,12 +33,33 @@
 #define NR_MOD_TABLE_QAM256_OFFSET 87
 
 extern c16_t nr_qpsk_mod_table[4];
-extern simde__m128i nr_qpsk_byte_mod_table[256];
+#ifdef FLT16_MAX
+extern cf16_t nr_qpsk_mod_table_fp16[4];
+#endif
 
 extern int32_t nr_16qam_mod_table[16];
-extern int64_t nr_16qam_byte_mod_table[256];
+#ifdef FLT16_MAX
+extern int32_t nr_16qam_mod_table_fp16[16];
+#endif
+#if defined(__SSE2__) || defined(__aarch64__)
+extern simde__m128i nr_qpsk_byte_mod_table[256];
+#ifdef FLT16_MAX
+extern simde__m128i nr_qpsk_byte_mod_table_fp16[256];
+#endif
+#endif
 
+extern int64_t nr_16qam_byte_mod_table[1024];
+#ifdef FLT16_MAX
+extern int64_t nr_16qam_byte_mod_table_fp16[1024];
+#endif
 extern int64_t nr_64qam_mod_table[4096];
 
-extern int32_t nr_256qam_mod_table[256];
+#ifdef FLT16_MAX
+extern int64_t nr_64qam_mod_table_fp16[4096];
+#endif
+
+extern int32_t nr_256qam_mod_table[256];
+#ifdef FLT16_MAX
+extern int32_t nr_256qam_mod_table_fp16[256];
+#endif
 #endif

openair1/PHY/NR_REFSIG/nr_refsig.h

@@ -39,7 +39,7 @@ int nr_pusch_dmrs_rx(PHY_VARS_gNB *gNB,
                      uint8_t dmrs_type,
                      int16_t dmrs_scaling);
 
-void nr_generate_modulation_table(void);
+void nr_generate_modulation_table();
 
 extern simde__m128i byte2m128i[256];
 

openair1/PHY/NR_TRANSPORT/nr_dci.c

@@ -34,6 +34,10 @@
 #include "nr_dci.h"
 #include "nr_dlsch.h"
 #include "nr_sch_dmrs.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 #include "PHY/MODULATION/nr_modulation.h"
 #include "common/utils/nr/nr_common.h"
 #include "SCHED_NR/sched_nr.h"
@@ -116,8 +120,15 @@ void nr_generate_dci(PHY_VARS_gNB *gNB,
     /// DMRS QPSK modulation
     for (int symb = cset_start_symb; symb < cset_start_symb + pdcch_pdu_rel15->DurationSymbols; symb++) {
       const uint32_t *gold = nr_gold_pdcch(frame_parms->N_RB_DL, frame_parms->symbols_per_slot, dci_pdu->ScramblingId, slot, symb);
-      nr_modulation(gold, dmrs_length, DMRS_MOD_ORDER, (int16_t *)mod_dmrs[symb]); // Qm = 2 as DMRS is QPSK modulated
+#ifdef FLT16_MAX
+      if (gNB->use_fp16)
+        nr_modulation(gold, dmrs_length, DMRS_MOD_ORDER, NULL,(_Float16 *)mod_dmrs[symb]); // Qm = 2 as DMRS is QPSK modulated
+      else
+        nr_modulation(gold, dmrs_length, DMRS_MOD_ORDER, (int16_t *)mod_dmrs[symb],NULL); // Qm = 2 as DMRS is QPSK modulated
 
+#else
+      nr_modulation(gold, dmrs_length, DMRS_MOD_ORDER, (int16_t *)mod_dmrs[symb]); // Qm = 2 as DMRS is QPSK modulated
+#endif
 #ifdef DEBUG_PDCCH_DMRS
       if(dci_pdu->RNTI!=0xFFFF) {
         for (int i=0; i<dmrs_length>>1; i++)
@@ -165,11 +176,22 @@ void nr_generate_dci(PHY_VARS_gNB *gNB,
 #endif
     /// QPSK modulation
     c16_t mod_dci[NR_MAX_DCI_SIZE / 2] __attribute__((aligned(16)));
-    nr_modulation(scrambled_output, encoded_length, DMRS_MOD_ORDER, (int16_t *)mod_dci); // Qm = 2 as DMRS is QPSK modulated
+#ifdef FLT16_MAX
+    if (gNB->use_fp16) 
+      nr_modulation(scrambled_output, encoded_length, DMRS_MOD_ORDER, NULL, (_Float16 *)mod_dci); // Qm = 2 as DMRS is QPSK modulated
+    else
+      nr_modulation(scrambled_output, encoded_length, DMRS_MOD_ORDER, (int16_t *)mod_dci,NULL); // Qm = 2 as DMRS is QPSK modulated
+#else
+      nr_modulation(scrambled_output, encoded_length, DMRS_MOD_ORDER, (int16_t *)mod_dci); // Qm = 2 as DMRS is QPSK modulated
+#endif
 #ifdef DEBUG_DCI
     
     for (int i=0; i<encoded_length>>1; i++)
-      printf("i %d mod_dci %d %d\n", i, mod_dci[i].r, mod_dci[i].i);
+#ifdef FLT16_MAX
+      if (gNB->use_fp16) printf("i %d mod_dci %f %f\n", i, ((cf16_t *)mod_dci)[i].r, ((cf16_t *)mod_dci)[i].i);
+      else 
+#endif
+        printf("i %d mod_dci %d %d\n", i, mod_dci[i].r, mod_dci[i].i);
 
 #endif
 
@@ -199,36 +221,68 @@ void nr_generate_dci(PHY_VARS_gNB *gNB,
           dmrs_idx = (reg_list[d][reg_count] + rb_offset) * 3;
 
         int k_prime = 0;
-
-        for (int m = 0; m < NR_NB_SC_PER_RB; m++) {
-          if (m == (k_prime << 2) + 1) { // DMRS if not already mapped
-            txdataF[l * frame_parms->ofdm_symbol_size + k] = c16mulRealShift(mod_dmrs[l][dmrs_idx], amp, 15);
+#ifdef FLT16_MAX
+        if (gNB->use_fp16) {
+          for (int m = 0; m < NR_NB_SC_PER_RB; m++) {
+            if (m == (k_prime << 2) + 1) { // DMRS if not already mapped
+              txdataF[l * frame_parms->ofdm_symbol_size + k] = mod_dmrs[l][dmrs_idx];
 
 #ifdef DEBUG_PDCCH_DMRS
-            LOG_I(NR_PHY_DCI,
-                  "PDCCH DMRS %d: l %d position %d => (%d,%d)\n",
-                  dmrs_idx,
-                  l,
-                  k,
-                  txdataF[l * frame_parms->ofdm_symbol_size + k].r,
-                  txdataF[l * frame_parms->ofdm_symbol_size + k].i);
+              LOG_I(NR_PHY_DCI,
+                    "PDCCH DMRS %d: l %d position %d => (%d,%d)\n",
+                    dmrs_idx,
+                    l,
+                    k,
+                    txdataF[l * frame_parms->ofdm_symbol_size + k].r,
+                    txdataF[l * frame_parms->ofdm_symbol_size + k].i);
 #endif
 
-            dmrs_idx++;
-            k_prime++;
+              dmrs_idx++;
+              k_prime++;
 
-          } else { // DCI payload
-            txdataF[l * frame_parms->ofdm_symbol_size + k] = c16mulRealShift(mod_dci[dci_idx], amp, 15);
-            dci_idx++;
-          }
+            } else { // DCI payload
+              txdataF[l * frame_parms->ofdm_symbol_size + k] = mod_dci[dci_idx];
+              dci_idx++;
+            }
 
-          k++;
+            k++;
 
-          if (k >= frame_parms->ofdm_symbol_size)
-            k -= frame_parms->ofdm_symbol_size;
-        } // m
-      } // reg_count
-    } // symbol_idx
+            if (k >= frame_parms->ofdm_symbol_size)
+              k -= frame_parms->ofdm_symbol_size;
+          } // m
+
+	}
+	else
+#endif
+          for (int m = 0; m < NR_NB_SC_PER_RB; m++) {
+            if (m == (k_prime << 2) + 1) { // DMRS if not already mapped
+              txdataF[l * frame_parms->ofdm_symbol_size + k] = c16mulRealShift(mod_dmrs[l][dmrs_idx], amp, 15);
+
+#ifdef DEBUG_PDCCH_DMRS
+              LOG_I(NR_PHY_DCI,
+                    "PDCCH DMRS %d: l %d position %d => (%d,%d)\n",
+                    dmrs_idx,
+                    l,
+                    k,
+                    txdataF[l * frame_parms->ofdm_symbol_size + k].r,
+                    txdataF[l * frame_parms->ofdm_symbol_size + k].i);
+#endif
+
+              dmrs_idx++;
+              k_prime++;
+
+            } else { // DCI payload
+              txdataF[l * frame_parms->ofdm_symbol_size + k] = c16mulRealShift(mod_dci[dci_idx], amp, 15);
+              dci_idx++;
+            }
+
+            k++;
+
+            if (k >= frame_parms->ofdm_symbol_size)
+              k -= frame_parms->ofdm_symbol_size;
+          } // m
+        } // reg_count
+      } // symbol_idx
 
     LOG_D(NR_PHY_DCI,
           "DCI: payloadSize = %d | payload = %llx\n",

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -1,7 +1,6 @@
 /*
  * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
+ * contributor license agreements.  See the NOTICE file dis* this work for additional information regarding copyright ownership.
  * The OpenAirInterface Software Alliance licenses this file to You under
  * the OAI Public License, Version 1.1  (the "License"); you may not use this file
  * except in compliance with the License.
@@ -56,8 +55,9 @@ static int do_ptrs_symbol(const nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15,
                           int symbol_sz,
                           c16_t *txF,
                           c16_t *tx_layer,
-                          int amp,
-                          c16_t *mod_ptrs)
+                          void *amp,
+                          c16_t *mod_ptrs,
+			  int use_fp16)
 {
   int ptrs_idx = 0;
   int k = start_sc;
@@ -69,7 +69,10 @@ static int do_ptrs_symbol(const nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15,
     if (is_ptrs_re) {
       /* check if cuurent RE is PTRS RE*/
       uint16_t beta_ptrs = 1;
-      txF[k] = c16mulRealShift(mod_ptrs[ptrs_idx], beta_ptrs * amp, 15);
+      if (use_fp16)
+        txF[k] = mod_ptrs[ptrs_idx];
+      else
+        txF[k] = c16mulRealShift(mod_ptrs[ptrs_idx], beta_ptrs * *(int16_t*)amp, 15);
 #ifdef DEBUG_DLSCH_MAPPING
       printf("ptrs_idx %d\t \t k %d \t \t txdataF: %d %d, mod_ptrs: %d %d\n",
              ptrs_idx,
@@ -81,7 +84,10 @@ static int do_ptrs_symbol(const nfapi_nr_dl_tti_pdsch_pdu_rel15_t *rel15,
 #endif
       ptrs_idx++;
     } else {
-      txF[k] = c16mulRealShift(*in++, amp, 15);
+      if (use_fp16)
+        txF[k] = *in++;
+      else
+        txF[k] = c16mulRealShift(*in++, *(int16_t*)amp, 15);
 #ifdef DEBUG_DLSCH_MAPPING
       printf("k %d \t txdataF: %d %d\n", k, txF[k].r, txF[k].i);
 #endif
@@ -97,59 +103,123 @@ typedef union {
   c16_t s[2];
 } amp_t;
 
-static inline int interleave_with_0_signal_first(c16_t *output, c16_t *mod_dmrs, const int16_t amp_dmrs, int sz)
+static inline int interleave_with_0_signal_first(c16_t *output, c16_t *mod_dmrs, void *amp_dmrs, int sz, int use_fp16)
 {
-#ifdef DEBUG_DLSCH_MAPPING
-  printf("doing DMRS pattern for port 0 : d0 0 d1 0 ... dNm2 0 dNm1 0 (ul %d, rr %d)\n", upper_limit, remaining_re);
-#endif
   // add filler to process all as SIMD
   c16_t *out = output;
   int i = 0;
   int end = sz / 2;
-#if defined(__AVX512BW__)
-  simde__m512i zeros512 = simde_mm512_setzero_si512(), amp_dmrs512 = simde_mm512_set1_epi16(amp_dmrs);
-  simde__m512i perml = simde_mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
-  simde__m512i permh = simde_mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
-  for (; i < (end & ~15); i += 16) {
-    simde__m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(mod_dmrs + i)), amp_dmrs512);
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, perml, zeros512));
-    out += 16;
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, permh, zeros512));
-    out += 16;
+  if (use_fp16) {
+#if defined(__AVX512FP16__) && defined(FLT16_MAX)
+    __m512i zeros512 = _mm512_setzero_si512();
+    __m512h amp_dmrs512 = _mm512_set1_ph(*(int16_t*)amp_dmrs);
+    __m512i perml = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    __m512i permh = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      __m512h d0 = _mm512_mul_ph(_mm512_loadu_ph((__m512h *)(mod_dmrs + i)), amp_dmrs512);
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32((__m512i)d0, perml, zeros512));
+      out += 16;
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32((__m512i)d0, permh, zeros512));
+      out += 16;
+    }
+    __m256i zeros256 = _mm256_setzero_si256();
+    __m256h amp_dmrs256 = _mm256_set1_ph(*(_Float16*)amp_dmrs);
+    for (; i < (end & ~7); i += 8) {
+      __m256h d0 = _mm256_mul_ph(_mm256_loadu_ph((__m256h *)(mod_dmrs + i)), amp_dmrs256);
+      __m256i d2 = _mm256_unpacklo_epi32((__m256i)d0, zeros256);
+      __m256i d3 = _mm256_unpackhi_epi32((__m256i)d0, zeros256);
+      _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256(d2, d3, 32));
+      out += 8;
+      _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256(d2, d3, 49));
+      out += 8;
+    }
+    __m128i zeros = _mm_setzero_si128();
+    __m128h amp_dmrs128 = _mm_set1_ph(*(_Float16*)amp_dmrs);
+    for (; i < (end & ~3); i += 4) {
+      __m128h d0 = _mm_mul_ph(_mm_loadu_ph((__m128h *)(mod_dmrs + i)), amp_dmrs128);
+      __m128i d2 = _mm_unpacklo_epi32((__m128i)d0, zeros);
+      __m128i d3 = _mm_unpackhi_epi32((__m128i)d0, zeros);
+      _mm_storeu_si128((__m128i *)out, d2);
+      out += 4;
+      _mm_storeu_si128((__m128i *)out, d3);
+      out += 4;
+    }
+#elif defined(__aarch64__)
+    uint32x4_t zeros = vdupq_n_u32(0); 
+    float16x8_t amp_dmrs128 = vdupq_n_f16(*(float16_t*)amp_dmrs);
+    //printf("Doing DMRS modulation for 2 CdmGrps, amp_dmrs %f, dmrs0 (%f %f)\n",*(float16_t*)amp_dmrs,((float16_t*)mod_dmrs)[0],((float16_t*)mod_dmrs)[1]);
+    for (; i < (end & ~3); i += 4) {
+      float16x8_t d0 = vmulq_f16(vld1q_f16((float16_t *)(mod_dmrs + i)), amp_dmrs128);
+      float16x8_t d2 = (float16x8_t)vzip1q_u32((uint32x4_t)d0, zeros);
+      float16x8_t d3 = (float16x8_t)vzip2q_u32((uint32x4_t)d0, zeros);
+      vst1q_f16((float16_t *)out, d2);
+      out += 4;
+      vst1q_f16((float16_t *)out, d3);
+      out += 4;
+    }
+#else
+    AssertFatal(1==0,"Architecture doesn't support fp16\n");
+#endif
+    cf16_t *out_cf16 = (cf16_t*)out;
+    for (; i < end; i++) {
+#ifdef __aarch64__
+      out_cf16->r = mod_dmrs[i].r * *(float16_t*)amp_dmrs;
+      out_cf16->i = mod_dmrs[i].i * *(float16_t*)amp_dmrs;
+#else
+      out_cf16->r = mod_dmrs[i].r * *(_Float16*)amp_dmrs;
+      out_cf16->i = mod_dmrs[i].i * *(_Float16*)amp_dmrs;
+#endif
+      out_cf16++;
+      *out_cf16++ = (cf16_t){};
+    }
   }
+  else {
+    //printf("interleave_with_0_signal_first : use_fp16 = 0\n");
+#if defined(__AVX512__) && defined(__AVX512BW__)
+    __m512i zeros512 = _mm512_setzero_si512(), amp_dmrs512 = simde_mm512_set1_epi16(((int16_t*)amp_dmrs);
+    __m512i perml = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    __m512i permh = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      __m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((__m512i *)(mod_dmrs + i)), amp_dmrs512);
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32(d0, perml, zeros512));
+      out += 16;
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32(d0, permh, zeros512));
+      out += 16;
+    }
 #endif
 #if defined(__AVX2__)
-  simde__m256i zeros256 = simde_mm256_setzero_si256(), amp_dmrs256 = simde_mm256_set1_epi16(amp_dmrs);
-  for (; i < (end & ~7); i += 8) {
-    simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(mod_dmrs + i)), amp_dmrs256);
-    simde__m256i d2 = simde_mm256_unpacklo_epi32(d0, zeros256);
-    simde__m256i d3 = simde_mm256_unpackhi_epi32(d0, zeros256);
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
-    out += 8;
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
-    out += 8;
-  }
+    simde__m256i zeros256 = simde_mm256_setzero_si256(), amp_dmrs256 = simde_mm256_set1_epi16(*(int16_t*)amp_dmrs);
+    for (; i < (end & ~7); i += 8) {
+      simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(mod_dmrs + i)), amp_dmrs256);
+      simde__m256i d2 = simde_mm256_unpacklo_epi32(d0, zeros256);
+      simde__m256i d3 = simde_mm256_unpackhi_epi32(d0, zeros256);
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
+      out += 8;
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
+      out += 8;
+    }
 #endif
 #if defined(USE128BIT)
-  simde__m128i zeros = simde_mm_setzero_si128(), amp_dmrs128 = simde_mm_set1_epi16(amp_dmrs);
-  for (; i < (end & ~3); i += 4) {
-    simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(mod_dmrs + i)), amp_dmrs128);
-    simde__m128i d2 = simde_mm_unpacklo_epi32(d0, zeros);
-    simde__m128i d3 = simde_mm_unpackhi_epi32(d0, zeros);
-    simde_mm_storeu_si128((simde__m128i *)out, d2);
-    out += 4;
-    simde_mm_storeu_si128((simde__m128i *)out, d3);
-    out += 4;
-  }
+    simde__m128i zeros = simde_mm_setzero_si128(), amp_dmrs128 = simde_mm_set1_epi16(*(int16_t*)amp_dmrs);
+    for (; i < (end & ~3); i += 4) {
+      simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(mod_dmrs + i)), amp_dmrs128);
+      simde__m128i d2 = simde_mm_unpacklo_epi32(d0, zeros);
+      simde__m128i d3 = simde_mm_unpackhi_epi32(d0, zeros);
+      simde_mm_storeu_si128((simde__m128i *)out, d2);
+      out += 4;
+      simde_mm_storeu_si128((simde__m128i *)out, d3);
+      out += 4;
+    }
 #endif
-  for (; i < end; i++) {
-    *out++ = c16mulRealShift(mod_dmrs[i], amp_dmrs, 15);
-    *out++ = (c16_t){};
+    for (; i < end; i++) {
+      *out++ = c16mulRealShift(mod_dmrs[i], *(int16_t*)amp_dmrs, 15);
+      *out++ = (c16_t){};
+    }
   }
   return 0;
 }
 
-static inline int interleave_with_0_start_with_0(c16_t *output, c16_t *mod_dmrs, const int16_t amp_dmrs, int sz)
+static inline int interleave_with_0_start_with_0(c16_t *output, c16_t *mod_dmrs, void *amp_dmrs, int sz, int use_fp16)
 {
 #ifdef DEBUG_DLSCH_MAPPING
   printf("doing DMRS pattern for port 2 : 0 d0 0 d1 ... 0 dNm2 0 dNm1\n");
@@ -157,50 +227,117 @@ static inline int interleave_with_0_start_with_0(c16_t *output, c16_t *mod_dmrs,
   c16_t *out = output;
   int i = 0;
   int end = sz / 2;
-#if defined(__AVX512BW__)
-  simde__m512i zeros512 = simde_mm512_setzero_si512(), amp_dmrs512 = simde_mm512_set1_epi16(amp_dmrs);
-  simde__m512i perml = simde_mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
-  simde__m512i permh = simde_mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
-  for (; i < (end & ~15); i += 16) {
-    simde__m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(mod_dmrs + i)), amp_dmrs512);
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(zeros512, perml, d0));
-    out += 16;
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(zeros512, permh, d0));
-    out += 16;
+  if (use_fp16) {
+#if defined(__AVX512FP16__) && defined(FLT16_MAX)
+    __m512i zeros512 = _mm512_setzero_si512();
+    __m512h amp_dmrs512 = _mm512_set1_ph(*(_Float16*)amp_dmrs);
+    __m512i perml = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    __m512i permh = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      __m512h d0 = _mm512_mul_ph(_mm512_loadu_ph((__m512h *)(mod_dmrs + i)), amp_dmrs512);
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32(zeros512, perml, (__m512i)d0));
+      out += 16;
+      _mm512_storeu_si512((__m512i *)out, _mm512_permutex2var_epi32(zeros512, permh, (__m512i)d0));
+      out += 16;
+    }
+    __m256i zeros256 = _mm256_setzero_si256();
+    __m256h amp_dmrs256 = _mm256_set1_ph(*(_Float16*)amp_dmrs);
+    for (; i < (end & ~7); i += 8) {
+      __m256h d0 = _mm256_mul_ph(_mm256_loadu_ph((__m256h *)(mod_dmrs + i)), amp_dmrs256);
+      __m256i d2 = _mm256_unpacklo_epi32(zeros256, (__m256i)d0);
+      __m256i d3 = _mm256_unpackhi_epi32(zeros256, (__m256i)d0);
+      _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256(d2, d3, 32));
+      out += 8;
+     _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256(d2, d3, 49));
+      out += 8;
+    }
+    __m128i zeros = _mm_setzero_si128();
+    __m128h amp_dmrs128 = _mm_set1_ph(*(_Float16*)amp_dmrs);
+    for (; i < (end & ~3); i += 4) {
+      __m128h d0 = _mm_mul_ph(_mm_loadu_ph((__m128h *)(mod_dmrs + i)), amp_dmrs128);
+      __m128i d2 = _mm_unpacklo_epi32(zeros, (__m128i)d0);
+      __m128i d3 = _mm_unpackhi_epi32(zeros, (__m128i)d0);
+      _mm_storeu_si128((__m128i *)out, d2);
+      out += 4;
+      _mm_storeu_si128((__m128i *)out, d3);
+      out += 4;
+    }
+#elif defined(__aarch64__)
+    uint32x4_t zeros = vdupq_n_u32(0); 
+    float16x8_t amp_dmrs128 = vdupq_n_f16(*(float16_t*)amp_dmrs);
+    for (; i < (end & ~3); i += 4) {
+      float16x8_t d0 = vmulq_f16(vld1q_f16((float16_t *)(mod_dmrs + i)), amp_dmrs128);
+      float16x8_t d2 = (float16x8_t)vzip1q_u32(zeros, (uint32x4_t)d0);
+      float16x8_t d3 = (float16x8_t)vzip2q_u32(zeros, (uint32x4_t)d0);
+      vst1q_f16((float16_t *)out, d2);
+      out += 4;
+      vst1q_f16((float16_t *)out, d3);
+      out += 4;
+    }
+#else
+    AssertFatal(1==0,"Architecture doesn't support fp16\n");
+#endif
+    cf16_t *out_cf16 = (cf16_t*)out;
+    for (; i < end; i++) {
+      *out_cf16++ = (cf16_t){};
+#ifdef __aarch64__
+      out_cf16->r = mod_dmrs[i].r * *(float16_t*)amp_dmrs;
+      out_cf16->i = mod_dmrs[i].i * *(float16_t*)amp_dmrs;
+#else
+      out_cf16->r = mod_dmrs[i].r * *(_Float16*)amp_dmrs;
+      out_cf16->i = mod_dmrs[i].i * *(_Float16*)amp_dmrs;
+#endif
+      out_cf16++;
+    }
   }
+  else 
+  {
+    //printf("interleave_with_0_start_with_0: use_fp16 = 0\n");
+#if defined(__AVX512__) && defined(__AVX512BW__)
+    simde__m512i zeros512 = simde_mm512_setzero_si512(), amp_dmrs512 = simde_mm512_set1_epi16(*(int16_t*)amp_dmrs);
+    simde__m512i perml = simde_mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    simde__m512i permh = simde_mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      simde__m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(mod_dmrs + i)), amp_dmrs512);
+      simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(zeros512, perml, d0));
+      out += 16;
+      simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(zeros512, permh, d0));
+      out += 16;
+    }
 #endif
 #if defined(__AVX2__)
-  simde__m256i zeros256 = simde_mm256_setzero_si256(), amp_dmrs256 = simde_mm256_set1_epi16(amp_dmrs);
-  for (; i < (end & ~7); i += 8) {
-    simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(mod_dmrs + i)), amp_dmrs256);
-    simde__m256i d2 = simde_mm256_unpacklo_epi32(zeros256, d0);
-    simde__m256i d3 = simde_mm256_unpackhi_epi32(zeros256, d0);
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
-    out += 8;
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
-    out += 8;
-  }
+    simde__m256i zeros256 = simde_mm256_setzero_si256(), amp_dmrs256 = simde_mm256_set1_epi16(*(int16_t*)amp_dmrs);
+    for (; i < (end & ~7); i += 8) {
+      simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(mod_dmrs + i)), amp_dmrs256);
+      simde__m256i d2 = simde_mm256_unpacklo_epi32(zeros256, d0);
+      simde__m256i d3 = simde_mm256_unpackhi_epi32(zeros256, d0);
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
+      out += 8;
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
+      out += 8;
+    }
 #endif
 #if defined(USE128BIT)
-  simde__m128i zeros = simde_mm_setzero_si128(), amp_dmrs128 = simde_mm_set1_epi16(amp_dmrs);
-  for (; i < (end & ~3); i += 4) {
-    simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(mod_dmrs + i)), amp_dmrs128);
-    simde__m128i d2 = simde_mm_unpacklo_epi32(zeros, d0);
-    simde__m128i d3 = simde_mm_unpackhi_epi32(zeros, d0);
-    simde_mm_storeu_si128((simde__m128i *)out, d2);
-    out += 4;
-    simde_mm_storeu_si128((simde__m128i *)out, d3);
-    out += 4;
-  }
+    simde__m128i zeros = simde_mm_setzero_si128(), amp_dmrs128 = simde_mm_set1_epi16(*(int16_t*)amp_dmrs);
+    for (; i < (end & ~3); i += 4) {
+      simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(mod_dmrs + i)), amp_dmrs128);
+      simde__m128i d2 = simde_mm_unpacklo_epi32(zeros, d0);
+      simde__m128i d3 = simde_mm_unpackhi_epi32(zeros, d0);
+      simde_mm_storeu_si128((simde__m128i *)out, d2);
+      out += 4;
+      simde_mm_storeu_si128((simde__m128i *)out, d3);
+      out += 4;
+    }
 #endif
-  for (; i < end; i++) {
-    *out++ = (c16_t){};
-    *out++ = c16mulRealShift(mod_dmrs[i], amp_dmrs, 15);
+    for (; i < end; i++) {
+      *out++ = (c16_t){};
+      *out++ = c16mulRealShift(mod_dmrs[i], *(int16_t*)amp_dmrs, 15);
+    }
   }
   return 0;
 }
 
-static inline int interleave_signals(c16_t *output, c16_t *signal1, const int amp, c16_t *signal2, const int amp2, int sz)
+static inline int interleave_signals(c16_t *output, c16_t *signal1, void *amp, c16_t *signal2, void *amp2, int sz, int use_fp16) 
 {
 #ifdef DEBUG_DLSCH_MAPPING
   printf("doing DMRS pattern for port 0 : d0 X0 d1 X1 ... dNm2 XNm2 dNm1 XNm1\n");
@@ -209,48 +346,123 @@ static inline int interleave_signals(c16_t *output, c16_t *signal1, const int am
   c16_t *out = output;
   int i = 0;
   int end = sz / 2;
-#if defined(__AVX512BW__)
-  simde__m512i amp2512 = simde_mm512_set1_epi16(amp2), amp512 = simde_mm512_set1_epi16(amp);
-  simde__m512i perml = simde_mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
-  simde__m512i permh = simde_mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
-  for (; i < (end & ~15); i += 16) {
-    simde__m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(signal2 + i)), amp2512);
-    simde__m512i d1 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(signal1 + i)), amp512);
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, perml, d1));
-    out += 16;
-    simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, permh, d1));
-    out += 16;
+  if (use_fp16) {
+#if defined(__AVX512FP16__) && defined(FLT16_MAX)
+    __m512h amp2512 = _mm512_set1_ph(*(_Float16*)amp2), amp512 = _mm512_set1_ph(*(_Float16*)amp);
+    __m512i perml = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    __m512i permh = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      __m512h d0 = _mm512_mul_ph((__m512h)_mm512_loadu_si512((__m512i *)(signal2 + i)), amp2512);
+      __m512h d1 = _mm512_mul_ph((__m512h)_mm512_loadu_si512((__m512i *)(signal1 + i)), amp512);
+      _mm512_storeu_si512((__m512h *)out, _mm512_permutex2var_epi32((__m512i)d0, perml, (__m512i)d1));
+      out += 16;
+      _mm512_storeu_si512((__m512h *)out, _mm512_permutex2var_epi32((__m512i)d0, permh, (__m512i)d1));
+      out += 16;
+    }
+    __m256h amp2256 = _mm256_set1_ph(*(_Float16*)amp2), amp256 = _mm256_set1_ph(*(_Float16*)amp);
+    for (; i < (end & ~7); i += 8) {
+      __m256h d0 = _mm256_mul_ph((__m256h)_mm256_loadu_si256((__m256i *)(signal2 + i)), amp2256);
+      __m256h d1 = _mm256_mul_ph((__m256h)_mm256_loadu_si256((__m256i *)(signal1 + i)), amp256);
+      __m256h d2 = (__m256h)_mm256_unpacklo_epi32((__m256i)d0, (__m256i)d1);
+      __m256h d3 = (__m256h)_mm256_unpackhi_epi32((__m256i)d0, (__m256i)d1);
+      _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256((__m256i)d2, (__m256i)d3, 32));
+      out += 8;
+      _mm256_storeu_si256((__m256i *)out, _mm256_permute2x128_si256((__m256i)d2, (__m256i)d3, 49));
+      out += 8;
+    }
+    __m128h amp2128 = _mm_set1_ph(*(_Float16*)amp2), amp128 = _mm_set1_ph(*(_Float16*)amp);
+    for (; i < (end & ~3); i += 4) {
+      __m128h d0 = _mm_mul_ph((__m128h)_mm_loadu_si128((__m128i *)(signal2 + i)), amp2128);
+      __m128h d1 = _mm_mul_ph((__m128h)_mm_loadu_si128((__m128i *)(signal1 + i)), amp128);
+      __m128h d2 = (__m128h)_mm_unpacklo_epi32((__m128i)d0, (__m128i)d1);
+      __m128h d3 = (__m128h)_mm_unpackhi_epi32((__m128i)d0, (__m128i)d1);
+      _mm_storeu_si128((__m128i *)out, (__m128i)d2);
+      out += 4;
+      _mm_storeu_si128((__m128i *)out, (__m128i)d3);
+      out += 4;
+    }
+#elif defined(__aarch64__)
+    float16x8_t amp2128 = vdupq_n_f16(*(float16_t*)amp2), amp128 = vdupq_n_f16(*(float16_t*)amp);
+    for (; i < (end & ~3); i += 4) {
+      float16x8_t d0 = vmulq_f16(vld1q_f16((float16_t *)(signal2 + i)), amp2128);
+      float16x8_t d1 = vmulq_f16(vld1q_f16((float16_t *)(signal1 + i)), amp128);
+      float16x8_t d2 = (float16x8_t)vzip1q_u32((uint32x4_t)d0,(uint32x4_t)d1);
+      float16x8_t d3 = (float16x8_t)vzip2q_u32((uint32x4_t)d0,(uint32x4_t)d1);
+      vst1q_f16((float16_t *)out, d2);
+      out += 4;
+      vst1q_f16((float16_t *)out, d3);
+      out += 4;
+    }
+#else
+    AssertFatal(1==0,"Architecture doesn't support fp16\n");
+#endif
+    cf16_t *out_cf16 = (cf16_t*)out;
+    
+    for (; i < end; i++) {
+      cf16_t *s1 = (cf16_t*)signal1,*s2=(cf16_t*)signal2;	    
+#ifdef __aarch64__
+      out_cf16->r = (float16_t)s2[i].r * *(float16_t*)amp2;
+      out_cf16->i = (float16_t)s2[i].i * *(float16_t*)amp2;
+      out_cf16++;
+      out_cf16->r = (float16_t)s1[i].r * *(float16_t*)amp;
+      out_cf16->i = (float16_t)s1[i].i * *(float16_t*)amp;
+      out_cf16++;
+#else
+      out_cf16->r = (_Float16)s2[i].r * *(_Float16*)amp2;
+      out_cf16->i = (_Float16)s2[i].i * *(_Float16*)amp2;
+      out_cf16++;
+      out_cf16->r = (_Float16)s1[i].r * *(_Float16*)amp;
+      out_cf16->i = (_Float16)s1[i].i * *(_Float16*)amp;
+      out_cf16++;
+#endif
+    }
   }
+  else
+  {  
+
+    //printf("interleave_signals: use_fp16 = 0\n");
+#if defined(__AVX512BW__)
+  
+    simde__m512i amp2512 = simde_mm512_set1_epi16(*(int16_t*)amp2), amp512 = simde_mm512_set1_epi16(*(int16_t*)amp);
+    simde__m512i perml = simde_mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+    simde__m512i permh = simde_mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+    for (; i < (end & ~15); i += 16) {
+      simde__m512i d0 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(signal2 + i)), amp2512);
+      simde__m512i d1 = simde_mm512_mulhrs_epi16(_mm512_loadu_si512((simde__m512i *)(signal1 + i)), amp512);
+      simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, perml, d1));
+      out += 16;
+      simde_mm512_storeu_si512((simde__m512i *)out, simde_mm512_permutex2var_epi32(d0, permh, d1));
+      out += 16;
+    }
 #endif
 #if defined(__AVX2__)
-  simde__m256i amp2256 = simde_mm256_set1_epi16(amp2), amp256 = simde_mm256_set1_epi16(amp);
-  for (; i < (end & ~7); i += 8) {
-    simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(signal2 + i)), amp2256);
-    simde__m256i d1 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(signal1 + i)), amp256);
-    simde__m256i d2 = simde_mm256_unpacklo_epi32(d0, d1);
-    simde__m256i d3 = simde_mm256_unpackhi_epi32(d0, d1);
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
-    out += 8;
-    simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
-    out += 8;
-  }
+    simde__m256i amp2256 = simde_mm256_set1_epi16(*(int16_t*)amp2), amp256 = simde_mm256_set1_epi16(*(int16_t*)amp);
+    for (; i < (end & ~7); i += 8) {
+      simde__m256i d0 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(signal2 + i)), amp2256);
+      simde__m256i d1 = simde_mm256_mulhrs_epi16(simde_mm256_loadu_si256((simde__m256i *)(signal1 + i)), amp256);
+      simde__m256i d2 = simde_mm256_unpacklo_epi32(d0, d1);
+      simde__m256i d3 = simde_mm256_unpackhi_epi32(d0, d1);
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 32));
+      out += 8;
+      simde_mm256_storeu_si256((simde__m256i *)out, simde_mm256_permute2x128_si256(d2, d3, 49));
+      out += 8;
+    }
 #endif
-#if defined(USE128BIT)
-  simde__m128i amp2128 = simde_mm_set1_epi16(amp2), amp128 = simde_mm_set1_epi16(amp);
-  for (; i < (end & ~3); i += 4) {
-    simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(signal2 + i)), amp2128);
-    simde__m128i d1 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(signal1 + i)), amp128);
-    simde__m128i d2 = simde_mm_unpacklo_epi32(d0, d1);
-    simde__m128i d3 = simde_mm_unpackhi_epi32(d0, d1);
-    simde_mm_storeu_si128((simde__m128i *)out, d2);
-    out += 4;
-    simde_mm_storeu_si128((simde__m128i *)out, d3);
-    out += 4;
-  }
-#endif
-  for (; i < end; i++) {
-    *out++ = c16mulRealShift(signal2[i], amp2, 15);
-    *out++ = c16mulRealShift(signal1[i], amp, 15);
+    simde__m128i amp2128 = simde_mm_set1_epi16(*(int16_t*)amp2), amp128 = simde_mm_set1_epi16(*(int16_t*)amp);
+    for (; i < (end & ~3); i += 4) {
+      simde__m128i d0 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(signal2 + i)), amp2128);
+      simde__m128i d1 = simde_mm_mulhrs_epi16(simde_mm_loadu_si128((simde__m128i *)(signal1 + i)), amp128);
+      simde__m128i d2 = simde_mm_unpacklo_epi32(d0, d1);
+      simde__m128i d3 = simde_mm_unpackhi_epi32(d0, d1);
+      simde_mm_storeu_si128((simde__m128i *)out, d2);
+      out += 4;
+      simde_mm_storeu_si128((simde__m128i *)out, d3);
+      out += 4;
+    }
+    for (; i < end; i++) {
+      *out++ = c16mulRealShift(signal2[i], *(int16_t*)amp2, 15);
+      *out++ = c16mulRealShift(signal1[i], *(int16_t*)amp, 15);
+    }
   }
   return sz / 2;
 }
@@ -258,8 +470,8 @@ static inline int interleave_signals(c16_t *output, c16_t *signal1, const int am
 static inline int dmrs_case00(c16_t *output,
                               c16_t *txl,
                               c16_t *mod_dmrs,
-                              const int16_t amp_dmrs,
-                              const int amp,
+                              void *amp_dmrs,
+                              void *amp,
                               int sz,
                               int start_sc,
                               int remaining_re,
@@ -267,7 +479,8 @@ static inline int dmrs_case00(c16_t *output,
                               const int dmrs_Type,
                               int symbol_sz,
                               int l_prime,
-                              uint8_t numDmrsCdmGrpsNoData)
+                              uint8_t numDmrsCdmGrpsNoData,
+			      int use_fp16)
 {
   // DMRS params for this dmrs port
   int Wt[2], Wf[2];
@@ -279,9 +492,23 @@ static inline int dmrs_case00(c16_t *output,
   c16_t *in = txl;
   uint8_t k_prime = 0;
   uint16_t n = 0;
+
   for (int i = 0; i < sz; i++) {
     if (k == ((start_sc + get_dmrs_freq_idx(n, k_prime, delta, dmrs_Type)) % (symbol_sz))) {
-      output[k] = c16mulRealShift(mod_dmrs[dmrs_idx], Wt[l_prime] * Wf[k_prime] * amp_dmrs, 15);
+#ifdef FLT16_MAX
+      if (use_fp16) {
+	cf16_t *output_cf16 = (cf16_t*)&output[k];
+	cf16_t *mod_dmrs_cf16 = (cf16_t*)&mod_dmrs[dmrs_idx];
+#ifdef __aarch64__
+        *output_cf16 = cf16mulReal(*mod_dmrs_cf16, Wt[l_prime] * Wf[k_prime] * *(float16_t*)amp_dmrs);
+#else
+        *output_cf16 = cf16mulReal(*mod_dmrs_cf16, Wt[l_prime] * Wf[k_prime] * *(_Float16*)amp_dmrs);
+#endif
+      }
+      else
+#else
+        output[k] = c16mulRealShift(mod_dmrs[dmrs_idx], Wt[l_prime] * Wf[k_prime] * *(int16_t*)amp_dmrs, 15);
+#endif
       dmrs_idx++;
       k_prime = (k_prime + 1) & 1;
       n += (k_prime ? 0 : 1);
@@ -289,7 +516,19 @@ static inline int dmrs_case00(c16_t *output,
     /* Map PTRS Symbol */
     /* Map DATA Symbol */
     else if (allowed_xlsch_re_in_dmrs_symbol(k, start_sc, symbol_sz, numDmrsCdmGrpsNoData, dmrs_Type)) {
-      output[k] = c16mulRealShift(*in++, amp, 15);
+#ifdef FLT16_MAX
+      if (use_fp16) {
+	cf16_t *output_cf16 = (cf16_t*)&output[k];
+	cf16_t *in_cf16 = (cf16_t*)in++;
+#ifdef __aarch64__
+        *output_cf16 = cf16mulReal(*in_cf16,*(float16_t*)amp);
+#else
+        *output_cf16 = cf16mulReal(*in_cf16,*(_Float16*)amp);
+#endif
+      }
+      else
+#endif
+        output[k] = c16mulRealShift(*in++, *(int16_t*)amp, 15);
     }
     /* mute RE */
     else {
@@ -300,41 +539,58 @@ static inline int dmrs_case00(c16_t *output,
   return in - txl;
 }
 
-static inline int no_ptrs_dmrs_case(c16_t *output, c16_t *txl, const int amp, const int sz)
+static inline int no_ptrs_dmrs_case(c16_t *output, c16_t *txl, void *amp, const int sz,int use_fp16)
 {
   // Loop Over SCs:
   int i = 0;
+#if (defined(__AVX512FP16__) && defined(FLT16_MAX))|| (defined(__aarch64__) && defined(FLT16_MAX))
+  if (use_fp16) 
+    memcpy((void*)output,(void*)txl,sz*sizeof(uint32_t));
+  else
+#endif
+  {
+    //printf("no_ptrs_dmrs_case: use_fp16=0\n");
 #if defined(__AVX512BW__)
-  simde__m512i amp512 = simde_mm512_set1_epi16(amp);
-  for (; i < (sz & ~15); i += 16) {
-    const simde__m512i txL = simde_mm512_loadu_si512((simde__m512i *)(txl + i));
-    simde_mm512_storeu_si512((simde__m512i *)(output + i), simde_mm512_mulhrs_epi16(amp512, txL));
-  }
+    simde__m512i amp512 = simde_mm512_set1_epi16(*(int16_t*)amp);
+    for (; i < (sz & ~15); i += 16) {
+      const simde__m512i txL = simde_mm512_loadu_si512((simde__m512i *)(txl + i));
+      simde_mm512_storeu_si512((simde__m512i *)(output + i), simde_mm512_mulhrs_epi16(amp512, txL));
+    }
 #endif
 #if defined(__AVX2__)
-  simde__m256i amp256 = simde_mm256_set1_epi16(amp);
-  for (; i < (sz & ~7); i += 8) {
-    const simde__m256i txL = simde_mm256_loadu_si256((simde__m256i *)(txl + i));
-    simde_mm256_storeu_si256((simde__m256i *)(output + i), _mm256_mulhrs_epi16(amp256, txL));
-  }
+    simde__m256i amp256 = simde_mm256_set1_epi16(*(int16_t*)amp);
+    for (; i < (sz & ~7); i += 8) {
+      const simde__m256i txL = simde_mm256_loadu_si256((simde__m256i *)(txl + i));
+      simde_mm256_storeu_si256((simde__m256i *)(output + i), _mm256_mulhrs_epi16(amp256, txL));
+    }
 #endif
 #if defined(USE128BIT)
-  simde__m128i amp128 = simde_mm_set1_epi16(amp);
-  for (; i < (sz & ~3); i += 4) {
-    const simde__m128i txL = simde_mm_loadu_si128((simde__m128i *)(txl + i));
-    simde_mm_storeu_si128((simde__m128i *)(output + i), simde_mm_mulhrs_epi16(amp128, txL));
-  }
+    simde__m128i amp128 = simde_mm_set1_epi16(*(int16_t*)amp);
+    for (; i < (sz & ~3); i += 4) {
+      const simde__m128i txL = simde_mm_loadu_si128((simde__m128i *)(txl + i));
+      simde_mm_storeu_si128((simde__m128i *)(output + i), simde_mm_mulhrs_epi16(amp128, txL));
+    }
+#endif
+#if !(defined(__AVX512FP16__) && defined(FLT16_MAX))
+    for (; i < sz; i++) {
+      output[i] = c16mulRealShift(txl[i], *(int16_t*)amp, 15);
+    }
 #endif
-  for (; i < sz; i++) {
-    output[i] = c16mulRealShift(txl[i], amp, 15);
   }
   return sz;
 }
 
-static inline void neg_dmrs(c16_t *in, c16_t *out, int sz)
+static inline void neg_dmrs(c16_t *in, c16_t *out, int sz, int use_fp16)
 {
-  for (int i = 0; i < sz; i++)
-    *out++ = i % 2 ? (c16_t){-in[i].r, -in[i].i} : in[i];
+  if (use_fp16) 
+    for (int i = 0; i < sz; i++) {
+      cf16_t *inp = (cf16_t*)&in[i];
+      cf16_t *outp = (cf16_t*)out++;      
+      *outp = i % 2 ? (cf16_t){-inp->r, -inp->i} : *inp;
+    }
+  else
+    for (int i = 0; i < sz; i++)
+      *out++ = i % 2 ? (c16_t){-in[i].r, -in[i].i} : in[i];
 }
 
 static inline int do_onelayer(NR_DL_FRAME_PARMS *frame_parms,
@@ -348,11 +604,12 @@ static inline int do_onelayer(NR_DL_FRAME_PARMS *frame_parms,
                               int l_symbol,
                               uint16_t dlPtrsSymPos,
                               int n_ptrs,
-                              int amp,
-                              int16_t amp_dmrs,
+                              void *amp,
+                              void *amp_dmrs,
                               int l_prime,
                               nfapi_nr_dmrs_type_e dmrs_Type,
-                              c16_t *dmrs_start)
+                              c16_t *dmrs_start,
+	                      int use_fp16)
 {
   c16_t *txl = txl_start;
   const uint sz = rel15->rbSize * NR_NB_SC_PER_RB;
@@ -376,8 +633,17 @@ static inline int do_onelayer(NR_DL_FRAME_PARMS *frame_parms,
         __attribute__((aligned(64))); // max only to please sanitizer, that kills if 0 even if it is not a error
     const uint32_t *gold =
         nr_gold_pdsch(frame_parms->N_RB_DL, frame_parms->symbols_per_slot, rel15->dlDmrsScramblingId, rel15->SCID, slot, l_symbol);
+#ifdef  FLT16_MAX
+    if (use_fp16)
+      nr_modulation(gold, n_ptrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, NULL, (_Float16 *)mod_ptrs);
+    else
+      nr_modulation(gold, n_ptrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, (int16_t *)mod_ptrs, NULL);
+#else
     nr_modulation(gold, n_ptrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, (int16_t *)mod_ptrs);
-    txl += do_ptrs_symbol(rel15, start_sc, symbol_sz, output, txl, amp, mod_ptrs);
+
+#endif
+
+    txl += do_ptrs_symbol(rel15, start_sc, symbol_sz, output, txl, amp, mod_ptrs,use_fp16);
 
   } else if (rel15->dlDmrsSymbPos & (1 << l_symbol)) {
     /* Map DMRS Symbol */
@@ -386,47 +652,47 @@ static inline int do_onelayer(NR_DL_FRAME_PARMS *frame_parms,
       if (rel15->numDmrsCdmGrpsNoData == 2) {
         switch (dmrs_port & 3) {
           case 0:
-            txl += interleave_with_0_signal_first(output + start_sc, dmrs_start, amp_dmrs, upper_limit);
-            txl += interleave_with_0_signal_first(output, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re);
+            txl += interleave_with_0_signal_first(output + start_sc, dmrs_start, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_with_0_signal_first(output, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
             break;
           case 1: {
             c16_t dmrs[sz / 2];
-            neg_dmrs(dmrs_start, dmrs, sz / 2);
-            txl += interleave_with_0_signal_first(output + start_sc, dmrs, amp_dmrs, upper_limit);
-            txl += interleave_with_0_signal_first(output, dmrs + upper_limit / 2, amp_dmrs, remaining_re);
+            neg_dmrs(dmrs_start, dmrs, sz / 2,use_fp16);
+            txl += interleave_with_0_signal_first(output + start_sc, dmrs, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_with_0_signal_first(output, dmrs + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
           } break;
           case 2:
-            txl += interleave_with_0_start_with_0(output + start_sc, dmrs_start, amp_dmrs, upper_limit);
-            txl += interleave_with_0_start_with_0(output, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re);
+            txl += interleave_with_0_start_with_0(output + start_sc, dmrs_start, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_with_0_start_with_0(output, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
             break;
           case 3: {
             c16_t dmrs[sz / 2];
-            neg_dmrs(dmrs_start, dmrs, sz / 2);
-            txl += interleave_with_0_start_with_0(output + start_sc, dmrs, amp_dmrs, upper_limit);
-            txl += interleave_with_0_start_with_0(output, dmrs + upper_limit / 2, amp_dmrs, remaining_re);
+            neg_dmrs(dmrs_start, dmrs, sz / 2,use_fp16);
+            txl += interleave_with_0_start_with_0(output + start_sc, dmrs, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_with_0_start_with_0(output, dmrs + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
           } break;
         }
       } else if (rel15->numDmrsCdmGrpsNoData == 1) {
         switch (dmrs_port & 3) {
           case 0:
-            txl += interleave_signals(output + start_sc, txl, amp, dmrs_start, amp_dmrs, upper_limit);
-            txl += interleave_signals(output, txl, amp, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re);
+            txl += interleave_signals(output + start_sc, txl, amp, dmrs_start, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_signals(output, txl, amp, dmrs_start + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
             break;
           case 1: {
             c16_t dmrs[sz / 2];
-            neg_dmrs(dmrs_start, dmrs, sz / 2);
-            txl += interleave_signals(output + start_sc, txl, amp, dmrs, amp_dmrs, upper_limit);
-            txl += interleave_signals(output, txl, amp, dmrs + upper_limit / 2, amp_dmrs, remaining_re);
+            neg_dmrs(dmrs_start, dmrs, sz / 2,use_fp16);
+            txl += interleave_signals(output + start_sc, txl, amp, dmrs, amp_dmrs, upper_limit,use_fp16);
+            txl += interleave_signals(output, txl, amp, dmrs + upper_limit / 2, amp_dmrs, remaining_re,use_fp16);
           } break;
           case 2:
-            txl += interleave_signals(output + start_sc, dmrs_start, amp_dmrs, txl, amp, upper_limit);
-            txl += interleave_signals(output, dmrs_start + upper_limit / 2, amp_dmrs, txl, amp, remaining_re);
+            txl += interleave_signals(output + start_sc, dmrs_start, amp_dmrs, txl, amp, upper_limit,use_fp16);
+            txl += interleave_signals(output, dmrs_start + upper_limit / 2, amp_dmrs, txl, amp, remaining_re,use_fp16);
             break;
           case 3: {
             c16_t dmrs[sz / 2];
-            neg_dmrs(dmrs_start, dmrs, sz / 2);
-            txl += interleave_signals(output + start_sc, dmrs, amp_dmrs, txl, amp, upper_limit);
-            txl += interleave_signals(output, dmrs + upper_limit / 2, amp_dmrs, txl, amp, remaining_re);
+            neg_dmrs(dmrs_start, dmrs, sz / 2,use_fp16);
+            txl += interleave_signals(output + start_sc, dmrs, amp_dmrs, txl, amp, upper_limit,use_fp16);
+            txl += interleave_signals(output, dmrs + upper_limit / 2, amp_dmrs, txl, amp, remaining_re,use_fp16);
           } break;
         }
       } else
@@ -444,12 +710,19 @@ static inline int do_onelayer(NR_DL_FRAME_PARMS *frame_parms,
                          dmrs_Type,
                          symbol_sz,
                          l_prime,
-                         rel15->numDmrsCdmGrpsNoData);
+                         rel15->numDmrsCdmGrpsNoData,
+			 use_fp16);
     } // generic DMRS case
   } else { // no PTRS or DMRS in this symbol
-    txl += no_ptrs_dmrs_case(output + start_sc, txl, amp, upper_limit);
-    txl += no_ptrs_dmrs_case(output, txl, amp, remaining_re);
+    txl += no_ptrs_dmrs_case(output + start_sc, txl, amp, upper_limit,use_fp16);
+    txl += no_ptrs_dmrs_case(output, txl, amp, remaining_re,use_fp16);
   } // no DMRS/PTRS in symbol
+   /* 
+  if (use_fp16) 
+    for (int i=0;i<72;i++) printf("symbol %d layer %d cdmGrps %d dmrs_port %d output[%d] = (%f,%f) amp %f, amp_dmrs %f\n",l_symbol, layer, rel15->numDmrsCdmGrpsNoData, get_dmrs_port(layer, rel15->dmrsPorts), i,(double)((cf16_t*)(output+start_sc))[i].r,(double)((cf16_t*)(output+start_sc))[i].i,(double)*(float16_t*)amp,(double)*(float16_t*)amp_dmrs); 
+  else
+    for (int i=0;i<72;i++) printf("symbol %d output[%d] = (%d,%d)\n",l_symbol,i,(output+start_sc)[i].r,(output+start_sc)[i].i); 
+    */
   return txl - txl_start;
 }
 
@@ -480,10 +753,15 @@ static inline void do_txdataF(c16_t **txdataF,
     const int re_cnt = NR_NB_SC_PER_RB * rb_step;
     if (pmi == 0) { // unitary Precoding
       if (subCarrier + re_cnt <= symbol_sz) { // RB does not cross DC
-        if (ant < rel15->nrOfLayers)
+        if (ant < rel15->nrOfLayers) {
           memcpy(&txdataF[ant][txdataF_offset_per_symbol + subCarrier],
                  &txdataF_precoding[ant][subCarrier],
                  re_cnt * sizeof(**txdataF));
+	  /*
+	  if (gNB->use_fp16) 
+	    for (int i=0;i<re_cnt;i++) printf("txdataF[%d][%d] %f.%f\n",ant,txdataF_offset_per_symbol + subCarrier + i,(double)(*(float16_t*)&txdataF[ant][txdataF_offset_per_symbol+subCarrier+i].r),(double)(*(float16_t*)&txdataF[ant][txdataF_offset_per_symbol+subCarrier+i].i));
+	    */
+	}
         else
           memset(&txdataF[ant][txdataF_offset_per_symbol + subCarrier], 0, re_cnt * sizeof(**txdataF));
       } else { // RB does cross DC
@@ -494,6 +772,10 @@ static inline void do_txdataF(c16_t **txdataF,
                  &txdataF_precoding[ant][subCarrier],
                  neg_length * sizeof(**txdataF));
           memcpy(&txdataF[ant][txdataF_offset_per_symbol], &txdataF_precoding[ant], pos_length * sizeof(**txdataF));
+	  /*
+	  if (gNB->use_fp16) 
+	    for (int i=0;i<neg_length;i++) printf("**txdataF[%d][%d] %f.%f\n",ant,txdataF_offset_per_symbol + subCarrier + i,(double)(*(float16_t*)&txdataF[ant][txdataF_offset_per_symbol+subCarrier+i].r),(double)(*(float16_t*)&txdataF[ant][txdataF_offset_per_symbol+i].i));
+	    */
         } else {
           memset(&txdataF[ant][txdataF_offset_per_symbol + subCarrier], 0, neg_length * sizeof(**txdataF));
           memset(&txdataF[ant][txdataF_offset_per_symbol], 0, pos_length * sizeof(**txdataF));
@@ -524,19 +806,27 @@ static inline void do_txdataF(c16_t **txdataF,
                                pmi_pdu,
                                subCarrier,
                                re_cnt,
-                               &txdataF[ant][txdataF_offset_per_symbol]);
+                               &txdataF[ant][txdataF_offset_per_symbol]
+#ifdef FLT16_MAX
+			       ,gNB->use_fp16
+#endif			       
+			       );
         subCarrier += re_cnt;
       } else { // crossing ofdm_symbol_size, use simple arithmetic operations
         for (int i = 0; i < re_cnt; i++) {
           txdataF[ant][txdataF_offset_per_symbol + subCarrier] =
-              nr_layer_precoder_cm(rel15->nrOfLayers, symbol_sz, txdataF_precoding, ant, pmi_pdu, subCarrier);
+              nr_layer_precoder_cm(rel15->nrOfLayers, symbol_sz, txdataF_precoding, ant, pmi_pdu, subCarrier
+#ifdef FLT16_MAX
+			      ,gNB->use_fp16
+#endif
+			      );
 #ifdef DEBUG_DLSCH_MAPPING
-          printf("antenna %d\t l %d \t subCarrier %d \t txdataF: %d %d\n",
+          printf("antenna %d\t off %d \t subCarrier %d \t txdataF: %d %d\n",
                  ant,
-                 l_symbol,
+                 txdataF_offset_per_symbol,
                  subCarrier,
-                 txdataF[ant][l_symbol * symbol_sz + subCarrier + txdataF_offset].r,
-                 txdataF[ant][l_symbol * symbol_sz + subCarrier + txdataF_offset].i);
+                 txdataF[ant][txdataF_offset_per_symbol + subCarrier].r,
+                 txdataF[ant][txdataF_offset_per_symbol + subCarrier].i);
 #endif
           if (++subCarrier >= symbol_sz) {
             subCarrier -= symbol_sz;
@@ -550,7 +840,6 @@ static inline void do_txdataF(c16_t **txdataF,
 }
 static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSCH_t *dlsch, int slot)
 {
-  const int16_t amp = gNB->TX_AMP;
   NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
 
   time_stats_t *dlsch_scrambling_stats = &gNB->dlsch_scrambling_stats;
@@ -560,7 +849,12 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
   const int symbol_sz=frame_parms->ofdm_symbol_size;
   const int dmrs_Type = rel15->dmrsConfigType;
   const int nb_re_dmrs = rel15->numDmrsCdmGrpsNoData * (rel15->dmrsConfigType == NFAPI_NR_DMRS_TYPE1 ? 6 : 4);
-  const int16_t amp_dmrs = min((double)amp * sqrt(rel15->numDmrsCdmGrpsNoData), INT16_MAX); // 3GPP TS 38.214 Section 4.1: Table 4.1-1
+#ifdef FLT16_MAX
+  const _Float16 amp_fp16 = 1.0f;
+  const _Float16 amp_dmrs_fp16 = (_Float16)sqrt(rel15->numDmrsCdmGrpsNoData);
+#endif
+  const int16_t amp = gNB->TX_AMP;
+  const int16_t amp_dmrs = min((double)gNB->TX_AMP * sqrt(rel15->numDmrsCdmGrpsNoData), INT16_MAX); // 3GPP TS 38.214 Section 4.1: Table 4.1-1
   LOG_D(PHY,
         "pdsch: BWPStart %d, BWPSize %d, rbStart %d, rbsize %d\n",
         rel15->BWPStart,
@@ -627,7 +921,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
     stop_meas(dlsch_scrambling_stats);
     /// Modulation
     start_meas(dlsch_modulation_stats);
+#ifdef  FLT16_MAX
+    if (gNB->use_fp16)
+      nr_modulation(scrambled_output, encoded_length, Qm, NULL, (_Float16 *)mod_symbs[codeWord]);
+    else
+      nr_modulation(scrambled_output, encoded_length, Qm, (int16_t *)mod_symbs[codeWord],NULL);
+#else
     nr_modulation(scrambled_output, encoded_length, Qm, (int16_t *)mod_symbs[codeWord]);
+#endif
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_gNB_PDSCH_MODULATION, 0);
     stop_meas(dlsch_modulation_stats);
 #ifdef DEBUG_DLSCH
@@ -718,13 +1019,20 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
                                            slot,
                                            l_symbol);
       // Qm = 1 as DMRS is QPSK modulated
+#ifdef FLT16_MAX
+      if (gNB->use_fp16) 
+        nr_modulation(gold, n_dmrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, NULL, (_Float16 *)mod_dmrs);
+      else 
+        nr_modulation(gold, n_dmrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, (int16_t *)mod_dmrs,NULL);
+#else
       nr_modulation(gold, n_dmrs * DMRS_MOD_ORDER, DMRS_MOD_ORDER, (int16_t *)mod_dmrs);
-
+#endif
 #ifdef DEBUG_DLSCH_MAPPING
       printf("DMRS modulation (symbol %d, %d symbols, type %d):\n", l_symbol, n_dmrs, dmrs_Type);
-      for (int i = 0; i < n_dmrs / 2; i += 8) {
+      for (int i = 0; i < n_dmrs ; i += 8) {
+	printf("%d:",i);
         for (int j = 0; j < 8; j++) {
-          printf("%d %d\t", mod_dmrs[i + j].r, mod_dmrs[i + j].i);
+          printf("%.2f %.2f ", (double)((cf16_t*)mod_dmrs)[i + j].r, (double)((cf16_t*)mod_dmrs)[i + j].i);
         }
         printf("\n");
       }
@@ -748,15 +1056,20 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
                              l_symbol,
                              dlPtrsSymPos,
                              n_ptrs,
-                             amp,
-                             amp_dmrs,
+#ifdef FLT16_MAX
+			     gNB->use_fp16?(void*)&amp_fp16:(void*)&amp,
+			     gNB->use_fp16?(void*)&amp_dmrs_fp16:(void*)&amp_dmrs,
+#else
+                             (void*)&amp,
+                             (void*)&amp_dmrs,
+#endif
                              l_prime,
                              dmrs_Type,
-                             mod_dmrs + dmrs_idx);
+                             mod_dmrs + dmrs_idx,
+			     gNB->use_fp16);
     } // layer loop
     re_beginning_of_symbol += layer_sz;
     stop_meas(&gNB->dlsch_resource_mapping_stats);
-
     start_meas(&gNB->dlsch_precoding_stats);
     for (int ant = 0; ant < frame_parms->nb_antennas_tx; ant++) {
       const size_t txdataF_offset_per_symbol = l_symbol * symbol_sz + txdataF_offset;
@@ -777,9 +1090,11 @@ void nr_generate_pdsch(PHY_VARS_gNB *gNB, int n_dlsch, NR_gNB_DLSCH_t *dlsch_arr
   NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
   time_stats_t *dlsch_encoding_stats = &gNB->dlsch_encoding_stats;
   time_stats_t *tinput = &gNB->tinput;
+  time_stats_t *tinput_memcpy = &gNB->tinput_memcpy;
   time_stats_t *tprep = &gNB->tprep;
   time_stats_t *tparity = &gNB->tparity;
   time_stats_t *toutput = &gNB->toutput;
+  time_stats_t *tconcat = &gNB->tconcat;
   time_stats_t *dlsch_rate_matching_stats = &gNB->dlsch_rate_matching_stats;
   time_stats_t *dlsch_interleaving_stats = &gNB->dlsch_interleaving_stats;
   time_stats_t *dlsch_segmentation_stats = &gNB->dlsch_segmentation_stats;
@@ -825,9 +1140,8 @@ void nr_generate_pdsch(PHY_VARS_gNB *gNB, int n_dlsch, NR_gNB_DLSCH_t *dlsch_arr
   }
 
   unsigned char output[size_output >> 3] __attribute__((aligned(64)));
-  bzero(output, sizeof(output));
-
   start_meas(dlsch_encoding_stats);
+  bzero(output, sizeof(output));
   if (nr_dlsch_encoding(gNB,
                         n_dlsch,
                         dlsch_array,
@@ -836,15 +1150,18 @@ void nr_generate_pdsch(PHY_VARS_gNB *gNB, int n_dlsch, NR_gNB_DLSCH_t *dlsch_arr
                         frame_parms,
                         output,
                         tinput,
+                        tinput_memcpy,
                         tprep,
                         tparity,
                         toutput,
+                        tconcat,
                         dlsch_rate_matching_stats,
                         dlsch_interleaving_stats,
                         dlsch_segmentation_stats)
       == -1) {
     return;
   }
+
   stop_meas(dlsch_encoding_stats);
 
   unsigned char *output_ptr = output;

openair1/PHY/NR_TRANSPORT/nr_dlsch.h

@@ -45,12 +45,14 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
                       NR_DL_FRAME_PARMS *frame_parms,
                       unsigned char *output,
                       time_stats_t *tinput,
+                      time_stats_t *tinput_memcpy,
                       time_stats_t *tprep,
                       time_stats_t *tparity,
                       time_stats_t *toutput,
                       time_stats_t *dlsch_rate_matching_stats,
                       time_stats_t *dlsch_interleaving_stats,
-                      time_stats_t *dlsch_segmentation_stats);
+                      time_stats_t *dlsch_segmentation_stats,
+                      time_stats_t *dlsch_output_stats);
 
 void dump_pdsch_stats(FILE *fd,PHY_VARS_gNB *gNB);
 

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -39,10 +39,13 @@
 #include <syscall.h>
 #include <openair2/UTIL/OPT/opt.h>
 
+#ifdef ENABLE_CUDA
+#include <cuda_runtime.h>
+#endif
 // #define DEBUG_DLSCH_CODING
 // #define DEBUG_DLSCH_FREE 1
 
-void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms)
+void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms, int use_gpumem)
 {
   int max_layers = (frame_parms->nb_antennas_tx < NR_MAX_NB_LAYERS) ? frame_parms->nb_antennas_tx : NR_MAX_NB_LAYERS;
   uint16_t a_segments = MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * max_layers;
@@ -53,7 +56,12 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM
   }
 
   if (dlsch->b) {
-    free16(dlsch->b, a_segments * 1056);
+#ifdef ENABLE_CUDA	  
+    if (use_gpumem) 
+      cudaFreeHost(dlsch->b);
+    else	  
+#endif  
+      free16(dlsch->b, a_segments * 1056);
     dlsch->b = NULL;
   }
   if (dlsch->f) {
@@ -67,7 +75,7 @@ void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARM
   free(dlsch->c);
 }
 
-NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
+NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB, int use_gpumem)
 {
   int max_layers = (frame_parms->nb_antennas_tx < NR_MAX_NB_LAYERS) ? frame_parms->nb_antennas_tx : NR_MAX_NB_LAYERS;
   uint16_t a_segments = MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * max_layers; // number of segments to be allocated
@@ -86,16 +94,35 @@ NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
   bzero(dlsch.b, dlsch_bytes);
 
   dlsch.c = (uint8_t **)malloc16(a_segments * sizeof(uint8_t *));
+#ifdef ENABLE_CUDA
+  uint8_t *c_devh[a_segments];
+#endif
   for (int r = 0; r < a_segments; r++) {
     // account for filler in first segment and CRCs for multiple segment case
     // [hna] 8448 is the maximum CB size in NR
     //       68*348 = 68*(maximum size of Zc)
     //       In section 5.3.2 in 38.212, the for loop is up to N + 2*Zc (maximum size of N is 66*Zc, therefore 68*Zc)
-    dlsch.c[r] = malloc16(8448);
-    AssertFatal(dlsch.c[r], "cannot allocate dlsch.c[%d]\n", r);
-    bzero(dlsch.c[r], 8448);
+#ifdef ENABLE_CUDA
+    if (use_gpumem) {
+      cudaHostAlloc((void**)&dlsch.c[r], 8448/8, cudaHostAllocMapped);
+      cudaError_t err=cudaHostAlloc((void**)&dlsch.c[r],(8448/8)*sizeof(uint8_t),cudaHostAllocMapped);
+      AssertFatal(err == cudaSuccess,"CUDA Error (dlsch.c[%d]): %s\n", r,cudaGetErrorString(err));
+      err=cudaHostGetDevicePointer((void**)&c_devh[r], dlsch.c[r], 0);
+      AssertFatal(err == cudaSuccess,"CUDA Error (cudaHostGetDevicePointer) dlsch.c_devh[%d]: %s\n", r,cudaGetErrorString(err));
+    }
+    else 
+#endif
+      dlsch.c[r] = malloc16(8448/8);
+    AssertFatal(dlsch.c[r], "cannot allocate dlsch.cc[%d]\n", r);
+    bzero(dlsch.c[r], 8448/8);
   }
-
+#ifdef ENABLE_CUDA
+  if (use_gpumem) {
+    cudaError_t err=cudaMalloc((void**)&dlsch.c_dev,a_segments*sizeof(uint8_t*));
+    err=cudaMemcpy(dlsch.c_dev,c_devh,a_segments*sizeof(uint8_t*),cudaMemcpyHostToDevice);
+    AssertFatal(err == cudaSuccess,"CUDA Error (memcpy c_devh -> input_dev): %s\n", cudaGetErrorString(err));
+  }
+#endif
   dlsch.f = malloc16(N_RB * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
   AssertFatal(dlsch.f, "cannot allocate dlsch->f\n");
   bzero(dlsch.f, N_RB * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
@@ -111,9 +138,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
                       NR_DL_FRAME_PARMS *frame_parms,
                       unsigned char *output,
                       time_stats_t *tinput,
+                      time_stats_t *tinput_memcpy,
                       time_stats_t *tprep,
                       time_stats_t *tparity,
                       time_stats_t *toutput,
+		      time_stats_t *tconcat,
                       time_stats_t *dlsch_rate_matching_stats,
                       time_stats_t *dlsch_interleaving_stats,
                       time_stats_t *dlsch_segmentation_stats)
@@ -159,6 +188,7 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
       phy_stats->dlsch_stats.current_Qm = rel15->qamModOrder[0];
     }
 
+    start_meas(dlsch_segmentation_stats);
     int max_bytes = MAX_NUM_NR_DLSCH_SEGMENTS_PER_LAYER * rel15->nrOfLayers * 1056;
     int B;
     if (A > NR_MAX_PDSCH_TBS) {
@@ -261,7 +291,9 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
       reset_meas(&segment_parameters->ts_rate_match);
       reset_meas(&segment_parameters->ts_ldpc_encode);
     }
-
+#ifdef ENABLE_CUDA
+    if (gNB->use_gpu) TB_parameters->c_dev = (uint8_t**)dlsch->c_dev;
+#endif  
     segments_offset += TB_parameters->C;
 
     /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)

openair1/PHY/NR_TRANSPORT/nr_prs.c

@@ -4,6 +4,7 @@
 #include "PHY/NR_REFSIG/nr_refsig.h"
 #include "PHY/sse_intrin.h"
 #include "openair1/PHY/NR_REFSIG/refsig_defs_ue.h"
+#include <float.h>
 #include "openair1/PHY/NR_REFSIG/nr_mod_table.h"
 //#define DEBUG_PRS_MOD
 //#define DEBUG_PRS_MAP

openair1/PHY/NR_TRANSPORT/nr_pss.c

@@ -21,6 +21,10 @@
 
 
 #include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 
 //#define NR_PSS_DEBUG
 
@@ -28,7 +32,11 @@ int nr_generate_pss(  c16_t *txdataF,
                       int16_t amp,
                       uint8_t ssb_start_symbol,
                       nfapi_nr_config_request_scf_t* config,
-                      NR_DL_FRAME_PARMS *frame_parms)
+                      NR_DL_FRAME_PARMS *frame_parms
+#ifdef FLT16_MAX
+		      ,int use_fp16
+#endif
+		      )
 {
   int16_t x[NR_PSS_LENGTH];
   const int x_initial[7] = {0, 1, 1 , 0, 1, 1, 1};
@@ -57,9 +65,19 @@ int nr_generate_pss(  c16_t *txdataF,
   uint8_t Nid2 = config->cell_config.phy_cell_id.value % 3;
   for (int i = 0; i < NR_PSS_LENGTH; i++) {
     int m = (i + 43*Nid2)%(NR_PSS_LENGTH);
-    int16_t d_pss = (1 - 2*x[m]) * 23170;
+#ifdef FLT16_MAX
+    if (use_fp16) {
+      double d_pss = (double)(1 - 2*x[m]) / sqrt(2.0f);
+      ((_Float16*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (_Float16)d_pss;
+    }
+    else
+#endif
+    {
+      int16_t d_pss = (1 - 2*x[m]) * 23170;
     //      printf("pss: writing position k %d / %d\n",k,frame_parms->ofdm_symbol_size);
-    ((int16_t*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (((int16_t)amp) * d_pss) >> 15;
+
+      ((int16_t*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (((int16_t)amp) * d_pss) >> 15;
+    }
     k++;
 
     if (k >= frame_parms->ofdm_symbol_size)

openair1/PHY/NR_TRANSPORT/nr_sss.c

@@ -20,6 +20,10 @@
  */
 
 #include "PHY/NR_TRANSPORT/nr_transport_proto.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 
 //#define NR_SSS_DEBUG
 
@@ -27,7 +31,11 @@ int nr_generate_sss(  c16_t *txdataF,
                       int16_t amp,
                       uint8_t ssb_start_symbol,
                       nfapi_nr_config_request_scf_t* config,
-                      NR_DL_FRAME_PARMS *frame_parms)
+                      NR_DL_FRAME_PARMS *frame_parms
+#ifdef FLT16_MAX
+		      ,int use_fp16
+#endif
+		      )
 {
   int16_t x0[NR_SSS_LENGTH];
   int16_t x1[NR_SSS_LENGTH];
@@ -63,8 +71,17 @@ int nr_generate_sss(  c16_t *txdataF,
   int l = ssb_start_symbol + 2;
 
   for (int i = 0; i < NR_SSS_LENGTH; i++) {
-    int16_t d_sss = (1 - 2*x0[(i + m0) % NR_SSS_LENGTH] ) * (1 - 2*x1[(i + m1) % NR_SSS_LENGTH] ) * 23170;
-    ((int16_t*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (((int16_t)amp) * d_sss) >> 15;
+#ifdef FLT16_MAX
+    if (use_fp16) {
+      double d_sss = (double)(1 - 2*x0[(i + m0) % NR_SSS_LENGTH] ) * (1 - 2*x1[(i + m1) % NR_SSS_LENGTH] ) / sqrt(2.0f);
+      ((_Float16*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (_Float16)d_sss;
+    }
+    else
+#endif
+    {
+      int16_t d_sss = (1 - 2*x0[(i + m0) % NR_SSS_LENGTH] ) * (1 - 2*x1[(i + m1) % NR_SSS_LENGTH] ) * 23170;
+      ((int16_t*)txdataF)[2*(l*frame_parms->ofdm_symbol_size + k)] = (((int16_t)amp) * d_sss) >> 15;
+    }
     k++;
 
     if (k >= frame_parms->ofdm_symbol_size)

openair1/PHY/NR_TRANSPORT/nr_transport_proto.h

@@ -57,7 +57,11 @@ int nr_generate_pss(c16_t *txdataF,
                     int16_t amp,
                     uint8_t ssb_start_symbol,
                     nfapi_nr_config_request_scf_t *config,
-                    NR_DL_FRAME_PARMS *frame_parms);
+                    NR_DL_FRAME_PARMS *frame_parms
+#ifdef FLT16_MAX
+		    ,int use_fp16
+#endif
+		    );
 
 /*!
 \fn int nr_generate_sss
@@ -69,7 +73,11 @@ int nr_generate_sss(c16_t *txdataF,
                     int16_t amp,
                     uint8_t ssb_start_symbol,
                     nfapi_nr_config_request_scf_t *config,
-                    NR_DL_FRAME_PARMS *frame_parms);
+                    NR_DL_FRAME_PARMS *frame_parms
+#ifdef FLT16_MAX
+		    ,int use_fp16
+#endif
+		   );
 
 /*!
 \fn void nr_generate_pbch_dmrs
@@ -106,9 +114,9 @@ void nr_generate_pbch(PHY_VARS_gNB *gNB,
 void nr_init_pbch_interleaver(uint8_t *interleaver);
 uint32_t nr_pbch_extra_byte_generation(int sfn, int n_hf, int ssb_index, int ssb_sc_offset, int Lmax);
 
-NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB);
+NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB,int use_gpu);
 
-void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms);
+void free_gNB_dlsch(NR_gNB_DLSCH_t *dlsch, uint16_t N_RB, const NR_DL_FRAME_PARMS *frame_parms,int use_gpu);
 
 /** \brief This function is the top-level entry point to PUSCH demodulation, after frequency-domain transformation and channel estimation.  It performs
     - RB extraction (signal and channel estimates)

openair1/PHY/NR_UE_TRANSPORT/csi_rx.c

@@ -875,7 +875,11 @@ void nr_ue_csi_rs_procedures(PHY_VARS_NR_UE *ue,
                      csirs_config_pdu->scramb_id,
                      csirs_config_pdu->power_control_offset_ss,
                      csirs_config_pdu->cdm_type,
-                     csi_info->csi_rs_generated_signal);
+                     csi_info->csi_rs_generated_signal
+#ifdef FLT16_MAX
+                     ,ue->use_fp16
+#endif
+		     );
 
   csi_info->csi_rs_generated_signal_bits = log2_approx(AMP);
 

openair1/PHY/NR_UE_TRANSPORT/nr_psbch_tx.c

@@ -24,6 +24,9 @@
 #include "PHY/NR_REFSIG/nr_mod_table.h"
 #include "PHY/NR_UE_TRANSPORT/nr_transport_proto_ue.h"
 #include "PHY/CODING/nrPolar_tools/nr_polar_psbch_defs.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
 #include "PHY/MODULATION/nr_modulation.h"
 
 // #define SL_DEBUG
@@ -158,7 +161,11 @@ void sl_generate_and_map_psbch(c16_t *txF,
                                uint16_t re_offset,
                                uint16_t scaling_factor,
                                uint16_t symbol_size,
-                               c16_t *psbch_dmrs)
+                               c16_t *psbch_dmrs
+#ifdef FLT16_MAX
+                               ,int use_fp16
+#endif
+	)
 {
   uint64_t psbch_a_reversed = 0;
   uint16_t num_psbch_modsym = 0, numsym = 0;
@@ -218,8 +225,14 @@ void sl_generate_and_map_psbch(c16_t *txF,
 #endif
 
   /// 38.211 QPSK modulation
+#ifdef FLT16_MAX
+  if (use_fp16)
+    nr_modulation(encoder_output, num_psbch_modsym * mod_order, mod_order, NULL,(_Float16 *)psbch_modsym);
+  else
+    nr_modulation(encoder_output, num_psbch_modsym * mod_order, mod_order, (int16_t *)psbch_modsym,NULL);
+#else
   nr_modulation(encoder_output, num_psbch_modsym * mod_order, mod_order, (int16_t *)psbch_modsym);
-
+#endif
   // RE MApping of PSBCH and PSBCH DMRS
   int index = 0, dmrs_index = 0;
   const int numre = SL_NR_NUM_PSBCH_RE_IN_ONE_SYMBOL;
@@ -363,7 +376,11 @@ void nr_tx_psbch(PHY_VARS_NR_UE *UE, uint32_t frame_tx, uint32_t slot_tx, sl_nr_
 
   struct complex16 *psbch_dmrs = &sl_ue_phy_params->init_params.psbch_dmrs_modsym[slss_id][0];
 
-  sl_generate_and_map_psbch(txF, &psbch_payload, slss_id, sl_fp->Ncp, re_offset, scaling_factor, symbol_size, psbch_dmrs);
+  sl_generate_and_map_psbch(txF, &psbch_payload, slss_id, sl_fp->Ncp, re_offset, scaling_factor, symbol_size, psbch_dmrs
+#ifdef FLT16_MAX
+                   ,UE->use_fp16
+#endif 
+		  );
 
 #ifdef SL_DEBUG
   printf("DEBUG PSBCH TX: txdataF Prepared\n");

openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_ue.c

@@ -35,6 +35,9 @@
 #include "PHY/NR_REFSIG/refsig_defs_ue.h"
 #include "PHY/NR_UE_TRANSPORT/nr_transport_ue.h"
 #include "PHY/NR_UE_TRANSPORT/nr_transport_proto_ue.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
 #include "PHY/MODULATION/nr_modulation.h"
 #include "PHY/MODULATION/modulation_common.h"
 #include "common/utils/assertions.h"
@@ -459,6 +462,31 @@ static void dmrs_amp_mult(const uint32_t dmrs_port,
   }
 }
 
+#ifdef FLT16_MAX
+
+static void dmrs_amp_mult_fp16(const uint32_t dmrs_port,
+                               const int Wt,
+                               const int Wf[2],
+                               const cf16_t *mod_dmrs,
+                               cf16_t *mod_dmrs_out,
+                               const uint32_t n_dmrs,
+                               const pusch_dmrs_type_t dmrs_type,
+                               const unsigned int num_cdm_groups_no_data)
+{
+  _Float16 beta_dmrs_pusch = (_Float16)get_beta_dmrs(num_cdm_groups_no_data, dmrs_type == pusch_dmrs_type2);
+  /* short array that hold amplitude for k_prime = 0 and k_prime = 1 */
+  int32_t alpha_dmrs[2] __attribute((aligned(16)));
+  for (int_fast8_t i = 0; i < sizeofArray(alpha_dmrs); i++) {
+    alpha_dmrs[i] = Wf[i] * Wt * beta_dmrs_pusch;
+  }
+
+  /* multiply amplitude with complex DMRS vector */
+  for (int_fast16_t i = 0; i < n_dmrs; i++) {
+    mod_dmrs_out[i].r= mod_dmrs[i].r * alpha_dmrs[i % 2];
+    mod_dmrs_out[i].i= mod_dmrs[i].i * alpha_dmrs[i % 2];
+  }
+}
+#endif
 /*
 Map ULSCH data and DMRS in all of the scheduled symbols and PRBs
 */
@@ -466,7 +494,11 @@ static void map_symbols(const nr_phy_pxsch_params_t p,
                         const unsigned int slot,
                         const c16_t *dmrs_seq,
                         const c16_t *data,
-                        c16_t *out)
+                        c16_t *out
+#ifdef FLT16_MAX
+                        ,int use_fp16
+#endif
+		)
 {
   // asign the function pointers
   map_dmrs_func_t map_dmrs_ptr = NULL;
@@ -493,17 +525,38 @@ static void map_symbols(const nr_phy_pxsch_params_t p,
     if (dmrs_symbol) {
       c16_t mod_dmrs[ALNARS_16_4(n_dmrs)] __attribute((aligned(16)));
       if (p.transform_precoding == transformPrecoder_disabled) {
+#ifdef FLT16_MAX
+	if (use_fp16) {
+          nr_modulation(gold, n_dmrs * 2, DMRS_MOD_ORDER, NULL,(_Float16 *)mod_dmrs);
+          dmrs_amp_mult_fp16(p.dmrs_port, p.Wt, p.Wf, mod_dmrs, mod_dmrs_amp, n_dmrs, p.dmrs_type, p.num_cdm_no_data);
+	}
+	else {
+          nr_modulation(gold, n_dmrs * 2, DMRS_MOD_ORDER, (int16_t *)mod_dmrs,NULL);
+          dmrs_amp_mult(p.dmrs_port, p.Wt, p.Wf, mod_dmrs, mod_dmrs_amp, n_dmrs, p.dmrs_type, p.num_cdm_no_data);
+	}
+#else
         nr_modulation(gold, n_dmrs * 2, DMRS_MOD_ORDER, (int16_t *)mod_dmrs);
         dmrs_amp_mult(p.dmrs_port, p.Wt, p.Wf, mod_dmrs, mod_dmrs_amp, n_dmrs, p.dmrs_type, p.num_cdm_no_data);
+#endif
       } else {
         dmrs_amp_mult(p.dmrs_port, p.Wt, p.Wf, dmrs_seq, mod_dmrs_amp, n_dmrs, p.dmrs_type, p.num_cdm_no_data);
       }
     } else if ((p.pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) && ptrs_symbol) {
       AssertFatal(p.transform_precoding == transformPrecoder_disabled, "PTRS NOT SUPPORTED IF TRANSFORM PRECODING IS ENABLED\n");
       c16_t mod_ptrs[ALNARS_16_4(p.nb_rb)] __attribute((aligned(16)));
+#ifdef FLT16_MAX
+      if (use_fp16)
+        nr_modulation(gold, p.nb_rb, DMRS_MOD_ORDER, NULL,(_Float16 *)mod_ptrs);
+      else {
+        nr_modulation(gold, p.nb_rb, DMRS_MOD_ORDER, (int16_t *)mod_ptrs,NULL);
+        const unsigned int beta_ptrs = 1; // temp value until power control is implemented
+        mult_complex_vector_real_scalar(mod_ptrs, beta_ptrs * AMP, mod_ptrs_amp, p.nb_rb);
+      }
+#else
       nr_modulation(gold, p.nb_rb, DMRS_MOD_ORDER, (int16_t *)mod_ptrs);
       const unsigned int beta_ptrs = 1; // temp value until power control is implemented
       mult_complex_vector_real_scalar(mod_ptrs, beta_ptrs * AMP, mod_ptrs_amp, p.nb_rb);
+#endif
     }
     map_current_symbol(p,
                        dmrs_symbol,
@@ -1286,12 +1339,24 @@ void nr_ue_ulsch_procedures(PHY_VARS_NR_UE *UE,
 
   int max_num_re = Nl * number_of_symbols * nb_rb * NR_NB_SC_PER_RB;
   c16_t d_mod[max_num_re] __attribute__((aligned(16)));
-
+#ifdef FLT16_MAX
+  if (UE->use_fp16)
+    nr_modulation(scrambled_output, // assume one codeword for the moment
+                  available_bits,
+                  mod_order,
+		  NULL,
+                  (_Float16 *)d_mod);
+  else 
+    nr_modulation(scrambled_output, // assume one codeword for the moment
+                  available_bits,
+                  mod_order,
+                  (int16_t *)d_mod,NULL);
+#else
   nr_modulation(scrambled_output, // assume one codeword for the moment
                 available_bits,
                 mod_order,
                 (int16_t *)d_mod);
-
+#endif
   /////////////////////////ULSCH layer mapping/////////////////////////
 
   const int sz = available_bits / mod_order / Nl;
@@ -1410,7 +1475,11 @@ void nr_ue_ulsch_procedures(PHY_VARS_NR_UE *UE,
                                     .delta = delta,
                                     .num_cdm_no_data = cdm_grps_no_data};
 
-    map_symbols(params, slot, dmrs_seq, data, tx_precoding[nl]);
+    map_symbols(params, slot, dmrs_seq, data, tx_precoding[nl]
+#ifdef FLT16_MAX
+		    ,UE->use_fp16
+#endif
+		    );
 
   } // for (nl=0; nl < Nl; nl++)
 
@@ -1519,7 +1588,11 @@ uint8_t nr_ue_pusch_common_procedures(PHY_VARS_NR_UE *UE,
       if (was_symbol_used[i] == false)
         continue;
       for (int ap = 0; ap < n_antenna_ports; ap++) {
-        apply_nr_rotation_TX(frame_parms, txdataF[ap], frame_parms->symbol_rotation[linktype], slot, N_RB, i, 1);
+        apply_nr_rotation_TX(frame_parms, txdataF[ap], frame_parms->symbol_rotation[linktype], slot, N_RB, i, 1
+#ifdef FLT16_MAX
+  		             ,UE->use_fp16,AMP
+#endif
+		            );
       }
     }
   }

openair1/PHY/TOOLS/tools_defs.h

@@ -39,6 +39,11 @@
 #include "common/utils/utils.h"
 #include "common/utils/LOG/log.h"
 
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
+
 #define simd_q15_t simde__m128i
 #define shiftright_int16(a,shift) simde_mm_srai_epi16(a,shift)
 #define set1_int16(a) simde_mm_set1_epi16(a)
@@ -244,6 +249,17 @@ extern "C" {
     };
   }
 
+#ifdef __aarch64__
+  __attribute__((always_inline)) inline cf16_t cf16mulReal(const cf16_t a, const float16_t b)
+  {
+    return (cf16_t){.r = (float16_t)(a.r * b), .i = (float16_t)(a.i * b)};
+  }
+#else
+  __attribute__((always_inline)) inline cf16_t cf16mulReal(const cf16_t a, const _Float16 b)
+  {
+    return (cf16_t){.r = (_Float16)(a.r * b), .i = (_Float16)(a.i * b)};
+  }
+#endif
   // On N complex numbers
   //   y.r += (x * alpha.r) >> 14
   //   y.i += (x * alpha.i) >> 14
@@ -970,6 +986,154 @@ void InitSinLUT(void);
 // ret.i == sinus << 14
 c16_t get_sin_cos(double phase);
 
+#ifdef FLT16_MAX
+// Interleaved input: [re0, im0, re1, im1, ...] as _Float16 (assumed to be -1...1
+// Interleaved output: c16_t Q15 with truncation rounding
+static inline void fp16_to_q15(const cf16_t *in_ri,
+                               c16_t *out_q15,
+                               int n_complex,
+		   	       int16_t amp)
+{
+   const int N = n_complex * 2;
+   int i = 0;
+   int16_t *out_q15_2 = (int16_t *)out_q15;
+#if defined(__AVX512FP16__) 
+   const _Float16 *in=(_Float16 *)in_ri;
+   const __m512h k = _mm512_set1_ph((_Float16)amp);     // FP16 scale
+   for (; i + 32 <= N; i += 32) {                  // 32 halves per iter
+      __m512h h  = _mm512_loadu_ph(in + i);    // unaligned ok
+      __m512h hs = _mm512_mul_ph(h, k);           // scale
+      __m512i q  = _mm512_cvtph_epi16(hs);        // trunc→i16, saturating
+      _mm512_storeu_si512((__m512i*)(out_q15_2 + i), q);
+   }
+   for (; i + 16 <= N; i += 16) {                  // 16 halves per iter
+      __m256h h  = _mm256_loadu_ph(in + i);    // unaligned ok
+      __m256h hs = _mm256_mul_ph(h, *(__m256h*)&k);           // scale
+      __m256i q  = _mm256_cvtph_epi16(hs);        // trunc→i16, saturating
+      _mm256_storeu_si256((__m256i*)(out_q15_2 + i), q);
+   }
+   for (; i + 8 <= N; i += 8) {                  // 16 halves per iter
+      __m128h h  = _mm_loadu_ph(in + i);    // unaligned ok
+      __m128h hs = _mm_mul_ph(h, *(__m128h*)&k);           // scale
+      __m128i q  = _mm_cvtph_epi16(hs);        // trunc→i16, saturating
+      _mm_storeu_si128((__m128i*)(out_q15_2 + i), q);
+   }
+#elif defined(__aarch64__)
+   const float16_t *in=(float16_t *)in_ri;
+   const float16x8_t k = vdupq_n_f16((float16_t)amp);
+   for (; i + 8 <= N; i += 8) {
+     float16x8_t h  = vld1q_f16((float16_t*)(in + i));     // 8 halves (4 complex)
+     float16x8_t hs = vmulq_f16(h, k);          // scale to Q15 domain
+     int16x8_t   q  = vcvtq_s16_f16(hs);        // truncate toward zero, saturate
+     vst1q_s16((int16_t*)(out_q15_2 + i), q);
+   }
+#endif
+   for (; i < N; ++i) {                           // tail
+     float f = (float)in[i] * (float)amp;
+     int   x = (int) (f > 0 ? floorf(f) : ceilf(f));  // trunc toward zero
+     if (x >  32767) x =  32767;                      // saturate
+     if (x < -32768) x = -32768;
+     out_q15_2[i] = (int16_t)x;
+   }
+}
+static inline void rotate_cpx_vector_fp16(const cf16_t *const x, const cf16_t *const alpha, cf16_t *y, uint32_t N)
+{
+    uint32_t i=0;
+#if defined(__aarch64__)
+    const float16x8_t zerosq=vdupq_n_f16(0.0f);
+    const float16x4_t zeros=vdup_n_f16(0.0f);
+    const uint32x4_t alpha16x8=vdupq_n_u32(*(uint32_t*)alpha);
+    for (; i + 4 <= N ; i+=4) {
+       float16x8_t x16x8 = vld1q_f16((float16_t*)(x + i));	    
+       float16x8_t y16x8 = vcmlaq_f16(zerosq,x16x8,*((float16x8_t*)&alpha16x8));
+       y16x8 = vcmlaq_rot90_f16(y16x8,x16x8,*((float16x8_t*)&alpha16x8));
+       vst1q_f16((float16_t*)(y + i),y16x8);
+    }
+    for (; i + 2 <= N ; i+=2) {
+       float16x4_t x16x4 = vld1_f16((float16_t*)(x + i));	    
+       float16x4_t y16x4 = vcmla_f16((float16x4_t)zeros,x16x4,*((float16x4_t*)&alpha16x8));
+       y16x4 = vcmla_rot90_f16(y16x4,x16x4,*((float16x4_t*)&alpha16x8));
+       vst1_f16((float16_t*)(y + i),y16x4);
+    }
+#elif defined(__AVX512FP16__) 
+    __m512h alpha512=_mm512_set1_pch(*(_Float16 _Complex*)alpha);
+    const __m512h alpha512c=_mm512_conj_pch(alpha512);
+    for (; i + 32 < N; i+=32) {
+    _mm512_storeu_ph((__m512h*)(y + i),_mm512_cmul_pch(_mm512_loadu_ph(x + i),alpha512c));
+    _mm512_storeu_ph((__m512h*)(y + i + 16),_mm512_cmul_pch(_mm512_loadu_ph(x + i + 16),alpha512c));
+	    
+//       if (i==0) { for (int j=0;j<2;j+=2) printf("%0.2f+(%0.2fj) ",(double)((_Float16*)&x512)[j],(double)((_Float16*)&x512)[j]); printf("\n");}
+       //if (i==0) { for (int j=0;j<32;j+=2) printf("%0.2f+(%0.2fj) ",(double)((_Float16*)&y512)[j],(double)((_Float16*)&y512)[j]); printf("\n");}
+//       _mm512_storeu_ph((__m512h*)(y + i),y512);
+    }
+    for (; i + 8 < N; i+=8) 
+       _mm256_storeu_ph((__m256h*)(y + i),_mm256_cmul_pch( _mm256_loadu_ph(x + i),*(__m256h*)&alpha512c));
+    
+    for (; i + 4 < N; i+=4) 
+       _mm_storeu_ph((__m128h*)(y + i),_mm_cmul_pch(_mm_loadu_ph(x + i),*(__m128h*)&alpha512c));
+    
+   /* 
+    printf("y:");
+    for (int j=0;j<32;j+=2) 
+       printf("%0.2f+(%0.2fj) ",(double)((_Float16*)y)[j],(double)((_Float16*)y)[j+1]);
+    printf("\n");*/
+#else
+    AssertFatal(1==0,"No support for fp16 complex multiplication and FP16 is requested\n");
+#endif
+}
+
+static inline void rotate_cpx_vector_fp16in_q15out(const cf16_t *const x, const cf16_t *const alpha, int16_t tx_amp, c16_t *y, uint32_t N)
+{
+    uint32_t i=0;
+#if defined(__aarch64__)
+    const float16x8_t zerosq=vdupq_n_f16(0.0f);
+    const float16x4_t zeros=vdup_n_f16(0.0f);
+    cf16_t alpha2;
+    alpha2.r = alpha->r * (float16_t)tx_amp;
+    alpha2.i = alpha->i * (float16_t)tx_amp;
+    const uint32x4_t alpha16x8=vdupq_n_u32(*(uint32_t*)&alpha2);
+    for (; i + 4 <= N ; i+=4) {
+       float16x8_t x16x8 = vld1q_f16((float16_t*)(x + i));	    
+       float16x8_t y16x8 = vcmlaq_f16(zerosq,x16x8,*((float16x8_t*)&alpha16x8));
+       y16x8 = vcmlaq_rot90_f16(y16x8,x16x8,*((float16x8_t*)&alpha16x8));
+       vst1q_s16((int16_t*)(y + i),vcvtq_s16_f16(y16x8));
+    }
+    for (; i + 2 <= N ; i+=2) {
+       float16x4_t x16x4 = vld1_f16((float16_t*)(x + i));	    
+       float16x4_t y16x4 = vcmla_f16((float16x4_t)zeros,x16x4,*((float16x4_t*)&alpha16x8));
+       y16x4 = vcmla_rot90_f16(y16x4,x16x4,*((float16x4_t*)&alpha16x8));
+       vst1_s16((int16_t*)(y + i),vcvt_s16_f16(y16x4));
+    }
+#elif defined(__AVX512FP16__) 
+    cf16_t alpha2;
+    alpha2.r = alpha->r * (_Float16)tx_amp;
+    alpha2.i = alpha->i * (_Float16)tx_amp;
+    __m512h alpha512=_mm512_set1_pch(*(_Float16 _Complex*)&alpha2);
+    const __m512h alpha512c=_mm512_conj_pch(alpha512);
+    for (; i + 32 < N; i+=32) {
+    _mm512_storeu_si512((__m512i*)(y + i),_mm512_cvtph_epi16(_mm512_cmul_pch(_mm512_loadu_ph(x + i),alpha512c)));
+    _mm512_storeu_si512((__m512i*)(y + i + 16),_mm512_cvtph_epi16(_mm512_cmul_pch(_mm512_loadu_ph(x + i + 16),alpha512c)));
+	    
+//       if (i==0) { for (int j=0;j<2;j+=2) printf("%0.2f+(%0.2fj) ",(double)((_Float16*)&x512)[j],(double)((_Float16*)&x512)[j]); printf("\n");}
+       //if (i==0) { for (int j=0;j<32;j+=2) printf("%0.2f+(%0.2fj) ",(double)((_Float16*)&y512)[j],(double)((_Float16*)&y512)[j]); printf("\n");}
+//       _mm512_storeu_ph((__m512h*)(y + i),y512);
+    }
+    for (; i + 8 < N; i+=8) 
+       _mm256_storeu_si256((__m256i*)(y + i),_mm256_cvtph_epi16(_mm256_cmul_pch( _mm256_loadu_ph(x + i),*(__m256h*)&alpha512c)));
+    
+    for (; i + 4 < N; i+=4) 
+       _mm_storeu_si128((__m128i*)(y + i),_mm_cvtph_epi16(_mm_cmul_pch(_mm_loadu_ph(x + i),*(__m128h*)&alpha512c)));
+    
+   /* 
+    printf("y:");
+    for (int j=0;j<32;j+=2) 
+       printf("%0.2f+(%0.2fj) ",(double)((_Float16*)y)[j],(double)((_Float16*)y)[j+1]);
+    printf("\n");*/
+#else
+    AssertFatal(1==0,"No support for fp16 complex multiplication and FP16 is requested\n");
+#endif
+}
+#endif
 #ifdef __cplusplus
 }
 #endif

openair1/PHY/defs_gNB.h

@@ -105,6 +105,9 @@ typedef struct {
   uint8_t *b;
   /// Pointers to transport block segments
   uint8_t **c;
+#ifdef ENABLE_CUDA
+  uint8_t *c_dev;
+#endif
   /// Interleaver outputs
   uint8_t *f;
   /// REs unavailable for DLSCH (overlapping with PTRS, CSIRS etc.)
@@ -458,9 +461,11 @@ typedef struct PHY_VARS_gNB_s {
   time_stats_t dlsch_resource_mapping_stats;
   time_stats_t dlsch_precoding_stats;
   time_stats_t tinput;
+  time_stats_t tinput_memcpy;
   time_stats_t tprep;
   time_stats_t tparity;
   time_stats_t toutput;
+  time_stats_t tconcat;
   
   time_stats_t dlsch_rate_matching_stats;
   time_stats_t dlsch_interleaving_stats;
@@ -501,6 +506,8 @@ typedef struct PHY_VARS_gNB_s {
   pthread_t L1_tx_thread;
   int L1_tx_thread_core;
   void *scopeData;
+  int use_gpu;
+  int use_fp16;
 } PHY_VARS_gNB;
 
 struct puschSymbolReqId {

openair1/PHY/defs_nr_UE.h

@@ -528,6 +528,8 @@ typedef struct PHY_VARS_NR_UE_s {
   ntn_config_message_t* ntn_config_message;
   pthread_t main_thread;
   pthread_t stat_thread;
+  int use_gpu;
+  int use_fp16;
 } PHY_VARS_NR_UE;
 
 typedef struct {

openair1/PHY/nr_phy_common/inc/nr_phy_common.h

@@ -372,7 +372,11 @@ bool generate_srs_nr(const NR_DL_FRAME_PARMS *frame_parms,
                      nr_srs_info_t *nr_srs_info,
                      int16_t amp,
                      frame_t frame_number,
-                     slot_t slot_number);
+                     slot_t slot_number
+#ifdef FLT16_MAX
+		     ,int use_fp16 
+#endif
+		     );
 void nr_generate_csi_rs(const NR_DL_FRAME_PARMS *frame_parms,
                         const csi_mapping_parms_t *phy_csi_parms,
                         const int16_t amp,
@@ -386,6 +390,10 @@ void nr_generate_csi_rs(const NR_DL_FRAME_PARMS *frame_parms,
                         const uint16_t scramb_id,
                         const uint8_t power_control_offset_ss,
                         const uint8_t cdm_type,
-                        c16_t **dataF);
+                        c16_t **dataF
+#ifdef FLT16_MAX
+			,int use_fp16
+#endif
+			);
 
 #endif

openair1/PHY/nr_phy_common/src/nr_phy_common_csirs.c

@@ -21,6 +21,10 @@
 
 #include "PHY/nr_phy_common/inc/nr_phy_common.h"
 
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 static void csi_rs_resource_mapping(c16_t **dataF,
                                     int csi_rs_length,
                                     int16_t mod_csi[][csi_rs_length >> 1],
@@ -94,27 +98,59 @@ static void get_modulated_csi_symbols(int symbols_per_slot,
                                       int l0,
                                       int l1,
                                       int row,
-                                      int scramb_id)
+                                      int scramb_id
+#ifdef FLT16_MAX
+				      ,int use_fp16
+#endif
+				      )
 {
   for (int lp = 0; lp <= lprime; lp++) {
     int symb = l0;
     const uint32_t *gold =
         nr_gold_csi_rs(N_RB_DL, symbols_per_slot, slot, symb + lp, scramb_id);
+#ifdef FLT16_MAX
+    if (use_fp16)
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, NULL, (_Float16*)mod_csi[symb + lp]);
+    else
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + lp],NULL);
+#else
     nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + lp]);
+#endif
     if ((row == 5) || (row == 7) || (row == 11) || (row == 13) || (row == 16)) {
       const uint32_t *gold =
           nr_gold_csi_rs(N_RB_DL, symbols_per_slot, slot, symb + 1, scramb_id);
+#ifdef FLT16_MAX
+    if (use_fp16)
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, NULL, (_Float16*)mod_csi[symb + 1]);
+    else
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + 1],NULL);
+#else
       nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + 1]);
+#endif
     }
     if ((row == 14) || (row == 13) || (row == 16) || (row == 17)) {
       symb = l1;
       const uint32_t *gold =
           nr_gold_csi_rs(N_RB_DL, symbols_per_slot, slot, symb + lp, scramb_id);
-      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + lp]);
-      if ((row == 13) || (row == 16)) {
+#ifdef FLT16_MAX
+    if (use_fp16)
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, NULL, (_Float16*)mod_csi[symb + lp]);
+    else
+      nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + lp],NULL);
+#else
+    nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + lp]);
+#endif
+    if ((row == 13) || (row == 16)) {
         const uint32_t *gold =
             nr_gold_csi_rs(N_RB_DL, symbols_per_slot, slot, symb + 1, scramb_id);
-        nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + 1]);
+#ifdef FLT16_MAX
+        if (use_fp16)
+          nr_modulation(gold, mod_length, DMRS_MOD_ORDER, NULL, (_Float16*)mod_csi[symb + 1]);
+	else
+          nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + 1],NULL);
+#else
+          nr_modulation(gold, mod_length, DMRS_MOD_ORDER, mod_csi[symb + 1]);
+#endif
       }
     }
   }
@@ -574,7 +610,11 @@ void nr_generate_csi_rs(const NR_DL_FRAME_PARMS *frame_parms,
                         const uint16_t scramb_id,
                         const uint8_t power_control_offset_ss,
                         const uint8_t cdm_type,
-                        c16_t **dataF)
+                        c16_t **dataF
+#ifdef FLT16_MAX
+			,int use_fp16
+#endif
+			)
 {
 #ifdef NR_CSIRS_DEBUG
   LOG_I(NR_PHY,
@@ -610,7 +650,11 @@ void nr_generate_csi_rs(const NR_DL_FRAME_PARMS *frame_parms,
                             symb_l0,
                             symb_l1,
                             row,
-                            scramb_id);
+                            scramb_id
+#ifdef FLT16_MAX
+			    ,use_fp16
+#endif
+			    );
 
   const uint32_t beta = get_csi_beta_amplitude(amp, power_control_offset_ss);
   const double alpha = (phy_csi_parms->ports == 1) ? rho : 2 * rho;

openair1/PHY/nr_phy_common/src/nr_phy_common_srs.c

@@ -146,7 +146,11 @@ bool generate_srs_nr(const NR_DL_FRAME_PARMS *frame_parms,
                      nr_srs_info_t *nr_srs_info,
                      int16_t amp,
                      frame_t frame_number,
-                     slot_t slot_number)
+                     slot_t slot_number
+#ifdef FLT16_MAX
+		     ,int use_fp16
+#endif
+		     )
 {
   uint8_t K_TC = 2 << nr_srs_info->comb_size;
   /* Number of antenna ports (M) can't be higher than number of physical antennas (N): M <= N */
@@ -329,6 +333,14 @@ bool generate_srs_nr(const NR_DL_FRAME_PARMS *frame_parms,
         LOG_I(NR_PHY, "(%d)  \t%i\t%i\n", subcarrier_log, r_amp.r, r_amp.i);
 #endif
 
+#ifdef FLT16_MAX
+	if (use_fp16) {
+          cf16_t r_amp = {((_Float16)(r.r / sqrt_N_ap)),
+                          ((_Float16)(r.i / sqrt_N_ap))};
+          *(cf16_t*)&txdataF[p_index][symbol_offset + l_line_offset + subcarrier] = r_amp;
+	}
+	else
+#endif
         txdataF[p_index][symbol_offset + l_line_offset + subcarrier] = r_amp;
 
         // Subcarrier increment

openair1/SCHED_NR/phy_procedures_nr_gNB.c

@@ -30,6 +30,10 @@
 #include "common/utils/LOG/log.h"
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include "PHY/INIT/nr_phy_init.h"
+#ifndef __STDC_WANT_IEC_60559_TYPES_EXT__
+#define __STDC_WANT_IEC_60559_TYPES_EXT__
+#endif
+#include <float.h>
 #include "PHY/MODULATION/nr_modulation.h"
 #include "T.h"
 #include "executables/nr-softmodem.h"
@@ -41,7 +45,6 @@
 #include <stdint.h>
 #include <openair1/PHY/TOOLS/phy_scope_interface.h>
 #include "PHY/log_tools.h"
-
 //#define DEBUG_RXDATA
 //#define SRS_IND_DEBUG
 static void nr_fill_indication(PHY_VARS_gNB *gNB,
@@ -165,8 +168,16 @@ void nr_common_signal_procedures(PHY_VARS_gNB *gNB, int frame, int slot, const n
                                       fp->symbols_per_slot,
                                       bitmap);
 
-  nr_generate_pss(&txdataF[beam_nb][0][txdataF_offset], gNB->TX_AMP, ssb_start_symbol, cfg, fp);
-  nr_generate_sss(&txdataF[beam_nb][0][txdataF_offset], gNB->TX_AMP, ssb_start_symbol, cfg, fp);
+  nr_generate_pss(&txdataF[beam_nb][0][txdataF_offset], gNB->TX_AMP, ssb_start_symbol, cfg, fp
+#ifdef FLT16_MAX
+		  ,gNB->use_fp16
+#endif
+		  );
+  nr_generate_sss(&txdataF[beam_nb][0][txdataF_offset], gNB->TX_AMP, ssb_start_symbol, cfg, fp
+#ifdef FLT16_MAX
+		  ,gNB->use_fp16
+#endif
+		  );
 
   uint16_t slots_per_hf = (fp->slots_per_frame) >> 1;
   int n_hf = slot < slots_per_hf ? 0 : 1;
@@ -246,7 +257,12 @@ static void nr_generate_csi_rs_gNB(PHY_VARS_gNB *gNB,
                      csi_params->scramb_id,
                      csi_params->power_control_offset_ss,
                      csi_params->cdm_type,
-                     gNB->common_vars.txdataF[beam_nb]);
+                     gNB->common_vars.txdataF[beam_nb]
+#ifdef FLT16_MAX
+		     ,gNB->use_fp16
+#endif
+		     );
+  
 }
 
 void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
@@ -343,7 +359,11 @@ void phy_procedures_gNB_TX(PHY_VARS_gNB *gNB,
                              slot,
                              fp->N_RB_DL,
                              0,
-                             fp->Ncp == EXTENDED ? 12 : 14);
+                             fp->Ncp == EXTENDED ? 12 : 14
+#ifdef FLT16_MAX
+			     ,gNB->use_fp16,gNB->TX_AMP
+#endif
+			     );
       }
       T(T_GNB_PHY_DL_OUTPUT_SIGNAL,
         T_INT(0),
@@ -1082,7 +1102,7 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, N
                       gNB->nr_srs_info[i],
                       AMP,
                       frame_rx,
-                      slot_rx);
+                      slot_rx,0);
     }
     stop_meas(&gNB->generate_srs_stats);
     c16_t **rxdataF = gNB->common_vars.rxdataF[srs->beam_nb];

openair1/SCHED_NR_UE/phy_procedures_nr_ue.c

@@ -38,6 +38,7 @@
 #include "PHY/defs_nr_UE.h"
 #include "PHY/NR_REFSIG/dmrs_nr.h"
 #include "PHY/MODULATION/modulation_UE.h"
+#include "PHY/TOOLS/tools_defs.h"
 #include "PHY/INIT/nr_phy_init.h"
 #include "PHY/nr_phy_common/inc/nr_phy_common.h"
 #include "PHY/NR_REFSIG/ptrs_nr.h"
@@ -360,7 +361,11 @@ static bool ue_srs_procedures_nr(PHY_VARS_NR_UE *ue,
                                    ue->nr_srs_info,
                                    AMP,
                                    proc->frame_tx,
-                                   proc->nr_slot_tx);
+                                   proc->nr_slot_tx
+#ifdef FLT16_MAX
+				   ,ue->use_fp16
+#endif
+				   );
   return generated;
 }
 

openair1/SIMULATION/NR_PHY/dlschsim.c

@@ -517,7 +517,7 @@ int main(int argc, char **argv)
         unsigned char output[nb_rb * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64)));
         bzero(output, sizeof(output));
 	if (input_fd == NULL) {
-	  nr_dlsch_encoding(gNB, 1, dlsch, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
+	  nr_dlsch_encoding(gNB, 1, dlsch, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
 	}
 
 	for (SNR = snr0; SNR < snr1 && !stop; SNR += snr_step) {

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -311,8 +311,10 @@ void validate_input_pmi(nfapi_nr_config_request_scf_t *gNB_config,
               num_antenna_ports, pmi_pdu->num_ant_ports, pmi);
 }
 
-
 configmodule_interface_t *uniqCfg = NULL;
+extern uint32_t use_gpu;
+extern uint32_t use_fp16;
+
 int main(int argc, char **argv)
 {
   stop = false;
@@ -408,7 +410,7 @@ int main(int argc, char **argv)
 
   FILE *scg_fd=NULL;
 
-  while ((c = getopt(argc, argv, "--:O:f:hA:p:f:g:i:n:s:S:t:v:x:y:z:o:H:M:N:F:GR:d:PI:L:a:b:e:m:w:T:U:q:X:Y:Z:")) != -1) {
+  while ((c = getopt(argc, argv, "--:O:f:hA:p:f:g:i:n:s:S:t:v:x:y:z:o:H:M:N:F:GR:d:PQVI:L:a:b:e:m:w:T:U:q:X:Y:Z:")) != -1) {
 
     /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
     /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
@@ -528,7 +530,14 @@ int main(int argc, char **argv)
       print_perf=1;
       cpu_meas_enabled = 1;
       break;
-      
+
+    case 'V':
+      use_fp16=1;
+      break; 
+
+    case 'Q':
+      use_gpu=1;
+      break; 
     case 'I':
       max_ldpc_iterations = atoi(optarg);
       break;
@@ -680,6 +689,7 @@ int main(int argc, char **argv)
   memset(RC.gNB[0],0,sizeof(PHY_VARS_gNB));
 
   gNB = RC.gNB[0];
+  gNB->use_fp16 = use_fp16;
   gNB->ofdm_offset_divisor = UINT_MAX;
   gNB->phase_comp = true; // we need to perform phase compensation, otherwise everything will fail
   frame_parms = &gNB->frame_parms; //to be initialized I suppose (maybe not necessary for PBCH)
@@ -920,6 +930,7 @@ int main(int argc, char **argv)
   PHY_vars_UE_g = malloc(sizeof(PHY_VARS_NR_UE**));
   PHY_vars_UE_g[0] = malloc(sizeof(PHY_VARS_NR_UE*));
   PHY_vars_UE_g[0][0] = UE;
+  printf("Copying gNB frame_parms to UE\n");
   memcpy(&UE->frame_parms,frame_parms,sizeof(NR_DL_FRAME_PARMS));
   UE->frame_parms.nb_antennas_rx = n_rx;
   UE->frame_parms.nb_antenna_ports_gNB = n_tx;
@@ -1041,9 +1052,11 @@ int main(int argc, char **argv)
     reset_meas(&gNB->dlsch_encoding_stats);
     reset_meas(&gNB->dci_generation_stats);
     reset_meas(&gNB->tinput);
+    reset_meas(&gNB->tinput_memcpy);
     reset_meas(&gNB->tprep);
     reset_meas(&gNB->tparity);
     reset_meas(&gNB->toutput);
+    reset_meas(&gNB->tconcat);
     reset_meas(&gNB->phase_comp_stats);
 
     uint32_t errors_scrambling[16] = {0};
@@ -1135,7 +1148,7 @@ int main(int argc, char **argv)
                   1);
         }
         if (n_trials == 1)
-          printf("slot_offset %d, txdataF_offset %d \n", slot_offset, txdataF_offset);
+          printf("frame %d slot %d slot_offset %d, txdataF_offset %d \n", frame, slot, slot_offset, txdataF_offset);
 
         //TODO: loop over slots
         for (aa=0; aa<gNB->frame_parms.nb_antennas_tx; aa++) {
@@ -1335,12 +1348,19 @@ int main(int argc, char **argv)
       printStatIndent2(&gNB->dci_generation_stats, "DCI encoding time");
       printStatIndent2(&gNB->dlsch_encoding_stats,"DLSCH encoding time");
       printStatIndent3(&gNB->dlsch_segmentation_stats,"DLSCH segmentation time");
+      gNB->tinput.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->tinput,"DLSCH LDPC input processing time");
-      printStatIndent3(&gNB->tprep,"DLSCH LDPC input preparation time");
+      gNB->tinput_memcpy.trials = gNB->dlsch_encoding_stats.trials;
+      printStatIndent3(&gNB->tinput_memcpy,"DLSCH LDPC input memcpy time");
+      gNB->tparity.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->tparity,"DLSCH LDPC parity generation time");
+      gNB->toutput.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->toutput,"DLSCH LDPC output generation time");
+      gNB->dlsch_rate_matching_stats.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->dlsch_rate_matching_stats,"DLSCH Rate Matching time");
+      gNB->dlsch_interleaving_stats.trials = gNB->dlsch_encoding_stats.trials;
       printStatIndent3(&gNB->dlsch_interleaving_stats,  "DLSCH Interleaving time");
+      printStatIndent3(&gNB->tconcat,  "DLSCH Segment Concatenation time");
       printStatIndent2(&gNB->dlsch_modulation_stats,"DLSCH modulation time");
       printStatIndent2(&gNB->dlsch_scrambling_stats, "DLSCH scrambling time");
       printStatIndent2(&gNB->dlsch_pdsch_generation_stats,"DLSCH PDSCH Generation time");

openair1/SIMULATION/NR_PHY/nr_unitary_common.c

@@ -32,6 +32,8 @@
 
 // Define signal handler to attempt graceful termination
 bool stop = false;
+uint32_t use_gpu=0;
+uint32_t use_fp16=0;
 
 void sigint_handler(int arg)
 {

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -1742,7 +1742,7 @@ int main(int argc, char *argv[])
       printStatIndent2(&gNB->rx_pusch_symbol_processing_stats, "RX PUSCH Symbol Processing time");
       printStatIndent(&gNB->ulsch_decoding_stats,"ULSCH total decoding time");
       printStatIndent2(&gNB->ts_deinterleave, "ULSCH segment deinterleaving time");
-      printStatIndent2(&gNB->ts_rate_unmatch, "ULSCH segment rate recovery time");
+      printStatIndent2(&gNB->ts_rate_unmatch, "ULSCH segment rate matching time");
       printStatIndent2(&gNB->ts_ldpc_decode, "ULSCH segments decoding time");
       printStatIndent(&gNB->rx_srs_stats,"RX SRS time");
       printStatIndent2(&gNB->generate_srs_stats,"Generate SRS sequence time");