Documentation update

CMakeLists.txt

@@ -587,17 +587,31 @@ target_link_libraries(benetel_5g PRIVATE pthread dl rt m numa)
 # LDPC offload library
 ##########################################################
 
-add_boolean_option(ENABLE_LDPC_T1 OFF "Build support for LDPC Offload to T1 library" OFF)
-if (ENABLE_LDPC_T1)
+add_boolean_option(ENABLE_LDPC_OFFLOAD_T1 OFF "Build support for LDPC Offload to T1 library" OFF)
+if (ENABLE_LDPC_OFFLOAD_T1)
   pkg_check_modules(LIBDPDK_T1 REQUIRED libdpdk=20.05.0)
   find_library(T1 NAMES rte_pmd_hpac_sdfec_pmd REQUIRED)
   if (NOT T1)
     message(FATAL_ERROR "Library rte_pmd_hpac_sdfec_pmd for T1 offload not found")
   endif()
-  add_library(ldpc_t1 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c)
-  set_target_properties(ldpc_t1 PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API")
-  target_compile_options(ldpc_t1 PRIVATE ${LIBDPDK_T1_CFLAGS})
-  target_link_libraries(ldpc_t1 ${LIBDPDK_T1_LDFLAGS} ${T1})
+  add_library(ldpc_offload_t1 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c)
+  set_target_properties(ldpc_offload_t1 PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API")
+  target_compile_options(ldpc_offload_t1 PRIVATE ${LIBDPDK_T1_CFLAGS})
+  target_link_libraries(ldpc_offload_t1 ${LIBDPDK_T1_LDFLAGS} ${T1})
+endif()
+
+add_boolean_option(ENABLE_LDPC_OFFLOAD_T2 OFF "Build support for LDPC Offload to T2 library" OFF)
+if (ENABLE_LDPC_OFFLOAD_T2)
+  set(ENV{PKG_CONFIG_PATH} "/usr/local/lib64/pkgconfig")
+  pkg_search_module(LIBDPDK_T2 REQUIRED libdpdk=20.11.3)
+  find_library(T2 NAMES rte_baseband_accl_ldpc REQUIRED)
+  if (NOT T2)
+    message(FATAL_ERROR "Library rte_baseband_accl_ldpc for T2 offload not found")
+  endif()
+  add_library(ldpc_offload_t2 MODULE ${OPENAIR1_DIR}/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c)
+  set_target_properties(ldpc_offload_t2 PROPERTIES COMPILE_FLAGS "-DALLOW_EXPERIMENTAL_API")
+  target_compile_options(ldpc_offload_t2 PRIVATE ${LIBDPDK_T2_CFLAGS})
+  target_link_libraries(ldpc_offload_t2 ${LIBDPDK_T2_LDFLAGS} ${T2})
 endif()
 
 ##########################################################
@@ -2270,8 +2284,12 @@ target_link_libraries(nr-softmodem PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
 
 add_dependencies(nr-softmodem ldpc_orig ldpc_optim ldpc_optim8seg ldpc)
 
-if (ENABLE_LDPC_T1)
-  add_dependencies(nr-softmodem ldpc_t1)
+if (ENABLE_LDPC_OFFLOAD_T1)
+  add_dependencies(nr-softmodem ldpc_offload_t1)
+endif()
+
+if (ENABLE_LDPC_OFFLOAD_T2)
+  add_dependencies(nr-softmodem ldpc_offload_t2)
 endif()
 
 # force the generation of ASN.1 so that we don't need to wait during the build
@@ -2508,10 +2526,15 @@ add_executable(nr_ulsim
   ${SHLIB_LOADER_SOURCES}
   )
 
-if (ENABLE_LDPC_T1)
-  add_dependencies(nr_ulsim ldpc_t1)
+if (ENABLE_LDPC_OFFLOAD_T1)
+  add_dependencies(nr_ulsim ldpc_offloa_offload_t1)
 endif()
 
+if (ENABLE_LDPC_OFFLOAD_T2)
+  add_dependencies(nr_ulsim ldpc_offload_t2)
+endif()
+
+
 target_link_libraries(nr_ulsim PRIVATE
   -Wl,--start-group UTIL SIMU SIMU_ETH PHY_COMMON PHY_NR_COMMON PHY_NR PHY_NR_UE SCHED_NR_LIB SCHED_NR_UE_LIB MAC_NR MAC_UE_NR MAC_NR_COMMON nr_rrc CONFIG_LIB L2_NR HASHTABLE x2ap SECU_CN ngap -lz -Wl,--end-group
   m pthread ${ATLAS_LIBRARIES} ${T_LIB} ITTI ${OPENSSL_LIBRARIES} dl

cmake_targets/build_oai

@@ -56,7 +56,14 @@ CMAKE_CMD="$CMAKE"
 MAKE_CMD=make
 BUILD_ECLIPSE=0
 NR="False"
-OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope nrqtscope ldpc_cuda ldpc_t1 websrv oai_iqplayer"
+OPTIONAL_LIBRARIES="telnetsrv enbscope uescope nrscope ldpc_cuda ldpc_offload_t1 ldpc_offload_t2"
+ulfiusdep=$(basename ./`find /usr/lib* -name libulfius.so`)
+jssondep=$(basename ./`find /usr/lib* -name libjansson.so`)
+if [ "$ulfiusdep" == "libulfius.so" -a "$jssondep" == "libjansson.so" ] ; then
+  OPTIONAL_LIBRARIES+=" websrv"
+else
+  echo_warning "  websrv optional build not included in build-lib option  as dependencies requirements not met "
+fi
 RU=0
 CMAKE_C_FLAGS=()
 CMAKE_CXX_FLAGS=()

doc/T1_T2_OFFLOAD_setup.md

@@ -0,0 +1,64 @@
+<table style="border-collapse: collapse; border: none;">
+  <tr style="border-collapse: collapse; border: none;">
+    <td style="border-collapse: collapse; border: none;">
+      <a href="http://www.openairinterface.org/">
+         <img src="./images/oai_final_logo.png" alt="" border=3 height=50 width=150>
+         </img>
+      </a>
+    </td>
+    <td style="border-collapse: collapse; border: none; vertical-align: center;">
+      <b><font size = "5">OAI T1/T2 LDPC offload</font></b>
+    </td>
+  </tr>
+</table>
+
+**Table of Contents**
+
+[[_TOC_]]
+
+This documentation aims to provide a tutorial for AMD Xilinx T1 and T2 Telco cards integration into OAI and its usage.
+## Prerequisites
+Offload of the channel decoding  was tested with T1 and T2 card in following setup:
+
+**AMD Xilinx T1 Telco card**
+
+ - bitstream image and drivers provided by VVDN
+ - DPDK 20.05 version used
+ - tested on RHEL7.9, RHEL9.1
+
+**AMD Xilinx T2 card**
+ - bitstream image and PMD driver provided by AccelerComm
+ - DPDK 20.11.3 version used
+ - tested on RHEL7.9, RHEL9.1, Ubuntu 20.04
+
+
+## DPDK setup
+
+ - check the presence of the card on the host
+	 - `lspci | grep "Xilinx"`
+ - binding of the device with igb_uio driver
+	 - `./dpdk-devbind.py -b igb_uio <address of the PCI of the card>`
+ - hugepages setup (10 x 1GB hugepages)
+	 - `./dpdk-hugepages.py -p 1G --setup 10G`
+
+*Note: Commands to run from dpdk/usertool folder*
+
+## Compilation
+Deployment of the OAI is precisely described in the following tutorials.
+ - [BUILD](https://gitlab.eurecom.fr/oai/openairinterface5g/-/blob/develop/doc/BUILD.md)
+ - [NR_SA_CN5G_gNB_USRP_COTS_UE_Tutorial](https://gitlab.eurecom.fr/oai/openairinterface5g/-/blob/develop/doc/NR_SA_Tutorial_COTS_UE.md)
+
+Shared object files *libldpc_offload_t1.so* or/and *libldpc_offload_t2.so* are created  during the compilation. These objects are conditionally compiled. Selection of the library to compile is done using *--build-lib ldpc_offload_t1 ldpc_offload_t2*. Sample command to build OAI with support for LDPC offload to T1 is:
+
+`./build_oai -P --gNB -w USRP --build-lib ldpc_offload_t1 --ninja`
+
+*Required DPDK version  has to be installed on the host, prior to the build of OAI*
+## nr_ulsim
+Offload of the channel decoding is in nr_ulsim specified by *-o* option followed by the version of the card *t1* or *t2*. Example command for running nr_ulsim with offload to the T1 card:
+
+`./nr_ulsim -n100 -s20 -m20 -r273 -o t1`
+
+## nr-softmodem
+Offload of the channel decoding in nr-softmodem is specified by *--ldpc-offload-enable 1* option, library for the LDPC decoder is selected by *--loader.ldpc_offload.shlibversion* followed by the version of the card *_t1* or *_t2*. Sample command for running nr_ulsim with offload to the T1 card:
+
+`sudo numactl --cpunodebind=1 --membind=1 ./nr-softmodem -O config_file.conf --sa --ldpc-offload-enable 1 --loader.ldpc_offload.shlibversion _t1`

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder_offload.c

@@ -55,32 +55,6 @@
 #define MAX_QUEUES RTE_MAX_LCORE
 #define TEST_REPETITIONS 1 
 
-#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
-#include <fpga_lte_fec.h>
-#define FPGA_LTE_PF_DRIVER_NAME ("intel_fpga_lte_fec_pf")
-#define FPGA_LTE_VF_DRIVER_NAME ("intel_fpga_lte_fec_vf")
-#define VF_UL_4G_QUEUE_VALUE 4
-#define VF_DL_4G_QUEUE_VALUE 4
-#define UL_4G_BANDWIDTH 3
-#define DL_4G_BANDWIDTH 3
-#define UL_4G_LOAD_BALANCE 128
-#define DL_4G_LOAD_BALANCE 128
-#define FLR_4G_TIMEOUT 610
-#endif
-
-#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
-#include <rte_pmd_fpga_5gnr_fec.h>
-#define FPGA_5GNR_PF_DRIVER_NAME ("intel_fpga_5gnr_fec_pf")
-#define FPGA_5GNR_VF_DRIVER_NAME ("intel_fpga_5gnr_fec_vf")
-#define VF_UL_5G_QUEUE_VALUE 4
-#define VF_DL_5G_QUEUE_VALUE 4
-#define UL_5G_BANDWIDTH 3
-#define DL_5G_BANDWIDTH 3
-#define UL_5G_LOAD_BALANCE 128
-#define DL_5G_LOAD_BALANCE 128
-#define FLR_5G_TIMEOUT 610
-#endif
-
 #define OPS_CACHE_SIZE 256U
 #define OPS_POOL_SIZE_MIN 511U /* 0.5K per queue */
 
@@ -362,97 +336,6 @@ add_dev(uint8_t dev_id, struct rte_bbdev_info *info)
 	unsigned int nb_queues;
 	enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC; 
 
-/* Configure fpga lte fec with PF & VF values
- * if '-i' flag is set and using fpga device
- */
-#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_LTE_FEC
-	if ((get_init_device() == true) &&
-		(!strcmp(info->drv.driver_name, FPGA_LTE_PF_DRIVER_NAME))) {
-		struct fpga_lte_fec_conf conf;
-		unsigned int i;
-
-		printf("Configure FPGA LTE FEC Driver %s with default values\n",
-				info->drv.driver_name);
-
-		/* clear default configuration before initialization */
-		memset(&conf, 0, sizeof(struct fpga_lte_fec_conf));
-
-		/* Set PF mode :
-		 * true if PF is used for data plane
-		 * false for VFs
-		 */
-		conf.pf_mode_en = true;
-
-		for (i = 0; i < FPGA_LTE_FEC_NUM_VFS; ++i) {
-			/* Number of UL queues per VF (fpga supports 8 VFs) */
-			conf.vf_ul_queues_number[i] = VF_UL_4G_QUEUE_VALUE;
-			/* Number of DL queues per VF (fpga supports 8 VFs) */
-			conf.vf_dl_queues_number[i] = VF_DL_4G_QUEUE_VALUE;
-		}
-
-		/* UL bandwidth. Needed for schedule algorithm */
-		conf.ul_bandwidth = UL_4G_BANDWIDTH;
-		/* DL bandwidth */
-		conf.dl_bandwidth = DL_4G_BANDWIDTH;
-
-		/* UL & DL load Balance Factor to 64 */
-		conf.ul_load_balance = UL_4G_LOAD_BALANCE;
-		conf.dl_load_balance = DL_4G_LOAD_BALANCE;
-
-		/**< FLR timeout value */
-		conf.flr_time_out = FLR_4G_TIMEOUT;
-
-		/* setup FPGA PF with configuration information */
-		ret = fpga_lte_fec_configure(info->dev_name, &conf);
-		TEST_ASSERT_SUCCESS(ret,
-				"Failed to configure 4G FPGA PF for bbdev %s",
-				info->dev_name);
-	}
-#endif
-#ifdef RTE_LIBRTE_PMD_BBDEV_FPGA_5GNR_FEC
-	if ((get_init_device() == true) &&
-		(!strcmp(info->drv.driver_name, FPGA_5GNR_PF_DRIVER_NAME))) {
-		struct fpga_5gnr_fec_conf conf;
-		unsigned int i;
-
-		printf("Configure FPGA 5GNR FEC Driver %s with default values\n",
-				info->drv.driver_name);
-
-		/* clear default configuration before initialization */
-		memset(&conf, 0, sizeof(struct fpga_5gnr_fec_conf));
-
-		/* Set PF mode :
-		 * true if PF is used for data plane
-		 * false for VFs
-		 */
-		conf.pf_mode_en = true;
-
-		for (i = 0; i < FPGA_5GNR_FEC_NUM_VFS; ++i) {
-			/* Number of UL queues per VF (fpga supports 8 VFs) */
-			conf.vf_ul_queues_number[i] = VF_UL_5G_QUEUE_VALUE;
-			/* Number of DL queues per VF (fpga supports 8 VFs) */
-			conf.vf_dl_queues_number[i] = VF_DL_5G_QUEUE_VALUE;
-		}
-
-		/* UL bandwidth. Needed for schedule algorithm */
-		conf.ul_bandwidth = UL_5G_BANDWIDTH;
-		/* DL bandwidth */
-		conf.dl_bandwidth = DL_5G_BANDWIDTH;
-
-		/* UL & DL load Balance Factor to 64 */
-		conf.ul_load_balance = UL_5G_LOAD_BALANCE;
-		conf.dl_load_balance = DL_5G_LOAD_BALANCE;
-
-		/**< FLR timeout value */
-		conf.flr_time_out = FLR_5G_TIMEOUT;
-
-		/* setup FPGA PF with configuration information */
-		ret = fpga_5gnr_fec_configure(info->dev_name, &conf);
-		TEST_ASSERT_SUCCESS(ret,
-				"Failed to configure 5G FPGA PF for bbdev %s",
-				info->dev_name);
-	}
-#endif
 	nb_queues = RTE_MIN(rte_lcore_count(), info->drv.max_num_queues);
 	nb_queues = RTE_MIN(nb_queues, (unsigned int) MAX_QUEUES);
 
@@ -817,7 +700,7 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 					ldpc_dec->tb_params.r;
 					printf("code block ea %d eb %d c %d cab %d r %d\n",ldpc_dec->tb_params.ea,ldpc_dec->tb_params.eb,ldpc_dec->tb_params.c, ldpc_dec->tb_params.cab, ldpc_dec->tb_params.r);
 		} else { */
-			ops[i]->ldpc_dec.cb_params.e = p_offloadParams->E; 
+		ops[i]->ldpc_dec.cb_params.e = p_offloadParams->E;
 		//}
 
 		ops[i]->ldpc_dec.basegraph = p_offloadParams->BG; 
@@ -825,15 +708,18 @@ set_ldpc_dec_op(struct rte_bbdev_dec_op **ops, unsigned int n,
 		ops[i]->ldpc_dec.q_m = p_offloadParams->Qm; 
 		ops[i]->ldpc_dec.n_filler = p_offloadParams->F; 
 		ops[i]->ldpc_dec.n_cb = p_offloadParams->n_cb;
-		ops[i]->ldpc_dec.iter_max = 20; 
+		ops[i]->ldpc_dec.iter_max = 10;
 		ops[i]->ldpc_dec.rv_index = p_offloadParams->rv; 
-		ops[i]->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE|RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE|RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE; //|RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP; 
+		ops[i]->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE |
+			                    RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE |
+					    RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE ;
+					    //RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE |
+					    //RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE |
+					    //RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP;
 		ops[i]->ldpc_dec.code_block_mode = 1; //ldpc_dec->code_block_mode;
-		//printf("set ldpc ulsch_id %d\n",ulsch_id);
 		ops[i]->ldpc_dec.harq_combined_input.offset = ulsch_id*(32*1024*1024)+harq_pid*(2*1024*1024)+r*(1024*32);
 		ops[i]->ldpc_dec.harq_combined_output.offset = ulsch_id*(32*1024*1024)+harq_pid*(2*1024*1024)+r*(1024*32);
 		
-
 		if (hard_outputs != NULL)
 			ops[i]->ldpc_dec.hard_output =
 					hard_outputs[start_idx + i];
@@ -907,10 +793,8 @@ retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 		int8_t* p_out)
 {
 	unsigned int i;
-	//int ret;
 	struct rte_bbdev_op_ldpc_dec *ops_td;
 	struct rte_bbdev_op_data *hard_output;
-	//struct rte_bbdev_op_ldpc_dec *ref_td = &ref_op->ldpc_dec;
 	struct rte_mbuf *m;  
 	char *data;
 
@@ -918,50 +802,16 @@ retrieve_ldpc_dec_op(struct rte_bbdev_dec_op **ops, const uint16_t n,
 		ops_td = &ops[i]->ldpc_dec;
 		hard_output = &ops_td->hard_output;
 		m = hard_output->data;
-	/*	ret = check_dec_status_and_ordering(ops[i], i, ref_op->status);
-		TEST_ASSERT_SUCCESS(ret,
-				"Checking status and ordering for decoder failed");
-		if (vector_mask & TEST_BBDEV_VF_EXPECTED_ITER_COUNT)
-			TEST_ASSERT(ops_td->iter_count <= ref_td->iter_count,
-					"Returned iter_count (%d) > expected iter_count (%d)",
-					ops_td->iter_count, ref_td->iter_count);
-	*/
 		uint16_t offset = hard_output->offset;
                 uint16_t data_len = rte_pktmbuf_data_len(m) - offset;
-
                 data = m->buf_addr;
                 memcpy(p_out, data+m->data_off, data_len);
 
 	}
-
 	return TEST_SUCCESS;
 }
 
 
-/*static int
-validate_ldpc_enc_op(struct rte_bbdev_enc_op **ops, const uint16_t n,
-		struct rte_bbdev_enc_op *ref_op)
-{
-	unsigned int i;
-	int ret;
-	struct op_data_entries *hard_data_orig =
-			&test_vector.entries[DATA_HARD_OUTPUT];
-
-	for (i = 0; i < n; ++i) {
-		ret = check_enc_status_and_ordering(ops[i], i, ref_op->status);
-		TEST_ASSERT_SUCCESS(ret,
-				"Checking status and ordering for encoder failed");
-		TEST_ASSERT_SUCCESS(validate_op_chain(
-				&ops[i]->ldpc_enc.output,
-				hard_data_orig),
-				"Output buffers (CB=%u) are not equal",
-				i);
-	}
-
-	return TEST_SUCCESS;
-}
-*/
-
 static void
 create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op, t_nrLDPCoffload_params *p_offloadParams)
 {
@@ -973,7 +823,10 @@ create_reference_ldpc_dec_op(struct rte_bbdev_dec_op *op, t_nrLDPCoffload_params
 		op->ldpc_dec.z_c = p_offloadParams->Z; 
 		op->ldpc_dec.n_filler = p_offloadParams->F; 
 		op->ldpc_dec.code_block_mode = 1;
-		op->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE|RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE|RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE|RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP;
+		op->ldpc_dec.op_flags = RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE|
+			                RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE|
+					RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE;
+					//RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP;
 }
 
 
@@ -1054,7 +907,6 @@ pmd_lcore_ldpc_dec(void *arg)
 	uint16_t num_to_enq;
  	int8_t *p_out = tp->p_out;	
 	t_nrLDPCoffload_params *p_offloadParams = tp->p_offloadParams;
-        
 	//struct rte_bbdev_op_data *hard_output;	
        
         //bool extDdr = check_bit(ldpc_cap_flags,
@@ -1082,8 +934,7 @@ pmd_lcore_ldpc_dec(void *arg)
 	ret = rte_bbdev_dec_op_alloc_bulk(tp->op_params->mp, ops_enq, num_ops);
 	TEST_ASSERT_SUCCESS(ret, "Allocation failed for %d ops", num_ops);
 
-	/* For throughput tests we need to disable early termination */
-	if (check_bit(ref_op->ldpc_dec.op_flags,
+	if (!check_bit(ref_op->ldpc_dec.op_flags,
 			RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE))
 		ref_op->ldpc_dec.op_flags -=
 				RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE;
@@ -1093,11 +944,9 @@ pmd_lcore_ldpc_dec(void *arg)
 	set_ldpc_dec_op(ops_enq, num_ops, 0, bufs->inputs,
 				bufs->hard_outputs, bufs->soft_outputs,
 			bufs->harq_inputs, bufs->harq_outputs, ref_op, r, harq_pid, ulsch_id, p_offloadParams);
-
 	/* Set counter to validate the ordering */
 	for (j = 0; j < num_ops; ++j)
 		ops_enq[j]->opaque_data = (void *)(uintptr_t)j;
-
 	for (i = 0; i < TEST_REPETITIONS; ++i) {
 		for (j = 0; j < num_ops; ++j) {
 			if (!loopback)
@@ -1107,8 +956,6 @@ pmd_lcore_ldpc_dec(void *arg)
 				mbuf_reset(
 				ops_enq[j]->ldpc_dec.harq_combined_output.data);
 		}
-		//	start_time = rte_rdtsc_precise();
-
 		for (enq = 0, deq = 0; enq < num_ops;) {
 			num_to_enq = burst_sz;
 
@@ -1120,9 +967,6 @@ pmd_lcore_ldpc_dec(void *arg)
 
 			deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
 					queue_id, &ops_deq[deq], enq - deq);
-
-			//printf("enq %d, deq %d\n",enq,deq);
-
                 }
 
 		/* dequeue the remaining */
@@ -1130,17 +974,15 @@ pmd_lcore_ldpc_dec(void *arg)
 		while (deq < enq) {
 			deq += rte_bbdev_dequeue_ldpc_dec_ops(tp->dev_id,
 					queue_id, &ops_deq[deq], enq - deq);
-			/*usleep(10);
-			trials++;
-			if (trials>=100) {
-			  printf("aborting decoding after 100 dequeue tries\n");
-			  break;
-			  }*/
+			//usleep(10);
+			//trials++;
+			//if (trials>=100) {
+			//  printf("aborting decoding after 100 dequeue tries\n");
+			//  break;
+			//  }
 		}
 
-		//total_time += rte_rdtsc_precise() - start_time;
 	}
-	//total_time = rte_rdtsc_precise() - start_time;
 	if (deq==enq) {
 	tp->iter_count = 0;
 	/* get the max of iter_count for all dequeued ops */
@@ -1148,7 +990,7 @@ pmd_lcore_ldpc_dec(void *arg)
 		tp->iter_count = RTE_MAX(ops_enq[i]->ldpc_dec.iter_count,
 				tp->iter_count);
 	}
-
+	//printf("ITER %d \n", tp->iter_count);
 	ret = retrieve_ldpc_dec_op(ops_deq, num_ops, ref_op,
 				tp->op_params->vector_mask, p_out);
 		TEST_ASSERT_SUCCESS(ret, "Validation failed!");
@@ -1173,6 +1015,10 @@ pmd_lcore_ldpc_dec(void *arg)
 			1000000.0) / ((double)total_time /
 			(double)rte_get_tsc_hz());
 	*/	
+	uint64_t start=rte_rdtsc_precise();
+	uint64_t end=rte_rdtsc_precise();
+        uint64_t total = end - start;
+        //printf("ldpc decode %lu\n", (total/2800));
 	return ret;
 }
 
@@ -1222,22 +1068,9 @@ start_pmd_dec(struct active_device *ad,
 	struct thread_params *tp;
 	//struct rte_bbdev_info info;
 	uint16_t num_lcores;
-	//uint64_t start_time, start_time1 ; //= rte_rdtsc_precise();
-	//uint64_t total_time=0, total_time1=0;
-	//rte_bbdev_info_get(ad->dev_id, &info);
-	//start_time = rte_rdtsc_precise();
-	/*printf("+ ------------------------------------------------------- +\n");
-	printf("== start pmd dec\ndev: %s, nb_queues: %u, burst size: %u, num ops: %u, num_lcores: %u,  itr mode: %s, GHz: %lg\n",
-			info.dev_name, ad->nb_queues, op_params->burst_sz,
-			op_params->num_to_process, op_params->num_lcores,
-			intr_enabled ? "Interrupt mode" : "PMD mode",
-			(double)rte_get_tsc_hz() / 1000000000.0);
-	*/	
-	/* Set number of lcores */
 	num_lcores = (ad->nb_queues < (op_params->num_lcores))
 			? ad->nb_queues
 			: op_params->num_lcores;
-
 	/* Allocate memory for thread parameters structure */
 	/*t_params = rte_zmalloc(NULL, num_lcores * sizeof(struct thread_params),
 			RTE_CACHE_LINE_SIZE);
@@ -1259,8 +1092,8 @@ start_pmd_dec(struct active_device *ad,
 	t_params[0].r = r;
 	t_params[0].harq_pid = harq_pid;
 	t_params[0].ulsch_id = ulsch_id;
-
-	RTE_LCORE_FOREACH_SLAVE(lcore_id) {
+        uint64_t start = rte_rdtsc_precise();
+	RTE_LCORE_FOREACH_WORKER(lcore_id) {
 		if (used_cores >= num_lcores)
 			break;
 		t_params[used_cores].dev_id = ad->dev_id;
@@ -1277,22 +1110,15 @@ start_pmd_dec(struct active_device *ad,
 		rte_eal_remote_launch(pmd_lcore_ldpc_dec,
 				&t_params[used_cores++], lcore_id);
 	}
-	
-	rte_atomic16_set(&op_params->sync, SYNC_START);
-	
-	//total_time = rte_rdtsc_precise() - start_time;
 
-	//if (total_time > 100*3000)
-	//LOG_E(PHY," start pmd 1st: %u\n",(uint) (total_time/3000));
+	rte_atomic16_set(&op_params->sync, SYNC_START);
 
 	ret = pmd_lcore_ldpc_dec(&t_params[0]);
-	//start_time1 = rte_rdtsc_precise();
 
 	/* Master core is always used */
 	//for (used_cores = 1; used_cores < num_lcores; used_cores++)
 	//	ret |= rte_eal_wait_lcore(t_params[used_cores].lcore_id);
 
-	/* Return if test failed */
 	if (ret) {
 		rte_free(t_params);
 		return ret;
@@ -1300,7 +1126,7 @@ start_pmd_dec(struct active_device *ad,
 
 	/* Print throughput if interrupts are disabled and test passed */
 	if (!intr_enabled) {
-	  ///print_dec_throughput(t_params, num_lcores);
+	  //print_dec_throughput(t_params, num_lcores);
 	  //rte_free(t_params);
 	  //	total_time1 = rte_rdtsc_precise() - start_time1;
 
@@ -1343,9 +1169,7 @@ start_pmd_dec(struct active_device *ad,
 	if (!ret) {
 			print_dec_throughput(t_params, num_lcores);
 	}
-
 	rte_free(t_params);
-
 	return ret;
 }
 
@@ -1407,250 +1231,208 @@ struct rte_mbuf *m_head[DATA_NUM_TYPES];
 struct thread_params *t_params;
 
 
-int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid, uint8_t ulsch_id, uint8_t C, uint8_t rv, uint16_t F, 
-			uint32_t E, uint8_t Qm, int8_t* p_llr, int8_t* p_out, uint8_t mode) 
+int32_t nrLDPC_decod_offload(t_nrLDPC_dec_params* p_decParams, uint8_t harq_pid,
+			     uint8_t ulsch_id, uint8_t C, uint8_t rv, uint16_t F,
+			     uint32_t E, uint8_t Qm, int8_t* p_llr, int8_t* p_out,
+			     uint8_t mode)
 {
-    	t_nrLDPCoffload_params offloadParams;
-    	t_nrLDPCoffload_params* p_offloadParams    = &offloadParams;
-    	uint64_t start=rte_rdtsc_precise();
-	/*uint64_t start_time;//=rte_rdtsc_precise();
-	uint64_t start_time1; //=rte_rdtsc_precise();
-	uint64_t total_time=0, total_time1=0;*/
-	uint32_t numIter = 0;
-	int ret;
-	/*uint64_t start_time_init;
-	  uint64_t total_time_init=0;*/
+  t_nrLDPCoffload_params offloadParams;
+  t_nrLDPCoffload_params* p_offloadParams    = &offloadParams;
+  uint32_t numIter = 0;
+  int ret;
+  int argc_re;
+  char *argv_re[10];
+  if (p_decParams->lib_version == 2) {
+    argc_re=6;
+    argv_re[0] = "/opt/accelercomm/ACL_BBDEV_latest/dpdk/build/app/testbbdev";
+    argv_re[1] = "-l";
+    argv_re[2] = "28";
+    argv_re[3] = "-a";
+    argv_re[4] = "21:00.0";
+    argv_re[5] = "--";
+  } else if (p_decParams->lib_version == 1 ) {
+    argc_re=7;
+    argv_re[0] = "/home/eurecom/hongzhi/dpdk-20.05orig/build/app/testbbdev";
+    argv_re[1] = "-l";
+    argv_re[2] = "31";
+    argv_re[3] = "-w";
+    argv_re[4] = "b0:00.0";
+    argv_re[5] = "--file-prefix=b6";
+    argv_re[6] = "--";
+  } else {
+    argc_re=1;
+    argv_re[0] = "/opt/accelercomm/ACL_BBDEV_latest/dpdk/build/app/testbbdev";
+  }
 
-	/*
-	int argc_re=2;
-	char *argv_re[2];
-	argv_re[0] = "/home/eurecom/hongzhi/dpdk-20.05orig/build/app/testbbdev";
-	argv_re[1] = "--";
-	*/
+  test_params.num_ops=1;
+  test_params.burst_sz=1;
+  test_params.num_lcores=2;
+  test_params.num_tests = 1;
+  struct active_device *ad;
+  ad = &active_devs[0];
+
+  int socket_id=0;
+  int i,f_ret;
+  struct rte_bbdev_info info;
+  enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
 	
-	int argc_re=7;
-        char *argv_re[7];
-        argv_re[0] = "/home/eurecom/hongzhi/dpdk-20.05orig/build/app/testbbdev";
-        argv_re[1] = "-l";
-        argv_re[2] = "31";
-        argv_re[3] = "-w";
-        argv_re[4] = "b0:00.0";
-        argv_re[5] = "--file-prefix=b6";
-        argv_re[6] = "--";
-	
-	test_params.num_ops=1; 
-	test_params.burst_sz=1;
-	test_params.num_lcores=1;		
-	test_params.num_tests = 1;
-	struct active_device *ad;
-        ad = &active_devs[0];
+  switch (mode) {
+    case 0:
+      ret = rte_eal_init(argc_re, argv_re);
+      if (ret<0) {
+        printf("Could not init EAL, ret %d\n",ret);
+        return(-1);
+      }
+      ret = device_setup();
+      if (ret != TEST_SUCCESS) {
+        printf("Couldn't setup device");
+        return(-1);
+      }
+      ret=ut_setup();
+      if (ret != TEST_SUCCESS) {
+        printf("Couldn't setup ut");
+        return(-1);
+      }
 
-	int socket_id=0;
-        int i,f_ret;
-        struct rte_bbdev_info info;
-        enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
-	
-	switch (mode) {
-	case 0:	
-          ret = rte_eal_init(argc_re, argv_re);
-          if (ret<0) {
-            printf("Could not init EAL, ret %d\n",ret);
-            return(-1);
-          }
-          ret = device_setup();
-          if (ret != TEST_SUCCESS) {
-            printf("Couldn't setup device");
-            return(-1);
-          }
-          ret=ut_setup();
-          if (ret != TEST_SUCCESS) {
-            printf("Couldn't setup ut");
-            return(-1);
-          }
+      p_offloadParams->E = E;
+      p_offloadParams->n_cb = (p_decParams->BG==1)?(66*p_decParams->Z):(50*p_decParams->Z);
+      p_offloadParams->BG = p_decParams->BG;
+      p_offloadParams->Z = p_decParams->Z;
+      p_offloadParams->rv = rv;
+      p_offloadParams->F = F;
+      p_offloadParams->Qm = Qm;
 
-	  p_offloadParams->E = E;
-          p_offloadParams->n_cb = (p_decParams->BG==1)?(66*p_decParams->Z):(50*p_decParams->Z);
-          p_offloadParams->BG = p_decParams->BG;
-          p_offloadParams->Z = p_decParams->Z;
-          p_offloadParams->rv = rv;
-          p_offloadParams->F = F;
-          p_offloadParams->Qm = Qm;
-
-          op_params = rte_zmalloc(NULL,
-                        sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
-          TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
-                               RTE_ALIGN(sizeof(struct test_op_params),
-                                         RTE_CACHE_LINE_SIZE));
+      op_params = rte_zmalloc(NULL, sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE);
+      TEST_ASSERT_NOT_NULL(op_params, "Failed to alloc %zuB for op_params",
+                           RTE_ALIGN(sizeof(struct test_op_params), RTE_CACHE_LINE_SIZE));
 	  
-	  rte_bbdev_info_get(ad->dev_id, &info);
-	  socket_id = GET_SOCKET(info.socket_id);
-	  f_ret = create_mempools(ad, socket_id, op_type,
-				  get_num_ops(),p_offloadParams);
-	  if (f_ret != TEST_SUCCESS) {
-	    printf("Couldn't create mempools");
-	  }
-	  f_ret = init_test_op_params(op_params, op_type,
+      rte_bbdev_info_get(ad->dev_id, &info);
+      socket_id = GET_SOCKET(info.socket_id);
+      f_ret = create_mempools(ad, socket_id, op_type, get_num_ops(),p_offloadParams);
+      if (f_ret != TEST_SUCCESS) {
+        printf("Couldn't create mempools");
+      }
+      f_ret = init_test_op_params(op_params, op_type,
 				      0,
 				      0,
 				      ad->ops_mempool,
 				      1,
 				      get_num_ops(),
 				      get_num_lcores());
-	  if (f_ret != TEST_SUCCESS) {
-	    printf("Couldn't init test op params");
-	  }
+      if (f_ret != TEST_SUCCESS) {
+        printf("Couldn't init test op params");
+      }
 	
-	  //const struct rte_bbdev_op_cap *capabilities = NULL;
-	  rte_bbdev_info_get(ad->dev_id, &info);
-	  socket_id = GET_SOCKET(info.socket_id);
-	  //enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;                                                                                                
-	  /*const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
+    //const struct rte_bbdev_op_cap *capabilities = NULL;
+      rte_bbdev_info_get(ad->dev_id, &info);
+      socket_id = GET_SOCKET(info.socket_id);
+    //enum rte_bbdev_op_type op_type = RTE_BBDEV_OP_LDPC_DEC;
+    /*const struct rte_bbdev_op_cap *cap = info.drv.capabilities;
+      for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
+        if (cap->type == op_type) {
+          capabilities = cap;
+          break;
+        }
+        cap++;
+      }*/
+      ad->nb_queues =  1;
+      enum op_data_type type;
 
-	  for (i = 0; i < RTE_BBDEV_OP_TYPE_COUNT; i++) {
-	    if (cap->type == op_type) {
-	      capabilities = cap;
-	      break;
-	    }
-	    cap++;
-	    }*/
-	  ad->nb_queues =  1;
-	  enum op_data_type type;
+      for (i = 0; i < ad->nb_queues; ++i) {
+        const uint16_t n = op_params->num_to_process;
+        struct rte_mempool *in_mp = ad->in_mbuf_pool;
+        struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool;
+        struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool;
+        struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool;
+        struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool;
+        struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {in_mp,
+	                                                  soft_out_mp,
+	                                                  hard_out_mp,
+	                                                  harq_in_mp,
+	                                                  harq_out_mp};
+        uint8_t queue_id =ad->queue_ids[i];
+        struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&op_params->q_bufs[socket_id][queue_id].inputs,
+	                                                        &op_params->q_bufs[socket_id][queue_id].soft_outputs,
+	                                                        &op_params->q_bufs[socket_id][queue_id].hard_outputs,
+	                                                        &op_params->q_bufs[socket_id][queue_id].harq_inputs,
+	                                                        &op_params->q_bufs[socket_id][queue_id].harq_outputs};
+	for (type = DATA_INPUT; type < 3; type+=2) {
+          ret = allocate_buffers_on_socket(queue_ops[type], n * sizeof(struct rte_bbdev_op_data), socket_id);
+	  TEST_ASSERT_SUCCESS(ret, "Couldn't allocate memory for rte_bbdev_op_data structs");
+	  m_head[type] = rte_pktmbuf_alloc(mbuf_pools[type]);
+	  TEST_ASSERT_NOT_NULL(m_head[type],
+			       "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
+			       op_type, 1,
+			       mbuf_pools[type]->size);
+        }
 
-	  for (i = 0; i < ad->nb_queues; ++i) {
+	/* Allocate memory for thread parameters structure */
+	t_params = rte_zmalloc(NULL,  sizeof(struct thread_params), RTE_CACHE_LINE_SIZE);
+	TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params", RTE_ALIGN(sizeof(struct thread_params), RTE_CACHE_LINE_SIZE));
+      }
+    break;
+    case 1:
+      p_offloadParams->E = E;
+      p_offloadParams->n_cb = (p_decParams->BG==1)?(66*p_decParams->Z):(50*p_decParams->Z);
+      p_offloadParams->BG = p_decParams->BG;
+      p_offloadParams->Z = p_decParams->Z;
+      p_offloadParams->rv = rv;
+      p_offloadParams->F = F;
+      p_offloadParams->Qm = Qm;
 
-	    const uint16_t n = op_params->num_to_process;
+      rte_bbdev_info_get(ad->dev_id, &info);
+      socket_id = GET_SOCKET(info.socket_id);
+      create_reference_ldpc_dec_op(op_params->ref_dec_op, p_offloadParams);
 
-	    struct rte_mempool *in_mp = ad->in_mbuf_pool;
-	    struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool;
-	    struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool;
-	    struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool;
-	    struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool;
+      struct rte_mempool *in_mp = ad->in_mbuf_pool;
+      struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool;
+      struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool;
+      struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool;
+      struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool;
 
-	    struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
-	      in_mp,
-	      soft_out_mp,
-	      hard_out_mp,
-	      harq_in_mp,
-	      harq_out_mp,
-	    };
+      struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {in_mp,
+                                                        soft_out_mp,
+                                                        hard_out_mp,
+                                                        harq_in_mp,
+                                                        harq_out_mp};
 
-	    uint8_t queue_id =ad->queue_ids[i];
-	    struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
-	      &op_params->q_bufs[socket_id][queue_id].inputs,
-	      &op_params->q_bufs[socket_id][queue_id].soft_outputs,
-	      &op_params->q_bufs[socket_id][queue_id].hard_outputs,
-	      &op_params->q_bufs[socket_id][queue_id].harq_inputs,
-	      &op_params->q_bufs[socket_id][queue_id].harq_outputs,
-	    };
+      uint8_t queue_id =ad->queue_ids[0];
+      struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {&op_params->q_bufs[socket_id][queue_id].inputs,
+                                                              &op_params->q_bufs[socket_id][queue_id].soft_outputs,
+                                                              &op_params->q_bufs[socket_id][queue_id].hard_outputs,
+                                                              &op_params->q_bufs[socket_id][queue_id].harq_inputs,
+                                                              &op_params->q_bufs[socket_id][queue_id].harq_outputs};
+      //start = rte_rdtsc_precise();
+      for (type = DATA_INPUT; type < 3; type+=2) {
+	ret = init_op_data_objs(*queue_ops[type], p_llr, p_offloadParams->E, m_head[type], mbuf_pools[type], 1, type, info.drv.min_alignment);
+	TEST_ASSERT_SUCCESS(ret, "Couldn't init rte_bbdev_op_data structs");
+      }
+      ret = start_pmd_dec(ad, op_params, t_params, p_offloadParams, C, harq_pid, ulsch_id, p_out);
+      //uint64_t end=rte_rdtsc_precise();
+      //uint64_t total = end - start;
+      //printf("ldpc decode %lu\n", (total/2800));
 
-	    for (type = DATA_INPUT; type < 3; type+=2) {
-
-	      ret = allocate_buffers_on_socket(queue_ops[type],
-					     n * sizeof(struct rte_bbdev_op_data),
-					     socket_id);
-	      TEST_ASSERT_SUCCESS(ret,
-				"Couldn't allocate memory for rte_bbdev_op_data structs");
-
-	      m_head[type] = rte_pktmbuf_alloc(mbuf_pools[type]);	  
-	
-	      TEST_ASSERT_NOT_NULL(m_head[type],
-			     "Not enough mbufs in %d data type mbuf pool (needed %u, available %u)",
-			     op_type, 1,
-			     mbuf_pools[type]->size);
-	
-	    }
-
-	    /* Allocate memory for thread parameters structure */
-	    t_params = rte_zmalloc(NULL,  sizeof(struct thread_params),
-				   RTE_CACHE_LINE_SIZE);
-	    TEST_ASSERT_NOT_NULL(t_params, "Failed to alloc %zuB for t_params",
-				 RTE_ALIGN(sizeof(struct thread_params),
-					   RTE_CACHE_LINE_SIZE));
-	  }
-
-	break;
-	case 1:
-	  //printf("offload param E %d BG %d F %d Z %d Qm %d rv %d\n", E,p_decParams->BG, F,p_decParams->Z, Qm,rv);
-	  //uint64_t start_time_init;
-	  //uint64_t total_time_init=0;
-
-	  //start_time_init = rte_rdtsc_precise();
-	  p_offloadParams->E = E;
-          p_offloadParams->n_cb = (p_decParams->BG==1)?(66*p_decParams->Z):(50*p_decParams->Z);
-          p_offloadParams->BG = p_decParams->BG;
-          p_offloadParams->Z = p_decParams->Z;
-          p_offloadParams->rv = rv;
-          p_offloadParams->F = F;
-          p_offloadParams->Qm = Qm;
-
-	  rte_bbdev_info_get(ad->dev_id, &info);
-	  socket_id = GET_SOCKET(info.socket_id);
-
-	  create_reference_ldpc_dec_op(op_params->ref_dec_op, p_offloadParams);
-
-	  struct rte_mempool *in_mp = ad->in_mbuf_pool;
-          struct rte_mempool *hard_out_mp = ad->hard_out_mbuf_pool;
-          struct rte_mempool *soft_out_mp = ad->soft_out_mbuf_pool;
-          struct rte_mempool *harq_in_mp = ad->harq_in_mbuf_pool;
-          struct rte_mempool *harq_out_mp = ad->harq_out_mbuf_pool;
-
-          struct rte_mempool *mbuf_pools[DATA_NUM_TYPES] = {
-            in_mp,
-            soft_out_mp,
-            hard_out_mp,
-            harq_in_mp,
-            harq_out_mp,
-          };
-
-          uint8_t queue_id =ad->queue_ids[0];
-          struct rte_bbdev_op_data **queue_ops[DATA_NUM_TYPES] = {
-            &op_params->q_bufs[socket_id][queue_id].inputs,
-            &op_params->q_bufs[socket_id][queue_id].soft_outputs,
-            &op_params->q_bufs[socket_id][queue_id].hard_outputs,
-            &op_params->q_bufs[socket_id][queue_id].harq_inputs,
-            &op_params->q_bufs[socket_id][queue_id].harq_outputs,
-          };
-	  //start_time1 = rte_rdtsc_precise();
-	  for (type = DATA_INPUT; type < 3; type+=2) {
-
-	    ret = init_op_data_objs(*queue_ops[type], p_llr, p_offloadParams->E,
-				    m_head[type], mbuf_pools[type], 1, type, info.drv.min_alignment);
-	    TEST_ASSERT_SUCCESS(ret,
-				"Couldn't init rte_bbdev_op_data structs");
-
-	  }
-	  /*total_time_init = rte_rdtsc_precise() - start_time_init;
-
-	  if (total_time_init > 100*3000)
-            LOG_E(PHY," ldpc decoder mode 1 first: %u\n",(uint) (total_time_init/3000));
-	  
-	    start_time1 = rte_rdtsc_precise();*/
-	    ret = start_pmd_dec(ad, op_params, t_params, p_offloadParams, C, harq_pid, ulsch_id, p_out);
-	  if (ret<0) {
-	    printf("Couldn't start pmd dec");
-	    return(-1);
-	  }
-	  /*total_time1 = rte_rdtsc_precise() - start_time1;
-	  if (total_time1 > 100*3000)
-	  LOG_E(PHY," ldpc decoder mode 1 second: %u\n",(uint) (total_time1/3000));*/
-	break;
-	case 2:
-
-	  free_buffers(ad, op_params);
-	  rte_free(op_params);
-	  rte_free(t_params);
-	  ut_teardown();
-	  testsuite_teardown();
-	break;
-	default:
-	  printf("Unknown mode: %d\n", mode);
-	  return(-1);
-	}	
-	/*uint64_t end=rte_rdtsc_precise();
-	
-        if (end - start > 200*3000)
-	  LOG_E(PHY," ldpc decode: %u\n",(uint) ((end - start)/3000));
-	*/
-    return numIter;
+      if (ret<0) {
+        printf("Couldn't start pmd dec");
+        return(-1);
+      }
+    break;
+    case 2:
+      free_buffers(ad, op_params);
+      rte_free(op_params);
+      rte_free(t_params);
+      ut_teardown();
+      testsuite_teardown();
+    break;
+    default:
+      printf("Unknown mode: %d\n", mode);
+      return(-1);
+    }
+    //uint64_t end=rte_rdtsc_precise();
+    //uint64_t total = end - start;
+    //printf("ldpc decode %lu\n", (total/3000));
+    //if (end - start > 200*3000)
+    //  LOG_E(PHY," ldpc decode: %u\n",(uint) ((end - start)/3000));
+  return numIter;
 }
 

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h

@@ -88,6 +88,7 @@ typedef struct nrLDPC_dec_params {
     uint8_t numMaxIter; /**< Maximum number of iterations */
     int block_length;
     e_nrLDPC_outMode outMode; /**< Output format */
+    int lib_version; /** Indication of T1 or T2 loaded lib, 1 -> libldpc_offload_T1.so, T1 card used, analogically for T2 card **/
 } t_nrLDPC_dec_params;
 
 /**

openair1/PHY/CODING/nrLDPC_extern.h

@@ -29,7 +29,7 @@ nrLDPC_dectopfunc_t top_testsuite;
 #else
 /* functions to load the LDPC shared lib, implemented in openair1/PHY/CODING/nrLDPC_load.c */
 extern int load_nrLDPClib(char *version) ;
-extern int load_nrLDPClib_offload(void) ;
+extern int load_nrLDPClib_offload(char *version) ;
 extern int free_nrLDPClib_offload(void) ;
 extern int load_nrLDPClib_ref(char *libversion, nrLDPC_encoderfunc_t * nrLDPC_encoder_ptr); // for ldpctest
 /* ldpc coder/decoder functions, as loaded by load_nrLDPClib(). */

openair1/PHY/CODING/nrLDPC_load.c

@@ -43,7 +43,8 @@
 
 /* function description array, to be used when loading the encoding/decoding shared lib */
 static loader_shlibfunc_t shlib_fdesc[3];
-
+/* function description array, to be used when loading LDPC decoding offload shared lib for T1/T2 card */
+static loader_shlibfunc_t shlib_decoffload_fdesc[1];
 /* arguments used when called from phy simulators exec's which do not use the config module */
 /* arg is used to initialize the config module so that the loader works as expected */
 char *arg[64]={"ldpctest",NULL};
@@ -71,33 +72,58 @@ int load_nrLDPClib(char *version) {
 return 0;
 }
 
-int load_nrLDPClib_offload(void) {
-     loader_shlibfunc_t shlib_decoffload_fdesc; 
-     
-     shlib_decoffload_fdesc.fname = "nrLDPC_decod_offload";
-     int ret=load_module_shlib("ldpc_t1",&shlib_decoffload_fdesc,1,NULL);
-     AssertFatal( (ret >= 0),"Error loading ldpc decoder offload");
-     nrLDPC_decoder_offload = (nrLDPC_decoffloadfunc_t)shlib_decoffload_fdesc.fptr;
+int load_nrLDPClib_offload(char *version) {
+  char *ptr = (char*)config_get_if();
+  char libname[64]="ldpc_offload";
+  char *shlibversion=NULL;
 
+  if (ptr == NULL) {
+    load_configmodule(1,arg,CONFIG_ENABLECMDLINEONLY);
+    logInit();
+  }
+
+  shlib_decoffload_fdesc[0].fname = "nrLDPC_decod_offload";
+  int ret;
+  if (version) {
+    ret = load_module_version_shlib(libname,version,shlib_decoffload_fdesc,sizeof(shlib_decoffload_fdesc)/sizeof(loader_shlibfunc_t),NULL);
+  } else if (version == NULL) {
+    ret=load_module_shlib(libname,shlib_decoffload_fdesc,sizeof(shlib_decoffload_fdesc)/sizeof(loader_shlibfunc_t),NULL);
+    paramdef_t LoaderParams[] ={{"shlibversion", NULL, 0, strptr:&shlibversion, defstrval:"", TYPE_STRING, 0, NULL}};
+    ret = config_get(LoaderParams,sizeof(LoaderParams)/sizeof(paramdef_t), "loader.ldpc_offload");
+    version = shlibversion;
+  }
+
+  AssertFatal( (ret >= 0),"Error loading ldpc offload decoder");
+  nrLDPC_decoder_offload = (nrLDPC_decoffloadfunc_t)shlib_decoffload_fdesc[0].fptr;
   t_nrLDPC_dec_params decParams;
   t_nrLDPC_dec_params* p_decParams    = &decParams;
-  int8_t   l[68*384];
+
+  int8_t   l[68*384+16];
   int8_t llrProcBuf[22*384];
 
   p_decParams->Z = 384;
   p_decParams->BG = 1;
 
+  //paramdef_t LoaderParams[] ={{"shlibversion", NULL, 0, strptr:&shlibversion, defstrval:"", TYPE_STRING, 0, NULL}};
+  //ret = config_get(LoaderParams,sizeof(LoaderParams)/sizeof(paramdef_t), "loader.ldpc_offload");
+  if (!strcmp(version,"_t1")) {
+    p_decParams->lib_version = 1;
+  } else if (!strcmp(version,"_t2")) {
+    p_decParams->lib_version = 2;
+  } else {
+    ret = 0;
+    AssertFatal( (ret = 0),"Incorrect library version");
+  }
+
   AssertFatal(nrLDPC_decoder_offload(p_decParams,0, 0,
-				     1,
-				     0,
-				     0,
-				     25344,
-				     8,
-				     l, 
-				     llrProcBuf, 0)>=0,
-	      "error loading LDPC decoder offload library\n");
-
-
+                                     1,
+                                     0,
+                                     0,
+                                     25344,
+                                     8,
+                                     l,
+                                     llrProcBuf, 0)>=0,
+              "error loading LDPC decoder offload library\n");
   return 0;
 }
 

openair1/PHY/INIT/nr_init.c

@@ -538,11 +538,19 @@ int phy_init_nr_gNB(PHY_VARS_gNB *gNB)
 
   load_nrLDPClib(NULL);
 
-  if (gNB->ldpc_offload_flag)
-    load_nrLDPClib_offload();
-
   gNB->max_nb_pdsch = MAX_MOBILES_PER_GNB;
-
+  if (gNB->ldpc_offload_flag){
+    //char *shlibversion=NULL;
+    //int ret = 0;
+    //paramdef_t LoaderParams[] ={{"shlibversion", NULL, 0, strptr:&shlibversion, defstrval:"", TYPE_STRING, 0, NULL}};
+    //ret = config_get(LoaderParams,sizeof(LoaderParams)/sizeof(paramdef_t), "loader.ldpc_offload");
+    if (!strcmp(gNB->ldpc_offload_version,"")) {
+      load_nrLDPClib_offload(NULL);
+    } else {
+      printf("VERSION: %s \n",gNB->ldpc_offload_version);
+      load_nrLDPClib_offload(gNB->ldpc_offload_version);
+    }
+  }
   init_codebook_gNB(gNB);
   init_ul_delay_table(fp);
 

openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c

@@ -286,7 +286,6 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
     LOG_E(PHY,"ulsch_decoding.c: NULL harq_process pointer\n");
     return 1;
   }
-  uint8_t dtx_det = 0;
 
   int Kr;
   int Kr_bytes;
@@ -309,8 +308,6 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING,1);
   harq_process->TBS = pusch_pdu->pusch_data.tb_size;
 
-  dtx_det = 0;
-
   uint32_t A = (harq_process->TBS) << 3;
 
   // target_code_rate is in 0.1 units
@@ -402,131 +399,66 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
 
   if (phy_vars_gNB->ldpc_offload_flag) {
     int8_t llrProcBuf[22 * 384];
-    //  if (dtx_det==0) {
-    int16_t z_ol[68 * 384];
-    int8_t l_ol[68 * 384];
+    int16_t z_ol[68 * 384 + 16] __attribute__((aligned(16)));
+    int8_t l_ol[68 * 384 + 16] __attribute__((aligned(16)));
 
-    int16_t z[68 * 384 + 16] __attribute__((aligned(16)));
-    int8_t l[68 * 384 + 16] __attribute__((aligned(16)));
-
-    __m128i *pv = (__m128i *)&z;
-    __m128i *pl = (__m128i *)&l;
     int crc_type;
     int length_dec;
 
     if (harq_process->C == 1) {
-      if (A > 3824)
-        crc_type = CRC24_A;
-      else
-        crc_type = CRC16;
-
+      crc_type = (A > 3824) ? CRC24_A : CRC16;
       length_dec = harq_process->B;
     } else {
       crc_type = CRC24_B;
       length_dec = (harq_process->B + 24 * harq_process->C) / harq_process->C;
     }
+
     int no_iteration_ldpc = 2;
+
     for (int r = 0; r < harq_process->C; r++) {
       int E = nr_get_E(G, harq_process->C, Qm, n_layers, r);
       memset(harq_process->c[r], 0, Kr_bytes);
 
       decParams.R = nr_get_R_ldpc_decoder(pusch_pdu->pusch_data.rv_index, E, decParams.BG, decParams.Z, &harq_process->llrLen, harq_process->round);
-
-      if ((dtx_det == 0) && (pusch_pdu->pusch_data.rv_index == 0)) {
-        // if (dtx_det==0){
-        if (mcs > 9) {
-          memcpy((&z_ol[0]), ulsch_llr + r_offset, E * sizeof(short));
-          __m128i *pv_ol128 = (__m128i *)&z_ol;
-          __m128i *pl_ol128 = (__m128i *)&l_ol;
-          for (int i = 0, j = 0; j < ((kc * harq_process->Z) >> 4) + 1; i += 2, j++) {
-            pl_ol128[j] = _mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]);
-          }
-
-          int ret = nrLDPC_decoder_offload(&decParams, harq_pid, ULSCH_id, r, pusch_pdu->pusch_data.rv_index, harq_process->F, E, Qm, (int8_t *)&pl_ol128[0], llrProcBuf, 1);
-          if (ret < 0) {
-            LOG_E(PHY, "ulsch_decoding.c: Problem in LDPC decoder offload\n");
-
-            no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
-            return 1;
-          }
-        } else {
-          int K_bits_F = Kr - harq_process->F;
-
-          t_nrLDPC_time_stats procTime = {0};
-          t_nrLDPC_time_stats *p_procTime = &procTime;
-          /// code blocks after bit selection in rate matching for LDPC code (38.212 V15.4.0 section 5.4.2.1)
-          int16_t harq_e[E];
-
-          nr_deinterleaving_ldpc(E, Qm, harq_e, ulsch_llr + r_offset);
-
-          if (nr_rate_matching_ldpc_rx(pusch_pdu->maintenance_parms_v3.tbSizeLbrmBytes,
-                                       decParams.BG,
-                                       decParams.Z,
-                                       harq_process->d[r],
-                                       harq_e,
-                                       harq_process->C,
-                                       pusch_pdu->pusch_data.rv_index,
-                                       harq_process->d_to_be_cleared[r],
-                                       E,
-                                       harq_process->F,
-                                       Kr - harq_process->F - 2 * (decParams.Z))
-              == -1) {
-            LOG_E(PHY, "ulsch_decoding.c: Problem in rate_matching\n");
-            no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
-            return 1;
-          }
-
-          harq_process->d_to_be_cleared[r] = false;
-
-          // set first 2*Z_c bits to zeros
-          memset(&z[0], 0, 2 * harq_process->Z * sizeof(int16_t));
-          // set Filler bits
-          memset((&z[0] + K_bits_F), 127, harq_process->F * sizeof(int16_t));
-          // Move coded bits before filler bits
-          memcpy((&z[0] + 2 * harq_process->Z), harq_process->d[r], (K_bits_F - 2 * harq_process->Z) * sizeof(int16_t));
-          // skip filler bits
-          memcpy((&z[0] + Kr), harq_process->d[r] + (Kr - 2 * harq_process->Z), (kc * harq_process->Z - Kr) * sizeof(int16_t));
-          // Saturate coded bits before decoding into 8 bits values
-          for (int i = 0, j = 0; j < ((kc * harq_process->Z) >> 4) + 1; i += 2, j++) {
-            pl[j] = _mm_packs_epi16(pv[i], pv[i + 1]);
-          }
-
-          no_iteration_ldpc = nrLDPC_decoder(&decParams, (int8_t *)pl, llrProcBuf, p_procTime);
-        }
-
-        for (int m = 0; m < Kr >> 3; m++) {
-          harq_process->c[r][m] = (uint8_t)llrProcBuf[m];
-        }
-
-        if (check_crc((uint8_t *)llrProcBuf, length_dec, harq_process->F, crc_type)) {
-#ifdef PRINT_CRC_CHECK
-          LOG_I(PHY, "Segment %d CRC OK\n", r);
-#endif
-          no_iteration_ldpc = 2;
-        } else {
-#ifdef PRINT_CRC_CHECK
-          LOG_I(PHY, "segment %d CRC NOK\n", r);
-#endif
-          no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
-        }
-        //}
-
-        r_offset += E;
-
-        /*for (int k=0;k<8;k++)
-          {
-          printf("output decoder [%d] =  0x%02x \n", k, harq_process->c[r][k]);
-          printf("llrprocbuf [%d] =  %x adr %p\n", k, llrProcBuf[k], llrProcBuf+k);
-          }
-        */
-      } else {
-        dtx_det = 0;
-        no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
+    //if ((dtx_det == 0) && (pusch_pdu->pusch_data.rv_index == 0)) {
+      memcpy((&z_ol[0]), ulsch_llr + r_offset, E * sizeof(short));
+      __m128i *pv_ol128 = (__m128i *)&z_ol;
+      __m128i *pl_ol128 = (__m128i *)&l_ol;
+      for (int i = 0, j = 0; j < ((kc * harq_process->Z) >> 4) + 1; i += 2, j++) {
+        pl_ol128[j] = _mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]);
       }
+
+      int ret = nrLDPC_decoder_offload(&decParams, harq_pid, ULSCH_id, r, pusch_pdu->pusch_data.rv_index, harq_process->F, E, Qm, (int8_t *)&pl_ol128[0], llrProcBuf, 1);
+      if (ret < 0) {
+        LOG_E(PHY, "ulsch_decoding.c: Problem in LDPC decoder offload\n");
+        no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
+        return 1;
+      }
+
+      for (int m = 0; m < Kr >> 3; m++) {
+        harq_process->c[r][m] = (uint8_t)llrProcBuf[m];
+      }
+
+      bool crc_check = check_crc((uint8_t *)llrProcBuf, length_dec, harq_process->F, crc_type);
+      no_iteration_ldpc = crc_check ? 2 : ulsch->max_ldpc_iterations + 1;
+
+#ifdef PRINT_CRC_CHECK
+      if (crc_check) {
+        LOG_I(PHY, "Segment %d CRC OK\n", r);
+      } else {
+        LOG_I(PHY, "segment %d CRC NOK\n", r);
+      }
+#endif
+      r_offset += E;
+  //} else {
+  //  dtx_det = 0;
+  //  no_iteration_ldpc = ulsch->max_ldpc_iterations + 1;
+  //}
       bool decodeSuccess = (no_iteration_ldpc <= ulsch->max_ldpc_iterations);
       if (decodeSuccess) {
-        memcpy(harq_process->b + offset, harq_process->c[r], Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
-        offset += (Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));
+        size_t copySize = Kr_bytes - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0);
+        memcpy(harq_process->b + offset, harq_process->c[r], copySize);
+        offset += copySize;
         harq_process->processedSegments++;
       } else {
         LOG_D(PHY, "uplink segment error %d/%d\n", r, harq_process->C);
@@ -534,17 +466,13 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
         break; // don't even attempt to decode other segments
       }
     }
-
-    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING, 0);
-
+    //VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING, 0);
     if (harq_process->processedSegments == harq_process->C) {
       LOG_D(PHY, "[gNB %d] ULSCH: Setting ACK for slot %d TBS %d\n", phy_vars_gNB->Mod_id, ulsch->slot, harq_process->TBS);
       ulsch->active = false;
       harq_process->round = 0;
-
       LOG_D(PHY, "ULSCH received ok \n");
       nr_fill_indication(phy_vars_gNB, ulsch->frame, ulsch->slot, ULSCH_id, harq_pid, 0, 0);
-
     } else {
       LOG_D(PHY,
             "[gNB %d] ULSCH: Setting NAK for SFN/SF %d/%d (pid %d, status %d, round %d, TBS %d)\n",
@@ -560,11 +488,11 @@ uint32_t nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
       LOG_D(PHY, "ULSCH %d in error\n", ULSCH_id);
       nr_fill_indication(phy_vars_gNB, ulsch->frame, ulsch->slot, ULSCH_id, harq_pid, 1, 0);
     }
+    VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING, 0);
     ulsch->last_iteration_cnt = no_iteration_ldpc;
   }
 
   else {
-    dtx_det = 0;
 
     for (int r = 0; r < harq_process->C; r++) {
       int E = nr_get_E(G, harq_process->C, Qm, n_layers, r);

openair1/PHY/defs_gNB.h

@@ -684,7 +684,7 @@ typedef struct PHY_VARS_gNB_s {
   int ldpc_offload_flag;
 
   int reorder_thread_disable;
-
+  char ldpc_offload_version[4];
   int max_ldpc_iterations;
   /// indicate the channel estimation technique in time domain
   int chest_time;

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -203,6 +203,8 @@ int main(int argc, char **argv)
   int ldpc_offload_flag = 0;
   uint8_t max_rounds = 4;
   int chest_type[2] = {0};
+  char *ldpc_offload_version=NULL;
+  ldpc_offload_version = "";
   int enable_ptrs = 0;
   int modify_dmrs = 0;
   /* L_PTRS = ptrs_arg[0], K_PTRS = ptrs_arg[1] */
@@ -237,7 +239,7 @@ int main(int argc, char **argv)
   /* initialize the sin-cos table */
    InitSinLUT();
 
-  while ((c = getopt(argc, argv, "a:b:c:d:ef:g:h:i:kl:m:n:op:q:r:s:t:u:v:w:y:z:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:")) != -1) {
+  while ((c = getopt(argc, argv, "a:b:c:d:ef:g:h:i:kl:m:n:o:p:q:r:s:t:u:v:w:y:z:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:")) != -1) {
     printf("handling optarg %c\n",c);
     switch (c) {
 
@@ -354,6 +356,7 @@ int main(int argc, char **argv)
 
     case 'o':
       ldpc_offload_flag = 1;
+      ldpc_offload_version = strdup(optarg);
       break;
       
     case 'p':
@@ -557,7 +560,6 @@ int main(int argc, char **argv)
       printf("-E {SRS: [0] Disabled, [1] Enabled} e.g. -E 1\n");
       exit(-1);
       break;
-
     }
   }
   
@@ -671,6 +673,20 @@ int main(int argc, char **argv)
   gNB->chest_freq = chest_type[0];
   gNB->chest_time = chest_type[1];
 
+  //phy_init_nr_gNB(gNB);
+
+  if (gNB->ldpc_offload_flag == 1) {
+    char version[128] = "_";
+    strcat(version,ldpc_offload_version);
+    if (!strcmp(ldpc_offload_version,"t1") || !strcmp(ldpc_offload_version,"t2")){
+       strcpy(gNB->ldpc_offload_version,version);
+    } else {
+      AssertFatal(strcmp(ldpc_offload_version,"") != 0, "Select offload library version: -o t1 / -o t2\n");
+      AssertFatal(strcmp(ldpc_offload_version,"t1") == 0 || !strcmp(ldpc_offload_version,"t2") == 0,
+                         "Unsupported offload library version\n");
+    }
+  }
+
   phy_init_nr_gNB(gNB);
   /* RU handles rxdataF, and gNB just has a pointer. Here, we don't have an RU,
    * so we need to allocate that memory as well. */