T2: test of decoder optim

Introduce new bit reversal implementation using simde_mm_gf2p8affine_epi64_epi8
instruction, based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/.
This implementation requires GFNI and AVX512VL support on the target machine.

common/utils/nr/nr_common.c

@@ -35,6 +35,7 @@
 #include "common/utils/assertions.h"
 #include "nr_common.h"
 #include <complex.h>
+#include <simde/x86/gfni.h>
 
 #define C_SRS_NUMBER (64)
 #define B_SRS_NUMBER (4)
@@ -131,8 +132,27 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out)
   DevAssert(in != NULL);
   DevAssert(out != NULL);
 
+// Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/
+#if defined(__GFNI__) && defined(__AVX512VL__)
+  int simde_sz = 16;
+  int remaining_bytes = sz % simde_sz;
+  int i = 0;
+  for (; i + 15 < sz; i += simde_sz) {
+    __m128i input = simde_mm_loadu_si128((__m128i*)&in[i]);
+    __m128i reversed = simde_mm_gf2p8affine_epi64_epi8(input, simde_mm_set1_epi64x(0x8040201008040201), 0);
+    simde_mm_storeu_si128((__m128i*)&out[i], reversed);
+  }
+
+  if (remaining_bytes > 0) {
+    for (; i < sz; ++i) {
+      out[i] = bit_reverse_table_256[in[i]];
+    }
+  }
+
+#else
   for(size_t i = 0; i < sz; ++i)
     out[i] = bit_reverse_table_256[in[i]];
+#endif
 }
 
 // Reverse bits implementation based on http://graphics.stanford.edu/~seander/bithacks.html

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h

@@ -253,6 +253,7 @@ typedef struct nrLDPC_coding_interface_s {
   nrLDPC_coding_shutdown_t *nrLDPC_coding_shutdown;
   nrLDPC_coding_decoder_t *nrLDPC_coding_decoder;
   nrLDPC_coding_encoder_t *nrLDPC_coding_encoder;
+  char version[64];
 } nrLDPC_coding_interface_t;
 
 int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *interface);

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface_load.c

@@ -63,6 +63,21 @@ int load_nrLDPC_coding_interface(char *version, nrLDPC_coding_interface_t *itf)
   itf->nrLDPC_coding_decoder = (nrLDPC_coding_decoder_t *)shlib_fdesc[2].fptr;
   itf->nrLDPC_coding_encoder = (nrLDPC_coding_encoder_t *)shlib_fdesc[3].fptr;
 
+  // Retrieve LDPC shlibversion
+  char *shlibpath = NULL;
+  char *shlibversion = NULL;
+  paramdef_t LoaderParams[] = {
+    {"shlibpath",    NULL, 0, .strptr = &shlibpath,    .defstrval = NULL, TYPE_STRING, 0, NULL},
+    {"shlibversion", NULL, 0, .strptr = &shlibversion, .defstrval = "",   TYPE_STRING, 0, NULL}
+  };
+
+  char cfgprefix[sizeof(LOADER_CONFIG_PREFIX)+strlen(libname)+16];
+  sprintf(cfgprefix,LOADER_CONFIG_PREFIX ".%s",libname);
+  ret = config_get(config_get_if(), LoaderParams, sizeofArray(LoaderParams), cfgprefix);
+
+  // Save the version string in the interface
+  snprintf(itf->version, sizeof(itf->version), "%s", shlibversion);
+
   AssertFatal(itf->nrLDPC_coding_init() == 0, "error starting LDPC library %s %s\n", libname, version);
 
   return 0;

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c

@@ -650,25 +650,40 @@ static int
 retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops,
                      nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
 {
+  uint8_t *p_out = NULL;
   unsigned int j = 0;
   for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){
+    int E_sum = 0;
+    int bit_offset = 0;
+    int byte_offset = 0;
+    p_out = nrLDPC_slot_encoding_parameters->TBs[h].segments[0].output;
     for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) {
       struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output;
       struct rte_mbuf *m = output->data;
       uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset;
-      uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].output;
       const char *data = m->buf_addr + m->data_off;
-      const char *end = data + data_len;
-      while (data < end) {
-        uint8_t byte = *data++; // get the current byte
-        for (int bit = 7; bit >= 0; --bit) {
-          *out++ = (byte >> bit) & 1; // extract each bit
+      if (bit_offset == 0) {
+        memcpy(&p_out[byte_offset], data, data_len);
+      } else {
+        uint8_t carry = 0;
+        p_out[byte_offset - 1] |= data[0] >> bit_offset;
+        for (size_t i = 0; i < data_len; i++) {
+          uint8_t current = *data++;
+          p_out[byte_offset + i] = (current << (8 - bit_offset));
+          if (i != 0) {
+            carry = current >> bit_offset;
+            p_out[byte_offset + i - 1] |= carry;
+          }
         }
       }
+      E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[i].E;
+      byte_offset = (E_sum + 7) / 8;
+      bit_offset = E_sum % 8;
       rte_pktmbuf_free(m);
       rte_pktmbuf_free(ops[j]->ldpc_enc.input.data);
       ++j;
     }
+    reverse_bits_u8(p_out, byte_offset, p_out);
   }
   return 0;
 }
@@ -815,6 +830,8 @@ static int pmd_lcore_ldpc_enc(void *arg)
   AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments);
 
   set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters);
+  if(nrLDPC_slot_encoding_parameters->tprep != NULL) stop_meas(nrLDPC_slot_encoding_parameters->tprep);
+  if(nrLDPC_slot_encoding_parameters->tparity != NULL) start_meas(nrLDPC_slot_encoding_parameters->tparity);
   for (enq = 0, deq = 0; enq < num_segments;) {
     num_to_enq = num_segments;
     if (unlikely(num_segments - enq < num_to_enq))
@@ -828,10 +845,11 @@ static int pmd_lcore_ldpc_enc(void *arg)
     time_out++;
     DevAssert(time_out <= TIME_OUT_POLL);
   }
-
+  if(nrLDPC_slot_encoding_parameters->tparity != NULL) stop_meas(nrLDPC_slot_encoding_parameters->tparity);
+  if(nrLDPC_slot_encoding_parameters->toutput != NULL) start_meas(nrLDPC_slot_encoding_parameters->toutput);
   ret = retrieve_ldpc_enc_op(ops_deq, nrLDPC_slot_encoding_parameters);
   AssertFatal(ret == 0, "Failed to retrieve LDPC encoding op!");
-
+  if(nrLDPC_slot_encoding_parameters->toutput != NULL) stop_meas(nrLDPC_slot_encoding_parameters->toutput);
   rte_bbdev_enc_op_free_bulk(ops_enq, num_segments);
   rte_free(ops_enq);
   rte_free(ops_deq);
@@ -1009,7 +1027,6 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
   for(uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){
     num_blocks += nrLDPC_slot_decoding_parameters->TBs[h].C;
   }
-  uint16_t z_ol[LDPC_MAX_CB_SIZE] __attribute__((aligned(16)));
   /* It is not unlikely that l_ol becomes big enough to overflow the stack
    * If you observe this behavior then move it to the heap
    * Then you would better do a persistent allocation to limit the overhead
@@ -1039,12 +1056,13 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
   int offset = 0;
   for(uint16_t h = 0; h < nrLDPC_slot_decoding_parameters->nb_TBs; ++h){
     for (int r = 0; r < nrLDPC_slot_decoding_parameters->TBs[h].C; r++) {
-      memcpy(z_ol, nrLDPC_slot_decoding_parameters->TBs[h].segments[r].llr, nrLDPC_slot_decoding_parameters->TBs[h].segments[r].E * sizeof(uint16_t));
-      simde__m128i *pv_ol128 = (simde__m128i *)z_ol;
+      nrLDPC_segment_decoding_parameters_t ldpc_segment =  nrLDPC_slot_decoding_parameters->TBs[h].segments[r];
+      simde__m128i *pv_ol128 = (simde__m128i *)ldpc_segment.llr;
       simde__m128i *pl_ol128 = (simde__m128i *)&l_ol[offset];
-      int kc = nrLDPC_slot_decoding_parameters->TBs[h].BG == 2 ? 52 : 68;
-      for (int i = 0, j = 0; j < ((kc * nrLDPC_slot_decoding_parameters->TBs[h].Z) >> 4) + 1; i += 2, j++) {
-        pl_ol128[j] = simde_mm_packs_epi16(pv_ol128[i], pv_ol128[i + 1]);
+      for (int i = 0, j = 0; j < (ldpc_segment.E >> 4) + 1; i += 2, j++) {
+        simde__m128i seg1 = simde_mm_loadu_si128(pv_ol128++);
+        simde__m128i seg2 = simde_mm_loadu_si128(pv_ol128++);
+        pl_ol128[j] = simde_mm_packs_epi16(seg1, seg2);
       }
       offset += LDPC_MAX_CB_SIZE;
     }
@@ -1074,6 +1092,7 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
 int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
 {
   pthread_mutex_lock(&encode_mutex);
+  if(nrLDPC_slot_encoding_parameters->tprep != NULL) start_meas(nrLDPC_slot_encoding_parameters->tprep);
   // hardcoded to use the first found board
   struct active_device *ad = active_devs;
   int ret;

openair1/PHY/CODING/nrLDPC_extern.h

@@ -29,6 +29,7 @@ typedef struct ldpc_interface_s {
   LDPC_shutdownfunc_t *LDPCshutdown;
   LDPC_decoderfunc_t *LDPCdecoder;
   LDPC_encoderfunc_t *LDPCencoder;
+  char version[64];
 } ldpc_interface_t;
 
 // Global var to limit the rework of the dirty legacy code

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -44,9 +44,13 @@
 //#define DEBUG_DLSCH
 //#define DEBUG_DLSCH_MAPPING
 
-static void nr_pdsch_codeword_scrambling(uint8_t *in, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI, uint32_t *out)
+static void nr_pdsch_codeword_scrambling(uint8_t *in, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI, uint32_t *out, char *ldpc_version)
 {
-  nr_codeword_scrambling(in, size, q, Nid, n_RNTI, out);
+  if (strcmp(ldpc_version, "_t2") == 0) {
+    nr_codeword_scrambling_t2(in, size, q, Nid, n_RNTI, out);
+  } else {
+    nr_codeword_scrambling(in, size, q, Nid, n_RNTI, out);
+  }
 }
 
 void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
@@ -187,7 +191,7 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
       uint32_t scrambled_output[(encoded_length>>5)+4]; // modulator acces by 4 bytes in some cases
       memset(scrambled_output, 0, sizeof(scrambled_output));
       if ( encoded_length > rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers) abort();
-      nr_pdsch_codeword_scrambling(&output[offset_output], encoded_length, codeWord, rel15->dataScramblingId, rel15->rnti, scrambled_output);
+      nr_pdsch_codeword_scrambling(&output[offset_output], encoded_length, codeWord, rel15->dataScramblingId, rel15->rnti, scrambled_output, gNB->nrLDPC_coding_interface.version);
 
 #ifdef DEBUG_DLSCH
       printf("PDSCH scrambling:\n");

openair1/PHY/NR_TRANSPORT/nr_scrambling.c

@@ -24,6 +24,27 @@
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #define DEBUG_SCRAMBLING(a)
 //#define DEBUG_SCRAMBLING(a) a
+
+void nr_codeword_scrambling_t2(uint8_t *in,
+                             uint32_t size,
+                             uint8_t q,
+                             uint32_t Nid,
+                             uint32_t n_RNTI,
+                             uint32_t* out)
+{
+  int numBytes = (size + 7) / 8;
+  const int roundedSz = (numBytes + 3) / 4;
+  uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz);  // Gold sequence for scrambling
+  for (int i = 0; i < roundedSz; i++) {
+    uint32_t in32 = 0;
+    in32 |= ((uint32_t)*in++ << (0 * 8));
+    in32 |= ((uint32_t)*in++ << (1 * 8));
+    in32 |= ((uint32_t)*in++ << (2 * 8));
+    in32 |= ((uint32_t)*in++ << (3 * 8));
+    out[i] = in32 ^ *seq++;
+  }
+}
+
 void nr_codeword_scrambling(uint8_t *in,
                             uint32_t size,
                             uint8_t q,

openair1/PHY/NR_TRANSPORT/nr_transport_common_proto.h

@@ -78,6 +78,13 @@ void nr_codeword_scrambling(uint8_t *in,
                             uint32_t n_RNTI,
                             uint32_t* out);
 
+void nr_codeword_scrambling_t2(uint8_t *in,
+                               uint32_t size,
+                               uint8_t q,
+                               uint32_t Nid,
+                               uint32_t n_RNTI,
+                               uint32_t* out);
+
 void nr_codeword_unscrambling(int16_t* llr, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI);
 void nr_codeword_unscrambling_init(int16_t *s, uint32_t size, uint8_t q, uint32_t Nid, uint32_t n_RNTI);
 

openair1/PHY/NR_UE_TRANSPORT/nr_transport_proto_ue.h

@@ -124,7 +124,8 @@ void nr_pusch_codeword_scrambling(uint8_t *in,
                                   uint32_t Nid,
                                   uint32_t n_RNTI,
                                   bool uci_on_pusch,
-                                  uint32_t* out);
+                                  uint32_t* out,
+                                  char *version);
 
 
 /** \brief Alternative entry point to UE uplink shared channels procedures.

openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_ue.c

@@ -77,12 +77,17 @@ void nr_pusch_codeword_scrambling_uci(uint8_t *in, uint32_t size, uint32_t Nid,
   }
 }
 
-void nr_pusch_codeword_scrambling(uint8_t *in, uint32_t size, uint32_t Nid, uint32_t n_RNTI, bool uci_on_pusch, uint32_t* out)
+void nr_pusch_codeword_scrambling(uint8_t *in, uint32_t size, uint32_t Nid, uint32_t n_RNTI, bool uci_on_pusch, uint32_t* out, char *ldpc_version)
 {
-  if (uci_on_pusch)
+  if (uci_on_pusch) {
     nr_pusch_codeword_scrambling_uci(in, size, Nid, n_RNTI, out);
-  else
-    nr_codeword_scrambling(in, size, 0, Nid, n_RNTI, out);
+  } else {
+    if (strcmp(ldpc_version, "_t2") == 0) {
+      nr_codeword_scrambling_t2(in, size, 0, Nid, n_RNTI, out);
+    } else {
+      nr_codeword_scrambling(in, size, 0, Nid, n_RNTI, out);
+    }
+  }
 }
 
 /*
@@ -599,7 +604,8 @@ void nr_ue_ulsch_procedures(PHY_VARS_NR_UE *UE,
                                pusch_pdu->data_scrambling_id,
                                rnti,
                                false,
-                               scrambled_output);
+                               scrambled_output,
+                               UE->nrLDPC_coding_interface.version);
 
   /////////////////////////ULSCH modulation/////////////////////////