added 128-bit implementations for interleaving of code segements in

ldpc_encoder_optim8seg

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_decoder.c

@@ -133,8 +133,9 @@
 #include "bnProc128/nrLDPC_bnProc_BG2_R23_128.h"
 #endif
 
-//#define NR_LDPC_PROFILER_DETAIL(a) a
-#define NR_LDPC_PROFILER_DETAIL(a)
+#define NR_LDPC_PROFILER_DETAIL(a) a
+//#define NR_LDPC_PROFILER_DETAIL(a)
+
 
 #include "openair1/PHY/CODING/nrLDPC_extern.h"
 

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c

@@ -42,6 +42,10 @@
 #include "ldpc_generate_coefficient.c"
 #include "PHY/sse_intrin.h"
 
+#ifdef __aarch64__
+#define USE128BIT
+#endif
+
 int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp)
 {
   int Zc = impp->Zc;
@@ -55,6 +59,7 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
   char temp;
   int simd_size;
 
+#ifndef USE128BIT
   simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
   simde__m256i andmask  = simde_mm256_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
   simde__m256i zero256   = simde_mm256_setzero_si256();
@@ -68,7 +73,21 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
   masks[5] = simde_mm256_set1_epi8(0x20);
   masks[6] = simde_mm256_set1_epi8(0x40);
   masks[7] = simde_mm256_set1_epi8(0x80);
-
+#else
+  simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+  simde__m128i andmask  = simde_mm_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
+  simde__m128i zero128   = simde_mm_setzero_si128();
+  simde__m128i masks[8];
+  register simde__m128i c128;
+  masks[0] = simde_mm_set1_epi8(0x1);
+  masks[1] = simde_mm_set1_epi8(0x2);
+  masks[2] = simde_mm_set1_epi8(0x4);
+  masks[3] = simde_mm_set1_epi8(0x8);
+  masks[4] = simde_mm_set1_epi8(0x10);
+  masks[5] = simde_mm_set1_epi8(0x20);
+  masks[6] = simde_mm_set1_epi8(0x40);
+  masks[7] = simde_mm_set1_epi8(0x80);
+#endif
   AssertFatal((impp->n_segments>0&&impp->n_segments<=8),"0 < n_segments %d <= 8\n",impp->n_segments);
 
   //determine number of bits in codeword
@@ -122,7 +141,7 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
       c[i] |= (temp << j);
     }
   }
-#else
+#elif !defined(USE128BIT)
   for (i=0; i<block_length>>5; i++) {
     c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero256),masks[0]);
     for (j=1; j<impp->n_segments; j++) {
@@ -130,14 +149,13 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
     }
     ((simde__m256i *)c)[i] = c256;
   }
-
-  for (i=(block_length>>5)<<5;i<block_length;i++) {
-    for (j=0; j<impp->n_segments; j++) {
-
-      temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
-      //printf("c(%d,%d)=%d\n",j,i,temp);
-      c[i] |= (temp << j);
+#else
+  for (i=0; i<block_length>>4; i++) {
+    c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero128),masks[0]);
+    for (j=1; j<impp->n_segments; j++) {
+      c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint32_t*)test_input[j])[i]), shufmask),andmask),zero128),masks[j]),c128);
     }
+    ((simde__m128i *)c)[i] = c128;
   }
 #endif
 
@@ -169,10 +187,10 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
     //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n");
     uint32_t l1 = (block_length-(2*Zc))>>5;
     uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5;
+#ifndef USE128BIT    
     simde__m256i *c256p = (simde__m256i *)&c[2*Zc];
     simde__m256i *d256p = (simde__m256i *)&d[0];
     //  if (((block_length-(2*Zc))&31)>0) l1++;
-    
     for (i=0;i<l1;i++)
       for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]);
     
@@ -180,6 +198,18 @@ int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implempar
     
     for (i1=0;i1<l2;i1++,i++)
       for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]);
+#else
+    simde__m128i *c128p = (simde__m128i *)&c[2*Zc];
+    simde__m128i *d128p = (simde__m128i *)&d[0];
+    //  if (((block_length-(2*Zc))&31)>0) l1++;
+    for (i=0;i<l1;i++)
+      for (j=0;j<impp->n_segments;j++) ((simde__m128i *)channel_input[j])[i] = simde_mm_and_si128(simde_mm_srai_epi16(c128p[i],j),masks[0]);
+    
+    //  if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++;
+    
+    for (i1=0;i1<l2;i1++,i++)
+      for (j=0;j<impp->n_segments;j++) ((simde__m128i *)channel_input[j])[i] = simde_mm_and_si128(simde_mm_srai_epi16(d128p[i1],j),masks[0]);
+#endif
   }
   else {
 #ifdef DEBUG_LDPC

openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c

@@ -13,6 +13,10 @@
 
 #define INVALID_VALUE 255
 
+#ifdef __aarch64__
+#define USE128BIT
+#endif
+
 void nr_idft(int32_t *z, uint32_t Msc_PUSCH)
 {
 
@@ -251,7 +255,7 @@ static void nr_ulsch_channel_compensation(c16_t *rxFext,
   int mod_order = rel15_ul->qam_mod_order;
   int nrOfLayers = rel15_ul->nrOfLayers;
   int nb_rx_ant = frame_parms->nb_antennas_rx;
-
+#ifndef USE128BIT 
   simde__m256i QAM_ampa_256 = simde_mm256_setzero_si256();
   simde__m256i QAM_ampb_256 = simde_mm256_setzero_si256();
   simde__m256i QAM_ampc_256 = simde_mm256_setzero_si256();
@@ -275,13 +279,45 @@ static void nr_ulsch_channel_compensation(c16_t *rxFext,
   simde__m256i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4;
   simde__m256i complex_shuffle256 = simde_mm256_set_epi8(29,28,31,30,25,24,27,26,21,20,23,22,17,16,19,18,13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2);
   simde__m256i conj256 = simde_mm256_set_epi16(1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1,1,-1);
+#else
+  simde__m128i QAM_ampa_128 = simde_mm_setzero_si128();
+  simde__m128i QAM_ampb_128 = simde_mm_setzero_si128();
+  simde__m128i QAM_ampc_128 = simde_mm_setzero_si128();
 
+  if (mod_order == 4) {
+    QAM_ampa_128 = simde_mm_set1_epi16(QAM16_n1);
+    QAM_ampb_128 = simde_mm_setzero_si128();
+    QAM_ampc_128 = simde_mm_setzero_si128();
+  }
+  else if (mod_order == 6) {
+    QAM_ampa_128 = simde_mm_set1_epi16(QAM64_n1);
+    QAM_ampb_128 = simde_mm_set1_epi16(QAM64_n2);
+    QAM_ampc_128 = simde_mm_setzero_si128();
+  }
+  else if (mod_order == 8) {
+    QAM_ampa_128 = simde_mm_set1_epi16(QAM256_n1);
+    QAM_ampb_128 = simde_mm_set1_epi16(QAM256_n2);
+    QAM_ampc_128 = simde_mm_set1_epi16(QAM256_n3);
+  }
+
+  simde__m128i xmmp0, xmmp1, xmmp2, xmmp3, xmmp4;
+  simde__m128i complex_shuffle128 = simde_mm_set_epi8(13,12,15,14,9,8,11,10,5,4,7,6,1,0,3,2);
+  simde__m128i conj128 = simde_mm_set_epi16(1,-1,1,-1,1,-1,1,-1);
+#endif
   for (int aatx = 0; aatx < nrOfLayers; aatx++) {
+#ifndef USE128BIT
     simde__m256i *rxComp_256 =     (simde__m256i*)     &rxComp[aatx * nb_rx_ant][symbol * buffer_length];
     simde__m256i *rxF_ch_maga_256 = (simde__m256i*)&ul_ch_maga[aatx * buffer_length];
     simde__m256i *rxF_ch_magb_256 = (simde__m256i*)&ul_ch_magb[aatx * buffer_length];
     simde__m256i *rxF_ch_magc_256 = (simde__m256i*)&ul_ch_magc[aatx * buffer_length];
+#else
+    simde__m128i *rxComp_128 =     (simde__m128i*)     &rxComp[aatx * nb_rx_ant][symbol * buffer_length];
+    simde__m128i *rxF_ch_maga_128 = (simde__m128i*)&ul_ch_maga[aatx * buffer_length];
+    simde__m128i *rxF_ch_magb_128 = (simde__m128i*)&ul_ch_magb[aatx * buffer_length];
+    simde__m128i *rxF_ch_magc_128 = (simde__m128i*)&ul_ch_magc[aatx * buffer_length];
+#endif
     for (int aarx = 0; aarx < nb_rx_ant; aarx++) {
+#ifndef USE128BIT 
       simde__m256i *rxF_256 = (simde__m256i*) &rxFext[aarx * buffer_length];
       simde__m256i *chF_256 = (simde__m256i*) &chFext[(aatx * nb_rx_ant + aarx) * buffer_length];
 
@@ -316,9 +352,46 @@ static void nr_ulsch_channel_compensation(c16_t *rxFext,
           rxF_ch_magb_256[i] = simde_mm256_add_epi16(rxF_ch_magb_256[i], xmmp3); 
         if (mod_order > 6)
           rxF_ch_magc_256[i] = simde_mm256_add_epi16(rxF_ch_magc_256[i], xmmp1); 
+#else
+      simde__m128i *rxF_128 = (simde__m128i*) &rxFext[aarx * buffer_length];
+      simde__m128i *chF_128 = (simde__m128i*) &chFext[(aatx * nb_rx_ant + aarx) * buffer_length];
+
+      for (int i = 0; i < buffer_length >> 2; i++) 
+      {
+        xmmp0  = simde_mm_madd_epi16(chF_128[i], rxF_128[i]);
+        // xmmp0 contains real part of 2 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i]
+        xmmp1  = simde_mm_shuffle_epi8(chF_128[i], complex_shuffle128);
+        xmmp1  = simde_mm_sign_epi16(xmmp1, conj128);
+        xmmp1  = simde_mm_madd_epi16(xmmp1, rxF_128[i]);
+        // xmmp1 contains imag part of 2 consecutive outputs (32-bit) of conj(H_m[i])*R_m[i]
+        xmmp0  = simde_mm_srai_epi32(xmmp0, output_shift);
+        xmmp1  = simde_mm_srai_epi32(xmmp1, output_shift);
+        xmmp2  = simde_mm_unpacklo_epi32(xmmp0, xmmp1);
+        xmmp3  = simde_mm_unpackhi_epi32(xmmp0, xmmp1);
+        xmmp4  = simde_mm_packs_epi32(xmmp2, xmmp3);
+
+        xmmp0 = simde_mm_madd_epi16(chF_128[i], chF_128[i]); // |h|^2
+        xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift); 
+        xmmp0 = simde_mm_packs_epi32(xmmp0, xmmp0);
+        xmmp1 = simde_mm_unpacklo_epi16(xmmp0, xmmp0);
+
+        xmmp2 = simde_mm_mulhrs_epi16(xmmp1, QAM_ampa_128);
+        xmmp3 = simde_mm_mulhrs_epi16(xmmp1, QAM_ampb_128);
+        xmmp1 = simde_mm_mulhrs_epi16(xmmp1, QAM_ampc_128);
+
+        // MRC
+        rxComp_128[i] = simde_mm_add_epi16(rxComp_128[i], xmmp4); 
+        //if (mod_order > 2)
+          rxF_ch_maga_128[i] = simde_mm_add_epi16(rxF_ch_maga_128[i], xmmp2); 
+        //if (mod_order > 4)
+          rxF_ch_magb_128[i] = simde_mm_add_epi16(rxF_ch_magb_128[i], xmmp3); 
+        //if (mod_order > 6)
+          rxF_ch_magc_128[i] = simde_mm_add_epi16(rxF_ch_magc_128[i], xmmp1); 
+#endif
       }
       if (rho != NULL) {
         for (int atx = 0; atx < nrOfLayers; atx++) {
+#ifndef USE128BIT 
           simde__m256i *rho_256  = (simde__m256i *   )&rho[(aatx * nrOfLayers + atx) * buffer_length];
           simde__m256i *chF_256  = (simde__m256i *)&chFext[(aatx * nb_rx_ant + aarx) * buffer_length];
           simde__m256i *chF2_256 = (simde__m256i *)&chFext[ (atx * nb_rx_ant + aarx) * buffer_length];
@@ -337,6 +410,27 @@ static void nr_ulsch_channel_compensation(c16_t *rxFext,
 
             rho_256[i] = simde_mm256_adds_epi16(rho_256[i], simde_mm256_packs_epi32(xmmp2, xmmp3));
           }
+#else
+          simde__m128i *rho_128  = (simde__m128i *   )&rho[(aatx * nrOfLayers + atx) * buffer_length];
+          simde__m128i *chF_128  = (simde__m128i *)&chFext[(aatx * nb_rx_ant + aarx) * buffer_length];
+          simde__m128i *chF2_128 = (simde__m128i *)&chFext[ (atx * nb_rx_ant + aarx) * buffer_length];
+          for (int i = 0; i < buffer_length >> 2; i++) {
+            // multiply by conjugated channel
+            xmmp0 = simde_mm_madd_epi16(chF_128[i], chF2_128[i]);
+            // xmmp0 contains real part of 2 consecutive outputs (32-bit)
+            xmmp1 = simde_mm_shuffle_epi8(chF_128[i], complex_shuffle128);
+            xmmp1 = simde_mm_sign_epi16(xmmp1, conj128);
+            xmmp1 = simde_mm_madd_epi16(xmmp1, chF2_128[i]);
+            // xmmp0 contains imag part of 2 consecutive outputs (32-bit)
+            xmmp0 = simde_mm_srai_epi32(xmmp0, output_shift);
+            xmmp1 = simde_mm_srai_epi32(xmmp1, output_shift);
+            xmmp2 = simde_mm_unpacklo_epi32(xmmp0, xmmp1);
+            xmmp3 = simde_mm_unpackhi_epi32(xmmp0, xmmp1);
+
+            rho_128[i] = simde_mm_adds_epi16(rho_128[i], simde_mm_packs_epi32(xmmp2, xmmp3));
+          }
+
+#endif
         }
       }
     }

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -1478,7 +1478,7 @@ int main(int argc, char *argv[])
     for (int r = 1; r < max_rounds; r++)
       printf(",%e", berStats[r]);
 
-    printf(") Avg round %.2f, Eff Rate %.4f bits/slot, Eff Throughput %.2f, TBS %u bits/slot\n", roundStats,effRate,effTP,TBS);
+    printf(") Avg round %.2f, Eff Rate %.4f bits/slot, Eff Throughput %.2f, TBS %u bits/slot, C %d\n", roundStats,effRate,effTP,TBS,UE->ul_harq_processes[harq_pid].C);
 
     double av_delay = (double)sum_pusch_delay / (2 * delay_pusch_est_count);
     printf("DMRS-PUSCH delay estimation: min %i, max %i, average %2.1f\n", min_pusch_delay >> 1, max_pusch_delay >> 1, av_delay);