fix(cnProc_gen_BG1_128): Avoid sign only for Arm

Add preprocessor directive to avoid the use of `simde_mm_sign_epi8` only on Arm targets.

CMakeLists.txt

@@ -146,9 +146,25 @@ elseif(CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
     set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -DSIMDE_X86_SSSE3_NATIVE")
   endif()
 elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "armv7l")
-  set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt")
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mfloat-abi=hard -mfpu=neon -lgcc -lrt")
 elseif (CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64")
-  set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -lgcc -lrt")
+  execute_process(COMMAND
+	          "lscpu"
+                  OUTPUT_VARIABLE
+                  CMAKE_NEOVERSE_INFO
+                  OUTPUT_STRIP_TRAILING_WHITESPACE
+                 )  
+  if (CMAKE_NEOVERSE_INFO MATCHES "Neoverse-V1")	
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v1 -lgcc -lrt")
+  elseif (CMAKE_NEOVERSE_INFO MATCHES "Neoverse-V2")
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-v2 -ftree-vectorize -lgcc -lrt")
+  elseif (CMAKE_NEOVERSE_INFO MATCHES "Neoverse-N1")
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n1  -lgcc -lrt")
+  elseif (CMAKE_NEOVERSE_INFO MATCHES "Neoverse-N2")
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=neoverse-n2 -ftree-vectorize -lgcc -lrt")
+  else ()
+    set(C_FLAGS_PROCESSOR "${C_FLAGS_PROCESSOR} -gdwarf-2 -mcpu=native -lgcc -lrt")
+  endif()
 else()
   message(FATAL_ERROR "compile for CPU architecture ${CPUARCH}, CMAKE_SYSTEM_PROCESSOR ${CMAKE_SYSTEM_PROCESSOR}")
 endif()

ci-scripts/datalog_rt_stats.100.2x2.yaml

@@ -11,8 +11,8 @@ Ref :
   feptx_prec : 0.0
   feptx_ofdm : 60.0
   feptx_total : 150.0
-  L1 Tx processing : 530.0
-  DLSCH encoding : 220.0
+  L1 Tx processing : 445.0
+  DLSCH encoding : 150.0
   L1 Rx processing : 530.0
   PUSCH inner-receiver : 360.0
   Schedule Response : 3.0

ci-scripts/datalog_rt_stats.1x1.60.yaml

@@ -11,8 +11,8 @@ Ref :
   feptx_prec : 15.0
   feptx_ofdm : 30.0
   feptx_total : 45.0
-  L1 Tx processing : 205.0
-  DLSCH encoding : 140.0
+  L1 Tx processing : 170.0
+  DLSCH encoding : 110.0
   L1 Rx processing : 345.0
   PUSCH inner-receiver : 150.0
   Schedule Response : 3.0

ci-scripts/datalog_rt_stats.60.2x2.yaml

@@ -11,8 +11,8 @@ Ref :
   feptx_prec : 14.0
   feptx_ofdm : 30.0
   feptx_total : 80.0
-  L1 Tx processing : 315.0
-  DLSCH encoding : 155.0
+  L1 Tx processing : 270.0
+  DLSCH encoding : 120.0
   L1 Rx processing : 345.0
   PUSCH inner-receiver : 155.0
   Schedule Response : 3.0

ci-scripts/datalog_rt_stats.default.yaml

@@ -11,8 +11,8 @@ Ref :
   feptx_prec : 13.0
   feptx_ofdm : 30.0
   feptx_total : 45.0
-  L1 Tx processing : 160.0
-  DLSCH encoding : 100.0
+  L1 Tx processing : 140.0
+  DLSCH encoding : 90.0
   L1 Rx processing : 290.0
   PUSCH inner-receiver : 115.0
   Schedule Response : 3.0

common/utils/nr/nr_common.c

@@ -37,6 +37,7 @@
 #include "nr_common.h"
 #include <limits.h>
 #include <math.h>
+#include <simde/x86/gfni.h>
 
 #define C_SRS_NUMBER (64)
 #define B_SRS_NUMBER (4)
@@ -131,8 +132,24 @@ void reverse_bits_u8(uint8_t const* in, size_t sz, uint8_t* out)
   DevAssert(in != NULL);
   DevAssert(out != NULL);
 
+// Bit reversal implementation based on https://wunkolo.github.io/post/2020/11/gf2p8affineqb-bit-reversal/
+#if defined(__GFNI__) && defined(__AVX512F__)
+  int simde_sz = 64;
+  int i = 0;
+  int simde_bound = sz - simde_sz;
+  for (; i <= simde_bound; i += simde_sz) {
+    __m512i input = _mm512_loadu_epi8(&in[i]);
+    __m512i reversed = _mm512_gf2p8affine_epi64_epi8(input, _mm512_set1_epi64(0x8040201008040201), 0);
+    _mm512_storeu_epi8(&out[i], reversed);
+  }
+
+  for (; i < sz; ++i) {
+    out[i] = bit_reverse_table_256[in[i]];
+  }
+#else
   for(size_t i = 0; i < sz; ++i)
     out[i] = bit_reverse_table_256[in[i]];
+#endif
 }
 
 // Reverse bits implementation based on http://graphics.stanford.edu/~seander/bithacks.html

doc/Doxyfile

@@ -2137,10 +2137,8 @@ INPUT                  = \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc176_byte.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc352_byte.c \
-@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc128_byte_128.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc224_byte.c \
-@CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_generate_coefficient.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc_BG2_Zc104_byte.c \
 @CMAKE_CURRENT_SOURCE_DIR@/../openair1/PHY/CODING/nrLDPC_encoder/ldpc320_byte_128.c \

openair1/PHY/CODING/CMakeLists.txt

@@ -17,20 +17,6 @@ add_library(ldpc_orig MODULE
 set_target_properties(ldpc_orig PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc_orig PRIVATE ldpc_segment ldpc_gen_HEADERS)
 
-add_library(ldpc_optim MODULE
-            nrLDPC_decoder/nrLDPC_decoder.c
-            nrLDPC_encoder/ldpc_encoder_optim.c
-)
-set_target_properties(ldpc_optim PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-target_link_libraries(ldpc_optim PRIVATE ldpc_segment ldpc_gen_HEADERS)
-
-add_library(ldpc_optim8seg MODULE
-            nrLDPC_decoder/nrLDPC_decoder.c
-            nrLDPC_encoder/ldpc_encoder_optim8seg.c
-)
-set_target_properties(ldpc_optim8seg PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
-target_link_libraries(ldpc_optim8seg PRIVATE ldpc_segment ldpc_gen_HEADERS)
-
 add_library(ldpc MODULE
             nrLDPC_decoder/nrLDPC_decoder.c
             nrLDPC_encoder/ldpc_encoder_optim8segmulti.c
@@ -38,13 +24,13 @@ add_library(ldpc MODULE
 set_target_properties(ldpc PROPERTIES LIBRARY_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR})
 target_link_libraries(ldpc PRIVATE ldpc_segment ldpc_gen_HEADERS)
 
-add_dependencies(ldpctest ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr-softmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr-uesoftmodem ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr_ulsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr_ulschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr_dlsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
-add_dependencies(nr_dlschsim ldpc ldpc_orig ldpc_optim ldpc_optim8seg)
+add_dependencies(ldpctest ldpc ldpc_orig)
+add_dependencies(nr-softmodem ldpc ldpc_orig)
+add_dependencies(nr-uesoftmodem ldpc ldpc_orig)
+add_dependencies(nr_ulsim ldpc ldpc_orig)
+add_dependencies(nr_ulschsim ldpc ldpc_orig)
+add_dependencies(nr_dlsim ldpc ldpc_orig)
+add_dependencies(nr_dlschsim ldpc ldpc_orig)
 
 add_custom_target(nrLDPC_decoder_kernels_CL
                   COMMAND gcc nrLDPC_decoder/nrLDPC_decoder_CL.c -dD -DNRLDPC_KERNEL_SOURCE -E -o ${CMAKE_CURRENT_BINARY_DIR}/nrLDPC_decoder_kernels_CL.clc

openair1/PHY/CODING/DOC/LDPCImplementation.md

@@ -120,16 +120,16 @@ By default the function `int load_nrLDPClib(void)` looks for `libldpc.so`, this
 
 #### Examples of ldpc shared lib selection when running nr softmodem's:
 
-loading `libldpc_optim8seg.so` instead of `libldpc.so`:
+loading `libldpc_orig.so` instead of `libldpc.so`:
 
 ```
-./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5  --loader.ldpc.shlibversion _optim8seg
+./nr-softmodem -O libconfig:gnb.band78.tm1.106PRB.usrpx300.conf:dbgl5  --loader.ldpc.shlibversion _orig
 .......................
 [CONFIG] loader.ldpc.shlibversion set to default value ""
 [LIBCONFIG] loader.ldpc: 2/2 parameters successfully set, (1 to default value)
-[CONFIG] shlibversion set to  _optim8seg from command line
+[CONFIG] shlibversion set to  _orig from command line
 [CONFIG] loader.ldpc 1 options set from command line
-[LOADER] library libldpc_optim8seg.so successfully loaded
+[LOADER] library libldpc_orig.so successfully loaded
 ........................
 ```
 

openair1/PHY/CODING/TESTBENCH/ldpctest.c

@@ -123,7 +123,7 @@ one_measurement_t test_ldpc(short max_iterations,
   uint8_t estimated_output[MAX_NUM_DLSCH_SEGMENTS][Kprime];
   memset(estimated_output, 0, sizeof(estimated_output));
   uint8_t *channel_input[MAX_NUM_DLSCH_SEGMENTS];
-  uint8_t *channel_input_optim[MAX_NUM_DLSCH_SEGMENTS];
+  uint8_t *channel_input_optim;
   // double channel_output[68 * 384];
   double modulated_input[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
   int8_t channel_output_fixed[MAX_NUM_DLSCH_SEGMENTS][68 * 384] = {0};
@@ -265,9 +265,9 @@ one_measurement_t test_ldpc(short max_iterations,
     memset(test_input[j], 0, ((K + 7) & ~7) / 8);
     channel_input[j] = malloc16(68 * 384);
     memset(channel_input[j], 0, 68 * 384);
-    channel_input_optim[j] = malloc16(68 * 384);
-    memset(channel_input_optim[j], 0, 68 * 384);
   }
+  channel_input_optim = malloc16(68 * 384);
+  memset(channel_input_optim, 0, 68 * 384);
 
   // Fill input segments with random values
   for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
@@ -282,10 +282,10 @@ one_measurement_t test_ldpc(short max_iterations,
   }
 
   encoder_implemparams_t impp = {.Zc = Zc, .Kb = Kb, .BG = BG, .K = K};
-  impp.gen_code = 2;
+  impp.gen_code = 1;
 
   if (ntrials == 0)
-    ldpc_orig.LDPCencoder(test_input, channel_input, &impp);
+    ldpc_orig.LDPCencoder(test_input, channel_input[0], &impp);
   impp.gen_code = 0;
   decode_abort_t dec_abort;
   init_abort(&dec_abort);
@@ -294,23 +294,21 @@ one_measurement_t test_ldpc(short max_iterations,
     //// encoder
     start_meas(&time);
     for (int j = 0; j < n_segments; j++) {
-      ldpc_orig.LDPCencoder(&test_input[j], &channel_input[j], &impp);
+      ldpc_orig.LDPCencoder(&test_input[j], channel_input[j], &impp);
     }
     stop_meas(&time);
 
     impp.n_segments = n_segments;
-    for (int j = 0; j < (n_segments / 8 + 1); j++) {
-      start_meas(&ret.time_optim);
-      impp.macro_num = j;
-      ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
-      stop_meas(&ret.time_optim);
-    }
+    start_meas(&ret.time_optim);
+    impp.first_seg = 0;
+    ldpc_toCompare.LDPCencoder(test_input, channel_input_optim, &impp);
+    stop_meas(&ret.time_optim);
 
     if (ntrials == 1)
       for (int j = 0; j < n_segments; j++)
         for (int i = 0; i < K + (nrows - no_punctured_columns) * Zc - removed_bit; i++) {
-          if (channel_input[j][i] != channel_input_optim[j][i]) {
-            printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], channel_input_optim[j][i]);
+          if (channel_input[j][i] != ((channel_input_optim[i] >> j) & 0x1)) {
+            printf("differ in seg %d pos %d (%u,%u)\n", j, i, channel_input[j][i], (channel_input_optim[i] >> j) & 0x1);
             return ret;
           }
         }
@@ -322,7 +320,7 @@ one_measurement_t test_ldpc(short max_iterations,
           printf("\ne %d..%d:    ", i, i + 15);
 #endif
 
-        if (channel_input_optim[j][i - 2 * Zc] == 0)
+        if (((channel_input_optim[i - 2 * Zc] >> j) & 0x1) == 0)
           modulated_input[j][i] = 1.0; /// sqrt(2);  //QPSK
         else
           modulated_input[j][i] = -1.0; /// sqrt(2);
@@ -332,7 +330,7 @@ one_measurement_t test_ldpc(short max_iterations,
 
         // Uncoded BER
         uint8_t channel_output_uncoded = channel_output_fixed[j][i] < 0 ? 1 /* QPSK demod */ : 0;
-        if (channel_output_uncoded != channel_input_optim[j][i - 2 * Zc])
+        if (channel_output_uncoded != ((channel_input_optim[i - 2 * Zc] >> j) & 0x1))
           ret.errors_bit_uncoded++;
       }
 
@@ -396,8 +394,8 @@ one_measurement_t test_ldpc(short max_iterations,
   for (int j = 0; j < MAX_NUM_DLSCH_SEGMENTS; j++) {
     free(test_input[j]);
     free(channel_input[j]);
-    free(channel_input_optim[j]);
   }
+  free(channel_input_optim);
 
   print_meas(&time, "ldpc_encoder", NULL, NULL);
   print_meas(&ret.time_optim, "ldpc_encoder_optim", NULL, NULL);

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_interface.h

@@ -140,9 +140,7 @@ typedef struct nrLDPC_slot_decoding_parameters_s{
  * \struct nrLDPC_segment_encoding_parameters_s
  * \brief encoding parameter of segments
  * \var E input llr segment size
- * \var output input llr segment array
  * \var c Pointers to code blocks before LDPC encoding (38.212 V15.4.0 section 5.2.2)
- * flag indicating that the decoding of the segment was successful
  * IT MUST BE FILLED BY THE IMPLEMENTATION
  * \var ts_interleave interleaving time stats
  * \var ts_rate_match rate matching time stats
@@ -150,7 +148,6 @@ typedef struct nrLDPC_slot_decoding_parameters_s{
  */
 typedef struct nrLDPC_segment_encoding_parameters_s{
   int E;
-  unsigned char *output;
   uint8_t *c;
   time_stats_t ts_interleave;
   time_stats_t ts_rate_match;
@@ -179,6 +176,7 @@ typedef struct nrLDPC_segment_encoding_parameters_s{
  * \var F filler bits size
  * \var C number of segments 
  * \var segments array of segments parameters
+ * \var output input llr TB array
  */
 typedef struct nrLDPC_TB_encoding_parameters_s{
 
@@ -202,6 +200,7 @@ typedef struct nrLDPC_TB_encoding_parameters_s{
 
   uint32_t C;
   nrLDPC_segment_encoding_parameters_t *segments;
+  unsigned char *output;
 } nrLDPC_TB_encoding_parameters_t;
 
 /**

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nrLDPC_coding_segment_encoder.c

@@ -41,12 +41,184 @@
 
 #include <syscall.h>
 
-// #define DEBUG_LDPC_ENCODING
-// #define DEBUG_LDPC_ENCODING_FREE 1
+#define DEBUG_LDPC_ENCODING
+//#define DEBUG_LDPC_ENCODING_FREE 1
 
+/**
+ * \brief write nrLDPC_coding_segment_encoder output with interleaver output
+ * the interleaver output has 8 bits from 8 different segments per byte
+ * nrLDPC_coding_segment_encoder is made of concatenated packed segments
+ * \param f input interleaved segment, 8 bits from 8 segments per byte
+ * \param E number of bits per segment in f
+ * \param f2 input interleaved segment, 8 bits from 8 segments per byte
+ * interleaved with E2 if there was a E shift in the segment group
+ * \param E2 number of bits per segment in f2
+ * \param Eshift flag indicating if there was a E shift within the segment group
+ * \param E2_first_segment index of the first segment of size E2 if there is a E shift in the segment group
+ * \param nb_segments number of segments in the group
+ * \param output nrLDPC_coding_segment_encoder with concatenated segments and packed bits
+ * \param Eoffset offset in number of bits of the first segment of the segment group within output
+ */
+static void write_task_output(uint8_t *f,
+                              uint32_t E,
+                              uint8_t *f2,
+                              uint32_t E2,
+                              bool Eshift,
+                              uint32_t E2_first_segment,
+                              uint32_t nb_segments,
+                              uint8_t *output,
+                              uint32_t Eoffset)
+{
+
+#if defined(__AVX512VBMI__)
+  uint64_t *output_p = (uint64_t*)output;
+  __m512i inc = _mm512_set1_epi8(0x1);
+
+  for (int i=0;i<E2;i+=64) {
+    uint32_t Eoffset2 = Eoffset;
+    __m512i bitperm = _mm512_set1_epi64(0x3830282018100800);
+    if (i<E) {
+      for (int j=0; j < E2_first_segment; j++) {
+        // Note: Here and below for AVX2, we are using the 64-bit SIMD instruction
+        // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
+        // and >> operations are undefined and in fact don't give "0" which is
+        // what we want here. The SIMD version do give 0 when the shift is 64
+        uint32_t Eoffset2_byte = Eoffset2 >> 6;
+        uint32_t Eoffset2_bit = Eoffset2 & 63;
+        __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f)[i >> 6],bitperm);
+        *(__m64*)(output_p + Eoffset2_byte)   = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
+        *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
+        Eoffset2 += E;
+        bitperm = _mm512_add_epi8(bitperm ,inc);
+      }
+    } else {
+      for (int j=0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+        bitperm = _mm512_add_epi8(bitperm ,inc);
+      }
+    }
+    for (int j=E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 6;
+      uint32_t Eoffset2_bit = Eoffset2 & 63;
+      __m64 tmp = (__m64)_mm512_bitshuffle_epi64_mask(((__m512i *)f2)[i >> 6],bitperm);
+      *(__m64*)(output_p + Eoffset2_byte)   = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte),_mm_slli_si64(tmp,Eoffset2_bit));
+      *(__m64*)(output_p + Eoffset2_byte+1) = _mm_or_si64(*(__m64*)(output_p + Eoffset2_byte+1),_mm_srli_si64(tmp,(64-Eoffset2_bit)));
+      Eoffset2 += E2;
+      bitperm = _mm512_add_epi8(bitperm ,inc);
+    }
+    output_p++;
+  }
+
+#elif defined(__aarch64__)
+  uint16_t *output_p = (uint16_t*)output;
+  const int8_t __attribute__ ((aligned (16))) ucShift[8][16] = {
+    {0,1,2,3,4,5,6,7,0,1,2,3,4,5,6,7},     // segment 0
+    {-1,0,1,2,3,4,5,6,-1,0,1,2,3,4,5,6},   // segment 1
+    {-2,-1,0,1,2,3,4,5,-2,-1,0,1,2,3,4,5}, // segment 2
+    {-3,-2,-1,0,1,2,3,4,-3,-2,-1,0,1,2,3,4}, // segment 3
+    {-4,-3,-2,-1,0,1,2,3,-4,-3,-2,-1,0,1,2,3}, // segment 4
+    {-5,-4,-3,-2,-1,0,1,2,-5,-4,-3,-2,-1,0,1,2}, // segment 5
+    {-6,-5,-4,-3,-2,-1,0,1,-6,-5,-4,-3,-2,-1,0,1}, // segment 6
+    {-7,-6,-5,-4,-3,-2,-1,0,-7,-6,-5,-4,-3,-2,-1,0}}; // segment 7
+  const uint8_t __attribute__ ((aligned (16))) masks[16] = 
+      {0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80,0x1,0x2,0x4,0x8,0x10,0x20,0x40,0x80};
+  int8x16_t vshift[8];
+  for (int n=0;n<8;n++) vshift[n] = vld1q_s8(ucShift[n]);
+  uint8x16_t vmask  = vld1q_u8(masks);
+
+  for (int i=0;i<E2;i+=16) {
+    uint32_t Eoffset2 = Eoffset;
+    if (i<E) {	
+      for (int j=0; j < E2_first_segment; j++) {
+        uint32_t Eoffset2_byte = Eoffset2 >> 4;
+        uint32_t Eoffset2_bit = Eoffset2 & 15;
+        uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f)[i >> 4],vshift[j]),vmask);
+        int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
+        tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
+        *(output_p + Eoffset2_byte)   |= (uint16_t)(tmp<<Eoffset2_bit);
+        *(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
+        Eoffset2 += E;
+      }
+    } else {
+      for (int j=0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+      }
+    }
+    for (int j=E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 4;
+      uint32_t Eoffset2_bit = Eoffset2 & 15;
+      uint8x16_t cshift = vandq_u8(vshlq_u8(((uint8x16_t*)f2)[i >> 4],vshift[j]),vmask);
+      int32_t tmp = (int)vaddv_u8(vget_low_u8(cshift));
+      tmp += (int)(vaddv_u8(vget_high_u8(cshift))<<8);
+      *(output_p + Eoffset2_byte)   |= (uint16_t)(tmp<<Eoffset2_bit);
+      *(output_p + Eoffset2_byte+1) |= (uint16_t)(tmp>>(16-Eoffset2_bit));
+      Eoffset2 += E2;
+    }
+    output_p++;
+  }
+       
+#else
+  uint32_t *output_p = (uint32_t*)output;
+
+  for (int i=0; i < E2; i += 32) {
+    uint32_t Eoffset2 = Eoffset;
+    if (i < E) {
+      for (int j = 0; j < E2_first_segment; j++) {
+        // Note: Here and below, we are using the 64-bit SIMD instruction
+        // instead of C >>/<< because when the Eoffset2_bit is 64 or 0, the <<
+        // and >> operations are undefined and in fact don't give "0" which is
+        // what we want here. The SIMD version do give 0 when the shift is 64
+        uint32_t Eoffset2_byte = Eoffset2 >> 5;
+        uint32_t Eoffset2_bit = Eoffset2 & 31;
+        int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f)[i >> 5], 7 - j));
+        __m64 tmp64 = _mm_set1_pi32(tmp);
+        __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
+        __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
+        __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
+        *(output_p + Eoffset2_byte) = _m_to_int(tmp64b);
+        *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
+        Eoffset2 += E;
+      }
+    } else {
+      for (int j = 0; j < E2_first_segment; j++) {
+        Eoffset2 += E;
+      }
+    } 
+    for (int j = E2_first_segment; j < nb_segments; j++) {
+      uint32_t Eoffset2_byte = Eoffset2 >> 5;
+      uint32_t Eoffset2_bit = Eoffset2 & 31;
+      int tmp = _mm256_movemask_epi8(_mm256_slli_epi16(((__m256i *)f2)[i >> 5], 7 - j));
+      __m64 tmp64 = _mm_set1_pi32(tmp);
+      __m64 out64 = _mm_set_pi32(*(output_p + Eoffset2_byte + 1), *(output_p + Eoffset2_byte));
+      __m64 tmp64b = _mm_or_si64(out64, _mm_slli_pi32(tmp64, Eoffset2_bit));
+      __m64 tmp64c = _mm_or_si64(out64, _mm_srli_pi32(tmp64, (32 - Eoffset2_bit)));
+      *(output_p + Eoffset2_byte)  = _m_to_int(tmp64b);
+      *(output_p + Eoffset2_byte + 1) = _m_to_int(_mm_srli_si64(tmp64c, 32));
+      Eoffset2 += E2;
+    }
+    output_p++;
+  }
+
+  
+#endif
+}
+
+/**
+ * \typedef ldpc8blocks_args_t
+ * \struct ldpc8blocks_args_s
+ * \brief Arguments of an encoding task
+ * encode up to 8 code blocks
+ * \var nrLDPC_TB_encoding_parameters TB encoding parameters as defined in the coding library interface
+ * \var impp encoder implementation specific parameters for the task
+ * \var f first interleaver output to be filled by the task
+ * \var f2 second interleaver output to be filled by the task
+ * in case of a shift of E in the code blocks group processed by the task
+ */
 typedef struct ldpc8blocks_args_s {
   nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters;
   encoder_implemparams_t impp;
+  uint8_t *f;
+  uint8_t *f2;
 } ldpc8blocks_args_t;
 
 static void ldpc8blocks(void *p)
@@ -64,156 +236,214 @@ static void ldpc8blocks(void *p)
 
   // nrLDPC_encoder output is in "d"
   // let's make this interface happy!
-  uint8_t tmp[8][68 * 384] __attribute__((aligned(32)));
-  uint8_t *d[impp->n_segments];
-  for (int rr = impp->macro_num * 8, i = 0; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++, i++)
-    d[rr] = tmp[i];
-
+  uint8_t d[68 * 384] __attribute__((aligned(64)));
   uint8_t *c[nrLDPC_TB_encoding_parameters->C];
+  unsigned int macro_segment, macro_segment_end;
+
+  
+  macro_segment = impp->first_seg;
+  macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
   for (int r = 0; r < nrLDPC_TB_encoding_parameters->C; r++)
     c[r] = nrLDPC_TB_encoding_parameters->segments[r].c;
-  start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
   LDPCencoder(c, d, impp);
-  stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->macro_num * 8].ts_ldpc_encode);
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[impp->first_seg].ts_ldpc_encode);
   // Compute where to place in output buffer that is concatenation of all segments
 
-  uint32_t r_offset = 0;
-  for (int i = 0; i < impp->macro_num * 8; i++)
-    r_offset += nrLDPC_TB_encoding_parameters->segments[i].E;
-  for (int rr = impp->macro_num * 8; rr < impp->n_segments && rr < (impp->macro_num + 1) * 8; rr++) {
-    if (impp->F > 0) {
-      // writing into positions d[r][k-2Zc] as in clause 5.3.2 step 2) in 38.212
-      memset(&d[rr][impp->K - impp->F - 2 * impp->Zc], NR_NULL, impp->F);
-    }
-
 #ifdef DEBUG_LDPC_ENCODING
-    LOG_D(PHY, "rvidx in encoding = %d\n", rel15->rvIndex[0]);
+  LOG_D(PHY, "rvidx in encoding = %d\n", nrLDPC_TB_encoding_parameters->rv_index);
 #endif
-    uint32_t E = nrLDPC_TB_encoding_parameters->segments[rr].E;
-    LOG_D(NR_PHY,
-          "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+  const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E;
+  uint32_t E2=E;
+  bool Eshift=false;
+  uint32_t Emax = E;
+  for (int s=macro_segment;s<macro_segment_end;s++)
+      if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
+	 E2=nrLDPC_TB_encoding_parameters->segments[s].E;
+         Eshift=true;
+         if(E2 > Emax)
+           Emax = E2;
+         break;
+      }	 
+    
+
+  LOG_D(NR_PHY,
+        "Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
           "%d,nrOfLayer %d)...\n",
-          rr,
+        macro_segment,
+        macro_segment_end,
+        impp->n_segments,
+        G,
+        E,E2,
+        impp->F,
+        impp->K - impp->F - 2 * impp->Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+/*
+  printf("Rate Matching, Code segment %d...%d/%d (coded bits (G) %u, E %d, E2 %d Filler bits %d, Filler offset %d mod_order %d, nb_rb "
+          "%d,nrOfLayer %d)...\n",
+        macro_segment,
+        macro_segment_end,
+        impp->n_segments,
+        G,
+        E,E2,
+        impp->F,
+        impp->K - impp->F - 2 * impp->Zc,
+        mod_order,
+        nb_rb,
+        nrLDPC_TB_encoding_parameters->nb_layers);
+*/
+  uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
+
+  uint8_t e[Emax] __attribute__((aligned(64)));
+  // Interleaver outputs are stored in the output arrays
+  uint8_t *f = args->f;
+  uint8_t *f2 = args->f2;
+  bzero(e, Emax);
+  bzero(f, ceil_mod(E, 64));
+  if (Eshift) {
+    bzero(f2, ceil_mod(E2, 64));
+  }
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
+  nr_rate_matching_ldpc(Tbslbrm,
+                        impp->BG,
+                        impp->Zc,
+                        d,
+                        e,
+                        impp->n_segments,
+                        impp->F,
+                        impp->K - impp->F - 2 * impp->Zc,
+                        nrLDPC_TB_encoding_parameters->rv_index,
+                        Emax);
+
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_rate_match);
+  if (impp->K - impp->F - 2 * impp->Zc > E) {
+    LOG_E(PHY,
+          "dlsch coding A %d  Kr %d G %d (nb_rb %d, mod_order %d)\n",
+          A,
+          impp->K,
+          G,
+          nb_rb,
+          (int)mod_order);
+
+    LOG_E(NR_PHY,
+          "Rate Matching, Code segments %d...%d/%d (coded bits (G) %u, E %d, Kr %d, Filler bits %d, Filler offset %d mod_order %d, "
+          "nb_rb %d)...\n",
+          macro_segment,
+	  macro_segment_end,
           impp->n_segments,
           G,
           E,
+          impp->K,
           impp->F,
           impp->K - impp->F - 2 * impp->Zc,
           mod_order,
-          nb_rb,
-          nrLDPC_TB_encoding_parameters->nb_layers);
-
-    uint32_t Tbslbrm = nrLDPC_TB_encoding_parameters->tbslbrm;
-
-    uint8_t e[E];
-    bzero(e, E);
-    start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match);
-    nr_rate_matching_ldpc(Tbslbrm,
-                          impp->BG,
-                          impp->Zc,
-                          d[rr],
-                          e,
-                          impp->n_segments,
-                          impp->F,
-                          impp->K - impp->F - 2 * impp->Zc,
-                          nrLDPC_TB_encoding_parameters->rv_index,
-                          E);
-    stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_rate_match);
-    if (impp->K - impp->F - 2 * impp->Zc > E) {
-      LOG_E(PHY, "dlsch coding A %d K %d G %d (nb_rb %d, mod_order %d)\n", A, impp->K, G, nb_rb, (int)mod_order);
-
-      LOG_E(NR_PHY,
-            "Rate Matching, Code segment %d/%d (coded bits (G) %u, E %d, K %d, Filler bits %d, Filler offset %d mod_order %d, "
-            "nb_rb %d)...\n",
-            rr,
-            impp->n_segments,
-            G,
-            E,
-            impp->K,
-            impp->F,
-            impp->K - impp->F - 2 * impp->Zc,
-            mod_order,
-            nb_rb);
-    }
-#ifdef DEBUG_LDPC_ENCODING
-
-    for (int i = 0; i < 16; i++)
-      printf("output ratematching e[%d]= %d r_offset %u\n", i, e[i], r_offset);
-
-#endif
-    start_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave);
-    nr_interleaving_ldpc(E, mod_order, e, impp->output + r_offset);
-    stop_meas(&nrLDPC_TB_encoding_parameters->segments[rr].ts_interleave);
-#ifdef DEBUG_LDPC_ENCODING
-
-    for (int i = 0; i < 16; i++)
-      printf("output interleaving f[%d]= %d r_offset %u\n", i, impp->output[i + r_offset], r_offset);
-
-    if (r == impp->n_segments - 1)
-      write_output("enc_output.m", "enc", impp->output, G, 1, 4);
-
-#endif
-    r_offset += E;
+          nb_rb);
   }
+  start_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave);
+  nr_interleaving_ldpc(E,
+                       mod_order,
+                       e,
+                       f);
+  if (Eshift)
+    nr_interleaving_ldpc(E2,
+                         mod_order,
+                         e,
+                         f2);
+  stop_meas(&nrLDPC_TB_encoding_parameters->segments[macro_segment].ts_interleave);
 
-  // Task running in // completed
   completed_task_ans(impp->ans);
 }
 
-static int nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
+static int nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters,
                                       int dlsch_id,
-                                      thread_info_tm_t *t_info)
+                                      thread_info_tm_t *t_info,
+                                      uint8_t **f,
+                                      uint8_t **f2)
 {
   nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
 
-  encoder_implemparams_t impp = {0};
+  encoder_implemparams_t common_segment_params = {
+    .n_segments = nrLDPC_TB_encoding_parameters->C,
+    .tinput = nrLDPC_slot_encoding_parameters->tinput,
+    .tprep = nrLDPC_slot_encoding_parameters->tprep,
+    .tparity = nrLDPC_slot_encoding_parameters->tparity,
+    .toutput = nrLDPC_slot_encoding_parameters->toutput,
+    .Kb = nrLDPC_TB_encoding_parameters->Kb,
+    .Zc = nrLDPC_TB_encoding_parameters->Z,
+    .BG = nrLDPC_TB_encoding_parameters->BG,
+    .output = nrLDPC_TB_encoding_parameters->output, 
+    .K = nrLDPC_TB_encoding_parameters->K,
+    .F = nrLDPC_TB_encoding_parameters->F,
+  };
 
-  impp.n_segments = nrLDPC_TB_encoding_parameters->C;
-  impp.tinput = nrLDPC_slot_encoding_parameters->tinput;
-  impp.tprep = nrLDPC_slot_encoding_parameters->tprep;
-  impp.tparity = nrLDPC_slot_encoding_parameters->tparity;
-  impp.toutput = nrLDPC_slot_encoding_parameters->toutput;
-  impp.Kb = nrLDPC_TB_encoding_parameters->Kb;
-  impp.Zc = nrLDPC_TB_encoding_parameters->Z;
-  impp.BG = nrLDPC_TB_encoding_parameters->BG;
-  impp.output = nrLDPC_TB_encoding_parameters->segments->output;
-  impp.K = nrLDPC_TB_encoding_parameters->K;
-  impp.F = nrLDPC_TB_encoding_parameters->F;
+  const size_t n_tasks = (common_segment_params.n_segments + 7) >> 3;
 
-  size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1));
-
-  for (int j = 0; j < n_seg; j++) {
+  for (int j = 0; j < n_tasks; j++) {
     ldpc8blocks_args_t *perJobImpp = &((ldpc8blocks_args_t *)t_info->buf)[t_info->len];
     DevAssert(t_info->len < t_info->cap);
-    impp.ans = t_info->ans;
     t_info->len += 1;
 
-    impp.macro_num = j;
-    perJobImpp->impp = impp;
+    perJobImpp->impp = common_segment_params;
+    perJobImpp->impp.first_seg = j * 8;
+    perJobImpp->impp.ans = t_info->ans;
     perJobImpp->nrLDPC_TB_encoding_parameters = nrLDPC_TB_encoding_parameters;
+    perJobImpp->f = f[j];
+    perJobImpp->f2 = f2[j];
 
     task_t t = {.func = ldpc8blocks, .args = perJobImpp};
     pushTpool(nrLDPC_slot_encoding_parameters->threadPool, t);
   }
-  return n_seg;
+  return n_tasks;
 }
 
 int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
 {
   int nbTasks = 0;
+
+  uint32_t Emax = 0;
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
+    // Compute number of tasks to encode TB
     nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
     size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
     nbTasks += n_seg;
+
+    // Search for maximum E for sizing encoder output f and f2
+    for (int seg_id = 0; seg_id < nrLDPC_TB_encoding_parameters->C; seg_id++) {
+      uint32_t E = nrLDPC_TB_encoding_parameters->segments[seg_id].E;
+      Emax = E > Emax ? E : Emax;
+    }
   }
+
+  // Create f and f2 to old encoding tasks outputs
+  uint32_t Emax_ceil_mod = ceil_mod(Emax, 64);
+  uint8_t f[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
+  uint8_t f2[nbTasks][Emax_ceil_mod] __attribute__((aligned(64)));
+
   ldpc8blocks_args_t arr[nbTasks];
   task_ans_t ans;
   init_task_ans(&ans, nbTasks);
   thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = nbTasks, .ans = &ans};
 
   int nbEncode = 0;
+  nbTasks = 0;
   for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
-    nbEncode += nrLDPC_prepare_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info);
+    // For easier indexing we store the pointers to sub arrays of f and f2 in pointer arrays
+    // Then a function to which we pass the pointer arrays can directly use f_2d[j] ans f2_2d[j]
+    nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
+    size_t n_seg = (nrLDPC_TB_encoding_parameters->C / 8 + ((nrLDPC_TB_encoding_parameters->C & 7) == 0 ? 0 : 1));
+    uint8_t *f_2d[n_seg];
+    uint8_t *f2_2d[n_seg];
+    for (int j = 0; j < n_seg; j++) {
+      f_2d[j] = &f[nbTasks + j][0];
+      f2_2d[j] = &f2[nbTasks + j][0];
+    }
+
+    // Launch tasks encoding TB
+    nbEncode += nrLDPC_launch_TB_encoding(nrLDPC_slot_encoding_parameters, dlsch_id, &t_info, f_2d, f2_2d);
+
+    nbTasks += n_seg;
   }
   if (nbEncode < nbTasks) {
     completed_many_task_ans(&ans, nbTasks - nbEncode);
@@ -221,5 +451,53 @@ int nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encodin
   // Execute thread pool tasks
   join_task_ans(&ans);
 
+  // Write output
+  nbTasks = 0;
+  for (int dlsch_id = 0; dlsch_id < nrLDPC_slot_encoding_parameters->nb_TBs; dlsch_id++) {
+    nrLDPC_TB_encoding_parameters_t *nrLDPC_TB_encoding_parameters = &nrLDPC_slot_encoding_parameters->TBs[dlsch_id];
+    uint32_t C = nrLDPC_TB_encoding_parameters->C;
+    size_t n_seg = (C / 8 + ((C & 7) == 0 ? 0 : 1));
+
+    time_stats_t *toutput = nrLDPC_slot_encoding_parameters->toutput;
+
+    for (int j = 0; j < n_seg; j++) {
+      unsigned int macro_segment = j * 8;
+      unsigned int macro_segment_end = (C > macro_segment + 8) ? macro_segment + 8 : C;
+      const uint32_t E = nrLDPC_TB_encoding_parameters->segments[macro_segment].E;
+      uint32_t E2 = E, E2_first_segment = macro_segment_end - macro_segment;
+      bool Eshift = false;
+      uint32_t Emax = E;
+      for (int s = macro_segment; s < macro_segment_end; s++) {
+        if (nrLDPC_TB_encoding_parameters->segments[s].E != E) {
+           E2 = nrLDPC_TB_encoding_parameters->segments[s].E;
+           Eshift = true;
+           E2_first_segment = s-macro_segment;
+           if(E2 > Emax)
+             Emax = E2;
+           break;
+        }
+      }
+
+      if(toutput != NULL) start_meas(toutput);
+ 
+      uint32_t Eoffset=0;
+      for (int s=0; s<macro_segment; s++)
+        Eoffset += (nrLDPC_TB_encoding_parameters->segments[s].E); 
+
+      write_task_output(&f[nbTasks + j][0],
+                        E,
+                        &f2[nbTasks + j][0],
+                        E2,
+                        Eshift,
+                        E2_first_segment,
+                        macro_segment_end - macro_segment,
+                        nrLDPC_TB_encoding_parameters->output,
+                        Eoffset);
+
+      if(toutput != NULL) stop_meas(toutput);
+    }
+    nbTasks += n_seg;
+  }
+
   return 0;
 }

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.c

@@ -29,274 +29,419 @@ static const uint8_t index_k0[2][4] = {{0, 17, 33, 56}, {0, 13, 25, 43}};
 
 void nr_interleaving_ldpc(uint32_t E, uint8_t Qm, uint8_t *e, uint8_t *f)
 {
-  uint32_t EQm;
-
-  EQm = E / Qm;
+  const uint32_t EQm = E / Qm;
   memset(f, 0, E * sizeof(uint8_t));
-  uint8_t *e0, *e1, *e2, *e3, *e4, *e5, *e6, *e7;
-  uint8_t *fp;
-#if 0 // def __WASAVX2__
-  simde__m256i tmp0,tmp1,tmp2,tmp0b,tmp1b,tmp3,tmp4,tmp5;
-  simde__m256i *e0_256,*e1_256,*e2_256,*e3_256,*e4_256,*e5_256,*e6_256,*e7_256;
-
-  simde__m256i *f_256=(simde__m256i *)f;
-
-  uint8_t *fp2;
   switch(Qm) {
-  case 2:
-    e0=e;
-    e1=e0+EQm;
-    e0_256=(simde__m256i *)e0;
-    e1_256=(simde__m256i *)e1;
-    for (int k=0,j=0;j<EQm>>5;j++,k+=2) {
-      f_256[k]   = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]);
-      f_256[k+1] = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); 
-    }
-    break;
-  case 4:
-    e0=e;
-    e1=e0+EQm;
-    e2=e1+EQm;
-    e3=e2+EQm;
-    e0_256=(simde__m256i *)e0;
-    e1_256=(simde__m256i *)e1;
-    e2_256=(simde__m256i *)e2;
-    e3_256=(simde__m256i *)e3;
-    for (int k=0,j=0;j<EQm>>5;j++,k+=4) {
-      tmp0   = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
-      tmp1   = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
-      f_256[k]   = simde_mm256_unpacklo_epi8(tmp0,tmp1);   // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
-      f_256[k+1] = simde_mm256_unpackhi_epi8(tmp0,tmp1);   // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
-      tmp0   = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
-      tmp1   = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
-      f_256[k+2] = simde_mm256_unpacklo_epi8(tmp0,tmp1);
-      f_256[k+3] = simde_mm256_unpackhi_epi8(tmp0,tmp1); 
-    }
-    break;
-  case 6:
-    e0=e;
-    e1=e0+EQm;
-    e2=e1+EQm;
-    e3=e2+EQm;
-    e4=e3+EQm;
-    e5=e4+EQm;
-    e0_256=(simde__m256i *)e0;
-    e1_256=(simde__m256i *)e1;
-    e2_256=(simde__m256i *)e2;
-    e3_256=(simde__m256i *)e3;
-    e4_256=(simde__m256i *)e4;
-    e5_256=(simde__m256i *)e5;
-
-    for (int j=0,k=0;j<EQm>>5;j++,k+=192) {
-      fp  = f+k;
-      fp2 = fp+96;
-
-      tmp0   = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
-      tmp1   = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
-      tmp0b  = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
-      tmp1b  = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
-      tmp0   = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
-      *((uint32_t*)fp)      = simde_mm256_extract_epi32(tmp0b,0);
-      *((uint16_t*)(fp+4))  = simde_mm256_extract_epi16(tmp0,0);
-      *((uint32_t*)(fp+6))  = simde_mm256_extract_epi32(tmp0b,1);
-      *((uint16_t*)(fp+10)) = simde_mm256_extract_epi16(tmp0,1);
-      *((uint32_t*)(fp+12)) = simde_mm256_extract_epi32(tmp0b,2);
-      *((uint16_t*)(fp+16)) = simde_mm256_extract_epi16(tmp0,2);
-      *((uint32_t*)(fp+18)) = simde_mm256_extract_epi32(tmp0b,3);
-      *((uint16_t*)(fp+22)) = simde_mm256_extract_epi16(tmp0,3);
-      *((uint32_t*)(fp+24)) = simde_mm256_extract_epi32(tmp0b,4);
-      *((uint16_t*)(fp+26)) = simde_mm256_extract_epi16(tmp0,4);
-      *((uint32_t*)(fp+30)) = simde_mm256_extract_epi32(tmp0b,5);
-      *((uint16_t*)(fp+34)) = simde_mm256_extract_epi16(tmp0,5);
-      *((uint32_t*)(fp+36)) = simde_mm256_extract_epi32(tmp0,6);
-      *((uint16_t*)(fp+40)) = simde_mm256_extract_epi16(tmp0,6);
-      *((uint32_t*)(fp+42)) = simde_mm256_extract_epi32(tmp0b,7);
-      *((uint16_t*)(fp+46)) = simde_mm256_extract_epi16(tmp0,7);
-
-      *((uint32_t*)(fp+48)) = simde_mm256_extract_epi32(tmp1b,0);
-      *((uint16_t*)(fp+52)) = simde_mm256_extract_epi16(tmp0,8);
-      *((uint32_t*)(fp+56)) = simde_mm256_extract_epi32(tmp1b,1);
-      *((uint16_t*)(fp+60)) = simde_mm256_extract_epi16(tmp0,9);
-      *((uint32_t*)(fp+62)) = simde_mm256_extract_epi32(tmp1b,2);
-      *((uint16_t*)(fp+66)) = simde_mm256_extract_epi16(tmp0,10);
-      *((uint32_t*)(fp+68)) = simde_mm256_extract_epi32(tmp1b,3);
-      *((uint16_t*)(fp+72)) = simde_mm256_extract_epi16(tmp0,11);
-      *((uint32_t*)(fp+74)) = simde_mm256_extract_epi32(tmp1b,4);
-      *((uint16_t*)(fp+76)) = simde_mm256_extract_epi16(tmp0,12);
-      *((uint32_t*)(fp+80)) = simde_mm256_extract_epi32(tmp1b,5);
-      *((uint16_t*)(fp+82)) = simde_mm256_extract_epi16(tmp0,13);
-      *((uint32_t*)(fp+86)) = simde_mm256_extract_epi32(tmp1b,6);
-      *((uint16_t*)(fp+90)) = simde_mm256_extract_epi16(tmp0,14);
-      *((uint32_t*)(fp+92)) = simde_mm256_extract_epi32(tmp1b,7);
-      *((uint16_t*)(fp+94)) = simde_mm256_extract_epi16(tmp0,15);
-
-      tmp0   = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
-      tmp1   = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
-      tmp0b  = simde_mm256_unpacklo_epi16(tmp0,tmp1); // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
-      tmp1b  = simde_mm256_unpackhi_epi16(tmp0,tmp1); // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
-      tmp0   = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
-      *((uint32_t*)fp2)      = simde_mm256_extract_epi32(tmp0b,0);
-      *((uint16_t*)(fp2+4))  = simde_mm256_extract_epi16(tmp0,0);
-      *((uint32_t*)(fp2+6))  = simde_mm256_extract_epi32(tmp0b,1);
-      *((uint16_t*)(fp2+10)) = simde_mm256_extract_epi16(tmp0,1);
-      *((uint32_t*)(fp2+12)) = simde_mm256_extract_epi32(tmp0b,2);
-      *((uint16_t*)(fp2+16)) = simde_mm256_extract_epi16(tmp0,2);
-      *((uint32_t*)(fp2+18)) = simde_mm256_extract_epi32(tmp0b,3);
-      *((uint16_t*)(fp2+22)) = simde_mm256_extract_epi16(tmp0,3);
-      *((uint32_t*)(fp2+24)) = simde_mm256_extract_epi32(tmp0b,4);
-      *((uint16_t*)(fp2+26)) = simde_mm256_extract_epi16(tmp0,4);
-      *((uint32_t*)(fp2+30)) = simde_mm256_extract_epi32(tmp0b,5);
-      *((uint16_t*)(fp2+34)) = simde_mm256_extract_epi16(tmp0,5);
-      *((uint32_t*)(fp2+36)) = simde_mm256_extract_epi32(tmp0,6);
-      *((uint16_t*)(fp2+40)) = simde_mm256_extract_epi16(tmp0,6);
-      *((uint32_t*)(fp2+42)) = simde_mm256_extract_epi32(tmp0b,7);
-      *((uint16_t*)(fp2+46)) = simde_mm256_extract_epi16(tmp0,7);
-
-      *((uint32_t*)(fp2+48)) = simde_mm256_extract_epi32(tmp1b,0);
-      *((uint16_t*)(fp2+52)) = simde_mm256_extract_epi16(tmp0,8);
-      *((uint32_t*)(fp2+56)) = simde_mm256_extract_epi32(tmp1b,1);
-      *((uint16_t*)(fp2+60)) = simde_mm256_extract_epi16(tmp0,9);
-      *((uint32_t*)(fp2+62)) = simde_mm256_extract_epi32(tmp1b,2);
-      *((uint16_t*)(fp2+66)) = simde_mm256_extract_epi16(tmp0,10);
-      *((uint32_t*)(fp2+68)) = simde_mm256_extract_epi32(tmp1b,3);
-      *((uint16_t*)(fp2+72)) = simde_mm256_extract_epi16(tmp0,11);
-      *((uint32_t*)(fp2+74)) = simde_mm256_extract_epi32(tmp1b,4);
-      *((uint16_t*)(fp2+76)) = simde_mm256_extract_epi16(tmp0,12);
-      *((uint32_t*)(fp2+80)) = simde_mm256_extract_epi32(tmp1b,5);
-      *((uint16_t*)(fp2+82)) = simde_mm256_extract_epi16(tmp0,13);
-      *((uint32_t*)(fp2+86)) = simde_mm256_extract_epi32(tmp1b,6);
-      *((uint16_t*)(fp2+90)) = simde_mm256_extract_epi16(tmp0,14);
-      *((uint32_t*)(fp2+92)) = simde_mm256_extract_epi32(tmp1b,7);
-      *((uint16_t*)(fp2+94)) = simde_mm256_extract_epi16(tmp0,15);
-    }
-    break;
-  case 8:
-    e0=e;
-    e1=e0+EQm;
-    e2=e1+EQm;
-    e3=e2+EQm;
-    e4=e3+EQm;
-    e5=e4+EQm;
-    e6=e5+EQm;
-    e7=e6+EQm;
-
-    e0_256=(simde__m256i *)e0;
-    e1_256=(simde__m256i *)e1;
-    e2_256=(simde__m256i *)e2;
-    e3_256=(simde__m256i *)e3;
-    e4_256=(simde__m256i *)e4;
-    e5_256=(simde__m256i *)e5;
-    e6_256=(simde__m256i *)e6;
-    e7_256=(simde__m256i *)e7;
-    for (int k=0,j=0;j<EQm>>5;j++,k+=8) {
-      tmp0   = simde_mm256_unpacklo_epi8(e0_256[j],e1_256[j]); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
-      tmp1   = simde_mm256_unpacklo_epi8(e2_256[j],e3_256[j]); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
-      tmp2   = simde_mm256_unpacklo_epi8(e4_256[j],e5_256[j]); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
-      tmp3   = simde_mm256_unpacklo_epi8(e6_256[j],e7_256[j]); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
-      tmp4   = simde_mm256_unpacklo_epi16(tmp0,tmp1);  // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
-      tmp5   = simde_mm256_unpacklo_epi16(tmp2,tmp3);  // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
-      f_256[k]   = simde_mm256_unpacklo_epi16(tmp4,tmp5);  // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3) e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
-      f_256[k+1] = simde_mm256_unpackhi_epi16(tmp4,tmp5);  // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)... e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
-
-      tmp4   = simde_mm256_unpackhi_epi16(tmp0,tmp1);  // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
-      tmp5   = simde_mm256_unpackhi_epi16(tmp2,tmp3);  // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
-      f_256[k+2]   = simde_mm256_unpacklo_epi16(tmp4,tmp5);  // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11) e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
-      f_256[k+3] = simde_mm256_unpackhi_epi16(tmp4,tmp5);  // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
-
-      tmp0   = simde_mm256_unpackhi_epi8(e0_256[j],e1_256[j]); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
-      tmp1   = simde_mm256_unpackhi_epi8(e2_256[j],e3_256[j]); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
-      tmp2   = simde_mm256_unpackhi_epi8(e4_256[j],e5_256[j]); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
-      tmp3   = simde_mm256_unpackhi_epi8(e6_256[j],e7_256[j]); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
-      tmp4   = simde_mm256_unpacklo_epi16(tmp0,tmp1);  // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
-      tmp5   = simde_mm256_unpacklo_epi16(tmp2,tmp3);  // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
-      f_256[k+4] = simde_mm256_unpacklo_epi16(tmp4,tmp5);  // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19) e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
-      f_256[k+5] = simde_mm256_unpackhi_epi16(tmp4,tmp5);  // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23) e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
-
-      tmp4   = simde_mm256_unpackhi_epi16(tmp0,tmp1);  // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
-      tmp5   = simde_mm256_unpackhi_epi16(tmp2,tmp3);  // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
-      f_256[k+6] = simde_mm256_unpacklo_epi16(tmp4,tmp5);  // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27) e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
-      f_256[k+7] = simde_mm256_unpackhi_epi16(tmp4,tmp5);  // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
-    }
-    break;
-  default: AssertFatal(1==0,"Should be here!\n");
-  }
-
-#else
-  // original unoptimized loops
-  /*
-  for (int j = 0; j< EQm; j++,j2+=2){
-    for (int i = 0; i< Qm; i++){
-    f[(i+j*Qm)] = e[(i*EQm + j)];
-  }
-  }
-  */
-
-  fp = f;
-  switch (Qm) {
-    case 2:
-      e0 = e;
-      e1 = e0 + EQm;
-      for (int j = 0, j2 = 0; j < EQm; j++, j2 += 2) {
-        fp = &f[j2];
-        fp[0] = e0[j];
-        fp[1] = e1[j];
+    case 2: {
+      uint8_t *e0 = e;
+      uint8_t *e1 = e0 + EQm;
+      int i = 0;
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24,
+                                          87, 23, 86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16,
+                                          79, 15, 78, 14, 77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8,
+                                          71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120,
+                                          56, 119, 55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112,
+                                          48, 111, 47, 110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104,
+                                          40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      // clang-format on
+      __m512i *f_512 = (__m512i *)f;
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8a, e1j));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi8(e0j, p8b, e1j));
       }
-      break;
-    case 4:
-      e0 = e;
-      e1 = e0 + EQm;
-      e2 = e1 + EQm;
-      e3 = e2 + EQm;
-      for (int j = 0, j2 = 0; j < EQm; j++, j2 += 4) {
-        fp = &f[j2];
-        fp[0] = e0[j];
-        fp[1] = e1[j];
-        fp[2] = e2[j];
-        fp[3] = e3[j];
-      }
-      break;
-    case 6:
-      e0 = e;
-      e1 = e0 + EQm;
-      e2 = e1 + EQm;
-      e3 = e2 + EQm;
-      e4 = e3 + EQm;
-      e5 = e4 + EQm;
-      fp = f;
-      for (int j = 0; j < EQm; j++) {
-        *fp++ = e0[j];
-        *fp++ = e1[j];
-        *fp++ = e2[j];
-        *fp++ = e3[j];
-        *fp++ = e4[j];
-        *fp++ = e5[j];
-      }
-      break;
-    case 8:
-      e0 = e;
-      e1 = e0 + EQm;
-      e2 = e1 + EQm;
-      e3 = e2 + EQm;
-      e4 = e3 + EQm;
-      e5 = e4 + EQm;
-      e6 = e5 + EQm;
-      e7 = e6 + EQm;
-      for (int j = 0, j2 = 0; j < EQm; j++, j2 += 8) {
-        fp = &f[j2];
-        fp[0] = e0[j];
-        fp[1] = e1[j];
-        fp[2] = e2[j];
-        fp[3] = e3[j];
-        fp[4] = e4[j];
-        fp[5] = e5[j];
-        fp[6] = e6[j];
-        fp[7] = e7[j];
-      }
-      break;
-    default:
-      AssertFatal(1 == 0, "Should never be here!\n");
-  }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      f = (uint8_t *)f_512;
 #endif
+#ifdef USE128BIT
+      simde__m128i *f_128 = (simde__m128i *)f;
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      for (; i < (EQm & ~15); i += 64) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi8(e0j, e1j));
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi8(e0j, e1j));
+      }
+      e0 = (uint8_t *)e0_128;
+      e1 = (uint8_t *)e1_128;
+      f = (uint8_t *)f_128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+      }
+    } break;
+    case 4: {
+      uint8_t *e0 = e;
+      uint8_t *e1 = e0 + EQm;
+      uint8_t *e2 = e1 + EQm;
+      uint8_t *e3 = e2 + EQm;
+      int i = 0;
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23, 86,
+                                          22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
+                                          76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
+                                          55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47,
+                                          110, 46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102,
+                                          38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6,
+                                            37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54,
+                                            22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16a, tmp1));
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp0, p16b, tmp1));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      f = (uint8_t *)f_512;
+#endif
+#ifdef USE128BIT
+      simde__m128i *e0_128 = (simde__m128i *)e0;
+      simde__m128i *e1_128 = (simde__m128i *)e1;
+      simde__m128i *e2_128 = (simde__m128i *)e2;
+      simde__m128i *e3_128 = (simde__m128i *)e3;
+      simde__m128i *f_128 = (simde__m128i *)f;
+      for (; i < (EQm & ~15); i += 16) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
+        simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
+        simde_mm_storeu_si128(f_128++,
+                              simde_mm_unpacklo_epi16(tmp0, tmp1)); // e0(i) e1(i) e2(i) e3(i) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        simde_mm_storeu_si128(
+            f_128++,
+            simde_mm_unpackhi_epi16(tmp0, tmp1)); // e0(i+4) e1(i+4) e2(i+4) e3(i+4) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
+        tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+9) e2(i+10) e3(i+10) .... e2(i+31) e3(i+31)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi16(tmp0, tmp1));
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi16(tmp0, tmp1));
+      }
+      e0 = (uint8_t *)e0_128;
+      e1 = (uint8_t *)e1_128;
+      e2 = (uint8_t *)e2_128;
+      e3 = (uint8_t *)e3_128;
+      f = (uint8_t *)f_128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+      }
+    } break;
+    case 6: {
+      uint8_t *e0 = e;
+      uint8_t *e1 = e0 + EQm;
+      uint8_t *e2 = e1 + EQm;
+      uint8_t *e3 = e2 + EQm;
+      uint8_t *e4 = e3 + EQm;
+      uint8_t *e5 = e4 + EQm;
+      int i = 0;
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
+                                          86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14,
+                                          77, 13, 76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119,
+                                          55, 118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110,
+                                          46, 109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+      const __m512i p16c = _mm512_set_epi16(21, 20, 41, 19, 18, 40, 17, 16, 39, 15, 14, 38, 13, 12, 37, 11, 10, 36, 9, 8, 35, 7, 6, 34, 5, 4, 33, 3, 2, 32, 1, 0);
+      const __m512i p16d = _mm512_set_epi16(10, 52, 9, 8, 51, 7, 6, 50, 5, 4, 49, 3, 2, 48, 1, 0, 47, 31, 30, 46, 29, 28, 45, 27, 26, 44, 25, 24, 43, 23, 22, 42);
+      const __m512i p16d2 = _mm512_set_epi16(32 + 10, 30, 32 + 9, 32 + 8, 27, 32 + 7, 32 + 6, 24, 32 + 5, 32 + 4, 21, 32 + 3, 32 + 2, 18, 32 + 1, 32 + 0,
+                                             15, 14, 13, 12, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1, 0);
+      const __m512i p16e = _mm512_set_epi16(63, 31, 30, 62, 29, 28, 61, 27, 26, 60, 25, 24, 59, 23, 22, 58, 21, 20, 57, 19, 18, 56, 17, 16, 55, 15, 14, 54, 13, 12, 53, 11);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *e4_512 = (__m512i *)e4;
+      __m512i *e5_512 = (__m512i *)e5;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i e4j = _mm512_loadu_si512(e4_512++);
+        __m512i e5j = _mm512_loadu_si512(e5_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+31) e1(i+31)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+31) e3(i+31)
+        __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+31) e5(i+31)
+        // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        __m512i tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15) e4(i+15)
+        // e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
+        __m512i tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16)  e0(i+20)
+        // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
+        // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
+
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+32) e1(i+32) e0(i+32) e1(i+32) .... e0(i+63) e1(i+63)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+32) e3(i+32) e2(i+32) e3(i+32) .... e2(i+63) e3(i+63)
+        tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+32) e5(i+32) e4(i+32) e5(i+32) .... e4(i+63) e5(i+63)
+        tmp3 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) ... e0(i+9) e1(i+9) e2(i+9) e3(i+9) e4(i+9) e5(i+9) e0(i+10) e1(i+10)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp3, p16c, tmp2));
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) x x x x e4(i+16) e5(i+16) x x x x .... e4(i+20) e5(i+20) x x
+        tmp5 = _mm512_permutex2var_epi16(tmp3, p16d, tmp2);
+        // e2(i+10) e3(i+10) e4(i+10) e5(i+10) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        // e4(i+15) e5(i+15) e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16)  e0(i+20)
+        // e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e0(i+21) e1(i+21)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp5, p16d2, tmp4));
+        // e2(i+21) e3(i+21) e4(i+21) e5(i+21) .... e0(i+31) e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31)
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi16(tmp4, p16e, tmp2));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      e4 = (uint8_t *)e4_512;
+      e5 = (uint8_t *)e5_512;
+      f = (uint8_t *)f_512;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+      }
+    } break;
+    case 8: {
+      uint8_t *e0 = e;
+      uint8_t *e1 = e0 + EQm;
+      uint8_t *e2 = e1 + EQm;
+      uint8_t *e3 = e2 + EQm;
+      uint8_t *e4 = e3 + EQm;
+      uint8_t *e5 = e4 + EQm;
+      uint8_t *e6 = e5 + EQm;
+      uint8_t *e7 = e6 + EQm;
+      
+      int i = 0;
+#ifdef __AVX512VBMI__
+      // clang-format off
+      const __m512i p8a = _mm512_set_epi8(95, 31, 94, 30, 93, 29, 92, 28, 91, 27, 90, 26, 89, 25, 88, 24, 87, 23,
+                                          86, 22, 85, 21, 84, 20, 83, 19, 82, 18, 81, 17, 80, 16, 79, 15, 78, 14, 77, 13,
+                                          76, 12, 75, 11, 74, 10, 73, 9, 72, 8, 71, 7, 70, 6, 69, 5, 68, 4, 67, 3, 66, 2, 65, 1, 64, 0);
+      const __m512i p8b = _mm512_set_epi8(127, 63, 126, 62, 125, 61, 124, 60, 123, 59, 122, 58, 121, 57, 120, 56, 119, 55,
+                                          118, 54, 117, 53, 116, 52, 115, 51, 114, 50, 113, 49, 112, 48, 111, 47, 110, 46,
+                                          109, 45, 108, 44, 107, 43, 106, 42, 105, 41, 104, 40, 103, 39, 102, 38, 101, 37, 100, 36, 99, 35, 98, 34, 97, 33, 96, 32);
+
+      const __m512i p16a = _mm512_set_epi16(47, 15, 46, 14, 45, 13, 44, 12, 43, 11, 42, 10, 41, 9, 40, 8, 39, 7, 38, 6, 37, 5, 36, 4, 35, 3, 34, 2, 33, 1, 32, 0);
+      const __m512i p16b = _mm512_set_epi16(63, 31, 62, 30, 61, 29, 60, 28, 59, 27, 58, 26, 57, 25, 56, 24, 55, 23, 54, 22, 53, 21, 52, 20, 51, 19, 50, 18, 49, 17, 48, 16);
+
+      const __m512i p32a = _mm512_set_epi32(23, 7, 22, 6, 21, 5, 20, 4, 19, 3, 18, 2, 17, 1, 16, 0);
+      const __m512i p32b = _mm512_set_epi32(31, 15, 30, 14, 29, 13, 28, 12, 27, 11, 26, 10, 25, 9, 24, 8);
+      // clang-format on
+      __m512i *e0_512 = (__m512i *)e0;
+      __m512i *e1_512 = (__m512i *)e1;
+      __m512i *e2_512 = (__m512i *)e2;
+      __m512i *e3_512 = (__m512i *)e3;
+      __m512i *e4_512 = (__m512i *)e4;
+      __m512i *e5_512 = (__m512i *)e5;
+      __m512i *e6_512 = (__m512i *)e6;
+      __m512i *e7_512 = (__m512i *)e7;
+      __m512i *f_512 = (__m512i *)f;
+      for (; i < (EQm & ~63); i += 64) {
+        __m512i e0j = _mm512_loadu_si512(e0_512++);
+        __m512i e1j = _mm512_loadu_si512(e1_512++);
+        __m512i e2j = _mm512_loadu_si512(e2_512++);
+        __m512i e3j = _mm512_loadu_si512(e3_512++);
+        __m512i e4j = _mm512_loadu_si512(e4_512++);
+        __m512i e5j = _mm512_loadu_si512(e5_512++);
+        __m512i e6j = _mm512_loadu_si512(e6_512++);
+        __m512i e7j = _mm512_loadu_si512(e7_512++);
+        __m512i tmp0 = _mm512_permutex2var_epi8(e0j, p8a, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+15) e1(i+15)
+        __m512i tmp1 = _mm512_permutex2var_epi8(e2j, p8a, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+15) e3(i+15)
+        __m512i tmp2 = _mm512_permutex2var_epi8(e4j, p8a, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+15) e5(i+15)
+        __m512i tmp3 = _mm512_permutex2var_epi8(e6j, p8a, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+15) e7(i+15)
+        __m512i tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1); // e0(i) e1(i) e2(i) e3(i) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        // e4(i) e5(i) e6(i) e7(i) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i)... e0(i+3) e1(i+3)
+        __m512i tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
+        // e2(i+3) e3(i+3) e4(i+3) e5(i+3) e6(i+3) e7(i+3))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7(i+4)...
+        // e0(i+7) e1(i+7) e2(i+7) e3(i+7) e4(i+7) e5(i+7) e6(i+7) e7(i+7))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+        // e0(i+8) e1(i+8) e2(i+8) e3(i+8) ... e0(i+15) e1(i+15) e2(i+15) e3(i+15)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e4(i+8) e5(i+8) e6(i+8) e7(i+8) ... e4(i+15) e5(i+15) e6(i+15) e7(i+15)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
+        // e0(i+8) e1(i+8) e2(i+8) e3(i+8) e4(i+8) e5(i+8) e6(i+8) e7(i+8)... e0(i+11)
+        // e1(i+11) e2(i+11) e3(i+11) e4(i+11) e5(i+11) e6(i+11) e7(i+11))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+12) e1(i+12) e2(i+12) e3(i+12) e4(i+12) e5(i+12) e6(i+12) e7(i+12)... e0(i+15)
+        // e1(i+15) e2(i+15) e3(i+15) e4(i+15) e5(i+15) e6(i+15) e7(i+15))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+
+        tmp0 = _mm512_permutex2var_epi8(e0j, p8b, e1j); // e0(i+16) e1(i+16) e0(i+17) e1(i+17) .... e0(i+31) e1(i+31)
+        tmp1 = _mm512_permutex2var_epi8(e2j, p8b, e3j); // e2(i+16) e3(i+16) e2(i+17) e3(i+17) .... e2(i+31) e3(i+31)
+        tmp2 = _mm512_permutex2var_epi8(e4j, p8b, e5j); // e4(i+16) e5(i+16) e4(i+17) e5(i+17) .... e4(i+31) e5(i+31)
+        tmp3 = _mm512_permutex2var_epi8(e6j, p8b, e7j); // e6(i+16) e7(i+16) e6(i+17) e7(i+17) .... e6(i+31) e7(i+31)
+        // e0(i+!6) e1(i+16) e2(i+16) e3(i+16) ... e0(i+23) e1(i+23) e2(i+23) e3(i+23)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16a, tmp1);
+        // e4(i+16) e5(i+16) e6(i+16) e7(i+16) ... e4(i+23) e5(i+23) e6(i+23) e7(i+23)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16a, tmp3);
+        // e0(i+16) e1(i+16) e2(i+16) e3(i+16) e4(i+16) e5(i+16) e6(i+16) e7(i+16)... e0(i+19)
+        // e1(i+19) e2(i+19) e3(i+19) e4(i+19) e5(i+19) e6(i+19) e7(i+19))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+20) e1(i+20) e2(i+20) e3(i+20) e4(i+20) e5(i+20) e6(i+20) e7(i+20)... e0(i+23)
+        // e1(i+23) e2(i+23) e3(i+23) e4(i+23) e5(i+23) e6(i+23) e7(i+23))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+        // e0(i+24) e1(i+24) e2(i+24) e3(i+24) ... e0(i+31) e1(i+31) e2(i+31) e3(i+31)
+        tmp4 = _mm512_permutex2var_epi16(tmp0, p16b, tmp1);
+        // e4(i+24) e5(i+24) e6(i+24) e7(i+24) ... e4(i+31) e5(i+31) e6(i+31) e7(i+31)
+        tmp5 = _mm512_permutex2var_epi16(tmp2, p16b, tmp3);
+        // e0(i+24) e1(i+24) e2(i+24) e3(i+24) e4(i+24) e5(i+24) e6(i+24) e7(i+24)... e0(i+27)
+        // e1(i+27) e2(i+27) e3(i+27) e4(i+27) e5(i+27) e6(i+27) e7(i+27))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32a, tmp5));
+        // e0(i+28) e1(i+28) e2(i+28) e3(i+28) e4(i+28) e5(i+28) e6(i+28) e7(i+28)... e0(i+31)
+        // e1(i+31) e2(i+31) e3(i+31) e4(i+31) e5(i+31) e6(i+31) e7(i+31))
+        _mm512_storeu_si512(f_512++, _mm512_permutex2var_epi32(tmp4, p32b, tmp5));
+      }
+      e0 = (uint8_t *)e0_512;
+      e1 = (uint8_t *)e1_512;
+      e2 = (uint8_t *)e2_512;
+      e3 = (uint8_t *)e3_512;
+      e4 = (uint8_t *)e4_512;
+      e5 = (uint8_t *)e5_512;
+      e6 = (uint8_t *)e6_512;
+      e7 = (uint8_t *)e7_512;
+      f = (uint8_t *)f_512;
+#endif
+#ifdef USE128BIT
+      e0_128 = (simde__m128i *)e0;
+      e1_128 = (simde__m128i *)e1;
+      e2_128 = (simde__m128i *)e2;
+      e3_128 = (simde__m128i *)e3;
+      e4_128 = (simde__m128i *)e4;
+      e5_128 = (simde__m128i *)e5;
+      e6_128 = (simde__m128i *)e6;
+      e7_128 = (simde__m128i *)e7;
+      for (; i < (EQm & ~15); i += 16) {
+        simde__m128i e0j = simde_mm_loadu_si128(e0_128++);
+        simde__m128i e1j = simde_mm_loadu_si128(e1_128++);
+        simde__m128i e2j = simde_mm_loadu_si128(e2_128++);
+        simde__m128i e3j = simde_mm_loadu_si128(e3_128++);
+        simde__m128i e4j = simde_mm_loadu_si128(e4_128++);
+        simde__m128i e5j = simde_mm_loadu_si128(e5_128++);
+        simde__m128i e6j = simde_mm_loadu_si128(e6_128++);
+        simde__m128i e7j = simde_mm_loadu_si128(e7_128++);
+        simde__m128i tmp0 = simde_mm_unpacklo_epi8(e0j, e1j); // e0(i) e1(i) e0(i+1) e1(i+1) .... e0(i+7) e1(i+7)
+        simde__m128i tmp1 = simde_mm_unpacklo_epi8(e2j, e3j); // e2(i) e3(i) e2(i+1) e3(i+1) .... e2(i+7) e3(i+7)
+        // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        simde__m128i tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        simde__m128i tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
+        simde__m128i tmp4 = simde_mm_unpacklo_epi8(e4j, e5j); // e4(i) e5(i) e4(i+1) e5(i+1) .... e4(i+7) e5(i+7)
+        simde__m128i tmp5 = simde_mm_unpacklo_epi8(e6j, e7j); // e6(i) e7(i) e6(i+1) e7(i+1) .... e6(i+7) e7(i+7)
+        // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
+        simde__m128i tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
+        // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
+        simde__m128i tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
+        // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
+        // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
+
+        tmp0 = simde_mm_unpackhi_epi8(e0j, e1j); // e0(i+8) e1(i+8) e0(i+9) e1(i+9) .... e0(i+15) e1(i+15)
+        tmp1 = simde_mm_unpackhi_epi8(e2j, e3j); // e2(i+8) e3(i+8) e2(i+9) e3(i+9) .... e2(i+15) e3(i+15)
+        // e0(i) e1(i) e2(i) e3(i) e0(i+1) e1(i+1) e2(i+1) e3(i+1) ... e0(i+3) e1(i+3) e2(i+3) e3(i+3)
+        tmp2 = simde_mm_unpacklo_epi16(tmp0, tmp1);
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e0(i+5) e1(i+5) e2(i+5) e3(i+5) ... e0(i+7) e1(i+7) e2(i+7) e3(i+7)
+        tmp3 = simde_mm_unpackhi_epi16(tmp0, tmp1);
+        tmp4 = simde_mm_unpackhi_epi8(e4j, e5j); // e4(i+8) e5(i+8) e4(i+9) e5(i+9) .... e4(i+15) e515i+7)
+        tmp5 = simde_mm_unpackhi_epi8(e6j, e7j); // e6(i+8) e7(i+8) e6(i+9) e7(i+9) .... e6(i+15) e7(i+15)
+        // e4(i) e5(i) e6(i) e7(i) e4(i+1) e5(i+1) e6(i+1) e7(i+1) ... e4(i+3) e5(i+3) e6(i+) e7(i+3)
+        tmp6 = simde_mm_unpacklo_epi16(tmp4, tmp5);
+        // e4(i+4) e5(i+4) e6(i+4) e7(i+4) e4(i+5) e5(i+5) e6(i+5) e7(i+5) ... e4(i+7) e5(i+7) e6(i+7) e7(i+7)
+        tmp7 = simde_mm_unpackhi_epi16(tmp4, tmp5);
+        // e0(i) e1(i) e2(i) e3(i) e4(i) e5(i) e6(i) e7(i) e0(i+1) ... e7(i+1)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp2, tmp6));
+        // e0(i+2) e1(i+2) e2(i+2) e3(i+2) e4(i+2) e5(i+2) e6(i+2) e7(i+2) e0(i+3) e1(i+3) ... e7(i+3)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp2, tmp6));
+        // e0(i+4) e1(i+4) e2(i+4) e3(i+4) e4(i+4) e5(i+4) e6(i+4) e7+4(i) e0(i+5) ... e7(i+5)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpacklo_epi32(tmp3, tmp7));
+        // e0(i+6) e1(i+6) e2(i+6) e3(i+6) e4(i+6) e5(i+6) e6(i+6) e7(i+6) e0(i+7) e0(i+7) ... e7(i+7)
+        simde_mm_storeu_si128(f_128++, simde_mm_unpackhi_epi32(tmp3, tmp7));
+      }
+      e0 = (uint8_t *)e0_128;
+      e1 = (uint8_t *)e1_128;
+      e2 = (uint8_t *)e2_128;
+      e3 = (uint8_t *)e3_128;
+      e4 = (uint8_t *)e4_128;
+      e5 = (uint8_t *)e5_128;
+      e6 = (uint8_t *)e6_128;
+      e7 = (uint8_t *)e7_128;
+      f = (uint8_t *)f_128;
+#endif
+      for (; i < EQm; i++) {
+        *f++ = *e0++;
+        *f++ = *e1++;
+        *f++ = *e2++;
+        *f++ = *e3++;
+        *f++ = *e4++;
+        *f++ = *e5++;
+        *f++ = *e6++;
+        *f++ = *e7++;
+      }
+    } break;
+    default:
+      AssertFatal(false, "Should be here!\n");
+  }
 }
 
 void nr_deinterleaving_ldpc(uint32_t E, uint8_t Qm, int16_t *e, int16_t *f)
@@ -454,8 +599,11 @@ int nr_rate_matching_ldpc(uint32_t Tbslbrm,
       printf("RM_TX k%u Ind: %u (%d)\n", k, ind, d[ind]);
 #endif
 
-      if (d[ind] != NR_NULL)
-        e[k++] = d[ind];
+      if (ind == Foffset)
+        ind = F + Foffset; // skip filler bits
+
+      e[k++] = d[ind];
+
     }
   }
 

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_segment/nr_rate_matching.h

@@ -24,8 +24,6 @@
 
 #include <stdint.h>
 
-#define NR_NULL 2
-
 /**
  * \brief interleave a code segment after encoding and rate matching
  * \param E size of the code segment in bits

openair1/PHY/CODING/nrLDPC_coding/nrLDPC_coding_t2/nrLDPC_coding_t2.c

@@ -650,25 +650,40 @@ static int
 retrieve_ldpc_enc_op(struct rte_bbdev_enc_op **ops,
                      nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
 {
+  uint8_t *p_out = NULL;
   unsigned int j = 0;
   for (unsigned int h = 0; h < nrLDPC_slot_encoding_parameters->nb_TBs; ++h){
+    int E_sum = 0;
+    int bit_offset = 0;
+    int byte_offset = 0;
+    p_out = nrLDPC_slot_encoding_parameters->TBs[h].output;
     for (unsigned int i = 0; i < nrLDPC_slot_encoding_parameters->TBs[h].C; ++i) {
       struct rte_bbdev_op_data *output = &ops[j]->ldpc_enc.output;
       struct rte_mbuf *m = output->data;
       uint16_t data_len = rte_pktmbuf_data_len(m) - output->offset;
-      uint8_t *out = nrLDPC_slot_encoding_parameters->TBs[h].segments[i].output;
       const char *data = m->buf_addr + m->data_off;
-      const char *end = data + data_len;
-      while (data < end) {
-        uint8_t byte = *data++; // get the current byte
-        for (int bit = 7; bit >= 0; --bit) {
-          *out++ = (byte >> bit) & 1; // extract each bit
+      if (bit_offset == 0) {
+        memcpy(&p_out[byte_offset], data, data_len);
+      } else {
+        uint8_t carry = 0;
+        p_out[byte_offset - 1] |= data[0] >> bit_offset;
+        for (size_t i = 0; i < data_len; i++) {
+          uint8_t current = *data++;
+          p_out[byte_offset + i] = (current << (8 - bit_offset));
+          if (i != 0) {
+            carry = current >> bit_offset;
+            p_out[byte_offset + i - 1] |= carry;
+          }
         }
       }
+      E_sum += nrLDPC_slot_encoding_parameters->TBs[h].segments[i].E;
+      byte_offset = (E_sum + 7) / 8;
+      bit_offset = E_sum % 8;
       rte_pktmbuf_free(m);
       rte_pktmbuf_free(ops[j]->ldpc_enc.input.data);
       ++j;
     }
+    reverse_bits_u8(p_out, byte_offset, p_out);
   }
   return 0;
 }
@@ -815,6 +830,10 @@ static int pmd_lcore_ldpc_enc(void *arg)
   AssertFatal(ret == 0, "Allocation failed for %d ops", num_segments);
 
   set_ldpc_enc_op(ops_enq, 0, bufs->inputs, bufs->hard_outputs, nrLDPC_slot_encoding_parameters);
+  if (nrLDPC_slot_encoding_parameters->tprep != NULL)
+    stop_meas(nrLDPC_slot_encoding_parameters->tprep);
+  if (nrLDPC_slot_encoding_parameters->tparity != NULL)
+    start_meas(nrLDPC_slot_encoding_parameters->tparity);
   for (enq = 0, deq = 0; enq < num_segments;) {
     num_to_enq = num_segments;
     if (unlikely(num_segments - enq < num_to_enq))
@@ -828,10 +847,14 @@ static int pmd_lcore_ldpc_enc(void *arg)
     time_out++;
     DevAssert(time_out <= TIME_OUT_POLL);
   }
-
+  if (nrLDPC_slot_encoding_parameters->tparity != NULL)
+    stop_meas(nrLDPC_slot_encoding_parameters->tparity);
+  if (nrLDPC_slot_encoding_parameters->toutput != NULL)
+    start_meas(nrLDPC_slot_encoding_parameters->toutput);
   ret = retrieve_ldpc_enc_op(ops_deq, nrLDPC_slot_encoding_parameters);
   AssertFatal(ret == 0, "Failed to retrieve LDPC encoding op!");
-
+  if (nrLDPC_slot_encoding_parameters->toutput != NULL)
+    stop_meas(nrLDPC_slot_encoding_parameters->toutput);
   rte_bbdev_enc_op_free_bulk(ops_enq, num_segments);
   rte_free(ops_enq);
   rte_free(ops_deq);
@@ -1074,6 +1097,8 @@ int32_t nrLDPC_coding_decoder(nrLDPC_slot_decoding_parameters_t *nrLDPC_slot_dec
 int32_t nrLDPC_coding_encoder(nrLDPC_slot_encoding_parameters_t *nrLDPC_slot_encoding_parameters)
 {
   pthread_mutex_lock(&encode_mutex);
+  if (nrLDPC_slot_encoding_parameters->tprep != NULL)
+    start_meas(nrLDPC_slot_encoding_parameters->tprep);
   // hardcoded to use the first found board
   struct active_device *ad = active_devs;
   int ret;

openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_tools/generator_cnProc/cnProc_gen_BG1_128.c

@@ -24,7 +24,9 @@
 #include <stdint.h>
 #include "../../nrLDPCdecoder_defs.h"
 
-#define AVOID_MM256_SIGN 1
+#ifdef __aarch64__
+#define AVOID_SIGN 1
+#endif
 #define DROP_MAXLLR 1
 void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 {
@@ -126,7 +128,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[0]>>4)+lut_idxCnProcG3[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -138,7 +144,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	  
 	  //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
+#ifdef AVOID_SIGN
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	  // Store result
 	  //                min = simde_mm_min_epu8(min, maxLLR); // 128 in epi8 is -127
 #ifndef DROP_MAXLLR
@@ -185,7 +195,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[1]>>4)+lut_idxCnProcG4[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -199,7 +213,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -252,7 +270,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[2]>>4)+lut_idxCnProcG5[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -266,7 +288,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -319,7 +345,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[3]>>4)+lut_idxCnProcG6[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -333,7 +363,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -389,7 +423,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[4]>>4)+lut_idxCnProcG7[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -403,7 +441,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -460,7 +502,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[5]>>4)+lut_idxCnProcG8[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -474,7 +520,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -531,7 +581,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[6]>>4)+lut_idxCnProcG9[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -545,7 +599,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -603,7 +661,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[7]>>4)+lut_idxCnProcG10[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -617,7 +679,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result
@@ -678,7 +744,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	  //                ymm0 = p_cnProcBuf[lut_idxCnProcG3[j][0] + i];
 	  fprintf(fd,"                ymm0 = ((simde__m128i*)cnProcBuf)[%d+i];\n",(lut_startAddrCnGroups[8]>>4)+lut_idxCnProcG19[j][0]*2);
 	  //                sgn  = simde_mm_sign_epi8(ones, ymm0);
+#ifdef AVOID_SIGN               
+	  fprintf(fd,"                sgn  = simde_mm_xor_si128(ones, ymm0);\n");
+#else
 	  fprintf(fd,"                sgn  = simde_mm_sign_epi8(ones, ymm0);\n");
+#endif
 	  //                min  = simde_mm_abs_epi8(ymm0);
 	  fprintf(fd,"                min  = simde_mm_abs_epi8(ymm0);\n");
 	  
@@ -692,7 +762,11 @@ void nrLDPC_cnProc_BG1_generator_128(const char* dir, int R)
 	      fprintf(fd,"                min  = simde_mm_min_epu8(min, simde_mm_abs_epi8(ymm0));\n");
 	      
 	      //                sgn  = simde_mm_sign_epi8(sgn, ymm0);
-  	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#ifdef AVOID_SIGN
+  	      fprintf(fd,"                sgn  = simde_mm_xor_si128(sgn, ymm0);\n");
+#else
+	      fprintf(fd,"                sgn  = simde_mm_sign_epi8(sgn, ymm0);\n");
+#endif
 	    }
 	  
 	  // Store result

openair1/PHY/CODING/nrLDPC_defs.h

@@ -28,7 +28,7 @@
 /**
    \brief LDPC encoder parameter structure
    \var n_segments number of segments in the transport block
-   \var macro_num index of the subset of segments to encode
+   \var first_seg index of the first segment of the subset to encode
    within the transport block 
    \var gen_code flag to generate parity check code
    0 -> encoding
@@ -52,7 +52,7 @@
 */
 typedef struct {
   unsigned int n_segments; // optim8seg
-  unsigned int macro_num;  // optim8segmulti
+  unsigned int first_seg;  // optim8segmulti
   unsigned char gen_code;  //orig
   time_stats_t *tinput;
   time_stats_t *tprep;
@@ -92,6 +92,6 @@ typedef int32_t(LDPC_decoderfunc_t)(t_nrLDPC_dec_params *p_decParams,
                                     int8_t *p_out,
                                     t_nrLDPC_time_stats *,
                                     decode_abort_t *ab);
-typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t **, encoder_implemparams_t *);
+typedef int32_t(LDPC_encoderfunc_t)(uint8_t **, uint8_t *, encoder_implemparams_t *);
 
 #endif

openair1/PHY/CODING/nrLDPC_encoder/ldpc384_simd512_byte.c

@@ -0,0 +1,151 @@
+#include "PHY/sse_intrin.h"
+// generated code for Zc=384, byte encoding
+static inline void ldpc384_byte(uint8_t *c,uint8_t *d) {
+  __m512i *csimd=(__m512i *)c,*dsimd=(__m512i *)d;
+
+  __m512i *c2,*d2;
+
+  int i2;
+  for (i2=0; i2<6; i2++) {
+     c2=&csimd[i2];
+     d2=&dsimd[i2];
+
+//row: 0
+     d2[0]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[3169],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[3193],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[2413],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[8500],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[4286],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2993],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[13310],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[10432],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[5957],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[6771],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[9421],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[9173],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13405],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[3137],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[12925],c2[10025]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 1
+     d2[6]=_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5028],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13224],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[12977],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14054],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6411],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16216],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8572],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12013],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11281],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13419],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[13958],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[2885],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7121],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 2
+     d2[12]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3435],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15327],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2141],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[64],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[8786],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[2992],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8546],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[16740],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[386],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2271],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6505],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14172],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[998],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[12891],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 3
+     d2[18]=_mm512_xor_si512(c2[13732],_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[3699],_mm512_xor_si512(c2[5284],_mm512_xor_si512(c2[5548],_mm512_xor_si512(c2[5292],_mm512_xor_si512(c2[15591],_mm512_xor_si512(c2[6085],_mm512_xor_si512(c2[6349],_mm512_xor_si512(c2[13488],_mm512_xor_si512(c2[3457],_mm512_xor_si512(c2[2405],_mm512_xor_si512(c2[13241],_mm512_xor_si512(c2[2677],_mm512_xor_si512(c2[36],_mm512_xor_si512(c2[300],_mm512_xor_si512(c2[8764],_mm512_xor_si512(c2[3485],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[5332],_mm512_xor_si512(c2[14318],_mm512_xor_si512(c2[4550],_mm512_xor_si512(c2[328],_mm512_xor_si512(c2[6675],_mm512_xor_si512(c2[9050],_mm512_xor_si512(c2[1923],_mm512_xor_si512(c2[2187],_mm512_xor_si512(c2[3257],_mm512_xor_si512(c2[3256],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[6950],_mm512_xor_si512(c2[3269],_mm512_xor_si512(c2[8810],_mm512_xor_si512(c2[14880],_mm512_xor_si512(c2[15144],_mm512_xor_si512(c2[16480],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[650],_mm512_xor_si512(c2[1178],_mm512_xor_si512(c2[1442],_mm512_xor_si512(c2[12277],_mm512_xor_si512(c2[10696],_mm512_xor_si512(c2[13072],_mm512_xor_si512(c2[13336],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[6221],_mm512_xor_si512(c2[10443],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[10193],_mm512_xor_si512(c2[2535],_mm512_xor_si512(c2[5705],_mm512_xor_si512(c2[5969],_mm512_xor_si512(c2[7035],_mm512_xor_si512(c2[6769],_mm512_xor_si512(c2[5451],_mm512_xor_si512(c2[5715],_mm512_xor_si512(c2[6519],_mm512_xor_si512(c2[9685],_mm512_xor_si512(c2[14436],_mm512_xor_si512(c2[11545],_mm512_xor_si512(c2[9437],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[13132],_mm512_xor_si512(c2[13669],_mm512_xor_si512(c2[1262],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[13683],_mm512_xor_si512(c2[13155],_mm512_xor_si512(c2[4180],_mm512_xor_si512(c2[4444],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[3401],_mm512_xor_si512(c2[3927],_mm512_xor_si512(c2[3149],_mm512_xor_si512(c2[3936],_mm512_xor_si512(c2[10536],_mm512_xor_si512(c2[10800],_mm512_xor_si512(c2[7385],_mm512_xor_si512(c2[13189],_mm512_xor_si512(c2[10025],c2[10289])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 4
+     d2[24]=_mm512_xor_si512(c2[3173],c2[14006]);
+
+//row: 5
+     d2[30]=_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[10827],_mm512_xor_si512(c2[11093],_mm512_xor_si512(c2[12936],_mm512_xor_si512(c2[795],_mm512_xor_si512(c2[12686],_mm512_xor_si512(c2[6084],_mm512_xor_si512(c2[13743],_mm512_xor_si512(c2[3708],_mm512_xor_si512(c2[3987],_mm512_xor_si512(c2[10851],_mm512_xor_si512(c2[9793],_mm512_xor_si512(c2[3734],_mm512_xor_si512(c2[10071],_mm512_xor_si512(c2[7694],_mm512_xor_si512(c2[13501],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[10873],_mm512_xor_si512(c2[12720],_mm512_xor_si512(c2[4817],_mm512_xor_si512(c2[11944],_mm512_xor_si512(c2[7716],_mm512_xor_si512(c2[14069],_mm512_xor_si512(c2[16444],_mm512_xor_si512(c2[9581],_mm512_xor_si512(c2[10645],_mm512_xor_si512(c2[10644],_mm512_xor_si512(c2[14344],_mm512_xor_si512(c2[10657],_mm512_xor_si512(c2[16204],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[6973],_mm512_xor_si512(c2[4073],_mm512_xor_si512(c2[7503],_mm512_xor_si512(c2[16224],_mm512_xor_si512(c2[8044],_mm512_xor_si512(c2[8836],_mm512_xor_si512(c2[2776],_mm512_xor_si512(c2[1189],_mm512_xor_si512(c2[3829],_mm512_xor_si512(c2[12290],_mm512_xor_si512(c2[13609],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[10178],_mm512_xor_si512(c2[686],_mm512_xor_si512(c2[9929],_mm512_xor_si512(c2[13357],_mm512_xor_si512(c2[14429],_mm512_xor_si512(c2[14163],_mm512_xor_si512(c2[13109],_mm512_xor_si512(c2[13913],_mm512_xor_si512(c2[184],_mm512_xor_si512(c2[4935],_mm512_xor_si512(c2[2044],_mm512_xor_si512(c2[16825],_mm512_xor_si512(c2[3625],_mm512_xor_si512(c2[13131],_mm512_xor_si512(c2[4168],_mm512_xor_si512(c2[8656],_mm512_xor_si512(c2[2581],_mm512_xor_si512(c2[4176],_mm512_xor_si512(c2[3648],_mm512_xor_si512(c2[11832],_mm512_xor_si512(c2[4721],_mm512_xor_si512(c2[10789],_mm512_xor_si512(c2[11321],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[11330],_mm512_xor_si512(c2[1299],_mm512_xor_si512(c2[14773],_mm512_xor_si512(c2[3688],_mm512_xor_si512(c2[782],c2[13716]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 6
+     d2[36]=_mm512_xor_si512(c2[5812],_mm512_xor_si512(c2[340],_mm512_xor_si512(c2[384],_mm512_xor_si512(c2[8321],_mm512_xor_si512(c2[7549],_mm512_xor_si512(c2[16311],_mm512_xor_si512(c2[4968],c2[8955])))))));
+
+//row: 7
+     d2[42]=_mm512_xor_si512(c2[2376],_mm512_xor_si512(c2[16380],_mm512_xor_si512(c2[15892],_mm512_xor_si512(c2[3521],_mm512_xor_si512(c2[9076],c2[13369])))));
+
+//row: 8
+     d2[48]=_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[7921],_mm512_xor_si512(c2[3962],_mm512_xor_si512(c2[14523],_mm512_xor_si512(c2[4228],_mm512_xor_si512(c2[14525],_mm512_xor_si512(c2[14789],_mm512_xor_si512(c2[6077],_mm512_xor_si512(c2[16368],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[13468],_mm512_xor_si512(c2[5821],_mm512_xor_si512(c2[16382],_mm512_xor_si512(c2[16120],_mm512_xor_si512(c2[9516],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[280],_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[13478],_mm512_xor_si512(c2[14017],_mm512_xor_si512(c2[7683],_mm512_xor_si512(c2[3986],_mm512_xor_si512(c2[14547],_mm512_xor_si512(c2[2928],_mm512_xor_si512(c2[13225],_mm512_xor_si512(c2[13489],_mm512_xor_si512(c2[13764],_mm512_xor_si512(c2[7430],_mm512_xor_si512(c2[3206],_mm512_xor_si512(c2[13767],_mm512_xor_si512(c2[829],_mm512_xor_si512(c2[11126],_mm512_xor_si512(c2[11390],_mm512_xor_si512(c2[9806],_mm512_xor_si512(c2[9293],_mm512_xor_si512(c2[2953],_mm512_xor_si512(c2[4008],_mm512_xor_si512(c2[14305],_mm512_xor_si512(c2[14569],_mm512_xor_si512(c2[5861],_mm512_xor_si512(c2[16152],_mm512_xor_si512(c2[16416],_mm512_xor_si512(c2[14847],_mm512_xor_si512(c2[8513],_mm512_xor_si512(c2[5079],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[857],_mm512_xor_si512(c2[11148],_mm512_xor_si512(c2[11412],_mm512_xor_si512(c2[7204],_mm512_xor_si512(c2[864],_mm512_xor_si512(c2[9579],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[2716],_mm512_xor_si512(c2[13013],_mm512_xor_si512(c2[13277],_mm512_xor_si512(c2[3780],_mm512_xor_si512(c2[14341],_mm512_xor_si512(c2[3785],_mm512_xor_si512(c2[14076],_mm512_xor_si512(c2[14340],_mm512_xor_si512(c2[7479],_mm512_xor_si512(c2[881],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[3792],_mm512_xor_si512(c2[14353],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[3005],_mm512_xor_si512(c2[15673],_mm512_xor_si512(c2[9075],_mm512_xor_si512(c2[9339],_mm512_xor_si512(c2[108],_mm512_xor_si512(c2[10669],_mm512_xor_si512(c2[14103],_mm512_xor_si512(c2[7769],_mm512_xor_si512(c2[638],_mm512_xor_si512(c2[10935],_mm512_xor_si512(c2[11199],_mm512_xor_si512(c2[9365],_mm512_xor_si512(c2[3025],_mm512_xor_si512(c2[1179],_mm512_xor_si512(c2[11476],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[1971],_mm512_xor_si512(c2[12268],_mm512_xor_si512(c2[12532],_mm512_xor_si512(c2[12806],_mm512_xor_si512(c2[6472],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[4885],_mm512_xor_si512(c2[13865],_mm512_xor_si512(c2[7261],_mm512_xor_si512(c2[7525],_mm512_xor_si512(c2[5425],_mm512_xor_si512(c2[15986],_mm512_xor_si512(c2[6744],_mm512_xor_si512(c2[410],_mm512_xor_si512(c2[11236],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[4896],_mm512_xor_si512(c2[10716],_mm512_xor_si512(c2[4382],_mm512_xor_si512(c2[3064],_mm512_xor_si512(c2[13361],_mm512_xor_si512(c2[13625],_mm512_xor_si512(c2[6492],_mm512_xor_si512(c2[16789],_mm512_xor_si512(c2[158],_mm512_xor_si512(c2[7564],_mm512_xor_si512(c2[1224],_mm512_xor_si512(c2[7298],_mm512_xor_si512(c2[700],_mm512_xor_si512(c2[964],_mm512_xor_si512(c2[6244],_mm512_xor_si512(c2[16541],_mm512_xor_si512(c2[16805],_mm512_xor_si512(c2[7048],_mm512_xor_si512(c2[708],_mm512_xor_si512(c2[10214],_mm512_xor_si512(c2[3880],_mm512_xor_si512(c2[14965],_mm512_xor_si512(c2[8367],_mm512_xor_si512(c2[8631],_mm512_xor_si512(c2[12074],_mm512_xor_si512(c2[5740],_mm512_xor_si512(c2[9960],_mm512_xor_si512(c2[3626],_mm512_xor_si512(c2[13661],_mm512_xor_si512(c2[7057],_mm512_xor_si512(c2[7321],_mm512_xor_si512(c2[10488],_mm512_xor_si512(c2[14198],_mm512_xor_si512(c2[7864],_mm512_xor_si512(c2[1791],_mm512_xor_si512(c2[12088],_mm512_xor_si512(c2[12352],_mm512_xor_si512(c2[12617],_mm512_xor_si512(c2[6013],_mm512_xor_si512(c2[6277],_mm512_xor_si512(c2[14212],_mm512_xor_si512(c2[7872],_mm512_xor_si512(c2[13684],_mm512_xor_si512(c2[7080],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[4973],_mm512_xor_si512(c2[15264],_mm512_xor_si512(c2[15528],_mm512_xor_si512(c2[14751],_mm512_xor_si512(c2[8417],_mm512_xor_si512(c2[3924],_mm512_xor_si512(c2[14485],_mm512_xor_si512(c2[4456],_mm512_xor_si512(c2[14753],_mm512_xor_si512(c2[15017],_mm512_xor_si512(c2[8679],_mm512_xor_si512(c2[3672],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[4465],_mm512_xor_si512(c2[14762],_mm512_xor_si512(c2[15026],_mm512_xor_si512(c2[11329],_mm512_xor_si512(c2[4731],_mm512_xor_si512(c2[4995],_mm512_xor_si512(c2[7908],_mm512_xor_si512(c2[1574],_mm512_xor_si512(c2[13718],_mm512_xor_si512(c2[7384],_mm512_xor_si512(c2[10812],_mm512_xor_si512(c2[4214],_mm512_xor_si512(c2[4478],c2[12929]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 9
+     d2[54]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[10575],_mm512_xor_si512(c2[389],_mm512_xor_si512(c2[1454],_mm512_xor_si512(c2[15204],_mm512_xor_si512(c2[12616],_mm512_xor_si512(c2[16848],c2[4993])))))));
+
+//row: 10
+     d2[60]=_mm512_xor_si512(c2[9781],_mm512_xor_si512(c2[5045],_mm512_xor_si512(c2[4804],_mm512_xor_si512(c2[12493],_mm512_xor_si512(c2[16733],c2[9149])))));
+
+//row: 11
+     d2[66]=_mm512_xor_si512(c2[268],_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[6864],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[8979],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[12672],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[2127],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[9780],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[10045],_mm512_xor_si512(c2[24],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[6888],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[5836],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[16672],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[6108],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[3737],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[12195],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[6916],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[8763],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[7981],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[3759],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[10106],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[12481],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5618],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[6688],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[10381],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[6700],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[12241],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[3016],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[110],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[12267],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[4081],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[4873],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[15708],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[14127],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[16767],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[8333],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[9652],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[14138],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[2256],_mm512_xor_si512(c2[13624],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[5966],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[10200],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[9146],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[9950],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[13116],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[14976],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[12868],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[16563],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[12600],_mm512_xor_si512(c2[205],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[4693],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[15519],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[219],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[16586],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[7875],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[758],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[6832],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[7358],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[6580],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[7373],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[14237],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[10816],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[16620],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[13720],_mm512_xor_si512(c2[6853],c2[16094])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 12
+     d2[72]=_mm512_xor_si512(c2[3433],_mm512_xor_si512(c2[15326],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[10695],_mm512_xor_si512(c2[13356],c2[2857])))));
+
+//row: 13
+     d2[78]=_mm512_xor_si512(c2[9772],_mm512_xor_si512(c2[10036],_mm512_xor_si512(c2[16632],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[1852],_mm512_xor_si512(c2[15052],_mm512_xor_si512(c2[1332],_mm512_xor_si512(c2[1596],_mm512_xor_si512(c2[11895],_mm512_xor_si512(c2[2653],_mm512_xor_si512(c2[9528],_mm512_xor_si512(c2[9792],_mm512_xor_si512(c2[16656],_mm512_xor_si512(c2[15604],_mm512_xor_si512(c2[9281],_mm512_xor_si512(c2[9545],_mm512_xor_si512(c2[15876],_mm512_xor_si512(c2[13505],_mm512_xor_si512(c2[12974],_mm512_xor_si512(c2[5068],_mm512_xor_si512(c2[16684],_mm512_xor_si512(c2[1636],_mm512_xor_si512(c2[10358],_mm512_xor_si512(c2[10622],_mm512_xor_si512(c2[854],_mm512_xor_si512(c2[13527],_mm512_xor_si512(c2[2715],_mm512_xor_si512(c2[2979],_mm512_xor_si512(c2[5354],_mm512_xor_si512(c2[15386],_mm512_xor_si512(c2[16456],_mm512_xor_si512(c2[16455],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[2728],_mm512_xor_si512(c2[16468],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[11448],_mm512_xor_si512(c2[12520],_mm512_xor_si512(c2[12784],_mm512_xor_si512(c2[9878],_mm512_xor_si512(c2[13308],_mm512_xor_si512(c2[4876],_mm512_xor_si512(c2[5140],_mm512_xor_si512(c2[13849],_mm512_xor_si512(c2[14641],_mm512_xor_si512(c2[8317],_mm512_xor_si512(c2[8581],_mm512_xor_si512(c2[7000],_mm512_xor_si512(c2[9640],_mm512_xor_si512(c2[936],_mm512_xor_si512(c2[1200],_mm512_xor_si512(c2[2525],_mm512_xor_si512(c2[7011],_mm512_xor_si512(c2[6233],_mm512_xor_si512(c2[6497],_mm512_xor_si512(c2[15734],_mm512_xor_si512(c2[2273],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[3073],_mm512_xor_si512(c2[2019],_mm512_xor_si512(c2[2559],_mm512_xor_si512(c2[2823],_mm512_xor_si512(c2[5989],_mm512_xor_si512(c2[10740],_mm512_xor_si512(c2[7585],_mm512_xor_si512(c2[7849],_mm512_xor_si512(c2[5741],_mm512_xor_si512(c2[9436],_mm512_xor_si512(c2[9973],_mm512_xor_si512(c2[14461],_mm512_xor_si512(c2[8392],_mm512_xor_si512(c2[9723],_mm512_xor_si512(c2[9987],_mm512_xor_si512(c2[9459],_mm512_xor_si512(c2[748],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10526],_mm512_xor_si512(c2[16600],_mm512_xor_si512(c2[231],_mm512_xor_si512(c2[16084],_mm512_xor_si512(c2[16348],_mm512_xor_si512(c2[240],_mm512_xor_si512(c2[7104],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[3425],_mm512_xor_si512(c2[3689],_mm512_xor_si512(c2[9493],c2[6593])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 14
+     d2[84]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[14931],_mm512_xor_si512(c2[2558],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[7337],c2[3420])))));
+
+//row: 15
+     d2[90]=_mm512_xor_si512(c2[267],_mm512_xor_si512(c2[6869],_mm512_xor_si512(c2[7129],_mm512_xor_si512(c2[8714],_mm512_xor_si512(c2[8978],_mm512_xor_si512(c2[12939],_mm512_xor_si512(c2[8728],_mm512_xor_si512(c2[2126],_mm512_xor_si512(c2[9521],_mm512_xor_si512(c2[9785],_mm512_xor_si512(c2[540],_mm512_xor_si512(c2[29],_mm512_xor_si512(c2[6893],_mm512_xor_si512(c2[5835],_mm512_xor_si512(c2[16671],_mm512_xor_si512(c2[6113],_mm512_xor_si512(c2[3472],_mm512_xor_si512(c2[3736],_mm512_xor_si512(c2[12194],_mm512_xor_si512(c2[6915],_mm512_xor_si512(c2[8498],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[853],_mm512_xor_si512(c2[7980],_mm512_xor_si512(c2[3758],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[12480],_mm512_xor_si512(c2[5353],_mm512_xor_si512(c2[5617],_mm512_xor_si512(c2[6687],_mm512_xor_si512(c2[6686],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[10380],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[12240],_mm512_xor_si512(c2[1421],_mm512_xor_si512(c2[1685],_mm512_xor_si512(c2[3015],_mm512_xor_si512(c2[109],_mm512_xor_si512(c2[3545],_mm512_xor_si512(c2[12266],_mm512_xor_si512(c2[4080],_mm512_xor_si512(c2[4608],_mm512_xor_si512(c2[4872],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[15713],_mm512_xor_si512(c2[14126],_mm512_xor_si512(c2[16502],_mm512_xor_si512(c2[16766],_mm512_xor_si512(c2[8332],_mm512_xor_si512(c2[9651],_mm512_xor_si512(c2[13873],_mm512_xor_si512(c2[14137],_mm512_xor_si512(c2[13623],_mm512_xor_si512(c2[5965],_mm512_xor_si512(c2[9135],_mm512_xor_si512(c2[9399],_mm512_xor_si512(c2[16528],_mm512_xor_si512(c2[10465],_mm512_xor_si512(c2[10205],_mm512_xor_si512(c2[8881],_mm512_xor_si512(c2[9145],_mm512_xor_si512(c2[9949],_mm512_xor_si512(c2[13121],_mm512_xor_si512(c2[977],_mm512_xor_si512(c2[14981],_mm512_xor_si512(c2[12867],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[16562],_mm512_xor_si512(c2[204],_mm512_xor_si512(c2[4692],_mm512_xor_si512(c2[15254],_mm512_xor_si512(c2[15518],_mm512_xor_si512(c2[218],_mm512_xor_si512(c2[16585],_mm512_xor_si512(c2[7610],_mm512_xor_si512(c2[7874],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[757],_mm512_xor_si512(c2[6831],_mm512_xor_si512(c2[7357],_mm512_xor_si512(c2[6579],_mm512_xor_si512(c2[7372],_mm512_xor_si512(c2[13972],_mm512_xor_si512(c2[14236],_mm512_xor_si512(c2[10815],_mm512_xor_si512(c2[16625],_mm512_xor_si512(c2[13455],c2[13719]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 16
+     d2[96]=_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[1322],_mm512_xor_si512(c2[1588],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3181],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[4238],_mm512_xor_si512(c2[3444],_mm512_xor_si512(c2[11377],_mm512_xor_si512(c2[1346],_mm512_xor_si512(c2[288],_mm512_xor_si512(c2[11124],_mm512_xor_si512(c2[566],_mm512_xor_si512(c2[15084],_mm512_xor_si512(c2[4793],_mm512_xor_si512(c2[6653],_mm512_xor_si512(c2[1368],_mm512_xor_si512(c2[3221],_mm512_xor_si512(c2[12207],_mm512_xor_si512(c2[2439],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[4564],_mm512_xor_si512(c2[6939],_mm512_xor_si512(c2[76],_mm512_xor_si512(c2[1140],_mm512_xor_si512(c2[1145],_mm512_xor_si512(c2[4839],_mm512_xor_si512(c2[1152],_mm512_xor_si512(c2[6699],_mm512_xor_si512(c2[13033],_mm512_xor_si512(c2[14369],_mm512_xor_si512(c2[11463],_mm512_xor_si512(c2[14893],_mm512_xor_si512(c2[6725],_mm512_xor_si512(c2[15434],_mm512_xor_si512(c2[16226],_mm512_xor_si512(c2[10166],_mm512_xor_si512(c2[8585],_mm512_xor_si512(c2[11225],_mm512_xor_si512(c2[15180],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[8596],_mm512_xor_si512(c2[8076],_mm512_xor_si512(c2[424],_mm512_xor_si512(c2[3852],_mm512_xor_si512(c2[4924],_mm512_xor_si512(c2[4658],_mm512_xor_si512(c2[3604],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[7574],_mm512_xor_si512(c2[12325],_mm512_xor_si512(c2[9434],_mm512_xor_si512(c2[7320],_mm512_xor_si512(c2[11021],_mm512_xor_si512(c2[11558],_mm512_xor_si512(c2[16046],_mm512_xor_si512(c2[9977],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[11044],_mm512_xor_si512(c2[2333],_mm512_xor_si512(c2[12111],_mm512_xor_si512(c2[1284],_mm512_xor_si512(c2[1816],_mm512_xor_si512(c2[1032],_mm512_xor_si512(c2[1825],_mm512_xor_si512(c2[8689],_mm512_xor_si512(c2[8956],_mm512_xor_si512(c2[5268],_mm512_xor_si512(c2[11078],c2[8172]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 17
+     d2[102]=_mm512_xor_si512(c2[1060],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[4681],_mm512_xor_si512(c2[10241],c2[14777]))));
+
+//row: 18
+     d2[108]=_mm512_xor_si512(c2[542],_mm512_xor_si512(c2[9386],_mm512_xor_si512(c2[6496],_mm512_xor_si512(c2[1274],c2[1284]))));
+
+//row: 19
+     d2[114]=_mm512_xor_si512(c2[4490],_mm512_xor_si512(c2[5559],_mm512_xor_si512(c2[6425],_mm512_xor_si512(c2[13299],c2[1443]))));
+
+//row: 20
+     d2[120]=_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[1850],_mm512_xor_si512(c2[2116],_mm512_xor_si512(c2[3965],_mm512_xor_si512(c2[15578],_mm512_xor_si512(c2[3709],_mm512_xor_si512(c2[14008],_mm512_xor_si512(c2[4766],_mm512_xor_si512(c2[11905],_mm512_xor_si512(c2[1874],_mm512_xor_si512(c2[816],_mm512_xor_si512(c2[11652],_mm512_xor_si512(c2[1094],_mm512_xor_si512(c2[15612],_mm512_xor_si512(c2[3735],_mm512_xor_si512(c2[7181],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[3749],_mm512_xor_si512(c2[12735],_mm512_xor_si512(c2[2967],_mm512_xor_si512(c2[15640],_mm512_xor_si512(c2[5092],_mm512_xor_si512(c2[7467],_mm512_xor_si512(c2[604],_mm512_xor_si512(c2[1668],_mm512_xor_si512(c2[1673],_mm512_xor_si512(c2[5367],_mm512_xor_si512(c2[1680],_mm512_xor_si512(c2[7227],_mm512_xor_si512(c2[13561],_mm512_xor_si512(c2[14897],_mm512_xor_si512(c2[11991],_mm512_xor_si512(c2[15421],_mm512_xor_si512(c2[2224],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[15962],_mm512_xor_si512(c2[16754],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[9113],_mm512_xor_si512(c2[11753],_mm512_xor_si512(c2[5681],_mm512_xor_si512(c2[3313],_mm512_xor_si512(c2[4632],_mm512_xor_si512(c2[9124],_mm512_xor_si512(c2[8604],_mm512_xor_si512(c2[952],_mm512_xor_si512(c2[4380],_mm512_xor_si512(c2[5452],_mm512_xor_si512(c2[5186],_mm512_xor_si512(c2[4132],_mm512_xor_si512(c2[4936],_mm512_xor_si512(c2[8102],_mm512_xor_si512(c2[12853],_mm512_xor_si512(c2[9962],_mm512_xor_si512(c2[7848],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[10505],_mm512_xor_si512(c2[12100],_mm512_xor_si512(c2[11572],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[12639],_mm512_xor_si512(c2[1812],_mm512_xor_si512(c2[2344],_mm512_xor_si512(c2[1560],_mm512_xor_si512(c2[2353],_mm512_xor_si512(c2[9217],_mm512_xor_si512(c2[5796],_mm512_xor_si512(c2[11606],c2[8700]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 21
+     d2[126]=_mm512_xor_si512(c2[3447],_mm512_xor_si512(c2[10093],_mm512_xor_si512(c2[2309],_mm512_xor_si512(c2[5787],c2[8965]))));
+
+//row: 22
+     d2[132]=_mm512_xor_si512(c2[7920],_mm512_xor_si512(c2[3048],_mm512_xor_si512(c2[10983],c2[6012])));
+
+//row: 23
+     d2[138]=_mm512_xor_si512(c2[6348],_mm512_xor_si512(c2[6625],_mm512_xor_si512(c2[16224],c2[7344])));
+
+//row: 24
+     d2[144]=_mm512_xor_si512(c2[7658],_mm512_xor_si512(c2[14260],_mm512_xor_si512(c2[14520],_mm512_xor_si512(c2[16369],_mm512_xor_si512(c2[11092],_mm512_xor_si512(c2[16119],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[281],_mm512_xor_si512(c2[7420],_mm512_xor_si512(c2[14284],_mm512_xor_si512(c2[13226],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[13504],_mm512_xor_si512(c2[11127],_mm512_xor_si512(c2[7958],_mm512_xor_si512(c2[2690],_mm512_xor_si512(c2[14306],_mm512_xor_si512(c2[16153],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[8244],_mm512_xor_si512(c2[15377],_mm512_xor_si512(c2[11149],_mm512_xor_si512(c2[601],_mm512_xor_si512(c2[2976],_mm512_xor_si512(c2[13008],_mm512_xor_si512(c2[14078],_mm512_xor_si512(c2[14077],_mm512_xor_si512(c2[876],_mm512_xor_si512(c2[14090],_mm512_xor_si512(c2[2736],_mm512_xor_si512(c2[9076],_mm512_xor_si512(c2[10406],_mm512_xor_si512(c2[7500],_mm512_xor_si512(c2[10936],_mm512_xor_si512(c2[2762],_mm512_xor_si512(c2[11477],_mm512_xor_si512(c2[12269],_mm512_xor_si512(c2[6209],_mm512_xor_si512(c2[4622],_mm512_xor_si512(c2[7262],_mm512_xor_si512(c2[5153],_mm512_xor_si512(c2[15723],_mm512_xor_si512(c2[147],_mm512_xor_si512(c2[4633],_mm512_xor_si512(c2[4119],_mm512_xor_si512(c2[13356],_mm512_xor_si512(c2[16790],_mm512_xor_si512(c2[961],_mm512_xor_si512(c2[701],_mm512_xor_si512(c2[16536],_mm512_xor_si512(c2[445],_mm512_xor_si512(c2[3617],_mm512_xor_si512(c2[8368],_mm512_xor_si512(c2[5477],_mm512_xor_si512(c2[3363],_mm512_xor_si512(c2[7058],_mm512_xor_si512(c2[7601],_mm512_xor_si512(c2[12089],_mm512_xor_si512(c2[6014],_mm512_xor_si512(c2[7609],_mm512_xor_si512(c2[7081],_mm512_xor_si512(c2[15265],_mm512_xor_si512(c2[8148],_mm512_xor_si512(c2[14222],_mm512_xor_si512(c2[14748],_mm512_xor_si512(c2[13970],_mm512_xor_si512(c2[14763],_mm512_xor_si512(c2[4732],_mm512_xor_si512(c2[1311],_mm512_xor_si512(c2[7121],c2[4215]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 25
+     d2[150]=_mm512_xor_si512(c2[2125],_mm512_xor_si512(c2[4560],_mm512_xor_si512(c2[16721],c2[14956])));
+
+//row: 26
+     d2[156]=_mm512_xor_si512(c2[1849],_mm512_xor_si512(c2[4513],_mm512_xor_si512(c2[3217],c2[2294])));
+
+//row: 27
+     d2[162]=_mm512_xor_si512(c2[543],_mm512_xor_si512(c2[603],c2[9865]));
+
+//row: 28
+     d2[168]=_mm512_xor_si512(c2[7923],_mm512_xor_si512(c2[5064],_mm512_xor_si512(c2[13959],c2[5008])));
+
+//row: 29
+     d2[174]=_mm512_xor_si512(c2[528],_mm512_xor_si512(c2[7130],_mm512_xor_si512(c2[7396],_mm512_xor_si512(c2[8981],_mm512_xor_si512(c2[9245],_mm512_xor_si512(c2[8989],_mm512_xor_si512(c2[2393],_mm512_xor_si512(c2[9782],_mm512_xor_si512(c2[10046],_mm512_xor_si512(c2[15855],_mm512_xor_si512(c2[290],_mm512_xor_si512(c2[7154],_mm512_xor_si512(c2[6096],_mm512_xor_si512(c2[37],_mm512_xor_si512(c2[6374],_mm512_xor_si512(c2[3733],_mm512_xor_si512(c2[3997],_mm512_xor_si512(c2[12461],_mm512_xor_si512(c2[7176],_mm512_xor_si512(c2[8765],_mm512_xor_si512(c2[9029],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[8247],_mm512_xor_si512(c2[4025],_mm512_xor_si512(c2[10372],_mm512_xor_si512(c2[12747],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[5884],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6953],_mm512_xor_si512(c2[10383],_mm512_xor_si512(c2[10647],_mm512_xor_si512(c2[6960],_mm512_xor_si512(c2[12507],_mm512_xor_si512(c2[1682],_mm512_xor_si512(c2[1946],_mm512_xor_si512(c2[3276],_mm512_xor_si512(c2[376],_mm512_xor_si512(c2[3806],_mm512_xor_si512(c2[12533],_mm512_xor_si512(c2[4347],_mm512_xor_si512(c2[4875],_mm512_xor_si512(c2[5139],_mm512_xor_si512(c2[15974],_mm512_xor_si512(c2[14393],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[132],_mm512_xor_si512(c2[8593],_mm512_xor_si512(c2[9912],_mm512_xor_si512(c2[14140],_mm512_xor_si512(c2[14404],_mm512_xor_si512(c2[13884],_mm512_xor_si512(c2[6232],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[9660],_mm512_xor_si512(c2[10732],_mm512_xor_si512(c2[10466],_mm512_xor_si512(c2[9148],_mm512_xor_si512(c2[9412],_mm512_xor_si512(c2[1488],_mm512_xor_si512(c2[10216],_mm512_xor_si512(c2[13382],_mm512_xor_si512(c2[1238],_mm512_xor_si512(c2[15242],_mm512_xor_si512(c2[13128],_mm512_xor_si512(c2[16565],_mm512_xor_si512(c2[16829],_mm512_xor_si512(c2[471],_mm512_xor_si512(c2[4959],_mm512_xor_si512(c2[15521],_mm512_xor_si512(c2[15785],_mm512_xor_si512(c2[485],_mm512_xor_si512(c2[16852],_mm512_xor_si512(c2[7877],_mm512_xor_si512(c2[8141],_mm512_xor_si512(c2[5234],_mm512_xor_si512(c2[1024],_mm512_xor_si512(c2[7092],_mm512_xor_si512(c2[7624],_mm512_xor_si512(c2[6840],_mm512_xor_si512(c2[7633],_mm512_xor_si512(c2[14233],_mm512_xor_si512(c2[14497],_mm512_xor_si512(c2[11076],_mm512_xor_si512(c2[16886],_mm512_xor_si512(c2[13716],c2[13980]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 30
+     d2[180]=_mm512_xor_si512(c2[3696],_mm512_xor_si512(c2[10298],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10564],_mm512_xor_si512(c2[12149],_mm512_xor_si512(c2[12413],_mm512_xor_si512(c2[8186],_mm512_xor_si512(c2[12157],_mm512_xor_si512(c2[5297],_mm512_xor_si512(c2[5561],_mm512_xor_si512(c2[12950],_mm512_xor_si512(c2[13214],_mm512_xor_si512(c2[3458],_mm512_xor_si512(c2[10322],_mm512_xor_si512(c2[9000],_mm512_xor_si512(c2[9264],_mm512_xor_si512(c2[3205],_mm512_xor_si512(c2[9542],_mm512_xor_si512(c2[6901],_mm512_xor_si512(c2[7165],_mm512_xor_si512(c2[15629],_mm512_xor_si512(c2[10080],_mm512_xor_si512(c2[10344],_mm512_xor_si512(c2[11933],_mm512_xor_si512(c2[12197],_mm512_xor_si512(c2[4288],_mm512_xor_si512(c2[11415],_mm512_xor_si512(c2[6929],_mm512_xor_si512(c2[7193],_mm512_xor_si512(c2[13540],_mm512_xor_si512(c2[15651],_mm512_xor_si512(c2[15915],_mm512_xor_si512(c2[8788],_mm512_xor_si512(c2[9052],_mm512_xor_si512(c2[10116],_mm512_xor_si512(c2[9857],_mm512_xor_si512(c2[10121],_mm512_xor_si512(c2[13551],_mm512_xor_si512(c2[13815],_mm512_xor_si512(c2[10128],_mm512_xor_si512(c2[15411],_mm512_xor_si512(c2[15675],_mm512_xor_si512(c2[4850],_mm512_xor_si512(c2[5114],_mm512_xor_si512(c2[6444],_mm512_xor_si512(c2[3544],_mm512_xor_si512(c2[6710],_mm512_xor_si512(c2[6974],_mm512_xor_si512(c2[15701],_mm512_xor_si512(c2[7251],_mm512_xor_si512(c2[7515],_mm512_xor_si512(c2[8043],_mm512_xor_si512(c2[8307],_mm512_xor_si512(c2[9891],_mm512_xor_si512(c2[2247],_mm512_xor_si512(c2[660],_mm512_xor_si512(c2[3036],_mm512_xor_si512(c2[3300],_mm512_xor_si512(c2[11761],_mm512_xor_si512(c2[13080],_mm512_xor_si512(c2[413],_mm512_xor_si512(c2[677],_mm512_xor_si512(c2[157],_mm512_xor_si512(c2[9136],_mm512_xor_si512(c2[9400],_mm512_xor_si512(c2[12564],_mm512_xor_si512(c2[12828],_mm512_xor_si512(c2[1216],_mm512_xor_si512(c2[13900],_mm512_xor_si512(c2[13370],_mm512_xor_si512(c2[13634],_mm512_xor_si512(c2[12316],_mm512_xor_si512(c2[12580],_mm512_xor_si512(c2[13384],_mm512_xor_si512(c2[16550],_mm512_xor_si512(c2[4142],_mm512_xor_si512(c2[4406],_mm512_xor_si512(c2[1515],_mm512_xor_si512(c2[16296],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[3096],_mm512_xor_si512(c2[3639],_mm512_xor_si512(c2[7863],_mm512_xor_si512(c2[8127],_mm512_xor_si512(c2[1788],_mm512_xor_si512(c2[2052],_mm512_xor_si512(c2[3653],_mm512_xor_si512(c2[2861],_mm512_xor_si512(c2[3125],_mm512_xor_si512(c2[11045],_mm512_xor_si512(c2[11309],_mm512_xor_si512(c2[4192],_mm512_xor_si512(c2[10260],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[10008],_mm512_xor_si512(c2[10537],_mm512_xor_si512(c2[10801],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[770],_mm512_xor_si512(c2[14244],_mm512_xor_si512(c2[3159],_mm512_xor_si512(c2[16884],c2[253])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 31
+     d2[186]=_mm512_xor_si512(c2[13994],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[3701],_mm512_xor_si512(c2[3437],_mm512_xor_si512(c2[3961],_mm512_xor_si512(c2[3697],_mm512_xor_si512(c2[5810],_mm512_xor_si512(c2[5282],_mm512_xor_si512(c2[5546],_mm512_xor_si512(c2[5560],_mm512_xor_si512(c2[5296],_mm512_xor_si512(c2[15853],_mm512_xor_si512(c2[15589],_mm512_xor_si512(c2[6617],_mm512_xor_si512(c2[6089],_mm512_xor_si512(c2[6353],_mm512_xor_si512(c2[9517],_mm512_xor_si512(c2[13756],_mm512_xor_si512(c2[13492],_mm512_xor_si512(c2[3725],_mm512_xor_si512(c2[3461],_mm512_xor_si512(c2[2667],_mm512_xor_si512(c2[2403],_mm512_xor_si512(c2[13503],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[2945],_mm512_xor_si512(c2[2681],_mm512_xor_si512(c2[568],_mm512_xor_si512(c2[40],_mm512_xor_si512(c2[304],_mm512_xor_si512(c2[9026],_mm512_xor_si512(c2[8762],_mm512_xor_si512(c2[3747],_mm512_xor_si512(c2[3483],_mm512_xor_si512(c2[5594],_mm512_xor_si512(c2[5066],_mm512_xor_si512(c2[5330],_mm512_xor_si512(c2[14580],_mm512_xor_si512(c2[14316],_mm512_xor_si512(c2[4812],_mm512_xor_si512(c2[4548],_mm512_xor_si512(c2[590],_mm512_xor_si512(c2[326],_mm512_xor_si512(c2[6937],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[9312],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[2449],_mm512_xor_si512(c2[1921],_mm512_xor_si512(c2[2185],_mm512_xor_si512(c2[3519],_mm512_xor_si512(c2[3255],_mm512_xor_si512(c2[3518],_mm512_xor_si512(c2[3254],_mm512_xor_si512(c2[7212],_mm512_xor_si512(c2[6684],_mm512_xor_si512(c2[6948],_mm512_xor_si512(c2[6159],_mm512_xor_si512(c2[3531],_mm512_xor_si512(c2[3267],_mm512_xor_si512(c2[9072],_mm512_xor_si512(c2[8808],_mm512_xor_si512(c2[15412],_mm512_xor_si512(c2[14884],_mm512_xor_si512(c2[15148],_mm512_xor_si512(c2[16742],_mm512_xor_si512(c2[16478],_mm512_xor_si512(c2[13836],_mm512_xor_si512(c2[13572],_mm512_xor_si512(c2[377],_mm512_xor_si512(c2[113],_mm512_xor_si512(c2[9098],_mm512_xor_si512(c2[8834],_mm512_xor_si512(c2[912],_mm512_xor_si512(c2[648],_mm512_xor_si512(c2[1704],_mm512_xor_si512(c2[1176],_mm512_xor_si512(c2[1440],_mm512_xor_si512(c2[12545],_mm512_xor_si512(c2[12281],_mm512_xor_si512(c2[10958],_mm512_xor_si512(c2[10694],_mm512_xor_si512(c2[13598],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[13334],_mm512_xor_si512(c2[5164],_mm512_xor_si512(c2[4900],_mm512_xor_si512(c2[6483],_mm512_xor_si512(c2[6219],_mm512_xor_si512(c2[10969],_mm512_xor_si512(c2[10441],_mm512_xor_si512(c2[10705],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[10191],_mm512_xor_si512(c2[2797],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[6231],_mm512_xor_si512(c2[5703],_mm512_xor_si512(c2[5967],_mm512_xor_si512(c2[7297],_mm512_xor_si512(c2[7033],_mm512_xor_si512(c2[7037],_mm512_xor_si512(c2[6773],_mm512_xor_si512(c2[5977],_mm512_xor_si512(c2[5449],_mm512_xor_si512(c2[5713],_mm512_xor_si512(c2[6781],_mm512_xor_si512(c2[6517],_mm512_xor_si512(c2[9953],_mm512_xor_si512(c2[9689],_mm512_xor_si512(c2[14704],_mm512_xor_si512(c2[14440],_mm512_xor_si512(c2[11813],_mm512_xor_si512(c2[11549],_mm512_xor_si512(c2[9699],_mm512_xor_si512(c2[9435],_mm512_xor_si512(c2[13394],_mm512_xor_si512(c2[12866],_mm512_xor_si512(c2[13130],_mm512_xor_si512(c2[13937],_mm512_xor_si512(c2[13673],_mm512_xor_si512(c2[1524],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[12350],_mm512_xor_si512(c2[11822],_mm512_xor_si512(c2[12086],_mm512_xor_si512(c2[13945],_mm512_xor_si512(c2[13681],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[13153],_mm512_xor_si512(c2[4706],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[14484],_mm512_xor_si512(c2[14220],_mm512_xor_si512(c2[3663],_mm512_xor_si512(c2[3399],_mm512_xor_si512(c2[4189],_mm512_xor_si512(c2[3925],_mm512_xor_si512(c2[3411],_mm512_xor_si512(c2[3147],_mm512_xor_si512(c2[4204],_mm512_xor_si512(c2[3940],_mm512_xor_si512(c2[11068],_mm512_xor_si512(c2[10540],_mm512_xor_si512(c2[10804],_mm512_xor_si512(c2[7647],_mm512_xor_si512(c2[7383],_mm512_xor_si512(c2[13457],_mm512_xor_si512(c2[13193],_mm512_xor_si512(c2[10551],_mm512_xor_si512(c2[10023],c2[10287]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 32
+     d2[192]=_mm512_xor_si512(c2[5019],_mm512_xor_si512(c2[11621],_mm512_xor_si512(c2[11617],_mm512_xor_si512(c2[11881],_mm512_xor_si512(c2[13466],_mm512_xor_si512(c2[13730],_mm512_xor_si512(c2[10033],_mm512_xor_si512(c2[13480],_mm512_xor_si512(c2[6614],_mm512_xor_si512(c2[6878],_mm512_xor_si512(c2[14273],_mm512_xor_si512(c2[14537],_mm512_xor_si512(c2[4781],_mm512_xor_si512(c2[11645],_mm512_xor_si512(c2[10323],_mm512_xor_si512(c2[10587],_mm512_xor_si512(c2[4528],_mm512_xor_si512(c2[10865],_mm512_xor_si512(c2[8224],_mm512_xor_si512(c2[8488],_mm512_xor_si512(c2[51],_mm512_xor_si512(c2[11403],_mm512_xor_si512(c2[11667],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[13514],_mm512_xor_si512(c2[5605],_mm512_xor_si512(c2[12732],_mm512_xor_si512(c2[8246],_mm512_xor_si512(c2[8510],_mm512_xor_si512(c2[14857],_mm512_xor_si512(c2[73],_mm512_xor_si512(c2[337],_mm512_xor_si512(c2[10105],_mm512_xor_si512(c2[10369],_mm512_xor_si512(c2[11439],_mm512_xor_si512(c2[11174],_mm512_xor_si512(c2[11438],_mm512_xor_si512(c2[14868],_mm512_xor_si512(c2[15132],_mm512_xor_si512(c2[11451],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[97],_mm512_xor_si512(c2[6173],_mm512_xor_si512(c2[6437],_mm512_xor_si512(c2[7767],_mm512_xor_si512(c2[4861],_mm512_xor_si512(c2[8033],_mm512_xor_si512(c2[8297],_mm512_xor_si512(c2[123],_mm512_xor_si512(c2[8568],_mm512_xor_si512(c2[8832],_mm512_xor_si512(c2[9360],_mm512_xor_si512(c2[9624],_mm512_xor_si512(c2[3564],_mm512_xor_si512(c2[1983],_mm512_xor_si512(c2[4359],_mm512_xor_si512(c2[4623],_mm512_xor_si512(c2[13084],_mm512_xor_si512(c2[14403],_mm512_xor_si512(c2[1730],_mm512_xor_si512(c2[1994],_mm512_xor_si512(c2[2523],_mm512_xor_si512(c2[1480],_mm512_xor_si512(c2[10453],_mm512_xor_si512(c2[10717],_mm512_xor_si512(c2[13887],_mm512_xor_si512(c2[14151],_mm512_xor_si512(c2[15217],_mm512_xor_si512(c2[14693],_mm512_xor_si512(c2[14957],_mm512_xor_si512(c2[13633],_mm512_xor_si512(c2[13897],_mm512_xor_si512(c2[12578],_mm512_xor_si512(c2[14701],_mm512_xor_si512(c2[972],_mm512_xor_si512(c2[5465],_mm512_xor_si512(c2[5729],_mm512_xor_si512(c2[2832],_mm512_xor_si512(c2[724],_mm512_xor_si512(c2[4155],_mm512_xor_si512(c2[4419],_mm512_xor_si512(c2[4956],_mm512_xor_si512(c2[9180],_mm512_xor_si512(c2[9444],_mm512_xor_si512(c2[3111],_mm512_xor_si512(c2[3375],_mm512_xor_si512(c2[4970],_mm512_xor_si512(c2[4178],_mm512_xor_si512(c2[4442],_mm512_xor_si512(c2[12362],_mm512_xor_si512(c2[12626],_mm512_xor_si512(c2[5509],_mm512_xor_si512(c2[11583],_mm512_xor_si512(c2[11845],_mm512_xor_si512(c2[12109],_mm512_xor_si512(c2[11331],_mm512_xor_si512(c2[11860],_mm512_xor_si512(c2[12124],_mm512_xor_si512(c2[1829],_mm512_xor_si512(c2[2093],_mm512_xor_si512(c2[15567],_mm512_xor_si512(c2[4476],_mm512_xor_si512(c2[1312],c2[1576])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 33
+     d2[198]=_mm512_xor_si512(c2[809],_mm512_xor_si512(c2[2136],_mm512_xor_si512(c2[10961],c2[11077])));
+
+//row: 34
+     d2[204]=_mm512_xor_si512(c2[10035],_mm512_xor_si512(c2[5366],_mm512_xor_si512(c2[2823],c2[14992])));
+
+//row: 35
+     d2[210]=_mm512_xor_si512(c2[16106],_mm512_xor_si512(c2[5813],_mm512_xor_si512(c2[6073],_mm512_xor_si512(c2[7922],_mm512_xor_si512(c2[7672],_mm512_xor_si512(c2[1070],_mm512_xor_si512(c2[8729],_mm512_xor_si512(c2[17],_mm512_xor_si512(c2[15868],_mm512_xor_si512(c2[5837],_mm512_xor_si512(c2[4779],_mm512_xor_si512(c2[15615],_mm512_xor_si512(c2[5057],_mm512_xor_si512(c2[2680],_mm512_xor_si512(c2[11138],_mm512_xor_si512(c2[5859],_mm512_xor_si512(c2[7706],_mm512_xor_si512(c2[16692],_mm512_xor_si512(c2[6924],_mm512_xor_si512(c2[2702],_mm512_xor_si512(c2[9049],_mm512_xor_si512(c2[11424],_mm512_xor_si512(c2[4561],_mm512_xor_si512(c2[4037],_mm512_xor_si512(c2[5631],_mm512_xor_si512(c2[5630],_mm512_xor_si512(c2[9324],_mm512_xor_si512(c2[5643],_mm512_xor_si512(c2[11184],_mm512_xor_si512(c2[629],_mm512_xor_si512(c2[1959],_mm512_xor_si512(c2[15948],_mm512_xor_si512(c2[2489],_mm512_xor_si512(c2[11210],_mm512_xor_si512(c2[3024],_mm512_xor_si512(c2[3816],_mm512_xor_si512(c2[14657],_mm512_xor_si512(c2[13070],_mm512_xor_si512(c2[15710],_mm512_xor_si512(c2[7276],_mm512_xor_si512(c2[8595],_mm512_xor_si512(c2[13081],_mm512_xor_si512(c2[672],_mm512_xor_si512(c2[12567],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[8343],_mm512_xor_si512(c2[9409],_mm512_xor_si512(c2[9149],_mm512_xor_si512(c2[8089],_mm512_xor_si512(c2[8893],_mm512_xor_si512(c2[12065],_mm512_xor_si512(c2[16816],_mm512_xor_si512(c2[13925],_mm512_xor_si512(c2[11811],_mm512_xor_si512(c2[15506],_mm512_xor_si512(c2[16049],_mm512_xor_si512(c2[3636],_mm512_xor_si512(c2[14462],_mm512_xor_si512(c2[16057],_mm512_xor_si512(c2[15529],_mm512_xor_si512(c2[6818],_mm512_xor_si512(c2[16596],_mm512_xor_si512(c2[5775],_mm512_xor_si512(c2[6301],_mm512_xor_si512(c2[5523],_mm512_xor_si512(c2[6316],_mm512_xor_si512(c2[13180],_mm512_xor_si512(c2[9759],_mm512_xor_si512(c2[15569],c2[12663])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 36
+     d2[216]=_mm512_xor_si512(c2[4755],_mm512_xor_si512(c2[15220],_mm512_xor_si512(c2[11008],c2[5760])));
+
+//row: 37
+     d2[222]=_mm512_xor_si512(c2[10032],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[3],_mm512_xor_si512(c2[269],_mm512_xor_si512(c2[2112],_mm512_xor_si512(c2[1598],_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[12161],_mm512_xor_si512(c2[2919],_mm512_xor_si512(c2[3448],_mm512_xor_si512(c2[9794],_mm512_xor_si512(c2[10058],_mm512_xor_si512(c2[27],_mm512_xor_si512(c2[15864],_mm512_xor_si512(c2[9541],_mm512_xor_si512(c2[9805],_mm512_xor_si512(c2[16142],_mm512_xor_si512(c2[13765],_mm512_xor_si512(c2[5328],_mm512_xor_si512(c2[49],_mm512_xor_si512(c2[1896],_mm512_xor_si512(c2[10624],_mm512_xor_si512(c2[10888],_mm512_xor_si512(c2[1120],_mm512_xor_si512(c2[13793],_mm512_xor_si512(c2[2981],_mm512_xor_si512(c2[3245],_mm512_xor_si512(c2[5620],_mm512_xor_si512(c2[15652],_mm512_xor_si512(c2[16716],_mm512_xor_si512(c2[16721],_mm512_xor_si512(c2[3520],_mm512_xor_si512(c2[16728],_mm512_xor_si512(c2[5380],_mm512_xor_si512(c2[11714],_mm512_xor_si512(c2[12780],_mm512_xor_si512(c2[13044],_mm512_xor_si512(c2[10144],_mm512_xor_si512(c2[13574],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5400],_mm512_xor_si512(c2[14115],_mm512_xor_si512(c2[14907],_mm512_xor_si512(c2[8583],_mm512_xor_si512(c2[8847],_mm512_xor_si512(c2[7260],_mm512_xor_si512(c2[9900],_mm512_xor_si512(c2[1202],_mm512_xor_si512(c2[1466],_mm512_xor_si512(c2[2785],_mm512_xor_si512(c2[7277],_mm512_xor_si512(c2[6493],_mm512_xor_si512(c2[6757],_mm512_xor_si512(c2[16000],_mm512_xor_si512(c2[2533],_mm512_xor_si512(c2[4909],_mm512_xor_si512(c2[3605],_mm512_xor_si512(c2[3339],_mm512_xor_si512(c2[2285],_mm512_xor_si512(c2[2825],_mm512_xor_si512(c2[3089],_mm512_xor_si512(c2[6255],_mm512_xor_si512(c2[11006],_mm512_xor_si512(c2[7851],_mm512_xor_si512(c2[8115],_mm512_xor_si512(c2[6001],_mm512_xor_si512(c2[9696],_mm512_xor_si512(c2[10239],_mm512_xor_si512(c2[14727],_mm512_xor_si512(c2[8652],_mm512_xor_si512(c2[9989],_mm512_xor_si512(c2[10253],_mm512_xor_si512(c2[9725],_mm512_xor_si512(c2[1008],_mm512_xor_si512(c2[10528],_mm512_xor_si512(c2[10792],_mm512_xor_si512(c2[16860],_mm512_xor_si512(c2[497],_mm512_xor_si512(c2[16344],_mm512_xor_si512(c2[16608],_mm512_xor_si512(c2[506],_mm512_xor_si512(c2[7370],_mm512_xor_si512(c2[3685],_mm512_xor_si512(c2[3949],_mm512_xor_si512(c2[9759],c2[6853])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 38
+     d2[228]=_mm512_xor_si512(c2[15050],_mm512_xor_si512(c2[13046],_mm512_xor_si512(c2[8836],c2[5955])));
+
+//row: 39
+     d2[234]=_mm512_xor_si512(c2[544],_mm512_xor_si512(c2[7693],_mm512_xor_si512(c2[6953],c2[11056])));
+
+//row: 40
+     d2[240]=_mm512_xor_si512(c2[12410],_mm512_xor_si512(c2[9864],c2[14992]));
+
+//row: 41
+     d2[246]=_mm512_xor_si512(c2[13740],_mm512_xor_si512(c2[15352],_mm512_xor_si512(c2[3014],c2[8668])));
+
+//row: 42
+     d2[252]=_mm512_xor_si512(c2[3698],_mm512_xor_si512(c2[10300],_mm512_xor_si512(c2[10296],_mm512_xor_si512(c2[10560],_mm512_xor_si512(c2[12145],_mm512_xor_si512(c2[12409],_mm512_xor_si512(c2[12937],_mm512_xor_si512(c2[12159],_mm512_xor_si512(c2[5293],_mm512_xor_si512(c2[5557],_mm512_xor_si512(c2[12952],_mm512_xor_si512(c2[13216],_mm512_xor_si512(c2[3460],_mm512_xor_si512(c2[10324],_mm512_xor_si512(c2[9002],_mm512_xor_si512(c2[9266],_mm512_xor_si512(c2[3207],_mm512_xor_si512(c2[9544],_mm512_xor_si512(c2[6903],_mm512_xor_si512(c2[7167],_mm512_xor_si512(c2[15625],_mm512_xor_si512(c2[10082],_mm512_xor_si512(c2[10346],_mm512_xor_si512(c2[11929],_mm512_xor_si512(c2[12193],_mm512_xor_si512(c2[3744],_mm512_xor_si512(c2[4284],_mm512_xor_si512(c2[11417],_mm512_xor_si512(c2[6925],_mm512_xor_si512(c2[7189],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[15653],_mm512_xor_si512(c2[15917],_mm512_xor_si512(c2[8784],_mm512_xor_si512(c2[9048],_mm512_xor_si512(c2[10118],_mm512_xor_si512(c2[9853],_mm512_xor_si512(c2[10117],_mm512_xor_si512(c2[13553],_mm512_xor_si512(c2[13817],_mm512_xor_si512(c2[10130],_mm512_xor_si512(c2[15413],_mm512_xor_si512(c2[15677],_mm512_xor_si512(c2[4852],_mm512_xor_si512(c2[5116],_mm512_xor_si512(c2[6446],_mm512_xor_si512(c2[3540],_mm512_xor_si512(c2[6712],_mm512_xor_si512(c2[6976],_mm512_xor_si512(c2[15697],_mm512_xor_si512(c2[7253],_mm512_xor_si512(c2[7517],_mm512_xor_si512(c2[8045],_mm512_xor_si512(c2[8309],_mm512_xor_si512(c2[2249],_mm512_xor_si512(c2[662],_mm512_xor_si512(c2[3038],_mm512_xor_si512(c2[3302],_mm512_xor_si512(c2[11763],_mm512_xor_si512(c2[13082],_mm512_xor_si512(c2[409],_mm512_xor_si512(c2[673],_mm512_xor_si512(c2[159],_mm512_xor_si512(c2[9132],_mm512_xor_si512(c2[9396],_mm512_xor_si512(c2[12566],_mm512_xor_si512(c2[12830],_mm512_xor_si512(c2[13896],_mm512_xor_si512(c2[13372],_mm512_xor_si512(c2[13636],_mm512_xor_si512(c2[12312],_mm512_xor_si512(c2[12576],_mm512_xor_si512(c2[13380],_mm512_xor_si512(c2[16552],_mm512_xor_si512(c2[4144],_mm512_xor_si512(c2[4408],_mm512_xor_si512(c2[1517],_mm512_xor_si512(c2[16298],_mm512_xor_si512(c2[2834],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[3641],_mm512_xor_si512(c2[7865],_mm512_xor_si512(c2[8129],_mm512_xor_si512(c2[1790],_mm512_xor_si512(c2[2054],_mm512_xor_si512(c2[3649],_mm512_xor_si512(c2[2857],_mm512_xor_si512(c2[3121],_mm512_xor_si512(c2[11041],_mm512_xor_si512(c2[11305],_mm512_xor_si512(c2[4188],_mm512_xor_si512(c2[10262],_mm512_xor_si512(c2[10524],_mm512_xor_si512(c2[10788],_mm512_xor_si512(c2[10010],_mm512_xor_si512(c2[10539],_mm512_xor_si512(c2[10803],_mm512_xor_si512(c2[508],_mm512_xor_si512(c2[772],_mm512_xor_si512(c2[14246],_mm512_xor_si512(c2[3161],_mm512_xor_si512(c2[16886],c2[255]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 43
+     d2[258]=_mm512_xor_si512(c2[7393],_mm512_xor_si512(c2[13995],_mm512_xor_si512(c2[14261],_mm512_xor_si512(c2[15840],_mm512_xor_si512(c2[16104],_mm512_xor_si512(c2[15854],_mm512_xor_si512(c2[9252],_mm512_xor_si512(c2[16647],_mm512_xor_si512(c2[16],_mm512_xor_si512(c2[12949],_mm512_xor_si512(c2[7155],_mm512_xor_si512(c2[14019],_mm512_xor_si512(c2[12961],_mm512_xor_si512(c2[6902],_mm512_xor_si512(c2[13239],_mm512_xor_si512(c2[10598],_mm512_xor_si512(c2[10862],_mm512_xor_si512(c2[2425],_mm512_xor_si512(c2[14041],_mm512_xor_si512(c2[15624],_mm512_xor_si512(c2[15888],_mm512_xor_si512(c2[7985],_mm512_xor_si512(c2[15112],_mm512_xor_si512(c2[10884],_mm512_xor_si512(c2[336],_mm512_xor_si512(c2[2717],_mm512_xor_si512(c2[12485],_mm512_xor_si512(c2[12749],_mm512_xor_si512(c2[13813],_mm512_xor_si512(c2[13812],_mm512_xor_si512(c2[353],_mm512_xor_si512(c2[617],_mm512_xor_si512(c2[13825],_mm512_xor_si512(c2[2477],_mm512_xor_si512(c2[8547],_mm512_xor_si512(c2[8811],_mm512_xor_si512(c2[10141],_mm512_xor_si512(c2[7241],_mm512_xor_si512(c2[10671],_mm512_xor_si512(c2[2497],_mm512_xor_si512(c2[11212],_mm512_xor_si512(c2[11740],_mm512_xor_si512(c2[12004],_mm512_xor_si512(c2[5944],_mm512_xor_si512(c2[4357],_mm512_xor_si512(c2[6733],_mm512_xor_si512(c2[6997],_mm512_xor_si512(c2[15458],_mm512_xor_si512(c2[16777],_mm512_xor_si512(c2[4104],_mm512_xor_si512(c2[4368],_mm512_xor_si512(c2[3854],_mm512_xor_si512(c2[13097],_mm512_xor_si512(c2[16261],_mm512_xor_si512(c2[16525],_mm512_xor_si512(c2[696],_mm512_xor_si512(c2[436],_mm512_xor_si512(c2[16013],_mm512_xor_si512(c2[16277],_mm512_xor_si512(c2[180],_mm512_xor_si512(c2[3352],_mm512_xor_si512(c2[8103],_mm512_xor_si512(c2[5212],_mm512_xor_si512(c2[3098],_mm512_xor_si512(c2[6529],_mm512_xor_si512(c2[6793],_mm512_xor_si512(c2[1250],_mm512_xor_si512(c2[7336],_mm512_xor_si512(c2[11824],_mm512_xor_si512(c2[5485],_mm512_xor_si512(c2[5749],_mm512_xor_si512(c2[7344],_mm512_xor_si512(c2[6816],_mm512_xor_si512(c2[14736],_mm512_xor_si512(c2[15000],_mm512_xor_si512(c2[13417],_mm512_xor_si512(c2[7889],_mm512_xor_si512(c2[13957],_mm512_xor_si512(c2[14489],_mm512_xor_si512(c2[13705],_mm512_xor_si512(c2[14498],_mm512_xor_si512(c2[4203],_mm512_xor_si512(c2[4467],_mm512_xor_si512(c2[1046],_mm512_xor_si512(c2[6856],_mm512_xor_si512(c2[3686],c2[3950]))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 44
+     d2[264]=_mm512_xor_si512(c2[1323],_mm512_xor_si512(c2[7925],_mm512_xor_si512(c2[8185],_mm512_xor_si512(c2[10034],_mm512_xor_si512(c2[4225],_mm512_xor_si512(c2[9784],_mm512_xor_si512(c2[3182],_mm512_xor_si512(c2[10841],_mm512_xor_si512(c2[1085],_mm512_xor_si512(c2[7949],_mm512_xor_si512(c2[6891],_mm512_xor_si512(c2[832],_mm512_xor_si512(c2[7169],_mm512_xor_si512(c2[4792],_mm512_xor_si512(c2[13250],_mm512_xor_si512(c2[7971],_mm512_xor_si512(c2[9818],_mm512_xor_si512(c2[1909],_mm512_xor_si512(c2[9036],_mm512_xor_si512(c2[4814],_mm512_xor_si512(c2[11161],_mm512_xor_si512(c2[13536],_mm512_xor_si512(c2[6673],_mm512_xor_si512(c2[7743],_mm512_xor_si512(c2[7742],_mm512_xor_si512(c2[11436],_mm512_xor_si512(c2[3781],_mm512_xor_si512(c2[7755],_mm512_xor_si512(c2[13296],_mm512_xor_si512(c2[2741],_mm512_xor_si512(c2[4071],_mm512_xor_si512(c2[1165],_mm512_xor_si512(c2[4601],_mm512_xor_si512(c2[9350],_mm512_xor_si512(c2[13322],_mm512_xor_si512(c2[5136],_mm512_xor_si512(c2[5928],_mm512_xor_si512(c2[16769],_mm512_xor_si512(c2[15182],_mm512_xor_si512(c2[927],_mm512_xor_si512(c2[9388],_mm512_xor_si512(c2[10707],_mm512_xor_si512(c2[15193],_mm512_xor_si512(c2[14679],_mm512_xor_si512(c2[7021],_mm512_xor_si512(c2[10455],_mm512_xor_si512(c2[11521],_mm512_xor_si512(c2[11261],_mm512_xor_si512(c2[10201],_mm512_xor_si512(c2[11005],_mm512_xor_si512(c2[14177],_mm512_xor_si512(c2[2033],_mm512_xor_si512(c2[16037],_mm512_xor_si512(c2[13923],_mm512_xor_si512(c2[723],_mm512_xor_si512(c2[1260],_mm512_xor_si512(c2[5748],_mm512_xor_si512(c2[16574],_mm512_xor_si512(c2[1274],_mm512_xor_si512(c2[746],_mm512_xor_si512(c2[8930],_mm512_xor_si512(c2[1813],_mm512_xor_si512(c2[7887],_mm512_xor_si512(c2[8413],_mm512_xor_si512(c2[7635],_mm512_xor_si512(c2[8428],_mm512_xor_si512(c2[15292],_mm512_xor_si512(c2[11871],_mm512_xor_si512(c2[780],c2[14775])))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))))));
+
+//row: 45
+     d2[270]=_mm512_xor_si512(c2[1862],_mm512_xor_si512(c2[5618],c2[4080]));
+  }
+}

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encode_parity_check.c

@@ -37,7 +37,11 @@
 #include "common/utils/LOG/log.h"
 
 
+#ifdef __AVX512F__
+#include "ldpc384_simd512_byte.c"
+#else
 #include "ldpc384_byte.c"
+#endif
 #include "ldpc384_byte_128.c"
 #include "ldpc352_byte.c"
 #include "ldpc352_byte_128.c"
@@ -87,7 +91,7 @@
 
 static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,short Zc,short Kb, int simd_size, int ncols)
 {
-  unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(32))); //double size matrix of c
+  unsigned char c[2 * 22 * Zc * simd_size] __attribute__((aligned(64))); //double size matrix of c
   for (int i1 = 0; i1 < ncols; i1++)   {
     memcpy(&c[2 * i1 * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
     memcpy(&c[(2 * i1 + 1) * Zc], &cc[i1 * Zc], Zc * sizeof(unsigned char));
@@ -95,7 +99,6 @@ static void encode_parity_check_part_optim(uint8_t *cc, uint8_t *d, short BG,sho
   for (int i1 = 1; i1 < simd_size; i1++) {
     memcpy(&c[(2 * ncols * Zc * i1)], &c[i1], (2 * ncols * Zc * sizeof(unsigned char)) - i1);
   }
-  
   if (BG == 1) {
     switch (Zc) {
       case 176:

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder.c

@@ -41,11 +41,11 @@
 #include "openair1/PHY/CODING/nrLDPC_extern.h"
 #include "ldpc_generate_coefficient.c"
 
-int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder_implemparams_t *impp)
+int LDPCencoder(unsigned char **inputArray, unsigned char *outputArray, encoder_implemparams_t *impp)
 {
   const unsigned char *input = inputArray[0];
   // channel input is the output of this function!
-  unsigned char *output = outputArray[0];
+  unsigned char *output = outputArray;
   const int Zc = impp->Zc;
   const int Kb = impp->Kb;
   const short block_length = impp->K;
@@ -114,7 +114,13 @@ int LDPCencoder(unsigned char **inputArray, unsigned char **outputArray, encoder
     fprintf(fd,"#include \"PHY/sse_intrin.h\"\n");
     fprintf(fd2,"#include \"PHY/sse_intrin.h\"\n");
 
-    if (gen_code == 1 && (Zc&31)==0) {
+    if (gen_code == 1 && (Zc&63)==0) {
+      shift=6;
+      mask=63;
+      strcpy(data_type,"__m512i");
+      strcpy(xor_command,"_mm512_xor_si512");
+    }
+    else if (gen_code == 1 && (Zc&31)==0) {
       shift=5; // AVX2 - 256-bit SIMD
       mask=31;
       strcpy(data_type,"simde__m256i");

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim.c

@@ -1,128 +0,0 @@
-/*
- * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The OpenAirInterface Software Alliance licenses this file to You under
- * the OAI Public License, Version 1.1  (the "License"); you may not use this file
- * except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.openairinterface.org/?page_id=698
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *-------------------------------------------------------------------------------
- * For more information about the OpenAirInterface (OAI) Software Alliance:
- *      contact@openairinterface.org
- */
-
-/*! \file ldpc_encoder_optim.c
- * \brief Defines the optimized LDPC encoder
- * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
- * \email openair_tech@eurecom.fr
- * \date 27-03-2018
- * \version 1.0
- * \note
- * \warning
- */
-
-#include <stdlib.h>
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
-#include "assertions.h"
-#include "common/utils/LOG/log.h"
-#include "time_meas.h"
-#include "openair1/PHY/CODING/nrLDPC_defs.h"
-#include "openair1/PHY/CODING/nrLDPC_extern.h"
-#include "ldpc_encode_parity_check.c"
-#include "ldpc_generate_coefficient.c"
-
-int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp)
-{
-  int Zc = impp->Zc;
-  int Kb = impp->Kb;
-  int block_length = impp->K;
-  int BG = impp->BG;
-  int nrows=0,ncols=0;
-  int rate=3;
-  int no_punctured_columns,removed_bit;
-
-  int simd_size;
-
-  //determine number of bits in codeword
-   //if (block_length>3840)
-   if (BG==1)
-     {
-       //BG=1;
-       nrows=46; //parity check bits
-       ncols=22; //info bits
-       rate=3;
-     }
-     //else if (block_length<=3840)
-    else if	(BG==2)
-     {
-       //BG=2;
-       nrows=42; //parity check bits
-       ncols=10; // info bits
-       rate=5;
-
-       }
-
-
-#ifdef DEBUG_LDPC
-  LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d\n",BG,Zc,Kb,block_length);
-  LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]);
-#endif
-
-  if ((Zc&31) > 0) simd_size = 16;
-  else simd_size = 32;
-
-  uint8_t c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
-  uint8_t d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
-
-  // calculate number of punctured bits
-  no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
-  removed_bit = (nrows - no_punctured_columns - 2) * Zc + block_length - block_length * rate;
-  // printf("%d\n",no_punctured_columns);
-  // printf("%d\n",removed_bit);
-  // unpack input
-  memset(c, 0, sizeof(c));
-  memset(d, 0, sizeof(d));
-
-  if(impp->tinput != NULL) start_meas(impp->tinput);
-  for (int i=0; i<block_length; i++) {
-    c[i] = (test_input[0][i/8]&(128>>(i&7)))>>(7-(i&7));
-      //printf("c(%d,%d)=%d\n",j,i,temp);
-    }
-
-  if(impp->tinput != NULL) stop_meas(impp->tinput);
-
-  if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
-    // extend matrix
-    if(impp->tprep != NULL) start_meas(impp->tprep);
-    if(impp->tprep != NULL) stop_meas(impp->tprep);
-    //parity check part
-    if(impp->tparity != NULL) start_meas(impp->tparity);
-    encode_parity_check_part_optim(c, d, BG, Zc, Kb,simd_size, ncols);
-    if(impp->tparity != NULL) stop_meas(impp->tparity);
-  }
-  else {
-    if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) {
-      printf("Problem with encoder\n");
-      return(-1);
-    }
-  }
-  if(impp->toutput != NULL) start_meas(impp->toutput);
-  // information part and puncture columns
-  memcpy(&channel_input[0][0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
-  memcpy(&channel_input[0][block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
-
-  if(impp->toutput != NULL) stop_meas(impp->toutput);
-  return 0;
-}
-
-

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8seg.c

@@ -1,201 +0,0 @@
-/*
- * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The OpenAirInterface Software Alliance licenses this file to You under
- * the OAI Public License, Version 1.1  (the "License"); you may not use this file
- * except in compliance with the License.
- * You may obtain a copy of the License at
- *
- *      http://www.openairinterface.org/?page_id=698
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- *-------------------------------------------------------------------------------
- * For more information about the OpenAirInterface (OAI) Software Alliance:
- *      contact@openairinterface.org
- */
-
-/*! \file ldpc_encoder_optim8seg.c
- * \brief Defines the optimized LDPC encoder
- * \author Florian Kaltenberger, Raymond Knopp, Kien le Trung (Eurecom)
- * \email openair_tech@eurecom.fr
- * \date 27-03-2018
- * \version 1.0
- * \note
- * \warning
- */
-
-#include <stdlib.h>
-#include <math.h>
-#include <stdio.h>
-#include <string.h>
-#include "assertions.h"
-#include "common/utils/LOG/log.h"
-#include "time_meas.h"
-#include "openair1/PHY/CODING/nrLDPC_defs.h"
-#include "openair1/PHY/CODING/nrLDPC_extern.h"
-#include "ldpc_encode_parity_check.c" 
-#include "ldpc_generate_coefficient.c"
-#include "PHY/sse_intrin.h"
-
-int LDPCencoder(uint8_t **test_input, uint8_t **channel_input, encoder_implemparams_t *impp)
-{
-  int Zc = impp->Zc;
-  int Kb = impp->Kb;
-  short block_length = impp->K;
-  short BG = impp->BG;
-
-  short nrows=0,ncols=0;
-  int i,i1,j,rate=3;
-  int no_punctured_columns,removed_bit;
-  char temp;
-  int simd_size;
-
-  simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
-  simde__m256i andmask  = simde_mm256_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
-  simde__m256i zero256   = simde_mm256_setzero_si256();
-  simde__m256i masks[8];
-  register simde__m256i c256;
-  masks[0] = simde_mm256_set1_epi8(0x1);
-  masks[1] = simde_mm256_set1_epi8(0x2);
-  masks[2] = simde_mm256_set1_epi8(0x4);
-  masks[3] = simde_mm256_set1_epi8(0x8);
-  masks[4] = simde_mm256_set1_epi8(0x10);
-  masks[5] = simde_mm256_set1_epi8(0x20);
-  masks[6] = simde_mm256_set1_epi8(0x40);
-  masks[7] = simde_mm256_set1_epi8(0x80);
-
-  AssertFatal((impp->n_segments>0&&impp->n_segments<=8),"0 < n_segments %d <= 8\n",impp->n_segments);
-
-  //determine number of bits in codeword
-  //if (block_length>3840)
-  if (BG==1)
-    {
-      nrows=46; //parity check bits
-      ncols=22; //info bits
-      rate=3;
-    }
-    //else if (block_length<=3840)
-   else if	(BG==2)
-    {
-      //BG=2;
-      nrows=42; //parity check bits
-      ncols=10; // info bits
-      rate=5;
-
-      }
-
-#ifdef DEBUG_LDPC
-  LOG_D(PHY,"ldpc_encoder_optim_8seg: BG %d, Zc %d, Kb %d, block_length %d, segments %d\n",BG,Zc,Kb,block_length,n_segments);
-  LOG_D(PHY,"ldpc_encoder_optim_8seg: PDU (seg 0) %x %x %x %x\n",test_input[0][0],test_input[0][1],test_input[0][2],test_input[0][3]);
-#endif
-
-  AssertFatal(Zc>0,"no valid Zc found for block length %d\n",block_length);
-
-  if ((Zc&31) > 0) simd_size = 16;
-  else          simd_size = 32;
-
-  unsigned char c[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
-  unsigned char d[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
-
-
-  // calculate number of punctured bits
-  no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
-  removed_bit=(nrows-no_punctured_columns-2) * Zc+block_length-(int)(block_length*rate);
-  // printf("%d\n",no_punctured_columns);
-  // printf("%d\n",removed_bit);
-  // unpack input
-  memset(c,0,sizeof(unsigned char) * ncols * Zc);
-  memset(d,0,sizeof(unsigned char) * nrows * Zc);
-
-  if(impp->tinput != NULL) start_meas(impp->tinput);
-#if 0
-  for (i=0; i<block_length; i++) {
-    for (j=0; j<n_segments; j++) {
-
-      temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
-      //printf("c(%d,%d)=%d\n",j,i,temp);
-      c[i] |= (temp << j);
-    }
-  }
-#else
-  for (i=0; i<block_length>>5; i++) {
-    c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[0])[i]), shufmask),andmask),zero256),masks[0]);
-    for (j=1; j<impp->n_segments; j++) {
-      c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)test_input[j])[i]), shufmask),andmask),zero256),masks[j]),c256);
-    }
-    ((simde__m256i *)c)[i] = c256;
-  }
-
-  for (i=(block_length>>5)<<5;i<block_length;i++) {
-    for (j=0; j<impp->n_segments; j++) {
-
-      temp = (test_input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
-      //printf("c(%d,%d)=%d\n",j,i,temp);
-      c[i] |= (temp << j);
-    }
-  }
-#endif
-
-  if(impp->tinput != NULL) stop_meas(impp->tinput);
-
-  if ((BG==1 && Zc>=176) || (BG==2 && Zc>=72)) {
-    // extend matrix
-    if(impp->tprep != NULL) start_meas(impp->tprep);
-    if(impp->tprep != NULL) stop_meas(impp->tprep);
-    //parity check part
-    if(impp->tparity != NULL) start_meas(impp->tparity);
-    encode_parity_check_part_optim(c, d, BG, Zc, Kb, simd_size, ncols);
-    if(impp->tparity != NULL) stop_meas(impp->tparity);
-  }
-  else {
-    if (encode_parity_check_part_orig(c, d, BG, Zc, Kb, block_length)!=0) {
-      printf("Problem with encoder\n");
-      return(-1);
-    }
-  }
-  if(impp->toutput != NULL) start_meas(impp->toutput);
-  // information part and puncture columns
-  /*
-  memcpy(&channel_input[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
-  memcpy(&channel_input[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
-  */
-  if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) {
-    //AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n");
-    //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n");
-    uint32_t l1 = (block_length-(2*Zc))>>5;
-    uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5;
-    simde__m256i *c256p = (simde__m256i *)&c[2*Zc];
-    simde__m256i *d256p = (simde__m256i *)&d[0];
-    //  if (((block_length-(2*Zc))&31)>0) l1++;
-    
-    for (i=0;i<l1;i++)
-      for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]);
-    
-    //  if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++;
-    
-    for (i1=0;i1<l2;i1++,i++)
-      for (j=0;j<impp->n_segments;j++) ((simde__m256i *)channel_input[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]);
-  }
-  else {
-#ifdef DEBUG_LDPC
-  LOG_W(PHY,"using non-optimized version\n");
-#endif
-    // do non-SIMD version
-    for (i=0;i<(block_length-2*Zc);i++) 
-      for (j=0; j<impp->n_segments; j++)
-	channel_input[j][i] = (c[2*Zc+i]>>j)&1;
-    for (i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++)
-      for (j=0; j<impp->n_segments; j++)
-	channel_input[j][block_length-2*Zc+i] = (d[i]>>j)&1;
-    }
-
-  if(impp->toutput != NULL) stop_meas(impp->toutput);
-  return 0;
-}
-
-

openair1/PHY/CODING/nrLDPC_encoder/ldpc_encoder_optim8segmulti.c

@@ -43,7 +43,7 @@
 #include "ldpc_encode_parity_check.c"
 #include "ldpc_generate_coefficient.c"
 
-int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
+int LDPCencoder(uint8_t **input, uint8_t *output, encoder_implemparams_t *impp)
 {
   //set_log(PHY, 4);
 
@@ -61,27 +61,11 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
   unsigned int macro_segment, macro_segment_end;
 
   
-  macro_segment = 8*impp->macro_num;
-  macro_segment_end = (impp->n_segments > 8*(impp->macro_num+1)) ? 8*(impp->macro_num+1) : impp->n_segments;
+  macro_segment = impp->first_seg;
+  macro_segment_end = (impp->n_segments > impp->first_seg + 8) ? impp->first_seg + 8 : impp->n_segments;
   ///printf("macro_segment: %d\n", macro_segment);
   ///printf("macro_segment_end: %d\n", macro_segment_end );
 
-  simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
-  simde__m256i andmask  = simde_mm256_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
-  simde__m256i zero256   = simde_mm256_setzero_si256();
-  simde__m256i masks[8];
-  register simde__m256i c256;
-  masks[0] = simde_mm256_set1_epi8(0x1);
-  masks[1] = simde_mm256_set1_epi8(0x2);
-  masks[2] = simde_mm256_set1_epi8(0x4);
-  masks[3] = simde_mm256_set1_epi8(0x8);
-  masks[4] = simde_mm256_set1_epi8(0x10);
-  masks[5] = simde_mm256_set1_epi8(0x20);
-  masks[6] = simde_mm256_set1_epi8(0x40);
-  masks[7] = simde_mm256_set1_epi8(0x80);
-
-
-
   //determine number of bits in codeword
   if (BG==1)
     {
@@ -103,10 +87,12 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
 
   AssertFatal(Zc > 0, "no valid Zc found for block length %d\n", block_length);
   if ((Zc&31) > 0) simd_size = 16;
-  else
-    simd_size = 32;
-  unsigned char cc[22*Zc] __attribute__((aligned(32))); //padded input, unpacked, max size
-  unsigned char dd[46*Zc] __attribute__((aligned(32))); //coded parity part output, unpacked, max size
+#ifdef __AVX512F__
+  else if ((BG==1) && Zc==384) simd_size =64;
+#endif
+  else simd_size=32;
+  unsigned char cc[22*Zc] __attribute__((aligned(64))); //padded input, unpacked, max size
+  unsigned char dd[46*Zc] __attribute__((aligned(64))); //coded parity part output, unpacked, max size
 
   // calculate number of punctured bits
   no_punctured_columns=(int)((nrows-2)*Zc+block_length-block_length*rate)/Zc;
@@ -114,40 +100,95 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
   //printf("%d\n",no_punctured_columns);
   //printf("%d\n",removed_bit);
   // unpack input
-  memset(cc,0,sizeof(unsigned char) * ncols * Zc);
-  memset(dd,0,sizeof(unsigned char) * nrows * Zc);
+  memset(cc,0,sizeof(cc));
+  memset(dd,0,sizeof(dd));
 
   if(impp->tinput != NULL) start_meas(impp->tinput);
-#if 0
-  for (i=0; i<block_length; i++) {
-	//for (j=0; j<n_segments; j++) {
-    for (j=macro_segment; j < macro_segment_end; j++) {
+  //interleave up to 8 transport-block segements at a time
 
-      temp = (input[j][i/8]&(1<<(i&7)))>>(i&7);
-      //printf("c(%d,%d)=%d\n",j,i,temp);
-      c[i] |= (temp << (j-macro_segment));
+  unsigned int i_byte = 0;
+
+#if defined(__AVX512F__) && defined(__AVX512BW__) && defined(__AVX512VBMI__)
+  const __m512i masks5[8] = { _mm512_set1_epi8(0x1), _mm512_set1_epi8(0x2),
+                              _mm512_set1_epi8(0x4), _mm512_set1_epi8(0x8),
+                              _mm512_set1_epi8(0x10), _mm512_set1_epi8(0x20),
+                              _mm512_set1_epi8(0x40), _mm512_set1_epi8(0x80)};
+  const __m512i zero512 = _mm512_setzero_si512();
+  const uint8_t perm[64]__attribute__((aligned(64))) = {7,6,5,4,3,2,1,0,         15,14,13,12,11,10,9,8,
+                                                        23,22,21,20,19,18,17,16, 31,30,29,28,27,26,25,24,
+                                                        39,38,37,36,35,34,33,32, 47,46,45,44,43,42,41,40,
+                                                        55,54,53,52,51,50,49,48, 63,62,61,60,59,58,57,56};
+  register __m512i c512;
+
+  for (; i_byte < ((block_length >> 6) << 6); i_byte += 64) {
+    unsigned int i = i_byte >> 6;
+    c512 = _mm512_mask_blend_epi8(((uint64_t *)&input[macro_segment][0])[i], zero512, masks5[0]);
+    for (int j = macro_segment + 1; j < macro_segment_end; j++) {
+      c512 = _mm512_or_si512(c512, _mm512_mask_blend_epi8(((uint64_t *)&input[j][0])[i], zero512, masks5[j - macro_segment]));
     }
+    c512 = _mm512_permutexvar_epi8(*(__m512i*)perm, c512);
+    ((__m512i *)cc)[i] = c512;
   }
-#else
-  for (int i=0; i<block_length>>5; i++) {
+#endif
+
+#ifndef __aarch64__
+  simde__m256i shufmask = simde_mm256_set_epi64x(0x0303030303030303, 0x0202020202020202,0x0101010101010101, 0x0000000000000000);
+  simde__m256i andmask  = simde_mm256_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
+  simde__m256i zero256   = simde_mm256_setzero_si256();
+  simde__m256i masks[8];
+  register simde__m256i c256;
+  masks[0] = simde_mm256_set1_epi8(0x1);
+  masks[1] = simde_mm256_set1_epi8(0x2);
+  masks[2] = simde_mm256_set1_epi8(0x4);
+  masks[3] = simde_mm256_set1_epi8(0x8);
+  masks[4] = simde_mm256_set1_epi8(0x10);
+  masks[5] = simde_mm256_set1_epi8(0x20);
+  masks[6] = simde_mm256_set1_epi8(0x40);
+  masks[7] = simde_mm256_set1_epi8(0x80);
+
+  for (; i_byte < ((block_length >> 5 ) << 5); i_byte += 32) {
+    unsigned int i = i_byte >> 5;
     c256 = simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[macro_segment])[i]), shufmask),andmask),zero256),masks[0]);
-    //for (j=1; j<n_segments; j++) {
     for (int j=macro_segment+1; j < macro_segment_end; j++) {    
       c256 = simde_mm256_or_si256(simde_mm256_and_si256(simde_mm256_cmpeq_epi8(simde_mm256_andnot_si256(simde_mm256_shuffle_epi8(simde_mm256_set1_epi32(((uint32_t*)input[j])[i]), shufmask),andmask),zero256),masks[j-macro_segment]),c256);
     }
     ((simde__m256i *)cc)[i] = c256;
   }
+#endif
 
-  for (int i=(block_length>>5)<<5;i<block_length;i++) {
-    //for (j=0; j<n_segments; j++) {
-	  for (int j=macro_segment; j < macro_segment_end; j++) {
+#ifdef __aarch64__
+  simde__m128i shufmask = simde_mm_set_epi64x(0x0101010101010101, 0x0000000000000000);
+  simde__m128i andmask  = simde_mm_set1_epi64x(0x0102040810204080);  // every 8 bits -> 8 bytes, pattern repeats.
+  simde__m128i zero128   = simde_mm_setzero_si128();
+  simde__m128i masks[8];
+  register simde__m128i c128;
+  masks[0] = simde_mm_set1_epi8(0x1);
+  masks[1] = simde_mm_set1_epi8(0x2);
+  masks[2] = simde_mm_set1_epi8(0x4);
+  masks[3] = simde_mm_set1_epi8(0x8);
+  masks[4] = simde_mm_set1_epi8(0x10);
+  masks[5] = simde_mm_set1_epi8(0x20);
+  masks[6] = simde_mm_set1_epi8(0x40);
+  masks[7] = simde_mm_set1_epi8(0x80);
 
-	    temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
-      //printf("c(%d,%d)=%d\n",j,i,temp);
+  for (; i_byte < ((block_length >> 4 ) << 4); i_byte += 16) {
+    unsigned int i = i_byte >> 4;
+    c128 = simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi16(((uint16_t*)input[macro_segment])[i]), shufmask),andmask),zero128),masks[0]);
+    for (int j=macro_segment+1; j < macro_segment_end; j++) {    
+      c128 = simde_mm_or_si128(simde_mm_and_si128(simde_mm_cmpeq_epi8(simde_mm_andnot_si128(simde_mm_shuffle_epi8(simde_mm_set1_epi32(((uint16_t*)input[j])[i]), shufmask),andmask),zero128),masks[j-macro_segment]),c128);
+    }
+    ((simde__m128i *)cc)[i] = c128;
+  }
+#endif
+
+  for (; i_byte < block_length; i_byte++) {
+    unsigned int i = i_byte;
+    for (int j = macro_segment; j < macro_segment_end; j++) {
+
+      temp = (input[j][i/8]&(128>>(i&7)))>>(7-(i&7));
       cc[i] |= (temp << (j-macro_segment));
     }
   }
-#endif
 
   if(impp->tinput != NULL) stop_meas(impp->tinput);
 
@@ -166,48 +207,8 @@ int LDPCencoder(uint8_t **input, uint8_t **output, encoder_implemparams_t *impp)
       return(-1);
     }
   }
-  if(impp->toutput != NULL) start_meas(impp->toutput);
-  // information part and puncture columns
-  /*
-  memcpy(&output[0], &c[2*Zc], (block_length-2*Zc)*sizeof(unsigned char));
-  memcpy(&output[block_length-2*Zc], &d[0], ((nrows-no_punctured_columns) * Zc-removed_bit)*sizeof(unsigned char));
-  */
-  if ((((2*Zc)&31) == 0) && (((block_length-(2*Zc))&31) == 0)) {
-    //AssertFatal(((2*Zc)&31) == 0,"2*Zc needs to be a multiple of 32 for now\n");
-    //AssertFatal(((block_length-(2*Zc))&31) == 0,"block_length-(2*Zc) needs to be a multiple of 32 for now\n");
-    uint32_t l1 = (block_length-(2*Zc))>>5;
-    uint32_t l2 = ((nrows-no_punctured_columns) * Zc-removed_bit)>>5;
-    simde__m256i *c256p = (simde__m256i *)&cc[2*Zc];
-    simde__m256i *d256p = (simde__m256i *)&dd[0];
-    //  if (((block_length-(2*Zc))&31)>0) l1++;
-
-    for (int i=0;i<l1;i++)
-      //for (j=0;j<n_segments;j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j),masks[0]);
-    	for (int j=macro_segment; j < macro_segment_end; j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(c256p[i],j-macro_segment),masks[0]);
-
-
-    //  if ((((nrows-no_punctured_columns) * Zc-removed_bit)&31)>0) l2++;
-
-    for (int i1=0, i=l1;i1<l2;i1++,i++)
-      //for (j=0;j<n_segments;j++) ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j),masks[0]);
-    	for (int j=macro_segment; j < macro_segment_end; j++)  ((simde__m256i *)output[j])[i] = simde_mm256_and_si256(simde_mm256_srai_epi16(d256p[i1],j-macro_segment),masks[0]);
-  }
-  else {
-#ifdef DEBUG_LDPC
-  LOG_W(PHY,"using non-optimized version\n");
-#endif
-    // do non-SIMD version
-    for (int i=0;i<(block_length-2*Zc);i++)
-      //for (j=0; j<n_segments; j++)
-      for (int j=macro_segment; j < macro_segment_end; j++)
-	output[j][i] = (cc[2*Zc+i]>>(j-macro_segment))&1;
-    for (int i=0;i<((nrows-no_punctured_columns) * Zc-removed_bit);i++)
-      //for (j=0; j<n_segments; j++)
-    	  for (int j=macro_segment; j < macro_segment_end; j++)
-	output[j][block_length-2*Zc+i] = (dd[i]>>(j-macro_segment))&1;
-    }
-
-  if(impp->toutput != NULL) stop_meas(impp->toutput);
+  memcpy(output,&cc[2*Zc],(block_length-(2*Zc)));
+  memcpy(output+block_length-(2*Zc),dd,((nrows-no_punctured_columns) * Zc-removed_bit));
   return 0;
 }
 

openair1/PHY/CODING/nr_segmentation.c

@@ -151,13 +151,8 @@ else
     for (r=0; r<*C; r++) {
 
       k = 0;
-
-      while (k<((Kprime - L)>>3)) {
-        output_buffers[r][k] = input_buffer[s];
-		//printf("encoding segment %d : byte %d (%d) => %d\n",r,k,(Kprime-L)>>3,input_buffer[s]);
-        k++;
-        s++;
-      }
+      memcpy(output_buffers[r],input_buffer+s,(Kprime-L)>>3);
+      s+=(Kprime-L)>>3;
 
       if (*C > 1) { // add CRC
         crc = crc24b(output_buffers[r],Kprime-L)>>8;

openair1/PHY/NR_TRANSPORT/nr_dlsch.c

@@ -600,14 +600,14 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
   printf("\nEncoded payload:\n");
   for (int i = 0; i < encoded_length; i += 8) {
     for (int j = 0; j < 8; j++)
-      printf("%d", input_ptr[i + j]);
+      printf("%d", (input_ptr[i >> 3] >> j) & 1);
     printf("\t");
   }
   printf("\n");
 #endif
 
   if (IS_SOFTMODEM_DLSIM)
-    memcpy(harq->f, input_ptr, encoded_length);
+    memcpy(harq->f, input_ptr, (encoded_length + 7) >> 3);
 
   c16_t mod_symbs[rel15->NrOfCodewords][encoded_length] __attribute__((aligned(64)));
   for (int codeWord = 0; codeWord < rel15->NrOfCodewords; codeWord++) {
@@ -763,11 +763,11 @@ static int do_one_dlsch(unsigned char *input_ptr, PHY_VARS_gNB *gNB, NR_gNB_DLSC
   }
 
   stop_meas(&gNB->dlsch_precoding_stats);
-  /* output and its parts for each dlsch should be aligned on 64 bytes
-   * should remain a multiple of 64 with enough offset to fit each dlsch
+  /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
+   * should remain a multiple of 8 * 64 with enough offset to fit each dlsch
    */
   uint32_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers;
-  return ceil_mod(size_output_tb,64);
+  return ((size_output_tb + 511) >> 9) << 6;
 }
 
 void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
@@ -817,15 +817,15 @@ void nr_generate_pdsch(processingData_L1tx_t *msgTx, int frame, int slot)
     /// CRC, coding, interleaving and rate matching
     AssertFatal(harq->pdu != NULL, "%4d.%2d no HARQ PDU for PDSCH generation\n", msgTx->frame, msgTx->slot);
 
-    /* output and its parts for each dlsch should be aligned on 64 bytes
-     * => size_output is a sum of parts sizes rounded up to a multiple of 64
+    /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
+     * => size_output is a sum of parts sizes rounded up to a multiple of 8 * 64
      */
     size_t size_output_tb = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * Qm * rel15->nrOfLayers;
-    size_output += ceil_mod(size_output_tb, 64);
+    size_output += ceil_mod(size_output_tb, 8 * 64);
   }
 
-  unsigned char output[size_output] __attribute__((aligned(64)));
-  bzero(output, size_output);
+  unsigned char output[size_output >> 3] __attribute__((aligned(64)));
+  bzero(output, sizeof(output));
 
   start_meas(dlsch_encoding_stats);
   if (nr_dlsch_encoding(gNB,

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -265,15 +265,13 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
 
     TB_parameters->tbslbrm = rel15->maintenance_parms_v3.tbSizeLbrmBytes;
 
+    TB_parameters->output = &output[dlsch_offset >> 3];
     TB_parameters->segments = &segments[segments_offset];
 
-    size_t r_offset = 0;
     for (int r = 0; r < TB_parameters->C; r++) {
       nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
       segment_parameters->c = harq->c[r];
       segment_parameters->E = nr_get_E(TB_parameters->G, TB_parameters->C, TB_parameters->Qm, rel15->nrOfLayers, r);
-      segment_parameters->output = &output[dlsch_offset + r_offset];
-      r_offset += segment_parameters->E;
 
       reset_meas(&segment_parameters->ts_interleave);
       reset_meas(&segment_parameters->ts_rate_match);
@@ -282,11 +280,11 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
 
     segments_offset += TB_parameters->C;
 
-    /* output and its parts for each dlsch should be aligned on 64 bytes
-     * => dlsch_offset should remain a multiple of 64 with enough offset to fit each dlsch
+    /* output and its parts for each dlsch should be aligned on 64 bytes (or 8 * 64 bits)
+     * => dlsch_offset should remain a multiple of 8 * 64 with enough offset to fit each dlsch
      */
     const size_t dlsch_size = rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * rel15->qamModOrder[0] * rel15->nrOfLayers;
-    dlsch_offset += ceil_mod(dlsch_size, 64);
+    dlsch_offset += ceil_mod(dlsch_size, 8 * 64);
   }
 
   gNB->nrLDPC_coding_interface.nrLDPC_coding_encoder(&slot_parameters);

openair1/PHY/NR_TRANSPORT/nr_scrambling.c

@@ -33,11 +33,26 @@ void nr_codeword_scrambling(uint8_t *in,
 {
   const int roundedSz = (size + 31) / 32;
   uint32_t *seq = gold_cache((n_RNTI << 15) + (q << 14) + Nid, roundedSz);
-  for (int i = 0; i < roundedSz; i++) {
-    simde__m256i c = ((simde__m256i*)in)[i];
-    uint32_t in32 = simde_mm256_movemask_epi8(simde_mm256_slli_epi16(c, 7));
-    out[i] = in32 ^ seq[i];
-    DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, in32, out[i]));
+  unsigned int i_32 = 0;
+#if defined(__AVX512F__) && defined(__AVX512VL__)
+  for (; i_32 < ((roundedSz >> 3) << 3); i_32 += 8) {
+    __m256i in_256 = _mm256_load_epi32(&((uint32_t *)in)[i_32]);
+    __m256i seq_256 = _mm256_load_epi32(&seq[i_32]);
+    _mm256_storeu_epi32(&out[i_32], _mm256_xor_si256(in_256, seq_256));
+  }
+#endif
+#if defined(__aarch64__)
+  for (; i_32 < ((roundedSz >> 2) << 2); i_32 += 4) {
+    unsigned int i = i_32 >> 2;
+    uint32x4_t in_32x4 = vld1q_u32(&((uint32_t *)in)[i_32]);
+    uint32x4_t seq_32x4 = vld1q_u32(&seq[i_32]);
+    ((uint32x4_t *)out)[i] = veorq_u32(in_32x4, seq_32x4);
+  }
+#endif
+  for (; i_32 < roundedSz; i_32++) { 
+    unsigned int i = i_32;
+    out[i] = ((uint32_t *)in)[i] ^ seq[i];
+    DEBUG_SCRAMBLING(LOG_D(PHY, "in[%d] %x => %x\n", i, ((uint32_t*)in)[i], out[i]));
   }
 }
 

openair1/PHY/NR_UE_TRANSPORT/nr_ulsch_coding.c

@@ -156,7 +156,11 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue,
     NR_UL_UE_HARQ_t *harq_process = &ue->ul_harq_processes[harq_pid];
     TB_parameters->segments = segments[pusch_id];
 
-    int r_offset = 0;
+    const nfapi_nr_ue_pusch_pdu_t *pusch_pdu = &ulsch->pusch_pdu;
+    uint16_t nb_rb = pusch_pdu->rb_size;
+    memset(harq_process->f, 0, 14 * nb_rb * 12 * 16);
+    TB_parameters->output = harq_process->f;
+
     for (int r = 0; r < TB_parameters->C; r++) {
       nrLDPC_segment_encoding_parameters_t *segment_parameters = &TB_parameters->segments[r];
       segment_parameters->c = harq_process->c[r];
@@ -165,8 +169,6 @@ int nr_ulsch_encoding(PHY_VARS_NR_UE *ue,
                                             TB_parameters->Qm,
                                             TB_parameters->nb_layers,
                                             r);
-      segment_parameters->output = harq_process->f + r_offset;
-      r_offset += segment_parameters->E;
 
       reset_meas(&segment_parameters->ts_interleave);
       reset_meas(&segment_parameters->ts_rate_match);

openair1/SIMULATION/NR_PHY/dlschsim.c

@@ -516,8 +516,8 @@ int main(int argc, char **argv)
 
 	//printf("crc32: [0]->0x%08x\n",crc24c(test_input, 32));
 	// generate signal
-        unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS] __attribute__((aligned(32)));
-        bzero(output,rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * 8 * NR_MAX_NB_LAYERS);
+        unsigned char output[rel15->rbSize * NR_SYMBOLS_PER_SLOT * NR_NB_SC_PER_RB * NR_MAX_NB_LAYERS] __attribute__((aligned(64)));
+        bzero(output, sizeof(output));
 	if (input_fd == NULL) {
     msgDataTx.num_pdsch_slot = 1;
 	  nr_dlsch_encoding(gNB, &msgDataTx, frame, slot, frame_parms, output, NULL, NULL, NULL, NULL, NULL, NULL, NULL);
@@ -537,12 +537,12 @@ int main(int argc, char **argv)
 				//if (i<16)
 				//   printf("encoder output f[%d] = %d\n",i,dlsch->harq_processes[0]->f[i]);
 
-				if (output[i] == 0)
+				if ((output[i >> 3] & (1 << (i & 7))) == 0)
 					modulated_input[i] = 1.0;        ///sqrt(2);  //QPSK
 				else
 					modulated_input[i] = -1.0;        ///sqrt(2);
 
-				//if (i<16) printf("modulated_input[%d] = %d\n",i,modulated_input[i]);
+				//if (i<16) printf("modulated_input[%d] = %f (%d)\n",i,modulated_input[i],output[i]);
 				//SNR =10;
 				SNR_lin = pow(10, SNR / 10.0);
 				sigma = 1.0 / sqrt(2 * SNR_lin);

openair1/SIMULATION/NR_PHY/ulschsim.c

@@ -546,7 +546,7 @@ int main(int argc, char **argv)
             }
         */
 
-        if (harq_process_ul_ue->f[i] == 0)
+        if ((harq_process_ul_ue->f[i >> 3] & (1 << (i & 7))) == 0)
           modulated_input[i] = 1.0;        ///sqrt(2);  //QPSK
         else
           modulated_input[i] = -1.0;        ///sqrt(2);

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -251,7 +251,9 @@ int main(int argc, char *argv[])
   InitSinLUT();
 
   int c;
-  while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:")) != -1) {
+  bool setAffinity=false;
+  char gNBthreads[128]="n";
+  while ((c = getopt(argc, argv, "--:O:a:b:c:d:ef:g:h:i:jk:m:n:p:q:r:s:t:u:v:w:y:z:A:C:F:G:H:I:M:N:PR:S:T:U:L:ZW:E:X:Y:")) != -1) {
     /* ignore long options starting with '--', option '-O' and their arguments that are handled by configmodule */
     /* with this opstring getopt returns 1 for non-option arguments, refer to 'man 3 getopt' */
     if (c == 1 || c == '-' || c == 'O')
@@ -496,6 +498,13 @@ int main(int argc, char *argv[])
       AssertFatal(filename_csv != NULL, "strdup() error: errno %d\n", errno);
       break;
 
+    case 'Y':
+      threadCnt = sizeof(gNBthreads)-1;
+      strncpy(gNBthreads, optarg, threadCnt);
+      gNBthreads[threadCnt]=0;
+      setAffinity=true;
+      break;
+
     case 'Z':
       transform_precoding = transformPrecoder_enabled;
       num_dmrs_cdm_grps_no_data = 2;
@@ -594,7 +603,11 @@ int main(int argc, char *argv[])
   gNB->RU_list[0] = calloc(1, sizeof(**gNB->RU_list));
   gNB->RU_list[0]->rfdevice.openair0_cfg = openair0_cfg;
 
-  initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool");
+  if (setAffinity == false)
+    initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool");
+  else
+    initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool");
+
   initNotifiedFIFO(&gNB->respDecode);
 
   initNotifiedFIFO(&gNB->respPuschSymb);