rework threadpool management

This work was done by Mikel Irazabal.

The original branch was too outdated to rebase properly on top of
develop, plus the commit messages were not informative. Also, the
code was adapted a bit to fit the project (no use of assert(),
clang formatting, change some names which were not easy to
understand).

CMakeLists.txt

@@ -38,7 +38,7 @@ include("cmake_targets/macros.cmake")
 option(CCACHE_ACTIVE "CCache" ON)
 find_program(CCACHE_FOUND ccache)
 if(CCACHE_FOUND AND CCACHE_ACTIVE)
-  if(${CMAKE_VERSION} VERSION_LESS "3.4.0") 
+  if(${CMAKE_VERSION} VERSION_LESS "3.4.0")
     set_property(GLOBAL PROPERTY RULE_LAUNCH_COMPILE "${CCACHE_PROGRAM}")
     message(STATUS "Found ccache in ${CCACHE_FOUND}. Using ccache. CMake < 3.4")
   else()
@@ -101,13 +101,13 @@ add_list_string_option(CMAKE_BUILD_TYPE "RelWithDebInfo" "Choose the type of bui
 
 # in case /proc/cpuinfo exists we want to inspect available Intrinsics
 # -so not to go always through SIMDE emulation
-# -so to avoid AVX512 instructions generation by gcc   
+# -so to avoid AVX512 instructions generation by gcc
 if(EXISTS "/proc/cpuinfo" AND NOT CROSS_COMPILE)
   file(STRINGS "/proc/cpuinfo" CPUFLAGS REGEX flags LIMIT_COUNT 1)
 else()
   message(WARNING "did not find /proc/cpuinfo -- not setting any x86-specific compilation variables")
 endif()
- 
+
 eval_boolean(AUTODETECT_AVX512 DEFINED CPUFLAGS AND CPUFLAGS MATCHES "avx512")
 add_boolean_option(AVX512 ${AUTODETECT_AVX512} "Whether AVX512 intrinsics is available on the host processor" ON)
 
@@ -633,6 +633,14 @@ include_directories(${OPENAIR_DIR}/common/utils/hashtable)
 add_library(UTIL
   ${OPENAIR_DIR}/common/utils/LOG/vcd_signal_dumper.c
   ${OPENAIR2_DIR}/UTIL/OPT/probe.c
+  ${OPENAIR_DIR}/common/utils/task_manager/task.c
+  ${OPENAIR_DIR}/common/utils/task_manager/task_ans.c
+  ${OPENAIR_DIR}/common/utils/task_manager/thread_pool/task_manager.c
+  ${OPENAIR_DIR}/common/utils/task_manager/threadPool/thread-pool.c
+  ${OPENAIR_DIR}/common/utils/utils.c
+  ${OPENAIR_DIR}/common/utils/system.c
+  ${OPENAIR_DIR}/common/utils/time_meas.c
+  ${OPENAIR_DIR}/common/utils/time_stat.c
 )
 if (ENABLE_LTTNG)
   find_package(LTTngUST 2.3.8 EXACT REQUIRED)
@@ -1228,7 +1236,7 @@ set(NR_SDAP_SRC
   ${OPENAIR2_DIR}/SDAP/nr_sdap/nr_sdap.c
   ${OPENAIR2_DIR}/SDAP/nr_sdap/nr_sdap_entity.c
   )
-  
+
 set(L2_SRC
   ${PDCP_DIR}/pdcp.c
   ${PDCP_DIR}/pdcp_fifo.c
@@ -1907,7 +1915,7 @@ add_executable(nfapi_test
   )
 
 add_executable(measurement_display
-  ${OPENAIR_DIR}/common/utils/threadPool/measurement_display.c)
+  ${OPENAIR_DIR}/common/utils/task_manager/threadPool/measurement_display.c)
 target_link_libraries (measurement_display minimal_lib)
 
 add_executable(test5Gnas
@@ -2077,7 +2085,7 @@ if(E2_AGENT)
   target_link_libraries(nr-cuup PRIVATE e2_agent e2_agent_arg e2_ran_func_cuup)
   target_compile_definitions(nr-cuup PRIVATE ${E2AP_VERSION} ${KPM_VERSION} E2_AGENT)
 endif()
-                     
+
 # nr-uesoftmodem is  UE implementation
 #######################################
 

common/utils/CMakeLists.txt

@@ -12,7 +12,7 @@ endif()
 add_subdirectory(T)
 add_subdirectory(nr)
 add_subdirectory(LOG)
-add_subdirectory(threadPool)
+add_subdirectory(task_manager/threadPool)
 add_library(utils utils.c system.c time_meas.c time_stat.c tun_if.c)
 target_include_directories(utils PUBLIC .)
 target_link_libraries(utils PRIVATE ${T_LIB})

common/utils/task_manager/README.md

@@ -0,0 +1,34 @@
+Task Manager
+=========================
+
+General
+-----------
+Task manager is a simple abstraction to handle tasks concurrently.
+It consists of the following 3 functions that can be found in
+task\_manager\_gen header file.
+
+1. init\_task\_manager
+  * Initialize the task manager. The first argument is the type, while the
+    second and third argument are a list of the cores where the threads
+    should be pinned to and its size. Only a range within the core id of the
+    machine is valid i.e., min. 0 max. output of the command \$nproc --all.
+    A -1 represents floating threads. There is no guarantee that the
+    underlying thread pool pins the threads to the cores.
+2.  free\_task\_manager
+  * Free the resources acquired by init. Terminate the running threads.
+3.  async\_task\_manager
+  * Asynchronously send a task to the task manager. The second argument is a
+    task that consists of a function pointer where the task will run and a
+    void* arg to the function. Similar syntax to c++ std::async.
+
+Joining tasks
+-----------
+
+For joining the tasks, a decoupled mechanism is also provided in the file
+task\_ans.h
+There are two methods:
+1. completed\_task\_ans
+  * Once the task is finished, it can itself announce that it finished
+2. join\_task\_ans
+  * This is a blocking join point. It will wait for all the tasks to be
+    completed before continuing.

common/utils/task_manager/task.c

@@ -0,0 +1,69 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#include "task.h"
+#include "assertions.h"
+#include <ctype.h>
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <string.h>
+#include <stdio.h>
+#include <sys/sysinfo.h>
+#include <time.h>
+
+// Compatibility with previous TPool
+void parse_num_threads(char const* params, span_core_id_t* out)
+{
+  DevAssert(params != NULL);
+
+  int const logical_cores = get_nprocs_conf();
+  DevAssert(logical_cores > 0);
+
+  char* saveptr = NULL;
+  char* params_cpy = strdup(params);
+  char* curptr = strtok_r(params_cpy, ",", &saveptr);
+
+  while (curptr != NULL) {
+    int const c = toupper(curptr[0]);
+
+    switch (c) {
+      case 'N': {
+        // pool->activated=false;
+        free(params_cpy);
+        out->core_id[out->sz++] = -1;
+        return;
+        break;
+      }
+
+      default: {
+        AssertFatal(out->sz != out->cap, "Capacity limit passed!. Please augment the span size");
+        int const core_id = atoi(curptr);
+        AssertFatal(core_id == -1 || core_id < logical_cores, "Invalid core ID passed");
+        out->core_id[out->sz++] = core_id;
+      }
+    }
+    curptr = strtok_r(NULL, ",", &saveptr);
+  }
+
+  free(params_cpy);
+}

common/utils/task_manager/task.h

@@ -0,0 +1,39 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#ifndef TASK_WORK_STEALING_THREAD_POOL_H
+#define TASK_WORK_STEALING_THREAD_POOL_H
+
+typedef struct {
+  void* args;
+  void (*func)(void* args);
+} task_t;
+
+// Compatibility with previous TPool
+typedef struct {
+  int* core_id;
+  int sz;
+  int const cap;
+} span_core_id_t;
+
+void parse_num_threads(char const* params, span_core_id_t* out);
+
+#endif

common/utils/task_manager/task_ans.c

@@ -0,0 +1,62 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#include "task_ans.h"
+#include "assertions.h"
+#include <limits.h>
+#include <stddef.h>
+#include <stdint.h>
+#include <stdlib.h>
+#include <time.h>
+
+void completed_task_ans(task_ans_t* task)
+{
+  DevAssert(task != NULL);
+
+  if (atomic_load_explicit(&task->status, memory_order_acquire) != 0)
+    AssertFatal(0, "Task already finished?");
+
+  atomic_store_explicit(&task->status, 1, memory_order_release);
+}
+
+void join_task_ans(task_ans_t* arr, size_t len)
+{
+  DevAssert(len < INT_MAX);
+  DevAssert(arr != NULL);
+
+  // Spin lock inspired by:
+  // The Art of Writing Efficient Programs:
+  // An advanced programmer's guide to efficient hardware utilization
+  // and compiler optimizations using C++ examples
+  const struct timespec ns = {0, 1};
+  uint64_t i = 0;
+  int j = len - 1;
+  for (; j != -1; i++) {
+    for (; j != -1; --j) {
+      int const task_completed = 1;
+      if (atomic_load_explicit(&arr[j].status, memory_order_acquire) != task_completed)
+        break;
+    }
+    if (i % 8 == 0) {
+      nanosleep(&ns, NULL);
+    }
+  }
+}

common/utils/task_manager/task_ans.h

@@ -0,0 +1,66 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#ifndef TASK_ANSWER_THREAD_POOL_H
+#define TASK_ANSWER_THREAD_POOL_H
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+#ifndef __cplusplus
+#include <stdalign.h>
+#include <stdatomic.h>
+#else
+#include <atomic>
+#define _Atomic(X) std::atomic<X>
+#define _Alignas(X) alignas(X)
+#endif
+
+#include <stddef.h>
+#include <stdint.h>
+
+#if defined(__i386__) || defined(__x86_64__)
+#define LEVEL1_DCACHE_LINESIZE 64
+#elif defined(__aarch64__)
+// This is not always true for ARM
+// in linux, you can obtain the size at runtime using sysconf (_SC_LEVEL1_DCACHE_LINESIZE)
+// or from the bash with the command $ getconf LEVEL1_DCACHE_LINESIZE
+// in c++ using std::hardware_destructive_interference_size
+#define LEVEL1_DCACHE_LINESIZE 64
+#else
+#error Unknown CPU architecture
+#endif
+
+typedef struct {
+  // Avoid false sharing
+  _Alignas(LEVEL1_DCACHE_LINESIZE) _Atomic(int) status;
+} task_ans_t;
+
+void join_task_ans(task_ans_t* arr, size_t len);
+
+void completed_task_ans(task_ans_t* task);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

common/utils/task_manager/task_manager_gen.h

@@ -0,0 +1,58 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#ifndef TASK_MANAGER_GENERIC_H
+#define TASK_MANAGER_GENERIC_H
+
+#include "task.h"
+#include "thread_pool/task_manager.h"
+#include "threadPool/thread-pool.h"
+
+#define THREAD_POOL_WORK_STEALING 1
+#define THREAD_POOL_SINGLE_QUEUE  2
+
+#ifndef THREAD_POOL_IMPLEMENTATION
+/* uncomment the version to use if none is defined (in make/cmake) */
+#define THREAD_POOL_IMPLEMENTATION THREAD_POOL_WORK_STEALING
+// #define THREAD_POOL_IMPLEMENTATION THREAD_POOL_SINGLE_QUEUE
+#endif
+
+#if THREAD_POOL_IMPLEMENTATION == THREAD_POOL_WORK_STEALING
+
+/* Work stealing thread pool */
+#define task_manager_t     ws_task_manager_t
+#define init_task_manager  init_ws_task_manager
+#define free_task_manager  free_ws_task_manager
+#define async_task_manager async_ws_task_manager
+
+#elif THREAD_POOL_IMPLEMENTATION == THREAD_POOL_SINGLE_QUEUE
+
+/* Previous single queue OAI thread pool */
+#define task_manager_t     tpool_t
+#define init_task_manager  init_sq_task_manager
+#define free_task_manager  free_sq_task_manager
+#define async_task_manager async_sq_task_manager
+
+#else
+#error unknown threadpool implmentation
+#endif
+
+#endif /* TASK_MANAGER_GENERIC_H */

common/utils/task_manager/threadPool/measurement_display.c

@@ -0,0 +1,68 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Author and copyright: Laurent Thomas, open-cells.com
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#include <common/utils/simple_executable.h>
+
+#include "thread-pool.h"
+
+#define SEP "\t"
+
+uint64_t cpuCyclesMicroSec;
+
+int main(int argc, char *argv[])
+{
+  if (argc != 2) {
+    printf("Need one parameter: the trace Linux pipe (fifo)");
+    exit(1);
+  }
+
+  mkfifo(argv[1], 0666);
+  int fd = open(argv[1], O_RDONLY);
+
+  if (fd == -1) {
+    perror("open read mode trace file:");
+    exit(1);
+  }
+
+  uint64_t deb = rdtsc_oai();
+  usleep(100000);
+  cpuCyclesMicroSec = (rdtsc_oai() - deb) / 100000;
+  printf("Cycles per microsecond: %lu\n", cpuCyclesMicroSec);
+  printf("Key" SEP "delay to process" SEP "processing time" SEP "delay to be read answer\n");
+  notifiedFIFO_elt_t doneRequest;
+
+  while (1) {
+    if (read(fd, &doneRequest, sizeof(doneRequest)) == sizeof(doneRequest)) {
+      printf("%lu" SEP "%lu" SEP "%lu" SEP
+             "%lu"
+             "\n",
+             doneRequest.key,
+             (doneRequest.startProcessingTime - doneRequest.creationTime) / cpuCyclesMicroSec,
+             (doneRequest.endProcessingTime - doneRequest.startProcessingTime) / cpuCyclesMicroSec,
+             (doneRequest.returnTime - doneRequest.endProcessingTime) / cpuCyclesMicroSec);
+    } else {
+      printf("no measurements\n");
+      sleep(1);
+    }
+  }
+}

common/utils/task_manager/threadPool/test/test_thread-pool.c

@@ -27,7 +27,7 @@
 #include <unistd.h>
 #include <ctype.h>
 #include <sys/sysinfo.h>
-#include <threadPool/thread-pool.h>
+#include <task_manager/threadPool/thread-pool.h>
 #include "log.h"
 
 void displayList(notifiedFIFO_t *nf)
@@ -105,7 +105,7 @@ int main()
   notifiedFIFO_t worker_back;
   initNotifiedFIFO(&worker_back);
 
-  //sleep(1);
+  // sleep(1);
   int cumulProcessTime = 0;
   struct timespec st, end;
   clock_gettime(CLOCK_MONOTONIC, &st);
@@ -116,6 +116,7 @@ int main()
       notifiedFIFO_elt_t *work = newNotifiedFIFO_elt(sizeof(struct testData), i, &worker_back, processing);
       struct testData *x = (struct testData *)NotifiedFifoData(work);
       x->id = i;
+      work->processingArg = NotifiedFifoData(work);
       pushTpool(&pool, work);
     }
     int sleepmax = 0;

common/utils/task_manager/threadPool/thread-pool.c

@@ -0,0 +1,337 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Author and copyright: Laurent Thomas, open-cells.com
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#define _GNU_SOURCE
+#include <sched.h>
+#include <sys/types.h>
+#include <sys/stat.h>
+#include <fcntl.h>
+#include <string.h>
+#include <unistd.h>
+#include <ctype.h>
+#include <sys/sysinfo.h>
+#include <task_manager/threadPool/thread-pool.h>
+
+static inline notifiedFIFO_elt_t *pullNotifiedFifoRemember(notifiedFIFO_t *nf, struct one_thread *thr)
+{
+  mutexlock(nf->lockF);
+
+  while (!nf->outF && !thr->terminate)
+    condwait(nf->notifF, nf->lockF);
+
+  if (thr->terminate) {
+    mutexunlock(nf->lockF);
+    return NULL;
+  }
+
+  notifiedFIFO_elt_t *ret = nf->outF;
+  nf->outF = nf->outF->next;
+
+  if (nf->outF == NULL)
+    nf->inF = NULL;
+
+  // For abort feature
+  thr->runningOnKey = ret->key;
+  thr->dropJob = false;
+  mutexunlock(nf->lockF);
+  return ret;
+}
+
+void *one_thread(void *arg)
+{
+  struct one_thread *myThread = (struct one_thread *)arg;
+  struct thread_pool *tp = myThread->pool;
+
+  // Infinite loop to process requests
+  do {
+    notifiedFIFO_elt_t *elt = pullNotifiedFifoRemember(&tp->incomingFifo, myThread);
+    if (elt == NULL) {
+      AssertFatal(myThread->terminate, "pullNotifiedFifoRemember() returned NULL although thread not aborted\n");
+      break;
+    }
+
+    if (tp->measurePerf)
+      elt->startProcessingTime = rdtsc_oai();
+
+    // elt->processingFunc(NotifiedFifoData(elt));
+    elt->processingFunc(elt->processingArg);
+
+    if (tp->measurePerf)
+      elt->endProcessingTime = rdtsc_oai();
+
+    if (elt->reponseFifo) {
+      // Check if the job is still alive, else it has been aborted
+      mutexlock(tp->incomingFifo.lockF);
+
+      if (myThread->dropJob)
+        delNotifiedFIFO_elt(elt);
+      else
+        pushNotifiedFIFO(elt->reponseFifo, elt);
+      myThread->runningOnKey = -1;
+      mutexunlock(tp->incomingFifo.lockF);
+    } else
+      delNotifiedFIFO_elt(elt);
+  } while (!myThread->terminate);
+  return NULL;
+}
+
+void initNamedTpool(char *params, tpool_t *pool, bool performanceMeas, char *name)
+{
+  memset(pool, 0, sizeof(*pool));
+  char *measr = getenv("OAI_THREADPOOLMEASUREMENTS");
+  pool->measurePerf = performanceMeas;
+  // force measurement if the output is defined
+  pool->measurePerf |= measr != NULL;
+
+  if (measr) {
+    mkfifo(measr, 0666);
+    AssertFatal(-1 != (pool->dummyKeepReadingTraceFd = open(measr, O_RDONLY | O_NONBLOCK)), "");
+    AssertFatal(-1 != (pool->traceFd = open(measr, O_WRONLY | O_APPEND | O_NOATIME | O_NONBLOCK)), "");
+  } else
+    pool->traceFd = -1;
+
+  pool->activated = true;
+  initNotifiedFIFO(&pool->incomingFifo);
+  char *saveptr, *curptr;
+  char *parms_cpy = strdup(params);
+  pool->nbThreads = 0;
+  curptr = strtok_r(parms_cpy, ",", &saveptr);
+  struct one_thread *ptr;
+  char *tname = (name == NULL ? "Tpool" : name);
+  while (curptr != NULL) {
+    int c = toupper(curptr[0]);
+
+    switch (c) {
+      case 'N':
+        pool->activated = false;
+        break;
+
+      default:
+        ptr = pool->allthreads;
+        pool->allthreads = (struct one_thread *)malloc(sizeof(struct one_thread));
+        pool->allthreads->next = ptr;
+        pool->allthreads->coreID = atoi(curptr);
+        pool->allthreads->id = pool->nbThreads;
+        pool->allthreads->pool = pool;
+        pool->allthreads->dropJob = false;
+        pool->allthreads->terminate = false;
+        // Configure the thread scheduler policy for Linux
+        //  set the thread name for debugging
+        sprintf(pool->allthreads->name, "%s%d_%d", tname, pool->nbThreads, pool->allthreads->coreID);
+        // we set the maximum priority for thread pool threads (which is close
+        // but not equal to Linux maximum). See also the corresponding commit
+        // message; initially introduced for O-RAN 7.2 fronthaul split
+        threadCreate(&pool->allthreads->threadID,
+                     one_thread,
+                     (void *)pool->allthreads,
+                     pool->allthreads->name,
+                     pool->allthreads->coreID,
+                     OAI_PRIORITY_RT_MAX);
+        pool->nbThreads++;
+    }
+
+    curptr = strtok_r(NULL, ",", &saveptr);
+  }
+  free(parms_cpy);
+  if (pool->activated && pool->nbThreads == 0) {
+    printf("No servers created in the thread pool, exit\n");
+    exit(1);
+  }
+}
+
+void initFloatingCoresTpool(int nbThreads, tpool_t *pool, bool performanceMeas, char *name)
+{
+  char threads[1024] = "n";
+  if (nbThreads) {
+    strcpy(threads, "-1");
+    for (int i = 1; i < nbThreads; i++)
+      strncat(threads, ",-1", sizeof(threads) - 1);
+  }
+  threads[sizeof(threads) - 1] = 0;
+  initNamedTpool(threads, pool, performanceMeas, name);
+}
+
+void init_sq_task_manager(tpool_t *pool, int *lst, size_t num_threads)
+{
+  DevAssert(pool != NULL);
+  DevAssert(lst != NULL);
+  DevAssert(num_threads > 0);
+
+  char str[1024] = {0};
+  int it = 0;
+  for (int i = 0; i < num_threads - 1; ++i) {
+    it += sprintf(&str[it], "%d,", lst[i]);
+    DevAssert(it < 1024);
+  }
+  it += sprintf(&str[it], "%d", lst[num_threads - 1]);
+  DevAssert(it < 1024);
+
+  bool performanceMeas = false;
+  // Single Queue thread pool
+  char name[] = "sq_TPool-";
+  initNamedTpool(str, pool, performanceMeas, name);
+}
+
+void async_sq_task_manager(tpool_t *pool, task_t t)
+{
+  DevAssert(pool != NULL);
+
+  int size = sizeof(void *);
+  uint64_t const key = 1021;
+  notifiedFIFO_t *responseFifo = NULL;
+  void (*processingFunc)(void *) = t.func;
+  notifiedFIFO_elt_t *elm = newNotifiedFIFO_elt(size, key, responseFifo, processingFunc);
+  elm->processingArg = t.args;
+
+  pushTpool(pool, elm);
+}
+
+void free_sq_task_manager(tpool_t *pool, void (*clean)(task_t *))
+{
+  DevAssert(pool != NULL);
+  abortTpool(pool);
+}
+
+#ifdef TEST_THREAD_POOL
+int oai_exit = 0;
+
+void exit_function(const char *file, const char *function, const int line, const char *s, const int assert)
+{
+  if (assert) {
+    abort();
+  } else {
+    exit(EXIT_SUCCESS);
+  }
+}
+
+struct testData {
+  int id;
+  int sleepTime;
+  char txt[30];
+};
+
+void processing(void *arg)
+{
+  struct testData *in = (struct testData *)arg;
+  // printf("doing: %d, %s, in thr %ld\n",in->id, in->txt,pthread_self() );
+  sprintf(in->txt, "Done by %ld, job %d", pthread_self(), in->id);
+  in->sleepTime = rand() % 1000;
+  usleep(in->sleepTime);
+  // printf("done: %d, %s, in thr %ld\n",in->id, in->txt,pthread_self() );
+}
+
+int main()
+{
+  notifiedFIFO_t myFifo;
+  initNotifiedFIFO(&myFifo);
+  pushNotifiedFIFO(&myFifo, newNotifiedFIFO_elt(sizeof(struct testData), 1234, NULL, NULL));
+
+  for (int i = 10; i > 1; i--) {
+    pushNotifiedFIFO(&myFifo, newNotifiedFIFO_elt(sizeof(struct testData), 1000 + i, NULL, NULL));
+  }
+
+  displayList(&myFifo);
+  notifiedFIFO_elt_t *tmp = pullNotifiedFIFO(&myFifo);
+  printf("pulled: %lu\n", tmp->key);
+  displayList(&myFifo);
+  tmp = pullNotifiedFIFO(&myFifo);
+  printf("pulled: %lu\n", tmp->key);
+  displayList(&myFifo);
+  pushNotifiedFIFO(&myFifo, newNotifiedFIFO_elt(sizeof(struct testData), 12345678, NULL, NULL));
+  displayList(&myFifo);
+
+  do {
+    tmp = pollNotifiedFIFO(&myFifo);
+
+    if (tmp) {
+      printf("pulled: %lu\n", tmp->key);
+      displayList(&myFifo);
+    } else
+      printf("Empty list \n");
+  } while (tmp);
+
+  tpool_t pool;
+  char params[] = "1,2,3,4,5";
+  initTpool(params, &pool, true);
+  notifiedFIFO_t worker_back;
+  initNotifiedFIFO(&worker_back);
+
+  sleep(1);
+  int cumulProcessTime = 0, cumulTime = 0;
+  struct timespec st, end;
+  clock_gettime(CLOCK_MONOTONIC, &st);
+  int nb_jobs = 4;
+  for (int i = 0; i < 1000; i++) {
+    int parall = nb_jobs;
+    for (int j = 0; j < parall; j++) {
+      notifiedFIFO_elt_t *work = newNotifiedFIFO_elt(sizeof(struct testData), i, &worker_back, processing);
+      struct testData *x = (struct testData *)NotifiedFifoData(work);
+      x->id = i;
+      pushTpool(&pool, work);
+    }
+    int sleepmax = 0;
+    while (parall) {
+      tmp = pullTpool(&worker_back, &pool);
+      if (tmp) {
+        parall--;
+        struct testData *dd = NotifiedFifoData(tmp);
+        if (dd->sleepTime > sleepmax)
+          sleepmax = dd->sleepTime;
+        delNotifiedFIFO_elt(tmp);
+      }
+    }
+    cumulProcessTime += sleepmax;
+  }
+  clock_gettime(CLOCK_MONOTONIC, &end);
+  long long dur = (end.tv_sec - st.tv_sec) * 1000 * 1000 + (end.tv_nsec - st.tv_nsec) / 1000;
+  printf("In microseconds, Total time per group of %d job:%lld, work time per job %d, overhead per job %lld\n",
+         nb_jobs,
+         dur / 1000,
+         cumulProcessTime / 1000,
+         (dur - cumulProcessTime) / (1000 * nb_jobs));
+
+  /*
+  for (int i=0; i <1000 ; i++) {
+    notifiedFIFO_elt_t *work=newNotifiedFIFO_elt(sizeof(struct testData), i, &worker_back, processing);
+    struct testData *x=(struct testData *)NotifiedFifoData(work);
+    x->id=i;
+    pushTpool(&pool, work);
+  }
+
+  do {
+    tmp=pullTpool(&worker_back,&pool);
+
+    if (tmp) {
+      struct testData *dd=NotifiedFifoData(tmp);
+      printf("Result: %s\n",dd->txt);
+      delNotifiedFIFO_elt(tmp);
+    } else
+      printf("Empty list \n");
+
+    abortTpoolJob(&pool,510);
+  } while(tmp);
+  */
+  return 0;
+}
+
+#endif

common/utils/task_manager/threadPool/thread-pool.h

@@ -1,29 +1,31 @@
 /*
-* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The OpenAirInterface Software Alliance licenses this file to You under
-* the OAI Public License, Version 1.1  (the "License"); you may not use this file
-* except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*      http://www.openairinterface.org/?page_id=698
-*
-* Author and copyright: Laurent Thomas, open-cells.com
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*-------------------------------------------------------------------------------
-* For more information about the OpenAirInterface (OAI) Software Alliance:
-*      contact@openairinterface.org
-*/
-
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Author and copyright: Laurent Thomas, open-cells.com
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
 
 #ifndef THREAD_POOL_H
 #define THREAD_POOL_H
+
+#include "../task.h"
+
 #include <stdbool.h>
 #include <stdint.h>
 #include <malloc.h>
@@ -31,36 +33,58 @@
 #include <pthread.h>
 #include <unistd.h>
 #include <sys/syscall.h>
-#include "assertions.h"
+#include "common/utils/assertions.h"
 #include "common/utils/time_meas.h"
 #include "common/utils/system.h"
 
 #ifdef DEBUG
-  #define THREADINIT   PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
+#define THREADINIT PTHREAD_ERRORCHECK_MUTEX_INITIALIZER_NP
 #else
-  #define THREADINIT   PTHREAD_MUTEX_INITIALIZER
+#define THREADINIT PTHREAD_MUTEX_INITIALIZER
 #endif
-#define mutexinit(mutex)   {int ret=pthread_mutex_init(&mutex,NULL); \
-                            AssertFatal(ret==0,"ret=%d\n",ret);}
-#define condinit(signal)   {int ret=pthread_cond_init(&signal,NULL); \
-                            AssertFatal(ret==0,"ret=%d\n",ret);}
-#define mutexlock(mutex)   {int ret=pthread_mutex_lock(&mutex); \
-                            AssertFatal(ret==0,"ret=%d\n",ret);}
-#define mutextrylock(mutex)   pthread_mutex_trylock(&mutex)
-#define mutexunlock(mutex) {int ret=pthread_mutex_unlock(&mutex); \
-                            AssertFatal(ret==0,"ret=%d\n",ret);}
-#define condwait(condition, mutex) {int ret=pthread_cond_wait(&condition, &mutex); \
-                                    AssertFatal(ret==0,"ret=%d\n",ret);}
-#define condbroadcast(signal) {int ret=pthread_cond_broadcast(&signal); \
-                               AssertFatal(ret==0,"ret=%d\n",ret);}
-#define condsignal(signal)    {int ret=pthread_cond_signal(&signal); \
-                               AssertFatal(ret==0,"ret=%d\n",ret);}
-#define tpool_nbthreads(tpool)   (tpool.nbThreads)
+#define mutexinit(mutex)                        \
+  {                                             \
+    int ret = pthread_mutex_init(&mutex, NULL); \
+    AssertFatal(ret == 0, "ret=%d\n", ret);     \
+  }
+#define condinit(signal)                        \
+  {                                             \
+    int ret = pthread_cond_init(&signal, NULL); \
+    AssertFatal(ret == 0, "ret=%d\n", ret);     \
+  }
+#define mutexlock(mutex)                    \
+  {                                         \
+    int ret = pthread_mutex_lock(&mutex);   \
+    AssertFatal(ret == 0, "ret=%d\n", ret); \
+  }
+#define mutextrylock(mutex) pthread_mutex_trylock(&mutex)
+#define mutexunlock(mutex)                  \
+  {                                         \
+    int ret = pthread_mutex_unlock(&mutex); \
+    AssertFatal(ret == 0, "ret=%d\n", ret); \
+  }
+#define condwait(condition, mutex)                   \
+  {                                                  \
+    int ret = pthread_cond_wait(&condition, &mutex); \
+    AssertFatal(ret == 0, "ret=%d\n", ret);          \
+  }
+#define condbroadcast(signal)                  \
+  {                                            \
+    int ret = pthread_cond_broadcast(&signal); \
+    AssertFatal(ret == 0, "ret=%d\n", ret);    \
+  }
+#define condsignal(signal)                  \
+  {                                         \
+    int ret = pthread_cond_signal(&signal); \
+    AssertFatal(ret == 0, "ret=%d\n", ret); \
+  }
+#define tpool_nbthreads(tpool) (tpool.nbThreads)
 typedef struct notifiedFIFO_elt_s {
   struct notifiedFIFO_elt_s *next;
-  uint64_t key; //To filter out elements
+  uint64_t key; // To filter out elements
   struct notifiedFIFO_s *reponseFifo;
   void (*processingFunc)(void *);
+  void *processingArg;
   bool malloced;
   oai_cputime_t creationTime;
   oai_cputime_t startProcessingTime;
@@ -70,39 +94,42 @@ typedef struct notifiedFIFO_elt_s {
   // user data behind it will be aligned to 32b as well
   // important! this needs to be the last member in the struct
   alignas(32) void *msgData;
-}  notifiedFIFO_elt_t;
+} notifiedFIFO_elt_t;
 
 typedef struct notifiedFIFO_s {
   notifiedFIFO_elt_t *outF;
   notifiedFIFO_elt_t *inF;
   pthread_mutex_t lockF;
-  pthread_cond_t  notifF;
+  pthread_cond_t notifF;
   bool abortFIFO; // if set, the FIFO always returns NULL -> abort condition
 } notifiedFIFO_t;
 
 // You can use this allocator or use any piece of memory
 static inline notifiedFIFO_elt_t *newNotifiedFIFO_elt(int size,
-    uint64_t key,
-    notifiedFIFO_t *reponseFifo,
-    void (*processingFunc)(void *)) {
+                                                      uint64_t key,
+                                                      notifiedFIFO_t *reponseFifo,
+                                                      void (*processingFunc)(void *))
+{
   notifiedFIFO_elt_t *ret = (notifiedFIFO_elt_t *)memalign(32, sizeof(notifiedFIFO_elt_t) + size);
   AssertFatal(NULL != ret, "out of memory\n");
-  ret->next=NULL;
-  ret->key=key;
-  ret->reponseFifo=reponseFifo;
-  ret->processingFunc=processingFunc;
+  ret->next = NULL;
+  ret->key = key;
+  ret->reponseFifo = reponseFifo;
+  ret->processingFunc = processingFunc;
   // We set user data piece aligend 32 bytes to be able to process it with SIMD
   // msgData is aligned to 32bytes, so everything after will be as well
   ret->msgData = ((uint8_t *)ret) + sizeof(notifiedFIFO_elt_t);
-  ret->malloced=true;
+  ret->malloced = true;
   return ret;
 }
 
-static inline void *NotifiedFifoData(notifiedFIFO_elt_t *elt) {
+static inline void *NotifiedFifoData(notifiedFIFO_elt_t *elt)
+{
   return elt->msgData;
 }
 
-static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt) {
+static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt)
+{
   if (elt->malloced) {
     elt->malloced = false;
     free(elt);
@@ -111,20 +138,23 @@ static inline void delNotifiedFIFO_elt(notifiedFIFO_elt_t *elt) {
    * the caller */
 }
 
-static inline void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
-  nf->inF=NULL;
-  nf->outF=NULL;
+static inline void initNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)
+{
+  nf->inF = NULL;
+  nf->outF = NULL;
   nf->abortFIFO = false;
 }
-static inline void initNotifiedFIFO(notifiedFIFO_t *nf) {
+static inline void initNotifiedFIFO(notifiedFIFO_t *nf)
+{
   mutexinit(nf->lockF);
-  condinit (nf->notifF);
+  condinit(nf->notifF);
   initNotifiedFIFO_nothreadSafe(nf);
   // No delete function: the creator has only to free the memory
 }
 
-static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
-  msg->next=NULL;
+static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)
+{
+  msg->next = NULL;
 
   if (nf->outF == NULL)
     nf->outF = msg;
@@ -135,48 +165,52 @@ static inline void pushNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf, notifiedFIF
   nf->inF = msg;
 }
 
-static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg) {
+static inline void pushNotifiedFIFO(notifiedFIFO_t *nf, notifiedFIFO_elt_t *msg)
+{
   mutexlock(nf->lockF);
   if (!nf->abortFIFO) {
-    pushNotifiedFIFO_nothreadSafe(nf,msg);
+    pushNotifiedFIFO_nothreadSafe(nf, msg);
     condsignal(nf->notifF);
   }
   mutexunlock(nf->lockF);
 }
 
-static inline  notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf) {
+static inline notifiedFIFO_elt_t *pullNotifiedFIFO_nothreadSafe(notifiedFIFO_t *nf)
+{
   if (nf->outF == NULL)
     return NULL;
   if (nf->abortFIFO)
     return NULL;
 
-  notifiedFIFO_elt_t *ret=nf->outF;
+  notifiedFIFO_elt_t *ret = nf->outF;
 
-  AssertFatal(nf->outF != nf->outF->next,"Circular list in thread pool: push several times the same buffer is forbidden\n");
+  AssertFatal(nf->outF != nf->outF->next, "Circular list in thread pool: push several times the same buffer is forbidden\n");
 
-  nf->outF=nf->outF->next;
+  nf->outF = nf->outF->next;
 
-  if (nf->outF==NULL)
-    nf->inF=NULL;
+  if (nf->outF == NULL)
+    nf->inF = NULL;
 
   return ret;
 }
 
-static inline  notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf) {
+static inline notifiedFIFO_elt_t *pullNotifiedFIFO(notifiedFIFO_t *nf)
+{
   mutexlock(nf->lockF);
   notifiedFIFO_elt_t *ret = NULL;
 
-  while((ret=pullNotifiedFIFO_nothreadSafe(nf)) == NULL && !nf->abortFIFO)
+  while ((ret = pullNotifiedFIFO_nothreadSafe(nf)) == NULL && !nf->abortFIFO)
     condwait(nf->notifF, nf->lockF);
 
   mutexunlock(nf->lockF);
   return ret;
 }
 
-static inline  notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf) {
-  int tmp=mutextrylock(nf->lockF);
+static inline notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf)
+{
+  int tmp = mutextrylock(nf->lockF);
 
-  if (tmp != 0 )
+  if (tmp != 0)
     return NULL;
 
   if (nf->abortFIFO) {
@@ -184,12 +218,12 @@ static inline  notifiedFIFO_elt_t *pollNotifiedFIFO(notifiedFIFO_t *nf) {
     return NULL;
   }
 
-  notifiedFIFO_elt_t *ret=pullNotifiedFIFO_nothreadSafe(nf);
+  notifiedFIFO_elt_t *ret = pullNotifiedFIFO_nothreadSafe(nf);
   mutexunlock(nf->lockF);
   return ret;
 }
 
-static inline time_stats_t exec_time_stats_NotifiedFIFO(const notifiedFIFO_elt_t* elt)
+static inline time_stats_t exec_time_stats_NotifiedFIFO(const notifiedFIFO_elt_t *elt)
 {
   time_stats_t ts = {0};
   if (elt->startProcessingTime == 0 && elt->endProcessingTime == 0)
@@ -205,11 +239,12 @@ static inline time_stats_t exec_time_stats_NotifiedFIFO(const notifiedFIFO_elt_t
 
 // This functions aborts all messages in the queue, and marks the queue as
 // "aborted", such that every call to it will return NULL
-static inline void abortNotifiedFIFO(notifiedFIFO_t *nf) {
+static inline void abortNotifiedFIFO(notifiedFIFO_t *nf)
+{
   mutexlock(nf->lockF);
   nf->abortFIFO = true;
   notifiedFIFO_elt_t **elt = &nf->outF;
-  while(*elt != NULL) {
+  while (*elt != NULL) {
     notifiedFIFO_elt_t *p = *elt;
     *elt = (*elt)->next;
     delNotifiedFIFO_elt(p);
@@ -222,7 +257,7 @@ static inline void abortNotifiedFIFO(notifiedFIFO_t *nf) {
 }
 
 struct one_thread {
-  pthread_t  threadID;
+  pthread_t threadID;
   int id;
   int coreID;
   char name[256];
@@ -244,19 +279,21 @@ typedef struct thread_pool {
   struct one_thread *allthreads;
 } tpool_t;
 
-static inline void pushTpool(tpool_t *t, notifiedFIFO_elt_t *msg) {
-  if (t->measurePerf) msg->creationTime=rdtsc_oai();
+static inline void pushTpool(tpool_t *t, notifiedFIFO_elt_t *msg)
+{
+  if (t->measurePerf)
+    msg->creationTime = rdtsc_oai();
 
-  if ( t->activated)
+  if (t->activated)
     pushNotifiedFIFO(&t->incomingFifo, msg);
   else {
     if (t->measurePerf)
-      msg->startProcessingTime=rdtsc_oai();
+      msg->startProcessingTime = rdtsc_oai();
 
     msg->processingFunc(NotifiedFifoData(msg));
 
     if (t->measurePerf)
-      msg->endProcessingTime=rdtsc_oai();
+      msg->endProcessingTime = rdtsc_oai();
 
     if (msg->reponseFifo)
       pushNotifiedFIFO(msg->reponseFifo, msg);
@@ -265,13 +302,14 @@ static inline void pushTpool(tpool_t *t, notifiedFIFO_elt_t *msg) {
   }
 }
 
-static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_t *t) {
-  notifiedFIFO_elt_t *msg= pullNotifiedFIFO(responseFifo);
+static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_t *t)
+{
+  notifiedFIFO_elt_t *msg = pullNotifiedFIFO(responseFifo);
   if (msg == NULL)
     return NULL;
   AssertFatal(t->traceFd != 0, "Thread pool used while not initialized");
   if (t->measurePerf)
-    msg->returnTime=rdtsc_oai();
+    msg->returnTime = rdtsc_oai();
 
   if (t->traceFd > 0) {
     ssize_t b = write(t->traceFd, msg, sizeof(*msg));
@@ -281,14 +319,15 @@ static inline notifiedFIFO_elt_t *pullTpool(notifiedFIFO_t *responseFifo, tpool_
   return msg;
 }
 
-static inline notifiedFIFO_elt_t *tryPullTpool(notifiedFIFO_t *responseFifo, tpool_t *t) {
-  notifiedFIFO_elt_t *msg= pollNotifiedFIFO(responseFifo);
+static inline notifiedFIFO_elt_t *tryPullTpool(notifiedFIFO_t *responseFifo, tpool_t *t)
+{
+  notifiedFIFO_elt_t *msg = pollNotifiedFIFO(responseFifo);
   AssertFatal(t->traceFd != 0, "Thread pool used while not initialized");
   if (msg == NULL)
     return NULL;
 
   if (t->measurePerf)
-    msg->returnTime=rdtsc_oai();
+    msg->returnTime = rdtsc_oai();
 
   if (t->traceFd > 0) {
     ssize_t b = write(t->traceFd, msg, sizeof(*msg));
@@ -298,15 +337,16 @@ static inline notifiedFIFO_elt_t *tryPullTpool(notifiedFIFO_t *responseFifo, tpo
   return msg;
 }
 
-static inline int abortTpool(tpool_t *t) {
-  int nbRemoved=0;
+static inline int abortTpool(tpool_t *t)
+{
+  int nbRemoved = 0;
   /* disables threading: if a message comes in now, we cannot have a race below
    * as each thread will simply execute the message itself */
   t->activated = false;
-  notifiedFIFO_t *nf=&t->incomingFifo;
+  notifiedFIFO_t *nf = &t->incomingFifo;
   mutexlock(nf->lockF);
   nf->abortFIFO = true;
-  notifiedFIFO_elt_t **start=&nf->outF;
+  notifiedFIFO_elt_t **start = &nf->outF;
 
   /* mark threads to abort them */
   struct one_thread *thread = t->allthreads;
@@ -318,20 +358,25 @@ static inline int abortTpool(tpool_t *t) {
   }
 
   /* clear FIFOs */
-  while(*start!=NULL) {
-    notifiedFIFO_elt_t **request=start;
-    *start=(*start)->next;
+  while (*start != NULL) {
+    notifiedFIFO_elt_t **request = start;
+    *start = (*start)->next;
     delNotifiedFIFO_elt(*request);
     *request = NULL;
     nbRemoved++;
   }
 
-  if (t->incomingFifo.outF==NULL)
-    t->incomingFifo.inF=NULL;
+  if (t->incomingFifo.outF == NULL)
+    t->incomingFifo.inF = NULL;
 
   condbroadcast(t->incomingFifo.notifF);
   mutexunlock(nf->lockF);
 
+  /* hack: let the threads finish and avoid a data race
+   * TODO: find a proper solution
+   */
+  sleep(1);
+
   /* join threads that are still runing */
   thread = t->allthreads;
   while (thread != NULL) {
@@ -340,10 +385,16 @@ static inline int abortTpool(tpool_t *t) {
     free(thread);
     thread = next;
   }
+  t->allthreads = NULL;
 
   return nbRemoved;
 }
-void initNamedTpool(char *params,tpool_t *pool, bool performanceMeas, char *name);
-void initFloatingCoresTpool(int nbThreads,tpool_t *pool, bool performanceMeas, char *name);
-#define  initTpool(PARAMPTR,TPOOLPTR, MEASURFLAG) initNamedTpool(PARAMPTR,TPOOLPTR, MEASURFLAG, NULL)
+
+void init_sq_task_manager(tpool_t *pool, int *lst, size_t num_threads);
+void async_sq_task_manager(tpool_t *pool, task_t t);
+void free_sq_task_manager(tpool_t *pool, void (*clean)(task_t *));
+
+void initNamedTpool(char *params, tpool_t *pool, bool performanceMeas, char *name);
+void initFloatingCoresTpool(int nbThreads, tpool_t *pool, bool performanceMeas, char *name);
+#define initTpool(PARAMPTR, TPOOLPTR, MEASURFLAG) initNamedTpool(PARAMPTR, TPOOLPTR, MEASURFLAG, NULL)
 #endif

common/utils/task_manager/thread_pool/README.md

@@ -0,0 +1,7 @@
+Thread Pool implemented in C following the talk of Sean Parent
+"Better Code: Concurrency" from 2016
+
+dyntickless contains the script to run if low-latency is needed
+Remember that isolcpus, rcu_nocbs-4-7, nohz_full=4-7 and rcu_nocb_poll
+is needed.
+The Kernel needs to be compiled appropiattely

common/utils/task_manager/thread_pool/dyntickless.sh

@@ -0,0 +1,61 @@
+#!/bin/bash
+
+# Full dyntick CPU on which we'll run the user loop,
+# it must be part of nohz_full kernel parameter
+TARGET=4
+
+# Migrate all possible tasks to CPU 0
+for P in $(ls /proc)
+do
+	if [ -x "/proc/$P/task/" ]
+	then
+		echo $P
+		taskset -acp 0 $P
+	fi
+done
+
+# Migrate irqs to CPU 0
+for D in $(ls /proc/irq)
+do
+	if [[ -x "/proc/irq/$D" && $D != "0" ]]
+	then
+		echo $D
+		echo 1 > /proc/irq/$D/smp_affinity
+	fi
+done
+
+# Delay the annoying vmstat timer far away
+sysctl vm.stat_interval=120
+
+# Shutdown nmi watchdog as it uses perf events
+sysctl -w kernel.watchdog=0
+
+# Remove -rt task runtime limit
+echo -1 > /proc/sys/kernel/sched_rt_runtime_us
+
+# Pin the writeback workqueue to CPU0
+echo 1 > /sys/bus/workqueue/devices/writeback/cpumask
+
+DIR=/sys/kernel/debug/tracing
+echo > $DIR/trace
+echo 0 > $DIR/tracing_on
+# Uncomment the below for more details on what disturbs the CPU
+echo 0 > $DIR/events/irq/enable
+echo 1 > $DIR/events/sched/sched_switch/enable
+echo 1 > $DIR/events/workqueue/workqueue_queue_work/enable
+echo 1 > $DIR/events/workqueue/workqueue_execute_start/enable
+echo 1 > $DIR/events/timer/hrtimer_expire_entry/enable
+echo 1 > $DIR/events/timer/tick_stop/enable
+echo nop > $DIR/current_tracer
+echo 1 > $DIR/tracing_on
+
+# Run a 10 secs user loop on target
+taskset -c 3-7 ./a.out
+#sleep 20
+#killall a.out
+
+# Checkout the trace in trace.* file
+cat /sys/kernel/debug/tracing/per_cpu/cpu4/trace > trace.4
+cat /sys/kernel/debug/tracing/per_cpu/cpu5/trace > trace.5
+cat /sys/kernel/debug/tracing/per_cpu/cpu6/trace > trace.6
+cat /sys/kernel/debug/tracing/per_cpu/cpu7/trace > trace.7

common/utils/task_manager/thread_pool/main.c

@@ -0,0 +1,131 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#include <fcntl.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <unistd.h>
+
+#include "task_manager.h"
+
+#define NUM_THREADS 4
+#define NUM_JOBS 1024 * 1024
+
+// To compile:
+// gcc main.c task_manager.c ../task_ans.c
+
+int64_t time_now_us(void)
+{
+  struct timespec tms;
+
+  if (clock_gettime(CLOCK_MONOTONIC_RAW, &tms)) {
+    return -1;
+  }
+  /* seconds, multiplied with 1 million */
+  int64_t micros = tms.tv_sec * 1000000;
+  /* Add full microseconds */
+  int64_t const tv_nsec = tms.tv_nsec;
+  micros += tv_nsec / 1000;
+  /* round up if necessary */
+  if (tv_nsec % 1000 >= 500) {
+    ++micros;
+  }
+  return micros;
+}
+
+typedef struct {
+  int64_t a;
+  int64_t time;
+  task_ans_t* ans;
+} pair_t;
+
+static inline int64_t naive_fibonnacci(int64_t a)
+{
+  DevAssert(a < 1000);
+  if (a < 2)
+    return a;
+
+  return naive_fibonnacci(a - 1) + naive_fibonnacci(a - 2);
+}
+
+static int marker_fd;
+
+void do_work(void* arg)
+{
+  // int64_t now = time_now_us();
+
+  pair_t* a = (pair_t*)arg;
+
+  naive_fibonnacci(23 + a->a);
+
+  // usleep(rand()%1024);
+  completed_task_ans(a->ans);
+
+  // printf("Task completed\n");
+  // int64_t stop = time_now_us();
+
+  // char buffer[100] = {0};
+  // int ret = snprintf(buffer, 100, "ID %lu Fib elapsed %ld start-stop %ld - %ld \n", pthread_self(),  stop - now, now, stop);
+  // DevAssert(ret > 0 && ret < 100);
+
+  //  write_marker_ft_mir(marker_fd, buffer);
+  // puts(buffer);
+}
+
+int main()
+{
+  ws_task_manager_t man = {0};
+  int arr_core_id[NUM_THREADS] = {0};
+  for (int i = 0; i < NUM_THREADS; ++i) {
+    arr_core_id[i] = -1;
+  }
+  init_ws_task_manager(&man, arr_core_id, NUM_THREADS);
+
+  pair_t* arr = calloc(NUM_JOBS, sizeof(pair_t));
+  DevAssert(arr != NULL);
+  task_ans_t* ans = calloc(NUM_JOBS, sizeof(task_ans_t));
+  DevAssert(ans != NULL);
+
+  int64_t now = time_now_us();
+
+  for (int i = 0; i < NUM_JOBS; ++i) {
+    pair_t* pa = &arr[i];
+    pa->a = 0; // i%10;
+    pa->time = 0;
+    pa->ans = &ans[i];
+    task_t t = {.args = pa, t.func = do_work};
+    async_ws_task_manager(&man, t);
+  }
+
+  printf("Waiting %ld \n", time_now_us());
+  join_task_ans(ans, NUM_JOBS);
+  int64_t end = time_now_us();
+  printf("Done %ld \n", end);
+
+  free_ws_task_manager(&man, NULL);
+
+  printf("Total elapsed %ld \n", end - now);
+
+  free(arr);
+  free(ans);
+  return EXIT_SUCCESS;
+}

common/utils/task_manager/thread_pool/task_manager.c

@@ -0,0 +1,607 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#define _GNU_SOURCE
+#include <unistd.h>
+
+#include "task_manager.h"
+
+#include "assertions.h"
+
+#include <errno.h>
+#include <fcntl.h>
+#include <limits.h>
+#include <stdbool.h>
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <poll.h>
+#include <sys/types.h>
+#include <sys/sysinfo.h>
+
+#include <linux/futex.h> /* Definition of FUTEX_* constants */
+#include <sys/syscall.h> /* Definition of SYS_* constants */
+#include <unistd.h>
+
+#include <ctype.h> // toupper
+
+// #define POLL_AND_SLEEP
+
+static void pin_thread_to_core(int core_num)
+{
+  cpu_set_t set = {0};
+  CPU_ZERO(&set);
+  CPU_SET(core_num, &set);
+  int ret = sched_setaffinity(gettid(), sizeof(set), &set);
+  DevAssert(ret != -1);
+  printf("Pining into core %d id %ld \n", core_num, pthread_self());
+}
+
+//////////////////////////////
+//////////////////////////////
+////////// RING //
+//////////////////////////////
+//////////////////////////////
+//////////////////////////////
+
+// For working correctly, maintain the default elements to a 2^N e.g., 2^5=32
+#define DEFAULT_ELM 256
+
+typedef struct seq_ring_buf_s {
+  task_t* array;
+  size_t cap;
+  uint32_t head;
+  uint32_t tail;
+  _Atomic uint64_t sz;
+} seq_ring_task_t;
+
+typedef void (*seq_free_func)(task_t*);
+
+static size_t size_seq_ring_task(seq_ring_task_t* r)
+{
+  DevAssert(r != NULL);
+
+  return r->head - r->tail;
+}
+
+static uint32_t mask(uint32_t cap, uint32_t val)
+{
+  return val & (cap - 1);
+}
+
+static bool full(seq_ring_task_t* r)
+{
+  return size_seq_ring_task(r) == r->cap - 1;
+}
+
+static void enlarge_buffer(seq_ring_task_t* r)
+{
+  DevAssert(r != NULL);
+  DevAssert(full(r));
+
+  const uint32_t factor = 2;
+  task_t* tmp_buffer = calloc(r->cap * factor, sizeof(task_t));
+  DevAssert(tmp_buffer != NULL);
+
+  const uint32_t head_pos = mask(r->cap, r->head);
+  const uint32_t tail_pos = mask(r->cap, r->tail);
+
+  if (head_pos > tail_pos) {
+    memcpy(tmp_buffer, r->array + tail_pos, (head_pos - tail_pos) * sizeof(task_t));
+  } else {
+    memcpy(tmp_buffer, r->array + tail_pos, (r->cap - tail_pos) * sizeof(task_t));
+    memcpy(tmp_buffer + (r->cap - tail_pos), r->array, head_pos * sizeof(task_t));
+  }
+  r->cap *= factor;
+  free(r->array);
+  r->array = tmp_buffer;
+  r->tail = 0;
+  r->head = r->cap / 2 - 1;
+}
+
+static void init_seq_ring_task(seq_ring_task_t* r)
+{
+  DevAssert(r != NULL);
+  task_t* tmp_buffer = calloc(DEFAULT_ELM, sizeof(task_t));
+  DevAssert(tmp_buffer != NULL);
+  seq_ring_task_t tmp = {.array = tmp_buffer, .head = 0, .tail = 0, .cap = DEFAULT_ELM};
+  memcpy(r, &tmp, sizeof(seq_ring_task_t));
+  r->sz = 0;
+}
+
+static void free_seq_ring_task(seq_ring_task_t* r, seq_free_func fp)
+{
+  DevAssert(r != NULL);
+  DevAssert(fp == NULL);
+  free(r->array);
+}
+
+static void push_back_seq_ring_task(seq_ring_task_t* r, task_t t)
+{
+  DevAssert(r != NULL);
+
+  if (full(r))
+    enlarge_buffer(r);
+
+  const uint32_t pos = mask(r->cap, r->head);
+  r->array[pos] = t;
+  r->head += 1;
+  r->sz += 1;
+}
+
+static task_t pop_seq_ring_task(seq_ring_task_t* r)
+{
+  DevAssert(r != NULL);
+  DevAssert(size_seq_ring_task(r) > 0);
+
+  const uint32_t pos = mask(r->cap, r->tail);
+  task_t t = r->array[pos];
+  r->tail += 1;
+  r->sz -= 1;
+  return t;
+}
+
+#undef DEFAULT_ELM
+
+//////////////////////////////
+//////////////////////////////
+////////// END RING //
+//////////////////////////////
+//////////////////////////////
+//////////////////////////////
+
+//////////////////////////////
+//////////////////////////////
+////////// Start Notification Queue //
+//////////////////////////////
+//////////////////////////////
+//////////////////////////////
+
+typedef struct {
+  pthread_mutex_t mtx;
+  pthread_cond_t cv;
+  seq_ring_task_t r;
+  size_t t_id;
+  size_t idx; // for debugginf
+  _Atomic int done;
+} not_q_t;
+
+typedef struct {
+  task_t t;
+  bool success;
+} ret_try_t;
+
+static void init_not_q(not_q_t* q, size_t idx, size_t t_id)
+{
+  DevAssert(q != NULL);
+  AssertFatal(t_id != 0, "Invalid thread id");
+
+  q->idx = idx;
+
+  q->done = 0;
+  init_seq_ring_task(&q->r);
+
+  pthread_mutexattr_t attr = {0};
+#ifdef _DEBUG
+  int const rc_mtx = pthread_mutexattr_settype(&attr, PTHREAD_MUTEX_ERRORCHECK);
+  DevAssert(rc_mtx == 0);
+#endif
+  int rc = pthread_mutex_init(&q->mtx, &attr);
+  AssertFatal(rc == 0, "Error while creating the mtx");
+
+  pthread_condattr_t* c_attr = NULL;
+  rc = pthread_cond_init(&q->cv, c_attr);
+  DevAssert(rc == 0);
+
+  q->t_id = t_id;
+}
+
+static void free_not_q(not_q_t* q, void (*clean)(task_t*))
+{
+  DevAssert(q != NULL);
+  DevAssert(q->done == 1);
+
+  free_seq_ring_task(&q->r, clean);
+
+  int rc = pthread_mutex_destroy(&q->mtx);
+  DevAssert(rc == 0);
+
+  rc = pthread_cond_destroy(&q->cv);
+  DevAssert(rc == 0);
+}
+
+static bool try_push_not_q(not_q_t* q, task_t t)
+{
+  DevAssert(q != NULL);
+  DevAssert(q->done == 0 || q->done == 1);
+  DevAssert(t.func != NULL);
+  DevAssert(t.args != NULL);
+
+#if 0
+  /* TODO: decide if it's a problem or not. For the moment, let's be quiet
+   * but keep this check to help debugging future problems.
+   * If something does not work well with the threadpool, activate this log,
+   * might be useful.
+   */
+  if (q->t_id == pthread_self()) {
+    printf("[TASK_MAN]: Cycle detected. Thread from tpool calling itself. Reentrancy should be forbidden. Most probably a bug \n");
+  }
+#endif
+
+  if (pthread_mutex_trylock(&q->mtx) != 0)
+    return false;
+
+  push_back_seq_ring_task(&q->r, t);
+
+  const size_t sz = size_seq_ring_task(&q->r);
+  DevAssert(sz > 0);
+
+  int const rc = pthread_mutex_unlock(&q->mtx);
+  DevAssert(rc == 0);
+
+  pthread_cond_signal(&q->cv);
+
+  return true;
+}
+
+static void push_not_q(not_q_t* q, task_t t)
+{
+  DevAssert(q != NULL);
+  DevAssert(q->done == 0 || q->done == 1);
+  DevAssert(t.func != NULL);
+
+  int const rc = pthread_mutex_lock(&q->mtx);
+  DevAssert(rc == 0);
+
+  push_back_seq_ring_task(&q->r, t);
+
+  DevAssert(size_seq_ring_task(&q->r) > 0);
+
+  pthread_mutex_unlock(&q->mtx);
+
+  pthread_cond_signal(&q->cv);
+}
+
+static ret_try_t try_pop_not_q(not_q_t* q)
+{
+  DevAssert(q != NULL);
+
+  ret_try_t ret = {.success = false};
+
+  int rc = pthread_mutex_trylock(&q->mtx);
+  DevAssert(rc == 0 || rc == EBUSY);
+
+  if (rc == EBUSY)
+    return ret;
+
+  DevAssert(q->done == 0 || q->done == 1);
+
+  size_t sz = size_seq_ring_task(&q->r);
+  if (sz == 0) {
+    rc = pthread_mutex_unlock(&q->mtx);
+    DevAssert(rc == 0);
+
+    return ret;
+  }
+
+  DevAssert(sz > 0);
+  ret.t = pop_seq_ring_task(&q->r);
+
+  rc = pthread_mutex_unlock(&q->mtx);
+  DevAssert(rc == 0);
+  ret.success = true;
+
+  return ret;
+}
+
+#ifdef POLL_AND_SLEEP
+
+static bool pop_not_q(not_q_t* q, ret_try_t* out)
+{
+  DevAssert(q != NULL);
+  DevAssert(out != NULL);
+  DevAssert(q->done == 0 || q->done == 1);
+
+  int rc = pthread_mutex_lock(&q->mtx);
+  DevAssert(rc == 0);
+  DevAssert(q->done == 0 || q->done == 1);
+
+  // Polling can be tunned using different combination
+  // of cnt values. it can also be done using pthread_cond_timedwait
+  const struct timespec ns = {0, 1};
+  int cnt = 0;
+  while (size_seq_ring_task(&q->r) == 0 && q->done == 0) {
+    rc = pthread_mutex_unlock(&q->mtx);
+    DevAssert(rc == 0);
+
+    cnt++;
+    if (cnt % 64)
+      nanosleep(&ns, NULL);
+
+    int rc = pthread_mutex_lock(&q->mtx);
+    DevAssert(rc == 0);
+
+    if (cnt == 4 * 1024) {
+      cnt = 0;
+      pthread_cond_wait(&q->cv, &q->mtx);
+    }
+    cnt++;
+  }
+
+  DevAssert(q->done == 0 || q->done == 1);
+  if (q->done == 1) {
+    int rc = pthread_mutex_unlock(&q->mtx);
+    DevAssert(rc == 0);
+    return false;
+  }
+
+  out->t = pop_seq_ring_task(&q->r);
+
+  rc = pthread_mutex_unlock(&q->mtx);
+  DevAssert(rc == 0);
+
+  return true;
+}
+
+#else
+
+static bool pop_not_q(not_q_t* q, ret_try_t* out)
+{
+  DevAssert(q != NULL);
+  DevAssert(out != NULL);
+  DevAssert(q->done == 0 || q->done == 1);
+
+  int rc = pthread_mutex_lock(&q->mtx);
+  DevAssert(rc == 0);
+  DevAssert(q->done == 0 || q->done == 1);
+
+  while (size_seq_ring_task(&q->r) == 0 && q->done == 0) {
+    pthread_cond_wait(&q->cv, &q->mtx);
+  }
+
+  DevAssert(q->done == 0 || q->done == 1);
+  if (q->done == 1) {
+    int rc = pthread_mutex_unlock(&q->mtx);
+    DevAssert(rc == 0);
+    return false;
+  }
+
+  out->t = pop_seq_ring_task(&q->r);
+
+  rc = pthread_mutex_unlock(&q->mtx);
+  DevAssert(rc == 0);
+
+  return true;
+}
+#endif
+
+static void done_not_q(not_q_t* q)
+{
+  DevAssert(q != NULL);
+
+  int rc = pthread_mutex_lock(&q->mtx);
+  DevAssert(rc == 0);
+
+  q->done = 1;
+
+  rc = pthread_cond_signal(&q->cv);
+  DevAssert(rc == 0);
+
+  // long r = syscall(SYS_futex, q->futex, FUTEX_WAKE_PRIVATE, INT_MAX, NULL, NULL, 0);
+  // DevAssert(r != -1);
+  rc = pthread_mutex_unlock(&q->mtx);
+  DevAssert(rc == 0);
+
+  //  q->futex++;
+}
+
+//////////////////////////////
+//////////////////////////////
+////////// END Notification Queue //
+//////////////////////////////
+//////////////////////////////
+//////////////////////////////
+
+typedef struct {
+  ws_task_manager_t* man;
+  int idx;
+  int core_id;
+} task_thread_args_t;
+
+// Just for debugging purposes, it is very slow!!!!
+// static
+//_Atomic int cnt_out = 0;
+
+// static
+//_Atomic int cnt_in = 0;
+
+static void* worker_thread(void* arg)
+{
+  DevAssert(arg != NULL);
+
+  task_thread_args_t* args = (task_thread_args_t*)arg;
+  int const idx = args->idx;
+
+  ws_task_manager_t* man = args->man;
+
+  uint32_t const len = man->len_thr;
+  uint32_t const num_it = 2 * (man->len_thr + idx);
+
+  not_q_t* q_arr = (not_q_t*)man->q_arr;
+
+  init_not_q(&q_arr[idx], idx, pthread_self());
+
+  int const logical_cores = get_nprocs_conf();
+  DevAssert(logical_cores > 0);
+  DevAssert(args->core_id > -2 && args->core_id < logical_cores);
+  if (args->core_id != -1)
+    pin_thread_to_core(args->core_id);
+
+  // Synchronize all threads
+  pthread_barrier_wait(&man->barrier);
+
+  size_t acc_num_task = 0;
+  for (;;) {
+    ret_try_t ret = {.success = false};
+
+    for (uint32_t i = idx; i < num_it; ++i) {
+      ret = try_pop_not_q(&q_arr[i % len]);
+      if (ret.success == true)
+        break;
+    }
+
+    if (ret.success == false) {
+      man->num_task -= acc_num_task;
+      acc_num_task = 0;
+      if (pop_not_q(&q_arr[idx], &ret) == false)
+        break;
+    }
+
+    // int64_t now = time_now_us();
+    // printf("Calling func \n");
+    ret.t.func(ret.t.args);
+    // printf("Returning from func \n");
+    // int64_t stop = time_now_us();
+
+    acc_num_task += 1;
+    // cnt_out++;
+  }
+
+  free(args);
+  return NULL;
+}
+
+void init_ws_task_manager(ws_task_manager_t* man, int* core_id, size_t num_threads)
+{
+  DevAssert(man != NULL);
+  AssertFatal(num_threads > 0 && num_threads < 33, "Do you have zero or more than 32 processors??");
+
+  man->q_arr = calloc(num_threads, sizeof(not_q_t));
+  AssertFatal(man->q_arr != NULL, "Memory exhausted");
+
+  man->t_arr = calloc(num_threads, sizeof(pthread_t));
+  AssertFatal(man->t_arr != NULL, "Memory exhausted");
+  man->len_thr = num_threads;
+
+  man->index = 0;
+
+  const pthread_barrierattr_t* barrier_attr = NULL;
+  int rc = pthread_barrier_init(&man->barrier, barrier_attr, num_threads + 1);
+  DevAssert(rc == 0);
+
+  for (size_t i = 0; i < num_threads; ++i) {
+    task_thread_args_t* args = malloc(sizeof(task_thread_args_t));
+    AssertFatal(args != NULL, "Memory exhausted");
+    args->idx = i;
+    args->man = man;
+    args->core_id = core_id[i];
+
+    pthread_attr_t attr = {0};
+
+    int ret = pthread_attr_init(&attr);
+    DevAssert(ret == 0);
+    ret = pthread_attr_setinheritsched(&attr, PTHREAD_EXPLICIT_SCHED);
+    DevAssert(ret == 0);
+    ret = pthread_attr_setschedpolicy(&attr, SCHED_RR);
+    DevAssert(ret == 0);
+    struct sched_param sparam = {0};
+    sparam.sched_priority = 97;
+    ret = pthread_attr_setschedparam(&attr, &sparam);
+
+    int rc = pthread_create(&man->t_arr[i], &attr, worker_thread, args);
+    if (rc != 0) {
+      printf("[TASK_MAN]: %s \n", strerror(rc));
+      printf("[TASK_MAN]: Could not create the pthread with attributtes, trying without attributes\n");
+      rc = pthread_create(&man->t_arr[i], NULL, worker_thread, args);
+      AssertFatal(rc == 0, "Error creating a thread");
+    }
+
+    char name[64];
+    sprintf(name, "Tpool%ld_%d", i, core_id[i]);
+    pthread_setname_np(man->t_arr[i], name);
+  }
+
+  // Syncronize thread pool threads. All the threads started
+  pthread_barrier_wait(&man->barrier);
+}
+
+void free_ws_task_manager(ws_task_manager_t* man, void (*clean)(task_t*))
+{
+  not_q_t* q_arr = (not_q_t*)man->q_arr;
+
+  for (uint32_t i = 0; i < man->len_thr; ++i) {
+    done_not_q(&q_arr[i]);
+  }
+
+  for (uint32_t i = 0; i < man->len_thr; ++i) {
+    int rc = pthread_join(man->t_arr[i], NULL);
+    DevAssert(rc == 0);
+  }
+
+  for (uint32_t i = 0; i < man->len_thr; ++i) {
+    free_not_q(&q_arr[i], clean);
+  }
+
+  int rc = pthread_barrier_destroy(&man->barrier);
+  DevAssert(rc == 0);
+
+  free(man->q_arr);
+
+  free(man->t_arr);
+}
+
+void async_ws_task_manager(ws_task_manager_t* man, task_t t)
+{
+  DevAssert(man != NULL);
+  DevAssert(man->len_thr > 0);
+  DevAssert(t.func != NULL);
+
+  size_t const index = man->index++;
+  size_t const len_thr = man->len_thr;
+
+  not_q_t* q_arr = (not_q_t*)man->q_arr;
+  // DevAssert(pthread_self() != q_arr[index%len_thr].t_id);
+
+  for (size_t i = 0; i < len_thr; ++i) {
+    if (try_push_not_q(&q_arr[(i + index) % len_thr], t)) {
+      man->num_task += 1;
+
+      // printf("Pushing idx %ld %ld \n",(i+index) % len_thr, time_now_us());
+
+      //  Debbugging purposes
+      // cnt_in++;
+      // printf(" async_task_manager t_id %ld Tasks in %d %ld num_task %ld idx %ld \n", pthread_self(), cnt_in, time_now_us(),
+      // man->num_task, (i+index) % len_thr );
+      return;
+    }
+  }
+
+  push_not_q(&q_arr[index % len_thr], t);
+
+  // printf("Pushing idx %ld %ld \n", index % len_thr, time_now_us());
+
+  man->num_task += 1;
+
+  // Debbugging purposes
+  // cnt_in++;
+  // printf("t_id %ld Tasks in %d %ld num_takss %ld idx %ld \n", pthread_self(), cnt_in, time_now_us(), man->num_task , index %
+  // len_thr );
+}

common/utils/task_manager/thread_pool/task_manager.h

@@ -0,0 +1,59 @@
+/*
+ * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The OpenAirInterface Software Alliance licenses this file to You under
+ * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ * except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.openairinterface.org/?page_id=698
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *-------------------------------------------------------------------------------
+ * For more information about the OpenAirInterface (OAI) Software Alliance:
+ *      contact@openairinterface.org
+ */
+
+#ifndef TASK_MANAGER_WORKING_STEALING_H
+#define TASK_MANAGER_WORKING_STEALING_H
+
+#include "../task.h"
+#include "../task_ans.h"
+
+#include <pthread.h>
+#include <stdbool.h>
+#include <stdint.h>
+
+typedef struct {
+  uint8_t* buf;
+  size_t len;
+  size_t cap; // capacity
+  task_ans_t* ans;
+} thread_info_tm_t;
+
+typedef struct {
+  pthread_t* t_arr;
+  size_t len_thr;
+
+  _Atomic(uint64_t) index;
+
+  void* q_arr;
+
+  _Atomic(uint64_t) num_task;
+
+  pthread_barrier_t barrier;
+
+} ws_task_manager_t;
+
+void init_ws_task_manager(ws_task_manager_t* man, int* core_id, size_t num_threads);
+
+void free_ws_task_manager(ws_task_manager_t* man, void (*clean)(task_t* args));
+
+void async_ws_task_manager(ws_task_manager_t* man, task_t t);
+
+#endif

common/utils/telnetsrv/telnetsrv.c

@@ -53,7 +53,7 @@
 #include <sys/resource.h>
 #include "common/utils/load_module_shlib.h"
 #include "common/config/config_userapi.h"
-#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/threadPool/thread-pool.h"
 #include "executables/softmodem-common.h"
 #include <readline/history.h>
 

common/utils/threadPool/measurement_display.c

@@ -1,66 +0,0 @@
-/*
-* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The OpenAirInterface Software Alliance licenses this file to You under
-* the OAI Public License, Version 1.1  (the "License"); you may not use this file
-* except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*      http://www.openairinterface.org/?page_id=698
-*
-* Author and copyright: Laurent Thomas, open-cells.com
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*-------------------------------------------------------------------------------
-* For more information about the OpenAirInterface (OAI) Software Alliance:
-*      contact@openairinterface.org
-*/
-
-#include <common/utils/simple_executable.h>
-
-#include "thread-pool.h"
-
-#define SEP "\t"
-
-uint64_t cpuCyclesMicroSec;
-
-int main(int argc, char *argv[]) {
-  if(argc != 2) {
-    printf("Need one parameter: the trace Linux pipe (fifo)");
-    exit(1);
-  }
-
-  mkfifo(argv[1],0666);
-  int fd=open(argv[1], O_RDONLY);
-
-  if ( fd == -1 ) {
-    perror("open read mode trace file:");
-    exit(1);
-  }
-
-  uint64_t deb=rdtsc_oai();
-  usleep(100000);
-  cpuCyclesMicroSec=(rdtsc_oai()-deb)/100000;
-  printf("Cycles per µs: %lu\n",cpuCyclesMicroSec);
-  printf("Key" SEP "delay to process" SEP "processing time" SEP "delay to be read answer\n");
-  notifiedFIFO_elt_t doneRequest;
-
-  while ( 1 ) {
-    if ( read(fd,&doneRequest, sizeof(doneRequest)) ==  sizeof(doneRequest)) {
-      printf("%lu" SEP "%lu" SEP "%lu" SEP "%lu" "\n",
-             doneRequest.key,
-             (doneRequest.startProcessingTime-doneRequest.creationTime)/cpuCyclesMicroSec,
-             (doneRequest.endProcessingTime-doneRequest.startProcessingTime)/cpuCyclesMicroSec,
-             (doneRequest.returnTime-doneRequest.endProcessingTime)/cpuCyclesMicroSec
-            );
-    } else {
-      printf("no measurements\n");
-      sleep(1);
-    }
-  }
-}

common/utils/threadPool/thread-pool.c

@@ -1,167 +0,0 @@
-/*
-* Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
-* contributor license agreements.  See the NOTICE file distributed with
-* this work for additional information regarding copyright ownership.
-* The OpenAirInterface Software Alliance licenses this file to You under
-* the OAI Public License, Version 1.1  (the "License"); you may not use this file
-* except in compliance with the License.
-* You may obtain a copy of the License at
-*
-*      http://www.openairinterface.org/?page_id=698
-*
-* Author and copyright: Laurent Thomas, open-cells.com
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*-------------------------------------------------------------------------------
-* For more information about the OpenAirInterface (OAI) Software Alliance:
-*      contact@openairinterface.org
-*/
-
-
-#define _GNU_SOURCE
-#include <sched.h>
-#include <sys/types.h>
-#include <sys/stat.h>
-#include <fcntl.h>
-#include <string.h>
-#include <unistd.h>
-#include <ctype.h>
-#include <sys/sysinfo.h>
-#include <threadPool/thread-pool.h>
-
-
-static inline  notifiedFIFO_elt_t *pullNotifiedFifoRemember( notifiedFIFO_t *nf, struct one_thread *thr) {
-  mutexlock(nf->lockF);
-
-  while(!nf->outF && !thr->terminate)
-    condwait(nf->notifF, nf->lockF);
-
-  if (thr->terminate) {
-    mutexunlock(nf->lockF);
-    return NULL;
-  }
-
-  notifiedFIFO_elt_t *ret=nf->outF;
-  nf->outF=nf->outF->next;
-
-  if (nf->outF==NULL)
-    nf->inF=NULL;
-
-  // For abort feature
-  thr->runningOnKey=ret->key;
-  thr->dropJob = false;
-  mutexunlock(nf->lockF);
-  return ret;
-}
-
-void *one_thread(void *arg) {
-  struct  one_thread *myThread=(struct  one_thread *) arg;
-  struct  thread_pool *tp=myThread->pool;
-
-  // Infinite loop to process requests
-  do {
-    notifiedFIFO_elt_t *elt=pullNotifiedFifoRemember(&tp->incomingFifo, myThread);
-    if (elt == NULL) {
-      AssertFatal(myThread->terminate, "pullNotifiedFifoRemember() returned NULL although thread not aborted\n");
-      break;
-    }
-
-    if (tp->measurePerf) elt->startProcessingTime=rdtsc_oai();
-
-    elt->processingFunc(NotifiedFifoData(elt));
-
-    if (tp->measurePerf) elt->endProcessingTime=rdtsc_oai();
-
-    if (elt->reponseFifo) {
-      // Check if the job is still alive, else it has been aborted
-      mutexlock(tp->incomingFifo.lockF);
-
-      if (myThread->dropJob)
-        delNotifiedFIFO_elt(elt);
-      else
-        pushNotifiedFIFO(elt->reponseFifo, elt);
-      myThread->runningOnKey=-1;
-      mutexunlock(tp->incomingFifo.lockF);
-    }
-    else
-      delNotifiedFIFO_elt(elt);
-  } while (!myThread->terminate);
-  return NULL;
-}
-
-void initNamedTpool(char *params,tpool_t *pool, bool performanceMeas, char *name) {
-  memset(pool,0,sizeof(*pool));
-  char *measr=getenv("OAI_THREADPOOLMEASUREMENTS");
-  pool->measurePerf=performanceMeas;
-  // force measurement if the output is defined
-  pool->measurePerf |= measr!=NULL;
-
-  if (measr) {
-    mkfifo(measr,0666);
-    AssertFatal(-1 != (pool->dummyKeepReadingTraceFd=
-                         open(measr, O_RDONLY| O_NONBLOCK)),"");
-    AssertFatal(-1 != (pool->traceFd=
-                         open(measr, O_WRONLY|O_APPEND|O_NOATIME|O_NONBLOCK)),"");
-  } else
-    pool->traceFd=-1;
-
-  pool->activated=true;
-  initNotifiedFIFO(&pool->incomingFifo);
-  char *saveptr, * curptr;
-  char *parms_cpy=strdup(params);
-  pool->nbThreads=0;
-  curptr=strtok_r(parms_cpy,",",&saveptr);
-  struct one_thread * ptr;
-  char *tname = (name == NULL ? "Tpool" : name);
-  while ( curptr!=NULL ) {
-    int c=toupper(curptr[0]);
-
-    switch (c) {
-
-      case 'N':
-        pool->activated=false;
-        break;
-
-      default:
-        ptr=pool->allthreads;
-        pool->allthreads=(struct one_thread *)malloc(sizeof(struct one_thread));
-        pool->allthreads->next=ptr;
-        pool->allthreads->coreID=atoi(curptr);
-        pool->allthreads->id=pool->nbThreads;
-        pool->allthreads->pool=pool;
-        pool->allthreads->dropJob = false;
-        pool->allthreads->terminate = false;
-        //Configure the thread scheduler policy for Linux
-        // set the thread name for debugging
-        sprintf(pool->allthreads->name,"%s%d_%d",tname,pool->nbThreads,pool->allthreads->coreID);
-        // we set the maximum priority for thread pool threads (which is close
-        // but not equal to Linux maximum). See also the corresponding commit
-        // message; initially introduced for O-RAN 7.2 fronthaul split
-        threadCreate(&pool->allthreads->threadID, one_thread, (void *)pool->allthreads,
-                     pool->allthreads->name, pool->allthreads->coreID, OAI_PRIORITY_RT_MAX);
-        pool->nbThreads++;
-    }
-
-    curptr=strtok_r(NULL,",",&saveptr);
-  }
-  free(parms_cpy);
-  if (pool->activated && pool->nbThreads==0) {
-    printf("No servers created in the thread pool, exit\n");
-    exit(1);
-  }
-}
-
-void initFloatingCoresTpool(int nbThreads,tpool_t *pool, bool performanceMeas, char *name) {
-  char threads[1024] = "n";
-  if (nbThreads) {
-    strcpy(threads,"-1");
-    for (int i=1; i < nbThreads; i++)
-      strncat(threads,",-1", sizeof(threads)-1);
-  }
-  threads[sizeof(threads)-1]=0;
-  initNamedTpool(threads, pool, performanceMeas, name);
-}

common/utils/time_meas.c

@@ -27,7 +27,7 @@
 #include "assertions.h"
 #include <pthread.h>
 #include "common/config/config_userapi.h"
-#include <common/utils/threadPool/thread-pool.h>
+#include <common/utils/task_manager/threadPool/thread-pool.h>
 // global var for openair performance profiler
 int opp_enabled = 0;
 double cpu_freq_GHz  __attribute__ ((aligned(32)));

executables/lte-softmodem.c

@@ -73,7 +73,7 @@ unsigned short config_frames[4] = {2,9,11,13};
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include "UTIL/OPT/opt.h"
 #include "enb_config.h"
-
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #include "create_tasks.h"
 
@@ -483,7 +483,7 @@ int main ( int argc, char **argv )
   if (RC.nb_inst > 0) {
     /* initializes PDCP and sets correct RLC Request/PDCP Indication callbacks */
    init_pdcp();
-    
+
     if (create_tasks(1) < 0) {
       printf("cannot create ITTI tasks\n");
       exit(-1);
@@ -538,24 +538,34 @@ int main ( int argc, char **argv )
     printf("Initializing eNB threads wait_for_sync:%d\n", get_softmodem_params()->wait_for_sync);
     init_eNB(get_softmodem_params()->wait_for_sync);
   }
+
   for (int x=0; x < RC.nb_L1_inst; x++)
     for (int CC_id=0; CC_id<RC.nb_L1_CC[x]; CC_id++) {
       L1_rxtx_proc_t *L1proc= &RC.eNB[x][CC_id]->proc.L1_proc;
-      L1_rxtx_proc_t *L1proctx= &RC.eNB[x][CC_id]->proc.L1_proc_tx;
-      L1proc->threadPool = (tpool_t *)malloc(sizeof(tpool_t));
+      L1_rxtx_proc_t *L1proctx = &RC.eNB[x][CC_id]->proc.L1_proc_tx;
       L1proc->respDecode=(notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
-      if ( strlen(get_softmodem_params()->threadPoolConfig) > 0 )
-       initTpool(get_softmodem_params()->threadPoolConfig, L1proc->threadPool, true);
-      else
-        initTpool("n", L1proc->threadPool, true);
+      if (strlen(get_softmodem_params()->threadPoolConfig) > 0) {
+        L1proc->thread_pool = calloc(1, sizeof(task_manager_t));
+        AssertFatal(L1proc->thread_pool != NULL, "Memory exhausted");
+
+        int core_id[128] = {0};
+        span_core_id_t out = {.cap = 128, .core_id = core_id};
+        parse_num_threads(get_softmodem_params()->threadPoolConfig, &out);
+        init_task_manager(L1proc->thread_pool, out.core_id, out.sz);
+      } else {
+        L1proc->thread_pool = calloc(1, sizeof(task_manager_t));
+        AssertFatal(L1proc->thread_pool != NULL, "Memory exhausted");
+        int lst_core_id = -1;
+        init_task_manager(L1proc->thread_pool, &lst_core_id, 1);
+      }
       initNotifiedFIFO(L1proc->respDecode);
-      L1proctx->threadPool = L1proc->threadPool;
-  }
+      L1proctx->thread_pool = L1proc->thread_pool;
+    }
 
   printf("wait_eNBs()\n");
   wait_eNBs();
   printf("About to Init RU threads RC.nb_RU:%d\n", RC.nb_RU);
-  
+
   // RU thread and some L1 procedure aren't necessary in VNF or L2 FAPI simulator.
   // but RU thread deals with pre_scd and this is necessary in VNF and simulator.
   // some initialization is necessary and init_ru_vnf do this.
@@ -565,7 +575,7 @@ int main ( int argc, char **argv )
           "number of L1 instances %d, number of RU %d, number of CPU cores %d: Initializing RU threads\n",
           RC.nb_L1_inst, RC.nb_RU, get_nprocs());
     init_RU(RC.ru,RC.nb_RU,RC.eNB,RC.nb_L1_inst,RC.nb_L1_CC,get_softmodem_params()->rf_config_file,get_softmodem_params()->send_dmrs_sync);
-    
+
     for (ru_id=0; ru_id<RC.nb_RU; ru_id++) {
       RC.ru[ru_id]->rf_map.card=0;
       RC.ru[ru_id]->rf_map.chain=CC_id+(get_softmodem_params()->chain_offset);
@@ -586,7 +596,7 @@ int main ( int argc, char **argv )
     LOG_I(ENB_APP,"RC.nb_RU:%d\n", RC.nb_RU);
     // once all RUs are ready intiailize the rest of the eNBs ((dependence on final RU parameters after configuration)
     printf("ALL RUs ready - init eNBs\n");
-    
+
     if (NFAPI_MODE!=NFAPI_MODE_PNF && NFAPI_MODE!=NFAPI_MODE_VNF) {
       LOG_I(ENB_APP,"Not NFAPI mode - call init_eNB_afterRU()\n");
       init_eNB_afterRU();

executables/nr-gnb.c

@@ -85,8 +85,7 @@
 #include <openair1/PHY/NR_TRANSPORT/nr_dlsch.h>
 #include <PHY/NR_ESTIMATION/nr_ul_estimation.h>
 
-// #define USRP_DEBUG 1
-//  Fix per CC openair rf/if device update
+#include "common/utils/task_manager/task_manager_gen.h"
 
 time_stats_t softmodem_stats_mt; // main thread
 time_stats_t softmodem_stats_hw; //  hw acquisition
@@ -170,7 +169,7 @@ static void tx_func(processingData_L1tx_t *info)
   deref_sched_response(info->sched_response_id);
 }
 
-void *L1_rx_thread(void *arg) 
+void *L1_rx_thread(void *arg)
 {
   PHY_VARS_gNB *gNB = (PHY_VARS_gNB*)arg;
 
@@ -359,18 +358,23 @@ void *nrL1_stats_thread(void *param) {
 void init_gNB_Tpool(int inst) {
   PHY_VARS_gNB *gNB;
   gNB = RC.gNB[inst];
+
+  // ULSCH decoding threadpool
+  int core_id[128] = {0};
+  span_core_id_t out = {.cap = 128, .core_id = core_id, .sz = 0};
+  parse_num_threads(get_softmodem_params()->threadPoolConfig, &out);
+  init_task_manager(&gNB->thread_pool, out.core_id, out.sz);
+
   gNB_L1_proc_t *proc = &gNB->proc;
   // PUSCH symbols per thread need to be calculated by how many threads we have
   gNB->num_pusch_symbols_per_thread = 1;
-  // ULSCH decoding threadpool
-  initTpool(get_softmodem_params()->threadPoolConfig, &gNB->threadPool, cpumeas(CPUMEAS_GETSTATE));
   // ULSCH decoder result FIFO
   initNotifiedFIFO(&gNB->respPuschSymb);
   initNotifiedFIFO(&gNB->respDecode);
 
   // L1 RX result FIFO
   initNotifiedFIFO(&gNB->resp_L1);
-  // L1 TX result FIFO 
+  // L1 TX result FIFO
   initNotifiedFIFO(&gNB->L1_tx_free);
   initNotifiedFIFO(&gNB->L1_tx_filled);
   initNotifiedFIFO(&gNB->L1_tx_out);
@@ -397,7 +401,7 @@ void init_gNB_Tpool(int inst) {
 
 void term_gNB_Tpool(int inst) {
   PHY_VARS_gNB *gNB = RC.gNB[inst];
-  abortTpool(&gNB->threadPool);
+
   abortNotifiedFIFO(&gNB->respDecode);
   abortNotifiedFIFO(&gNB->resp_L1);
   abortNotifiedFIFO(&gNB->L1_tx_free);
@@ -405,6 +409,9 @@ void term_gNB_Tpool(int inst) {
   abortNotifiedFIFO(&gNB->L1_tx_out);
   abortNotifiedFIFO(&gNB->L1_rx_out);
 
+  void (*clean)(task_t *) = NULL;
+  free_task_manager(&gNB->thread_pool, clean);
+
   gNB_L1_proc_t *proc = &gNB->proc;
   if (!get_softmodem_params()->emulate_l1)
     pthread_join(proc->L1_stats_thread, NULL);
@@ -463,7 +470,7 @@ void init_eNB_afterRU(void) {
       AssertFatal(gNB->RU_list[ru_id]->prach_rxsigF!=NULL,
 		  "RU %d : prach_rxsigF is NULL\n",
 		  gNB->RU_list[ru_id]->idx);
-      
+
       for (i=0; i<gNB->RU_list[ru_id]->nb_rx; aa++,i++) {
         LOG_I(PHY,"Attaching RU %d antenna %d to gNB antenna %d\n",gNB->RU_list[ru_id]->idx,i,aa);
         gNB->prach_vars.rxsigF[aa]    =  gNB->RU_list[ru_id]->prach_rxsigF[0][i];
@@ -526,7 +533,6 @@ void init_gNB(int wait_for_sync) {
     gNB->chest_freq = get_softmodem_params()->chest_freq;
 
   }
-  
 
   LOG_I(PHY,"[nr-gnb.c] gNB structure allocated\n");
 }

executables/nr-ru.c

@@ -53,6 +53,9 @@
 #include "common/utils/LOG/vcd_signal_dumper.h"
 
 #include <executables/softmodem-common.h>
+
+#include "common/utils/task_manager/task_manager_gen.h"
+
 /* these variables have to be defined before including ENB_APP/enb_paramdef.h and GNB_APP/gnb_paramdef.h */
 static int DEFBANDS[] = {7};
 static int DEFENBS[] = {0};
@@ -1845,14 +1848,18 @@ void init_NR_RU(configmodule_interface_t *cfg, char *rf_config_file)
       int threadCnt = ru->num_tpcores;
       if (threadCnt < 2) LOG_E(PHY,"Number of threads for gNB should be more than 1. Allocated only %d\n",threadCnt);
       else LOG_I(PHY,"RU Thread pool size %d\n",threadCnt);
-      char pool[80];
+      char pool[512] = {0};
       int s_offset = sprintf(pool,"%d",ru->tpcores[0]);
       for (int icpu=1; icpu<threadCnt; icpu++) {
          s_offset+=sprintf(pool+s_offset,",%d",ru->tpcores[icpu]);
       }
       LOG_I(PHY,"RU thread-pool core string %s\n",pool);
-      ru->threadPool = (tpool_t*)malloc(sizeof(tpool_t));
-      initTpool(pool, ru->threadPool, cpumeas(CPUMEAS_GETSTATE));
+
+      int core_id[128] = {0};
+      span_core_id_t out = {.cap = 128, .core_id = core_id};
+      parse_num_threads(pool, &out);
+      init_task_manager(&ru->thread_pool, out.core_id, out.sz);
+
       // FEP RX result FIFO
       ru->respfeprx = (notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
       initNotifiedFIFO(ru->respfeprx);

executables/nr-softmodem-common.h

@@ -28,7 +28,7 @@
 #undef MALLOC
 #include "assertions.h"
 #include "PHY/types.h"
-#include <threadPool/thread-pool.h>
+#include <task_manager/threadPool/thread-pool.h>
 
 /* help strings definition for command line options, used in CMDLINE_XXX_DESC macros and printed when -h option is used */
 #define CONFIG_HLP_RFCFGF        "Configuration file for front-end (e.g. LMS7002M)\n"

executables/nr-ue.c

@@ -37,6 +37,7 @@
 #include "LAYER2/nr_pdcp/nr_pdcp_oai_api.h"
 #include "LAYER2/nr_rlc/nr_rlc_oai_api.h"
 #include "RRC/NR/MESSAGES/asn1_msg.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 /*
  *  NR SLOT PROCESSING SEQUENCE
@@ -344,6 +345,7 @@ typedef struct {
   nr_gscn_info_t gscnInfo[MAX_GSCN_BAND];
   int numGscn;
   int rx_offset;
+  notifiedFIFO_elt_t *elt;
 } syncData_t;
 
 static int nr_ue_adjust_rx_gain(PHY_VARS_NR_UE *UE, openair0_config_t *cfg0, int gain_change)
@@ -427,6 +429,11 @@ static void UE_synch(void *arg) {
     else
       LOG_E(PHY, "synch Failed: \n");
   }
+
+  if (syncD->elt->reponseFifo)
+    pushNotifiedFIFO(syncD->elt->reponseFifo, syncD->elt);
+  else
+    delNotifiedFIFO_elt(syncD->elt);
 }
 
 static void RU_write(nr_rxtx_thread_data_t *rxtxD, bool sl_tx_action)
@@ -593,6 +600,11 @@ void processSlotTX(void *arg)
   *msgData = newslot;
   pushNotifiedFIFO(UE->tx_resume_ind_fifo + newslot, newElt);
   RU_write(rxtxD, sl_tx_action);
+
+  if (rxtxD->elt->reponseFifo)
+    pushNotifiedFIFO(rxtxD->elt->reponseFifo, rxtxD->elt);
+  else
+    delNotifiedFIFO_elt(rxtxD->elt);
 }
 
 static int UE_dl_preprocessing(PHY_VARS_NR_UE *UE, const UE_nr_rxtx_proc_t *proc, int *tx_wait_for_dlsch, nr_phy_data_t *phy_data)
@@ -669,6 +681,8 @@ void UE_dl_processing(void *arg) {
 
   if (!UE->sl_mode)
     pdsch_processing(UE, proc, phy_data);
+
+  free(rxtxD);
 }
 
 void dummyWrite(PHY_VARS_NR_UE *UE,openair0_timestamp timestamp, int writeBlockSize) {
@@ -819,7 +833,7 @@ void *UE_thread(void *arg)
 
   while (!oai_exit) {
     if (syncRunning) {
-      notifiedFIFO_elt_t *res=tryPullTpool(&nf,&(get_nrUE_params()->Tpool));
+      notifiedFIFO_elt_t *res = pollNotifiedFIFO(&nf);
 
       if (res) {
         syncRunning = false;
@@ -876,7 +890,10 @@ void *UE_thread(void *arg)
       }
       syncMsg->UE = UE;
       memset(&syncMsg->proc, 0, sizeof(syncMsg->proc));
-      pushTpool(&(get_nrUE_params()->Tpool), Msg);
+      syncMsg->elt = Msg;
+      task_t t = {.func = UE_synch, .args = syncMsg};
+      async_task_manager(&(get_nrUE_params()->thread_pool), t);
+
       trashed_frames = 0;
       syncRunning = true;
       continue;
@@ -995,13 +1012,15 @@ void *UE_thread(void *arg)
       nr_ue_rrc_timer_trigger(UE->Mod_id, curMsg.proc.frame_tx, curMsg.proc.gNB_id);
 
     // RX slot processing. We launch and forget.
-    notifiedFIFO_elt_t *newRx = newNotifiedFIFO_elt(sizeof(nr_rxtx_thread_data_t), curMsg.proc.nr_slot_rx, NULL, UE_dl_processing);
-    nr_rxtx_thread_data_t *curMsgRx = (nr_rxtx_thread_data_t *)NotifiedFifoData(newRx);
+    // Memory ownership is transferred to the function UE_dl_processing
+    nr_rxtx_thread_data_t *curMsgRx = calloc(1, sizeof(nr_rxtx_thread_data_t));
+    AssertFatal(curMsgRx != NULL, "Memory exhausted");
     *curMsgRx = (nr_rxtx_thread_data_t){.proc = curMsg.proc, .UE = UE};
     int ret = UE_dl_preprocessing(UE, &curMsgRx->proc, tx_wait_for_dlsch, &curMsgRx->phy_data);
     if (ret != INT_MAX)
       shiftForNextFrame = ret;
-    pushTpool(&(get_nrUE_params()->Tpool), newRx);
+    task_t t = {.func = UE_dl_processing, .args = curMsgRx};
+    async_task_manager(&(get_nrUE_params()->thread_pool), t);
 
     // Start TX slot processing here. It runs in parallel with RX slot processing
     // in current code, DURATION_RX_TO_TX constant is the limit to get UL data to encode from a RX slot
@@ -1015,7 +1034,9 @@ void *UE_thread(void *arg)
     curMsgTx->stream_status = stream_status;
     stream_status = STREAM_STATUS_SYNCED;
     tx_wait_for_dlsch[curMsgTx->proc.nr_slot_tx] = 0;
-    pushTpool(&(get_nrUE_params()->Tpool), newTx);
+    curMsgTx->elt = newTx;
+    t = (task_t){.func = processSlotTX, .args = curMsgTx};
+    async_task_manager(&(get_nrUE_params()->thread_pool), t);
   }
 
   return NULL;

executables/nr-uesoftmodem.c

@@ -32,7 +32,7 @@
 #include "SCHED_NR_UE/defs.h"
 #include "common/ran_context.h"
 #include "common/config/config_userapi.h"
-//#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 #include "common/utils/load_module_shlib.h"
 //#undef FRAME_LENGTH_COMPLEX_SAMPLES //there are two conflicting definitions, so we better make sure we don't use it at all
 #include "common/utils/nr/nr_common.h"
@@ -89,7 +89,7 @@ unsigned short config_frames[4] = {2,9,11,13};
 
 extern const char *duplex_mode[];
 THREAD_STRUCT thread_struct;
-nrUE_params_t nrUE_params = {0};
+static nrUE_params_t nrUE_params = {0};
 
 // Thread variables
 pthread_cond_t nfapi_sync_cond;
@@ -409,7 +409,11 @@ int main(int argc, char **argv)
 #if T_TRACER
   T_Config_Init();
 #endif
-  initTpool(get_softmodem_params()->threadPoolConfig, &(nrUE_params.Tpool), cpumeas(CPUMEAS_GETSTATE));
+
+  int core_id[128] = {0};
+  span_core_id_t out = {.cap = 128, .core_id = core_id};
+  parse_num_threads(get_softmodem_params()->threadPoolConfig, &out);
+  init_task_manager(&nrUE_params.thread_pool, out.core_id, out.sz);
   //randominit (0);
   set_taus_seed (0);
 

executables/nr-uesoftmodem.h

@@ -3,6 +3,7 @@
 #include <executables/nr-softmodem-common.h>
 #include <executables/softmodem-common.h>
 #include "PHY/defs_nr_UE.h"
+#include "common/utils/task_manager/thread_pool/task_manager.h"
 
 #define  CONFIG_HLP_IF_FREQ                "IF frequency for RF, if needed\n"
 #define  CONFIG_HLP_IF_FREQ_OFF            "UL IF frequency offset for RF, if needed\n"
@@ -78,7 +79,7 @@ typedef struct {
   uint64_t optmask; // mask to store boolean config options
   uint32_t ofdm_offset_divisor; // Divisor for sample offset computation for each OFDM symbol
   int max_ldpc_iterations; // number of maximum LDPC iterations
-  tpool_t Tpool; // thread pool
+  task_manager_t thread_pool;
   int UE_scan_carrier;
   int UE_fo_compensation;
   uint64_t if_freq;

openair1/PHY/CODING/nrLDPC_defs.h

@@ -24,6 +24,7 @@
 #define __NRLDPC_DEFS__H__
 #include <openair1/PHY/defs_nr_common.h>
 #include "openair1/PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
+#include "common/utils/task_manager/thread_pool/task_manager.h"
 
 /**
    \brief LDPC encoder
@@ -64,6 +65,7 @@ typedef struct {
   nrLDPC_params_per_cb_t perCB[NR_LDPC_MAX_NUM_CB];
   // Redundancy version index
   uint8_t rv;
+  task_ans_t *ans;
 } encoder_implemparams_t;
 
 typedef int32_t(LDPC_initfunc_t)(void);

openair1/PHY/LTE_TRANSPORT/dlsch_coding.c

@@ -42,7 +42,7 @@
 #include "common/utils/LOG/log.h"
 #include "executables/lte-softmodem.h"
 #include <syscall.h>
-#include <common/utils/threadPool/thread-pool.h>
+#include "common/utils/task_manager/task_manager_gen.h"
 
 //#define DEBUG_DLSCH_CODING
 //#define DEBUG_DLSCH_FREE 1
@@ -280,6 +280,9 @@ static void TPencode(void * arg) {
 			  rdata->r,
 			  hadlsch->nb_rb);
     stop_meas(rdata->rm_stats);
+
+    // Task completed in parallel
+    completed_task_ans(rdata->ans);
 }
 
 int dlsch_encoding(PHY_VARS_eNB *eNB,
@@ -317,8 +320,6 @@ int dlsch_encoding(PHY_VARS_eNB *eNB,
 	    num_pdcch_symbols,
 	    frame,subframe,beamforming_mode);
 
-  int nbEncode = 0;
-
   //  if (hadlsch->Ndi == 1) {  // this is a new packet
   if (hadlsch->round == 0) {  // this is a new packet
     // Add 24-bit crc (polynomial A) to payload
@@ -345,13 +346,14 @@ int dlsch_encoding(PHY_VARS_eNB *eNB,
       return(-1);
   }
 
-  notifiedFIFO_t respEncode;
-  initNotifiedFIFO(&respEncode);
-  for (int r=0, r_offset=0; r<hadlsch->C; r++) {
-    
-    union turboReqUnion id= {.s={dlsch->rnti,frame,subframe,r,0}};
-    notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(turboEncode_t), id.p, &respEncode, TPencode);
-    turboEncode_t * rdata=(turboEncode_t *) NotifiedFifoData(req);
+  turboEncode_t arr[hadlsch->C];
+  task_ans_t ans[hadlsch->C];
+  memset(ans, 0, hadlsch->C * sizeof(task_ans_t));
+
+  for (int r = 0, r_offset = 0; r < hadlsch->C; r++) {
+    turboEncode_t *rdata = &arr[r];
+    rdata->ans = &ans[r];
+
     rdata->input=hadlsch->c[r];
     rdata->Kr_bytes= ( r<hadlsch->Cminus ? hadlsch->Kminus : hadlsch->Kplus) >>3;
     rdata->filler=(r==0) ? hadlsch->F : 0;
@@ -365,8 +367,8 @@ int dlsch_encoding(PHY_VARS_eNB *eNB,
     rdata->r_offset=r_offset;
     rdata->G=G;
 
-    pushTpool(proc->threadPool, req);
-    nbEncode++;
+    task_t t = {.func = TPencode, .args = rdata};
+    async_task_manager(proc->thread_pool, t);
 
     int Qm=hadlsch->Qm;
     int C=hadlsch->C;
@@ -378,14 +380,9 @@ int dlsch_encoding(PHY_VARS_eNB *eNB,
     else
       r_offset += Nl*Qm * ((GpmodC==0?0:1) + (Gp/C));
   }
-  // Wait all other threads finish to process
-  while (nbEncode) {
-    notifiedFIFO_elt_t *res = pullTpool(&respEncode, proc->threadPool);
-    if (res == NULL)
-      break; // Tpool has been stopped
-    delNotifiedFIFO_elt(res);
-    nbEncode--;
-  }
+
+  join_task_ans(ans, hadlsch->C);
+
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_ENB_DLSCH_ENCODING, VCD_FUNCTION_OUT);
   return(0);
 }
@@ -450,14 +447,14 @@ int dlsch_encoding_fembms_pmch(PHY_VARS_eNB *eNB,
                          &hadlsch->F)<0)
       return(-1);
   }
-  int nbEncode = 0;
-  notifiedFIFO_t respEncode;
-  initNotifiedFIFO(&respEncode);
-  for (int r=0, r_offset=0; r<hadlsch->C; r++) {
-    
-    union turboReqUnion id= {.s={dlsch->rnti,frame,subframe,r,0}};
-    notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(turboEncode_t), id.p, &respEncode, TPencode);
-    turboEncode_t * rdata=(turboEncode_t *) NotifiedFifoData(req);
+  turboEncode_t arr[hadlsch->C];
+  task_ans_t ans[hadlsch->C];
+  memset(ans, 0, hadlsch->C * sizeof(task_ans_t));
+
+  for (int r = 0, r_offset = 0; r < hadlsch->C; r++) {
+    turboEncode_t *rdata = &arr[r];
+    rdata->ans = &ans[r];
+
     rdata->input=hadlsch->c[r];
     rdata->Kr_bytes= ( r<hadlsch->Cminus ? hadlsch->Kminus : hadlsch->Kplus) >>3;
     rdata->filler=(r==0) ? hadlsch->F : 0;
@@ -471,8 +468,8 @@ int dlsch_encoding_fembms_pmch(PHY_VARS_eNB *eNB,
     rdata->r_offset=r_offset;
     rdata->G=G;
 
-    pushTpool(proc->threadPool, req);
-    nbEncode++;
+    task_t t = {.func = TPencode, .args = rdata};
+    async_task_manager(proc->thread_pool, t);
 
     int Qm=hadlsch->Qm;
     int C=hadlsch->C;
@@ -484,14 +481,9 @@ int dlsch_encoding_fembms_pmch(PHY_VARS_eNB *eNB,
     else
       r_offset += Nl*Qm * ((GpmodC==0?0:1) + (Gp/C));
   }
-  // Wait all other threads finish to process
-  while (nbEncode) {
-    notifiedFIFO_elt_t *res = pullTpool(&respEncode, proc->threadPool);
-    if (res == NULL)
-      break; // Tpool has been stopped
-    delNotifiedFIFO_elt(res);
-    nbEncode--;
-  }
+
+  join_task_ans(ans, hadlsch->C);
+
   return(0);
 }
 

openair1/PHY/LTE_TRANSPORT/transport_proto.h

@@ -36,6 +36,8 @@
 #include "nfapi_interface.h"
 #include "transport_common_proto.h"
 
+#include "common/utils/task_manager/thread_pool/task_manager.h"
+
 // Functions below implement 36-211 and 36-212
 
 /** @addtogroup _PHY_TRANSPORT_
@@ -520,12 +522,13 @@ void rx_ulsch(PHY_VARS_eNB *eNB,
   @param llr8_flag If 1, indicate that the 8-bit turbo decoder should be used
   @returns 0 on success
 */
-unsigned int  ulsch_decoding(PHY_VARS_eNB *phy_vars_eNB,
-                             L1_rxtx_proc_t *proc,
-                             uint8_t UE_id,
-                             uint8_t control_only_flag,
-                             uint8_t Nbundled,
-                             uint8_t llr8_flag);
+unsigned int ulsch_decoding(PHY_VARS_eNB *phy_vars_eNB,
+                            L1_rxtx_proc_t *proc,
+                            uint8_t UE_id,
+                            uint8_t control_only_flag,
+                            uint8_t Nbundled,
+                            uint8_t llr8_flag,
+                            thread_info_tm_t *t_info);
 
 void generate_phich_top(PHY_VARS_eNB *phy_vars_eNB,
                         L1_rxtx_proc_t *proc,

openair1/PHY/LTE_TRANSPORT/ulsch_decoding.c

@@ -41,6 +41,7 @@
 #include "RRC/LTE/rrc_extern.h"
 #include "PHY_INTERFACE/phy_interface.h"
 #include "transport_proto.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 extern int oai_exit;
 
@@ -239,8 +240,10 @@ void processULSegment(void * arg) {
                                    1,
                                    r,
                                    &E)==-1) {
-      LOG_E(PHY,"ulsch_decoding.c: Problem in rate matching\n");
-      return;
+    // Task completed in parallel
+    completed_task_ans(rdata->ans);
+    LOG_E(PHY, "ulsch_decoding.c: Problem in rate matching\n");
+    return;
    }
    stop_meas(&eNB->ulsch_rate_unmatching_stats);
    int max_Ncb = 3*ulsch_harq->RTC[r]*32 ;
@@ -284,6 +287,9 @@ void processULSegment(void * arg) {
                                             &eNB->ulsch_tc_intl1_stats,
                                             &eNB->ulsch_tc_intl2_stats,
                                             &ulsch_harq->abort_decode);
+
+  // Task completed in parallel
+  completed_task_ans(rdata->ans);
 }
 
 /*!
@@ -295,7 +301,12 @@ void processULSegment(void * arg) {
   @returns 0 on success
 */
 
-static int ulsch_decoding_data(PHY_VARS_eNB *eNB, L1_rxtx_proc_t *proc, int UE_id, int harq_pid, int llr8_flag)
+static int ulsch_decoding_data(PHY_VARS_eNB *eNB,
+                               L1_rxtx_proc_t *proc,
+                               int UE_id,
+                               int harq_pid,
+                               int llr8_flag,
+                               thread_info_tm_t *t_info)
 {
   unsigned int r_offset=0;
   int offset = 0;
@@ -330,13 +341,11 @@ static int ulsch_decoding_data(PHY_VARS_eNB *eNB, L1_rxtx_proc_t *proc, int UE_i
       E = ulsch_harq->Qm * (Gp/ulsch_harq->C);
     else
       E = ulsch_harq->Qm * ((GpmodC==0?0:1) + (Gp/ulsch_harq->C));
-    
-    union turboReqUnion id= {.s={ulsch->rnti,proc->frame_rx,proc->subframe_rx,0,0}};
-    notifiedFIFO_elt_t *req=newNotifiedFIFO_elt(sizeof(turboDecode_t),
-                                                id.p,
-                                                proc->respDecode,
-                                                processULSegment);
-    turboDecode_t * rdata=(turboDecode_t *) NotifiedFifoData(req);
+
+    turboDecode_t *rdata = &((turboDecode_t *)t_info->buf)[t_info->len];
+    DevAssert(t_info->len < t_info->cap);
+    rdata->ans = &t_info->ans[t_info->len];
+    t_info->len += 1;
 
     rdata->eNB=eNB;
     rdata->frame=proc->frame_rx;
@@ -355,12 +364,16 @@ static int ulsch_decoding_data(PHY_VARS_eNB *eNB, L1_rxtx_proc_t *proc, int UE_i
     rdata->function=td;
     int Fbytes=(r==0) ? rdata->Fbits>>3 : 0;
     int sz=Kr_bytes - Fbytes - ((ulsch_harq->C>1)?3:0);
-    pushTpool(proc->threadPool,req);
+
+    task_t t = {.func = &processULSegment, .args = rdata};
+    async_task_manager(proc->thread_pool, t);
+
     proc->nbDecode++;
     LOG_D(PHY,"Added a block to decode, in pipe: %d\n",proc->nbDecode);
     r_offset+=E;
     offset+=sz;	    
   }
+
   return(ret);
 }
 
@@ -390,12 +403,13 @@ static inline unsigned int lte_gold_unscram(unsigned int *x1, unsigned int *x2,
   //  printf("n=%d : c %x\n",n,x1^x2);
 }
 
-unsigned int  ulsch_decoding(PHY_VARS_eNB *eNB,
-                             L1_rxtx_proc_t *proc,
-                             uint8_t UE_id,
-                             uint8_t control_only_flag,
-                             uint8_t Nbundled,
-                             uint8_t llr8_flag)
+unsigned int ulsch_decoding(PHY_VARS_eNB *eNB,
+                            L1_rxtx_proc_t *proc,
+                            uint8_t UE_id,
+                            uint8_t control_only_flag,
+                            uint8_t Nbundled,
+                            uint8_t llr8_flag,
+                            thread_info_tm_t *t_info)
 {
   int16_t *ulsch_llr = eNB->pusch_vars[UE_id]->llr;
   LTE_DL_FRAME_PARMS *frame_parms = &eNB->frame_parms;
@@ -1082,7 +1096,7 @@ unsigned int  ulsch_decoding(PHY_VARS_eNB *eNB,
 
   LOG_D(PHY,"frame %d subframe %d O_ACK:%d o_ACK[]=%d:%d:%d:%d\n",frame,subframe,ulsch_harq->O_ACK,ulsch_harq->o_ACK[0],ulsch_harq->o_ACK[1],ulsch_harq->o_ACK[2],ulsch_harq->o_ACK[3]);
   // Do ULSCH Decoding for data portion
-  ret = ulsch_decoding_data(eNB, proc, UE_id, harq_pid, llr8_flag);
+  ret = ulsch_decoding_data(eNB, proc, UE_id, harq_pid, llr8_flag, t_info);
   return(ret);
 }
 

openair1/PHY/NR_TRANSPORT/nr_dlsch_coding.c

@@ -44,6 +44,7 @@
 #include "common/utils/nr/nr_common.h"
 #include <syscall.h>
 #include <openair2/UTIL/OPT/opt.h>
+#include "common/utils/task_manager/task_manager_gen.h"
 
 //#define DEBUG_DLSCH_CODING
 //#define DEBUG_DLSCH_FREE 1
@@ -137,7 +138,7 @@ NR_gNB_DLSCH_t new_gNB_dlsch(NR_DL_FRAME_PARMS *frame_parms, uint16_t N_RB)
   return(dlsch);
 }
 
-void ldpc8blocks(void *p)
+static void ldpc8blocks(void *p)
 {
   encoder_implemparams_t *impp=(encoder_implemparams_t *) p;
   NR_DL_gNB_HARQ_t *harq = (NR_DL_gNB_HARQ_t *)impp->harq;
@@ -252,6 +253,9 @@ void ldpc8blocks(void *p)
 #endif
     r_offset += E;
   }
+
+  // Task running in // completed
+  completed_task_ans(impp->ans);
 }
 
 int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
@@ -389,24 +393,23 @@ int nr_dlsch_encoding(PHY_VARS_gNB *gNB,
     }
     ldpc_interface_offload.LDPCencoder(harq->c, &impp.output, &impp);
   } else {
-    notifiedFIFO_t nf;
-    initNotifiedFIFO(&nf);
-    int nbJobs = 0;
-    for (int j = 0; j < (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1)); j++) {
-      notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(impp), j, &nf, ldpc8blocks);
-      encoder_implemparams_t *perJobImpp = (encoder_implemparams_t *)NotifiedFifoData(req);
+    size_t const n_seg = (impp.n_segments / 8 + ((impp.n_segments & 7) == 0 ? 0 : 1));
+
+    encoder_implemparams_t arr[n_seg];
+    task_ans_t ans[n_seg];
+    memset(ans, 0, n_seg * sizeof(task_ans_t));
+
+    for (int j = 0; j < n_seg; j++) {
+      encoder_implemparams_t *perJobImpp = &arr[j];
       *perJobImpp = impp;
       perJobImpp->macro_num = j;
-      pushTpool(&gNB->threadPool, req);
-      nbJobs++;
-    }
-    while (nbJobs) {
-      notifiedFIFO_elt_t *req = pullTpool(&nf, &gNB->threadPool);
-      if (req == NULL)
-        break; // Tpool has been stopped
-      delNotifiedFIFO_elt(req);
-      nbJobs--;
+      perJobImpp->ans = &ans[j];
+
+      task_t t = {.func = ldpc8blocks, .args = perJobImpp};
+      async_task_manager(&gNB->thread_pool, t);
     }
+
+    join_task_ans(ans, n_seg);
   }
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_gNB_DLSCH_ENCODING, VCD_FUNCTION_OUT);
   return 0;

openair1/PHY/NR_TRANSPORT/nr_ulsch.h

@@ -34,7 +34,8 @@
 #define NR_ULSCH_H_
 
 #include "PHY/defs_gNB.h"
-#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/threadPool/thread-pool.h"
+#include "common/utils/task_manager/thread_pool/task_manager.h"
 
 #define NUMBER_FRAMES_PHY_UE_INACTIVE 10
 
@@ -62,7 +63,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
                       uint32_t frame,
                       uint8_t nr_tti_rx,
                       uint8_t harq_pid,
-                      uint32_t G);
+                      uint32_t G,
+                      thread_info_tm_t *t_info);
 
 /*! \brief Perform PUSCH unscrambling. TS 38.211 V15.4.0 subclause 6.3.1.1
   @param llr, Pointer to llr bits

openair1/PHY/NR_TRANSPORT/nr_ulsch_decoding.c

@@ -50,6 +50,11 @@
 //#define DEBUG_ULSCH_DECODING
 //#define gNB_DEBUG_TRACE
 
+#include "common/utils/task_manager/task_manager_gen.h"
+#include <stdint.h>
+#include <time.h>
+#include <stdalign.h>
+
 #define OAI_UL_LDPC_MAX_NUM_LLR 27000//26112 // NR_LDPC_NCOL_BG1*NR_LDPC_ZMAX = 68*384
 //#define DEBUG_CRC
 #ifdef DEBUG_CRC
@@ -181,6 +186,9 @@ static void nr_processULSegment(void *arg)
     LOG_E(PHY, "ulsch_decoding.c: Problem in rate_matching\n");
     rdata->decodeIterations = max_ldpc_iterations + 1;
     set_abort(&ulsch_harq->abort_decode, true);
+
+    // Task completed
+    completed_task_ans(rdata->ans);
     return;
   }
 
@@ -221,6 +229,9 @@ static void nr_processULSegment(void *arg)
 
   if (rdata->decodeIterations <= p_decoderParms->numMaxIter)
     memcpy(ulsch_harq->c[r],llrProcBuf,  Kr>>3);
+
+  // Task completed
+  completed_task_ans(rdata->ans);
 }
 
 int decode_offload(PHY_VARS_gNB *phy_vars_gNB,
@@ -321,7 +332,8 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
                       uint32_t frame,
                       uint8_t nr_tti_rx,
                       uint8_t harq_pid,
-                      uint32_t G)
+                      uint32_t G,
+                      thread_info_tm_t *t_info)
 {
   if (!ulsch_llr) {
     LOG_E(PHY, "ulsch_decoding.c: NULL ulsch_llr pointer\n");
@@ -431,9 +443,12 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
   set_abort(&harq_process->abort_decode, false);
   for (int r = 0; r < harq_process->C; r++) {
     int E = nr_get_E(G, harq_process->C, Qm, n_layers, r);
-    union ldpcReqUnion id = {.s = {ulsch->rnti, frame, nr_tti_rx, 0, 0}};
-    notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(ldpcDecode_t), id.p, &phy_vars_gNB->respDecode, &nr_processULSegment);
-    ldpcDecode_t *rdata = (ldpcDecode_t *)NotifiedFifoData(req);
+
+    ldpcDecode_t *rdata = &((ldpcDecode_t *)t_info->buf)[t_info->len];
+    DevAssert(t_info->len < t_info->cap);
+    rdata->ans = &t_info->ans[t_info->len];
+    t_info->len += 1;
+
     decParams.R = nr_get_R_ldpc_decoder(pusch_pdu->pusch_data.rv_index,
                                         E,
                                         decParams.BG,
@@ -458,7 +473,10 @@ int nr_ulsch_decoding(PHY_VARS_gNB *phy_vars_gNB,
     rdata->ulsch = ulsch;
     rdata->ulsch_id = ULSCH_id;
     rdata->tbslbrm = pusch_pdu->maintenance_parms_v3.tbSizeLbrmBytes;
-    pushTpool(&phy_vars_gNB->threadPool, req);
+
+    task_t t = {.func = &nr_processULSegment, .args = rdata};
+    async_task_manager(&phy_vars_gNB->thread_pool, t);
+
     LOG_D(PHY, "Added a block to decode, in pipe: %d\n", r);
     r_offset += E;
     offset += ((harq_process->K >> 3) - (harq_process->F >> 3) - ((harq_process->C > 1) ? 3 : 0));

openair1/PHY/NR_TRANSPORT/nr_ulsch_demodulation.c

@@ -10,6 +10,7 @@
 #include "common/utils/nr/nr_common.h"
 #include <openair1/PHY/TOOLS/phy_scope_interface.h>
 #include "PHY/sse_intrin.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #define INVALID_VALUE 255
 
@@ -1395,6 +1396,7 @@ typedef struct puschSymbolProc_s {
   int16_t **llr_layers;
   int16_t *scramblingSequence;
   uint32_t nvar;
+  task_ans_t *ans;
 } puschSymbolProc_t;
 
 static void nr_pusch_symbol_processing(void *arg)
@@ -1443,6 +1445,9 @@ static void nr_pusch_symbol_processing(void *arg)
     for (int i = 0; i < end; i++)
       llr16[i] = llr_ptr[i] * s[i];
   }
+
+  // Task running in // completed
+  completed_task_ans(rdata->ans);
 }
 
 
@@ -1648,6 +1653,13 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
   start_meas(&gNB->rx_pusch_symbol_processing_stats);
   int numSymbols = gNB->num_pusch_symbols_per_thread;
 
+  int const loop_iter = rel15_ul->nr_of_symbols / numSymbols;
+  puschSymbolProc_t arr[loop_iter];
+  task_ans_t arr_ans[loop_iter];
+
+  memset(arr_ans, 0, loop_iter * sizeof(task_ans_t));
+  int sz_arr = 0;
+
   for(uint8_t symbol = rel15_ul->start_symbol_index; symbol < end_symbol; symbol += numSymbols) {
     int total_res = 0;
     for (int s = 0; s < numSymbols; s++) { 
@@ -1658,10 +1670,9 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
       total_res+=pusch_vars->ul_valid_re_per_slot[symbol+s];
     }
     if (total_res > 0) {
-      union puschSymbolReqUnion id = {.s={ulsch_id,frame,slot,0}};
-      id.p=1+symbol;
-      notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(puschSymbolProc_t), id.p, &gNB->respPuschSymb, &nr_pusch_symbol_processing); // create a job for Tpool
-      puschSymbolProc_t *rdata = (puschSymbolProc_t*)NotifiedFifoData(req); // data for the job
+      puschSymbolProc_t *rdata = &arr[sz_arr];
+      rdata->ans = &arr_ans[sz_arr];
+      ++sz_arr;
 
       rdata->gNB = gNB;
       rdata->frame_parms = frame_parms;
@@ -1678,7 +1689,8 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
       if (rel15_ul->pdu_bit_map & PUSCH_PDU_BITMAP_PUSCH_PTRS) {
         nr_pusch_symbol_processing(rdata);
       } else {
-        pushTpool(&gNB->threadPool, req);
+        task_t t = {.func = &nr_pusch_symbol_processing, .args = rdata};
+        async_task_manager(&gNB->thread_pool, t);
         nbSymb++;
       }
 
@@ -1686,12 +1698,9 @@ int nr_rx_pusch_tp(PHY_VARS_gNB *gNB,
     }
   } // symbol loop
 
-  while (nbSymb) {
-    notifiedFIFO_elt_t *req = pullTpool(&gNB->respPuschSymb, &gNB->threadPool);
-    nbSymb--;
-    delNotifiedFIFO_elt(req);
+  if (nbSymb > 0) {
+    join_task_ans(arr_ans, sz_arr);
   }
-
   stop_meas(&gNB->rx_pusch_symbol_processing_stats);
   return 0;
 }

openair1/PHY/NR_UE_TRANSPORT/nr_dlsch_decoding.c

@@ -44,6 +44,7 @@
 #include "common/utils/nr/nr_common.h"
 #include "openair1/PHY/TOOLS/phy_scope_interface.h"
 #include "nfapi/open-nFAPI/nfapi/public_inc/nfapi_nr_interface.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 //#define ENABLE_PHY_PAYLOAD_DEBUG 1
 
@@ -70,15 +71,13 @@ void nr_dlsch_unscrambling(int16_t *llr, uint32_t size, uint8_t q, uint32_t Nid,
 }
 
 static bool nr_ue_postDecode(PHY_VARS_NR_UE *phy_vars_ue,
-                             notifiedFIFO_elt_t *req,
-                             notifiedFIFO_t *nf_p,
-                             const bool last,
+                             ldpcDecode_ue_t *rdata,
+                             bool last,
                              int b_size,
                              uint8_t b[b_size],
                              int *num_seg_ok,
-                             const UE_nr_rxtx_proc_t *proc)
+                             UE_nr_rxtx_proc_t const *proc)
 {
-  ldpcDecode_ue_t *rdata = (ldpcDecode_ue_t*) NotifiedFifoData(req);
   NR_DL_UE_HARQ_t *harq_process = rdata->harq_process;
   NR_UE_DLSCH_t *dlsch = (NR_UE_DLSCH_t *) rdata->dlsch;
   int r = rdata->segment_r;
@@ -110,7 +109,7 @@ static bool nr_ue_postDecode(PHY_VARS_NR_UE *phy_vars_ue,
       if (harq_process->C > 1) {
         int A = dlsch->dlsch_config.TBS;
         /* check global CRC */
-	// we have regrouped the transport block, so it is "1" segment
+        // we have regrouped the transport block, so it is "1" segment
         if (!check_crc(b, lenWithCrc(1, A), crcType(1, A))) {
           harq_process->ack = 0;
           dlsch->last_iteration_cnt = dlsch->max_ldpc_iterations + 1;
@@ -119,7 +118,7 @@ static bool nr_ue_postDecode(PHY_VARS_NR_UE *phy_vars_ue,
           LOG_D(PHY, "DLSCH received nok \n");
           return true; //stop
         }
-	const int sz=A / 8;
+        const int sz = A / 8;
         if (b[sz] == 0 && b[sz + 1] == 0) { // We search only a reccuring OAI error that propagates all 0 packets with a 0 CRC, so we
                                           // do the check only if the 2 first bytes of the CRC are 0 (it can be CRC16 or CRC24)
           int i = 0;
@@ -134,7 +133,7 @@ static bool nr_ue_postDecode(PHY_VARS_NR_UE *phy_vars_ue,
           }
         }
       }
-	
+
       //LOG_D(PHY,"[UE %d] DLSCH: Setting ACK for nr_slot_rx %d TBS %d mcs %d nb_rb %d harq_process->round %d\n",
       //      phy_vars_ue->Mod_id,nr_slot_rx,harq_process->TBS,harq_process->mcs,harq_process->nb_rb, harq_process->round);
       harq_process->status = SCH_IDLE;
@@ -234,6 +233,9 @@ static void nr_processDLSegment(void *arg)
     //VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_DLSCH_RATE_MATCHING, VCD_FUNCTION_OUT);
     stop_meas(&rdata->ts_rate_unmatch);
     LOG_E(PHY,"dlsch_decoding.c: Problem in rate_matching\n");
+
+    // Task completed in parallel
+    completed_task_ans(rdata->ans);
     return;
   }
   stop_meas(&rdata->ts_rate_unmatch);
@@ -275,6 +277,9 @@ static void nr_processDLSegment(void *arg)
       memcpy(harq_process->c[r], LDPCoutput, Kr >> 3);
     stop_meas(&rdata->ts_ldpc_decode);
   }
+
+  // Task completed in parallel
+  completed_task_ans(rdata->ans);
 }
 
 uint32_t nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
@@ -410,16 +415,18 @@ uint32_t nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
   Kr = harq_process->K;
   Kr_bytes = Kr>>3;
   offset = 0;
-  notifiedFIFO_t nf;
-  initNotifiedFIFO(&nf);
   set_abort(&harq_process->abort_decode, false);
+
+  ldpcDecode_ue_t arr[harq_process->C];
+  task_ans_t ans[harq_process->C];
+  memset(ans, 0, harq_process->C * sizeof(task_ans_t));
+
   for (r=0; r<harq_process->C; r++) {
     //printf("start rx segment %d\n",r);
     uint32_t E = nr_get_E(G, harq_process->C, dlsch->dlsch_config.qamModOrder, dlsch->Nl, r);
     decParams.R = nr_get_R_ldpc_decoder(dlsch->dlsch_config.rv, E, decParams.BG, decParams.Z, &harq_process->llrLen, harq_process->DLround);
-    union ldpcReqUnion id = {.s={dlsch->rnti,frame,nr_slot_rx,0,0}};
-    notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(ldpcDecode_ue_t), id.p, &nf, &nr_processDLSegment);
-    ldpcDecode_ue_t * rdata=(ldpcDecode_ue_t *) NotifiedFifoData(req);
+    ldpcDecode_ue_t *rdata = &arr[r];
+    rdata->ans = &ans[r];
 
     rdata->phy_vars_ue = phy_vars_ue;
     rdata->harq_process = harq_process;
@@ -443,7 +450,9 @@ uint32_t nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
     reset_meas(&rdata->ts_deinterleave);
     reset_meas(&rdata->ts_rate_unmatch);
     reset_meas(&rdata->ts_ldpc_decode);
-    pushTpool(&get_nrUE_params()->Tpool,req);
+
+    task_t t = {.args = rdata, .func = nr_processDLSegment};
+    async_task_manager(&get_nrUE_params()->thread_pool, t);
     LOG_D(PHY, "Added a block to decode, in pipe: %d\n", r);
     r_offset += E;
     offset += (Kr_bytes - (harq_process->F>>3) - ((harq_process->C>1)?3:0));
@@ -451,13 +460,11 @@ uint32_t nr_dlsch_decoding(PHY_VARS_NR_UE *phy_vars_ue,
   }
   int num_seg_ok = 0;
   int nbDecode = harq_process->C;
-  while (nbDecode) {
-    notifiedFIFO_elt_t *req=pullTpool(&nf,  &get_nrUE_params()->Tpool);
-    if (req == NULL)
-      break; // Tpool has been stopped
-    nr_ue_postDecode(phy_vars_ue, req, &nf, nbDecode == 1, b_size, b, &num_seg_ok, proc);
-    delNotifiedFIFO_elt(req);
-    nbDecode--;
+  if (nbDecode > 0) {
+    join_task_ans(ans, nbDecode);
+    for (size_t i = 0; i < nbDecode; ++i) {
+      nr_ue_postDecode(phy_vars_ue, &arr[i], i == nbDecode - 1, b_size, b, &num_seg_ok, proc);
+    }
   }
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_DLSCH_COMBINE_SEG, VCD_FUNCTION_OUT);

openair1/PHY/NR_UE_TRANSPORT/nr_initial_sync.c

@@ -293,6 +293,8 @@ void nr_scan_ssb(void *arg)
       }
     }
   }
+
+  completed_task_ans(ssbInfo->ans);
 }
 
 nr_initial_sync_t nr_initial_sync(UE_nr_rxtx_proc_t *proc,
@@ -304,18 +306,19 @@ nr_initial_sync_t nr_initial_sync(UE_nr_rxtx_proc_t *proc,
 {
   NR_DL_FRAME_PARMS *fp = &ue->frame_parms;
 
-  notifiedFIFO_t nf;
-  initNotifiedFIFO(&nf);
-
   // Perform SSB scanning in parallel. One GSCN per thread.
   LOG_I(NR_PHY,
         "Starting cell search with center freq: %ld, bandwidth: %d. Scanning for %d number of GSCN.\n",
         fp->dl_CarrierFreq,
         fp->N_RB_DL,
         numGscn);
+
+  nr_ue_ssb_scan_t arr[numGscn];
+  task_ans_t ans[numGscn];
+  memset(ans, 0, numGscn * sizeof(task_ans_t));
+
   for (int s = 0; s < numGscn; s++) {
-    notifiedFIFO_elt_t *req = newNotifiedFIFO_elt(sizeof(nr_ue_ssb_scan_t), gscnInfo[s].gscn, &nf, &nr_scan_ssb);
-    nr_ue_ssb_scan_t *ssbInfo = (nr_ue_ssb_scan_t *)NotifiedFifoData(req);
+    nr_ue_ssb_scan_t *ssbInfo = &arr[s];
     *ssbInfo = (nr_ue_ssb_scan_t){.gscnInfo = gscnInfo[s],
                                   .fp = &ue->frame_parms,
                                   .proc = proc,
@@ -334,32 +337,34 @@ nr_initial_sync_t nr_initial_sync(UE_nr_rxtx_proc_t *proc,
           ssbInfo->gscnInfo.gscn,
           ssbInfo->gscnInfo.ssbFirstSC,
           ssbInfo->gscnInfo.ssRef);
-    pushTpool(&get_nrUE_params()->Tpool, req);
+    ssbInfo->ans = &ans[s];
+    task_t t = {.func = nr_scan_ssb, .args = ssbInfo};
+    async_task_manager(&get_nrUE_params()->thread_pool, t);
   }
 
   // Collect the scan results
   nr_ue_ssb_scan_t res = {0};
-  while (numGscn) {
-    notifiedFIFO_elt_t *req = pullTpool(&nf, &get_nrUE_params()->Tpool);
-    nr_ue_ssb_scan_t *ssbInfo = (nr_ue_ssb_scan_t *)NotifiedFifoData(req);
-    if (ssbInfo->syncRes.cell_detected) {
-      LOG_A(NR_PHY,
-            "Cell Detected with GSCN: %d, SSB SC offset: %d, SSB Ref: %lf, PSS Corr peak: %d dB, PSS Corr Average: %d\n",
-            ssbInfo->gscnInfo.gscn,
-            ssbInfo->gscnInfo.ssbFirstSC,
-            ssbInfo->gscnInfo.ssRef,
-            ssbInfo->pssCorrPeakPower,
-            ssbInfo->pssCorrAvgPower);
-      if (!res.syncRes.cell_detected) { // take the first cell detected
-        res = *ssbInfo;
+  if (numGscn > 0) {
+    join_task_ans(ans, numGscn);
+    for (int i = 0; i < numGscn; i++) {
+      nr_ue_ssb_scan_t *ssbInfo = &arr[i];
+      if (ssbInfo->syncRes.cell_detected) {
+        LOG_A(NR_PHY,
+              "Cell Detected with GSCN: %d, SSB SC offset: %d, SSB Ref: %lf, PSS Corr peak: %d dB, PSS Corr Average: %d\n",
+              ssbInfo->gscnInfo.gscn,
+              ssbInfo->gscnInfo.ssbFirstSC,
+              ssbInfo->gscnInfo.ssRef,
+              ssbInfo->pssCorrPeakPower,
+              ssbInfo->pssCorrAvgPower);
+        if (!res.syncRes.cell_detected) { // take the first cell detected
+          res = *ssbInfo;
+        }
       }
+      for (int ant = 0; ant < fp->nb_antennas_rx; ant++) {
+        free(ssbInfo->rxdata[ant]);
+      }
+      free(ssbInfo->rxdata);
     }
-    for (int ant = 0; ant < fp->nb_antennas_rx; ant++) {
-      free(ssbInfo->rxdata[ant]);
-    }
-    free(ssbInfo->rxdata);
-    delNotifiedFIFO_elt(req);
-    numGscn--;
   }
 
   // Set globals based on detected cell

openair1/PHY/defs_RU.h

@@ -39,8 +39,8 @@
 #include "time_meas.h"
 #include "defs_common.h"
 #include "nfapi_nr_interface_scf.h"
-#include <common/utils/threadPool/thread-pool.h>
 #include <executables/rt_profiling.h>
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #define MAX_BANDS_PER_RRU 4
 #define MAX_RRU_CONFIG_SIZE 1024
@@ -175,7 +175,8 @@ typedef struct {
  struct RU_t_s *ru;
  int startSymbol;
  int endSymbol;
- int slot; 
+ int slot;
+ task_ans_t *ans;
 } feprx_cmd_t;
 
 typedef struct {
@@ -184,6 +185,7 @@ typedef struct {
  int slot; 
  int startSymbol;
  int numSymbols;
+ task_ans_t *ans;
 } feptx_cmd_t;
 
 typedef struct {
@@ -426,8 +428,7 @@ typedef enum {
 
 
 typedef struct RU_t_s {
-  /// ThreadPool for RU	
-  tpool_t *threadPool;
+  task_manager_t thread_pool;
   /// index of this ru
   uint32_t idx;
   /// pointer to first RU

openair1/PHY/defs_eNB.h

@@ -66,7 +66,7 @@
 #include "PHY/LTE_TRANSPORT/transport_eNB.h"
 #include "openair2/PHY_INTERFACE/IF_Module.h"
 #include "common/openairinterface5g_limits.h"
-
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #define PBCH_A 24
 #define MAX_NUM_RU_PER_eNB 64
@@ -252,7 +252,7 @@ typedef struct {
   pthread_cond_t cond_RUs;
   /// mutex for RXn-TXnp4 processing thread
   pthread_mutex_t mutex_RUs;
-  tpool_t *threadPool;
+  task_manager_t *thread_pool; // non-owning ptr
   int nbDecode;
   notifiedFIFO_t *respDecode;
   pthread_mutex_t mutex_emulateRF;
@@ -786,6 +786,7 @@ typedef struct TurboDecode_s {
     int offset;
     int maxIterations;
     int decodeIterations;
+    task_ans_t *ans;
 } turboDecode_t;
 
 #define TURBO_SIMD_SOFTBITS   96+12+3+3*6144
@@ -802,6 +803,7 @@ typedef struct turboEncode_s {
   time_stats_t *rm_stats;
   time_stats_t *te_stats;
   time_stats_t *i_stats;
+  task_ans_t *ans;
 } turboEncode_t;
 
 

openair1/PHY/defs_gNB.h

@@ -43,6 +43,7 @@
 #include "PHY/CODING/nrLDPC_decoder/nrLDPC_types.h"
 #include "executables/rt_profiling.h"
 #include "nfapi_nr_interface_scf.h"
+#include "common/utils/task_manager/thread_pool/task_manager.h"
 
 #define MAX_NUM_RU_PER_gNB 8
 #define MAX_PUCCH0_NID 8
@@ -690,7 +691,6 @@ typedef struct PHY_VARS_gNB_s {
   notifiedFIFO_t L1_tx_out;
   notifiedFIFO_t L1_rx_out;
   notifiedFIFO_t resp_RU_tx;
-  tpool_t threadPool;
   int num_pusch_symbols_per_thread;
   pthread_t L1_rx_thread;
   int L1_rx_thread_core;
@@ -699,7 +699,9 @@ typedef struct PHY_VARS_gNB_s {
   struct processingData_L1tx *msgDataTx;
   void *scopeData;
   /// structure for analyzing high-level RT measurements
-  rt_L1_profiling_t rt_L1_profiling; 
+  rt_L1_profiling_t rt_L1_profiling;
+
+  task_manager_t thread_pool;
 } PHY_VARS_gNB;
 
 struct puschSymbolReqId {
@@ -734,6 +736,7 @@ typedef struct LDPCDecode_s {
   int offset;
   int decodeIterations;
   uint32_t tbslbrm;
+  task_ans_t *ans;
 } ldpcDecode_t;
 
 struct ldpcReqId {
@@ -754,6 +757,7 @@ typedef struct processingData_L1 {
   int slot_rx;
   openair0_timestamp timestamp_tx;
   PHY_VARS_gNB *gNB;
+  notifiedFIFO_elt_t *elt;
 } processingData_L1_t;
 
 typedef enum {

openair1/PHY/defs_nr_UE.h

@@ -51,6 +51,7 @@
 #include "fapi_nr_ue_interface.h"
 #include "assertions.h"
 //#include "openair1/SCHED_NR_UE/defs.h"
+#include "common/utils/task_manager/thread_pool/task_manager.h"
 
 #ifdef MEX
   #define msg mexPrintf
@@ -580,6 +581,7 @@ typedef struct {
   int pssCorrPeakPower;
   int pssCorrAvgPower;
   int adjust_rxgain;
+  task_ans_t *ans;
 } nr_ue_ssb_scan_t;
 
 typedef struct nr_phy_data_tx_s {
@@ -613,6 +615,7 @@ typedef struct nr_rxtx_thread_data_s {
   int tx_wait_for_dlsch;
   int rx_offset;
   enum stream_status_e stream_status;
+  notifiedFIFO_elt_t *elt;
 } nr_rxtx_thread_data_t;
 
 typedef struct LDPCDecode_ue_s {
@@ -639,6 +642,7 @@ typedef struct LDPCDecode_ue_s {
   time_stats_t ts_rate_unmatch;
   time_stats_t ts_ldpc_decode;
   UE_nr_rxtx_proc_t proc;
+  task_ans_t *ans;
 } ldpcDecode_ue_t;
 
 static inline void start_meas_nr_ue_phy(PHY_VARS_NR_UE *ue, int meas_index) {

openair1/SCHED/phy_procedures_lte_eNb.c

@@ -41,6 +41,7 @@
 #include <common/utils/system.h>
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include <nfapi/oai_integration/nfapi_pnf.h>
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #include "assertions.h"
 
@@ -457,7 +458,7 @@ void phy_procedures_eNB_TX(PHY_VARS_eNB *eNB,
 
   // clear the transmit data array for the current subframe
   for (int aa = 0; aa < fp->nb_antenna_ports_eNB; aa++) {
-    if (eNB->use_DTX==0) 
+    if (eNB->use_DTX == 0)
       memcpy(&eNB->common_vars.txdataF[aa][subframe * fp->ofdm_symbol_size * (fp->symbols_per_tti)],
 	     &eNB->subframe_mask[aa][subframe*fp->ofdm_symbol_size*fp->symbols_per_tti],
 	     fp->ofdm_symbol_size * (fp->symbols_per_tti) * sizeof (int32_t));
@@ -679,7 +680,7 @@ void fill_sr_indication(int UEid, PHY_VARS_eNB *eNB,uint16_t rnti,int frame,int
   pthread_mutex_lock(&eNB->UL_INFO_mutex);
   nfapi_sr_indication_t       *sr_ind =         &eNB->UL_INFO.sr_ind;
   nfapi_sr_indication_body_t  *sr_ind_body =    &sr_ind->sr_indication_body;
-  assert(sr_ind_body->number_of_srs <= NFAPI_SR_IND_MAX_PDU);
+  DevAssert(sr_ind_body->number_of_srs <= NFAPI_SR_IND_MAX_PDU);
   nfapi_sr_indication_pdu_t *pdu =   &sr_ind_body->sr_pdu_list[sr_ind_body->number_of_srs];
   sr_ind->sfn_sf = frame<<4|subframe;
   sr_ind->header.message_id = NFAPI_RX_SR_INDICATION;
@@ -723,7 +724,7 @@ uci_procedures(PHY_VARS_eNB *eNB,
   LTE_DL_FRAME_PARMS *fp = &(eNB->frame_parms);
 
   calc_pucch_1x_interference(eNB, frame, subframe, 0);
-  
+
   for (int i = 0; i < NUMBER_OF_UCI_MAX; i++) {
     uci = &(eNB->uci_vars[i]);
 
@@ -1232,14 +1233,12 @@ uci_procedures(PHY_VARS_eNB *eNB,
   } // end loop for (int i = 0; i < NUMBER_OF_UCI_MAX; i++) {
 }
 
-void postDecode(L1_rxtx_proc_t *proc, notifiedFIFO_elt_t *req)
+void postDecode(L1_rxtx_proc_t *proc, turboDecode_t *rdata)
 {
-  turboDecode_t * rdata=(turboDecode_t *) NotifiedFifoData(req);
-
   LTE_eNB_ULSCH_t *ulsch = rdata->eNB->ulsch[rdata->UEid];
   LTE_UL_eNB_HARQ_t *ulsch_harq = rdata->ulsch_harq;
   PHY_VARS_eNB *eNB=rdata->eNB;
-    
+
   bool decodeSucess=rdata->decodeIterations <= rdata->maxIterations;
   ulsch_harq->processedSegments++;
   LOG_D(PHY, "processing result of segment: %d, ue %d, processed %d/%d\n",
@@ -1282,7 +1281,7 @@ void postDecode(L1_rxtx_proc_t *proc, notifiedFIFO_elt_t *req)
 	      ulsch->Mlimit,
 	      ulsch_harq->o_ACK[0],
 	      ulsch_harq->o_ACK[1]);
-	  
+
 	if (ulsch_harq->round >= 3)  {
 	  ulsch_harq->status  = SCH_IDLE;
 	  ulsch_harq->handled = 0;
@@ -1330,9 +1329,13 @@ void pusch_procedures(PHY_VARS_eNB *eNB,L1_rxtx_proc_t *proc) {
   const int frame    = proc->frame_rx;
   uint32_t harq_pid0 = subframe2harq_pid(&eNB->frame_parms,frame,subframe);
 
+  turboDecode_t arr[64] = {0};
+  task_ans_t ans[64] = {0};
+  thread_info_tm_t t_info = {.ans = ans, .cap = 64, .len = 0, .buf = (uint8_t *)arr};
+
   for (i = 0; i < NUMBER_OF_ULSCH_MAX; i++) {
     ulsch = eNB->ulsch[i];
-    if (!ulsch) continue; 
+    if (!ulsch) continue;
 
     if (ulsch->ue_type > 0) harq_pid = 0;
     else harq_pid=harq_pid0;
@@ -1387,7 +1390,8 @@ void pusch_procedures(PHY_VARS_eNB *eNB,L1_rxtx_proc_t *proc) {
                      i,
                      0, // control_only_flag
                      ulsch_harq->V_UL_DAI,
-                     ulsch_harq->nb_rb > 20 ? 1 : 0);
+                     ulsch_harq->nb_rb > 20 ? 1 : 0,
+                     &t_info);
     }
     else if ((ulsch) &&
              (ulsch->rnti>0) &&
@@ -1404,14 +1408,12 @@ void pusch_procedures(PHY_VARS_eNB *eNB,L1_rxtx_proc_t *proc) {
   }   //   for (i=0; i<NUMBER_OF_ULSCH_MAX; i++)
 
   const bool decode = proc->nbDecode;
-  while (proc->nbDecode > 0) {
-    notifiedFIFO_elt_t *req=pullTpool(proc->respDecode, proc->threadPool);
-    if (req == NULL)
-      break; // Tpool has been stopped
-    postDecode(proc, req);
-    const time_stats_t ts = exec_time_stats_NotifiedFIFO(req);
-    merge_meas(&eNB->ulsch_turbo_decoding_stats, &ts);
-    delNotifiedFIFO_elt(req);
+  DevAssert(t_info.len == proc->nbDecode);
+  if (proc->nbDecode > 0) {
+    join_task_ans(t_info.ans, t_info.len);
+    for (size_t i = 0; i < t_info.len; ++i) {
+      postDecode(proc, &arr[i]);
+    }
   }
   if (decode)
     stop_meas(&eNB->ulsch_decoding_stats);
@@ -1437,7 +1439,7 @@ void fill_rx_indication(PHY_VARS_eNB *eNB,
   pthread_mutex_lock(&eNB->UL_INFO_mutex);
   eNB->UL_INFO.rx_ind.sfn_sf                    = frame<<4| subframe;
   eNB->UL_INFO.rx_ind.rx_indication_body.tl.tag = NFAPI_RX_INDICATION_BODY_TAG;
-  assert(eNB->UL_INFO.rx_ind.rx_indication_body.number_of_pdus <= NFAPI_RX_IND_MAX_PDU);
+  DevAssert(eNB->UL_INFO.rx_ind.rx_indication_body.number_of_pdus <= NFAPI_RX_IND_MAX_PDU);
   pdu                                    = &eNB->UL_INFO.rx_ind.rx_indication_body.rx_pdu_list[eNB->UL_INFO.rx_ind.rx_indication_body.number_of_pdus];
   //  pdu->rx_ue_information.handle          = eNB->ulsch[ULSCH_id]->handle;
   pdu->rx_ue_information.tl.tag          = NFAPI_RX_UE_INFORMATION_TAG;
@@ -1445,18 +1447,18 @@ void fill_rx_indication(PHY_VARS_eNB *eNB,
   pdu->rx_indication_rel8.tl.tag         = NFAPI_RX_INDICATION_REL8_TAG;
   pdu->rx_indication_rel8.length         = eNB->ulsch[ULSCH_id]->harq_processes[harq_pid]->TBS>>3;
   pdu->rx_indication_rel8.offset         = 1;   // DJP - I dont understand - but broken unless 1 ????  0;  // filled in at the end of the UL_INFO formation
-  assert(pdu->rx_indication_rel8.length <= NFAPI_RX_IND_DATA_MAX);
+  DevAssert(pdu->rx_indication_rel8.length <= NFAPI_RX_IND_DATA_MAX);
   memcpy(pdu->rx_ind_data,
          eNB->ulsch[ULSCH_id]->harq_processes[harq_pid]->decodedBytes,
          pdu->rx_indication_rel8.length);
 
   // estimate timing advance for MAC
   sync_pos                               = lte_est_timing_advance_pusch(&eNB->frame_parms, eNB->pusch_vars[ULSCH_id]->drs_ch_estimates_time);
-  timing_advance_update                  = sync_pos; 
+  timing_advance_update = sync_pos;
 
-  for (int i=0;i<NUMBER_OF_SCH_STATS_MAX;i++) 
-      if (eNB->ulsch_stats[i].rnti == eNB->ulsch[ULSCH_id]->rnti) 
-         eNB->ulsch_stats[i].timing_offset = sync_pos;
+  for (int i = 0; i < NUMBER_OF_SCH_STATS_MAX; i++)
+    if (eNB->ulsch_stats[i].rnti == eNB->ulsch[ULSCH_id]->rnti)
+      eNB->ulsch_stats[i].timing_offset = sync_pos;
   //  if (timing_advance_update > 10) { dump_ulsch(eNB,frame,subframe,ULSCH_id); exit(-1);}
   //  if (timing_advance_update < -10) { dump_ulsch(eNB,frame,subframe,ULSCH_id); exit(-1);}
   switch (eNB->frame_parms.N_RB_DL) {
@@ -1497,14 +1499,13 @@ void fill_rx_indication(PHY_VARS_eNB *eNB,
     timing_advance_update = 63;
 
   pdu->rx_indication_rel8.timing_advance = timing_advance_update;
-  // estimate UL_CQI for MAC 
-  int total_power=0, avg_noise_power=0;  
+  // estimate UL_CQI for MAC
+  int total_power = 0, avg_noise_power = 0;
   for (int i=0;i<eNB->frame_parms.nb_antennas_rx;i++) {
      total_power+=(eNB->pusch_vars[ULSCH_id]->ulsch_power[i]);
      avg_noise_power+=(eNB->pusch_vars[ULSCH_id]->ulsch_noise_power[i])/eNB->frame_parms.nb_antennas_rx;
   }
-  int SNRtimes10 = dB_fixed_x10(total_power) - 
-                   dB_fixed_x10(avg_noise_power);
+  int SNRtimes10 = dB_fixed_x10(total_power) - dB_fixed_x10(avg_noise_power);
 
   if (SNRtimes10 < -640)
     pdu->rx_indication_rel8.ul_cqi = 0;
@@ -1694,7 +1695,7 @@ int getM(PHY_VARS_eNB *eNB,int frame,int subframe) {
 
 void fill_ulsch_cqi_indication (PHY_VARS_eNB *eNB, uint16_t frame, uint8_t subframe, LTE_UL_eNB_HARQ_t *ulsch_harq, uint16_t rnti) {
   pthread_mutex_lock (&eNB->UL_INFO_mutex);
-  assert(eNB->UL_INFO.cqi_ind.cqi_indication_body.number_of_cqis <= NFAPI_CQI_IND_MAX_PDU);
+  DevAssert(eNB->UL_INFO.cqi_ind.cqi_indication_body.number_of_cqis <= NFAPI_CQI_IND_MAX_PDU);
   nfapi_cqi_indication_pdu_t *pdu         = &eNB->UL_INFO.cqi_ind.cqi_indication_body.cqi_pdu_list[eNB->UL_INFO.cqi_ind.cqi_indication_body.number_of_cqis];
   nfapi_cqi_indication_raw_pdu_t *raw_pdu = &eNB->UL_INFO.cqi_ind.cqi_indication_body.cqi_raw_pdu_list[eNB->UL_INFO.cqi_ind.cqi_indication_body.number_of_cqis];
   pdu->instance_length = 0;
@@ -1741,7 +1742,7 @@ void fill_ulsch_harq_indication (PHY_VARS_eNB *eNB, LTE_UL_eNB_HARQ_t *ulsch_har
 
   //AssertFatal(DLSCH_id>=0,"DLSCH_id doesn't exist\n");
   pthread_mutex_lock(&eNB->UL_INFO_mutex);
-  assert(eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs <= NFAPI_HARQ_IND_MAX_PDU);
+  DevAssert(eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs <= NFAPI_HARQ_IND_MAX_PDU);
   nfapi_harq_indication_pdu_t *pdu =   &eNB->UL_INFO.harq_ind.harq_indication_body.harq_pdu_list[eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs];
   int M;
   int i;
@@ -1811,7 +1812,7 @@ void fill_uci_harq_indication (int UEid, PHY_VARS_eNB *eNB, LTE_eNB_UCI *uci, in
   bool goodPacket=true;
   nfapi_harq_indication_t *ind       = &eNB->UL_INFO.harq_ind;
   nfapi_harq_indication_body_t *body = &ind->harq_indication_body;
-  assert(eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs <= NFAPI_HARQ_IND_MAX_PDU);
+  DevAssert(eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs <= NFAPI_HARQ_IND_MAX_PDU);
   nfapi_harq_indication_pdu_t *pdu   = &body->harq_pdu_list[eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs];
   ind->sfn_sf = frame<<4|subframe;
   ind->header.message_id = NFAPI_HARQ_INDICATION;
@@ -1987,7 +1988,7 @@ void fill_uci_harq_indication (int UEid, PHY_VARS_eNB *eNB, LTE_eNB_UCI *uci, in
             } else {
               pdu->harq_indication_tdd_rel13.harq_data[0].bundling.value_0 = 0;
             }
-	    
+
             break;
         }
 
@@ -1996,20 +1997,20 @@ void fill_uci_harq_indication (int UEid, PHY_VARS_eNB *eNB, LTE_eNB_UCI *uci, in
       }
     }
   }                             //TDD
-  
+
   if (goodPacket) {
     eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs++;
     LOG_D(PHY,"Incremented eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs:%d\n", eNB->UL_INFO.harq_ind.harq_indication_body.number_of_harqs);
   } else {
     LOG_W(PHY,"discarded a PUCCH because the decoded values are impossible\n");
   }
-    
+
   pthread_mutex_unlock(&eNB->UL_INFO_mutex);
 }
 
 void fill_crc_indication (PHY_VARS_eNB *eNB, int ULSCH_id, int frame, int subframe, uint8_t crc_flag) {
   pthread_mutex_lock(&eNB->UL_INFO_mutex);
-  assert(eNB->UL_INFO.crc_ind.crc_indication_body.number_of_crcs <= NFAPI_CRC_IND_MAX_PDU);
+  DevAssert(eNB->UL_INFO.crc_ind.crc_indication_body.number_of_crcs <= NFAPI_CRC_IND_MAX_PDU);
   nfapi_crc_indication_pdu_t *pdu =   &eNB->UL_INFO.crc_ind.crc_indication_body.crc_pdu_list[eNB->UL_INFO.crc_ind.crc_indication_body.number_of_crcs];
   eNB->UL_INFO.crc_ind.sfn_sf                         = frame<<4 | subframe;
   eNB->UL_INFO.crc_ind.header.message_id              = NFAPI_CRC_INDICATION;

openair1/SCHED_NR/nr_ru_procedures.c

@@ -38,6 +38,7 @@
 #include "common/utils/LOG/log.h"
 #include "common/utils/system.h"
 #include "common/utils/LOG/vcd_signal_dumper.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 #include "T.h"
 
@@ -74,7 +75,7 @@ void nr_feptx0(RU_t *ru,int tti_tx,int first_symbol, int num_symbols, int aa) {
   slot_offset += fp->ofdm_symbol_size*first_symbol;
 
   LOG_D(PHY,"SFN/SF:RU:TX:%d/%d aa %d Generating slot %d (first_symbol %d num_symbols %d) slot_offset %d, slot_offsetF %d\n",ru->proc.frame_tx, ru->proc.tti_tx,aa,slot,first_symbol,num_symbols,slot_offset,slot_offsetF);
-  
+
   if (fp->Ncp == 1) {
     PHY_ofdm_mod(&ru->common.txdataF_BF[aa][slot_offsetF],
                  (int*)&ru->common.txdata[aa][slot_offset],
@@ -84,7 +85,6 @@ void nr_feptx0(RU_t *ru,int tti_tx,int first_symbol, int num_symbols, int aa) {
                  CYCLIC_PREFIX);
   } else {
     if (fp->numerology_index != 0) {
-      
       if (!(slot%(fp->slots_per_subframe/2))&&(first_symbol==0)) { // case where first symbol in slot has longer prefix
         PHY_ofdm_mod(&ru->common.txdataF_BF[aa][slot_offsetF],
                      (int*)&ru->common.txdata[aa][slot_offset],
@@ -136,13 +136,12 @@ void nr_feptx0(RU_t *ru,int tti_tx,int first_symbol, int num_symbols, int aa) {
   }
 
   if (aa==0 && first_symbol==0) stop_meas(&ru->ofdm_mod_stats);
-        
+
   //VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPTX_OFDM+(first_symbol!=0?1:0), 0);
 }
 
 // RU FEP TX OFDM modulation, single-thread
 void nr_feptx_ofdm(RU_t *ru,int frame_tx,int tti_tx) {
-     
   nfapi_nr_config_request_scf_t *cfg = &ru->gNB_list[0]->gNB_config;
   NR_DL_FRAME_PARMS *fp=ru->nr_frame_parms;
   int cyclic_prefix_type = NFAPI_CP_NORMAL;
@@ -231,8 +230,8 @@ void nr_fep_full(RU_t *ru, int slot) {
   int l, aa;
   NR_DL_FRAME_PARMS *fp = ru->nr_frame_parms;
 
-  // if ((fp->frame_type == TDD) && 
-     // (subframe_select(fp,proc->tti_rx) != NR_UPLINK_SLOT)) return;
+  // if ((fp->frame_type == TDD) &&
+  // (subframe_select(fp,proc->tti_rx) != NR_UPLINK_SLOT)) return;
 
   LOG_D(PHY,"In fep_full for slot = %d\n", proc->tti_rx);
 
@@ -256,11 +255,9 @@ void nr_fep_full(RU_t *ru, int slot) {
 
   if (ru->idx == 0) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX, 0 );
   stop_meas(&ru->ofdm_demod_stats);
-  
-  
 }
 
-// core routine for FEP TX, called from threads in RU TX thread-pool 
+// core routine for FEP TX, called from threads in RU TX thread-pool
 void nr_feptx(void *arg) {
 
   feptx_cmd_t *feptx = (feptx_cmd_t *)arg;
@@ -271,13 +268,13 @@ void nr_feptx(void *arg) {
   int  startSymbol = feptx->startSymbol;
   NR_DL_FRAME_PARMS  *fp    = ru->nr_frame_parms;
   int  numSymbols  = feptx->numSymbols;
-  int  numSamples  = feptx->numSymbols*fp->ofdm_symbol_size; 
+  int numSamples = feptx->numSymbols * fp->ofdm_symbol_size;
   int txdataF_offset = (slot*fp->samples_per_slot_wCP) + startSymbol*fp->ofdm_symbol_size;
   int txdataF_BF_offset = startSymbol*fp->ofdm_symbol_size;
 
       ////////////precoding////////////
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPTX_PREC+feptx->aid , 1);
-      
+
   if (aa==0) start_meas(&ru->precoding_stats);
 
   if (ru->do_precoding == 1) {
@@ -319,45 +316,58 @@ void nr_feptx(void *arg) {
 
       ////////////FEPTX////////////
   nr_feptx0(ru,slot,startSymbol,numSymbols,aa);
+
+  // Task completed in //
+  completed_task_ans(feptx->ans);
 }
 
 // RU FEP TX using thread-pool
 void nr_feptx_tp(RU_t *ru, int frame_tx, int slot) {
-
   nfapi_nr_config_request_scf_t *cfg = &ru->gNB_list[0]->gNB_config;
-  int nbfeptx=0;
-  if (nr_slot_select(cfg,frame_tx,slot) == NR_UPLINK_SLOT) return;
-//  for (int aa=0; aa<ru->nb_tx; aa++) memset(ru->common.txdataF[aa],0,ru->nr_frame_parms->samples_per_slot_wCP*sizeof(int32_t));
+  if (nr_slot_select(cfg, frame_tx, slot) == NR_UPLINK_SLOT)
+     return;
 
   if (ru->idx == 0) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPTX_OFDM, 1 );
   start_meas(&ru->ofdm_total_stats);
-  for (int aid=0;aid<ru->nb_tx;aid++) {
-       notifiedFIFO_elt_t *req=newNotifiedFIFO_elt(sizeof(feptx_cmd_t), 2000 + aid,ru->respfeptx,nr_feptx);
-       feptx_cmd_t *feptx_cmd=(feptx_cmd_t*)NotifiedFifoData(req);       
-       feptx_cmd->aid          = aid;
-       feptx_cmd->ru           = ru;
-       feptx_cmd->slot         = slot;
-       feptx_cmd->startSymbol  = 0;
-       feptx_cmd->numSymbols   = (ru->half_slot_parallelization>0)?ru->nr_frame_parms->symbols_per_slot>>1:ru->nr_frame_parms->symbols_per_slot;
-       pushTpool(ru->threadPool,req);
-       nbfeptx++;
-       if (ru->half_slot_parallelization>0) {
-         notifiedFIFO_elt_t *req=newNotifiedFIFO_elt(sizeof(feptx_cmd_t), 2000 + aid + ru->nb_tx,ru->respfeptx,nr_feptx);
-         feptx_cmd_t *feptx_cmd=(feptx_cmd_t*)NotifiedFifoData(req);       
-         feptx_cmd->aid          = aid;
-         feptx_cmd->ru           = ru;
-         feptx_cmd->slot         = slot;
-         feptx_cmd->startSymbol  = ru->nr_frame_parms->symbols_per_slot>>1;
-         feptx_cmd->numSymbols   = ru->nr_frame_parms->symbols_per_slot>>1;
-         pushTpool(ru->threadPool,req);
-         nbfeptx++;
-       }
-  }
-  while (nbfeptx>0) {
-    notifiedFIFO_elt_t *req=pullTpool(ru->respfeptx, ru->threadPool);
-    delNotifiedFIFO_elt(req);
-    nbfeptx--;
+
+  size_t const sz = ru->nb_tx + (ru->half_slot_parallelization > 0) * ru->nb_tx;
+  AssertFatal(sz < 64, "Please, increase the buffer size");
+  feptx_cmd_t arr[64] = {0};
+  task_ans_t ans[64] = {0};
+
+  int nbfeptx = 0;
+  for (int aid = 0; aid < ru->nb_tx; aid++) {
+    feptx_cmd_t *feptx_cmd = &arr[nbfeptx];
+    feptx_cmd->ans = &ans[nbfeptx];
+
+    feptx_cmd->aid = aid;
+    feptx_cmd->ru = ru;
+    feptx_cmd->slot = slot;
+    feptx_cmd->startSymbol = 0;
+    feptx_cmd->numSymbols =
+        (ru->half_slot_parallelization > 0) ? ru->nr_frame_parms->symbols_per_slot >> 1 : ru->nr_frame_parms->symbols_per_slot;
+
+    task_t t = {.func = nr_feptx, .args = feptx_cmd};
+    async_task_manager(&ru->thread_pool, t);
+    nbfeptx++;
+    if (ru->half_slot_parallelization > 0) {
+      feptx_cmd_t *feptx_cmd = &arr[nbfeptx];
+      feptx_cmd->ans = &ans[nbfeptx];
+
+      feptx_cmd->aid = aid;
+      feptx_cmd->ru = ru;
+      feptx_cmd->slot = slot;
+      feptx_cmd->startSymbol = ru->nr_frame_parms->symbols_per_slot >> 1;
+      feptx_cmd->numSymbols = ru->nr_frame_parms->symbols_per_slot >> 1;
+
+      task_t t = {.func = nr_feptx, .args = feptx_cmd};
+      async_task_manager(&ru->thread_pool, t);
+      nbfeptx++;
+    }
   }
+
+  join_task_ans(ans, nbfeptx);
+
   stop_meas(&ru->ofdm_total_stats);
   if (ru->idx == 0) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPTX_OFDM, 0 );
 }
@@ -373,20 +383,18 @@ void nr_fep(void* arg) {
   int startSymbol  = feprx_cmd->startSymbol;
   int endSymbol    = feprx_cmd->endSymbol;
   NR_DL_FRAME_PARMS *fp = ru->nr_frame_parms;
-  
+
   LOG_D(PHY,"In nr_fep for aid %d, slot = %d, startSymbol %d, endSymbol %d\n", aid, tti_rx,startSymbol,endSymbol);
 
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX+aid, 1);
 
   int offset = (tti_rx % RU_RX_SLOT_DEPTH) * fp->symbols_per_slot * fp->ofdm_symbol_size;
-  for (int l = startSymbol; l <= endSymbol; l++) 
-      nr_slot_fep_ul(fp,
-                     ru->common.rxdata[aid],
-                     &ru->common.rxdataF[aid][offset],
-                     l,
-                     tti_rx,
-                     ru->N_TA_offset);
+  for (int l = startSymbol; l <= endSymbol; l++)
+     nr_slot_fep_ul(fp, ru->common.rxdata[aid], &ru->common.rxdataF[aid][offset], l, tti_rx, ru->N_TA_offset);
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX+aid, 0);
+
+  // Task completed in //
+  completed_task_ans(feprx_cmd->ans);
 }
 
 // RU RX FEP using thread-pool
@@ -395,34 +403,45 @@ void nr_fep_tp(RU_t *ru, int slot) {
   int nbfeprx=0;
   if (ru->idx == 0) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX, 1 );
   start_meas(&ru->ofdm_demod_stats);
+
+  size_t const sz = ru->nb_rx + (ru->half_slot_parallelization > 0) * ru->nb_rx;
+  AssertFatal(sz < 64, "Please, increase buffer size");
+  feprx_cmd_t arr[64] = {0};
+  task_ans_t ans[64] = {0};
+
   for (int aid=0;aid<ru->nb_rx;aid++) {
-       notifiedFIFO_elt_t *req=newNotifiedFIFO_elt(sizeof(feprx_cmd_t), 1000 + aid,ru->respfeprx,nr_fep);
-       feprx_cmd_t *feprx_cmd=(feprx_cmd_t*)NotifiedFifoData(req);       
-       feprx_cmd->aid          = aid;
-       feprx_cmd->ru           = ru;
-       feprx_cmd->slot         = ru->proc.tti_rx;
-       feprx_cmd->startSymbol  = 0;
-       feprx_cmd->endSymbol    = (ru->half_slot_parallelization > 0)?(ru->nr_frame_parms->symbols_per_slot>>1)-1:(ru->nr_frame_parms->symbols_per_slot-1);
-       pushTpool(ru->threadPool,req);
-       nbfeprx++;
-       if (ru->half_slot_parallelization>0) {
-         notifiedFIFO_elt_t *req=newNotifiedFIFO_elt(sizeof(feprx_cmd_t), 1000 + aid + ru->nb_rx,ru->respfeprx,nr_fep);
-         feprx_cmd_t *feprx_cmd=(feprx_cmd_t*)NotifiedFifoData(req);       
-         feprx_cmd->aid          = aid;
-         feprx_cmd->ru           = ru;
-         feprx_cmd->slot         = ru->proc.tti_rx;
-	 feprx_cmd->startSymbol  = ru->nr_frame_parms->symbols_per_slot>>1;
-         feprx_cmd->endSymbol    = ru->nr_frame_parms->symbols_per_slot-1;
-         pushTpool(ru->threadPool,req);
-         nbfeprx++;
-       }
-  }
-  while (nbfeprx>0) {
-    notifiedFIFO_elt_t *req=pullTpool(ru->respfeprx, ru->threadPool);
-    delNotifiedFIFO_elt(req);
-    nbfeprx--;
+    feprx_cmd_t *feprx_cmd = &arr[nbfeprx];
+    feprx_cmd->ans = &ans[nbfeprx];
+
+    feprx_cmd->aid = aid;
+    feprx_cmd->ru = ru;
+    feprx_cmd->slot = ru->proc.tti_rx;
+    feprx_cmd->startSymbol = 0;
+    feprx_cmd->endSymbol = (ru->half_slot_parallelization > 0) ? (ru->nr_frame_parms->symbols_per_slot >> 1) - 1
+                                                               : (ru->nr_frame_parms->symbols_per_slot - 1);
+
+    task_t t = {.func = nr_fep, .args = feprx_cmd};
+    async_task_manager(&ru->thread_pool, t);
+    nbfeprx++;
+    if (ru->half_slot_parallelization > 0) {
+      feprx_cmd_t *feprx_cmd = &arr[nbfeprx];
+      feprx_cmd->ans = &ans[nbfeprx];
+
+      feprx_cmd->aid = aid;
+      feprx_cmd->ru = ru;
+      feprx_cmd->slot = ru->proc.tti_rx;
+      feprx_cmd->startSymbol = ru->nr_frame_parms->symbols_per_slot >> 1;
+      feprx_cmd->endSymbol = ru->nr_frame_parms->symbols_per_slot - 1;
+
+      task_t t = {.func = nr_fep, .args = feprx_cmd};
+      async_task_manager(&ru->thread_pool, t);
+
+      nbfeprx++;
+    }
   }
+
+  join_task_ans(ans, nbfeprx);
+
   stop_meas(&ru->ofdm_demod_stats);
   if (ru->idx == 0) VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME( VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_RU_FEPRX, 0 );
 }
-

openair1/SCHED_NR/phy_procedures_nr_gNB.c

@@ -39,6 +39,7 @@
 #include "nfapi/oai_integration/vendor_ext.h"
 #include "assertions.h"
 #include <time.h>
+#include <stdint.h>
 
 //#define DEBUG_RXDATA
 //#define SRS_IND_DEBUG
@@ -261,9 +262,8 @@ void phy_procedures_gNB_TX(processingData_L1tx_t *msgTx,
   VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_PROCEDURES_gNB_TX + gNB->CC_id, 0);
 }
 
-static void nr_postDecode(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req)
+static void nr_postDecode(PHY_VARS_gNB *gNB, ldpcDecode_t *rdata)
 {
-  ldpcDecode_t *rdata = (ldpcDecode_t*) NotifiedFifoData(req);
   NR_UL_gNB_HARQ_t *ulsch_harq = rdata->ulsch_harq;
   NR_gNB_ULSCH_t *ulsch = rdata->ulsch;
   int r = rdata->segment_r;
@@ -336,6 +336,7 @@ static void nr_postDecode(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req)
       //      dumpsig=1;
     }
     ulsch->last_iteration_cnt = rdata->decodeIterations;
+
     /*
         if (ulsch_harq->ulsch_pdu.mcs_index == 0 && dumpsig==1) {
           int off = ((ulsch_harq->ulsch_pdu.rb_size&1) == 1)? 4:0;
@@ -373,12 +374,18 @@ static void nr_postDecode(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req)
 
         }
     */
+
     ulsch->last_iteration_cnt = rdata->decodeIterations;
     VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_PHY_gNB_ULSCH_DECODING,0);
   }
 }
 
-static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int ULSCH_id, uint8_t harq_pid)
+static int nr_ulsch_procedures(PHY_VARS_gNB *gNB,
+                               int frame_rx,
+                               int slot_rx,
+                               int ULSCH_id,
+                               uint8_t harq_pid,
+                               thread_info_tm_t *t_info)
 {
   NR_DL_FRAME_PARMS *frame_parms = &gNB->frame_parms;
   nfapi_nr_pusch_pdu_t *pusch_pdu = &gNB->ulsch[ULSCH_id].harq_process->ulsch_pdu;
@@ -429,9 +436,17 @@ static int nr_ulsch_procedures(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx, int
    * measurement per processed TB.*/
   if (gNB->max_nb_pusch == 1)
     start_meas(&gNB->ulsch_decoding_stats);
-  int nbDecode =
-      nr_ulsch_decoding(gNB, ULSCH_id, gNB->pusch_vars[ULSCH_id].llr, frame_parms, pusch_pdu, frame_rx, slot_rx, harq_pid, G);
 
+  int const nbDecode = nr_ulsch_decoding(gNB,
+                                         ULSCH_id,
+                                         gNB->pusch_vars[ULSCH_id].llr,
+                                         frame_parms,
+                                         pusch_pdu,
+                                         frame_rx,
+                                         slot_rx,
+                                         harq_pid,
+                                         G,
+                                         t_info);
   return nbDecode;
 }
 
@@ -810,6 +825,11 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx)
     }
   }
 
+  ldpcDecode_t arr[64];
+  task_ans_t ans[64] = {0};
+  thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .len = 0, .cap = 64, .ans = ans};
+
+  // int64_t const t0 = time_now_ns();
   int totalDecode = 0;
   for (int ULSCH_id = 0; ULSCH_id < gNB->max_nb_pusch; ULSCH_id++) {
     NR_gNB_ULSCH_t *ulsch = &gNB->ulsch[ULSCH_id];
@@ -907,27 +927,26 @@ int phy_procedures_gNB_uespec_RX(PHY_VARS_gNB *gNB, int frame_rx, int slot_rx)
       // LOG_M("rxdataF_comp.m","rxF_comp",gNB->pusch_vars[0]->rxdataF_comp[0],6900,1,1);
       // LOG_M("rxdataF_ext.m","rxF_ext",gNB->pusch_vars[0]->rxdataF_ext[0],6900,1,1);
       VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_ULSCH_PROCEDURES_RX, 1);
-      int const tasks_added = nr_ulsch_procedures(gNB, frame_rx, slot_rx, ULSCH_id, ulsch->harq_pid);
+      int const tasks_added = nr_ulsch_procedures(gNB, frame_rx, slot_rx, ULSCH_id, ulsch->harq_pid, &t_info);
       if (tasks_added > 0)
-        totalDecode += tasks_added; 
-
+        totalDecode += tasks_added;
       VCD_SIGNAL_DUMPER_DUMP_FUNCTION_BY_NAME(VCD_SIGNAL_DUMPER_FUNCTIONS_NR_ULSCH_PROCEDURES_RX, 0);
     }
   }
-    while (totalDecode > 0) {
-      notifiedFIFO_elt_t *req = pullTpool(&gNB->respDecode, &gNB->threadPool);
-      if (req == NULL)
-        break; // Tpool has been stopped
-      nr_postDecode(gNB, req);
-      delNotifiedFIFO_elt(req);
-      totalDecode--;
-    }
+
+  DevAssert(totalDecode == t_info.len);
+
+  join_task_ans(t_info.ans, t_info.len);
+  for (int i = 0; i < t_info.len; ++i) {
+    nr_postDecode(gNB, &arr[i]);
+  }
   /* Do ULSCH decoding time measurement only when number of PUSCH is limited to 1
    * (valid for unitary physical simulators). ULSCH processing loop is then executed
    * only once, which ensures exactly one start and stop of the ULSCH decoding time
    * measurement per processed TB.*/
   if (gNB->max_nb_pusch == 1)
     stop_meas(&gNB->ulsch_decoding_stats);
+
   for (int i = 0; i < gNB->max_nb_srs; i++) {
     NR_gNB_SRS_t *srs = &gNB->srs[i];
     if (srs) {

openair1/SIMULATION/LTE_PHY/dlsim.c

@@ -56,6 +56,7 @@
 #include "dummy_functions.c"
 #include "executables/thread-common.h"
 #include "common/ran_context.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 void feptx_ofdm(RU_t *ru, int frame, int subframe);
 void feptx_prec(RU_t *ru, int frame, int subframe);
 
@@ -589,6 +590,7 @@ int main(int argc, char **argv) {
   nfapi_tx_request_t TX_req;
   Sched_Rsp_t sched_resp;
   int pa=dB0;
+  task_manager_t thread_pool = {0};
   cpu_freq_GHz = get_cpu_freq_GHz();
   printf("Detected cpu_freq %f GHz\n",cpu_freq_GHz);
   memset((void *)&sched_resp,0,sizeof(sched_resp));
@@ -1262,11 +1264,13 @@ int main(int argc, char **argv) {
   }
 
   L1_rxtx_proc_t *proc_eNB = &eNB->proc.L1_proc;
-  proc_eNB->threadPool = (tpool_t *)malloc(sizeof(tpool_t));
-  proc_eNB->respDecode=(notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
-  initTpool("n", proc_eNB->threadPool, true);
+  proc_eNB->respDecode = (notifiedFIFO_t *)malloc(sizeof(notifiedFIFO_t));
   initNotifiedFIFO(proc_eNB->respDecode);
 
+  int lst_core_id = -1;
+  init_task_manager(&thread_pool, &lst_core_id, 1);
+  proc_eNB->thread_pool = &thread_pool;
+
   proc_eNB->frame_tx=0;
 
   if (input_fd==NULL) {

openair1/SIMULATION/LTE_PHY/ulsim.c

@@ -55,6 +55,7 @@
 #include "PHY/LTE_ESTIMATION/lte_estimation.h"
 #include "openair1/PHY/LTE_TRANSPORT/dlsch_tbs_full.h"
 #include "PHY/phy_extern.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 const char *__asan_default_options()
 {
@@ -795,9 +796,13 @@ int main(int argc, char **argv) {
   proc_rxtx_ue->frame_rx = (subframe<4)?(proc_rxtx->frame_tx-1):(proc_rxtx->frame_tx);
   proc_rxtx_ue->subframe_tx = proc_rxtx->subframe_rx;
   proc_rxtx_ue->subframe_rx = (proc_rxtx->subframe_tx+6)%10;
-  proc_rxtx->threadPool = (tpool_t *)malloc(sizeof(tpool_t));
-  proc_rxtx->respDecode=(notifiedFIFO_t*) malloc(sizeof(notifiedFIFO_t));
-  initTpool("n", proc_rxtx->threadPool, true);
+
+  int lst_core_id = -1;
+  proc_rxtx->thread_pool = calloc(1, sizeof(task_manager_t));
+  AssertFatal(proc_rxtx->thread_pool != NULL, "Memory exhausted");
+  init_task_manager(proc_rxtx->thread_pool, &lst_core_id, 1);
+
+  proc_rxtx->respDecode = (notifiedFIFO_t *)malloc(sizeof(notifiedFIFO_t));
   initNotifiedFIFO(proc_rxtx->respDecode);
 
   printf("Init UL hopping UE\n");

openair1/SIMULATION/NR_PHY/dlschsim.c

@@ -49,6 +49,7 @@
 #include "openair2/LAYER2/NR_MAC_COMMON/nr_mac_common.h"
 #include "executables/nr-uesoftmodem.h"
 #include "nfapi/oai_integration/vendor_ext.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 //#define DEBUG_NR_DLSCHSIM
 
@@ -379,11 +380,18 @@ int main(int argc, char **argv)
 	RC.gNB = (PHY_VARS_gNB **) malloc(sizeof(PHY_VARS_gNB *));
 	RC.gNB[0] = calloc(1, sizeof(PHY_VARS_gNB));
 	gNB = RC.gNB[0];
-	initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool");
-        initFloatingCoresTpool(dlsch_threads, &nrUE_params.Tpool, false, "UE-tpool");
-	//gNB_config = &gNB->gNB_config;
-	frame_parms = &gNB->frame_parms; //to be initialized I suppose (maybe not necessary for PBCH)
-	frame_parms->nb_antennas_tx = n_tx;
+
+  int core_id[128] = {0};
+  span_core_id_t out = {.cap = 128, .core_id = core_id};
+  parse_num_threads(gNBthreads, &out);
+  init_task_manager(&gNB->thread_pool, out.core_id, out.sz);
+
+  int lst_core_id[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
+  DevAssert(dlsch_threads < 8);
+  init_task_manager(&nrUE_params.thread_pool, lst_core_id, max(dlsch_threads, 1));
+
+  frame_parms = &gNB->frame_parms; // to be initialized I suppose (maybe not necessary for PBCH)
+  frame_parms->nb_antennas_tx = n_tx;
 	frame_parms->nb_antennas_rx = n_rx;
 	frame_parms->N_RB_DL = N_RB_DL;
 	frame_parms->Ncp = extended_prefix_flag ? EXTENDED : NORMAL;
@@ -661,6 +669,10 @@ int main(int argc, char **argv)
     free(gNB->gNB_config.tdd_table.max_tdd_periodicity_list[i].max_num_of_symbol_per_slot_list);
   free(gNB->gNB_config.tdd_table.max_tdd_periodicity_list);
 
+  void (*clean)(task_t *) = NULL;
+  free_task_manager(&nrUE_params.thread_pool, clean);
+  free_task_manager(&gNB->thread_pool, clean);
+
   phy_free_nr_gNB(gNB);
   free(RC.gNB[0]);
   free(RC.gNB);

openair1/SIMULATION/NR_PHY/dlsim.c

@@ -30,6 +30,8 @@
 #include "common/utils/nr/nr_common.h"
 #include "common/utils/var_array.h"
 #include "common/utils/LOG/log.h"
+#include "common/utils/task_manager/task_manager_gen.h"
+
 #include "LAYER2/NR_MAC_gNB/nr_mac_gNB.h"
 #include "LAYER2/NR_MAC_UE/mac_defs.h"
 #include "LAYER2/NR_MAC_UE/mac_extern.h"
@@ -72,6 +74,7 @@
 #include <executables/softmodem-common.h>
 #include <openair3/ocp-gtpu/gtp_itf.h>
 #include <executables/nr-uesoftmodem.h>
+#include "common/utils/task_manager/task_manager_gen.h"
 
 const char *__asan_default_options()
 {
@@ -861,9 +864,11 @@ int main(int argc, char **argv)
   unsigned char *test_input_bit;
   unsigned int errors_bit = 0;
 
-  initFloatingCoresTpool(dlsch_threads, &nrUE_params.Tpool, false, "UE-tpool");
+  int lst_core_id[8] = {-1, -1, -1, -1, -1, -1, -1, -1};
+  DevAssert(dlsch_threads < 9);
+  init_task_manager(&nrUE_params.thread_pool, lst_core_id, max(1, dlsch_threads));
 
-  test_input_bit = (unsigned char *) malloc16(sizeof(unsigned char) * 16 * 68 * 384);
+  test_input_bit = (unsigned char *)malloc16(sizeof(unsigned char) * 16 * 68 * 384);
   estimated_output_bit = (unsigned char *) malloc16(sizeof(unsigned char) * 16 * 68 * 384);
   
   // generate signal
@@ -890,7 +895,11 @@ int main(int argc, char **argv)
   //NR_COMMON_channels_t *cc = RC.nrmac[0]->common_channels;
   int n_errs = 0;
 
-  initNamedTpool(gNBthreads, &gNB->threadPool, true, "gNB-tpool");
+  int core_id[128] = {0};
+  span_core_id_t out = {.cap = 128, .core_id = core_id};
+  parse_num_threads(gNBthreads, &out);
+  init_task_manager(&gNB->thread_pool, out.core_id, out.sz);
+
   initNotifiedFIFO(&gNB->L1_tx_free);
   initNotifiedFIFO(&gNB->L1_tx_filled);
   initNotifiedFIFO(&gNB->L1_tx_out);
@@ -1273,6 +1282,10 @@ int main(int argc, char **argv)
     free(r_im[i]);
   }
 
+  void (*clean)(task_t *) = NULL;
+  free_task_manager(&nrUE_params.thread_pool, clean);
+  free_task_manager(&gNB->thread_pool, clean);
+
   free(s_re);
   free(s_im);
   free(r_re);

openair1/SIMULATION/NR_PHY/pbchsim.c

@@ -503,7 +503,16 @@ int main(int argc, char **argv)
   set_tdd_config_nr(&gNB->gNB_config, mu, 7, 6, 2, 4);
   phy_init_nr_gNB(gNB);
   frame_parms->ssb_start_subcarrier = 12 * gNB->gNB_config.ssb_table.ssb_offset_point_a.value + ssb_subcarrier_offset;
-  initFloatingCoresTpool(ssb_scan_threads, &nrUE_params.Tpool, false, "UE-tpool");
+
+  if (ssb_scan_threads <= 0) {
+    LOG_W(PHY, "ssb_scan_threads is %d, should be > 0, setting it to 1\n", ssb_scan_threads);
+    ssb_scan_threads = 1;
+  }
+
+  int core_id[ssb_scan_threads];
+  for (int i = 0; i < ssb_scan_threads; i++)
+    core_id[i] = -1;
+  init_task_manager(&nrUE_params.thread_pool, core_id, ssb_scan_threads);
 
   int n_hf = 0;
   int cyclic_prefix_type = NFAPI_CP_NORMAL;
@@ -889,6 +898,9 @@ int main(int argc, char **argv)
     free(txdata[i]);
   }
 
+  void (*clean)(task_t *) = NULL;
+  free_task_manager(&nrUE_params.thread_pool, clean);
+
   free(s_re);
   free(s_im);
   free(r_re);

openair1/SIMULATION/NR_PHY/ulschsim.c

@@ -45,10 +45,11 @@
 #include "openair1/SIMULATION/TOOLS/sim.h"
 #include "openair1/SIMULATION/RF/rf.h"
 #include "openair1/SIMULATION/NR_PHY/nr_unitary_defs.h"
-#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 #include "openair2/LAYER2/NR_MAC_COMMON/nr_mac_common.h"
 #include "executables/nr-uesoftmodem.h"
 #include "nfapi/oai_integration/vendor_ext.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 
 //#define DEBUG_NR_ULSCHSIM
 
@@ -91,9 +92,8 @@ void deref_sched_response(int _)
   exit(1);
 }
 
-int nr_postDecode_sim(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req, int *nb_ok)
+int nr_postDecode_sim(PHY_VARS_gNB *gNB, ldpcDecode_t *rdata, int *nb_ok)
 {
-  ldpcDecode_t *rdata = (ldpcDecode_t*) NotifiedFifoData(req);
   NR_UL_gNB_HARQ_t *ulsch_harq = rdata->ulsch_harq;
   int r = rdata->segment_r;
 
@@ -107,8 +107,10 @@ int nr_postDecode_sim(PHY_VARS_gNB *gNB, notifiedFIFO_elt_t *req, int *nb_ok)
   }
 
   // if all segments are done
-  if (rdata->nbSegments == ulsch_harq->processedSegments)
+  if (rdata->nbSegments == ulsch_harq->processedSegments) {
     return *nb_ok == rdata->nbSegments;
+  }
+
   return 0;
 }
 
@@ -414,7 +416,9 @@ int main(int argc, char **argv)
   gNB = RC.gNB[0];
   //gNB_config = &gNB->gNB_config;
 
-  initTpool("n", &gNB->threadPool, true);
+  int lst_core_id = -1;
+  init_task_manager(&gNB->thread_pool, &lst_core_id, 1);
+
   initNotifiedFIFO(&gNB->respDecode);
   frame_parms = &gNB->frame_parms; //to be initialized I suppose (maybe not necessary for PBCH)
   frame_parms->N_RB_DL = N_RB_DL;
@@ -595,20 +599,23 @@ int main(int argc, char **argv)
       exit(-1);
 #endif
 
-     int nbDecode = nr_ulsch_decoding(gNB, UE_id, channel_output_fixed, frame_parms, rel15_ul, frame, subframe, harq_pid, G);
-     int nb_ok = 0;
-     if (nbDecode > 0)
-       while (nbDecode > 0) {
-         notifiedFIFO_elt_t *req = pullTpool(&gNB->respDecode, &gNB->threadPool);
-         ret = nr_postDecode_sim(gNB, req, &nb_ok);
-         delNotifiedFIFO_elt(req);
-         nbDecode--;
-       }
+      ldpcDecode_t arr[16] = {0};
+      task_ans_t ans[16] = {0};
+      thread_info_tm_t t_info = {.buf = (uint8_t *)arr, .cap = 16, .len = 0, .ans = ans};
+      int nbDecode =
+          nr_ulsch_decoding(gNB, UE_id, channel_output_fixed, frame_parms, rel15_ul, frame, subframe, harq_pid, G, &t_info);
+      DevAssert(nbDecode > 0);
 
+      int nb_ok = 0;
+      join_task_ans(t_info.ans, t_info.len);
+      for (size_t i = 0; i < nbDecode; ++i) {
+        ret = nr_postDecode_sim(gNB, &arr[i], &nb_ok);
+      }
+      nbDecode = 0;
       if (ret)
         n_errors++;
     }
-    
+
     printf("*****************************************\n");
     printf("SNR %f, BLER %f (false positive %f)\n", SNR,
            (float) n_errors / (float) n_trials,
@@ -631,6 +638,9 @@ int main(int argc, char **argv)
     free(gNB->gNB_config.tdd_table.max_tdd_periodicity_list[i].max_num_of_symbol_per_slot_list);
   free(gNB->gNB_config.tdd_table.max_tdd_periodicity_list);
 
+  void (*clean)(task_t *args) = NULL;
+  free_task_manager(&gNB->thread_pool, clean);
+
   term_nr_ue_signal(UE, 1);
   free(UE);
 

openair1/SIMULATION/NR_PHY/ulsim.c

@@ -56,7 +56,7 @@
 #include "openair2/RRC/NR/nr_rrc_config.h"
 #include "openair2/LAYER2/NR_MAC_UE/mac_proto.h"
 #include "openair2/LAYER2/NR_MAC_gNB/mac_proto.h"
-#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 #include "PHY/NR_REFSIG/ptrs_nr.h"
 #define inMicroS(a) (((double)(a))/(get_cpu_freq_GHz()*1000.0))
 #include "SIMULATION/LTE_PHY/common_sim.h"
@@ -67,6 +67,7 @@
 #include "PHY/NR_REFSIG/ul_ref_seq_nr.h"
 #include <openair3/ocp-gtpu/gtp_itf.h>
 #include "executables/nr-uesoftmodem.h"
+#include "common/utils/task_manager/task_manager_gen.h"
 //#define DEBUG_ULSIM
 
 const char *__asan_default_options()
@@ -562,7 +563,14 @@ int main(int argc, char *argv[])
   gNB->RU_list[0] = calloc(1, sizeof(**gNB->RU_list));
   gNB->RU_list[0]->rfdevice.openair0_cfg = openair0_cfg;
 
-  initFloatingCoresTpool(threadCnt, &gNB->threadPool, false, "gNB-tpool");
+  const int max_cid = 32;
+  int lst_core_id[max_cid];
+  for (int i = 0; i < max_cid; ++i) {
+    lst_core_id[i] = -1;
+  }
+  DevAssert(threadCnt < max_cid);
+  init_task_manager(&gNB->thread_pool, lst_core_id, max(threadCnt, 1));
+
   initNotifiedFIFO(&gNB->respDecode);
 
   initNotifiedFIFO(&gNB->respPuschSymb);

openair2/PHY_INTERFACE/IF_Module.h

@@ -37,7 +37,7 @@
 //#include "openair1/PHY/LTE_TRANSPORT/transport_eNB.h"
 #include "nfapi_interface.h"
 #include "common/platform_types.h"
-#include <common/utils/threadPool/thread-pool.h>
+#include <common/utils/task_manager/threadPool/thread-pool.h>
 #include <radio/COMMON/common_lib.h>
 
 #define MAX_NUM_DL_PDU 100

radio/COMMON/common_lib.h

@@ -37,7 +37,7 @@
 #include <sys/types.h>
 #include <openair1/PHY/TOOLS/tools_defs.h>
 #include "record_player.h"
-#include <common/utils/threadPool/thread-pool.h>
+#include <common/utils/task_manager/threadPool/thread-pool.h>
 
 /* default name of shared library implementing the radio front end */
 #define OAI_RF_LIBNAME        "oai_device"

radio/ETHERNET/eth_udp.c

@@ -50,7 +50,7 @@
 #include "common_lib.h"
 #include "ethernet_lib.h"
 #include "openair1/PHY/sse_intrin.h"
-#include "common/utils/threadPool/thread-pool.h"
+#include "common/utils/task_manager/threadPool/thread-pool.h"
 
 //#define DEBUG 1
 

radio/ETHERNET/ethernet_lib.h

@@ -41,7 +41,7 @@
 #include <sys/socket.h>
 #include <net/if.h>
 #include <netinet/ether.h>
-#include <common/utils/threadPool/thread-pool.h>
+#include <common/utils/task_manager/threadPool/thread-pool.h>
 
 #define MAX_INST 4
 #define DEFAULT_IF "lo"

radio/fhi_72/oran_isolate.c

@@ -31,7 +31,6 @@
 #include "common/utils/LOG/log.h"
 #include "common/utils/LOG/vcd_signal_dumper.h"
 #include "openair1/PHY/defs_gNB.h"
-#include "common/utils/threadPool/thread-pool.h"
 #include "oaioran.h"
 
 // include the following file for VERSIONX, version of xran lib, to print it during