Use cmake to enable a C/C++ compiler warning in a subdirectory

Integration `2026.w09`

* !3944 Fail early if configuration module can not initialize properly
* !3945 gNB: Reset NDI only after usage in case of disable HARQ
* !3822 NR UE: fix PRACH k1 calculation at MAC when handover
* !3874 print better message for rsrp: invalid doesn't mean it is not measures, it...
* !3928 NR UE: correctly compute writeTimestamp for SCS > 30 kHz with rf-simulator
* !3949 fix: removed the nssai_sd parameter in the ue.conf file
* !3897 Change Aerial config file in targets to match Nvidia's CI setup
* !3951 Fix handling of IPv4v6 PDU sessions
* !3940 IPV6 support for telnet socket binding
* !3947 Add missing packing for CONFIG.request parameter BetaPSS
* !3871 Support for up to 12 ports in NR gNB
* !3943 Remove L1 dependence from NR band number
* !3892 Synchronize multiple gNBs by aliging their the slot boundaries
* !3952 Fixes for Initial UE Context Setup
* !3953 Improvements in compilation dependence
* change dl_absoluteFrequencyPointA in targets/PROJECTS/GENERIC-NR-5GC/CONF/gnb.sa.band78.fr1.24PRB.usrpb210.conf to 640032
* !3900 [CI] Pull Ubuntu base image from internal registry
* !3910 GPU Channel Simulation: nr_dlsim/nr_ulsim Integration + CI Test
* !3942 CI: rework log analysis to simplify and allow new use cases
* !3870 Configure CSI-RS according to beam_id used by UE

Closes #897 and #834

See merge request oai/openairinterface5g!3950

CMakeLists.txt

@@ -434,7 +434,9 @@ add_library(ngap
   ${NGAP_DIR}/ngap_gNB_handlers.c
   ${NGAP_DIR}/ngap_gNB_itti_messaging.c
   ${NGAP_DIR}/ngap_gNB_management_procedures.c
+  ${NGAP_DIR}/ngap_gNB_paging.c
   ${NGAP_DIR}/ngap_gNB_nas_procedures.c
+  ${NGAP_DIR}/ngap_gNB_pdu_session_management.c
   ${NGAP_DIR}/ngap_gNB_nnsf.c
   ${NGAP_DIR}/ngap_gNB_overload.c
   ${NGAP_DIR}/ngap_gNB_trace.c
@@ -442,6 +444,9 @@ add_library(ngap
   )
 target_link_libraries(ngap PUBLIC asn1_ngap)
 target_link_libraries(ngap PRIVATE asn1_lte_rrc_hdrs asn1_nr_rrc_hdrs)
+if(ENABLE_TESTS)
+  target_compile_definitions(ngap PRIVATE ENABLE_TESTS)
+endif()
 
 #M2AP
 ##############
@@ -544,8 +549,10 @@ endif()
 
 # Hardware dependant options
 ###################################
-add_list1_option(NB_ANTENNAS_RX "4" "Number of antennas in reception" "1" "2" "4")
-add_list1_option(NB_ANTENNAS_TX "4" "Number of antennas in transmission" "1" "2" "4")
+set(NB_ANTENNAS_RX "4" CACHE STRING "Number of antennas in reception")
+set_property(CACHE NB_ANTENNAS_RX PROPERTY STRINGS "1" "2" "4" "8")
+set(NB_ANTENNAS_TX "4" CACHE STRING "Number of antennas in transmission")
+set_property(CACHE NB_ANTENNAS_TX PROPERTY STRINGS "1" "2" "4" "8")
 
 set (CONFIG_ROOTDIR ${OPENAIR_DIR}/common/config)
 add_library(config_internals OBJECT
@@ -648,7 +655,6 @@ include_directories("${OPENAIR3_DIR}/S1AP")
 include_directories("${OPENAIR2_DIR}/X2AP")
 include_directories("${OPENAIR2_DIR}/M2AP")
 include_directories("${OPENAIR2_DIR}/F1AP")
-include_directories("${OPENAIR3_DIR}/ocp-gtpu")
 include_directories("${OPENAIR3_DIR}/M3AP")
 include_directories("${OPENAIR3_DIR}/MME_APP")
 include_directories("${OPENAIR_DIR}/radio/COMMON")
@@ -926,7 +932,6 @@ set(PHY_SRC_UE
   ${OPENAIR1_DIR}/PHY/LTE_TRANSPORT/sss_gen.c
   ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/dlsch_demodulation.c
   ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/dlsch_llr_computation.c
-  ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/linear_preprocessing_rec.c
   ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/dlsch_decoding.c
   ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/dci_tools_ue.c
   ${OPENAIR1_DIR}/PHY/LTE_UE_TRANSPORT/uci_tools_ue.c
@@ -1072,16 +1077,8 @@ add_dependencies(PHY_COMMON dfts)
 add_library(PHY ${PHY_SRC})
 target_link_libraries(PHY PRIVATE asn1_lte_rrc_hdrs asn1_nr_rrc_hdrs)
 
-pkg_check_modules(blas REQUIRED blas)
-pkg_check_modules(lapacke REQUIRED lapacke)
-
 add_library(PHY_UE ${PHY_SRC_UE})
 target_link_libraries(PHY_UE PRIVATE asn1_lte_rrc_hdrs asn1_nr_rrc_hdrs UTIL)
-# RHEL needs also cblas, but Ubuntu does not have it. So `cblas_LIBRARIES` and
-# `cblas_INCLUDE_DIRS` will be empty for Ubuntu, a no-op
-pkg_check_modules(cblas cblas)
-target_link_libraries(PHY_UE PRIVATE ${blas_LIBRARIES} ${cblas_LIBRARIES} ${lapacke_LIBRARIES})
-target_include_directories(PHY_UE PRIVATE ${blas_INCLUDE_DIRS} ${cblas_INCLUDE_DIRS} ${lapacke_INCLUDE_DIRS})
 
 add_library(PHY_NR_COMMON ${PHY_NR_SRC_COMMON})
 target_link_libraries(PHY_NR_COMMON PUBLIC UTIL)
@@ -1158,7 +1155,7 @@ set(L2_SRC
   ${RRC_DIR}/rrc_eNB_S1AP.c
   ${RRC_DIR}/rrc_eNB_M2AP.c
   ${RRC_DIR}/rrc_eNB_UE_context.c
-  ${NR_RRC_DIR}/rrc_gNB_UE_context.c
+  ${RRC_DIR}/rrc_eNB_GTPV1U.c
   ${RRC_DIR}/rrc_common.c
   ${RRC_DIR}/L2_interface.c
   ${RRC_DIR}/L2_interface_common.c
@@ -1405,15 +1402,6 @@ include_directories("${OPENAIR1_DIR}/SCHED_NR_UE")
 # CN libs
 ##########################
 
-add_library (GTPV1U
-  ${RRC_DIR}/rrc_eNB_GTPV1U.c
-  ${OPENAIR3_DIR}/ocp-gtpu/gtp_itf.cpp
-  ${OPENAIR3_DIR}/ocp-gtpu/gtpu_extensions.c
-  )
-target_link_libraries(GTPV1U PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
-target_link_libraries(GTPV1U PRIVATE SIMU ds)
-include_directories(${OPENAIR3_DIR}/ocp-gtp)
-
 set (MME_APP_SRC
   ${OPENAIR3_DIR}/MME_APP/mme_app.c
   ${OPENAIR3_DIR}/MME_APP/mme_config.c
@@ -1773,7 +1761,6 @@ target_link_libraries(lte-softmodem PRIVATE
 target_link_libraries(lte-softmodem PRIVATE pthread m CONFIG_LIB rt)
 target_link_libraries(lte-softmodem PRIVATE ${T_LIB})
 target_link_libraries(lte-softmodem PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
-target_link_libraries(lte-softmodem PRIVATE ${blas_LIBRARIES} ${cblas_LIBRARIES} ${lapacke_LIBRARIES} ${lapack_LIBRARIES})
 
 if(E2_AGENT)
   target_link_libraries(lte-softmodem PUBLIC e2_agent e2_agent_arg e2_ran_func_du_cucp_cuup)
@@ -1824,7 +1811,6 @@ target_link_libraries(lte-uesoftmodem PRIVATE
 target_link_libraries(lte-uesoftmodem PRIVATE pthread m CONFIG_LIB rt)
 target_link_libraries(lte-uesoftmodem PRIVATE ${T_LIB})
 target_link_libraries(lte-uesoftmodem PRIVATE asn1_nr_rrc_hdrs asn1_lte_rrc_hdrs)
-target_link_libraries(lte-uesoftmodem PRIVATE ${blas_LIBRARIES} ${cblas_LIBRARIES} ${lapacke_LIBRARIES} ${lapack_LIBRARIES})
 
 # force the generation of ASN.1 so that we don't need to wait during the build
 target_link_libraries(lte-uesoftmodem PRIVATE

ci-scripts/args_parse.py

@@ -33,7 +33,6 @@
 #-----------------------------------------------------------
 import sys		# arg
 import re		# reg
-import yaml
 import constants as CONST
 
 #-----------------------------------------------------------
@@ -44,6 +43,7 @@ import constants as CONST
 def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER):
 
     force_local = False
+    date_fmt = None
     while len(argvs) > 1:
         myArgv = argvs.pop(1)	# 0th is this file's name
 
@@ -56,6 +56,9 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER):
 
 
 	    #consider inline parameters
+        elif re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE):
+            matchReg = re.match(r'^\-\-datefmt=(.+)$', myArgv, re.IGNORECASE)
+            date_fmt = matchReg.group(1)
         elif re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE):
             matchReg = re.match(r'^\-\-mode=(.+)$', myArgv, re.IGNORECASE)
             mode = matchReg.group(1)
@@ -158,9 +161,6 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER):
         elif re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE):
             matchReg = re.match(r'^\-\-OCRegistry=(.+)$', myArgv, re.IGNORECASE)
             CLUSTER.OCRegistry = matchReg.group(1)
-        elif re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE):
-            matchReg = re.match(r'^\-\-BuildId=(.+)$', myArgv, re.IGNORECASE)
-            RAN.BuildId = matchReg.group(1)
         elif re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE):
             matchReg = re.match(r'^\-\-FlexRicTag=(.+)$', myArgv, re.IGNORECASE)
             CONTAINERS.flexricTag = matchReg.group(1)
@@ -168,4 +168,4 @@ def ArgsParse(argvs,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER):
             HELP.GenericHelp(CONST.Version)
             sys.exit('Invalid Parameter: ' + myArgv)
 
-    return mode, force_local
+    return mode, force_local, date_fmt

ci-scripts/cls_analysis.py

@@ -22,10 +22,15 @@
 
 import re
 import os
+import logging
+import yaml
+import signal
 
 import xml.etree.ElementTree as ET
 import json
 
+import importlib, inspect
+
 class Analysis():
 
 	def _get_test_description(properties):
@@ -109,3 +114,134 @@ class Analysis():
 		test_summary['Nbpass'] =  nb_tests - nb_failed
 		test_summary['Nbfail'] =  nb_failed
 		return nb_failed == 0, test_summary, test_result
+
+	def analyze_rt_stats(thresholds, L1_stats, MAC_stats):
+		with open(thresholds, 'r') as f:
+			datalog_rt_stats = yaml.load(f, Loader=yaml.FullLoader)
+
+		rt_keys = datalog_rt_stats['Ref']
+		real_time_stats = {}
+		for sf in [L1_stats, MAC_stats]:
+			with open(sf, 'r') as f:
+				for line in f.readlines():
+					for k in rt_keys:
+						result = re.search(k, line)
+						if result is not None:
+							tmp = re.match(rf'^.*?(\b{k}\b.*)', line.rstrip())
+							if tmp is not None:
+								real_time_stats[k] = tmp.group(1)
+
+		# datalog_rt_stats format must align with HTML.CreateHtmlDataLogTable()
+		datalog_rt_stats['Data']={}
+		for k in real_time_stats:
+			tmp = re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+(?P<count>\d+);\s+(?P<max>\d+\.\d+) us;', real_time_stats[k])
+			if tmp is not None:
+				metric = tmp.group('metric')
+				avg = float(tmp.group('avg'))
+				max = float(tmp.group('max'))
+				count = int(tmp.group('count'))
+				datalog_rt_stats['Data'][metric] = ["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg / datalog_rt_stats['Ref'][metric])]
+
+		success = True
+		for k in datalog_rt_stats['Data']:
+			valnorm = float(datalog_rt_stats['Data'][k][3])
+			dev = datalog_rt_stats['DeviationThreshold'][k]
+			if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail: avg/ref deviates by more than "deviation threshold"
+				logging.debug(f'\u001B[1;30;43m normalized metric {k}={valnorm} deviates by more than {dev}\u001B[0m')
+				success = False
+		return success, datalog_rt_stats
+
+
+# returns tuple of (service, analyzer, option string)
+def _lookupServiceAnalyzerOpt(s, analyzers):
+	res = s.split("=", 2)
+	name = res[0]
+	if len(res) == 1:
+		return res[0], analyzers["Default"], None
+	opt = res[2] if len(res) > 2 else None
+	func = res[1]
+	a = analyzers[func] if func in analyzers else None
+	return name, a, opt
+
+# groups requested service analysis (service=func[=option]) on per service
+# basis and looks up analyzer func. If log analysis is requested, will always
+# run Default log analysis
+def _groupServices(to_analyze):
+	req_analysis = {}
+	# get content of cls_loganalysis module, then get all analyzers (classes) in this module
+	mod = importlib.import_module("cls_loganalysis")
+	analyzers = {name:cl for name, cl in inspect.getmembers(mod, inspect.isclass)}
+	for req in to_analyze:
+		s, func, opt = _lookupServiceAnalyzerOpt(req, analyzers)
+		logging.debug(f"requested check '{req}' => service {s}, function {func.__name__}, options '{opt}'")
+		# always put default analyzer first
+		l = req_analysis[s] if s in req_analysis else [(analyzers["Default"], None, "default")]
+		if func is not analyzers["Default"]:
+			l.append((func, opt, req))
+		req_analysis[s] = l
+	return req_analysis
+
+def _describe_exit_code(code):
+	if code > 128:
+		sig = code - 128
+		try:
+			return f"terminated by signal {sig} ({signal.Signals(sig).name})"
+		except ValueError:
+			return f"terminated by unknown signal {sig}"
+	else:
+		return f"custom exit code"
+
+def AnalyzeServices(HTML, service_desc, to_analyze):
+	success = True
+	# hack: we want to give as a description "log analysis", but the description
+	# is set outside, so retain what was set before
+	orig_html_desc = HTML.desc
+	# group analysis on a per-service basis, then iterate
+	for serv, list_analysis in _groupServices(to_analyze).items():
+		HTML.desc = f"Log analysis for service {serv}"
+		if serv not in service_desc:
+			success = False
+			logging.error(f"requested service {serv} not in list of services")
+			HTML.CreateHtmlTestRowQueue("N/A", 'KO', ["service not detected"])
+			continue
+		# pre-initialize with return code
+		rc = service_desc[serv]["returncode"]
+		logging.info(f"analyze service {serv}: return code {rc}")
+		service_success = True # TODO rc == 0: too many functions (eNB, lteUE, RIC, nrUE) fail non-zero
+		logs = []
+		if rc != 0:
+			logs.append(f"=> return code {rc}, likely " + _describe_exit_code(rc) + " [ignored by CI]")
+		# skip if the file is too big
+		logfile = service_desc[serv]["logfile"]
+		filename = os.path.basename(logfile)
+		b = os.path.getsize(logfile)
+		logging.debug(f"using logfile {logfile} of size {b} bytes")
+		if b > 10 * 1024 * 1024:
+			success = False
+			logs.append(f"logfile too big (>10MB)")
+			logging.error(logs)
+			HTML.CreateHtmlTestRowQueue(filename, 'KO', ["\n".join(logs)])
+			continue
+		# run each analyzer with its options on the logfile
+		for (func, opt, desc) in list_analysis:
+			if func is None:
+				service_success = False
+				logging.error(f"request analysis function for desc {desc} not found")
+				HTML.CreateHtmlTestRowQueue(serv, 'KO', [f"no analysis function for {desc}"])
+				continue
+			result, l = func.run(logfile, opt)
+			logging.info(f"service {serv}: analysis with func {func}, result {result}, logs '{l}'")
+			service_success = service_success and result
+			if not result:
+				logs.append(f"=> {func.__name__} check (options '{opt}') FAILED")
+				logs.append(l)
+		logs = '\n'.join(logs)
+		if not service_success:
+			logging.error(l)
+		else:
+			logging.info(l)
+		all_funcs = ", ".join([f.__name__ for (f, _, _) in list_analysis])
+		HTML.CreateHtmlTestRowQueue(f"Check {all_funcs} on {filename}", 'OK' if service_success else 'KO', [logs])
+		success = success and service_success
+	HTML.desc = orig_html_desc
+	return success

ci-scripts/cls_containerize.py

@@ -31,23 +31,17 @@
 #-----------------------------------------------------------
 # Import
 #-----------------------------------------------------------
-import sys	      # arg
 import re	       # reg
 import logging
 import os
-import shutil
-import time
-from zipfile import ZipFile
 
 #-----------------------------------------------------------
 # OAI Testing modules
 #-----------------------------------------------------------
 import cls_cmd
-import helpreadme as HELP
 import constants as CONST
-import cls_oaicitest
+import cls_analysis
 from cls_ci_helper import archiveArtifact
-from collections import deque
 
 #-----------------------------------------------------------
 # Helper functions used here and in other classes
@@ -186,62 +180,9 @@ def GetDeployedServices(ssh, file):
 		else:
 			c = ret.stdout
 			logging.info(f'service {s} with container id {c}')
-			deployed_services.append(s)
+			deployed_services.append((s, c))
 	return deployed_services
 
-def CheckLogs(self, filename, HTML, RAN):
-	success = True
-	name = os.path.basename(filename)
-	if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
-		logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
-		logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN)
-		opt = f"UE log analysis ({name})"
-		# usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files
-		# abuse HTML to store their reports, and we here want to put custom options,
-		# which is not possible with CreateHtmlTestRow
-		# solution: use HTML templates, where we don't need different HTML write funcs
-		if (logStatus < 0):
-			HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmlUEFailureMsg])
-			success = False
-		else:
-			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg])
-		HTML.htmlUEFailureMsg = ""
-	elif 'nv-cubb' in name:
-		msg = 'Undeploy PNF/Nvidia CUBB'
-		HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK)
-	elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb','vnf'])):
-		logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m')
-		logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
-		opt = f"xNB log analysis ({name})"
-		if (logStatus < 0):
-			HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg])
-			success = False
-		else:
-			HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
-		HTML.htmleNBFailureMsg = ""
-	elif 'xapp' in name:
-		opt = f"Undeploy {name}"
-		with open(f'{filename}', "r") as f:
-			last_line = deque(f, maxlen=1).pop()
-		if ('Test xApp run SUCCESSFULLY' in last_line):
-			HTML.CreateHtmlTestRowQueue(opt, 'OK', ["xApp run successfully"])
-		else:
-			HTML.CreateHtmlTestRowQueue(opt, 'KO', ["xApp didn't run successfully"])
-			success = False
-	elif 'RIC' in name:
-		opt = f"Undeploy {name}"
-		with open(f'{filename}', 'r') as f:
-			last_line = deque(f, maxlen=1).pop()
-		if ('Removing E2 Node' in last_line):
-			HTML.CreateHtmlTestRowQueue(opt, 'OK', ["nearRT-RIC run successfully"])
-		else:
-			HTML.CreateHtmlTestRowQueue(opt, 'KO', ["nearRT-RIC didn't run successfully"])
-			success = False
-	else:
-		logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE")
-	logging.debug(f"log check: file {filename} passed analysis {success}")
-	return success
-
 #-----------------------------------------------------------
 # Class Declaration
 #-----------------------------------------------------------
@@ -266,8 +207,6 @@ class Containerize():
 		self.dockerfileprefix = ''
 		self.host = ''
 
-		#checkers from xml
-		self.ran_checkers={}
 		self.num_attempts = 1
 
 		self.flexricTag = ''
@@ -372,12 +311,16 @@ class Containerize():
 		for image,pattern,name,option in imageNames:
 			cmd.run(f"{self.cli} image rm {name}:{imageTag}", reportNonZero=False)
 
+		cmd.run(f'docker login -u oaicicd -p oaicicd {DEFAULT_REGISTRY}')
+		ubuntuImage = "ubuntu:noble"
 		# Build the base image only on Push Events (not on Merge Requests)
 		# On when the base image docker file is being modified.
 		if forceBaseImageBuild:
 			cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
 			logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log'
-			cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600)
+			if self.host == 'Ubuntu':
+				option = f" --build-arg UBUNTU_IMAGE={DEFAULT_REGISTRY}/{ubuntuImage}"
+			cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} {option} . &> {logfile}", timeout=1600)
 			t = ("ran-base", archiveArtifact(cmd, ctx, logfile))
 			log_files.append(t)
 
@@ -420,6 +363,8 @@ class Containerize():
 			if image == 'oai-gnb-aerial':
 				cmd.run('cp -f /opt/nvidia-ipc/nvipc_src.2026.01.07.tar.gz .')
 			logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log'
+			if self.host == 'Ubuntu':
+				option = option + f" --build-arg UBUNTU_IMAGE={DEFAULT_REGISTRY}/{ubuntuImage}"
 			ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200)
 			t = (name, archiveArtifact(cmd, ctx, logfile))
 			log_files.append(t)
@@ -446,7 +391,7 @@ class Containerize():
 					allImagesSize[name] = 'unknown'
 			# Now pruning dangling images in between target builds
 			cmd.run(f"{self.cli} image prune --force")
-
+		cmd.run(f'docker logout {DEFAULT_REGISTRY}')
 		# Remove all intermediate build images and clean up
 		cmd.run(f"{self.cli} image rm ran-build:{imageTag} ran-build-asan:{imageTag} ran-build-fhi72:{imageTag} || true")
 		cmd.run(f"{self.cli} volume prune --force")
@@ -790,7 +735,7 @@ class Containerize():
 		wd_yaml = f'{wd}/docker-compose.y*ml'
 		with cls_cmd.getConnection(node) as ssh:
 			ExistEnvFilePrint(ssh, wd)
-			services = GetDeployedServices(ssh, wd_yaml)
+			services = [s for s, _ in GetDeployedServices(ssh, wd_yaml)]
 			success = []
 			fail = []
 			for s in reqServices:
@@ -808,7 +753,7 @@ class Containerize():
 			HTML.CreateHtmlTestRowQueue(self.services, 'KO', [f'Failed stopping {" ".join(fail)}, succeeded {" ".join(success)}'])
 		return success
 
-	def UndeployObject(self, ctx, node, HTML, RAN):
+	def UndeployObject(self, ctx, node, HTML, to_analyze):
 		lSourcePath = self.eNBSourceCodePath
 		logging.info(f'\u001B[1m Undeploying all objects from server {node}\u001B[0m')
 		yaml = self.yamlPath.strip('/')
@@ -817,23 +762,49 @@ class Containerize():
 		with cls_cmd.getConnection(node) as ssh:
 			ExistEnvFilePrint(ssh, wd)
 			services = GetDeployedServices(ssh, wd_yaml)
-			copyin_res = None
+			all_logs = True
 			ssh.run(f'docker compose -f {wd_yaml} stop')
 			if services is not None:
-				copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx) for s in services]
+				service_desc = {}
+				for s, c in services:
+					ret = ssh.run(f'docker inspect {c} --format="{{{{.State.ExitCode}}}}"')
+					rc = int(ret.stdout.strip())
+					f = CopyinServiceLog(ssh, lSourcePath, s, wd_yaml, ctx)
+					all_logs = all_logs and f is not None
+					service_desc[s] = {'returncode': rc, 'logfile': f}
 			else:
 				logging.warning('could not identify services to stop => no log file')
 			ssh.run(f'docker compose -f {wd_yaml} down -v')
+			HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful'])
 			ssh.run(f'rm {wd}/.env')
-		if not copyin_res:
+		if not all_logs:
 			HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
-			logging.error(f"could not copy all files: {copyin_res=} {services=}")
+			logging.error(f"could not copy all files: {all_logs=} {services=}")
 			success = False
 		else:
-			log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res]
-			success = all(log_results)
+			success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze)
 		if success:
 			logging.info('\u001B[1m Undeploying objects Pass\u001B[0m')
 		else:
 			logging.error('\u001B[1m Undeploying objects Failed\u001B[0m')
 		return success
+
+	def AnalyzeRTStatsObject(self, HTML, node, ctx, thresholds):
+		logging.info(f'Analyzing realtime stats from server: {node}')
+		yaml = self.yamlPath.strip('/')
+		wd = f'{self.eNBSourceCodePath}/{yaml}'
+		wd_yaml = f'{wd}/docker-compose.y*ml'
+
+		with cls_cmd.getConnection(node) as cmd:
+			services = GetDeployedServices(cmd, wd_yaml)
+			s, _ = services[0] # we simply assume something is deployed, if not Exception will report
+			# similar to BuildRunTests(), use docker cp to avoid problems with permissions
+			cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrL1_stats.log {wd}/')
+			l1_file = archiveArtifact(cmd, ctx, f"{wd}/nrL1_stats.log")
+			cmd.run(f'docker compose -f {wd_yaml} cp {s}:/opt/oai-gnb/nrMAC_stats.log {wd}/')
+			mac_file = archiveArtifact(cmd, ctx, f"{wd}/nrMAC_stats.log")
+
+		logging.info(f"check against thresholds from {thresholds}")
+		success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file)
+		HTML.CreateHtmlDataLogTable(datalog_rt_stats)
+		return success

ci-scripts/cls_loganalysis.py

@@ -0,0 +1,161 @@
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *	  http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *	  contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+
+import re
+import os
+import logging
+from collections import deque
+
+class Default:
+	def run(file, opt=None):
+		success = True
+		logs = []
+		with open(file, "r") as f:
+			for line in f.readlines():
+				result = re.search('[Aa]ssertion', line)
+				if result:
+					logs.append(line)
+					success = False
+		return success, "\n".join(logs)
+
+class ContainsString:
+	def run(file, needle):
+		with open(file, "r") as f:
+			for line in f.readlines():
+				if re.search(needle, line):
+					return True, ""
+		return False, f"could not find '{needle}' in logs"
+
+class LastLineContains:
+	def run(file, string):
+		with open(file, "r") as f:
+			last_line = deque(f, maxlen=1).pop()
+		success = string in last_line
+		log = "" if success else f"could not find '{string}' in last line"
+		return success, log
+
+class EndsWithBye:
+	def run(file, option):
+		# no option: we ignore any
+		with open(file, "r") as f:
+			n = 0
+			for line in reversed(f.readlines()):
+				if re.search(r'^Bye.\n', line, re.MULTILINE) is not None:
+					return True, ""
+				n = n + 1
+				if n > 50: # if not found in last 50 lines, assume it does not exist
+					break
+		return False, "No Bye. message found, did not stop properly"
+
+class RetxCheck():
+	def _parseList(s):
+		if not s:
+			return []
+		return [int(n) for n in s.split(",")]
+
+	def _get_ul_dl(s):
+		dl, ul = (None, None)
+		for o in s.split(";", 1):
+			if o.startswith("dl="):
+				dl = o[3:]
+			elif o.startswith("ul="):
+				ul = o[3:]
+			else:
+				raise ValueError(f"unrecognized option {s}: needs format '[ul=,dl=]X,Y,...[;ul=Z,V,...]'")
+		return dl, ul
+
+	def _parseRetxCheckers(options):
+		if not "=" in options:
+			dlul = RetxCheck._parseList(options)
+			return dlul, dlul
+		dl, ul = RetxCheck._get_ul_dl(options)
+		return RetxCheck._parseList(dl), RetxCheck._parseList(ul)
+
+	def _analyzeUeRetx(rounds, checkers, regex):
+		perc = list(0 for i in checkers) # results in %
+		stats = list(False for i in checkers) # status if succeeded
+		tmp = re.match(regex, rounds)
+		if tmp is None:
+			logging.error('_analyzeUeRetx: did not match regex for DL retx analysis')
+			return stats
+		retx_data = [float(x) for x in tmp.groups()]
+		for i in range(0, len(perc)):
+			#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
+			perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i]
+			#treating % > 100 , % > requirement
+			stats[i] = perc[i] <= 100 and perc[i] <= checkers[i]
+		return stats
+
+	def _filter_retx_rounds(l):
+		# map to round number if False, else None
+		l = [i+1 if x is False else None for i, x in enumerate(l)]
+		# filter out None
+		return [x for x in l if x is not None]
+
+	def run(file, options):
+		# aggregate statistics per UE
+		dlsch_ulsch_stats = {}
+		status = True
+		with open(file, "r") as f:
+			for line in f.readlines():
+				keys = {'dlsch_rounds','ulsch_rounds'}
+				for k in keys:
+					result = re.search(k, line)
+					if result is None:
+						continue
+					result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line)
+					if result is None:
+						logging.error(f'did not find RNTI while matching key {k}')
+						status = False
+						continue
+					rnti = result.group(1)
+					if not rnti in dlsch_ulsch_stats:
+						dlsch_ulsch_stats[rnti] = {}
+					dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
+
+		dl, ul = RetxCheck._parseRetxCheckers(options)
+		msgs = []
+		for ue in dlsch_ulsch_stats:
+			dlulstat = dlsch_ulsch_stats[ue]
+
+			for key in dlulstat:
+				msgs.append(dlulstat[key])
+
+			retx_dl = RetxCheck._analyzeUeRetx(dlulstat['dlsch_rounds'], dl, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)')
+			retx_ul = RetxCheck._analyzeUeRetx(dlulstat['ulsch_rounds'], ul, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)')
+
+			if False in retx_dl or False in retx_ul:
+				dlstr = RetxCheck._filter_retx_rounds(retx_dl)
+				ulstr = RetxCheck._filter_retx_rounds(retx_ul)
+				msg = f"UE {ue}: retx rounds"
+				if dlstr:
+					msg = f"{msg} DL {dlstr}"
+				if ulstr:
+					msg = f"{msg} UL {ulstr}"
+				msg = f"{msg} exceeded threshold"
+				msgs.append('!!! Failure: ' + msg)
+				logging.error(f'\u001B[1;37;41m {msg}\u001B[0m')
+				status = False
+			else:
+				msg = f"UE {ue} retransmissions within bounds"
+				logging.debug(msg)
+		return status, "\n".join(msgs)

ci-scripts/cls_oai_html.py

@@ -218,7 +218,7 @@ class HTMLManagement():
 				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__//' test_results.html"
 				subprocess.run(cmd, shell=True)
 			else:
-				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + "__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
+				cmd = "sed -i -e 's/__STATE_" + self.htmlTabNames[0] + r"__/<span class=\"glyphicon glyphicon-remove\"><\/span>/' test_results.html"
 				subprocess.run(cmd, shell=True)
 		self.htmlFooterCreated = False
 

ci-scripts/cls_oaicitest.py

@@ -32,7 +32,6 @@
 #-----------------------------------------------------------
 # Import Libs
 #-----------------------------------------------------------
-import sys		# arg
 import re		# reg
 import time		# sleep
 import os
@@ -41,7 +40,6 @@ import concurrent.futures
 import json
 
 #import our libs
-import helpreadme as HELP
 import constants as CONST
 
 import cls_module
@@ -301,7 +299,6 @@ class OaiCiTest():
 		self.iperf_profile = ''
 		self.iperf_tcp_rate_target = ''
 		self.finalStatus = False
-		self.air_interface=''
 		self.ue_ids = []
 		self.svr_node = None
 		self.svr_id = None
@@ -579,277 +576,6 @@ class OaiCiTest():
 			HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', [f'{ue_header}\n{msg}'])
 		return success
 
-	def AnalyzeLogFile_UE(self, UElogFile,HTML,RAN):
-		if (not os.path.isfile(f'{UElogFile}')):
-			return -1
-		ue_log_file = open(f'{UElogFile}', 'r')
-		exitSignalReceived = False
-		foundAssertion = False
-		msgAssertion = ''
-		msgLine = 0
-		foundSegFault = False
-		foundRealTimeIssue = False
-		uciStatMsgCount = 0
-		pdcpDataReqFailedCount = 0
-		badDciCount = 0
-		f1aRetransmissionCount = 0
-		fatalErrorCount = 0
-		macBsrTimerExpiredCount = 0
-		rrcConnectionRecfgComplete = 0
-		no_cell_sync_found = False
-		mib_found = False
-		frequency_found = False
-		plmn_found = False
-		nrUEFlag = False
-		nrDecodeMib = 0
-		nrFoundDCI = 0
-		nrCRCOK = 0
-		mbms_messages = 0
-		nbPduSessAccept = 0
-		nbPduDiscard = 0
-		HTML.htmlUEFailureMsg=''
-		global_status = CONST.ALL_PROCESSES_OK
-		for line in ue_log_file.readlines():
-			result = re.search('nr_synchro_time|Starting NR UE soft modem', str(line))
-			sidelink = re.search('sl-mode', str(line))
-			if result is not None:
-				nrUEFlag = True
-			if sidelink is not None:
-				nrUEFlag = False
-			if nrUEFlag:
-				result = re.search('decode mib', str(line))
-				if result is not None:
-					nrDecodeMib += 1
-				result = re.search('found 1 DCIs', str(line))
-				if result is not None:
-					nrFoundDCI += 1
-				result = re.search('CRC OK', str(line))
-				if result is not None:
-					nrCRCOK += 1
-				result = re.search('Received PDU Session Establishment Accept', str(line))
-				if result is not None:
-					nbPduSessAccept += 1
-				result = re.search('warning: discard PDU, sn out of window', str(line))
-				if result is not None:
-					nbPduDiscard += 1
-				result = re.search('--nfapi STANDALONE_PNF --node-number 2', str(line))
-				if result is not None:
-					frequency_found = True
-			result = re.search('Exiting OAI softmodem', str(line))
-			if result is not None:
-				exitSignalReceived = True
-			result = re.search('System error|[Ss]egmentation [Ff]ault|======= Backtrace: =========|======= Memory map: ========', str(line))
-			if result is not None and not exitSignalReceived:
-				foundSegFault = True
-			result = re.search('[Cc]ore [dD]ump', str(line))
-			if result is not None and not exitSignalReceived:
-				foundSegFault = True
-			result = re.search('[Aa]ssertion', str(line))
-			if result is not None and not exitSignalReceived:
-				foundAssertion = True
-			result = re.search('LLL', str(line))
-			if result is not None and not exitSignalReceived:
-				foundRealTimeIssue = True
-			if foundAssertion and (msgLine < 3):
-				msgLine += 1
-				msgAssertion += str(line)
-			result = re.search('uci->stat', str(line))
-			if result is not None and not exitSignalReceived:
-				uciStatMsgCount += 1
-			result = re.search('PDCP data request failed', str(line))
-			if result is not None and not exitSignalReceived:
-				pdcpDataReqFailedCount += 1
-			result = re.search('bad DCI 1', str(line))
-			if result is not None and not exitSignalReceived:
-				badDciCount += 1
-			result = re.search('Format1A Retransmission but TBS are different', str(line))
-			if result is not None and not exitSignalReceived:
-				f1aRetransmissionCount += 1
-			result = re.search('FATAL ERROR', str(line))
-			if result is not None and not exitSignalReceived:
-				fatalErrorCount += 1
-			result = re.search('MAC BSR Triggered ReTxBSR Timer expiry', str(line))
-			if result is not None and not exitSignalReceived:
-				macBsrTimerExpiredCount += 1
-			result = re.search('Generating RRCConnectionReconfigurationComplete', str(line))
-			if result is not None:
-				rrcConnectionRecfgComplete += 1
-			# No cell synchronization found, abandoning
-			result = re.search('No cell synchronization found, abandoning', str(line))
-			if result is not None:
-				no_cell_sync_found = True
-			if RAN.eNBmbmsEnables[0]:
-				result = re.search('TRIED TO PUSH MBMS DATA', str(line))
-				if result is not None:
-					mbms_messages += 1
-			result = re.search(r"MIB Information => ([a-zA-Z]{1,10}), ([a-zA-Z]{1,10}), NidCell (?P<nidcell>\d{1,3}), N_RB_DL (?P<n_rb_dl>\d{1,3}), PHICH DURATION (?P<phich_duration>\d), PHICH RESOURCE (?P<phich_resource>.{1,4}), TX_ANT (?P<tx_ant>\d)", str(line))
-			if result is not None and (not mib_found):
-				try:
-					mibMsg = "MIB Information: " + result.group(1) + ', ' + result.group(2)
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mibMsg = "    nidcell = " + result.group('nidcell')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mibMsg = "    n_rb_dl = " + result.group('n_rb_dl')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mibMsg = "    phich_duration = " + result.group('phich_duration')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mibMsg = "    phich_resource = " + result.group('phich_resource')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mibMsg = "    tx_ant = " + result.group('tx_ant')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					mib_found = True
-				except Exception as e:
-					logging.error(f'\033[91m MIB marker was not found \033[0m')
-			result = re.search("Initial sync: pbch decoded sucessfully", str(line))
-			if result is not None and (not frequency_found):
-				try:
-					mibMsg = f"UE decoded PBCH successfully"
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					frequency_found = True
-				except Exception as e:
-					logging.error(f'\033[91m UE did not find PBCH\033[0m')
-			result = re.search(r"PLMN MCC (?P<mcc>\d{1,3}), MNC (?P<mnc>\d{1,3}), TAC", str(line))
-			if result is not None and (not plmn_found):
-				try:
-					mibMsg = f"PLMN MCC = {result.group('mcc')} MNC = {result.group('mnc')}"
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-					plmn_found = True
-				except Exception as e:
-					logging.error(f'\033[91m PLMN not found \033[0m')
-			result = re.search(r"Found (?P<operator>[\w,\s]{1,15}) \(name from internal table\)", str(line))
-			if result is not None:
-				try:
-					mibMsg = f"The operator is: {result.group('operator')}"
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + '\n'
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-				except Exception as e:
-					logging.error(f'\033[91m Operator name not found \033[0m')
-			result = re.search("SIB5 InterFreqCarrierFreq element (.{1,4})/(.{1,4})", str(line))
-			if result is not None:
-				try:
-					mibMsg = f'SIB5 InterFreqCarrierFreq element {result.group(1)}/{result.group(2)}'
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + mibMsg + ' -> '
-					logging.debug(f'\033[94m{mibMsg}\033[0m')
-				except Exception as e:
-					logging.error(f'\033[91m SIB5 InterFreqCarrierFreq element not found \033[0m')
-			result = re.search(r"DL Carrier Frequency/ARFCN : \-*(?P<carrier_frequency>\d{1,15}/\d{1,4})", str(line))
-			if result is not None:
-				try:
-					freq = result.group('carrier_frequency')
-					new_freq = re.sub('/[0-9]+','',freq)
-					float_freq = float(new_freq) / 1000000
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'DL Freq: ' + ('%.1f' % float_freq) + ' MHz'
-					logging.debug(f'\033[94m    DL Carrier Frequency is:  {freq}\033[0m')
-				except Exception as e:
-					logging.error(f'\033[91m    DL Carrier Frequency not found \033[0m')
-			result = re.search(r"AllowedMeasBandwidth : (?P<allowed_bandwidth>\d{1,7})", str(line))
-			if result is not None:
-				try:
-					prb = result.group('allowed_bandwidth')
-					HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + ' -- PRB: ' + prb + '\n'
-					logging.debug(f'\033[94m    AllowedMeasBandwidth: {prb}\033[0m')
-				except Exception as e:
-					logging.error(f'\033[91m    AllowedMeasBandwidth not found \033[0m')
-		ue_log_file.close()
-		if rrcConnectionRecfgComplete > 0:
-			statMsg = f'UE connected to eNB ({rrcConnectionRecfgComplete}) RRCConnectionReconfigurationComplete message(s) generated)'
-			logging.debug(f'\033[94m{statMsg}\033[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if nrUEFlag:
-			if nrDecodeMib > 0:
-				statMsg = f'UE showed {nrDecodeMib} "MIB decode" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-			if nrFoundDCI > 0:
-				statMsg = f'UE showed {nrFoundDCI} "DCI found" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-			if nrCRCOK > 0:
-				statMsg = f'UE showed {nrCRCOK} "PDSCH decoding" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-			if not frequency_found:
-				statMsg = 'NR-UE could NOT synch!'
-				logging.error(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-			if nbPduSessAccept > 0:
-				statMsg = f'UE showed {nbPduSessAccept} "Received PDU Session Establishment Accept" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-			if nbPduDiscard > 0:
-				statMsg = f'UE showed {nbPduDiscard} "warning: discard PDU, sn out of window" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if uciStatMsgCount > 0:
-			statMsg = f'UE showed {uciStatMsgCount} "uci->stat" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if pdcpDataReqFailedCount > 0:
-			statMsg = f'UE showed {pdcpDataReqFailedCount} "PDCP data request failed" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if badDciCount > 0:
-			statMsg = f'UE showed {badDciCount} "bad DCI 1(A)" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if f1aRetransmissionCount > 0:
-			statMsg = f'UE showed {f1aRetransmissionCount} "Format1A Retransmission but TBS are different" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if fatalErrorCount > 0:
-			statMsg = f'UE showed {fatalErrorCount} "FATAL ERROR:" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if macBsrTimerExpiredCount > 0:
-			statMsg = f'UE showed {fatalErrorCount} "MAC BSR Triggered ReTxBSR Timer expiry" message(s)'
-			logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if RAN.eNBmbmsEnables[0]:
-			if mbms_messages > 0:
-				statMsg = f'UE showed {mbms_messages} "TRIED TO PUSH MBMS DATA" message(s)'
-				logging.debug(f'\u001B[1;30;43m{statMsg}\u001B[0m')
-			else:
-				statMsg = 'UE did NOT SHOW "TRIED TO PUSH MBMS DATA" message(s)'
-				logging.debug(f'\u001B[1;30;41m{statMsg}\u001B[0m')
-				global_status = CONST.OAI_UE_PROCESS_NO_MBMS_MSGS
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + statMsg + '\n'
-		if foundSegFault:
-			logging.debug('\u001B[1;37;41m UE ended with a Segmentation Fault! \u001B[0m')
-			if not nrUEFlag:
-				global_status = CONST.OAI_UE_PROCESS_SEG_FAULT
-			else:
-				if not frequency_found:
-					global_status = CONST.OAI_UE_PROCESS_SEG_FAULT
-		if foundAssertion:
-			logging.debug('\u001B[1;30;43m UE showed an assertion! \u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE showed an assertion!\n'
-			if not nrUEFlag:
-				if not mib_found or not frequency_found:
-					global_status = CONST.OAI_UE_PROCESS_ASSERTION
-			else:
-				if not frequency_found:
-					global_status = CONST.OAI_UE_PROCESS_ASSERTION
-		if foundRealTimeIssue:
-			logging.debug('\u001B[1;37;41m UE faced real time issues! \u001B[0m')
-			HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE faced real time issues!\n'
-		if nrUEFlag:
-			if not frequency_found:
-				global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
-		else:
-			if no_cell_sync_found and not mib_found:
-				logging.debug('\u001B[1;37;41m UE could not synchronize ! \u001B[0m')
-				HTML.htmlUEFailureMsg=HTML.htmlUEFailureMsg + 'UE could not synchronize!\n'
-				global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
-		return global_status
-
 	def TerminateUE(self, ctx, node, HTML):
 		ues = [cls_module.Module_UE(n.strip(), node) for n in self.ue_ids]
 		with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:

ci-scripts/cls_static_code_analysis.py

@@ -89,7 +89,7 @@ class StaticCodeAnalysis():
 		logging.debug('Building on server: ' + node)
 		cmd = cls_cmd.getConnection(node)
 		# on RedHat/CentOS .git extension is mandatory
-		result = re.search('([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
+		result = re.search(r'([a-zA-Z0-9\:\-\.\/])+\.git', self.ranRepository)
 		if result is not None:
 			full_ran_repo_name = self.ranRepository.replace('git/', 'git')
 		else:
@@ -121,7 +121,7 @@ class StaticCodeAnalysis():
 				xmlStart = False
 				with open(filename, 'r') as logfile:
 					for line in logfile:
-						ret = re.search('cppcheck version="(?P<version>[0-9\.]+)"', str(line))
+						ret = re.search(r'cppcheck version="(?P<version>[0-9\.]+)"', str(line))
 						if ret is not None:
 						   CCR.versions[vId] = ret.group('version')
 						if re.search('RUN cat cmake_targets/log/cppcheck.xml', str(line)) is not None:

ci-scripts/conf_files/gnb.sa.band77.273prb.fhi72.4x4-4L-vvdn.conf

@@ -0,0 +1,266 @@
+Active_gNBs = ( "gNB-OAI");
+# Asn1_verbosity, choice in: none, info, annoying
+Asn1_verbosity = "none";
+
+gNBs =
+(
+ {
+    ////////// Identification parameters:
+    gNB_ID    =  0xe00;
+    gNB_name  =  "gNB-OAI";
+
+    // Tracking area code, 0x0000 and 0xfffe are reserved values
+    tracking_area_code  =  1;
+    plmn_list = ({ mcc = 208; mnc = 99; mnc_length = 2; snssaiList = ( { sst = 1; }); });
+
+    nr_cellid = 1;
+
+    ////////// Physical parameters:
+
+    pdsch_AntennaPorts_XP = 2;
+    pdsch_AntennaPorts_N1 = 2;
+    maxMIMO_layers        = 4;
+    pusch_AntennaPorts    = 4;
+    do_CSIRS              = 0;
+    do_SRS                = 0;
+
+    servingCellConfigCommon = (
+    {
+ #spCellConfigCommon
+      physCellId                                                    = 0;
+#  downlinkConfigCommon
+    #frequencyInfoDL
+      # center frequency = 4049.76 MHz
+      # selected SSB frequency = 4049.76 MHz
+      absoluteFrequencySSB                                          = 668640; #669984;
+      dl_frequencyBand                                              = 77;
+      # frequency point A = 4000.62 MHz
+      dl_absoluteFrequencyPointA                                    = 666708;
+      #scs-SpecificCarrierList
+        dl_offstToCarrier                                           = 0;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+        dl_subcarrierSpacing                                        = 1;
+        dl_carrierBandwidth                                         = 273;
+     #initialDownlinkBWP
+      #genericParameters
+       initialDLBWPlocationAndBandwidth                             = 1099; #38.101-1 Table 5.3.2-1
+       #
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+        initialDLBWPsubcarrierSpacing                               = 1;
+      #pdcch-ConfigCommon
+        initialDLBWPcontrolResourceSetZero                          = 11;
+        initialDLBWPsearchSpaceZero                                 = 0;
+
+  #uplinkConfigCommon
+     #frequencyInfoUL
+      ul_frequencyBand                                              = 77;
+      #scs-SpecificCarrierList
+      ul_offstToCarrier                                             = 0;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+      ul_subcarrierSpacing                                          = 1;
+      ul_carrierBandwidth                                           = 273;
+      pMax                                                          = 23;
+     #initialUplinkBWP
+      #genericParameters
+        initialULBWPlocationAndBandwidth                            = 1099;
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+        initialULBWPsubcarrierSpacing                               = 1;
+      #rach-ConfigCommon
+        #rach-ConfigGeneric
+          prach_ConfigurationIndex                                  = 159;
+#prach_msg1_FDM
+#0 = one, 1=two, 2=four, 3=eight
+          prach_msg1_FDM                                            = 0;
+          prach_msg1_FrequencyStart                                 = 0;
+          zeroCorrelationZoneConfig                                 = 0;
+          preambleReceivedTargetPower                               = -100;
+#preamblTransMax (0...10) = (3,4,5,6,7,8,10,20,50,100,200)
+          preambleTransMax                                          = 7;
+#powerRampingStep
+# 0=dB0,1=dB2,2=dB4,3=dB6
+        powerRampingStep                                            = 2;
+#ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR
+#1=oneeighth,2=onefourth,3=half,4=one,5=two,6=four,7=eight,8=sixteen
+        ssb_perRACH_OccasionAndCB_PreamblesPerSSB_PR                = 4;
+#one (0..15) 4,8,12,16,...60,64
+        ssb_perRACH_OccasionAndCB_PreamblesPerSSB                   = 15;
+#ra_ContentionResolutionTimer
+#(0..7) 8,16,24,32,40,48,56,64
+        ra_ContentionResolutionTimer                                = 7;
+        rsrp_ThresholdSSB                                           = 19;
+#prach-RootSequenceIndex_PR
+#1 = 839, 2 = 139
+        prach_RootSequenceIndex_PR                                  = 2;
+        prach_RootSequenceIndex                                     = 1;
+        # SCS for msg1, can only be 15 for 30 kHz < 6 GHz, takes precendence over the one derived from prach-ConfigIndex
+        #
+        msg1_SubcarrierSpacing                                      = 1,
+# restrictedSetConfig
+# 0=unrestricted, 1=restricted type A, 2=restricted type B
+        restrictedSetConfig                                         = 0,
+
+# this is the offset between the last PRACH preamble power and the Msg3 PUSCH, 2 times the field value in dB
+        msg3_DeltaPreamble                                          = 2;
+        p0_NominalWithGrant                                         = -100;
+
+# pucch-ConfigCommon setup :
+# pucchGroupHopping
+# 0 = neither, 1= group hopping, 2=sequence hopping
+        pucchGroupHopping                                           = 0;
+        hoppingId                                                   = 0;
+        p0_nominal                                                  = -96;
+
+      ssb_PositionsInBurst_Bitmap                                   = 0x1;
+
+# ssb_periodicityServingCell
+# 0 = ms5, 1=ms10, 2=ms20, 3=ms40, 4=ms80, 5=ms160, 6=spare2, 7=spare1
+      ssb_periodicityServingCell                                    = 2;
+
+# dmrs_TypeA_position
+# 0 = pos2, 1 = pos3
+      dmrs_TypeA_Position                                           = 0;
+
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+      subcarrierSpacing                                             = 1;
+
+
+  #tdd-UL-DL-ConfigurationCommon
+# subcarrierSpacing
+# 0=kHz15, 1=kHz30, 2=kHz60, 3=kHz120
+      referenceSubcarrierSpacing                                    = 1;
+      # pattern1
+      # dl_UL_TransmissionPeriodicity
+      # 0=ms0p5, 1=ms0p625, 2=ms1, 3=ms1p25, 4=ms2, 5=ms2p5, 6=ms5, 7=ms10
+      dl_UL_TransmissionPeriodicity                                 = 5;
+      nrofDownlinkSlots                                             = 3;
+      nrofDownlinkSymbols                                           = 6;
+      nrofUplinkSlots                                               = 1;
+      nrofUplinkSymbols                                             = 4;
+
+  ssPBCH_BlockPower                                                 = -11;
+  }
+
+  );
+
+
+    # ------- SCTP definitions
+    SCTP :
+    {
+        # Number of streams to use in input/output
+        SCTP_INSTREAMS  = 2;
+        SCTP_OUTSTREAMS = 2;
+    };
+
+
+    ////////// AMF parameters:
+    amf_ip_address = ({ ipv4 = "172.21.6.5"; });
+
+    NETWORK_INTERFACES :
+    {
+        GNB_IPV4_ADDRESS_FOR_NG_AMF              = "172.21.16.209";
+        GNB_IPV4_ADDRESS_FOR_NGU                 = "172.21.16.209";
+        GNB_PORT_FOR_S1U                         = 2152; # Spec 2152
+    };
+  }
+);
+
+MACRLCs = (
+{
+  num_cc                      = 1;
+  tr_s_preference             = "local_L1";
+  tr_n_preference             = "local_RRC";
+  pusch_TargetSNRx10          = 220;
+  pucch_TargetSNRx10          = 220;
+  pusch_FailureThres          = 100;
+}
+);
+
+L1s = (
+{
+  num_cc = 1;
+  tr_n_preference = "local_mac";
+  prach_dtx_threshold = 120
+  pucch0_dtx_threshold = 80;
+  pusch_dtx_threshold = -100;
+  tx_amp_backoff_dB = 3;
+  L1_rx_thread_core = 8;
+  L1_tx_thread_core = 9;
+  max_ldpc_iterations = 20;
+  phase_compensation = 0; # needs to match O-RU configuration
+}
+);
+
+RUs = (
+{
+  local_rf       = "no";
+  nb_tx          = 4;
+  nb_rx          = 4;
+  att_tx         = 0;
+  att_rx         = 0;
+  bands          = [77];
+  max_pdschReferenceSignalPower = -27;
+  max_rxgain                    = 75;
+  sf_extension                  = 0;
+  eNB_instances  = [0];
+  ru_thread_core = 10;
+  sl_ahead       = 5;
+  tr_preference  = "raw_if4p5"; # important: activate FHI7.2
+  do_precoding   = 0; # needs to match O-RU configuration
+}
+);
+
+security = {
+  # preferred ciphering algorithms
+  # the first one of the list that an UE supports in chosen
+  # valid values: nea0, nea1, nea2, nea3
+  ciphering_algorithms = ( "nea0" );
+
+  # preferred integrity algorithms
+  # the first one of the list that an UE supports in chosen
+  # valid values: nia0, nia1, nia2, nia3
+  integrity_algorithms = ( "nia2", "nia0" );
+
+  # setting 'drb_ciphering' to "no" disables ciphering for DRBs, no matter
+  # what 'ciphering_algorithms' configures; same thing for 'drb_integrity'
+  drb_ciphering = "yes";
+  drb_integrity = "no";
+};
+
+log_config :
+{
+  global_log_level = "info";
+  hw_log_level     = "info";
+  phy_log_level    = "info";
+  mac_log_level    = "info";
+  rlc_log_level    = "info";
+  pdcp_log_level   = "info";
+  rrc_log_level    = "info";
+  ngap_log_level   = "info";
+  f1ap_log_level   = "info";
+};
+
+fhi_72 = {
+  dpdk_devices = ("0000:01:01.0");
+  dpdk_iova_mode = "VA";
+  system_core = 11;
+  io_core = 12;
+  worker_cores = (13);
+  ru_addr = ("98:ae:71:04:83:e3");
+  mtu = 9600;
+  fh_config = ({
+    T1a_cp_dl = (285, 470);
+    T1a_cp_ul = (285, 429);
+    T1a_up = (125, 350);
+    Ta4 = (110, 180);
+    ru_config = {
+      iq_width = 9;
+      iq_width_prach = 9;
+    };
+  });
+};
+

ci-scripts/datalog_rt_stats.100.4x4.fhi72.yaml

@@ -0,0 +1,28 @@
+#this is a configuration file
+#used to build real time processing statistics
+#for 5G NR phy test (gNB terminate)
+Title : Processing Time (us) from datalog_rt_stats.4x4.fhi72.yaml
+ColNames :
+  - Metric
+  - Average; Max; Count
+  - Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)
+Ref :
+  tx_fhaul : 40.2
+  feptx_prec (per port, half_slot) : 12.3
+  L1 Tx processing : 221.0
+  DLSCH encoding : 141.0
+  L1 Rx processing : 332.5
+  UL segments decoding : 55.2
+  UL Indication : 0.4
+  Slot Indication : 14.3
+  PUSCH inner-receiver : 421.4
+DeviationThreshold :
+  tx_fhaul : 0.25
+  feptx_prec (per port, half_slot) : 0.25
+  L1 Tx processing : 0.25
+  DLSCH encoding : 0.25
+  L1 Rx processing : 0.25
+  UL segments decoding : 0.25
+  UL Indication : 1.00
+  Slot Indication : 0.50
+  PUSCH inner-receiver : 0.25

ci-scripts/docker/Dockerfile.physim.cuda.ubuntu

@@ -0,0 +1,52 @@
+#/*
+# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+# * contributor license agreements.  See the NOTICE file distributed with
+# * this work for additional information regarding copyright ownership.
+# * The OpenAirInterface Software Alliance licenses this file to You under
+# * the OAI Public License, Version 1.1  (the "License"); you may not use this file
+# * except in compliance with the License.
+# * You may obtain a copy of the License at
+# *
+# *      http://www.openairinterface.org/?page_id=698
+# *
+# * Unless required by applicable law or agreed to in writing, software
+# * distributed under the License is distributed on an "AS IS" BASIS,
+# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# * See the License for the specific language governing permissions and
+# * limitations under the License.
+# *-------------------------------------------------------------------------------
+# * For more information about the OpenAirInterface (OAI) Software Alliance:
+# *      contact@openairinterface.org
+# */
+#---------------------------------------------------------------------
+#
+# Dockerfile for the Open-Air-Interface BUILD service
+#   Valid for Ubuntu 24.04
+#
+#---------------------------------------------------------------------
+FROM nvidia/cuda:12.9.1-devel-ubuntu22.04 AS cuda-image
+FROM ran-base:develop AS ran-tests
+
+RUN rm -Rf /oai-ran
+COPY --from=cuda-image /usr/local/cuda/ /usr/local/cuda/
+
+# Set the LD_LIBRARY_PATH to ensure the system can find the copied libraries.
+# This is crucial for applications that use CUDA.
+ENV LD_LIBRARY_PATH=/usr/local/cuda/lib64:/usr/local/cuda/targets/sbsa-linux/lib
+ENV PATH=/usr/local/cuda/bin:$PATH
+ENV C_INCLUDE_PATH=/usr/local/cuda/include
+
+WORKDIR /oai-ran
+COPY . .
+
+WORKDIR /oai-ran/build
+RUN cmake -GNinja -DENABLE_PHYSIM_TESTS=ON -DENABLE_TESTS=ON \
+        -DSANITIZE_UNDEFINED=OFF -DSANITIZE_ADDRESS=OFF \
+        -DCMAKE_C_FLAGS=-Werror -DCMAKE_CXX_FLAGS=-Werror \
+        -DPHYSIM_CHECK_FILES="ThresholdsCuda.cmake" \
+        -DCUDA_ENABLE=ON \
+        -DUSE_UNIFIED_MEMORY=ON \
+        -DUSE_ATS_MEMORY=OFF \
+        -DCMAKE_RUNTIME_OUTPUT_DIRECTORY=/oai-ran/cmake_targets/ran_build/build \
+        .. && \
+        ninja nr_ulsim nr_dlsim

ci-scripts/main.py

@@ -62,13 +62,6 @@ import lxml.etree as ET
 import logging
 import signal
 import traceback
-logging.basicConfig(
-	level=logging.DEBUG,
-	stream=sys.stdout,
-	format="[%(asctime)s] %(levelname)8s: %(message)s"
-)
-
-
 
 
 #-----------------------------------------------------------
@@ -109,41 +102,20 @@ def ExecuteActionWithParam(action, ctx, node):
 			success = CONTAINERS.BuildRunTests(ctx, node, HTML)
 
 	elif action == 'Initialize_eNB':
-		datalog_rt_stats_file=test.findtext('rt_stats_cfg')
-		if datalog_rt_stats_file is None:
-			RAN.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
-		else:
-			RAN.datalog_rt_stats_file=datalog_rt_stats_file
 		RAN.Initialize_eNB_args=test.findtext('Initialize_eNB_args')
-		USRPIPAddress = test.findtext('USRP_IPAddress') or ''
-
-		#local variable air_interface
-		air_interface = test.findtext('air_interface')		
-		if (air_interface is None) or (air_interface.lower() not in ['nr','lte']):
-			RAN.air_interface = 'lte-softmodem'
-		else:
-			RAN.air_interface = air_interface.lower() +'-softmodem'
-
 		cmd_prefix = test.findtext('cmd_prefix')
 		if cmd_prefix is not None: RAN.cmd_prefix = cmd_prefix
 		success = RAN.InitializeeNB(ctx, node, HTML)
 
 	elif action == 'Terminate_eNB':
-		#retx checkers
-		string_field = test.findtext('d_retx_th')
-		if (string_field is not None):
-			RAN.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')]
-		string_field=test.findtext('u_retx_th')
-		if (string_field is not None):
-			RAN.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')]
-
-		#local variable air_interface
-		air_interface = test.findtext('air_interface')		
-		if (air_interface is None) or (air_interface.lower() not in ['nr','lte']):
-			RAN.air_interface = 'lte-softmodem'
-		else:
-			RAN.air_interface = air_interface.lower() +'-softmodem'
-		success = RAN.TerminateeNB(ctx, node, HTML)
+		services = []
+		analysis = test.find("analysis")
+		if analysis is not None:
+			# services: multiple services to analyse, separated by whitespace
+			services = analysis.findtext("services", default="").split()
+			# service: individual services to analyze, in case they have whitespace
+			services = services + [s.text for s in analysis.findall("service")]
+		success = RAN.TerminateeNB(ctx, node, HTML, services)
 
 	elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE':
 		CiTestObj.ue_ids = test.findtext('id').split(' ')
@@ -216,12 +188,6 @@ def ExecuteActionWithParam(action, ctx, node):
 
 	elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace" or action == "Stop_Object":
 		CONTAINERS.yamlPath = test.findtext('yaml_path')
-		string_field=test.findtext('d_retx_th')
-		if (string_field is not None):
-			CONTAINERS.ran_checkers['d_retx_th'] = [float(x) for x in string_field.split(',')]
-		string_field=test.findtext('u_retx_th')
-		if (string_field is not None):
-			CONTAINERS.ran_checkers['u_retx_th'] = [float(x) for x in string_field.split(',')]
 		CONTAINERS.services = test.findtext('services')
 		CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
 		CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
@@ -230,7 +196,14 @@ def ExecuteActionWithParam(action, ctx, node):
 		elif action == 'Stop_Object':
 			success = CONTAINERS.StopObject(ctx, node, HTML)
 		elif action == 'Undeploy_Object':
-			success = CONTAINERS.UndeployObject(ctx, node, HTML, RAN)
+			analysis = test.find("analysis")
+			services = []
+			if analysis is not None:
+				# services: multiple services to analyse, separated by whitespace
+				services = analysis.findtext("services", default="").split()
+				# service: individual services to analyze, in case they have whitespace
+				services = services + [s.text for s in analysis.findall("service")]
+			success = CONTAINERS.UndeployObject(ctx, node, HTML, services)
 		elif action == 'Create_Workspace':
 			if force_local:
 				# Do not create a working directory when running locally. Current repo directory will be used
@@ -280,21 +253,20 @@ def ExecuteActionWithParam(action, ctx, node):
 		images = test.findtext('images').split()
 		success = CLUSTER.PullClusterImage(HTML, node, images, tag_prefix=tag_prefix)
 
+	elif action == 'AnalyzeRTStats':
+		yaml = test.findtext('stats_cfg')
+		success = RAN.AnalyzeRTStats(HTML, node, ctx, yaml)
+
+	elif action == 'AnalyzeRTStats_Object':
+		yaml = test.findtext('stats_cfg')
+		success = CONTAINERS.AnalyzeRTStatsObject(HTML, node, ctx, yaml)
+
 	else:
 		logging.warning(f"unknown action {action}, skip step")
 		success = True # by default, we skip the step and print a warning
 
 	return success
 
-#check if given test is in list
-#it is in list if one of the strings in 'list' is at the beginning of 'test'
-def test_in_list(test, list):
-	for check in list:
-		check=check.replace('+','')
-		if (test.startswith(check)):
-			return True
-	return False
-
 test_runner_abort = False
 def receive_signal(signum, frame):
     global test_runner_abort
@@ -349,7 +321,12 @@ CLUSTER = cls_cluster.Cluster()
 import args_parse
 # Force local execution, move all execution targets to localhost
 force_local = False
-mode, force_local = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER)
+mode, force_local, date_fmt = args_parse.ArgsParse(sys.argv,CiTestObj,RAN,HTML,CONTAINERS,HELP,SCA,CLUSTER)
+fmt = "%(levelname)8s: %(message)s"
+if date_fmt:
+    fmt = "[%(asctime)s] %(levelname)s %(message)s"
+logging.basicConfig(level=logging.DEBUG, stream=sys.stdout, format=fmt, datefmt=date_fmt,)
+
 
 #-----------------------------------------------------------
 # mode amd XML class (action) analysis
@@ -367,18 +344,7 @@ elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
 elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
 	logging.warning("Option LogCollectBuild ignored")
 elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
-	if RAN.eNBSourceCodePath == '':
-		HELP.GenericHelp(CONST.Version)
-		sys.exit('Insufficient Parameter')
-	if os.path.isdir('cmake_targets/log'):
-		cmd = 'zip -r enb.log.' + RAN.BuildId + '.zip cmake_targets/log'
-		logging.info(cmd)
-		try:
-			zipStatus = subprocess.check_output(cmd, shell=True, stderr=subprocess.STDOUT, universal_newlines=True, timeout=60)
-		except subprocess.CalledProcessError as e:
-			logging.error("Command '{}' returned non-zero exit status {}.".format(e.cmd, e.returncode))
-			logging.error("Error output:\n{}".format(e.output))
-		sys.exit(0)
+	logging.warning("Option LogCollecteNB ignored")
 elif re.match('^LogCollectHSS$', mode, re.IGNORECASE):
 	logging.warning("Option LogCollectHSS ignored")
 elif re.match('^LogCollectMME$', mode, re.IGNORECASE):

ci-scripts/ran.py

@@ -31,21 +31,16 @@
 #-----------------------------------------------------------
 # Import
 #-----------------------------------------------------------
-import sys              # arg
 import re               # reg
 import logging
 import os
 import time
-import yaml
-import cls_cmd
-
 
 #-----------------------------------------------------------
 # OAI Testing modules
 #-----------------------------------------------------------
 import cls_cmd
-import helpreadme as HELP
-import constants as CONST
+import cls_analysis
 from cls_ci_helper import archiveArtifact
 
 #-----------------------------------------------------------
@@ -63,16 +58,9 @@ class RANManagement():
 		self.eNBSourceCodePath = ''
 		self.Initialize_eNB_args = ''
 		self.imageKind = ''
-		self.air_interface = ''
 		self.eNBOptions = ['', '', '']
-		self.eNBmbmsEnables = [False, False, False]
 		self.eNBstatuses = [-1, -1, -1]
 		self.runtime_stats= ''
-		self.datalog_rt_stats={}
-		self.datalog_rt_stats_file='datalog_rt_stats.default.yaml'
-		self.USRPIPAddress = ''
-		#checkers from xml
-		self.ran_checkers={}
 		self.cmd_prefix = '' # prefix before {lte,nr}-softmodem
 		self.node = ''
 		self.command = ''
@@ -102,7 +90,7 @@ class RANManagement():
 
 		logfile = f'{lSourcePath}/cmake_targets/enb.log'
 		cmd.cd(f"{lSourcePath}/cmake_targets/") # important: set wd so nrL1_stats.log etc are logged here
-		cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/{self.air_interface} -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &')
+		cmd.run(f'sudo -E stdbuf -o0 {self.cmd_prefix} {lSourcePath}/cmake_targets/ran_build/build/nr-softmodem -O {lSourcePath}/{full_config_file} {extra_options} > {logfile} 2>&1 &')
 
 		if extra_options != '':
 			self.eNBOptions = extra_options
@@ -116,12 +104,12 @@ class RANManagement():
 				enbDidSync = True
 				break
 		if not enbDidSync:
-			cmd.run(f'sudo killall -9 {self.air_interface}') # in case it did not stop automatically
+			cmd.run(f'sudo killall -9 nr-softmodem') # in case it did not stop automatically
 			archiveArtifact(cmd, ctx, logfile)
 
 		cmd.close()
 
-		msg = f'{self.cmd_prefix} {self.air_interface} -O {config_file} {extra_options}'
+		msg = f'{self.cmd_prefix} nr-softmodem -O {config_file} {extra_options}'
 		if enbDidSync:
 			logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
 			HTML.CreateHtmlTestRowQueue(msg, 'OK', [])
@@ -131,7 +119,7 @@ class RANManagement():
 
 		return enbDidSync
 
-	def TerminateeNB(self, ctx, node, HTML):
+	def TerminateeNB(self, ctx, node, HTML, to_analyze):
 		logging.debug('Stopping eNB/gNB on server: ' + node)
 		lSourcePath = self.eNBSourceCodePath
 		cmd = cls_cmd.getConnection(node)
@@ -146,14 +134,13 @@ class RANManagement():
 		if result is not None:
 			cmd.run('sudo -S killall --signal SIGKILL -r .*-softmodem')
 			time.sleep(5)
+		HTML.CreateHtmlTestRowQueue(node, 'OK', ['Undeployment successful'])
 
 		# see InitializeeNB()
 		logfile = f'{lSourcePath}/cmake_targets/enb.log'
 		logdir = os.path.dirname(logfile)
 
 		file = archiveArtifact(cmd, ctx, logfile)
-		archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log")
-		archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log")
 		cmd.close()
 		if file is None:
 			logging.debug('\u001B[1;37;41m Could not copy xNB logfile to analyze it! \u001B[0m')
@@ -162,633 +149,21 @@ class RANManagement():
 			return False
 
 		logging.debug('\u001B[1m Analyzing xNB logfile \u001B[0m ' + file)
-		logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
-		if logStatus < 0:
-			HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
-		else:
-			HTML.CreateHtmlTestRow(self.runtime_stats, 'OK', CONST.ALL_PROCESSES_OK)
+		service_desc = {}
+		service_desc["nr-softmodem"] = {'returncode': 0, 'logfile': file}
+		success = cls_analysis.AnalyzeServices(HTML, service_desc, to_analyze)
+		return success
 
-		#display rt stats for gNB only
-		if len(self.datalog_rt_stats) != 0:
-			HTML.CreateHtmlDataLogTable(self.datalog_rt_stats)
+	def AnalyzeRTStats(self, HTML, node, ctx, thresholds):
+		logging.info(f'Analyzing realtime stats from server: {node}')
+		lSourcePath = self.eNBSourceCodePath
 
-		return logStatus >= 0
+		logdir = f'{lSourcePath}/cmake_targets'
+		with cls_cmd.getConnection(node) as cmd:
+			l1_file = archiveArtifact(cmd, ctx, f"{logdir}/nrL1_stats.log")
+			mac_file = archiveArtifact(cmd, ctx, f"{logdir}/nrMAC_stats.log")
 
-	def _analyzeUeRetx(self, rounds, checkers, regex):
-		if len(rounds) == 0 or len(checkers) == 0:
-			logging.warning(f'warning: rounds={rounds} checkers={checkers}')
-			return []
-
-		perc = list(0 for i in checkers) # results in %
-		stats = list(False for i in checkers) # status if succeeded
-		tmp = re.match(regex, rounds)
-		if tmp is None:
-			logging.error('_analyzeUeRetx: did not match regex for DL retx analysis')
-			return stats
-		retx_data = [float(x) for x in tmp.groups()]
-		for i in range(0, len(perc)):
-			#case where numerator > denumerator with denum ==0 is disregarded, cannot hapen in principle, will lead to 0%
-			perc[i] = 0 if (retx_data[i] == 0) else 100 * retx_data[i + 1] / retx_data[i]
-			#treating % > 100 , % > requirement
-			stats[i] = perc[i] <= 100 and perc[i] <= checkers[i]
-		return stats
-
-	def AnalyzeLogFile_eNB(self, eNBlogFile, HTML, checkers={}):
-		if (not os.path.isfile(eNBlogFile)):
-			return -1
-		enb_log_file = open(eNBlogFile, 'r')
-		exitSignalReceived = False
-		foundAssertion = False
-		msgAssertion = ''
-		msgLine = 0
-		foundSegFault = False
-		foundRealTimeIssue = False
-		foundRealTimeIssue_cnt = 0
-		rrcSetupComplete = 0
-		rrcReleaseRequest = 0
-		rrcReconfigRequest = 0
-		rrcReconfigComplete = 0
-		rrcReestablishRequest = 0
-		rrcReestablishComplete = 0
-		rrcReestablishReject = 0
-		rlcDiscardBuffer = 0
-		rachCanceledProcedure = 0
-		uciStatMsgCount = 0
-		pdcpFailure = 0
-		ulschFailure = 0
-		ulschAllocateCCEerror = 0
-		uplinkSegmentsAborted = 0
-		ulschReceiveOK = 0
-		gnbRxTxWakeUpFailure = 0
-		gnbTxWriteThreadEnabled = False
-		cdrxActivationMessageCount = 0
-		dropNotEnoughRBs = 0
-		mbmsRequestMsg = 0
-		htmleNBFailureMsg = ''
-		isRRU = False
-		isSlave = False
-		slaveReceivesFrameResyncCmd = False
-		X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
-		X2HO_inNbProcedures = 0
-		X2HO_outNbProcedures = 0
-		global_status = CONST.ALL_PROCESSES_OK
-		# Runtime statistics
-		runTime = ''
-		userTime = ''
-		systemTime = ''
-		maxPhyMemUsage = ''
-		nbContextSwitches = ''
-		#NSA FR1 check
-		NSA_RAPROC_PUSCH_check = 0
-		#dlsch and ulsch statistics (dictionary)
-		dlsch_ulsch_stats = {}
-		#real time statistics (dictionary)
-		real_time_stats = {}
-		#count "problem receiving samples" msg
-		pb_receiving_samples_cnt = 0
-		#count "removing UE" msg
-		removing_ue = 0
-		#count"X2AP-PDU"
-		x2ap_pdu = 0
-		#gnb specific log markers
-		gnb_markers ={'SgNBReleaseRequestAcknowledge': [],'FAILURE': [], 'scgFailureInformationNR-r15': [], 'SgNBReleaseRequest': [], 'Detected UL Failure on PUSCH':[]}
-		nodeB_prefix_found = False
-		RealTimeProcessingIssue = False
-		retx_status = {}
-		nrRrcRcfgComplete = 0
-		harqFeedbackPast = 0
-		showedByeMsg = False # last line is Bye. -> stopped properly
-	
-		line_cnt=0 #log file line counter
-		for line in enb_log_file.readlines():
-			line_cnt+=1
-			# Detection of eNB/gNB from a container log
-			result = re.search('Starting eNB soft modem', str(line))
-			if result is not None:
-				nodeB_prefix_found = True
-				nodeB_prefix = 'e'
-			result = re.search('Starting gNB soft modem', str(line))
-			if result is not None:
-				nodeB_prefix_found = True
-				nodeB_prefix = 'g'
-			result = re.search('Run time:' ,str(line))
-			# Runtime statistics
-			result = re.search('Run time:' ,str(line))
-			if result is not None:
-				runTime = str(line).strip()
-			if runTime != '':
-				result = re.search('Time executing user inst', str(line))
-				if result is not None:
-					fields=line.split(':')
-					userTime = 'userTime : ' + fields[1].replace('\n','')
-				result = re.search('Time executing system inst', str(line))
-				if result is not None:
-					fields=line.split(':')
-					systemTime = 'systemTime : ' + fields[1].replace('\n','')
-				result = re.search('Max. Phy. memory usage:', str(line))
-				if result is not None:
-					fields=line.split(':')
-					maxPhyMemUsage = 'maxPhyMemUsage : ' + fields[1].replace('\n','')
-				result = re.search('Number of context switch.*process origin', str(line))
-				if result is not None:
-					fields=line.split(':')
-					nbContextSwitches = 'nbContextSwitches : ' + fields[1].replace('\n','')
-			if X2HO_state == CONST.X2_HO_REQ_STATE__IDLE:
-				result = re.search('target eNB Receives X2 HO Req X2AP_HANDOVER_REQ', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ
-				result = re.search('source eNB receives the X2 HO ACK X2AP_HANDOVER_REQ_ACK', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK
-			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RECEIVES_REQ:
-				result = re.search('Received LTE_RRCConnectionReconfigurationComplete from UE', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE
-			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_RRC_RECFG_COMPLETE:
-				result = re.search('issue rrc_eNB_send_PATH_SWITCH_REQ', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ
-			if X2HO_state == CONST.X2_HO_REQ_STATE__TARGET_SENDS_SWITCH_REQ:
-				result = re.search('received path switch ack S1AP_PATH_SWITCH_REQ_ACK', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
-					X2HO_inNbProcedures += 1
-			if X2HO_state == CONST.X2_HO_REQ_STATE__SOURCE_RECEIVES_REQ_ACK:
-				result = re.search('source eNB receives the X2 UE CONTEXT RELEASE X2AP_UE_CONTEXT_RELEASE', str(line))
-				if result is not None:
-					X2HO_state = CONST.X2_HO_REQ_STATE__IDLE
-					X2HO_outNbProcedures += 1
-
-			if self.eNBOptions[0] != '':
-				res1 = re.search('max_rxgain (?P<requested_option>[0-9]+)', self.eNBOptions[0])
-				res2 = re.search('max_rxgain (?P<applied_option>[0-9]+)',  str(line))
-				if res1 is not None and res2 is not None:
-					requested_option = int(res1.group('requested_option'))
-					applied_option = int(res2.group('applied_option'))
-					if requested_option == applied_option:
-						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ok-circle"></span> Command line option(s) correctly applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[0] + '\n\n'
-					else:
-						htmleNBFailureMsg += '<span class="glyphicon glyphicon-ban-circle"></span> Command line option(s) NOT applied <span class="glyphicon glyphicon-arrow-right"></span> ' + self.eNBOptions[0] + '\n\n'
-			result = re.search('Exiting OAI softmodem|Caught SIGTERM, shutting down', str(line))
-			if result is not None:
-				exitSignalReceived = True
-			result = re.search('[Ss]egmentation [Ff]ault', str(line))
-			if result is not None and not exitSignalReceived:
-				foundSegFault = True
-			result = re.search('[Cc]ore [dD]ump', str(line))
-			if result is not None and not exitSignalReceived:
-				foundSegFault = True
-			result = re.search('[Aa]ssertion', str(line))
-			if result is not None and not exitSignalReceived:
-				foundAssertion = True
-			result = re.search('LLL', str(line))
-			if result is not None and not exitSignalReceived:
-				foundRealTimeIssue = True
-				foundRealTimeIssue_cnt += 1
-			if foundAssertion and (msgLine < 3):
-				msgLine += 1
-				msgAssertion += str(line)
-			result = re.search('Setting function for RU', str(line))
-			if result is not None:
-				isRRU = True
-			if isRRU:
-				result = re.search('RU 0 is_slave=yes', str(line))
-				if result is not None:
-					isSlave = True
-				if isSlave:
-					result = re.search('Received RRU_frame_resynch command', str(line))
-					if result is not None:
-						slaveReceivesFrameResyncCmd = True
-			result = re.search('LTE_RRCConnectionSetupComplete from UE', str(line))
-			if result is not None:
-				rrcSetupComplete += 1
-			result = re.search('Generate LTE_RRCConnectionRelease|Generate RRCConnectionRelease', str(line))
-			if result is not None:				rrcReleaseRequest += 1
-			result = re.search('Generate LTE_RRCConnectionReconfiguration', str(line))
-			if result is not None:
-				rrcReconfigRequest += 1
-			result = re.search('LTE_RRCConnectionReconfigurationComplete from UE rnti', str(line))
-			if result is not None:
-				rrcReconfigComplete += 1
-			result = re.search('LTE_RRCConnectionReestablishmentRequest', str(line))
-			if result is not None:
-				rrcReestablishRequest += 1
-			result = re.search('LTE_RRCConnectionReestablishmentComplete', str(line))
-			if result is not None:
-				rrcReestablishComplete += 1
-			result = re.search('LTE_RRCConnectionReestablishmentReject', str(line))
-			if result is not None:
-				rrcReestablishReject += 1
-			result = re.search('CDRX configuration activated after RRC Connection', str(line))
-			if result is not None:
-				cdrxActivationMessageCount += 1
-			result = re.search('uci->stat', str(line))
-			if result is not None:
-				uciStatMsgCount += 1
-			result = re.search('PDCP.*Out of Resources.*reason', str(line))
-			if result is not None:
-				pdcpFailure += 1
-			result = re.search('could not wakeup gNB rxtx process', str(line))
-			if result is not None:
-				gnbRxTxWakeUpFailure += 1
-			result = re.search('tx write thread ready', str(line))
-			if result is not None:
-				gnbTxWriteThreadEnabled = True
-			result = re.search('ULSCH in error in round|ULSCH 0 in error', str(line))
-			if result is not None:
-				ulschFailure += 1
-			result = re.search('ERROR ALLOCATING CCEs', str(line))
-			if result is not None:
-				ulschAllocateCCEerror += 1
-			result = re.search('uplink segment error.*aborted [1-9] segments', str(line))
-			if result is not None:
-				uplinkSegmentsAborted += 1
-			result = re.search('ULSCH received ok', str(line))
-			if result is not None:
-				ulschReceiveOK += 1
-			result = re.search('BAD all_segments_received', str(line))
-			if result is not None:
-				rlcDiscardBuffer += 1
-			result = re.search('Canceled RA procedure for UE rnti', str(line))
-			if result is not None:
-				rachCanceledProcedure += 1
-			result = re.search('dropping, not enough RBs', str(line))
-			if result is not None:
-				dropNotEnoughRBs += 1
-			if self.eNBmbmsEnables[0]:
-				result = re.search('MBMS USER-PLANE.*Requesting.*bytes from RLC', str(line))
-				if result is not None:
-					mbmsRequestMsg += 1
-			#FR1 NSA test : add new markers to make sure gNB is used
-			result = re.search('\[gNB [0-9]+\]\[RAPROC\] PUSCH with TC_RNTI 0x[0-9a-fA-F]+ received correctly, adding UE MAC Context RNTI 0x[0-9a-fA-F]+', str(line))
-			if result is not None:
-				NSA_RAPROC_PUSCH_check = 1
-
-			# Collect information on UE DLSCH and ULSCH statistics
-			keys = {'dlsch_rounds','ulsch_rounds'}
-			for k in keys:
-				result = re.search(k, line)
-				if result is None:
-					continue
-				result = re.search('UE (?:RNTI )?([0-9a-f]{4})', line)
-				if result is None:
-					logging.error(f'did not find RNTI while matching key {k}')
-					continue
-				rnti = result.group(1)
-
-				#remove 1- all useless char before relevant info (ulsch or dlsch) 2- trailing char
-				if not rnti in dlsch_ulsch_stats: dlsch_ulsch_stats[rnti] = {}
-				dlsch_ulsch_stats[rnti][k]=re.sub(r'^.*\]\s+', r'' , line.rstrip())
-
-			result = re.search('Received NR_RRCReconfigurationComplete from UE', str(line))
-			if result is not None:
-				nrRrcRcfgComplete += 1
-			result = re.search('HARQ feedback is in the past', str(line))
-			if result is not None:
-				harqFeedbackPast += 1
-
-
-			#count "problem receiving samples" msg
-			result = re.search('\[PHY\]\s+problem receiving samples', str(line))
-			if result is not None:
-				pb_receiving_samples_cnt += 1
-			#count "Removing UE" msg
-			result = re.search('\[MAC\]\s+Removing UE', str(line))
-			if result is not None:
-				removing_ue += 1
-			#count "X2AP-PDU"
-			result = re.search('X2AP-PDU', str(line))
-			if result is not None:
-				x2ap_pdu += 1
-			#gnb markers logging
-			for k in gnb_markers:
-				result = re.search(k, line)
-				if result is not None:
-					gnb_markers[k].append(line_cnt)
-
-			# check whether e/gNB log finishes with "Bye." message
-			showedByeMsg |= re.search(r'^Bye.\n', str(line), re.MULTILINE) is not None
-
-		enb_log_file.close()
-
-
-		#the following part takes the *_stats.log files as source (not the stdout log file)
-
-		#the datalog config file has to be loaded
-		datalog_rt_stats_file=self.datalog_rt_stats_file
-		if (os.path.isfile(datalog_rt_stats_file)):
-			yaml_file=datalog_rt_stats_file
-		elif (os.path.isfile('ci-scripts/'+datalog_rt_stats_file)):
-			yaml_file='ci-scripts/'+datalog_rt_stats_file
-		else:
-			logging.error("Datalog RT stats yaml file cannot be found")
-			sys.exit("Datalog RT stats yaml file cannot be found")
-
-		with open(yaml_file,'r') as f:
-			datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
-		rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field  
-
-		# nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file
-		# currently the link is only implicit as below based on pattern matching
-		# I will rework this to give the file explicitly
-		l1_stats_fn = re.sub(r'-enb.log$', '-nrL1_stats.log', eNBlogFile)
-		mac_stats_fn = re.sub(r'-enb.log$', '-nrMAC_stats.log', eNBlogFile)
-		if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn):
-			# don't use CI-nrL1_stats.log, as this will increase the processing time for
-			# no reason, we just need the last occurence
-			nrL1_stats = open(l1_stats_fn, 'r')
-			nrMAC_stats = open(mac_stats_fn, 'r')
-			for line in nrL1_stats.readlines():
-				for k in rt_keys:
-					result = re.search(k, line)     
-					if result is not None:
-						#remove 1- all useless char before relevant info  2- trailing char
-						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
-						if tmp!=None: 
-							real_time_stats[k]=tmp.group(1)
-			for line in nrMAC_stats.readlines():
-				for k in rt_keys:
-					result = re.search(k, line)     
-					if result is not None:
-						#remove 1- all useless char before relevant info  2- trailing char
-						tmp=re.match(rf'^.*?(\b{k}\b.*)',line.rstrip()) #from python 3.6 we can use literal string interpolation for the variable k, using rf' in the regex
-						if tmp!=None: 
-							real_time_stats[k]=tmp.group(1)
-			nrL1_stats.close()
-			nrMAC_stats.close()
-		else:
-			logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}")
-
-		#stdout log file and stat log files analysis completed
-		logging.debug('   File analysis (stdout, stats) completed')
-
-		#post processing depending on the node type
-		if not nodeB_prefix_found:
-			if self.air_interface == 'lte-softmodem':
-				nodeB_prefix = 'e'
-			else:
-				nodeB_prefix = 'g'
-
-		if nodeB_prefix == 'g':
-			if ulschReceiveOK > 0:
-				statMsg = nodeB_prefix + 'NB showed ' + str(ulschReceiveOK) + ' "ULSCH received ok" message(s)'
-				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-				htmleNBFailureMsg += statMsg + '\n'
-			if gnbRxTxWakeUpFailure > 0:
-				statMsg = nodeB_prefix + 'NB showed ' + str(gnbRxTxWakeUpFailure) + ' "could not wakeup gNB rxtx process" message(s)'
-				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-				htmleNBFailureMsg += statMsg + '\n'
-			if gnbTxWriteThreadEnabled:
-				statMsg = nodeB_prefix + 'NB ran with TX Write thread enabled'
-				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-				htmleNBFailureMsg += statMsg + '\n'
-			if nrRrcRcfgComplete > 0:
-				statMsg = nodeB_prefix + 'NB showed ' + str(nrRrcRcfgComplete) + ' "Received NR_RRCReconfigurationComplete from UE" message(s)'
-				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-				htmleNBFailureMsg += statMsg + '\n'
-			if harqFeedbackPast > 0:
-				statMsg = nodeB_prefix + 'NB showed ' + str(harqFeedbackPast) + ' "HARQ feedback is in the past" message(s)'
-				logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-				htmleNBFailureMsg += statMsg + '\n'
-			#FR1 NSA test : add new markers to make sure gNB is used
-			if NSA_RAPROC_PUSCH_check:
-				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : PASS '
-				htmlMsg = statMsg+'\n'
-			else:
-				statMsg = '[RAPROC] PUSCH with TC_RNTI message check for ' + nodeB_prefix + 'NB : FAIL or not relevant'
-				htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			#problem receiving samples log
-			statMsg = '[PHY] problem receiving samples msg count =  '+str(pb_receiving_samples_cnt)
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			#gnb markers
-			statMsg = 'logfile line count = ' + str(line_cnt)			
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			if len(gnb_markers['SgNBReleaseRequestAcknowledge'])!=0:
-				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequestAcknowledge'][0])
-			else:
-				statMsg = 'SgNBReleaseRequestAcknowledge = ' + str(len(gnb_markers['SgNBReleaseRequestAcknowledge'])) + ' occurences' 
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			statMsg = 'FAILURE = ' + str(len(gnb_markers['FAILURE'])) + ' occurences'
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			statMsg = 'Detected UL Failure on PUSCH = ' + str(len(gnb_markers['Detected UL Failure on PUSCH'])) + ' occurences'
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-
-			#ulsch and dlsch statistics and checkers
-			for ue in dlsch_ulsch_stats:
-				dlulstat = dlsch_ulsch_stats[ue]
-				#print statistics into html
-				statMsg=''
-				for key in dlulstat:
-					statMsg += dlulstat[key] + '\n'
-					logging.debug(dlulstat[key])
-				htmleNBFailureMsg += statMsg
-
-				retx_status[ue] = {}
-				dlcheckers = [] if 'd_retx_th' not in checkers else checkers['d_retx_th']
-				retx_status[ue]['dl'] = self._analyzeUeRetx(dlulstat['dlsch_rounds'], dlcheckers, r'^.*dlsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+dlsch_errors\s+(\d+)')
-				ulcheckers = [] if 'u_retx_th' not in checkers else checkers['u_retx_th']
-				retx_status[ue]['ul'] = self._analyzeUeRetx(dlulstat['ulsch_rounds'], ulcheckers, r'^.*ulsch_rounds\s+(\d+)\/(\d+)\/(\d+)\/(\d+),\s+ulsch_errors\s+(\d+)')
-
-
-			#real time statistics
-			datalog_rt_stats['Data']={}
-			if len(real_time_stats)!=0: #check if dictionary is not empty
-				for k in real_time_stats:
-					tmp=re.match(r'^(?P<metric>.*):\s+(?P<avg>\d+\.\d+) us;\s+(?P<count>\d+);\s+(?P<max>\d+\.\d+) us;',real_time_stats[k])
-					if tmp is not None:
-						metric=tmp.group('metric')
-						avg=float(tmp.group('avg'))
-						max=float(tmp.group('max'))
-						count=int(tmp.group('count'))
-						datalog_rt_stats['Data'][metric]=["{:.0f}".format(avg),"{:.0f}".format(max),"{:d}".format(count),"{:.2f}".format(avg/datalog_rt_stats['Ref'][metric])]
-				#once all metrics are collected, store the data as a class attribute to build a dedicated HTML table afterward
-				self.datalog_rt_stats=datalog_rt_stats
-				#check if there is a fail => will render the test as failed
-				for k in datalog_rt_stats['Data']:
-					valnorm = float(datalog_rt_stats['Data'][k][3])
-					dev = datalog_rt_stats['DeviationThreshold'][k]
-					if valnorm > 1.0 + dev or valnorm < 1.0 - dev: # condition for fail : avg/ref deviates by more than "deviation threshold"
-						logging.debug(f'\u001B[1;30;43m normalized datalog_rt_stats metric {k}={valnorm} deviates by more than {dev}\u001B[0m')
-						RealTimeProcessingIssue = True
-			else:
-				statMsg = 'No real time stats found in the log file\n'
-				logging.debug('No real time stats found in the log file')
-				htmleNBFailureMsg += statMsg
-
-			if not showedByeMsg:
-				logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB did not show "Bye." message at end, it likely did not stop properly! \u001B[0m')
-				htmleNBFailureMsg += 'No Bye. message found, did not stop properly\n'
-				global_status = CONST.ENB_SHUTDOWN_NO_BYE
-			else:
-				logging.debug('"Bye." message found at end.')
-
-		else:
-			#Removing UE log
-			statMsg = '[MAC] Removing UE msg count =  '+str(removing_ue)
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			#X2AP-PDU log
-			statMsg = 'X2AP-PDU msg count =  '+str(x2ap_pdu)
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			#nsa markers
-			statMsg = 'logfile line count = ' + str(line_cnt)			
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			if len(gnb_markers['SgNBReleaseRequest'])!=0:
-				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences , starting line ' + str(gnb_markers['SgNBReleaseRequest'][0])
-			else:
-				statMsg = 'SgNBReleaseRequest = ' + str(len(gnb_markers['SgNBReleaseRequest'])) + ' occurences'
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg
-			statMsg = 'scgFailureInformationNR-r15 = ' + str(len(gnb_markers['scgFailureInformationNR-r15'])) + ' occurences'
-			htmlMsg = statMsg+'\n'
-			logging.debug(statMsg)
-			htmleNBFailureMsg += htmlMsg			
-
-		for ue in retx_status:
-			msg = f"retransmissions for UE {ue}: DL {retx_status[ue]['dl']} UL {retx_status[ue]['ul']}"
-			if False in retx_status[ue]['dl'] or False in retx_status[ue]['ul']:
-				msg = 'Failure: ' + msg
-				logging.error(f'\u001B[1;37;41m {msg}\u001B[0m')
-				htmleNBFailureMsg += f'{msg}\n'
-				global_status = CONST.ENB_RETX_ISSUE
-			else:
-				logging.debug(msg)
-
-		if RealTimeProcessingIssue:
-			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with real time processing issue! \u001B[0m')
-			htmleNBFailureMsg += 'Fail due to real time processing issue\n'
-			global_status = CONST.ENB_REAL_TIME_PROCESSING_ISSUE
-		if uciStatMsgCount > 0:
-			statMsg = nodeB_prefix + 'NB showed ' + str(uciStatMsgCount) + ' "uci->stat" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if pdcpFailure > 0:
-			statMsg = nodeB_prefix + 'NB showed ' + str(pdcpFailure) + ' "PDCP Out of Resources" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if ulschFailure > 0:
-			statMsg = nodeB_prefix + 'NB showed ' + str(ulschFailure) + ' "ULSCH in error in round" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if ulschAllocateCCEerror > 0:
-			statMsg = nodeB_prefix + 'NB showed ' + str(ulschAllocateCCEerror) + ' "eNB_dlsch_ulsch_scheduler(); ERROR ALLOCATING CCEs" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if uplinkSegmentsAborted > 0:
-			statMsg = nodeB_prefix + 'NB showed ' + str(uplinkSegmentsAborted) + ' "uplink segment error 0/2, aborted * segments" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if dropNotEnoughRBs > 0:
-			statMsg = 'eNB showed ' + str(dropNotEnoughRBs) + ' "dropping, not enough RBs" message(s)'
-			logging.debug('\u001B[1;30;43m ' + statMsg + ' \u001B[0m')
-			htmleNBFailureMsg += statMsg + '\n'
-		if rrcSetupComplete > 0:
-			rrcMsg = nodeB_prefix + 'NB completed ' + str(rrcSetupComplete) + ' RRC Connection Setup(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-			rrcMsg = ' -- ' + str(rrcSetupComplete) + ' were completed'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if rrcReleaseRequest > 0:
-			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReleaseRequest) + ' RRC Connection Release(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if rrcReconfigRequest > 0 or rrcReconfigComplete > 0:
-			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReconfigRequest) + ' RRC Connection Reconfiguration(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-			rrcMsg = ' -- ' + str(rrcReconfigComplete) + ' were completed'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if rrcReestablishRequest > 0 or rrcReestablishComplete > 0 or rrcReestablishReject > 0:
-			rrcMsg = nodeB_prefix + 'NB requested ' + str(rrcReestablishRequest) + ' RRC Connection Reestablishment(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-			rrcMsg = ' -- ' + str(rrcReestablishComplete) + ' were completed'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-			rrcMsg = ' -- ' + str(rrcReestablishReject) + ' were rejected'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if self.eNBmbmsEnables[0]:
-			if mbmsRequestMsg > 0:
-				rrcMsg = 'eNB requested ' + str(mbmsRequestMsg) + ' times the RLC for MBMS USER-PLANE'
-				logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-				htmleNBFailureMsg += rrcMsg + '\n'
-		if X2HO_inNbProcedures > 0:
-			rrcMsg = 'eNB completed ' + str(X2HO_inNbProcedures) + ' X2 Handover Connection procedure(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if X2HO_outNbProcedures > 0:
-			rrcMsg = 'eNB completed ' + str(X2HO_outNbProcedures) + ' X2 Handover Release procedure(s)'
-			logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rrcMsg + '\n'
-		if self.eNBOptions[0] != '':
-			res1 = re.search('drx_Config_present prSetup', self.eNBOptions[0])
-			if res1 is not None:
-				if cdrxActivationMessageCount > 0:
-					rrcMsg = 'eNB activated the CDRX Configuration for ' + str(cdrxActivationMessageCount) + ' time(s)'
-					logging.debug('\u001B[1;30;43m ' + rrcMsg + ' \u001B[0m')
-					htmleNBFailureMsg += rrcMsg + '\n'
-				else:
-					rrcMsg = 'eNB did NOT ACTIVATE the CDRX Configuration'
-					logging.debug('\u001B[1;37;43m ' + rrcMsg + ' \u001B[0m')
-					htmleNBFailureMsg += rrcMsg + '\n'
-		if rachCanceledProcedure > 0:
-			rachMsg = nodeB_prefix + 'NB cancelled ' + str(rachCanceledProcedure) + ' RA procedure(s)'
-			logging.debug('\u001B[1;30;43m ' + rachMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rachMsg + '\n'
-		if isRRU:
-			if isSlave:
-				if slaveReceivesFrameResyncCmd:
-					rruMsg = 'Slave RRU received the RRU_frame_resynch command from RAU'
-					logging.debug('\u001B[1;30;43m ' + rruMsg + ' \u001B[0m')
-					htmleNBFailureMsg += rruMsg + '\n'
-				else:
-					rruMsg = 'Slave RRU DID NOT receive the RRU_frame_resynch command from RAU'
-					logging.debug('\u001B[1;37;41m ' + rruMsg + ' \u001B[0m')
-					htmleNBFailureMsg += rruMsg + '\n'
-					global_status = CONST.ENB_PROCESS_SLAVE_RRU_NOT_SYNCED
-		if foundSegFault:
-			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with a Segmentation Fault! \u001B[0m')
-			global_status = CONST.ENB_PROCESS_SEG_FAULT
-		if foundAssertion:
-			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB ended with an assertion! \u001B[0m')
-			htmleNBFailureMsg += msgAssertion
-			global_status = CONST.ENB_PROCESS_ASSERTION
-		if foundRealTimeIssue:
-			logging.debug('\u001B[1;37;41m ' + nodeB_prefix + 'NB faced real time issues! \u001B[0m')
-			htmleNBFailureMsg += nodeB_prefix + 'NB faced real time issues! COUNT = '+ str(foundRealTimeIssue_cnt) +' lines\n'
-		if rlcDiscardBuffer > 0:
-			rlcMsg = nodeB_prefix + 'NB RLC discarded ' + str(rlcDiscardBuffer) + ' buffer(s)'
-			logging.debug('\u001B[1;37;41m ' + rlcMsg + ' \u001B[0m')
-			htmleNBFailureMsg += rlcMsg + '\n'
-			global_status = CONST.ENB_PROCESS_REALTIME_ISSUE
-		HTML.htmleNBFailureMsg=htmleNBFailureMsg
-		# Runtime statistics for console output and HTML
-		if runTime != '':
-			logging.debug(runTime)
-			logging.debug(userTime)
-			logging.debug(systemTime)
-			logging.debug(maxPhyMemUsage)
-			logging.debug(nbContextSwitches)
-			self.runtime_stats='<pre>'+runTime + '\n'+ userTime + '\n' + systemTime + '\n' + maxPhyMemUsage + '\n' + nbContextSwitches+'</pre>'
-		return global_status
+		logging.info(f"check against thresholds from {thresholds}")
+		success, datalog_rt_stats = cls_analysis.Analysis.analyze_rt_stats(thresholds, l1_file, mac_file)
+		HTML.CreateHtmlDataLogTable(datalog_rt_stats)
+		return success

ci-scripts/run_locally.sh

@@ -1,6 +1,5 @@
 #!/bin/bash
 
-set -e
 SHORT_COMMIT_SHA=$(git rev-parse --short=8 HEAD)
 COMMIT_SHA=$(git rev-parse HEAD)
 CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
@@ -13,6 +12,8 @@ if [ $# -eq 0 ]
     exit 1
 fi
 
+set -x
+
 # The script assumes you've build the following images:
 #
 # docker build . -f docker/Dockerfile.gNB.ubuntu -t oai-gnb
@@ -31,15 +32,15 @@ docker tag oai-nr-cuup oai-ci/oai-nr-cuup:develop-${SHORT_COMMIT_SHA}
 python3 main.py --mode=InitiateHtml --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \
     --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \
     --ranTargetBranch=NONE \
-    --XMLTestFile=xml_files/${TESTCASE} --local
+    --XMLTestFile=xml_files/${TESTCASE} --local --datefmt="%H:%M:%S"
 
 python3 main.py --mode=TesteNB --ranRepository=NONE --ranBranch=${CURRENT_BRANCH} \
     --ranCommitID=${COMMIT_SHA} --ranAllowMerge=false \
     --ranTargetBranch=NONE \
     --eNBSourceCodePath=${REPO_PATH} \
-    --XMLTestFile=${TESTCASE} --local
+    --XMLTestFile=${TESTCASE} --local --datefmt="%H:%M:%S"
 RET=$?
 
-python3 main.py --mode=FinalizeHtml --local
+python3 main.py --mode=FinalizeHtml --local --datefmt="%H:%M:%S"
 
 exit ${RET}

ci-scripts/scripts/docker-build-and-deploy-physims-cuda.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+function die() { echo $@; exit 1; }
+[ $# -ge 1 ] || die "usage: $0 <directory> [ctest-options]"
+
+IMAGE=ran-physimtests:ci-temp
+CONTAINER=ran-physimtests
+function cleanup-docker() {
+  docker stop ${CONTAINER}
+  docker rm ${CONTAINER}
+  docker rmi ${IMAGE}
+  docker volume prune --force
+}
+trap cleanup-docker EXIT
+
+set -x
+DIR=$1
+RESULT_DIR=${DIR}/
+shift
+CTEST_OPT=$@
+
+# build physims
+docker build --progress=plain --tag ${IMAGE} --file ${DIR}/ci-scripts/docker/Dockerfile.physim.cuda.ubuntu ${DIR} &>> ${RESULT_DIR}/physim_log.txt
+if [ $? -ne 0 ]; then
+  echo "build of physims failed"
+  exit 1
+fi
+
+# get a JSON description of all tests to run
+docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm --name ${CONTAINER} ${IMAGE} ctest ${CTEST_OPT} --show-only=json-v1 &> ${RESULT_DIR}/desc-tests.json
+JSON_RES=$?
+
+# run the actual tests: we don't suppy --rm as we have to copy the files
+# similar to unit tests, we can't mount the file where we write physims-5g-run.xml to
+# as it would write a file as root, but this script is run as a normal user
+docker run -a STDOUT --gpus all --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --name ${CONTAINER} ${IMAGE} ctest ${CTEST_OPT} --output-junit results-run.xml --test-output-size-passed 100000 --test-output-size-failed 100000 &>> ${RESULT_DIR}/physim_log.txt
+RUN_RES=$?
+docker cp ${CONTAINER}:/oai-ran/build/results-run.xml ${RESULT_DIR}/
+docker cp ${CONTAINER}:/oai-ran/build/Testing/Temporary/LastTestsFailed.log ${RESULT_DIR}/
+docker cp ${CONTAINER}:/oai-ran/build/Testing/Temporary/LastTest.log ${RESULT_DIR}/
+
+# if both were successful, return 0
+# TODO not sure
+#[[ $JSON_RES -eq 0 && $RUN_RES -eq 0 ]] && exit 0
+
+exit 0

ci-scripts/tests/README.md

@@ -10,12 +10,15 @@ directory, i.e., `ci-scripts/`, and run
 
 To run individual unit tests, start them like so:
 
+    python tests/analysis.py -v
     python tests/build.py -v
     python tests/cmd.py -v
     python tests/corenetwork.py -v
     python tests/deployment.py -v
     python tests/iperf-analysis.py -v
+    python tests/log-analysis.py -v
     python tests/ping-iperf.py -v
+    python tests/pull-clean-int-registry.py -v
 
 The logs will indicate if all tests passed. `tests/deployment.py` requires
 these images to be present:

ci-scripts/tests/analysis.py

@@ -0,0 +1,71 @@
+import sys
+import logging
+logging.basicConfig(
+	level=logging.DEBUG,
+	stream=sys.stdout,
+	format="[%(asctime)s] %(levelname)8s: %(message)s"
+)
+import os
+import unittest
+import yaml
+
+sys.path.append('./') # to find OAI imports below
+import cls_analysis
+import cls_oai_html
+import constants
+
+class TestAnalysis(unittest.TestCase):
+	def setUp(self):
+		self.html = cls_oai_html.HTMLManagement()
+		self.html.testCaseId = "000000"
+		self.maxDiff = 10000 # to compare long files below
+
+	def test_gnb_phytest(self):
+		rtsf = "datalog_rt_stats.100.2x2.yaml"
+		l1f = "tests/analysis/gnb_phytest.success.nrL1.log"
+		macf = "tests/analysis/gnb_phytest.success.nrMAC.log"
+		status, s = cls_analysis.Analysis.analyze_rt_stats(rtsf, l1f, macf)
+		self.assertTrue(status)
+		self.assertEqual(s['Title'], "Processing Time (us) from datalog_rt_stats.100.2x2.yaml")
+		self.assertEqual(s['ColNames'], ["Metric", "Average; Max; Count", "Average vs Reference Deviation (Reference Value; Acceptability Deviation Threshold)"])
+		with open(rtsf, 'r') as f:
+			rt_stats = yaml.load(f, Loader=yaml.FullLoader)
+		# check that analyze_rt_stats() picked up all Ref/DeviationThreshold values from yaml file, e.g.
+		# s['Ref']['L1 Tx processing'] == corresponding value in yaml
+		# s['DeviationThreshold']['L1 Tx processing'] == corresponding value in yaml
+		for k, v in rt_stats['Ref'].items():
+			self.assertEqual(s['Ref'][k], v)
+		for k, v in rt_stats['DeviationThreshold'].items():
+			self.assertEqual(s['DeviationThreshold'][k], v)
+		for k, v in s['Ref'].items():
+			self.assertEqual(v, rt_stats['Ref'][k])
+		for k, v in s['DeviationThreshold'].items():
+			self.assertEqual(v, rt_stats['DeviationThreshold'][k])
+		self.assertEqual(s['Data']['L1 Tx processing'], ['476', '872', '102933', '1.00'])
+		self.assertEqual(s['Data']['DLSCH encoding'], ['216', '544', '77201', '1.08'])
+		self.assertEqual(s['Data']['L1 Rx processing'], ['488', '868', '38598', '0.92'])
+		self.assertEqual(s['Data']['UL Indication'], ['1', '10', '38598', '0.76'])
+		self.assertEqual(s['Data']['Slot Indication'], ['18', '94', '128667', '1.38'])
+		self.assertEqual(s['Data']['PUSCH inner-receiver'], ['307', '530', '25730', '0.85'])
+		self.assertEqual(s['Data']['feprx'], ['169', '310', '38600', '1.12'])
+		self.assertEqual(s['Data']['feptx_ofdm (per port, half_slot)'], ['56', '88', '102933', '0.98'])
+		self.assertEqual(s['Data']['feptx_total'], ['148', '299', '102933', '1.00'])
+		# TODO this one is not found, the CI does not recover this??
+		#self.assertEqual(s['Data']['DL & UL scheduling timing'], ['17', '97', '159371', '1.14'])
+
+	def test_analze_services_no_service(self):
+		service_desc = {} # nothing to analyze
+		to_analyze = ["testserv"] # some service requested to be analyze that does not exist
+		status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze)
+		self.assertFalse(status)
+
+	# TODO this does not work: we don't evaluate return codes yet
+	#def test_analze_service_exit_nonnull(self):
+	#	service_desc = {}
+	#	service_desc["testserv"] = {'returncode': 129, 'logfile': 'tests/log-analysis/empty.log'}
+	#	to_analyze = ["testserv"] # some service requested to be analyze that does not exist
+	#	status = cls_analysis.AnalyzeServices(self.html, service_desc, to_analyze)
+	#	self.assertFalse(status)
+
+if __name__ == '__main__':
+	unittest.main()

ci-scripts/tests/analysis/gnb_phytest.success.nrL1.log

@@ -0,0 +1,37 @@
+Blacklisted PRBs 0/273
+-5.-6.-4.-4.-4.-6.-6.-6.-8.-6.-6.-4.-4.-4.-3.-6.-6.-11.-8.-8.-5.-4.-5.-4.-5.
+-6.-11.-11.-8.-8.-5.-5.-4.-4.-6.-8.-11.-11.-8.-8.-6.-5.-6.-6.-8.-11.-11.-11.-8.-6.
+-6.-6.-6.-11.-11.-11.-11.-11.-11.-6.-6.-6.-8.-3.-11.-11.-11.-8.-8.-6.-6.-6.-11.-11.-11.
+-11.-8.-6.-5.-6.-6.-8.-11.-11.-11.-6.-6.-5.-4.-4.-5.-8.-8.-8.-6.-5.-4.-3.-3.-4.
+-4.-6.-6.-5.-3.-3.-2.-2.-2.-3.-3.-4.-5.-5.-3.-2.-1.-1.-2.-5.-4.-4.-4.-3.-2.
+ 0.-1.-1.-1.-2.-2.-3.-2.-1. 0. 2.-11.13. 8. 7. 6. 5. 0. 0.-1. 0. 0. 0.-1.-3.
+-3.-3.-3.-1.-1.-1.-1.-1.-1.-2.-3.-1.-1.-1.-1.-2.-1.-3.-4.-5.-8.-6.-3.-3.-3.
+-3.-3.-5.-8.-8.-8.-6.-4.-4.-3.-4.-6.-6.-8.-8.-8.-6.-5.-6.-5.-6.-8.-11.-11.-11.
+-8.-5.-5.-8.-6.-6.-8.-11.-11.-11.-8.-6.-6.-6.-8.-11.-11.-11.-8.-8.-6.-6.-5.-5.-8.
+-11.-11.-8.-11.-6.-5.-5.-5.-5.-6.-11.-11.-8.-6.-3.-4.-5.-5.-6.-8.-6.-6.-5.-4.-4.
+-4.-5.-6.-6.-11.-8.-6.-4.-3.-4.-3.-4.-5.-6.-8.-6.-5.-4.-3.-4.-3.-6.-5.
+max_IO = 24 (137), min_I0 = 0 (17), avg_I0 = 11 dB(8.8.)
+PRACH I0 = 0.0 dB
+DLSCH RNTI 1234: current_Qm 6, current_RI 2, total_bytes TX -1130807110
+ULSCH RNTI 1234, 389: ulsch_power[0] 41,5 ulsch_noise_power[0] 7.8, sync_pos 0
+                  ulsch_power[1] 26.7, ulsch_noise_power[1] 7.8
+                 round_trials 6434(1.0e+00):6432(1.0e+00):6432(1.0e+00):6432, DTX 0, current_Qm 6, current_RI 1, total_bytes RX/SCHED 0/385630560
+                L1 Tx job:          401.013 us;          128667;         895.809 us;
+                L1 Rx job:          183.883 us;          128667;         980.814 us;
+         L1 Tx processing:          476.247 us;          102933;         872.099 us;
+           DLSCH encoding:          216.371 us;           77201;         544.272 us;
+         DLSCH scrambling:            5.524 us;           77201;          19.192 us;
+         DLSCH modulation:           21.137 us;           77201;          36.428 us;
+   DLSCH resource mapping:            1.127 us;         1003613;          12.027 us;
+          DLSCH precoding:            3.667 us;         1003613;          17.411 us;
+         L1 Rx processing:          488.478 us;           38598;         867.672 us;
+UL segment deinterleaving:            5.063 us;          488870;          27.005 us;
+ UL segment rate recovery:           11.554 us;          488870;         517.998 us;
+     UL segments decoding:          134.035 us;          488870;         759.164 us;
+            UL Indication:            0.763 us;           38598;           9.588 us;
+          Slot Indication:           17.891 us;          128667;          94.446 us;
+     PUSCH inner-receiver:          307.193 us;           25730;         529.580 us;
+                    feprx:          169.305 us;           38600;         310.489 us;
+feptx_ofdm (per port, half_slot):           56.093 us;          102933;          88.125 us;
+              feptx_total:          147.889 us;          102933;         299.307 us;
+

ci-scripts/tests/deployment.py

@@ -16,7 +16,6 @@ import cls_oai_html
 import cls_oaicitest
 import cls_containerize
 from cls_ci_helper import TestCaseCtx
-import ran
 import cls_cmd
 
 class TestDeploymentMethods(unittest.TestCase):
@@ -39,7 +38,6 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.html.testCaseId = "000000"
 		self.ci = cls_oaicitest.OaiCiTest()
 		self.cont = cls_containerize.Containerize()
-		self.ran = ran.RANManagement()
 		self.cont.yamlPath = ''
 		self.cont.ranAllowMerge = True
 		self.cont.ranBranch = ''
@@ -56,7 +54,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.yamlPath = 'tests/simple-dep/'
 		self.cont.deploymentTag = "noble"
 		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
-		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -79,7 +77,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		stopB = self.cont.StopObject(self.ctx, self.node, self.html)
 		# should not undeploy anything (everything already stopped)
 		self.cont.services = None
-		undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		undeployAll = self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertTrue(deploy)
 		self.assertFalse(stopC)
 		self.assertTrue(stopA)
@@ -92,7 +90,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		old = self.cont.yamlPath
 		self.cont.yamlPath = 'tests/simple-fail/'
 		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
-		self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -101,7 +99,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		old = self.cont.yamlPath
 		self.cont.yamlPath = 'tests/simple-fail-2svc/'
 		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
-		self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertFalse(deploy)
 		self.cont.yamlPath = old
 
@@ -110,7 +108,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.services = "oai-gnb"
 		self.cont.deploymentTag = 'develop-12345678'
 		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
-		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -119,7 +117,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		self.cont.services = "oai-gnb oai-nr-ue"
 		self.cont.deploymentTag = 'develop-12345678'
 		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
-		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertTrue(deploy)
 		self.assertTrue(undeploy)
 
@@ -130,7 +128,7 @@ class TestDeploymentMethods(unittest.TestCase):
 		deploy1 = self.cont.DeployObject(self.ctx, self.node, self.html)
 		self.cont.services = "oai-nr-ue"
 		deploy2 = self.cont.DeployObject(self.ctx, self.node, self.html)
-		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, self.ran)
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, [])
 		self.assertTrue(deploy1)
 		self.assertTrue(deploy2)
 		self.assertTrue(undeploy)
@@ -145,5 +143,37 @@ class TestDeploymentMethods(unittest.TestCase):
 			cmd.run(f"rm -rf {self.cont.eNBSourceCodePath}")
 		self.assertTrue(ws)
 
+	def test_undeploy_loganalysis(self):
+		self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora'
+		self.cont.services = "oai-gnb"
+		self.cont.deploymentTag = 'develop-12345678'
+		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
+		analyze = ["oai-gnb"]
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze)
+		self.assertTrue(deploy)
+		self.assertTrue(undeploy)
+
+	def test_undeploy_multi_loganalysis(self):
+		self.cont.yamlPath = 'yaml_files/5g_rfsimulator_tdd_dora'
+		self.cont.services = "oai-gnb"
+		self.cont.deploymentTag = 'develop-12345678'
+		deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
+		analyze = ["oai-gnb", "oai-gnb=ContainsString=NOTFOUND"]
+		undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, analyze)
+		self.assertTrue(deploy)
+		self.assertFalse(undeploy)
+
+	# TODO this does not work: we don't evaluate return codes yet
+	#def test_undeploy_rc_nonnull(self):
+	#	# fails reliably
+	#	old = self.cont.yamlPath
+	#	self.cont.yamlPath = 'tests/simple-fail/'
+	#	deploy = self.cont.DeployObject(self.ctx, self.node, self.html)
+	#	# should be false because wrong exit code
+	#	undeploy = self.cont.UndeployObject(self.ctx, self.node, self.html, ["test"])
+	#	self.assertFalse(deploy)
+	#	self.assertFalse(undeploy)
+	#	self.cont.yamlPath = old
+
 if __name__ == '__main__':
 	unittest.main()

ci-scripts/tests/log-analysis.py

@@ -0,0 +1,131 @@
+import sys
+import logging
+logging.basicConfig(
+	level=logging.DEBUG,
+	stream=sys.stdout,
+	format="[%(asctime)s] %(levelname)8s: %(message)s"
+)
+import os
+import unittest
+
+sys.path.append('./') # to find OAI imports below
+import cls_loganalysis
+
+class TestLogAnalysis(unittest.TestCase):
+    def setUp(self):
+        self.maxDiff = None
+
+    #def test_no_file(self):
+    #    f = "/total/fantasy/path/file.log"
+    #    result, _ = cls_loganalysis.Default.run(f, None)
+    #    self.assertFalse(result)
+
+    def test_no_error(self):
+        f = "tests/log-analysis/empty.log"
+        result, _ = cls_loganalysis.Default.run(f, None)
+        self.assertTrue(result)
+
+    def test_retx_check_good(self):
+        f = "tests/log-analysis/retx-check-good.log"
+        opt = "dl=1,0,0,0;ul=1,0,0,0"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3
+UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 51.0 dB CCE fail 0"""
+        self.assertTrue(result)
+        self.assertEqual(log, expected)
+
+    def test_retx_check_good_single_val(self):
+        f = "tests/log-analysis/retx-check-good.log"
+        opt = "1,0,0,0"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3
+UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 51.0 dB CCE fail 0"""
+        self.assertTrue(result)
+        self.assertEqual(log, expected)
+
+    def test_retx_check_good_ul_only(self):
+        f = "tests/log-analysis/retx-check-good.log"
+        opt = "ul=1,0,0,0"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3
+UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 51.0 dB CCE fail 0"""
+        self.assertTrue(result)
+        self.assertEqual(log, expected)
+
+    def test_retx_check_bad(self):
+        f = "tests/log-analysis/retx-check-bad.log"
+        opt = "dl=10,100,100,100;ul=10,100,100,100"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0
+UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 34.0 dB CCE fail 0
+UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64
+UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104  SNR 38.0 dB CCE fail 0
+!!! Failure: UE bf9b: retx rounds DL [1] UL [1] exceeded threshold
+UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0
+UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5  SNR 29.0 dB CCE fail 0"""
+        self.assertFalse(result)
+        self.assertEqual(log, expected)
+
+    def test_retx_check_dl_only_bad(self):
+        f = "tests/log-analysis/retx-check-bad.log"
+        opt = "dl=10,80,100,100"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0
+UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 34.0 dB CCE fail 0
+UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64
+UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104  SNR 38.0 dB CCE fail 0
+!!! Failure: UE bf9b: retx rounds DL [1, 2] exceeded threshold
+UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0
+UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5  SNR 29.0 dB CCE fail 0"""
+        self.assertFalse(result)
+        self.assertEqual(log, expected)
+
+    def test_retx_check_ul_bad(self):
+        f = "tests/log-analysis/retx-check-bad.log"
+        opt = "dl=100,100,100,100;ul=10,100,100,100"
+        result, log = cls_loganalysis.RetxCheck.run(f, opt)
+        expected = """UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0
+UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 34.0 dB CCE fail 0
+UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64
+UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104  SNR 38.0 dB CCE fail 0
+!!! Failure: UE bf9b: retx rounds UL [1] exceeded threshold
+UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0
+UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5  SNR 29.0 dB CCE fail 0"""
+        self.assertFalse(result)
+        self.assertEqual(log, expected)
+
+    def test_last_line_contains_false(self):
+        f = "tests/log-analysis/arbitrary.log"
+        opt = "Bye."
+        result, log = cls_loganalysis.LastLineContains.run(f, opt)
+        expected = f"could not find '{opt}' in last line"
+        self.assertFalse(result)
+        self.assertEqual(log, expected)
+
+    def test_last_line_contains_false(self):
+        f = "tests/log-analysis/arbitrary.log"
+        opt = "got sync"
+        result, log = cls_loganalysis.LastLineContains.run(f, opt)
+        expected = ""
+        self.assertTrue(result)
+        self.assertEqual(log, expected)
+
+    def test_ends_no_bye(self):
+        f = "tests/log-analysis/arbitrary.log"
+        opt = None
+        result, log = cls_loganalysis.EndsWithBye.run(f, opt)
+        expected = "No Bye. message found, did not stop properly"
+        self.assertFalse(result)
+        self.assertEqual(log, expected)
+
+    def test_ends_with_bye(self):
+        f = "tests/log-analysis/with-bye.log"
+        opt = None
+        result, log = cls_loganalysis.EndsWithBye.run(f, opt)
+        expected = ""
+        self.assertTrue(result)
+        self.assertEqual(log, expected)
+
+
+if __name__ == '__main__':
+	unittest.main()

ci-scripts/tests/log-analysis/arbitrary.log

@@ -0,0 +1,9 @@
+[UTIL]   threadCreate() for L1_rx_thread: creating thread (no affinity, default priority)
+[UTIL]   threadCreate() for L1_tx_thread: creating thread (no affinity, default priority)
+[UTIL]   threadCreate() for L1_stats: creating thread (no affinity, default priority)
+TYPE <CTRL-C> TO TERMINATE
+[PHY]    got sync (ru_thread)
+[PHY]    RU 0 rf device ready
+[PHY]    RU 0 RF started cpu_meas_enabled 0
+[HW]     No connected device, generating void samples...
+[PHY]    got sync (L1_stats_thread)

ci-scripts/tests/log-analysis/retx-check-bad.log

@@ -0,0 +1,54 @@
+9185541.336575 [NR_RRC] I UE 1 Logical Channel DL-DCCH, Generate SecurityModeCommand (bytes 3)
+9185541.356794 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received Security Mode Complete
+9185541.357264 [NR_RRC] I UE 1: Logical Channel DL-DCCH, Generate NR UECapabilityEnquiry (bytes 8, xid 2)
+9185541.373199 [NR_RRC] A [UL] (cellID bc614e, UE ID 1 RNTI a491) Received UE capabilities
+9185541.373330 [NR_RRC] I Send message to ngap: NGAP_UE_CAPABILITIES_IND
+9185541.373356 [NR_RRC] A [DL] (cellID bc614e, UE ID 1 RNTI a491) Send DL Information Transfer [53 bytes]
+9185541.373368 [NR_RRC] I Send message to sctp: NGAP_InitialContextSetupResponse
+9185541.390034 [NR_MAC] I Frame.Slot 384.0
+UE RNTI a491 CU-UE-ID 1 in-sync PH 59 dB PCMAX 21 dBm, average RSRP -101 (4 meas)
+UE a491: CQI 2, RI 2, PMI (0,0)
+UE a491: dlsch_rounds 13/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.11000 MCS (0) 0 CCE fail 0
+UE a491: ulsch_rounds 23/1/0/0, ulsch_errors 0, ulsch_DTX 1, BLER 0.08447 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 34.0 dB CCE fail 0
+UE a491: MAC:    TX            367 RX           3664 bytes
+UE a491: LCID 1: TX            213 RX           1649 bytes
+
+9185559.310044 [NR_MAC] I Frame.Slot 128.0
+UE RNTI bf9b CU-UE-ID 1 in-sync PH 48 dB PCMAX 21 dBm
+UE bf9b: dlsch_rounds 370/368/368/368, dlsch_errors 368, pucch0_DTX 1377, BLER 1.00000 MCS (0) 0 CCE fail 64
+UE bf9b: ulsch_rounds 150/148/147/147, ulsch_errors 147, ulsch_DTX 589, BLER 1.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 104  SNR 38.0 dB CCE fail 0
+UE bf9b: MAC:    TX          10145 RX             82 bytes
+UE bf9b: LCID 1: TX           5283 RX             53 bytes
+
+9185582.350069 [NR_MAC] I Frame.Slot 384.0
+UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -83 (16 meas)
+UE 257a: CQI 15, RI 2, PMI (0,1)
+UE 257a: UL-RI 2, TPMI 0
+UE 257a: dlsch_rounds 153/1/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00000 MCS (1) 0 CCE fail 0
+UE 257a: ulsch_rounds 562/2/1/0, ulsch_errors 0, ulsch_DTX 3, BLER 0.00000 MCS (1) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 29.5 dB CCE fail 0
+UE 257a: MAC:    TX          13416 RX          26519 bytes
+UE 257a: LCID 1: TX            523 RX           2046 bytes
+UE 257a: LCID 2: TX              0 RX              0 bytes
+UE 257a: LCID 4: TX           9800 RX           9945 bytes
+
+9185687.310202 [NR_MAC] I Frame.Slot 640.0
+UE RNTI 257a CU-UE-ID 2 in-sync PH 61 dB PCMAX 21 dBm, average RSRP -83 (16 meas)
+UE 257a: CQI 15, RI 2, PMI (0,1)
+UE 257a: UL-RI 2, TPMI 0
+UE 257a: dlsch_rounds 64992/129/13/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00242 MCS (1) 27 CCE fail 0
+UE 257a: ulsch_rounds 39328/2610/8/0, ulsch_errors 0, ulsch_DTX 14, BLER 0.11139 MCS (1) 9 (Qm 4 deltaMCS 0 dB) NPRB 273  SNR 25.5 dB CCE fail 0
+UE 257a: MAC:    TX     3344707955 RX      818603547 bytes
+UE 257a: LCID 1: TX            832 RX           4312 bytes
+UE 257a: LCID 2: TX              0 RX              0 bytes
+UE 257a: LCID 4: TX     3322610250 RX      815642955 bytes
+
+9185692.430254 [NR_MAC] I Frame.Slot 128.0
+UE RNTI 257a CU-UE-ID 2 in-sync PH 48 dB PCMAX 21 dBm, average RSRP -84 (16 meas)
+UE 257a: CQI 15, RI 2, PMI (0,1)
+UE 257a: UL-RI 2, TPMI 0
+UE 257a: dlsch_rounds 68071/142/14/0, dlsch_errors 0, pucch0_DTX 10, BLER 0.00427 MCS (1) 25 CCE fail 0
+UE 257a: ulsch_rounds 41777/2863/10/0, ulsch_errors 0, ulsch_DTX 17, BLER 0.11456 MCS (1) 10 (Qm 4 deltaMCS 0 dB) NPRB 5  SNR 29.0 dB CCE fail 0
+UE 257a: MAC:    TX     3424886787 RX      879242724 bytes
+UE 257a: LCID 1: TX            847 RX           4422 bytes
+UE 257a: LCID 2: TX              0 RX              0 bytes
+UE 257a: LCID 4: TX     3401067134 RX      875911216 bytes

ci-scripts/tests/log-analysis/retx-check-good.log

@@ -0,0 +1,8 @@
+7799905.793901 [NR_MAC] I Frame.Slot 256.0
+UE RNTI f40a CU-UE-ID 2 in-sync PH 48 dB PCMAX 20 dBm, average RSRP -44 (16 meas)
+UE f40a: dlsch_rounds 86/0/0/0, dlsch_errors 0, pucch0_DTX 0, BLER 0.00052 MCS (0) 0 CCE fail 3
+UE f40a: ulsch_rounds 518/0/0/0, ulsch_errors 0, ulsch_DTX 0, BLER 0.00000 MCS (0) 0 (Qm 2 deltaMCS 0 dB) NPRB 5  SNR 51.0 dB CCE fail 0
+UE f40a: MAC:    TX           3836 RX          13684 bytes
+UE f40a: LCID 1: TX            530 RX            281 bytes
+UE f40a: LCID 2: TX              3 RX             35 bytes
+UE f40a: LCID 4: TX           1886 RX           2100 bytes

ci-scripts/tests/log-analysis/with-bye.log

@@ -0,0 +1,3 @@
+Some other logs
+Bye.
+process exited.

ci-scripts/xml_class_list.yml

@@ -32,3 +32,5 @@
   - Custom_Command
   - Custom_Script
   - Create_Workspace
+  - AnalyzeRTStats
+  - AnalyzeRTStats_Object

ci-scripts/xml_files/container_4g_l2sim_tdd.xml

@@ -129,6 +129,9 @@
     <node>localhost</node>
     <desc>Undeploy all OAI 4G stack</desc>
     <yaml_path>ci-scripts/yaml_files/4g_l2sim_fdd</yaml_path>
+    <analysis>
+      <services>oai_enb=EndsWithBye oai_ue1</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz.xml

@@ -147,6 +147,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_fdd_05MHz_noS1.xml

@@ -118,6 +118,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_fdd_05MHz_noS1</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_fdd_10MHz.xml

@@ -147,6 +147,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_fdd_10MHz</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_fdd_20MHz.xml

@@ -147,6 +147,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_fdd_20MHz</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_fembms.xml

@@ -84,6 +84,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_fembms</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_mbms.xml

@@ -84,6 +84,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_mbms</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_4g_rfsim_tdd_05MHz.xml

@@ -146,6 +146,9 @@
     <desc>Undeploy all OAI 4G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/4g_rfsimulator_tdd_05MHz</yaml_path>
+    <analysis>
+      <services>oai_enb0=EndsWithBye oai_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_e1_rfsim.xml

@@ -86,8 +86,9 @@
     <node>localhost</node>
     <desc>Undeploy all OAI 5G stack</desc>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_e1</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-cucp=EndsWithBye oai-cuup=EndsWithBye oai-cuup2=EndsWithBye oai-cuup3=EndsWithBye oai-du=RetxCheck=1,0,0,0 oai-du=EndsWithBye oai-du2=RetxCheck=1,0,0,0 oai-du2=EndsWithBye oai-du3=RetxCheck=1,0,0,0 oai-du3=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_f1_rfsim.xml

@@ -311,8 +311,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_f1_rfsimulator</yaml_path>
-    <d_retx_th>30,100,100,100</d_retx_th>
-    <u_retx_th>30,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-cu=EndsWithBye oai-du-pci0=RetxCheck=30,100,100,100 oai-du-pci0=EndsWithBye oai-du-pci1=RetxCheck=30,100,100,100 oai-du-pci1=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_fdd_rfsim.xml

@@ -113,8 +113,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_fdd_rfsimulator</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_flexric_rfsim.xml

@@ -197,6 +197,14 @@
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_flexric</yaml_path>
     <always_exec>true</always_exec>
+    <analysis>
+      <service>nearRT-RIC=LastLineContains=Removing E2 Node</service>
+      <service>xapp-rc-moni=LastLineContains=Test xApp run SUCCESSFULLY</service>
+      <service>xapp-kpm-moni=LastLineContains=Test xApp run SUCCESSFULLY</service>
+      <service>xapp-kpm-rc=LastLineContains=Test xApp run SUCCESSFULLY</service>
+      <service>xapp-gtp-mac-rlc-pdcp-moni=LastLineContains=Test xApp run SUCCESSFULLY</service>
+      <services>oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim.xml

@@ -243,8 +243,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_24prb.xml

@@ -257,6 +257,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_24prb</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_2x2.xml

@@ -113,8 +113,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_2x2</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_fdd_phytest.xml

@@ -113,8 +113,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_fdd_phytest</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_fr2_66prb.xml

@@ -89,8 +89,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_fr2_66prb</yaml_path>
-    <d_retx_th>10,0,0,0</d_retx_th>
-    <u_retx_th>10,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=10,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_multiue.xml

@@ -159,10 +159,11 @@
     <class>Undeploy_Object</class>
     <always_exec>true</always_exec>
     <desc>Undeploy all OAI 5G stack</desc>
-    <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_multiue</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
     <node>localhost</node>
+    <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_multiue</yaml_path>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_n2_ho.xml

@@ -227,8 +227,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_n2_ho</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-cucp-0=EndsWithBye oai-cuup-0=EndsWithBye oai-du-0=RetxCheck=10,100,100,100 oai-du-0=EndsWithBye oai-nr-ue oai-cucp-1=EndsWithBye oai-cuup-1=EndsWithBye oai-du-1=RetxCheck=10,100,100,100 oai-du-1=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_ntn_geo.xml

@@ -138,6 +138,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_ntn_geo</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_ntn_leo.xml

@@ -138,6 +138,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_ntn_leo</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_prs.xml

@@ -74,8 +74,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_prs</yaml_path>
-    <d_retx_th>30,100,100,100</d_retx_th>
-    <u_retx_th>30,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=30,100,100,100 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_sidelink.xml

@@ -60,21 +60,16 @@
     <idle_sleep_time_in_sec>20</idle_sleep_time_in_sec>
   </testCase>
 
-  <testCase>
-    <class>Custom_Command</class>
-    <desc>Check that UE synched</desc>
-    <node>localhost</node>
-    <command>docker logs rfsim5g-oai-nr-ue-2 | grep -m 1 "PSBCH RX:OK"</command>
-  </testCase>
-
   <testCase>
     <class>Undeploy_Object</class>
     <always_exec>true</always_exec>
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_sidelink</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-nr-ue-1</services>
+      <service>oai-nr-ue-2=ContainsString=PSBCH RX:OK</service>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_simple.xml

@@ -80,8 +80,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue oai-nr-ue2 oai-nr-ue3 oai-nr-ue4 oai-nr-ue5 oai-nr-ue6 oai-nr-ue7 oai-nr-ue8 oai-nr-ue9 oai-nr-ue10</services>
+    </analysis>
   </testCase>
 
 </testCaseList>

ci-scripts/xml_files/container_5g_rfsim_tdd_dora.xml

@@ -95,8 +95,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_tdd_dora</yaml_path>
-    <d_retx_th>0,0,0,0</d_retx_th>
-    <u_retx_th>0,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=0,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_rfsim_u0_25prb.xml

@@ -174,8 +174,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_rfsimulator_u0_25prb</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-vnf=RetxCheck=0,0,0,0 oai-vnf=EndsWithBye oai-pnf=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_vrtsim_chanmod.xml

@@ -113,8 +113,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_vrtsim_chanmod</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_vrtsim_chanmod_gh.xml

@@ -114,8 +114,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_vrtsim_chanmod</yaml_path>
-    <d_retx_th>1,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_5g_vrtsim_cirdb.xml

@@ -114,8 +114,9 @@
     <desc>Undeploy all OAI 5G stack</desc>
     <node>localhost</node>
     <yaml_path>ci-scripts/yaml_files/5g_vrtsim_cirdb</yaml_path>
-    <d_retx_th>10,0,0,0</d_retx_th>
-    <u_retx_th>1,0,0,0</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=dl=10,0,0,0;ul=1,0,0,0 oai-gnb=EndsWithBye oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1.xml

@@ -183,6 +183,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_if4_5.xml

@@ -165,6 +165,9 @@
     <desc>Undeploy RCC/RRU</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_05Mhz_if4.5</yaml_path>
+    <analysis>
+      <services>rru_fdd=EndsWithBye rcc_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_05Mhz_tm1_no_rrc_activity.xml

@@ -156,6 +156,9 @@ Replaces xml_files/enb_usrp210_band7_test_05mhz_tm1_rrc_inactivity_no_flexran.xm
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_05Mhz_tm1_no_rrc_activity</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1.xml

@@ -168,6 +168,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_cdrx.xml

@@ -116,6 +116,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_cdrx</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_10Mhz_tm1_oaiue.xml

@@ -133,6 +133,9 @@
     <desc>Undeploy LTE-UE</desc>
     <node>carabe</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_10Mhz_oai_ue_magma</yaml_path>
+    <analysis>
+      <services>lte_ue0</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -147,6 +150,9 @@
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_10Mhz_tm1_magma</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_fdd_20Mhz_tm1.xml

@@ -140,6 +140,9 @@ Replaces xml_files/enb_usrp210_band7_test_10mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>hutch</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_fdd_20Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1.xml

@@ -194,6 +194,9 @@ Replaces xml_files/enb_usrp210_band40_test_05mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>starsky</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_tdd_05Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_tdd_05Mhz_tm1_if4_5.xml

@@ -174,6 +174,9 @@
     <desc>Undeploy RRU/RCC</desc>
     <node>starsky</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_tdd_05Mhz_if4.5</yaml_path>
+    <analysis>
+      <services>rru_tdd=EndsWithBye rcc_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_tdd_10Mhz_tm1.xml

@@ -187,6 +187,9 @@ Replaces xml_files/enb_usrp210_band40_test_10mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>starsky</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_tdd_10Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1.xml

@@ -133,6 +133,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1.xml
     <desc>Undeploy eNB</desc>
     <node>starsky</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1</yaml_path>
+    <analysis>
+      <services>enb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_b200_tdd_20Mhz_tm1_default_scheduler.xml

@@ -189,6 +189,9 @@ Replaces xml_files/enb_usrp210_band40_test_20mhz_tm1_default_scheduler.xml
     <desc>Undeploy eNB</desc>
     <node>starsky</node>
     <yaml_path>ci-scripts/yaml_files/lte_b200_tdd_20Mhz_tm1_default_scheduler</yaml_path>
+    <analysis>
+      <services>enb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm1.xml

@@ -125,6 +125,9 @@
     <desc>Undeploy gNB</desc>
     <node>obelix</node>
     <yaml_path>ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm1</yaml_path>
+    <analysis>
+      <services>oai-enb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_lte_n3xx_tdd_2x2_tm2.xml

@@ -125,6 +125,9 @@
     <desc>Undeploy gNB</desc>
     <node>obelix</node>
     <yaml_path>ci-scripts/yaml_files/lte_n3xx_tdd_2x2_tm2</yaml_path>
+    <analysis>
+      <services>oai-enb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_nsa_b200_quectel.xml

@@ -192,8 +192,9 @@
     <desc>Undeploy gNB</desc>
     <node>ofqot</node>
     <yaml_path>ci-scripts/yaml_files/nsa_b200_gnb</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -202,6 +203,9 @@
     <desc>Undeploy eNB</desc>
     <node>nepes</node>
     <yaml_path>ci-scripts/yaml_files/nsa_b200_enb</yaml_path>
+    <analysis>
+      <services>enb_mono_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_aerial_quectel.xml

@@ -132,8 +132,9 @@
     <desc>Undeploy gNB</desc>
     <node>gracehopper1-oai</node>
     <yaml_path>ci-scripts/yaml_files/sa_gnb_aerial</yaml_path>
-    <d_retx_th>20,100,100,100</d_retx_th>
-    <u_retx_th>20,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_aerial_quectel_ul_heavy.xml

@@ -121,8 +121,9 @@
     <desc>Undeploy gNB</desc>
     <node>gracehopper1-oai</node>
     <yaml_path>ci-scripts/yaml_files/sa_gnb_aerial_ul</yaml_path>
-    <d_retx_th>20,100,100,100</d_retx_th>
-    <u_retx_th>20,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb-aerial=RetxCheck=20,100,100,100 oai-gnb-aerial=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_aw2s_asue.xml

@@ -177,6 +177,9 @@
     <desc>Undeploy gNB</desc>
     <node>avra</node>
     <yaml_path>ci-scripts/yaml_files/sa_aw2s_gnb</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_aw2s_asue_2x2.xml

@@ -235,6 +235,9 @@
     <desc>Undeploy gNB</desc>
     <node>avra</node>
     <yaml_path>ci-scripts/yaml_files/sa_aw2s_2x2_gnb</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_b200_quectel.xml

@@ -229,8 +229,9 @@
     <desc>Undeploy gNB</desc>
     <node>ofqot</node>
     <yaml_path>ci-scripts/yaml_files/sa_b200_gnb</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_e1_b200_quectel.xml

@@ -255,8 +255,9 @@
     <desc>Undeploy CUCP/CU-UP/DU</desc>
     <node>ofqot</node>
     <yaml_path>ci-scripts/yaml_files/sa_e1_b200_gnb</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>gnb_cucp=EndsWithBye gnb_cuup=EndsWithBye gnb_du_tdd=RetxCheck=10,100,100,100 gnb_du_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_f1_b200_quectel.xml

@@ -263,8 +263,9 @@
     <desc>Undeploy CU-DU</desc>
     <node>ofqot</node>
     <yaml_path>ci-scripts/yaml_files/sa_f1_b200_gnb</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>gnb_cu=EndsWithBye gnb_du_fdd=RetxCheck=10,100,100,100 gnb_du_fdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_f1_ho_b210_quectel.xml

@@ -191,6 +191,9 @@
     <desc>Undeploy CU-DU0</desc>
     <node>groot</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_f1_b210_ho</yaml_path>
+    <analysis>
+      <services>oai-cu=EndsWithBye oai-du0=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -199,6 +202,9 @@
     <desc>Undeploy DU1</desc>
     <node>rocket</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_f1_b210_ho</yaml_path>
+    <analysis>
+      <services>oai-du1=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_fhi72_metanoia_up2_2x2.xml

@@ -127,6 +127,9 @@
     <desc>Undeploy gNB</desc>
     <node>cacofonix</node>
     <yaml_path>ci-scripts/yaml_files/sa_fhi_7.2_metanoia_2x2_gnb</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_fhi72_vvdn_up2.xml

@@ -138,6 +138,9 @@
     <desc>Undeploy gNB</desc>
     <node>cacofonix</node>
     <yaml_path>ci-scripts/yaml_files/sa_fhi_7.2_vvdn_gnb</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n2_ho_b210_quectel.xml

@@ -160,6 +160,9 @@
     <desc>Undeploy Monolithic gNB-PCI0</desc>
     <node>groot</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n2_ho_b210</yaml_path>
+    <analysis>
+      <services>oai-gnb-pci0=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -168,6 +171,9 @@
     <desc>Undeploy Monolithic gNB-PCI1</desc>
     <node>rocket</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n2_ho_b210</yaml_path>
+    <analysis>
+      <services>oai-gnb-pci1=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n310_2X2_100MHz_quectel.xml

@@ -123,8 +123,9 @@
     <desc>Undeploy gNB</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_100MHz</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=10,100,100,100 oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n310_2X2_60MHz_quectel.xml

@@ -123,8 +123,9 @@
     <desc>Undeploy gNB</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_2x2_60MHz</yaml_path>
-    <d_retx_th>15,100,100,100</d_retx_th>
-    <u_retx_th>20,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=dl=15,100,100,100;ul=20,100,100,100 oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n310_4X4_60MHz_quectel.xml

@@ -123,8 +123,9 @@
     <desc>Undeploy VNF-PNF</desc>
     <node>matix</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_4x4_60MHz</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-vnf=RetxCheck=10,100,100,100 oai-vnf=EndsWithBye oai-pnf=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n310_nrue.xml

@@ -125,6 +125,9 @@
     <desc>Undeploy nr UE</desc>
     <node>caracal</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_nrue</yaml_path>
+    <analysis>
+      <services>oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -133,8 +136,9 @@
     <desc>Undeploy gNB</desc>
     <node>avra</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_gnb</yaml_path>
-    <d_retx_th>20,100,100,100</d_retx_th>
-    <u_retx_th>20,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_n310_nrue_longrun.xml

@@ -205,6 +205,9 @@
     <desc>Undeploy nr UE</desc>
     <node>caracal</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_nrue</yaml_path>
+    <analysis>
+      <services>oai-nr-ue</services>
+    </analysis>
   </testCase>
 
   <testCase>
@@ -213,8 +216,9 @@
     <desc>Undeploy gNB</desc>
     <node>avra</node>
     <yaml_path>ci-scripts/yaml_files/5g_sa_n310_gnb</yaml_path>
-    <d_retx_th>20,100,100,100</d_retx_th>
-    <u_retx_th>20,100,100,100</u_retx_th>
+    <analysis>
+      <services>oai-gnb=RetxCheck=20,100,100,100 oai-gnb=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/container_sa_sc_b200_quectel.xml

@@ -183,8 +183,9 @@
     <desc>Undeploy gNB</desc>
     <node>ofqot</node>
     <yaml_path>ci-scripts/yaml_files/sa_sc_b200_gnb</yaml_path>
-    <d_retx_th>10,100,100,100</d_retx_th>
-    <u_retx_th>10,100,100,100</u_retx_th>
+    <analysis>
+      <services>gnb_mono_tdd=RetxCheck=10,100,100,100 gnb_mono_tdd=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/gnb_phytest_fhi7.2_docker.xml

@@ -0,0 +1,88 @@
+<!--
+
+ Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
+ contributor license agreements.  See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The OpenAirInterface Software Alliance licenses this file to You under
+ the OAI Public License, Version 1.1  (the "License"); you may not use this file
+ except in compliance with the License.
+ You may obtain a copy of the License at
+
+      http://www.openairinterface.org/?page_id=698
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+
+ For more information about the OpenAirInterface (OAI) Software Alliance:
+      contact@openairinterface.org
+
+-->
+<testCaseList>
+  <htmlTabRef>gNB-PHY-Test-100-4x4-FHI72</htmlTabRef>
+  <htmlTabName>Timing phytest 100 MHz 4x4 FHI7.2</htmlTabName>
+  <htmlTabIcon>tasks</htmlTabIcon>
+
+  <testCase>
+    <class>Custom_Script</class>
+    <desc>Setup sriov and network interfaces for dummy (no RU)</desc>
+    <node>bulbul</node>
+    <script>yaml_files/phytest_fhi72/setup_sriov_dummy.sh</script>
+  </testCase>
+
+  <testCase>
+    <class>Pull_Local_Registry</class>
+    <desc>Pull Images from Local Registry</desc>
+    <node>bulbul</node>
+    <images>oai-gnb-fhi72</images>
+  </testCase>
+
+  <testCase>
+    <class>Create_Workspace</class>
+    <desc>Create new Workspace</desc>
+    <node>bulbul</node>
+  </testCase>
+
+  <testCase>
+    <class>Deploy_Object</class>
+    <desc>Initialize gNB FHI7.2</desc>
+    <node>bulbul</node>
+    <yaml_path>ci-scripts/yaml_files/phytest_fhi72</yaml_path>
+    <services>oai-gnb</services>
+  </testCase>
+
+  <testCase>
+    <class>IdleSleep</class>
+    <desc>Sleep</desc>
+    <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
+  </testCase>
+
+  <testCase>
+    <class>AnalyzeRTStats_Object</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.100.4x4.fhi72.yaml</stats_cfg>
+    <node>bulbul</node>
+  </testCase>
+
+  <testCase>
+    <class>Undeploy_Object</class>
+    <node>bulbul</node>
+    <always_exec>true</always_exec>
+    <desc>Terminate gNB</desc>
+    <yaml_path>ci-scripts/yaml_files/phytest_fhi72</yaml_path>
+    <analysis>
+      <services>oai-gnb=EndsWithBye</services>
+    </analysis>
+  </testCase>
+
+  <testCase>
+    <class>Clean_Test_Server_Images</class>
+    <always_exec>true</always_exec>
+    <desc>Clean Test Images on Test Server</desc>
+    <node>bulbul</node>
+    <images>oai-gnb-fhi72</images>
+  </testCase>
+
+</testCaseList>

ci-scripts/xml_files/gnb_phytest_rfemulator_run_100_2x2.xml

@@ -37,7 +37,6 @@
     <desc>Initialize gNB USRP</desc>
     <node>caracal</node>
     <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --device.name rf_emulator --rf_emulator.noise_level_dBFS -36 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
-    <rt_stats_cfg>datalog_rt_stats.100.2x2.yaml</rt_stats_cfg>
     <air_interface>NR</air_interface>
     <cmd_prefix>numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0</cmd_prefix>
   </testCase>
@@ -48,12 +47,21 @@
     <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
   </testCase>
 
+  <testCase>
+    <class>AnalyzeRTStats</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.100.2x2.yaml</stats_cfg>
+    <node>caracal</node>
+  </testCase>
+
   <testCase>
     <class>Terminate_eNB</class>
     <node>caracal</node>
     <always_exec>true</always_exec>
     <desc>Terminate gNB</desc>
-    <air_interface>NR</air_interface>
+    <analysis>
+      <services>nr-softmodem=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/gnb_phytest_usrp_run.xml

@@ -37,9 +37,6 @@
     <desc>Initialize gNB USRP</desc>
     <node>caracal</node>
     <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band79.106prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 106 -t 23 -D 127 -m 28 -M 106 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
-    <rt_stats_cfg>datalog_rt_stats.default.yaml</rt_stats_cfg>
-    <air_interface>NR</air_interface>
-    <USRP_IPAddress>192.168.20.2</USRP_IPAddress>
     <cmd_prefix>numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0</cmd_prefix>
   </testCase>
 
@@ -49,12 +46,21 @@
     <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
   </testCase>
 
+  <testCase>
+    <class>AnalyzeRTStats</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.default.yaml</stats_cfg>
+    <node>caracal</node>
+  </testCase>
+
   <testCase>
     <class>Terminate_eNB</class>
     <always_exec>true</always_exec>
     <desc>Terminate gNB</desc>
     <node>caracal</node>
-    <air_interface>NR</air_interface>
+    <analysis>
+      <services>nr-softmodem=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/gnb_phytest_usrp_run_100_2x2.xml

@@ -37,9 +37,6 @@
     <desc>Initialize gNB USRP</desc>
     <node>caracal</node>
     <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band79.273prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 273 -t 23 -D 127 -m 23 -M 273 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
-    <rt_stats_cfg>datalog_rt_stats.100.2x2.yaml</rt_stats_cfg>
-    <air_interface>NR</air_interface>
-    <USRP_IPAddress>192.168.20.2</USRP_IPAddress>
     <cmd_prefix>numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0</cmd_prefix>
   </testCase>
 
@@ -49,12 +46,21 @@
     <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
   </testCase>
 
+  <testCase>
+    <class>AnalyzeRTStats</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.100.2x2.yaml</stats_cfg>
+    <node>caracal</node>
+  </testCase>
+
   <testCase>
     <class>Terminate_eNB</class>
     <always_exec>true</always_exec>
     <desc>Terminate gNB</desc>
     <node>caracal</node>
-    <air_interface>NR</air_interface>
+    <analysis>
+      <services>nr-softmodem=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/gnb_phytest_usrp_run_60.xml

@@ -37,9 +37,6 @@
     <desc>Initialize gNB USRP</desc>
     <node>caracal</node>
     <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
-    <rt_stats_cfg>datalog_rt_stats.1x1.60.yaml</rt_stats_cfg>
-    <air_interface>NR</air_interface>
-    <USRP_IPAddress>192.168.20.2</USRP_IPAddress>
     <cmd_prefix>numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0</cmd_prefix>
   </testCase>
 
@@ -49,12 +46,21 @@
     <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
   </testCase>
 
+  <testCase>
+    <class>AnalyzeRTStats</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.1x1.60.yaml</stats_cfg>
+    <node>caracal</node>
+  </testCase>
+
   <testCase>
     <class>Terminate_eNB</class>
     <always_exec>true</always_exec>
     <desc>Terminate gNB</desc>
     <node>caracal</node>
-    <air_interface>NR</air_interface>
+    <analysis>
+      <services>nr-softmodem=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>

ci-scripts/xml_files/gnb_phytest_usrp_run_60_2x2.xml

@@ -37,9 +37,6 @@
     <desc>Initialize gNB USRP</desc>
     <node>caracal</node>
     <Initialize_eNB_args>-O ci-scripts/conf_files/gnb.band79.162prb.usrpn300.phytest-dora.conf --phy-test --gNBs.[0].pdsch_AntennaPorts_XP 2 --RUs.[0].nb_tx 2 --RUs.[0].nb_rx 2 -q -U 768 -T 162 -t 23 -D 127 -m 23 -M 162 -l 2 --usrp-tx-thread-config 1 --log_config.global_log_options level,nocolor,time</Initialize_eNB_args>
-    <rt_stats_cfg>datalog_rt_stats.60.2x2.yaml</rt_stats_cfg>
-    <air_interface>NR</air_interface>
-    <USRP_IPAddress>192.168.20.2</USRP_IPAddress>
     <cmd_prefix>numactl --cpunodebind=netdev:enp94s0f0np0 --membind=netdev:enp94s0f0np0</cmd_prefix>
   </testCase>
 
@@ -49,12 +46,21 @@
     <idle_sleep_time_in_sec>60</idle_sleep_time_in_sec>
   </testCase>
 
+  <testCase>
+    <class>AnalyzeRTStats</class>
+    <desc>Analyze Real-Time Stats</desc>
+    <stats_cfg>datalog_rt_stats.60.2x2.yaml</stats_cfg>
+    <node>caracal</node>
+  </testCase>
+
   <testCase>
     <class>Terminate_eNB</class>
     <always_exec>true</always_exec>
     <desc>Terminate gNB</desc>
     <node>caracal</node>
-    <air_interface>NR</air_interface>
+    <analysis>
+      <services>nr-softmodem=EndsWithBye</services>
+    </analysis>
   </testCase>
 
   <testCase>