yigid/openairinterface5g
1
0
Fork 0
mirror of https://gitlab.eurecom.fr/oai/openairinterface5g.git synced 2026-07-13 04:30:28 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge remote-tracking branch 'origin/ci-refactor-core-handling' into integration_2025_w14_p1 (!3322)

Browse Source 
Refactor core network handling in CI, harmonize code

Prior to change set, there is a highly heterogeneous situation for core
network management in CI code: some log collection is done from groovy
files (when it could be done in python, which is easier for local
development), there are multiple commands to start a CN from XML files,
the code for starting a CN is always different (no one unified way to
start "any" CN), Ping/Iperf require some support in ci_infra.yaml, ...

To help with the above, refactor the handling of core networks in the CI
code as follows:

- define all CNs in ci_infra.yaml (basically how to deploy, undeploy,
  collect logs, and on which node)
- make one Python class that reads from ci_infra.yaml and that allows to
  start/stop previously defined CNs (and add unit tests...)
- make one XML command for start/stop of CNs, leveraging above class
- leverage the CNs defined in ci_infra.yaml consistently in
  Ping()/Iperf()
- delete old code

Furthermore, the following supporting changes/bugfixes are done:

- repair Iperf2_Unidir() and add unit test
- fix a number of invalid escape sequence errors (there are some
  remaining, which I will do soon)
- various bug fixes
- remove unused XML steps
This commit is contained in:
Robert Schmidt
2025-04-03 10:10:58 +02:00
parent 30c6c23dc8 ee3601df5d
commit 1449bdd410
108 changed files with 1070 additions and 3380 deletions
Download Patch File Download Diff File Expand all files Collapse all files

62
ci-scripts/cls_cluster.py
View File

@@ -43,9 +43,6 @@ NAMESPACE = "oaicicd-ran"
OCUrl = "https://api.oai.cs.eurecom.fr:6443"
OCRegistry = "default-route-openshift-image-registry.apps.oai.cs.eurecom.fr"
CI_OC_RAN_NAMESPACE = "oaicicd-ran"
CN_IMAGES = ["mysql", "oai-nrf", "oai-amf", "oai-smf", "oai-upf", "oai-ausf", "oai-udm", "oai-udr", "oai-traffic-server"]
CN_CONTAINERS = ["", "-c nrf", "-c amf", "-c smf", "-c upf", "-c ausf", "-c udm", "-c udr", ""]
def OC_login(cmd, ocUserName, ocPassword, ocProjectName):
if ocUserName == '' or ocPassword == '' or ocProjectName == '':
@@ -67,49 +64,6 @@ def OC_login(cmd, ocUserName, ocPassword, ocProjectName):
def OC_logout(cmd):
cmd.run(f'oc logout')
def OC_deploy_CN(cmd, ocUserName, ocPassword, ocNamespace, path):
logging.debug(f'OC OAI CN5G: Deploying OAI CN5G on Openshift Cluster: {ocNamespace}')
succeeded = OC_login(cmd, ocUserName, ocPassword, ocNamespace)
if not succeeded:
return False, CONST.OC_LOGIN_FAIL
cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait')
ret = cmd.run(f'helm install --wait oai5gcn {path}/ci-scripts/charts/oai-5g-basic/.')
if ret.returncode != 0:
logging.error('OC OAI CN5G: Deployment failed')
OC_logout(cmd)
return False, CONST.OC_PROJECT_FAIL
report = cmd.run('oc get pods')
OC_logout(cmd)
return True, report
def OC_undeploy_CN(cmd, ocUserName, ocPassword, ocNamespace, path):
logging.debug(f'OC OAI CN5G: Terminating CN on Openshift Cluster: {ocNamespace}')
succeeded = OC_login(cmd, ocUserName, ocPassword, ocNamespace)
if not succeeded:
return False, CONST.OC_LOGIN_FAIL
cmd.run(f'rm -Rf {path}/logs')
cmd.run(f'mkdir -p {path}/logs')
logging.debug('OC OAI CN5G: Collecting log files to workspace')
cmd.run(f'oc describe pod &> {path}/logs/describe-pods-post-test.log')
cmd.run(f'oc get pods.metrics.k8s &> {path}/logs/nf-resource-consumption.log')
for ii, ci in zip(CN_IMAGES, CN_CONTAINERS):
podName = cmd.run(f"oc get pods | grep {ii} | awk \'{{print $1}}\'").stdout.strip()
if not podName:
logging.debug(f'{ii} pod not found!')
else:
cmd.run(f'oc logs -f {podName} {ci} &> {path}/logs/{ii}.log &')
cmd.run(f'cd {path}/logs && zip -r -qq test_logs_CN.zip *.log')
cmd.copyin(f'{path}/logs/test_logs_CN.zip','test_logs_CN.zip')
ret = cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait')
if ret.returncode != 0:
logging.error('OC OAI CN5G: Undeployment failed')
cmd.run(f'helm list -aq -n {ocNamespace} | xargs -r helm uninstall -n {ocNamespace} --wait')
OC_logout(cmd)
return False, CONST.OC_PROJECT_FAIL
report = cmd.run('oc get pods')
OC_logout(cmd)
return True, report
class Cluster:
def __init__(self):
self.eNBIPAddress = ""
@@ -140,7 +94,7 @@ class Cluster:
self._retag_image_statement(name, name, newTag, filename)
self.cmd.run(f'oc delete -f {filename}')
ret = self.cmd.run(f'oc create -f {filename}')
if re.search('buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', ret.stdout) is not None:
if re.search(r'buildconfig.build.openshift.io/[a-zA-Z\-0-9]+ created', ret.stdout) is not None:
return True
logging.error('error while creating buildconfig: ' + ret.stdout)
return False
@@ -210,7 +164,7 @@ class Cluster:
def _deploy_pod(self, filename, timeout = 120):
ret = self.cmd.run(f'oc create -f {filename}')
result = re.search(f'pod/(?P<pod>[a-zA-Z0-9_\-]+) created', ret.stdout)
result = re.search(r'pod/(?P<pod>[a-zA-Z0-9_\-]+) created', ret.stdout)
if result is None:
logging.error(f'could not deploy pod: {ret.stdout}')
return None
@@ -368,7 +322,7 @@ class Cluster:
self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log')
self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log')
self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log')
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
if status:
self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
@@ -428,7 +382,7 @@ class Cluster:
self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log')
self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log')
self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log')
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
if status:
self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml')
@@ -441,7 +395,7 @@ class Cluster:
if not wait: logging.error('error during build of ranbuildfhi72_job')
status = status and wait
self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log')
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
if status:
self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml')
@@ -456,7 +410,7 @@ class Cluster:
status = status and wait
# recover logs
self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log')
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log', '\$', 10)
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
# split and analyze logs
imageSize = {}
@@ -472,9 +426,9 @@ class Cluster:
imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
grep_exp = "\|".join(attemptedImages)
grep_exp = r"\|".join(attemptedImages)
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log');
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done', '\$', 60)
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done')
build_log_name = f'build_log_{self.testCase_id}'
cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name)