mirror of
https://gitlab.eurecom.fr/oai/openairinterface5g.git
synced 2026-07-13 04:30:28 +00:00
Merge remote-tracking branch 'origin/ci-collect-ue-logs' into integration_2025_w32 (!3560)
CI: Refactor log collection, collect also (COTS) UE logs Refactor log handling in CI: collect according to currently used XML files. This makes it homogeneous in the pipelines, and avoids accidental overwriting (as happens in phytest-timing), as well as harmonizing the log placement (e.g., no YAML lookup in Ping/Iperf). Allow to trace and archive UE logs (e.g., Quectel via QLog, Amarisoft UE logs). By default, no logs will be collected as of this MR, as many files are big. Various cleanup in Jenkinsfiles, some bugfixes, add unit tests.
This commit is contained in:
Jaroslava Fiedlerova
@@ -216,24 +216,6 @@ pipeline {
|
||||
}
|
||||
stage('Log Collection') {
|
||||
parallel {
|
||||
stage('Log Collection (eNB - Build)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
|
||||
]) {
|
||||
echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
|
||||
sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
|
||||
|
||||
echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
|
||||
sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
|
||||
}
|
||||
script {
|
||||
if(fileExists("build.log.${env.BUILD_ID}.zip")) {
|
||||
archiveArtifacts "build.log.${env.BUILD_ID}.zip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Log Collection (eNB - Run)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
|
||||
@@ -222,24 +222,6 @@ pipeline {
|
||||
}
|
||||
stage('Log Collection') {
|
||||
parallel {
|
||||
stage('Log Collection (eNB - Build)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
|
||||
]) {
|
||||
echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
|
||||
sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
|
||||
|
||||
echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
|
||||
sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
|
||||
}
|
||||
script {
|
||||
if(fileExists("build.log.${env.BUILD_ID}.zip")) {
|
||||
archiveArtifacts "build.log.${env.BUILD_ID}.zip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Log Collection (eNB - Run)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
|
||||
@@ -210,24 +210,6 @@ pipeline {
|
||||
}
|
||||
stage('Log Collection') {
|
||||
parallel {
|
||||
stage('Log Collection (eNB - Build)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
|
||||
]) {
|
||||
echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
|
||||
sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
|
||||
|
||||
echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
|
||||
sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
|
||||
}
|
||||
script {
|
||||
if(fileExists("build.log.${env.BUILD_ID}.zip")) {
|
||||
archiveArtifacts "build.log.${env.BUILD_ID}.zip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Log Collection (eNB - Run)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
|
||||
@@ -227,24 +227,6 @@ pipeline {
|
||||
}
|
||||
stage('Log Collection') {
|
||||
parallel {
|
||||
stage('Log Collection (eNB - Build)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
[$class: 'UsernamePasswordMultiBinding', credentialsId: "${params.eNB_Credentials}", usernameVariable: 'eNB_Username', passwordVariable: 'eNB_Password']
|
||||
]) {
|
||||
echo '\u2705 \u001B[32mLog Collection (eNB - Build)\u001B[0m'
|
||||
sh "python3 ci-scripts/main.py --mode=LogCollectBuild --eNBIPAddress=${params.eNB_IPAddress} --eNBUserName=${eNB_Username} --eNBPassword=${eNB_Password} --eNBSourceCodePath=${params.eNB_SourceCodePath}"
|
||||
|
||||
echo '\u2705 \u001B[32mLog Transfer (eNB - Build)\u001B[0m'
|
||||
sh "sshpass -p \'${eNB_Password}\' scp -o 'StrictHostKeyChecking no' -o 'ConnectTimeout 10' ${eNB_Username}@${params.eNB_IPAddress}:${eNB_SourceCodePath}/cmake_targets/build.log.zip ./build.log.${env.BUILD_ID}.zip || true"
|
||||
}
|
||||
script {
|
||||
if(fileExists("build.log.${env.BUILD_ID}.zip")) {
|
||||
archiveArtifacts "build.log.${env.BUILD_ID}.zip"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
stage('Log Collection (eNB - Run)') {
|
||||
steps {
|
||||
withCredentials([
|
||||
|
||||
@@ -5,10 +5,12 @@ idefix:
|
||||
AttachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 wup
|
||||
DetachScript: sudo python3 ci_ctl_qtel.py /dev/ttyUSB2 detach
|
||||
NetworkScript: ip a show dev wwan0
|
||||
#Tracing:
|
||||
# Start: 'sudo rm -f ci_qlog/*; nohup sudo -b QLog/QLog -s ci_qlog -f NR5G.cfg &'
|
||||
# Stop: 'sudo killall QLog'
|
||||
# Collect: 'sudo mv ci_qlog/* %%log_dir%%/'
|
||||
IF: wwan0
|
||||
MTU: 1500
|
||||
Trace: True
|
||||
LogStore: /media/usb-drive/ci_qlogs
|
||||
|
||||
up2:
|
||||
Host: up2
|
||||
@@ -46,7 +48,6 @@ adb_ue_1:
|
||||
NetworkScript: adb -s R3CM40LZPHT shell "ip address show | grep rmnet_data0"
|
||||
CmdPrefix: adb -s R3CM40LZPHT shell
|
||||
MTU: 1500
|
||||
LogStore: /media/usb-drive/ci_adb_1-logs
|
||||
adb_ue_2:
|
||||
Host: nano
|
||||
InitScript: /home/oaicicd/ci_ctl_adb.sh initialize 5200c00db4413517
|
||||
@@ -59,7 +60,6 @@ adb_ue_2:
|
||||
NetworkScript: adb -s 5200c00db4413517 shell "ip address show | grep rmnet"
|
||||
CmdPrefix: adb -s 5200c00db4413517 shell
|
||||
MTU: 1500
|
||||
LogStore: /media/usb-drive/ci_adb_2-logs
|
||||
|
||||
oc-cn5g:
|
||||
Host: avra
|
||||
@@ -150,6 +150,10 @@ amarisoft_ue:
|
||||
InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz/aw2s-multi-00102-20.cfg &
|
||||
TermScript: /root/lteue-linux-2025-03-15/ws.js -t 10 127.0.0.1:9002 '{"message":"quit"}' || killall -KILL lteue-avx2
|
||||
NetworkScript: ip netns exec ue1 ip a show dev pdn0
|
||||
#Tracing:
|
||||
# Start: '' # nothing to be done
|
||||
# Stop: '' # nothing to be done
|
||||
# Collect: 'cp /tmp/ue.log %%log_dir%%'
|
||||
amarisoft_ue_2x2:
|
||||
Host: amariue
|
||||
InitScript: /root/lteue-linux-2025-03-15/lteue /root/oaicicd/ran_sa_aw2s_asue_20MHz_2x2/aw2s-multi-00102-2x2-v2.cfg &
|
||||
|
||||
@@ -84,7 +84,7 @@ class Analysis():
|
||||
|
||||
return success,msg
|
||||
|
||||
def analyze_oc_physim(result_junit, details_json):
|
||||
def analyze_oc_physim(result_junit, details_json, logPath):
|
||||
try:
|
||||
tree = ET.parse(result_junit)
|
||||
root = tree.getroot()
|
||||
@@ -124,7 +124,7 @@ class Analysis():
|
||||
output = test.findtext("system-out")
|
||||
output_check = "exceeds the threshold" not in output
|
||||
# collect logs
|
||||
log_dir = f'../cmake_targets/log/{test_exec}'
|
||||
log_dir = f'{logPath}/{test_exec}'
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
with open(f'{log_dir}/{test_name}.log', 'w') as f:
|
||||
f.write(output)
|
||||
|
||||
44
ci-scripts/cls_ci_helper.py
Normal file
44
ci-scripts/cls_ci_helper.py
Normal file
@@ -0,0 +1,44 @@
|
||||
#/*
|
||||
# * Licensed to the OpenAirInterface (OAI) Software Alliance under one or more
|
||||
# * contributor license agreements. See the NOTICE file distributed with
|
||||
# * this work for additional information regarding copyright ownership.
|
||||
# * The OpenAirInterface Software Alliance licenses this file to You under
|
||||
# * the OAI Public License, Version 1.1 (the "License"); you may not use this file
|
||||
# * except in compliance with the License.
|
||||
# * You may obtain a copy of the License at
|
||||
# *
|
||||
# * http://www.openairinterface.org/?page_id=698
|
||||
# *
|
||||
# * Unless required by applicable law or agreed to in writing, software
|
||||
# * distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# * See the License for the specific language governing permissions and
|
||||
# * limitations under the License.
|
||||
# *-------------------------------------------------------------------------------
|
||||
# * For more information about the OpenAirInterface (OAI) Software Alliance:
|
||||
# * contact@openairinterface.org
|
||||
# */
|
||||
|
||||
import os
|
||||
from typing import NamedTuple
|
||||
import logging
|
||||
|
||||
class TestCaseCtx(NamedTuple):
|
||||
count: int
|
||||
test_id: int
|
||||
logPath: str
|
||||
|
||||
def Default(logPath):
|
||||
return TestCaseCtx(123, 112233, logPath)
|
||||
def baseFilename(self):
|
||||
# typically, the test ID is of form 001234 (6 digits)
|
||||
return f"{self.logPath}/{self.count}-{self.test_id:06d}"
|
||||
|
||||
def archiveArtifact(cmd, ctx, remote_path):
|
||||
base = os.path.basename(remote_path)
|
||||
local = f"{ctx.baseFilename()}-{base}"
|
||||
logging.info(f"Archive artifact '{local}'")
|
||||
success = cmd.copyin(remote_path, local)
|
||||
if success:
|
||||
cmd.run(f'rm {remote_path}', silent=True)
|
||||
return local if success else None
|
||||
@@ -39,6 +39,7 @@ import constants as CONST
|
||||
import helpreadme as HELP
|
||||
import cls_containerize
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
IMAGE_REGISTRY_SERVICE_NAME = "image-registry.openshift-image-registry.svc"
|
||||
NAMESPACE = "oaicicd-ran"
|
||||
@@ -207,7 +208,12 @@ class Cluster:
|
||||
HTML.CreateHtmlTestRowQueue(param, 'KO', [msg])
|
||||
return success
|
||||
|
||||
def BuildClusterImage(self, HTML):
|
||||
def _retrieveOCLog(self, ctx, job, lSourcePath, image):
|
||||
fn = f'{lSourcePath}/cmake_targets/log/{image}.log'
|
||||
self.cmd.run(f'oc logs {job} &> {fn}')
|
||||
return (image, archiveArtifact(self.cmd, ctx, fn))
|
||||
|
||||
def BuildClusterImage(self, ctx, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {ranBranch} ranCommitID {self.ranCommitID}')
|
||||
@@ -280,22 +286,27 @@ class Cluster:
|
||||
self._recreate_entitlements()
|
||||
|
||||
status = True # flag to abandon compiling if any image fails
|
||||
attemptedImages = []
|
||||
log_files = []
|
||||
build_metrics = f"{lSourcePath}/cmake_targets/log/build-metrics.log"
|
||||
if forceBaseImageBuild:
|
||||
self._recreate_is_tag('ran-base', baseTag, 'openshift/ran-base-is.yaml')
|
||||
self._recreate_bc('ran-base', baseTag, 'openshift/ran-base-bc.yaml')
|
||||
ranbase_job = self._start_build('ran-base')
|
||||
attemptedImages += ['ran-base']
|
||||
status = ranbase_job is not None and self._wait_build_end([ranbase_job], 1000)
|
||||
if not status: logging.error('failure during build of ran-base')
|
||||
self.cmd.run(f'oc logs {ranbase_job} &> cmake_targets/log/ran-base.log') # cannot use cmd.run because of redirect
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbase_job, lSourcePath, 'ran-base'))
|
||||
|
||||
# recover logs by mounting image
|
||||
self._retag_image_statement('ran-base', 'ran-base', baseTag, 'openshift/ran-base-log-retrieval.yaml')
|
||||
pod = self._deploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
if pod is not None:
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/ran-base')
|
||||
self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ cmake_targets/log/ran-base')
|
||||
logdir = f'{lSourcePath}/cmake_targets/log/ran-base'
|
||||
self.cmd.run(f'mkdir -p {logdir}')
|
||||
self.cmd.run(f'oc rsync {pod}:/oai-ran/cmake_targets/log/ {logdir}')
|
||||
self._undeploy_pod('openshift/ran-base-log-retrieval.yaml')
|
||||
ret = self.cmd.run(f'ls {logdir}')
|
||||
for f in ret.stdout.splitlines():
|
||||
archiveArtifact(self.cmd, ctx, f"{logdir}/{f}")
|
||||
else:
|
||||
status = False
|
||||
|
||||
@@ -304,27 +315,24 @@ class Cluster:
|
||||
self._recreate_bc('oai-physim', imageTag, 'openshift/oai-physim-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.phySim.rhel9')
|
||||
physim_job = self._start_build('oai-physim')
|
||||
attemptedImages += ['oai-physim']
|
||||
|
||||
self._recreate_is_tag('ran-build', imageTag, 'openshift/ran-build-is.yaml')
|
||||
self._recreate_bc('ran-build', imageTag, 'openshift/ran-build-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.rhel9')
|
||||
ranbuild_job = self._start_build('ran-build')
|
||||
attemptedImages += ['ran-build']
|
||||
|
||||
self._recreate_is_tag('oai-clang', imageTag, 'openshift/oai-clang-is.yaml')
|
||||
self._recreate_bc('oai-clang', imageTag, 'openshift/oai-clang-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.clang.rhel9')
|
||||
clang_job = self._start_build('oai-clang')
|
||||
attemptedImages += ['oai-clang']
|
||||
|
||||
wait = ranbuild_job is not None and physim_job is not None and clang_job is not None and self._wait_build_end([ranbuild_job, physim_job, clang_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuild_job or physim_job or clang_job')
|
||||
status = status and wait
|
||||
self.cmd.run(f'oc logs {ranbuild_job} &> cmake_targets/log/ran-build.log')
|
||||
self.cmd.run(f'oc logs {physim_job} &> cmake_targets/log/oai-physim.log')
|
||||
self.cmd.run(f'oc logs {clang_job} &> cmake_targets/log/oai-clang.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbuild_job, lSourcePath, 'ran-build'))
|
||||
log_files.append(self._retrieveOCLog(ctx, physim_job, lSourcePath, 'oai-physim'))
|
||||
log_files.append(self._retrieveOCLog(ctx, clang_job, lSourcePath, 'oai-clang'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('oai-enb', imageTag, 'openshift/oai-enb-is.yaml')
|
||||
@@ -332,72 +340,66 @@ class Cluster:
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.eNB.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.eNB.rhel9')
|
||||
enb_job = self._start_build('oai-enb')
|
||||
attemptedImages += ['oai-enb']
|
||||
|
||||
self._recreate_is_tag('oai-gnb', imageTag, 'openshift/oai-gnb-is.yaml')
|
||||
self._recreate_bc('oai-gnb', imageTag, 'openshift/oai-gnb-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.rhel9')
|
||||
gnb_job = self._start_build('oai-gnb')
|
||||
attemptedImages += ['oai-gnb']
|
||||
|
||||
self._recreate_is_tag('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-is.yaml')
|
||||
self._recreate_bc('oai-gnb-aw2s', imageTag, 'openshift/oai-gnb-aw2s-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.gNB.aw2s.rhel9')
|
||||
gnb_aw2s_job = self._start_build('oai-gnb-aw2s')
|
||||
attemptedImages += ['oai-gnb-aw2s']
|
||||
|
||||
wait = enb_job is not None and gnb_job is not None and gnb_aw2s_job is not None and self._wait_build_end([enb_job, gnb_job, gnb_aw2s_job], 800)
|
||||
if not wait: logging.error('error during build of eNB/gNB')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {enb_job} &> cmake_targets/log/oai-enb.log')
|
||||
self.cmd.run(f'oc logs {gnb_job} &> cmake_targets/log/oai-gnb.log')
|
||||
self.cmd.run(f'oc logs {gnb_aw2s_job} &> cmake_targets/log/oai-gnb-aw2s.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, enb_job, lSourcePath, 'oai-enb'))
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_job, lSourcePath, 'oai-gnb'))
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_aw2s_job, lSourcePath, 'oai-gnb-aw2s'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
self._recreate_is_tag('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-is.yaml')
|
||||
self._recreate_bc('oai-nr-cuup', imageTag, 'openshift/oai-nr-cuup-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nr-cuup.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nr-cuup.rhel9')
|
||||
nr_cuup_job = self._start_build('oai-nr-cuup')
|
||||
attemptedImages += ['oai-nr-cuup']
|
||||
|
||||
self._recreate_is_tag('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-is.yaml')
|
||||
self._recreate_bc('oai-lte-ue', imageTag, 'openshift/oai-lte-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.lteUE.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.lteUE.rhel9')
|
||||
lteue_job = self._start_build('oai-lte-ue')
|
||||
attemptedImages += ['oai-lte-ue']
|
||||
|
||||
self._recreate_is_tag('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-is.yaml')
|
||||
self._recreate_bc('oai-nr-ue', imageTag, 'openshift/oai-nr-ue-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.nrUE.rhel9')
|
||||
self._retag_image_statement('ran-build', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build', imageTag, 'docker/Dockerfile.nrUE.rhel9')
|
||||
nrue_job = self._start_build('oai-nr-ue')
|
||||
attemptedImages += ['oai-nr-ue']
|
||||
|
||||
wait = nr_cuup_job is not None and lteue_job is not None and nrue_job is not None and self._wait_build_end([nr_cuup_job, lteue_job, nrue_job], 800)
|
||||
if not wait: logging.error('error during build of nr-cuup/lteUE/nrUE')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {nr_cuup_job} &> cmake_targets/log/oai-nr-cuup.log')
|
||||
self.cmd.run(f'oc logs {lteue_job} &> cmake_targets/log/oai-lte-ue.log')
|
||||
self.cmd.run(f'oc logs {nrue_job} &> cmake_targets/log/oai-nr-ue.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, nr_cuup_job, lSourcePath, 'oai-nr-cuup'))
|
||||
log_files.append(self._retrieveOCLog(ctx, lteue_job, lSourcePath, 'oai-lte-ue'))
|
||||
log_files.append(self._retrieveOCLog(ctx, nrue_job, lSourcePath, 'oai-nr-ue'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-is.yaml')
|
||||
self._recreate_bc('ran-build-fhi72', imageTag, 'openshift/ran-build-fhi72-bc.yaml')
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.build.fhi72.rhel9')
|
||||
ranbuildfhi72_job = self._start_build('ran-build-fhi72')
|
||||
attemptedImages += ['ran-build-fhi72']
|
||||
|
||||
wait = ranbuildfhi72_job is not None and self._wait_build_end([ranbuildfhi72_job], 1200)
|
||||
if not wait: logging.error('error during build of ranbuildfhi72_job')
|
||||
status = status and wait
|
||||
self.cmd.run(f'oc logs {ranbuildfhi72_job} &> cmake_targets/log/ran-build-fhi72.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, ranbuildfhi72_job, lSourcePath, 'ran-build-fhi72'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
if status:
|
||||
self._recreate_is_tag('oai-gnb-fhi72', imageTag, 'openshift/oai-gnb-fhi72-is.yaml')
|
||||
@@ -405,19 +407,17 @@ class Cluster:
|
||||
self._retag_image_statement('ran-base', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-base', baseTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
|
||||
self._retag_image_statement('ran-build-fhi72', 'image-registry.openshift-image-registry.svc:5000/oaicicd-ran/ran-build-fhi72', imageTag, 'docker/Dockerfile.gNB.fhi72.rhel9')
|
||||
gnb_fhi72_job = self._start_build('oai-gnb-fhi72')
|
||||
attemptedImages += ['oai-gnb-fhi72']
|
||||
|
||||
wait = gnb_fhi72_job is not None and self._wait_build_end([gnb_fhi72_job], 600)
|
||||
if not wait: logging.error('error during build of gNB-fhi72')
|
||||
status = status and wait
|
||||
# recover logs
|
||||
self.cmd.run(f'oc logs {gnb_fhi72_job} &> cmake_targets/log/oai-gnb-fhi72.log')
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> cmake_targets/log/build-metrics.log')
|
||||
log_files.append(self._retrieveOCLog(ctx, gnb_fhi72_job, lSourcePath, 'oai-gnb-fhi72'))
|
||||
self.cmd.run(f'oc get pods.metrics.k8s.io &>> {build_metrics}')
|
||||
|
||||
# split and analyze logs
|
||||
imageSize = {}
|
||||
for image in attemptedImages:
|
||||
self.cmd.run(f'mkdir -p cmake_targets/log/{image}')
|
||||
for image, _ in log_files:
|
||||
tag = imageTag if image != 'ran-base' else baseTag
|
||||
size = self._get_image_size(image, tag)
|
||||
if size <= 0:
|
||||
@@ -428,12 +428,14 @@ class Cluster:
|
||||
imageSize[image] = f'{sizeMb:.1f} Mbytes (uncompressed: ~{sizeMb*2.5:.1f} Mbytes)'
|
||||
logging.info(f'\u001B[1m{image} size is {imageSize[image]}\u001B[0m')
|
||||
|
||||
grep_exp = r"\|".join(attemptedImages)
|
||||
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> cmake_targets/log/image_registry.log');
|
||||
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> cmake_targets/log/build_pod_summary.log; done')
|
||||
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
cls_containerize.CopyLogsToExecutor(self.cmd, lSourcePath, build_log_name)
|
||||
archiveArtifact(self.cmd, ctx, build_metrics)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/image_registry.log'
|
||||
grep_exp = r"\|".join([i for i,f in log_files])
|
||||
self.cmd.run(f'oc get images | grep -e \'{grep_exp}\' &> {logfile}');
|
||||
archiveArtifact(self.cmd, ctx, logfile)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/build_pod_summary.log'
|
||||
self.cmd.run(f'for pod in $(oc get pods | tail -n +2 | awk \'{{print $1}}\'); do oc get pod $pod -o json &>> {logfile}; done')
|
||||
archiveArtifact(self.cmd, ctx, logfile)
|
||||
|
||||
self.cmd.run('for pod in $(oc get pods | tail -n +2 | awk \'{print $1}\'); do oc delete pod ${pod}; done')
|
||||
|
||||
@@ -442,7 +444,7 @@ class Cluster:
|
||||
self.cmd.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
collectInfo = cls_containerize.AnalyzeBuildLogs(log_files, status)
|
||||
for img in collectInfo:
|
||||
for f in collectInfo[img]:
|
||||
status = status and collectInfo[img][f]['status']
|
||||
@@ -458,9 +460,16 @@ class Cluster:
|
||||
|
||||
HTML.CreateHtmlNextTabHeaderTestRow(collectInfo, imageSize)
|
||||
|
||||
# TODO fix groovy script, remove the following.
|
||||
# the groovy scripts expects all logs in
|
||||
# <jenkins-workspace>/<pipeline>/ci-scripts, so copy it there
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'mkdir -p {os.getcwd()}/test_log_{ctx.test_id}/')
|
||||
c.run(f'cp -r {ctx.logPath} {os.getcwd()}/test_log_{ctx.test_id}/')
|
||||
|
||||
return status
|
||||
|
||||
def deploy_oc_physim(self, HTML, oc_release, svr_id):
|
||||
def deploy_oc_physim(self, ctx, HTML, oc_release, svr_id):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
raise ValueError(f'Insufficient Parameter: ranRepository {self.ranRepository} ranBranch {self.ranBranch} ranCommitID {self.ranCommitID}')
|
||||
@@ -470,17 +479,13 @@ class Cluster:
|
||||
options = f"oaicicd-core-for-ci-ran {oc_release} {image_tag} {self.eNBSourceCodePath}"
|
||||
ret = cls_cmd.runScript(svr_id, script, 600, options)
|
||||
logging.debug(f'"{script}" finished with code {ret.returncode}, output:\n{ret.stdout}')
|
||||
log_dir = f'{os.getcwd()}/../cmake_targets/log'
|
||||
os.makedirs(log_dir, exist_ok=True)
|
||||
result_junit = f'{oc_release}-run.xml'
|
||||
details_json = f'{oc_release}-tests.json'
|
||||
with cls_cmd.getConnection(svr_id) as ssh:
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{details_json}', tgt=f'{log_dir}/{details_json}')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/{result_junit}', tgt=f'{log_dir}/{result_junit}')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt', tgt=f'{log_dir}/physim_log.txt')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt', tgt=f'{log_dir}/physim_pods_summary.txt')
|
||||
ssh.copyin(src=f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log', tgt=f'{log_dir}/LastTestsFailed.log')
|
||||
test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(f'{log_dir}/{result_junit}', f'{log_dir}/{details_json}')
|
||||
details_json = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-tests.json')
|
||||
result_junit = archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/{oc_release}-run.xml')
|
||||
archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_log.txt')
|
||||
archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/physim_pods_summary.txt')
|
||||
archiveArtifact(ssh, ctx, f'{self.eNBSourceCodePath}/ci-scripts/LastTestsFailed.log')
|
||||
test_status, test_summary, test_result = cls_analysis.Analysis.analyze_oc_physim(result_junit, details_json, ctx.logPath)
|
||||
if test_summary:
|
||||
if test_status:
|
||||
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
|
||||
@@ -46,6 +46,7 @@ import cls_cmd
|
||||
import helpreadme as HELP
|
||||
import constants as CONST
|
||||
import cls_oaicitest
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Helper functions used here and in other classes
|
||||
@@ -81,67 +82,26 @@ def CreateTag(ranCommitID, ranBranch, ranAllowMerge):
|
||||
tagToUse = f'develop-{shortCommit}'
|
||||
return tagToUse
|
||||
|
||||
def CopyLogsToExecutor(cmd, sourcePath, log_name):
|
||||
cmd.cd(f'{sourcePath}/cmake_targets')
|
||||
cmd.run(f'rm -f {log_name}.zip')
|
||||
cmd.run(f'mkdir -p {log_name}')
|
||||
cmd.run(f'mv log/* {log_name}')
|
||||
cmd.run(f'zip -r -qq {log_name}.zip {log_name}')
|
||||
|
||||
# copy zip to executor for analysis
|
||||
if (os.path.isfile(f'./{log_name}.zip')):
|
||||
os.remove(f'./{log_name}.zip')
|
||||
if (os.path.isdir(f'./{log_name}')):
|
||||
shutil.rmtree(f'./{log_name}')
|
||||
cmd.copyin(src=f'{sourcePath}/cmake_targets/{log_name}.zip', tgt=f'{os.getcwd()}/{log_name}.zip')
|
||||
cmd.run(f'rm -f {log_name}.zip')
|
||||
ZipFile(f'{log_name}.zip').extractall('.')
|
||||
|
||||
def AnalyzeBuildLogs(buildRoot, images, globalStatus):
|
||||
def AnalyzeBuildLogs(logfiles, globalStatus):
|
||||
collectInfo = {}
|
||||
for image in images:
|
||||
for image, lf in logfiles:
|
||||
files = {}
|
||||
file_list = [f for f in os.listdir(f'{buildRoot}/{image}') if os.path.isfile(os.path.join(f'{buildRoot}/{image}', f)) and f.endswith('.txt')]
|
||||
# Analyze the "sub-logs" of every target image
|
||||
for fil in file_list:
|
||||
errorandwarnings = {}
|
||||
warningsNo = 0
|
||||
errorsNo = 0
|
||||
with open(f'{buildRoot}/{image}/{fil}', mode='r') as inputfile:
|
||||
for line in inputfile:
|
||||
result = re.search(' ERROR ', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' error:', str(line))
|
||||
if result is not None:
|
||||
errorsNo += 1
|
||||
result = re.search(' WARNING ', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
result = re.search(' warning:', str(line))
|
||||
if result is not None:
|
||||
warningsNo += 1
|
||||
errorandwarnings['errors'] = errorsNo
|
||||
errorandwarnings['warnings'] = warningsNo
|
||||
errorandwarnings['status'] = globalStatus
|
||||
files[fil] = errorandwarnings
|
||||
# Analyze the target image
|
||||
if os.path.isfile(f'{buildRoot}/{image}.log'):
|
||||
errorandwarnings = {}
|
||||
committed = False
|
||||
tagged = False
|
||||
with open(f'{buildRoot}/{image}.log', mode='r') as inputfile:
|
||||
for line in inputfile:
|
||||
lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
|
||||
lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
|
||||
tagged = tagged or lineHasTag or lineHasTag2
|
||||
# the OpenShift Cluster builder prepends image registry URL
|
||||
lineHasCommit = re.search(f'COMMIT [a-zA-Z0-9\.:/\-]*{image}', str(line)) is not None
|
||||
committed = committed or lineHasCommit
|
||||
errorandwarnings['errors'] = 0 if committed or tagged else 1
|
||||
errorandwarnings['warnings'] = 0
|
||||
errorandwarnings['status'] = committed or tagged
|
||||
files['Target Image Creation'] = errorandwarnings
|
||||
errorandwarnings = {}
|
||||
committed = False
|
||||
tagged = False
|
||||
with open(lf, mode='r') as inputfile:
|
||||
for line in inputfile:
|
||||
lineHasTag = re.search(f'Successfully tagged {image}:', str(line)) is not None
|
||||
lineHasTag2 = re.search(f'naming to docker.io/library/{image}:', str(line)) is not None
|
||||
tagged = tagged or lineHasTag or lineHasTag2
|
||||
# the OpenShift Cluster builder prepends image registry URL
|
||||
lineHasCommit = re.search(r'COMMIT [a-zA-Z0-9\.:/\-]*' + image, str(line)) is not None
|
||||
committed = committed or lineHasCommit
|
||||
errorandwarnings['errors'] = 0 if committed or tagged else 1
|
||||
errorandwarnings['warnings'] = 0
|
||||
errorandwarnings['status'] = committed or tagged
|
||||
logging.info(f"Analyzing {image}, file {lf}: {errorandwarnings}")
|
||||
files['Target Image Creation'] = errorandwarnings
|
||||
collectInfo[image] = files
|
||||
return collectInfo
|
||||
|
||||
@@ -202,12 +162,10 @@ def GetServices(ssh, requested, file):
|
||||
else:
|
||||
return requested
|
||||
|
||||
def CopyinServiceLog(ssh, lSourcePath, yaml, svcName, wd_yaml, filename):
|
||||
remote_filename = f"{lSourcePath}/cmake_targets/log/{filename}"
|
||||
def CopyinServiceLog(ssh, lSourcePath, svcName, wd_yaml, ctx):
|
||||
remote_filename = f"{lSourcePath}/cmake_targets/log/{svcName}.logs"
|
||||
ssh.run(f'docker compose -f {wd_yaml} logs {svcName} --no-log-prefix &> {remote_filename}')
|
||||
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml}"
|
||||
local_filename = f"{local_dir}/{filename}"
|
||||
return ssh.copyin(remote_filename, local_filename)
|
||||
return archiveArtifact(ssh, ctx, remote_filename)
|
||||
|
||||
def GetRunningServices(ssh, file):
|
||||
ret = ssh.run(f'docker compose -f {file} config --services')
|
||||
@@ -229,14 +187,13 @@ def GetRunningServices(ssh, file):
|
||||
logging.info(f'stopping services: {running_services}')
|
||||
return running_services
|
||||
|
||||
def CheckLogs(self, yaml, service_name, HTML, RAN):
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/{yaml}'
|
||||
filename = f'{logPath}/{service_name}-{HTML.testCase_id}.log'
|
||||
def CheckLogs(self, filename, HTML, RAN):
|
||||
success = True
|
||||
if (any(sub in service_name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
|
||||
name = os.path.basename(filename)
|
||||
if (any(sub in name for sub in ['oai_ue','oai-nr-ue','lte_ue'])):
|
||||
logging.debug(f'\u001B[1m Analyzing UE logfile {filename} \u001B[0m')
|
||||
logStatus = cls_oaicitest.OaiCiTest().AnalyzeLogFile_UE(filename, HTML, RAN)
|
||||
opt = f"UE log analysis for service {service_name}"
|
||||
opt = f"UE log analysis ({name})"
|
||||
# usage of htmlUEFailureMsg/htmleNBFailureMsg is because Analyze log files
|
||||
# abuse HTML to store their reports, and we here want to put custom options,
|
||||
# which is not possible with CreateHtmlTestRow
|
||||
@@ -247,13 +204,13 @@ def CheckLogs(self, yaml, service_name, HTML, RAN):
|
||||
else:
|
||||
HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmlUEFailureMsg])
|
||||
HTML.htmlUEFailureMsg = ""
|
||||
elif service_name == 'nv-cubb':
|
||||
elif 'nv-cubb' in name:
|
||||
msg = 'Undeploy PNF/Nvidia CUBB'
|
||||
HTML.CreateHtmlTestRow(msg, 'OK', CONST.ALL_PROCESSES_OK)
|
||||
elif (any(sub in service_name for sub in ['enb','rru','rcc','cu','du','gnb'])):
|
||||
elif (any(sub in name for sub in ['enb','rru','rcc','cu','du','gnb'])):
|
||||
logging.debug(f'\u001B[1m Analyzing XnB logfile {filename}\u001B[0m')
|
||||
logStatus = RAN.AnalyzeLogFile_eNB(filename, HTML, self.ran_checkers)
|
||||
opt = f"xNB log analysis for service {service_name}"
|
||||
opt = f"xNB log analysis ({name})"
|
||||
if (logStatus < 0):
|
||||
HTML.CreateHtmlTestRowQueue(opt, 'KO', [HTML.htmleNBFailureMsg])
|
||||
success = False
|
||||
@@ -261,9 +218,9 @@ def CheckLogs(self, yaml, service_name, HTML, RAN):
|
||||
HTML.CreateHtmlTestRowQueue(opt, 'OK', [HTML.htmleNBFailureMsg])
|
||||
HTML.htmleNBFailureMsg = ""
|
||||
else:
|
||||
logging.info(f'Skipping to analyze log for service name {service_name}')
|
||||
HTML.CreateHtmlTestRowQueue(f"service {service_name}", 'OK', ["no analysis function"])
|
||||
logging.debug(f"log check: service {service_name} passed analysis {success}")
|
||||
logging.info(f"Skipping analysis of log '{filename}': no submatch for xNB/UE")
|
||||
HTML.CreateHtmlTestRowQueue(f"file {name}", 'OK', ["no analysis function"])
|
||||
logging.debug(f"log check: file {filename} passed analysis {success}")
|
||||
return success
|
||||
|
||||
#-----------------------------------------------------------
|
||||
@@ -299,8 +256,6 @@ class Containerize():
|
||||
self.services = ['', '', '']
|
||||
self.deploymentTag = ''
|
||||
|
||||
self.testCase_id = ''
|
||||
|
||||
self.cli = ''
|
||||
self.cliBuildOptions = ''
|
||||
self.dockerfileprefix = ''
|
||||
@@ -330,11 +285,12 @@ class Containerize():
|
||||
raise ValueError(f'Insufficient Parameter: IP/node {ip}, path {path}')
|
||||
return (ip, path)
|
||||
|
||||
def BuildImage(self, HTML):
|
||||
def BuildImage(self, ctx, HTML):
|
||||
svr = self.eNB_serverId[self.eNB_instance]
|
||||
lIpAddr, lSourcePath = self.GetCredentials(svr)
|
||||
logging.debug('Building on server: ' + lIpAddr)
|
||||
cmd = cls_cmd.getConnection(lIpAddr)
|
||||
log_files = []
|
||||
|
||||
# Checking the hostname to get adapted on cli and dockerfileprefixes
|
||||
cmd.run('hostnamectl')
|
||||
@@ -390,7 +346,6 @@ class Containerize():
|
||||
imageNames.append(('oai-nr-ue', 'nrUE', 'oai-nr-ue', ''))
|
||||
imageNames.append(('oai-gnb-aerial', 'gNB.aerial', 'oai-gnb-aerial', ''))
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
cmd.cd(lSourcePath)
|
||||
# if asterix, copy the entitlement and subscription manager configurations
|
||||
if self.host == 'Red Hat':
|
||||
@@ -420,13 +375,24 @@ class Containerize():
|
||||
# Let's remove any previous run artifacts if still there
|
||||
cmd.run(f"{self.cli} image prune --force")
|
||||
for image,pattern,name,option in imageNames:
|
||||
cmd.run(f"{self.cli} image rm {name}:{imageTag}")
|
||||
cmd.run(f"{self.cli} image rm {name}:{imageTag}", reportNonZero=False)
|
||||
|
||||
# Build the base image only on Push Events (not on Merge Requests)
|
||||
# On when the base image docker file is being modified.
|
||||
if forceBaseImageBuild:
|
||||
cmd.run(f"{self.cli} image rm {baseImage}:{baseTag}")
|
||||
cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> cmake_targets/log/ran-base.log", timeout=1600)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/ran-base.docker.log'
|
||||
cmd.run(f"{self.cli} build {self.cliBuildOptions} --target {baseImage} --tag {baseImage}:{baseTag} --file docker/Dockerfile.base{self.dockerfileprefix} . &> {logfile}", timeout=1600)
|
||||
t = ("ran-base", archiveArtifact(cmd, ctx, logfile))
|
||||
log_files.append(t)
|
||||
|
||||
# Recover build logs, for the moment only possible when build is successful
|
||||
cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
|
||||
cmd.run("mkdir -p cmake_targets/log/ran-base")
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/ran-base.log'
|
||||
cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/all.txt {logfile}")
|
||||
cmd.run(f"{self.cli} rm -f test")
|
||||
archiveArtifact(cmd, ctx, logfile)
|
||||
# First verify if the base image was properly created.
|
||||
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {baseImage}:{baseTag}")
|
||||
allImagesSize = {}
|
||||
@@ -441,7 +407,7 @@ class Containerize():
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
return False
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
@@ -450,17 +416,9 @@ class Containerize():
|
||||
else:
|
||||
logging.debug('ran-base size is unknown')
|
||||
|
||||
# Recover build logs, for the moment only possible when build is successful
|
||||
cmd.run(f"{self.cli} create --name test {baseImage}:{baseTag}")
|
||||
cmd.run("mkdir -p cmake_targets/log/ran-base")
|
||||
cmd.run(f"{self.cli} cp test:/oai-ran/cmake_targets/log/. cmake_targets/log/ran-base")
|
||||
cmd.run(f"{self.cli} rm -f test")
|
||||
|
||||
# Build the target image(s)
|
||||
status = True
|
||||
attemptedImages = ['ran-base']
|
||||
for image,pattern,name,option in imageNames:
|
||||
attemptedImages += [name]
|
||||
# the archived Dockerfiles have "ran-base:latest" as base image
|
||||
# we need to update them with proper tag
|
||||
cmd.run(f'git checkout -- docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
||||
@@ -474,15 +432,18 @@ class Containerize():
|
||||
cmd.run(f'sed -i -e "s#ran-build:latest#ran-build:{imageTag}#" docker/Dockerfile.{pattern}{self.dockerfileprefix}')
|
||||
if image == 'oai-gnb-aerial':
|
||||
cmd.run('cp -f /opt/nvidia-ipc/nvipc.src.2025.05.20.tar.gz .')
|
||||
ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > cmake_targets/log/{name}.log 2>&1', timeout=1200)
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/{name}.docker.log'
|
||||
ret = cmd.run(f'{self.cli} build {self.cliBuildOptions} --target {image} --tag {name}:{imageTag} --file docker/Dockerfile.{pattern}{self.dockerfileprefix} {option} . > {logfile} 2>&1', timeout=1200)
|
||||
t = (name, archiveArtifact(cmd, ctx, logfile))
|
||||
log_files.append(t)
|
||||
if image == 'oai-gnb-aerial':
|
||||
cmd.run('rm -f nvipc.src.2025.05.20.tar.gz')
|
||||
if image == 'ran-build' and ret.returncode == 0:
|
||||
if image == 'ran-build' or image == 'ran-build-asan' or image == 'ran-build-fhi72':
|
||||
cmd.run(f"docker run --name test-log -d {name}:{imageTag} /bin/true")
|
||||
cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/ cmake_targets/log/{name}/")
|
||||
logfile = f'{lSourcePath}/{image}.ninja.log'
|
||||
cmd.run(f"docker cp test-log:/oai-ran/cmake_targets/log/all.txt {logfile}")
|
||||
cmd.run(f"docker rm -f test-log")
|
||||
else:
|
||||
cmd.run(f"mkdir -p cmake_targets/log/{name}")
|
||||
archiveArtifact(cmd, ctx, logfile)
|
||||
# check the status of the build
|
||||
ret = cmd.run(f"{self.cli} image inspect --format=\'Size = {{{{.Size}}}} bytes\' {name}:{imageTag}")
|
||||
if ret.returncode != 0:
|
||||
@@ -493,7 +454,7 @@ class Containerize():
|
||||
allImagesSize[name] = 'N/A -- Build Failed'
|
||||
break
|
||||
else:
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', cmd.getBefore())
|
||||
if result is not None:
|
||||
size = float(result.group("size")) / 1000000 # convert to MB
|
||||
imageSizeStr = f'{size:.1f}'
|
||||
@@ -516,13 +477,10 @@ class Containerize():
|
||||
logging.debug(cmd.run("df -h").stdout)
|
||||
logging.debug(cmd.run("docker system df").stdout)
|
||||
|
||||
# create a zip with all logs
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
||||
cmd.close()
|
||||
|
||||
# Analyze the logs
|
||||
collectInfo = AnalyzeBuildLogs(build_log_name, attemptedImages, status)
|
||||
collectInfo = AnalyzeBuildLogs(log_files, status)
|
||||
|
||||
if status:
|
||||
logging.info('\u001B[1m Building OAI Image(s) Pass\u001B[0m')
|
||||
@@ -536,13 +494,12 @@ class Containerize():
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
return False
|
||||
|
||||
def BuildProxy(self, HTML):
|
||||
def BuildProxy(self, ctx, HTML):
|
||||
svr = self.eNB_serverId[self.eNB_instance]
|
||||
lIpAddr, lSourcePath = self.GetCredentials(svr)
|
||||
logging.debug('Building on server: ' + lIpAddr)
|
||||
ssh = cls_cmd.getConnection(lIpAddr)
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
oldRanCommidID = self.ranCommitID
|
||||
oldRanRepository = self.ranRepository
|
||||
oldRanAllowMerge = self.ranAllowMerge
|
||||
@@ -568,15 +525,10 @@ class Containerize():
|
||||
if not success:
|
||||
raise Exception("could not clone proxy repository")
|
||||
|
||||
filename = f'build_log_{self.testCase_id}'
|
||||
fullpath = f'{lSourcePath}/{filename}'
|
||||
fullpath = f'{lSourcePath}/proxy_build.log'
|
||||
|
||||
ssh.run(f'docker build --target oai-lte-multi-ue-proxy --tag proxy:{tag} --file {lSourcePath}/docker/Dockerfile.ubuntu18.04 {lSourcePath} > {fullpath} 2>&1')
|
||||
ssh.run(f'zip -r -qq {fullpath}.zip {fullpath}')
|
||||
local_file = f"{os.getcwd()}/../cmake_targets/log/{filename}.zip"
|
||||
ssh.copyin(f'{fullpath}.zip', local_file)
|
||||
# don't delete such that we might recover the zips
|
||||
#ssh.run(f'rm -f {fullpath}.zip')
|
||||
archiveArtifact(ssh, ctx, fullpath)
|
||||
|
||||
ssh.run('docker image prune --force')
|
||||
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
|
||||
@@ -616,7 +568,7 @@ class Containerize():
|
||||
collectInfo = {}
|
||||
collectInfo['proxy'] = files
|
||||
ret = ssh.run(f'docker image inspect --format=\'Size = {{{{.Size}}}} bytes\' proxy:{tag}')
|
||||
result = re.search('Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
|
||||
result = re.search(r'Size *= *(?P<size>[0-9\-]+) *bytes', ret.stdout)
|
||||
# Cleaning any created tmp volume
|
||||
ssh.run('docker volume prune --force')
|
||||
ssh.close()
|
||||
@@ -638,7 +590,7 @@ class Containerize():
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
return False
|
||||
|
||||
def BuildRunTests(self, HTML):
|
||||
def BuildRunTests(self, ctx, HTML):
|
||||
svr = self.eNB_serverId[self.eNB_instance]
|
||||
lIpAddr, lSourcePath = self.GetCredentials(svr)
|
||||
logging.debug('Building on server: ' + lIpAddr)
|
||||
@@ -655,7 +607,6 @@ class Containerize():
|
||||
|
||||
if self.forcedWorkspaceCleanup:
|
||||
cmd.run(f'sudo -S rm -Rf {lSourcePath}')
|
||||
self.testCase_id = HTML.testCase_id
|
||||
|
||||
# check that ran-base image exists as we expect it
|
||||
baseImage = 'ran-base'
|
||||
@@ -675,10 +626,10 @@ class Containerize():
|
||||
|
||||
# build ran-unittests image
|
||||
dockerfile = "ci-scripts/docker/Dockerfile.unittest.ubuntu22"
|
||||
ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {lSourcePath}/cmake_targets/log/unittest-build.log')
|
||||
logfile = f'{lSourcePath}/cmake_targets/log/unittest-build.log'
|
||||
ret = cmd.run(f'docker build --progress=plain --tag ran-unittests:{baseTag} --file {dockerfile} . &> {logfile}')
|
||||
archiveArtifact(cmd, ctx, logfile)
|
||||
if ret.returncode != 0:
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
||||
logging.error(f'Cannot build unit tests')
|
||||
HTML.CreateHtmlTestRow("Unit test build failed", 'KO', [dockerfile])
|
||||
HTML.CreateHtmlTabFooter(False)
|
||||
@@ -687,10 +638,15 @@ class Containerize():
|
||||
HTML.CreateHtmlTestRowQueue("Build unit tests", 'OK', [dockerfile])
|
||||
|
||||
# it worked, build and execute tests, and close connection
|
||||
ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --rm ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)')
|
||||
cmd.run(f'docker rmi ran-unittests:{baseTag}')
|
||||
build_log_name = f'build_log_{self.testCase_id}'
|
||||
CopyLogsToExecutor(cmd, lSourcePath, build_log_name)
|
||||
# I would like to run it with --rm and mount the ctest result directory to avoid 'docker cp'
|
||||
# below, but then permissions are messed up and we can't remove the directory without sudo
|
||||
# making the next pipeline fail
|
||||
ret = cmd.run(f'docker run -a STDOUT --workdir /oai-ran/build/ --env LD_LIBRARY_PATH=/oai-ran/build/ --name ran-unittests ran-unittests:{baseTag} ctest --no-label-summary -j$(nproc)')
|
||||
cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTest.log .')
|
||||
archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTest.log')
|
||||
cmd.run('docker cp ran-unittests:/oai-ran/build/Testing/Temporary/LastTestsFailed.log .')
|
||||
archiveArtifact(cmd, ctx, f'{lSourcePath}/LastTestsFailed.log')
|
||||
cmd.run('docker rm ran-unittests')
|
||||
cmd.close()
|
||||
|
||||
if ret.returncode == 0:
|
||||
@@ -823,18 +779,14 @@ class Containerize():
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ["cannot create workspace"])
|
||||
return success
|
||||
|
||||
def DeployObject(self, HTML):
|
||||
def DeployObject(self, ctx, HTML):
|
||||
svr = self.eNB_serverId[self.eNB_instance]
|
||||
num_attempts = self.num_attempts
|
||||
lIpAddr, lSourcePath = self.GetCredentials(svr)
|
||||
logging.debug(f'Deploying OAI Object on server: {lIpAddr}')
|
||||
yaml = self.yamlPath[self.eNB_instance].strip('/')
|
||||
# creating the log folder by default
|
||||
local_dir = f"{os.getcwd()}/../cmake_targets/log/{yaml.split('/')[-1]}"
|
||||
os.system(f'mkdir -p {local_dir}')
|
||||
wd = f'{lSourcePath}/{yaml}'
|
||||
wd_yaml = f'{wd}/docker-compose.y*ml'
|
||||
yaml_dir = yaml.split('/')[-1]
|
||||
with cls_cmd.getConnection(lIpAddr) as ssh:
|
||||
services = GetServices(ssh, self.services[self.eNB_instance], wd_yaml)
|
||||
if services == [] or services == ' ' or services == None:
|
||||
@@ -861,7 +813,7 @@ class Containerize():
|
||||
logging.warning(warning_msg)
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'NOK', [warning_msg])
|
||||
for svc in services.split():
|
||||
CopyinServiceLog(ssh, lSourcePath, yaml_dir, svc, wd_yaml, f'{svc}-{HTML.testCase_id}-attempt{attempt}.log')
|
||||
CopyinServiceLog(ssh, lSourcePath, svc, wd_yaml, ctx)
|
||||
ssh.run(f'docker compose -f {wd_yaml} down -- {services}')
|
||||
imagesInfo = info.stdout.splitlines()[1:]
|
||||
logging.debug(f'{info.stdout.splitlines()[1:]}')
|
||||
@@ -871,13 +823,12 @@ class Containerize():
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['\n'.join(imagesInfo)])
|
||||
return deployed
|
||||
|
||||
def UndeployObject(self, HTML, RAN):
|
||||
def UndeployObject(self, ctx, HTML, RAN):
|
||||
svr = self.eNB_serverId[self.eNB_instance]
|
||||
lIpAddr, lSourcePath = self.GetCredentials(svr)
|
||||
logging.debug(f'\u001B[1m Undeploying OAI Object from server: {lIpAddr}\u001B[0m')
|
||||
yaml = self.yamlPath[self.eNB_instance].strip('/')
|
||||
wd = f'{lSourcePath}/{yaml}'
|
||||
yaml_dir = yaml.split('/')[-1]
|
||||
with cls_cmd.getConnection(lIpAddr) as ssh:
|
||||
ExistEnvFilePrint(ssh, wd)
|
||||
services = GetRunningServices(ssh, f"{wd}/docker-compose.y*ml")
|
||||
@@ -885,7 +836,7 @@ class Containerize():
|
||||
if services is not None:
|
||||
all_serv = " ".join([s for s, _ in services])
|
||||
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml stop -- {all_serv}')
|
||||
copyin_res = all(CopyinServiceLog(ssh, lSourcePath, yaml_dir, s, f"{wd}/docker-compose.y*ml", f'{s}-{HTML.testCase_id}.log') for s, c in services)
|
||||
copyin_res = [CopyinServiceLog(ssh, lSourcePath, s, f"{wd}/docker-compose.y*ml", ctx) for s, _ in services]
|
||||
else:
|
||||
logging.warning('could not identify services to stop => no log file')
|
||||
ssh.run(f'docker compose -f {wd}/docker-compose.y*ml down -v')
|
||||
@@ -894,7 +845,7 @@ class Containerize():
|
||||
HTML.CreateHtmlTestRowQueue('N/A', 'KO', ['Could not copy logfile(s)'])
|
||||
return False
|
||||
else:
|
||||
log_results = [CheckLogs(self, yaml_dir, s, HTML, RAN) for s, _ in services]
|
||||
log_results = [CheckLogs(self, f, HTML, RAN) for f in copyin_res]
|
||||
success = all(log_results)
|
||||
if success:
|
||||
logging.info('\u001B[1m Undeploying OAI Object Pass\u001B[0m')
|
||||
|
||||
@@ -29,6 +29,7 @@ import yaml
|
||||
import re
|
||||
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
def listify(s):
|
||||
if s is None:
|
||||
@@ -114,8 +115,7 @@ class CoreNetwork:
|
||||
logging.info(f'deployed core network {self}, pingable IP address {ip}')
|
||||
return True, output
|
||||
|
||||
def _collect_logs(self, log_dir):
|
||||
logging.info(f'collecting logs into (local) {log_dir}')
|
||||
def _collect_logs(self, ctx):
|
||||
remote_dir = "/tmp/cn-undeploy-logs"
|
||||
with cls_cmd.getConnection(self._host) as c:
|
||||
# create a directory for log collection
|
||||
@@ -134,18 +134,17 @@ class CoreNetwork:
|
||||
logging.error("cannot enumerate log files")
|
||||
return []
|
||||
log_files = []
|
||||
# copy them to the executor one by one, and store in log_dir
|
||||
# copy them to the executor one by one
|
||||
for f in ret.stdout.split("\n"):
|
||||
l = f.replace(remote_dir, log_dir)
|
||||
c.copyin(f, l)
|
||||
log_files.append(l)
|
||||
name = archiveArtifact(c, ctx, f)
|
||||
log_files.append(name)
|
||||
c.run(f'rm -rf {remote_dir}')
|
||||
return log_files
|
||||
|
||||
def undeploy(self, log_dir=None):
|
||||
def undeploy(self, ctx=None):
|
||||
log_files = []
|
||||
if log_dir is not None:
|
||||
log_files = self._collect_logs(log_dir)
|
||||
if ctx is not None:
|
||||
log_files = self._collect_logs(ctx)
|
||||
else:
|
||||
logging.warning("no directory for log collection specified, cannot retrieve core network logs")
|
||||
logging.info(f'undeploy core network {self}')
|
||||
|
||||
@@ -32,6 +32,7 @@ import re
|
||||
import yaml
|
||||
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
class Module_UE:
|
||||
|
||||
@@ -59,26 +60,31 @@ class Module_UE:
|
||||
}
|
||||
self.interface = m.get('IF')
|
||||
self.MTU = m.get('MTU')
|
||||
self.trace = m.get('trace') == True
|
||||
self.logStore = m.get('LogStore')
|
||||
self.cmd_prefix = m.get('CmdPrefix')
|
||||
logging.info(f'initialized UE {self} from {filename}')
|
||||
|
||||
t = m.get('Tracing')
|
||||
self.trace = t is not None
|
||||
if self.trace:
|
||||
if t.get('Start') is None or t.get('Stop') is None or t.get('Collect')is None :
|
||||
raise ValueError("need to have Start/Stop/Collect for tracing")
|
||||
self.cmd_dict["traceStart"] = t.get('Start')
|
||||
self.cmd_dict["traceStop"] = t.get('Stop')
|
||||
self._logCollect = t.get('Collect')
|
||||
if "%%log_dir%%" not in self._logCollect:
|
||||
raise ValueError(f"(At least one) LogCollect expression for {module_name} must contain \"%%log_dir%%\"")
|
||||
|
||||
def __str__(self):
|
||||
return f"{self.module_name}@{self.host} [IP: {self.getIP()}]"
|
||||
return f"{self.module_name}@{self.host}"
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
def _command(self, cmd, silent = False):
|
||||
def _command(self, cmd, silent=False, reportNonZero=True):
|
||||
if cmd is None:
|
||||
raise Exception("no command provided")
|
||||
if self.host == "" or self.host == "localhost":
|
||||
c = cls_cmd.LocalCmd()
|
||||
else:
|
||||
c = cls_cmd.RemoteCmd(self.host)
|
||||
response = c.run(cmd, silent=silent)
|
||||
c.close()
|
||||
with cls_cmd.getConnection(self.host) as c:
|
||||
response = c.run(cmd, silent=silent, reportNonZero=reportNonZero)
|
||||
return response
|
||||
|
||||
#-----------------$
|
||||
@@ -86,25 +92,23 @@ class Module_UE:
|
||||
#-----------------$
|
||||
|
||||
def initialize(self):
|
||||
if self.trace:
|
||||
raise Exception("UE tracing not implemented yet")
|
||||
self._enableTrace()
|
||||
# we first terminate to make sure the UE has been stopped
|
||||
if self.cmd_dict["detach"]:
|
||||
self._command(self.cmd_dict["detach"], silent=True)
|
||||
self._command(self.cmd_dict["terminate"], silent=True)
|
||||
ret = self._command(self.cmd_dict["initialize"])
|
||||
logging.info(f'For command: {ret.args} | return output: {ret.stdout} | Code: {ret.returncode}')
|
||||
if self.trace:
|
||||
self._enableTrace()
|
||||
# Here each UE returns differently for the successful initialization, requires check based on UE
|
||||
return ret.returncode == 0
|
||||
|
||||
|
||||
def terminate(self):
|
||||
def terminate(self, ctx=None):
|
||||
self._command(self.cmd_dict["terminate"])
|
||||
if self.trace:
|
||||
raise Exception("UE tracing not implemented yet")
|
||||
if self.trace and ctx is not None:
|
||||
self._disableTrace()
|
||||
return self._logCollect()
|
||||
return self._collectTrace(ctx)
|
||||
return None
|
||||
|
||||
def attach(self, attach_tries = 4, attach_timeout = 60):
|
||||
@@ -113,10 +117,11 @@ class Module_UE:
|
||||
self._command(self.cmd_dict["attach"])
|
||||
timeout = attach_timeout
|
||||
logging.debug("Waiting for IP address to be assigned")
|
||||
ip = self.getIP(silent=False, reportNonZero=True)
|
||||
while timeout > 0 and not ip:
|
||||
time.sleep(5)
|
||||
timeout -= 5
|
||||
ip = self.getIP()
|
||||
time.sleep(1)
|
||||
timeout -= 1
|
||||
ip = self.getIP(silent=True, reportNonZero=False)
|
||||
if ip:
|
||||
break
|
||||
logging.warning(f"UE did not receive IP address after {attach_timeout} s, detaching")
|
||||
@@ -160,8 +165,8 @@ class Module_UE:
|
||||
logging.error(message)
|
||||
return False
|
||||
|
||||
def getIP(self):
|
||||
output = self._command(self.cmd_dict["getNetwork"], silent=True)
|
||||
def getIP(self, silent=True, reportNonZero=True):
|
||||
output = self._command(self.cmd_dict["getNetwork"], silent=silent, reportNonZero=reportNonZero)
|
||||
result = re.search(r'inet (?P<ip>[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+)', output.stdout)
|
||||
if result and result.group('ip'):
|
||||
ip = result.group('ip')
|
||||
@@ -191,10 +196,33 @@ class Module_UE:
|
||||
return self.cmd_prefix if self.cmd_prefix else ""
|
||||
|
||||
def _enableTrace(self):
|
||||
raise Exception("not implemented")
|
||||
logging.info(f'UE {self}: start UE tracing')
|
||||
self._command(self.cmd_dict["traceStart"])
|
||||
|
||||
def _disableTrace(self):
|
||||
raise Exception("not implemented")
|
||||
logging.info(f'UE {self}: stop UE tracing')
|
||||
self._command(self.cmd_dict["traceStop"])
|
||||
|
||||
def _logCollect(self):
|
||||
raise Exception("not implemented")
|
||||
def _collectTrace(self, ctx):
|
||||
remote_dir = "/tmp/ue-trace-logs"
|
||||
with cls_cmd.getConnection(self.host) as c:
|
||||
# create a directory for log collection
|
||||
c.run(f'rm -rf {remote_dir}')
|
||||
ret = c.run(f'mkdir {remote_dir}')
|
||||
if ret.returncode != 0:
|
||||
logging.error("cannot create directory for log collection")
|
||||
return []
|
||||
log_cmd = self._logCollect.replace('%%log_dir%%', remote_dir)
|
||||
self._command(log_cmd)
|
||||
# enumerate collected files
|
||||
ret = c.run(f'ls {remote_dir}/*')
|
||||
if ret.returncode != 0:
|
||||
logging.error("cannot enumerate log files")
|
||||
return []
|
||||
log_files = []
|
||||
# copy them to the executor one by one, and store in log_dir
|
||||
for f in ret.stdout.split("\n"):
|
||||
name = archiveArtifact(c, ctx, f)
|
||||
log_files.append(name)
|
||||
c.run(f'rm -rf {remote_dir}')
|
||||
return log_files
|
||||
|
||||
@@ -28,27 +28,24 @@ import cls_cmd
|
||||
import cls_oai_html
|
||||
import cls_analysis
|
||||
import constants as CONST
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
LOG_PATH_PHYSIM = 'phy_sim_logs'
|
||||
DPDK_PATH = '/opt/dpdk-t2-22.11.0'
|
||||
|
||||
class Native():
|
||||
|
||||
def Build(test_case, HTML, host, directory, options):
|
||||
def Build(ctx, test_case, HTML, host, directory, options):
|
||||
logging.debug(f'Building on server: {host}')
|
||||
HTML.testCase_id = test_case
|
||||
|
||||
with cls_cmd.getConnection(host) as ssh:
|
||||
base = f"{directory}/cmake_targets"
|
||||
ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/log/build_oai.log", timeout=900)
|
||||
ret = ssh.run(f"C_INCLUDE_PATH={DPDK_PATH}/include/ PKG_CONFIG_PATH={DPDK_PATH}/lib64/pkgconfig/ {base}/build_oai {options} > {base}/build_oai.log", timeout=900)
|
||||
success = ret.returncode == 0
|
||||
logs = ssh.run(f"cat {base}/log/build_oai.log", silent=True)
|
||||
logs = ssh.run(f"cat {base}/build_oai.log", silent=True)
|
||||
logging.debug(f"build finished with code {ret.returncode}, output:\n{logs.stdout}")
|
||||
|
||||
# create log directory, and copy build logs
|
||||
target = f"{base}/build_log_{test_case}/"
|
||||
ssh.run(f"mkdir -p {target}")
|
||||
ssh.run(f"mv {base}/log/* {target}")
|
||||
archiveArtifact(ssh, ctx, f'{base}/build_oai.log')
|
||||
|
||||
# check if build artifacts are there
|
||||
# NOTE: build_oai should fail with exit code if it could not build, but it does not
|
||||
@@ -68,15 +65,14 @@ class Native():
|
||||
HTML.CreateHtmlTestRow(options, 'KO', CONST.ALL_PROCESSES_OK)
|
||||
return success
|
||||
|
||||
def Run_Physim(HTML, host, directory, options, physim_test, threshold):
|
||||
def Run_Physim(ctx, HTML, host, directory, options, physim_test, threshold):
|
||||
logging.debug(f'Runnin {physim_test} on server: {host}')
|
||||
workSpacePath = f'{directory}/cmake_targets'
|
||||
os.system(f'mkdir -p ./{LOG_PATH_PHYSIM}')
|
||||
runLogFile=f'physim_{HTML.testCase_id}.log'
|
||||
runLogFile=f'{workSpacePath}/physim.log'
|
||||
with cls_cmd.getConnection(host) as cmd:
|
||||
cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {workSpacePath}/{runLogFile}')
|
||||
cmd.copyin(src=f'{workSpacePath}/{runLogFile}', tgt=f'{LOG_PATH_PHYSIM}/{runLogFile}')
|
||||
success, msg = cls_analysis.Analysis.analyze_physim(f'{LOG_PATH_PHYSIM}/{runLogFile}', physim_test, options, threshold)
|
||||
cmd.run(f'sudo LD_LIBRARY_PATH=.:{DPDK_PATH}/lib64/ {workSpacePath}/ran_build/build/{physim_test} {options} >> {runLogFile}')
|
||||
physim_file = archiveArtifact(cmd, ctx, runLogFile)
|
||||
success, msg = cls_analysis.Analysis.analyze_physim(physim_file, physim_test, options, threshold)
|
||||
if success:
|
||||
HTML.CreateHtmlTestRowQueue(options, 'OK', [msg])
|
||||
else:
|
||||
|
||||
@@ -47,6 +47,7 @@ import constants as CONST
|
||||
import cls_module
|
||||
import cls_corenetwork
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Helper functions used here and in other classes
|
||||
@@ -259,9 +260,7 @@ class OaiCiTest():
|
||||
self.ranAllowMerge = False
|
||||
self.ranTargetBranch = ''
|
||||
|
||||
self.testCase_id = ''
|
||||
self.testXMLfiles = []
|
||||
self.desc = ''
|
||||
self.ping_args = ''
|
||||
self.ping_packetloss_threshold = ''
|
||||
self.ping_rttavg_threshold =''
|
||||
@@ -355,7 +354,7 @@ class OaiCiTest():
|
||||
HTML.CreateHtmlTestRowQueue('NA', 'OK', messages)
|
||||
return True
|
||||
|
||||
def Ping_common(self, cn, ue, logPath):
|
||||
def Ping_common(self, ctx, cn, ue):
|
||||
ping_status = 0
|
||||
ueIP = ue.getIP()
|
||||
if not ueIP:
|
||||
@@ -363,13 +362,12 @@ class OaiCiTest():
|
||||
svrIP = cn.getIP()
|
||||
if not svrIP:
|
||||
return (False, f"CN {cn.getName()} has no IP address")
|
||||
ping_log_file = f'ping_{self.testCase_id}_{ue.getName()}.log'
|
||||
ping_log_file = f'/tmp/ping_{ue.getName()}.log'
|
||||
ping_time = re.findall(r"-c *(\d+)",str(self.ping_args))
|
||||
local_ping_log_file = f'{logPath}/{ping_log_file}'
|
||||
if re.search('%cn_ip%', self.ping_args) or re.search(r'[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+', self.ping_args):
|
||||
raise Exception(f"ping_args should not have IP address: {self.ping_args}")
|
||||
interface = f'-I {ue.getIFName()}' if ue.getIFName() else ''
|
||||
ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee /tmp/{ping_log_file}'
|
||||
ping_cmd = f'{ue.getCmdPrefix()} ping {interface} {self.ping_args} {svrIP} 2>&1 | tee {ping_log_file}'
|
||||
cmd = cls_cmd.getConnection(ue.getHost())
|
||||
response = cmd.run(ping_cmd, timeout=int(ping_time[0])*1.5)
|
||||
ue_header = f'UE {ue.getName()} ({ueIP})'
|
||||
@@ -377,8 +375,7 @@ class OaiCiTest():
|
||||
message = ue_header + ': ping crashed: TIMEOUT?'
|
||||
return (False, message)
|
||||
|
||||
#copy the ping log file to have it locally for analysis (ping stats)
|
||||
cmd.copyin(src=f'/tmp/{ping_log_file}', tgt=local_ping_log_file)
|
||||
local_ping_log_file = archiveArtifact(cmd, ctx, ping_log_file)
|
||||
cmd.close()
|
||||
|
||||
with open(local_ping_log_file, 'r') as f:
|
||||
@@ -418,18 +415,13 @@ class OaiCiTest():
|
||||
|
||||
return (True, message)
|
||||
|
||||
def Ping(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
|
||||
def Ping(self, ctx, HTML, infra_file="ci_infra.yaml"):
|
||||
if self.ue_ids == [] or self.svr_id == None:
|
||||
raise Exception("no module names in self.ue_ids or/and self.svr_id provided")
|
||||
# Creating destination log folder if needed on the python executor workspace
|
||||
with cls_cmd.getConnection('localhost') as local:
|
||||
ymlPath = CONTAINERS.yamlPath[0].split('/')
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
|
||||
local.run(f'mkdir -p {logPath}', silent=True)
|
||||
ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)]
|
||||
cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
|
||||
futures = [executor.submit(self.Ping_common, cn, ue, logPath) for ue in ues]
|
||||
futures = [executor.submit(self.Ping_common, ctx, cn, ue) for ue in ues]
|
||||
results = [f.result() for f in futures]
|
||||
# each result in results is a tuple, first member goes to successes, second to messages
|
||||
successes, messages = map(list, zip(*results))
|
||||
@@ -450,7 +442,7 @@ class OaiCiTest():
|
||||
HTML.CreateHtmlTestRowQueue(self.ping_args, 'KO', messages)
|
||||
return success
|
||||
|
||||
def Iperf_Module(self, cn, ue, idx, ue_num, logPath):
|
||||
def Iperf_Module(self, ctx, cn, ue, idx, ue_num):
|
||||
ueIP = ue.getIP()
|
||||
if not ueIP:
|
||||
return (False, f"UE {ue.getName()} has no IP address")
|
||||
@@ -463,7 +455,7 @@ class OaiCiTest():
|
||||
serverReport = ""
|
||||
udpIperf = re.search('-u', iperf_opt) is not None
|
||||
bidirIperf = re.search('--bidir', iperf_opt) is not None
|
||||
client_filename = f'iperf_client_{self.testCase_id}_{ue.getName()}.log'
|
||||
client_filename = f'/tmp/iperf_client_{ue.getName()}.log'
|
||||
if udpIperf:
|
||||
target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(idx, ue_num, self.iperf_profile, self.iperf_args)
|
||||
# note: for UDP testing we don't want to use json report - reports 0 Mbps received bitrate
|
||||
@@ -478,14 +470,11 @@ class OaiCiTest():
|
||||
port = 5002 + idx
|
||||
# note: some core setups start an iperf3 server automatically, indicated in ci_infra by runIperf3Server: False`
|
||||
t = iperf_time * 2.5
|
||||
cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True)
|
||||
cmd_ue.run(f'rm {client_filename}', reportNonZero=False, silent=True)
|
||||
if cn.runIperf3Server():
|
||||
cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf3 -s -B {svrIP} -p {port} -1 {jsonReport} >> /dev/null &', timeout=t)
|
||||
cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> /tmp/{client_filename}', timeout=t)
|
||||
# note: copy iperf3 log to the current directory for log analysis and log collection
|
||||
dest_filename = f'{logPath}/{client_filename}'
|
||||
cmd_ue.copyin(f'/tmp/{client_filename}', dest_filename)
|
||||
cmd_ue.run(f'rm /tmp/{client_filename}', reportNonZero=False, silent=True)
|
||||
cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} {iperf_ue} -B {ueIP} -c {svrIP} -p {port} {iperf_opt} {jsonReport} {serverReport} -O 5 >> {client_filename}', timeout=t)
|
||||
dest_filename = archiveArtifact(cmd_ue, ctx, client_filename)
|
||||
if udpIperf:
|
||||
status, msg = Iperf_analyzeV3UDP(dest_filename, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
|
||||
elif bidirIperf:
|
||||
@@ -495,20 +484,15 @@ class OaiCiTest():
|
||||
|
||||
return (status, f'{ue_header}\n{msg}')
|
||||
|
||||
def Iperf(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
|
||||
def Iperf(self, ctx, HTML, infra_file="ci_infra.yaml"):
|
||||
logging.debug(f'Iperf: iperf_args "{self.iperf_args}" iperf_packetloss_threshold "{self.iperf_packetloss_threshold}" iperf_bitrate_threshold "{self.iperf_bitrate_threshold}" iperf_profile "{self.iperf_profile}" iperf_options "{self.iperf_options}"')
|
||||
|
||||
if self.ue_ids == [] or self.svr_id == None:
|
||||
raise Exception("no module names in self.ue_ids or/and self.svr_id provided")
|
||||
# create log directory on executor node
|
||||
with cls_cmd.getConnection('localhost') as local:
|
||||
ymlPath = CONTAINERS.yamlPath[0].split('/')
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
|
||||
local.run(f'mkdir -p {logPath}', silent=True)
|
||||
ues = [cls_module.Module_UE(ue_id, server_name, infra_file) for ue_id, server_name in zip(self.ue_ids, self.nodes)]
|
||||
cn = cls_corenetwork.CoreNetwork(self.svr_id, self.svr_node, filename=infra_file)
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
|
||||
futures = [executor.submit(self.Iperf_Module, cn, ue, i, len(ues), logPath) for i, ue in enumerate(ues)]
|
||||
futures = [executor.submit(self.Iperf_Module, ctx, cn, ue, i, len(ues)) for i, ue in enumerate(ues)]
|
||||
results = [f.result() for f in futures]
|
||||
# each result in results is a tuple, first member goes to successes, second to messages
|
||||
successes, messages = map(list, zip(*results))
|
||||
@@ -529,7 +513,7 @@ class OaiCiTest():
|
||||
HTML.CreateHtmlTestRowQueue(self.iperf_args, 'KO', messages)
|
||||
return success
|
||||
|
||||
def Iperf2_Unidir(self, HTML, CONTAINERS, infra_file="ci_infra.yaml"):
|
||||
def Iperf2_Unidir(self, ctx, HTML, infra_file="ci_infra.yaml"):
|
||||
if self.ue_ids == [] or self.svr_id == None or len(self.ue_ids) != 1:
|
||||
raise Exception("no module names in self.ue_ids or/and self.svr_id provided, multi UE scenario not supported")
|
||||
ue = cls_module.Module_UE(self.ue_ids[0].strip(),self.nodes[0].strip(), infra_file)
|
||||
@@ -540,23 +524,17 @@ class OaiCiTest():
|
||||
svrIP = cn.getIP()
|
||||
if not svrIP:
|
||||
return False
|
||||
server_filename = f'iperf_server_{self.testCase_id}_{ue.getName()}.log'
|
||||
ymlPath = CONTAINERS.yamlPath[0].split('/')
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/{ymlPath[-1]}'
|
||||
server_filename = f'/tmp/iperf_server_{ue.getName()}.log'
|
||||
iperf_time = Iperf_ComputeTime(self.iperf_args)
|
||||
target_bitrate, iperf_opt = Iperf_ComputeModifiedBW(0, 1, self.iperf_profile, self.iperf_args)
|
||||
t = iperf_time*2.5
|
||||
with cls_cmd.getConnection('localhost') as local:
|
||||
local.run(f'mkdir -p {logPath}')
|
||||
with cls_cmd.getConnection(ue.getHost()) as cmd_ue, cls_cmd.getConnection(cn.getHost()) as cmd_svr:
|
||||
cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
|
||||
cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> /tmp/{server_filename} &', timeout=t)
|
||||
cmd_ue.run(f'rm {server_filename}', reportNonZero=False)
|
||||
cmd_ue.run(f'{ue.getCmdPrefix()} timeout -vk3 {t} iperf -B {ueIP} -s -u -i1 >> {server_filename} &', timeout=t)
|
||||
cmd_svr.run(f'{cn.getCmdPrefix()} timeout -vk3 {t} iperf -c {ueIP} -B {svrIP} {iperf_opt} -i1 >> /dev/null', timeout=t)
|
||||
localPath = f'{os.getcwd()}'
|
||||
# note: copy iperf2 log to the directory for log collection
|
||||
cmd_ue.copyin(f'/tmp/{server_filename}', f'{logPath}/{server_filename}')
|
||||
cmd_ue.run(f'rm /tmp/{server_filename}', reportNonZero=False)
|
||||
success, msg = Iperf_analyzeV2UDP(f'{logPath}/{server_filename}', self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
|
||||
local = archiveArtifact(cmd_ue, ctx, server_filename)
|
||||
success, msg = Iperf_analyzeV2UDP(local, self.iperf_bitrate_threshold, self.iperf_packetloss_threshold, target_bitrate)
|
||||
ue_header = f'UE {ue.getName()} ({ueIP})'
|
||||
logging.info(f'\u001B[1;37;45m iperf result for {ue_header}\u001B[0m')
|
||||
for l in msg.split('\n'):
|
||||
@@ -838,17 +816,17 @@ class OaiCiTest():
|
||||
global_status = CONST.OAI_UE_PROCESS_COULD_NOT_SYNC
|
||||
return global_status
|
||||
|
||||
def TerminateUE(self, HTML):
|
||||
def TerminateUE(self, ctx, HTML):
|
||||
ues = [cls_module.Module_UE(n.strip()) for n in self.ue_ids]
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=64) as executor:
|
||||
futures = [executor.submit(ue.terminate) for ue in ues]
|
||||
futures = [executor.submit(ue.terminate, ctx) for ue in ues]
|
||||
archives = [f.result() for f in futures]
|
||||
archive_info = [f'Log at: {a}' if a else 'No log available' for a in archives]
|
||||
messages = [f"UE {ue.getName()}: {log}" for (ue, log) in zip(ues, archive_info)]
|
||||
HTML.CreateHtmlTestRowQueue(f'N/A', 'OK', messages)
|
||||
return True
|
||||
|
||||
def DeployCoreNetwork(cn_id, HTML):
|
||||
def DeployCoreNetwork(cn_id, ctx, HTML):
|
||||
core_name = cn_id.strip()
|
||||
cn = cls_corenetwork.CoreNetwork(core_name)
|
||||
success, output = cn.deploy()
|
||||
@@ -862,41 +840,11 @@ class OaiCiTest():
|
||||
HTML.CreateHtmlTestRowQueue(core_name, 'KO', [msg])
|
||||
return success
|
||||
|
||||
def UndeployCoreNetwork(cn_id, HTML):
|
||||
# Ping, Iperf, DeployObject put logs into a path based on YAML. We
|
||||
# can't do this here (because there is no yaml), so hardcode a path for
|
||||
# "cn_logs" for the moment
|
||||
logPath = f'{os.getcwd()}/../cmake_targets/log/cn_logs'
|
||||
with cls_cmd.getConnection('localhost') as local:
|
||||
local.run(f'mkdir -p {logPath}', silent=True)
|
||||
def UndeployCoreNetwork(cn_id, ctx, HTML):
|
||||
core_name = cn_id.strip()
|
||||
cn = cls_corenetwork.CoreNetwork(core_name)
|
||||
logs, output = cn.undeploy(log_dir=logPath)
|
||||
logs, output = cn.undeploy(ctx=ctx)
|
||||
logging.info(f"undeployed core network {core_name}, logs {logs}, output:\n{output}")
|
||||
message = "Log files: " + ", ".join([os.path.basename(l) for l in logs])
|
||||
message = "Log files:\n" + "\n".join([os.path.basename(l) for l in logs])
|
||||
HTML.CreateHtmlTestRowQueue(core_name, 'OK', [message])
|
||||
return True
|
||||
|
||||
def LogCollectBuild(self,RAN):
|
||||
# Some pipelines are using "none" IP / Credentials
|
||||
# In that case, just forget about it
|
||||
if RAN.eNBIPAddress == 'none':
|
||||
sys.exit(0)
|
||||
|
||||
if (RAN.eNBIPAddress != '' and RAN.eNBUserName != '' and RAN.eNBPassword != ''):
|
||||
IPAddress = RAN.eNBIPAddress
|
||||
UserName = RAN.eNBUserName
|
||||
Password = RAN.eNBPassword
|
||||
SourceCodePath = RAN.eNBSourceCodePath
|
||||
else:
|
||||
sys.exit('Insufficient Parameter')
|
||||
with cls_cmd.getConnection(IPAddress) as cmd:
|
||||
d = f'{SourceCodePath}/cmake_targets'
|
||||
cmd.run(f'rm -f {d}/build.log.zip')
|
||||
cmd.run(f'cd {d} && zip -r build.log.zip build_log_*/*')
|
||||
|
||||
def ShowTestID(self):
|
||||
logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
|
||||
logging.info(f'\u001B[1m Test ID: {self.testCase_id} \u001B[0m')
|
||||
logging.info(f'\u001B[1m {self.desc} \u001B[0m')
|
||||
logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
|
||||
|
||||
@@ -43,6 +43,7 @@ from pathlib import Path
|
||||
import helpreadme as HELP
|
||||
import constants as CONST
|
||||
import cls_cmd
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Class Declaration
|
||||
@@ -80,7 +81,7 @@ class StaticCodeAnalysis():
|
||||
self.eNBPassword = ''
|
||||
self.eNBSourceCodePath = ''
|
||||
|
||||
def CppCheckAnalysis(self, HTML):
|
||||
def CppCheckAnalysis(self, ctx, HTML):
|
||||
if self.ranRepository == '' or self.ranBranch == '' or self.ranCommitID == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
@@ -103,7 +104,7 @@ class StaticCodeAnalysis():
|
||||
full_ran_repo_name = self.ranRepository + '.git'
|
||||
|
||||
cmd.cd(lSourcePath)
|
||||
logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
|
||||
logDir = f'{lSourcePath}/cmake_targets/log'
|
||||
cmd.run(f'mkdir -p {logDir}')
|
||||
cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
|
||||
cmd.run(f'sed -e "s@xenial@bionic@" {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.xenial > {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.bionic')
|
||||
@@ -112,31 +113,21 @@ class StaticCodeAnalysis():
|
||||
cmd.run(f'docker build --tag oai-cppcheck:focal --file {lSourcePath}/ci-scripts/docker/Dockerfile.cppcheck.focal . > {logDir}/cppcheck-focal.txt 2>&1')
|
||||
cmd.run('docker image rm oai-cppcheck:bionic oai-cppcheck:focal')
|
||||
|
||||
# Analyzing the logs
|
||||
cmd.copyin(f'{logDir}/cppcheck-bionic.txt', 'cppcheck-bionic.txt')
|
||||
cmd.copyin(f'{logDir}/cppcheck-focal.txt', 'cppcheck-focal.txt')
|
||||
bionic = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-bionic.txt')
|
||||
focal = archiveArtifact(cmd, ctx, f'{logDir}/cppcheck-focal.txt')
|
||||
cmd.close()
|
||||
|
||||
CCR = CppCheckResults()
|
||||
CCR_ref = CppCheckResults()
|
||||
vId = 0
|
||||
for variant in CCR.variants:
|
||||
refAvailable = False
|
||||
if self.ranAllowMerge:
|
||||
refFolder = str(Path.home()) + '/cppcheck-references'
|
||||
if (os.path.isfile(refFolder + '/cppcheck-'+ variant + '.txt')):
|
||||
refAvailable = True
|
||||
with open(refFolder + '/cppcheck-'+ variant + '.txt', 'r') as refFile:
|
||||
for line in refFile:
|
||||
ret = re.search(' (?P<nb_errors>[0-9\.]+) errors', str(line))
|
||||
if ret is not None:
|
||||
CCR_ref.nbErrors[vId] = int(ret.group('nb_errors'))
|
||||
ret = re.search(' (?P<nb_warnings>[0-9\.]+) warnings', str(line))
|
||||
if ret is not None:
|
||||
CCR_ref.nbWarnings[vId] = int(ret.group('nb_warnings'))
|
||||
if (os.path.isfile('./cppcheck-'+ variant + '.txt')):
|
||||
filename = ctx.baseFilename() + '-cppcheck-'+ variant + '.txt'
|
||||
logging.info(f"will check file '{filename}'")
|
||||
if not os.path.isfile(filename):
|
||||
raise FileNotFoundError(f"{filename} is not a file")
|
||||
else:
|
||||
xmlStart = False
|
||||
with open('./cppcheck-'+ variant + '.txt', 'r') as logfile:
|
||||
with open(filename, 'r') as logfile:
|
||||
for line in logfile:
|
||||
ret = re.search('cppcheck version="(?P<version>[0-9\.]+)"', str(line))
|
||||
if ret is not None:
|
||||
@@ -185,26 +176,6 @@ class StaticCodeAnalysis():
|
||||
vMsg += ' Wrong Scanf Nb Args: ' + str(CCR.nbWrongScanfArg[vId]) + '\n'
|
||||
for vLine in vMsg.split('\n'):
|
||||
logging.debug(vLine)
|
||||
if self.ranAllowMerge and refAvailable:
|
||||
if CCR_ref.nbErrors[vId] == CCR.nbErrors[vId]:
|
||||
logging.debug(' No change in number of errors')
|
||||
elif CCR_ref.nbErrors[vId] > CCR.nbErrors[vId]:
|
||||
logging.debug(' Good! Decrease in number of errors')
|
||||
else:
|
||||
logging.debug(' Bad! increase in number of errors')
|
||||
if CCR_ref.nbWarnings[vId] == CCR.nbWarnings[vId]:
|
||||
logging.debug(' No change in number of warnings')
|
||||
elif CCR_ref.nbWarnings[vId] > CCR.nbWarnings[vId]:
|
||||
logging.debug(' Good! Decrease in number of warnings')
|
||||
else:
|
||||
logging.debug(' Bad! increase in number of warnings')
|
||||
# Create new reference file
|
||||
if not self.ranAllowMerge:
|
||||
refFolder = str(Path.home()) + '/cppcheck-references'
|
||||
if not os.path.isdir(refFolder):
|
||||
os.mkdir(refFolder)
|
||||
with open(refFolder + '/cppcheck-'+ variant + '.txt', 'w') as refFile:
|
||||
refFile.write(vMsg)
|
||||
vId += 1
|
||||
|
||||
HTML.CreateHtmlTestRow('N/A', 'OK', CONST.ALL_PROCESSES_OK)
|
||||
@@ -213,7 +184,7 @@ class StaticCodeAnalysis():
|
||||
|
||||
return True
|
||||
|
||||
def LicenceAndFormattingCheck(self, HTML):
|
||||
def LicenceAndFormattingCheck(self, ctx, HTML):
|
||||
# Workspace is no longer recreated from scratch.
|
||||
# It implies that this method shall be called last within a build pipeline
|
||||
# where workspace is already created
|
||||
@@ -238,7 +209,7 @@ class StaticCodeAnalysis():
|
||||
else:
|
||||
check_options += f' --build-arg TARGET_BRANCH={self.ranTargetBranch}'
|
||||
|
||||
logDir = f'{lSourcePath}/cmake_targets/build_log_{self.testCase_id}'
|
||||
logDir = f'{lSourcePath}/cmake_targets/log/'
|
||||
cmd.run(f'mkdir -p {logDir}')
|
||||
cmd.run('docker image rm oai-formatting-check:latest')
|
||||
cmd.run(f'docker build --target oai-formatting-check --tag oai-formatting-check:latest {check_options} --file {lSourcePath}/ci-scripts/docker/Dockerfile.formatting.bionic {lSourcePath} > {logDir}/oai-formatting-check.txt 2>&1')
|
||||
@@ -247,12 +218,11 @@ class StaticCodeAnalysis():
|
||||
cmd.run('docker image prune --force')
|
||||
cmd.run('docker volume prune --force')
|
||||
|
||||
# Analyzing the logs
|
||||
cmd.copyin(f'{logDir}/oai-formatting-check.txt', 'oai-formatting-check.txt')
|
||||
file = archiveArtifact(cmd, ctx, f'{logDir}/oai-formatting-check.txt')
|
||||
cmd.close()
|
||||
|
||||
finalStatus = 0
|
||||
if (os.path.isfile('./oai-formatting-check.txt')):
|
||||
if (os.path.isfile(file)):
|
||||
analyzed = False
|
||||
nbFilesNotFormatted = 0
|
||||
listFiles = False
|
||||
@@ -263,7 +233,7 @@ class StaticCodeAnalysis():
|
||||
gnuGplLicenceFiles = []
|
||||
suspectLicence = False
|
||||
suspectLicenceFiles = []
|
||||
with open('./oai-formatting-check.txt', 'r') as logfile:
|
||||
with open(file, 'r') as logfile:
|
||||
for line in logfile:
|
||||
ret = re.search('./ci-scripts/checkCodingFormattingRules.sh', str(line))
|
||||
if ret is not None:
|
||||
|
||||
@@ -43,8 +43,10 @@ import cls_containerize #class Containerize for all container-based operations
|
||||
import cls_static_code_analysis #class for static code analysis
|
||||
import cls_cluster # class for building/deploying on cluster
|
||||
import cls_native # class for all native/source-based operations
|
||||
from cls_ci_helper import TestCaseCtx
|
||||
|
||||
import ran
|
||||
import cls_cmd
|
||||
import cls_oai_html
|
||||
|
||||
|
||||
@@ -91,7 +93,7 @@ def AssignParams(params_dict):
|
||||
setattr(RAN, key, value)
|
||||
setattr(HTML, key, value)
|
||||
|
||||
def ExecuteActionWithParam(action):
|
||||
def ExecuteActionWithParam(action, ctx):
|
||||
global RAN
|
||||
global HTML
|
||||
global CONTAINERS
|
||||
@@ -134,15 +136,15 @@ def ExecuteActionWithParam(action):
|
||||
if proxy_commit is not None:
|
||||
CONTAINERS.proxyCommit = proxy_commit
|
||||
if action == 'Build_eNB':
|
||||
success = cls_native.Native.Build(HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
|
||||
success = cls_native.Native.Build(ctx, HTML.testCase_id, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, RAN.Build_eNB_args)
|
||||
elif action == 'Build_Image':
|
||||
success = CONTAINERS.BuildImage(HTML)
|
||||
success = CONTAINERS.BuildImage(ctx, HTML)
|
||||
elif action == 'Build_Proxy':
|
||||
success = CONTAINERS.BuildProxy(HTML)
|
||||
success = CONTAINERS.BuildProxy(ctx, HTML)
|
||||
elif action == 'Build_Cluster_Image':
|
||||
success = CLUSTER.BuildClusterImage(HTML)
|
||||
success = CLUSTER.BuildClusterImage(ctx, HTML)
|
||||
elif action == 'Build_Run_Tests':
|
||||
success = CONTAINERS.BuildRunTests(HTML)
|
||||
success = CONTAINERS.BuildRunTests(ctx, HTML)
|
||||
|
||||
elif action == 'Initialize_eNB':
|
||||
datalog_rt_stats_file=test.findtext('rt_stats_cfg')
|
||||
@@ -204,7 +206,7 @@ def ExecuteActionWithParam(action):
|
||||
RAN.air_interface[RAN.eNB_instance] = 'lte-softmodem'
|
||||
else:
|
||||
RAN.air_interface[RAN.eNB_instance] = air_interface.lower() +'-softmodem'
|
||||
success = RAN.TerminateeNB(HTML)
|
||||
success = RAN.TerminateeNB(ctx, HTML)
|
||||
|
||||
elif action == 'Initialize_UE' or action == 'Attach_UE' or action == 'Detach_UE' or action == 'Terminate_UE' or action == 'CheckStatusUE' or action == 'DataEnable_UE' or action == 'DataDisable_UE':
|
||||
CiTestObj.ue_ids = test.findtext('id').split(' ')
|
||||
@@ -226,7 +228,7 @@ def ExecuteActionWithParam(action):
|
||||
elif action == 'Detach_UE':
|
||||
success = CiTestObj.DetachUE(HTML)
|
||||
elif action == 'Terminate_UE':
|
||||
success = CiTestObj.TerminateUE(HTML)
|
||||
success = CiTestObj.TerminateUE(ctx, HTML)
|
||||
elif action == 'CheckStatusUE':
|
||||
success = CiTestObj.CheckStatusUE(HTML)
|
||||
elif action == 'DataEnable_UE':
|
||||
@@ -253,7 +255,7 @@ def ExecuteActionWithParam(action):
|
||||
else:
|
||||
CiTestObj.nodes = [None] * len(CiTestObj.ue_ids)
|
||||
ping_rttavg_threshold = test.findtext('ping_rttavg_threshold') or ''
|
||||
success = CiTestObj.Ping(HTML, CONTAINERS)
|
||||
success = CiTestObj.Ping(ctx, HTML)
|
||||
|
||||
elif action == 'Iperf' or action == 'Iperf2_Unidir':
|
||||
CiTestObj.iperf_args = test.findtext('iperf_args')
|
||||
@@ -284,9 +286,9 @@ def ExecuteActionWithParam(action):
|
||||
logging.error('test-case has wrong option ' + CiTestObj.iperf_options)
|
||||
CiTestObj.iperf_options = 'check'
|
||||
if action == 'Iperf':
|
||||
success = CiTestObj.Iperf(HTML, CONTAINERS)
|
||||
success = CiTestObj.Iperf(ctx, HTML)
|
||||
elif action == 'Iperf2_Unidir':
|
||||
success = CiTestObj.Iperf2_Unidir(HTML, CONTAINERS)
|
||||
success = CiTestObj.Iperf2_Unidir(ctx, HTML)
|
||||
|
||||
elif action == 'IdleSleep':
|
||||
st = test.findtext('idle_sleep_time_in_sec') or "5"
|
||||
@@ -295,12 +297,12 @@ def ExecuteActionWithParam(action):
|
||||
elif action == 'Deploy_Run_PhySim':
|
||||
oc_release = test.findtext('oc_release')
|
||||
svr_id = test.findtext('svr_id') or None
|
||||
success = CLUSTER.deploy_oc_physim(HTML, oc_release, svr_id)
|
||||
success = CLUSTER.deploy_oc_physim(ctx, HTML, oc_release, svr_id)
|
||||
|
||||
elif action == 'DeployCoreNetwork' or action == 'UndeployCoreNetwork':
|
||||
cn_id = test.findtext('cn_id')
|
||||
core_op = getattr(cls_oaicitest.OaiCiTest, action)
|
||||
success = core_op(cn_id, HTML)
|
||||
success = core_op(cn_id, ctx, HTML)
|
||||
|
||||
elif action == 'Deploy_Object' or action == 'Undeploy_Object' or action == "Create_Workspace":
|
||||
eNB_instance=test.findtext('eNB_instance')
|
||||
@@ -328,9 +330,9 @@ def ExecuteActionWithParam(action):
|
||||
CONTAINERS.num_attempts = int(test.findtext('num_attempts') or 1)
|
||||
CONTAINERS.deploymentTag = cls_containerize.CreateTag(CONTAINERS.ranCommitID, CONTAINERS.ranBranch, CONTAINERS.ranAllowMerge)
|
||||
if action == 'Deploy_Object':
|
||||
success = CONTAINERS.DeployObject(HTML)
|
||||
success = CONTAINERS.DeployObject(ctx, HTML)
|
||||
elif action == 'Undeploy_Object':
|
||||
success = CONTAINERS.UndeployObject(HTML, RAN)
|
||||
success = CONTAINERS.UndeployObject(ctx, HTML, RAN)
|
||||
elif action == 'Create_Workspace':
|
||||
if force_local:
|
||||
# Do not create a working directory when running locally. Current repo directory will be used
|
||||
@@ -341,13 +343,13 @@ def ExecuteActionWithParam(action):
|
||||
physim_options = test.findtext('physim_run_args')
|
||||
physim_test = test.findtext('physim_test')
|
||||
physim_threshold = test.findtext('physim_time_threshold') or 'inf'
|
||||
success = cls_native.Native.Run_Physim(HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
|
||||
success = cls_native.Native.Run_Physim(ctx, HTML, RAN.eNBIPAddress, RAN.eNBSourceCodePath, physim_options, physim_test, physim_threshold)
|
||||
|
||||
elif action == 'LicenceAndFormattingCheck':
|
||||
success = SCA.LicenceAndFormattingCheck(HTML)
|
||||
success = SCA.LicenceAndFormattingCheck(ctx, HTML)
|
||||
|
||||
elif action == 'Cppcheck_Analysis':
|
||||
success = SCA.CppCheckAnalysis(HTML)
|
||||
success = SCA.CppCheckAnalysis(ctx, HTML)
|
||||
|
||||
elif action == 'Push_Local_Registry':
|
||||
svr_id = test.findtext('svr_id')
|
||||
@@ -407,10 +409,11 @@ def test_in_list(test, list):
|
||||
def receive_signal(signum, frame):
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
|
||||
|
||||
|
||||
def ShowTestID(ctx, desc):
|
||||
logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
|
||||
logging.info(f'\u001B[1m Test ID: {ctx.test_id} (#{ctx.count}) \u001B[0m')
|
||||
logging.info(f'\u001B[1m {desc} \u001B[0m')
|
||||
logging.info(f'\u001B[1m----------------------------------------\u001B[0m')
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# MAIN PART
|
||||
@@ -466,15 +469,7 @@ if py_param_file_present == True:
|
||||
cwd = os.getcwd()
|
||||
|
||||
if re.match('^TerminateeNB$', mode, re.IGNORECASE):
|
||||
if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
sys.exit('Insufficient Parameter')
|
||||
if RAN.eNBIPAddress == 'none':
|
||||
sys.exit(0)
|
||||
RAN.eNB_instance=0
|
||||
RAN.eNB_serverId[0]='0'
|
||||
RAN.eNBSourceCodePath='/tmp/'
|
||||
RAN.TerminateeNB(HTML)
|
||||
logging.warning("Option TerminateeNB ignored")
|
||||
elif re.match('^TerminateHSS$', mode, re.IGNORECASE):
|
||||
logging.warning("Option TerminateHSS ignored")
|
||||
elif re.match('^TerminateMME$', mode, re.IGNORECASE):
|
||||
@@ -482,12 +477,7 @@ elif re.match('^TerminateMME$', mode, re.IGNORECASE):
|
||||
elif re.match('^TerminateSPGW$', mode, re.IGNORECASE):
|
||||
logging.warning("Option TerminateSPGW ignored")
|
||||
elif re.match('^LogCollectBuild$', mode, re.IGNORECASE):
|
||||
if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
|
||||
logging.warning("nothing to collect (eNBIPAddress/eNBUserName/eNBPassword/eNBSourceCodePath is '')")
|
||||
sys.exit(0)
|
||||
if RAN.eNBIPAddress == 'none':
|
||||
sys.exit(0)
|
||||
CiTestObj.LogCollectBuild(RAN)
|
||||
logging.warning("Option LogCollectBuild ignored")
|
||||
elif re.match('^LogCollecteNB$', mode, re.IGNORECASE):
|
||||
if RAN.eNBIPAddress == '' or RAN.eNBUserName == '' or RAN.eNBPassword == '' or RAN.eNBSourceCodePath == '':
|
||||
HELP.GenericHelp(CONST.Version)
|
||||
@@ -565,6 +555,16 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
|
||||
else:
|
||||
xml_test_file = cwd + "/" + CiTestObj.testXMLfiles[0]
|
||||
|
||||
# directory where all log artifacts will be placed
|
||||
logPath = f"{cwd}/../cmake_targets/log/{CiTestObj.testXMLfiles[0].split('/')[-1]}.d"
|
||||
# we run from within ci-scripts, but the logPath is absolute, so replace
|
||||
# the ci-scripts/..; if it does not exist, nothing will happen
|
||||
logPath = logPath.replace(r'/ci-scripts/..', '')
|
||||
logging.info(f"placing all artifacts for this run in {logPath}/")
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f"rm -rf {logPath}")
|
||||
c.run(f"mkdir -p {logPath}")
|
||||
|
||||
xmlTree = ET.parse(xml_test_file)
|
||||
xmlRoot = xmlTree.getroot()
|
||||
|
||||
@@ -615,29 +615,32 @@ elif re.match('^TesteNB$', mode, re.IGNORECASE) or re.match('^TestUE$', mode, re
|
||||
task_set_succeeded = True
|
||||
HTML.startTime=int(round(time.time() * 1000))
|
||||
|
||||
i = 0
|
||||
for test_case_id in todo_tests:
|
||||
for test in all_tests:
|
||||
id = test.get('id')
|
||||
if test_case_id != id:
|
||||
continue
|
||||
i += 1
|
||||
CiTestObj.testCase_id = id
|
||||
ctx = TestCaseCtx(i, int(id), logPath)
|
||||
HTML.testCase_id=CiTestObj.testCase_id
|
||||
CiTestObj.desc = test.findtext('desc')
|
||||
desc = test.findtext('desc')
|
||||
always_exec = test.findtext('always_exec') in ['True', 'true', 'Yes', 'yes']
|
||||
may_fail = test.findtext('may_fail') in ['True', 'true', 'Yes', 'yes']
|
||||
HTML.desc=CiTestObj.desc
|
||||
HTML.desc = desc
|
||||
action = test.findtext('class')
|
||||
if (CheckClassValidity(xml_class_list, action, id) == False):
|
||||
task_set_succeeded = False
|
||||
continue
|
||||
CiTestObj.ShowTestID()
|
||||
ShowTestID(ctx, desc)
|
||||
if not task_set_succeeded and not always_exec:
|
||||
msg = f"skipping test due to prior error"
|
||||
logging.warning(msg)
|
||||
HTML.CreateHtmlTestRowQueue(msg, "SKIP", [])
|
||||
break
|
||||
try:
|
||||
test_succeeded = ExecuteActionWithParam(action)
|
||||
test_succeeded = ExecuteActionWithParam(action, ctx)
|
||||
if not test_succeeded and may_fail:
|
||||
logging.warning(f"test ID {test_case_id} action {action} may or may not fail, proceeding despite error")
|
||||
elif not test_succeeded:
|
||||
|
||||
@@ -46,6 +46,7 @@ import cls_cmd
|
||||
import sshconnection as SSH
|
||||
import helpreadme as HELP
|
||||
import constants as CONST
|
||||
from cls_ci_helper import archiveArtifact
|
||||
|
||||
#-----------------------------------------------------------
|
||||
# Class Declaration
|
||||
@@ -124,7 +125,6 @@ class RANManagement():
|
||||
|
||||
self.testCase_id = HTML.testCase_id
|
||||
mySSH = SSH.SSHConnection()
|
||||
cwd = os.getcwd()
|
||||
|
||||
mySSH.open(lIpAddr, lUserName, lPassWord)
|
||||
mySSH.command('cd ' + lSourcePath, '\$', 5)
|
||||
@@ -249,42 +249,7 @@ class RANManagement():
|
||||
logging.debug('\u001B[1m Initialize eNB/gNB Completed\u001B[0m')
|
||||
return enbDidSync
|
||||
|
||||
def CheckeNBProcess(self, status_queue):
|
||||
# At least the instance 0 SHALL be on!
|
||||
if self.eNBstatuses[0] == 0:
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lUserName = self.eNBUserName
|
||||
lPassWord = self.eNBPassword
|
||||
elif self.eNBstatuses[0] == 1:
|
||||
lIpAddr = self.eNB1IPAddress
|
||||
lUserName = self.eNB1UserName
|
||||
lPassWord = self.eNB1Password
|
||||
elif self.eNBstatuses[0] == 2:
|
||||
lIpAddr = self.eNB2IPAddress
|
||||
lUserName = self.eNB2UserName
|
||||
lPassWord = self.eNB2Password
|
||||
else:
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lUserName = self.eNBUserName
|
||||
lPassWord = self.eNBPassword
|
||||
mySSH = SSH.SSHConnection()
|
||||
mySSH.open(lIpAddr, lUserName, lPassWord)
|
||||
if self.air_interface[self.eNB_instance] == '':
|
||||
pattern = 'softmodem'
|
||||
else:
|
||||
pattern = self.air_interface[self.eNB_instance]
|
||||
mySSH.command('stdbuf -o0 ps -aux | grep --color=never ' + pattern + ' | grep -v grep', '\$', 5)
|
||||
result = re.search(pattern, mySSH.getBefore())
|
||||
success = result is not None
|
||||
if not success:
|
||||
logging.debug('\u001B[1;37;41m eNB Process Not Found! \u001B[0m')
|
||||
status_queue.put(CONST.ENB_PROCESS_FAILED)
|
||||
else:
|
||||
status_queue.put(CONST.ENB_PROCESS_OK)
|
||||
mySSH.close()
|
||||
return success
|
||||
|
||||
def TerminateeNB(self, HTML):
|
||||
def TerminateeNB(self, ctx, HTML):
|
||||
if self.eNB_serverId[self.eNB_instance] == '0':
|
||||
lIpAddr = self.eNBIPAddress
|
||||
lUserName = self.eNBUserName
|
||||
@@ -356,13 +321,11 @@ class RANManagement():
|
||||
fileToAnalyze = self.eNBLogFiles[int(self.eNB_instance)]
|
||||
self.eNBLogFiles[int(self.eNB_instance)] = ''
|
||||
if analyzeFile:
|
||||
#*stats.log files + pickle + png
|
||||
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*stats.log', '.')
|
||||
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.pickle', '.')
|
||||
mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/*.png', '.')
|
||||
#
|
||||
copyin_res = mySSH.copyin(lIpAddr, lUserName, lPassWord, lSourcePath + '/cmake_targets/' + fileToAnalyze, '.')
|
||||
if (copyin_res == -1):
|
||||
with cls_cmd.getConnection(lIpAddr) as cmd:
|
||||
file = archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/' + fileToAnalyze)
|
||||
archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrL1_stats.log')
|
||||
archiveArtifact(cmd, ctx, lSourcePath + '/cmake_targets/nrMAC_stats.log')
|
||||
if file is None:
|
||||
logging.debug('\u001B[1;37;41m Could not copy ' + nodeB_prefix + 'NB logfile to analyze it! \u001B[0m')
|
||||
HTML.htmleNBFailureMsg='Could not copy ' + nodeB_prefix + 'NB logfile to analyze it!'
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', CONST.ENB_PROCESS_NOLOGFILE_TO_ANALYZE)
|
||||
@@ -375,8 +338,8 @@ class RANManagement():
|
||||
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrL1_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
|
||||
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './nrMAC_stats.log', self.eNBSourceCodePath + '/cmake_targets/')
|
||||
mySSH.copyout(self.eNBIPAddress, self.eNBUserName, self.eNBPassword, './' + fileToAnalyze, self.eNBSourceCodePath + '/cmake_targets/')
|
||||
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + fileToAnalyze)
|
||||
logStatus = self.AnalyzeLogFile_eNB(fileToAnalyze, HTML, self.ran_checkers)
|
||||
logging.debug('\u001B[1m Analyzing ' + nodeB_prefix + 'NB logfile \u001B[0m ' + file)
|
||||
logStatus = self.AnalyzeLogFile_eNB(file, HTML, self.ran_checkers)
|
||||
if (logStatus < 0):
|
||||
HTML.CreateHtmlTestRow('N/A', 'KO', logStatus)
|
||||
#display rt stats for gNB only
|
||||
@@ -734,11 +697,16 @@ class RANManagement():
|
||||
datalog_rt_stats = yaml.load(f,Loader=yaml.FullLoader)
|
||||
rt_keys = datalog_rt_stats['Ref'] #we use the keys from the Ref field
|
||||
|
||||
if os.path.isfile('./nrL1_stats.log') and os.path.isfile('./nrMAC_stats.log'):
|
||||
# nrL1_stats.log/nrMAC_stats.log should be in the same directory as main log file
|
||||
# currently the link is only implicit as below based on pattern matching
|
||||
# I will rework this to give the file explicitly
|
||||
l1_stats_fn = re.sub(r'enb_[0-9]+', 'nrL1_stats', eNBlogFile)
|
||||
mac_stats_fn = re.sub(r'enb_[0-9]+', 'nrMAC_stats', eNBlogFile)
|
||||
if os.path.isfile(l1_stats_fn) and os.path.isfile(mac_stats_fn):
|
||||
# don't use CI-nrL1_stats.log, as this will increase the processing time for
|
||||
# no reason, we just need the last occurence
|
||||
nrL1_stats = open('./nrL1_stats.log', 'r')
|
||||
nrMAC_stats = open('./nrMAC_stats.log', 'r')
|
||||
nrL1_stats = open(l1_stats_fn, 'r')
|
||||
nrMAC_stats = open(mac_stats_fn, 'r')
|
||||
for line in nrL1_stats.readlines():
|
||||
for k in rt_keys:
|
||||
result = re.search(k, line)
|
||||
@@ -758,7 +726,7 @@ class RANManagement():
|
||||
nrL1_stats.close()
|
||||
nrMAC_stats.close()
|
||||
else:
|
||||
logging.debug("NR Stats files for RT analysis not found")
|
||||
logging.debug(f"NR Stats files for RT analysis not found: {l1_stats_fn}, {mac_stats_fn}")
|
||||
|
||||
#stdout log file and stat log files analysis completed
|
||||
logging.debug(' File analysis (stdout, stats) completed')
|
||||
|
||||
34
ci-scripts/tests/config/test_module_infra.yaml
Normal file
34
ci-scripts/tests/config/test_module_infra.yaml
Normal file
@@ -0,0 +1,34 @@
|
||||
test:
|
||||
Host: localhost
|
||||
InitScript: ""
|
||||
TermScript: ""
|
||||
AttachScript: "true"
|
||||
DetachScript: "true"
|
||||
NetworkScript: ip a show dev lo
|
||||
IF: lo
|
||||
MTU: 65536
|
||||
|
||||
test-fail:
|
||||
Host: localhost
|
||||
InitScript: ""
|
||||
TermScript: ""
|
||||
AttachScript: "false"
|
||||
DetachScript: "true"
|
||||
NetworkScript: ip a show dev TESTDEVICEDOESNOTEXIST
|
||||
IF: TESTDEVICEDOESNOTEXIST
|
||||
MTU: 123
|
||||
|
||||
test-trace:
|
||||
Host: localhost
|
||||
InitScript: ""
|
||||
TermScript: ""
|
||||
AttachScript: "true"
|
||||
DetachScript: "true"
|
||||
Tracing:
|
||||
# tttf: temp-trace-test-file
|
||||
Start: "rm -f /tmp/tttf; for i in {1..8}; do echo $i >> /tmp/tttf; sleep 1; done &"
|
||||
Stop: "while [[ $(wc -l /tmp/tttf) < 4 ]]; do sleep 1; done"
|
||||
Collect: "cp /tmp/tttf %%log_dir%%"
|
||||
NetworkScript: ip a show dev lo
|
||||
IF: lo
|
||||
MTU: 65536
|
||||
@@ -6,14 +6,22 @@ logging.basicConfig(
|
||||
format="[%(asctime)s] %(levelname)8s: %(message)s"
|
||||
)
|
||||
import os
|
||||
import tempfile
|
||||
|
||||
import unittest
|
||||
|
||||
sys.path.append('./') # to find OAI imports below
|
||||
import cls_corenetwork
|
||||
from cls_ci_helper import TestCaseCtx
|
||||
import cls_cmd
|
||||
|
||||
class TestCoreNetwork(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
|
||||
def tearDown(self):
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'rm -rf {self.ctx.logPath}')
|
||||
|
||||
def test_simple_core(self):
|
||||
c = cls_corenetwork.CoreNetwork("test", filename="tests/config/test_core_infra.yaml")
|
||||
success, output = c.deploy()
|
||||
@@ -22,14 +30,14 @@ class TestCoreNetwork(unittest.TestCase):
|
||||
# should not have it set in config
|
||||
self.assertTrue(c.runIperf3Server())
|
||||
self.assertEqual(c.getIP(), "127.0.0.1")
|
||||
log_files, output = c.undeploy(log_dir="/tmp")
|
||||
log_files, output = c.undeploy(self.ctx)
|
||||
self.assertEqual(output, "undeploy")
|
||||
with cls_cmd.LocalCmd() as cmd:
|
||||
# there must be one log file for this (test) core
|
||||
self.assertEqual(len(log_files), 1)
|
||||
# test core writes to %%log_dir%%/logs, which expands to /tmp/logs
|
||||
# undeploy uses archiveArtifact(), which writes to {prefix}-logs
|
||||
l = log_files[0]
|
||||
self.assertEqual(l, "/tmp/logs")
|
||||
self.assertEqual(l, f"{self.ctx.baseFilename()}-logs")
|
||||
ret = cmd.run(f"cat {l}")
|
||||
self.assertEqual(ret.returncode, 0) # command must succeed
|
||||
self.assertEqual(ret.stdout, "logs") # output should be "logs"
|
||||
@@ -39,14 +47,14 @@ class TestCoreNetwork(unittest.TestCase):
|
||||
success, _ = c.deploy()
|
||||
self.assertTrue(success)
|
||||
self.assertFalse(c.runIperf3Server())
|
||||
c.undeploy(log_dir="/tmp")
|
||||
c.undeploy(None)
|
||||
|
||||
def test_core_fail(self):
|
||||
c = cls_corenetwork.CoreNetwork("test_fail", filename="tests/config/test_core_infra.yaml")
|
||||
success, _ = c.deploy()
|
||||
self.assertFalse(success)
|
||||
# undeployment should still work
|
||||
c.undeploy(log_dir="/tmp")
|
||||
c.undeploy(None)
|
||||
|
||||
def test_core_script(self):
|
||||
c = cls_corenetwork.CoreNetwork("test_script", filename="tests/config/test_core_infra.yaml")
|
||||
|
||||
@@ -6,6 +6,7 @@ logging.basicConfig(
|
||||
format="[%(asctime)s] %(levelname)8s: %(message)s"
|
||||
)
|
||||
import os
|
||||
import tempfile
|
||||
os.system(f'rm -rf cmake_targets')
|
||||
os.system(f'mkdir -p cmake_targets/log')
|
||||
import unittest
|
||||
@@ -14,6 +15,7 @@ sys.path.append('./') # to find OAI imports below
|
||||
import cls_oai_html
|
||||
import cls_oaicitest
|
||||
import cls_containerize
|
||||
from cls_ci_helper import TestCaseCtx
|
||||
import ran
|
||||
import cls_cmd
|
||||
|
||||
@@ -48,12 +50,16 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
self.cont.eNBPassword = None
|
||||
self.cont.eNBSourceCodePath = os.getcwd()
|
||||
self.cont.num_attempts = 3
|
||||
self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
|
||||
def tearDown(self):
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'rm -rf {self.ctx.logPath}')
|
||||
|
||||
def test_deploy(self):
|
||||
self.cont.yamlPath[0] = 'tests/simple-dep/'
|
||||
self.cont.deploymentTag = "noble"
|
||||
deploy = self.cont.DeployObject(self.html)
|
||||
undeploy = self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy = self.cont.DeployObject(self.ctx, self.html)
|
||||
undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertTrue(deploy)
|
||||
self.assertTrue(undeploy)
|
||||
|
||||
@@ -61,8 +67,8 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
# fails reliably
|
||||
old = self.cont.yamlPath
|
||||
self.cont.yamlPath[0] = 'tests/simple-fail/'
|
||||
deploy = self.cont.DeployObject(self.html)
|
||||
self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy = self.cont.DeployObject(self.ctx, self.html)
|
||||
self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertFalse(deploy)
|
||||
self.cont.yamlPath = old
|
||||
|
||||
@@ -70,8 +76,8 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
# fails reliably
|
||||
old = self.cont.yamlPath
|
||||
self.cont.yamlPath[0] = 'tests/simple-fail-2svc/'
|
||||
deploy = self.cont.DeployObject(self.html)
|
||||
self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy = self.cont.DeployObject(self.ctx, self.html)
|
||||
self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertFalse(deploy)
|
||||
self.cont.yamlPath = old
|
||||
|
||||
@@ -79,8 +85,8 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
|
||||
self.cont.services[0] = "oai-gnb"
|
||||
self.cont.deploymentTag = 'develop-12345678'
|
||||
deploy = self.cont.DeployObject(self.html)
|
||||
undeploy = self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy = self.cont.DeployObject(self.ctx, self.html)
|
||||
undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertTrue(deploy)
|
||||
self.assertTrue(undeploy)
|
||||
|
||||
@@ -88,8 +94,8 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
|
||||
self.cont.services[0] = "oai-gnb oai-nr-ue"
|
||||
self.cont.deploymentTag = 'develop-12345678'
|
||||
deploy = self.cont.DeployObject(self.html)
|
||||
undeploy = self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy = self.cont.DeployObject(self.ctx, self.html)
|
||||
undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertTrue(deploy)
|
||||
self.assertTrue(undeploy)
|
||||
|
||||
@@ -97,10 +103,10 @@ class TestDeploymentMethods(unittest.TestCase):
|
||||
self.cont.yamlPath[0] = 'yaml_files/5g_rfsimulator_tdd_dora'
|
||||
self.cont.services[0] = "oai-gnb"
|
||||
self.cont.deploymentTag = 'develop-12345678'
|
||||
deploy1 = self.cont.DeployObject(self.html)
|
||||
deploy1 = self.cont.DeployObject(self.ctx, self.html)
|
||||
self.cont.services[0] = "oai-nr-ue"
|
||||
deploy2 = self.cont.DeployObject(self.html)
|
||||
undeploy = self.cont.UndeployObject(self.html, self.ran)
|
||||
deploy2 = self.cont.DeployObject(self.ctx, self.html)
|
||||
undeploy = self.cont.UndeployObject(self.ctx, self.html, self.ran)
|
||||
self.assertTrue(deploy1)
|
||||
self.assertTrue(deploy2)
|
||||
self.assertTrue(undeploy)
|
||||
|
||||
64
ci-scripts/tests/module.py
Normal file
64
ci-scripts/tests/module.py
Normal file
@@ -0,0 +1,64 @@
|
||||
import sys
|
||||
import logging
|
||||
import tempfile
|
||||
logging.basicConfig(
|
||||
level=logging.DEBUG,
|
||||
stream=sys.stdout,
|
||||
format="[%(asctime)s] %(levelname)8s: %(message)s"
|
||||
)
|
||||
import os
|
||||
|
||||
import unittest
|
||||
|
||||
sys.path.append('./') # to find OAI imports below
|
||||
import cls_module
|
||||
from cls_ci_helper import TestCaseCtx
|
||||
import cls_cmd
|
||||
|
||||
class TestModule(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
|
||||
def tearDown(self):
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'rm -rf {self.ctx.logPath}')
|
||||
|
||||
def test_simple_module(self):
|
||||
c = cls_module.Module_UE("test", filename="tests/config/test_module_infra.yaml")
|
||||
self.assertFalse(c.trace)
|
||||
success = c.initialize()
|
||||
self.assertTrue(success)
|
||||
ip = c.attach()
|
||||
self.assertEqual(ip, "127.0.0.1")
|
||||
self.assertTrue(c.checkMTU())
|
||||
c.detach()
|
||||
logs = c.terminate()
|
||||
self.assertEqual(logs, None) # no tracing
|
||||
|
||||
@unittest.skip("this test takes long: it verifies the UE cannot attach")
|
||||
def test_simple_fail(self):
|
||||
c = cls_module.Module_UE("test-fail", filename="tests/config/test_module_infra.yaml")
|
||||
success = c.initialize()
|
||||
self.assertTrue(success)
|
||||
ip = c.attach()
|
||||
self.assertEqual(ip, None)
|
||||
self.assertFalse(c.checkMTU())
|
||||
c.detach()
|
||||
logs = c.terminate()
|
||||
self.assertEqual(logs, None) # no tracing
|
||||
|
||||
def test_simple_trace(self):
|
||||
c = cls_module.Module_UE("test-trace", filename="tests/config/test_module_infra.yaml")
|
||||
self.assertTrue(c.trace)
|
||||
success = c.initialize()
|
||||
self.assertTrue(success)
|
||||
ip = c.attach()
|
||||
self.assertEqual(ip, "127.0.0.1")
|
||||
self.assertTrue(c.checkMTU())
|
||||
c.detach()
|
||||
tmp = tempfile.mkdtemp()
|
||||
log_file = c.terminate(self.ctx)
|
||||
# undeploy uses archiveArtifact(), which writes to {prefix}-logs
|
||||
self.assertEqual(log_file, [f"{self.ctx.baseFilename()}-tttf"]) # matches test-trace UE collection file
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest.main()
|
||||
@@ -9,11 +9,13 @@ import os
|
||||
os.system(f'rm -rf cmake_targets')
|
||||
os.system(f'mkdir -p cmake_targets/log')
|
||||
import unittest
|
||||
import tempfile
|
||||
|
||||
sys.path.append('./') # to find OAI imports below
|
||||
import cls_oai_html
|
||||
from cls_ci_helper import TestCaseCtx
|
||||
import cls_oaicitest
|
||||
import cls_containerize
|
||||
import cls_cmd
|
||||
|
||||
class TestPingIperf(unittest.TestCase):
|
||||
def setUp(self):
|
||||
@@ -22,7 +24,10 @@ class TestPingIperf(unittest.TestCase):
|
||||
self.ci = cls_oaicitest.OaiCiTest()
|
||||
self.ci.ue_ids = ["test"]
|
||||
self.ci.nodes = ["localhost"]
|
||||
self.cont = cls_containerize.Containerize()
|
||||
self.ctx = TestCaseCtx.Default(tempfile.mkdtemp())
|
||||
def tearDown(self):
|
||||
with cls_cmd.LocalCmd() as c:
|
||||
c.run(f'rm -rf {self.ctx.logPath}')
|
||||
|
||||
def test_ping(self):
|
||||
self.ci.ping_args = "-c3"
|
||||
@@ -30,7 +35,7 @@ class TestPingIperf(unittest.TestCase):
|
||||
self.ci.svr_id = "test"
|
||||
infra_file = "tests/config/infra_ping_iperf.yaml"
|
||||
# TODO Should need nothing but options and UE(s) to use
|
||||
success = self.ci.Ping(self.html, self.cont, infra_file=infra_file)
|
||||
success = self.ci.Ping(self.ctx, self.html, infra_file=infra_file)
|
||||
self.assertTrue(success)
|
||||
|
||||
def test_iperf(self):
|
||||
@@ -45,7 +50,7 @@ class TestPingIperf(unittest.TestCase):
|
||||
self.ci.iperf_profile = "balanced"
|
||||
infra_file = "tests/config/infra_ping_iperf.yaml"
|
||||
# TODO Should need nothing but options and UE(s) to use
|
||||
success = self.ci.Iperf(self.html, self.cont, infra_file=infra_file)
|
||||
success = self.ci.Iperf(self.ctx, self.html, infra_file=infra_file)
|
||||
self.assertTrue(success)
|
||||
|
||||
def test_iperf2_unidir(self):
|
||||
@@ -57,7 +62,7 @@ class TestPingIperf(unittest.TestCase):
|
||||
self.ci.iperf_profile = "balanced"
|
||||
infra_file = "tests/config/infra_ping_iperf.yaml"
|
||||
# TODO Should need nothing but options and UE(s) to use
|
||||
success = self.ci.Iperf2_Unidir(self.html, self.cont, infra_file=infra_file)
|
||||
success = self.ci.Iperf2_Unidir(self.ctx, self.html, infra_file=infra_file)
|
||||
self.assertTrue(success)
|
||||
|
||||
def test_iperf_highrate(self):
|
||||
|
||||
@@ -26,7 +26,6 @@
|
||||
<htmlTabIcon>wrench</htmlTabIcon>
|
||||
<TestCaseRequestedList>
|
||||
111111
|
||||
100001
|
||||
800813
|
||||
000001
|
||||
000002
|
||||
|
||||
@@ -26,7 +26,6 @@
|
||||
<htmlTabIcon>wrench</htmlTabIcon>
|
||||
<TestCaseRequestedList>
|
||||
111111
|
||||
100001
|
||||
800813
|
||||
000001
|
||||
000002
|
||||
|
||||
@@ -47,6 +47,7 @@
|
||||
<testCase id="100000">
|
||||
<class>Custom_Command</class>
|
||||
<desc>Reboot USRP</desc>
|
||||
<may_fail>true</may_fail>
|
||||
<node>caracal</node>
|
||||
<command>ssh root@192.168.20.2 reboot</command>
|
||||
</testCase>
|
||||
|
||||
@@ -78,7 +78,7 @@ RUN /bin/sh oaienv && \
|
||||
cd cmake_targets && \
|
||||
mkdir -p log && \
|
||||
./build_oai \
|
||||
--gNB \
|
||||
--ninja --gNB \
|
||||
--build-lib "telnetsrv enbscope uescope nrscope" \
|
||||
-t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
|
||||
--build-e2 --cmake-opt -DXAPP_MULTILANGUAGE=OFF --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
|
||||
|
||||
@@ -58,7 +58,7 @@ RUN /bin/sh oaienv && \
|
||||
mkdir -p log && \
|
||||
scl enable gcc-toolset-13 "\
|
||||
./build_oai \
|
||||
--gNB \
|
||||
--ninja --gNB \
|
||||
--build-lib 'telnetsrv enbscope uescope nrscope' \
|
||||
-t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
|
||||
--build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION" &&\
|
||||
|
||||
@@ -68,7 +68,7 @@ RUN /bin/sh oaienv && \
|
||||
cd cmake_targets && \
|
||||
mkdir -p log && \
|
||||
./build_oai \
|
||||
--gNB \
|
||||
--ninja --gNB \
|
||||
--build-lib "telnetsrv enbscope uescope nrscope" \
|
||||
-t oran_fhlib_5g --cmake-opt -Dxran_LOCATION=/opt/phy/fhi_lib/lib \
|
||||
--build-e2 --cmake-opt -DKPM_VERSION=$KPM_VERSION --cmake-opt -DE2AP_VERSION=$E2AP_VERSION \
|
||||
|
||||
Reference in New Issue
Block a user