maple/prplos
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
a49a1afa0c
prplos/.gitlab/testbed.yml
Petr Štetiar b0aa5e3e34 ci: cram: size component job timeouts 
Component jobs currently inherit the full-tier 85-minute timeout.
Measured runs peak at 35.8 minutes for components and 80.8 minutes for
the full tier.

Set each tier to the smallest five-minute multiple at least four
minutes above its maximum observed duration. This gives component jobs
40 minutes and retains 85 minutes for full jobs.

Keep the full-job limit explicit because those jobs would otherwise
inherit the new component limit. A hung component job now releases its
device roughly 45 minutes sooner.

Assisted-by: claude:opus-5:orchestrator
Assisted-by: codex:gpt-5.6-sol:medium-fast
References: CI-7
Signed-off-by: Petr Štetiar <petr.stetiar@prplfoundation.org>
2026-08-19 10:14:27 +00:00

289 lines
11 KiB
YAML
Raw Blame History

.testbed:
stage: run
image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
timeout: 87 minutes
variables:
DUT_WAN_INTERFACE: eth1
TESTBED_MNG_INTERFACE: enp1s0
TESTBED_LAN_INTERFACE: enp2s0
TESTBED_WAN_INTERFACE: enp3s0
TESTBED_TFTP_PATH: /var/lib/tftpboot
TFTP_IMAGE_PATH: bin/targets/$DUT_TARGET/$DUT_SUBTARGET
PACKAGE_DIR: bin/packages
DUT_ARCH_PACKAGES_PATH: $PACKAGE_DIR/${DUT_ARCH}_${DUT_CPU_TYPE}_${DUT_CPU_SUBTYPE}
TARGET_LAN_IP: 192.168.1.1
TARGET_LAN_TEST_HOST: 192.168.1.2
DUT_SLEEP_AFTER_BOOT: 90
DUT_SSH_CONNECT_TIMEOUT: 30
DUT_SSH_CONNECT_ATTEMPTS: 3
DUT_TR181_WAN_INTERFACE: Device.Ethernet.Link.2.
DUT_TR181_LAN_INTERFACE: Device.Ethernet.Link.3.
DUT_OBUSPA_VERBOSE_LEVEL: ''
DUT_OBUSPA_PROTOCOL_TRACE: ''
TEST_SETUP_TEARDOWN_DURATION: 360
CRAM_REMOTE_COMMAND: ssh -n root@$TARGET_LAN_IP
CRAM_REMOTE_COPY: scp -O -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
CRAM_TEST_SUITE: |
.gitlab/tests/cram/generic/acceleration-plan-components
.gitlab/tests/cram/$DUT_BOARD/acceleration-plan-components
.gitlab/tests/cram/generic/lcm
.gitlab/tests/cram/generic
.gitlab/tests/cram/$DUT_BOARD
CRAM_TEST_SUITE_POST: |
.gitlab/tests/cram/post/generic
DUT_INIT_SCRIPTS: |
.gitlab/scripts/dut-init/dut-change-fail-action-to-restart.sh
before_script:
- source .gitlab/scripts/helpers.sh
- section_start "cdrouter-cleanup" "CDRouter Cleanup" "collapsed"
- >
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
fi
- .gitlab/scripts/testbed-cdrouter.py check_connectivity
- .gitlab/scripts/testbed-cdrouter.py package_stop
- section_end "cdrouter-cleanup"
- section_start "testbed-setup" "Testbed Setup" "collapsed"
- touch .run_failed
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
- sleep 10
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- |
for f in ${TFTP_IMAGE_FILENAMES:-$TFTP_IMAGE_FILENAME}; do
echo "Using firmware image $TFTP_IMAGE_PATH/$f"
cp "$TFTP_IMAGE_PATH/$f" "$TESTBED_TFTP_PATH"
done
if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
eval "$TFTP_IMAGE_UNPACK_COMMAND"
fi
- >
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
fi
- section_end "testbed-setup"
- section_start "dut-bringup" "DUT Bringup" "collapsed"
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
- >
.gitlab/scripts/testbed-device.py
--target $LABGRID_TARGET check_network
--network lan
--remote-host $TARGET_LAN_TEST_HOST
- sleep $DUT_SLEEP_AFTER_BOOT
- section_end "dut-bringup"
- section_start "dut-system-info" "DUT System Information" "collapsed"
- echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
- time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
- >
if [ -n "${DUT_OBUSPA_VERBOSE_LEVEL:-}" ] ||
[ -n "${DUT_OBUSPA_PROTOCOL_TRACE:-}" ]; then
.gitlab/scripts/dut-init/dut-configure-obuspa-logging.sh;
fi
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
- ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
- ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json
- >
for dut_init_script in $DUT_INIT_SCRIPTS;
do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
done
- section_end "dut-system-info"
script:
- source .gitlab/scripts/helpers.sh
- section_start "test-execution" "Test Execution${COMPONENT:+ [$COMPONENT]}" "expanded"
- |
TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"
- |
rm -fr .run_check_failed
run_check() {
"$@" || {
local statuses=("${PIPESTATUS[@]}")
local command=("$@")
local test_count=0
while [ "${#command[@]}" -gt 0 ] && [[ "${command[-1]}" == *.t ]]; do
unset 'command[-1]'
((test_count += 1))
done
if [ "$test_count" -gt 0 ]; then
command+=("($test_count tests)")
fi
for status in "${statuses[@]}"; do
if [ "$status" -ne 0 ]; then
log_error "run_check: ${command[*]} | FAILED! (status: $status)" | tee -a .run_check_failed
fi
done
}
}
run_check ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"'
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"'
run_check \
timeout --kill-after=10s --verbose ${TEST_CUTOFF_TIMEOUT}s \
python3 -m cram --xunit-file=cram-result-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE $CRAM_TEST_SUITE_EXTRA | tee cram-result-$LABGRID_TARGET.txt
run_check python3 -m cram --xunit-file=cram-result-post-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE_POST | tee --append cram-result-$LABGRID_TARGET.txt
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"'
- section_end "test-execution"
- |
[ -f .run_check_failed ] && cat .run_check_failed && exit 1
rm .run_failed
after_script:
- >-
curl --fail \
--output .gitlab/scripts/cram-xunit-merger.py \
https://gitlab.com/prpl-foundation/tooling/cram-xunit-merger/-/raw/9bc14318ab88c2a024ad0ff8bf09c21d048e1e3b/cram-xunit-merger.py
- python3 .gitlab/scripts/cram-xunit-merger.py \
--output "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.junit.xml" \
"${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.xunit.xml" \
"${CI_PROJECT_DIR}/cram-result-post-$LABGRID_TARGET.xunit.xml" || true
- >
[ "$TMATE_ENABLE_SHELL" = "YES" ] && {
echo "Starting tmate session..."
tmate -n "${TMATE_SESSION_TOKEN}-$CI_JOB_ID" -k "$TMATE_API_KEY" -F new "ssh root@$TARGET_LAN_IP" > /tmp/tmate-session.log 2>&1
true
}
- mkdir -p ~/.ssh
- chmod 700 ~/.ssh
- >
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
fi
- >
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
} || true
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
- >
ssh root@$TARGET_LAN_IP exit && {
ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
scp -O -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
scp -O -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
scp -O -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
} || true
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
- tar xzf debug-*_all.tar.gz || true
- mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
- ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
- mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
- mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
artifacts:
expire_in: 1 month
when: always
paths:
- .gitlab/tests/cram/**/*.t.err
- .gitlab/tests/cram/**/*.actual
- .gitlab/tests/cram/**/*.diff
- etc
- prplmesh_beerocks_logs
- debug-information-$LABGRID_TARGET.txt
- processes-$LABGRID_TARGET.txt
- dmesg-$LABGRID_TARGET.txt
- syslog-$LABGRID_TARGET.txt
- system-$LABGRID_TARGET.json
- console_$LABGRID_TARGET.txt
- cram-result-*
- opkg_list-$LABGRID_TARGET.txt
- coredumps-$LABGRID_TARGET.tar.gz
reports:
junit: cram-result-$LABGRID_TARGET.junit.xml
.cram_component:
extends: .testbed
timeout: 40 minutes
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- if: '$CI_COMMIT_TAG =~ /^prplware-v[0-9]+\.[0-9]+\.[0-9]+.*$/ && $COMPONENT == "full"'
- if: '$COMPONENT == "full" && $CRAM_COMPONENTS =~ /(^|,)full($|,)/'
- if: '$COMPONENT == "prplmesh" && $CRAM_COMPONENTS =~ /(^|,)prplmesh($|,)/'
- if: '$COMPONENT == "lcm" && $CRAM_COMPONENTS =~ /(^|,)lcm($|,)/'
- if: '$COMPONENT == "prplos" && $CRAM_COMPONENTS =~ /(^|,)prplos($|,)/'
- when: manual
allow_failure: true
variables:
COMPONENT: prplos
script:
- |
CRAM_TEST_SUITE="$(
.gitlab/tests/cram/scripts/resolve-component-tests.py \
--test-root .gitlab/tests/cram \
--manifest components.yml \
--board "$DUT_BOARD" \
--component "$COMPONENT"
)"
export CRAM_TEST_SUITE
- !reference [.testbed, script]
lint cram component manifest:
stage: run
image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
needs: []
rules:
- if: '$CI_PIPELINE_SOURCE == "schedule"'
when: never
- when: manual
allow_failure: true
script:
- |
export GIT_CONFIG_COUNT=1
export GIT_CONFIG_KEY_0=safe.directory
export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"
base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
if [ -n "$base_ref" ]; then
echo "Comparing manifest coverage against MR diff base $base_ref"
git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
git fetch --no-tags origin "$base_ref"
set -- --base-ref "$base_ref"
else
set --
fi
.gitlab/tests/cram/scripts/check-component-coverage.py \
--test-root .gitlab/tests/cram \
--manifest components.yml \
"$@"
.testbed true.cz:
extends: .cram_component
Reference in New Issue View Git Blame Copy Permalink