647353c929
Merge request changes currently leave all component CRAM jobs dormant. Reviewers cannot see the selected coverage without resolving the matrix themselves, and playing a bucket job cannot combine affected components into one boot. Add per-board changes anchors and six automatic component jobs. Resolve their content from refreshed merge request labels and the diff before any testbed setup. Exit cleanly without DUT cleanup when selection is empty. An automatic selection resolves a union of components that can become a full run, so give these jobs the full-tier 85-minute budget instead of the 40-minute component budget they would otherwise inherit. Add an MR-only report job that prints the complete selection matrix without a build or hardware dependency. Keep all 24 existing bucket jobs unchanged. The automatic template originally extended .cram_component as the later parent after each board template. GitLab's parent precedence therefore reapplied generic .testbed variables over board-specific values. This changed DUT_WAN_INTERFACE on Freedom, Omnia, Haze, Mozart, and Valyrian, and DUT_ARCH_PACKAGES_PATH on OSPv2, Haze, Mozart, and Valyrian. The resulting Freedom failures exposed the wrong interface: ```text 014-firewall-levels.t: grep $DUT_WAN_INTERFACE -> [1] 045-qos.t: grep $DUT_WAN_INTERFACE -> [1] ``` Make .cram_component_auto a standalone overlay because the board parent already supplies the component script and testbed chain. Add a committed guard that resolves all six extends chains and requires each automatic job to preserve its full-bucket sibling's variables. Assisted-by: codex:gpt-5.6-sol References: CI-9 Signed-off-by: Petr Štetiar <petr.stetiar@prplfoundation.org>
405 lines
15 KiB
YAML
405 lines
15 KiB
YAML
.testbed:
|
|
stage: run
|
|
image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
|
|
timeout: 87 minutes
|
|
variables:
|
|
DUT_WAN_INTERFACE: eth1
|
|
TESTBED_MNG_INTERFACE: enp1s0
|
|
TESTBED_LAN_INTERFACE: enp2s0
|
|
TESTBED_WAN_INTERFACE: enp3s0
|
|
TESTBED_TFTP_PATH: /var/lib/tftpboot
|
|
TFTP_IMAGE_PATH: bin/targets/$DUT_TARGET/$DUT_SUBTARGET
|
|
PACKAGE_DIR: bin/packages
|
|
DUT_ARCH_PACKAGES_PATH: $PACKAGE_DIR/${DUT_ARCH}_${DUT_CPU_TYPE}_${DUT_CPU_SUBTYPE}
|
|
TARGET_LAN_IP: 192.168.1.1
|
|
TARGET_LAN_TEST_HOST: 192.168.1.2
|
|
DUT_SLEEP_AFTER_BOOT: 90
|
|
DUT_SSH_CONNECT_TIMEOUT: 30
|
|
DUT_SSH_CONNECT_ATTEMPTS: 3
|
|
DUT_TR181_WAN_INTERFACE: Device.Ethernet.Link.2.
|
|
DUT_TR181_LAN_INTERFACE: Device.Ethernet.Link.3.
|
|
DUT_OBUSPA_VERBOSE_LEVEL: ''
|
|
DUT_OBUSPA_PROTOCOL_TRACE: ''
|
|
TEST_SETUP_TEARDOWN_DURATION: 360
|
|
CRAM_REMOTE_COMMAND: ssh -n root@$TARGET_LAN_IP
|
|
CRAM_REMOTE_COPY: scp -O -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
CRAM_TEST_SUITE: |
|
|
.gitlab/tests/cram/generic/acceleration-plan-components
|
|
.gitlab/tests/cram/$DUT_BOARD/acceleration-plan-components
|
|
.gitlab/tests/cram/generic/lcm
|
|
.gitlab/tests/cram/generic
|
|
.gitlab/tests/cram/$DUT_BOARD
|
|
CRAM_TEST_SUITE_POST: |
|
|
.gitlab/tests/cram/post/generic
|
|
DUT_INIT_SCRIPTS: |
|
|
.gitlab/scripts/dut-init/dut-change-fail-action-to-restart.sh
|
|
|
|
before_script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "cdrouter-cleanup" "CDRouter Cleanup" "collapsed"
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
|
|
fi
|
|
|
|
- .gitlab/scripts/testbed-cdrouter.py check_connectivity
|
|
- .gitlab/scripts/testbed-cdrouter.py package_stop
|
|
|
|
- section_end "cdrouter-cleanup"
|
|
- section_start "testbed-setup" "Testbed Setup" "collapsed"
|
|
|
|
- touch .run_failed
|
|
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
|
|
- sleep 10
|
|
|
|
- mkdir -p ~/.ssh
|
|
- chmod 700 ~/.ssh
|
|
|
|
- |
|
|
for f in ${TFTP_IMAGE_FILENAMES:-$TFTP_IMAGE_FILENAME}; do
|
|
echo "Using firmware image $TFTP_IMAGE_PATH/$f"
|
|
cp "$TFTP_IMAGE_PATH/$f" "$TESTBED_TFTP_PATH"
|
|
done
|
|
|
|
if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
|
|
echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
|
|
eval "$TFTP_IMAGE_UNPACK_COMMAND"
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- section_end "testbed-setup"
|
|
- section_start "dut-bringup" "DUT Bringup" "collapsed"
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
|
|
- >
|
|
.gitlab/scripts/testbed-device.py
|
|
--target $LABGRID_TARGET check_network
|
|
--network lan
|
|
--remote-host $TARGET_LAN_TEST_HOST
|
|
|
|
- sleep $DUT_SLEEP_AFTER_BOOT
|
|
|
|
- section_end "dut-bringup"
|
|
- section_start "dut-system-info" "DUT System Information" "collapsed"
|
|
|
|
- echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
|
|
- time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- >
|
|
if [ -n "${DUT_OBUSPA_VERBOSE_LEVEL:-}" ] ||
|
|
[ -n "${DUT_OBUSPA_PROTOCOL_TRACE:-}" ]; then
|
|
.gitlab/scripts/dut-init/dut-configure-obuspa-logging.sh;
|
|
fi
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
|
|
- ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
|
|
- ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json
|
|
|
|
- >
|
|
for dut_init_script in $DUT_INIT_SCRIPTS;
|
|
do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
|
|
done
|
|
|
|
- section_end "dut-system-info"
|
|
|
|
script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "test-execution" "Test Execution${COMPONENT:+ [$COMPONENT]}" "expanded"
|
|
|
|
- |
|
|
TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
|
|
echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"
|
|
|
|
- |
|
|
rm -fr .run_check_failed
|
|
|
|
run_check() {
|
|
"$@" || {
|
|
local statuses=("${PIPESTATUS[@]}")
|
|
local command=("$@")
|
|
local test_count=0
|
|
while [ "${#command[@]}" -gt 0 ] && [[ "${command[-1]}" == *.t ]]; do
|
|
unset 'command[-1]'
|
|
((test_count += 1))
|
|
done
|
|
if [ "$test_count" -gt 0 ]; then
|
|
command+=("($test_count tests)")
|
|
fi
|
|
for status in "${statuses[@]}"; do
|
|
if [ "$status" -ne 0 ]; then
|
|
log_error "run_check: ${command[*]} | FAILED! (status: $status)" | tee -a .run_check_failed
|
|
fi
|
|
done
|
|
}
|
|
}
|
|
|
|
run_check ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"'
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"'
|
|
run_check \
|
|
timeout --kill-after=10s --verbose ${TEST_CUTOFF_TIMEOUT}s \
|
|
python3 -m cram --xunit-file=cram-result-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE $CRAM_TEST_SUITE_EXTRA | tee cram-result-$LABGRID_TARGET.txt
|
|
run_check python3 -m cram --xunit-file=cram-result-post-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE_POST | tee --append cram-result-$LABGRID_TARGET.txt
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"'
|
|
|
|
- section_end "test-execution"
|
|
|
|
- |
|
|
[ -f .run_check_failed ] && cat .run_check_failed && exit 1
|
|
rm .run_failed
|
|
|
|
after_script:
|
|
- >-
|
|
curl --fail \
|
|
--output .gitlab/scripts/cram-xunit-merger.py \
|
|
https://gitlab.com/prpl-foundation/tooling/cram-xunit-merger/-/raw/9bc14318ab88c2a024ad0ff8bf09c21d048e1e3b/cram-xunit-merger.py
|
|
|
|
- python3 .gitlab/scripts/cram-xunit-merger.py \
|
|
--output "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.junit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.xunit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-post-$LABGRID_TARGET.xunit.xml" || true
|
|
|
|
- >
|
|
[ "$TMATE_ENABLE_SHELL" = "YES" ] && {
|
|
echo "Starting tmate session..."
|
|
tmate -n "${TMATE_SESSION_TOKEN}-$CI_JOB_ID" -k "$TMATE_API_KEY" -F new "ssh root@$TARGET_LAN_IP" > /tmp/tmate-session.log 2>&1
|
|
true
|
|
}
|
|
|
|
- mkdir -p ~/.ssh
|
|
- chmod 700 ~/.ssh
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- >
|
|
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
|
|
} || true
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- >
|
|
ssh root@$TARGET_LAN_IP exit && {
|
|
ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
|
|
script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
|
|
scp -O -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
|
|
scp -O -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
|
|
scp -O -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
|
|
} || true
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
|
|
- tar xzf debug-*_all.tar.gz || true
|
|
- mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
|
|
- ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
|
|
- mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
|
|
- mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
|
|
|
|
artifacts:
|
|
expire_in: 1 month
|
|
when: always
|
|
paths:
|
|
- .gitlab/tests/cram/**/*.t.err
|
|
- .gitlab/tests/cram/**/*.actual
|
|
- .gitlab/tests/cram/**/*.diff
|
|
- etc
|
|
- prplmesh_beerocks_logs
|
|
- debug-information-$LABGRID_TARGET.txt
|
|
- processes-$LABGRID_TARGET.txt
|
|
- dmesg-$LABGRID_TARGET.txt
|
|
- syslog-$LABGRID_TARGET.txt
|
|
- system-$LABGRID_TARGET.json
|
|
- console_$LABGRID_TARGET.txt
|
|
- cram-result-*
|
|
- opkg_list-$LABGRID_TARGET.txt
|
|
- coredumps-$LABGRID_TARGET.tar.gz
|
|
reports:
|
|
junit: cram-result-$LABGRID_TARGET.junit.xml
|
|
|
|
.cram_component:
|
|
extends: .testbed
|
|
timeout: 40 minutes
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
when: never
|
|
- if: '$CI_COMMIT_TAG =~ /^prplware-v[0-9]+\.[0-9]+\.[0-9]+.*$/ && $COMPONENT == "full"'
|
|
- if: '$COMPONENT == "full" && $CRAM_COMPONENTS =~ /(^|,)full($|,)/'
|
|
- if: '$COMPONENT == "prplmesh" && $CRAM_COMPONENTS =~ /(^|,)prplmesh($|,)/'
|
|
- if: '$COMPONENT == "lcm" && $CRAM_COMPONENTS =~ /(^|,)lcm($|,)/'
|
|
- if: '$COMPONENT == "prplos" && $CRAM_COMPONENTS =~ /(^|,)prplos($|,)/'
|
|
- when: manual
|
|
allow_failure: true
|
|
variables:
|
|
COMPONENT: prplos
|
|
script:
|
|
- |
|
|
CRAM_TEST_SUITE="$(
|
|
.gitlab/tests/cram/scripts/resolve-component-tests.py \
|
|
--test-root .gitlab/tests/cram \
|
|
--manifest components.yml \
|
|
--board "$DUT_BOARD" \
|
|
--component "$COMPONENT"
|
|
)"
|
|
export CRAM_TEST_SUITE
|
|
- !reference [.testbed, script]
|
|
|
|
.cram_component_auto:
|
|
variables:
|
|
COMPONENT: null
|
|
before_script:
|
|
- |
|
|
export GIT_CONFIG_COUNT=1
|
|
export GIT_CONFIG_KEY_0=safe.directory
|
|
export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"
|
|
|
|
runtime_labels="${CI_MERGE_REQUEST_LABELS:-}"
|
|
labels_json="$(mktemp)"
|
|
if curl --fail --silent --show-error \
|
|
--header "JOB-TOKEN: $CI_JOB_TOKEN" \
|
|
--output "$labels_json" \
|
|
"$CI_API_V4_URL/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID"; then
|
|
runtime_labels="$(
|
|
python3 - "$labels_json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
print(",".join(json.load(source)["labels"]))
|
|
PY
|
|
)"
|
|
else
|
|
echo "WARNING: cannot refresh MR labels; using pipeline snapshot" >&2
|
|
fi
|
|
rm -f "$labels_json"
|
|
|
|
base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
|
|
if [ -z "$base_ref" ]; then
|
|
echo "error: CI_MERGE_REQUEST_DIFF_BASE_SHA is unset" >&2
|
|
exit 1
|
|
fi
|
|
git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
|
|
git fetch --no-tags origin "$base_ref"
|
|
|
|
set -- \
|
|
.gitlab/tests/cram/scripts/resolve-changed-components.py \
|
|
--test-root .gitlab/tests/cram \
|
|
--manifest components.yml \
|
|
--base-ref "$base_ref" \
|
|
--board "$DUT_BOARD" \
|
|
--labels "$runtime_labels"
|
|
if [ "${CRAM_COMPONENTS+x}" = x ]; then
|
|
set -- "$@" --cram-components "$CRAM_COMPONENTS"
|
|
fi
|
|
if ! COMPONENT="$("$@")"; then
|
|
echo "error: cram component selection failed" >&2
|
|
exit 1
|
|
fi
|
|
if [ -z "$COMPONENT" ]; then
|
|
exit 0
|
|
fi
|
|
export COMPONENT
|
|
touch .cram-auto-selected
|
|
- !reference [.testbed, before_script]
|
|
after_script:
|
|
- |
|
|
if [ ! -f .cram-auto-selected ]; then
|
|
echo "Skipping DUT cleanup: automatic selection was empty"
|
|
exit 0
|
|
fi
|
|
- !reference [.testbed, after_script]
|
|
|
|
lint cram component manifest:
|
|
stage: run
|
|
image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
|
|
needs: []
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
when: never
|
|
- when: manual
|
|
allow_failure: true
|
|
script:
|
|
- |
|
|
export GIT_CONFIG_COUNT=1
|
|
export GIT_CONFIG_KEY_0=safe.directory
|
|
export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"
|
|
|
|
base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
|
|
if [ -n "$base_ref" ]; then
|
|
echo "Comparing manifest coverage against MR diff base $base_ref"
|
|
git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
|
|
git fetch --no-tags origin "$base_ref"
|
|
set -- --base-ref "$base_ref"
|
|
else
|
|
set --
|
|
fi
|
|
|
|
.gitlab/tests/cram/scripts/check-component-coverage.py \
|
|
--test-root .gitlab/tests/cram \
|
|
--manifest components.yml \
|
|
"$@"
|
|
|
|
.gitlab/tests/cram/scripts/check-auto-job-variables.py
|
|
|
|
.report_cram_selection:
|
|
stage: run
|
|
image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
|
|
script:
|
|
- |
|
|
export GIT_CONFIG_COUNT=1
|
|
export GIT_CONFIG_KEY_0=safe.directory
|
|
export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"
|
|
|
|
runtime_labels="${CI_MERGE_REQUEST_LABELS:-}"
|
|
labels_json="$(mktemp)"
|
|
if curl --fail --silent --show-error \
|
|
--header "JOB-TOKEN: $CI_JOB_TOKEN" \
|
|
--output "$labels_json" \
|
|
"$CI_API_V4_URL/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID"; then
|
|
runtime_labels="$(
|
|
python3 - "$labels_json" <<'PY'
|
|
import json
|
|
import sys
|
|
|
|
with open(sys.argv[1], encoding="utf-8") as source:
|
|
print(",".join(json.load(source)["labels"]))
|
|
PY
|
|
)"
|
|
else
|
|
echo "WARNING: cannot refresh MR labels; using pipeline snapshot" >&2
|
|
fi
|
|
rm -f "$labels_json"
|
|
|
|
base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
|
|
if [ -z "$base_ref" ]; then
|
|
echo "error: CI_MERGE_REQUEST_DIFF_BASE_SHA is unset" >&2
|
|
exit 1
|
|
fi
|
|
git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
|
|
git fetch --no-tags origin "$base_ref"
|
|
|
|
set -- \
|
|
.gitlab/tests/cram/scripts/resolve-changed-components.py \
|
|
--test-root .gitlab/tests/cram \
|
|
--manifest components.yml \
|
|
--base-ref "$base_ref" \
|
|
--report \
|
|
--labels "$runtime_labels"
|
|
if [ "${CRAM_COMPONENTS+x}" = x ]; then
|
|
set -- "$@" --cram-components "$CRAM_COMPONENTS"
|
|
fi
|
|
"$@"
|
|
|
|
.testbed true.cz:
|
|
extends: .cram_component
|