6792ba6b95
Currently the gmap upnp and dnssd cram tests (066-gmap-mod-upnp.t,
068-gmap-mod-dnssd.t) each duplicate apt source fixing and package
installation inline, which is fragile, slow and inconsistent with
the established pattern of pre-installing test dependencies.
So lets fix this by moving the runtime apt package installation
(minidlna, avahi-daemon, avahi-utils, dbus) into the testbed
before_script, following the pattern from commit d247c51ed663
("ci: cram: remove runtime pip install calls from tests").
The proper fix is to pre-install these packages in the testbed
Docker image, which is blocked on the container pipeline rebuild.
References: PPW-1258
Signed-off-by: Petr Štetiar <petr.stetiar@prplfoundation.org>
233 lines
9.3 KiB
YAML
233 lines
9.3 KiB
YAML
.testbed:
|
|
stage: run
|
|
image: "$CI_REGISTRY_IMAGE/$CI_DESIGNATED_BRANCH_SLUG/testbed:latest"
|
|
timeout: 64 minutes
|
|
rules:
|
|
- if: '$CI_PIPELINE_SOURCE == "schedule"'
|
|
when: never
|
|
- when: on_success
|
|
variables:
|
|
DUT_WAN_INTERFACE: eth1
|
|
TESTBED_MNG_INTERFACE: enp1s0
|
|
TESTBED_LAN_INTERFACE: enp2s0
|
|
TESTBED_WAN_INTERFACE: enp3s0
|
|
TESTBED_TFTP_PATH: /var/lib/tftpboot
|
|
TFTP_IMAGE_PATH: bin/targets/$DUT_TARGET/$DUT_SUBTARGET
|
|
PACKAGE_DIR: bin/packages
|
|
DUT_ARCH_PACKAGES_PATH: $PACKAGE_DIR/${DUT_ARCH}_${DUT_CPU_TYPE}_${DUT_CPU_SUBTYPE}
|
|
TARGET_LAN_IP: 192.168.1.1
|
|
TARGET_LAN_TEST_HOST: 192.168.1.2
|
|
DUT_SLEEP_AFTER_BOOT: 90
|
|
DUT_SSH_CONNECT_TIMEOUT: 30
|
|
DUT_SSH_CONNECT_ATTEMPTS: 3
|
|
DUT_TR181_WAN_INTERFACE: Device.Ethernet.Link.2.
|
|
DUT_TR181_LAN_INTERFACE: Device.Ethernet.Link.3.
|
|
TEST_SETUP_TEARDOWN_DURATION: 360
|
|
CRAM_REMOTE_COMMAND: ssh root@$TARGET_LAN_IP
|
|
CRAM_REMOTE_COPY: scp -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
|
|
CRAM_TEST_SUITE: |
|
|
.gitlab/tests/cram/generic/acceleration-plan-components
|
|
.gitlab/tests/cram/$DUT_BOARD/acceleration-plan-components
|
|
.gitlab/tests/cram/generic
|
|
.gitlab/tests/cram/$DUT_BOARD
|
|
CRAM_TEST_SUITE_POST: |
|
|
.gitlab/tests/cram/post/generic
|
|
DUT_INIT_SCRIPTS: |
|
|
.gitlab/scripts/dut-init/dut-change-fail-action-to-restart.sh
|
|
|
|
before_script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "cdrouter-cleanup" "CDRouter Cleanup" "collapsed"
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
|
|
fi
|
|
|
|
- .gitlab/scripts/testbed-cdrouter.py check_connectivity
|
|
- .gitlab/scripts/testbed-cdrouter.py package_stop
|
|
|
|
- section_end "cdrouter-cleanup"
|
|
- section_start "testbed-setup" "Testbed Setup" "collapsed"
|
|
|
|
- touch .run_failed
|
|
|
|
- sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
|
|
- sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
|
|
- sleep 10
|
|
|
|
- sudo sed -i 's|http://deb.debian.org/debian|http://archive.debian.org/debian|g' /etc/apt/sources.list 2>/dev/null
|
|
- sudo sed -i 's|http://deb.debian.org/debian-security|http://archive.debian.org/debian-security|g' /etc/apt/sources.list 2>/dev/null
|
|
- sudo apt-get update && sudo apt-get install --yes --no-install-recommends minidlna avahi-daemon avahi-utils dbus
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- |
|
|
echo "Using firmware image $TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME"
|
|
cp "$TFTP_IMAGE_PATH/$TFTP_IMAGE_FILENAME" "$TESTBED_TFTP_PATH"
|
|
|
|
if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
|
|
echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
|
|
eval "$TFTP_IMAGE_UNPACK_COMMAND"
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- section_end "testbed-setup"
|
|
- section_start "dut-bringup" "DUT Bringup" "collapsed"
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
|
|
- >
|
|
.gitlab/scripts/testbed-device.py
|
|
--target $LABGRID_TARGET check_network
|
|
--network lan
|
|
--remote-host $TARGET_LAN_TEST_HOST
|
|
|
|
- sleep $DUT_SLEEP_AFTER_BOOT
|
|
|
|
- section_end "dut-bringup"
|
|
- section_start "dut-system-info" "DUT System Information" "collapsed"
|
|
|
|
- echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
|
|
- time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
|
|
- ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
|
|
- ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json
|
|
|
|
- >
|
|
for dut_init_script in $DUT_INIT_SCRIPTS;
|
|
do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
|
|
done
|
|
|
|
- section_end "dut-system-info"
|
|
|
|
script:
|
|
- source .gitlab/scripts/helpers.sh
|
|
- section_start "test-execution" "Test Execution" "expanded"
|
|
|
|
- |
|
|
TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
|
|
echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"
|
|
|
|
- |
|
|
rm -fr .run_check_failed
|
|
|
|
run_check() {
|
|
"$@" || {
|
|
local statuses=("${PIPESTATUS[@]}")
|
|
for status in "${statuses[@]}"; do
|
|
if [ "$status" -ne 0 ]; then
|
|
log_error "run_check: $* | FAILED! (status: $status)" | tee -a .run_check_failed
|
|
fi
|
|
done
|
|
}
|
|
}
|
|
|
|
run_check ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"'
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"'
|
|
run_check \
|
|
timeout --kill-after=10s --verbose ${TEST_CUTOFF_TIMEOUT}s \
|
|
python3 -m cram --xunit-file=cram-result-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE $CRAM_TEST_SUITE_EXTRA | tee cram-result-$LABGRID_TARGET.txt
|
|
run_check python3 -m cram --xunit-file=cram-result-post-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE_POST | tee --append cram-result-$LABGRID_TARGET.txt
|
|
run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"'
|
|
|
|
- section_end "test-execution"
|
|
|
|
- |
|
|
[ -f .run_check_failed ] && cat .run_check_failed && exit 1
|
|
rm .run_failed
|
|
|
|
after_script:
|
|
- >-
|
|
curl --fail \
|
|
--output .gitlab/scripts/cram-xunit-merger.py \
|
|
https://gitlab.com/prpl-foundation/tooling/cram-xunit-merger/-/raw/9bc14318ab88c2a024ad0ff8bf09c21d048e1e3b/cram-xunit-merger.py
|
|
|
|
- python3 .gitlab/scripts/cram-xunit-merger.py \
|
|
--output "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.junit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.xunit.xml" \
|
|
"${CI_PROJECT_DIR}/cram-result-post-$LABGRID_TARGET.xunit.xml" || true
|
|
|
|
- >
|
|
[ "$TMATE_ENABLE_SHELL" = "YES" ] && {
|
|
echo "Starting tmate session..."
|
|
tmate -n "${TMATE_SESSION_TOKEN}-$CI_JOB_ID" -k "$TMATE_API_KEY" -F new "ssh root@$TARGET_LAN_IP" > /tmp/tmate-session.log 2>&1
|
|
true
|
|
}
|
|
|
|
- mkdir -p ~/.ssh; chmod 700 ~/.ssh
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
|
|
eval $(ssh-agent -s)
|
|
echo "$TESTBED_SSH_PRIVATE_KEY" | tr -d '\r' | ssh-add -
|
|
ssh-keyscan $TESTBED_UART_RELAY_HOST > ~/.ssh/known_hosts 2> /dev/null
|
|
chmod 644 ~/.ssh/known_hosts
|
|
fi
|
|
|
|
- >
|
|
if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
|
|
export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
|
|
fi
|
|
|
|
- >
|
|
ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
|
|
.gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
|
|
} || true
|
|
|
|
- ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
|
|
- >
|
|
ssh root@$TARGET_LAN_IP exit && {
|
|
ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
|
|
ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
|
|
script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
|
|
scp -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
|
|
} || true
|
|
|
|
- .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
|
|
- tar xzf debug-*_all.tar.gz || true
|
|
- mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
|
|
- ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
|
|
- mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
|
|
- mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
|
|
|
|
artifacts:
|
|
expire_in: 1 month
|
|
when: always
|
|
paths:
|
|
- .gitlab/tests/cram/**/*.t.err
|
|
- etc
|
|
- prplmesh_beerocks_logs
|
|
- debug-information-$LABGRID_TARGET.txt
|
|
- processes-$LABGRID_TARGET.txt
|
|
- dmesg-$LABGRID_TARGET.txt
|
|
- syslog-$LABGRID_TARGET.txt
|
|
- system-$LABGRID_TARGET.json
|
|
- console_$LABGRID_TARGET.txt
|
|
- cram-result-*
|
|
- opkg_list-$LABGRID_TARGET.txt
|
|
- coredumps-$LABGRID_TARGET.tar.gz
|
|
reports:
|
|
junit: cram-result-$LABGRID_TARGET.junit.xml
|
|
|
|
.testbed true.cz:
|
|
extends: .testbed
|
|
variables:
|
|
TESTBED_UART_RELAY_HOST: uart-relay.testbed.vpn.true.cz
|