prplos/.gitlab/cdrouter.yml

.cdrouter:
  stage: cdrouter
  timeout: 40 minutes
  image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - when: manual
  variables:
    CDROUTER_CONFIG_WIFI_BEACON_PHY: 'ax b g n'
    CDROUTER_CONFIG_WIFI_BEACON_RSN_CIPHER: 'CCMP-128'
    CDROUTER_CONFIG_WIFI_LAN_CHANNEL: 6
    CDROUTER_CONFIG_WIFI_LAN_SSID: 'prplOS'
    CDROUTER_CONFIG_USP_CONTROLLER_MTP: mqtt
    CDROUTER_CONFIG_USP_CONTROLLER_MTP_ENCRYPTION: 'yes'
    CDROUTER_CONFIG_USP_CONTROLLER_DOMAIN: controller.cdroutertest.com
    CDROUTER_CONFIG_USP_CONTROLLER_MTP_CERT_PATH: /usr/cdrouter-data/prplos-ci/usp-ssl-certs/server.crt
    CDROUTER_CONFIG_USP_CONTROLLER_MTP_CA_CERT_PATH: /usr/cdrouter-data/prplos-ci/usp-ssl-certs/ca.crt
    CDROUTER_TEST_CONFIG: generic
    CDROUTER_DUT_INIT_USP_MTP_SCRIPT: .gitlab/scripts/dut-init/cdrouter-dut-configure-usp-mqtt.sh
    DUT_OBUSPA_VERBOSE_LEVEL: ''
    DUT_OBUSPA_PROTOCOL_TRACE: ''
    DUT_SLEEP_AFTER_BOOT: 90
    TEST_SETUP_TEARDOWN_DURATION: 360
    CDROUTER_LAN_INTERFACE: eth1
    CDROUTER_WAN_INTERFACE: eth2
    CDROUTER_ETH0_MAC_ADDRESS: 00:03:2d:49:2e:d8
    CDROUTER_ETH0_MAC_ADDRESS_TESTBED_02: 00:03:2d:49:2e:c8

  before_script:
    - export TEST_PACKAGE="$(echo $CI_JOB_NAME | sed 's/\(\w\) .*/\1/')"
    - .gitlab/scripts/testbed-cdrouter.py package_validate $TEST_PACKAGE

    - source .gitlab/scripts/helpers.sh
    - section_start "testbed-setup" "Testbed Setup" "collapsed"

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-01; then
        sudo etherwake -i $TESTBED_MNG_INTERFACE $CDROUTER_ETH0_MAC_ADDRESS 2> /dev/null
      else
        sudo etherwake -i $TESTBED_MNG_INTERFACE $CDROUTER_ETH0_MAC_ADDRESS_TESTBED_02 2> /dev/null
      fi      
    - sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
    - sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
    - sleep 10

    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    - |
      for f in ${TFTP_IMAGE_FILENAMES:-$TFTP_IMAGE_FILENAME}; do
        echo "Using firmware image $TFTP_IMAGE_PATH/$f"
        cp "$TFTP_IMAGE_PATH/$f" "$TESTBED_TFTP_PATH"
      done

      if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
        echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
        eval "$TFTP_IMAGE_UNPACK_COMMAND"
      fi      

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
      fi      

    - section_end "testbed-setup"
    - section_start "dut-bringup" "DUT Bringup" "collapsed"

    - .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
    - sudo ip link set $TESTBED_WAN_INTERFACE down 2> /dev/null
    - >
      .gitlab/scripts/testbed-device.py
      --target $LABGRID_TARGET check_network
      --network lan
      --remote-host $TARGET_LAN_TEST_HOST      

    - sleep $DUT_SLEEP_AFTER_BOOT

    - section_end "dut-bringup"
    - section_start "dut-system-info" "DUT System Information" "collapsed"

    - echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
    - time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'

    - ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
    - >
      if [ -n "${DUT_OBUSPA_VERBOSE_LEVEL:-}" ] ||
         [ -n "${DUT_OBUSPA_PROTOCOL_TRACE:-}" ]; then
        .gitlab/scripts/dut-init/dut-configure-obuspa-logging.sh;
      fi      
    - ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
    - ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
    - ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json

    - section_end "dut-system-info"
    - section_start "dut-init" "DUT Initialization" "collapsed"

    - >
      if [ -n "${CDROUTER_DUT_INIT_USP_MTP_SCRIPT:-}" ]; then
        echo "Running $CDROUTER_DUT_INIT_USP_MTP_SCRIPT";
        "${CI_PROJECT_DIR}/$CDROUTER_DUT_INIT_USP_MTP_SCRIPT";
      fi      

    - >
      for dut_init_script in $DUT_INIT_SCRIPTS;
        do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
      done      

    - section_end "dut-init"
    - section_start "cdrouter-configuration" "CDRouter Configuration" "collapsed"

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
        export CDROUTER_CONFIG_WIFI_LAN_CHANNEL=12
        export CDROUTER_CONFIG_WIFI_LAN_SSID=prplOS-2g-12
      else
        export CDROUTER_CONFIG_WIFI_LAN_CHANNEL=6
        export CDROUTER_CONFIG_WIFI_LAN_SSID=prplOS-2g-6
      fi      

    - .gitlab/scripts/testbed-cdrouter.py check_connectivity
    - .gitlab/scripts/testbed-cdrouter.py package_stop
    - .gitlab/scripts/testbed-cdrouter.py wait_for_netif $CDROUTER_LAN_INTERFACE
    - .gitlab/scripts/testbed-cdrouter.py wait_for_netif $CDROUTER_WAN_INTERFACE

  script:
    - source .gitlab/scripts/helpers.sh

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
      fi      

    - export TEST_PACKAGE="$(echo $CI_JOB_NAME | sed 's/\(\w\) .*/\1/')"
    - .gitlab/scripts/testbed-cdrouter.py config_import $CDROUTER_TEST_CONFIG
    - .gitlab/scripts/testbed-cdrouter.py package_apply $TEST_PACKAGE

    - section_end "cdrouter-configuration"
    - >
      ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"' || true      
    - ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"' || true
    - sudo ip link set $TESTBED_LAN_INTERFACE down 2> /dev/null

    - |
      TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
      echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"      

    - section_start "test-execution" "Test Execution" "expanded"
    - |
      .gitlab/scripts/testbed-cdrouter.py \
        package_run $TEST_PACKAGE \
        --device generic \
        --configuration $CDROUTER_TEST_CONFIG \
        --system-info system-$LABGRID_TARGET.json \
        --timeout ${TEST_CUTOFF_TIMEOUT} || cdrouter_exit_code=$?      
    - section_end "test-execution"

    - sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null || true
    - sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null || true
    - sleep 5

    - python3 -m cram --verbose .gitlab/tests/cram/post/generic/50-check-for-crashes.t || cram_exit_code=$?

    - >
      if test -n "$cdrouter_exit_code"; then
        log_error "CDRouter exited with $cdrouter_exit_code exit code."
        exit "$cdrouter_exit_code"
      fi

      if test -n "$cram_exit_code"; then
        log_error "cram exited with $cram_exit_code exit code."
        exit "$cram_exit_code"
      fi      

  after_script:
    - >
      test -f *-logdir.tgz &&
      mkdir -p cdrouter-results &&
      tar xf *-logdir.tgz --strip-components=1 --directory=cdrouter-results &&
      cat cdrouter-results/final.txt &&
      .gitlab/scripts/cdrouter-pretty-failures.awk cdrouter-results/*.txt      

    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
      fi      

    - >
      ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
        .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
        .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
      } || true      

    - ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
    - ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"' || true
    - >
      ssh root@$TARGET_LAN_IP exit && {
        ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
        script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
        scp -O -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
        scp -O -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
        scp -O -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
      } || true      

    - .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
    - mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true
    - tar xzf debug-*_all.tar.gz || true
    - mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
    - ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
    - mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
      fi      
    - .gitlab/scripts/testbed-cdrouter.py package_stop

  artifacts:
    expire_in: 1 month
    when: always
    paths:
      - cdrouter-results
      - etc
      - prplmesh_beerocks_logs
      - debug-information-$LABGRID_TARGET.txt
      - processes-$LABGRID_TARGET.txt
      - dmesg-$LABGRID_TARGET.txt
      - syslog-$LABGRID_TARGET.txt
      - system-$LABGRID_TARGET.json
      - console_$LABGRID_TARGET.txt
      - opkg_list-$LABGRID_TARGET.txt
      - coredumps-$LABGRID_TARGET.tar.gz