prplos/.gitlab/testbed.yml

.testbed:
  stage: run
  image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
  timeout: 87 minutes
  variables:
    DUT_WAN_INTERFACE: eth1
    TESTBED_MNG_INTERFACE: enp1s0
    TESTBED_LAN_INTERFACE: enp2s0
    TESTBED_WAN_INTERFACE: enp3s0
    TESTBED_TFTP_PATH: /var/lib/tftpboot
    TFTP_IMAGE_PATH: bin/targets/$DUT_TARGET/$DUT_SUBTARGET
    PACKAGE_DIR: bin/packages
    DUT_ARCH_PACKAGES_PATH: $PACKAGE_DIR/${DUT_ARCH}_${DUT_CPU_TYPE}_${DUT_CPU_SUBTYPE}
    TARGET_LAN_IP: 192.168.1.1
    TARGET_LAN_TEST_HOST: 192.168.1.2
    DUT_SLEEP_AFTER_BOOT: 90
    DUT_SSH_CONNECT_TIMEOUT: 30
    DUT_SSH_CONNECT_ATTEMPTS: 3
    DUT_TR181_WAN_INTERFACE: Device.Ethernet.Link.2.
    DUT_TR181_LAN_INTERFACE: Device.Ethernet.Link.3.
    DUT_OBUSPA_VERBOSE_LEVEL: ''
    DUT_OBUSPA_PROTOCOL_TRACE: ''
    TEST_SETUP_TEARDOWN_DURATION: 360
    CRAM_REMOTE_COMMAND: ssh -n root@$TARGET_LAN_IP
    CRAM_REMOTE_COPY: scp -O -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null
    CRAM_TEST_SUITE: |
      .gitlab/tests/cram/generic/acceleration-plan-components
      .gitlab/tests/cram/$DUT_BOARD/acceleration-plan-components
      .gitlab/tests/cram/generic/lcm
      .gitlab/tests/cram/generic
      .gitlab/tests/cram/$DUT_BOARD      
    CRAM_TEST_SUITE_POST: |
      .gitlab/tests/cram/post/generic      
    DUT_INIT_SCRIPTS: |
      .gitlab/scripts/dut-init/dut-change-fail-action-to-restart.sh      

  before_script:
    - source .gitlab/scripts/helpers.sh
    - section_start "cdrouter-cleanup" "CDRouter Cleanup" "collapsed"

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export CDROUTER_API_TOKEN="$CDROUTER_API_TOKEN_TESTBED_02"
      fi      

    - .gitlab/scripts/testbed-cdrouter.py check_connectivity
    - .gitlab/scripts/testbed-cdrouter.py package_stop

    - section_end "cdrouter-cleanup"
    - section_start "testbed-setup" "Testbed Setup" "collapsed"

    - touch .run_failed

    - sudo ip link set $TESTBED_LAN_INTERFACE up 2> /dev/null
    - sudo ip link set $TESTBED_WAN_INTERFACE up 2> /dev/null
    - sleep 10

    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    - |
      for f in ${TFTP_IMAGE_FILENAMES:-$TFTP_IMAGE_FILENAME}; do
        echo "Using firmware image $TFTP_IMAGE_PATH/$f"
        cp "$TFTP_IMAGE_PATH/$f" "$TESTBED_TFTP_PATH"
      done

      if [ -n "$TFTP_IMAGE_UNPACK_COMMAND" ]; then
        echo "Running $TFTP_IMAGE_UNPACK_COMMAND"
        eval "$TFTP_IMAGE_UNPACK_COMMAND"
      fi      

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
      fi      

    - section_end "testbed-setup"
    - section_start "dut-bringup" "DUT Bringup" "collapsed"

    - .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET boot_into shell
    - >
      .gitlab/scripts/testbed-device.py
      --target $LABGRID_TARGET check_network
      --network lan
      --remote-host $TARGET_LAN_TEST_HOST      

    - sleep $DUT_SLEEP_AFTER_BOOT

    - section_end "dut-bringup"
    - section_start "dut-system-info" "DUT System Information" "collapsed"

    - echo "Waiting for SSH connection availability with ${DUT_SSH_CONNECT_TIMEOUT}sec timeout, with $DUT_SSH_CONNECT_ATTEMPTS attempts..."
    - time ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectionAttempts=$DUT_SSH_CONNECT_ATTEMPTS -o ConnectTimeout=$DUT_SSH_CONNECT_TIMEOUT root@$TARGET_LAN_IP 'exit 0'

    - ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
    - >
      if [ -n "${DUT_OBUSPA_VERBOSE_LEVEL:-}" ] ||
         [ -n "${DUT_OBUSPA_PROTOCOL_TRACE:-}" ]; then
        .gitlab/scripts/dut-init/dut-configure-obuspa-logging.sh;
      fi      
    - ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "System accessible over SSH, $(uptime -p)"' || true
    - ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt 2> /dev/null || true
    - ssh root@$TARGET_LAN_IP "ubus call system board" | tee system-$LABGRID_TARGET.json

    - >
      for dut_init_script in $DUT_INIT_SCRIPTS;
        do echo "Running $dut_init_script"; "${CI_PROJECT_DIR}/$dut_init_script";
      done      

    - section_end "dut-system-info"

  script:
    - source .gitlab/scripts/helpers.sh
    - section_start "test-execution" "Test Execution${COMPONENT:+ [$COMPONENT]}" "expanded"

    - |
      TEST_CUTOFF_TIMEOUT=$((CI_JOB_TIMEOUT - DUT_SLEEP_AFTER_BOOT - TEST_SETUP_TEARDOWN_DURATION))
      echo "Job timeout: $CI_JOB_TIMEOUT seconds, test cutoff timeout: $TEST_CUTOFF_TIMEOUT seconds"      

    - |
      rm -fr .run_check_failed

      run_check() {
        "$@" || {
          local statuses=("${PIPESTATUS[@]}")
          local command=("$@")
          local test_count=0
          while [ "${#command[@]}" -gt 0 ] && [[ "${command[-1]}" == *.t ]]; do
            unset 'command[-1]'
            ((test_count += 1))
          done
          if [ "$test_count" -gt 0 ]; then
            command+=("($test_count tests)")
          fi
          for status in "${statuses[@]}"; do
            if [ "$status" -ne 0 ]; then
              log_error "run_check: ${command[*]} | FAILED! (status: $status)" | tee -a .run_check_failed
            fi
          done
        }
      }

      run_check ssh root@$TARGET_LAN_IP 'f=$(logread | grep "procd.*init complete"); echo "${f:-Warning: init is not complete yet,} $(uptime -p)"'
      run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing starts, $(uptime -p)"'
      run_check \
        timeout --kill-after=10s --verbose ${TEST_CUTOFF_TIMEOUT}s \
        python3 -m cram --xunit-file=cram-result-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE $CRAM_TEST_SUITE_EXTRA | tee cram-result-$LABGRID_TARGET.txt
      run_check python3 -m cram --xunit-file=cram-result-post-$LABGRID_TARGET.xunit.xml --verbose $CRAM_TEST_SUITE_POST | tee --append cram-result-$LABGRID_TARGET.txt
      run_check ssh root@$TARGET_LAN_IP 'logger -t CI -p local0.info "Testing finished, $(uptime -p)"'      

    - section_end "test-execution"

    - |
      [ -f .run_check_failed ] && cat .run_check_failed && exit 1
      rm .run_failed      

  after_script:
    - >-
      curl --fail \
        --output .gitlab/scripts/cram-xunit-merger.py \
        https://gitlab.com/prpl-foundation/tooling/cram-xunit-merger/-/raw/9bc14318ab88c2a024ad0ff8bf09c21d048e1e3b/cram-xunit-merger.py      

    - python3 .gitlab/scripts/cram-xunit-merger.py \
      --output "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.junit.xml" \
      "${CI_PROJECT_DIR}/cram-result-$LABGRID_TARGET.xunit.xml" \
      "${CI_PROJECT_DIR}/cram-result-post-$LABGRID_TARGET.xunit.xml" || true

    - >
      [ "$TMATE_ENABLE_SHELL" = "YES" ] && {
        echo "Starting tmate session..."
        tmate -n "${TMATE_SESSION_TOKEN}-$CI_JOB_ID" -k "$TMATE_API_KEY" -F new "ssh root@$TARGET_LAN_IP" > /tmp/tmate-session.log 2>&1
        true
      }      

    - mkdir -p ~/.ssh
    - chmod 700 ~/.ssh

    - >
      if echo "$CI_RUNNER_DESCRIPTION" | grep -q testbed-02; then
        export TB_CONFIG=".testbed/labgrid/testbed-02.yaml"
      fi      

    - >
      ssh -o BatchMode=yes -o StrictHostKeyChecking=no -o ConnectTimeout=15 root@$TARGET_LAN_IP 'exit 0' || {
        .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_dump_system_state
        .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET console_recover_ssh_access
      } || true      

    - ssh-keyscan $TARGET_LAN_IP >> ~/.ssh/known_hosts 2> /dev/null
    - >
      ssh root@$TARGET_LAN_IP exit && {
        ssh root@$TARGET_LAN_IP ps > processes-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP dmesg > dmesg-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP "cat /var/log/messages" > syslog-$LABGRID_TARGET.txt
        ssh root@$TARGET_LAN_IP opkg list > opkg_list-$LABGRID_TARGET.txt
        script --quiet --command "ssh -t root@$TARGET_LAN_IP /bin/getDebugInformation --all"
        scp -O -r root@${TARGET_LAN_IP}:/tmp/debug-*_all.tar.gz . > /dev/null
        scp -O -r root@${TARGET_LAN_IP}:/etc etc > /dev/null
        scp -O -r root@${TARGET_LAN_IP}:/tmp/beerocks/logs prplmesh_beerocks_logs > /dev/null
      } || true      

    - .gitlab/scripts/testbed-device.py --target $LABGRID_TARGET power off
    - tar xzf debug-*_all.tar.gz || true
    - mv debug-*.txt debug-information-$LABGRID_TARGET.txt || true
    - ( [ $(tar -tf coredumps_all.tar.gz | wc -l) -eq 0 ] && rm coredumps_all.tar.gz || true ) 2>/dev/null
    - mv coredumps_all.tar.gz coredumps-$LABGRID_TARGET.tar.gz 2> /dev/null || true
    - mv console_$LABGRID_TARGET console_$LABGRID_TARGET.txt || true

  artifacts:
    expire_in: 1 month
    when: always
    paths:
      - .gitlab/tests/cram/**/*.t.err
      - .gitlab/tests/cram/**/*.actual
      - .gitlab/tests/cram/**/*.diff
      - etc
      - prplmesh_beerocks_logs
      - debug-information-$LABGRID_TARGET.txt
      - processes-$LABGRID_TARGET.txt
      - dmesg-$LABGRID_TARGET.txt
      - syslog-$LABGRID_TARGET.txt
      - system-$LABGRID_TARGET.json
      - console_$LABGRID_TARGET.txt
      - cram-result-*
      - opkg_list-$LABGRID_TARGET.txt
      - coredumps-$LABGRID_TARGET.tar.gz
    reports:
      junit: cram-result-$LABGRID_TARGET.junit.xml

.cram_component:
  extends: .testbed
  timeout: 40 minutes
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - if: '$CI_COMMIT_TAG =~ /^prplware-v[0-9]+\.[0-9]+\.[0-9]+.*$/ && $COMPONENT == "full"'
    - if: '$COMPONENT == "full" && $CRAM_COMPONENTS =~ /(^|,)full($|,)/'
    - if: '$COMPONENT == "prplmesh" && $CRAM_COMPONENTS =~ /(^|,)prplmesh($|,)/'
    - if: '$COMPONENT == "lcm" && $CRAM_COMPONENTS =~ /(^|,)lcm($|,)/'
    - if: '$COMPONENT == "prplos" && $CRAM_COMPONENTS =~ /(^|,)prplos($|,)/'
    - when: manual
      allow_failure: true
  variables:
    COMPONENT: prplos
  script:
    - |
      CRAM_TEST_SUITE="$(
        .gitlab/tests/cram/scripts/resolve-component-tests.py \
          --test-root .gitlab/tests/cram \
          --manifest components.yml \
          --board "$DUT_BOARD" \
          --component "$COMPONENT"
      )"
      export CRAM_TEST_SUITE      
    - !reference [.testbed, script]

.cram_component_auto:
  variables:
    COMPONENT: null
  before_script:
    - |
      export GIT_CONFIG_COUNT=1
      export GIT_CONFIG_KEY_0=safe.directory
      export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"

      runtime_labels="${CI_MERGE_REQUEST_LABELS:-}"
      labels_json="$(mktemp)"
      if curl --fail --silent --show-error \
        --header "JOB-TOKEN: $CI_JOB_TOKEN" \
        --output "$labels_json" \
        "$CI_API_V4_URL/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID"; then
        runtime_labels="$(
          python3 - "$labels_json" <<'PY'
      import json
      import sys

      with open(sys.argv[1], encoding="utf-8") as source:
          print(",".join(json.load(source)["labels"]))
      PY
        )"
      else
        echo "WARNING: cannot refresh MR labels; using pipeline snapshot" >&2
      fi
      rm -f "$labels_json"

      base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
      if [ -z "$base_ref" ]; then
        echo "error: CI_MERGE_REQUEST_DIFF_BASE_SHA is unset" >&2
        exit 1
      fi
      git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
        git fetch --no-tags origin "$base_ref"

      set -- \
        .gitlab/tests/cram/scripts/resolve-changed-components.py \
        --test-root .gitlab/tests/cram \
        --manifest components.yml \
        --base-ref "$base_ref" \
        --board "$DUT_BOARD" \
        --labels "$runtime_labels"
      if [ "${CRAM_COMPONENTS+x}" = x ]; then
        set -- "$@" --cram-components "$CRAM_COMPONENTS"
      fi
      if ! COMPONENT="$("$@")"; then
        echo "error: cram component selection failed" >&2
        exit 1
      fi
      if [ -z "$COMPONENT" ]; then
        exit 0
      fi
      export COMPONENT
      touch .cram-auto-selected      
    - !reference [.testbed, before_script]
  after_script:
    - |
      if [ ! -f .cram-auto-selected ]; then
        echo "Skipping DUT cleanup: automatic selection was empty"
        exit 0
      fi      
    - !reference [.testbed, after_script]

lint cram component manifest:
  stage: run
  image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
  needs: []
  rules:
    - if: '$CI_PIPELINE_SOURCE == "schedule"'
      when: never
    - when: manual
      allow_failure: true
  script:
    - |
      export GIT_CONFIG_COUNT=1
      export GIT_CONFIG_KEY_0=safe.directory
      export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"

      base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
      if [ -n "$base_ref" ]; then
        echo "Comparing manifest coverage against MR diff base $base_ref"
        git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
          git fetch --no-tags origin "$base_ref"
        set -- --base-ref "$base_ref"
      else
        set --
      fi

      .gitlab/tests/cram/scripts/check-component-coverage.py \
        --test-root .gitlab/tests/cram \
        --manifest components.yml \
        "$@"

      .gitlab/tests/cram/scripts/check-auto-job-variables.py      

.report_cram_selection:
  stage: run
  image: "registry.gitlab.com/prpl-foundation/prplos/ci/containers/testbed/testbed:v1.3.0@sha256:7f50dde2279c201542569b7f31e965d43a308e245a4ec973df52d581560b0c28"
  script:
    - |
      export GIT_CONFIG_COUNT=1
      export GIT_CONFIG_KEY_0=safe.directory
      export GIT_CONFIG_VALUE_0="$CI_PROJECT_DIR"

      runtime_labels="${CI_MERGE_REQUEST_LABELS:-}"
      labels_json="$(mktemp)"
      if curl --fail --silent --show-error \
        --header "JOB-TOKEN: $CI_JOB_TOKEN" \
        --output "$labels_json" \
        "$CI_API_V4_URL/projects/$CI_PROJECT_ID/merge_requests/$CI_MERGE_REQUEST_IID"; then
        runtime_labels="$(
          python3 - "$labels_json" <<'PY'
      import json
      import sys

      with open(sys.argv[1], encoding="utf-8") as source:
          print(",".join(json.load(source)["labels"]))
      PY
        )"
      else
        echo "WARNING: cannot refresh MR labels; using pipeline snapshot" >&2
      fi
      rm -f "$labels_json"

      base_ref="${CI_MERGE_REQUEST_DIFF_BASE_SHA:-}"
      if [ -z "$base_ref" ]; then
        echo "error: CI_MERGE_REQUEST_DIFF_BASE_SHA is unset" >&2
        exit 1
      fi
      git cat-file -e "$base_ref^{commit}" 2>/dev/null ||
        git fetch --no-tags origin "$base_ref"

      set -- \
        .gitlab/tests/cram/scripts/resolve-changed-components.py \
        --test-root .gitlab/tests/cram \
        --manifest components.yml \
        --base-ref "$base_ref" \
        --report \
        --labels "$runtime_labels"
      if [ "${CRAM_COMPONENTS+x}" = x ]; then
        set -- "$@" --cram-components "$CRAM_COMPONENTS"
      fi
      "$@"      

.testbed true.cz:
  extends: .cram_component