From e0e930d7cb7d16025246ef213017af9a908fb063 Mon Sep 17 00:00:00 2001 From: Maarten De Decker Date: Tue, 3 Jun 2025 13:15:30 +0200 Subject: [PATCH] CI: easymesh: add prplMesh smoke test Copy over files from the prplMesh repository to flash devices in the R1 Easymesh testbed. Adds an easyMesh R1 certification test on Freedom as a smoke test (MAP-4.4.2_BHWIFI_FH5GH). Closes: PPM-3316 Signed-off-by: Maarten De Decker --- .gitlab-ci.yml | 2 + .gitlab/prplmesh.yml | 38 ++ .../scripts/prplMesh/R1_marvell_station.sh | 15 + .gitlab/scripts/prplMesh/capi.py | 279 +++++++++++++ .gitlab/scripts/prplMesh/configure_device.py | 44 +++ .gitlab/scripts/prplMesh/deploy_firmware.py | 108 +++++ .gitlab/scripts/prplMesh/device/__init__.py | 0 .../scripts/prplMesh/device/configuration.py | 91 +++++ .gitlab/scripts/prplMesh/device/freedom.py | 65 ++++ .gitlab/scripts/prplMesh/device/generic.py | 236 +++++++++++ .gitlab/scripts/prplMesh/device/get_device.py | 49 +++ .gitlab/scripts/prplMesh/device/haze.py | 65 ++++ .gitlab/scripts/prplMesh/device/logs.py | 89 +++++ .gitlab/scripts/prplMesh/device/prplos.py | 161 ++++++++ .gitlab/scripts/prplMesh/device/serial.py | 106 +++++ .../scripts/prplMesh/device/turris_prplos.py | 156 ++++++++ .../scripts/prplMesh/device/turris_rdk_b.py | 368 ++++++++++++++++++ .gitlab/scripts/prplMesh/device/urx_osp.py | 73 ++++ .gitlab/scripts/prplMesh/device/utils.py | 85 ++++ .../prplMesh/dut-configuration/R1/freedom.sh | 100 +++++ .../prplMesh/dut-configuration/R1/haze.sh | 95 +++++ .../dut-configuration/R1/turris-omnia.sh | 94 +++++ .../scripts/prplMesh/easymesh_cert_version | 1 + .gitlab/scripts/prplMesh/git-clean-reset.sh | 14 + .gitlab/scripts/prplMesh/opts.py | 43 ++ 25 files changed, 2377 insertions(+) create mode 100644 .gitlab/prplmesh.yml create mode 100755 .gitlab/scripts/prplMesh/R1_marvell_station.sh create mode 100755 .gitlab/scripts/prplMesh/capi.py create mode 100755 .gitlab/scripts/prplMesh/configure_device.py create mode 100755 .gitlab/scripts/prplMesh/deploy_firmware.py create mode 100644 .gitlab/scripts/prplMesh/device/__init__.py create mode 100644 .gitlab/scripts/prplMesh/device/configuration.py create mode 100644 .gitlab/scripts/prplMesh/device/freedom.py create mode 100644 .gitlab/scripts/prplMesh/device/generic.py create mode 100644 .gitlab/scripts/prplMesh/device/get_device.py create mode 100644 .gitlab/scripts/prplMesh/device/haze.py create mode 100644 .gitlab/scripts/prplMesh/device/logs.py create mode 100644 .gitlab/scripts/prplMesh/device/prplos.py create mode 100644 .gitlab/scripts/prplMesh/device/serial.py create mode 100644 .gitlab/scripts/prplMesh/device/turris_prplos.py create mode 100644 .gitlab/scripts/prplMesh/device/turris_rdk_b.py create mode 100644 .gitlab/scripts/prplMesh/device/urx_osp.py create mode 100644 .gitlab/scripts/prplMesh/device/utils.py create mode 100644 .gitlab/scripts/prplMesh/dut-configuration/R1/freedom.sh create mode 100644 .gitlab/scripts/prplMesh/dut-configuration/R1/haze.sh create mode 100644 .gitlab/scripts/prplMesh/dut-configuration/R1/turris-omnia.sh create mode 100644 .gitlab/scripts/prplMesh/easymesh_cert_version create mode 100755 .gitlab/scripts/prplMesh/git-clean-reset.sh create mode 100644 .gitlab/scripts/prplMesh/opts.py diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 025747adaa..5036ca12df 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -6,6 +6,7 @@ include: - local: .gitlab/build.yml - local: .gitlab/release.yml - local: .gitlab/cdrouter.yml + - local: .gitlab/prplmesh.yml - local: .gitlab/cdrouter/prpl-haze.yml - local: .gitlab/cdrouter/turris-omnia.yml - local: .gitlab/cdrouter/urx851-hdk-3.yml @@ -39,6 +40,7 @@ stages: - docker IB LCM container - run - cdrouter smoke + - easymesh smoke - cdrouter - coverity diff --git a/.gitlab/prplmesh.yml b/.gitlab/prplmesh.yml new file mode 100644 index 0000000000..48cc6ea668 --- /dev/null +++ b/.gitlab/prplmesh.yml @@ -0,0 +1,38 @@ +EasyMesh-Smoke-test-R1-Freedom: + stage: easymesh smoke + variables: + RUN_ON_DEVICE: "freedom" + EASYMESH_REVISION: "R1" + EASYMESH_TESTS_SIMPLE: "MAP-4.4.2_BHWIFI_FH5GH" + interruptible: false + allow_failure: true + timeout: 40min + script: + - | + EASYMESH_TESTS=$EASYMESH_TESTS_SIMPLE + echo -e "${TXT_RED}Executing Easymesh tests: ${TXT_PRPL}$EASYMESH_TESTS${TXT_CLEAR}" + - mkdir -p build/freedom/ + - cp bin/targets/ipq95xx/generic/prplos-ipq95xx-generic-prpl_freedom-squashfs-sysupgrade.bin build/freedom/prplos-ipq95xx-generic-prpl_freedom-squashfs-sysupgrade.bin + - .gitlab/scripts/prplMesh/deploy_firmware.py --device "$RUN_ON_DEVICE" --full --target-name "$RUN_ON_DEVICE" --image "prplos-ipq95xx-generic-prpl_freedom-squashfs-sysupgrade.bin" --whm --configuration ".gitlab/scripts/prplMesh/dut-configuration/R1/freedom.sh" + - .gitlab/scripts/prplMesh/git-clean-reset.sh ~/easymesh_cert "$(<".gitlab/scripts/prplMesh/easymesh_cert_version")" + - ~/easymesh_cert/run_test_file.py --verbose --log-folder logs --easymesh-revision "$EASYMESH_REVISION" --device-under-test $RUN_ON_DEVICE "$EASYMESH_TESTS" + after_script: + - .gitlab/scripts/prplMesh/R1_marvell_station.sh + needs: + - job: build test qca_ipq95xx prpl cellular security thread + optional: true + retry: 2 + rules: + - if: '$CI_PIPELINE_SOURCE == "schedule"' + when: never + - when: manual + - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH + artifacts: + paths: + - logs + when: always + tags: + - dut-prplmesh-freedom + - prplmesh-R1 + + diff --git a/.gitlab/scripts/prplMesh/R1_marvell_station.sh b/.gitlab/scripts/prplMesh/R1_marvell_station.sh new file mode 100755 index 0000000000..5df4b705ea --- /dev/null +++ b/.gitlab/scripts/prplMesh/R1_marvell_station.sh @@ -0,0 +1,15 @@ +#!/bin/sh + +# Check if the Marvell station driver ever failed to load. +# If it did, reboot the host. + +if ! journalctl -n -k --no-pager > /dev/null ; then + echo "Unable to get system logs." + echo "You either don't have permissions to read system logs, or you are not using systemd." + exit 1 +fi + +if journalctl -k -b | grep -q -E 'wlan_pcie: probe of .* failed with error -14' ; then + echo "The Marvell station driver failed to load properly, rebooting." + reboot +fi diff --git a/.gitlab/scripts/prplMesh/capi.py b/.gitlab/scripts/prplMesh/capi.py new file mode 100755 index 0000000000..1d6f54dcba --- /dev/null +++ b/.gitlab/scripts/prplMesh/capi.py @@ -0,0 +1,279 @@ +#!/usr/bin/env python3 +import argparse +import logging +import socket +import re +import time +from enum import Enum +from typing import Dict, Union +from opts import debug + +logger = logging.getLogger(__name__) + + +class CAPIReply(Enum): + """An enum representing the possible CAPI replies.""" + RUNNING = "status,RUNNING" + COMPLETE = "status,COMPLETE" # parameters may follow + INVALID = "status,INVALID" + ERROR = "status,ERROR" + + +class tlv: + """Representation of an 1905.1 TLV.""" + + def __init__(self, type_: int, value: str): + """A TLV has a type, a length and a value. + + Parameters + ---------- + type_ : int + The TLV type. + + length : int + The length of the TLV. This must correspond to the length of `value`. + + value : str + The TLV value, as a string. It must be formatted according to the UCC rules, i.e. with + curly braces and hexadecimal numbers. + """ + self.type = type_ + self.length = self.get_byte_length(value) + self.value = value + + def get_byte_length(self, value: str) -> int: + """Return tlv length for a given value + + Parameters + ---------- + value : str + The TLV value, as a string. It must be formatted according to the UCC rules, i.e. with + curly braces and hexadecimal numbers. + + Returns + ------- + int + The length of the given value + """ + byte_length = 0 + hex_values = re.split(' |:', re.sub('({|})', '', value)) + for num in hex_values: + if num: + if len(num) > 4: + byte_length += len(num[4:])/2 + byte_length += 1 + return int(byte_length) + + def format(self, tlv_num: int = 0) -> str: + """Format the TLV for the dev_send_1905 CAPI command. + + Parameters + ---------- + tlv_num : int + The TLV counter. In the CAPI command, if there are multiple TLVs, they must be numbered + starting from 1. `tlv_num` is that number. If there is only a single TLV, set it to + zero. + + Returns + ------- + str + The string representation of the TLV according to CAPI definition. + """ + if tlv_num: + tlv_num_str = str(tlv_num) + else: + tlv_num_str = '' + return "tlv_type{tlv_num_str},0x{tlv_type:02x}," \ + "tlv_length{tlv_num_str},0x{tlv_length:04x}," \ + "tlv_value{tlv_num_str},{tlv_value}".format(tlv_num_str=tlv_num_str, + tlv_type=self.type, + tlv_length=self.length, + tlv_value=self.value) + + +class UCCSocket: + """Abstraction of the target listening socket. + + It connects to the listener and it sends and receives + CAPI commands from it. + """ + + def __init__(self, host: str, port: int, timeout: int = 30): + """Constructor for UCCSocket + + Parameters + ---------- + host: str + The host to connect to. Can either be an ip or a hostname. + port: str + The port to connect to. + timeout: int + (optional) The timeout for both creating a connection, + and receiving or sending data. + """ + self.host = host + self.port = port + self.timeout = timeout + + def __enter__(self): + self.conn = socket.create_connection((self.host, self.port), self.timeout) + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + self.conn.close() + + def send_cmd(self, command: str) -> None: + """Send a new CAPI command to the device. + If previous replies from the server were available, they will be discarded. + + Parameters + ---------- + command : str + The command to send. If it does not end with a newline, a new line will be appended. + """ + if command[-1] != "\n": + command += "\n" + self.conn.send(command.encode("utf-8")) + + def get_reply(self, verbose: bool = False, timeout: Union[int, None] = 120) -> Dict[str, str]: + """Wait until the server replies with a `CAPIReply` message other than `CAPIReply.RUNNING`. + + The replies from the server will be printed as they are received. + Note that this method only returns once a `CAPIReply.COMPLETE`, `CAPIReply.INVALID`, + or `CAPIReply.ERROR` message has been received from the server. + + Parameters + ---------- + verbose : bool + If True, print out the valid replies (RUNNING and COMPLETE) as they arrive. + timeout : int + The maximum amount of time (in seconds) to wait for a reply. Defaults to 120. + Note that this method is not guaranteed to return exactly after `timeout` seconds. It + will abort after the next read call returns if the timeout is reached, which means that + it can take more than `timeout` to execute depending on what data is received. + A value of None can be used to wait until a CAPIReply other than RUNNING is received. + Regardless of the value of `timeout`, at least one read is always done. + + Returns + ------- + Dict[str, str] + A mapping of parameter -> value. The CAPI COMPLETE message is followed by + parameter,value pairs. These are converted to a dict and returned. If the COMPLETE + message has no parameters, an empty dict is returned. + + Raises + ------ + ValueError + If the reply was something other than `CAPIReply.RUNNING`, or `CAPIReply.COMPLETE`, or + `timeout` was reached. + """ + data = bytearray() + start_time = time.time() + timed_out = False + while not timed_out: + # resetting data to the next line: + data = data[data.find(b"\n") + 1:] + while not timed_out and b"\n" not in data: + # reading until there is a newline + data.extend(self.conn.recv(256)) + timed_out = time.time() > start_time + timeout if timeout else False + replies = data.decode("utf-8").split("\n") + for r in replies: + if not r: + pass # server replied with an empty line + elif CAPIReply.RUNNING.value in r: + if verbose: + print(r) + elif CAPIReply.COMPLETE.value in r: + if verbose: + print(r) + reply_value_str = r[len(CAPIReply.COMPLETE.value) + 1:].strip() + reply_values = reply_value_str.split(',') + return {k: v for k, v in zip(reply_values[::2], reply_values[1::2])} + elif CAPIReply.INVALID.value in r or CAPIReply.ERROR.value in r: + raise ValueError("Server replied with {}".format(r)) + else: + raise ValueError("Received an unknown reply from the server:\n {}".format(r)) + if timed_out: + raise ValueError("Timed out when waiting for a reply from the server.") + + def cmd_reply(self, command: str, verbose: bool = False, timeout: int = 120) -> Dict[str, str]: + """Open the connection, send a command and wait for the reply.""" + + debug("Sending CAPI command: '{}'".format(command)) + with self: + self.send_cmd(command) + return self.get_reply(verbose, timeout) + + def start_wps_registration(self, band: str) -> None: + """Call start_wps_registration on a given band. + + Parameters + ---------- + band : str + The band on which to start wps + + """ + command = "start_wps_registration,band,{},WpsConfigMethod,PBC".format(band) + self.cmd_reply(command) + + def dev_get_parameter(self, parameter: str, **additional_parameters: str) -> str: + """Call dev_get_parameter and return the parameter, or raise KeyError if it is missing. + + Parameters + ---------- + parameter : str + The parameter to query. It is the "parameter" argument of the dev_get_parameter command. + + additional_parameters : str + Additional keyword arguments are passed as additional parameters to the + dev_get_parameter command. This is needed for example to get the "macaddr" parameter, + which needs additional "ssid" and "ruid" parameters in the command. + + Returns + ------- + str + The value of the requested parameter. + """ + command = "dev_get_parameter,program,map,parameter,{}".format(parameter) + if additional_parameters: + command += ',' + ','.join([','.join(param) for param in additional_parameters.items()]) + return self.cmd_reply(command)[parameter] + + def dev_send_1905(self, dest: str, message_type: int, *tlvs: tlv) -> int: + """Call dev_send_1905 to `dest` with CMDU type `message_type` and additional `tlvs`. + + Parameters + ---------- + dest : str + The AL-MAC address of the recipient, as a string. + + message_type : int + The message type of the 1905.1 message to be sent, as an integer. + + tlvs : tlv + Additional arguments are the TLVs in the 1905.1 message, as `tlv` objects. + + Returns + ------- + The MID of the message, as an integer. + """ + cmd = "DEV_SEND_1905,DestALid,{dest:s},MessageTypeValue,0x{message_type:04x}"\ + .format(**locals()) + if len(tlvs) > 1: + formatted_tlvs = [tlv.format(tlv_num + 1) for (tlv_num, tlv) in enumerate(tlvs)] + cmd += ',' + ','.join(formatted_tlvs) + elif tlvs: + cmd += ',' + tlvs[0].format() + return int(self.cmd_reply(cmd)["mid"], base=0) + + +if __name__ == "__main__": + parser = argparse.ArgumentParser(description="Simulated UCC") + parser.add_argument("host", help="The device hostname or IP.", type=str) + parser.add_argument("port", help="The listening port on the device.", type=int) + parser.add_argument("command", help="The CAPI command to send.") + args = parser.parse_args() + socket.gethostbyname(args.host) + + UCCSocket(args.host, args.port).cmd_reply(args.command, True) diff --git a/.gitlab/scripts/prplMesh/configure_device.py b/.gitlab/scripts/prplMesh/configure_device.py new file mode 100755 index 0000000000..52e1405ab5 --- /dev/null +++ b/.gitlab/scripts/prplMesh/configure_device.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +"""This script can be used to configure a device (prplOS, RDK-B, etc) +over a serial conntection.""" + +import argparse +import sys + +from pathlib import Path + +from device.configuration import configure_device +from device.get_device import device_from_name + + +def main(): + parser = argparse.ArgumentParser(prog=sys.argv[0], + description="""Configure a device over serial.""") + parser.add_argument('-d', '--device', + help="""Device to configure. For a list of the supported devices, see the + get_device module.""", required=True) + parser.add_argument( + '-t', + '--target-name', + help="Name of the target to upgrade.", required=True) + + parser.add_argument( + '-c', + '--configuration', + help="The path to the configuration file.", required=True) + + args = parser.parse_args() + + dev = device_from_name(args.device, args.target_name) + configure_device(dev, Path(args.configuration)) + + +if __name__ == '__main__': + main() diff --git a/.gitlab/scripts/prplMesh/deploy_firmware.py b/.gitlab/scripts/prplMesh/deploy_firmware.py new file mode 100755 index 0000000000..05cd8dcd52 --- /dev/null +++ b/.gitlab/scripts/prplMesh/deploy_firmware.py @@ -0,0 +1,108 @@ +#!/usr/bin/env python3 +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2020-2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import argparse +import time +import sys +from pathlib import Path + +# Third party +from device.configuration import configure_device +from device.get_device import device_from_name + + +def main(): + parser = argparse.ArgumentParser(prog=sys.argv[0], + description="""Update a prplOS device, either through u-boot + or using sysupgrade, depending on the target device.""") + parser.add_argument('-d', '--device', + help="""Device to upgrade. Currently supported targets are: + turris-omnia haze urx_osp urx_ospv2""", required=True) + parser.add_argument( + '-t', + '--target-name', + help="Name of the target to upgrade (make sure it's reachable through ssh).", required=True) + + parser.add_argument( + '-i', + '--image', + help="Name of the image to use for the upgrade (should exist in the artifacts folder).", + required=True) + + parser.add_argument( + '-f', + '--full', + action='store_true', + help="Always flash the full image (even if it's not required).") + + parser.add_argument( + '-w', + '--whm', + action='store_true', + help="Flash using the WHM build") + + parser.add_argument( + '-c', + '--configuration', + help="The path to an optional configuration file.", required=False) + + args = parser.parse_args() + + dev = device_from_name(args.device, args.target_name, args.image) + + def do_upgrade(dev): + try: + dev.upgrade_bootloader() + except NotImplementedError: + dev.sysupgrade() + + needs_upgrade = False + if args.full: + print("--full was provided, the device {} will be upgraded".format(dev.name)) + needs_upgrade = True + else: + print("Checking if the device needs to be upgraded") + needs_upgrade = False + try: + needs_upgrade = dev.needs_upgrade() + except Exception: # pylint: disable=broad-except + print("Couldn't determine if the device needs to be ugpgraded. Upgrading anyway.") + needs_upgrade = True + if needs_upgrade: + print("The device {} will be upgraded".format(dev.name)) + do_upgrade(dev) + # do_upgrade will wait for the specified initialisation time + else: + print("The device is already using the same version, no upgrade will be done.") + + # Apply the configuration if there is one: + if args.configuration: + print("A configuration file was provided, it will be applied.") + configure_device(dev, Path(args.configuration)) + + if args.configuration: + # If the device was configured, give it some time to initialize: + print("Waiting for the device to initialize.") + time.sleep(dev.configuration_initialization_time) + + print("Checking if the device is reachable.") + if not dev.reach(attempts=10): + raise ValueError("The device was not reachable after the upgrade!") + + # If the device had to be upgraded, check if the upgrade was successful: + if needs_upgrade: + print("Checking if the device was properly updated") + if dev.needs_upgrade(): + print("Something went wrong with the update!") + sys.exit(1) + print("Done") + + +if __name__ == '__main__': + main() diff --git a/.gitlab/scripts/prplMesh/device/__init__.py b/.gitlab/scripts/prplMesh/device/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/.gitlab/scripts/prplMesh/device/configuration.py b/.gitlab/scripts/prplMesh/device/configuration.py new file mode 100644 index 0000000000..6e7d5f80f5 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/configuration.py @@ -0,0 +1,91 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### +"""This module facilitates configuring devices over serial.""" + +# Standard library +import hashlib +import time +from pathlib import Path + +# Third party +import pexpect +from device.generic import GenericDevice +from device.serial import SerialDevice + + +def configure_device(device: GenericDevice, configuration_file: Path): + """Configure a device over a serial line. + + It is assumed that the device is accessible over serial at + /dev/device_name. + Note that a udev rule can be used to create a symlink pointing to + the right serial device. + + To configure the device, the configuration file is written to the + device over the serial line. As serial lines can be unreliable, an + md5 hash of the file is computed to try to make sure the + configuration that we apply corresponds to what we expect. Note + that this means that the md5sum utility should be present on the + device. + + Parameters + ---------- + device + The device to configure. + configuration_file + The shell file to use to configure the device (relative to the + repository's top-level directory). + """ + configuration_file = device.rootdir / configuration_file + if not configuration_file.is_file(): + raise ValueError(f"Missing configuration file {configuration_file}!") + + print(f"Applying configuration {str(configuration_file)} on device {device.name}.") + conf_file_location = "/tmp/config_file.sh" + md5 = hashlib.md5() + with SerialDevice(device.baudrate, device.name, device.serial_prompt, + expect_prompt_on_connect=False) as shell: + # Workaround for PCF-600: kill leftover network restarts, and + # remove NetDev neighbors. + shell.sendline("") + time.sleep(1) + shell.send("\003") + shell.expect(device.serial_prompt) + shell.sendline("pgrep -f 'ubus -t 30 wait_for network.interface' | xargs kill -s KILL") + shell.sendline("/etc/init.d/netdev-plugin restart") + shell.expect(device.serial_prompt) + shell.sendline("") + shell.expect(device.serial_prompt) + shell.sendline("sysctl -w kernel.printk='0 4 1 7'") + shell.expect(device.serial_prompt) + shell.sendline(f"rm -rf {conf_file_location} && touch {conf_file_location}") + shell.expect(device.serial_prompt) + shell.sendline(f"cat << 'END_OF_TRANSMISSION' > {conf_file_location}") + shell.expect("> ") + with open(str(configuration_file)) as config_file: + # Write the configuration file line by line on the device: + for line in config_file.readlines(): + shell.sendstring(line) + shell.expect("> ") + md5.update(line.encode()) + shell.sendline("END_OF_TRANSMISSION") + shell.expect(device.serial_prompt) + shell.sendline(f"md5sum {conf_file_location}") + shell.expect(f"md5sum {conf_file_location}") + shell.expect(device.serial_prompt) + md5_device = shell.before.decode().splitlines()[1] + if md5_device != f"{md5.hexdigest()} {conf_file_location}": + raise ValueError("The md5 of the configuration file doesn't match! Aborting.") + + print(md5.hexdigest()) + shell.sendline(f"sh -x {conf_file_location}") + shell.expect(device.serial_prompt, timeout=150) + shell.sendline("echo conf_file_exit_code=$?") + try: + shell.expect("conf_file_exit_code=0", timeout=1) + except pexpect.TIMEOUT as err: + raise ValueError("Failed to apply the configuration on the device!") from err diff --git a/.gitlab/scripts/prplMesh/device/freedom.py b/.gitlab/scripts/prplMesh/device/freedom.py new file mode 100644 index 0000000000..09a863c5bf --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/freedom.py @@ -0,0 +1,65 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import time + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.prplos import GenericPrplOS + + +class Freedom(GenericPrplOS): + """A WNC Freedom running prplOS. + + A tftp server must be running to serve images, and both 'serverip' + and 'ipaddr' should already be set in the bootloader. + """ + + bootloader_prompt = r"IPQ9574# " + """The u-boot prompt on the target.""" + + def upgrade_from_u_boot(self, shell: pexpect.fdpexpect.fdspawn): + """Upgrade from u-boot and remove the overlay. + + Parameters + ---------- + shell: pexpect.fdpexpect.fdspawn + The serial console to send commands to. + It's assumed that the console is already stopped in u-boot. + """ + shell.sendline("") + shell.expect(self.bootloader_prompt) + # Give the ethernet interfaces some time to initialize: + time.sleep(10) + + shell.sendline(f"tftpboot 0x44000000 {self.image}") + shell.sendline("") + shell.expect("Loading: ") + shell.expect("done") + shell.expect(self.bootloader_prompt) + + shell.sendline("setenv untar_addr_kernel; setenv untar_addr_root") + shell.expect(self.bootloader_prompt) + + shell.sendline("untar 0x$fileaddr 0x$filesize kernel root") + shell.expect("filename: sysupgrade-prpl_freedom/CONTROL") + shell.expect("filename: sysupgrade-prpl_freedom/kernel") + shell.expect("filename: sysupgrade-prpl_freedom/root") + shell.expect(self.bootloader_prompt) + + shell.sendline("if test -n $untar_addr_kernel; then flash '0:HLOS' 0x$untar_addr_kernel 0x$untar_size_kernel; else echo 'kernel not found'; fi") # noqa E501 + shell.expect("blocks erased: OK", timeout=10) + shell.expect("blocks written: OK", timeout=10) + shell.expect(self.bootloader_prompt) + + shell.sendline("if test -n $untar_addr_root; then flash 'rootfs' 0x$untar_addr_root 0x$untar_size_root; else echo 'rootfs not found'; fi") # noqa E501 + shell.expect("blocks erased: OK", timeout=10) + shell.expect("blocks written: OK", timeout=10) + shell.expect(self.bootloader_prompt) diff --git a/.gitlab/scripts/prplMesh/device/generic.py b/.gitlab/scripts/prplMesh/device/generic.py new file mode 100644 index 0000000000..054750c37b --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/generic.py @@ -0,0 +1,236 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import difflib +import os +import re +import time +from typing import List, Union + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.serial import SerialDevice +from device.utils import ShellType + + +class GenericDevice(): + """Base class that all the devices class should extend. + + This class relies on access to the artifacts of a build job to + determine when an upgrade is needed (see `artifacts_dir`). + + It's also expected that: + + - The device is reachable over SSH without a + password. + - A serial line for the device is accessible at + '/dev/name'. Note that a udev rule can be used to create symlinks + for serial devices. + """ + + serial_prompt = r'root@[^\s]+:[^\s]+# ' + """The OS prompt for serial connections.""" + + baudrate = 115200 + """The baudrate of the serial connection to the device.""" + + initialization_time = 200 + """The time (in seconds) the device needs to initialize when it boots + for the first time after flashing a new image.""" + + configuration_initialization_time = 30 + """The time (in seconds) the device needs to initialize after configuring it""" + + bootloader_prompt = "=> " + """The prompt of the bootloader.""" + + boot_stop_expression = "Hit any key to stop autoboot" + """The expression signaling the device can be stopped in its bootloader.""" + + boot_stop_sequence = "\n" + """The sequence to use to stop the device in its bootloader.""" + + bootloader_reboot_command = "reset" + """The command to reboot the device in u-boot""" + + tftp_dir = "/srv/tftp" + """The root directory of the tftp server. OS images will be copied there.""" + + def __init__(self, device: str, name: str, image: Union[str, None] = None, + username: str = "root"): + """ + + Parameters + ----------- + device: str + The name of the platform (example: nec-wx3000hp). + name: str + The name of the device (it should ne reachable through ssh without a password). + image: Union[str, None] + The name of the image that can be used to upgrade the device. + username: str, optional + The username to use when connecting to the device over SSH. + """ + self.device = device + self.name = name + self.image = image + self.username = username + + self.rootdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../../../..") + self.artifacts_dir = os.path.join(self.rootdir, "build/{}".format(self.device)) + """The directory where artifacts are stored. It's expected to contain + a version file, and the image file.""" + + def reach(self, attempts: int = 5, wait: int = 5) -> bool: + """Check if the device is reachable via SSH (and optionally try multiple times). + + Parameters + ---------- + attempts: int, optional + How many times we try before concluding that the device is unreachable. + wait: int, optional + The number of seconds to wait between attempts. + + Returns + ------- + bool + True if the device is reachable via SSH, false otherwise. + """ + for _ in range(attempts): + try: + with pexpect.pxssh.pxssh(options={"StrictHostKeyChecking": "no", + "UserKnownHostsFile": "/dev/null"}) as shell: + shell.login(self.name, self.username, login_timeout=5) + return True + except (pexpect.pxssh.ExceptionPxssh, pexpect.exceptions.EOF): + print("Waiting for the device to be reachable") + time.sleep(wait) + return False + + def needs_upgrade(self): + return False + """Check if a device needs to be updated + + The check is done by comparing prplwrt-version on the target + with a local one (see `read_artifacts_dir_version). + """ + artifacts_dir_version = self.read_artifacts_dir_version() + remote_version = self.read_remote_version() + diff = difflib.unified_diff(artifacts_dir_version, remote_version, + fromfile='artifacts', tofile='device') + diff_str = '\n'.join(diff) + if diff_str: + print(diff_str) + return bool(diff_str) + + def reboot(self, serial_type: ShellType, stop_in_bootloader: bool = False): + """Reboot the device. + + Note that this method handles both the cases where the device + is currently booted into a Linux OS, or stopped in its + bootloader. + + Parameters + ----------- + serial_type: ShellType + Type of the serial connection as enum ShellType(uboot, rdkb, prplOS) + stop_in_bootloader: bool + Whether to stop the device when it enters its bootloader or not. + """ + with SerialDevice(self.baudrate, self.name, + self.serial_prompt, expect_prompt_on_connect=False) as shell: + print("Reset board.") + + if serial_type == ShellType.UBOOT: + if stop_in_bootloader: + shell.sendline("reset") + else: + shell.sendline(self.bootloader_reboot_command) + elif serial_type in [ShellType.PRPLOS, ShellType.RDKB, ShellType.LINUX_UNKNOWN]: + shell.sendline("reboot ; sleep 15 && echo force rebooting && reboot -f") + if stop_in_bootloader: + print("Device will be stopped in its bootloader.") + shell.expect(self.boot_stop_expression, timeout=180) + shell.sendline(self.boot_stop_sequence) + shell.expect(self.bootloader_prompt) + print("Device stopped in bootloader.") + + def check_serial_type(self) -> ShellType: + """ Checks type of the serial terminal. + + Returns + ------- + ShellType + Enum for rdkb, prplOS or uboot shell. + + Raises + ------- + ValueError + If connecting to the serial device failed or the serial + type could not be determined. + """ + + OSTYPE_RE = r"NAME=[^\s]*" + + with SerialDevice(self.baudrate, self.name, self.serial_prompt, + expect_prompt_on_connect=False) as shell: + shell.send('\003') + shell.expect([self.bootloader_prompt, pexpect.TIMEOUT], timeout=1) + if shell.match is not pexpect.TIMEOUT: + return ShellType.UBOOT + + # Silence kernel messages: + shell.sendline("sysctl -w kernel.printk='0 4 1 7'") + shell.expect([self.serial_prompt, pexpect.TIMEOUT], timeout=1) + shell.sendline("") + shell.sendline("cat /etc/os-release") + + os_name = "" + + shell.expect([OSTYPE_RE, pexpect.TIMEOUT], timeout=2) + os_name = str(re.search(OSTYPE_RE, shell.buffer.decode("utf-8"))) + if os_name: + if re.findall(r"OpenWrt", os_name): + return ShellType.PRPLOS + if re.findall(r"RDK", os_name): + return ShellType.RDKB + shell.sendline("") + shell.sendline("dmesg --help") + try: + shell.expect("Usage: dmesg") + return ShellType.LINUX_UNKNOWN + except pexpect.TIMEOUT as err: + raise ValueError("Unknown serial type!") from err + + def read_artifacts_dir_version(self) -> List[str]: + """Reads the local version file. + + The version file is read from the artifacts (see `artifacts_dir`). + + Returns + ------- + List[str] + The content of the local version file as a list of strings. + """ + raise NotImplementedError("Reading the local version is not implemented for this device.") + + def read_remote_version(self) -> List[str]: + """Read the version file on a remote device. + + Returns + ------- + List[str] + The content of the version file on the device as a list of strings. + """ + raise NotImplementedError("Reading the remote version is not implemented for this device.") + + def upgrade_bootloader(self): + """Upgrade the device through uboot.""" + raise NotImplementedError("u-boot upgrades is not implemented for this device.") diff --git a/.gitlab/scripts/prplMesh/device/get_device.py b/.gitlab/scripts/prplMesh/device/get_device.py new file mode 100644 index 0000000000..81471a636c --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/get_device.py @@ -0,0 +1,49 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2020-2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### +"""Utility module to get the correct device.""" + +# Standard library +from typing import Union + +# Third party +import device.generic +import device.prplos +import device.turris_prplos +import device.turris_rdk_b +import device.haze +import device.urx_osp +import device.freedom + + +def device_from_name(name: str, target_name: str, image: Union[str, None] = None + ) -> device.generic.GenericDevice: + """Construct a device based on its name and type. + + Parameters + ---------- + name: str + The name of the device + target_name: str + The name of the target. + image: image: Union[str, None] + The name of the image (optional, defaults to None). + """ + if name == "turris-omnia-rdk": + dev = device.turris_rdk_b.TurrisRdkb(name, target_name, image) + elif name == "turris-omnia": + dev = device.turris_prplos.TurrisPrplOS(name, target_name, image) + elif name == "haze": + dev = device.haze.Haze(name, target_name, image) + elif name == "urx_osp" or name == "urx_ospv2": + dev = device.urx_osp.URXOSP(name, target_name, image) + elif name == "freedom": + dev = device.freedom.Freedom(name, target_name, image) + else: + # if no device matched, try the generic prplOS (sysupgrade) + print("No specific device matched, using GenericPrplOS.") + dev = device.prplos.GenericPrplOS(name, target_name, image) + return dev diff --git a/.gitlab/scripts/prplMesh/device/haze.py b/.gitlab/scripts/prplMesh/device/haze.py new file mode 100644 index 0000000000..b8c3f708e2 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/haze.py @@ -0,0 +1,65 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import time + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.prplos import GenericPrplOS + + +class Haze(GenericPrplOS): + """A WNC Haze running prplOS. + + A tftp server must be running to serve images, and both 'serverip' + and 'ipaddr' should already be set in the bootloader. + """ + + bootloader_prompt = r"IPQ807x# " + """The u-boot prompt on the target.""" + + def upgrade_from_u_boot(self, shell: pexpect.fdpexpect.fdspawn): + """Upgrade from u-boot and remove the overlay. + + Parameters + ---------- + shell: pexpect.fdpexpect.fdspawn + The serial console to send commands to. + It's assumed that the console is already stopped in u-boot. + """ + shell.sendline("") + shell.expect(self.bootloader_prompt) + # Give the ethernet interfaces some time to initialize: + time.sleep(10) + + shell.sendline(f"tftpboot 0x44000000 {self.image}") + shell.sendline("") + shell.expect("Loading: ") + shell.expect("done") + shell.expect(self.bootloader_prompt) + + shell.sendline("setenv untar_addr_kernel; setenv untar_addr_root") + shell.expect(self.bootloader_prompt) + + shell.sendline("untar 0x$fileaddr 0x$filesize kernel root") + shell.expect("filename: sysupgrade-prpl_haze/CONTROL") + shell.expect("filename: sysupgrade-prpl_haze/kernel") + shell.expect("filename: sysupgrade-prpl_haze/root") + shell.expect(self.bootloader_prompt) + + shell.sendline("if test -n $untar_addr_kernel; then flash '0:HLOS' 0x$untar_addr_kernel 0x$untar_size_kernel; else echo 'kernel not found'; fi") # noqa E501 + shell.expect("blocks erased: OK", timeout=10) + shell.expect("blocks written: OK", timeout=10) + shell.expect(self.bootloader_prompt) + + shell.sendline("if test -n $untar_addr_root; then flash 'rootfs' 0x$untar_addr_root 0x$untar_size_root; else echo 'rootfs not found'; fi") # noqa E501 + shell.expect("blocks erased: OK", timeout=10) + shell.expect("blocks written: OK", timeout=10) + shell.expect(self.bootloader_prompt) diff --git a/.gitlab/scripts/prplMesh/device/logs.py b/.gitlab/scripts/prplMesh/device/logs.py new file mode 100644 index 0000000000..f765f18078 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/logs.py @@ -0,0 +1,89 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### +"""This module facilitates capturing logs from a device over serial.""" + +# Standard library +import os +import time +from collections import namedtuple +from datetime import datetime + +# Third party +import pexpect +from device.generic import GenericDevice +from device.serial import SerialDevice +from device.utils import ShellType + +LogCommand = namedtuple("LogCommand", "cmd, filename") + +LOG_COMMANDS = \ + [ + LogCommand("ip address show", "ip-addr.txt"), + LogCommand("cat /etc/config/network", "config-network.txt"), + LogCommand('ubus call IP _get \'{ "depth": "10" }\'', "dm-IP.txt"), + LogCommand('ubus call Bridging _get \'{ "depth": "10" }\'', "dm-Bridge.txt"), + LogCommand('ubus call Ethernet _get \'{ "depth": "10" }\'', "dm-Ethernet.txt"), + LogCommand('ubus call DHCPv4 _get \'{ "depth": "10" }\'', "dm-DHCPv4.txt"), + LogCommand('ubus call DHCPv6 _get \'{ "depth": "10" }\'', "dm-DHCPv6.txt"), + LogCommand('ubus call Firewall _get \'{ "depth": "10" }\'', "dm-Firewall.txt"), + LogCommand('ubus call WiFi _get \'{ "depth": "10" }\'', "dm-WiFi.txt"), + LogCommand('logread', "syslog.txt"), + LogCommand('iptables -L', "iptables-filter.txt"), + LogCommand('find /etc/config/tr181-bridging/ -type f -print -exec cat {} \';\'', + "config-tr181-bridging.txt"), + LogCommand('find /etc/amx/tr181-bridging/ -type f -print -exec cat {} \';\'', + "amx-tr181-bridging.txt"), + ] + + +def capture_logs(device: GenericDevice, path: str): + """Capture logs from a device over a serial line. + + Commands from LOG_COMMANDS are executed, and the results are saved + in individual files (one file per command). + + It is assumed that the device is accessible over serial at + /dev/device_name. + Note that a udev rule can be used to create a symlink pointing to + the right serial device. + + The logs to capture are defined statically in this file. + + Parameters + ---------- + device + The device to configure. + path + The path to save the log files to. If it doesn't exist, it + will be created. + """ + + print("Checking that the device is booted into an OS.") + if device.check_serial_type is ShellType.UBOOT: + raise ValueError("The device is currently in bootloader, logs \ + cannot be captured.") + print(f"Capturing logs from device {device.name}.") + os.makedirs(path, exist_ok=True) + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + with open(f"{path}/all-console-{timestamp}.txt", "wb") as logfile: + with SerialDevice(device.baudrate, device.name, device.serial_prompt, + expect_prompt_on_connect=False, logfile=logfile) as shell: + shell.sendline("") + time.sleep(1) + # Interrupt any running command: + shell.send("\003") + shell.expect(device.serial_prompt) + # There might have been two prompts, if no command had to be interrupted: + shell.expect([device.serial_prompt, pexpect.TIMEOUT]) + for log in LOG_COMMANDS: + shell.sendline(log.cmd) + shell.expect(log.cmd) + shell.expect(device.serial_prompt) + cmd_output = shell.before.decode("utf-8") + with open(f"{path}/{log.filename}", "w", encoding="utf-8") as output_file: + output_file.writelines(cmd_output) + print("Done") diff --git a/.gitlab/scripts/prplMesh/device/prplos.py b/.gitlab/scripts/prplMesh/device/prplos.py new file mode 100644 index 0000000000..ad9aaf5e0a --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/prplos.py @@ -0,0 +1,161 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import os +import shutil +import subprocess +import sys +import time +from typing import List + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.generic import GenericDevice +from device.serial import SerialDevice + + +class GenericPrplOS(GenericDevice): + """A generic PrplOSDevice. + + Since upgrading through uboot is generally target-specific, it + only offers the sysupgrade option. + """ + + def read_artifacts_dir_version(self) -> List[str]: + """Reads the local prplwrt-version. + + The version file is read from the artifacts (see `artifacts_dir`). + + Returns + ------- + List[str] + The content of the local prplwrt-version as a list of strings. + """ + with open(os.path.join(self.artifacts_dir, "prplwrt-version")) as version_file: + return version_file.read().splitlines() + + def read_remote_prplos_version(self) -> List[str]: + """Read prplos build.version on a remote device + + Returns + ------- + List[str] + The content of the build.version file on the device as a list of strings. + """ + version = None + with pexpect.pxssh.pxssh(logfile=sys.stdout.buffer) as shell: + shell.login(self.name, self.username) + shell.sendline("cat /etc/build.version") + shell.expect("cat /etc/build.version") + shell.prompt() + # remove the first (empty) line: + version = shell.before.decode().splitlines()[1:] + return version + + def read_remote_version(self) -> List[str]: + """Read prplwrt-version on a remote device + + Returns + ------- + List[str] + The content of the prplwrt-version file on the device as a list of strings. + """ + version = None + with pexpect.pxssh.pxssh(logfile=sys.stdout.buffer) as shell: + shell.login(self.name, self.username) + shell.sendline("cat /etc/prplwrt-version") + shell.expect("cat /etc/prplwrt-version") + shell.prompt() + # remove the first (empty) line: + version = shell.before.decode().splitlines()[1:] + return version + + def upgrade_bootloader(self): + """Upgrade the device through u-boot. + + It requires a running tftp server listenning on the IP the + target device will expect (see `ipaddr` in the uboot environment). + """ + serial_path = "/dev/{}".format(self.name) + if not os.path.exists(serial_path): + raise ValueError("The serial device {} does not exist!\n".format(serial_path) + + "Please make sure you have an appropriate udev rule for it.") + print("Copying image '{}' to '{}'".format(self.image, self.tftp_dir)) + shutil.copy(os.path.join(self.artifacts_dir, self.image), self.tftp_dir) + print("Image copied to {}.".format(self.tftp_dir)) + with SerialDevice(self.baudrate, self.name, self.serial_prompt, + expect_prompt_on_connect=False) as shell: + self.reboot(self.check_serial_type(), + stop_in_bootloader=True) + + try: + self.upgrade_from_u_boot(shell) + except pexpect.exceptions.TIMEOUT: + print("The upgrade timed out, trying one more time after a reboot.") + # Interrupt any running command: + shell.send('\003') + time.sleep(1) + self.reboot(self.check_serial_type(), + stop_in_bootloader=True) + try: + self.upgrade_from_u_boot(shell) + except pexpect.exceptions.TIMEOUT as err: + print("The upgrade failed again, aborting.") + raise err + print("The upgrade is done, rebooting.") + self.reboot(self.check_serial_type()) + shell.expect("Please press Enter to activate this console", timeout=180) + # activate the console: + shell.sendline("") + shell.expect(self.serial_prompt) + # Wait at least for the CAC timer: + time.sleep(self.initialization_time) + + def sysupgrade(self): + serial_path = "/dev/{}".format(self.name) + if not os.path.exists(serial_path): + raise ValueError("The serial device {} does not exist!\n".format(serial_path) + + "Please make sure you have an appropriate udev rule for it.") + print("Copying image '{}' to the target".format(self.image)) + try: + subprocess.check_output(["scp", + "{}/{}".format(self.artifacts_dir, self.image), + "{}:/tmp/{}".format(self.name, self.image)]) + except subprocess.CalledProcessError as exc: + print("Failed to copy the image to the target:\n{}".format(exc.output)) + raise exc + with SerialDevice(self.baudrate, self.name, self.serial_prompt) as shell: + # Turn off the wifi to make sure it doesn't prevent the upgrade: + shell.sendline("wifi down") + # Do the upgrade + shell.sendline("sysupgrade -v /tmp/{}".format(self.image)) + # first give it 30 seconds, and fail early if the upgrade didn't start: + shell.expect(r"Performing system upgrade\.\.\.", timeout=30) + # now give it more time to apply the upgrade and reboot: + shell.expect(r"Rebooting system\.\.\.", timeout=300) + shell.expect("Please press Enter to activate this console", timeout=180) + # activate the console: + shell.sendline("") + shell.expect(self.serial_prompt) + time.sleep(self.initialization_time) + + def upgrade_from_u_boot(self, shell: pexpect.fdpexpect.fdspawn): + """Upgrades the OS image from the bootloader once it's ready. + Since the exact procedure to upgrade devices can vary from + devices to devices, this method should be overridden in sub + classes. + Subclasses can assume the serial shell is ready to receive u-boot commands + + Parameters + ---------- + shell + The pexpect-based shell (in u-boot). + """ + raise NotImplementedError("Not implemented for this device.") diff --git a/.gitlab/scripts/prplMesh/device/serial.py b/.gitlab/scripts/prplMesh/device/serial.py new file mode 100644 index 0000000000..8b9c659ea4 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/serial.py @@ -0,0 +1,106 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### +"""This module contains utilities for shells over a serial line.""" +# Standard library +import os +import sys +import time + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +import serial + + +class SerialDevice(pexpect.fdpexpect.fdspawn): + """A base class representing a serial device using pexpect. + + It assumes the device will be accessible at /dev/ (use a + udev rule to make sure it's the case). + """ + + send_delay = 4 + """The delay (in milliseconds) to wait before sending another character. + This is used for cases where the serial console cannot handle a regular pexpect + "sendline" because the characters are sent too fast. + """ + + def __init__(self, baudrate: int, name: str, prompt, expect_prompt_on_connect=True, + logfile=sys.stdout.buffer): + self.baudrate = baudrate + self.name = name + self.prompt_expr = prompt + self.expect_prompt_on_connect = expect_prompt_on_connect + self.logfile = logfile + + self.serial_path = "/dev/{}".format(self.name) + self.serial = None + self.shell = None + + def __enter__(self): + self.connect() + return self + + def __exit__(self, *exc_details): + self.disconnect() + + def connect(self): + """Create the serial connection and the pexpect shell. + + Raises + ------ + ValueError + If the shell is not alive after its creation. + """ + if not os.path.exists(self.serial_path): + raise ValueError(f"The serial device {self.serial_path} does not exist!\n" + + "Please make sure you have an appropriate udev rule for it.") + if self.serial is not None: + raise ValueError("Serial already connected!") + self.serial = serial.Serial(self.serial_path, self.baudrate, xonxoff=True) + self.serial.flushInput() + super().__init__(self.serial, logfile=self.logfile) + + # Unlike pxssh, fdspawn doesn't have a spawn method. For ease + # of use, create one: + def prompt(): + self.expect(self.prompt_expr) + self.prompt = prompt + + if not self.isalive(): + raise ValueError("Unable to connect to the serial device!") + self.sendline("") + if self.expect_prompt_on_connect: + self.prompt() + + def disconnect(self): + """Close the serial connection.""" + if self.serial.is_open: + self.serial.close() + + def sendline(self, s: str): + """Imitates pexpect sendline(), but waits for a delay between each + characters to give the device some time to process them. + """ + if not self.serial: + raise ValueError("There is no serial! Use connect() to create it first.") + for char in s: + self.send(char) + time.sleep(self.send_delay / 1000.) + super().sendline("") + time.sleep(self.send_delay / 1000.) + + def sendstring(self, s: str): + """Imitates pexpect send(), but waits for a delay between each + characters to give the device some time to process them. + """ + if not self.serial: + raise ValueError("There is no serial! Use connect() to create it first.") + for char in s: + self.send(char) + time.sleep(self.send_delay / 1000.) diff --git a/.gitlab/scripts/prplMesh/device/turris_prplos.py b/.gitlab/scripts/prplMesh/device/turris_prplos.py new file mode 100644 index 0000000000..7f728e57d8 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/turris_prplos.py @@ -0,0 +1,156 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import re +import subprocess +import time +import traceback + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.prplos import GenericPrplOS + + +class TurrisPrplOS(GenericPrplOS): + """A Turris Omnia running prplOS.""" + + bootloader_prompt = "=> " + """The u-boot prompt on the target.""" + + rescue_prompt = "~ # " + + def _upgrade_from_u_boot_write_emmc(self, shell: pexpect.fdpexpect.fdspawn): + """Do the upgrade in the bootloader by loading the image over tftp, + and write it to the emmc. + + Note that this might not work on some recent Turris Omnias, + where u-boot is not able to write to the emmc. For those, see + `_upgrade_from_rescue_shell`. + """ + uncompressed_image = self.image.split(".gz")[0] + shell.sendline("setenv bootargs earlyprintk console=ttyS0,115200") + shell.expect(self.bootloader_prompt) + time.sleep(1) + shell.sendline("setenv set_blkcnt 'setexpr blkcnt ${filesize} + 0x7ffff && setexpr blkcnt ${blkcnt} / 0x80000 && setexpr blkcnt ${blkcnt} * 0x400'") # noqa E50 + shell.expect(self.bootloader_prompt) + shell.sendline("tftpboot ${kernel_addr_r} " + f"{uncompressed_image}") + shell.expect("Loading: ") + shell.expect("done") + shell.expect(self.bootloader_prompt) + # Write to the emmc: + shell.sendline("run set_blkcnt && mmc dev 0 0 && mmc erase 0 ${blkcnt} && mmc write ${kernel_addr_r} 0 ${blkcnt}") # noqa E501 + res = shell.expect(["0 blocks written: ERROR", r"\d+ blocks written: OK"], timeout=60) + if res == 0: + raise ValueError("Failed to write to the emmc: 0 blocks written!") + # Still check that a non-zero number of blocks were written: + try: + blkcnt_line = re.search(r"(\d+) blocks written: OK", shell.after.decode()) + blkcnt = blkcnt_line.group(1) + if int(blkcnt) <= 0: + raise ValueError("Failed to write to the emmc") + except ValueError as err: + raise ValueError("Failed to write to the emmc") from err + + def _upgrade_from_rescue_shell(self, shell: pexpect.fdpexpect.fdspawn): + """Use the device's "rescue shell to load the image over SSH, and + write it back to NAND flash. + """ + shell.sendline("") + shell.expect(self.bootloader_prompt) + shell.sendline("setenv omnia_reset 6") + shell.expect(self.bootloader_prompt) + + bootcmd = """setenv bootcmd 'setenv bootargs "earlyprintk console=ttyS0,115200 omniarescue=$omnia_reset rescue_mode=$omnia_reset"; sf probe; sf read 0x1000000 0x100000 0x700000; lzmadec 0x1000000 0x1700000; if gpio input gpio@71_4; then bootm 0x1700000#sfp; else bootm 0x1700000; fi;bootz 0x1000000'""" # noqa E501 + shell.sendline(bootcmd) + shell.expect(self.bootloader_prompt) + shell.sendline("run bootcmd") + shell.expect("Initializing the system", timeout=120) + shell.expect(self.rescue_prompt) + + # We need to get the IP address of the device for the upgrade. + # Since we rely on an SSH configuration to be available, get it from there. + ssh_config = subprocess.check_output(("ssh", "-G", self.name)).decode() + hostname = re.search(r"\nhostname ([^\n]+)", ssh_config) + try: + ipaddr = hostname.group(1) + except (AttributeError, IndexError) as err: + raise ValueError("The IP address of the device could not be found.\n" + + "Make sure you have the IP of the device in your SSH " + + "configuration and try again") from err + + # We cannot know the subnet but we don't care, it will only be + # used by the device for the upgrade: + shell.sendline(f"ip addr add {ipaddr}/1 dev eth2") + shell.expect(self.rescue_prompt) + + shell.sendline("ip link set eth2 up") + shell.expect(self.rescue_prompt) + # Start dropbear from the rescue shell: + shell.sendline("rm -f /etc/dropbear/dropbear_rsa_host_key && dropbear -R") + shell.expect(self.rescue_prompt) + + # Give the ethernet interface some time to initialize: + time.sleep(10) + + # Try to copy the uncompressed image to the device: + uncompressed_image = self.image.split(".gz")[0] + try: + subprocess.check_output(["scp", self.tftp_dir + "/" + + uncompressed_image, + f"{self.name}:/tmp/"]) + except subprocess.CalledProcessError as exc: + print("Failed to copy the image to the target:\n{}".format(exc.output)) + raise exc + + # Finally, write the image to the flash: + shell.sendline(f'dd if="/tmp/{uncompressed_image}" ' + + 'of=/dev/mmcblk0 bs=4096 conv=fsync') + shell.expect(self.rescue_prompt) + + def upgrade_from_u_boot(self, shell: pexpect.fdpexpect.fdspawn): + """Upgrade from u-boot, discarding the overlay. + + A first attempt is made to load the image in u-boot using + tftp, then write it back to the emmc. For some devices (some + recent Turris Omnias), writing to the emmc from u-boot doesn't + work. + To make sure we can always upgrade the device anyway, in this + case we fall back to using the rescue shell of the device + instead. The rescue shell is entered just like if the user + pressed the reset button until 7 leds are on. From there, the + prplOS image is copied over SSH and then written to flash + using 'dd'. + + Parameters + ---------- + shell: pexpect.fdpexpect.fdspawn + The serial console to send commands to. + It's assumed that the console is already stopped in u-boot. + """ + + # Before anything else, we need to decompress the image. + # gunzip will always have a non-zero exit-code, because there + # is trailing data at the end of the archive. Instead of + # checking gunzip's exit code, just check that there is an + # extracted file. + subprocess.run(["gunzip", "-f", f"{self.tftp_dir}/{self.image}"], check=False) + try: + subprocess.check_output(["test", "-f", f"{self.tftp_dir}/" + + self.image.split(".gz")[0]]) + except subprocess.CalledProcessError as err: + raise ValueError("An error occured while decompressing the image!") from err + + try: + self._upgrade_from_u_boot_write_emmc(shell) + except ValueError: + print("Unable to flash the device through u-boot:") + print(traceback.format_exc()) + print("Trying through rescue shell.") + self._upgrade_from_rescue_shell(shell) diff --git a/.gitlab/scripts/prplMesh/device/turris_rdk_b.py b/.gitlab/scripts/prplMesh/device/turris_rdk_b.py new file mode 100644 index 0000000000..cbbc635f39 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/turris_rdk_b.py @@ -0,0 +1,368 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import os +import re +import subprocess +import time +from pathlib import Path + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.generic import GenericDevice +from device.serial import SerialDevice +from device.utils import check_uboot_var, serial_cmd_err, ShellType + + +class TurrisRdkb(GenericDevice): + """Represents a RDKB device. + + Offers methods to check if a device needs to be upgraded and to do the actual upgrade. + + It needs to have access to the artifacts of a build job to determine when an upgrade + is needed (see `artifacts_dir`). + """ + + TURRIS_DTB = "armada-385-turris-omnia.dtb" + """ Device Tree Blob (Flat Device Tree) for Turris Omnia. + The DTB is a database that represents the hardware components on a given board. """ + + KERNEL_PARTITION = "mmcblk0p3" + """ eMMC partiotion used for RDKB kernel (zImage) and DTB file.""" + + ROOTFS_PARTITION = "mmcblk0p5" + """ eMMC partition used for RDKB rootfs.""" + + serial_prompt = r'root@[^\s]+:[^\s]+# ' + """ Regular expression for root prompt.""" + + UBOOT_PROMPT = "=>" + """ Standard UBoot prompt.""" + + kernel = "zImage" + """ The name of the kernel binary that can be used to upgrade kernel on device.""" + + def __init__(self, device: str, name: str, rdkbfs: str, username: str = "root"): + """ + + Parameters + ----------- + device: str + The name of the platform (example: turris-omnia). + name: str + The name of the device (it should be reachable through ssh without a password). + rdkbfs: str + The name of the rdkbfs tarball that can be used to upgrade the device. + username: str, optional + The username to use when connecting to the device over SSH. + """ + self.device = device + self.name = name + self.rdkbfs = rdkbfs + self.username = username + + self.rootdir = os.path.join(os.path.dirname(os.path.realpath(__file__)), "../..") + self.artifacts_dir = os.path.join(self.rootdir, "build/{}".format(self.device)) + """The directory where artifacts are stored. It's expected to contain the + image, kernel, dtb files.""" + + def is_prplos_ready(self) -> bool: + """ Checks if prplOS propmpt ready to use after reboot""" + + with SerialDevice(self.baudrate, self.name, self.serial_prompt, + expect_prompt_on_connect=False) as shell: + shell.expect(["Please press Enter to activate this console.", pexpect.TIMEOUT], + timeout=180) + if shell.match == pexpect.TIMEOUT: + return False + + # Give the shell a few seconds to be ready: + time.sleep(30) + shell.sendline("") + time.sleep(2) + + return True + + def check_images_on_board(self): + """Check images on the Turris Omnia. + + If images was not copied return False, otherwise True. + """ + + with SerialDevice(self.baudrate, self.name, self.serial_prompt) as shell: + + def find_file(file_name: str): + shell.sendline("") + shell.expect(self.serial_prompt) + shell.sendline(f"find /tmp -maxdepth 1 -name {file_name}") + shell.expect([f"/tmp/{file_name}", pexpect.TIMEOUT]) + if shell.match == pexpect.TIMEOUT: + raise ValueError(f"File {file_name} not found.") + + shell.sendline("") + find_file(self.rdkbfs) + find_file(self.kernel) + find_file(self.TURRIS_DTB) + + def load_rdkb_firmware(self): + """Copy RDKB rootfs and kernel to the Turris Omnia. + """ + + print(f"Copying '{self.rdkbfs}'\n'{self.kernel}'\n'{self.TURRIS_DTB}'\n to the target") + try: + subprocess.check_output(["scp", "-o StrictHostKeyChecking=no", + f"{self.artifacts_dir}/{self.rdkbfs}", + f"{self.username}@{self.name}:/tmp/{self.rdkbfs}"]) + subprocess.check_output(["scp", "-o StrictHostKeyChecking=no", + f"{self.artifacts_dir}/{self.kernel}", + f"{self.username}@{self.name}:/tmp/{self.kernel}"]) + subprocess.check_output(["scp", "-o StrictHostKeyChecking=no", + f"{self.artifacts_dir}/{self.TURRIS_DTB}", + f"{self.username}@{self.name}:/tmp/{self.TURRIS_DTB}"]) + except subprocess.CalledProcessError as exc: + print(f"Failed to copy the image to the target:\n{exc.output}") + raise exc + + def burn_rdkb_on_board(self): + """Burn RDKB image on Turris Omnia. + """ + # We need to make sure we're not running currently running + # RDK-B! Otherwise, the rest of this method will try to "rm-rf" + # the whole rootfs on the running system. + serial_type = self.check_serial_type() + if serial_type == ShellType.RDKB: + raise ValueError("The board is currently running RDK-B! Aborting.") + # If we're currently in u-boot, don't even try the upgrade + # either, as there is no way it could succeed. + if serial_type == ShellType.UBOOT: + raise ValueError("The board is not currently in u-boot, aborting.") + + self.check_images_on_board() + + with SerialDevice(self.baudrate, self.name, self.serial_prompt) as shell: + + def mount_mmc(partition: str): + serial_cmd_err(shell, self.serial_prompt, f"mount /dev/{partition} /mnt") + shell.sendline("mount") + shell.expect(f"/dev/{partition} on /mnt") + + def umount_mmc(): + serial_cmd_err(shell, self.serial_prompt, "umount /mnt") + shell.sendline("du -sh /mnt") + shell.expect("0") + + def copy_to_mmc(src: str, dst: str): + try: + serial_cmd_err(shell, self.serial_prompt, f"cp -v {src} {dst}") + except ValueError: + umount_mmc() + raise + + def check_partition(partition: str): + shell.sendline("") + shell.expect(self.serial_prompt) + shell.sendline(f"find /dev/ -maxdepth 1 -name {partition}") + shell.expect([f"/dev/{partition}", pexpect.TIMEOUT]) + if shell.match == pexpect.TIMEOUT: + raise ValueError( + f"Partition {partition} not found, need to create it.") + + shell.sendline("") + print("Start to burn RDKB on Turris Omnia") + + check_partition(self.KERNEL_PARTITION) + check_partition(self.ROOTFS_PARTITION) + + shell.sendline("") + + mount_mmc(self.KERNEL_PARTITION) + + print("Install kernel.") + copy_to_mmc(f"/tmp/{self.kernel}", "/mnt/zImage") + + shell.sendline("") + + print("Install Device Tree Blob") + copy_to_mmc(f"/tmp/{self.TURRIS_DTB}", f"/mnt/{self.TURRIS_DTB}") + + shell.sendline("") + + umount_mmc() + + shell.sendline("") + + print("Install RDKB rootfs.") + mount_mmc(self.ROOTFS_PARTITION) + + shell.sendline("") + + serial_cmd_err(shell, self.serial_prompt, "rm -rf /mnt/*") + shell.sendline("du -sh /mnt") + shell.expect("0") + + serial_cmd_err(shell, self.serial_prompt, f"tar -xzf /tmp/{self.rdkbfs} -C /mnt/") + + shell.sendline("") + + umount_mmc() + + def load_rdkb(self): + """ Launch RDKB on Turris Omnia. + + Raises + ----------- + ValueError + If serial does not exist or unable to connect. + """ + + self.reboot(self.check_serial_type()) + + with SerialDevice(self.baudrate, self.name, + self.serial_prompt, expect_prompt_on_connect=False) as shell: + shell.expect("Hit any key to stop autoboot") + shell.sendline("") + shell.expect(self.UBOOT_PROMPT) + + check_uboot_var(shell, "yocto_bootargs", "yocto_bootargs=earlyprintk") + check_uboot_var(shell, "mmcboot", "mmcboot=run") + check_uboot_var(shell, "yocto_mmcload", "yocto_mmcload=setenv") + + shell.sendline("run mmcboot") + shell.expect(["TurrisOmnia-GW login", pexpect.TIMEOUT]) + time.sleep(30) + + # Set ip to erouter0 to disable some error messages + # This causes the CcspPandMSsp process to not interrupt the configuration after flashing + # See PPM-2247 + shell.sendline("ifconfig erouter0 10.0.0.10") + + def sysupgrade(self): + """Upgrade RDKB image on Turris Omnia and launch it. + """ + + # Currently RDK-B can only be upgraded when the device has booted into prplOS: + serial_type = self.check_serial_type() + if serial_type != ShellType.PRPLOS: + print("The device is not running prplOS, rebooting.") + self.reboot(serial_type) + if not self.is_prplos_ready(): + raise ValueError("Failed to get ready prplOS serial.") + + self.load_rdkb_firmware() + + self.burn_rdkb_on_board() + + self.load_rdkb() + + def read_rdkb_rootfs_version(self) -> str: + """ Read new image version. + + Returns + ----------- + String with image build date otherwise empty string. + """ + + image_name_re = r"rdkb[^\s]+[\d*]+[^\s]\.rootfs\.tar\.gz" + """ Find correct RDKB rootfs file.""" + + image_date_re = r"0*[1-9]\d{4,}" + """ Retrieve RDKB rootfs build date.""" + + image = "" + for artifact in Path(self.artifacts_dir).iterdir(): + image = re.findall(image_name_re, str(artifact)) + if image: + break + + date_list = re.findall(image_date_re, str(image)) + + date = "" + for i in date_list: + date = str(i) + + return date + + def read_remote_rdkb_version(self) -> str: + """ Read current RDKB rootfs version. + + Returns + ----------- + str + RDKB rootfs version. + + Raises + ----------- + ValueError + If version of rootfs not found. + """ + + RDKB_VERSION_PATH = "/mnt/etc/version" + IMAGE_DATE_RE = r"0*[1-9]\d{4,}" + """ Retrieve RDKB rootfs build date.""" + + with pexpect.pxssh.pxssh(options={"StrictHostKeyChecking": "no", + "UserKnownHostsFile": "/dev/null"}) as shell: + shell.login(self.name, self.username) + + shell.sendline(f"mount /dev/{self.ROOTFS_PARTITION} /mnt") + shell.prompt() + shell.sendline("mount") + shell.expect(f"/dev/{self.ROOTFS_PARTITION} on /mnt") + + shell.sendline(f"cat {RDKB_VERSION_PATH}") + shell.expect([f"cat {RDKB_VERSION_PATH}", pexpect.TIMEOUT]) + shell.prompt() + version = re.findall(IMAGE_DATE_RE, shell.before.decode()) + if not version: + # We need to add dummy version for handling case when we flash new board or board + # where RDKB rootfs was brocken and version file does not exist. + version = "1" + + shell.sendline("umount /mnt") + shell.expect("umount /mnt") + shell.prompt() + + for i in version: + version = str(i) + + return version + + def needs_upgrade(self) -> bool: + """ Check if we need to upgrade the board or not. + + Returns + ----------- + bool + True if upgrade required otherwise False. + + Raises + ----------- + ValueError + If failed to get serial connection or prplOS is not ready for use + """ + + serial_type = self.check_serial_type() + if serial_type == ShellType.UBOOT: + self.reboot(serial_type) + if not self.is_prplos_ready(): + raise ValueError("Failed to get ready prplOS serial.") + + current_version = int(self.read_remote_rdkb_version()) + new_version = int(self.read_rdkb_rootfs_version()) + + print(f"Current RDKB version is: {current_version} \nNew RDKB version is: {new_version}") + + will_upgrade = new_version != current_version + if will_upgrade and serial_type == ShellType.RDKB: + self.reboot(self.check_serial_type()) + if not self.is_prplos_ready(): + raise ValueError("Failed to get ready prplOS serial.") + + return will_upgrade diff --git a/.gitlab/scripts/prplMesh/device/urx_osp.py b/.gitlab/scripts/prplMesh/device/urx_osp.py new file mode 100644 index 0000000000..cfb2203bb2 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/urx_osp.py @@ -0,0 +1,73 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +import time + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh +from device.prplos import GenericPrplOS + + +class URXOSP(GenericPrplOS): + """An MXL Open Service Platform URX device running prplOS. + + A tftp server must be running to serve images, and both 'serverip' + and 'ipaddr' should already be set in the bootloader. + """ + + initialization_time = 280 + """The time (in seconds) the device needs to initialize when it boots + for the first time after flashing a new image.""" + + bootloader_prompt = r"Lightning # " + """The u-boot prompt on the target.""" + + bootloader_reboot_command = "run bootcmd" + """The command to reboot the device in u-boot""" + + def upgrade_from_u_boot(self, shell: pexpect.fdpexpect.fdspawn): + """Upgrade from u-boot and remove the overlay. + + Parameters + ---------- + shell: pexpect.fdpexpect.fdspawn + The serial console to send commands to. + It's assumed that the console is already stopped in u-boot. + """ + shell.sendline("") + shell.expect(self.bootloader_prompt) + # Give the ethernet interfaces some time to initialize: + time.sleep(5) + + # Changes the default image name to upgrade + # shell.sendline(f"setenv fullimage {self.image}") + # shell.sendline("") + # shell.sendline("saveenv") + # shell.expect("OK") + + shell.sendline("run update_fullimage") + shell.expect("done", timeout=30) + # Image transfer successful + shell.expect("Found device tree image", timeout=15) + shell.expect("Saving Environment to MMC", timeout=15) + shell.expect("OK", timeout=15) + # Image written to MMC + shell.sendline("") + shell.expect(self.bootloader_prompt) + + time.sleep(10) + shell.sendline( + "mmc erase ${overlay_container_a_block_start} ${overlay_container_a_block_size}") + shell.expect("blocks erased: OK", timeout=15) + time.sleep(5) + shell.sendline( + "mmc erase ${overlay_container_a_block_start} ${overlay_container_a_block_size}") + shell.expect("blocks erased: OK", timeout=15) + shell.expect(self.bootloader_prompt) diff --git a/.gitlab/scripts/prplMesh/device/utils.py b/.gitlab/scripts/prplMesh/device/utils.py new file mode 100644 index 0000000000..f4f9286864 --- /dev/null +++ b/.gitlab/scripts/prplMesh/device/utils.py @@ -0,0 +1,85 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2022 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +# Standard library +from enum import Enum + +# Third party +import pexpect +import pexpect.fdpexpect +import pexpect.pxssh + + +class ShellType(Enum): + """ ShellType enum contains 3 types of the possible shell on the device: + + UBOOT, PRPLOS, RDKB + """ + UBOOT = 1 + PRPLOS = 2 + RDKB = 3 + LINUX_UNKNOWN = 4 + + +def check_uboot_var(shell, variable: str, expectation: str): + """ Check content of the UBoot variable. + + Parameters + ---------- + shell : pexpect.fdpexpect.fdspawn + Shell file descriptor. + + variable : str + The number of seconds to wait between attempts. + + expectation: str + Expected content of the variable. + + Raises + ------- + ValueError + If the getting content of the variable failed. + """ + + UBOOT_PROMPT = "=>" + # Standard UBoot prompt. + + shell.sendline(f"printenv {variable}") + shell.expect([expectation, pexpect.TIMEOUT]) + shell.expect([UBOOT_PROMPT, pexpect.TIMEOUT]) + if shell.match == pexpect.TIMEOUT: + raise ValueError(f"Failed to get {variable} variable.") + + +def serial_cmd_err(shell, cmd_prompt: str, command: str): + """ Execute command via serial port and check error code. + + Parameters + ---------- + shell : int + Shell file descriptor. + + cmd_prompt : + Serial prompt. + + command : str + Command which should be executed. + + Raises + ------- + ValueError + If command prompt not found or failed to execute command. + """ + shell.sendline("") + shell.expect([cmd_prompt, pexpect.TIMEOUT]) + if shell.match == pexpect.TIMEOUT: + raise ValueError("Failed to get serial prompt!") + + shell.sendline(f"{command};echo err_code $?") + shell.expect(["err_code 0", pexpect.TIMEOUT]) + if shell.match == pexpect.TIMEOUT: + raise ValueError(f"Failed to execute {command}!") diff --git a/.gitlab/scripts/prplMesh/dut-configuration/R1/freedom.sh b/.gitlab/scripts/prplMesh/dut-configuration/R1/freedom.sh new file mode 100644 index 0000000000..5cd0a4342c --- /dev/null +++ b/.gitlab/scripts/prplMesh/dut-configuration/R1/freedom.sh @@ -0,0 +1,100 @@ +#!/bin/sh + +# We need to source some files which are only available on prplWrt +# devices, so prevent shellcheck from trying to read them: +# shellcheck disable=SC1091 + +set -e + +# Start with a new log file: +rm -f /var/log/messages && syslog-ng-ctl reload + +/etc/init.d/tr181-upnp stop || true +rm -f /etc/rc.d/S*tr181-upnp + +/etc/init.d/obuspa stop || true +rm -f /etc/rc.d/S*obuspa + +# Stop the default ssh server on the lan-bridge +/etc/init.d/ssh-server stop || true +rm -f /etc/rc.d/S*ssh-server + +# Stop and disable the firewall: +/etc/init.d/tr181-firewall stop || true +rm -f /etc/rc.d/S*tr181-firewall + +# Disable restarting failing serivces by default +/etc/init.d/amx-processmonitor stop || true + +ubus wait_for IP.Interface + +# Stop and disable the DHCP clients and servers: +ba-cli DHCPv4Client.Client.wan.Enable=0 +ba-cli DHCPv6Client.Client.wan.Enable=0 +ba-cli DHCPv4Server.Enable=0 +ba-cli DHCPv6Server.Enable=0 + +# We use WAN for the control interface. +# Add the IP address if there is none yet: +ba-cli IP.Interface.wan.IPv4Address.primary.? | grep -Eq "No data found|ERROR" && { + echo "Adding IP address $IP" + ba-cli 'IP.Interface.wan.IPv4Address.+{Alias="primary", AddressingType="Static"}' +} +# Configure it: +ba-cli 'IP.Interface.wan.IPv4Address.primary.{IPAddress="192.168.250.150", SubnetMask="255.255.255.0", AddressingType="Static", Enable=1}' +# Enable it: +ba-cli IP.Interface.wan.IPv4Enable=1 + +# Set the LAN bridge IP: +ba-cli "IP.Interface.[Name == \"br-lan\"].IPv4Address.lan.IPAddress=192.165.100.150" + +# Workaround for PPM-3339: Setting prplMesh' BackhaulWireIface doesn't always persist on first boot +/etc/init.d/prplmesh stop && sleep 1 +/etc/init.d/prplmesh start && sleep 5 + +# Set the wired backhaul interface: +if ba-cli "X_PRPLWARE-COM_Agent.Configuration.?" | grep -Eq "No data found|ERROR"; then + # Prplmesh agent is not running. Data model isn't up. + echo "Prplmesh agent is not running" +else + # Prplmesh agent is running, configure it over the bus + echo "Setting prplMesh BackhaulWireInterface over DM" + ba-cli X_PRPLWARE-COM_Agent.Configuration.BackhaulWireInterface="lan4" +fi + +# enable Wi-Fi radios +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"2.4GHz\"].", "parameters": { "Enable": "true" } }' +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"5GHz\"].", "parameters": { "Enable": "true" } }' + +ba-cli WiFi.Radio.*.RegulatoryDomain="US" + +# tshark on the sniffer can not handle RM capabilities, and thinks beacons containing them are malformed +ba-cli WiFi.AccessPoint.*.IEEE80211kEnabled=0 + +# Set multiAP profile for primary_vlan_id support +ubus-cli WiFi.AccessPoint.*.MultiAPProfile=3 + +# Enable when hostapd on this target supports it +ubus-cli "WiFi.AccessPoint.*.MBOEnable=1" + +# Restrict channel bandwidth or the certification test could miss beacons +# (see PPM-258) +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"2.4GHz\"].OperatingChannelBandwidth=20MHz" +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"5GHz\"].OperatingChannelBandwidth=20MHz" + +# Commands to start a new SSH server on the control port +# Allow all incoming connections (allows SSH/CAPI connections on WAN) +start_ssh_commands="iptables -P INPUT ACCEPT +killall -9 dropbear +dropbear -F -T 10 -p192.168.250.150:22 &" + +sleep 5 + +# Copy generated SSH host keys +cp /etc/config/ssh_server/*_key /etc/dropbear/ + +# Add command to start dropbear to rc.local to allow SSH access after reboot +bootscript="/etc/rc.local" +boot_cmd="sleep 60 && $start_ssh_commands" +if ! grep -q "$boot_cmd" "$bootscript"; then { head -n -2 "$bootscript"; echo "$boot_cmd"; tail -2 "$bootscript"; } >> btscript.tmp; mv btscript.tmp "$bootscript"; fi +set +e && eval "$start_ssh_commands" diff --git a/.gitlab/scripts/prplMesh/dut-configuration/R1/haze.sh b/.gitlab/scripts/prplMesh/dut-configuration/R1/haze.sh new file mode 100644 index 0000000000..abdd35d7b2 --- /dev/null +++ b/.gitlab/scripts/prplMesh/dut-configuration/R1/haze.sh @@ -0,0 +1,95 @@ +#!/bin/sh + +# We need to source some files which are only available on prplWrt +# devices, so prevent shellcheck from trying to read them: +# shellcheck disable=SC1091 + +set -e + +# Start with a new log file: +rm -f /var/log/messages && syslog-ng-ctl reload + +sh /etc/init.d/tr181-upnp stop || true +rm -f /etc/rc.d/S*tr181-upnp + +sh /etc/init.d/obuspa stop || true +rm -f /etc/rc.d/S*obuspa + +# Stop the default ssh server on the lan-bridge +sh /etc/init.d/ssh-server stop || true +rm -f /etc/rc.d/S*ssh-server + +# Stop and disable the firewall: +sh /etc/init.d/tr181-firewall stop || true +rm -f /etc/rc.d/S*tr181-firewall + +# Disable restarting failing serivces by default +sh /etc/init.d/amx-processmonitor stop || true + +ubus wait_for IP.Interface + +# Stop and disable the DHCP clients and servers: +ba-cli DHCPv4Client.Client.wan.Enable=0 +ba-cli DHCPv6Client.Client.wan.Enable=0 +ba-cli DHCPv4Server.Enable=0 +ba-cli DHCPv6Server.Enable=0 + +# We use WAN for the control interface. +# Add the IP address if there is none yet: +ba-cli IP.Interface.wan.IPv4Address.primary.? | grep -Eq "No data found|ERROR" && { + echo "Adding IP address $IP" + ba-cli 'IP.Interface.wan.IPv4Address.+{Alias="primary", AddressingType="Static"}' +} +# Configure it: +ba-cli 'IP.Interface.wan.IPv4Address.primary.{IPAddress="192.168.250.130", SubnetMask="255.255.255.0", AddressingType="Static", Enable=1}' +# Enable it: +ba-cli IP.Interface.wan.IPv4Enable=1 + +# Set the LAN bridge IP: +ba-cli "IP.Interface.[Name == \"br-lan\"].IPv4Address.lan.IPAddress=192.165.100.130" + +# Set the wired backhaul interface: +if ba-cli "X_PRPLWARE-COM_Agent.Configuration.?" | grep -Eq "No data found|ERROR"; then + # Prplmesh agent is not running. Data model isn't up. + echo "Prplmesh agent is not running" +else + # Prplmesh agent is running, configure it over the bus + echo "Setting prplMesh BackhaulWireInterface over DM" + ba-cli X_PRPLWARE-COM_Agent.Configuration.BackhaulWireInterface="lan3" +fi + +# all pwhm default configuration can be found in /etc/amx/wld/wld_defaults.odl.uc + +# Enable when hostapd on this target supports it +# ubus-cli "WiFi.AccessPoint.*.MBOEnable=1" + +# Make sure specific channels are configured. If channel is set to 0, +# ACS will be configured. If ACS is configured hostapd will refuse to +# switch channels when we ask it to. Channels 1 and 48 were chosen +# because they are NOT used in the WFA certification tests (this +# allows to verify that the device actually switches channel as part +# of the test). +# See also PPM-1928. +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"2.4GHz\"].", "parameters": { "Channel": "1" } }' +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"5GHz\"].", "parameters": { "Channel": "48" } }' + +# Restrict channel bandwidth or the certification test could miss beacons +# (see PPM-258) +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"2.4GHz\"].OperatingChannelBandwidth=20MHz" +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"5GHz\"].OperatingChannelBandwidth=20MHz" + + +# Commands to start a new SSH server +start_ssh_commands="killall -9 dropbear +dropbear -F -T 10 -p192.168.250.130:22 &" + +sleep 5 + +# Copy generated SSH host keys +cp /etc/config/ssh_server/*_key /etc/dropbear/ + +# Add command to start dropbear to rc.local to allow SSH access after reboot +bootscript="/etc/rc.local" +boot_cmd="sleep 60 && $start_ssh_commands" +if ! grep -q "$boot_cmd" "$bootscript"; then { head -n -2 "$bootscript"; echo "$boot_cmd"; tail -2 "$bootscript"; } >> btscript.tmp; mv btscript.tmp "$bootscript"; fi +set +e && eval "$start_ssh_commands" diff --git a/.gitlab/scripts/prplMesh/dut-configuration/R1/turris-omnia.sh b/.gitlab/scripts/prplMesh/dut-configuration/R1/turris-omnia.sh new file mode 100644 index 0000000000..a1cec98ae4 --- /dev/null +++ b/.gitlab/scripts/prplMesh/dut-configuration/R1/turris-omnia.sh @@ -0,0 +1,94 @@ +#!/bin/sh + +# We need to source some files which are only available on prplWrt +# devices, so prevent shellcheck from trying to read them: +# shellcheck disable=SC1091 + +set -e + +# Start with a new log file: +rm -f /var/log/messages && syslog-ng-ctl reload + +sh /etc/init.d/tr181-upnp stop || true +rm -f /etc/rc.d/S*tr181-upnp + +sh /etc/init.d/obuspa stop || true +rm -f /etc/rc.d/S*obuspa + +# Stop the default ssh server on the lan-bridge +sh /etc/init.d/ssh-server stop || true +rm -f /etc/rc.d/S*ssh-server + +# Stop and disable the firewall: +sh /etc/init.d/tr181-firewall stop || true +rm -f /etc/rc.d/S*tr181-firewall + +# Disable restarting failing serivces by default +sh /etc/init.d/amx-processmonitor stop || true + +ubus wait_for IP.Interface + +# Stop and disable the DHCP clients and servers: +ba-cli DHCPv4Client.Client.wan.Enable=0 +ba-cli DHCPv6Client.Client.wan.Enable=0 +ba-cli DHCPv4Server.Enable=0 +ba-cli DHCPv6Server.Enable=0 + +# We use WAN for the control interface. +# Add the IP address if there is none yet: +ba-cli IP.Interface.wan.IPv4Address.primary.? | grep -Eq "No data found|ERROR" && { + echo "Adding IP address $IP" + ba-cli 'IP.Interface.wan.IPv4Address.+{Alias="primary", AddressingType="Static"}' +} +# Configure it: +ba-cli 'IP.Interface.wan.IPv4Address.primary.{IPAddress="192.168.250.190", SubnetMask="255.255.255.0", AddressingType="Static", Enable=1}' +# Enable it: +ba-cli IP.Interface.wan.IPv4Enable=1 + +# Set the LAN bridge IP: +ba-cli "IP.Interface.[Name == \"br-lan\"].IPv4Address.lan.IPAddress=192.165.100.190" + +# Set the wired backhaul interface: +if ba-cli "X_PRPLWARE-COM_Agent.Configuration.?" | grep -Eq "No data found|ERROR"; then + # Prplmesh agent is not running. Data model isn't up. + echo "Prplmesh agent is not running" +else + # Prplmesh agent is running, configure it over the bus + echo "Setting prplMesh BackhaulWireInterface over DM" + ba-cli X_PRPLWARE-COM_Agent.Configuration.BackhaulWireInterface="lan0" +fi + +# all pwhm default configuration can be found in /etc/amx/wld/wld_defaults.odl.uc + +# Enable when hostapd on this target supports it +# ubus-cli "WiFi.AccessPoint.*.MBOEnable=1" + +# Make sure specific channels are configured. If channel is set to 0, +# ACS will be configured. If ACS is configured hostapd will refuse to +# switch channels when we ask it to. Channels 1 and 48 were chosen +# because they are NOT used in the WFA certification tests (this +# allows to verify that the device actually switches channel as part +# of the test). +# See also PPM-1928. +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"2.4GHz\"].", "parameters": { "Channel": "1" } }' +ubus call "WiFi.Radio" _set '{ "rel_path": ".[OperatingFrequencyBand == \"5GHz\"].", "parameters": { "Channel": "48" } }' + +# Restrict channel bandwidth or the certification test could miss beacons +# (see PPM-258) +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"2.4GHz\"].OperatingChannelBandwidth=20MHz" +ba-cli "WiFi.Radio.[OperatingFrequencyBand == \"5GHz\"].OperatingChannelBandwidth=20MHz" + +# Commands to start a new SSH server on the control port +start_ssh_commands="killall -9 dropbear +dropbear -F -T 10 -p192.168.250.190:22 &" + +sleep 5 + +# Copy generated SSH host keys +cp /etc/config/ssh_server/*_key /etc/dropbear/ + +# Add command to start dropbear to rc.local to allow SSH access after reboot +bootscript="/etc/rc.local" +boot_cmd="sleep 60 && $start_ssh_commands" +if ! grep -q "$boot_cmd" "$bootscript"; then { head -n -2 "$bootscript"; echo "$boot_cmd"; tail -2 "$bootscript"; } >> btscript.tmp; mv btscript.tmp "$bootscript"; fi +set +e && eval "$start_ssh_commands" diff --git a/.gitlab/scripts/prplMesh/easymesh_cert_version b/.gitlab/scripts/prplMesh/easymesh_cert_version new file mode 100644 index 0000000000..3a1a14a589 --- /dev/null +++ b/.gitlab/scripts/prplMesh/easymesh_cert_version @@ -0,0 +1 @@ +5b6b3efec429c5fa757676e148e64087fe830f2d \ No newline at end of file diff --git a/.gitlab/scripts/prplMesh/git-clean-reset.sh b/.gitlab/scripts/prplMesh/git-clean-reset.sh new file mode 100755 index 0000000000..4b92c0eb1c --- /dev/null +++ b/.gitlab/scripts/prplMesh/git-clean-reset.sh @@ -0,0 +1,14 @@ +#!/bin/sh -e + +# Reset the repository pointed by $1 to the revision given by $2 + +cd "$1" || { + echo "Failed to cd into \"$1 \". Skipping reset." + exit 1 +} + +git fetch --recurse-submodules +git clean -fd +git submodule foreach --recursive git clean -fd +git checkout --force "$2" +git submodule update --init --force diff --git a/.gitlab/scripts/prplMesh/opts.py b/.gitlab/scripts/prplMesh/opts.py new file mode 100644 index 0000000000..712ff89be4 --- /dev/null +++ b/.gitlab/scripts/prplMesh/opts.py @@ -0,0 +1,43 @@ +############################################################### +# SPDX-License-Identifier: BSD-2-Clause-Patent +# SPDX-FileCopyrightText: 2020 the prplMesh contributors (see AUTHORS.md) +# This code is subject to the terms of the BSD+Patent license. +# See LICENSE file for more details. +############################################################### + +from datetime import datetime + + +class opts: + '''Static class that encodes the global options.''' + verbose = True + tcpdump_dir = '' + stop_on_failure = False + + +def message(msg: str, color: int = 0, end='\n'): + '''Print a message, optionally in a color.''' + # Get local time and transform it to string. Remove the 3 least significant microsecods digits + # to get milliseconds value. + time_str = datetime.now().strftime("%H:%M:%S:%f")[:-3] + + if color: + print('\x1b[1;{}m{}: {}\x1b[0m'.format(color, time_str, msg), end=end) + else: + print('{}: {}'.format(time_str, msg), end=end) + + +def debug(msg: str, end='\n'): + '''Print a debug message if verbose is enabled.''' + if opts.verbose: + message(msg, end=end) + + +def status(msg: str): + '''Print a purple status message.''' + message(msg, 35) + + +def err(msg: str): + '''Print a red error message.''' + message(msg, 31)