From 7cfb0ae5fd599c5c65e27d3d95eac0ee64c34a4d Mon Sep 17 00:00:00 2001 From: Sathishkumar Selvam Date: Tue, 2 Sep 2025 13:09:09 +0000 Subject: [PATCH] profiles: prpl_core.yml: feed_prplos: Integrating Amx Process Monitor Enhancement features with prplos (FEAT-70, FEAT-77 & FEAT-88) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ci:cram Add/Update cram testcase for amx-processmonitoring Integrate below enhancements to Amx Process Monitor - Redesigned Process Monitor to allow it to respect its configuration, regardless of the configured time intervals and number of processes, allowing to execute multiple tests in parallel. - Add appropriate configuration for critical prplWare service which would required GW REBOOT to recover multiple failures. Service restart would be taken care by procd under FEAT-067. - Added capability to detect service exits using procd events - Collected and reported procd event statistics - Additionally collected and reported procd respawn configuration for configured services - Added a per-test parameter that records largest observed value of ProcessMonitor.Test.{i}.NumFailed when the “Test Failed” condition in the flow diagram of process monitor is false - Added a per-test maximum response time measured for a successful test. - Allowed to reset these values through the reset() Command. References: PCF-2011, PPM-3493, PCF-2033 Signed-off-by: Sathishkumar Selvam Closes PCF-2011, PPM-3493, PCF-2033 --- .../005-amx-respawn-reset.t | 158 +++++++++++++++ .../006-amx-monitoring-change-subject.t | 133 +++++++++++++ .../007-amx-monitoring-reset.t | 115 +++++++++++ .../008-verify-amx-monitoring-enabled.t | 43 +++++ .../009-amx-process-fail.t | 182 ++++++++++++++++++ .../025-process-monitor.t | 41 ++-- .../script_functions_amx.sh | 109 +++++++++++ profiles/prpl_core.yml | 2 +- 8 files changed, 765 insertions(+), 18 deletions(-) create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/005-amx-respawn-reset.t create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/006-amx-monitoring-change-subject.t create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/007-amx-monitoring-reset.t create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/008-verify-amx-monitoring-enabled.t create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/009-amx-process-fail.t mode change 100644 => 100755 .gitlab/tests/cram/generic/acceleration-plan-components/025-process-monitor.t create mode 100644 .gitlab/tests/cram/generic/acceleration-plan-components/script_functions_amx.sh diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/005-amx-respawn-reset.t b/.gitlab/tests/cram/generic/acceleration-plan-components/005-amx-respawn-reset.t new file mode 100644 index 0000000000..3b413b76ab --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/005-amx-respawn-reset.t @@ -0,0 +1,158 @@ +Create R alias: + + $ alias R="${CRAM_REMOTE_COMMAND:-}" + + $ alias C="${CRAM_REMOTE_COPY:-}" + +Set script parameters and copy the script to device: + + $ S=". /tmp/script_functions_amx.sh" + + $ C ${TESTDIR}/script_functions_amx.sh root@${TARGET_LAN_IP}:/tmp/script_functions_amx.sh + Warning: Permanently added '*' (*) to the list of known hosts* (glob) + +Pre-test actions, Restart process service to clear the respawns and other failures before starting with tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service tr181-qos restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + +Wait 5 seconds for the process to turn functional: + + $ sleep 5 + +Initialize the ProcessMonitor.Test.i Id for required processes: + + $ Tr181McastdId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-mcastd | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Tr181PcpId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-pcp | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ WanManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-qos | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Dhcpv4ManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep dhcpv4-manager | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + +Get the initial NumProcessRespawn for all the process: + + $ Tr181McastdMaxRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastdId.NumProcessRespawn? | sed '/^$/d'") + $ Tr181PcpRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.NumProcessRespawn? | sed '/^$/d'") + $ WanManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.NumProcessRespawn? | sed '/^$/d'") + $ Dhcpv4ManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.NumProcessRespawn? | sed '/^$/d'") + +Get the initial MaxFailNum for all the process: + + $ Tr181McastdMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastdId.MaxFailNum? | sed '/^$/d'") + $ Tr181PcpMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum? | sed '/^$/d'") + $ WanManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum? | sed '/^$/d'") + $ Dhcpv4ManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum? | sed '/^$/d'") + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" \ + > "dhcpv4-manager"; do R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Change MaxFail parameter for the processes to higher value: + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181McastdId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum=30 | sed '/^$/d'" + 30 + +Kill the processes and wait - Frist kill attempt: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" \ + > "dhcpv4-manager"; do R "${S} && kill_process \"$process_name\""; done + + $ sleep 14 + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" \ + > "dhcpv4-manager"; do R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Verify amx-processmonitoring has updated the NumProcessRespawn after process respawn: + + $ R "${S} && verify_respawn_value $Tr181McastdId $((Tr181McastdMaxRespawn+1))" + tr181-mcastd NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Tr181PcpId $((Tr181PcpRespawn+1))" + tr181-pcp NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $WanManagerId $((WanManagerRespawn+1))" + tr181-qos NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Dhcpv4ManagerId $((Dhcpv4ManagerRespawn+1))" + dhcpv4-manager NumProcessRespawn PASS + +Stop start all the process: + + $ R "service tr181-mcastd stop > /dev/null 2>&1" + + $ R "service tr181-mcastd start > /dev/null 2>&1" + + $ R "service tr181-pcp stop > /dev/null 2>&1" + + $ R "service tr181-pcp start > /dev/null 2>&1" + + $ R "service tr181-qos stop > /dev/null 2>&1" + + $ R "service tr181-qos start > /dev/null 2>&1" + + $ R "service dhcpv4-manager stop > /dev/null 2>&1" + + $ R "service dhcpv4-manager start > /dev/null 2>&1" + +Wait for the processes to come up: + + $ sleep 5 + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" \ + > "dhcpv4-manager"; do R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Verify amx-process monitor has reset NumProcessRespawn to 0: + + $ R "${S} && verify_respawn_value $Tr181McastdId 0" + tr181-mcastd NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Tr181PcpId 0" + tr181-pcp NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $WanManagerId 0" + tr181-qos NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Dhcpv4ManagerId 0" + dhcpv4-manager NumProcessRespawn PASS + +Clean-up, Revert MaxFail parameter for the process to initial value: + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181McastdId.MaxFailNum=$Tr181McastdMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum=$Tr181PcpMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum=$WanManagerMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum=$Dhcpv4ManagerMaxFail | sed '/^$/d'" + \d+ (re) + + $ R logger -t cram "Amx-processmonitoring respawn attribute reset test finished" \ No newline at end of file diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/006-amx-monitoring-change-subject.t b/.gitlab/tests/cram/generic/acceleration-plan-components/006-amx-monitoring-change-subject.t new file mode 100644 index 0000000000..ee3c92b189 --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/006-amx-monitoring-change-subject.t @@ -0,0 +1,133 @@ +Create aliases: + + $ alias R="${CRAM_REMOTE_COMMAND:-}" + + $ alias C="${CRAM_REMOTE_COPY:-}" + +Set script parameters and copy the script to device: + + $ S=". /tmp/script_functions_amx.sh" + + $ C ${TESTDIR}/script_functions_amx.sh root@${TARGET_LAN_IP}:/tmp/script_functions_amx.sh + Warning: Permanently added '*' (*) to the list of known hosts* (glob) + + $ R logger -t cram "Starting with amx-processmonitoring reset cram test" + +Pre-test actions, Restart the process service to clear the respawns and other failures before starting with tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service tr181-qos restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + +Wait 5 seconds for the process to turn functional: + + $ sleep 5 + +Initialize the ProcessMonitor.Test.i Id for required processes: + + $ Tr181McastId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-mcastd | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Tr181PcpId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-pcp | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ WanManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-qos | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Dhcpv4ManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep dhcpv4-manager | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + +Get the initial NumProcessRespawn for all the process: + + $ Tr181McastRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.NumProcessRespawn? | sed '/^$/d'") + $ Tr181PcpRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.NumProcessRespawn? | sed '/^$/d'") + $ WanManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.NumProcessRespawn? | sed '/^$/d'") + $ Dhcpv4ManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.NumProcessRespawn? | sed '/^$/d'") + +Get the initial MaxFailNum for all the process: + + $ Tr181McastMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.MaxFailNum? | sed '/^$/d'") + $ Tr181PcpMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum? | sed '/^$/d'") + $ WanManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum? | sed '/^$/d'") + $ Dhcpv4ManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum? | sed '/^$/d'") + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Kill the processes - Frist kill attempt: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 15 + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Verify amx-process monitor has updated the NumProcessRespawn after process respawn: + + $ R "${S} && verify_process_fail_update $Tr181McastId $((Tr181McastRespawn+1))" + tr181-mcastd NumProcessRespawn PASS + tr181-mcastd MaxNumFailed PASS + + $ R "${S} && verify_process_fail_update $Tr181PcpId $((Tr181PcpRespawn+1))" + tr181-pcp NumProcessRespawn PASS + tr181-pcp MaxNumFailed PASS + + $ R "${S} && verify_process_fail_update $WanManagerId $((WanManagerRespawn+1))" + tr181-qos NumProcessRespawn PASS + tr181-qos MaxNumFailed PASS + + $ R "${S} && verify_process_fail_update $Dhcpv4ManagerId $((Dhcpv4ManagerRespawn+1))" + dhcpv4-manager NumProcessRespawn PASS + dhcpv4-manager MaxNumFailed PASS + +Change test subject of amx-processmonitor and verify process monitoring parameters are reset: + + $ for process_id in $Tr181McastId $Tr181PcpId $WanManagerId $Dhcpv4ManagerId; do + > R "${S} && change_process_subject" $process_id; done + tr181-mcastd subject change OK + tr181-pcp subject change OK + tr181-qos subject change OK + dhcpv4-manager subject change OK + +Verify ProcessMonitoring parameter reset after calling reset method: + + $ R "${S} && verify_value_reset $Tr181McastId" + tr181-mcastd ProcessMonitoringEnabled PASS + tr181-mcastd MaxNumFailed PASS + tr181-mcastd NumProcessFail PASS + tr181-mcastd NumProcessRespawn PASS + + $ R "${S} && verify_value_reset $Tr181PcpId" + tr181-pcp ProcessMonitoringEnabled PASS + tr181-pcp MaxNumFailed PASS + tr181-pcp NumProcessFail PASS + tr181-pcp NumProcessRespawn PASS + + $ R "${S} && verify_value_reset $WanManagerId" + tr181-qos ProcessMonitoringEnabled PASS + tr181-qos MaxNumFailed PASS + tr181-qos NumProcessFail PASS + tr181-qos NumProcessRespawn PASS + + $ R "${S} && verify_value_reset $Dhcpv4ManagerId" + dhcpv4-manager ProcessMonitoringEnabled PASS + dhcpv4-manager MaxNumFailed PASS + dhcpv4-manager NumProcessFail PASS + dhcpv4-manager NumProcessRespawn PASS + +Restart the process service to clear the respawns from above tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service tr181-qos restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + + $ R logger -t cram "Amx-processmonitoring reset cram test finished" diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/007-amx-monitoring-reset.t b/.gitlab/tests/cram/generic/acceleration-plan-components/007-amx-monitoring-reset.t new file mode 100644 index 0000000000..8adf215d6e --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/007-amx-monitoring-reset.t @@ -0,0 +1,115 @@ +Create aliases: + + $ alias R="${CRAM_REMOTE_COMMAND:-}" + + $ alias C="${CRAM_REMOTE_COPY:-}" + +Set script parameters and copy the script to device: + + $ S=". /tmp/script_functions_amx.sh" + + $ C ${TESTDIR}/script_functions_amx.sh root@${TARGET_LAN_IP}:/tmp/amx_script_functions.sh + Warning: Permanently added '*' (*) to the list of known hosts* (glob) + + $ R logger -t cram "Starting with amx-processmonitoring reset cram test" + +Pre-test actions, Restart the process service to clear the respawns and other failures before starting with tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + +Wait 5 seconds for the process to turn functional: + + $ sleep 5 + +Initialize the ProcessMonitor.Test.i Id for required processes: + + $ Tr181McastId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-mcastd | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Tr181PcpId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-pcp | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Dhcpv4ManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep dhcpv4-manager | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + +Get the initial NumProcessRespawn for all the process: + + $ Tr181McastRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.NumProcessRespawn? | sed '/^$/d'") + $ Tr181PcpRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.NumProcessRespawn? | sed '/^$/d'") + $ Dhcpv4ManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.NumProcessRespawn? | sed '/^$/d'") + +Get the initial MaxFailNum for all the process: + + $ Tr181McastMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.MaxFailNum? | sed '/^$/d'") + $ Tr181PcpMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum? | sed '/^$/d'") + $ Dhcpv4ManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum? | sed '/^$/d'") + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Kill the processes - Frist kill attempt: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "dhcpv4-manager"; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 15 + +Get the Process ID and verify all expected process are running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Verify amx-process monitor has updated the NumProcessRespawn after process respawn: + + $ R "${S} && verify_process_fail_update $Tr181McastId $((Tr181McastRespawn+1))" + tr181-mcastd NumProcessRespawn PASS + tr181-mcastd MaxNumFailed PASS + + $ R "${S} && verify_process_fail_update $Tr181PcpId $((Tr181PcpRespawn+1))" + tr181-pcp NumProcessRespawn PASS + tr181-pcp MaxNumFailed PASS + + $ R "${S} && verify_process_fail_update $Dhcpv4ManagerId $((Dhcpv4ManagerRespawn+1))" + dhcpv4-manager NumProcessRespawn PASS + dhcpv4-manager MaxNumFailed PASS + +Call reset method of amx-processmonitor and verify process monitoring parameters are reset: + + $ for process_id in $Tr181McastId $Tr181PcpId $Dhcpv4ManagerId; do + > R "${S} && reset_amx_process_monitoring $process_id"; done + tr181-mcastd reset OK + tr181-pcp reset OK + dhcpv4-manager reset OK + +Verify ProcessMonitoring parameter reset after calling reset method: + + $ R "${S} && verify_value_reset $Tr181McastId" + tr181-mcastd ProcessMonitoringEnabled PASS + tr181-mcastd MaxNumFailed PASS + tr181-mcastd NumProcessFail PASS + tr181-mcastd NumProcessRespawn PASS + + $ R "${S} && verify_value_reset $Tr181PcpId" + tr181-pcp ProcessMonitoringEnabled PASS + tr181-pcp MaxNumFailed PASS + tr181-pcp NumProcessFail PASS + tr181-pcp NumProcessRespawn PASS + + $ R "${S} && verify_value_reset $Dhcpv4ManagerId" + dhcpv4-manager ProcessMonitoringEnabled PASS + dhcpv4-manager MaxNumFailed PASS + dhcpv4-manager NumProcessFail PASS + dhcpv4-manager NumProcessRespawn PASS + +Restart the process service to clear the respawns from above tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + + $ R logger -t cram "Amx-processmonitoring reset cram test finished" diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/008-verify-amx-monitoring-enabled.t b/.gitlab/tests/cram/generic/acceleration-plan-components/008-verify-amx-monitoring-enabled.t new file mode 100644 index 0000000000..37cd6f4528 --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/008-verify-amx-monitoring-enabled.t @@ -0,0 +1,43 @@ +Create R alias: + + $ alias R="${CRAM_REMOTE_COMMAND:-}" + + $ R logger -t "Starting with amx-processmonitoring enable test" + +This Helper method finds the Instance Id and return the ProcessMonitoringEnabled parameter, #Param1 - name of the process: + + $ get_amx_process_monitoring() { InstanceId=$(R "ba-cli " \ + > "ProcessMonitor.Test.*.Name? | grep $1 | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'"); \ + > amx_enable=$(R "ba-cli -l ProcessMonitor.Test.$InstanceId.ProcessMonitoringEnabled? | sed '/^$/d'"); \ + > echo "$1=$amx_enable";} + +Verify all expected processes are enabled for monitoring by amx-processmonitor, value of 1 specifies enabled for monitoring: + + $ for process in tr181-device tr181-pcp tr181-firewall tr181-qos tr181-mcastd; do get_amx_process_monitoring $process; done + tr181-device=1 + tr181-pcp=1 + tr181-firewall=1 + tr181-qos=1 + tr181-mcastd=1 + + $ for process in tr181-mqttbroker tr181-bridging tr181-ppp tr181-dns tr181-xpon; do get_amx_process_monitoring $process; done + tr181-mqttbroker=1 + tr181-bridging=1 + tr181-ppp=1 + tr181-dns=1 + tr181-xpon=1 + + $ for process in deviceinfo-manager tr181-dns gmap-server cwmp_plugin reboot-service; do get_amx_process_monitoring $process; done + deviceinfo-manager=1 + tr181-dns=1 + gmap-server=1 + cwmp_plugin=1 + reboot-service=1 + + $ for process in odhcpd hosts-manager cellular-manager dhcpv4-manager; do get_amx_process_monitoring $process; done + odhcpd=1 + hosts-manager=1 + cellular-manager=1 + dhcpv4-manager=1 + + $ R logger -t "Amx-processmonitoring enable test finished" diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/009-amx-process-fail.t b/.gitlab/tests/cram/generic/acceleration-plan-components/009-amx-process-fail.t new file mode 100644 index 0000000000..09d6be16f8 --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/009-amx-process-fail.t @@ -0,0 +1,182 @@ +Create R alias: + + $ alias R="${CRAM_REMOTE_COMMAND:-}" + + $ alias C="${CRAM_REMOTE_COPY:-}" + +Set script parameters and copy the script to device: + + $ S=". /tmp/script_functions_amx.sh" + + $ C ${TESTDIR}/script_functions_amx.sh root@${TARGET_LAN_IP}:/tmp/amx_script_functions.sh + Warning: Permanently added '*' (*) to the list of known hosts* (glob) + + $ R logger -t cram "Starting amx-processmonitoring process fail test" + +Pre-test actions, Restart the process service to clear the respawns and other failures before starting with tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service tr181-qos restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + +Wait 5 seconds for the process to become functional: + + $ sleep 5 + +Initialize the ProcessMonitor.Test.i Id for required processes: + + $ Tr181McastId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-mcastd | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Tr181PcpId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-pcp | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ WanManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep tr181-qos | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + $ Dhcpv4ManagerId=$(R "ba-cli ProcessMonitor.Test.*.Name? | grep dhcpv4-manager | sed -n 's/.*Test\.\([0-9]\+\)\..*/\1/p'") + +Get the initial NumProcessRespawn for all the process: + + $ Tr181McastRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.NumProcessRespawn? | sed '/^$/d'") + $ Tr181PcpRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.NumProcessRespawn? | sed '/^$/d'") + $ WanManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.NumProcessRespawn? | sed '/^$/d'") + $ Dhcpv4ManagerRespawn=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.NumProcessRespawn? | sed '/^$/d'") + +Get the initial NumProcessFail for all the process: + + $ Tr181McastFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.NumProcessFail? | sed '/^$/d'") + $ Tr181PcpFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.NumProcessFail? | sed '/^$/d'") + $ WanManagerFail=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.NumProcessFail? | sed '/^$/d'") + $ Dhcpv4ManagerFail=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.NumProcessFail? | sed '/^$/d'") + +Get the initial MaxFailNum for all the process: + + $ Tr181McastMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.MaxFailNum? | sed '/^$/d'") + $ Tr181PcpMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum? | sed '/^$/d'") + $ WanManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum? | sed '/^$/d'") + $ Dhcpv4ManagerMaxFail=$(R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum? | sed '/^$/d'") + +Verify process are up and running: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Change MaxFail parameter for the processes to higher value: + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum=30 | sed '/^$/d'" + 30 + + $ R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum=30 | sed '/^$/d'" + 30 + +Kill the processes - first kill attempt: + + $ for process_name in tr181-mcastd tr181-pcp tr181-qos dhcpv4-manager; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 5 + +Verify process respawn: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Kill the processes - Second kill attempt: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 5 + +Verify process respawn: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Kill the processes - third kill attempt: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 5 + +Verify process respawn: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && get_pid \"$process_name\""; done + tr181-mcastd.* \d+ (re) + tr181-pcp.* \d+ (re) + tr181-qos.* \d+ (re) + dhcpv4-manager.* \d+ (re) + +Kill the processes - Fourth kill attempt, no more respawns of failed process by procd and NumProcessFail will increment: + + $ for process_name in "tr181-mcastd" "tr181-pcp" "tr181-qos" "dhcpv4-manager"; do + > R "${S} && kill_process \"$process_name\""; done + + $ sleep 5 + +Verify NumProcessFail is incremented for process fail: + + $ R "${S} && verify_num_Process_fail $Tr181McastId $((Tr181McastFail+1))" + tr181-mcastd NumProcessFail PASS + + $ R "${S} && verify_num_Process_fail $Tr181PcpId $((Tr181PcpFail+1))" + tr181-pcp NumProcessFail PASS + + $ R "${S} && verify_num_Process_fail $WanManagerId $((WanManagerFail+1))" + tr181-qos NumProcessFail PASS + + $ R "${S} && verify_num_Process_fail $Dhcpv4ManagerId $((Dhcpv4ManagerFail+1))" + dhcpv4-manager NumProcessFail PASS + +Verify amx-process monitor has updated the NumProcessRespawn after process respawn for all kill attempts above: + + $ R "${S} && verify_respawn_value $Tr181McastId $((Tr181McastRespawn+3))" + tr181-mcastd NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Tr181PcpId $((Tr181PcpRespawn+3))" + tr181-pcp NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $WanManagerId $((WanManagerRespawn+3))" + tr181-qos NumProcessRespawn PASS + + $ R "${S} && verify_respawn_value $Dhcpv4ManagerId $((Dhcpv4ManagerRespawn+3))" + dhcpv4-manager NumProcessRespawn PASS + +Clean-up Revert MaxFail parameter for the process to initial value: + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181McastId.MaxFailNum=$Tr181McastMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$Tr181PcpId.MaxFailNum=$Tr181PcpMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$WanManagerId.MaxFailNum=$WanManagerMaxFail | sed '/^$/d'" + \d+ (re) + + $ R "ba-cli -l ProcessMonitor.Test.$Dhcpv4ManagerId.MaxFailNum=$Dhcpv4ManagerMaxFail | sed '/^$/d'" + \d+ (re) + +Restart process service to clear the respawns from above tests: + + $ R "service tr181-mcastd restart > /dev/null 2>&1" + $ R "service tr181-pcp restart > /dev/null 2>&1" + $ R "service tr181-qos restart > /dev/null 2>&1" + $ R "service dhcpv4-manager restart > /dev/null 2>&1" + + $ R logger -t cram "Amx-processmonitoring process fail test finished" diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/025-process-monitor.t b/.gitlab/tests/cram/generic/acceleration-plan-components/025-process-monitor.t old mode 100644 new mode 100755 index 26a4e425f2..b372391c8e --- a/.gitlab/tests/cram/generic/acceleration-plan-components/025-process-monitor.t +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/025-process-monitor.t @@ -18,6 +18,11 @@ Check the root datamodel settings: } } +Change Respawn timeout on tr181-led to allow testing with Amx ProcessMonitor: + + $ R "export respawn_timeout=60" + $ R "/etc/init.d/tr181-led restart" + Check that only one instance of tr181-led manager is running: $ R "pgrep -cf 'tr181-led'" @@ -29,9 +34,10 @@ Get current tr181-led manager PID: Add test for checking tr181-leds manager using PID: - $ R "ba-cli 'ProcessMonitor.Test+{Type=Process,Name=tr181-led,Subject=/var/run/tr181-led.pid,FailAction=RESTART,TestInterval=2,MaxFailNum=1}' | grep -v '^>'" + $ R "ba-cli 'ProcessMonitor.Test+{Type=Process,Name=tr181-led,Subject=/var/run/tr181-led.pid,FailAction=RESTART,TestInterval=30,MaxFailNum=1,ProcessMonitoringEnabled=0}' | grep -v '^>'" ProcessMonitor.Test.\d+. (re) + Check the LED manager check datamodel settings: $ R "ba-cli --json ProcessMonitor.Test.[Name==\\\"tr181-led\\\"].? | sed -n '2p'" | jq --sort-keys '.[0]' @@ -45,28 +51,32 @@ Check the LED manager check datamodel settings: "LastFailAction": "0001-01-01T00:00:00Z", "LastFailReason": "Error_None", "LastSuccess": "0001-01-01T00:00:00Z", + "LogEntryEnabled": 1, "MaxFailDuration": -1, "MaxFailNum": 1, + "MaxFailedDuration": 0, + "MaxNumFailed": 0, "Name": "tr181-led", "NumFailActions": 0, "NumFailed": 0, + "NumProcessFail": -1, + "NumProcessRespawn": -1, + "ProcessMonitoringEnabled": 0, "RebootAfterRestartThreshold": 0, "Subject": "/var/run/tr181-led.pid", "SuccessfulSince": "0001-01-01T00:00:00Z", - "TestInterval": 2, + "TestInterval": 30, "TestIntervalMultiplier": 1, "TestResetInterval": 3600, "Type": "Process" + }, + "ProcessMonitor.Test.\d+.ProcessRespawnParams.": { (re) + "RetryAttempts": -*\d+, (re) + "Threshold": -*\d+, (re) + "Timeout": -*\d+ (re) } } -Shorthen the test cycle duration to 1 second: - - $ current_cycle_duration="$(R 'ba-cli -lj "ProcessMonitor.CycleDuration?"' | jq -e '.[] | .[] | .CycleDuration')" - $ R "ba-cli -l -j 'ProcessMonitor.CycleDuration=1'" - - [{"ProcessMonitor.":{"CycleDuration":1}}] - Kill the LED manager service: $ R "kill \$(cat /var/run/tr181-led.pid)" @@ -78,10 +88,9 @@ Check that LED manager is not running: 0 [1] -Calculate a timeout with (number_of_tests * test_interval * 2s): +Calculate a timeout with tr181-led current test interval: - $ number_of_tests="$(R 'ba-cli -lj "ProcessMonitor.NumberOfTest?"' | jq -e '.[] | .[] | .NumberOfTest')" - $ reactivation_timeout=$((number_of_tests*2*10)) + $ reactivation_timeout="$(R 'ba-cli -lj "ProcessMonitor.Test.[Name==\"tr181-led\"].CurrentTestInterval?"' | jq -e '.[] | .[] | .CurrentTestInterval')" Check that ProcessMonitor have restarted the LED manager properly: @@ -101,9 +110,7 @@ Cleanup: $ R "ba-cli --json ProcessMonitor.Test.[Name==\\\"tr181-led\\\"].-" >/dev/null -Revert back the test cycle duration: +Revert back the respawn timeout on tr181-led: - $ R "ba-cli -l 'ProcessMonitor.CycleDuration=$current_cycle_duration'" - - \d+ (re) - + $ R "export respawn_timeout=5" + $ R "/etc/init.d/tr181-led restart" diff --git a/.gitlab/tests/cram/generic/acceleration-plan-components/script_functions_amx.sh b/.gitlab/tests/cram/generic/acceleration-plan-components/script_functions_amx.sh new file mode 100644 index 0000000000..6bc8dacf27 --- /dev/null +++ b/.gitlab/tests/cram/generic/acceleration-plan-components/script_functions_amx.sh @@ -0,0 +1,109 @@ +#!/bin/ash +CLI="ba-cli -l " +CLI_JSON="ba-cli -j -l" + +# Performs pkill for the given process name +kill_process() { + pkill -P 1 -f "$1" +} + +# Retrieves the process ID for given process name +get_pid() { + echo "$1 "$(pgrep -P 1 -f "$1") +} + +# Verifies NumProcessRespawn against provided value +verify_process_fail_update() { + respawn=$(${CLI} "ProcessMonitor.Test.$1.NumProcessRespawn?" | sed '/^$/d') + process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ $respawn -eq $2 ]; then + echo "$process_name NumProcessRespawn PASS" + else + echo "FAIL $process_name, expected ProcessMonitor.Test.$1.NumProcessRespawn value: $2 found: $respawn" + fi + MaxNumFailed=$(${CLI} "ProcessMonitor.Test.$1.MaxNumFailed?" | sed '/^$/d') + if [ $MaxNumFailed -eq 0 ]; then + echo "FAIL $process_name, Expected ProcessMonitor.Test.$1.MaxNumFailed to be updated but found $MaxNumFailed" + else + echo "$process_name MaxNumFailed PASS" + fi +} + +# Verifies ProcessMonitoringEnabled is enabled, LastFailReason cleared +# MaxNumFailed, NumProcessFail and NumProcessRespawn are cleared +verify_value_reset() { + ProcessMonitoringEnabled=$(${CLI} "ProcessMonitor.Test.$1.ProcessMonitoringEnabled?" | sed '/^$/d') + process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ $ProcessMonitoringEnabled -eq 1 ]; then + echo "$process_name ProcessMonitoringEnabled PASS" + else + echo "FAIL $process_name, ProcessMonitor.Test.$1.ProcessMonitoringEnabled - $ProcessMonitoringEnabled " \ + "not enabled after reset" + fi + + MaxNumFailed=$(${CLI} "ProcessMonitor.Test.$1.MaxNumFailed?" | sed '/^$/d') + if [ $MaxNumFailed -eq 0 ]; then + echo "$process_name MaxNumFailed PASS" + else + echo "FAIL $process_name, ProcessMonitor.Test.$1.MaxNumFailed - $MaxNumFailed not reset" + fi + + NumProcessFail=$(${CLI} "ProcessMonitor.Test.$1.NumProcessFail?" | sed '/^$/d') + if [ $NumProcessFail -eq 0 ]; then + echo "$process_name NumProcessFail PASS" + else + echo "FAIL $process_name, ProcessMonitor.Test.$1.NumProcessFail - $NumProcessFail not reset" + fi + + NumProcessRespawn=$(${CLI} "ProcessMonitor.Test.$1.NumProcessRespawn?" | sed '/^$/d') + if [ $NumProcessRespawn -eq 0 ]; then + echo "$process_name NumProcessRespawn PASS" + else + echo "FAIL $process_name, ProcessMonitor.Test.$1.NumProcessRespawn - $NumProcessRespawn not reset" + fi +} + +# Calls reset method for given ProcessMonitor.Test.Id +reset_amx_process_monitoring() { + reset_output=$(${CLI_JSON} "ProcessMonitor.Test.$1.reset()" | sed '/^$/d' | tail -n 1) + process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ "[\"\"]" == $reset_output ]; then + echo "$process_name reset OK" + else + echo "$process_name reset Failed" + fi +} + +# Changes subject of the given ProcessMonitor.Test.Id +change_process_subject() { + current_time_stamp=$(date +"%Y%m%d_%H%M%S") + change_process_output=$(${CLI} "ProcessMonitor.Test.$1.Subject=\"test_subject_$1_$current_time_stamp\"" | sed '/^$/d') + change_process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ test_subject_$1_$current_time_stamp == $change_process_output ]; then + echo "$change_process_name subject change OK" + else + echo "$change_process_name subject change Failed" + fi +} + +# Verify NumProcessRespawn against given value +verify_respawn_value() { + respawn=$(${CLI} "ProcessMonitor.Test.$1.NumProcessRespawn?" | sed '/^$/d') + process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ $respawn -eq $2 ]; then + echo "$process_name NumProcessRespawn PASS" + else + echo "FAIL $process_name, expected ProcessMonitor.Test.$1.NumProcessRespawn value: $2 found: $respawn" + fi +} + +# Verifies NumProcessFail against expected value +verify_num_Process_fail() { + num_process_fail=$(${CLI} "ProcessMonitor.Test.$1.NumProcessFail?" | sed '/^$/d') + process_name=$(${CLI} "ProcessMonitor.Test.$1.Name?" | sed '/^$/d') + if [ $num_process_fail -eq $2 ]; then + echo "$process_name NumProcessFail PASS" + else + echo "Fail $process_name, Expected value for ProcessMonitor.Test.$1.NumProcessFail: $2, found: $num_process_fail" + fi +} diff --git a/profiles/prpl_core.yml b/profiles/prpl_core.yml index ae7278a17e..c2190427a6 100644 --- a/profiles/prpl_core.yml +++ b/profiles/prpl_core.yml @@ -11,7 +11,7 @@ feeds: - name: feed_prplos uri: https://gitlab.com/prpl-foundation/prplos/feeds/feed-prplos.git - revision: main@c8aa66f717226272627c6ec08f4758245b41bbc7 + revision: main@66da6abf41800504cc9a0c0bbe57628f914bec08 additional_packages: - feed: packages