From ac392556729cc020fff511facf705b8f87609c31 Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Wed, 24 Dec 2025 12:01:16 +0000 Subject: [PATCH] test(android-test-app): unify presentation framework with evidence collection Implement P0-P5 directives for operator clarity, consistent outcomes, and easy evidence capture across all test phases. Changes: - alarm-test-lib.sh: Add evidence collection (capture_alarms, capture_logcat, capture_screenshot), verdict functions (verdict_pass/warn/fail), run directory management, and release gating support (RELEASE_GATE_PHASE3) - test-phase1.sh: Refactor to unified framework with CLI modes (--setup, --run, --smoke, --all, --ci), micro-prompts, evidence capture, and verdict blocks for all 5 tests - test-phase2.sh: Add evidence capture, verdict blocks, and STRICTNESS policy (soft/hard) for warn vs fail behavior - test-phase3.sh: Add evidence capture, verdict blocks, release gating (--gate-phase3), and fatigue reduction (time estimates, automation hints) - RUNBOOK-TESTING.md: New comprehensive operator guide (669 lines) covering prerequisites, all phases, evidence locations, verdict interpretation, common failures, and troubleshooting All test scripts now use consistent UI helpers (section, substep, info, ok, warn, error), standardized evidence collection, and clear verdict reporting. Evidence is saved to timestamped run directories (runs//) with alarms, logs, and screenshots organized by test phase and scenario. Tests pass with consistent presentation and reproducible evidence collection. --- test-apps/android-test-app/alarm-test-lib.sh | 329 +++- .../android-test-app/docs/RUNBOOK-TESTING.md | 669 +++++++ test-apps/android-test-app/test-phase1.sh | 1619 +++++++++-------- test-apps/android-test-app/test-phase2.sh | 275 ++- test-apps/android-test-app/test-phase3.sh | 351 +++- 5 files changed, 2373 insertions(+), 870 deletions(-) create mode 100644 test-apps/android-test-app/docs/RUNBOOK-TESTING.md diff --git a/test-apps/android-test-app/alarm-test-lib.sh b/test-apps/android-test-app/alarm-test-lib.sh index ae590c3..7f59ccf 100644 --- a/test-apps/android-test-app/alarm-test-lib.sh +++ b/test-apps/android-test-app/alarm-test-lib.sh @@ -12,6 +12,12 @@ # # Configuration can be overridden before sourcing: # APP_ID="custom.package" source "${SCRIPT_DIR}/alarm-test-lib.sh" +# +# STRICT MODE NOTE: +# This library does NOT set strict mode itself (set -euo pipefail) because +# it's a library file. Scripts that source this library SHOULD set strict mode: +# set -euo pipefail +# IFS=$'\n\t' # --- Config Defaults (can be overridden before sourcing) --- @@ -27,6 +33,13 @@ : "${SCREENSHOT_ROOT:=screenshots}" : "${ENABLE_SCREENSHOTS:=1}" +# Run folder configuration (P1) +: "${RUN_ID:=$(date '+%Y%m%d_%H%M%S' 2>/dev/null || echo 'unknown')}" +: "${RUN_DIR:=runs/${RUN_ID}}" + +# Release gating configuration (P4) +: "${RELEASE_GATE_PHASE3:=0}" + # Derived config (for backward compatibility with Phase 1) PACKAGE="${APP_ID}" ACTIVITY="${APP_ID}/.MainActivity" @@ -38,7 +51,11 @@ YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color -# --- UI/Log Helpers --- +# ======================================== +# PUBLIC API - UI/Log Helpers +# ======================================== +# These are the primary functions that all scripts should use. +# Deprecated functions (print_*, wait_for_*) are kept for backward compatibility. section() { echo @@ -84,8 +101,54 @@ ui_prompt() { echo } -# Phase 1 compatibility aliases (print_* functions) +# ======================================== +# PUBLIC API - Command Execution Helpers +# ======================================== + +run_cmd() { + # Execute a command and capture output + # Usage: run_cmd "description" command [args...] + # Returns: exit code of command + local desc="$1" + shift + local cmd=("$@") + + info "Running: $desc" + if "${cmd[@]}"; then + ok "$desc completed" + return 0 + else + local exit_code=$? + error "$desc failed (exit code: $exit_code)" + return $exit_code + fi +} + +require_cmd() { + # Execute a command and exit on failure + # Usage: require_cmd "description" command [args...] + # Exits script if command fails + local desc="$1" + shift + local cmd=("$@") + + info "Required: $desc" + if ! "${cmd[@]}"; then + local exit_code=$? + error "$desc failed (exit code: $exit_code)" + exit $exit_code + fi + ok "$desc completed" +} + +# ======================================== +# DEPRECATED - Phase 1 Compatibility Aliases +# ======================================== +# These functions are kept for backward compatibility but should not be used +# in new code. Use the public API functions above instead. + print_header() { + # DEPRECATED: Use section() instead echo "" echo -e "${BLUE}========================================${NC}" echo -e "${BLUE}$1${NC}" @@ -94,36 +157,44 @@ print_header() { } print_step() { + # DEPRECATED: Use substep() instead echo -e "${GREEN}→ Step $1:${NC} $2" } print_wait() { + # DEPRECATED: Use info() or warn() instead echo -e "${YELLOW}⏳ $1${NC}" } print_success() { + # DEPRECATED: Use ok() instead echo -e "${GREEN}✅ $1${NC}" } print_error() { + # DEPRECATED: Use error() instead echo -e "${RED}❌ $1${NC}" } print_info() { + # DEPRECATED: Use info() instead echo -e "${BLUE}ℹ️ $1${NC}" } print_warn() { + # DEPRECATED: Use warn() instead echo -e "${YELLOW}⚠️ $1${NC}" } wait_for_user() { + # DEPRECATED: Use pause() instead echo "" read -p "Press Enter when ready to continue..." echo "" } wait_for_ui_action() { + # DEPRECATED: Use ui_prompt() instead ui_prompt "$1" } @@ -611,3 +682,257 @@ should_run_test() { return 1 } +# ======================================== +# PUBLIC API - Run Folder & Evidence Helpers (P1) +# ======================================== + +ensure_run_dir() { + # Create run directory structure if it doesn't exist + # Creates: RUN_DIR/logs, RUN_DIR/alarms, RUN_DIR/screens, RUN_DIR/notes + # Returns: 0 on success, 1 on failure + local base_dir + if [ -n "$SCRIPT_DIR" ] && [ -d "$SCRIPT_DIR" ]; then + base_dir="${SCRIPT_DIR}/${RUN_DIR}" + elif [ -n "${BASH_SOURCE[0]}" ]; then + local lib_dir + lib_dir="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd 2>/dev/null)" || lib_dir="" + if [ -n "$lib_dir" ]; then + base_dir="${lib_dir}/${RUN_DIR}" + else + base_dir="${RUN_DIR}" + fi + else + base_dir="${RUN_DIR}" + fi + + mkdir -p "${base_dir}/logs" "${base_dir}/alarms" "${base_dir}/screens" "${base_dir}/notes" 2>/dev/null || { + error "Failed to create run directory: ${base_dir}" + return 1 + } + + # Export for use by capture functions + export RUN_DIR_ABS="${base_dir}" + return 0 +} + +get_run_dir() { + # Get absolute path to current run directory + # Returns: absolute path, or empty string if not initialized + echo "${RUN_DIR_ABS:-}" +} + +capture_alarms() { + # Capture AlarmManager dump to run folder + # Usage: capture_alarms "