ac39255672
Implement P0-P5 directives for operator clarity, consistent outcomes, and easy evidence capture across all test phases. Changes: - alarm-test-lib.sh: Add evidence collection (capture_alarms, capture_logcat, capture_screenshot), verdict functions (verdict_pass/warn/fail), run directory management, and release gating support (RELEASE_GATE_PHASE3) - test-phase1.sh: Refactor to unified framework with CLI modes (--setup, --run, --smoke, --all, --ci), micro-prompts, evidence capture, and verdict blocks for all 5 tests - test-phase2.sh: Add evidence capture, verdict blocks, and STRICTNESS policy (soft/hard) for warn vs fail behavior - test-phase3.sh: Add evidence capture, verdict blocks, release gating (--gate-phase3), and fatigue reduction (time estimates, automation hints) - RUNBOOK-TESTING.md: New comprehensive operator guide (669 lines) covering prerequisites, all phases, evidence locations, verdict interpretation, common failures, and troubleshooting All test scripts now use consistent UI helpers (section, substep, info, ok, warn, error), standardized evidence collection, and clear verdict reporting. Evidence is saved to timestamped run directories (runs/<RUN_ID>/) with alarms, logs, and screenshots organized by test phase and scenario. Tests pass with consistent presentation and reproducible evidence collection.
1295 lines
47 KiB
Bash
Executable File
1295 lines
47 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
|
|
# Phase 1 Testing Script - Interactive Test Runner
|
|
# Guides through all Phase 1 tests with clear prompts for UI interaction
|
|
|
|
# Strict mode (P0.2)
|
|
set -euo pipefail
|
|
IFS=$'\n\t'
|
|
|
|
# Source shared library
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "${SCRIPT_DIR}/alarm-test-lib.sh"
|
|
|
|
# Initialize run directory (P1)
|
|
ensure_run_dir || {
|
|
error "Failed to initialize run directory"
|
|
exit 1
|
|
}
|
|
|
|
# Phase 1 specific configuration
|
|
VERIFY_FIRE=${VERIFY_FIRE:-false} # Set VERIFY_FIRE=true to enable alarm fire verification
|
|
APP_DIR="${SCRIPT_DIR}"
|
|
PROJECT_ROOT="$(cd "${APP_DIR}/../.." && pwd)"
|
|
|
|
# Allow selecting specific tests on the command line (e.g. ./test-phase1.sh 1 2)
|
|
SELECTED_TESTS=()
|
|
|
|
check_plugin_configured() {
|
|
print_info "Checking if plugin is already configured..."
|
|
|
|
# Wait a moment for app to fully load
|
|
sleep 3
|
|
|
|
# Check if database exists (indicates plugin has been used)
|
|
DB_EXISTS=$($ADB_BIN shell run-as "${APP_ID}" ls databases/ 2>/dev/null | grep -c "daily_notification" || echo "0")
|
|
|
|
# Check if SharedPreferences has configuration (more reliable)
|
|
# The plugin stores config in SharedPreferences
|
|
PREFS_EXISTS=$($ADB_BIN shell run-as "${APP_ID}" ls shared_prefs/ 2>/dev/null | grep -c "DailyNotification" || echo "0")
|
|
|
|
# Check recent logs for configuration activity
|
|
RECENT_CONFIG=$($ADB_BIN logcat -d -t 50 | grep -E "Plugin configured|configurePlugin|Configuration" | tail -3)
|
|
|
|
if [ "${DB_EXISTS}" -gt "0" ] || [ "${PREFS_EXISTS}" -gt "0" ]; then
|
|
print_success "Plugin appears to be configured (database or preferences exist)"
|
|
|
|
# Show user what to check in UI
|
|
print_info "Please verify in the app UI that you see:"
|
|
echo " ⚙️ Plugin Settings: ✅ Configured"
|
|
echo " 🔌 Native Fetcher: ✅ Configured"
|
|
echo ""
|
|
echo "If both show ✅, the plugin is configured and you can skip configuration."
|
|
echo "If either shows ❌ or 'Not configured', you'll need to click 'Configure Plugin'."
|
|
echo ""
|
|
|
|
return 0
|
|
else
|
|
print_info "Plugin not configured (no database or preferences found)"
|
|
print_info "You will need to click 'Configure Plugin' in the app UI"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
check_permissions() {
|
|
print_info "Checking notification permissions..."
|
|
|
|
# Check if POST_NOTIFICATIONS permission is granted (Android 13+)
|
|
PERM_CHECK=$($ADB_BIN shell dumpsys package "${APP_ID}" | grep -A 5 "granted=true" | grep -c "android.permission.POST_NOTIFICATIONS" || echo "0")
|
|
|
|
# Also check via app's permission status if available
|
|
PERM_GRANTED=false
|
|
if [ "${PERM_CHECK}" -gt "0" ]; then
|
|
PERM_GRANTED=true
|
|
else
|
|
# Check if we're on Android 12 or below (permission not required)
|
|
SDK_VERSION=$($ADB_BIN shell getprop ro.build.version.sdk)
|
|
if [ "${SDK_VERSION}" -lt "33" ]; then
|
|
PERM_GRANTED=true # Pre-Android 13 doesn't need runtime permission
|
|
fi
|
|
fi
|
|
|
|
if [ "${PERM_GRANTED}" = true ]; then
|
|
print_success "Notification permissions granted"
|
|
return 0
|
|
else
|
|
print_warn "Notification permissions may not be granted"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
ensure_permissions() {
|
|
if check_permissions; then
|
|
print_success "Permissions already granted"
|
|
return 0
|
|
else
|
|
print_info "Notification permissions needed"
|
|
wait_for_ui_action "In the app UI, click the 'Request Permissions' button.
|
|
|
|
This will show a system permission dialog.
|
|
|
|
Steps:
|
|
1. Click 'Request Permissions' button
|
|
2. In the system dialog, tap 'Allow' to grant notification permission
|
|
3. Return to the app and verify the status shows:
|
|
- 🔔 Notifications: ✅ Granted (or similar)
|
|
|
|
Once permission is granted, press Enter to continue."
|
|
|
|
# Re-check permissions
|
|
sleep 2
|
|
if check_permissions; then
|
|
print_success "Permissions granted"
|
|
return 0
|
|
else
|
|
print_warn "Permissions may still not be granted - continuing anyway"
|
|
print_info "If notifications fail, you may need to grant permissions manually"
|
|
return 1
|
|
fi
|
|
fi
|
|
}
|
|
|
|
ensure_plugin_configured() {
|
|
if check_plugin_configured; then
|
|
# Plugin might be configured, but let user verify
|
|
wait_for_ui_action "Please check the Plugin Status section at the top of the app.
|
|
|
|
If you see:
|
|
- ⚙️ Plugin Settings: ✅ Configured
|
|
- 🔌 Native Fetcher: ✅ Configured
|
|
- 🔔 Notifications: ✅ Granted (or similar)
|
|
|
|
Then the plugin is already configured - just press Enter to continue.
|
|
|
|
If any show ❌ or 'Not configured':
|
|
- Click 'Request Permissions' if notifications are not granted
|
|
- Click 'Configure Plugin' if settings/fetcher are not configured
|
|
- Wait for all to show ✅, then press Enter."
|
|
|
|
# Give a moment for any configuration that just happened
|
|
sleep 2
|
|
print_success "Continuing with tests (plugin configuration verified or skipped)"
|
|
return 0
|
|
else
|
|
# Plugin definitely needs configuration
|
|
print_info "Plugin needs configuration"
|
|
|
|
# First ensure permissions
|
|
ensure_permissions
|
|
|
|
wait_for_ui_action "Click the 'Configure Plugin' button in the app UI.
|
|
|
|
Wait for the status to update:
|
|
- ⚙️ Plugin Settings: Should change to ✅ Configured
|
|
- 🔌 Native Fetcher: Should change to ✅ Configured
|
|
|
|
Once both show ✅, press Enter to continue."
|
|
|
|
# Verify configuration completed
|
|
sleep 2
|
|
print_success "Plugin configuration completed (or verified)"
|
|
fi
|
|
}
|
|
|
|
# kill_app is now provided by the library, but Phase 1 uses it with PACKAGE variable
|
|
# The library version uses APP_ID, which is set to PACKAGE, so it should work
|
|
# But we keep this as a compatibility wrapper if needed
|
|
|
|
# Phase 1 specific helper: get_current_time (used for fire verification)
|
|
get_current_time() {
|
|
$ADB_BIN shell date +%s
|
|
}
|
|
|
|
# ========================================
|
|
# CLI Mode Parsing (P2.1)
|
|
# ========================================
|
|
|
|
parse_cli_args() {
|
|
local mode="all"
|
|
SELECTED_TESTS=()
|
|
|
|
while [[ $# -gt 0 ]]; do
|
|
case "$1" in
|
|
--setup)
|
|
mode="setup"
|
|
shift
|
|
;;
|
|
--run)
|
|
mode="run"
|
|
shift
|
|
;;
|
|
--smoke)
|
|
mode="smoke"
|
|
shift
|
|
;;
|
|
--all)
|
|
mode="all"
|
|
shift
|
|
;;
|
|
--ci)
|
|
mode="ci"
|
|
shift
|
|
;;
|
|
-h|--help)
|
|
show_help
|
|
exit 0
|
|
;;
|
|
[0-9]*)
|
|
SELECTED_TESTS+=("$1")
|
|
shift
|
|
;;
|
|
*)
|
|
error "Unknown option: $1"
|
|
show_help
|
|
exit 1
|
|
;;
|
|
esac
|
|
done
|
|
|
|
echo "$mode"
|
|
}
|
|
|
|
show_help() {
|
|
cat <<EOF
|
|
Usage: $0 [MODE] [TEST_IDS...]
|
|
|
|
Modes:
|
|
--setup Run setup only (permissions, app install, plugin config checks)
|
|
--run Run actual Phase 1 tests (requires setup first)
|
|
--smoke Minimal test: schedule one notification + verify pending
|
|
--all Run setup + all tests (default)
|
|
--ci Non-interactive: automated checks only, fail fast
|
|
|
|
Test Selection:
|
|
[TEST_IDS] Optional test numbers to run (e.g., 0 1 2)
|
|
If omitted, all tests run
|
|
|
|
Examples:
|
|
$0 # Run all (setup + tests)
|
|
$0 --setup # Setup only
|
|
$0 --run # Run tests only
|
|
$0 --smoke # Quick smoke test
|
|
$0 --ci # CI mode (non-interactive)
|
|
$0 --run 1 2 # Run only tests 1 and 2
|
|
$0 --smoke 0 # Run smoke test for test 0
|
|
|
|
EOF
|
|
}
|
|
|
|
# ========================================
|
|
# Setup Mode (P2.1)
|
|
# ========================================
|
|
|
|
run_setup() {
|
|
section "Phase 1 Setup"
|
|
|
|
info "This mode performs pre-flight checks and setup:"
|
|
echo " • ADB device connection"
|
|
echo " • Emulator readiness"
|
|
echo " • App build and installation"
|
|
echo " • Permission checks"
|
|
echo " • Plugin configuration checks"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Pre-flight checks
|
|
section "Pre-Flight Checks"
|
|
require_adb_device
|
|
|
|
# Build and install
|
|
build_app
|
|
install_app
|
|
|
|
# Clear logs
|
|
clear_logs
|
|
|
|
# Launch app for setup checks
|
|
launch_app
|
|
sleep 2
|
|
|
|
# Check permissions
|
|
section "Permission Checks"
|
|
if check_permissions; then
|
|
ok "Notification permissions already granted"
|
|
else
|
|
if [ "${MODE}" = "ci" ]; then
|
|
error "Permissions not granted - cannot proceed in CI mode"
|
|
exit 1
|
|
fi
|
|
ensure_permissions
|
|
fi
|
|
|
|
# Check plugin configuration
|
|
section "Plugin Configuration"
|
|
if check_plugin_configured; then
|
|
ok "Plugin appears to be configured"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "Please verify in the app UI that you see:
|
|
|
|
1) ⚙️ Plugin Settings: ✅ Configured
|
|
2) 🔌 Native Fetcher: ✅ Configured
|
|
3) 🔔 Notifications: ✅ Granted
|
|
|
|
If all show ✅, press Enter to continue.
|
|
If any show ❌, click 'Configure Plugin' first."
|
|
fi
|
|
else
|
|
if [ "${MODE}" = "ci" ]; then
|
|
error "Plugin not configured - cannot proceed in CI mode"
|
|
exit 1
|
|
fi
|
|
ensure_plugin_configured
|
|
fi
|
|
|
|
ok "Setup complete!"
|
|
|
|
# Capture initial state
|
|
capture_alarms "setup_initial"
|
|
capture_logcat "setup_initial" "DNP" 100
|
|
|
|
evidence_block "phase1_setup"
|
|
}
|
|
|
|
# ========================================
|
|
# Smoke Test Mode (P2.1)
|
|
# ========================================
|
|
|
|
run_smoke_test() {
|
|
section "Phase 1 Smoke Test"
|
|
|
|
info "Minimal test: schedule one notification + verify pending"
|
|
echo "Expected time: 2-4 minutes"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Ensure setup is done
|
|
if ! check_permissions >/dev/null 2>&1; then
|
|
warn "Setup not complete - running setup first..."
|
|
run_setup
|
|
fi
|
|
|
|
# Capture before state
|
|
capture_alarms "smoke_before"
|
|
|
|
# Schedule notification
|
|
section "Smoke Test: Schedule Notification"
|
|
launch_app
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "1) In the app UI, click 'Test Notification' to schedule a notification for 4 minutes in the future."
|
|
else
|
|
info "CI mode: Assuming notification is scheduled manually or via automation"
|
|
sleep 3
|
|
fi
|
|
|
|
# Wait for alarm to be registered
|
|
sleep 3
|
|
|
|
# Capture after state
|
|
capture_alarms "smoke_after"
|
|
capture_logcat "smoke_after" "DNP-SCHEDULE|DNP-NOTIFY" 50
|
|
|
|
# Verify alarm exists
|
|
local alarm_count
|
|
alarm_count="$(get_plugin_alarm_count)"
|
|
|
|
if [ "${alarm_count}" -eq "1" ]; then
|
|
verdict_pass "phase1_smoke" "Smoke test passed: 1 alarm scheduled"
|
|
elif [ "${alarm_count}" -gt "1" ]; then
|
|
verdict_fail "phase1_smoke" "Smoke test failed: ${alarm_count} alarms found (expected: 1)"
|
|
else
|
|
verdict_warn "phase1_smoke" "Smoke test inconclusive: ${alarm_count} alarms found (expected: 1, may be race condition)"
|
|
fi
|
|
|
|
evidence_block "phase1_smoke"
|
|
}
|
|
|
|
# ========================================
|
|
# Main Test Execution
|
|
# ========================================
|
|
|
|
main() {
|
|
# Parse CLI arguments
|
|
MODE="$(parse_cli_args "$@")"
|
|
|
|
section "Phase 1 Testing Script"
|
|
echo "Mode: ${MODE}"
|
|
echo "Run ID: ${RUN_ID}"
|
|
echo "Evidence directory: $(get_run_dir)"
|
|
echo ""
|
|
|
|
if [ "${MODE}" = "setup" ]; then
|
|
run_setup
|
|
return 0
|
|
elif [ "${MODE}" = "smoke" ]; then
|
|
run_smoke_test
|
|
return 0
|
|
elif [ "${MODE}" = "ci" ]; then
|
|
info "CI mode: Non-interactive, automated checks only"
|
|
run_setup
|
|
# In CI mode, we might want to run a subset of tests
|
|
# For now, just run smoke test
|
|
run_smoke_test
|
|
return 0
|
|
fi
|
|
|
|
# For --run and --all modes, continue with full test suite
|
|
if [ "${MODE}" = "all" ]; then
|
|
info "Running full test suite (setup + tests)"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
run_setup
|
|
elif [ "${MODE}" = "run" ]; then
|
|
info "Running tests only (assuming setup already done)"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi
|
|
|
|
# ============================================
|
|
# TEST 0: Daily Rollover (Core Contract Verification)
|
|
# ============================================
|
|
if should_run_test "0" SELECTED_TESTS; then
|
|
section "TEST 0: Daily Rollover Verification"
|
|
|
|
info "Purpose: Verify that after a notification fires, the next day's"
|
|
info " schedule is correctly computed and only ONE alarm exists."
|
|
info ""
|
|
info "Expected time: 5-10 minutes"
|
|
info "Automatable: Partial (requires manual clock manipulation or waiting)"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Capture initial state
|
|
capture_alarms "test0_initial"
|
|
capture_logcat "test0_initial" "DNP" 50
|
|
|
|
substep "Step 1: Schedule a test notification"
|
|
launch_app
|
|
ensure_plugin_configured
|
|
|
|
# Capture pre-schedule state
|
|
capture_alarms "test0_before_schedule"
|
|
capture_screenshot "test0_before_schedule"
|
|
|
|
local initial_count system_count
|
|
initial_count="$(get_plugin_alarm_count)"
|
|
system_count="$(get_system_alarm_count)"
|
|
info "Current notification alarms: ${initial_count} (expected before scheduling: 0)"
|
|
info "System/other alarms: ${system_count} (for context)"
|
|
|
|
if [ "${initial_count}" -eq "0" ] 2>/dev/null; then
|
|
ok "No existing notification alarms (clean state)"
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "1) In the app UI, schedule a daily notification.
|
|
|
|
For this test, schedule it for a time very soon (e.g., 1-2 minutes from now)
|
|
to observe the rollover behavior.
|
|
|
|
This will schedule:
|
|
• 1 notification alarm (AlarmManager) for the specified time
|
|
• 1 prefetch job (WorkManager) for 2 minutes before that time"
|
|
else
|
|
info "CI mode: Assuming notification is scheduled manually"
|
|
sleep 3
|
|
fi
|
|
|
|
sleep 3 # Give alarm time to be registered
|
|
|
|
# Capture post-schedule state
|
|
capture_alarms "test0_after_schedule"
|
|
capture_screenshot "test0_after_schedule"
|
|
|
|
local post_schedule_count
|
|
post_schedule_count="$(get_plugin_alarm_count)"
|
|
|
|
# Preliminary check (final verdict comes after rollover)
|
|
if [ "${post_schedule_count}" -eq "0" ] 2>/dev/null; then
|
|
warn "Found 0 plugin alarms right after scheduling (may be race condition)"
|
|
info "Alarm may not be visible in dumpsys yet - will verify after rollover"
|
|
elif [ "${post_schedule_count}" -eq "1" ] 2>/dev/null; then
|
|
ok "Found 1 notification alarm (expected: 1) - preliminary check passed"
|
|
else
|
|
warn "Found ${post_schedule_count} notification alarms (expected: 1) - DUPLICATES DETECTED!"
|
|
error "This indicates duplicate NOTIFICATION alarms were created (BUG)"
|
|
show_plugin_alarms_compact
|
|
fi
|
|
initial_count="${post_schedule_count}"
|
|
fi
|
|
|
|
# Show alarm details if we found exactly 1 alarm
|
|
if [ "${initial_count}" -eq "1" ] 2>/dev/null; then
|
|
ok "Single notification alarm scheduled (one per day)"
|
|
|
|
local alarm_details
|
|
alarm_details="$($ADB_BIN shell dumpsys alarm | grep -A 3 "com.timesafari.dailynotification" | grep -A 3 "com.timesafari.daily.NOTIFICATION" | head -10)"
|
|
if [ -n "${alarm_details}" ]; then
|
|
info "Notification alarm details:"
|
|
echo "${alarm_details}" | head -5
|
|
echo ""
|
|
fi
|
|
elif [ "${initial_count}" -gt "1" ] 2>/dev/null; then
|
|
warn "Found ${initial_count} notification alarms (expected: 1) - DUPLICATES DETECTED!"
|
|
fi
|
|
|
|
substep "Step 2: Wait for notification to fire"
|
|
info "To complete this test:"
|
|
echo " 1. Wait for notification to fire (or advance emulator clock)"
|
|
echo " 2. Verify plugin computed next day's time (24 hours later)"
|
|
echo " 3. Verify exactly ONE notification alarm exists for tomorrow"
|
|
echo " 4. Verify no duplicate alarms were created"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "2) After the notification fires (or you advance the clock), verify:
|
|
|
|
• Only ONE alarm exists (one per day)
|
|
• Alarm time is for tomorrow (24 hours later)
|
|
• No duplicate alarms were created
|
|
|
|
Press Enter when you've verified this (or to skip this test)."
|
|
else
|
|
info "CI mode: Skipping manual verification step"
|
|
sleep 5 # Give time for rollover if automated
|
|
fi
|
|
|
|
substep "Step 3: Verify rollover state"
|
|
info "Polling for stable alarm count (allowing up to ~10 seconds for Android to settle)..."
|
|
|
|
local post_rollover_count system_final
|
|
post_rollover_count="$(wait_for_stable_plugin_alarm_count 5 2)"
|
|
system_final="$(get_system_alarm_count)"
|
|
|
|
# Capture post-rollover state
|
|
capture_alarms "test0_after_rollover"
|
|
capture_logcat "test0_after_rollover" "DNP-SCHEDULE|DNP-NOTIFY|ROLLOVER" 100
|
|
capture_screenshot "test0_after_rollover"
|
|
|
|
info "Notification alarms after rollover: ${post_rollover_count} (expected: 1)"
|
|
info "System/other alarms: ${system_final} (for context)"
|
|
|
|
# Final verdict
|
|
if [ "${post_rollover_count}" -eq "1" ] 2>/dev/null; then
|
|
verdict_pass "test0_daily_rollover" "Daily rollover created exactly one NOTIFICATION alarm for tomorrow"
|
|
elif [ "${post_rollover_count}" -gt "1" ] 2>/dev/null; then
|
|
error "Daily rollover created ${post_rollover_count} NOTIFICATION alarms (duplicates)"
|
|
show_plugin_alarms_compact
|
|
verdict_fail "test0_daily_rollover" "Duplicate alarms detected after rollover (expected: 1, got: ${post_rollover_count})"
|
|
else
|
|
warn "No NOTIFICATION alarm found after rollover (expected: 1, got: 0)"
|
|
show_plugin_alarms_compact
|
|
verdict_warn "test0_daily_rollover" "Rollover may have failed - no alarm found (expected: 1, got: 0)"
|
|
fi
|
|
|
|
evidence_block "test0_daily_rollover"
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi # End of "if should_run_test 0" block
|
|
|
|
# ============================================
|
|
# TEST 1: Force-Stop Recovery - Database Restoration
|
|
# ============================================
|
|
if should_run_test "1" SELECTED_TESTS; then
|
|
section "TEST 1: Force-Stop Recovery - Database Restoration"
|
|
|
|
info "Purpose: Verify that after force-stop (which clears alarms), recovery"
|
|
info " uses the database to rebuild alarms on app relaunch."
|
|
info ""
|
|
info "Expected time: 5-8 minutes"
|
|
info "Automatable: Partial (requires manual force-stop verification)"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Capture initial state
|
|
capture_alarms "test1_initial"
|
|
capture_logcat "test1_initial" "DNP" 50
|
|
|
|
substep "Step 1: Clean start - verify no lingering alarms"
|
|
local lingering_count system_count
|
|
lingering_count="$(get_plugin_alarm_count)"
|
|
system_count="$(get_system_alarm_count)"
|
|
info "Current plugin notification alarms: ${lingering_count}"
|
|
info "System/other alarms: ${system_count} (for context)"
|
|
|
|
local goto_test1_end=false
|
|
|
|
if [ "${lingering_count}" -gt "0" ] 2>/dev/null; then
|
|
warn "Found ${lingering_count} lingering plugin alarm(s) - these will interfere with TEST 1"
|
|
info "TEST 1 needs a clean state (no existing plugin alarms)"
|
|
info "Resetting app state via uninstall + reinstall..."
|
|
|
|
# Uninstall existing app
|
|
info "Uninstalling existing app..."
|
|
set +e
|
|
local uninstall_output uninstall_status
|
|
uninstall_output="$($ADB_BIN uninstall "$APP_ID" 2>&1)"
|
|
uninstall_status=$?
|
|
set -e
|
|
|
|
if [ $uninstall_status -eq 0 ]; then
|
|
ok "App uninstall succeeded (clean slate)"
|
|
else
|
|
if grep -q "DELETE_FAILED_INTERNAL_ERROR" <<<"$uninstall_output"; then
|
|
info "No existing app to uninstall (continuing)"
|
|
else
|
|
warn "App uninstall reported an error: $uninstall_output (continuing anyway)"
|
|
fi
|
|
fi
|
|
|
|
# Reinstall APK
|
|
info "Reinstalling APK..."
|
|
if $ADB_BIN install -r "$APK_PATH" >/dev/null 2>&1; then
|
|
ok "App reinstall succeeded"
|
|
else
|
|
error "App reinstall FAILED - cannot proceed with TEST 1"
|
|
warn "Marking TEST 1 as INCONCLUSIVE due to dirty starting state"
|
|
capture_screenshot "test1_dirty_state_reinstall_failed"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
goto_test1_end=true
|
|
fi
|
|
|
|
if [ "${goto_test1_end}" != "true" ]; then
|
|
# Verify install
|
|
if ! $ADB_BIN shell pm list packages | grep -q "$APP_ID"; then
|
|
error "App not found in package list after reinstall - aborting TEST 1"
|
|
capture_screenshot "test1_dirty_state_verify_failed"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
goto_test1_end=true
|
|
fi
|
|
fi
|
|
|
|
if [ "${goto_test1_end}" != "true" ]; then
|
|
# Clear logs
|
|
info "Clearing logcat buffer after reinstall..."
|
|
$ADB_BIN logcat -c || true
|
|
|
|
# Re-check plugin alarms
|
|
info "Rechecking plugin alarms after reset..."
|
|
sleep 2
|
|
lingering_count="$(get_plugin_alarm_count)"
|
|
info "Plugin alarms after reset: ${lingering_count} (expected: 0)"
|
|
|
|
if [ "${lingering_count}" -ne "0" ] 2>/dev/null; then
|
|
warn "TEST 1 starting with non-zero alarm count even after reset; treating as INCONCLUSIVE"
|
|
info "This may indicate device-specific behavior where alarms persist across uninstall"
|
|
capture_screenshot "test1_unexpected_alarms_after_reset"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
goto_test1_end=true
|
|
fi
|
|
fi
|
|
|
|
if [ "${goto_test1_end}" = "true" ]; then
|
|
warn "TEST 1 skipped due to inability to achieve clean starting state"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
else
|
|
ok "App state reset complete. TEST 1 starting from clean state"
|
|
fi
|
|
else
|
|
ok "No lingering plugin alarms found (clean state)"
|
|
fi
|
|
|
|
# Skip remaining TEST 1 steps if we couldn't achieve clean state
|
|
if [ "${goto_test1_end}" != "true" ]; then
|
|
|
|
substep "Step 2: Schedule a known future alarm"
|
|
launch_app
|
|
ensure_plugin_configured
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "1) In the app UI, click the 'Test Notification' button.
|
|
|
|
This will schedule ONE notification for 4 minutes in the future.
|
|
(App supports one alarm per day)
|
|
|
|
The alarm will be stored in the database AND scheduled in AlarmManager."
|
|
else
|
|
info "CI mode: Assuming notification is scheduled manually"
|
|
sleep 3
|
|
fi
|
|
|
|
sleep 2
|
|
|
|
# Capture before force-stop state
|
|
capture_alarms "test1_before_force_stop"
|
|
capture_logcat "test1_before_force_stop" "DNP-SCHEDULE|DNP-NOTIFY" 50
|
|
|
|
substep "Step 3: Verify alarm exists in AlarmManager (BEFORE force-stop)"
|
|
local alarm_count_before system_count_before
|
|
alarm_count_before="$(get_plugin_alarm_count)"
|
|
system_count_before="$(get_system_alarm_count)"
|
|
info "Plugin alarms: ${alarm_count_before} (expected: 1)"
|
|
info "System/other alarms: ${system_count_before} (for context)"
|
|
|
|
if [ "${alarm_count_before}" -eq "0" ] 2>/dev/null; then
|
|
error "No plugin alarms found in AlarmManager - cannot test force-stop recovery"
|
|
info "Make sure you clicked 'Test Notification' and wait a moment"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
# Re-check
|
|
alarm_count_before="$(get_plugin_alarm_count)"
|
|
if [ "${alarm_count_before}" -eq "0" ] 2>/dev/null; then
|
|
error "Still no alarms found - aborting test"
|
|
verdict_fail "test1_force_stop_recovery" "No alarms found before force-stop - cannot test recovery"
|
|
evidence_block "test1_force_stop_recovery"
|
|
goto_test1_end=true
|
|
fi
|
|
fi
|
|
|
|
if [ "${alarm_count_before}" -eq "1" ] 2>/dev/null; then
|
|
ok "Single plugin alarm confirmed in AlarmManager (one per day)"
|
|
else
|
|
warn "Found ${alarm_count_before} plugin alarms (expected: 1) - continuing anyway"
|
|
fi
|
|
|
|
# Extract trigger time for later verification
|
|
local alarm_details_before alarm_trigger_ms alarm_trigger_sec alarm_readable
|
|
alarm_details_before="$($ADB_BIN shell dumpsys alarm | grep -A 3 "$APP_ID" | head -10)"
|
|
if [ -n "${alarm_details_before}" ]; then
|
|
info "Alarm details:"
|
|
echo "${alarm_details_before}" | head -5
|
|
echo ""
|
|
fi
|
|
|
|
alarm_trigger_ms=$(echo "${alarm_details_before}" | grep -oE "origWhen [0-9]+" | head -1 | awk '{print $2}')
|
|
if [ -n "${alarm_trigger_ms}" ]; then
|
|
alarm_trigger_sec=$((alarm_trigger_ms / 1000))
|
|
alarm_readable=$(date -d "@${alarm_trigger_sec}" 2>/dev/null || echo "${alarm_trigger_sec}")
|
|
info "Alarm scheduled for: ${alarm_readable} (${alarm_trigger_ms} ms)"
|
|
fi
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
substep "Step 4: Force-stop the app (clears alarms)"
|
|
warn "Force-stop will clear ALL alarms from AlarmManager"
|
|
info "Executing: $ADB_BIN shell am force-stop ${APP_ID}"
|
|
force_stop_app
|
|
sleep 2
|
|
|
|
# Capture after force-stop state
|
|
capture_alarms "test1_after_force_stop"
|
|
|
|
substep "Step 5: Verify alarms are MISSING (cleared by OS)"
|
|
sleep 1
|
|
local alarm_count_after system_count_after
|
|
alarm_count_after="$(get_plugin_alarm_count)"
|
|
system_count_after="$(get_system_alarm_count)"
|
|
info "Plugin alarms after force-stop: ${alarm_count_after} (expected: 0)"
|
|
info "System/other alarms: ${system_count_after} (for context)"
|
|
|
|
if [ "${alarm_count_after}" -eq "0" ] 2>/dev/null; then
|
|
ok "Plugin alarms cleared by force-stop (count: ${alarm_count_after})"
|
|
info "This confirms: Force-stop cleared alarms from AlarmManager"
|
|
else
|
|
warn "Plugin alarms still present after force-stop (count: ${alarm_count_after})"
|
|
info "Some devices/OS versions may not clear alarms on force-stop"
|
|
info "Continuing test anyway - recovery should still work"
|
|
fi
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
substep "Step 6: Relaunch app (triggers recovery from database)"
|
|
clear_logs
|
|
launch_app
|
|
sleep 4 # Give recovery time to run
|
|
|
|
substep "Step 7: Verify recovery rebuilt alarms from database"
|
|
sleep 2
|
|
|
|
# Capture after recovery state
|
|
capture_alarms "test1_after_recovery"
|
|
capture_logcat "test1_after_recovery" "DNP-REACTIVATION" 100
|
|
capture_screenshot "test1_after_recovery"
|
|
|
|
local alarm_count_recovered system_count_recovered
|
|
alarm_count_recovered="$(get_plugin_alarm_count)"
|
|
system_count_recovered="$(get_system_alarm_count)"
|
|
info "Plugin alarms after recovery: ${alarm_count_recovered} (expected: 1)"
|
|
info "System/other alarms: ${system_count_recovered} (for context)"
|
|
|
|
info "Checking recovery logs..."
|
|
check_recovery_logs
|
|
|
|
info "Expected log output:"
|
|
echo " DNP-REACTIVATION: Starting app launch recovery"
|
|
echo " DNP-REACTIVATION: Rescheduled alarm: <id> for <time>"
|
|
echo " DNP-REACTIVATION: Cold start recovery complete: ... rescheduled>=1, ..."
|
|
echo ""
|
|
|
|
# Check recovery logs for rescheduling and recovery source
|
|
local recovery_result rescheduled_count verified_count recovery_source
|
|
recovery_result="$($ADB_BIN logcat -d | grep "Cold start recovery complete\|Boot recovery complete" | tail -1)"
|
|
rescheduled_count=$(echo "${recovery_result}" | grep -oE "rescheduled=[0-9]+" | grep -oE "[0-9]+" || echo "0")
|
|
verified_count=$(echo "${recovery_result}" | grep -oE "verified=[0-9]+" | grep -oE "[0-9]+" || echo "0")
|
|
recovery_source="$($ADB_BIN logcat -d | grep -E "recovery source|from database|DATABASE" | tail -1 || echo "")"
|
|
|
|
# Pass/fail criteria
|
|
local test1_passed=false
|
|
|
|
if [ "${alarm_count_recovered}" -gt "0" ] 2>/dev/null; then
|
|
ok "Alarms restored in AlarmManager (count: ${alarm_count_recovered})"
|
|
if [ "${rescheduled_count}" -gt "0" ] 2>/dev/null; then
|
|
ok "Recovery logs confirm rescheduling (rescheduled=${rescheduled_count})"
|
|
test1_passed=true
|
|
else
|
|
warn "Alarms restored but logs show rescheduled=0"
|
|
info "This might be okay if alarms were verified instead of rescheduled"
|
|
# Still pass if alarms are restored, even if rescheduled=0
|
|
test1_passed=true
|
|
fi
|
|
else
|
|
error "No alarms restored (count: ${alarm_count_recovered})"
|
|
info "Recovery may have failed or alarms were not in database"
|
|
fi
|
|
|
|
# Final verdict
|
|
if [ "${test1_passed}" = "true" ]; then
|
|
verdict_pass "test1_force_stop_recovery" "Recovery successfully rebuilt alarms from database (before: ${alarm_count_before}, after stop: ${alarm_count_after}, after recovery: ${alarm_count_recovered}, rescheduled: ${rescheduled_count})"
|
|
else
|
|
verdict_fail "test1_force_stop_recovery" "Recovery did not rebuild alarms correctly (before: ${alarm_count_before}, after stop: ${alarm_count_after}, after recovery: ${alarm_count_recovered})"
|
|
fi
|
|
|
|
evidence_block "test1_force_stop_recovery"
|
|
|
|
# Optional: Verify alarm fires (controlled by VERIFY_FIRE flag)
|
|
if [ "${VERIFY_FIRE}" = "true" ] && [ -n "${alarm_trigger_ms}" ] && [ "${goto_test1_end}" != "true" ]; then
|
|
substep "Step 8: Verify alarm fires at scheduled time (optional)"
|
|
|
|
local current_time_sec current_time_ms wait_ms wait_sec
|
|
current_time_sec=$(get_current_time)
|
|
current_time_ms=$((current_time_sec * 1000))
|
|
wait_ms=$((alarm_trigger_ms - current_time_ms))
|
|
|
|
if [ "${wait_ms}" -lt 0 ]; then
|
|
warn "Alarm time already passed (${wait_ms} ms ago); skipping fire verification"
|
|
else
|
|
wait_sec=$((wait_ms / 1000))
|
|
|
|
# Clamp upper bound to prevent accidentally waiting 30+ minutes
|
|
if [ "${wait_sec}" -gt 600 ]; then
|
|
warn "Alarm is >10 minutes away (${wait_sec}s); skipping fire verification"
|
|
info "To test fire verification, schedule alarm closer to current time"
|
|
else
|
|
info "Alarm scheduled for: ${alarm_readable}"
|
|
info "Current time: $(date -d "@${current_time_sec}" 2>/dev/null || echo "${current_time_sec}")"
|
|
info "Waiting ~${wait_sec} seconds for alarm to fire..."
|
|
|
|
clear_logs
|
|
sleep ${wait_sec}
|
|
sleep 2
|
|
|
|
info "Checking logs for fired alarm..."
|
|
local alarm_fired
|
|
alarm_fired="$($ADB_BIN logcat -d | grep -E "DNP-RECEIVE|DNP-NOTIFY|DNP-WORK|Alarm fired|Notification displayed" | tail -10)"
|
|
|
|
if [ -n "${alarm_fired}" ]; then
|
|
ok "Alarm fired! Logs:"
|
|
echo "${alarm_fired}"
|
|
else
|
|
warn "No alarm fire logs found"
|
|
info "Check notification tray manually or review recent logs"
|
|
fi
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
fi # End of "if goto_test1_end != true" block
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi # End of "if should_run_test 1" block
|
|
|
|
# ============================================
|
|
# TEST 2: Schedule Update (One-Per-Day Semantics)
|
|
# ============================================
|
|
if should_run_test "2" SELECTED_TESTS; then
|
|
section "TEST 2: Schedule Update Verification"
|
|
|
|
info "Purpose: Verify that updating the schedule time maintains 'one per day' semantics"
|
|
info ""
|
|
info "Expected time: 4-6 minutes"
|
|
info "Automatable: Partial (requires manual schedule update)"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Capture initial state
|
|
capture_alarms "test2_initial"
|
|
capture_logcat "test2_initial" "DNP" 50
|
|
|
|
substep "Step 1: Launch app and verify initial schedule"
|
|
launch_app
|
|
ensure_plugin_configured
|
|
|
|
# Get initial alarm count
|
|
local initial_alarm_count system_alarm_count
|
|
initial_alarm_count="$(get_plugin_alarm_count)"
|
|
system_alarm_count="$(get_system_alarm_count)"
|
|
info "Plugin alarms: ${initial_alarm_count} (expected: 1)"
|
|
info "System/other alarms: ${system_alarm_count} (for context)"
|
|
|
|
if [ "${initial_alarm_count}" -eq "1" ] 2>/dev/null; then
|
|
ok "Initial alarm confirmed (one per day)"
|
|
elif [ "${initial_alarm_count}" -eq "0" ] 2>/dev/null; then
|
|
warn "No initial alarm found - scheduling one first..."
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "1) In the app UI, schedule a daily notification (e.g., click 'Test Notification')."
|
|
else
|
|
info "CI mode: Assuming notification is scheduled manually"
|
|
sleep 2
|
|
fi
|
|
sleep 2
|
|
initial_alarm_count="$(get_plugin_alarm_count)"
|
|
if [ "${initial_alarm_count}" -eq "1" ] 2>/dev/null; then
|
|
ok "Alarm scheduled"
|
|
else
|
|
error "Failed to schedule initial alarm"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi
|
|
else
|
|
warn "Found ${initial_alarm_count} plugin alarms (expected: 1) - continuing anyway"
|
|
fi
|
|
|
|
# Capture before update state
|
|
capture_alarms "test2_before_update"
|
|
|
|
substep "Step 2: Update schedule time"
|
|
if [ "${MODE}" != "ci" ]; then
|
|
ui_prompt "2) In the app UI, change the schedule time (e.g., from 06:50 to 07:10)
|
|
and apply the schedule.
|
|
|
|
This should cancel the old alarm and schedule a new one at the new time.
|
|
You should still have exactly 1 alarm (one per day)."
|
|
else
|
|
info "CI mode: Assuming schedule is updated manually"
|
|
sleep 3
|
|
fi
|
|
|
|
sleep 3
|
|
|
|
# Capture after update state
|
|
capture_alarms "test2_after_update"
|
|
capture_logcat "test2_after_update" "DNP-SCHEDULE|DNP-NOTIFY" 50
|
|
|
|
local updated_alarm_count
|
|
updated_alarm_count="$(get_plugin_alarm_count)"
|
|
info "Plugin alarms after update: ${updated_alarm_count} (expected: 1)"
|
|
|
|
local test2_failed=false
|
|
|
|
if [ "${updated_alarm_count}" -eq "1" ] 2>/dev/null; then
|
|
ok "Single alarm confirmed after schedule update (one per day maintained)"
|
|
elif [ "${updated_alarm_count}" -gt "1" ] 2>/dev/null; then
|
|
error "TEST 2 FAILED: Found ${updated_alarm_count} plugin alarms after update (expected: 1)"
|
|
error "Old alarm was NOT canceled - violates 'one per day' semantics"
|
|
info "This indicates the plugin did not clean up existing schedules before creating new one"
|
|
test2_failed=true
|
|
else
|
|
warn "Found ${updated_alarm_count} plugin alarms (expected: 1) - no alarm scheduled after update"
|
|
fi
|
|
|
|
substep "Step 3: Kill app and relaunch (triggers recovery)"
|
|
kill_app
|
|
clear_logs
|
|
launch_app
|
|
|
|
substep "Step 4: Check recovery logs and verify alarm count"
|
|
sleep 3
|
|
|
|
# Capture after recovery state
|
|
capture_alarms "test2_after_recovery"
|
|
capture_logcat "test2_after_recovery" "DNP-REACTIVATION" 100
|
|
|
|
check_recovery_logs
|
|
|
|
info "Expected log output (either):"
|
|
echo " DNP-REACTIVATION: Verified scheduled alarm: <id> at <time>"
|
|
echo " OR"
|
|
echo " DNP-REACTIVATION: Rescheduled missing alarm: <id> at <time>"
|
|
echo " DNP-REACTIVATION: Cold start recovery complete: ..., verified=1 or rescheduled=1, ..."
|
|
echo ""
|
|
|
|
local recovery_result rescheduled_count verified_count alarm_count_after_recovery
|
|
recovery_result="$($ADB_BIN logcat -d | grep "Cold start recovery complete" | tail -1)"
|
|
rescheduled_count=$(echo "${recovery_result}" | grep -oE "rescheduled=[0-9]+" | grep -oE "[0-9]+" || echo "0")
|
|
verified_count=$(echo "${recovery_result}" | grep -oE "verified=[0-9]+" | grep -oE "[0-9]+" || echo "0")
|
|
alarm_count_after_recovery="$(get_plugin_alarm_count)"
|
|
info "Plugin alarms after recovery: ${alarm_count_after_recovery} (expected: 1)"
|
|
|
|
# CRITICAL: Test fails if alarm count > 1 after recovery (violates "one per day")
|
|
if [ "${alarm_count_after_recovery}" -gt "1" ] 2>/dev/null; then
|
|
error "TEST 2 FAILED: Found ${alarm_count_after_recovery} plugin alarms after recovery (expected: 1)"
|
|
error "Multiple schedules in database caused recovery to reschedule duplicates"
|
|
error "This violates 'one per day' semantics - only one notification per day should exist"
|
|
test2_failed=true
|
|
fi
|
|
|
|
# Final verdict
|
|
if [ "${test2_failed}" = "true" ]; then
|
|
verdict_fail "test2_schedule_update" "Multiple alarms detected - violates 'one per day' semantics (after update: ${updated_alarm_count}, after recovery: ${alarm_count_after_recovery})"
|
|
elif [ "${alarm_count_after_recovery}" -eq "1" ] 2>/dev/null; then
|
|
if [ "${rescheduled_count}" -gt "0" ] 2>/dev/null; then
|
|
verdict_pass "test2_schedule_update" "Schedule update maintained one per day semantics (rescheduled: ${rescheduled_count})"
|
|
elif [ "${verified_count}" -gt "0" ] 2>/dev/null; then
|
|
verdict_pass "test2_schedule_update" "Schedule update maintained one per day semantics (verified: ${verified_count})"
|
|
else
|
|
verdict_pass "test2_schedule_update" "Schedule update maintained one per day semantics (no recovery needed)"
|
|
fi
|
|
else
|
|
verdict_warn "test2_schedule_update" "Schedule update result inconclusive (after update: ${updated_alarm_count}, after recovery: ${alarm_count_after_recovery})"
|
|
fi
|
|
|
|
evidence_block "test2_schedule_update"
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi # End of "if should_run_test 2" block
|
|
|
|
# ============================================
|
|
# TEST 3: Recovery Timeout
|
|
# ============================================
|
|
if should_run_test "3" SELECTED_TESTS; then
|
|
section "TEST 3: Recovery Timeout"
|
|
|
|
info "Purpose: Verify recovery times out gracefully"
|
|
info ""
|
|
info "Expected time: 2-3 minutes"
|
|
info "Automatable: Yes (code verification only)"
|
|
info "Note: Full test (100+ schedules) requires manual execution"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Capture initial state
|
|
capture_alarms "test3_initial"
|
|
|
|
substep "Step 1: Check recovery timeout implementation"
|
|
local timeout_found timeout_protection_found
|
|
timeout_found=false
|
|
timeout_protection_found=false
|
|
|
|
if grep -q "RECOVERY_TIMEOUT_SECONDS.*2L" "${PROJECT_ROOT}/android/src/main/java/com/timesafari/dailynotification/ReactivationManager.kt" 2>/dev/null; then
|
|
ok "Timeout is set to 2 seconds"
|
|
timeout_found=true
|
|
else
|
|
error "Timeout not found in code"
|
|
fi
|
|
|
|
if grep -q "withTimeout" "${PROJECT_ROOT}/android/src/main/java/com/timesafari/dailynotification/ReactivationManager.kt" 2>/dev/null; then
|
|
ok "Timeout protection is implemented"
|
|
timeout_protection_found=true
|
|
else
|
|
error "Timeout protection not found"
|
|
fi
|
|
|
|
# Capture evidence
|
|
capture_logcat "test3_timeout_check" "DNP-REACTIVATION" 50
|
|
|
|
# Final verdict
|
|
if [ "${timeout_found}" = "true" ] && [ "${timeout_protection_found}" = "true" ]; then
|
|
verdict_pass "test3_recovery_timeout" "Timeout mechanism verified in code (2 seconds, withTimeout protection)"
|
|
else
|
|
verdict_fail "test3_recovery_timeout" "Timeout mechanism not properly implemented (timeout found: ${timeout_found}, protection found: ${timeout_protection_found})"
|
|
fi
|
|
|
|
evidence_block "test3_recovery_timeout"
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi # End of "if should_run_test 3" block
|
|
|
|
# ============================================
|
|
# TEST 4: Invalid Data Handling
|
|
# ============================================
|
|
if should_run_test "4" SELECTED_TESTS; then
|
|
section "TEST 4: Invalid Data Handling"
|
|
|
|
info "Purpose: Verify invalid data doesn't crash recovery"
|
|
info ""
|
|
info "Expected time: 5-8 minutes"
|
|
info "Automatable: Partial (requires debuggable app or manual injection)"
|
|
info "Note: This test injects invalid data (empty IDs, null nextRunAt) and"
|
|
info " verifies that recovery handles it gracefully without crashing"
|
|
echo ""
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
|
|
# Capture initial state
|
|
capture_alarms "test4_initial"
|
|
capture_logcat "test4_initial" "DNP" 50
|
|
|
|
substep "Step 1: Inject invalid test data"
|
|
|
|
# Clear logs before test
|
|
clear_logs
|
|
|
|
# Check if app is debuggable (look for DEBUGGABLE flag)
|
|
local is_debuggable=false
|
|
if $ADB_BIN shell dumpsys package "${APP_ID}" | grep -qi "DEBUGGABLE"; then
|
|
is_debuggable=true
|
|
fi
|
|
|
|
if [ "${is_debuggable}" = "true" ]; then
|
|
ok "App is debuggable - can inject data via database"
|
|
|
|
substep "Step 2: Inject invalid data via direct database access"
|
|
|
|
# Launch app first to ensure database is initialized
|
|
info "Launching app to initialize database..."
|
|
$ADB_BIN shell am start -n "${APP_ID}/.MainActivity" > /dev/null 2>&1
|
|
sleep 2
|
|
|
|
# Stop app before database injection (prevents locking issues)
|
|
info "Stopping app before database injection..."
|
|
$ADB_BIN shell am force-stop "${APP_ID}"
|
|
sleep 1
|
|
|
|
# Inject invalid data via direct database access
|
|
info "Injecting invalid test data into database..."
|
|
|
|
# Calculate next run time (24 hours from now in milliseconds)
|
|
local next_run past_time now_time
|
|
next_run=$(($(date +%s) * 1000 + 86400000))
|
|
past_time=$(($(date +%s) * 1000 - 3600000))
|
|
now_time=$(($(date +%s) * 1000))
|
|
|
|
# Inject schedule with empty ID
|
|
info " - Injecting schedule with empty ID..."
|
|
$ADB_BIN shell "run-as ${APP_ID} sqlite3 databases/daily_notification_plugin.db \"INSERT OR REPLACE INTO schedules (id, kind, cron, clockTime, enabled, nextRunAt, jitterMs, backoffPolicy) VALUES ('', 'notify', '0 9 * * *', '09:00', 1, ${next_run}, 0, 'exp');\"" 2>&1 || true
|
|
|
|
# Inject schedule with null nextRunAt
|
|
info " - Injecting schedule with null nextRunAt..."
|
|
$ADB_BIN shell "run-as ${APP_ID} sqlite3 databases/daily_notification_plugin.db \"INSERT OR REPLACE INTO schedules (id, kind, cron, clockTime, enabled, nextRunAt, jitterMs, backoffPolicy) VALUES ('test_null_nextrunat', 'notify', '0 9 * * *', '09:00', 1, NULL, 0, 'exp');\"" 2>&1 || true
|
|
|
|
# Checkpoint WAL file to ensure changes are visible
|
|
info " - Checkpointing WAL file..."
|
|
$ADB_BIN shell "run-as ${APP_ID} sqlite3 databases/daily_notification_plugin.db \"PRAGMA wal_checkpoint(FULL);\"" 2>&1 || true
|
|
|
|
# Verify data was inserted
|
|
info "Verifying data injection..."
|
|
local schedule_count
|
|
schedule_count="$($ADB_BIN shell "run-as ${APP_ID} sqlite3 databases/daily_notification_plugin.db \"SELECT COUNT(*) FROM schedules;\"" 2>&1 | tr -d '\r\n')"
|
|
|
|
info " - Schedules in database: ${schedule_count}"
|
|
|
|
if [ "${schedule_count}" -gt "0" ] 2>/dev/null; then
|
|
ok "Invalid test data injected successfully"
|
|
else
|
|
warn "No schedules found after injection - data may not have been inserted"
|
|
fi
|
|
|
|
substep "Step 3: Trigger recovery with invalid data"
|
|
$ADB_BIN shell am start -n "${APP_ID}/.MainActivity" > /dev/null 2>&1
|
|
sleep 3 # Give recovery time to run
|
|
|
|
info "Waiting for recovery to complete..."
|
|
sleep 2
|
|
|
|
# Capture after recovery state
|
|
capture_alarms "test4_after_recovery"
|
|
capture_logcat "test4_after_recovery" "DNP-REACTIVATION|Skipping invalid" 100
|
|
capture_screenshot "test4_after_recovery"
|
|
|
|
substep "Step 4: Check recovery logs for invalid data handling"
|
|
|
|
# Check logs
|
|
local recovery_logs
|
|
recovery_logs="$($ADB_BIN logcat -d | grep -E "DNP-REACTIVATION|Skipping invalid|TEST:" | tail -30)"
|
|
|
|
echo ""
|
|
info "Recovery logs:"
|
|
echo "${recovery_logs}"
|
|
echo ""
|
|
|
|
# Check for invalid data handling
|
|
local test4_passed=false
|
|
local test4_failed=false
|
|
|
|
if echo "${recovery_logs}" | grep -q "Skipping invalid"; then
|
|
ok "Invalid data was detected and skipped"
|
|
echo "${recovery_logs}" | grep "Skipping invalid"
|
|
test4_passed=true
|
|
else
|
|
warn "No 'Skipping invalid' logs found"
|
|
info "This could mean:"
|
|
echo " - Invalid data wasn't injected (database constraints prevented it)"
|
|
echo " - Recovery didn't encounter invalid data"
|
|
echo " - Logs were cleared"
|
|
fi
|
|
|
|
if echo "${recovery_logs}" | grep -q "recovery complete\|Recovery completed"; then
|
|
ok "Recovery completed successfully"
|
|
if [ "${test4_passed}" = "false" ]; then
|
|
test4_passed=true # Recovery completed = passed (even if no invalid data found)
|
|
fi
|
|
else
|
|
warn "Recovery completion message not found in logs"
|
|
fi
|
|
|
|
if echo "${recovery_logs}" | grep -qiE "crash|fatal|exception.*recovery|Failed.*recovery"; then
|
|
error "TEST 4 FAILED: Recovery crashed or threw fatal exception"
|
|
test4_failed=true
|
|
fi
|
|
|
|
# Final verdict
|
|
if [ "${test4_failed}" = "true" ]; then
|
|
verdict_fail "test4_invalid_data_handling" "Recovery crashed or threw fatal exception when handling invalid data"
|
|
elif [ "${test4_passed}" = "true" ]; then
|
|
verdict_pass "test4_invalid_data_handling" "Recovery handled invalid data gracefully (or no invalid data found)"
|
|
else
|
|
verdict_warn "test4_invalid_data_handling" "Could not verify invalid data handling"
|
|
fi
|
|
|
|
else
|
|
info "App is not debuggable - cannot inject data via database"
|
|
info "TEST 4: Code review confirms invalid data handling exists"
|
|
info " - ReactivationManager.kt checks for empty IDs"
|
|
info " - Errors are logged but don't crash recovery"
|
|
verdict_warn "test4_invalid_data_handling" "App not debuggable - cannot inject invalid data for testing"
|
|
fi
|
|
|
|
evidence_block "test4_invalid_data_handling"
|
|
|
|
if [ "${MODE}" != "ci" ]; then
|
|
pause
|
|
fi
|
|
fi # End of "if should_run_test 4" block
|
|
|
|
# ============================================
|
|
# Summary
|
|
# ============================================
|
|
section "Testing Complete"
|
|
|
|
info "Test Results Summary:"
|
|
echo ""
|
|
echo "All test verdicts are shown above with evidence locations."
|
|
echo "Review evidence in: $(get_run_dir)"
|
|
echo ""
|
|
|
|
info "All recovery logs:"
|
|
echo ""
|
|
$ADB_BIN logcat -d | grep "$REACTIVATION_TAG" | tail -20
|
|
echo ""
|
|
|
|
ok "Phase 1 testing script complete!"
|
|
echo ""
|
|
echo "Next steps:"
|
|
echo " - Review evidence in: $(get_run_dir)"
|
|
echo " - Verify all test verdicts above"
|
|
echo " - Check database if needed (debuggable app)"
|
|
echo " - Update documentation with test results"
|
|
}
|
|
|
|
# Run main function
|
|
main "$@"
|
|
|