#!/bin/bash # Test Stability Runner for TimeSafari # Executes the full test suite 10 times and analyzes failure patterns # Author: Matthew Raymer set -euo pipefail # Configuration TOTAL_RUNS=10 RESULTS_DIR="test-stability-results" TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Initialize results tracking declare -A test_results declare -A test_failures declare -A test_successes declare -A run_times # Create results directory mkdir -p "${RESULTS_DIR}" # Logging functions log_info() { echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } # Function to extract test names from Playwright output extract_test_names() { local output_file="$1" # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq } # Function to check if test passed in a run test_passed_in_run() { local test_name="$1" local run_output="$2" grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to check if test failed in a run test_failed_in_run() { local test_name="$1" local run_output="$2" grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to get test duration get_test_duration() { local test_name="$1" local run_output="$2" local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) echo "${duration:-unknown}" } # Function to analyze test results analyze_results() { log_info "Analyzing test results..." # Initialize summary data local summary_data="{ \"timestamp\": \"$(date -Iseconds)\", \"total_runs\": $TOTAL_RUNS, \"test_results\": {}, \"summary_stats\": { \"total_tests\": 0, \"always_passing\": 0, \"always_failing\": 0, \"intermittent\": 0, \"success_rate\": 0.0 } }" # Analyze each test for test_name in "${!test_results[@]}"; do local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} local total=$((passes + fails)) local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0") # Determine test stability local stability="" if [ "$fails" -eq 0 ]; then stability="always_passing" elif [ "$passes" -eq 0 ]; then stability="always_failing" else stability="intermittent" fi # Add to summary summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \ --arg stability "$stability" \ --arg passes "$passes" \ --arg fails "$fails" \ --arg total "$total" \ --arg rate "$success_rate" \ '.test_results[$test] = { "stability": $stability, "passes": ($passes | tonumber), "fails": ($fails | tonumber), "total": ($total | tonumber), "success_rate": ($rate | tonumber) }') done # Calculate summary statistics local total_tests=$(echo "$summary_data" | jq '.test_results | length') local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length') local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length') local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length') summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \ --arg passing "$always_passing" \ --arg failing "$always_failing" \ --arg intermittent "$intermittent" \ '.summary_stats.total_tests = ($total | tonumber) | .summary_stats.always_passing = ($passing | tonumber) | .summary_stats.always_failing = ($failing | tonumber) | .summary_stats.intermittent = ($intermittent | tonumber)') # Save summary echo "$summary_data" | jq '.' > "${SUMMARY_FILE}" log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" } # Function to generate detailed report generate_report() { log_info "Generating detailed stability report..." local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" { echo "# TimeSafari Test Stability Report" echo "" echo "**Generated:** $(date)" echo "**Total Runs:** $TOTAL_RUNS" # Calculate duration with proper error handling local current_time=$(date +%s) local duration=0 if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then duration=$((current_time - START_TIME)) fi echo "**Duration:** ${duration} seconds" echo "" # Summary statistics echo "## Summary Statistics" echo "" local summary_data=$(cat "${SUMMARY_FILE}") local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') echo "- **Total Tests:** $total_tests" echo "- **Always Passing:** $always_passing" echo "- **Always Failing:** $always_failing" echo "- **Intermittent:** $intermittent" echo "" # Always failing tests echo "## Always Failing Tests" echo "" local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"') if [ -n "$failing_tests" ]; then echo "$failing_tests" else echo "No always failing tests found." fi echo "" # Intermittent tests echo "## Intermittent Tests (Most Unstable First)" echo "" local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"') if [ -n "$intermittent_tests" ]; then echo "$intermittent_tests" else echo "No intermittent tests found." fi echo "" # Always passing tests echo "## Always Passing Tests" echo "" local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key') if [ -n "$passing_tests" ]; then echo "$passing_tests" else echo "No always passing tests found." fi echo "" # Detailed test results echo "## Detailed Test Results" echo "" echo "| Test Name | Stability | Passes | Fails | Success Rate |" echo "|-----------|-----------|--------|-------|--------------|" echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"' echo "" # Run-by-run summary echo "## Run-by-Run Summary" echo "" for ((i=1; i<=TOTAL_RUNS; i++)); do local run_file="${RESULTS_DIR}/run-${i}.txt" if [ -f "$run_file" ]; then # Extract passed and failed counts using the same method as the main script local passed=0 local failed=0 local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) if [ -n "$passed_line" ]; then passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") fi local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) if [ -n "$failed_line" ]; then failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi local total=$((passed + failed)) echo "**Run $i:** $passed passed, $failed failed ($total total)" fi done } > "$report_file" log_success "Detailed report generated: $report_file" } # Main execution main() { START_TIME=$(date +%s) log_info "Starting TimeSafari Test Stability Runner" log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" log_info "Log file: ${LOG_FILE}" # Check prerequisites log_info "Checking prerequisites..." if ! command -v jq &> /dev/null; then log_error "jq is required but not installed. Please install jq." exit 1 fi if ! command -v bc &> /dev/null; then log_error "bc is required but not installed. Please install bc." exit 1 fi # Check if Playwright is available if ! npx playwright --version &> /dev/null; then log_error "Playwright is not available. Please install dependencies." exit 1 fi log_success "Prerequisites check passed" # Run tests multiple times for ((run=1; run<=TOTAL_RUNS; run++)); do log_info "Starting run $run/$TOTAL_RUNS" local run_start=$(date +%s) local run_output="${RESULTS_DIR}/run-${run}.txt" # Run the test suite if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then log_success "Run $run completed successfully" else log_warning "Run $run completed with failures" fi local run_end=$(date +%s) local run_duration=$((run_end - run_start)) run_times[$run]=$run_duration log_info "Run $run completed in ${run_duration}s" # Extract and track test results local test_names=$(extract_test_names "$run_output") for test_name in $test_names; do if test_passed_in_run "$test_name" "$run_output"; then test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) test_results[$test_name]="pass" elif test_failed_in_run "$test_name" "$run_output"; then test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) test_results[$test_name]="fail" # Log failure details echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true echo "" >> "$FAILURE_LOG" fi done # Brief summary for this run - extract from Playwright summary lines local passed=0 local failed=0 # Extract passed count from the last line containing "passed" local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) if [ -n "$passed_line" ]; then passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") fi # Extract failed count from the last line containing "failed" local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) if [ -n "$failed_line" ]; then failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi log_info "Run $run summary: $passed passed, $failed failed" # Show failed tests for this run if [ "$failed" -gt 0 ]; then log_warning "Failed tests in run $run:" # Extract failed test names from the summary section sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') log_warning " - $test_name" done else log_success "All tests passed in run $run" fi done # Analyze results analyze_results # Generate detailed report generate_report # Final summary local total_duration=$(($(date +%s) - START_TIME)) log_success "Test stability analysis complete!" log_info "Total duration: ${total_duration}s" log_info "Results saved to: ${RESULTS_DIR}" log_info "Summary: ${SUMMARY_FILE}" log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" log_info "Failure details: ${FAILURE_LOG}" # Display quick summary echo "" echo "=== QUICK SUMMARY ===" local summary_data=$(cat "${SUMMARY_FILE}") local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') echo "Total Tests: $total_tests" echo "Always Passing: $always_passing" echo "Always Failing: $always_failing" echo "Intermittent: $intermittent" # Show run-by-run failure summary echo "" echo "=== RUN-BY-RUN FAILURE SUMMARY ===" for ((i=1; i<=TOTAL_RUNS; i++)); do local run_file="${RESULTS_DIR}/run-${i}.txt" if [ -f "$run_file" ]; then local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) local failed_count=0 if [ -n "$failed_line" ]; then failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi if [ "$failed_count" -gt 0 ]; then echo "Run $i: $failed_count failed" # Extract failed test names from the summary section sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') echo " - $test_name" done else echo "Run $i: All tests passed" fi fi done if [ "$always_failing" -gt 0 ]; then echo "" echo "🚨 ALWAYS FAILING TESTS:" echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key' fi if [ "$intermittent" -gt 0 ]; then echo "" echo "⚠️ INTERMITTENT TESTS (most unstable first):" echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"' fi } # Run the main function main "$@"