#!/bin/bash # Test Stability Runner for TimeSafari (Simple Version) # Executes the full test suite 10 times and analyzes failure patterns # Author: Matthew Raymer set -euo pipefail # Configuration TOTAL_RUNS=10 RESULTS_DIR="test-stability-results" TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.txt" FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" REPORT_FILE="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' # No Color # Initialize results tracking declare -A test_successes declare -A test_failures declare -A test_names # Create results directory mkdir -p "${RESULTS_DIR}" # Logging functions log_info() { echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } # Function to extract test names from Playwright output extract_test_names() { local output_file="$1" # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq } # Function to check if test passed in a run test_passed_in_run() { local test_name="$1" local run_output="$2" grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to check if test failed in a run test_failed_in_run() { local test_name="$1" local run_output="$2" grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to calculate percentage calculate_percentage() { local passes="$1" local total="$2" if [ "$total" -eq 0 ]; then echo "0" else echo "$((passes * 100 / total))" fi } # Function to analyze test results analyze_results() { log_info "Analyzing test results..." # Count total tests local total_tests=0 local always_passing=0 local always_failing=0 local intermittent=0 # Analyze each test for test_name in "${!test_names[@]}"; do total_tests=$((total_tests + 1)) local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} local total=$((passes + fails)) local success_rate=$(calculate_percentage "$passes" "$total") # Determine test stability if [ "$fails" -eq 0 ]; then always_passing=$((always_passing + 1)) elif [ "$passes" -eq 0 ]; then always_failing=$((always_failing + 1)) else intermittent=$((intermittent + 1)) fi # Save to summary file echo "$test_name|$passes|$fails|$total|$success_rate" >> "${SUMMARY_FILE}" done # Save summary statistics echo "SUMMARY_STATS|$total_tests|$always_passing|$always_failing|$intermittent" >> "${SUMMARY_FILE}" log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" } # Function to generate detailed report generate_report() { log_info "Generating detailed stability report..." { echo "# TimeSafari Test Stability Report" echo "" echo "**Generated:** $(date)" echo "**Total Runs:** $TOTAL_RUNS" # Calculate duration with proper error handling local current_time=$(date +%s) local duration=0 if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then duration=$((current_time - START_TIME)) fi echo "**Duration:** ${duration} seconds" echo "" # Summary statistics echo "## Summary Statistics" echo "" local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}") local total_tests=$(echo "$summary_line" | cut -d'|' -f2) local always_passing=$(echo "$summary_line" | cut -d'|' -f3) local always_failing=$(echo "$summary_line" | cut -d'|' -f4) local intermittent=$(echo "$summary_line" | cut -d'|' -f5) echo "- **Total Tests:** $total_tests" echo "- **Always Passing:** $always_passing" echo "- **Always Failing:** $always_failing" echo "- **Intermittent:** $intermittent" echo "" # Always failing tests echo "## Always Failing Tests" echo "" local failing_found=false while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then echo "- $test_name ($fails/$total fails)" failing_found=true fi done < "${SUMMARY_FILE}" if [ "$failing_found" = false ]; then echo "No always failing tests found." fi echo "" # Intermittent tests (sorted by success rate) echo "## Intermittent Tests (Most Unstable First)" echo "" local intermittent_found=false # Create temporary file for sorting local temp_file=$(mktemp) while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then echo "$rate|$test_name|$passes|$fails|$total" >> "$temp_file" intermittent_found=true fi done < "${SUMMARY_FILE}" if [ "$intermittent_found" = true ]; then sort -n "$temp_file" | while IFS='|' read -r rate test_name passes fails total; do echo "- $test_name ($rate% success rate)" done else echo "No intermittent tests found." fi rm -f "$temp_file" echo "" # Always passing tests echo "## Always Passing Tests" echo "" local passing_found=false while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -eq "$TOTAL_RUNS" ]; then echo "- $test_name" passing_found=true fi done < "${SUMMARY_FILE}" if [ "$passing_found" = false ]; then echo "No always passing tests found." fi echo "" # Detailed test results echo "## Detailed Test Results" echo "" echo "| Test Name | Stability | Passes | Fails | Success Rate |" echo "|-----------|-----------|--------|-------|--------------|" while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ]; then local stability="" if [ "$fails" -eq 0 ]; then stability="always_passing" elif [ "$passes" -eq 0 ]; then stability="always_failing" else stability="intermittent" fi echo "| $test_name | $stability | $passes | $fails | ${rate}% |" fi done < "${SUMMARY_FILE}" echo "" # Run-by-run summary echo "## Run-by-Run Summary" echo "" for ((i=1; i<=TOTAL_RUNS; i++)); do local run_file="${RESULTS_DIR}/run-${i}.txt" if [ -f "$run_file" ]; then # Extract passed and failed counts using the same method as the main script local passed=0 local failed=0 local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) if [ -n "$passed_line" ]; then passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") fi local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) if [ -n "$failed_line" ]; then failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi local total=$((passed + failed)) echo "**Run $i:** $passed passed, $failed failed ($total total)" fi done } > "$REPORT_FILE" log_success "Detailed report generated: $REPORT_FILE" } # Main execution main() { START_TIME=$(date +%s) log_info "Starting TimeSafari Test Stability Runner (Simple Version)" log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" log_info "Log file: ${LOG_FILE}" # Check prerequisites log_info "Checking prerequisites..." # Check if Playwright is available if ! npx playwright --version &> /dev/null; then log_error "Playwright is not available. Please install dependencies." exit 1 fi log_success "Prerequisites check passed" # Run tests multiple times for ((run=1; run<=TOTAL_RUNS; run++)); do log_info "Starting run $run/$TOTAL_RUNS" local run_start=$(date +%s) local run_output="${RESULTS_DIR}/run-${run}.txt" # Run the test suite if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then log_success "Run $run completed successfully" else log_warning "Run $run completed with failures" fi local run_end=$(date +%s) local run_duration=$((run_end - run_start)) log_info "Run $run completed in ${run_duration}s" # Extract and track test results local test_names_list=$(extract_test_names "$run_output") for test_name in $test_names_list; do test_names[$test_name]=1 if test_passed_in_run "$test_name" "$run_output"; then test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) elif test_failed_in_run "$test_name" "$run_output"; then test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) # Log failure details echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true echo "" >> "$FAILURE_LOG" fi done # Brief summary for this run - extract from Playwright summary lines local passed=0 local failed=0 # Extract passed count from the last line containing "passed" local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) if [ -n "$passed_line" ]; then passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") fi # Extract failed count from the last line containing "failed" local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) if [ -n "$failed_line" ]; then failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi log_info "Run $run summary: $passed passed, $failed failed" # Show failed tests for this run if [ "$failed" -gt 0 ]; then log_warning "Failed tests in run $run:" # Extract failed test names from the summary section sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') log_warning " - $test_name" done else log_success "All tests passed in run $run" fi done # Analyze results analyze_results # Generate detailed report generate_report # Final summary local total_duration=$(($(date +%s) - START_TIME)) log_success "Test stability analysis complete!" log_info "Total duration: ${total_duration}s" log_info "Results saved to: ${RESULTS_DIR}" log_info "Summary: ${SUMMARY_FILE}" log_info "Detailed report: ${REPORT_FILE}" log_info "Failure details: ${FAILURE_LOG}" # Display quick summary echo "" echo "=== QUICK SUMMARY ===" local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}") local total_tests=$(echo "$summary_line" | cut -d'|' -f2) local always_passing=$(echo "$summary_line" | cut -d'|' -f3) local always_failing=$(echo "$summary_line" | cut -d'|' -f4) local intermittent=$(echo "$summary_line" | cut -d'|' -f5) echo "Total Tests: $total_tests" echo "Always Passing: $always_passing" echo "Always Failing: $always_failing" echo "Intermittent: $intermittent" # Show run-by-run failure summary echo "" echo "=== RUN-BY-RUN FAILURE SUMMARY ===" for ((i=1; i<=TOTAL_RUNS; i++)); do local run_file="${RESULTS_DIR}/run-${i}.txt" if [ -f "$run_file" ]; then local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) local failed_count=0 if [ -n "$failed_line" ]; then failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") fi if [ "$failed_count" -gt 0 ]; then echo "Run $i: $failed_count failed" # Extract failed test names from the summary section sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') echo " - $test_name" done else echo "Run $i: All tests passed" fi fi done if [ "$always_failing" -gt 0 ]; then echo "" echo "🚨 ALWAYS FAILING TESTS:" while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then echo " - $test_name" fi done < "${SUMMARY_FILE}" fi if [ "$intermittent" -gt 0 ]; then echo "" echo "⚠️ INTERMITTENT TESTS (most unstable first):" local temp_file=$(mktemp) while IFS='|' read -r test_name passes fails total rate; do if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then echo "$rate|$test_name" >> "$temp_file" fi done < "${SUMMARY_FILE}" sort -n "$temp_file" | while IFS='|' read -r rate test_name; do echo " - $test_name ($rate% success)" done rm -f "$temp_file" fi } # Run the main function main "$@"