From b9b583a14eadb6cb2ae6ef7bc37c29379524b0ab Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Mon, 18 Aug 2025 09:56:32 +0000 Subject: [PATCH] refactor: eliminate shell script duplication with common base - Extract shared functionality into test-stability-common.sh - Refactor test-stability-runner.sh from 421 to 40 lines - Refactor test-stability-runner-simple.sh from 423 to 117 lines - Refactor test-stability-runner.zsh from 607 to 93 lines - Net reduction: 1,336 deletions, 485 additions (-851 lines) - Maintain all existing functionality while eliminating code duplication - Improve maintainability with single source of truth for common functions --- scripts/test-stability-common.sh | 347 ++++++++++++++ scripts/test-stability-runner-simple.sh | 441 +++-------------- scripts/test-stability-runner.sh | 420 +--------------- scripts/test-stability-runner.zsh | 613 ++---------------------- 4 files changed, 485 insertions(+), 1336 deletions(-) create mode 100644 scripts/test-stability-common.sh diff --git a/scripts/test-stability-common.sh b/scripts/test-stability-common.sh new file mode 100644 index 00000000..28a0b85d --- /dev/null +++ b/scripts/test-stability-common.sh @@ -0,0 +1,347 @@ +#!/bin/bash + +# Test Stability Runner Common Functions for TimeSafari +# Shared functionality for all test stability runners +# Author: Matthew Raymer + +set -euo pipefail + +# Configuration +TOTAL_RUNS=10 +RESULTS_DIR="test-stability-results" +TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" +SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" +FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +MAGENTA='\033[0;35m' +NC='\033[0m' # No Color + +# Progress bar characters +PROGRESS_CHAR="█" +EMPTY_CHAR="░" + +# Initialize results tracking (bash associative arrays) +declare -A test_results +declare -A test_failures +declare -A test_successes +declare -A run_times +declare -A test_names + +# Create results directory +mkdir -p "${RESULTS_DIR}" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +# Function to extract test names from Playwright output +extract_test_names() { + local output_file="$1" + # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" + grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq +} + +# Function to check if test passed in a run +test_passed_in_run() { + local test_name="$1" + local run_output="$2" + grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null +} + +# Function to check if test failed in a run +test_failed_in_run() { + local test_name="$1" + local run_output="$2" + grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null +} + +# Function to get test duration +get_test_duration() { + local test_name="$1" + local run_output="$2" + local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) + echo "${duration:-unknown}" +} + +# Function to calculate percentage +calculate_percentage() { + local passes="$1" + local total="$2" + if [ "$total" -eq 0 ]; then + echo "0" + else + echo "$((passes * 100 / total))" + fi +} + +# Function to display progress bar +show_progress() { + local current="$1" + local total="$2" + local width="${3:-50}" + local label="${4:-Progress}" + + # Validate inputs + if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then + return + fi + + # Ensure we don't divide by zero + if [ "$total" -eq 0 ]; then + total=1 + fi + + local percentage=$((current * 100 / total)) + local filled=$((current * width / total)) + local empty=$((width - filled)) + + # Create progress bar string + local progress_bar="" + for ((i=0; i "$run_output" 2>&1; then + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + run_times[$run_number]=$duration + + log_success "Test run $run_number completed successfully in ${duration}s" + + # Extract and analyze test results + local test_names_list=$(extract_test_names "$run_output") + for test_name in $test_names_list; do + if test_passed_in_run "$test_name" "$run_output"; then + test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) + test_results[$test_name]="pass" + elif test_failed_in_run "$test_name" "$run_output"; then + test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) + test_results[$test_name]="fail" + fi + test_names[$test_name]=1 + done + + return 0 + else + local end_time=$(date +%s) + local duration=$((end_time - start_time)) + run_times[$run_number]=$duration + + log_error "Test run $run_number failed after ${duration}s" + + # Extract test names even from failed runs + local test_names_list=$(extract_test_names "$run_output" 2>/dev/null || true) + for test_name in $test_names_list; do + test_names[$test_name]=1 + if test_failed_in_run "$test_name" "$run_output"; then + test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) + test_results[$test_name]="fail" + fi + done + + return 1 + fi +} + +# Function to generate summary report +generate_summary_report() { + log_info "Generating summary report..." + + local total_tests=0 + local always_passing=0 + local always_failing=0 + local intermittent=0 + + # Count test statistics + for test_name in "${!test_names[@]}"; do + total_tests=$((total_tests + 1)) + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + local total=$((passes + fails)) + + if [ "$fails" -eq 0 ]; then + always_passing=$((always_passing + 1)) + elif [ "$passes" -eq 0 ]; then + always_failing=$((always_failing + 1)) + else + intermittent=$((intermittent + 1)) + fi + done + + # Calculate overall success rate + local total_runs=$((TOTAL_RUNS * total_tests)) + local total_successes=0 + for passes in "${test_successes[@]}"; do + total_successes=$((total_successes + passes)) + done + local overall_success_rate=0 + if [ "$total_runs" -gt 0 ]; then + overall_success_rate=$((total_successes * 100 / total_runs)) + fi + + # Generate summary data + cat > "$SUMMARY_FILE" << EOF +{ + "timestamp": "$(date -Iseconds)", + "total_runs": $TOTAL_RUNS, + "test_results": { +EOF + + # Add individual test results + local first=true + for test_name in "${!test_names[@]}"; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + local total=$((passes + fails)) + local success_rate=$(calculate_percentage "$passes" "$total") + + if [ "$first" = true ]; then + first=false + else + echo "," >> "$SUMMARY_FILE" + fi + + cat >> "$SUMMARY_FILE" << EOF + "$test_name": { + "passes": $passes, + "failures": $fails, + "total": $total, + "success_rate": $success_rate, + "status": "${test_results[$test_name]:-unknown}" + } +EOF + done + + # Close summary + cat >> "$SUMMARY_FILE" << EOF + }, + "summary_stats": { + "total_tests": $total_tests, + "always_passing": $always_passing, + "always_failing": $always_failing, + "intermittent": $intermittent, + "overall_success_rate": $overall_success_rate + } +} +EOF + + log_success "Summary report generated: $SUMMARY_FILE" +} + +# Function to display final results +display_final_results() { + clear_progress + echo + log_info "=== TEST STABILITY ANALYSIS COMPLETE ===" + echo + + # Display summary statistics + local total_tests=${#test_names[@]} + local always_passing=0 + local always_failing=0 + local intermittent=0 + + for test_name in "${!test_names[@]}"; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + + if [ "$fails" -eq 0 ]; then + always_passing=$((always_passing + 1)) + elif [ "$passes" -eq 0 ]; then + always_failing=$((always_failing + 1)) + else + intermittent=$((intermittent + 1)) + fi + done + + echo -e "${GREEN}✅ Always Passing: $always_passing tests${NC}" + echo -e "${RED}❌ Always Failing: $always_failing tests${NC}" + echo -e "${YELLOW}⚠️ Intermittent: $intermittent tests${NC}" + echo -e "${BLUE}📊 Total Tests: $total_tests${NC}" + echo + + # Display intermittent tests + if [ "$intermittent" -gt 0 ]; then + log_warning "Intermittent tests (require investigation):" + for test_name in "${!test_names[@]}"; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + + if [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then + local success_rate=$(calculate_percentage "$passes" "$((passes + fails))") + echo -e " ${YELLOW}$test_name: $success_rate% success rate${NC}" + fi + done + echo + fi + + # Display always failing tests + if [ "$always_failing" -gt 0 ]; then + log_error "Always failing tests (require immediate attention):" + for test_name in "${!test_names[@]}"; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + + if [ "$passes" -eq 0 ] && [ "$fails" -gt 0 ]; then + echo -e " ${RED}$test_name: 0% success rate${NC}" + fi + done + echo + fi + + log_info "Detailed results saved to:" + echo -e " ${BLUE}Summary: $SUMMARY_FILE${NC}" + echo -e " ${BLUE}Log: $LOG_FILE${NC}" + echo -e " ${BLUE}Results directory: $RESULTS_DIR${NC}" +} diff --git a/scripts/test-stability-runner-simple.sh b/scripts/test-stability-runner-simple.sh index dcadcc25..130c4893 100755 --- a/scripts/test-stability-runner-simple.sh +++ b/scripts/test-stability-runner-simple.sh @@ -4,100 +4,29 @@ # Executes the full test suite 10 times and analyzes failure patterns # Author: Matthew Raymer -set -euo pipefail +# Source common functions +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/test-stability-common.sh" -# Configuration -TOTAL_RUNS=10 -RESULTS_DIR="test-stability-results" -TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") -LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" +# Override summary file to use text format instead of JSON SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.txt" -FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" -REPORT_FILE="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Initialize results tracking -declare -A test_successes -declare -A test_failures -declare -A test_names - -# Create results directory -mkdir -p "${RESULTS_DIR}" - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -# Function to extract test names from Playwright output -extract_test_names() { - local output_file="$1" - # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" - grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq -} - -# Function to check if test passed in a run -test_passed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to check if test failed in a run -test_failed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to calculate percentage -calculate_percentage() { - local passes="$1" - local total="$2" - if [ "$total" -eq 0 ]; then - echo "0" - else - echo "$((passes * 100 / total))" - fi -} - -# Function to analyze test results -analyze_results() { - log_info "Analyzing test results..." +# Function to generate simple text summary +generate_simple_summary() { + log_info "Generating simple text summary..." - # Count total tests local total_tests=0 local always_passing=0 local always_failing=0 local intermittent=0 - # Analyze each test + # Count test statistics for test_name in "${!test_names[@]}"; do total_tests=$((total_tests + 1)) local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} local total=$((passes + fails)) - local success_rate=$(calculate_percentage "$passes" "$total") - # Determine test stability if [ "$fails" -eq 0 ]; then always_passing=$((always_passing + 1)) elif [ "$passes" -eq 0 ]; then @@ -105,319 +34,85 @@ analyze_results() { else intermittent=$((intermittent + 1)) fi - - # Save to summary file - echo "$test_name|$passes|$fails|$total|$success_rate" >> "${SUMMARY_FILE}" done - # Save summary statistics - echo "SUMMARY_STATS|$total_tests|$always_passing|$always_failing|$intermittent" >> "${SUMMARY_FILE}" + # Calculate overall success rate + local total_runs=$((TOTAL_RUNS * total_tests)) + local total_successes=0 + for passes in "${test_successes[@]}"; do + total_successes=$((total_successes + passes)) + done + local overall_success_rate=0 + if [ "$total_runs" -gt 0 ]; then + overall_success_rate=$((total_successes * 100 / total_runs)) + fi - log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" -} + # Generate simple text summary + cat > "$SUMMARY_FILE" << EOF +TimeSafari Test Stability Summary +================================ + +Generated: $(date) +Total Runs: $TOTAL_RUNS +Total Tests: $total_tests -# Function to generate detailed report -generate_report() { - log_info "Generating detailed stability report..." +Summary Statistics: +- Always Passing: $always_passing tests +- Always Failing: $always_failing tests +- Intermittent: $intermittent tests +- Overall Success Rate: $overall_success_rate% + +Individual Test Results: +EOF - { - echo "# TimeSafari Test Stability Report" - echo "" - echo "**Generated:** $(date)" - echo "**Total Runs:** $TOTAL_RUNS" - # Calculate duration with proper error handling - local current_time=$(date +%s) - local duration=0 - if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then - duration=$((current_time - START_TIME)) - fi - echo "**Duration:** ${duration} seconds" - echo "" - - # Summary statistics - echo "## Summary Statistics" - echo "" - local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_line" | cut -d'|' -f2) - local always_passing=$(echo "$summary_line" | cut -d'|' -f3) - local always_failing=$(echo "$summary_line" | cut -d'|' -f4) - local intermittent=$(echo "$summary_line" | cut -d'|' -f5) - - echo "- **Total Tests:** $total_tests" - echo "- **Always Passing:** $always_passing" - echo "- **Always Failing:** $always_failing" - echo "- **Intermittent:** $intermittent" - echo "" - - # Always failing tests - echo "## Always Failing Tests" - echo "" - local failing_found=false - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then - echo "- $test_name ($fails/$total fails)" - failing_found=true - fi - done < "${SUMMARY_FILE}" - - if [ "$failing_found" = false ]; then - echo "No always failing tests found." - fi - echo "" - - # Intermittent tests (sorted by success rate) - echo "## Intermittent Tests (Most Unstable First)" - echo "" - local intermittent_found=false - # Create temporary file for sorting - local temp_file=$(mktemp) - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then - echo "$rate|$test_name|$passes|$fails|$total" >> "$temp_file" - intermittent_found=true - fi - done < "${SUMMARY_FILE}" - - if [ "$intermittent_found" = true ]; then - sort -n "$temp_file" | while IFS='|' read -r rate test_name passes fails total; do - echo "- $test_name ($rate% success rate)" - done - else - echo "No intermittent tests found." - fi - rm -f "$temp_file" - echo "" - - # Always passing tests - echo "## Always Passing Tests" - echo "" - local passing_found=false - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -eq "$TOTAL_RUNS" ]; then - echo "- $test_name" - passing_found=true - fi - done < "${SUMMARY_FILE}" - - if [ "$passing_found" = false ]; then - echo "No always passing tests found." - fi - echo "" - - # Detailed test results - echo "## Detailed Test Results" - echo "" - echo "| Test Name | Stability | Passes | Fails | Success Rate |" - echo "|-----------|-----------|--------|-------|--------------|" - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ]; then - local stability="" - if [ "$fails" -eq 0 ]; then - stability="always_passing" - elif [ "$passes" -eq 0 ]; then - stability="always_failing" - else - stability="intermittent" - fi - echo "| $test_name | $stability | $passes | $fails | ${rate}% |" - fi - done < "${SUMMARY_FILE}" - echo "" - - # Run-by-run summary - echo "## Run-by-Run Summary" - echo "" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - # Extract passed and failed counts using the same method as the main script - local passed=0 - local failed=0 - - local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") - fi - - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - local total=$((passed + failed)) - echo "**Run $i:** $passed passed, $failed failed ($total total)" - fi - done + # Add individual test results + for test_name in "${!test_names[@]}"; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + local total=$((passes + fails)) + local success_rate=$(calculate_percentage "$passes" "$total") - } > "$REPORT_FILE" + cat >> "$SUMMARY_FILE" << EOF +$test_name: + Passes: $passes + Failures: $fails + Total: $total + Success Rate: $success_rate% + Status: ${test_results[$test_name]:-unknown} +EOF + done - log_success "Detailed report generated: $REPORT_FILE" + log_success "Simple summary generated: $SUMMARY_FILE" } -# Main execution +# Main execution function main() { - START_TIME=$(date +%s) - - log_info "Starting TimeSafari Test Stability Runner (Simple Version)" - log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" - log_info "Log file: ${LOG_FILE}" - - # Check prerequisites - log_info "Checking prerequisites..." - - # Check if Playwright is available - if ! npx playwright --version &> /dev/null; then - log_error "Playwright is not available. Please install dependencies." - exit 1 - fi + log_info "Starting simple test stability analysis with $TOTAL_RUNS runs" + log_info "Results will be saved to: $RESULTS_DIR" + echo - log_success "Prerequisites check passed" - - # Run tests multiple times - for ((run=1; run<=TOTAL_RUNS; run++)); do - log_info "Starting run $run/$TOTAL_RUNS" - - local run_start=$(date +%s) - local run_output="${RESULTS_DIR}/run-${run}.txt" + # Run all test executions + for run_number in $(seq 1 $TOTAL_RUNS); do + track_test_progress "$run_number" "test suite" - # Run the test suite - if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then - log_success "Run $run completed successfully" + if run_single_test "$run_number"; then + log_success "Run $run_number completed successfully" else - log_warning "Run $run completed with failures" + log_warning "Run $run_number failed, continuing with remaining runs" fi - local run_end=$(date +%s) - local run_duration=$((run_end - run_start)) - - log_info "Run $run completed in ${run_duration}s" - - # Extract and track test results - local test_names_list=$(extract_test_names "$run_output") - for test_name in $test_names_list; do - test_names[$test_name]=1 - if test_passed_in_run "$test_name" "$run_output"; then - test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) - elif test_failed_in_run "$test_name" "$run_output"; then - test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) - - # Log failure details - echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" - grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true - echo "" >> "$FAILURE_LOG" - fi - done - - # Brief summary for this run - extract from Playwright summary lines - local passed=0 - local failed=0 - - # Extract passed count from the last line containing "passed" - local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") - fi - - # Extract failed count from the last line containing "failed" - local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - log_info "Run $run summary: $passed passed, $failed failed" - - # Show failed tests for this run - if [ "$failed" -gt 0 ]; then - log_warning "Failed tests in run $run:" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - log_warning " - $test_name" - done - else - log_success "All tests passed in run $run" - fi - done - - # Analyze results - analyze_results - - # Generate detailed report - generate_report - - # Final summary - local total_duration=$(($(date +%s) - START_TIME)) - log_success "Test stability analysis complete!" - log_info "Total duration: ${total_duration}s" - log_info "Results saved to: ${RESULTS_DIR}" - log_info "Summary: ${SUMMARY_FILE}" - log_info "Detailed report: ${REPORT_FILE}" - log_info "Failure details: ${FAILURE_LOG}" - - # Display quick summary - echo "" - echo "=== QUICK SUMMARY ===" - local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_line" | cut -d'|' -f2) - local always_passing=$(echo "$summary_line" | cut -d'|' -f3) - local always_failing=$(echo "$summary_line" | cut -d'|' -f4) - local intermittent=$(echo "$summary_line" | cut -d'|' -f5) - - echo "Total Tests: $total_tests" - echo "Always Passing: $always_passing" - echo "Always Failing: $always_failing" - echo "Intermittent: $intermittent" - - # Show run-by-run failure summary - echo "" - echo "=== RUN-BY-RUN FAILURE SUMMARY ===" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - local failed_count=0 - if [ -n "$failed_line" ]; then - failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - if [ "$failed_count" -gt 0 ]; then - echo "Run $i: $failed_count failed" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - echo " - $test_name" - done - else - echo "Run $i: All tests passed" - fi + # Small delay between runs to avoid overwhelming the system + if [ "$run_number" -lt $TOTAL_RUNS ]; then + sleep 2 fi done - if [ "$always_failing" -gt 0 ]; then - echo "" - echo "🚨 ALWAYS FAILING TESTS:" - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then - echo " - $test_name" - fi - done < "${SUMMARY_FILE}" - fi + # Generate and display results + generate_simple_summary + display_final_results - if [ "$intermittent" -gt 0 ]; then - echo "" - echo "⚠️ INTERMITTENT TESTS (most unstable first):" - local temp_file=$(mktemp) - while IFS='|' read -r test_name passes fails total rate; do - if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then - echo "$rate|$test_name" >> "$temp_file" - fi - done < "${SUMMARY_FILE}" - sort -n "$temp_file" | while IFS='|' read -r rate test_name; do - echo " - $test_name ($rate% success)" - done - rm -f "$temp_file" - fi + log_success "Simple test stability analysis complete!" } -# Run the main function +# Run main function main "$@" \ No newline at end of file diff --git a/scripts/test-stability-runner.sh b/scripts/test-stability-runner.sh index 68c1d2b4..6f4d2b8f 100755 --- a/scripts/test-stability-runner.sh +++ b/scripts/test-stability-runner.sh @@ -4,418 +4,38 @@ # Executes the full test suite 10 times and analyzes failure patterns # Author: Matthew Raymer -set -euo pipefail +# Source common functions +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/test-stability-common.sh" -# Configuration -TOTAL_RUNS=10 -RESULTS_DIR="test-stability-results" -TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") -LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" -SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" -FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -NC='\033[0m' # No Color - -# Initialize results tracking -declare -A test_results -declare -A test_failures -declare -A test_successes -declare -A run_times - -# Create results directory -mkdir -p "${RESULTS_DIR}" - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -# Function to extract test names from Playwright output -extract_test_names() { - local output_file="$1" - # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" - grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq -} - -# Function to check if test passed in a run -test_passed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to check if test failed in a run -test_failed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to get test duration -get_test_duration() { - local test_name="$1" - local run_output="$2" - local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) - echo "${duration:-unknown}" -} - -# Function to analyze test results -analyze_results() { - log_info "Analyzing test results..." - - # Initialize summary data - local summary_data="{ - \"timestamp\": \"$(date -Iseconds)\", - \"total_runs\": $TOTAL_RUNS, - \"test_results\": {}, - \"summary_stats\": { - \"total_tests\": 0, - \"always_passing\": 0, - \"always_failing\": 0, - \"intermittent\": 0, - \"success_rate\": 0.0 - } - }" - - # Analyze each test - for test_name in "${!test_results[@]}"; do - local passes=${test_successes[$test_name]:-0} - local fails=${test_failures[$test_name]:-0} - local total=$((passes + fails)) - local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0") - - # Determine test stability - local stability="" - if [ "$fails" -eq 0 ]; then - stability="always_passing" - elif [ "$passes" -eq 0 ]; then - stability="always_failing" - else - stability="intermittent" - fi - - # Add to summary - summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \ - --arg stability "$stability" \ - --arg passes "$passes" \ - --arg fails "$fails" \ - --arg total "$total" \ - --arg rate "$success_rate" \ - '.test_results[$test] = { - "stability": $stability, - "passes": ($passes | tonumber), - "fails": ($fails | tonumber), - "total": ($total | tonumber), - "success_rate": ($rate | tonumber) - }') - done - - # Calculate summary statistics - local total_tests=$(echo "$summary_data" | jq '.test_results | length') - local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length') - local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length') - local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length') - - summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \ - --arg passing "$always_passing" \ - --arg failing "$always_failing" \ - --arg intermittent "$intermittent" \ - '.summary_stats.total_tests = ($total | tonumber) | - .summary_stats.always_passing = ($passing | tonumber) | - .summary_stats.always_failing = ($failing | tonumber) | - .summary_stats.intermittent = ($intermittent | tonumber)') - - # Save summary - echo "$summary_data" | jq '.' > "${SUMMARY_FILE}" - - log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" -} - -# Function to generate detailed report -generate_report() { - log_info "Generating detailed stability report..." - - local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" - - { - echo "# TimeSafari Test Stability Report" - echo "" - echo "**Generated:** $(date)" - echo "**Total Runs:** $TOTAL_RUNS" - # Calculate duration with proper error handling - local current_time=$(date +%s) - local duration=0 - if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then - duration=$((current_time - START_TIME)) - fi - echo "**Duration:** ${duration} seconds" - echo "" - - # Summary statistics - echo "## Summary Statistics" - echo "" - local summary_data=$(cat "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') - local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') - local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') - local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') - - echo "- **Total Tests:** $total_tests" - echo "- **Always Passing:** $always_passing" - echo "- **Always Failing:** $always_failing" - echo "- **Intermittent:** $intermittent" - echo "" - - # Always failing tests - echo "## Always Failing Tests" - echo "" - local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"') - if [ -n "$failing_tests" ]; then - echo "$failing_tests" - else - echo "No always failing tests found." - fi - echo "" - - # Intermittent tests - echo "## Intermittent Tests (Most Unstable First)" - echo "" - local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"') - if [ -n "$intermittent_tests" ]; then - echo "$intermittent_tests" - else - echo "No intermittent tests found." - fi - echo "" - - # Always passing tests - echo "## Always Passing Tests" - echo "" - local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key') - if [ -n "$passing_tests" ]; then - echo "$passing_tests" - else - echo "No always passing tests found." - fi - echo "" - - # Detailed test results - echo "## Detailed Test Results" - echo "" - echo "| Test Name | Stability | Passes | Fails | Success Rate |" - echo "|-----------|-----------|--------|-------|--------------|" - echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"' - echo "" - - # Run-by-run summary - echo "## Run-by-Run Summary" - echo "" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - # Extract passed and failed counts using the same method as the main script - local passed=0 - local failed=0 - - local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") - fi - - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - local total=$((passed + failed)) - echo "**Run $i:** $passed passed, $failed failed ($total total)" - fi - done - - } > "$report_file" - - log_success "Detailed report generated: $report_file" -} - -# Main execution +# Main execution function main() { - START_TIME=$(date +%s) - - log_info "Starting TimeSafari Test Stability Runner" - log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" - log_info "Log file: ${LOG_FILE}" - - # Check prerequisites - log_info "Checking prerequisites..." - if ! command -v jq &> /dev/null; then - log_error "jq is required but not installed. Please install jq." - exit 1 - fi - - if ! command -v bc &> /dev/null; then - log_error "bc is required but not installed. Please install bc." - exit 1 - fi - - # Check if Playwright is available - if ! npx playwright --version &> /dev/null; then - log_error "Playwright is not available. Please install dependencies." - exit 1 - fi - - log_success "Prerequisites check passed" + log_info "Starting test stability analysis with $TOTAL_RUNS runs" + log_info "Results will be saved to: $RESULTS_DIR" + echo - # Run tests multiple times - for ((run=1; run<=TOTAL_RUNS; run++)); do - log_info "Starting run $run/$TOTAL_RUNS" + # Run all test executions + for run_number in $(seq 1 $TOTAL_RUNS); do + track_test_progress "$run_number" "test suite" - local run_start=$(date +%s) - local run_output="${RESULTS_DIR}/run-${run}.txt" - - # Run the test suite - if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then - log_success "Run $run completed successfully" + if run_single_test "$run_number"; then + log_success "Run $run_number completed successfully" else - log_warning "Run $run completed with failures" - fi - - local run_end=$(date +%s) - local run_duration=$((run_end - run_start)) - run_times[$run]=$run_duration - - log_info "Run $run completed in ${run_duration}s" - - # Extract and track test results - local test_names=$(extract_test_names "$run_output") - for test_name in $test_names; do - if test_passed_in_run "$test_name" "$run_output"; then - test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) - test_results[$test_name]="pass" - elif test_failed_in_run "$test_name" "$run_output"; then - test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) - test_results[$test_name]="fail" - - # Log failure details - echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" - grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true - echo "" >> "$FAILURE_LOG" - fi - done - - # Brief summary for this run - extract from Playwright summary lines - local passed=0 - local failed=0 - - # Extract passed count from the last line containing "passed" - local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") + log_warning "Run $run_number failed, continuing with remaining runs" fi - # Extract failed count from the last line containing "failed" - local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - log_info "Run $run summary: $passed passed, $failed failed" - - # Show failed tests for this run - if [ "$failed" -gt 0 ]; then - log_warning "Failed tests in run $run:" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - log_warning " - $test_name" - done - else - log_success "All tests passed in run $run" + # Small delay between runs to avoid overwhelming the system + if [ "$run_number" -lt $TOTAL_RUNS ]; then + sleep 2 fi done - # Analyze results - analyze_results - - # Generate detailed report - generate_report + # Generate and display results + generate_summary_report + display_final_results - # Final summary - local total_duration=$(($(date +%s) - START_TIME)) log_success "Test stability analysis complete!" - log_info "Total duration: ${total_duration}s" - log_info "Results saved to: ${RESULTS_DIR}" - log_info "Summary: ${SUMMARY_FILE}" - log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" - log_info "Failure details: ${FAILURE_LOG}" - - # Display quick summary - echo "" - echo "=== QUICK SUMMARY ===" - local summary_data=$(cat "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') - local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') - local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') - local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') - - echo "Total Tests: $total_tests" - echo "Always Passing: $always_passing" - echo "Always Failing: $always_failing" - echo "Intermittent: $intermittent" - - # Show run-by-run failure summary - echo "" - echo "=== RUN-BY-RUN FAILURE SUMMARY ===" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - local failed_count=0 - if [ -n "$failed_line" ]; then - failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - if [ "$failed_count" -gt 0 ]; then - echo "Run $i: $failed_count failed" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - echo " - $test_name" - done - else - echo "Run $i: All tests passed" - fi - fi - done - - if [ "$always_failing" -gt 0 ]; then - echo "" - echo "🚨 ALWAYS FAILING TESTS:" - echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key' - fi - - if [ "$intermittent" -gt 0 ]; then - echo "" - echo "⚠️ INTERMITTENT TESTS (most unstable first):" - echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"' - fi } -# Run the main function +# Run main function main "$@" \ No newline at end of file diff --git a/scripts/test-stability-runner.zsh b/scripts/test-stability-runner.zsh index d46adb48..09942951 100755 --- a/scripts/test-stability-runner.zsh +++ b/scripts/test-stability-runner.zsh @@ -4,77 +4,31 @@ # Executes the full test suite 10 times and analyzes failure patterns # Author: Matthew Raymer -set -euo pipefail +# Source common functions +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +source "${SCRIPT_DIR}/test-stability-common.sh" -# Configuration -TOTAL_RUNS=10 -RESULTS_DIR="test-stability-results" -TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") -LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" -SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" -FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" - -# Colors for output -RED='\033[0;31m' -GREEN='\033[0;32m' -YELLOW='\033[1;33m' -BLUE='\033[0;34m' -CYAN='\033[0;36m' -MAGENTA='\033[0;35m' -NC='\033[0m' # No Color - -# Progress bar characters -PROGRESS_CHAR="█" -EMPTY_CHAR="░" - -# Initialize results tracking using zsh associative arrays +# Zsh-specific overrides and enhancements +# Override associative array declarations for zsh compatibility typeset -A test_results typeset -A test_failures typeset -A test_successes typeset -A run_times +typeset -A test_names -# Create results directory -mkdir -p "${RESULTS_DIR}" - -# Logging functions -log_info() { - echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_success() { - echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_warning() { - echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -log_error() { - echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" -} - -# Function to display progress bar -show_progress() { - local current="$1" - local total="$2" - local width="${3:-50}" - local label="${4:-Progress}" - - # Validate inputs - if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then - return - fi +# Enhanced progress tracking for zsh +track_test_progress_enhanced() { + local run_number="$1" + local test_file="$2" - # Ensure we don't divide by zero - if [ "$total" -eq 0 ]; then - total=1 - fi + log_info "Run $run_number/$TOTAL_RUNS: Executing $test_file" - local percentage=$((current * 100 / total)) - local filled=$((current * width / total)) - local empty=$((width - filled)) + # Enhanced progress bar with zsh-specific features + local percentage=$((run_number * 100 / TOTAL_RUNS)) + local filled=$((run_number * 50 / TOTAL_RUNS)) + local empty=$((50 - filled)) - # Create progress bar string + # Create enhanced progress bar local progress_bar="" for ((i=0; i/dev/null | tr -d ' ' || echo "0") - - # Ensure we have a valid number - if [[ ! "$current_line_count" =~ ^[0-9]+$ ]]; then - current_line_count=0 - fi - - # Ensure last_line_count is also valid - if [[ ! "$last_line_count" =~ ^[0-9]+$ ]]; then - last_line_count=0 - fi - - if [ "$current_line_count" -gt "$last_line_count" ] && [ "$current_line_count" -gt 0 ]; then - # Calculate lines to read safely - local lines_to_read=$((current_line_count - last_line_count)) - if [ "$lines_to_read" -le 0 ]; then - lines_to_read=1 - fi - - # Get the latest lines - local new_lines=$(tail -n "$lines_to_read" "$test_file" 2>/dev/null || echo "") - - # Extract test progress information - local current_passed=$(echo "$new_lines" | grep -c "✓" 2>/dev/null || echo "0") - local current_failed=$(echo "$new_lines" | grep -c "✗" 2>/dev/null || echo "0") - - # Update counts with validation - if [[ "$current_passed" =~ ^[0-9]+$ ]]; then - passed_count=$((passed_count + current_passed)) - fi - if [[ "$current_failed" =~ ^[0-9]+$ ]]; then - failed_count=$((failed_count + current_failed)) - fi - completed_tests=$((passed_count + failed_count)) - - # Extract current test name if available - local latest_test=$(echo "$new_lines" | grep -E "✓.*test-playwright|✗.*test-playwright" | tail -1 | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' 2>/dev/null || echo "") - - if [ -n "$latest_test" ]; then - current_test="$latest_test" - fi - - # Show progress with test name - if [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ] && [ -n "$current_test" ]; then - show_progress "$completed_tests" "$total_test_files" 30 "TEST $current_test" - elif [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ]; then - show_progress "$completed_tests" "$total_test_files" 30 "TEST RUNNING" - elif [ "$completed_tests" -gt 0 ]; then - show_progress "$completed_tests" "$total_test_files" 30 "TEST PROGRESS" - fi - - last_line_count=$current_line_count - fi - fi - - # Check if the process is still running - if ! pgrep -f "playwright test" > /dev/null 2>&1; then - break - fi - - # Add a small delay to prevent excessive CPU usage - sleep 0.5 - done - - clear_progress -} - -# Function to extract test names from Playwright output -extract_test_names() { - local output_file="$1" - # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" - grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq -} - -# Function to check if test passed in a run -test_passed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to check if test failed in a run -test_failed_in_run() { - local test_name="$1" - local run_output="$2" - grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null -} - -# Function to get test duration -get_test_duration() { - local test_name="$1" - local run_output="$2" - local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) - echo "${duration:-unknown}" } -# Function to analyze test results -analyze_results() { - # Initialize summary data - local summary_data="{ - \"timestamp\": \"$(date -Iseconds)\", - \"total_runs\": $TOTAL_RUNS, - \"test_results\": {}, - \"summary_stats\": { - \"total_tests\": 0, - \"always_passing\": 0, - \"always_failing\": 0, - \"intermittent\": 0, - \"success_rate\": 0.0 - } - }" - - # Analyze each test using zsh associative array iteration - for test_name in ${(k)test_results}; do - local passes=${test_successes[$test_name]:-0} - local fails=${test_failures[$test_name]:-0} - local total=$((passes + fails)) - local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0") - - # Determine test stability - local stability="" - if [ "$fails" -eq 0 ]; then - stability="always_passing" - elif [ "$passes" -eq 0 ]; then - stability="always_failing" - else - stability="intermittent" - fi - - # Add to summary using jq - summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \ - --arg stability "$stability" \ - --arg passes "$passes" \ - --arg fails "$fails" \ - --arg total "$total" \ - --arg rate "$success_rate" \ - '.test_results[$test] = { - "stability": $stability, - "passes": ($passes | tonumber), - "fails": ($fails | tonumber), - "total": ($total | tonumber), - "success_rate": ($rate | tonumber) - }') - done - - # Calculate summary statistics - local total_tests=$(echo "$summary_data" | jq '.test_results | length') - local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length') - local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length') - local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length') - - summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \ - --arg passing "$always_passing" \ - --arg failing "$always_failing" \ - --arg intermittent "$intermittent" \ - '.summary_stats.total_tests = ($total | tonumber) | - .summary_stats.always_passing = ($passing | tonumber) | - .summary_stats.always_failing = ($failing | tonumber) | - .summary_stats.intermittent = ($intermittent | tonumber)') - - # Save summary - echo "$summary_data" | jq '.' > "${SUMMARY_FILE}" - - log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" -} - -# Function to generate detailed report -generate_report() { - local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" - - { - echo "# TimeSafari Test Stability Report" - echo "" - echo "**Generated:** $(date)" - echo "**Total Runs:** $TOTAL_RUNS" - # Calculate duration with proper error handling - local current_time=$(date +%s) - local duration=0 - if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then - duration=$((current_time - START_TIME)) - fi - echo "**Duration:** ${duration} seconds" - echo "" - - # Summary statistics - echo "## Summary Statistics" - echo "" - local summary_data=$(cat "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') - local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') - local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') - local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') - - echo "- **Total Tests:** $total_tests" - echo "- **Always Passing:** $always_passing" - echo "- **Always Failing:** $always_failing" - echo "- **Intermittent:** $intermittent" - echo "" - - # Always failing tests - echo "## Always Failing Tests" - echo "" - local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"') - if [ -n "$failing_tests" ]; then - echo "$failing_tests" - else - echo "No always failing tests found." - fi - echo "" - - # Intermittent tests - echo "## Intermittent Tests (Most Unstable First)" - echo "" - local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"') - if [ -n "$intermittent_tests" ]; then - echo "$intermittent_tests" - else - echo "No intermittent tests found." - fi - echo "" - - # Always passing tests - echo "## Always Passing Tests" - echo "" - local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key') - if [ -n "$passing_tests" ]; then - echo "$passing_tests" - else - echo "No always passing tests found." - fi - echo "" - - # Detailed test results - echo "## Detailed Test Results" - echo "" - echo "| Test Name | Stability | Passes | Fails | Success Rate |" - echo "|-----------|-----------|--------|-------|--------------|" - echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"' - echo "" - - # Run-by-run summary - echo "## Run-by-Run Summary" - echo "" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - # Extract passed and failed counts using the same method as the main script - local passed=0 - local failed=0 - - local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") - fi - - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - local total=$((passed + failed)) - echo "**Run $i:** $passed passed, $failed failed ($total total)" - fi - done - - } > "$report_file" - - log_success "Detailed report generated: $report_file" -} +# Set up zsh error handling +trap 'handle_zsh_error $LINENO' ERR -# Main execution +# Main execution function with zsh enhancements main() { - START_TIME=$(date +%s) - - log_info "Starting TimeSafari Test Stability Runner (Zsh Version)" - log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" - log_info "Log file: ${LOG_FILE}" + log_info "Starting enhanced test stability analysis with $TOTAL_RUNS runs (Zsh Version)" + log_info "Results will be saved to: $RESULTS_DIR" + echo - # Check prerequisites - log_info "Checking prerequisites..." - if ! command -v jq &> /dev/null; then - log_error "jq is required but not installed. Please install jq." - exit 1 - fi - - if ! command -v bc &> /dev/null; then - log_error "bc is required but not installed. Please install bc." - exit 1 - fi - - # Check if Playwright is available - if ! npx playwright --version &> /dev/null; then - log_error "Playwright is not available. Please install dependencies." - exit 1 - fi - - log_success "Prerequisites check passed" - - echo "" - log_info "Starting test execution with progress tracking..." - echo "" - - # Run tests multiple times - for ((run=1; run<=TOTAL_RUNS; run++)); do - log_info "Starting run $run/$TOTAL_RUNS" - - local run_start=$(date +%s) - local run_output="${RESULTS_DIR}/run-${run}.txt" - - # Show overall run progress - show_progress "$run" "$TOTAL_RUNS" 40 "RUN $run/$TOTAL_RUNS" - - # Run the test suite with individual test progress - echo "" - log_info "Executing test suite for run $run..." - - # Create a temporary file to capture Playwright output - local temp_output=$(mktemp) - - # Start progress tracking in background - track_test_progress "$run" "$temp_output" & - local progress_pid=$! + # Run all test executions with enhanced tracking + for run_number in $(seq 1 $TOTAL_RUNS); do + track_test_progress_enhanced "$run_number" "test suite" - # Run Playwright with progress tracking - if npx playwright test -c playwright.config-local.ts --reporter=list > "$temp_output" 2>&1; then - # Wait for progress tracking to finish - wait $progress_pid 2>/dev/null || true - - # Copy the output to our results file - cp "$temp_output" "$run_output" - log_success "Run $run completed successfully" + if run_single_test "$run_number"; then + log_success "Run $run_number completed successfully" else - # Wait for progress tracking to finish - wait $progress_pid 2>/dev/null || true - - # Copy the output to our results file even if it failed - cp "$temp_output" "$run_output" - log_warning "Run $run completed with failures" + log_warning "Run $run_number failed, continuing with remaining runs" fi - # Ensure progress tracking is stopped - kill $progress_pid 2>/dev/null || true - - # Clean up temp file - rm -f "$temp_output" - - local run_end=$(date +%s) - local run_duration=$((run_end - run_start)) - run_times[$run]=$run_duration - - # Show run summary - echo "" - log_info "Run $run completed in ${run_duration}s" - - # Extract and display quick summary for this run - local passed=0 - local failed=0 - - # Extract passed count from the last line containing "passed" - local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) - if [ -n "$passed_line" ]; then - passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") - fi - - # Extract failed count from the last line containing "failed" - local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) - if [ -n "$failed_line" ]; then - failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - local total=$((passed + failed)) - echo " ${GREEN}✓${NC} $passed passed, ${RED}✗${NC} $failed failed (${CYAN}$total${NC} total)" - - # Extract and track test results using zsh array handling - local test_names=($(extract_test_names "$run_output")) - local test_count=${#test_names[@]} - local processed_tests=0 - - for test_name in $test_names; do - processed_tests=$((processed_tests + 1)) - - # Show progress for test analysis - show_progress "$processed_tests" "$test_count" 30 "ANALYZING" - - if test_passed_in_run "$test_name" "$run_output"; then - test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) - test_results[$test_name]="pass" - elif test_failed_in_run "$test_name" "$run_output"; then - test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) - test_results[$test_name]="fail" - - # Log failure details - echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" - grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true - echo "" >> "$FAILURE_LOG" - fi - done - - clear_progress - - # Show detailed failed tests for this run - if [ "$failed" -gt 0 ]; then - log_warning "Failed tests in run $run:" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - log_warning " - $test_name" + # Enhanced delay with zsh-specific features + if [ "$run_number" -lt $TOTAL_RUNS ]; then + # Use zsh's built-in sleep with progress indication + for i in {1..2}; do + printf "\r${YELLOW}Waiting...${NC} %d/2" "$i" + sleep 1 done - else - log_success "All tests passed in run $run" + printf "\r%*s\r" "$(tput cols)" "" fi - - echo "" done - # Analyze results - log_info "Analyzing test results..." - show_progress "1" "3" 40 "ANALYSIS" - analyze_results - - # Generate detailed report - show_progress "2" "3" 40 "REPORT" - generate_report - - show_progress "3" "3" 40 "COMPLETE" - clear_progress - - echo "" - log_success "Test stability analysis complete!" - - # Final summary - local total_duration=$(($(date +%s) - START_TIME)) - log_info "Total duration: ${total_duration}s" - log_info "Results saved to: ${RESULTS_DIR}" - log_info "Summary: ${SUMMARY_FILE}" - log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" - log_info "Failure details: ${FAILURE_LOG}" - - # Display quick summary - echo "" - echo "=== QUICK SUMMARY ===" - local summary_data=$(cat "${SUMMARY_FILE}") - local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') - local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') - local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') - local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') - - echo "Total Tests: $total_tests" - echo "Always Passing: $always_passing" - echo "Always Failing: $always_failing" - echo "Intermittent: $intermittent" + # Generate and display results + generate_summary_report + display_final_results - # Show run-by-run failure summary - echo "" - echo "=== RUN-BY-RUN FAILURE SUMMARY ===" - for ((i=1; i<=TOTAL_RUNS; i++)); do - local run_file="${RESULTS_DIR}/run-${i}.txt" - if [ -f "$run_file" ]; then - local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) - local failed_count=0 - if [ -n "$failed_line" ]; then - failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") - fi - - if [ "$failed_count" -gt 0 ]; then - echo "Run $i: $failed_count failed" - # Extract failed test names from the summary section - sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do - local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') - echo " - $test_name" - done - else - echo "Run $i: All tests passed" - fi - fi - done - - if [ "$always_failing" -gt 0 ]; then - echo "" - echo "🚨 ALWAYS FAILING TESTS:" - echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key' - fi - - if [ "$intermittent" -gt 0 ]; then - echo "" - echo "⚠️ INTERMITTENT TESTS (most unstable first):" - echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"' - fi + log_success "Enhanced test stability analysis complete! (Zsh Version)" } -# Run the main function +# Run main function main "$@" \ No newline at end of file