crowd-funder-for-time-pwa/scripts/test-stability-common.sh

#!/bin/bash

# Test Stability Runner Common Functions for TimeSafari
# Shared functionality for all test stability runners
# Author: Matthew Raymer

set -euo pipefail

# Configuration
TOTAL_RUNS=10
RESULTS_DIR="test-stability-results"
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"

# Colors for output
RED='\033[0;31m'
GREEN='\033[0;32m'
YELLOW='\033[1;33m'
BLUE='\033[0;34m'
CYAN='\033[0;36m'
MAGENTA='\033[0;35m'
NC='\033[0m' # No Color

# Progress bar characters
PROGRESS_CHAR="█"
EMPTY_CHAR="░"

# Initialize results tracking (bash associative arrays)
declare -A test_results
declare -A test_failures
declare -A test_successes
declare -A run_times
declare -A test_names

# Create results directory
mkdir -p "${RESULTS_DIR}"

# Logging functions
log_info() {
    echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
}

log_success() {
    echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
}

log_warning() {
    echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
}

log_error() {
    echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
}

# Function to extract test names from Playwright output
extract_test_names() {
    local output_file="$1"
    # Extract test names from lines like "✓  13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
    grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
}

# Function to check if test passed in a run
test_passed_in_run() {
    local test_name="$1"
    local run_output="$2"
    grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
}

# Function to check if test failed in a run
test_failed_in_run() {
    local test_name="$1"
    local run_output="$2"
    grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
}

# Function to get test duration
get_test_duration() {
    local test_name="$1"
    local run_output="$2"
    local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
    echo "${duration:-unknown}"
}

# Function to calculate percentage
calculate_percentage() {
    local passes="$1"
    local total="$2"
    if [ "$total" -eq 0 ]; then
        echo "0"
    else
        echo "$((passes * 100 / total))"
    fi
}

# Function to display progress bar
show_progress() {
    local current="$1"
    local total="$2"
    local width="${3:-50}"
    local label="${4:-Progress}"
    
    # Validate inputs
    if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then
        return
    fi
    
    # Ensure we don't divide by zero
    if [ "$total" -eq 0 ]; then
        total=1
    fi
    
    local percentage=$((current * 100 / total))
    local filled=$((current * width / total))
    local empty=$((width - filled))
    
    # Create progress bar string
    local progress_bar=""
    for ((i=0; i<filled; i++)); do
        progress_bar+="$PROGRESS_CHAR"
    done
    for ((i=0; i<empty; i++)); do
        progress_bar+="$EMPTY_CHAR"
    done
    
    # Print progress bar with carriage return to overwrite
    printf "\r${CYAN}[%s]${NC} %s [%s] %d%% (%d/%d)" \
        "$label" "$progress_bar" "$percentage" "$current" "$total"
}

# Function to clear progress bar line
clear_progress() {
    printf "\r%*s\r" "$(tput cols)" ""
}

# Function to track test execution progress
track_test_progress() {
    local run_number="$1"
    local test_file="$2"
    
    log_info "Run $run_number/$TOTAL_RUNS: Executing $test_file"
    show_progress "$run_number" "$TOTAL_RUNS" 50 "Test Run"
}

# Function to run a single test execution
run_single_test() {
    local run_number="$1"
    local run_output="${RESULTS_DIR}/run-${run_number}.txt"
    local start_time=$(date +%s)
    
    log_info "Starting test run $run_number/$TOTAL_RUNS"
    
    # Run the test suite
    if npm run test:playwright > "$run_output" 2>&1; then
        local end_time=$(date +%s)
        local duration=$((end_time - start_time))
        run_times[$run_number]=$duration
        
        log_success "Test run $run_number completed successfully in ${duration}s"
        
        # Extract and analyze test results
        local test_names_list=$(extract_test_names "$run_output")
        for test_name in $test_names_list; do
            if test_passed_in_run "$test_name" "$run_output"; then
                test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
                test_results[$test_name]="pass"
            elif test_failed_in_run "$test_name" "$run_output"; then
                test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
                test_results[$test_name]="fail"
            fi
            test_names[$test_name]=1
        done
        
        return 0
    else
        local end_time=$(date +%s)
        local duration=$((end_time - start_time))
        run_times[$run_number]=$duration
        
        log_error "Test run $run_number failed after ${duration}s"
        
        # Extract test names even from failed runs
        local test_names_list=$(extract_test_names "$run_output" 2>/dev/null || true)
        for test_name in $test_names_list; do
            test_names[$test_name]=1
            if test_failed_in_run "$test_name" "$run_output"; then
                test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
                test_results[$test_name]="fail"
            fi
        done
        
        return 1
    fi
}

# Function to generate summary report
generate_summary_report() {
    log_info "Generating summary report..."
    
    local total_tests=0
    local always_passing=0
    local always_failing=0
    local intermittent=0
    
    # Count test statistics
    for test_name in "${!test_names[@]}"; do
        total_tests=$((total_tests + 1))
        local passes=${test_successes[$test_name]:-0}
        local fails=${test_failures[$test_name]:-0}
        local total=$((passes + fails))
        
        if [ "$fails" -eq 0 ]; then
            always_passing=$((always_passing + 1))
        elif [ "$passes" -eq 0 ]; then
            always_failing=$((always_failing + 1))
        else
            intermittent=$((intermittent + 1))
        fi
    done
    
    # Calculate overall success rate
    local total_runs=$((TOTAL_RUNS * total_tests))
    local total_successes=0
    for passes in "${test_successes[@]}"; do
        total_successes=$((total_successes + passes))
    done
    local overall_success_rate=0
    if [ "$total_runs" -gt 0 ]; then
        overall_success_rate=$((total_successes * 100 / total_runs))
    fi
    
    # Generate summary data
    cat > "$SUMMARY_FILE" << EOF
{
    "timestamp": "$(date -Iseconds)",
    "total_runs": $TOTAL_RUNS,
    "test_results": {
EOF
    
    # Add individual test results
    local first=true
    for test_name in "${!test_names[@]}"; do
        local passes=${test_successes[$test_name]:-0}
        local fails=${test_failures[$test_name]:-0}
        local total=$((passes + fails))
        local success_rate=$(calculate_percentage "$passes" "$total")
        
        if [ "$first" = true ]; then
            first=false
        else
            echo "," >> "$SUMMARY_FILE"
        fi
        
        cat >> "$SUMMARY_FILE" << EOF
        "$test_name": {
            "passes": $passes,
            "failures": $fails,
            "total": $total,
            "success_rate": $success_rate,
            "status": "${test_results[$test_name]:-unknown}"
        }
EOF
    done
    
    # Close summary
    cat >> "$SUMMARY_FILE" << EOF
    },
    "summary_stats": {
        "total_tests": $total_tests,
        "always_passing": $always_passing,
        "always_failing": $always_failing,
        "intermittent": $intermittent,
        "overall_success_rate": $overall_success_rate
    }
}
EOF
    
    log_success "Summary report generated: $SUMMARY_FILE"
}

# Function to display final results
display_final_results() {
    clear_progress
    echo
    log_info "=== TEST STABILITY ANALYSIS COMPLETE ==="
    echo
    
    # Display summary statistics
    local total_tests=${#test_names[@]}
    local always_passing=0
    local always_failing=0
    local intermittent=0
    
    for test_name in "${!test_names[@]}"; do
        local passes=${test_successes[$test_name]:-0}
        local fails=${test_failures[$test_name]:-0}
        
        if [ "$fails" -eq 0 ]; then
            always_passing=$((always_passing + 1))
        elif [ "$passes" -eq 0 ]; then
            always_failing=$((always_failing + 1))
        else
            intermittent=$((intermittent + 1))
        fi
    done
    
    echo -e "${GREEN}✅ Always Passing: $always_passing tests${NC}"
    echo -e "${RED}❌ Always Failing: $always_failing tests${NC}"
    echo -e "${YELLOW}⚠️  Intermittent: $intermittent tests${NC}"
    echo -e "${BLUE}📊 Total Tests: $total_tests${NC}"
    echo
    
    # Display intermittent tests
    if [ "$intermittent" -gt 0 ]; then
        log_warning "Intermittent tests (require investigation):"
        for test_name in "${!test_names[@]}"; do
            local passes=${test_successes[$test_name]:-0}
            local fails=${test_failures[$test_name]:-0}
            
            if [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then
                local success_rate=$(calculate_percentage "$passes" "$((passes + fails))")
                echo -e "  ${YELLOW}$test_name: $success_rate% success rate${NC}"
            fi
        done
        echo
    fi
    
    # Display always failing tests
    if [ "$always_failing" -gt 0 ]; then
        log_error "Always failing tests (require immediate attention):"
        for test_name in "${!test_names[@]}"; do
            local passes=${test_successes[$test_name]:-0}
            local fails=${test_failures[$test_name]:-0}
            
            if [ "$passes" -eq 0 ] && [ "$fails" -gt 0 ]; then
                echo -e "  ${RED}$test_name: 0% success rate${NC}"
            fi
        done
        echo
    fi
    
    log_info "Detailed results saved to:"
    echo -e "  ${BLUE}Summary: $SUMMARY_FILE${NC}"
    echo -e "  ${BLUE}Log: $LOG_FILE${NC}"
    echo -e "  ${BLUE}Results directory: $RESULTS_DIR${NC}"
}