#!/bin/bash # Test Stability Runner Common Functions for TimeSafari # Shared functionality for all test stability runners # Author: Matthew Raymer set -euo pipefail # Configuration TOTAL_RUNS=10 RESULTS_DIR="test-stability-results" TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' CYAN='\033[0;36m' MAGENTA='\033[0;35m' NC='\033[0m' # No Color # Progress bar characters PROGRESS_CHAR="█" EMPTY_CHAR="░" # Initialize results tracking (bash associative arrays) declare -A test_results declare -A test_failures declare -A test_successes declare -A run_times declare -A test_names # Create results directory mkdir -p "${RESULTS_DIR}" # Logging functions log_info() { echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_success() { echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_warning() { echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } log_error() { echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" } # Function to extract test names from Playwright output extract_test_names() { local output_file="$1" # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq } # Function to check if test passed in a run test_passed_in_run() { local test_name="$1" local run_output="$2" grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to check if test failed in a run test_failed_in_run() { local test_name="$1" local run_output="$2" grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null } # Function to get test duration get_test_duration() { local test_name="$1" local run_output="$2" local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) echo "${duration:-unknown}" } # Function to calculate percentage calculate_percentage() { local passes="$1" local total="$2" if [ "$total" -eq 0 ]; then echo "0" else echo "$((passes * 100 / total))" fi } # Function to display progress bar show_progress() { local current="$1" local total="$2" local width="${3:-50}" local label="${4:-Progress}" # Validate inputs if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then return fi # Ensure we don't divide by zero if [ "$total" -eq 0 ]; then total=1 fi local percentage=$((current * 100 / total)) local filled=$((current * width / total)) local empty=$((width - filled)) # Create progress bar string local progress_bar="" for ((i=0; i "$run_output" 2>&1; then local end_time=$(date +%s) local duration=$((end_time - start_time)) run_times[$run_number]=$duration log_success "Test run $run_number completed successfully in ${duration}s" # Extract and analyze test results local test_names_list=$(extract_test_names "$run_output") for test_name in $test_names_list; do if test_passed_in_run "$test_name" "$run_output"; then test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) test_results[$test_name]="pass" elif test_failed_in_run "$test_name" "$run_output"; then test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) test_results[$test_name]="fail" fi test_names[$test_name]=1 done return 0 else local end_time=$(date +%s) local duration=$((end_time - start_time)) run_times[$run_number]=$duration log_error "Test run $run_number failed after ${duration}s" # Extract test names even from failed runs local test_names_list=$(extract_test_names "$run_output" 2>/dev/null || true) for test_name in $test_names_list; do test_names[$test_name]=1 if test_failed_in_run "$test_name" "$run_output"; then test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) test_results[$test_name]="fail" fi done return 1 fi } # Function to generate summary report generate_summary_report() { log_info "Generating summary report..." local total_tests=0 local always_passing=0 local always_failing=0 local intermittent=0 # Count test statistics for test_name in "${!test_names[@]}"; do total_tests=$((total_tests + 1)) local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} local total=$((passes + fails)) if [ "$fails" -eq 0 ]; then always_passing=$((always_passing + 1)) elif [ "$passes" -eq 0 ]; then always_failing=$((always_failing + 1)) else intermittent=$((intermittent + 1)) fi done # Calculate overall success rate local total_runs=$((TOTAL_RUNS * total_tests)) local total_successes=0 for passes in "${test_successes[@]}"; do total_successes=$((total_successes + passes)) done local overall_success_rate=0 if [ "$total_runs" -gt 0 ]; then overall_success_rate=$((total_successes * 100 / total_runs)) fi # Generate summary data cat > "$SUMMARY_FILE" << EOF { "timestamp": "$(date -Iseconds)", "total_runs": $TOTAL_RUNS, "test_results": { EOF # Add individual test results local first=true for test_name in "${!test_names[@]}"; do local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} local total=$((passes + fails)) local success_rate=$(calculate_percentage "$passes" "$total") if [ "$first" = true ]; then first=false else echo "," >> "$SUMMARY_FILE" fi cat >> "$SUMMARY_FILE" << EOF "$test_name": { "passes": $passes, "failures": $fails, "total": $total, "success_rate": $success_rate, "status": "${test_results[$test_name]:-unknown}" } EOF done # Close summary cat >> "$SUMMARY_FILE" << EOF }, "summary_stats": { "total_tests": $total_tests, "always_passing": $always_passing, "always_failing": $always_failing, "intermittent": $intermittent, "overall_success_rate": $overall_success_rate } } EOF log_success "Summary report generated: $SUMMARY_FILE" } # Function to display final results display_final_results() { clear_progress echo log_info "=== TEST STABILITY ANALYSIS COMPLETE ===" echo # Display summary statistics local total_tests=${#test_names[@]} local always_passing=0 local always_failing=0 local intermittent=0 for test_name in "${!test_names[@]}"; do local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} if [ "$fails" -eq 0 ]; then always_passing=$((always_passing + 1)) elif [ "$passes" -eq 0 ]; then always_failing=$((always_failing + 1)) else intermittent=$((intermittent + 1)) fi done echo -e "${GREEN}✅ Always Passing: $always_passing tests${NC}" echo -e "${RED}❌ Always Failing: $always_failing tests${NC}" echo -e "${YELLOW}⚠️ Intermittent: $intermittent tests${NC}" echo -e "${BLUE}📊 Total Tests: $total_tests${NC}" echo # Display intermittent tests if [ "$intermittent" -gt 0 ]; then log_warning "Intermittent tests (require investigation):" for test_name in "${!test_names[@]}"; do local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} if [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then local success_rate=$(calculate_percentage "$passes" "$((passes + fails))") echo -e " ${YELLOW}$test_name: $success_rate% success rate${NC}" fi done echo fi # Display always failing tests if [ "$always_failing" -gt 0 ]; then log_error "Always failing tests (require immediate attention):" for test_name in "${!test_names[@]}"; do local passes=${test_successes[$test_name]:-0} local fails=${test_failures[$test_name]:-0} if [ "$passes" -eq 0 ] && [ "$fails" -gt 0 ]; then echo -e " ${RED}$test_name: 0% success rate${NC}" fi done echo fi log_info "Detailed results saved to:" echo -e " ${BLUE}Summary: $SUMMARY_FILE${NC}" echo -e " ${BLUE}Log: $LOG_FILE${NC}" echo -e " ${BLUE}Results directory: $RESULTS_DIR${NC}" }