From 33ba03d208462b3f6c786026759d803a0bfe08b7 Mon Sep 17 00:00:00 2001 From: Matthew Raymer Date: Tue, 5 Aug 2025 12:19:27 +0000 Subject: [PATCH] Fix math expression errors in Zsh test stability runner - Add input validation for all numeric values before math operations - Implement safe math calculations with zero-division protection - Add error redirection (2>/dev/null) to suppress command errors - Improve process management with proper background process cleanup - Add fallback values when commands return invalid output - Fix progress bar display with better validation and error handling - Ensure all math expressions use validated numeric inputs Resolves "bad math expression: operator expected" errors in track_test_progress function. --- scripts/test-stability-runner.zsh | 607 ++++++++++++++++++++++++++++++ 1 file changed, 607 insertions(+) create mode 100755 scripts/test-stability-runner.zsh diff --git a/scripts/test-stability-runner.zsh b/scripts/test-stability-runner.zsh new file mode 100755 index 00000000..d46adb48 --- /dev/null +++ b/scripts/test-stability-runner.zsh @@ -0,0 +1,607 @@ +#!/bin/zsh + +# Test Stability Runner for TimeSafari (Zsh Version) +# Executes the full test suite 10 times and analyzes failure patterns +# Author: Matthew Raymer + +set -euo pipefail + +# Configuration +TOTAL_RUNS=10 +RESULTS_DIR="test-stability-results" +TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") +LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" +SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" +FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" + +# Colors for output +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +MAGENTA='\033[0;35m' +NC='\033[0m' # No Color + +# Progress bar characters +PROGRESS_CHAR="█" +EMPTY_CHAR="░" + +# Initialize results tracking using zsh associative arrays +typeset -A test_results +typeset -A test_failures +typeset -A test_successes +typeset -A run_times + +# Create results directory +mkdir -p "${RESULTS_DIR}" + +# Logging functions +log_info() { + echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_success() { + echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_warning() { + echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +log_error() { + echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" +} + +# Function to display progress bar +show_progress() { + local current="$1" + local total="$2" + local width="${3:-50}" + local label="${4:-Progress}" + + # Validate inputs + if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then + return + fi + + # Ensure we don't divide by zero + if [ "$total" -eq 0 ]; then + total=1 + fi + + local percentage=$((current * 100 / total)) + local filled=$((current * width / total)) + local empty=$((width - filled)) + + # Create progress bar string + local progress_bar="" + for ((i=0; i/dev/null | tr -d ' ' || echo "0") + + # Ensure we have a valid number + if [[ ! "$current_line_count" =~ ^[0-9]+$ ]]; then + current_line_count=0 + fi + + # Ensure last_line_count is also valid + if [[ ! "$last_line_count" =~ ^[0-9]+$ ]]; then + last_line_count=0 + fi + + if [ "$current_line_count" -gt "$last_line_count" ] && [ "$current_line_count" -gt 0 ]; then + # Calculate lines to read safely + local lines_to_read=$((current_line_count - last_line_count)) + if [ "$lines_to_read" -le 0 ]; then + lines_to_read=1 + fi + + # Get the latest lines + local new_lines=$(tail -n "$lines_to_read" "$test_file" 2>/dev/null || echo "") + + # Extract test progress information + local current_passed=$(echo "$new_lines" | grep -c "✓" 2>/dev/null || echo "0") + local current_failed=$(echo "$new_lines" | grep -c "✗" 2>/dev/null || echo "0") + + # Update counts with validation + if [[ "$current_passed" =~ ^[0-9]+$ ]]; then + passed_count=$((passed_count + current_passed)) + fi + if [[ "$current_failed" =~ ^[0-9]+$ ]]; then + failed_count=$((failed_count + current_failed)) + fi + completed_tests=$((passed_count + failed_count)) + + # Extract current test name if available + local latest_test=$(echo "$new_lines" | grep -E "✓.*test-playwright|✗.*test-playwright" | tail -1 | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' 2>/dev/null || echo "") + + if [ -n "$latest_test" ]; then + current_test="$latest_test" + fi + + # Show progress with test name + if [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ] && [ -n "$current_test" ]; then + show_progress "$completed_tests" "$total_test_files" 30 "TEST $current_test" + elif [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ]; then + show_progress "$completed_tests" "$total_test_files" 30 "TEST RUNNING" + elif [ "$completed_tests" -gt 0 ]; then + show_progress "$completed_tests" "$total_test_files" 30 "TEST PROGRESS" + fi + + last_line_count=$current_line_count + fi + fi + + # Check if the process is still running + if ! pgrep -f "playwright test" > /dev/null 2>&1; then + break + fi + + # Add a small delay to prevent excessive CPU usage + sleep 0.5 + done + + clear_progress +} + +# Function to extract test names from Playwright output +extract_test_names() { + local output_file="$1" + # Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" + grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq +} + +# Function to check if test passed in a run +test_passed_in_run() { + local test_name="$1" + local run_output="$2" + grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null +} + +# Function to check if test failed in a run +test_failed_in_run() { + local test_name="$1" + local run_output="$2" + grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null +} + +# Function to get test duration +get_test_duration() { + local test_name="$1" + local run_output="$2" + local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) + echo "${duration:-unknown}" +} + +# Function to analyze test results +analyze_results() { + # Initialize summary data + local summary_data="{ + \"timestamp\": \"$(date -Iseconds)\", + \"total_runs\": $TOTAL_RUNS, + \"test_results\": {}, + \"summary_stats\": { + \"total_tests\": 0, + \"always_passing\": 0, + \"always_failing\": 0, + \"intermittent\": 0, + \"success_rate\": 0.0 + } + }" + + # Analyze each test using zsh associative array iteration + for test_name in ${(k)test_results}; do + local passes=${test_successes[$test_name]:-0} + local fails=${test_failures[$test_name]:-0} + local total=$((passes + fails)) + local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0") + + # Determine test stability + local stability="" + if [ "$fails" -eq 0 ]; then + stability="always_passing" + elif [ "$passes" -eq 0 ]; then + stability="always_failing" + else + stability="intermittent" + fi + + # Add to summary using jq + summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \ + --arg stability "$stability" \ + --arg passes "$passes" \ + --arg fails "$fails" \ + --arg total "$total" \ + --arg rate "$success_rate" \ + '.test_results[$test] = { + "stability": $stability, + "passes": ($passes | tonumber), + "fails": ($fails | tonumber), + "total": ($total | tonumber), + "success_rate": ($rate | tonumber) + }') + done + + # Calculate summary statistics + local total_tests=$(echo "$summary_data" | jq '.test_results | length') + local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length') + local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length') + local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length') + + summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \ + --arg passing "$always_passing" \ + --arg failing "$always_failing" \ + --arg intermittent "$intermittent" \ + '.summary_stats.total_tests = ($total | tonumber) | + .summary_stats.always_passing = ($passing | tonumber) | + .summary_stats.always_failing = ($failing | tonumber) | + .summary_stats.intermittent = ($intermittent | tonumber)') + + # Save summary + echo "$summary_data" | jq '.' > "${SUMMARY_FILE}" + + log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" +} + +# Function to generate detailed report +generate_report() { + local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" + + { + echo "# TimeSafari Test Stability Report" + echo "" + echo "**Generated:** $(date)" + echo "**Total Runs:** $TOTAL_RUNS" + # Calculate duration with proper error handling + local current_time=$(date +%s) + local duration=0 + if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then + duration=$((current_time - START_TIME)) + fi + echo "**Duration:** ${duration} seconds" + echo "" + + # Summary statistics + echo "## Summary Statistics" + echo "" + local summary_data=$(cat "${SUMMARY_FILE}") + local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') + local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') + local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') + local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') + + echo "- **Total Tests:** $total_tests" + echo "- **Always Passing:** $always_passing" + echo "- **Always Failing:** $always_failing" + echo "- **Intermittent:** $intermittent" + echo "" + + # Always failing tests + echo "## Always Failing Tests" + echo "" + local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"') + if [ -n "$failing_tests" ]; then + echo "$failing_tests" + else + echo "No always failing tests found." + fi + echo "" + + # Intermittent tests + echo "## Intermittent Tests (Most Unstable First)" + echo "" + local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"') + if [ -n "$intermittent_tests" ]; then + echo "$intermittent_tests" + else + echo "No intermittent tests found." + fi + echo "" + + # Always passing tests + echo "## Always Passing Tests" + echo "" + local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key') + if [ -n "$passing_tests" ]; then + echo "$passing_tests" + else + echo "No always passing tests found." + fi + echo "" + + # Detailed test results + echo "## Detailed Test Results" + echo "" + echo "| Test Name | Stability | Passes | Fails | Success Rate |" + echo "|-----------|-----------|--------|-------|--------------|" + echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"' + echo "" + + # Run-by-run summary + echo "## Run-by-Run Summary" + echo "" + for ((i=1; i<=TOTAL_RUNS; i++)); do + local run_file="${RESULTS_DIR}/run-${i}.txt" + if [ -f "$run_file" ]; then + # Extract passed and failed counts using the same method as the main script + local passed=0 + local failed=0 + + local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) + if [ -n "$passed_line" ]; then + passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") + fi + + local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) + if [ -n "$failed_line" ]; then + failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") + fi + + local total=$((passed + failed)) + echo "**Run $i:** $passed passed, $failed failed ($total total)" + fi + done + + } > "$report_file" + + log_success "Detailed report generated: $report_file" +} + +# Main execution +main() { + START_TIME=$(date +%s) + + log_info "Starting TimeSafari Test Stability Runner (Zsh Version)" + log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" + log_info "Log file: ${LOG_FILE}" + + # Check prerequisites + log_info "Checking prerequisites..." + if ! command -v jq &> /dev/null; then + log_error "jq is required but not installed. Please install jq." + exit 1 + fi + + if ! command -v bc &> /dev/null; then + log_error "bc is required but not installed. Please install bc." + exit 1 + fi + + # Check if Playwright is available + if ! npx playwright --version &> /dev/null; then + log_error "Playwright is not available. Please install dependencies." + exit 1 + fi + + log_success "Prerequisites check passed" + + echo "" + log_info "Starting test execution with progress tracking..." + echo "" + + # Run tests multiple times + for ((run=1; run<=TOTAL_RUNS; run++)); do + log_info "Starting run $run/$TOTAL_RUNS" + + local run_start=$(date +%s) + local run_output="${RESULTS_DIR}/run-${run}.txt" + + # Show overall run progress + show_progress "$run" "$TOTAL_RUNS" 40 "RUN $run/$TOTAL_RUNS" + + # Run the test suite with individual test progress + echo "" + log_info "Executing test suite for run $run..." + + # Create a temporary file to capture Playwright output + local temp_output=$(mktemp) + + # Start progress tracking in background + track_test_progress "$run" "$temp_output" & + local progress_pid=$! + + # Run Playwright with progress tracking + if npx playwright test -c playwright.config-local.ts --reporter=list > "$temp_output" 2>&1; then + # Wait for progress tracking to finish + wait $progress_pid 2>/dev/null || true + + # Copy the output to our results file + cp "$temp_output" "$run_output" + log_success "Run $run completed successfully" + else + # Wait for progress tracking to finish + wait $progress_pid 2>/dev/null || true + + # Copy the output to our results file even if it failed + cp "$temp_output" "$run_output" + log_warning "Run $run completed with failures" + fi + + # Ensure progress tracking is stopped + kill $progress_pid 2>/dev/null || true + + # Clean up temp file + rm -f "$temp_output" + + local run_end=$(date +%s) + local run_duration=$((run_end - run_start)) + run_times[$run]=$run_duration + + # Show run summary + echo "" + log_info "Run $run completed in ${run_duration}s" + + # Extract and display quick summary for this run + local passed=0 + local failed=0 + + # Extract passed count from the last line containing "passed" + local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) + if [ -n "$passed_line" ]; then + passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") + fi + + # Extract failed count from the last line containing "failed" + local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) + if [ -n "$failed_line" ]; then + failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") + fi + + local total=$((passed + failed)) + echo " ${GREEN}✓${NC} $passed passed, ${RED}✗${NC} $failed failed (${CYAN}$total${NC} total)" + + # Extract and track test results using zsh array handling + local test_names=($(extract_test_names "$run_output")) + local test_count=${#test_names[@]} + local processed_tests=0 + + for test_name in $test_names; do + processed_tests=$((processed_tests + 1)) + + # Show progress for test analysis + show_progress "$processed_tests" "$test_count" 30 "ANALYZING" + + if test_passed_in_run "$test_name" "$run_output"; then + test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) + test_results[$test_name]="pass" + elif test_failed_in_run "$test_name" "$run_output"; then + test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) + test_results[$test_name]="fail" + + # Log failure details + echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" + grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true + echo "" >> "$FAILURE_LOG" + fi + done + + clear_progress + + # Show detailed failed tests for this run + if [ "$failed" -gt 0 ]; then + log_warning "Failed tests in run $run:" + # Extract failed test names from the summary section + sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do + local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') + log_warning " - $test_name" + done + else + log_success "All tests passed in run $run" + fi + + echo "" + done + + # Analyze results + log_info "Analyzing test results..." + show_progress "1" "3" 40 "ANALYSIS" + analyze_results + + # Generate detailed report + show_progress "2" "3" 40 "REPORT" + generate_report + + show_progress "3" "3" 40 "COMPLETE" + clear_progress + + echo "" + log_success "Test stability analysis complete!" + + # Final summary + local total_duration=$(($(date +%s) - START_TIME)) + log_info "Total duration: ${total_duration}s" + log_info "Results saved to: ${RESULTS_DIR}" + log_info "Summary: ${SUMMARY_FILE}" + log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" + log_info "Failure details: ${FAILURE_LOG}" + + # Display quick summary + echo "" + echo "=== QUICK SUMMARY ===" + local summary_data=$(cat "${SUMMARY_FILE}") + local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') + local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') + local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') + local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') + + echo "Total Tests: $total_tests" + echo "Always Passing: $always_passing" + echo "Always Failing: $always_failing" + echo "Intermittent: $intermittent" + + # Show run-by-run failure summary + echo "" + echo "=== RUN-BY-RUN FAILURE SUMMARY ===" + for ((i=1; i<=TOTAL_RUNS; i++)); do + local run_file="${RESULTS_DIR}/run-${i}.txt" + if [ -f "$run_file" ]; then + local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) + local failed_count=0 + if [ -n "$failed_line" ]; then + failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") + fi + + if [ "$failed_count" -gt 0 ]; then + echo "Run $i: $failed_count failed" + # Extract failed test names from the summary section + sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do + local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') + echo " - $test_name" + done + else + echo "Run $i: All tests passed" + fi + fi + done + + if [ "$always_failing" -gt 0 ]; then + echo "" + echo "🚨 ALWAYS FAILING TESTS:" + echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key' + fi + + if [ "$intermittent" -gt 0 ]; then + echo "" + echo "⚠️ INTERMITTENT TESTS (most unstable first):" + echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"' + fi +} + +# Run the main function +main "$@" \ No newline at end of file