Browse Source
- Add input validation for all numeric values before math operations - Implement safe math calculations with zero-division protection - Add error redirection (2>/dev/null) to suppress command errors - Improve process management with proper background process cleanup - Add fallback values when commands return invalid output - Fix progress bar display with better validation and error handling - Ensure all math expressions use validated numeric inputs Resolves "bad math expression: operator expected" errors in track_test_progress function.pull/159/head
Matthew Raymer
2 weeks ago
1 changed files with 607 additions and 0 deletions
@ -0,0 +1,607 @@ |
|||||
|
#!/bin/zsh |
||||
|
|
||||
|
# Test Stability Runner for TimeSafari (Zsh Version) |
||||
|
# Executes the full test suite 10 times and analyzes failure patterns |
||||
|
# Author: Matthew Raymer |
||||
|
|
||||
|
set -euo pipefail |
||||
|
|
||||
|
# Configuration |
||||
|
TOTAL_RUNS=10 |
||||
|
RESULTS_DIR="test-stability-results" |
||||
|
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S") |
||||
|
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log" |
||||
|
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json" |
||||
|
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log" |
||||
|
|
||||
|
# Colors for output |
||||
|
RED='\033[0;31m' |
||||
|
GREEN='\033[0;32m' |
||||
|
YELLOW='\033[1;33m' |
||||
|
BLUE='\033[0;34m' |
||||
|
CYAN='\033[0;36m' |
||||
|
MAGENTA='\033[0;35m' |
||||
|
NC='\033[0m' # No Color |
||||
|
|
||||
|
# Progress bar characters |
||||
|
PROGRESS_CHAR="█" |
||||
|
EMPTY_CHAR="░" |
||||
|
|
||||
|
# Initialize results tracking using zsh associative arrays |
||||
|
typeset -A test_results |
||||
|
typeset -A test_failures |
||||
|
typeset -A test_successes |
||||
|
typeset -A run_times |
||||
|
|
||||
|
# Create results directory |
||||
|
mkdir -p "${RESULTS_DIR}" |
||||
|
|
||||
|
# Logging functions |
||||
|
log_info() { |
||||
|
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" |
||||
|
} |
||||
|
|
||||
|
log_success() { |
||||
|
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" |
||||
|
} |
||||
|
|
||||
|
log_warning() { |
||||
|
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" |
||||
|
} |
||||
|
|
||||
|
log_error() { |
||||
|
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}" |
||||
|
} |
||||
|
|
||||
|
# Function to display progress bar |
||||
|
show_progress() { |
||||
|
local current="$1" |
||||
|
local total="$2" |
||||
|
local width="${3:-50}" |
||||
|
local label="${4:-Progress}" |
||||
|
|
||||
|
# Validate inputs |
||||
|
if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then |
||||
|
return |
||||
|
fi |
||||
|
|
||||
|
# Ensure we don't divide by zero |
||||
|
if [ "$total" -eq 0 ]; then |
||||
|
total=1 |
||||
|
fi |
||||
|
|
||||
|
local percentage=$((current * 100 / total)) |
||||
|
local filled=$((current * width / total)) |
||||
|
local empty=$((width - filled)) |
||||
|
|
||||
|
# Create progress bar string |
||||
|
local progress_bar="" |
||||
|
for ((i=0; i<filled; i++)); do |
||||
|
progress_bar+="$PROGRESS_CHAR" |
||||
|
done |
||||
|
for ((i=0; i<empty; i++)); do |
||||
|
progress_bar+="$EMPTY_CHAR" |
||||
|
done |
||||
|
|
||||
|
# Print progress bar with carriage return to overwrite |
||||
|
printf "\r${CYAN}[%s]${NC} %s [%s] %d%% (%d/%d)" \ |
||||
|
"$label" "$progress_bar" "$percentage" "$current" "$total" |
||||
|
} |
||||
|
|
||||
|
# Function to clear progress bar line |
||||
|
clear_progress() { |
||||
|
printf "\r%*s\r" "$(tput cols)" "" |
||||
|
} |
||||
|
|
||||
|
# Function to track test execution progress |
||||
|
track_test_progress() { |
||||
|
local run_number="$1" |
||||
|
local test_file="$2" |
||||
|
|
||||
|
# Get total number of test files for progress calculation |
||||
|
local total_test_files=$(find test-playwright -name "*.spec.ts" | wc -l | tr -d ' ') |
||||
|
if [[ ! "$total_test_files" =~ ^[0-9]+$ ]] || [ "$total_test_files" -eq 0 ]; then |
||||
|
total_test_files=1 |
||||
|
fi |
||||
|
local completed_tests=0 |
||||
|
local current_test="" |
||||
|
|
||||
|
# Monitor the test file for progress |
||||
|
local last_line_count=0 |
||||
|
local passed_count=0 |
||||
|
local failed_count=0 |
||||
|
|
||||
|
while true; do |
||||
|
if [ -f "$test_file" ]; then |
||||
|
local current_line_count=$(wc -l < "$test_file" 2>/dev/null | tr -d ' ' || echo "0") |
||||
|
|
||||
|
# Ensure we have a valid number |
||||
|
if [[ ! "$current_line_count" =~ ^[0-9]+$ ]]; then |
||||
|
current_line_count=0 |
||||
|
fi |
||||
|
|
||||
|
# Ensure last_line_count is also valid |
||||
|
if [[ ! "$last_line_count" =~ ^[0-9]+$ ]]; then |
||||
|
last_line_count=0 |
||||
|
fi |
||||
|
|
||||
|
if [ "$current_line_count" -gt "$last_line_count" ] && [ "$current_line_count" -gt 0 ]; then |
||||
|
# Calculate lines to read safely |
||||
|
local lines_to_read=$((current_line_count - last_line_count)) |
||||
|
if [ "$lines_to_read" -le 0 ]; then |
||||
|
lines_to_read=1 |
||||
|
fi |
||||
|
|
||||
|
# Get the latest lines |
||||
|
local new_lines=$(tail -n "$lines_to_read" "$test_file" 2>/dev/null || echo "") |
||||
|
|
||||
|
# Extract test progress information |
||||
|
local current_passed=$(echo "$new_lines" | grep -c "✓" 2>/dev/null || echo "0") |
||||
|
local current_failed=$(echo "$new_lines" | grep -c "✗" 2>/dev/null || echo "0") |
||||
|
|
||||
|
# Update counts with validation |
||||
|
if [[ "$current_passed" =~ ^[0-9]+$ ]]; then |
||||
|
passed_count=$((passed_count + current_passed)) |
||||
|
fi |
||||
|
if [[ "$current_failed" =~ ^[0-9]+$ ]]; then |
||||
|
failed_count=$((failed_count + current_failed)) |
||||
|
fi |
||||
|
completed_tests=$((passed_count + failed_count)) |
||||
|
|
||||
|
# Extract current test name if available |
||||
|
local latest_test=$(echo "$new_lines" | grep -E "✓.*test-playwright|✗.*test-playwright" | tail -1 | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' 2>/dev/null || echo "") |
||||
|
|
||||
|
if [ -n "$latest_test" ]; then |
||||
|
current_test="$latest_test" |
||||
|
fi |
||||
|
|
||||
|
# Show progress with test name |
||||
|
if [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ] && [ -n "$current_test" ]; then |
||||
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST $current_test" |
||||
|
elif [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ]; then |
||||
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST RUNNING" |
||||
|
elif [ "$completed_tests" -gt 0 ]; then |
||||
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST PROGRESS" |
||||
|
fi |
||||
|
|
||||
|
last_line_count=$current_line_count |
||||
|
fi |
||||
|
fi |
||||
|
|
||||
|
# Check if the process is still running |
||||
|
if ! pgrep -f "playwright test" > /dev/null 2>&1; then |
||||
|
break |
||||
|
fi |
||||
|
|
||||
|
# Add a small delay to prevent excessive CPU usage |
||||
|
sleep 0.5 |
||||
|
done |
||||
|
|
||||
|
clear_progress |
||||
|
} |
||||
|
|
||||
|
# Function to extract test names from Playwright output |
||||
|
extract_test_names() { |
||||
|
local output_file="$1" |
||||
|
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given" |
||||
|
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq |
||||
|
} |
||||
|
|
||||
|
# Function to check if test passed in a run |
||||
|
test_passed_in_run() { |
||||
|
local test_name="$1" |
||||
|
local run_output="$2" |
||||
|
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null |
||||
|
} |
||||
|
|
||||
|
# Function to check if test failed in a run |
||||
|
test_failed_in_run() { |
||||
|
local test_name="$1" |
||||
|
local run_output="$2" |
||||
|
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null |
||||
|
} |
||||
|
|
||||
|
# Function to get test duration |
||||
|
get_test_duration() { |
||||
|
local test_name="$1" |
||||
|
local run_output="$2" |
||||
|
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1) |
||||
|
echo "${duration:-unknown}" |
||||
|
} |
||||
|
|
||||
|
# Function to analyze test results |
||||
|
analyze_results() { |
||||
|
# Initialize summary data |
||||
|
local summary_data="{ |
||||
|
\"timestamp\": \"$(date -Iseconds)\", |
||||
|
\"total_runs\": $TOTAL_RUNS, |
||||
|
\"test_results\": {}, |
||||
|
\"summary_stats\": { |
||||
|
\"total_tests\": 0, |
||||
|
\"always_passing\": 0, |
||||
|
\"always_failing\": 0, |
||||
|
\"intermittent\": 0, |
||||
|
\"success_rate\": 0.0 |
||||
|
} |
||||
|
}" |
||||
|
|
||||
|
# Analyze each test using zsh associative array iteration |
||||
|
for test_name in ${(k)test_results}; do |
||||
|
local passes=${test_successes[$test_name]:-0} |
||||
|
local fails=${test_failures[$test_name]:-0} |
||||
|
local total=$((passes + fails)) |
||||
|
local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0") |
||||
|
|
||||
|
# Determine test stability |
||||
|
local stability="" |
||||
|
if [ "$fails" -eq 0 ]; then |
||||
|
stability="always_passing" |
||||
|
elif [ "$passes" -eq 0 ]; then |
||||
|
stability="always_failing" |
||||
|
else |
||||
|
stability="intermittent" |
||||
|
fi |
||||
|
|
||||
|
# Add to summary using jq |
||||
|
summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \ |
||||
|
--arg stability "$stability" \ |
||||
|
--arg passes "$passes" \ |
||||
|
--arg fails "$fails" \ |
||||
|
--arg total "$total" \ |
||||
|
--arg rate "$success_rate" \ |
||||
|
'.test_results[$test] = { |
||||
|
"stability": $stability, |
||||
|
"passes": ($passes | tonumber), |
||||
|
"fails": ($fails | tonumber), |
||||
|
"total": ($total | tonumber), |
||||
|
"success_rate": ($rate | tonumber) |
||||
|
}') |
||||
|
done |
||||
|
|
||||
|
# Calculate summary statistics |
||||
|
local total_tests=$(echo "$summary_data" | jq '.test_results | length') |
||||
|
local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length') |
||||
|
local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length') |
||||
|
local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length') |
||||
|
|
||||
|
summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \ |
||||
|
--arg passing "$always_passing" \ |
||||
|
--arg failing "$always_failing" \ |
||||
|
--arg intermittent "$intermittent" \ |
||||
|
'.summary_stats.total_tests = ($total | tonumber) | |
||||
|
.summary_stats.always_passing = ($passing | tonumber) | |
||||
|
.summary_stats.always_failing = ($failing | tonumber) | |
||||
|
.summary_stats.intermittent = ($intermittent | tonumber)') |
||||
|
|
||||
|
# Save summary |
||||
|
echo "$summary_data" | jq '.' > "${SUMMARY_FILE}" |
||||
|
|
||||
|
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}" |
||||
|
} |
||||
|
|
||||
|
# Function to generate detailed report |
||||
|
generate_report() { |
||||
|
local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" |
||||
|
|
||||
|
{ |
||||
|
echo "# TimeSafari Test Stability Report" |
||||
|
echo "" |
||||
|
echo "**Generated:** $(date)" |
||||
|
echo "**Total Runs:** $TOTAL_RUNS" |
||||
|
# Calculate duration with proper error handling |
||||
|
local current_time=$(date +%s) |
||||
|
local duration=0 |
||||
|
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then |
||||
|
duration=$((current_time - START_TIME)) |
||||
|
fi |
||||
|
echo "**Duration:** ${duration} seconds" |
||||
|
echo "" |
||||
|
|
||||
|
# Summary statistics |
||||
|
echo "## Summary Statistics" |
||||
|
echo "" |
||||
|
local summary_data=$(cat "${SUMMARY_FILE}") |
||||
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') |
||||
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') |
||||
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') |
||||
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') |
||||
|
|
||||
|
echo "- **Total Tests:** $total_tests" |
||||
|
echo "- **Always Passing:** $always_passing" |
||||
|
echo "- **Always Failing:** $always_failing" |
||||
|
echo "- **Intermittent:** $intermittent" |
||||
|
echo "" |
||||
|
|
||||
|
# Always failing tests |
||||
|
echo "## Always Failing Tests" |
||||
|
echo "" |
||||
|
local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"') |
||||
|
if [ -n "$failing_tests" ]; then |
||||
|
echo "$failing_tests" |
||||
|
else |
||||
|
echo "No always failing tests found." |
||||
|
fi |
||||
|
echo "" |
||||
|
|
||||
|
# Intermittent tests |
||||
|
echo "## Intermittent Tests (Most Unstable First)" |
||||
|
echo "" |
||||
|
local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"') |
||||
|
if [ -n "$intermittent_tests" ]; then |
||||
|
echo "$intermittent_tests" |
||||
|
else |
||||
|
echo "No intermittent tests found." |
||||
|
fi |
||||
|
echo "" |
||||
|
|
||||
|
# Always passing tests |
||||
|
echo "## Always Passing Tests" |
||||
|
echo "" |
||||
|
local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key') |
||||
|
if [ -n "$passing_tests" ]; then |
||||
|
echo "$passing_tests" |
||||
|
else |
||||
|
echo "No always passing tests found." |
||||
|
fi |
||||
|
echo "" |
||||
|
|
||||
|
# Detailed test results |
||||
|
echo "## Detailed Test Results" |
||||
|
echo "" |
||||
|
echo "| Test Name | Stability | Passes | Fails | Success Rate |" |
||||
|
echo "|-----------|-----------|--------|-------|--------------|" |
||||
|
echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"' |
||||
|
echo "" |
||||
|
|
||||
|
# Run-by-run summary |
||||
|
echo "## Run-by-Run Summary" |
||||
|
echo "" |
||||
|
for ((i=1; i<=TOTAL_RUNS; i++)); do |
||||
|
local run_file="${RESULTS_DIR}/run-${i}.txt" |
||||
|
if [ -f "$run_file" ]; then |
||||
|
# Extract passed and failed counts using the same method as the main script |
||||
|
local passed=0 |
||||
|
local failed=0 |
||||
|
|
||||
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1) |
||||
|
if [ -n "$passed_line" ]; then |
||||
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") |
||||
|
fi |
||||
|
|
||||
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) |
||||
|
if [ -n "$failed_line" ]; then |
||||
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") |
||||
|
fi |
||||
|
|
||||
|
local total=$((passed + failed)) |
||||
|
echo "**Run $i:** $passed passed, $failed failed ($total total)" |
||||
|
fi |
||||
|
done |
||||
|
|
||||
|
} > "$report_file" |
||||
|
|
||||
|
log_success "Detailed report generated: $report_file" |
||||
|
} |
||||
|
|
||||
|
# Main execution |
||||
|
main() { |
||||
|
START_TIME=$(date +%s) |
||||
|
|
||||
|
log_info "Starting TimeSafari Test Stability Runner (Zsh Version)" |
||||
|
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}" |
||||
|
log_info "Log file: ${LOG_FILE}" |
||||
|
|
||||
|
# Check prerequisites |
||||
|
log_info "Checking prerequisites..." |
||||
|
if ! command -v jq &> /dev/null; then |
||||
|
log_error "jq is required but not installed. Please install jq." |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
if ! command -v bc &> /dev/null; then |
||||
|
log_error "bc is required but not installed. Please install bc." |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
# Check if Playwright is available |
||||
|
if ! npx playwright --version &> /dev/null; then |
||||
|
log_error "Playwright is not available. Please install dependencies." |
||||
|
exit 1 |
||||
|
fi |
||||
|
|
||||
|
log_success "Prerequisites check passed" |
||||
|
|
||||
|
echo "" |
||||
|
log_info "Starting test execution with progress tracking..." |
||||
|
echo "" |
||||
|
|
||||
|
# Run tests multiple times |
||||
|
for ((run=1; run<=TOTAL_RUNS; run++)); do |
||||
|
log_info "Starting run $run/$TOTAL_RUNS" |
||||
|
|
||||
|
local run_start=$(date +%s) |
||||
|
local run_output="${RESULTS_DIR}/run-${run}.txt" |
||||
|
|
||||
|
# Show overall run progress |
||||
|
show_progress "$run" "$TOTAL_RUNS" 40 "RUN $run/$TOTAL_RUNS" |
||||
|
|
||||
|
# Run the test suite with individual test progress |
||||
|
echo "" |
||||
|
log_info "Executing test suite for run $run..." |
||||
|
|
||||
|
# Create a temporary file to capture Playwright output |
||||
|
local temp_output=$(mktemp) |
||||
|
|
||||
|
# Start progress tracking in background |
||||
|
track_test_progress "$run" "$temp_output" & |
||||
|
local progress_pid=$! |
||||
|
|
||||
|
# Run Playwright with progress tracking |
||||
|
if npx playwright test -c playwright.config-local.ts --reporter=list > "$temp_output" 2>&1; then |
||||
|
# Wait for progress tracking to finish |
||||
|
wait $progress_pid 2>/dev/null || true |
||||
|
|
||||
|
# Copy the output to our results file |
||||
|
cp "$temp_output" "$run_output" |
||||
|
log_success "Run $run completed successfully" |
||||
|
else |
||||
|
# Wait for progress tracking to finish |
||||
|
wait $progress_pid 2>/dev/null || true |
||||
|
|
||||
|
# Copy the output to our results file even if it failed |
||||
|
cp "$temp_output" "$run_output" |
||||
|
log_warning "Run $run completed with failures" |
||||
|
fi |
||||
|
|
||||
|
# Ensure progress tracking is stopped |
||||
|
kill $progress_pid 2>/dev/null || true |
||||
|
|
||||
|
# Clean up temp file |
||||
|
rm -f "$temp_output" |
||||
|
|
||||
|
local run_end=$(date +%s) |
||||
|
local run_duration=$((run_end - run_start)) |
||||
|
run_times[$run]=$run_duration |
||||
|
|
||||
|
# Show run summary |
||||
|
echo "" |
||||
|
log_info "Run $run completed in ${run_duration}s" |
||||
|
|
||||
|
# Extract and display quick summary for this run |
||||
|
local passed=0 |
||||
|
local failed=0 |
||||
|
|
||||
|
# Extract passed count from the last line containing "passed" |
||||
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1) |
||||
|
if [ -n "$passed_line" ]; then |
||||
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+") |
||||
|
fi |
||||
|
|
||||
|
# Extract failed count from the last line containing "failed" |
||||
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1) |
||||
|
if [ -n "$failed_line" ]; then |
||||
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") |
||||
|
fi |
||||
|
|
||||
|
local total=$((passed + failed)) |
||||
|
echo " ${GREEN}✓${NC} $passed passed, ${RED}✗${NC} $failed failed (${CYAN}$total${NC} total)" |
||||
|
|
||||
|
# Extract and track test results using zsh array handling |
||||
|
local test_names=($(extract_test_names "$run_output")) |
||||
|
local test_count=${#test_names[@]} |
||||
|
local processed_tests=0 |
||||
|
|
||||
|
for test_name in $test_names; do |
||||
|
processed_tests=$((processed_tests + 1)) |
||||
|
|
||||
|
# Show progress for test analysis |
||||
|
show_progress "$processed_tests" "$test_count" 30 "ANALYZING" |
||||
|
|
||||
|
if test_passed_in_run "$test_name" "$run_output"; then |
||||
|
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1)) |
||||
|
test_results[$test_name]="pass" |
||||
|
elif test_failed_in_run "$test_name" "$run_output"; then |
||||
|
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1)) |
||||
|
test_results[$test_name]="fail" |
||||
|
|
||||
|
# Log failure details |
||||
|
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG" |
||||
|
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true |
||||
|
echo "" >> "$FAILURE_LOG" |
||||
|
fi |
||||
|
done |
||||
|
|
||||
|
clear_progress |
||||
|
|
||||
|
# Show detailed failed tests for this run |
||||
|
if [ "$failed" -gt 0 ]; then |
||||
|
log_warning "Failed tests in run $run:" |
||||
|
# Extract failed test names from the summary section |
||||
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do |
||||
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') |
||||
|
log_warning " - $test_name" |
||||
|
done |
||||
|
else |
||||
|
log_success "All tests passed in run $run" |
||||
|
fi |
||||
|
|
||||
|
echo "" |
||||
|
done |
||||
|
|
||||
|
# Analyze results |
||||
|
log_info "Analyzing test results..." |
||||
|
show_progress "1" "3" 40 "ANALYSIS" |
||||
|
analyze_results |
||||
|
|
||||
|
# Generate detailed report |
||||
|
show_progress "2" "3" 40 "REPORT" |
||||
|
generate_report |
||||
|
|
||||
|
show_progress "3" "3" 40 "COMPLETE" |
||||
|
clear_progress |
||||
|
|
||||
|
echo "" |
||||
|
log_success "Test stability analysis complete!" |
||||
|
|
||||
|
# Final summary |
||||
|
local total_duration=$(($(date +%s) - START_TIME)) |
||||
|
log_info "Total duration: ${total_duration}s" |
||||
|
log_info "Results saved to: ${RESULTS_DIR}" |
||||
|
log_info "Summary: ${SUMMARY_FILE}" |
||||
|
log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md" |
||||
|
log_info "Failure details: ${FAILURE_LOG}" |
||||
|
|
||||
|
# Display quick summary |
||||
|
echo "" |
||||
|
echo "=== QUICK SUMMARY ===" |
||||
|
local summary_data=$(cat "${SUMMARY_FILE}") |
||||
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests') |
||||
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing') |
||||
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing') |
||||
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent') |
||||
|
|
||||
|
echo "Total Tests: $total_tests" |
||||
|
echo "Always Passing: $always_passing" |
||||
|
echo "Always Failing: $always_failing" |
||||
|
echo "Intermittent: $intermittent" |
||||
|
|
||||
|
# Show run-by-run failure summary |
||||
|
echo "" |
||||
|
echo "=== RUN-BY-RUN FAILURE SUMMARY ===" |
||||
|
for ((i=1; i<=TOTAL_RUNS; i++)); do |
||||
|
local run_file="${RESULTS_DIR}/run-${i}.txt" |
||||
|
if [ -f "$run_file" ]; then |
||||
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1) |
||||
|
local failed_count=0 |
||||
|
if [ -n "$failed_line" ]; then |
||||
|
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+") |
||||
|
fi |
||||
|
|
||||
|
if [ "$failed_count" -gt 0 ]; then |
||||
|
echo "Run $i: $failed_count failed" |
||||
|
# Extract failed test names from the summary section |
||||
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do |
||||
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//') |
||||
|
echo " - $test_name" |
||||
|
done |
||||
|
else |
||||
|
echo "Run $i: All tests passed" |
||||
|
fi |
||||
|
fi |
||||
|
done |
||||
|
|
||||
|
if [ "$always_failing" -gt 0 ]; then |
||||
|
echo "" |
||||
|
echo "🚨 ALWAYS FAILING TESTS:" |
||||
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key' |
||||
|
fi |
||||
|
|
||||
|
if [ "$intermittent" -gt 0 ]; then |
||||
|
echo "" |
||||
|
echo "⚠️ INTERMITTENT TESTS (most unstable first):" |
||||
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"' |
||||
|
fi |
||||
|
} |
||||
|
|
||||
|
# Run the main function |
||||
|
main "$@" |
||||
Loading…
Reference in new issue