You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
607 lines
22 KiB
607 lines
22 KiB
#!/bin/zsh
|
|
|
|
# Test Stability Runner for TimeSafari (Zsh Version)
|
|
# Executes the full test suite 10 times and analyzes failure patterns
|
|
# Author: Matthew Raymer
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
TOTAL_RUNS=10
|
|
RESULTS_DIR="test-stability-results"
|
|
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
|
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
|
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
|
|
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
CYAN='\033[0;36m'
|
|
MAGENTA='\033[0;35m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Progress bar characters
|
|
PROGRESS_CHAR="█"
|
|
EMPTY_CHAR="░"
|
|
|
|
# Initialize results tracking using zsh associative arrays
|
|
typeset -A test_results
|
|
typeset -A test_failures
|
|
typeset -A test_successes
|
|
typeset -A run_times
|
|
|
|
# Create results directory
|
|
mkdir -p "${RESULTS_DIR}"
|
|
|
|
# Logging functions
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
# Function to display progress bar
|
|
show_progress() {
|
|
local current="$1"
|
|
local total="$2"
|
|
local width="${3:-50}"
|
|
local label="${4:-Progress}"
|
|
|
|
# Validate inputs
|
|
if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then
|
|
return
|
|
fi
|
|
|
|
# Ensure we don't divide by zero
|
|
if [ "$total" -eq 0 ]; then
|
|
total=1
|
|
fi
|
|
|
|
local percentage=$((current * 100 / total))
|
|
local filled=$((current * width / total))
|
|
local empty=$((width - filled))
|
|
|
|
# Create progress bar string
|
|
local progress_bar=""
|
|
for ((i=0; i<filled; i++)); do
|
|
progress_bar+="$PROGRESS_CHAR"
|
|
done
|
|
for ((i=0; i<empty; i++)); do
|
|
progress_bar+="$EMPTY_CHAR"
|
|
done
|
|
|
|
# Print progress bar with carriage return to overwrite
|
|
printf "\r${CYAN}[%s]${NC} %s [%s] %d%% (%d/%d)" \
|
|
"$label" "$progress_bar" "$percentage" "$current" "$total"
|
|
}
|
|
|
|
# Function to clear progress bar line
|
|
clear_progress() {
|
|
printf "\r%*s\r" "$(tput cols)" ""
|
|
}
|
|
|
|
# Function to track test execution progress
|
|
track_test_progress() {
|
|
local run_number="$1"
|
|
local test_file="$2"
|
|
|
|
# Get total number of test files for progress calculation
|
|
local total_test_files=$(find test-playwright -name "*.spec.ts" | wc -l | tr -d ' ')
|
|
if [[ ! "$total_test_files" =~ ^[0-9]+$ ]] || [ "$total_test_files" -eq 0 ]; then
|
|
total_test_files=1
|
|
fi
|
|
local completed_tests=0
|
|
local current_test=""
|
|
|
|
# Monitor the test file for progress
|
|
local last_line_count=0
|
|
local passed_count=0
|
|
local failed_count=0
|
|
|
|
while true; do
|
|
if [ -f "$test_file" ]; then
|
|
local current_line_count=$(wc -l < "$test_file" 2>/dev/null | tr -d ' ' || echo "0")
|
|
|
|
# Ensure we have a valid number
|
|
if [[ ! "$current_line_count" =~ ^[0-9]+$ ]]; then
|
|
current_line_count=0
|
|
fi
|
|
|
|
# Ensure last_line_count is also valid
|
|
if [[ ! "$last_line_count" =~ ^[0-9]+$ ]]; then
|
|
last_line_count=0
|
|
fi
|
|
|
|
if [ "$current_line_count" -gt "$last_line_count" ] && [ "$current_line_count" -gt 0 ]; then
|
|
# Calculate lines to read safely
|
|
local lines_to_read=$((current_line_count - last_line_count))
|
|
if [ "$lines_to_read" -le 0 ]; then
|
|
lines_to_read=1
|
|
fi
|
|
|
|
# Get the latest lines
|
|
local new_lines=$(tail -n "$lines_to_read" "$test_file" 2>/dev/null || echo "")
|
|
|
|
# Extract test progress information
|
|
local current_passed=$(echo "$new_lines" | grep -c "✓" 2>/dev/null || echo "0")
|
|
local current_failed=$(echo "$new_lines" | grep -c "✗" 2>/dev/null || echo "0")
|
|
|
|
# Update counts with validation
|
|
if [[ "$current_passed" =~ ^[0-9]+$ ]]; then
|
|
passed_count=$((passed_count + current_passed))
|
|
fi
|
|
if [[ "$current_failed" =~ ^[0-9]+$ ]]; then
|
|
failed_count=$((failed_count + current_failed))
|
|
fi
|
|
completed_tests=$((passed_count + failed_count))
|
|
|
|
# Extract current test name if available
|
|
local latest_test=$(echo "$new_lines" | grep -E "✓.*test-playwright|✗.*test-playwright" | tail -1 | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' 2>/dev/null || echo "")
|
|
|
|
if [ -n "$latest_test" ]; then
|
|
current_test="$latest_test"
|
|
fi
|
|
|
|
# Show progress with test name
|
|
if [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ] && [ -n "$current_test" ]; then
|
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST $current_test"
|
|
elif [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ]; then
|
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST RUNNING"
|
|
elif [ "$completed_tests" -gt 0 ]; then
|
|
show_progress "$completed_tests" "$total_test_files" 30 "TEST PROGRESS"
|
|
fi
|
|
|
|
last_line_count=$current_line_count
|
|
fi
|
|
fi
|
|
|
|
# Check if the process is still running
|
|
if ! pgrep -f "playwright test" > /dev/null 2>&1; then
|
|
break
|
|
fi
|
|
|
|
# Add a small delay to prevent excessive CPU usage
|
|
sleep 0.5
|
|
done
|
|
|
|
clear_progress
|
|
}
|
|
|
|
# Function to extract test names from Playwright output
|
|
extract_test_names() {
|
|
local output_file="$1"
|
|
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
|
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
|
}
|
|
|
|
# Function to check if test passed in a run
|
|
test_passed_in_run() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
|
}
|
|
|
|
# Function to check if test failed in a run
|
|
test_failed_in_run() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
|
}
|
|
|
|
# Function to get test duration
|
|
get_test_duration() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
|
|
echo "${duration:-unknown}"
|
|
}
|
|
|
|
# Function to analyze test results
|
|
analyze_results() {
|
|
# Initialize summary data
|
|
local summary_data="{
|
|
\"timestamp\": \"$(date -Iseconds)\",
|
|
\"total_runs\": $TOTAL_RUNS,
|
|
\"test_results\": {},
|
|
\"summary_stats\": {
|
|
\"total_tests\": 0,
|
|
\"always_passing\": 0,
|
|
\"always_failing\": 0,
|
|
\"intermittent\": 0,
|
|
\"success_rate\": 0.0
|
|
}
|
|
}"
|
|
|
|
# Analyze each test using zsh associative array iteration
|
|
for test_name in ${(k)test_results}; do
|
|
local passes=${test_successes[$test_name]:-0}
|
|
local fails=${test_failures[$test_name]:-0}
|
|
local total=$((passes + fails))
|
|
local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0")
|
|
|
|
# Determine test stability
|
|
local stability=""
|
|
if [ "$fails" -eq 0 ]; then
|
|
stability="always_passing"
|
|
elif [ "$passes" -eq 0 ]; then
|
|
stability="always_failing"
|
|
else
|
|
stability="intermittent"
|
|
fi
|
|
|
|
# Add to summary using jq
|
|
summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \
|
|
--arg stability "$stability" \
|
|
--arg passes "$passes" \
|
|
--arg fails "$fails" \
|
|
--arg total "$total" \
|
|
--arg rate "$success_rate" \
|
|
'.test_results[$test] = {
|
|
"stability": $stability,
|
|
"passes": ($passes | tonumber),
|
|
"fails": ($fails | tonumber),
|
|
"total": ($total | tonumber),
|
|
"success_rate": ($rate | tonumber)
|
|
}')
|
|
done
|
|
|
|
# Calculate summary statistics
|
|
local total_tests=$(echo "$summary_data" | jq '.test_results | length')
|
|
local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length')
|
|
local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length')
|
|
local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length')
|
|
|
|
summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \
|
|
--arg passing "$always_passing" \
|
|
--arg failing "$always_failing" \
|
|
--arg intermittent "$intermittent" \
|
|
'.summary_stats.total_tests = ($total | tonumber) |
|
|
.summary_stats.always_passing = ($passing | tonumber) |
|
|
.summary_stats.always_failing = ($failing | tonumber) |
|
|
.summary_stats.intermittent = ($intermittent | tonumber)')
|
|
|
|
# Save summary
|
|
echo "$summary_data" | jq '.' > "${SUMMARY_FILE}"
|
|
|
|
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}"
|
|
}
|
|
|
|
# Function to generate detailed report
|
|
generate_report() {
|
|
local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
|
|
|
{
|
|
echo "# TimeSafari Test Stability Report"
|
|
echo ""
|
|
echo "**Generated:** $(date)"
|
|
echo "**Total Runs:** $TOTAL_RUNS"
|
|
# Calculate duration with proper error handling
|
|
local current_time=$(date +%s)
|
|
local duration=0
|
|
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then
|
|
duration=$((current_time - START_TIME))
|
|
fi
|
|
echo "**Duration:** ${duration} seconds"
|
|
echo ""
|
|
|
|
# Summary statistics
|
|
echo "## Summary Statistics"
|
|
echo ""
|
|
local summary_data=$(cat "${SUMMARY_FILE}")
|
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
|
|
|
echo "- **Total Tests:** $total_tests"
|
|
echo "- **Always Passing:** $always_passing"
|
|
echo "- **Always Failing:** $always_failing"
|
|
echo "- **Intermittent:** $intermittent"
|
|
echo ""
|
|
|
|
# Always failing tests
|
|
echo "## Always Failing Tests"
|
|
echo ""
|
|
local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"')
|
|
if [ -n "$failing_tests" ]; then
|
|
echo "$failing_tests"
|
|
else
|
|
echo "No always failing tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Intermittent tests
|
|
echo "## Intermittent Tests (Most Unstable First)"
|
|
echo ""
|
|
local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"')
|
|
if [ -n "$intermittent_tests" ]; then
|
|
echo "$intermittent_tests"
|
|
else
|
|
echo "No intermittent tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Always passing tests
|
|
echo "## Always Passing Tests"
|
|
echo ""
|
|
local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key')
|
|
if [ -n "$passing_tests" ]; then
|
|
echo "$passing_tests"
|
|
else
|
|
echo "No always passing tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Detailed test results
|
|
echo "## Detailed Test Results"
|
|
echo ""
|
|
echo "| Test Name | Stability | Passes | Fails | Success Rate |"
|
|
echo "|-----------|-----------|--------|-------|--------------|"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"'
|
|
echo ""
|
|
|
|
# Run-by-run summary
|
|
echo "## Run-by-Run Summary"
|
|
echo ""
|
|
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
|
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
|
if [ -f "$run_file" ]; then
|
|
# Extract passed and failed counts using the same method as the main script
|
|
local passed=0
|
|
local failed=0
|
|
|
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1)
|
|
if [ -n "$passed_line" ]; then
|
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
|
if [ -n "$failed_line" ]; then
|
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
local total=$((passed + failed))
|
|
echo "**Run $i:** $passed passed, $failed failed ($total total)"
|
|
fi
|
|
done
|
|
|
|
} > "$report_file"
|
|
|
|
log_success "Detailed report generated: $report_file"
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
START_TIME=$(date +%s)
|
|
|
|
log_info "Starting TimeSafari Test Stability Runner (Zsh Version)"
|
|
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}"
|
|
log_info "Log file: ${LOG_FILE}"
|
|
|
|
# Check prerequisites
|
|
log_info "Checking prerequisites..."
|
|
if ! command -v jq &> /dev/null; then
|
|
log_error "jq is required but not installed. Please install jq."
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v bc &> /dev/null; then
|
|
log_error "bc is required but not installed. Please install bc."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if Playwright is available
|
|
if ! npx playwright --version &> /dev/null; then
|
|
log_error "Playwright is not available. Please install dependencies."
|
|
exit 1
|
|
fi
|
|
|
|
log_success "Prerequisites check passed"
|
|
|
|
echo ""
|
|
log_info "Starting test execution with progress tracking..."
|
|
echo ""
|
|
|
|
# Run tests multiple times
|
|
for ((run=1; run<=TOTAL_RUNS; run++)); do
|
|
log_info "Starting run $run/$TOTAL_RUNS"
|
|
|
|
local run_start=$(date +%s)
|
|
local run_output="${RESULTS_DIR}/run-${run}.txt"
|
|
|
|
# Show overall run progress
|
|
show_progress "$run" "$TOTAL_RUNS" 40 "RUN $run/$TOTAL_RUNS"
|
|
|
|
# Run the test suite with individual test progress
|
|
echo ""
|
|
log_info "Executing test suite for run $run..."
|
|
|
|
# Create a temporary file to capture Playwright output
|
|
local temp_output=$(mktemp)
|
|
|
|
# Start progress tracking in background
|
|
track_test_progress "$run" "$temp_output" &
|
|
local progress_pid=$!
|
|
|
|
# Run Playwright with progress tracking
|
|
if npx playwright test -c playwright.config-local.ts --reporter=list > "$temp_output" 2>&1; then
|
|
# Wait for progress tracking to finish
|
|
wait $progress_pid 2>/dev/null || true
|
|
|
|
# Copy the output to our results file
|
|
cp "$temp_output" "$run_output"
|
|
log_success "Run $run completed successfully"
|
|
else
|
|
# Wait for progress tracking to finish
|
|
wait $progress_pid 2>/dev/null || true
|
|
|
|
# Copy the output to our results file even if it failed
|
|
cp "$temp_output" "$run_output"
|
|
log_warning "Run $run completed with failures"
|
|
fi
|
|
|
|
# Ensure progress tracking is stopped
|
|
kill $progress_pid 2>/dev/null || true
|
|
|
|
# Clean up temp file
|
|
rm -f "$temp_output"
|
|
|
|
local run_end=$(date +%s)
|
|
local run_duration=$((run_end - run_start))
|
|
run_times[$run]=$run_duration
|
|
|
|
# Show run summary
|
|
echo ""
|
|
log_info "Run $run completed in ${run_duration}s"
|
|
|
|
# Extract and display quick summary for this run
|
|
local passed=0
|
|
local failed=0
|
|
|
|
# Extract passed count from the last line containing "passed"
|
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1)
|
|
if [ -n "$passed_line" ]; then
|
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
# Extract failed count from the last line containing "failed"
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1)
|
|
if [ -n "$failed_line" ]; then
|
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
local total=$((passed + failed))
|
|
echo " ${GREEN}✓${NC} $passed passed, ${RED}✗${NC} $failed failed (${CYAN}$total${NC} total)"
|
|
|
|
# Extract and track test results using zsh array handling
|
|
local test_names=($(extract_test_names "$run_output"))
|
|
local test_count=${#test_names[@]}
|
|
local processed_tests=0
|
|
|
|
for test_name in $test_names; do
|
|
processed_tests=$((processed_tests + 1))
|
|
|
|
# Show progress for test analysis
|
|
show_progress "$processed_tests" "$test_count" 30 "ANALYZING"
|
|
|
|
if test_passed_in_run "$test_name" "$run_output"; then
|
|
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
|
test_results[$test_name]="pass"
|
|
elif test_failed_in_run "$test_name" "$run_output"; then
|
|
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
|
test_results[$test_name]="fail"
|
|
|
|
# Log failure details
|
|
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG"
|
|
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true
|
|
echo "" >> "$FAILURE_LOG"
|
|
fi
|
|
done
|
|
|
|
clear_progress
|
|
|
|
# Show detailed failed tests for this run
|
|
if [ "$failed" -gt 0 ]; then
|
|
log_warning "Failed tests in run $run:"
|
|
# Extract failed test names from the summary section
|
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do
|
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
|
log_warning " - $test_name"
|
|
done
|
|
else
|
|
log_success "All tests passed in run $run"
|
|
fi
|
|
|
|
echo ""
|
|
done
|
|
|
|
# Analyze results
|
|
log_info "Analyzing test results..."
|
|
show_progress "1" "3" 40 "ANALYSIS"
|
|
analyze_results
|
|
|
|
# Generate detailed report
|
|
show_progress "2" "3" 40 "REPORT"
|
|
generate_report
|
|
|
|
show_progress "3" "3" 40 "COMPLETE"
|
|
clear_progress
|
|
|
|
echo ""
|
|
log_success "Test stability analysis complete!"
|
|
|
|
# Final summary
|
|
local total_duration=$(($(date +%s) - START_TIME))
|
|
log_info "Total duration: ${total_duration}s"
|
|
log_info "Results saved to: ${RESULTS_DIR}"
|
|
log_info "Summary: ${SUMMARY_FILE}"
|
|
log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
|
log_info "Failure details: ${FAILURE_LOG}"
|
|
|
|
# Display quick summary
|
|
echo ""
|
|
echo "=== QUICK SUMMARY ==="
|
|
local summary_data=$(cat "${SUMMARY_FILE}")
|
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
|
|
|
echo "Total Tests: $total_tests"
|
|
echo "Always Passing: $always_passing"
|
|
echo "Always Failing: $always_failing"
|
|
echo "Intermittent: $intermittent"
|
|
|
|
# Show run-by-run failure summary
|
|
echo ""
|
|
echo "=== RUN-BY-RUN FAILURE SUMMARY ==="
|
|
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
|
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
|
if [ -f "$run_file" ]; then
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
|
local failed_count=0
|
|
if [ -n "$failed_line" ]; then
|
|
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
echo "Run $i: $failed_count failed"
|
|
# Extract failed test names from the summary section
|
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do
|
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
|
echo " - $test_name"
|
|
done
|
|
else
|
|
echo "Run $i: All tests passed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$always_failing" -gt 0 ]; then
|
|
echo ""
|
|
echo "🚨 ALWAYS FAILING TESTS:"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key'
|
|
fi
|
|
|
|
if [ "$intermittent" -gt 0 ]; then
|
|
echo ""
|
|
echo "⚠️ INTERMITTENT TESTS (most unstable first):"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"'
|
|
fi
|
|
}
|
|
|
|
# Run the main function
|
|
main "$@"
|