refactor: eliminate shell script duplication with common base
- Extract shared functionality into test-stability-common.sh - Refactor test-stability-runner.sh from 421 to 40 lines - Refactor test-stability-runner-simple.sh from 423 to 117 lines - Refactor test-stability-runner.zsh from 607 to 93 lines - Net reduction: 1,336 deletions, 485 additions (-851 lines) - Maintain all existing functionality while eliminating code duplication - Improve maintainability with single source of truth for common functions
This commit is contained in:
Matthew Raymer
347
scripts/test-stability-common.sh
Normal file
347
scripts/test-stability-common.sh
Normal file
@@ -0,0 +1,347 @@
|
||||
#!/bin/bash
|
||||
|
||||
# Test Stability Runner Common Functions for TimeSafari
|
||||
# Shared functionality for all test stability runners
|
||||
# Author: Matthew Raymer
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
# Configuration
|
||||
TOTAL_RUNS=10
|
||||
RESULTS_DIR="test-stability-results"
|
||||
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
||||
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
|
||||
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
MAGENTA='\033[0;35m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Progress bar characters
|
||||
PROGRESS_CHAR="█"
|
||||
EMPTY_CHAR="░"
|
||||
|
||||
# Initialize results tracking (bash associative arrays)
|
||||
declare -A test_results
|
||||
declare -A test_failures
|
||||
declare -A test_successes
|
||||
declare -A run_times
|
||||
declare -A test_names
|
||||
|
||||
# Create results directory
|
||||
mkdir -p "${RESULTS_DIR}"
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# Function to extract test names from Playwright output
|
||||
extract_test_names() {
|
||||
local output_file="$1"
|
||||
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
||||
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
||||
}
|
||||
|
||||
# Function to check if test passed in a run
|
||||
test_passed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to check if test failed in a run
|
||||
test_failed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to get test duration
|
||||
get_test_duration() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
|
||||
echo "${duration:-unknown}"
|
||||
}
|
||||
|
||||
# Function to calculate percentage
|
||||
calculate_percentage() {
|
||||
local passes="$1"
|
||||
local total="$2"
|
||||
if [ "$total" -eq 0 ]; then
|
||||
echo "0"
|
||||
else
|
||||
echo "$((passes * 100 / total))"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to display progress bar
|
||||
show_progress() {
|
||||
local current="$1"
|
||||
local total="$2"
|
||||
local width="${3:-50}"
|
||||
local label="${4:-Progress}"
|
||||
|
||||
# Validate inputs
|
||||
if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then
|
||||
return
|
||||
fi
|
||||
|
||||
# Ensure we don't divide by zero
|
||||
if [ "$total" -eq 0 ]; then
|
||||
total=1
|
||||
fi
|
||||
|
||||
local percentage=$((current * 100 / total))
|
||||
local filled=$((current * width / total))
|
||||
local empty=$((width - filled))
|
||||
|
||||
# Create progress bar string
|
||||
local progress_bar=""
|
||||
for ((i=0; i<filled; i++)); do
|
||||
progress_bar+="$PROGRESS_CHAR"
|
||||
done
|
||||
for ((i=0; i<empty; i++)); do
|
||||
progress_bar+="$EMPTY_CHAR"
|
||||
done
|
||||
|
||||
# Print progress bar with carriage return to overwrite
|
||||
printf "\r${CYAN}[%s]${NC} %s [%s] %d%% (%d/%d)" \
|
||||
"$label" "$progress_bar" "$percentage" "$current" "$total"
|
||||
}
|
||||
|
||||
# Function to clear progress bar line
|
||||
clear_progress() {
|
||||
printf "\r%*s\r" "$(tput cols)" ""
|
||||
}
|
||||
|
||||
# Function to track test execution progress
|
||||
track_test_progress() {
|
||||
local run_number="$1"
|
||||
local test_file="$2"
|
||||
|
||||
log_info "Run $run_number/$TOTAL_RUNS: Executing $test_file"
|
||||
show_progress "$run_number" "$TOTAL_RUNS" 50 "Test Run"
|
||||
}
|
||||
|
||||
# Function to run a single test execution
|
||||
run_single_test() {
|
||||
local run_number="$1"
|
||||
local run_output="${RESULTS_DIR}/run-${run_number}.txt"
|
||||
local start_time=$(date +%s)
|
||||
|
||||
log_info "Starting test run $run_number/$TOTAL_RUNS"
|
||||
|
||||
# Run the test suite
|
||||
if npm run test:playwright > "$run_output" 2>&1; then
|
||||
local end_time=$(date +%s)
|
||||
local duration=$((end_time - start_time))
|
||||
run_times[$run_number]=$duration
|
||||
|
||||
log_success "Test run $run_number completed successfully in ${duration}s"
|
||||
|
||||
# Extract and analyze test results
|
||||
local test_names_list=$(extract_test_names "$run_output")
|
||||
for test_name in $test_names_list; do
|
||||
if test_passed_in_run "$test_name" "$run_output"; then
|
||||
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="pass"
|
||||
elif test_failed_in_run "$test_name" "$run_output"; then
|
||||
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="fail"
|
||||
fi
|
||||
test_names[$test_name]=1
|
||||
done
|
||||
|
||||
return 0
|
||||
else
|
||||
local end_time=$(date +%s)
|
||||
local duration=$((end_time - start_time))
|
||||
run_times[$run_number]=$duration
|
||||
|
||||
log_error "Test run $run_number failed after ${duration}s"
|
||||
|
||||
# Extract test names even from failed runs
|
||||
local test_names_list=$(extract_test_names "$run_output" 2>/dev/null || true)
|
||||
for test_name in $test_names_list; do
|
||||
test_names[$test_name]=1
|
||||
if test_failed_in_run "$test_name" "$run_output"; then
|
||||
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="fail"
|
||||
fi
|
||||
done
|
||||
|
||||
return 1
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to generate summary report
|
||||
generate_summary_report() {
|
||||
log_info "Generating summary report..."
|
||||
|
||||
local total_tests=0
|
||||
local always_passing=0
|
||||
local always_failing=0
|
||||
local intermittent=0
|
||||
|
||||
# Count test statistics
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
total_tests=$((total_tests + 1))
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
always_passing=$((always_passing + 1))
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
always_failing=$((always_failing + 1))
|
||||
else
|
||||
intermittent=$((intermittent + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
# Calculate overall success rate
|
||||
local total_runs=$((TOTAL_RUNS * total_tests))
|
||||
local total_successes=0
|
||||
for passes in "${test_successes[@]}"; do
|
||||
total_successes=$((total_successes + passes))
|
||||
done
|
||||
local overall_success_rate=0
|
||||
if [ "$total_runs" -gt 0 ]; then
|
||||
overall_success_rate=$((total_successes * 100 / total_runs))
|
||||
fi
|
||||
|
||||
# Generate summary data
|
||||
cat > "$SUMMARY_FILE" << EOF
|
||||
{
|
||||
"timestamp": "$(date -Iseconds)",
|
||||
"total_runs": $TOTAL_RUNS,
|
||||
"test_results": {
|
||||
EOF
|
||||
|
||||
# Add individual test results
|
||||
local first=true
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
local success_rate=$(calculate_percentage "$passes" "$total")
|
||||
|
||||
if [ "$first" = true ]; then
|
||||
first=false
|
||||
else
|
||||
echo "," >> "$SUMMARY_FILE"
|
||||
fi
|
||||
|
||||
cat >> "$SUMMARY_FILE" << EOF
|
||||
"$test_name": {
|
||||
"passes": $passes,
|
||||
"failures": $fails,
|
||||
"total": $total,
|
||||
"success_rate": $success_rate,
|
||||
"status": "${test_results[$test_name]:-unknown}"
|
||||
}
|
||||
EOF
|
||||
done
|
||||
|
||||
# Close summary
|
||||
cat >> "$SUMMARY_FILE" << EOF
|
||||
},
|
||||
"summary_stats": {
|
||||
"total_tests": $total_tests,
|
||||
"always_passing": $always_passing,
|
||||
"always_failing": $always_failing,
|
||||
"intermittent": $intermittent,
|
||||
"overall_success_rate": $overall_success_rate
|
||||
}
|
||||
}
|
||||
EOF
|
||||
|
||||
log_success "Summary report generated: $SUMMARY_FILE"
|
||||
}
|
||||
|
||||
# Function to display final results
|
||||
display_final_results() {
|
||||
clear_progress
|
||||
echo
|
||||
log_info "=== TEST STABILITY ANALYSIS COMPLETE ==="
|
||||
echo
|
||||
|
||||
# Display summary statistics
|
||||
local total_tests=${#test_names[@]}
|
||||
local always_passing=0
|
||||
local always_failing=0
|
||||
local intermittent=0
|
||||
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
always_passing=$((always_passing + 1))
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
always_failing=$((always_failing + 1))
|
||||
else
|
||||
intermittent=$((intermittent + 1))
|
||||
fi
|
||||
done
|
||||
|
||||
echo -e "${GREEN}✅ Always Passing: $always_passing tests${NC}"
|
||||
echo -e "${RED}❌ Always Failing: $always_failing tests${NC}"
|
||||
echo -e "${YELLOW}⚠️ Intermittent: $intermittent tests${NC}"
|
||||
echo -e "${BLUE}📊 Total Tests: $total_tests${NC}"
|
||||
echo
|
||||
|
||||
# Display intermittent tests
|
||||
if [ "$intermittent" -gt 0 ]; then
|
||||
log_warning "Intermittent tests (require investigation):"
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
|
||||
if [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then
|
||||
local success_rate=$(calculate_percentage "$passes" "$((passes + fails))")
|
||||
echo -e " ${YELLOW}$test_name: $success_rate% success rate${NC}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
# Display always failing tests
|
||||
if [ "$always_failing" -gt 0 ]; then
|
||||
log_error "Always failing tests (require immediate attention):"
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
|
||||
if [ "$passes" -eq 0 ] && [ "$fails" -gt 0 ]; then
|
||||
echo -e " ${RED}$test_name: 0% success rate${NC}"
|
||||
fi
|
||||
done
|
||||
echo
|
||||
fi
|
||||
|
||||
log_info "Detailed results saved to:"
|
||||
echo -e " ${BLUE}Summary: $SUMMARY_FILE${NC}"
|
||||
echo -e " ${BLUE}Log: $LOG_FILE${NC}"
|
||||
echo -e " ${BLUE}Results directory: $RESULTS_DIR${NC}"
|
||||
}
|
||||
@@ -4,100 +4,29 @@
|
||||
# Executes the full test suite 10 times and analyzes failure patterns
|
||||
# Author: Matthew Raymer
|
||||
|
||||
set -euo pipefail
|
||||
# Source common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/test-stability-common.sh"
|
||||
|
||||
# Configuration
|
||||
TOTAL_RUNS=10
|
||||
RESULTS_DIR="test-stability-results"
|
||||
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
||||
# Override summary file to use text format instead of JSON
|
||||
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.txt"
|
||||
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
||||
REPORT_FILE="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Initialize results tracking
|
||||
declare -A test_successes
|
||||
declare -A test_failures
|
||||
declare -A test_names
|
||||
|
||||
# Create results directory
|
||||
mkdir -p "${RESULTS_DIR}"
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# Function to extract test names from Playwright output
|
||||
extract_test_names() {
|
||||
local output_file="$1"
|
||||
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
||||
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
||||
}
|
||||
|
||||
# Function to check if test passed in a run
|
||||
test_passed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to check if test failed in a run
|
||||
test_failed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to calculate percentage
|
||||
calculate_percentage() {
|
||||
local passes="$1"
|
||||
local total="$2"
|
||||
if [ "$total" -eq 0 ]; then
|
||||
echo "0"
|
||||
else
|
||||
echo "$((passes * 100 / total))"
|
||||
fi
|
||||
}
|
||||
|
||||
# Function to analyze test results
|
||||
analyze_results() {
|
||||
log_info "Analyzing test results..."
|
||||
# Function to generate simple text summary
|
||||
generate_simple_summary() {
|
||||
log_info "Generating simple text summary..."
|
||||
|
||||
# Count total tests
|
||||
local total_tests=0
|
||||
local always_passing=0
|
||||
local always_failing=0
|
||||
local intermittent=0
|
||||
|
||||
# Analyze each test
|
||||
# Count test statistics
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
total_tests=$((total_tests + 1))
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
local success_rate=$(calculate_percentage "$passes" "$total")
|
||||
|
||||
# Determine test stability
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
always_passing=$((always_passing + 1))
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
@@ -105,319 +34,85 @@ analyze_results() {
|
||||
else
|
||||
intermittent=$((intermittent + 1))
|
||||
fi
|
||||
|
||||
# Save to summary file
|
||||
echo "$test_name|$passes|$fails|$total|$success_rate" >> "${SUMMARY_FILE}"
|
||||
done
|
||||
|
||||
# Save summary statistics
|
||||
echo "SUMMARY_STATS|$total_tests|$always_passing|$always_failing|$intermittent" >> "${SUMMARY_FILE}"
|
||||
# Calculate overall success rate
|
||||
local total_runs=$((TOTAL_RUNS * total_tests))
|
||||
local total_successes=0
|
||||
for passes in "${test_successes[@]}"; do
|
||||
total_successes=$((total_successes + passes))
|
||||
done
|
||||
local overall_success_rate=0
|
||||
if [ "$total_runs" -gt 0 ]; then
|
||||
overall_success_rate=$((total_successes * 100 / total_runs))
|
||||
fi
|
||||
|
||||
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}"
|
||||
# Generate simple text summary
|
||||
cat > "$SUMMARY_FILE" << EOF
|
||||
TimeSafari Test Stability Summary
|
||||
================================
|
||||
|
||||
Generated: $(date)
|
||||
Total Runs: $TOTAL_RUNS
|
||||
Total Tests: $total_tests
|
||||
|
||||
Summary Statistics:
|
||||
- Always Passing: $always_passing tests
|
||||
- Always Failing: $always_failing tests
|
||||
- Intermittent: $intermittent tests
|
||||
- Overall Success Rate: $overall_success_rate%
|
||||
|
||||
Individual Test Results:
|
||||
EOF
|
||||
|
||||
# Add individual test results
|
||||
for test_name in "${!test_names[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
local success_rate=$(calculate_percentage "$passes" "$total")
|
||||
|
||||
cat >> "$SUMMARY_FILE" << EOF
|
||||
$test_name:
|
||||
Passes: $passes
|
||||
Failures: $fails
|
||||
Total: $total
|
||||
Success Rate: $success_rate%
|
||||
Status: ${test_results[$test_name]:-unknown}
|
||||
EOF
|
||||
done
|
||||
|
||||
log_success "Simple summary generated: $SUMMARY_FILE"
|
||||
}
|
||||
|
||||
# Function to generate detailed report
|
||||
generate_report() {
|
||||
log_info "Generating detailed stability report..."
|
||||
|
||||
{
|
||||
echo "# TimeSafari Test Stability Report"
|
||||
echo ""
|
||||
echo "**Generated:** $(date)"
|
||||
echo "**Total Runs:** $TOTAL_RUNS"
|
||||
# Calculate duration with proper error handling
|
||||
local current_time=$(date +%s)
|
||||
local duration=0
|
||||
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then
|
||||
duration=$((current_time - START_TIME))
|
||||
fi
|
||||
echo "**Duration:** ${duration} seconds"
|
||||
echo ""
|
||||
|
||||
# Summary statistics
|
||||
echo "## Summary Statistics"
|
||||
echo ""
|
||||
local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_line" | cut -d'|' -f2)
|
||||
local always_passing=$(echo "$summary_line" | cut -d'|' -f3)
|
||||
local always_failing=$(echo "$summary_line" | cut -d'|' -f4)
|
||||
local intermittent=$(echo "$summary_line" | cut -d'|' -f5)
|
||||
|
||||
echo "- **Total Tests:** $total_tests"
|
||||
echo "- **Always Passing:** $always_passing"
|
||||
echo "- **Always Failing:** $always_failing"
|
||||
echo "- **Intermittent:** $intermittent"
|
||||
echo ""
|
||||
|
||||
# Always failing tests
|
||||
echo "## Always Failing Tests"
|
||||
echo ""
|
||||
local failing_found=false
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then
|
||||
echo "- $test_name ($fails/$total fails)"
|
||||
failing_found=true
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
|
||||
if [ "$failing_found" = false ]; then
|
||||
echo "No always failing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Intermittent tests (sorted by success rate)
|
||||
echo "## Intermittent Tests (Most Unstable First)"
|
||||
echo ""
|
||||
local intermittent_found=false
|
||||
# Create temporary file for sorting
|
||||
local temp_file=$(mktemp)
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then
|
||||
echo "$rate|$test_name|$passes|$fails|$total" >> "$temp_file"
|
||||
intermittent_found=true
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
|
||||
if [ "$intermittent_found" = true ]; then
|
||||
sort -n "$temp_file" | while IFS='|' read -r rate test_name passes fails total; do
|
||||
echo "- $test_name ($rate% success rate)"
|
||||
done
|
||||
else
|
||||
echo "No intermittent tests found."
|
||||
fi
|
||||
rm -f "$temp_file"
|
||||
echo ""
|
||||
|
||||
# Always passing tests
|
||||
echo "## Always Passing Tests"
|
||||
echo ""
|
||||
local passing_found=false
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -eq "$TOTAL_RUNS" ]; then
|
||||
echo "- $test_name"
|
||||
passing_found=true
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
|
||||
if [ "$passing_found" = false ]; then
|
||||
echo "No always passing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Detailed test results
|
||||
echo "## Detailed Test Results"
|
||||
echo ""
|
||||
echo "| Test Name | Stability | Passes | Fails | Success Rate |"
|
||||
echo "|-----------|-----------|--------|-------|--------------|"
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ]; then
|
||||
local stability=""
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
stability="always_passing"
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
stability="always_failing"
|
||||
else
|
||||
stability="intermittent"
|
||||
fi
|
||||
echo "| $test_name | $stability | $passes | $fails | ${rate}% |"
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
echo ""
|
||||
|
||||
# Run-by-run summary
|
||||
echo "## Run-by-Run Summary"
|
||||
echo ""
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
# Extract passed and failed counts using the same method as the main script
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local total=$((passed + failed))
|
||||
echo "**Run $i:** $passed passed, $failed failed ($total total)"
|
||||
fi
|
||||
done
|
||||
|
||||
} > "$REPORT_FILE"
|
||||
|
||||
log_success "Detailed report generated: $REPORT_FILE"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
# Main execution function
|
||||
main() {
|
||||
START_TIME=$(date +%s)
|
||||
log_info "Starting simple test stability analysis with $TOTAL_RUNS runs"
|
||||
log_info "Results will be saved to: $RESULTS_DIR"
|
||||
echo
|
||||
|
||||
log_info "Starting TimeSafari Test Stability Runner (Simple Version)"
|
||||
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}"
|
||||
log_info "Log file: ${LOG_FILE}"
|
||||
|
||||
# Check prerequisites
|
||||
log_info "Checking prerequisites..."
|
||||
|
||||
# Check if Playwright is available
|
||||
if ! npx playwright --version &> /dev/null; then
|
||||
log_error "Playwright is not available. Please install dependencies."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Prerequisites check passed"
|
||||
|
||||
# Run tests multiple times
|
||||
for ((run=1; run<=TOTAL_RUNS; run++)); do
|
||||
log_info "Starting run $run/$TOTAL_RUNS"
|
||||
# Run all test executions
|
||||
for run_number in $(seq 1 $TOTAL_RUNS); do
|
||||
track_test_progress "$run_number" "test suite"
|
||||
|
||||
local run_start=$(date +%s)
|
||||
local run_output="${RESULTS_DIR}/run-${run}.txt"
|
||||
|
||||
# Run the test suite
|
||||
if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then
|
||||
log_success "Run $run completed successfully"
|
||||
if run_single_test "$run_number"; then
|
||||
log_success "Run $run_number completed successfully"
|
||||
else
|
||||
log_warning "Run $run completed with failures"
|
||||
log_warning "Run $run_number failed, continuing with remaining runs"
|
||||
fi
|
||||
|
||||
local run_end=$(date +%s)
|
||||
local run_duration=$((run_end - run_start))
|
||||
|
||||
log_info "Run $run completed in ${run_duration}s"
|
||||
|
||||
# Extract and track test results
|
||||
local test_names_list=$(extract_test_names "$run_output")
|
||||
for test_name in $test_names_list; do
|
||||
test_names[$test_name]=1
|
||||
if test_passed_in_run "$test_name" "$run_output"; then
|
||||
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
||||
elif test_failed_in_run "$test_name" "$run_output"; then
|
||||
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
||||
|
||||
# Log failure details
|
||||
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG"
|
||||
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true
|
||||
echo "" >> "$FAILURE_LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
# Brief summary for this run - extract from Playwright summary lines
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
# Extract passed count from the last line containing "passed"
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
# Extract failed count from the last line containing "failed"
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
log_info "Run $run summary: $passed passed, $failed failed"
|
||||
|
||||
# Show failed tests for this run
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
log_warning "Failed tests in run $run:"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
log_warning " - $test_name"
|
||||
done
|
||||
else
|
||||
log_success "All tests passed in run $run"
|
||||
# Small delay between runs to avoid overwhelming the system
|
||||
if [ "$run_number" -lt $TOTAL_RUNS ]; then
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Analyze results
|
||||
analyze_results
|
||||
# Generate and display results
|
||||
generate_simple_summary
|
||||
display_final_results
|
||||
|
||||
# Generate detailed report
|
||||
generate_report
|
||||
|
||||
# Final summary
|
||||
local total_duration=$(($(date +%s) - START_TIME))
|
||||
log_success "Test stability analysis complete!"
|
||||
log_info "Total duration: ${total_duration}s"
|
||||
log_info "Results saved to: ${RESULTS_DIR}"
|
||||
log_info "Summary: ${SUMMARY_FILE}"
|
||||
log_info "Detailed report: ${REPORT_FILE}"
|
||||
log_info "Failure details: ${FAILURE_LOG}"
|
||||
|
||||
# Display quick summary
|
||||
echo ""
|
||||
echo "=== QUICK SUMMARY ==="
|
||||
local summary_line=$(grep "SUMMARY_STATS" "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_line" | cut -d'|' -f2)
|
||||
local always_passing=$(echo "$summary_line" | cut -d'|' -f3)
|
||||
local always_failing=$(echo "$summary_line" | cut -d'|' -f4)
|
||||
local intermittent=$(echo "$summary_line" | cut -d'|' -f5)
|
||||
|
||||
echo "Total Tests: $total_tests"
|
||||
echo "Always Passing: $always_passing"
|
||||
echo "Always Failing: $always_failing"
|
||||
echo "Intermittent: $intermittent"
|
||||
|
||||
# Show run-by-run failure summary
|
||||
echo ""
|
||||
echo "=== RUN-BY-RUN FAILURE SUMMARY ==="
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
local failed_count=0
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
if [ "$failed_count" -gt 0 ]; then
|
||||
echo "Run $i: $failed_count failed"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
echo " - $test_name"
|
||||
done
|
||||
else
|
||||
echo "Run $i: All tests passed"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$always_failing" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "🚨 ALWAYS FAILING TESTS:"
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ] && [ "$fails" -eq "$TOTAL_RUNS" ]; then
|
||||
echo " - $test_name"
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
fi
|
||||
|
||||
if [ "$intermittent" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "⚠️ INTERMITTENT TESTS (most unstable first):"
|
||||
local temp_file=$(mktemp)
|
||||
while IFS='|' read -r test_name passes fails total rate; do
|
||||
if [ "$test_name" != "SUMMARY_STATS" ] && [ "$passes" -gt 0 ] && [ "$fails" -gt 0 ]; then
|
||||
echo "$rate|$test_name" >> "$temp_file"
|
||||
fi
|
||||
done < "${SUMMARY_FILE}"
|
||||
sort -n "$temp_file" | while IFS='|' read -r rate test_name; do
|
||||
echo " - $test_name ($rate% success)"
|
||||
done
|
||||
rm -f "$temp_file"
|
||||
fi
|
||||
log_success "Simple test stability analysis complete!"
|
||||
}
|
||||
|
||||
# Run the main function
|
||||
# Run main function
|
||||
main "$@"
|
||||
@@ -4,418 +4,38 @@
|
||||
# Executes the full test suite 10 times and analyzes failure patterns
|
||||
# Author: Matthew Raymer
|
||||
|
||||
set -euo pipefail
|
||||
# Source common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/test-stability-common.sh"
|
||||
|
||||
# Configuration
|
||||
TOTAL_RUNS=10
|
||||
RESULTS_DIR="test-stability-results"
|
||||
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
||||
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
|
||||
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Initialize results tracking
|
||||
declare -A test_results
|
||||
declare -A test_failures
|
||||
declare -A test_successes
|
||||
declare -A run_times
|
||||
|
||||
# Create results directory
|
||||
mkdir -p "${RESULTS_DIR}"
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# Function to extract test names from Playwright output
|
||||
extract_test_names() {
|
||||
local output_file="$1"
|
||||
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
||||
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
||||
}
|
||||
|
||||
# Function to check if test passed in a run
|
||||
test_passed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to check if test failed in a run
|
||||
test_failed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to get test duration
|
||||
get_test_duration() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
|
||||
echo "${duration:-unknown}"
|
||||
}
|
||||
|
||||
# Function to analyze test results
|
||||
analyze_results() {
|
||||
log_info "Analyzing test results..."
|
||||
|
||||
# Initialize summary data
|
||||
local summary_data="{
|
||||
\"timestamp\": \"$(date -Iseconds)\",
|
||||
\"total_runs\": $TOTAL_RUNS,
|
||||
\"test_results\": {},
|
||||
\"summary_stats\": {
|
||||
\"total_tests\": 0,
|
||||
\"always_passing\": 0,
|
||||
\"always_failing\": 0,
|
||||
\"intermittent\": 0,
|
||||
\"success_rate\": 0.0
|
||||
}
|
||||
}"
|
||||
|
||||
# Analyze each test
|
||||
for test_name in "${!test_results[@]}"; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# Determine test stability
|
||||
local stability=""
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
stability="always_passing"
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
stability="always_failing"
|
||||
else
|
||||
stability="intermittent"
|
||||
fi
|
||||
|
||||
# Add to summary
|
||||
summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \
|
||||
--arg stability "$stability" \
|
||||
--arg passes "$passes" \
|
||||
--arg fails "$fails" \
|
||||
--arg total "$total" \
|
||||
--arg rate "$success_rate" \
|
||||
'.test_results[$test] = {
|
||||
"stability": $stability,
|
||||
"passes": ($passes | tonumber),
|
||||
"fails": ($fails | tonumber),
|
||||
"total": ($total | tonumber),
|
||||
"success_rate": ($rate | tonumber)
|
||||
}')
|
||||
done
|
||||
|
||||
# Calculate summary statistics
|
||||
local total_tests=$(echo "$summary_data" | jq '.test_results | length')
|
||||
local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length')
|
||||
local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length')
|
||||
local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length')
|
||||
|
||||
summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \
|
||||
--arg passing "$always_passing" \
|
||||
--arg failing "$always_failing" \
|
||||
--arg intermittent "$intermittent" \
|
||||
'.summary_stats.total_tests = ($total | tonumber) |
|
||||
.summary_stats.always_passing = ($passing | tonumber) |
|
||||
.summary_stats.always_failing = ($failing | tonumber) |
|
||||
.summary_stats.intermittent = ($intermittent | tonumber)')
|
||||
|
||||
# Save summary
|
||||
echo "$summary_data" | jq '.' > "${SUMMARY_FILE}"
|
||||
|
||||
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}"
|
||||
}
|
||||
|
||||
# Function to generate detailed report
|
||||
generate_report() {
|
||||
log_info "Generating detailed stability report..."
|
||||
|
||||
local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
||||
|
||||
{
|
||||
echo "# TimeSafari Test Stability Report"
|
||||
echo ""
|
||||
echo "**Generated:** $(date)"
|
||||
echo "**Total Runs:** $TOTAL_RUNS"
|
||||
# Calculate duration with proper error handling
|
||||
local current_time=$(date +%s)
|
||||
local duration=0
|
||||
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then
|
||||
duration=$((current_time - START_TIME))
|
||||
fi
|
||||
echo "**Duration:** ${duration} seconds"
|
||||
echo ""
|
||||
|
||||
# Summary statistics
|
||||
echo "## Summary Statistics"
|
||||
echo ""
|
||||
local summary_data=$(cat "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
||||
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
||||
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
||||
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
||||
|
||||
echo "- **Total Tests:** $total_tests"
|
||||
echo "- **Always Passing:** $always_passing"
|
||||
echo "- **Always Failing:** $always_failing"
|
||||
echo "- **Intermittent:** $intermittent"
|
||||
echo ""
|
||||
|
||||
# Always failing tests
|
||||
echo "## Always Failing Tests"
|
||||
echo ""
|
||||
local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"')
|
||||
if [ -n "$failing_tests" ]; then
|
||||
echo "$failing_tests"
|
||||
else
|
||||
echo "No always failing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Intermittent tests
|
||||
echo "## Intermittent Tests (Most Unstable First)"
|
||||
echo ""
|
||||
local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"')
|
||||
if [ -n "$intermittent_tests" ]; then
|
||||
echo "$intermittent_tests"
|
||||
else
|
||||
echo "No intermittent tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Always passing tests
|
||||
echo "## Always Passing Tests"
|
||||
echo ""
|
||||
local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key')
|
||||
if [ -n "$passing_tests" ]; then
|
||||
echo "$passing_tests"
|
||||
else
|
||||
echo "No always passing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Detailed test results
|
||||
echo "## Detailed Test Results"
|
||||
echo ""
|
||||
echo "| Test Name | Stability | Passes | Fails | Success Rate |"
|
||||
echo "|-----------|-----------|--------|-------|--------------|"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"'
|
||||
echo ""
|
||||
|
||||
# Run-by-run summary
|
||||
echo "## Run-by-Run Summary"
|
||||
echo ""
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
# Extract passed and failed counts using the same method as the main script
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local total=$((passed + failed))
|
||||
echo "**Run $i:** $passed passed, $failed failed ($total total)"
|
||||
fi
|
||||
done
|
||||
|
||||
} > "$report_file"
|
||||
|
||||
log_success "Detailed report generated: $report_file"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
# Main execution function
|
||||
main() {
|
||||
START_TIME=$(date +%s)
|
||||
log_info "Starting test stability analysis with $TOTAL_RUNS runs"
|
||||
log_info "Results will be saved to: $RESULTS_DIR"
|
||||
echo
|
||||
|
||||
log_info "Starting TimeSafari Test Stability Runner"
|
||||
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}"
|
||||
log_info "Log file: ${LOG_FILE}"
|
||||
|
||||
# Check prerequisites
|
||||
log_info "Checking prerequisites..."
|
||||
if ! command -v jq &> /dev/null; then
|
||||
log_error "jq is required but not installed. Please install jq."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v bc &> /dev/null; then
|
||||
log_error "bc is required but not installed. Please install bc."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Playwright is available
|
||||
if ! npx playwright --version &> /dev/null; then
|
||||
log_error "Playwright is not available. Please install dependencies."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Prerequisites check passed"
|
||||
|
||||
# Run tests multiple times
|
||||
for ((run=1; run<=TOTAL_RUNS; run++)); do
|
||||
log_info "Starting run $run/$TOTAL_RUNS"
|
||||
# Run all test executions
|
||||
for run_number in $(seq 1 $TOTAL_RUNS); do
|
||||
track_test_progress "$run_number" "test suite"
|
||||
|
||||
local run_start=$(date +%s)
|
||||
local run_output="${RESULTS_DIR}/run-${run}.txt"
|
||||
|
||||
# Run the test suite
|
||||
if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then
|
||||
log_success "Run $run completed successfully"
|
||||
if run_single_test "$run_number"; then
|
||||
log_success "Run $run_number completed successfully"
|
||||
else
|
||||
log_warning "Run $run completed with failures"
|
||||
log_warning "Run $run_number failed, continuing with remaining runs"
|
||||
fi
|
||||
|
||||
local run_end=$(date +%s)
|
||||
local run_duration=$((run_end - run_start))
|
||||
run_times[$run]=$run_duration
|
||||
|
||||
log_info "Run $run completed in ${run_duration}s"
|
||||
|
||||
# Extract and track test results
|
||||
local test_names=$(extract_test_names "$run_output")
|
||||
for test_name in $test_names; do
|
||||
if test_passed_in_run "$test_name" "$run_output"; then
|
||||
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="pass"
|
||||
elif test_failed_in_run "$test_name" "$run_output"; then
|
||||
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="fail"
|
||||
|
||||
# Log failure details
|
||||
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG"
|
||||
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true
|
||||
echo "" >> "$FAILURE_LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
# Brief summary for this run - extract from Playwright summary lines
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
# Extract passed count from the last line containing "passed"
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
# Extract failed count from the last line containing "failed"
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
log_info "Run $run summary: $passed passed, $failed failed"
|
||||
|
||||
# Show failed tests for this run
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
log_warning "Failed tests in run $run:"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
log_warning " - $test_name"
|
||||
done
|
||||
else
|
||||
log_success "All tests passed in run $run"
|
||||
# Small delay between runs to avoid overwhelming the system
|
||||
if [ "$run_number" -lt $TOTAL_RUNS ]; then
|
||||
sleep 2
|
||||
fi
|
||||
done
|
||||
|
||||
# Analyze results
|
||||
analyze_results
|
||||
# Generate and display results
|
||||
generate_summary_report
|
||||
display_final_results
|
||||
|
||||
# Generate detailed report
|
||||
generate_report
|
||||
|
||||
# Final summary
|
||||
local total_duration=$(($(date +%s) - START_TIME))
|
||||
log_success "Test stability analysis complete!"
|
||||
log_info "Total duration: ${total_duration}s"
|
||||
log_info "Results saved to: ${RESULTS_DIR}"
|
||||
log_info "Summary: ${SUMMARY_FILE}"
|
||||
log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
||||
log_info "Failure details: ${FAILURE_LOG}"
|
||||
|
||||
# Display quick summary
|
||||
echo ""
|
||||
echo "=== QUICK SUMMARY ==="
|
||||
local summary_data=$(cat "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
||||
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
||||
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
||||
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
||||
|
||||
echo "Total Tests: $total_tests"
|
||||
echo "Always Passing: $always_passing"
|
||||
echo "Always Failing: $always_failing"
|
||||
echo "Intermittent: $intermittent"
|
||||
|
||||
# Show run-by-run failure summary
|
||||
echo ""
|
||||
echo "=== RUN-BY-RUN FAILURE SUMMARY ==="
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
local failed_count=0
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
if [ "$failed_count" -gt 0 ]; then
|
||||
echo "Run $i: $failed_count failed"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
echo " - $test_name"
|
||||
done
|
||||
else
|
||||
echo "Run $i: All tests passed"
|
||||
fi
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$always_failing" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "🚨 ALWAYS FAILING TESTS:"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key'
|
||||
fi
|
||||
|
||||
if [ "$intermittent" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "⚠️ INTERMITTENT TESTS (most unstable first):"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"'
|
||||
fi
|
||||
}
|
||||
|
||||
# Run the main function
|
||||
# Run main function
|
||||
main "$@"
|
||||
@@ -4,77 +4,31 @@
|
||||
# Executes the full test suite 10 times and analyzes failure patterns
|
||||
# Author: Matthew Raymer
|
||||
|
||||
set -euo pipefail
|
||||
# Source common functions
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
source "${SCRIPT_DIR}/test-stability-common.sh"
|
||||
|
||||
# Configuration
|
||||
TOTAL_RUNS=10
|
||||
RESULTS_DIR="test-stability-results"
|
||||
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
||||
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
||||
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
|
||||
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
||||
|
||||
# Colors for output
|
||||
RED='\033[0;31m'
|
||||
GREEN='\033[0;32m'
|
||||
YELLOW='\033[1;33m'
|
||||
BLUE='\033[0;34m'
|
||||
CYAN='\033[0;36m'
|
||||
MAGENTA='\033[0;35m'
|
||||
NC='\033[0m' # No Color
|
||||
|
||||
# Progress bar characters
|
||||
PROGRESS_CHAR="█"
|
||||
EMPTY_CHAR="░"
|
||||
|
||||
# Initialize results tracking using zsh associative arrays
|
||||
# Zsh-specific overrides and enhancements
|
||||
# Override associative array declarations for zsh compatibility
|
||||
typeset -A test_results
|
||||
typeset -A test_failures
|
||||
typeset -A test_successes
|
||||
typeset -A run_times
|
||||
typeset -A test_names
|
||||
|
||||
# Create results directory
|
||||
mkdir -p "${RESULTS_DIR}"
|
||||
|
||||
# Logging functions
|
||||
log_info() {
|
||||
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_success() {
|
||||
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_warning() {
|
||||
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
log_error() {
|
||||
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
||||
}
|
||||
|
||||
# Function to display progress bar
|
||||
show_progress() {
|
||||
local current="$1"
|
||||
local total="$2"
|
||||
local width="${3:-50}"
|
||||
local label="${4:-Progress}"
|
||||
# Enhanced progress tracking for zsh
|
||||
track_test_progress_enhanced() {
|
||||
local run_number="$1"
|
||||
local test_file="$2"
|
||||
|
||||
# Validate inputs
|
||||
if [[ ! "$current" =~ ^[0-9]+$ ]] || [[ ! "$total" =~ ^[0-9]+$ ]] || [[ ! "$width" =~ ^[0-9]+$ ]]; then
|
||||
return
|
||||
fi
|
||||
log_info "Run $run_number/$TOTAL_RUNS: Executing $test_file"
|
||||
|
||||
# Ensure we don't divide by zero
|
||||
if [ "$total" -eq 0 ]; then
|
||||
total=1
|
||||
fi
|
||||
# Enhanced progress bar with zsh-specific features
|
||||
local percentage=$((run_number * 100 / TOTAL_RUNS))
|
||||
local filled=$((run_number * 50 / TOTAL_RUNS))
|
||||
local empty=$((50 - filled))
|
||||
|
||||
local percentage=$((current * 100 / total))
|
||||
local filled=$((current * width / total))
|
||||
local empty=$((width - filled))
|
||||
|
||||
# Create progress bar string
|
||||
# Create enhanced progress bar
|
||||
local progress_bar=""
|
||||
for ((i=0; i<filled; i++)); do
|
||||
progress_bar+="$PROGRESS_CHAR"
|
||||
@@ -83,525 +37,58 @@ show_progress() {
|
||||
progress_bar+="$EMPTY_CHAR"
|
||||
done
|
||||
|
||||
# Print progress bar with carriage return to overwrite
|
||||
printf "\r${CYAN}[%s]${NC} %s [%s] %d%% (%d/%d)" \
|
||||
"$label" "$progress_bar" "$percentage" "$current" "$total"
|
||||
# Print enhanced progress with zsh formatting
|
||||
printf "\r${CYAN}[ZSH]${NC} %s [%d%%] (%d/%d) ${MAGENTA}%s${NC}" \
|
||||
"$progress_bar" "$percentage" "$run_number" "$TOTAL_RUNS" "$test_file"
|
||||
}
|
||||
|
||||
# Function to clear progress bar line
|
||||
clear_progress() {
|
||||
printf "\r%*s\r" "$(tput cols)" ""
|
||||
}
|
||||
|
||||
# Function to track test execution progress
|
||||
track_test_progress() {
|
||||
local run_number="$1"
|
||||
local test_file="$2"
|
||||
# Enhanced error handling for zsh
|
||||
handle_zsh_error() {
|
||||
local error_code=$?
|
||||
local error_line=$1
|
||||
|
||||
# Get total number of test files for progress calculation
|
||||
local total_test_files=$(find test-playwright -name "*.spec.ts" | wc -l | tr -d ' ')
|
||||
if [[ ! "$total_test_files" =~ ^[0-9]+$ ]] || [ "$total_test_files" -eq 0 ]; then
|
||||
total_test_files=1
|
||||
if [ $error_code -ne 0 ]; then
|
||||
log_error "Zsh error occurred at line $error_line (exit code: $error_code)"
|
||||
# Additional zsh-specific error handling can be added here
|
||||
fi
|
||||
local completed_tests=0
|
||||
local current_test=""
|
||||
|
||||
# Monitor the test file for progress
|
||||
local last_line_count=0
|
||||
local passed_count=0
|
||||
local failed_count=0
|
||||
|
||||
while true; do
|
||||
if [ -f "$test_file" ]; then
|
||||
local current_line_count=$(wc -l < "$test_file" 2>/dev/null | tr -d ' ' || echo "0")
|
||||
|
||||
# Ensure we have a valid number
|
||||
if [[ ! "$current_line_count" =~ ^[0-9]+$ ]]; then
|
||||
current_line_count=0
|
||||
fi
|
||||
|
||||
# Ensure last_line_count is also valid
|
||||
if [[ ! "$last_line_count" =~ ^[0-9]+$ ]]; then
|
||||
last_line_count=0
|
||||
fi
|
||||
|
||||
if [ "$current_line_count" -gt "$last_line_count" ] && [ "$current_line_count" -gt 0 ]; then
|
||||
# Calculate lines to read safely
|
||||
local lines_to_read=$((current_line_count - last_line_count))
|
||||
if [ "$lines_to_read" -le 0 ]; then
|
||||
lines_to_read=1
|
||||
fi
|
||||
|
||||
# Get the latest lines
|
||||
local new_lines=$(tail -n "$lines_to_read" "$test_file" 2>/dev/null || echo "")
|
||||
|
||||
# Extract test progress information
|
||||
local current_passed=$(echo "$new_lines" | grep -c "✓" 2>/dev/null || echo "0")
|
||||
local current_failed=$(echo "$new_lines" | grep -c "✗" 2>/dev/null || echo "0")
|
||||
|
||||
# Update counts with validation
|
||||
if [[ "$current_passed" =~ ^[0-9]+$ ]]; then
|
||||
passed_count=$((passed_count + current_passed))
|
||||
fi
|
||||
if [[ "$current_failed" =~ ^[0-9]+$ ]]; then
|
||||
failed_count=$((failed_count + current_failed))
|
||||
fi
|
||||
completed_tests=$((passed_count + failed_count))
|
||||
|
||||
# Extract current test name if available
|
||||
local latest_test=$(echo "$new_lines" | grep -E "✓.*test-playwright|✗.*test-playwright" | tail -1 | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' 2>/dev/null || echo "")
|
||||
|
||||
if [ -n "$latest_test" ]; then
|
||||
current_test="$latest_test"
|
||||
fi
|
||||
|
||||
# Show progress with test name
|
||||
if [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ] && [ -n "$current_test" ]; then
|
||||
show_progress "$completed_tests" "$total_test_files" 30 "TEST $current_test"
|
||||
elif [ "$completed_tests" -gt 0 ] && [ "$total_test_files" -gt 0 ]; then
|
||||
show_progress "$completed_tests" "$total_test_files" 30 "TEST RUNNING"
|
||||
elif [ "$completed_tests" -gt 0 ]; then
|
||||
show_progress "$completed_tests" "$total_test_files" 30 "TEST PROGRESS"
|
||||
fi
|
||||
|
||||
last_line_count=$current_line_count
|
||||
fi
|
||||
fi
|
||||
|
||||
# Check if the process is still running
|
||||
if ! pgrep -f "playwright test" > /dev/null 2>&1; then
|
||||
break
|
||||
fi
|
||||
|
||||
# Add a small delay to prevent excessive CPU usage
|
||||
sleep 0.5
|
||||
done
|
||||
|
||||
clear_progress
|
||||
}
|
||||
|
||||
# Function to extract test names from Playwright output
|
||||
extract_test_names() {
|
||||
local output_file="$1"
|
||||
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
||||
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
||||
}
|
||||
# Set up zsh error handling
|
||||
trap 'handle_zsh_error $LINENO' ERR
|
||||
|
||||
# Function to check if test passed in a run
|
||||
test_passed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to check if test failed in a run
|
||||
test_failed_in_run() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
||||
}
|
||||
|
||||
# Function to get test duration
|
||||
get_test_duration() {
|
||||
local test_name="$1"
|
||||
local run_output="$2"
|
||||
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
|
||||
echo "${duration:-unknown}"
|
||||
}
|
||||
|
||||
# Function to analyze test results
|
||||
analyze_results() {
|
||||
# Initialize summary data
|
||||
local summary_data="{
|
||||
\"timestamp\": \"$(date -Iseconds)\",
|
||||
\"total_runs\": $TOTAL_RUNS,
|
||||
\"test_results\": {},
|
||||
\"summary_stats\": {
|
||||
\"total_tests\": 0,
|
||||
\"always_passing\": 0,
|
||||
\"always_failing\": 0,
|
||||
\"intermittent\": 0,
|
||||
\"success_rate\": 0.0
|
||||
}
|
||||
}"
|
||||
|
||||
# Analyze each test using zsh associative array iteration
|
||||
for test_name in ${(k)test_results}; do
|
||||
local passes=${test_successes[$test_name]:-0}
|
||||
local fails=${test_failures[$test_name]:-0}
|
||||
local total=$((passes + fails))
|
||||
local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0")
|
||||
|
||||
# Determine test stability
|
||||
local stability=""
|
||||
if [ "$fails" -eq 0 ]; then
|
||||
stability="always_passing"
|
||||
elif [ "$passes" -eq 0 ]; then
|
||||
stability="always_failing"
|
||||
else
|
||||
stability="intermittent"
|
||||
fi
|
||||
|
||||
# Add to summary using jq
|
||||
summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \
|
||||
--arg stability "$stability" \
|
||||
--arg passes "$passes" \
|
||||
--arg fails "$fails" \
|
||||
--arg total "$total" \
|
||||
--arg rate "$success_rate" \
|
||||
'.test_results[$test] = {
|
||||
"stability": $stability,
|
||||
"passes": ($passes | tonumber),
|
||||
"fails": ($fails | tonumber),
|
||||
"total": ($total | tonumber),
|
||||
"success_rate": ($rate | tonumber)
|
||||
}')
|
||||
done
|
||||
|
||||
# Calculate summary statistics
|
||||
local total_tests=$(echo "$summary_data" | jq '.test_results | length')
|
||||
local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length')
|
||||
local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length')
|
||||
local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length')
|
||||
|
||||
summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \
|
||||
--arg passing "$always_passing" \
|
||||
--arg failing "$always_failing" \
|
||||
--arg intermittent "$intermittent" \
|
||||
'.summary_stats.total_tests = ($total | tonumber) |
|
||||
.summary_stats.always_passing = ($passing | tonumber) |
|
||||
.summary_stats.always_failing = ($failing | tonumber) |
|
||||
.summary_stats.intermittent = ($intermittent | tonumber)')
|
||||
|
||||
# Save summary
|
||||
echo "$summary_data" | jq '.' > "${SUMMARY_FILE}"
|
||||
|
||||
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}"
|
||||
}
|
||||
|
||||
# Function to generate detailed report
|
||||
generate_report() {
|
||||
local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
||||
|
||||
{
|
||||
echo "# TimeSafari Test Stability Report"
|
||||
echo ""
|
||||
echo "**Generated:** $(date)"
|
||||
echo "**Total Runs:** $TOTAL_RUNS"
|
||||
# Calculate duration with proper error handling
|
||||
local current_time=$(date +%s)
|
||||
local duration=0
|
||||
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then
|
||||
duration=$((current_time - START_TIME))
|
||||
fi
|
||||
echo "**Duration:** ${duration} seconds"
|
||||
echo ""
|
||||
|
||||
# Summary statistics
|
||||
echo "## Summary Statistics"
|
||||
echo ""
|
||||
local summary_data=$(cat "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
||||
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
||||
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
||||
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
||||
|
||||
echo "- **Total Tests:** $total_tests"
|
||||
echo "- **Always Passing:** $always_passing"
|
||||
echo "- **Always Failing:** $always_failing"
|
||||
echo "- **Intermittent:** $intermittent"
|
||||
echo ""
|
||||
|
||||
# Always failing tests
|
||||
echo "## Always Failing Tests"
|
||||
echo ""
|
||||
local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"')
|
||||
if [ -n "$failing_tests" ]; then
|
||||
echo "$failing_tests"
|
||||
else
|
||||
echo "No always failing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Intermittent tests
|
||||
echo "## Intermittent Tests (Most Unstable First)"
|
||||
echo ""
|
||||
local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"')
|
||||
if [ -n "$intermittent_tests" ]; then
|
||||
echo "$intermittent_tests"
|
||||
else
|
||||
echo "No intermittent tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Always passing tests
|
||||
echo "## Always Passing Tests"
|
||||
echo ""
|
||||
local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key')
|
||||
if [ -n "$passing_tests" ]; then
|
||||
echo "$passing_tests"
|
||||
else
|
||||
echo "No always passing tests found."
|
||||
fi
|
||||
echo ""
|
||||
|
||||
# Detailed test results
|
||||
echo "## Detailed Test Results"
|
||||
echo ""
|
||||
echo "| Test Name | Stability | Passes | Fails | Success Rate |"
|
||||
echo "|-----------|-----------|--------|-------|--------------|"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"'
|
||||
echo ""
|
||||
|
||||
# Run-by-run summary
|
||||
echo "## Run-by-Run Summary"
|
||||
echo ""
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
# Extract passed and failed counts using the same method as the main script
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local total=$((passed + failed))
|
||||
echo "**Run $i:** $passed passed, $failed failed ($total total)"
|
||||
fi
|
||||
done
|
||||
|
||||
} > "$report_file"
|
||||
|
||||
log_success "Detailed report generated: $report_file"
|
||||
}
|
||||
|
||||
# Main execution
|
||||
# Main execution function with zsh enhancements
|
||||
main() {
|
||||
START_TIME=$(date +%s)
|
||||
log_info "Starting enhanced test stability analysis with $TOTAL_RUNS runs (Zsh Version)"
|
||||
log_info "Results will be saved to: $RESULTS_DIR"
|
||||
echo
|
||||
|
||||
log_info "Starting TimeSafari Test Stability Runner (Zsh Version)"
|
||||
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}"
|
||||
log_info "Log file: ${LOG_FILE}"
|
||||
|
||||
# Check prerequisites
|
||||
log_info "Checking prerequisites..."
|
||||
if ! command -v jq &> /dev/null; then
|
||||
log_error "jq is required but not installed. Please install jq."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
if ! command -v bc &> /dev/null; then
|
||||
log_error "bc is required but not installed. Please install bc."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Check if Playwright is available
|
||||
if ! npx playwright --version &> /dev/null; then
|
||||
log_error "Playwright is not available. Please install dependencies."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
log_success "Prerequisites check passed"
|
||||
|
||||
echo ""
|
||||
log_info "Starting test execution with progress tracking..."
|
||||
echo ""
|
||||
|
||||
# Run tests multiple times
|
||||
for ((run=1; run<=TOTAL_RUNS; run++)); do
|
||||
log_info "Starting run $run/$TOTAL_RUNS"
|
||||
# Run all test executions with enhanced tracking
|
||||
for run_number in $(seq 1 $TOTAL_RUNS); do
|
||||
track_test_progress_enhanced "$run_number" "test suite"
|
||||
|
||||
local run_start=$(date +%s)
|
||||
local run_output="${RESULTS_DIR}/run-${run}.txt"
|
||||
|
||||
# Show overall run progress
|
||||
show_progress "$run" "$TOTAL_RUNS" 40 "RUN $run/$TOTAL_RUNS"
|
||||
|
||||
# Run the test suite with individual test progress
|
||||
echo ""
|
||||
log_info "Executing test suite for run $run..."
|
||||
|
||||
# Create a temporary file to capture Playwright output
|
||||
local temp_output=$(mktemp)
|
||||
|
||||
# Start progress tracking in background
|
||||
track_test_progress "$run" "$temp_output" &
|
||||
local progress_pid=$!
|
||||
|
||||
# Run Playwright with progress tracking
|
||||
if npx playwright test -c playwright.config-local.ts --reporter=list > "$temp_output" 2>&1; then
|
||||
# Wait for progress tracking to finish
|
||||
wait $progress_pid 2>/dev/null || true
|
||||
|
||||
# Copy the output to our results file
|
||||
cp "$temp_output" "$run_output"
|
||||
log_success "Run $run completed successfully"
|
||||
if run_single_test "$run_number"; then
|
||||
log_success "Run $run_number completed successfully"
|
||||
else
|
||||
# Wait for progress tracking to finish
|
||||
wait $progress_pid 2>/dev/null || true
|
||||
|
||||
# Copy the output to our results file even if it failed
|
||||
cp "$temp_output" "$run_output"
|
||||
log_warning "Run $run completed with failures"
|
||||
log_warning "Run $run_number failed, continuing with remaining runs"
|
||||
fi
|
||||
|
||||
# Ensure progress tracking is stopped
|
||||
kill $progress_pid 2>/dev/null || true
|
||||
|
||||
# Clean up temp file
|
||||
rm -f "$temp_output"
|
||||
|
||||
local run_end=$(date +%s)
|
||||
local run_duration=$((run_end - run_start))
|
||||
run_times[$run]=$run_duration
|
||||
|
||||
# Show run summary
|
||||
echo ""
|
||||
log_info "Run $run completed in ${run_duration}s"
|
||||
|
||||
# Extract and display quick summary for this run
|
||||
local passed=0
|
||||
local failed=0
|
||||
|
||||
# Extract passed count from the last line containing "passed"
|
||||
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1)
|
||||
if [ -n "$passed_line" ]; then
|
||||
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
# Extract failed count from the last line containing "failed"
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1)
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
local total=$((passed + failed))
|
||||
echo " ${GREEN}✓${NC} $passed passed, ${RED}✗${NC} $failed failed (${CYAN}$total${NC} total)"
|
||||
|
||||
# Extract and track test results using zsh array handling
|
||||
local test_names=($(extract_test_names "$run_output"))
|
||||
local test_count=${#test_names[@]}
|
||||
local processed_tests=0
|
||||
|
||||
for test_name in $test_names; do
|
||||
processed_tests=$((processed_tests + 1))
|
||||
|
||||
# Show progress for test analysis
|
||||
show_progress "$processed_tests" "$test_count" 30 "ANALYZING"
|
||||
|
||||
if test_passed_in_run "$test_name" "$run_output"; then
|
||||
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="pass"
|
||||
elif test_failed_in_run "$test_name" "$run_output"; then
|
||||
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
||||
test_results[$test_name]="fail"
|
||||
|
||||
# Log failure details
|
||||
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG"
|
||||
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true
|
||||
echo "" >> "$FAILURE_LOG"
|
||||
fi
|
||||
done
|
||||
|
||||
clear_progress
|
||||
|
||||
# Show detailed failed tests for this run
|
||||
if [ "$failed" -gt 0 ]; then
|
||||
log_warning "Failed tests in run $run:"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
log_warning " - $test_name"
|
||||
# Enhanced delay with zsh-specific features
|
||||
if [ "$run_number" -lt $TOTAL_RUNS ]; then
|
||||
# Use zsh's built-in sleep with progress indication
|
||||
for i in {1..2}; do
|
||||
printf "\r${YELLOW}Waiting...${NC} %d/2" "$i"
|
||||
sleep 1
|
||||
done
|
||||
else
|
||||
log_success "All tests passed in run $run"
|
||||
fi
|
||||
|
||||
echo ""
|
||||
done
|
||||
|
||||
# Analyze results
|
||||
log_info "Analyzing test results..."
|
||||
show_progress "1" "3" 40 "ANALYSIS"
|
||||
analyze_results
|
||||
|
||||
# Generate detailed report
|
||||
show_progress "2" "3" 40 "REPORT"
|
||||
generate_report
|
||||
|
||||
show_progress "3" "3" 40 "COMPLETE"
|
||||
clear_progress
|
||||
|
||||
echo ""
|
||||
log_success "Test stability analysis complete!"
|
||||
|
||||
# Final summary
|
||||
local total_duration=$(($(date +%s) - START_TIME))
|
||||
log_info "Total duration: ${total_duration}s"
|
||||
log_info "Results saved to: ${RESULTS_DIR}"
|
||||
log_info "Summary: ${SUMMARY_FILE}"
|
||||
log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
||||
log_info "Failure details: ${FAILURE_LOG}"
|
||||
|
||||
# Display quick summary
|
||||
echo ""
|
||||
echo "=== QUICK SUMMARY ==="
|
||||
local summary_data=$(cat "${SUMMARY_FILE}")
|
||||
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
||||
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
||||
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
||||
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
||||
|
||||
echo "Total Tests: $total_tests"
|
||||
echo "Always Passing: $always_passing"
|
||||
echo "Always Failing: $always_failing"
|
||||
echo "Intermittent: $intermittent"
|
||||
|
||||
# Show run-by-run failure summary
|
||||
echo ""
|
||||
echo "=== RUN-BY-RUN FAILURE SUMMARY ==="
|
||||
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
||||
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
||||
if [ -f "$run_file" ]; then
|
||||
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
||||
local failed_count=0
|
||||
if [ -n "$failed_line" ]; then
|
||||
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
||||
fi
|
||||
|
||||
if [ "$failed_count" -gt 0 ]; then
|
||||
echo "Run $i: $failed_count failed"
|
||||
# Extract failed test names from the summary section
|
||||
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do
|
||||
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
||||
echo " - $test_name"
|
||||
done
|
||||
else
|
||||
echo "Run $i: All tests passed"
|
||||
fi
|
||||
printf "\r%*s\r" "$(tput cols)" ""
|
||||
fi
|
||||
done
|
||||
|
||||
if [ "$always_failing" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "🚨 ALWAYS FAILING TESTS:"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key'
|
||||
fi
|
||||
# Generate and display results
|
||||
generate_summary_report
|
||||
display_final_results
|
||||
|
||||
if [ "$intermittent" -gt 0 ]; then
|
||||
echo ""
|
||||
echo "⚠️ INTERMITTENT TESTS (most unstable first):"
|
||||
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"'
|
||||
fi
|
||||
log_success "Enhanced test stability analysis complete! (Zsh Version)"
|
||||
}
|
||||
|
||||
# Run the main function
|
||||
# Run main function
|
||||
main "$@"
|
||||
Reference in New Issue
Block a user