You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
421 lines
16 KiB
421 lines
16 KiB
#!/bin/bash
|
|
|
|
# Test Stability Runner for TimeSafari
|
|
# Executes the full test suite 10 times and analyzes failure patterns
|
|
# Author: Matthew Raymer
|
|
|
|
set -euo pipefail
|
|
|
|
# Configuration
|
|
TOTAL_RUNS=10
|
|
RESULTS_DIR="test-stability-results"
|
|
TIMESTAMP=$(date +"%Y-%m-%d_%H-%M-%S")
|
|
LOG_FILE="${RESULTS_DIR}/stability-run-${TIMESTAMP}.log"
|
|
SUMMARY_FILE="${RESULTS_DIR}/stability-summary-${TIMESTAMP}.json"
|
|
FAILURE_LOG="${RESULTS_DIR}/failure-details-${TIMESTAMP}.log"
|
|
|
|
# Colors for output
|
|
RED='\033[0;31m'
|
|
GREEN='\033[0;32m'
|
|
YELLOW='\033[1;33m'
|
|
BLUE='\033[0;34m'
|
|
NC='\033[0m' # No Color
|
|
|
|
# Initialize results tracking
|
|
declare -A test_results
|
|
declare -A test_failures
|
|
declare -A test_successes
|
|
declare -A run_times
|
|
|
|
# Create results directory
|
|
mkdir -p "${RESULTS_DIR}"
|
|
|
|
# Logging functions
|
|
log_info() {
|
|
echo -e "${BLUE}[INFO]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_success() {
|
|
echo -e "${GREEN}[SUCCESS]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_warning() {
|
|
echo -e "${YELLOW}[WARNING]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
log_error() {
|
|
echo -e "${RED}[ERROR]${NC} $(date '+%Y-%m-%d %H:%M:%S') - $1" | tee -a "${LOG_FILE}"
|
|
}
|
|
|
|
# Function to extract test names from Playwright output
|
|
extract_test_names() {
|
|
local output_file="$1"
|
|
# Extract test names from lines like "✓ 13 [chromium] › test-playwright/30-record-gift.spec.ts:84:5 › Record something given"
|
|
grep -E "✓.*test-playwright" "$output_file" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//' | sort | uniq
|
|
}
|
|
|
|
# Function to check if test passed in a run
|
|
test_passed_in_run() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
grep -q "✓.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
|
}
|
|
|
|
# Function to check if test failed in a run
|
|
test_failed_in_run() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
grep -q "✗.*test-playwright/$test_name" "$run_output" 2>/dev/null
|
|
}
|
|
|
|
# Function to get test duration
|
|
get_test_duration() {
|
|
local test_name="$1"
|
|
local run_output="$2"
|
|
local duration=$(grep -A 1 "✓ $test_name\|✗ $test_name" "$run_output" | grep -o "[0-9]\+ms" | head -1)
|
|
echo "${duration:-unknown}"
|
|
}
|
|
|
|
# Function to analyze test results
|
|
analyze_results() {
|
|
log_info "Analyzing test results..."
|
|
|
|
# Initialize summary data
|
|
local summary_data="{
|
|
\"timestamp\": \"$(date -Iseconds)\",
|
|
\"total_runs\": $TOTAL_RUNS,
|
|
\"test_results\": {},
|
|
\"summary_stats\": {
|
|
\"total_tests\": 0,
|
|
\"always_passing\": 0,
|
|
\"always_failing\": 0,
|
|
\"intermittent\": 0,
|
|
\"success_rate\": 0.0
|
|
}
|
|
}"
|
|
|
|
# Analyze each test
|
|
for test_name in "${!test_results[@]}"; do
|
|
local passes=${test_successes[$test_name]:-0}
|
|
local fails=${test_failures[$test_name]:-0}
|
|
local total=$((passes + fails))
|
|
local success_rate=$(echo "scale=2; $passes * 100 / $total" | bc -l 2>/dev/null || echo "0")
|
|
|
|
# Determine test stability
|
|
local stability=""
|
|
if [ "$fails" -eq 0 ]; then
|
|
stability="always_passing"
|
|
elif [ "$passes" -eq 0 ]; then
|
|
stability="always_failing"
|
|
else
|
|
stability="intermittent"
|
|
fi
|
|
|
|
# Add to summary
|
|
summary_data=$(echo "$summary_data" | jq --arg test "$test_name" \
|
|
--arg stability "$stability" \
|
|
--arg passes "$passes" \
|
|
--arg fails "$fails" \
|
|
--arg total "$total" \
|
|
--arg rate "$success_rate" \
|
|
'.test_results[$test] = {
|
|
"stability": $stability,
|
|
"passes": ($passes | tonumber),
|
|
"fails": ($fails | tonumber),
|
|
"total": ($total | tonumber),
|
|
"success_rate": ($rate | tonumber)
|
|
}')
|
|
done
|
|
|
|
# Calculate summary statistics
|
|
local total_tests=$(echo "$summary_data" | jq '.test_results | length')
|
|
local always_passing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_passing")) | length')
|
|
local always_failing=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "always_failing")) | length')
|
|
local intermittent=$(echo "$summary_data" | jq '.test_results | to_entries | map(select(.value.stability == "intermittent")) | length')
|
|
|
|
summary_data=$(echo "$summary_data" | jq --arg total "$total_tests" \
|
|
--arg passing "$always_passing" \
|
|
--arg failing "$always_failing" \
|
|
--arg intermittent "$intermittent" \
|
|
'.summary_stats.total_tests = ($total | tonumber) |
|
|
.summary_stats.always_passing = ($passing | tonumber) |
|
|
.summary_stats.always_failing = ($failing | tonumber) |
|
|
.summary_stats.intermittent = ($intermittent | tonumber)')
|
|
|
|
# Save summary
|
|
echo "$summary_data" | jq '.' > "${SUMMARY_FILE}"
|
|
|
|
log_success "Analysis complete. Results saved to ${SUMMARY_FILE}"
|
|
}
|
|
|
|
# Function to generate detailed report
|
|
generate_report() {
|
|
log_info "Generating detailed stability report..."
|
|
|
|
local report_file="${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
|
|
|
{
|
|
echo "# TimeSafari Test Stability Report"
|
|
echo ""
|
|
echo "**Generated:** $(date)"
|
|
echo "**Total Runs:** $TOTAL_RUNS"
|
|
# Calculate duration with proper error handling
|
|
local current_time=$(date +%s)
|
|
local duration=0
|
|
if [ -n "$START_TIME" ] && [ "$START_TIME" -gt 0 ]; then
|
|
duration=$((current_time - START_TIME))
|
|
fi
|
|
echo "**Duration:** ${duration} seconds"
|
|
echo ""
|
|
|
|
# Summary statistics
|
|
echo "## Summary Statistics"
|
|
echo ""
|
|
local summary_data=$(cat "${SUMMARY_FILE}")
|
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
|
|
|
echo "- **Total Tests:** $total_tests"
|
|
echo "- **Always Passing:** $always_passing"
|
|
echo "- **Always Failing:** $always_failing"
|
|
echo "- **Intermittent:** $intermittent"
|
|
echo ""
|
|
|
|
# Always failing tests
|
|
echo "## Always Failing Tests"
|
|
echo ""
|
|
local failing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | "- " + .key + " (" + (.value.fails | tostring) + "/" + (.value.total | tostring) + " fails)"')
|
|
if [ -n "$failing_tests" ]; then
|
|
echo "$failing_tests"
|
|
else
|
|
echo "No always failing tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Intermittent tests
|
|
echo "## Intermittent Tests (Most Unstable First)"
|
|
echo ""
|
|
local intermittent_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | "- " + .key + " (" + (.value.success_rate | tostring) + "% success rate)"')
|
|
if [ -n "$intermittent_tests" ]; then
|
|
echo "$intermittent_tests"
|
|
else
|
|
echo "No intermittent tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Always passing tests
|
|
echo "## Always Passing Tests"
|
|
echo ""
|
|
local passing_tests=$(echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_passing")) | .[] | "- " + .key')
|
|
if [ -n "$passing_tests" ]; then
|
|
echo "$passing_tests"
|
|
else
|
|
echo "No always passing tests found."
|
|
fi
|
|
echo ""
|
|
|
|
# Detailed test results
|
|
echo "## Detailed Test Results"
|
|
echo ""
|
|
echo "| Test Name | Stability | Passes | Fails | Success Rate |"
|
|
echo "|-----------|-----------|--------|-------|--------------|"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | sort_by(.key) | .[] | "| " + .key + " | " + .value.stability + " | " + (.value.passes | tostring) + " | " + (.value.fails | tostring) + " | " + (.value.success_rate | tostring) + "% |"'
|
|
echo ""
|
|
|
|
# Run-by-run summary
|
|
echo "## Run-by-Run Summary"
|
|
echo ""
|
|
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
|
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
|
if [ -f "$run_file" ]; then
|
|
# Extract passed and failed counts using the same method as the main script
|
|
local passed=0
|
|
local failed=0
|
|
|
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_file" | tail -1)
|
|
if [ -n "$passed_line" ]; then
|
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
|
if [ -n "$failed_line" ]; then
|
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
local total=$((passed + failed))
|
|
echo "**Run $i:** $passed passed, $failed failed ($total total)"
|
|
fi
|
|
done
|
|
|
|
} > "$report_file"
|
|
|
|
log_success "Detailed report generated: $report_file"
|
|
}
|
|
|
|
# Main execution
|
|
main() {
|
|
START_TIME=$(date +%s)
|
|
|
|
log_info "Starting TimeSafari Test Stability Runner"
|
|
log_info "Configuration: $TOTAL_RUNS runs, results in ${RESULTS_DIR}"
|
|
log_info "Log file: ${LOG_FILE}"
|
|
|
|
# Check prerequisites
|
|
log_info "Checking prerequisites..."
|
|
if ! command -v jq &> /dev/null; then
|
|
log_error "jq is required but not installed. Please install jq."
|
|
exit 1
|
|
fi
|
|
|
|
if ! command -v bc &> /dev/null; then
|
|
log_error "bc is required but not installed. Please install bc."
|
|
exit 1
|
|
fi
|
|
|
|
# Check if Playwright is available
|
|
if ! npx playwright --version &> /dev/null; then
|
|
log_error "Playwright is not available. Please install dependencies."
|
|
exit 1
|
|
fi
|
|
|
|
log_success "Prerequisites check passed"
|
|
|
|
# Run tests multiple times
|
|
for ((run=1; run<=TOTAL_RUNS; run++)); do
|
|
log_info "Starting run $run/$TOTAL_RUNS"
|
|
|
|
local run_start=$(date +%s)
|
|
local run_output="${RESULTS_DIR}/run-${run}.txt"
|
|
|
|
# Run the test suite
|
|
if npx playwright test -c playwright.config-local.ts --reporter=list > "$run_output" 2>&1; then
|
|
log_success "Run $run completed successfully"
|
|
else
|
|
log_warning "Run $run completed with failures"
|
|
fi
|
|
|
|
local run_end=$(date +%s)
|
|
local run_duration=$((run_end - run_start))
|
|
run_times[$run]=$run_duration
|
|
|
|
log_info "Run $run completed in ${run_duration}s"
|
|
|
|
# Extract and track test results
|
|
local test_names=$(extract_test_names "$run_output")
|
|
for test_name in $test_names; do
|
|
if test_passed_in_run "$test_name" "$run_output"; then
|
|
test_successes[$test_name]=$((${test_successes[$test_name]:-0} + 1))
|
|
test_results[$test_name]="pass"
|
|
elif test_failed_in_run "$test_name" "$run_output"; then
|
|
test_failures[$test_name]=$((${test_failures[$test_name]:-0} + 1))
|
|
test_results[$test_name]="fail"
|
|
|
|
# Log failure details
|
|
echo "=== Run $run - $test_name ===" >> "$FAILURE_LOG"
|
|
grep -A 10 -B 5 "✗ $test_name" "$run_output" >> "$FAILURE_LOG" 2>/dev/null || true
|
|
echo "" >> "$FAILURE_LOG"
|
|
fi
|
|
done
|
|
|
|
# Brief summary for this run - extract from Playwright summary lines
|
|
local passed=0
|
|
local failed=0
|
|
|
|
# Extract passed count from the last line containing "passed"
|
|
local passed_line=$(grep -E "[0-9]+ passed" "$run_output" | tail -1)
|
|
if [ -n "$passed_line" ]; then
|
|
passed=$(echo "$passed_line" | grep -o "[0-9]\+ passed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
# Extract failed count from the last line containing "failed"
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_output" | tail -1)
|
|
if [ -n "$failed_line" ]; then
|
|
failed=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
log_info "Run $run summary: $passed passed, $failed failed"
|
|
|
|
# Show failed tests for this run
|
|
if [ "$failed" -gt 0 ]; then
|
|
log_warning "Failed tests in run $run:"
|
|
# Extract failed test names from the summary section
|
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_output" | grep "test-playwright" | while read -r line; do
|
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
|
log_warning " - $test_name"
|
|
done
|
|
else
|
|
log_success "All tests passed in run $run"
|
|
fi
|
|
done
|
|
|
|
# Analyze results
|
|
analyze_results
|
|
|
|
# Generate detailed report
|
|
generate_report
|
|
|
|
# Final summary
|
|
local total_duration=$(($(date +%s) - START_TIME))
|
|
log_success "Test stability analysis complete!"
|
|
log_info "Total duration: ${total_duration}s"
|
|
log_info "Results saved to: ${RESULTS_DIR}"
|
|
log_info "Summary: ${SUMMARY_FILE}"
|
|
log_info "Detailed report: ${RESULTS_DIR}/stability-report-${TIMESTAMP}.md"
|
|
log_info "Failure details: ${FAILURE_LOG}"
|
|
|
|
# Display quick summary
|
|
echo ""
|
|
echo "=== QUICK SUMMARY ==="
|
|
local summary_data=$(cat "${SUMMARY_FILE}")
|
|
local total_tests=$(echo "$summary_data" | jq '.summary_stats.total_tests')
|
|
local always_passing=$(echo "$summary_data" | jq '.summary_stats.always_passing')
|
|
local always_failing=$(echo "$summary_data" | jq '.summary_stats.always_failing')
|
|
local intermittent=$(echo "$summary_data" | jq '.summary_stats.intermittent')
|
|
|
|
echo "Total Tests: $total_tests"
|
|
echo "Always Passing: $always_passing"
|
|
echo "Always Failing: $always_failing"
|
|
echo "Intermittent: $intermittent"
|
|
|
|
# Show run-by-run failure summary
|
|
echo ""
|
|
echo "=== RUN-BY-RUN FAILURE SUMMARY ==="
|
|
for ((i=1; i<=TOTAL_RUNS; i++)); do
|
|
local run_file="${RESULTS_DIR}/run-${i}.txt"
|
|
if [ -f "$run_file" ]; then
|
|
local failed_line=$(grep -E "[0-9]+ failed" "$run_file" | tail -1)
|
|
local failed_count=0
|
|
if [ -n "$failed_line" ]; then
|
|
failed_count=$(echo "$failed_line" | grep -o "[0-9]\+ failed" | grep -o "[0-9]\+")
|
|
fi
|
|
|
|
if [ "$failed_count" -gt 0 ]; then
|
|
echo "Run $i: $failed_count failed"
|
|
# Extract failed test names from the summary section
|
|
sed -n '/^ 1 failed$/,/^ 37 passed/p' "$run_file" | grep "test-playwright" | while read -r line; do
|
|
local test_name=$(echo "$line" | sed 's/.*test-playwright\///' | sed 's/:[0-9]*:[0-9]*.*$//')
|
|
echo " - $test_name"
|
|
done
|
|
else
|
|
echo "Run $i: All tests passed"
|
|
fi
|
|
fi
|
|
done
|
|
|
|
if [ "$always_failing" -gt 0 ]; then
|
|
echo ""
|
|
echo "🚨 ALWAYS FAILING TESTS:"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "always_failing")) | .[] | " - " + .key'
|
|
fi
|
|
|
|
if [ "$intermittent" -gt 0 ]; then
|
|
echo ""
|
|
echo "⚠️ INTERMITTENT TESTS (most unstable first):"
|
|
echo "$summary_data" | jq -r '.test_results | to_entries | map(select(.value.stability == "intermittent")) | sort_by(.value.success_rate) | .[] | " - " + .key + " (" + (.value.success_rate | tostring) + "% success)"'
|
|
fi
|
|
}
|
|
|
|
# Run the main function
|
|
main "$@"
|