🧪 test: enhance validation script with separate unit/BDD tests and better failure parsing

scripts/validate-test-suite.sh

@@ -1,16 +1,14 @@
 #!/bin/bash
 
 # Test Suite Validation Script
-# Runs tests N times and collects failure metrics
+# Runs tests N times with separate unit and BDD test phases
-# Usage: ./scripts/validate-test-suite.sh [N] [test_path]
+# Usage: ./scripts/validate-test-suite.sh [N]
 #   N          - Number of times to run tests (default: 20)
-#   test_path  - Test path (default: ./...)
 
 set -e
 
 # Default values
 RUN_COUNT=${1:-20}
-TEST_PATH=${2:-./...}
 SCRIPTS_DIR=$(dirname "$(realpath "${BASH_SOURCE[0]}")")
 
 # Colors for output
@@ -21,26 +19,29 @@ BLUE='\033[0;34m'
 NC='\033[0m' # No Color
 
 # Temporary files
-FAILURE_LOG=$(mktemp)
+UNIT_FAILURE_LOG=$(mktemp)
-UNIQUE_FAILURES=$(mktemp)
+BDD_FAILURE_LOG=$(mktemp)
 SUMMARY_REPORT=$(mktemp)
 
 # Cleanup temporary files on exit
 cleanup() {
-    rm -f "$FAILURE_LOG" "$UNIQUE_FAILURES" "$SUMMARY_REPORT"
+    rm -f "$UNIT_FAILURE_LOG" "$BDD_FAILURE_LOG" "$SUMMARY_REPORT"
 }
 trap cleanup EXIT
 
 echo "🧪 Test Suite Validation Script"
 echo "=============================="
 echo "Runs: $RUN_COUNT"
-echo "Tests: $TEST_PATH"
+echo "Unit Tests: ./cmd/... ./pkg/..."
+echo "BDD Tests: ./features/..."
 echo "Date: $(date)"
 echo
 
 # Initialize counters
-SUCCESS_COUNT=0
+UNIT_SUCCESS=0
-FAILURE_COUNT=0
+UNIT_FAILURE=0
+BDD_SUCCESS=0
+BDD_FAILURE=0
 START_TIME=$(date +%s)
 
 echo "Starting validation runs..."
@@ -50,28 +51,48 @@ echo
 for (( run=1; run<=$RUN_COUNT; run++ )); do
     echo "Run $run/$RUN_COUNT..."
     
-    # Clean test cache for each run
+    # ===== UNIT TESTS =====
+    echo "  🧪 Unit tests..."
     go clean -testcache > /dev/null 2>&1
     
-    # Run tests and capture output
     set +e  # Temporarily disable exit on error
-    TEST_OUTPUT=$(go test $TEST_PATH -v 2>&1)
+    UNIT_OUTPUT=$(go test ./cmd/... ./pkg/... -v 2>&1)
-    TEST_EXIT_CODE=$?
+    UNIT_EXIT_CODE=$?
     set -e  # Re-enable exit on error
     
-    if [ $TEST_EXIT_CODE -eq 0 ]; then
+    if [ $UNIT_EXIT_CODE -eq 0 ]; then
         echo "    ✅ Passed"
-        ((SUCCESS_COUNT++))
+        ((UNIT_SUCCESS++))
     else
         echo "    ❌ Failed"
-        ((FAILURE_COUNT++))
+        ((UNIT_FAILURE++))
         
-        # Extract failing test names and errors
+        # Extract detailed unit test failures
-        echo "$TEST_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$FAILURE_LOG"
+        echo "$UNIT_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$UNIT_FAILURE_LOG"
+        echo "$UNIT_OUTPUT" | grep -A 10 "FAIL.*\.go" >> "$UNIT_FAILURE_LOG"
+        echo "---" >> "$UNIT_FAILURE_LOG"
+    fi
     
-        # Extract specific test failures with errors
+    # ===== BDD TESTS =====
-        echo "$TEST_OUTPUT" | grep -A 5 "FAIL.*\.go" | head -6 >> "$FAILURE_LOG"
+    echo "  🧪 BDD tests..."
-        echo "---" >> "$FAILURE_LOG"
+    go clean -testcache > /dev/null 2>&1
+    
+    set +e  # Temporarily disable exit on error
+    BDD_OUTPUT=$(go test ./features/... -v 2>&1)
+    BDD_EXIT_CODE=$?
+    set -e  # Re-enable exit on error
+    
+    if [ $BDD_EXIT_CODE -eq 0 ]; then
+        echo "    ✅ Passed"
+        ((BDD_SUCCESS++))
+    else
+        echo "    ❌ Failed"
+        ((BDD_FAILURE++))
+        
+        # Extract detailed BDD test failures with actual test names
+        echo "$BDD_OUTPUT" | grep -E "^(FAIL|--- FAIL)" | sed 's/^\*\*\* //' >> "$BDD_FAILURE_LOG"
+        echo "$BDD_OUTPUT" | grep -A 10 "FAIL.*Test" >> "$BDD_FAILURE_LOG"
+        echo "---" >> "$BDD_FAILURE_LOG"
     fi
 done
 
@@ -82,13 +103,19 @@ DURATION=$((END_TIME - START_TIME))
 echo "Validation Complete"
 echo "=================="
 echo "Total Runs: $RUN_COUNT"
-echo "Success: ${GREEN}$SUCCESS_COUNT${NC}"
+echo "Unit Tests:"
-echo "Failures: ${RED}$FAILURE_COUNT${NC}"
+echo "  Success: ${GREEN}$UNIT_SUCCESS${NC}"
+echo "  Failures: ${RED}$UNIT_FAILURE${NC}"
+echo "BDD Tests:"
+echo "  Success: ${GREEN}$BDD_SUCCESS${NC}"
+echo "  Failures: ${RED}$BDD_FAILURE${NC}"
 echo "Duration: $DURATION seconds"
 echo
 
-# Check if there were any failures
+# Check overall success
-if [ $FAILURE_COUNT -eq 0 ]; then
+TOTAL_FAILURES=$((UNIT_FAILURE + BDD_FAILURE))
+
+if [ $TOTAL_FAILURES -eq 0 ]; then
     echo "${GREEN}✅ All tests passed successfully!${NC}"
     echo "Test suite is stable and ready for production"
     exit 0
@@ -96,40 +123,57 @@ else
     echo "${RED}❌ Some tests failed during validation${NC}"
     echo
     
-    # Process failure log to get unique failures with counts
+    # Process unit test failures
-    if [ -s "$FAILURE_LOG" ]; then
+    if [ -s "$UNIT_FAILURE_LOG" ]; then
-        echo "Failure Analysis"
+        echo "Unit Test Failures:"
+        echo "=================="
+        
+        # Count unit test failures
+        UNIT_FAILURES=$(grep "FAIL" "$UNIT_FAILURE_LOG" | sort | uniq -c | sort -rn)
+        if [ -n "$UNIT_FAILURES" ]; then
+            echo "$UNIT_FAILURES"
+        else
+            echo "  None (check log for details)"
+        fi
+        
+        echo
+    fi
+    
+    # Process BDD test failures
+    if [ -s "$BDD_FAILURE_LOG" ]; then
+        echo "BDD Test Failures:"
         echo "================"
         
-        # Count occurrences of each failing test
+        # Count BDD test failures with granularity
-        echo "Failing Test Summary:"
+        BDD_FAILURES=$(grep "FAIL" "$BDD_FAILURE_LOG" | sort | uniq -c | sort -rn)
-        grep "FAIL" "$FAILURE_LOG" | sort | uniq -c | sort -rn | while read count test; do
+        if [ -n "$BDD_FAILURES" ]; then
-            test_name=$(echo "$test" | sed 's/FAIL[[:space:]]*//')
+            echo "Summary:"
-            echo "  $count × $test_name"
+            while IFS= read -r line; do
-        done
+                count=$(echo "$line" | awk '{print $1}')
-        
+                test=$(echo "$line" | sed 's/^[0-9]*[[:space:]]*//')
-        echo
+                echo "  $count × $test"
-        echo "Unique Failure Patterns:"
+            done <<< "$BDD_FAILURES"
-        
+        else
-        # Extract unique failure patterns
+            echo "  None (check log for details)"
-        grep -E "^(FAIL|---)" "$FAILURE_LOG" | sort | uniq | while read line; do
-            if [[ "$line" == FAIL* ]]; then
-                echo "  • $line"
         fi
-        done
         
         echo
-        echo "Detailed Failure Log:"
+        echo "Detailed BDD Failure Log (first 20 lines):"
-        echo "======================"
+        echo "=========================================="
-        cat "$FAILURE_LOG"
+        # Show only the relevant failure lines with actual test names
+        grep -E "(FAIL.*Test|--- FAIL)" "$BDD_FAILURE_LOG" | head -20
+        
+        echo
+        echo "Full failure log saved in: $BDD_FAILURE_LOG"
+    fi
     
     echo
     echo "Recommendations:"
-        echo "  1. Mark flaky tests with @flaky tag"
+    echo "  1. Mark flaky BDD tests with @flaky tag"
-        echo "  2. Investigate and fix failing tests"
+    echo "  2. Investigate unit test failures first (faster to fix)"
-        echo "  3. Run with FIXED_TEST_PORT=true to debug port issues"
+    echo "  3. Check for race conditions in failing tests"
-        echo "  4. Check for race conditions in failing tests"
+    echo "  4. Run with FIXED_TEST_PORT=true for debugging"
-    fi
+    echo "  5. Use ./scripts/run-bdd-tests.sh list-tags to see available tags"
     
     exit 1
 fi