julienrbrt · Nov 24, 2025 · Nov 20, 2025 · Nov 20, 2025 · Nov 20, 2025 · Nov 20, 2025
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -15,6 +15,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
+- Improved EVM execution client payload status validation with proper retry logic for SYNCING states in `InitChain`, `ExecuteTxs`, and `SetFinal` methods. The implementation now follows Engine API specification by retrying SYNCING/ACCEPTED status with exponential backoff and failing immediately on INVALID status, preventing unnecessary node shutdowns during transient execution engine sync operations.
+
 - Remove GasPrice and GasMultiplier from DA interface and configuration to use celestia-node's native fee estimation. ([#2822](https://github.com/evstack/ev-node/pull/2822))
 - Use cache instead of in memory store for reaper. Persist cache on reload. Autoclean after 24 hours. ([#2811](https://github.com/evstack/ev-node/pull/2811))
 

diff --git a/execution/evm/execution.go b/execution/evm/execution.go
@@ -22,14 +22,94 @@ import (
 	"github.com/evstack/ev-node/core/execution"
 )
 
+const (
+	// MaxPayloadStatusRetries is the maximum number of retries for SYNCING status.
+	// According to the Engine API specification, SYNCING indicates temporary unavailability
+	// and should be retried with exponential backoff.
+	MaxPayloadStatusRetries = 3
+	// InitialRetryBackoff is the initial backoff duration for retries.
+	// The backoff doubles on each retry attempt (exponential backoff).
+	InitialRetryBackoff = 1 * time.Second
+)
+
 var (
-	// ErrInvalidPayloadStatus indicates that EVM returned status != VALID
+	// ErrInvalidPayloadStatus indicates that the execution engine returned a permanent
+	// failure status (INVALID or unknown status). This error should not be retried.
 	ErrInvalidPayloadStatus = errors.New("invalid payload status")
+	// ErrPayloadSyncing indicates that the execution engine is temporarily syncing.
+	// According to the Engine API specification, this is a transient condition that
+	// should be handled with retry logic rather than immediate failure.
+	ErrPayloadSyncing = errors.New("payload syncing")
 )
 
 // Ensure EngineAPIExecutionClient implements the execution.Execute interface
 var _ execution.Executor = (*EngineClient)(nil)
 
+// validatePayloadStatus checks the payload status and returns appropriate errors.
+// It implements the Engine API specification's status handling:
+//   - VALID: Operation succeeded, return nil
+//   - SYNCING/ACCEPTED: Temporary unavailability, return ErrPayloadSyncing for retry
+//   - INVALID: Permanent failure, return ErrInvalidPayloadStatus (no retry)
+//   - Unknown: Treat as permanent failure (no retry)
+func validatePayloadStatus(status engine.PayloadStatusV1, operation string) error {
+	switch status.Status {
+	case engine.VALID:
+		return nil
+	case engine.SYNCING, engine.ACCEPTED:
+		// SYNCING and ACCEPTED indicate temporary unavailability - should retry
+		return ErrPayloadSyncing
+	case engine.INVALID:
+		// INVALID is a permanent failure - should not retry
+		return ErrInvalidPayloadStatus
+	default:
+		// Unknown status - treat as invalid
+		return ErrInvalidPayloadStatus
+	}
+}
+
+// retryWithBackoff executes a function with exponential backoff retry logic.
+// It implements the Engine API specification's recommendation to retry SYNCING
+// status with exponential backoff. The function:
+//   - Retries only on ErrPayloadSyncing (transient failures)
+//   - Fails immediately on ErrInvalidPayloadStatus (permanent failures)
+//   - Respects context cancellation for graceful shutdown
+//   - Uses exponential backoff that doubles on each attempt
+func retryWithBackoff(ctx context.Context, fn func() error, maxRetries int, initialBackoff time.Duration, operation string) error {
+	backoff := initialBackoff
+
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		err := fn()
+		if err == nil {
+			return nil
+		}
+
+		// Don't retry on invalid status
+		if errors.Is(err, ErrInvalidPayloadStatus) {
+			return err
+		}
+
+		// Only retry on syncing status
+		if !errors.Is(err, ErrPayloadSyncing) {
+			return err
+		}
+
+		// Check if we've exhausted retries
+		if attempt >= maxRetries {
+			return fmt.Errorf("max retries (%d) exceeded for %s: %w", maxRetries, operation, err)
+		}
+
+		// Wait with exponential backoff
+		select {
+		case <-ctx.Done():
+			return fmt.Errorf("context cancelled during retry for %s: %w", operation, ctx.Err())
+		case <-time.After(backoff):
+			backoff *= 2
+		}
+	}
+
+	return fmt.Errorf("max retries (%d) exceeded for %s", maxRetries, operation)
+}
+
 // EngineClient represents a client that interacts with an Ethereum execution engine
 // through the Engine API. It manages connections to both the engine and standard Ethereum
 // APIs, and maintains state related to block processing.
@@ -105,28 +185,35 @@ func (c *EngineClient) InitChain(ctx context.Context, genesisTime time.Time, ini
 		return nil, 0, fmt.Errorf("initialHeight must be 1, got %d", initialHeight)
 	}
 
-	// Acknowledge the genesis block
-	var forkchoiceResult engine.ForkChoiceResponse
-	err := c.engineClient.CallContext(ctx, &forkchoiceResult, "engine_forkchoiceUpdatedV3",
-		engine.ForkchoiceStateV1{
-			HeadBlockHash:      c.genesisHash,
-			SafeBlockHash:      c.genesisHash,
-			FinalizedBlockHash: c.genesisHash,
-		},
-		nil,
-	)
-	if err != nil {
-		return nil, 0, fmt.Errorf("engine_forkchoiceUpdatedV3 failed: %w", err)
-	}
+	// Acknowledge the genesis block with retry logic for SYNCING status
+	err := retryWithBackoff(ctx, func() error {
+		var forkchoiceResult engine.ForkChoiceResponse
+		err := c.engineClient.CallContext(ctx, &forkchoiceResult, "engine_forkchoiceUpdatedV3",
+			engine.ForkchoiceStateV1{
+				HeadBlockHash:      c.genesisHash,
+				SafeBlockHash:      c.genesisHash,
+				FinalizedBlockHash: c.genesisHash,
+			},
+			nil,
+		)
+		if err != nil {
+			return fmt.Errorf("engine_forkchoiceUpdatedV3 failed: %w", err)
+		}
 
-	// Validate payload status
-	if forkchoiceResult.PayloadStatus.Status != engine.VALID {
-		c.logger.Warn().
-			Str("status", string(forkchoiceResult.PayloadStatus.Status)).
-			Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
-			Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
-			Msg("InitChain: engine_forkchoiceUpdatedV3 returned non-VALID status")
-		return nil, 0, fmt.Errorf("%w: status=%s", ErrInvalidPayloadStatus, forkchoiceResult.PayloadStatus.Status)
+		// Validate payload status
+		if err := validatePayloadStatus(forkchoiceResult.PayloadStatus, "InitChain"); err != nil {
+			c.logger.Warn().
+				Str("status", forkchoiceResult.PayloadStatus.Status).
+				Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
+				Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
+				Msg("InitChain: engine_forkchoiceUpdatedV3 returned non-VALID status")
+			return err
+		}
+
+		return nil
+	}, MaxPayloadStatusRetries, InitialRetryBackoff, "InitChain")
+	if err != nil {
+		return nil, 0, err
 	}
 
 	_, stateRoot, gasLimit, _, err := c.getBlockInfo(ctx, 0)
@@ -208,35 +295,56 @@ func (c *EngineClient) ExecuteTxs(ctx context.Context, txs [][]byte, blockHeight
 		return nil, 0, fmt.Errorf("forkchoice update failed: %w", err)
 	}
 
-	// Validate payload status
-	if forkchoiceResult.PayloadStatus.Status != engine.VALID {
-		c.logger.Warn().
-			Str("status", string(forkchoiceResult.PayloadStatus.Status)).
-			Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
-			Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
-			Uint64("blockHeight", blockHeight).
-			Msg("ExecuteTxs: engine_forkchoiceUpdatedV3 returned non-VALID status")
-		return nil, 0, fmt.Errorf("%w: status=%s", ErrInvalidPayloadStatus, forkchoiceResult.PayloadStatus.Status)
-	}
+	// Validate payload status with retry logic for SYNCING
+	var payloadID *engine.PayloadID
+	err = retryWithBackoff(ctx, func() error {
+		// Validate payload status
+		if err := validatePayloadStatus(forkchoiceResult.PayloadStatus, "ExecuteTxs forkchoice"); err != nil {
+			c.logger.Warn().
+				Str("status", forkchoiceResult.PayloadStatus.Status).
+				Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
+				Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
+				Uint64("blockHeight", blockHeight).
+				Msg("ExecuteTxs: engine_forkchoiceUpdatedV3 returned non-VALID status")
+
+			// If syncing, retry the entire forkchoice call
+			if errors.Is(err, ErrPayloadSyncing) {
+				// Re-call forkchoiceUpdatedV3 on retry
+				var retryResult engine.ForkChoiceResponse
+				if callErr := c.engineClient.CallContext(ctx, &retryResult, "engine_forkchoiceUpdatedV3", args, evPayloadAttrs); callErr != nil {
+					return fmt.Errorf("forkchoice update retry failed: %w", callErr)
+				}
+				forkchoiceResult = retryResult
+				return err // Return ErrPayloadSyncing to trigger retry
+			}
+			return err
+		}
+
+		if forkchoiceResult.PayloadID == nil {
+			c.logger.Error().
+				Str("status", forkchoiceResult.PayloadStatus.Status).
+				Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
+				Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
+				Interface("forkchoiceState", args).
+				Interface("payloadAttributes", evPayloadAttrs).
+				Uint64("blockHeight", blockHeight).
+				Msg("returned nil PayloadID")
+
+			return fmt.Errorf("returned nil PayloadID - (status: %s, latestValidHash: %s)",
+				forkchoiceResult.PayloadStatus.Status,
+				forkchoiceResult.PayloadStatus.LatestValidHash.Hex())
+		}
 
-	if forkchoiceResult.PayloadID == nil {
-		c.logger.Error().
-			Str("status", string(forkchoiceResult.PayloadStatus.Status)).
-			Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
-			Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
-			Interface("forkchoiceState", args).
-			Interface("payloadAttributes", evPayloadAttrs).
-			Uint64("blockHeight", blockHeight).
-			Msg("returned nil PayloadID")
-
-		return nil, 0, fmt.Errorf("returned nil PayloadID - (status: %s, latestValidHash: %s)",
-			forkchoiceResult.PayloadStatus.Status,
-			forkchoiceResult.PayloadStatus.LatestValidHash.Hex())
+		payloadID = forkchoiceResult.PayloadID
+		return nil
+	}, MaxPayloadStatusRetries, InitialRetryBackoff, "ExecuteTxs forkchoice")
+	if err != nil {
+		return nil, 0, err
 	}
 
 	// get payload
 	var payloadResult engine.ExecutionPayloadEnvelope
-	err = c.engineClient.CallContext(ctx, &payloadResult, "engine_getPayloadV4", *forkchoiceResult.PayloadID)
+	err = c.engineClient.CallContext(ctx, &payloadResult, "engine_getPayloadV4", *payloadID)
 	if err != nil {
 		return nil, 0, fmt.Errorf("get payload failed: %w", err)
 	}
@@ -253,13 +361,36 @@ func (c *EngineClient) ExecuteTxs(ctx context.Context, txs [][]byte, blockHeight
 		return nil, 0, fmt.Errorf("new payload submission failed: %w", err)
 	}
 
-	if newPayloadResult.Status != engine.VALID {
-		c.logger.Warn().
-			Str("status", string(newPayloadResult.Status)).
-			Str("latestValidHash", newPayloadResult.LatestValidHash.Hex()).
-			Interface("validationError", newPayloadResult.ValidationError).
-			Msg("engine_newPayloadV4 returned non-VALID status")
-		return nil, 0, ErrInvalidPayloadStatus
+	// Validate new payload status with retry logic
+	err = retryWithBackoff(ctx, func() error {
+		if err := validatePayloadStatus(newPayloadResult, "ExecuteTxs newPayload"); err != nil {
+			c.logger.Warn().
+				Str("status", newPayloadResult.Status).
+				Str("latestValidHash", newPayloadResult.LatestValidHash.Hex()).
+				Interface("validationError", newPayloadResult.ValidationError).
+				Uint64("blockHeight", blockHeight).
+				Msg("engine_newPayloadV4 returned non-VALID status")
+
+			// If syncing, retry the newPayload call
+			if errors.Is(err, ErrPayloadSyncing) {
+				var retryResult engine.PayloadStatusV1
+				if callErr := c.engineClient.CallContext(ctx, &retryResult, "engine_newPayloadV4",
+					payloadResult.ExecutionPayload,
+					[]string{},
+					common.Hash{}.Hex(),
+					[][]byte{},
+				); callErr != nil {
+					return fmt.Errorf("new payload retry failed: %w", callErr)
+				}
+				newPayloadResult = retryResult
+				return err // Return ErrPayloadSyncing to trigger retry
+			}
+			return err
+		}
+		return nil
+	}, MaxPayloadStatusRetries, InitialRetryBackoff, "ExecuteTxs newPayload")
+	if err != nil {
+		return nil, 0, err
 	}
 
 	// forkchoice update
@@ -296,13 +427,30 @@ func (c *EngineClient) setFinal(ctx context.Context, blockHash common.Hash, isFi
 		return fmt.Errorf("forkchoice update failed with error: %w", err)
 	}
 
-	if forkchoiceResult.PayloadStatus.Status != engine.VALID {
-		c.logger.Warn().
-			Str("status", string(forkchoiceResult.PayloadStatus.Status)).
-			Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
-			Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
-			Msg("forkchoiceUpdatedV3 returned non-VALID status")
-		return ErrInvalidPayloadStatus
+	// Validate payload status with retry logic
+	err = retryWithBackoff(ctx, func() error {
+		if err := validatePayloadStatus(forkchoiceResult.PayloadStatus, "setFinal"); err != nil {
+			c.logger.Warn().
+				Str("status", forkchoiceResult.PayloadStatus.Status).
+				Str("latestValidHash", forkchoiceResult.PayloadStatus.LatestValidHash.Hex()).
+				Interface("validationError", forkchoiceResult.PayloadStatus.ValidationError).
+				Msg("forkchoiceUpdatedV3 returned non-VALID status")
+
+			// If syncing, retry the forkchoice call
+			if errors.Is(err, ErrPayloadSyncing) {
+				var retryResult engine.ForkChoiceResponse
+				if callErr := c.engineClient.CallContext(ctx, &retryResult, "engine_forkchoiceUpdatedV3", args, nil); callErr != nil {
+					return fmt.Errorf("forkchoice update retry failed: %w", callErr)
+				}
+				forkchoiceResult = retryResult
+				return err // Return ErrPayloadSyncing to trigger retry
+			}
+			return err
+		}
+		return nil
+	}, MaxPayloadStatusRetries, InitialRetryBackoff, "setFinal")
+	if err != nil {
+		return err
 	}
 
 	return nil