Skip to content
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
60 commits
Select commit Hold shift + click to select a range
0b04da3
wip
alpe Nov 26, 2025
044903f
x
alpe Nov 27, 2025
2c9a212
Review feedback
alpe Nov 28, 2025
e2b0520
Encapsulate hint in sync package
alpe Nov 28, 2025
aaacbde
Async DA pull
alpe Nov 28, 2025
74be668
Merge branch 'main' into alex/2609_hints
alpe Dec 1, 2025
b9b5f5b
Minor cleanup
alpe Dec 1, 2025
e3336ce
Merge branch 'main' into alex/2609_hints
alpe Dec 8, 2025
c40b96b
Indipendent types for p2p store
alpe Dec 15, 2025
f3fb315
Merge branch 'main' into alex/2609_hints
alpe Dec 15, 2025
56c278f
Merge updates
alpe Dec 15, 2025
413b40d
Merge branch 'main' into alex/2609_hints
alpe Dec 15, 2025
a585190
Bump sonic version
alpe Dec 15, 2025
d09c8ab
Make tidy-all
alpe Dec 15, 2025
f0a505f
Merge branch 'main' into alex/2609_hints
alpe Dec 16, 2025
4ecf0a0
Use envelope for p2p store
alpe Dec 19, 2025
6c85d8d
Merge branch 'main' into alex/2609_hints
alpe Dec 19, 2025
7abfecc
Minor cleanup
alpe Dec 19, 2025
e593848
Better test data
alpe Dec 19, 2025
993c2b1
Merge branch 'main' into alex/2609_hints
alpe Dec 22, 2025
8b99828
Merge branch 'main' into alex/2609_hints
alpe Jan 8, 2026
544c0b9
Linter
alpe Jan 8, 2026
e1446a3
Merge branch 'main' into alex/2609_hints
alpe Jan 19, 2026
570ac06
Resolve merge conflicts
alpe Jan 19, 2026
1f6c405
Merge branch 'main' into alex/2609_hints
alpe Jan 19, 2026
4907c92
Tidy all
alpe Jan 19, 2026
6e4554d
Merge branch 'main' into alex/2609_hints
alpe Jan 19, 2026
6962e6f
Merge branch 'main' into alex/2609_hints
alpe Jan 20, 2026
158bcad
Merge branch 'main' into alex/2609_hints
alpe Jan 21, 2026
66b6db8
Integrate changes
alpe Jan 21, 2026
94fe911
Merge branch 'main' into alex/2609_hints
alpe Jan 22, 2026
a20e483
Merge branch 'main' into alex/2609_hints
alpe Jan 26, 2026
69bf91b
Merge branch 'main' into alex/2609_hints
alpe Jan 27, 2026
b8ec42f
Merge branch 'main' into alex/2609_hints
alpe Jan 28, 2026
eb5aff5
Merge branch 'main' into alex/2609_hints
julienrbrt Feb 2, 2026
9c95101
updates
julienrbrt Feb 2, 2026
19835e8
build fixes
julienrbrt Feb 2, 2026
c6e5696
fixes
julienrbrt Feb 2, 2026
b7b2100
cleanup
julienrbrt Feb 2, 2026
7b1c802
updates
julienrbrt Feb 2, 2026
14b83ff
implement setting
julienrbrt Feb 2, 2026
3a2ba3f
limit abstractions
julienrbrt Feb 2, 2026
c9d3251
linting
julienrbrt Feb 2, 2026
fcb3e75
use same key as sequencer
julienrbrt Feb 2, 2026
1b3802e
fix unit test
julienrbrt Feb 2, 2026
aac3f47
rename working pool to avoid naming confusion
julienrbrt Feb 2, 2026
68b3bcb
persist da/height for da synced nodes as well for consistency
julienrbrt Feb 2, 2026
1b394e0
Merge branch 'main' into alex/2609_hints
julienrbrt Feb 2, 2026
4f4ad36
feat(syncer): verify force inclusion for p2p blocks
julienrbrt Feb 2, 2026
36526ec
fix tests
julienrbrt Feb 2, 2026
025ee3f
fixes
julienrbrt Feb 2, 2026
3abb188
Merge branch 'alex/2609_hints' into julien/fi-p2p
julienrbrt Feb 2, 2026
0298cea
use height instead of hashes
julienrbrt Feb 3, 2026
4905677
Merge branch 'main' into alex/2609_hints
julienrbrt Feb 3, 2026
d6ba01d
fixes
julienrbrt Feb 3, 2026
f662a50
add docs
julienrbrt Feb 3, 2026
0a8e542
remove worker pool
julienrbrt Feb 3, 2026
a729ac2
Merge branch 'alex/2609_hints' into julien/fi-p2p
julienrbrt Feb 3, 2026
f728ec2
fix test
julienrbrt Feb 3, 2026
956c898
Merge branch 'main' into julien/fi-p2p
julienrbrt Feb 3, 2026
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rename working pool to avoid naming confusion
  • Loading branch information
julienrbrt committed Feb 2, 2026
commit aac3f47928eb98fc7d6f22acdded33b26b739df9
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import (
"github.com/rs/zerolog"
)

// AsyncDARetriever handles concurrent DA retrieval operations.
type AsyncDARetriever struct {
// DARetrievalWorkerPool handles concurrent on-demand DA retrieval operations
// triggered by P2P hints. Unlike the background prefetcher (AsyncBlockRetriever),
// this worker pool processes explicit retrieval requests for specific DA heights.
type DARetrievalWorkerPool struct {
retriever DARetriever
resultCh chan<- common.DAHeightEvent
workCh chan uint64
Expand All @@ -21,44 +23,44 @@ type AsyncDARetriever struct {
cancel context.CancelFunc
}

// NewAsyncDARetriever creates a new AsyncDARetriever.
func NewAsyncDARetriever(
// NewDARetrievalWorkerPool creates a new DARetrievalWorkerPool.
func NewDARetrievalWorkerPool(
retriever DARetriever,
resultCh chan<- common.DAHeightEvent,
logger zerolog.Logger,
) *AsyncDARetriever {
return &AsyncDARetriever{
) *DARetrievalWorkerPool {
return &DARetrievalWorkerPool{
retriever: retriever,
resultCh: resultCh,
workCh: make(chan uint64, 100), // Buffer size 100
inFlight: make(map[uint64]struct{}),
logger: logger.With().Str("component", "async_da_retriever").Logger(),
logger: logger.With().Str("component", "da_retrieval_worker_pool").Logger(),
}
}

// Start starts the worker pool.
func (r *AsyncDARetriever) Start(ctx context.Context) {
func (r *DARetrievalWorkerPool) Start(ctx context.Context) {
r.ctx, r.cancel = context.WithCancel(ctx)
// Start 5 workers
for i := 0; i < 5; i++ {
r.wg.Add(1)
go r.worker()
}
r.logger.Info().Msg("AsyncDARetriever started")
r.logger.Info().Msg("DARetrievalWorkerPool started")
}

// Stop stops the worker pool.
func (r *AsyncDARetriever) Stop() {
func (r *DARetrievalWorkerPool) Stop() {
if r.cancel != nil {
r.cancel()
}
r.wg.Wait()
r.logger.Info().Msg("AsyncDARetriever stopped")
r.logger.Info().Msg("DARetrievalWorkerPool stopped")
}

// RequestRetrieval requests a DA retrieval for the given height.
// It is non-blocking and idempotent.
func (r *AsyncDARetriever) RequestRetrieval(height uint64) {
func (r *DARetrievalWorkerPool) RequestRetrieval(height uint64) {
r.mu.Lock()
defer r.mu.Unlock()

Expand All @@ -75,7 +77,7 @@ func (r *AsyncDARetriever) RequestRetrieval(height uint64) {
}
}

func (r *AsyncDARetriever) worker() {
func (r *DARetrievalWorkerPool) worker() {
defer r.wg.Done()

for {
Expand All @@ -88,7 +90,7 @@ func (r *AsyncDARetriever) worker() {
}
}

func (r *AsyncDARetriever) processRetrieval(height uint64) {
func (r *DARetrievalWorkerPool) processRetrieval(height uint64) {
defer func() {
r.mu.Lock()
delete(r.inFlight, height)
Expand All @@ -97,7 +99,7 @@ func (r *AsyncDARetriever) processRetrieval(height uint64) {

events, err := r.retriever.RetrieveFromDA(r.ctx, height)
if err != nil {
r.logger.Debug().Err(err).Uint64("height", height).Msg("async DA retrieval failed")
r.logger.Debug().Err(err).Uint64("height", height).Msg("DA retrieval failed")
return
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,23 +11,23 @@ import (
"github.com/stretchr/testify/mock"
)

func TestAsyncDARetriever_RequestRetrieval(t *testing.T) {
func TestDARetrievalWorkerPool_RequestRetrieval(t *testing.T) {
logger := zerolog.Nop()
mockRetriever := NewMockDARetriever(t)
resultCh := make(chan common.DAHeightEvent, 10)

asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger)
workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

asyncRetriever.Start(ctx)
defer asyncRetriever.Stop()
workerPool.Start(ctx)
defer workerPool.Stop()

// 1. Test successful retrieval
height1 := uint64(100)
mockRetriever.EXPECT().RetrieveFromDA(mock.Anything, height1).Return([]common.DAHeightEvent{{DaHeight: height1}}, nil).Once()

asyncRetriever.RequestRetrieval(height1)
workerPool.RequestRetrieval(height1)

select {
case event := <-resultCh:
Expand All @@ -52,7 +52,7 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) {
}).Once() // Should be called only once despite multiple requests

// Send first request
asyncRetriever.RequestRetrieval(height2)
workerPool.RequestRetrieval(height2)

// Wait for the worker to pick it up
select {
Expand All @@ -62,8 +62,8 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) {
}

// Send duplicate requests while the first one is still in flight
asyncRetriever.RequestRetrieval(height2)
asyncRetriever.RequestRetrieval(height2)
workerPool.RequestRetrieval(height2)
workerPool.RequestRetrieval(height2)

// Unblock the worker
close(unblockCh)
Expand All @@ -84,17 +84,17 @@ func TestAsyncDARetriever_RequestRetrieval(t *testing.T) {
}
}

func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) {
func TestDARetrievalWorkerPool_WorkerPoolLimit(t *testing.T) {
logger := zerolog.Nop()
mockRetriever := NewMockDARetriever(t)
resultCh := make(chan common.DAHeightEvent, 100)

asyncRetriever := NewAsyncDARetriever(mockRetriever, resultCh, logger)
workerPool := NewDARetrievalWorkerPool(mockRetriever, resultCh, logger)
ctx, cancel := context.WithCancel(context.Background())
defer cancel()

asyncRetriever.Start(ctx)
defer asyncRetriever.Stop()
workerPool.Start(ctx)
defer workerPool.Stop()

// We have 5 workers. We'll block them all.
unblockCh := make(chan struct{})
Expand All @@ -106,7 +106,7 @@ func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) {
<-unblockCh
return []common.DAHeightEvent{{DaHeight: h}}, nil
}).Once()
asyncRetriever.RequestRetrieval(h)
workerPool.RequestRetrieval(h)
}

// Give workers time to pick up tasks
Expand All @@ -120,7 +120,7 @@ func TestAsyncDARetriever_WorkerPoolLimit(t *testing.T) {
return []common.DAHeightEvent{{DaHeight: h}}, nil
}).Once()

asyncRetriever.RequestRetrieval(height6)
workerPool.RequestRetrieval(height6)

// Ensure 6th request is NOT processed yet
select {
Expand Down
10 changes: 5 additions & 5 deletions block/internal/syncing/syncer.go
Original file line number Diff line number Diff line change
Expand Up @@ -127,8 +127,8 @@ type Syncer struct {
// defaults to self, but can be wrapped with tracing.
blockSyncer BlockSyncer

// Async DA retriever
asyncDARetriever *AsyncDARetriever
// Worker pool for on-demand DA retrieval triggered by P2P hints
daRetrievalWorkerPool *DARetrievalWorkerPool
}

// pendingForcedInclusionTx represents a forced inclusion transaction that hasn't been included yet
Expand Down Expand Up @@ -216,8 +216,8 @@ func (s *Syncer) Start(ctx context.Context) error {

// Initialize handlers
s.daRetriever = NewDARetriever(s.daClient, s.cache, s.genesis, s.logger)
s.asyncDARetriever = NewAsyncDARetriever(s.daRetriever, s.heightInCh, s.logger)
s.asyncDARetriever.Start(s.ctx)
s.daRetrievalWorkerPool = NewDARetrievalWorkerPool(s.daRetriever, s.heightInCh, s.logger)
s.daRetrievalWorkerPool.Start(s.ctx)
if s.config.Instrumentation.IsTracingEnabled() {
s.daRetriever = WithTracingDARetriever(s.daRetriever)
}
Expand Down Expand Up @@ -634,7 +634,7 @@ func (s *Syncer) processHeightEvent(ctx context.Context, event *common.DAHeightE
Msg("P2P event with DA height hint, triggering targeted DA retrieval")

// Trigger targeted DA retrieval in background via worker pool
s.asyncDARetriever.RequestRetrieval(daHeightHint)
s.daRetrievalWorkerPool.RequestRetrieval(daHeightHint)
}
}
}
Expand Down
12 changes: 6 additions & 6 deletions block/internal/syncing/syncer_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -741,14 +741,14 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) {
require.NoError(t, s.initializeState())
s.ctx = context.Background()

// Mock AsyncDARetriever
// Mock DARetrievalWorkerPool
mockRetriever := NewMockDARetriever(t)
asyncRetriever := NewAsyncDARetriever(mockRetriever, s.heightInCh, zerolog.Nop())
// We don't start the async retriever to avoid race conditions in test,
workerPool := NewDARetrievalWorkerPool(mockRetriever, s.heightInCh, zerolog.Nop())
// We don't start the worker pool to avoid race conditions in test,
// we just want to verify RequestRetrieval queues the request.
// However, RequestRetrieval writes to a channel, so we need a consumer or a buffered channel.
// The workCh is buffered (100), so we are good.
s.asyncDARetriever = asyncRetriever
s.daRetrievalWorkerPool = workerPool

// Create event with DA height hint
evt := common.DAHeightEvent{
Expand Down Expand Up @@ -778,9 +778,9 @@ func TestProcessHeightEvent_TriggersAsyncDARetrieval(t *testing.T) {

s.processHeightEvent(t.Context(), &evt)

// Verify that the request was queued in the async retriever
// Verify that the request was queued in the worker pool
select {
case h := <-asyncRetriever.workCh:
case h := <-workerPool.workCh:
assert.Equal(t, uint64(100), h)
default:
t.Fatal("expected DA retrieval request to be queued")
Expand Down
Loading