op-supervisor,op-node: Handle non-genesis Interop (#16008)

* op-supervisor,op-node: Handle non-interop genesis activation

* op-supervisor: fix OpenBlock error case

* op-e2e: Skip tests that depend on multi-node setup

Multi-node will be completely removed via #16174.
Other tests will be fixed by switching to multi-supervisor setup (#16187).

---------

Co-authored-by: protolambda <[email protected]>

Author: sebastianst

op-acceptance-tests/tests/interop/sync/redundant_interop/interop_sync_test.go

@@ -13,6 +13,7 @@ import (
 // TestUnsafeChainKnownToL2CL tests the below scenario:
 // supervisor cross-safe ahead of L2CL cross-safe, aka L2CL can "skip" forward to match safety of supervisor.
 func TestUnsafeChainKnownToL2CL(gt *testing.T) {
+	gt.Skip() // TODO(#16187): Fix to use multi-supervisor (not multi-node) setup.
 	t := devtest.SerialT(gt)
 
 	// Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.
@@ -84,6 +85,7 @@ func TestUnsafeChainKnownToL2CL(gt *testing.T) {
 // TestUnsafeChainUnknownToL2CL tests the below scenario:
 // supervisor unsafe ahead of L2CL unsafe, aka L2CL processes new blocks first.
 func TestUnsafeChainUnknownToL2CL(gt *testing.T) {
+	gt.Skip() // TODO(#16187): Fix to use multi-supervisor (not multi-node) setup.
 	t := devtest.SerialT(gt)
 
 	// Sequencer and verifier are connected via P2P, which makes their unsafe heads in sync.

op-e2e/actions/interop/interop_fork_test.go

@@ -77,27 +77,19 @@ func TestInteropUpgrade(gt *testing.T) {
 			c.Sequencer.ActL2PipelineFull(t)
 		}, dsl.WithFinalizedAdvancesTo(1)) // assert unsafe head does not move, but update finalized head
 
-		// Interop
+		////////////////////////////
+		// Unsafe Interop Upgrade (for the current chain in iteration)
+		////////////////////////////
+
 		syncAsserter.RequireSeqSyncStatus(func() {
 			// Build another L2 block so that Interop activates
 			system.AddL2Block(c, dsl.WithL2BlocksUntilTimestamp(*c.Sequencer.RollupCfg.InteropTime))
 		}, dsl.WithUnsafeAdvancesTo(2), dsl.WithCrossUnsafeAdvancesTo(2)) // assert unsafe and crossUnsafe advance by one
 
-		////////////////////////////
-		// After Interop Upgrade (for the current chain in iteration)
-		////////////////////////////
-
 		dsl.RequireUnsafeTimeOffset(t, c, 4) // interop should be enabled
-		// Assert for the first time on supervisor heads
-		dsl.RequireSupervisorChainHeads(t, svr, c, syncAsserter.PrevStatus.UnsafeL2.ID(), syncAsserter.PrevStatus.CrossUnsafeL2.ID(), syncAsserter.PrevStatus.LocalSafeL2.ID(), syncAsserter.PrevStatus.SafeL2.ID(), eth.BlockID{Hash: common.BytesToHash([]byte{}), Number: 0})
-
-		// TODO(15863): The initial state of the finalized head differs between supervisor
-		// and sequencer. In an action test setup, the sequencer considers
-		// genesis head as finalized from the start. The supervisor however sets
-		// the finalized head to a nil value until it it set. Seems worth fixing.
-		// The following will fail.
-		// system.ActSyncSupernode(t, dsl.WithChains(c), dsl.WithFinalizedSignal(), dsl.WithLatestSignal(), dsl.WithRequireFinalizedAdvances())
-		// syncAsserter.RequireSupChainHeadsBySyncStatus()
+
+		// Safe head hasn't moved yet.
+		dsl.RequireSupervisorChainHeads(t, svr, c, syncAsserter.PrevStatus.UnsafeL2.ID(), syncAsserter.PrevStatus.CrossUnsafeL2.ID(), eth.BlockID{}, eth.BlockID{}, eth.BlockID{})
 	}
 
 	// Settle both chains to DA again and have supervisor ingest
@@ -107,6 +99,9 @@ func TestInteropUpgrade(gt *testing.T) {
 	dsl.RequireL1Heads(t, system, 3, 1)
 
 	// // TODO: Why doesn't the cross head promotion happen here?
+	// Seb: I think because there's always one round of back and forth between supervisor and node required
+	// per L1 block. The node processed until an ExhaustL1Event, then needs the supervisor
+	// to give it the next L1 block.
 	superchainSyncAsserter.RequireAllSeqSyncStatuses(func() {
 		system.ActSyncSupernode(t, dsl.WithLatestSignal())
 	}, dsl.WithMapChainAssertions(dsl.WithLocalSafeAdvancesTo(2)))
@@ -115,14 +110,21 @@ func TestInteropUpgrade(gt *testing.T) {
 		system.ActSyncSupernode(t)
 	}, dsl.WithMapChainAssertions(dsl.WithSafeAdvancesTo(2)))
 
-	// Note that finalized remains nil
+	// TODO(15863): The initial state of the finalized head differs between supervisor
+	// and sequencer. In an action test setup, the sequencer considers
+	// genesis head as finalized from the start. The supervisor however sets
+	// the finalized head to a nil value until it it set. Seems worth fixing.
+	// The following will fail.
+	// system.ActSyncSupernode(t, dsl.WithChains(c), dsl.WithFinalizedSignal(), dsl.WithLatestSignal(), dsl.WithRequireFinalizedAdvances())
+	// syncAsserter.RequireSupChainHeadsBySyncStatus()
 	for _, c := range chains {
 		syncAsserter := superchainSyncAsserter.ChainAsserters[c.ChainID]
-		dsl.RequireSupervisorChainHeads(t, svr, c, syncAsserter.PrevStatus.UnsafeL2.ID(), syncAsserter.PrevStatus.CrossUnsafeL2.ID(), syncAsserter.PrevStatus.LocalSafeL2.ID(), syncAsserter.PrevStatus.SafeL2.ID(), eth.BlockID{Hash: common.BytesToHash([]byte{}), Number: 0})
+		dsl.RequireSupervisorChainHeads(t, svr, c, syncAsserter.PrevStatus.UnsafeL2.ID(), syncAsserter.PrevStatus.CrossUnsafeL2.ID(), syncAsserter.PrevStatus.LocalSafeL2.ID(), syncAsserter.PrevStatus.SafeL2.ID(), eth.BlockID{})
 	}
 
 	// Verify proofs agree
-	assertProgramOutputMatchesDerivationForBlockTimestamp(gt, system, system.Actors.ChainA.Sequencer.L2Unsafe().Time)
+	// TODO(#16166): Fix non-genesis Interop activation proofs
+	// assertProgramOutputMatchesDerivationForBlockTimestamp(gt, system, system.Actors.ChainA.Sequencer.L2Safe().Time)
 
 	superchainSyncAsserter.RequireAllSeqSyncStatuses(func() {
 		// Advance L1 safe head and finalized head
@@ -140,9 +142,6 @@ func TestInteropUpgrade(gt *testing.T) {
 		syncAsserter.RequireSupChainHeadsBySyncStatus()
 	}
 
-	// Verify proofs agree on prev blocks
-	assertProgramOutputMatchesDerivationForBlockTimestamp(gt, system, system.Actors.ChainA.Sequencer.L2Safe().Time)
-
 	for _, c := range chains {
 		// Advance unsafe head again
 		syncAsserter := superchainSyncAsserter.ChainAsserters[c.ChainID]

op-e2e/actions/interop/proofs_test.go

@@ -176,6 +176,8 @@ func TestInteropFaultProofs_ConsolidateValidCrossChainMessage(gt *testing.T) {
 }
 
 func TestInteropFaultProofs_PreForkActivation(gt *testing.T) {
+	// TODO(#16166): Fix non-genesis Interop activation proofs
+	gt.Skip()
 	t := helpers.NewDefaultTesting(gt)
 	system := dsl.NewInteropDSL(t, dsl.SetInteropForkScheduledButInactive())
 

op-e2e/interop/interop_test.go

@@ -364,6 +364,7 @@ func TestInteropBlockBuilding(t *testing.T) {
 }
 
 func TestMultiNode(t *testing.T) {
+	t.Skip() // TODO(#16174): Decide on future of multi-node support
 	t.Parallel()
 	test := func(t *testing.T, s2 SuperSystem) {
 		supervisor := s2.SupervisorClient()

op-node/rollup/interop/managed/api.go

@@ -39,6 +39,7 @@ func (ib *InteropAPI) InvalidateBlock(ctx context.Context, seal supervisortypes.
 	return ib.backend.InvalidateBlock(ctx, seal)
 }
 
+// TODO(#16140): remove
 func (ib *InteropAPI) AnchorPoint(ctx context.Context) (supervisortypes.DerivedBlockRefPair, error) {
 	return ib.backend.AnchorPoint(ctx)
 }
@@ -47,6 +48,10 @@ func (ib *InteropAPI) Reset(ctx context.Context, lUnsafe, xUnsafe, lSafe, xSafe,
 	return ib.backend.Reset(ctx, lUnsafe, xUnsafe, lSafe, xSafe, finalized)
 }
 
+func (ib *InteropAPI) ResetPreInterop(ctx context.Context) error {
+	return ib.backend.ResetPreInterop(ctx)
+}
+
 func (ib *InteropAPI) FetchReceipts(ctx context.Context, blockHash common.Hash) (types.Receipts, error) {
 	return ib.backend.FetchReceipts(ctx, blockHash)
 }

op-node/rollup/interop/managed/system.go

@@ -54,6 +54,7 @@ type ManagedMode struct {
 }
 
 func NewManagedMode(log log.Logger, cfg *rollup.Config, addr string, port int, jwtSecret eth.Bytes32, l1 L1Source, l2 L2Source, m opmetrics.RPCMetricer) *ManagedMode {
+	log = log.With("mode", "managed", "chainId", cfg.L2ChainID)
 	out := &ManagedMode{
 		log:       log,
 		cfg:       cfg,
@@ -113,21 +114,34 @@ func (m *ManagedMode) AttachEmitter(em event.Emitter) {
 	m.emitter = em
 }
 
+// Outgoing events to supervisor
 func (m *ManagedMode) OnEvent(ev event.Event) bool {
 	switch x := ev.(type) {
 	case rollup.ResetEvent:
 		msg := x.Err.Error()
 		m.events.Send(&supervisortypes.ManagedEvent{Reset: &msg})
 	case engine.UnsafeUpdateEvent:
+		if !m.cfg.IsInterop(x.Ref.Time) {
+			m.log.Debug("Ignoring non-Interop local unsafe update", "unsafe", x.Ref)
+			return false
+		}
 		ref := x.Ref.BlockRef()
 		m.events.Send(&supervisortypes.ManagedEvent{UnsafeBlock: &ref})
 	case engine.LocalSafeUpdateEvent:
+		if !m.cfg.IsInterop(x.Ref.Time) {
+			m.log.Debug("Ignoring non-Interop local safe update", "derivedFrom", x.Source, "derived", x.Ref)
+			return false
+		}
 		m.log.Info("Emitting local safe update because of L2 block", "derivedFrom", x.Source, "derived", x.Ref)
 		m.events.Send(&supervisortypes.ManagedEvent{DerivationUpdate: &supervisortypes.DerivedBlockRefPair{
 			Source:  x.Source,
 			Derived: x.Ref.BlockRef(),
 		}})
 	case derive.DeriverL1StatusEvent:
+		if !m.cfg.IsInterop(x.LastL2.Time) {
+			m.log.Debug("Ignoring non-Interop L1 traversal", "origin", x.Origin, "lastL2", x.LastL2)
+			return false
+		}
 		m.log.Info("Emitting local safe update because of L1 traversal", "derivedFrom", x.Origin, "derived", x.LastL2)
 		m.events.Send(&supervisortypes.ManagedEvent{
 			DerivationUpdate: &supervisortypes.DerivedBlockRefPair{
@@ -240,6 +254,15 @@ func (m *ManagedMode) InvalidateBlock(ctx context.Context, seal supervisortypes.
 }
 
 func (m *ManagedMode) AnchorPoint(ctx context.Context) (supervisortypes.DerivedBlockRefPair, error) {
+	// TODO: maybe cache non-genesis anchor point when seeing safe Interop activation block?
+	//  Only needed if we don't test for activation block in the supervisor.
+	if !m.cfg.IsInterop(m.cfg.Genesis.L2Time) {
+		return supervisortypes.DerivedBlockRefPair{}, &gethrpc.JsonError{
+			Code:    InteropInactiveRPCErrCode,
+			Message: "Interop inactive at genesis",
+		}
+	}
+
 	l1Ref, err := m.l1.L1BlockRefByHash(ctx, m.cfg.Genesis.L1.Hash)
 	if err != nil {
 		return supervisortypes.DerivedBlockRefPair{}, fmt.Errorf("failed to fetch L1 block ref: %w", err)
@@ -258,8 +281,16 @@ const (
 	InternalErrorRPCErrcode    = -32603
 	BlockNotFoundRPCErrCode    = -39001
 	ConflictingBlockRPCErrCode = -39002
+	InteropInactiveRPCErrCode  = -39003
 )
 
+// TODO: add ResetPreInterop, called by supervisor if bisection went pre-Interop. Emit ResetEngineRequestEvent.
+func (m *ManagedMode) ResetPreInterop(ctx context.Context) error {
+	m.log.Info("Received pre-interop reset request")
+	m.emitter.Emit(engine.ResetEngineRequestEvent{})
+	return nil
+}
+
 func (m *ManagedMode) Reset(ctx context.Context, lUnsafe, xUnsafe, lSafe, xSafe, finalized eth.BlockID) error {
 	logger := m.log.New(
 		"localUnsafe", lUnsafe,

op-node/rollup/sync/start.go

@@ -48,9 +48,11 @@ type L2Chain interface {
 	L2BlockRefByLabel(ctx context.Context, label eth.BlockLabel) (eth.L2BlockRef, error)
 }
 
-var ReorgFinalizedErr = errors.New("cannot reorg finalized block")
-var WrongChainErr = errors.New("wrong chain")
-var TooDeepReorgErr = errors.New("reorg is too deep")
+var (
+	ReorgFinalizedErr = errors.New("cannot reorg finalized block")
+	WrongChainErr     = errors.New("wrong chain")
+	TooDeepReorgErr   = errors.New("reorg is too deep")
+)
 
 const MaxReorgSeqWindows = 5
 
@@ -156,6 +158,8 @@ func FindL2Heads(ctx context.Context, cfg *rollup.Config, l1 L1Chain, l2 L2Chain
 	// Once we pass the previous safe head and we have seen enough canonical L1 origins to fill a sequence window worth of data,
 	// then we return the last L2 block of the epoch before that as safe head.
 	// Each loop iteration we traverse a single L2 block, and we check if the L1 origins are consistent.
+	// TODO(#16141): maybe check inside here not to go over Interop activation, this should be handled by supervisor
+	// Fine to keep unsafe chain. Safe chain should be capped to stay pre-Interop.
 	for {
 		// Fetch L1 information if we never had it, or if we do not have it for the current origin.
 		// Optimization: as soon as we have a previous L1 block, try to traverse L1 by hash instead of by number, to fill the cache.

op-supervisor/supervisor/backend/backend.go

@@ -81,8 +81,10 @@ type SupervisorBackend struct {
 	rpcVerificationWarnings bool
 }
 
-var _ event.AttachEmitter = (*SupervisorBackend)(nil)
-var _ frontend.Backend = (*SupervisorBackend)(nil)
+var (
+	_ event.AttachEmitter = (*SupervisorBackend)(nil)
+	_ frontend.Backend    = (*SupervisorBackend)(nil)
+)
 
 var (
 	errAlreadyStopped        = errors.New("already stopped")
@@ -97,7 +99,8 @@ var (
 var verifyAccessWithRPCTimeout = 10 * time.Second
 
 func NewSupervisorBackend(ctx context.Context, logger log.Logger,
-	m Metrics, cfg *config.Config, eventExec event.Executor) (*SupervisorBackend, error) {
+	m Metrics, cfg *config.Config, eventExec event.Executor,
+) (*SupervisorBackend, error) {
 	// attempt to prepare the data directory
 	if err := db.PrepDataDir(cfg.Datadir); err != nil {
 		return nil, err
@@ -176,6 +179,17 @@ func NewSupervisorBackend(ctx context.Context, logger log.Logger,
 func (su *SupervisorBackend) OnEvent(ev event.Event) bool {
 	switch x := ev.(type) {
 	case superevents.LocalUnsafeReceivedEvent:
+		if !su.cfgSet.IsInterop(x.ChainID, x.NewLocalUnsafe.Time) {
+			su.logger.Warn("ignoring local unsafe received event for pre-interop block", "chainID", x.ChainID, "unsafe", x.NewLocalUnsafe)
+			return false
+		} else if su.cfgSet.IsInteropActivationBlock(x.ChainID, x.NewLocalUnsafe.Time) {
+			su.emitter.Emit(superevents.UnsafeActivationBlockEvent{
+				ChainID: x.ChainID,
+				Unsafe:  x.NewLocalUnsafe,
+			})
+			// don't process events of the activation block
+			return true
+		}
 		su.emitter.Emit(superevents.ChainProcessEvent{
 			ChainID: x.ChainID,
 			Target:  x.NewLocalUnsafe.Number,
@@ -188,6 +202,16 @@ func (su *SupervisorBackend) OnEvent(ev event.Event) bool {
 		su.emitter.Emit(superevents.UpdateCrossUnsafeRequestEvent{
 			ChainID: x.ChainID,
 		})
+	case superevents.LocalDerivedEvent:
+		if !su.cfgSet.IsInterop(x.ChainID, x.Derived.Derived.Time) {
+			su.logger.Warn("ignoring local derived event for pre-interop block", "chainID", x.ChainID, "derived", x.Derived.Derived)
+			return false
+		} else if su.cfgSet.IsInteropActivationBlock(x.ChainID, x.Derived.Derived.Time) {
+			su.emitter.Emit(superevents.SafeActivationBlockEvent{
+				ChainID: x.ChainID,
+				Safe:    x.Derived,
+			})
+		}
 	case superevents.LocalSafeUpdateEvent:
 		su.emitter.Emit(superevents.ChainProcessEvent{
 			ChainID: x.ChainID,
@@ -299,6 +323,20 @@ func (su *SupervisorBackend) openChainDBs(chainID eth.ChainID) error {
 
 	su.chainDBs.AddCrossUnsafeTracker(chainID)
 
+	// If Interop is active at genesis, emit SafeActivationBlockEvent so that the DB
+	// can initialize, if needed.
+	genesis := su.cfgSet.Genesis(chainID)
+	if su.cfgSet.IsInterop(chainID, genesis.L2.Timestamp) {
+		su.emitter.Emit(superevents.SafeActivationBlockEvent{
+			ChainID: chainID,
+			Safe: types.DerivedBlockRefPair{
+				// Initialization skips parent checks, so zero parents are ok.
+				Source:  genesis.L1.WithZeroParent(),
+				Derived: genesis.L2.WithZeroParent(),
+			},
+		})
+	}
+
 	return nil
 }
 
@@ -314,11 +352,6 @@ func (su *SupervisorBackend) AttachSyncNode(ctx context.Context, src syncnode.Sy
 	if !su.cfgSet.HasChain(chainID) {
 		return nil, fmt.Errorf("chain %s is not part of the interop dependency set: %w", chainID, types.ErrUnknownChain)
 	}
-	// before attaching the sync source to the backend at all,
-	// query the anchor point to initialize the database
-	if err := su.QueryAnchorpoint(chainID, src); err != nil {
-		return nil, fmt.Errorf("failed to query anchor point: %w", err)
-	}
 	err = su.AttachProcessorSource(chainID, src)
 	if err != nil {
 		return nil, fmt.Errorf("failed to attach sync source to processor: %w", err)
@@ -330,18 +363,6 @@ func (su *SupervisorBackend) AttachSyncNode(ctx context.Context, src syncnode.Sy
 	return su.syncNodesController.AttachNodeController(chainID, src, noSubscribe)
 }
 
-func (su *SupervisorBackend) QueryAnchorpoint(chainID eth.ChainID, src syncnode.SyncNode) error {
-	anchor, err := src.AnchorPoint(context.Background())
-	if err != nil {
-		return fmt.Errorf("failed to get anchor point: %w", err)
-	}
-	su.emitter.Emit(superevents.AnchorEvent{
-		ChainID: chainID,
-		Anchor:  anchor,
-	})
-	return nil
-}
-
 func (su *SupervisorBackend) AttachProcessorSource(chainID eth.ChainID, src processors.Source) error {
 	proc, ok := su.chainProcessors.Get(chainID)
 	if !ok {
@@ -364,7 +385,7 @@ func (su *SupervisorBackend) attachL1RPC(ctx context.Context, l1RPCAddr string)
 	su.logger.Info("attaching L1 RPC to L1 processor", "rpc", l1RPCAddr)
 
 	logger := su.logger.New("l1-rpc", l1RPCAddr)
-	l1RPC, err := client.NewRPC(ctx, logger, l1RPCAddr)
+	l1RPC, err := client.NewRPC(ctx, logger, l1RPCAddr, client.WithLazyDial())
 	if err != nil {
 		return fmt.Errorf("failed to setup L1 RPC: %w", err)
 	}
@@ -523,7 +544,8 @@ func (su *SupervisorBackend) checkSafety(chainID eth.ChainID, blockID eth.BlockI
 }
 
 func (su *SupervisorBackend) CheckAccessList(ctx context.Context, inboxEntries []common.Hash,
-	minSafety types.SafetyLevel, executingDescriptor types.ExecutingDescriptor) error {
+	minSafety types.SafetyLevel, executingDescriptor types.ExecutingDescriptor,
+) error {
 	switch minSafety {
 	case types.LocalUnsafe, types.CrossUnsafe, types.LocalSafe, types.CrossSafe, types.Finalized:
 		// valid safety level
@@ -662,7 +684,7 @@ func (su *SupervisorBackend) FinalizedL1(ctx context.Context) (eth.BlockRef, err
 	return v, nil
 }
 
-func (su *SupervisorBackend) AnchorPoint(ctx context.Context, chainID eth.ChainID) (types.DerivedBlockSealPair, error) {
+func (su *SupervisorBackend) ActivationBlock(ctx context.Context, chainID eth.ChainID) (types.DerivedBlockSealPair, error) {
 	return su.chainDBs.AnchorPoint(chainID)
 }