diff --git a/core/capabilities/registry.go b/core/capabilities/registry.go index c0790a21ebb..6be10c9c194 100644 --- a/core/capabilities/registry.go +++ b/core/capabilities/registry.go @@ -63,6 +63,16 @@ func (r *Registry) DONsForCapability(ctx context.Context, capabilityID string) ( return r.metadataRegistry.DONsForCapability(ctx, capabilityID) } +func (r *Registry) DONByID(ctx context.Context, donID uint32) (capabilities.DON, error) { + r.mu.RLock() + defer r.mu.RUnlock() + if r.metadataRegistry == nil { + return capabilities.DON{}, errors.New("metadataRegistry information not available") + } + + return r.metadataRegistry.DONByID(ctx, donID) +} + // SetLocalRegistry sets a local copy of the offchain registry for the registry to use. // This is only public for testing purposes; the only production use should be from the CapabilitiesLauncher. func (r *Registry) SetLocalRegistry(lr core.CapabilitiesRegistryMetadata) { diff --git a/core/capabilities/vault/capability.go b/core/capabilities/vault/capability.go index 48b28e7fbae..3c25b849e8d 100644 --- a/core/capabilities/vault/capability.go +++ b/core/capabilities/vault/capability.go @@ -31,6 +31,7 @@ type Capability struct { capabilitiesRegistry core.CapabilitiesRegistry publicKey *LazyPublicKey lifecycle *RequestLifecycleTracker + zoneBRestrictor *zoneBRestrictor *RequestValidator } @@ -70,6 +71,10 @@ func (s *Capability) Close() error { err = errors.Join(err, fmt.Errorf("error closing request validator: %w", lerr)) } + if lerr := s.zoneBRestrictor.close(); lerr != nil { + err = errors.Join(err, lerr) + } + return err } @@ -99,6 +104,13 @@ func (s *Capability) Execute(ctx context.Context, request capabilities.Capabilit return capabilities.CapabilityResponse{}, errors.New("unsupported method: can only call GetSecrets via capability interface") } + // Reject GetSecrets reads from a restricted zone-b workflow DON before the + // request enters the OCR queue, so non-allowlisted owners get a deterministic + // rejection and zone-a / gateway paths are unaffected. + if err := s.zoneBRestrictor.enforce(ctx, request.Metadata.WorkflowDonID); err != nil { + return capabilities.CapabilityResponse{}, err + } + r := &vaultcommon.GetSecretsRequest{} err := request.Payload.UnmarshalTo(r) if err != nil { @@ -310,6 +322,13 @@ func NewCapability( if err != nil { return nil, err } + zoneBRestrictor, err := newZoneBRestrictor(lggr, limitsFactory, capabilitiesRegistry) + if err != nil { + return nil, err + } + if zoneBRestrictor == nil { + return nil, errors.New("vault capability requires a non-nil zone-b restrictor") + } return &Capability{ lggr: logger.Named(lggr, "VaultCapability"), clock: clock, @@ -318,6 +337,7 @@ func NewCapability( capabilitiesRegistry: capabilitiesRegistry, publicKey: publicKey, lifecycle: lifecycle, + zoneBRestrictor: zoneBRestrictor, RequestValidator: requestValidator, }, nil } diff --git a/core/capabilities/vault/zone_b_restriction.go b/core/capabilities/vault/zone_b_restriction.go new file mode 100644 index 00000000000..2faefb919e2 --- /dev/null +++ b/core/capabilities/vault/zone_b_restriction.go @@ -0,0 +1,146 @@ +package vault + +import ( + "context" + "errors" + "fmt" + "slices" + "strings" + "sync" + + "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" +) + +// zoneBFamily is the DON family (in the capabilities registry) identifying the +// zone-b workflow DON whose vault GetSecrets reads are restricted to an +// allowlist of workflow owners. +const zoneBFamily = "zone-b" + +// zoneBRestrictor enforces that GetSecrets reads originating from a zone-b +// workflow DON are limited to an allowlist of workflow owners. Zone membership +// is resolved authoritatively from the node's own capabilities registry view, +// never from caller-supplied metadata. +type zoneBRestrictor struct { + lggr logger.Logger + capabilitiesRegistry core.CapabilitiesRegistry + // restrictEnabled is the master gate. When open, GetSecrets reads from a + // zone-b workflow DON are restricted to allowlisted workflow owners. + restrictEnabled limits.GateLimiter + // ownerAllowed is the owner-scoped allowlist gate consulted only for zone-b + // callers when the master gate is open. + ownerAllowed limits.GateLimiter + + // zoneCacheMu guards zoneCache. + zoneCacheMu sync.RWMutex + // zoneCache holds the last successfully-resolved zone-b membership per + // WorkflowDonID. It is the fallback when the capabilities registry view is + // transiently unavailable, so a registry blip does not fail every vault read + // DON-wide (see isZoneBWorkflowDON). + zoneCache map[uint32]bool +} + +func newZoneBRestrictor(lggr logger.Logger, limitsFactory limits.Factory, capabilitiesRegistry core.CapabilitiesRegistry) (*zoneBRestrictor, error) { + restrictEnabled, err := limits.MakeGateLimiter(limitsFactory, cresettings.Default.VaultZoneBWorkflowGetSecretsRestrictEnabled) + if err != nil { + return nil, fmt.Errorf("failed to create zone-b restrict gate limiter: %w", err) + } + ownerAllowed, err := limits.MakeGateLimiter(limitsFactory, cresettings.Default.PerOwner.VaultZoneBGetSecretsAllowed) + if err != nil { + return nil, fmt.Errorf("failed to create zone-b owner allowlist gate limiter: %w", err) + } + if restrictEnabled == nil || ownerAllowed == nil { + return nil, errors.New("zone-b restrictor requires non-nil gate limiters") + } + return &zoneBRestrictor{ + lggr: logger.Named(lggr, "ZoneBRestrictor"), + capabilitiesRegistry: capabilitiesRegistry, + restrictEnabled: restrictEnabled, + ownerAllowed: ownerAllowed, + zoneCache: make(map[uint32]bool), + }, nil +} + +// enforce denies GetSecrets reads originating from a zone-b workflow DON unless +// the calling workflow owner is allowlisted. It is a no-op unless the master +// gate (VaultZoneBWorkflowGetSecretsRestrictEnabled) is open and the caller +// resolves to a zone-b DON. The owner is read from ctx, which must already carry +// the (normalized) CRE owner via RequestMetadata.ContextWithCRE. +func (z *zoneBRestrictor) enforce(ctx context.Context, workflowDonID uint32) error { + enabled, err := z.restrictEnabled.Limit(ctx) + if err != nil { + return fmt.Errorf("could not evaluate zone-b vault read restriction gate: %w", err) + } + if !enabled { + return nil + } + + isZoneB, err := z.isZoneBWorkflowDON(ctx, workflowDonID) + if err != nil { + // Fail closed: if we cannot authoritatively resolve the caller's zone, do + // not proceed. The registry is in-process, so this only fires for an + // unknown/unregistered WorkflowDonID. + return err + } + if !isZoneB { + return nil + } + + if err := z.ownerAllowed.AllowErr(ctx); err != nil { + return fmt.Errorf("zone-b workflow DON may only read vault secrets for allowlisted workflow owners: %w", err) + } + return nil +} + +// isZoneBWorkflowDON authoritatively resolves the caller's WorkflowDonID against +// the node's own capabilities registry view and reports whether that DON is in +// the zone-b family. This does not trust any caller-supplied zone data. +func (z *zoneBRestrictor) isZoneBWorkflowDON(ctx context.Context, workflowDonID uint32) (bool, error) { + don, err := z.capabilitiesRegistry.DONByID(ctx, workflowDonID) + if err != nil { + // The registry view can be transiently unavailable (e.g. not yet synced + // after startup, or nil mid-update: DONByID returns "metadataRegistry + // information not available"). That error is not specific to zone-b + // callers, so failing closed here would block every vault GetSecrets read + // DON-wide. Fall back to the last successfully-resolved membership for this + // DON; only a never-before-resolved DON fails closed. + if cached, ok := z.cachedZoneMembership(workflowDonID); ok { + z.lggr.Warnw("capabilities registry lookup failed; using cached zone-b membership", + "workflowDonID", workflowDonID, "isZoneB", cached, "err", err) + return cached, nil + } + return false, fmt.Errorf("could not resolve caller workflow DON %d for zone-b vault read restriction: %w", workflowDonID, err) + } + // Case-insensitive match: family casing may vary across registry sources. + isZoneB := slices.ContainsFunc(don.Families, func(family string) bool { + return strings.EqualFold(family, zoneBFamily) + }) + z.storeZoneMembership(workflowDonID, isZoneB) + return isZoneB, nil +} + +func (z *zoneBRestrictor) cachedZoneMembership(workflowDonID uint32) (bool, bool) { + z.zoneCacheMu.RLock() + defer z.zoneCacheMu.RUnlock() + isZoneB, ok := z.zoneCache[workflowDonID] + return isZoneB, ok +} + +func (z *zoneBRestrictor) storeZoneMembership(workflowDonID uint32, isZoneB bool) { + z.zoneCacheMu.Lock() + defer z.zoneCacheMu.Unlock() + z.zoneCache[workflowDonID] = isZoneB +} + +func (z *zoneBRestrictor) close() error { + var err error + if cerr := z.restrictEnabled.Close(); cerr != nil { + err = errors.Join(err, fmt.Errorf("error closing zone-b restrict gate limiter: %w", cerr)) + } + if cerr := z.ownerAllowed.Close(); cerr != nil { + err = errors.Join(err, fmt.Errorf("error closing zone-b owner allowlist gate limiter: %w", cerr)) + } + return err +} diff --git a/core/capabilities/vault/zone_b_restriction_test.go b/core/capabilities/vault/zone_b_restriction_test.go new file mode 100644 index 00000000000..17f027ba4e8 --- /dev/null +++ b/core/capabilities/vault/zone_b_restriction_test.go @@ -0,0 +1,290 @@ +package vault + +import ( + "context" + "errors" + "fmt" + "sync/atomic" + "testing" + "time" + + "github.com/jonboulle/clockwork" + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/types/known/anypb" + + "github.com/smartcontractkit/chainlink-common/pkg/capabilities" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/actions/vault" + "github.com/smartcontractkit/chainlink-common/pkg/capabilities/consensus/requests" + "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" + "github.com/smartcontractkit/chainlink-common/pkg/settings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" + "github.com/smartcontractkit/chainlink-common/pkg/types/core" + coreCapabilities "github.com/smartcontractkit/chainlink/v2/core/capabilities" + "github.com/smartcontractkit/chainlink/v2/core/capabilities/vault/vaulttypes" + "github.com/smartcontractkit/chainlink/v2/core/logger" +) + +// fakeMetadataRegistry resolves DONByID from an in-memory map, standing in for +// the node's local capabilities registry view. +type fakeMetadataRegistry struct { + core.UnimplementedCapabilitiesRegistryMetadata + dons map[uint32]capabilities.DON +} + +func (f *fakeMetadataRegistry) DONByID(_ context.Context, donID uint32) (capabilities.DON, error) { + d, ok := f.dons[donID] + if !ok { + return capabilities.DON{}, fmt.Errorf("could not find don %d", donID) + } + return d, nil +} + +const ( + zoneBDonID = uint32(10) + zoneADonID = uint32(20) + // zoneBMixedCaseDonID is a zone-b DON whose registry family casing differs + // from the zoneBFamily constant, guarding the case-insensitive match. + zoneBMixedCaseDonID = uint32(30) +) + +func newZoneBTestCapability(t *testing.T, settingsJSON string) *Capability { + t.Helper() + lggr := logger.TestLogger(t) + clock := clockwork.NewFakeClock() + expiry := 10 * time.Second + store := requests.NewStore[*vaulttypes.Request]() + handler := requests.NewHandler(lggr, store, clock, expiry) + + reg := coreCapabilities.NewRegistry(lggr) + reg.SetLocalRegistry(&fakeMetadataRegistry{dons: map[uint32]capabilities.DON{ + zoneBDonID: {ID: zoneBDonID, Name: "workflow_1_zone-b", Families: []string{"zone-b"}}, + zoneADonID: {ID: zoneADonID, Name: "workflow_1_zone-a", Families: []string{"zone-a"}}, + zoneBMixedCaseDonID: {ID: zoneBMixedCaseDonID, Name: "workflow_1_zone-b_mixed", Families: []string{"Zone-B"}}, + }}) + + getter, err := settings.NewJSONGetter([]byte(settingsJSON)) + require.NoError(t, err) + lf := limits.Factory{Settings: getter} + + capability, err := NewCapability(lggr, clock, expiry, handler, reg, nil, lf, newTestRequestLifecycleTracker(t)) + require.NoError(t, err) + servicetest.Run(t, capability) + return capability +} + +// allowlistedOwner is the normalized (no 0x, lowercase) form of the workflow +// owner used across these tests. +const allowlistedOwner = "1111111111111111111111111111111111111111" + +func zoneBGetSecretsRequest(t *testing.T, owner string) capabilities.CapabilityRequest { + t.Helper() + gsr := &vault.GetSecretsRequest{ + Requests: []*vault.SecretRequest{ + { + Id: &vault.SecretIdentifier{Key: "Foo", Namespace: "Bar", Owner: owner}, + EncryptionKeys: []string{"k"}, + }, + }, + } + anyproto, err := anypb.New(gsr) + require.NoError(t, err) + return capabilities.CapabilityRequest{ + Payload: anyproto, + Method: vault.MethodGetSecrets, + Metadata: capabilities.RequestMetadata{ + WorkflowOwner: owner, + WorkflowID: "wf-id", + WorkflowExecutionID: "exec-id", + ReferenceID: "ref-id", + WorkflowDonID: zoneBDonID, + }, + } +} + +func TestCapability_Execute_ZoneBRestriction_DeniesNonAllowlistedOwner(t *testing.T) { + t.Parallel() + // Master gate enabled, owner NOT allowlisted (default deny). + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}`) + + _, err := capability.Execute(t.Context(), zoneBGetSecretsRequest(t, "0x"+allowlistedOwner)) + require.Error(t, err) + require.ErrorContains(t, err, "zone-b workflow DON may only read vault secrets for allowlisted workflow owners") +} + +// MUST-2: exercises the allow path through Execute() rather than calling +// enforce() directly, covering the CRE-context population wiring. An allowlisted +// zone-b owner must pass the gate; with no OCR oracle in this unit test the +// request then blocks in handleRequest until the context deadline, so we assert +// it got past the gate (no zone-b denial) and reached the handler (timeout). +func TestCapability_Execute_ZoneBRestriction_AllowsAllowlistedOwner(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"},"owner":{"`+allowlistedOwner+`":{"PerOwner":{"VaultZoneBGetSecretsAllowed":"true"}}}}`) + + ctx, cancel := context.WithTimeout(t.Context(), 500*time.Millisecond) + defer cancel() + _, err := capability.Execute(ctx, zoneBGetSecretsRequest(t, "0x"+allowlistedOwner)) + require.Error(t, err) + require.NotContains(t, err.Error(), "zone-b workflow DON may only read vault secrets for allowlisted workflow owners") + require.ErrorIs(t, err, context.DeadlineExceeded) +} + +func TestCapability_enforceZoneBWorkflowRestriction(t *testing.T) { + t.Parallel() + ctxWithOwner := func(donID uint32, owner string) (context.Context, uint32) { + md := capabilities.RequestMetadata{WorkflowOwner: owner, WorkflowDonID: donID} + return md.ContextWithCRE(t.Context()), donID + } + + t.Run("gate disabled: allows zone-b caller regardless of owner", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"false"}}`) + ctx, donID := ctxWithOwner(zoneBDonID, "0x"+allowlistedOwner) + require.NoError(t, capability.zoneBRestrictor.enforce(ctx, donID)) + }) + + t.Run("gate enabled: allows non-zone-b (zone-a) caller", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}`) + ctx, donID := ctxWithOwner(zoneADonID, "0xdeadbeef") + require.NoError(t, capability.zoneBRestrictor.enforce(ctx, donID)) + }) + + t.Run("gate enabled: denies zone-b caller with non-allowlisted owner", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}`) + ctx, donID := ctxWithOwner(zoneBDonID, "0x"+allowlistedOwner) + err := capability.zoneBRestrictor.enforce(ctx, donID) + require.ErrorContains(t, err, "allowlisted workflow owners") + }) + + t.Run("gate enabled: allows zone-b caller with allowlisted owner", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"},"owner":{"`+allowlistedOwner+`":{"PerOwner":{"VaultZoneBGetSecretsAllowed":"true"}}}}`) + ctx, donID := ctxWithOwner(zoneBDonID, "0x"+allowlistedOwner) + require.NoError(t, capability.zoneBRestrictor.enforce(ctx, donID)) + }) + + t.Run("gate enabled: fails closed for unknown caller DON", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}`) + md := capabilities.RequestMetadata{WorkflowOwner: "0x" + allowlistedOwner, WorkflowDonID: 999} + err := capability.zoneBRestrictor.enforce(md.ContextWithCRE(t.Context()), 999) + require.ErrorContains(t, err, "could not resolve caller workflow DON 999") + }) + + // MUST-3: guards the case-insensitive family match. If isZoneBWorkflowDON + // regressed to an exact (==) comparison, a "Zone-B" family would be treated + // as non-zone-b and this caller would be allowed, failing this test. + t.Run("gate enabled: denies zone-b caller identified by case-insensitive family match", func(t *testing.T) { + t.Parallel() + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}`) + ctx, donID := ctxWithOwner(zoneBMixedCaseDonID, "0x"+allowlistedOwner) + err := capability.zoneBRestrictor.enforce(ctx, donID) + require.ErrorContains(t, err, "allowlisted workflow owners") + }) + + // MUST-4: the allowlist is keyed by the normalized owner (0x stripped, + // lowercased), while callers supply the standard "0x"-prefixed, possibly + // mixed-case address. If owner matching regressed to a bare string compare, + // this allowlisted caller would be denied. Uses a hex owner with letters in + // mixed case so both the 0x-strip and the lowercasing are exercised. + t.Run("gate enabled: allowlist match normalizes 0x prefix and case of caller owner", func(t *testing.T) { + t.Parallel() + const mixedCaseOwner = "AbCdEf0000000000000000000000000000000001" + const normalizedOwner = "abcdef0000000000000000000000000000000001" + capability := newZoneBTestCapability(t, `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"},"owner":{"`+normalizedOwner+`":{"PerOwner":{"VaultZoneBGetSecretsAllowed":"true"}}}}`) + ctx, donID := ctxWithOwner(zoneBDonID, "0x"+mixedCaseOwner) + require.NoError(t, capability.zoneBRestrictor.enforce(ctx, donID)) + }) +} + +// toggleableMetadataRegistry resolves DONByID from an in-memory map but can be +// switched to return the same "information not available" error the real +// registry surfaces while its local view is nil/mid-sync. +type toggleableMetadataRegistry struct { + core.UnimplementedCapabilitiesRegistryMetadata + dons map[uint32]capabilities.DON + fail atomic.Bool +} + +func (f *toggleableMetadataRegistry) DONByID(_ context.Context, donID uint32) (capabilities.DON, error) { + if f.fail.Load() { + return capabilities.DON{}, errors.New("metadataRegistry information not available") + } + d, ok := f.dons[donID] + if !ok { + return capabilities.DON{}, fmt.Errorf("could not find don %d", donID) + } + return d, nil +} + +// newOutageTestRestrictor builds a zone-b restrictor backed by a toggleable +// registry so tests can warm the cache and then simulate a registry outage. +func newOutageTestRestrictor(t *testing.T, settingsJSON string) (*zoneBRestrictor, *toggleableMetadataRegistry) { + t.Helper() + lggr := logger.TestLogger(t) + reg := coreCapabilities.NewRegistry(lggr) + fake := &toggleableMetadataRegistry{dons: map[uint32]capabilities.DON{ + zoneADonID: {ID: zoneADonID, Name: "workflow_1_zone-a", Families: []string{"zone-a"}}, + zoneBDonID: {ID: zoneBDonID, Name: "workflow_1_zone-b", Families: []string{"zone-b"}}, + }} + reg.SetLocalRegistry(fake) + + getter, err := settings.NewJSONGetter([]byte(settingsJSON)) + require.NoError(t, err) + z, err := newZoneBRestrictor(lggr, limits.Factory{Settings: getter}, reg) + require.NoError(t, err) + t.Cleanup(func() { require.NoError(t, z.close()) }) + return z, fake +} + +// MUST-2: a transient capabilities registry outage must not fail every vault +// read DON-wide. Once a DON's zone membership has been resolved, a subsequent +// registry lookup failure falls back to the cached value instead of erroring; +// a DON never resolved before still fails closed. Crucially, the cached value +// preserves the security decision in both directions: a cached zone-b DON stays +// denied during the outage rather than slipping through. +func TestZoneBRestrictor_RegistryOutageUsesCachedMembership(t *testing.T) { + t.Parallel() + const restrictEnabled = `{"global":{"VaultZoneBWorkflowGetSecretsRestrictEnabled":"true"}}` + + t.Run("cached zone-a membership survives outage (still allowed)", func(t *testing.T) { + t.Parallel() + z, fake := newOutageTestRestrictor(t, restrictEnabled) + md := capabilities.RequestMetadata{WorkflowOwner: "0xdeadbeef", WorkflowDonID: zoneADonID} + ctx := md.ContextWithCRE(t.Context()) + + // Warm the cache while healthy: zone-a caller is allowed. + require.NoError(t, z.enforce(ctx, zoneADonID)) + // Outage: cached (non-zone-b) membership keeps the read allowed. + fake.fail.Store(true) + require.NoError(t, z.enforce(ctx, zoneADonID)) + }) + + t.Run("cached zone-b membership survives outage (still denied)", func(t *testing.T) { + t.Parallel() + z, fake := newOutageTestRestrictor(t, restrictEnabled) + md := capabilities.RequestMetadata{WorkflowOwner: "0x" + allowlistedOwner, WorkflowDonID: zoneBDonID} + ctx := md.ContextWithCRE(t.Context()) + + // Warm the cache while healthy: zone-b + non-allowlisted owner is denied, + // and the zone-b membership is cached. + require.ErrorContains(t, z.enforce(ctx, zoneBDonID), "allowlisted workflow owners") + + // Outage: the cached zone-b membership must still be enforced. The caller + // is denied via the allowlist, not allowed through by a resolve failure. + fake.fail.Store(true) + err := z.enforce(ctx, zoneBDonID) + require.ErrorContains(t, err, "allowlisted workflow owners") + require.NotContains(t, err.Error(), "could not resolve caller workflow DON") + }) + + t.Run("DON never resolved before outage fails closed", func(t *testing.T) { + t.Parallel() + z, fake := newOutageTestRestrictor(t, restrictEnabled) + fake.fail.Store(true) + md := capabilities.RequestMetadata{WorkflowOwner: "0x" + allowlistedOwner, WorkflowDonID: zoneBDonID} + err := z.enforce(md.ContextWithCRE(t.Context()), zoneBDonID) + require.ErrorContains(t, err, "could not resolve caller workflow DON") + }) +} diff --git a/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-vault-workflow-don-binding-enabled.toml b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-vault-workflow-don-binding-enabled.toml new file mode 100644 index 00000000000..d8d84c5dd67 --- /dev/null +++ b/core/scripts/cre/environment/configs/workflow-gateway-capabilities-don-vault-workflow-don-binding-enabled.toml @@ -0,0 +1,103 @@ +# Custom local CRE topology for Vault remote-executable WorkflowDonID binding smoke coverage. +# Enables RemoteExecutableWorkflowDONBindingEnabled on the workflow (caller) and +# capabilities (vault remote-executable server) nodes so the DON2DON binding check +# is exercised end-to-end for Vault GetSecrets. +[chip_router] + image = "local-cre-chip-router:v1.0.1" + +[[blockchains]] + type = "anvil" + chain_id = "1337" + container_name = "anvil-1337" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[[blockchains]] + type = "anvil" + chain_id = "2337" + container_name = "anvil-2337" + port = "8546" + docker_cmd_params = ["-b", "0.5", "--mixed-mining"] + +[jd] + csa_encryption_key = "d1093c0060d50a3c89c189b2e485da5a3ce57f3dcb38ab7e2c0d5f0bb2314a44" + image = "job-distributor:0.28.0" + +[fake] + port = 8171 + +[fake_http] + port = 8666 + +[infra] + type = "docker" + +[[nodesets]] + nodes = 4 + name = "workflow" + don_family = "test-don-family" + don_types = ["workflow"] + override_mode = "all" + http_port_range_start = 10100 + supported_evm_chains = [1337, 2337] + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}', CL_CRE_SETTINGS_DEFAULT = '{"RemoteExecutableWorkflowDONBindingEnabled":"true"}' } + capabilities = ["cron", "http-action", "http-trigger", "consensus", "don-time", "evm-1337"] + registry_based_launch_allowlist = ["cron-trigger@1.0.0"] + + [nodesets.db] + image = "postgres:12.0" + port = 13000 + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + user_config_overrides = "" + +[[nodesets]] + nodes = 4 + name = "capabilities" + don_family = "test-don-family" + don_types = ["capabilities"] + exposes_remote_capabilities = true + override_mode = "all" + http_port_range_start = 10200 + supported_evm_chains = [1337, 2337] + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}', CL_CRE_SETTINGS_DEFAULT = '{"RemoteExecutableWorkflowDONBindingEnabled":"true"}' } + capabilities = ["vault", "evm-2337"] + + [nodesets.db] + image = "postgres:12.0" + port = 13100 + + [[nodesets.node_specs]] + roles = ["plugin"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + user_config_overrides = "" + +[[nodesets]] + nodes = 1 + name = "bootstrap-gateway" + don_family = "test-don-family" + don_types = ["bootstrap", "gateway"] + override_mode = "each" + http_port_range_start = 10300 + supported_evm_chains = [1337, 2337] + env_vars = { CL_EVM_CMD = "", OTEL_SERVICE_NAME = "chainlink-node", CL_CRE_SETTINGS = '{"global":{"PerOrg":{"BaseTriggerRetransmitEnabled":"true"}}}' } + + [nodesets.db] + image = "postgres:12.0" + port = 13200 + + [[nodesets.node_specs]] + roles = ["bootstrap", "gateway"] + [nodesets.node_specs.node] + docker_ctx = "../../../.." + docker_file = "core/chainlink.Dockerfile" + docker_build_args = { "CL_IS_PROD_BUILD" = "false" } + custom_ports = ["5002:5002","15002:15002"] + user_config_overrides = "" diff --git a/core/services/registrysyncer/local_registry.go b/core/services/registrysyncer/local_registry.go index 0fe15dce519..b32b701ecfa 100644 --- a/core/services/registrysyncer/local_registry.go +++ b/core/services/registrysyncer/local_registry.go @@ -352,6 +352,17 @@ func (l *LocalRegistry) nodesForDON(ctx context.Context, don capabilities.DON) ( return nodes, nil } +func (l *LocalRegistry) DONByID(ctx context.Context, donID uint32) (capabilities.DON, error) { + if err := l.ensureNotEmpty(); err != nil { + return capabilities.DON{}, err + } + d, ok := l.IDsToDONs[DonID(donID)] + if !ok { + return capabilities.DON{}, fmt.Errorf("could not find don %d", donID) + } + return d.DON, nil +} + func (l *LocalRegistry) ConfigForCapability(ctx context.Context, capabilityID string, donID uint32) (capabilities.CapabilityConfiguration, error) { err := l.ensureNotEmpty() if err != nil { diff --git a/core/services/registrysyncer/local_registry_test.go b/core/services/registrysyncer/local_registry_test.go index 4fa56a1871c..0fe00d179fd 100644 --- a/core/services/registrysyncer/local_registry_test.go +++ b/core/services/registrysyncer/local_registry_test.go @@ -162,6 +162,54 @@ func TestLocalRegistry_DONsForCapability(t *testing.T) { require.ErrorContains(t, err, "could not find node for peerID") } +func TestLocalRegistry_DONByID(t *testing.T) { + t.Parallel() + lggr := logger.Test(t) + getPeerID := func() (types.PeerID, error) { + return types.PeerID{0: 1}, nil + } + idsToDons := map[DonID]DON{ + 1: { + DON: capabilities.DON{ + Name: "workflow_1_zone-b", + ID: 1, + F: 1, + Families: []string{"zone-b"}, + Members: []types.PeerID{{0: 1}}, + }, + }, + 2: { + DON: capabilities.DON{ + Name: "workflow_1_zone-a", + ID: 2, + F: 1, + Members: []types.PeerID{{0: 2}}, + }, + }, + } + idsToNodes := map[types.PeerID]NodeInfo{ + {0: 1}: {NodeOperatorID: 0}, + } + idsToCapabilities := map[string]Capability{ + "capabilityID@1.0.0": {ID: "capabilityID@1.0.0", CapabilityType: capabilities.CapabilityTypeAction}, + } + lr := NewLocalRegistry(lggr, getPeerID, idsToDons, idsToNodes, idsToCapabilities) + + got, err := lr.DONByID(t.Context(), 1) + require.NoError(t, err) + assert.Equal(t, idsToDons[1].DON, got) + assert.Contains(t, got.Families, "zone-b") + + got2, err := lr.DONByID(t.Context(), 2) + require.NoError(t, err) + assert.Equal(t, idsToDons[2].DON, got2) + assert.NotContains(t, got2.Families, "zone-b") + + // Unknown DON ID errors. + _, err = lr.DONByID(t.Context(), 99) + require.ErrorContains(t, err, "could not find don 99") +} + func mustMarshalProto(t *testing.T, msg proto.Message) []byte { t.Helper() b, err := proto.Marshal(msg) diff --git a/core/services/workflows/syncer/v2/handler.go b/core/services/workflows/syncer/v2/handler.go index 6ad16ba4af6..e21a6cfff6c 100644 --- a/core/services/workflows/syncer/v2/handler.go +++ b/core/services/workflows/syncer/v2/handler.go @@ -779,7 +779,7 @@ func (h *eventHandler) engineFactoryFn(ctx context.Context, workflowID string, o binaryHash := v2.ComputeBinaryHash(binary) confLggr := logger.Named(h.lggr, "WorkflowEngine.ConfidentialModule") confLggr = logger.With(confLggr, "workflowID", workflowID, "workflowName", name, "workflowOwner", owner) - confidential, err := v2.NewConfidentialModule(h.capRegistry, h.executionHandlers, binaryURL, binaryHash, workflowID, owner, name.String(), tag, h.fetchOrganizationID, h.engineLimiters.ConfidentialWorkflowsEnabled, confLggr) + confidential, err := v2.NewConfidentialModule(h.capRegistry, h.executionHandlers, binaryURL, binaryHash, workflowID, owner, name.String(), tag, h.fetchOrganizationID, h.engineLimiters.ConfidentialWorkflowsEnabled, h.engineLimiters.Settings, confLggr) if err != nil { return nil, fmt.Errorf("failed to create confidential module: %w", err) } diff --git a/core/services/workflows/v2/confidential_module.go b/core/services/workflows/v2/confidential_module.go index bdd5200790e..0be75d69703 100644 --- a/core/services/workflows/v2/confidential_module.go +++ b/core/services/workflows/v2/confidential_module.go @@ -16,6 +16,8 @@ import ( "github.com/smartcontractkit/chainlink-common/pkg/capabilities" "github.com/smartcontractkit/chainlink-common/pkg/logger" + "github.com/smartcontractkit/chainlink-common/pkg/settings" + "github.com/smartcontractkit/chainlink-common/pkg/settings/cresettings" "github.com/smartcontractkit/chainlink-common/pkg/settings/limits" "github.com/smartcontractkit/chainlink-common/pkg/types/core" "github.com/smartcontractkit/chainlink-common/pkg/workflows/host" @@ -70,12 +72,13 @@ type ConfidentialModule struct { provider func(tee *sdkpb.Tee) bool executionHandlers *confidentialrelay.ExecutionHandlers enabledGate limits.GateLimiter + creSettingsGetter settings.Getter } var _ host.RequirementEnforcingModule = (*ConfidentialModule)(nil) var _ host.RestrictionAwareModule = (*ConfidentialModule)(nil) -func NewConfidentialModule(capRegistry core.CapabilitiesRegistry, executionHandlers *confidentialrelay.ExecutionHandlers, binaryURL string, binaryHash []byte, workflowID, workflowOwner, workflowName, workflowTag string, resolveOrgID func(ctx context.Context, owner string) (string, error), enabledGate limits.GateLimiter, lggr logger.Logger) (*ConfidentialModule, error) { +func NewConfidentialModule(capRegistry core.CapabilitiesRegistry, executionHandlers *confidentialrelay.ExecutionHandlers, binaryURL string, binaryHash []byte, workflowID, workflowOwner, workflowName, workflowTag string, resolveOrgID func(ctx context.Context, owner string) (string, error), enabledGate limits.GateLimiter, creSettingsGetter settings.Getter, lggr logger.Logger) (*ConfidentialModule, error) { if enabledGate == nil { return nil, errors.New("enabledGate must not be nil") } @@ -93,6 +96,7 @@ func NewConfidentialModule(capRegistry core.CapabilitiesRegistry, executionHandl workflowTag: workflowTag, resolveOrgID: resolveOrgID, enabledGate: enabledGate, + creSettingsGetter: creSettingsGetter, lggr: lggr, }, nil } @@ -204,19 +208,37 @@ func doRequest[I, O proto.Message]( config, _ := anypb.New(&emptypb.Empty{}) + metadata := capabilities.RequestMetadata{ + WorkflowID: m.workflowID, + WorkflowOwner: m.workflowOwner, + WorkflowName: m.workflowName, + WorkflowTag: m.workflowTag, + WorkflowExecutionID: execID, + OrgID: orgID, + } + // When the WorkflowDonID binding gate is enabled, the remote executable + // capability server rejects requests whose RequestMetadata.WorkflowDonID + // does not match the authenticated calling DON. Set it to the calling + // (local) workflow DON ID so the request is accepted. Any failure here must + // fail the whole call rather than silently sending a zero WorkflowDonID. + bindingEnabled, err := cresettings.Default.RemoteExecutableWorkflowDONBindingEnabled.GetOrDefault(ctx, m.creSettingsGetter) + if err != nil { + return fmt.Errorf("failed to read RemoteExecutableWorkflowDONBindingEnabled setting: %w", err) + } + if bindingEnabled { + localNode, lnErr := m.capRegistry.LocalNode(ctx) + if lnErr != nil { + return fmt.Errorf("failed to get local node for confidential-workflows request metadata: %w", lnErr) + } + metadata.WorkflowDonID = localNode.WorkflowDON.ID + } + capReq := capabilities.CapabilityRequest{ Payload: payload, ConfigPayload: config, Method: method, CapabilityId: confidentialWorkflowsCapabilityID, - Metadata: capabilities.RequestMetadata{ - WorkflowID: m.workflowID, - WorkflowOwner: m.workflowOwner, - WorkflowName: m.workflowName, - WorkflowTag: m.workflowTag, - WorkflowExecutionID: execID, - OrgID: orgID, - }, + Metadata: metadata, } capResp, err := executable.Execute(ctx, capReq) diff --git a/core/services/workflows/v2/confidential_module_test.go b/core/services/workflows/v2/confidential_module_test.go index 0d439c583f8..ba7711d3082 100644 --- a/core/services/workflows/v2/confidential_module_test.go +++ b/core/services/workflows/v2/confidential_module_test.go @@ -673,7 +673,7 @@ func TestConfidentialModule_InterfaceMethods(t *testing.T) { func mustNewConfidentialModule(t *testing.T, capRegistry *regmocks.CapabilitiesRegistry, executionHandlers *confidentialrelay.ExecutionHandlers, binaryURL string, binaryHash []byte, workflowID, workflowOwner, workflowName, workflowTag string, enabledGate limits.GateLimiter, lggr logger.Logger) *ConfidentialModule { t.Helper() - m, err := NewConfidentialModule(capRegistry, executionHandlers, binaryURL, binaryHash, workflowID, workflowOwner, workflowName, workflowTag, func(context.Context, string) (string, error) { return "org-test", nil }, enabledGate, lggr) + m, err := NewConfidentialModule(capRegistry, executionHandlers, binaryURL, binaryHash, workflowID, workflowOwner, workflowName, workflowTag, func(context.Context, string) (string, error) { return "org-test", nil }, enabledGate, nil, lggr) require.NoError(t, err) return m } @@ -681,6 +681,6 @@ func mustNewConfidentialModule(t *testing.T, capRegistry *regmocks.CapabilitiesR func TestNewConfidentialModule_NilGate(t *testing.T) { t.Parallel() capReg := regmocks.NewCapabilitiesRegistry(t) - _, err := NewConfidentialModule(capReg, &confidentialrelay.ExecutionHandlers{}, "", nil, "wf", "owner", "name", "tag", func(context.Context, string) (string, error) { return "org-test", nil }, nil, logger.Test(t)) + _, err := NewConfidentialModule(capReg, &confidentialrelay.ExecutionHandlers{}, "", nil, "wf", "owner", "name", "tag", func(context.Context, string) (string, error) { return "org-test", nil }, nil, nil, logger.Test(t)) require.Error(t, err) } diff --git a/core/services/workflows/v2/secrets.go b/core/services/workflows/v2/secrets.go index 4234ddf9b4a..4f86cf62deb 100644 --- a/core/services/workflows/v2/secrets.go +++ b/core/services/workflows/v2/secrets.go @@ -119,19 +119,35 @@ func keyFor(owner, namespace, id string) string { return fmt.Sprintf("%s::%s::%s", owner, namespace, id) } -func (s *secretsFetcher) vaultGetSecretsMetadata(ctx context.Context, callbackID int64) capabilities.RequestMetadata { +func (s *secretsFetcher) vaultGetSecretsMetadata(ctx context.Context, callbackID int64) (capabilities.RequestMetadata, error) { metadata := capabilities.RequestMetadata{ WorkflowOwner: s.workflowOwner, WorkflowName: s.workflowName, WorkflowExecutionID: sha(s.phaseID, strconv.FormatInt(callbackID, 10)), ReferenceID: strconv.FormatInt(callbackID, 10), } + // When the WorkflowDonID binding gate is enabled, the remote executable + // capability server rejects requests whose RequestMetadata.WorkflowDonID + // does not match the authenticated calling DON. Set it to the calling + // (local) workflow DON ID so the request is accepted. Any failure here must + // fail the whole call rather than silently sending a zero WorkflowDonID. + bindingEnabled, err := cresettings.Default.RemoteExecutableWorkflowDONBindingEnabled.GetOrDefault(ctx, s.creSettingsGetter) + if err != nil { + return capabilities.RequestMetadata{}, fmt.Errorf("failed to read RemoteExecutableWorkflowDONBindingEnabled setting: %w", err) + } + if bindingEnabled { + localNode, lnErr := s.capRegistry.LocalNode(ctx) + if lnErr != nil { + return capabilities.RequestMetadata{}, fmt.Errorf("failed to get local node for vault request metadata: %w", lnErr) + } + metadata.WorkflowDonID = localNode.WorkflowDON.ID + } if propagateOrgIDMeta, _ := cresettings.Default.PropagateOrgIDInRequestMetadata.GetOrDefault(ctx, s.creSettingsGetter); propagateOrgIDMeta && s.orgID != "" { metadata.OrgID = s.orgID // WorkflowID is under this gate because we previously skipped setting workflowID on SecretsFetcher entirely. Now setting it safely. metadata.WorkflowID = s.workflowID } - return metadata + return metadata, nil } func (s *secretsFetcher) GetSecrets(ctx context.Context, request *sdkpb.GetSecretsRequest) ([]*sdkpb.SecretResponse, error) { @@ -144,7 +160,10 @@ func (s *secretsFetcher) GetSecrets(ctx context.Context, request *sdkpb.GetSecre if request != nil { callbackID = int64(request.CallbackId) } - metadata := s.vaultGetSecretsMetadata(ctx, callbackID) + metadata, err := s.vaultGetSecretsMetadata(ctx, callbackID) + if err != nil { + return nil, err + } vaultRequestID := vaultutils.BuildWorkflowGetSecretsRequestID(metadata) s.lggr.Debugw("get secrets request received", "vaultRequestID", vaultRequestID, "metadata", metadata) s.callCounter.mu.Lock() @@ -285,7 +304,10 @@ func (s *secretsFetcher) GetRawSecrets(ctx context.Context, request *sdkpb.GetSe if err != nil { return nil, fmt.Errorf("failed to get encryption keys: %w", err) } - metadata := s.vaultGetSecretsMetadata(ctx, int64(request.CallbackId)) + metadata, err := s.vaultGetSecretsMetadata(ctx, int64(request.CallbackId)) + if err != nil { + return nil, err + } vaultRequestID := vaultutils.BuildWorkflowGetSecretsRequestID(metadata) vp := &vault.GetSecretsRequest{ Requests: make([]*vault.SecretRequest, 0), diff --git a/system-tests/tests/smoke/cre/cre_suite_test.go b/system-tests/tests/smoke/cre/cre_suite_test.go index ae9888d4030..750ae752a14 100644 --- a/system-tests/tests/smoke/cre/cre_suite_test.go +++ b/system-tests/tests/smoke/cre/cre_suite_test.go @@ -97,6 +97,9 @@ func runSuiteScenario(t *testing.T, topology string, scenario suite_config.Suite } else if isVaultOptimizationsEnabledTopology(topology) { vaultConfig = getVaultOptimizationsEnabledTestConfig(t) allowlistSubtestName = "allowlist_auth_when_vault_optimizations_enabled" + } else if isVaultWorkflowDONBindingEnabledTopology(topology) { + vaultConfig = getVaultWorkflowDONBindingEnabledTestConfig(t) + allowlistSubtestName = "allowlist_auth_when_workflow_don_binding_enabled" } fixture := setupVaultSharedScenarioFixture(t, vaultConfig) diff --git a/system-tests/tests/smoke/cre/vault_don_test_helpers.go b/system-tests/tests/smoke/cre/vault_don_test_helpers.go index 572602779a9..bfd1010944b 100644 --- a/system-tests/tests/smoke/cre/vault_don_test_helpers.go +++ b/system-tests/tests/smoke/cre/vault_don_test_helpers.go @@ -54,10 +54,11 @@ import ( ) const ( - vaultDefaultConfigPath = "/configs/workflow-gateway-capabilities-don.toml" - vaultJWTAuthEnabledConfigPath = "/configs/workflow-gateway-capabilities-don-vault-jwt_auth-enabled.toml" - vaultOptimizationsEnabledConfigPath = "/configs/workflow-gateway-capabilities-don-vault-optimizations-enabled.toml" - vaultJWTIssuerListenAddr = "0.0.0.0:18123" + vaultDefaultConfigPath = "/configs/workflow-gateway-capabilities-don.toml" + vaultJWTAuthEnabledConfigPath = "/configs/workflow-gateway-capabilities-don-vault-jwt_auth-enabled.toml" + vaultOptimizationsEnabledConfigPath = "/configs/workflow-gateway-capabilities-don-vault-optimizations-enabled.toml" + vaultWorkflowDONBindingEnabledConfigPath = "/configs/workflow-gateway-capabilities-don-vault-workflow-don-binding-enabled.toml" + vaultJWTIssuerListenAddr = "0.0.0.0:18123" // vaultJWTTestTenantID is the tenant_id / urn:chainlink:tenant_id claim for Vault JWT tests and // matches the org_id passed to DeriveJWTAuthorizedVaultWorkflowOwner. vaultJWTTestTenantID uint64 = 1 @@ -255,6 +256,12 @@ func getVaultOptimizationsEnabledTestConfig(t *testing.T) *ttypes.TestConfig { return t_helpers.GetTestConfig(t, vaultOptimizationsEnabledConfigPath) } +func getVaultWorkflowDONBindingEnabledTestConfig(t *testing.T) *ttypes.TestConfig { + t.Helper() + + return t_helpers.GetTestConfig(t, vaultWorkflowDONBindingEnabledConfigPath) +} + func isVaultJWTAuthEnabledTopology(topologyName string) bool { return strings.Contains(topologyName, "vault-jwt_auth-enabled") } @@ -263,6 +270,10 @@ func isVaultOptimizationsEnabledTopology(topologyName string) bool { return strings.Contains(topologyName, "vault-optimizations-enabled") } +func isVaultWorkflowDONBindingEnabledTopology(topologyName string) bool { + return strings.Contains(topologyName, "vault-workflow-don-binding-enabled") +} + func setupVaultScenarioFixture(t *testing.T, baseConfig *ttypes.TestConfig, usePerTestKeys bool) *vaultScenarioFixture { t.Helper()