Skip to content

build: Add per-step CPU and memory resource limits #3876

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
tonistiigi merged 1 commit into from
Jun 10, 2026
Merged

build: Add per-step CPU and memory resource limits #3876

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
79 changes: 53 additions & 26 deletions bake/bake.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -606,6 +606,13 @@ func (c Config) newOverrides(v []string) (map[string]map[string]Override, error)
override.Append = appendTo
override.ArrValue = append(override.ArrValue, parts[1])
}
case "resources":
if len(keys) != 3 {
return nil, errors.Errorf("invalid key %s, resources requires name", parts[0])
}
if len(parts) == 2 {

@tonistiigi tonistiigi Jun 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

nit: this is already guaranteed by the len check in L582

override.Value = parts[1]
}
case "args":
if len(keys) != 3 {
return nil, errors.Errorf("invalid key %s, args requires name", parts[0])
Expand Down Expand Up @@ -745,32 +752,33 @@ type Target struct {
// Inherits is the only field that cannot be overridden with --set
Inherits []string `json:"inherits,omitempty" hcl:"inherits,optional" cty:"inherits"`

Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest buildflags.Attests `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets buildflags.Secrets `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH buildflags.SSHKeys `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs buildflags.Exports `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional" cty:"shm-size"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional" cty:"ulimits"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
ExtraHosts map[string]*string `json:"extra-hosts,omitempty" hcl:"extra-hosts,optional" cty:"extra-hosts"`
Policy buildflags.PolicyConfigs `json:"policy,omitempty" hcl:"policy,optional" cty:"policy"`
Annotations []string `json:"annotations,omitempty" hcl:"annotations,optional" cty:"annotations"`
Attest buildflags.Attests `json:"attest,omitempty" hcl:"attest,optional" cty:"attest"`
Context *string `json:"context,omitempty" hcl:"context,optional" cty:"context"`
Contexts map[string]string `json:"contexts,omitempty" hcl:"contexts,optional" cty:"contexts"`
Dockerfile *string `json:"dockerfile,omitempty" hcl:"dockerfile,optional" cty:"dockerfile"`
DockerfileInline *string `json:"dockerfile-inline,omitempty" hcl:"dockerfile-inline,optional" cty:"dockerfile-inline"`
Args map[string]*string `json:"args,omitempty" hcl:"args,optional" cty:"args"`
Labels map[string]*string `json:"labels,omitempty" hcl:"labels,optional" cty:"labels"`
Tags []string `json:"tags,omitempty" hcl:"tags,optional" cty:"tags"`
CacheFrom buildflags.CacheOptions `json:"cache-from,omitempty" hcl:"cache-from,optional" cty:"cache-from"`
CacheTo buildflags.CacheOptions `json:"cache-to,omitempty" hcl:"cache-to,optional" cty:"cache-to"`
Target *string `json:"target,omitempty" hcl:"target,optional" cty:"target"`
Secrets buildflags.Secrets `json:"secret,omitempty" hcl:"secret,optional" cty:"secret"`
SSH buildflags.SSHKeys `json:"ssh,omitempty" hcl:"ssh,optional" cty:"ssh"`
Platforms []string `json:"platforms,omitempty" hcl:"platforms,optional" cty:"platforms"`
Outputs buildflags.Exports `json:"output,omitempty" hcl:"output,optional" cty:"output"`
Pull *bool `json:"pull,omitempty" hcl:"pull,optional" cty:"pull"`
NoCache *bool `json:"no-cache,omitempty" hcl:"no-cache,optional" cty:"no-cache"`
NetworkMode *string `json:"network,omitempty" hcl:"network,optional" cty:"network"`
NoCacheFilter []string `json:"no-cache-filter,omitempty" hcl:"no-cache-filter,optional" cty:"no-cache-filter"`
ShmSize *string `json:"shm-size,omitempty" hcl:"shm-size,optional" cty:"shm-size"`
Ulimits []string `json:"ulimits,omitempty" hcl:"ulimits,optional" cty:"ulimits"`
Resources *buildflags.ResourcesConfig `json:"resources,omitempty" hcl:"resources,optional" cty:"resources"`
Call *string `json:"call,omitempty" hcl:"call,optional" cty:"call"`
Entitlements []string `json:"entitlements,omitempty" hcl:"entitlements,optional" cty:"entitlements"`
ExtraHosts map[string]*string `json:"extra-hosts,omitempty" hcl:"extra-hosts,optional" cty:"extra-hosts"`
Policy buildflags.PolicyConfigs `json:"policy,omitempty" hcl:"policy,optional" cty:"policy"`
// IMPORTANT: if you add more fields here, do not forget to update newOverrides/AddOverrides and docs/bake-reference.md.

// linked is a private field to mark a target used as a linked one
Expand Down Expand Up @@ -944,6 +952,9 @@ func (t *Target) Merge(t2 *Target) {
if t2.Ulimits != nil { // merge
t.Ulimits = append(t.Ulimits, t2.Ulimits...)
}
if t2.Resources != nil { // merge
t.Resources = t.Resources.Merge(t2.Resources)
}
if t2.Description != "" {
t.Description = t2.Description
}
Expand Down Expand Up @@ -1138,6 +1149,16 @@ func (t *Target) AddOverrides(overrides map[string]Override, ent *EntitlementCon
} else {
t.Ulimits = o.ArrValue
}
case "resources":

@tonistiigi tonistiigi Jun 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The override keys seem to be undocumented in docs/reference/bake

@crazy-max crazy-max Jun 10, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ah yeah indeed should be done in https://docs.docker.com/build/bake/overrides/

if len(keys) != 2 {
return errors.Errorf("invalid format for resources, expecting resources.<name>=<value>")
}
if t.Resources == nil {
t.Resources = &buildflags.ResourcesConfig{}
}
if err := t.Resources.SetField(keys[1], value); err != nil {
return err
}
case "network":
t.NetworkMode = &value
case "pull":
Expand Down Expand Up @@ -1612,6 +1633,12 @@ func toBuildOpt(t *Target, inp *Input) (*build.Options, error) {
}
bo.Ulimits = ulimits

resourceLimits, err := build.ParseResourceLimits(t.Resources.ToEntries())
if err != nil {
return nil, err
}
bo.ResourceLimits = resourceLimits

bo.Allow = append(bo.Allow, t.Entitlements...)

return bo, nil
Expand Down
126 changes: 126 additions & 0 deletions bake/bake_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,15 @@ target "webDEP" {
no-cache = true
shm-size = "128m"
ulimits = ["nofile=1024:1024"]
resources = {
memory = "2g"
memory-swap = "4g"
cpu-shares = 1024
cpu-period = 100000
cpu-quota = 50000
cpuset-cpus = "0-3"
cpuset-mems = "0,1"
}
extra-hosts = {
my_hostname = "8.8.8.8"
}
Expand Down Expand Up @@ -67,6 +76,14 @@ target "webapp" {
require.Equal(t, true, *m["webapp"].NoCache)
require.Equal(t, "128m", *m["webapp"].ShmSize)
require.Equal(t, []string{"nofile=1024:1024"}, m["webapp"].Ulimits)
require.NotNil(t, m["webapp"].Resources)
require.Equal(t, ptrstr("2g"), m["webapp"].Resources.Memory)
require.Equal(t, ptrstr("4g"), m["webapp"].Resources.MemorySwap)
require.Equal(t, int64(1024), *m["webapp"].Resources.CPUShares)
require.Equal(t, int64(100000), *m["webapp"].Resources.CPUPeriod)
require.Equal(t, int64(50000), *m["webapp"].Resources.CPUQuota)
require.Equal(t, ptrstr("0-3"), m["webapp"].Resources.CPUSetCPUs)
require.Equal(t, ptrstr("0,1"), m["webapp"].Resources.CPUSetMems)
require.Equal(t, map[string]*string{"my_hostname": ptrstr("8.8.8.8")}, m["webapp"].ExtraHosts)
require.Nil(t, m["webapp"].Pull)

Expand Down Expand Up @@ -226,6 +243,21 @@ target "webapp" {
require.Equal(t, "256m", *m["webapp"].ShmSize)
})

t.Run("ResourceLimitsOverride", func(t *testing.T) {
m, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{
"webapp.resources.memory=512m",
"webapp.resources.cpu-quota=25000",
}, nil, nil, &EntitlementConf{})
require.NoError(t, err)
require.Equal(t, ptrstr("512m"), m["webapp"].Resources.Memory)
require.Equal(t, int64(25000), *m["webapp"].Resources.CPUQuota)
})

t.Run("ResourceLimitsInvalidOverride", func(t *testing.T) {
_, _, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.resources.cpu-quota=notanumber"}, nil, nil, &EntitlementConf{})
require.Error(t, err)
})

t.Run("PullOverride", func(t *testing.T) {
t.Parallel()
m, g, err := ReadTargets(ctx, []File{fp}, []string{"webapp"}, []string{"webapp.pull=false"}, nil, nil, &EntitlementConf{})
Expand Down Expand Up @@ -1834,6 +1866,100 @@ func TestExtraHostsDeterministicOrder(t *testing.T) {
}
}

func TestResourceLimitsToBuildOpt(t *testing.T) {
newInt64 := func(v int64) *int64 { return &v }
bo, err := toBuildOpt(&Target{
DockerfileInline: ptrstr("FROM scratch"),
Resources: &buildflags.ResourcesConfig{
Memory: ptrstr("2g"),
MemorySwap: ptrstr("4g"),
CPUShares: newInt64(1024),
CPUPeriod: newInt64(100000),
CPUQuota: newInt64(50000),
CPUSetCPUs: ptrstr("0-3"),
CPUSetMems: ptrstr("0,1"),
},
}, &Input{})
require.NoError(t, err)
require.Equal(t, int64(2*1024*1024*1024), bo.ResourceLimits.Memory.Value())
require.Equal(t, int64(4*1024*1024*1024), bo.ResourceLimits.MemorySwap.Value())
require.Equal(t, int64(1024), bo.ResourceLimits.CPUShares)
require.Equal(t, int64(100000), bo.ResourceLimits.CPUPeriod)
require.Equal(t, int64(50000), bo.ResourceLimits.CPUQuota)
require.Equal(t, "0-3", bo.ResourceLimits.CPUSetCPUs)
require.Equal(t, "0,1", bo.ResourceLimits.CPUSetMems)
}

func TestResourceLimitsUnlimitedSwap(t *testing.T) {
bo, err := toBuildOpt(&Target{
DockerfileInline: ptrstr("FROM scratch"),
Resources: &buildflags.ResourcesConfig{MemorySwap: ptrstr("-1")},
}, &Input{})
require.NoError(t, err)
require.Equal(t, int64(-1), bo.ResourceLimits.MemorySwap.Value())
}

func TestResourceLimitsInvalid(t *testing.T) {
_, err := toBuildOpt(&Target{
DockerfileInline: ptrstr("FROM scratch"),
Resources: &buildflags.ResourcesConfig{Memory: ptrstr("notabyte")},
}, &Input{})
require.Error(t, err)
}

func TestResourcesObject(t *testing.T) {
ctx := context.TODO()

cpuShares, cpuQuota := int64(1024), int64(50000)
want := &buildflags.ResourcesConfig{
Memory: ptrstr("2g"),
MemorySwap: ptrstr("4g"),
CPUShares: &cpuShares,
CPUQuota: &cpuQuota,
CPUSetCPUs: ptrstr("0-3"),
}

data := `target "app" {
resources = {
memory = "2g"
memory-swap = "4g"
cpu-shares = 1024
cpu-quota = 50000
cpuset-cpus = "0-3"
}
}`

m, _, err := ReadTargets(ctx, []File{{Name: "docker-bake.hcl", Data: []byte(data)}}, []string{"app"}, nil, nil, nil, &EntitlementConf{})
require.NoError(t, err)
require.Equal(t, want, m["app"].Resources)
}

// TestResourcesInheritIsolation ensures merging resources from a shared base
// target doesn't leak a child's override into the base or a sibling.
func TestResourcesInheritIsolation(t *testing.T) {
ctx := context.TODO()

data := `target "base" {
resources = {
memory = "1g"
}
}
target "child1" {
inherits = ["base"]
resources = {
memory = "2g"
}
}
target "child2" {
inherits = ["base"]
}`

m, _, err := ReadTargets(ctx, []File{{Name: "docker-bake.hcl", Data: []byte(data)}}, []string{"child1", "child2"}, nil, nil, nil, &EntitlementConf{})
require.NoError(t, err)
require.Equal(t, ptrstr("2g"), m["child1"].Resources.Memory)
require.Equal(t, ptrstr("1g"), m["child2"].Resources.Memory)
}

func TestAnnotations(t *testing.T) {
fp := File{
Name: "docker-bake.hcl",
Expand Down
13 changes: 13 additions & 0 deletions build/build.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -87,6 +87,7 @@ type Options struct {
Tags []string
Target string
Ulimits *opts.UlimitOpt
ResourceLimits ResourceLimits

Session []session.Attachable
Linked bool // Linked marks this target as exclusively linked (not requested by the user).
Expand All @@ -98,6 +99,18 @@ type Options struct {
Policy []buildflags.PolicyConfig
}

// ResourceLimits holds the cgroup resource constraints applied to individual
// build steps (RUN instructions). They don't affect the build cache key.
type ResourceLimits struct {
Memory opts.MemBytes
MemorySwap opts.MemSwapBytes
CPUShares int64
CPUPeriod int64
CPUQuota int64
CPUSetCPUs string
CPUSetMems string
}

type CallFunc struct {
Name string
Format string
Expand Down
3 changes: 3 additions & 0 deletions build/opt.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -610,6 +610,9 @@ func toSolveOpt(ctx context.Context, np *noderesolver.ResolvedNode, multiDriver
so.FrontendAttrs["ulimit"] = ulimits
}

// setup per-step resource limits
addResourceLimits(opt.ResourceLimits, so.FrontendAttrs)

// mark call request as internal
if opt.CallFunc != nil {
so.Internal = true
Expand Down
Loading
Loading