Skip to content

Commit 3de69a5

Browse files
dssengdsseng
committed
fix: re-create cgroups when restarting runners
Make sure processes are only launched into freshly-created cgroups with all limits set when they are restarted. This also allows processes to restart after cgroup being killed via the cgroup.kill mechanism. Fixes #11785 Signed-off-by: Dmitrii Sharshakov <[email protected]>
1 parent 4d876d9 commit 3de69a5

File tree

4 files changed

+179
-156
lines changed

4 files changed

+179
-156
lines changed

‎internal/app/machined/pkg/startup/cgroups.go‎

Lines changed: 8 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -12,30 +12,15 @@ import (
1212

1313
"github.com/containerd/cgroups/v3"
1414
"github.com/containerd/cgroups/v3/cgroup1"
15-
"github.com/containerd/cgroups/v3/cgroup2"
1615
"github.com/opencontainers/runtime-spec/specs-go"
17-
"github.com/siderolabs/go-debug"
18-
"github.com/siderolabs/go-pointer"
1916
"go.uber.org/zap"
2017

2118
"github.com/siderolabs/talos/internal/app/machined/pkg/runtime"
2219
"github.com/siderolabs/talos/internal/pkg/cgroup"
2320
"github.com/siderolabs/talos/pkg/machinery/constants"
2421
)
2522

26-
func zeroIfRace[T any](v T) T {
27-
if debug.RaceEnabled {
28-
var zeroT T
29-
30-
return zeroT
31-
}
32-
33-
return v
34-
}
35-
3623
// CreateSystemCgroups creates system cgroups.
37-
//
38-
//nolint:gocyclo
3924
func CreateSystemCgroups(ctx context.Context, log *zap.Logger, rt runtime.Runtime, next NextTaskFunc) error {
4025
// in container mode cgroups mode depends on cgroups provided by the container runtime
4126
if !rt.State().Platform().Mode().InContainer() {
@@ -53,157 +38,25 @@ func CreateSystemCgroups(ctx context.Context, log *zap.Logger, rt runtime.Runtim
5338

5439
log.Info("initializing cgroups", zap.String("root", cgroup.Root()))
5540

56-
groups := []struct {
57-
name string
58-
resources *cgroup2.Resources
59-
}{
60-
{
61-
name: constants.CgroupInit,
62-
resources: &cgroup2.Resources{
63-
Memory: &cgroup2.Memory{
64-
Min: pointer.To[int64](constants.CgroupInitReservedMemory),
65-
Low: pointer.To[int64](constants.CgroupInitReservedMemory * 2),
66-
},
67-
CPU: &cgroup2.CPU{
68-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupInitMillicores))),
69-
},
70-
},
71-
},
72-
{
73-
name: constants.CgroupSystem,
74-
resources: &cgroup2.Resources{
75-
Memory: &cgroup2.Memory{
76-
Min: pointer.To[int64](constants.CgroupSystemReservedMemory),
77-
Low: pointer.To[int64](constants.CgroupSystemReservedMemory * 2),
78-
},
79-
CPU: &cgroup2.CPU{
80-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupSystemMillicores))),
81-
},
82-
},
83-
},
84-
{
85-
name: constants.CgroupSystemRuntime,
86-
resources: &cgroup2.Resources{
87-
Memory: &cgroup2.Memory{
88-
Min: pointer.To[int64](constants.CgroupSystemRuntimeReservedMemory),
89-
Low: pointer.To[int64](constants.CgroupSystemRuntimeReservedMemory * 2),
90-
},
91-
CPU: &cgroup2.CPU{
92-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupSystemRuntimeMillicores))),
93-
},
94-
},
95-
},
96-
{
97-
name: constants.CgroupUdevd,
98-
resources: &cgroup2.Resources{
99-
Memory: &cgroup2.Memory{
100-
Min: pointer.To[int64](constants.CgroupUdevdReservedMemory),
101-
Low: pointer.To[int64](constants.CgroupUdevdReservedMemory * 2),
102-
},
103-
CPU: &cgroup2.CPU{
104-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupUdevdMillicores))),
105-
},
106-
},
107-
},
108-
{
109-
name: constants.CgroupPodRuntimeRoot,
110-
resources: &cgroup2.Resources{
111-
CPU: &cgroup2.CPU{
112-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupPodRuntimeRootMillicores))),
113-
},
114-
},
115-
},
116-
{
117-
name: constants.CgroupPodRuntime,
118-
resources: &cgroup2.Resources{
119-
Memory: &cgroup2.Memory{
120-
Min: pointer.To[int64](constants.CgroupPodRuntimeReservedMemory),
121-
Low: pointer.To[int64](constants.CgroupPodRuntimeReservedMemory * 2),
122-
},
123-
CPU: &cgroup2.CPU{
124-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupPodRuntimeMillicores))),
125-
},
126-
},
127-
},
128-
{
129-
name: constants.CgroupKubelet,
130-
resources: &cgroup2.Resources{
131-
Memory: &cgroup2.Memory{
132-
Min: pointer.To[int64](constants.CgroupKubeletReservedMemory),
133-
Low: pointer.To[int64](constants.CgroupKubeletReservedMemory * 2),
134-
},
135-
CPU: &cgroup2.CPU{
136-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupKubeletMillicores))),
137-
},
138-
},
139-
},
140-
{
141-
name: constants.CgroupDashboard,
142-
resources: &cgroup2.Resources{
143-
Memory: &cgroup2.Memory{
144-
Max: zeroIfRace(pointer.To[int64](constants.CgroupDashboardMaxMemory)),
145-
},
146-
CPU: &cgroup2.CPU{
147-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupDashboardMillicores))),
148-
},
149-
},
150-
},
151-
{
152-
name: constants.CgroupApid,
153-
resources: &cgroup2.Resources{
154-
Memory: &cgroup2.Memory{
155-
Min: pointer.To[int64](constants.CgroupApidReservedMemory),
156-
Low: pointer.To[int64](constants.CgroupApidReservedMemory * 2),
157-
Max: zeroIfRace(pointer.To[int64](constants.CgroupApidMaxMemory)),
158-
Swap: pointer.To[int64](0),
159-
},
160-
CPU: &cgroup2.CPU{
161-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupApidMillicores))),
162-
},
163-
},
164-
},
165-
{
166-
name: constants.CgroupTrustd,
167-
resources: &cgroup2.Resources{
168-
Memory: &cgroup2.Memory{
169-
Min: pointer.To[int64](constants.CgroupTrustdReservedMemory),
170-
Low: pointer.To[int64](constants.CgroupTrustdReservedMemory * 2),
171-
Max: zeroIfRace(pointer.To[int64](constants.CgroupTrustdMaxMemory)),
172-
Swap: pointer.To[int64](0),
173-
},
174-
CPU: &cgroup2.CPU{
175-
Weight: pointer.To[uint64](cgroup.MillicoresToCPUWeight(cgroup.MilliCores(constants.CgroupTrustdMillicores))),
176-
},
177-
},
178-
},
41+
groups := []string{
42+
constants.CgroupInit,
43+
constants.CgroupSystem,
44+
constants.CgroupPodRuntimeRoot,
17945
}
18046

18147
for _, c := range groups {
18248
if cgroups.Mode() == cgroups.Unified {
183-
resources := c.resources
184-
185-
if rt.State().Platform().Mode().InContainer() {
186-
// don't attempt to set resources in container mode, as they might conflict with the parent cgroup tree
187-
resources = &cgroup2.Resources{}
188-
}
189-
190-
cg, err := cgroup2.NewManager(constants.CgroupMountPath, cgroup.Path(c.name), resources)
49+
_, err := cgroup.CreateCgroupV2(c)
19150
if err != nil {
192-
return fmt.Errorf("failed to create cgroup: %w", err)
193-
}
194-
195-
if c.name == constants.CgroupInit {
196-
if err := cg.AddProc(uint64(os.Getpid())); err != nil {
197-
return fmt.Errorf("failed to move init process to cgroup: %w", err)
198-
}
51+
return err
19952
}
20053
} else {
201-
cg, err := cgroup1.New(cgroup1.StaticPath(c.name), &specs.LinuxResources{})
54+
cg, err := cgroup1.New(cgroup1.StaticPath(c), &specs.LinuxResources{})
20255
if err != nil {
20356
return fmt.Errorf("failed to create cgroup: %w", err)
20457
}
20558

206-
if c.name == constants.CgroupInit {
59+
if c == constants.CgroupInit {
20760
if err := cg.Add(cgroup1.Process{
20861
Pid: os.Getpid(),
20962
}); err != nil {

‎internal/app/machined/pkg/system/runner/containerd/containerd.go‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,11 @@ import (
1010
"fmt"
1111
"io"
1212
"log"
13+
"os"
1314
"syscall"
1415
"time"
1516

17+
"github.com/containerd/cgroups/v3"
1618
containerd "github.com/containerd/containerd/v2/client"
1719
"github.com/containerd/containerd/v2/contrib/seccomp"
1820
"github.com/containerd/containerd/v2/pkg/cio"
@@ -168,6 +170,23 @@ func (c *containerdRunner) Run(eventSink events.Recorder) error {
168170
return fmt.Errorf("error creating log: %w", err)
169171
}
170172

173+
if cgroups.Mode() == cgroups.Unified {
174+
cg, err := cgroup.CreateCgroupV2(c.opts.CgroupPath)
175+
if err != nil {
176+
return fmt.Errorf("error creating cgroup: %w", err)
177+
}
178+
179+
// If the task is not cleaned up by containerd or another error
180+
// happens during the lifecycle, remove the cgroup before exiting
181+
// if one still exists
182+
defer func() {
183+
err := cg.Delete()
184+
if err != nil && !os.IsNotExist(err) {
185+
eventSink(events.StateStopping, "Failed to remove cgroup for %s, %s", c, err)
186+
}
187+
}()
188+
}
189+
171190
defer logW.Close() //nolint:errcheck
172191

173192
var w io.Writer = logW

‎internal/app/machined/pkg/system/runner/process/process.go‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -413,7 +413,22 @@ func setSchedulingPolicy(p *processRunner, pid int, schedulingPolicy uint) error
413413
return nil
414414
}
415415

416+
//nolint:gocyclo
416417
func (p *processRunner) run(eventSink events.Recorder) error {
418+
if cgroups.Mode() == cgroups.Unified {
419+
cg, err := cgroup.CreateCgroupV2(p.opts.CgroupPath)
420+
if err != nil {
421+
return fmt.Errorf("error creating cgroup: %w", err)
422+
}
423+
424+
defer func() {
425+
err := cg.Delete()
426+
if err != nil {
427+
eventSink(events.StateStopping, "Failed to remove cgroup for %s, %s", p, err)
428+
}
429+
}()
430+
}
431+
417432
cmdWrapper, err := p.build()
418433
if err != nil {
419434
return fmt.Errorf("error building command: %w", err)

0 commit comments

Comments
 (0)