Skip to content

Commit 56f77d5

Browse files
erikstmartinerikstmartin
committed
Implementing support for --cpu-rt-period and --cpu-rt-runtime so that
containers may specify these cgroup values at runtime. This will allow processes to change their priority to real-time within the container when CONFIG_RT_GROUP_SCHED is enabled in the kernel. See moby#22380. Also added sanity checks for the new --cpu-rt-runtime and --cpu-rt-period flags to ensure that that the kernel supports these features and that runtime is not greater than period. Daemon will support a --cpu-rt-runtime flag to initialize the parent cgroup on startup, this prevents the administrator from alotting runtime to docker after each restart. There are additional checks that could be added but maybe too far? Check parent cgroups to ensure values are <= parent, inspecting rtprio ulimit and issuing a warning. Signed-off-by: Erik St. Martin <[email protected]>
1 parent a6a38f8 commit 56f77d5

25 files changed

Lines changed: 404 additions & 115 deletions

‎api/types/container/host_config.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -243,8 +243,10 @@ type Resources struct {
243243
BlkioDeviceWriteBps []*blkiodev.ThrottleDevice
244244
BlkioDeviceReadIOps []*blkiodev.ThrottleDevice
245245
BlkioDeviceWriteIOps []*blkiodev.ThrottleDevice
246-
CPUPeriod int64 `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period
247-
CPUQuota int64 `json:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota
246+
CPUPeriod int64 `json:"CpuPeriod"` // CPU CFS (Completely Fair Scheduler) period
247+
CPUQuota int64 `json:"CpuQuota"` // CPU CFS (Completely Fair Scheduler) quota
248+
CPURealtimePeriod int64 `json:"CpuRealtimePeriod"` // CPU real-time period
249+
CPURealtimeRuntime int64 `json:"CpuRealtimeRuntime"` // CPU real-time runtime
248250
CpusetCpus string // CpusetCpus 0-2, 0,1
249251
CpusetMems string // CpusetMems 0-2, 0,1
250252
Devices []DeviceMapping // List of devices to map inside the container

‎cli/command/container/update.go‎

Lines changed: 27 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -15,17 +15,19 @@ import (
1515
)
1616

1717
type updateOptions struct {
18-
blkioWeight uint16
19-
cpuPeriod int64
20-
cpuQuota int64
21-
cpusetCpus string
22-
cpusetMems string
23-
cpuShares int64
24-
memoryString string
25-
memoryReservation string
26-
memorySwap string
27-
kernelMemory string
28-
restartPolicy string
18+
blkioWeight uint16
19+
cpuPeriod int64
20+
cpuQuota int64
21+
cpuRealtimePeriod int64
22+
cpuRealtimeRuntime int64
23+
cpusetCpus string
24+
cpusetMems string
25+
cpuShares int64
26+
memoryString string
27+
memoryReservation string
28+
memorySwap string
29+
kernelMemory string
30+
restartPolicy string
2931

3032
nFlag int
3133

@@ -51,6 +53,8 @@ func NewUpdateCommand(dockerCli *command.DockerCli) *cobra.Command {
5153
flags.Uint16Var(&opts.blkioWeight, "blkio-weight", 0, "Block IO (relative weight), between 10 and 1000")
5254
flags.Int64Var(&opts.cpuPeriod, "cpu-period", 0, "Limit CPU CFS (Completely Fair Scheduler) period")
5355
flags.Int64Var(&opts.cpuQuota, "cpu-quota", 0, "Limit CPU CFS (Completely Fair Scheduler) quota")
56+
flags.Int64Var(&opts.cpuRealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds")
57+
flags.Int64Var(&opts.cpuRealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds")
5458
flags.StringVar(&opts.cpusetCpus, "cpuset-cpus", "", "CPUs in which to allow execution (0-3, 0,1)")
5559
flags.StringVar(&opts.cpusetMems, "cpuset-mems", "", "MEMs in which to allow execution (0-3, 0,1)")
5660
flags.Int64VarP(&opts.cpuShares, "cpu-shares", "c", 0, "CPU shares (relative weight)")
@@ -115,16 +119,18 @@ func runUpdate(dockerCli *command.DockerCli, opts *updateOptions) error {
115119
}
116120

117121
resources := containertypes.Resources{
118-
BlkioWeight: opts.blkioWeight,
119-
CpusetCpus: opts.cpusetCpus,
120-
CpusetMems: opts.cpusetMems,
121-
CPUShares: opts.cpuShares,
122-
Memory: memory,
123-
MemoryReservation: memoryReservation,
124-
MemorySwap: memorySwap,
125-
KernelMemory: kernelMemory,
126-
CPUPeriod: opts.cpuPeriod,
127-
CPUQuota: opts.cpuQuota,
122+
BlkioWeight: opts.blkioWeight,
123+
CpusetCpus: opts.cpusetCpus,
124+
CpusetMems: opts.cpusetMems,
125+
CPUShares: opts.cpuShares,
126+
Memory: memory,
127+
MemoryReservation: memoryReservation,
128+
MemorySwap: memorySwap,
129+
KernelMemory: kernelMemory,
130+
CPUPeriod: opts.cpuPeriod,
131+
CPUQuota: opts.cpuQuota,
132+
CPURealtimePeriod: opts.cpuRealtimePeriod,
133+
CPURealtimeRuntime: opts.cpuRealtimeRuntime,
128134
}
129135

130136
updateConfig := containertypes.UpdateConfig{

‎contrib/completion/bash/docker‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1316,6 +1316,8 @@ _docker_container_run() {
13161316
--cidfile
13171317
--cpu-period
13181318
--cpu-quota
1319+
--cpu-rt-period
1320+
--cpu-rt-runtime
13191321
--cpuset-cpus
13201322
--cpuset-mems
13211323
--cpu-shares -c
@@ -1667,6 +1669,8 @@ _docker_container_update() {
16671669
--blkio-weight
16681670
--cpu-period
16691671
--cpu-quota
1672+
--cpu-rt-period
1673+
--cpu-rt-runtime
16701674
--cpuset-cpus
16711675
--cpuset-mems
16721676
--cpu-shares -c

‎contrib/completion/zsh/_docker‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1433,6 +1433,8 @@ __docker_subcommand() {
14331433
"($help -c --cpu-shares)"{-c=,--cpu-shares=}"[CPU shares (relative weight)]:CPU shares:(0 10 100 200 500 800 1000)"
14341434
"($help)--cpu-period=[Limit the CPU CFS (Completely Fair Scheduler) period]:CPU period: "
14351435
"($help)--cpu-quota=[Limit the CPU CFS (Completely Fair Scheduler) quota]:CPU quota: "
1436+
"($help)--cpu-rt-period=[Limit the CPU real-time period]:CPU real-time period in microseconds: "
1437+
"($help)--cpu-rt-runtime=[Limit the CPU real-time runtime]:CPU real-time runtime in microseconds: "
14361438
"($help)--cpuset-cpus=[CPUs in which to allow execution]:CPUs: "
14371439
"($help)--cpuset-mems=[MEMs in which to allow execution]:MEMs: "
14381440
"($help -m --memory)"{-m=,--memory=}"[Memory limit]:Memory limit: "

‎daemon/config_unix.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,8 @@ type Config struct {
3434
Ulimits map[string]*units.Ulimit `json:"default-ulimits,omitempty"`
3535
Runtimes map[string]types.Runtime `json:"runtimes,omitempty"`
3636
DefaultRuntime string `json:"default-runtime,omitempty"`
37+
CPURealtimePeriod int64 `json:"cpu-rt-period,omitempty"`
38+
CPURealtimeRuntime int64 `json:"cpu-rt-runtime,omitempty"`
3739
OOMScoreAdjust int `json:"oom-score-adjust,omitempty"`
3840
Init bool `json:"init,omitempty"`
3941
InitPath string `json:"init-path,omitempty"`
@@ -97,6 +99,8 @@ func (config *Config) InstallFlags(flags *pflag.FlagSet) {
9799
flags.IntVar(&config.OOMScoreAdjust, "oom-score-adjust", -500, "Set the oom_score_adj for the daemon")
98100
flags.BoolVar(&config.Init, "init", false, "Run an init in the container to forward signals and reap processes")
99101
flags.StringVar(&config.InitPath, "init-path", "", "Path to the docker-init binary")
102+
flags.Int64Var(&config.CPURealtimePeriod, "cpu-rt-period", 0, "Limit the CPU real-time period in microseconds")
103+
flags.Int64Var(&config.CPURealtimeRuntime, "cpu-rt-runtime", 0, "Limit the CPU real-time runtime in microseconds")
100104

101105
config.attachExperimentalFlags(flags)
102106
}

‎daemon/daemon_unix.go‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,11 @@ import (
3636
"github.com/docker/libnetwork/options"
3737
lntypes "github.com/docker/libnetwork/types"
3838
"github.com/golang/protobuf/ptypes"
39+
"github.com/opencontainers/runc/libcontainer/cgroups"
3940
"github.com/opencontainers/runc/libcontainer/label"
4041
rsystem "github.com/opencontainers/runc/libcontainer/system"
4142
"github.com/opencontainers/runc/libcontainer/user"
42-
"github.com/opencontainers/runtime-spec/specs-go"
43+
specs "github.com/opencontainers/runtime-spec/specs-go"
4344
"github.com/vishvananda/netlink"
4445
)
4546

@@ -118,6 +119,16 @@ func getCPUResources(config containertypes.Resources) *specs.CPU {
118119
cpu.Quota = &quota
119120
}
120121

122+
if config.CPURealtimePeriod != 0 {
123+
period := uint64(config.CPURealtimePeriod)
124+
cpu.RealtimePeriod = &period
125+
}
126+
127+
if config.CPURealtimeRuntime != 0 {
128+
runtime := uint64(config.CPURealtimeRuntime)
129+
cpu.RealtimeRuntime = &runtime
130+
}
131+
121132
return &cpu
122133
}
123134

@@ -1184,3 +1195,34 @@ func setupOOMScoreAdj(score int) error {
11841195
f.Close()
11851196
return err
11861197
}
1198+
1199+
func (daemon *Daemon) initCgroupsPath(path string) error {
1200+
if path == "/" || path == "." {
1201+
return nil
1202+
}
1203+
1204+
daemon.initCgroupsPath(filepath.Dir(path))
1205+
1206+
_, root, err := cgroups.FindCgroupMountpointAndRoot("cpu")
1207+
if err != nil {
1208+
return err
1209+
}
1210+
1211+
path = filepath.Join(root, path)
1212+
sysinfo := sysinfo.New(false)
1213+
if err := os.MkdirAll(path, 0755); err != nil && !os.IsExist(err) {
1214+
return err
1215+
}
1216+
if sysinfo.CPURealtimePeriod && daemon.configStore.CPURealtimePeriod != 0 {
1217+
if err := ioutil.WriteFile(filepath.Join(path, "cpu.rt_period_us"), []byte(strconv.FormatInt(daemon.configStore.CPURealtimePeriod, 10)), 0700); err != nil {
1218+
return err
1219+
}
1220+
}
1221+
if sysinfo.CPURealtimeRuntime && daemon.configStore.CPURealtimeRuntime != 0 {
1222+
if err := ioutil.WriteFile(filepath.Join(path, "cpu.rt_runtime_us"), []byte(strconv.FormatInt(daemon.configStore.CPURealtimeRuntime, 10)), 0700); err != nil {
1223+
return err
1224+
}
1225+
}
1226+
1227+
return nil
1228+
}

‎daemon/oci_linux.go‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,9 +21,10 @@ import (
2121
"github.com/docker/docker/pkg/symlink"
2222
"github.com/docker/docker/volume"
2323
"github.com/opencontainers/runc/libcontainer/apparmor"
24+
"github.com/opencontainers/runc/libcontainer/cgroups"
2425
"github.com/opencontainers/runc/libcontainer/devices"
2526
"github.com/opencontainers/runc/libcontainer/user"
26-
"github.com/opencontainers/runtime-spec/specs-go"
27+
specs "github.com/opencontainers/runtime-spec/specs-go"
2728
)
2829

2930
func setResources(s *specs.Spec, r containertypes.Resources) error {
@@ -655,6 +656,29 @@ func (daemon *Daemon) createSpec(c *container.Container) (*specs.Spec, error) {
655656
}
656657
s.Linux.Resources.OOMScoreAdj = &c.HostConfig.OomScoreAdj
657658
s.Linux.Sysctl = c.HostConfig.Sysctls
659+
660+
p := *s.Linux.CgroupsPath
661+
if useSystemd {
662+
initPath, err := cgroups.GetInitCgroupDir("cpu")
663+
if err != nil {
664+
return nil, err
665+
}
666+
p, _ = cgroups.GetThisCgroupDir("cpu")
667+
if err != nil {
668+
return nil, err
669+
}
670+
p = filepath.Join(initPath, p)
671+
}
672+
673+
// Clean path to guard against things like ../../../BAD
674+
parentPath := filepath.Dir(p)
675+
if !filepath.IsAbs(parentPath) {
676+
parentPath = filepath.Clean("/" + parentPath)
677+
}
678+
679+
if err := daemon.initCgroupsPath(parentPath); err != nil {
680+
return nil, fmt.Errorf("linux init cgroups path: %v", err)
681+
}
658682
if err := setDevices(&s, c); err != nil {
659683
return nil, fmt.Errorf("linux runtime spec devices: %v", err)
660684
}

‎docs/reference/api/docker_remote_api.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -160,6 +160,7 @@ This section lists each version from latest to oldest. Each listing includes a
160160
* `POST /volumes/prune` prunes unused volumes.
161161
* `POST /networks/prune` prunes unused networks.
162162
* Every API response now includes a `Docker-Experimental` header specifying if experimental features are enabled (value can be `true` or `false`).
163+
* The `hostConfig` option now accepts the fields `CpuRealtimePeriod` and `CpuRtRuntime` to allocate cpu runtime to rt tasks when `CONFIG_RT_GROUP_SCHED` is enabled in the kernel.
163164

164165

165166
### v1.24 API changes

‎docs/reference/api/docker_remote_api_v1.25.md‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -304,6 +304,8 @@ Create a container
304304
"CpuPercent": 80,
305305
"CpuShares": 512,
306306
"CpuPeriod": 100000,
307+
"CpuRealtimePeriod": 1000000,
308+
"CpuRealtimeRuntime": 10000,
307309
"CpuQuota": 50000,
308310
"CpusetCpus": "0,1",
309311
"CpusetMems": "0,1",
@@ -426,6 +428,8 @@ Create a container
426428
- **CpuShares** - An integer value containing the container's CPU Shares
427429
(ie. the relative weight vs other containers).
428430
- **CpuPeriod** - The length of a CPU period in microseconds.
431+
- **CpuRealtimePeriod** - The length of a CPU real-time period in microseconds (0=no time allocated for rt tasks)
432+
- **CpuRealtimeRuntime** - The length of a CPU real-time runtime in microseconds (0=no time allocated for rt tasks)
429433
- **CpuQuota** - Microseconds of CPU time that the container can get in a CPU period.
430434
- **CpusetCpus** - String value containing the `cgroups CpusetCpus` to use.
431435
- **CpusetMems** - Memory nodes (MEMs) in which to allow execution (0-3, 0,1). Only effective on NUMA systems.
@@ -615,6 +619,8 @@ Return low-level information on the container `id`
615619
"CpuPercent": 80,
616620
"CpuShares": 0,
617621
"CpuPeriod": 100000,
622+
"CpuRealtimePeriod": 1000000,
623+
"CpuRealtimeRuntime": 10000,
618624
"Devices": [],
619625
"Dns": null,
620626
"DnsOptions": null,
@@ -1191,6 +1197,8 @@ Update configuration of one or more containers.
11911197
"BlkioWeight": 300,
11921198
"CpuShares": 512,
11931199
"CpuPeriod": 100000,
1200+
"CpuRealtimePeriod": 1000000,
1201+
"CpuRealtimeRuntime": 10000,
11941202
"CpuQuota": 50000,
11951203
"CpusetCpus": "0,1",
11961204
"CpusetMems": "0",

‎docs/reference/commandline/create.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,8 @@ Options:
3535
--cpu-period int Limit CPU CFS (Completely Fair Scheduler) period
3636
--cpu-quota int Limit CPU CFS (Completely Fair Scheduler) quota
3737
-c, --cpu-shares int CPU shares (relative weight)
38+
--cpu-rt-period int Limit the CPU real-time period in microseconds
39+
--cpu-rt-runtime int Limit the CPU real-time runtime in microseconds
3840
--cpuset-cpus string CPUs in which to allow execution (0-3, 0,1)
3941
--cpuset-mems string MEMs in which to allow execution (0-3, 0,1)
4042
--device value Add a host device to the container (default [])

0 commit comments

Comments
 (0)