[chore]: Bump github.com/KimMachineGun/automemlimit from 0.4.0 to 0.5.0 (#2560)
This commit is contained in:
parent
4e0488acfe
commit
a858831387
3
go.mod
3
go.mod
|
@ -21,7 +21,7 @@ require (
|
||||||
codeberg.org/gruf/go-structr v0.1.1
|
codeberg.org/gruf/go-structr v0.1.1
|
||||||
codeberg.org/superseriousbusiness/exif-terminator v0.7.0
|
codeberg.org/superseriousbusiness/exif-terminator v0.7.0
|
||||||
github.com/DmitriyVTitov/size v1.5.0
|
github.com/DmitriyVTitov/size v1.5.0
|
||||||
github.com/KimMachineGun/automemlimit v0.4.0
|
github.com/KimMachineGun/automemlimit v0.5.0
|
||||||
github.com/abema/go-mp4 v1.1.1
|
github.com/abema/go-mp4 v1.1.1
|
||||||
github.com/buckket/go-blurhash v1.1.0
|
github.com/buckket/go-blurhash v1.1.0
|
||||||
github.com/coreos/go-oidc/v3 v3.9.0
|
github.com/coreos/go-oidc/v3 v3.9.0
|
||||||
|
@ -143,6 +143,7 @@ require (
|
||||||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||||
github.com/opencontainers/runtime-spec v1.0.2 // indirect
|
github.com/opencontainers/runtime-spec v1.0.2 // indirect
|
||||||
|
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
|
||||||
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
|
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
|
||||||
github.com/pkg/errors v0.9.1 // indirect
|
github.com/pkg/errors v0.9.1 // indirect
|
||||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||||
|
|
6
go.sum
6
go.sum
|
@ -79,8 +79,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
||||||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||||
github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
|
github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
|
||||||
github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
|
github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
|
||||||
github.com/KimMachineGun/automemlimit v0.4.0 h1:qOjSDbAUENEL6fiKmRKuAVhPaLijpoEHFDTE+I+prp0=
|
github.com/KimMachineGun/automemlimit v0.5.0 h1:BeOe+BbJc8L5chL3OwzVYjVzyvPALdd5wxVVOWuUZmQ=
|
||||||
github.com/KimMachineGun/automemlimit v0.4.0/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0=
|
github.com/KimMachineGun/automemlimit v0.5.0/go.mod h1:di3GCKiu9Y+1fs92erCbUvKzPkNyViN3mA0vti/ykEQ=
|
||||||
github.com/abema/go-mp4 v1.1.1 h1:OfzkdMO6SWTBR1ltNSVwlTHatrAK9I3iYLQfkdEMMuc=
|
github.com/abema/go-mp4 v1.1.1 h1:OfzkdMO6SWTBR1ltNSVwlTHatrAK9I3iYLQfkdEMMuc=
|
||||||
github.com/abema/go-mp4 v1.1.1/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
|
github.com/abema/go-mp4 v1.1.1/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
|
||||||
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
|
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
|
||||||
|
@ -416,6 +416,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia
|
||||||
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
||||||
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
|
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
|
||||||
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
|
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
|
||||||
|
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
|
||||||
|
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
|
||||||
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
|
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
|
||||||
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
|
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
|
||||||
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||||
|
|
|
@ -8,6 +8,14 @@ Automatically set `GOMEMLIMIT` to match Linux [cgroups(7)](https://man7.org/linu
|
||||||
|
|
||||||
See more details about `GOMEMLIMIT` [here](https://tip.golang.org/doc/gc-guide#Memory_limit).
|
See more details about `GOMEMLIMIT` [here](https://tip.golang.org/doc/gc-guide#Memory_limit).
|
||||||
|
|
||||||
|
## Notice
|
||||||
|
|
||||||
|
Version `v0.5.0` introduces a fallback to system memory limits as an experimental feature when cgroup limits are unavailable. Activate this by setting `AUTOMEMLIMIT_EXPERIMENT=system`.
|
||||||
|
You can also use system memory limits via `memlimit.FromSystem` provider directly.
|
||||||
|
|
||||||
|
This feature is under evaluation and might become a default or be removed based on user feedback.
|
||||||
|
If you have any feedback about this feature, please open an issue.
|
||||||
|
|
||||||
## Installation
|
## Installation
|
||||||
|
|
||||||
```shell
|
```shell
|
||||||
|
@ -34,9 +42,17 @@ import "github.com/KimMachineGun/automemlimit/memlimit"
|
||||||
func init() {
|
func init() {
|
||||||
memlimit.SetGoMemLimitWithOpts(
|
memlimit.SetGoMemLimitWithOpts(
|
||||||
memlimit.WithRatio(0.9),
|
memlimit.WithRatio(0.9),
|
||||||
memlimit.WithEnv(),
|
|
||||||
memlimit.WithProvider(memlimit.FromCgroup),
|
memlimit.WithProvider(memlimit.FromCgroup),
|
||||||
)
|
)
|
||||||
|
memlimit.SetGoMemLimitWithOpts(
|
||||||
|
memlimit.WithRatio(0.9),
|
||||||
|
memlimit.WithProvider(
|
||||||
|
memlimit.ApplyFallback(
|
||||||
|
memlimit.FromCgroup,
|
||||||
|
memlimit.FromSystem,
|
||||||
|
),
|
||||||
|
),
|
||||||
|
)
|
||||||
memlimit.SetGoMemLimitWithEnv()
|
memlimit.SetGoMemLimitWithEnv()
|
||||||
memlimit.SetGoMemLimit(0.9)
|
memlimit.SetGoMemLimit(0.9)
|
||||||
memlimit.SetGoMemLimitWithProvider(memlimit.Limit(1024*1024), 0.9)
|
memlimit.SetGoMemLimitWithProvider(memlimit.Limit(1024*1024), 0.9)
|
||||||
|
|
|
@ -1,91 +1,12 @@
|
||||||
//go:build linux
|
|
||||||
// +build linux
|
|
||||||
|
|
||||||
package memlimit
|
package memlimit
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"path/filepath"
|
"errors"
|
||||||
|
|
||||||
"github.com/containerd/cgroups/v3"
|
|
||||||
"github.com/containerd/cgroups/v3/cgroup1"
|
|
||||||
"github.com/containerd/cgroups/v3/cgroup2"
|
|
||||||
)
|
)
|
||||||
|
|
||||||
const (
|
var (
|
||||||
cgroupMountPoint = "/sys/fs/cgroup"
|
// ErrNoCgroup is returned when the process is not in cgroup.
|
||||||
|
ErrNoCgroup = errors.New("process is not in cgroup")
|
||||||
|
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
|
||||||
|
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
|
||||||
)
|
)
|
||||||
|
|
||||||
// FromCgroup returns the memory limit based on the cgroups version on this system.
|
|
||||||
func FromCgroup() (uint64, error) {
|
|
||||||
switch cgroups.Mode() {
|
|
||||||
case cgroups.Legacy:
|
|
||||||
return FromCgroupV1()
|
|
||||||
case cgroups.Hybrid:
|
|
||||||
return FromCgroupHybrid()
|
|
||||||
case cgroups.Unified:
|
|
||||||
return FromCgroupV2()
|
|
||||||
}
|
|
||||||
return 0, ErrNoCgroup
|
|
||||||
}
|
|
||||||
|
|
||||||
// FromCgroupV1 returns the memory limit from the cgroup v1.
|
|
||||||
func FromCgroupV1() (uint64, error) {
|
|
||||||
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
|
|
||||||
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
|
|
||||||
))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 {
|
|
||||||
return limit, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0, ErrNoLimit
|
|
||||||
}
|
|
||||||
|
|
||||||
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
|
|
||||||
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
|
|
||||||
func FromCgroupHybrid() (uint64, error) {
|
|
||||||
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
|
|
||||||
if err == nil {
|
|
||||||
return limit, nil
|
|
||||||
} else if err != ErrNoLimit {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
return FromCgroupV1()
|
|
||||||
}
|
|
||||||
|
|
||||||
// FromCgroupV2 returns the memory limit from the cgroup v2.
|
|
||||||
func FromCgroupV2() (uint64, error) {
|
|
||||||
return fromCgroupV2(cgroupMountPoint)
|
|
||||||
}
|
|
||||||
|
|
||||||
func fromCgroupV2(mountPoint string) (uint64, error) {
|
|
||||||
path, err := cgroup2.NestedGroupPath("")
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
stats, err := m.Stat()
|
|
||||||
if err != nil {
|
|
||||||
return 0, err
|
|
||||||
}
|
|
||||||
|
|
||||||
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 {
|
|
||||||
return limit, nil
|
|
||||||
}
|
|
||||||
|
|
||||||
return 0, ErrNoLimit
|
|
||||||
}
|
|
||||||
|
|
98
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go
generated
vendored
Normal file
98
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go
generated
vendored
Normal file
|
@ -0,0 +1,98 @@
|
||||||
|
//go:build linux
|
||||||
|
// +build linux
|
||||||
|
|
||||||
|
package memlimit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"math"
|
||||||
|
"os"
|
||||||
|
"path/filepath"
|
||||||
|
|
||||||
|
"github.com/containerd/cgroups/v3"
|
||||||
|
"github.com/containerd/cgroups/v3/cgroup1"
|
||||||
|
"github.com/containerd/cgroups/v3/cgroup2"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
cgroupMountPoint = "/sys/fs/cgroup"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FromCgroup returns the memory limit based on the cgroups version on this system.
|
||||||
|
func FromCgroup() (uint64, error) {
|
||||||
|
switch cgroups.Mode() {
|
||||||
|
case cgroups.Legacy:
|
||||||
|
return FromCgroupV1()
|
||||||
|
case cgroups.Hybrid:
|
||||||
|
return FromCgroupHybrid()
|
||||||
|
case cgroups.Unified:
|
||||||
|
return FromCgroupV2()
|
||||||
|
}
|
||||||
|
return 0, ErrNoCgroup
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromCgroupV1 returns the memory limit from the cgroup v1.
|
||||||
|
func FromCgroupV1() (uint64, error) {
|
||||||
|
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
|
||||||
|
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
|
||||||
|
))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 && limit != getCgroupV1NoLimit() {
|
||||||
|
return limit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0, ErrNoLimit
|
||||||
|
}
|
||||||
|
|
||||||
|
func getCgroupV1NoLimit() uint64 {
|
||||||
|
ps := uint64(os.Getpagesize())
|
||||||
|
return math.MaxInt64 / ps * ps
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
|
||||||
|
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
|
||||||
|
func FromCgroupHybrid() (uint64, error) {
|
||||||
|
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
|
||||||
|
if err == nil {
|
||||||
|
return limit, nil
|
||||||
|
} else if err != ErrNoLimit {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
return FromCgroupV1()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FromCgroupV2 returns the memory limit from the cgroup v2.
|
||||||
|
func FromCgroupV2() (uint64, error) {
|
||||||
|
return fromCgroupV2(cgroupMountPoint)
|
||||||
|
}
|
||||||
|
|
||||||
|
func fromCgroupV2(mountPoint string) (uint64, error) {
|
||||||
|
path, err := cgroup2.NestedGroupPath("")
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
stats, err := m.Stat()
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 && limit != math.MaxUint64 {
|
||||||
|
return limit, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return 0, ErrNoLimit
|
||||||
|
}
|
14
vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go
generated
vendored
Normal file
14
vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
||||||
|
package memlimit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/pbnjay/memory"
|
||||||
|
)
|
||||||
|
|
||||||
|
// FromSystem returns the total memory of the system.
|
||||||
|
func FromSystem() (uint64, error) {
|
||||||
|
limit := memory.TotalMemory()
|
||||||
|
if limit == 0 {
|
||||||
|
return 0, ErrNoLimit
|
||||||
|
}
|
||||||
|
return limit, nil
|
||||||
|
}
|
59
vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go
generated
vendored
Normal file
59
vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go
generated
vendored
Normal file
|
@ -0,0 +1,59 @@
|
||||||
|
package memlimit
|
||||||
|
|
||||||
|
import (
|
||||||
|
"fmt"
|
||||||
|
"os"
|
||||||
|
"reflect"
|
||||||
|
"strings"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT"
|
||||||
|
)
|
||||||
|
|
||||||
|
// Experiments is a set of experiment flags.
|
||||||
|
// It is used to enable experimental features.
|
||||||
|
//
|
||||||
|
// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT.
|
||||||
|
// The value of the environment variable is a comma-separated list of experiment names.
|
||||||
|
//
|
||||||
|
// The following experiment names are known:
|
||||||
|
//
|
||||||
|
// - none: disable all experiments
|
||||||
|
// - system: enable fallback to system memory limit
|
||||||
|
type Experiments struct {
|
||||||
|
// System enables fallback to system memory limit.
|
||||||
|
System bool
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseExperiments() (Experiments, error) {
|
||||||
|
var exp Experiments
|
||||||
|
|
||||||
|
// Create a map of known experiment names.
|
||||||
|
names := make(map[string]func(bool))
|
||||||
|
rv := reflect.ValueOf(&exp).Elem()
|
||||||
|
rt := rv.Type()
|
||||||
|
for i := 0; i < rt.NumField(); i++ {
|
||||||
|
field := rv.Field(i)
|
||||||
|
names[strings.ToLower(rt.Field(i).Name)] = field.SetBool
|
||||||
|
}
|
||||||
|
|
||||||
|
// Parse names.
|
||||||
|
for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") {
|
||||||
|
if f == "" {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
if f == "none" {
|
||||||
|
exp = Experiments{}
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
val := true
|
||||||
|
set, ok := names[f]
|
||||||
|
if !ok {
|
||||||
|
return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f)
|
||||||
|
}
|
||||||
|
set(val)
|
||||||
|
}
|
||||||
|
|
||||||
|
return exp, nil
|
||||||
|
}
|
|
@ -22,16 +22,11 @@ const (
|
||||||
var (
|
var (
|
||||||
// ErrNoLimit is returned when the memory limit is not set.
|
// ErrNoLimit is returned when the memory limit is not set.
|
||||||
ErrNoLimit = errors.New("memory is not limited")
|
ErrNoLimit = errors.New("memory is not limited")
|
||||||
// ErrNoCgroup is returned when the process is not in cgroup.
|
|
||||||
ErrNoCgroup = errors.New("process is not in cgroup")
|
|
||||||
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
|
|
||||||
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
|
|
||||||
)
|
)
|
||||||
|
|
||||||
type config struct {
|
type config struct {
|
||||||
logger *log.Logger
|
logger *log.Logger
|
||||||
ratio float64
|
ratio float64
|
||||||
env bool
|
|
||||||
provider Provider
|
provider Provider
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -50,10 +45,10 @@ func WithRatio(ratio float64) Option {
|
||||||
// WithEnv configures whether to use environment variables.
|
// WithEnv configures whether to use environment variables.
|
||||||
//
|
//
|
||||||
// Default: false
|
// Default: false
|
||||||
|
//
|
||||||
|
// Deprecated: currently this does nothing.
|
||||||
func WithEnv() Option {
|
func WithEnv() Option {
|
||||||
return func(cfg *config) {
|
return func(cfg *config) {}
|
||||||
cfg.env = true
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
// WithProvider configures the provider.
|
// WithProvider configures the provider.
|
||||||
|
@ -65,17 +60,24 @@ func WithProvider(provider Provider) Option {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options.
|
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables.
|
||||||
|
//
|
||||||
|
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
|
||||||
|
// through AUTOMEMLIMIT envrironment variable in the half-open range (0.0,1.0].
|
||||||
|
//
|
||||||
|
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
|
||||||
|
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
|
||||||
|
//
|
||||||
|
// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features.
|
||||||
|
// Please see the documentation of Experiments for more details.
|
||||||
//
|
//
|
||||||
// Options:
|
// Options:
|
||||||
// - WithRatio
|
// - WithRatio
|
||||||
// - WithEnv (see more SetGoMemLimitWithEnv)
|
|
||||||
// - WithProvider
|
// - WithProvider
|
||||||
func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
||||||
cfg := &config{
|
cfg := &config{
|
||||||
logger: log.New(io.Discard, "", log.LstdFlags),
|
logger: log.New(io.Discard, "", log.LstdFlags),
|
||||||
ratio: defaultAUTOMEMLIMIT,
|
ratio: defaultAUTOMEMLIMIT,
|
||||||
env: false,
|
|
||||||
provider: FromCgroup,
|
provider: FromCgroup,
|
||||||
}
|
}
|
||||||
if os.Getenv(envAUTOMEMLIMIT_DEBUG) == "true" {
|
if os.Getenv(envAUTOMEMLIMIT_DEBUG) == "true" {
|
||||||
|
@ -90,6 +92,15 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
||||||
}
|
}
|
||||||
}()
|
}()
|
||||||
|
|
||||||
|
exps, err := parseExperiments()
|
||||||
|
if err != nil {
|
||||||
|
return 0, fmt.Errorf("failed to parse experiments: %w", err)
|
||||||
|
}
|
||||||
|
if exps.System {
|
||||||
|
cfg.logger.Println("system experiment is enabled: using system memory limit as a fallback")
|
||||||
|
cfg.provider = ApplyFallback(cfg.provider, FromSystem)
|
||||||
|
}
|
||||||
|
|
||||||
snapshot := debug.SetMemoryLimit(-1)
|
snapshot := debug.SetMemoryLimit(-1)
|
||||||
defer func() {
|
defer func() {
|
||||||
err := recover()
|
err := recover()
|
||||||
|
@ -122,6 +133,10 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
||||||
|
|
||||||
limit, err := setGoMemLimit(ApplyRatio(cfg.provider, ratio))
|
limit, err := setGoMemLimit(ApplyRatio(cfg.provider, ratio))
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if errors.Is(err, ErrNoLimit) {
|
||||||
|
cfg.logger.Printf("memory is not limited, skipping: %v\n", err)
|
||||||
|
return 0, nil
|
||||||
|
}
|
||||||
return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err)
|
return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -130,14 +145,8 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
||||||
return limit, nil
|
return limit, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variable.
|
|
||||||
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
|
|
||||||
// through AUTOMEMLIMIT in the half-open range (0.0,1.0].
|
|
||||||
//
|
|
||||||
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
|
|
||||||
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
|
|
||||||
func SetGoMemLimitWithEnv() {
|
func SetGoMemLimitWithEnv() {
|
||||||
_, _ = SetGoMemLimitWithOpts(WithEnv())
|
_, _ = SetGoMemLimitWithOpts()
|
||||||
}
|
}
|
||||||
|
|
||||||
// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio.
|
// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio.
|
||||||
|
|
|
@ -0,0 +1,29 @@
|
||||||
|
BSD 3-Clause License
|
||||||
|
|
||||||
|
Copyright (c) 2017, Jeremy Jay
|
||||||
|
All rights reserved.
|
||||||
|
|
||||||
|
Redistribution and use in source and binary forms, with or without
|
||||||
|
modification, are permitted provided that the following conditions are met:
|
||||||
|
|
||||||
|
* Redistributions of source code must retain the above copyright notice, this
|
||||||
|
list of conditions and the following disclaimer.
|
||||||
|
|
||||||
|
* Redistributions in binary form must reproduce the above copyright notice,
|
||||||
|
this list of conditions and the following disclaimer in the documentation
|
||||||
|
and/or other materials provided with the distribution.
|
||||||
|
|
||||||
|
* Neither the name of the copyright holder nor the names of its
|
||||||
|
contributors may be used to endorse or promote products derived from
|
||||||
|
this software without specific prior written permission.
|
||||||
|
|
||||||
|
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||||
|
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||||
|
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||||
|
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||||
|
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||||
|
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||||
|
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||||
|
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||||
|
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||||
|
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@ -0,0 +1,41 @@
|
||||||
|
# memory
|
||||||
|
|
||||||
|
Package `memory` provides two methods reporting total physical system memory
|
||||||
|
accessible to the kernel, and free memory available to the running application.
|
||||||
|
|
||||||
|
This package has no external dependency besides the standard library and default operating system tools.
|
||||||
|
|
||||||
|
Documentation:
|
||||||
|
[![GoDoc](https://godoc.org/github.com/pbnjay/memory?status.svg)](https://godoc.org/github.com/pbnjay/memory)
|
||||||
|
|
||||||
|
This is useful for dynamic code to minimize thrashing and other contention, similar to the stdlib `runtime.NumCPU`
|
||||||
|
See some history of the proposal at https://github.com/golang/go/issues/21816
|
||||||
|
|
||||||
|
|
||||||
|
## Example
|
||||||
|
|
||||||
|
```go
|
||||||
|
fmt.Printf("Total system memory: %d\n", memory.TotalMemory())
|
||||||
|
fmt.Printf("Free memory: %d\n", memory.FreeMemory())
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## Testing
|
||||||
|
|
||||||
|
Tested/working on:
|
||||||
|
- macOS 10.12.6 (16G29), 10.15.7 (19H2)
|
||||||
|
- Windows 10 1511 (10586.1045)
|
||||||
|
- Linux RHEL (3.10.0-327.3.1.el7.x86_64)
|
||||||
|
- Raspberry Pi 3 (ARMv8) on Raspbian, ODROID-C1+ (ARMv7) on Ubuntu, C.H.I.P
|
||||||
|
(ARMv7).
|
||||||
|
- Amazon Linux 2 aarch64 (m6a.large, 4.14.203-156.332.amzn2.aarch64)
|
||||||
|
|
||||||
|
Tested on virtual machines:
|
||||||
|
- Windows 7 SP1 386
|
||||||
|
- Debian stretch 386
|
||||||
|
- NetBSD 7.1 amd64 + 386
|
||||||
|
- OpenBSD 6.1 amd64 + 386
|
||||||
|
- FreeBSD 11.1 amd64 + 386
|
||||||
|
- DragonFly BSD 4.8.1 amd64
|
||||||
|
|
||||||
|
If you have access to untested systems please test and report success/bugs.
|
|
@ -0,0 +1,24 @@
|
||||||
|
// Package memory provides a single method reporting total system memory
|
||||||
|
// accessible to the kernel.
|
||||||
|
package memory
|
||||||
|
|
||||||
|
// TotalMemory returns the total accessible system memory in bytes.
|
||||||
|
//
|
||||||
|
// The total accessible memory is installed physical memory size minus reserved
|
||||||
|
// areas for the kernel and hardware, if such reservations are reported by
|
||||||
|
// the operating system.
|
||||||
|
//
|
||||||
|
// If accessible memory size could not be determined, then 0 is returned.
|
||||||
|
func TotalMemory() uint64 {
|
||||||
|
return sysTotalMemory()
|
||||||
|
}
|
||||||
|
|
||||||
|
// FreeMemory returns the total free system memory in bytes.
|
||||||
|
//
|
||||||
|
// The total free memory is installed physical memory size minus reserved
|
||||||
|
// areas for other applications running on the same system.
|
||||||
|
//
|
||||||
|
// If free memory size could not be determined, then 0 is returned.
|
||||||
|
func FreeMemory() uint64 {
|
||||||
|
return sysFreeMemory()
|
||||||
|
}
|
|
@ -0,0 +1,19 @@
|
||||||
|
// +build freebsd openbsd dragonfly netbsd
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
func sysTotalMemory() uint64 {
|
||||||
|
s, err := sysctlUint64("hw.physmem")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysFreeMemory() uint64 {
|
||||||
|
s, err := sysctlUint64("hw.usermem")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
|
@ -0,0 +1,49 @@
|
||||||
|
// +build darwin
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"os/exec"
|
||||||
|
"regexp"
|
||||||
|
"strconv"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sysTotalMemory() uint64 {
|
||||||
|
s, err := sysctlUint64("hw.memsize")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return s
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysFreeMemory() uint64 {
|
||||||
|
cmd := exec.Command("vm_stat")
|
||||||
|
outBytes, err := cmd.Output()
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
|
||||||
|
rePageSize := regexp.MustCompile("page size of ([0-9]*) bytes")
|
||||||
|
reFreePages := regexp.MustCompile("Pages free: *([0-9]*)\\.")
|
||||||
|
|
||||||
|
// default: page size of 4096 bytes
|
||||||
|
matches := rePageSize.FindSubmatchIndex(outBytes)
|
||||||
|
pageSize := uint64(4096)
|
||||||
|
if len(matches) == 4 {
|
||||||
|
pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// ex: Pages free: 1126961.
|
||||||
|
matches = reFreePages.FindSubmatchIndex(outBytes)
|
||||||
|
freePages := uint64(0)
|
||||||
|
if len(matches) == 4 {
|
||||||
|
freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return freePages * pageSize
|
||||||
|
}
|
|
@ -0,0 +1,29 @@
|
||||||
|
// +build linux
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import "syscall"
|
||||||
|
|
||||||
|
func sysTotalMemory() uint64 {
|
||||||
|
in := &syscall.Sysinfo_t{}
|
||||||
|
err := syscall.Sysinfo(in)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// If this is a 32-bit system, then these fields are
|
||||||
|
// uint32 instead of uint64.
|
||||||
|
// So we always convert to uint64 to match signature.
|
||||||
|
return uint64(in.Totalram) * uint64(in.Unit)
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysFreeMemory() uint64 {
|
||||||
|
in := &syscall.Sysinfo_t{}
|
||||||
|
err := syscall.Sysinfo(in)
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// If this is a 32-bit system, then these fields are
|
||||||
|
// uint32 instead of uint64.
|
||||||
|
// So we always convert to uint64 to match signature.
|
||||||
|
return uint64(in.Freeram) * uint64(in.Unit)
|
||||||
|
}
|
|
@ -0,0 +1,60 @@
|
||||||
|
// +build windows
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
// omitting a few fields for brevity...
|
||||||
|
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx
|
||||||
|
type memStatusEx struct {
|
||||||
|
dwLength uint32
|
||||||
|
dwMemoryLoad uint32
|
||||||
|
ullTotalPhys uint64
|
||||||
|
ullAvailPhys uint64
|
||||||
|
unused [5]uint64
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysTotalMemory() uint64 {
|
||||||
|
kernel32, err := syscall.LoadDLL("kernel32.dll")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// GetPhysicallyInstalledSystemMemory is simpler, but broken on
|
||||||
|
// older versions of windows (and uses this under the hood anyway).
|
||||||
|
globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
msx := &memStatusEx{
|
||||||
|
dwLength: 64,
|
||||||
|
}
|
||||||
|
r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
|
||||||
|
if r == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return msx.ullTotalPhys
|
||||||
|
}
|
||||||
|
|
||||||
|
func sysFreeMemory() uint64 {
|
||||||
|
kernel32, err := syscall.LoadDLL("kernel32.dll")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
// GetPhysicallyInstalledSystemMemory is simpler, but broken on
|
||||||
|
// older versions of windows (and uses this under the hood anyway).
|
||||||
|
globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
|
||||||
|
if err != nil {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
msx := &memStatusEx{
|
||||||
|
dwLength: 64,
|
||||||
|
}
|
||||||
|
r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
|
||||||
|
if r == 0 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
return msx.ullAvailPhys
|
||||||
|
}
|
|
@ -0,0 +1,21 @@
|
||||||
|
// +build darwin freebsd openbsd dragonfly netbsd
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
import (
|
||||||
|
"syscall"
|
||||||
|
"unsafe"
|
||||||
|
)
|
||||||
|
|
||||||
|
func sysctlUint64(name string) (uint64, error) {
|
||||||
|
s, err := syscall.Sysctl(name)
|
||||||
|
if err != nil {
|
||||||
|
return 0, err
|
||||||
|
}
|
||||||
|
// hack because the string conversion above drops a \0
|
||||||
|
b := []byte(s)
|
||||||
|
if len(b) < 8 {
|
||||||
|
b = append(b, 0)
|
||||||
|
}
|
||||||
|
return *(*uint64)(unsafe.Pointer(&b[0])), nil
|
||||||
|
}
|
|
@ -0,0 +1,10 @@
|
||||||
|
// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd
|
||||||
|
|
||||||
|
package memory
|
||||||
|
|
||||||
|
func sysTotalMemory() uint64 {
|
||||||
|
return 0
|
||||||
|
}
|
||||||
|
func sysFreeMemory() uint64 {
|
||||||
|
return 0
|
||||||
|
}
|
|
@ -68,7 +68,7 @@ codeberg.org/superseriousbusiness/exif-terminator
|
||||||
# github.com/DmitriyVTitov/size v1.5.0
|
# github.com/DmitriyVTitov/size v1.5.0
|
||||||
## explicit; go 1.14
|
## explicit; go 1.14
|
||||||
github.com/DmitriyVTitov/size
|
github.com/DmitriyVTitov/size
|
||||||
# github.com/KimMachineGun/automemlimit v0.4.0
|
# github.com/KimMachineGun/automemlimit v0.5.0
|
||||||
## explicit; go 1.19
|
## explicit; go 1.19
|
||||||
github.com/KimMachineGun/automemlimit
|
github.com/KimMachineGun/automemlimit
|
||||||
github.com/KimMachineGun/automemlimit/memlimit
|
github.com/KimMachineGun/automemlimit/memlimit
|
||||||
|
@ -418,6 +418,9 @@ github.com/oklog/ulid
|
||||||
# github.com/opencontainers/runtime-spec v1.0.2
|
# github.com/opencontainers/runtime-spec v1.0.2
|
||||||
## explicit
|
## explicit
|
||||||
github.com/opencontainers/runtime-spec/specs-go
|
github.com/opencontainers/runtime-spec/specs-go
|
||||||
|
# github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
|
||||||
|
## explicit; go 1.16
|
||||||
|
github.com/pbnjay/memory
|
||||||
# github.com/pelletier/go-toml/v2 v2.1.1
|
# github.com/pelletier/go-toml/v2 v2.1.1
|
||||||
## explicit; go 1.16
|
## explicit; go 1.16
|
||||||
github.com/pelletier/go-toml/v2
|
github.com/pelletier/go-toml/v2
|
||||||
|
|
Loading…
Reference in New Issue