[chore]: Bump github.com/KimMachineGun/automemlimit from 0.4.0 to 0.5.0 (#2560)
This commit is contained in:
parent
4e0488acfe
commit
a858831387
3
go.mod
3
go.mod
|
@ -21,7 +21,7 @@ require (
|
|||
codeberg.org/gruf/go-structr v0.1.1
|
||||
codeberg.org/superseriousbusiness/exif-terminator v0.7.0
|
||||
github.com/DmitriyVTitov/size v1.5.0
|
||||
github.com/KimMachineGun/automemlimit v0.4.0
|
||||
github.com/KimMachineGun/automemlimit v0.5.0
|
||||
github.com/abema/go-mp4 v1.1.1
|
||||
github.com/buckket/go-blurhash v1.1.0
|
||||
github.com/coreos/go-oidc/v3 v3.9.0
|
||||
|
@ -143,6 +143,7 @@ require (
|
|||
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
|
||||
github.com/modern-go/reflect2 v1.0.2 // indirect
|
||||
github.com/opencontainers/runtime-spec v1.0.2 // indirect
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
|
||||
github.com/pelletier/go-toml/v2 v2.1.1 // indirect
|
||||
github.com/pkg/errors v0.9.1 // indirect
|
||||
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
|
||||
|
|
6
go.sum
6
go.sum
|
@ -79,8 +79,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03
|
|||
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
|
||||
github.com/DmitriyVTitov/size v1.5.0 h1:/PzqxYrOyOUX1BXj6J9OuVRVGe+66VL4D9FlUaW515g=
|
||||
github.com/DmitriyVTitov/size v1.5.0/go.mod h1:le6rNI4CoLQV1b9gzp1+3d7hMAD/uu2QcJ+aYbNgiU0=
|
||||
github.com/KimMachineGun/automemlimit v0.4.0 h1:qOjSDbAUENEL6fiKmRKuAVhPaLijpoEHFDTE+I+prp0=
|
||||
github.com/KimMachineGun/automemlimit v0.4.0/go.mod h1:pJhTW/nWJMj6SnWSU2TEKSlCaM+1N5Mej+IfS/5/Ol0=
|
||||
github.com/KimMachineGun/automemlimit v0.5.0 h1:BeOe+BbJc8L5chL3OwzVYjVzyvPALdd5wxVVOWuUZmQ=
|
||||
github.com/KimMachineGun/automemlimit v0.5.0/go.mod h1:di3GCKiu9Y+1fs92erCbUvKzPkNyViN3mA0vti/ykEQ=
|
||||
github.com/abema/go-mp4 v1.1.1 h1:OfzkdMO6SWTBR1ltNSVwlTHatrAK9I3iYLQfkdEMMuc=
|
||||
github.com/abema/go-mp4 v1.1.1/go.mod h1:vPl9t5ZK7K0x68jh12/+ECWBCXoWuIDtNgPtU2f04ws=
|
||||
github.com/ajg/form v1.5.1 h1:t9c7v8JUKu/XxOGBU0yjNpaMloxGEJhUkqFRq0ibGeU=
|
||||
|
@ -416,6 +416,8 @@ github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNia
|
|||
github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0=
|
||||
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e h1:s2RNOM/IGdY0Y6qfTeUKhDawdHDpK9RGBdx80qN4Ttw=
|
||||
github.com/orcaman/writerseeker v0.0.0-20200621085525-1d3f536ff85e/go.mod h1:nBdnFKj15wFbf94Rwfq4m30eAcyY9V/IyKAGQFtqkW0=
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
|
||||
github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
|
||||
github.com/pelletier/go-toml/v2 v2.0.1/go.mod h1:r9LEWfGN8R5k0VXJ+0BkIe7MYkRdwZOjgMj2KwnJFUo=
|
||||
github.com/pelletier/go-toml/v2 v2.1.1 h1:LWAJwfNvjQZCFIDKWYQaM62NcYeYViCmWIwmOStowAI=
|
||||
github.com/pelletier/go-toml/v2 v2.1.1/go.mod h1:tJU2Z3ZkXwnxa4DPO899bsyIoywizdUvyaeZurnPPDc=
|
||||
|
|
|
@ -8,6 +8,14 @@ Automatically set `GOMEMLIMIT` to match Linux [cgroups(7)](https://man7.org/linu
|
|||
|
||||
See more details about `GOMEMLIMIT` [here](https://tip.golang.org/doc/gc-guide#Memory_limit).
|
||||
|
||||
## Notice
|
||||
|
||||
Version `v0.5.0` introduces a fallback to system memory limits as an experimental feature when cgroup limits are unavailable. Activate this by setting `AUTOMEMLIMIT_EXPERIMENT=system`.
|
||||
You can also use system memory limits via `memlimit.FromSystem` provider directly.
|
||||
|
||||
This feature is under evaluation and might become a default or be removed based on user feedback.
|
||||
If you have any feedback about this feature, please open an issue.
|
||||
|
||||
## Installation
|
||||
|
||||
```shell
|
||||
|
@ -34,9 +42,17 @@ import "github.com/KimMachineGun/automemlimit/memlimit"
|
|||
func init() {
|
||||
memlimit.SetGoMemLimitWithOpts(
|
||||
memlimit.WithRatio(0.9),
|
||||
memlimit.WithEnv(),
|
||||
memlimit.WithProvider(memlimit.FromCgroup),
|
||||
)
|
||||
memlimit.SetGoMemLimitWithOpts(
|
||||
memlimit.WithRatio(0.9),
|
||||
memlimit.WithProvider(
|
||||
memlimit.ApplyFallback(
|
||||
memlimit.FromCgroup,
|
||||
memlimit.FromSystem,
|
||||
),
|
||||
),
|
||||
)
|
||||
memlimit.SetGoMemLimitWithEnv()
|
||||
memlimit.SetGoMemLimit(0.9)
|
||||
memlimit.SetGoMemLimitWithProvider(memlimit.Limit(1024*1024), 0.9)
|
||||
|
|
|
@ -1,91 +1,12 @@
|
|||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package memlimit
|
||||
|
||||
import (
|
||||
"path/filepath"
|
||||
|
||||
"github.com/containerd/cgroups/v3"
|
||||
"github.com/containerd/cgroups/v3/cgroup1"
|
||||
"github.com/containerd/cgroups/v3/cgroup2"
|
||||
"errors"
|
||||
)
|
||||
|
||||
const (
|
||||
cgroupMountPoint = "/sys/fs/cgroup"
|
||||
var (
|
||||
// ErrNoCgroup is returned when the process is not in cgroup.
|
||||
ErrNoCgroup = errors.New("process is not in cgroup")
|
||||
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
|
||||
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
|
||||
)
|
||||
|
||||
// FromCgroup returns the memory limit based on the cgroups version on this system.
|
||||
func FromCgroup() (uint64, error) {
|
||||
switch cgroups.Mode() {
|
||||
case cgroups.Legacy:
|
||||
return FromCgroupV1()
|
||||
case cgroups.Hybrid:
|
||||
return FromCgroupHybrid()
|
||||
case cgroups.Unified:
|
||||
return FromCgroupV2()
|
||||
}
|
||||
return 0, ErrNoCgroup
|
||||
}
|
||||
|
||||
// FromCgroupV1 returns the memory limit from the cgroup v1.
|
||||
func FromCgroupV1() (uint64, error) {
|
||||
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
|
||||
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
|
||||
))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 {
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
return 0, ErrNoLimit
|
||||
}
|
||||
|
||||
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
|
||||
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
|
||||
func FromCgroupHybrid() (uint64, error) {
|
||||
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
|
||||
if err == nil {
|
||||
return limit, nil
|
||||
} else if err != ErrNoLimit {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return FromCgroupV1()
|
||||
}
|
||||
|
||||
// FromCgroupV2 returns the memory limit from the cgroup v2.
|
||||
func FromCgroupV2() (uint64, error) {
|
||||
return fromCgroupV2(cgroupMountPoint)
|
||||
}
|
||||
|
||||
func fromCgroupV2(mountPoint string) (uint64, error) {
|
||||
path, err := cgroup2.NestedGroupPath("")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
stats, err := m.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 {
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
return 0, ErrNoLimit
|
||||
}
|
||||
|
|
98
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go
generated
vendored
Normal file
98
vendor/github.com/KimMachineGun/automemlimit/memlimit/cgroups_linux.go
generated
vendored
Normal file
|
@ -0,0 +1,98 @@
|
|||
//go:build linux
|
||||
// +build linux
|
||||
|
||||
package memlimit
|
||||
|
||||
import (
|
||||
"math"
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/containerd/cgroups/v3"
|
||||
"github.com/containerd/cgroups/v3/cgroup1"
|
||||
"github.com/containerd/cgroups/v3/cgroup2"
|
||||
)
|
||||
|
||||
const (
|
||||
cgroupMountPoint = "/sys/fs/cgroup"
|
||||
)
|
||||
|
||||
// FromCgroup returns the memory limit based on the cgroups version on this system.
|
||||
func FromCgroup() (uint64, error) {
|
||||
switch cgroups.Mode() {
|
||||
case cgroups.Legacy:
|
||||
return FromCgroupV1()
|
||||
case cgroups.Hybrid:
|
||||
return FromCgroupHybrid()
|
||||
case cgroups.Unified:
|
||||
return FromCgroupV2()
|
||||
}
|
||||
return 0, ErrNoCgroup
|
||||
}
|
||||
|
||||
// FromCgroupV1 returns the memory limit from the cgroup v1.
|
||||
func FromCgroupV1() (uint64, error) {
|
||||
cg, err := cgroup1.Load(cgroup1.RootPath, cgroup1.WithHiearchy(
|
||||
cgroup1.SingleSubsystem(cgroup1.Default, cgroup1.Memory),
|
||||
))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
metrics, err := cg.Stat(cgroup1.IgnoreNotExist)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if limit := metrics.GetMemory().GetHierarchicalMemoryLimit(); limit != 0 && limit != getCgroupV1NoLimit() {
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
return 0, ErrNoLimit
|
||||
}
|
||||
|
||||
func getCgroupV1NoLimit() uint64 {
|
||||
ps := uint64(os.Getpagesize())
|
||||
return math.MaxInt64 / ps * ps
|
||||
}
|
||||
|
||||
// FromCgroupHybrid returns the memory limit from the cgroup v1 or v2.
|
||||
// It checks the cgroup v2 first, and if it fails, it falls back to cgroup v1.
|
||||
func FromCgroupHybrid() (uint64, error) {
|
||||
limit, err := fromCgroupV2(filepath.Join(cgroupMountPoint, "unified"))
|
||||
if err == nil {
|
||||
return limit, nil
|
||||
} else if err != ErrNoLimit {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
return FromCgroupV1()
|
||||
}
|
||||
|
||||
// FromCgroupV2 returns the memory limit from the cgroup v2.
|
||||
func FromCgroupV2() (uint64, error) {
|
||||
return fromCgroupV2(cgroupMountPoint)
|
||||
}
|
||||
|
||||
func fromCgroupV2(mountPoint string) (uint64, error) {
|
||||
path, err := cgroup2.NestedGroupPath("")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
m, err := cgroup2.Load(path, cgroup2.WithMountpoint(mountPoint))
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
stats, err := m.Stat()
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
if limit := stats.GetMemory().GetUsageLimit(); limit != 0 && limit != math.MaxUint64 {
|
||||
return limit, nil
|
||||
}
|
||||
|
||||
return 0, ErrNoLimit
|
||||
}
|
14
vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go
generated
vendored
Normal file
14
vendor/github.com/KimMachineGun/automemlimit/memlimit/exp_system.go
generated
vendored
Normal file
|
@ -0,0 +1,14 @@
|
|||
package memlimit
|
||||
|
||||
import (
|
||||
"github.com/pbnjay/memory"
|
||||
)
|
||||
|
||||
// FromSystem returns the total memory of the system.
|
||||
func FromSystem() (uint64, error) {
|
||||
limit := memory.TotalMemory()
|
||||
if limit == 0 {
|
||||
return 0, ErrNoLimit
|
||||
}
|
||||
return limit, nil
|
||||
}
|
59
vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go
generated
vendored
Normal file
59
vendor/github.com/KimMachineGun/automemlimit/memlimit/experiment.go
generated
vendored
Normal file
|
@ -0,0 +1,59 @@
|
|||
package memlimit
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"reflect"
|
||||
"strings"
|
||||
)
|
||||
|
||||
const (
|
||||
envAUTOMEMLIMIT_EXPERIMENT = "AUTOMEMLIMIT_EXPERIMENT"
|
||||
)
|
||||
|
||||
// Experiments is a set of experiment flags.
|
||||
// It is used to enable experimental features.
|
||||
//
|
||||
// You can set the flags by setting the environment variable AUTOMEMLIMIT_EXPERIMENT.
|
||||
// The value of the environment variable is a comma-separated list of experiment names.
|
||||
//
|
||||
// The following experiment names are known:
|
||||
//
|
||||
// - none: disable all experiments
|
||||
// - system: enable fallback to system memory limit
|
||||
type Experiments struct {
|
||||
// System enables fallback to system memory limit.
|
||||
System bool
|
||||
}
|
||||
|
||||
func parseExperiments() (Experiments, error) {
|
||||
var exp Experiments
|
||||
|
||||
// Create a map of known experiment names.
|
||||
names := make(map[string]func(bool))
|
||||
rv := reflect.ValueOf(&exp).Elem()
|
||||
rt := rv.Type()
|
||||
for i := 0; i < rt.NumField(); i++ {
|
||||
field := rv.Field(i)
|
||||
names[strings.ToLower(rt.Field(i).Name)] = field.SetBool
|
||||
}
|
||||
|
||||
// Parse names.
|
||||
for _, f := range strings.Split(os.Getenv(envAUTOMEMLIMIT_EXPERIMENT), ",") {
|
||||
if f == "" {
|
||||
continue
|
||||
}
|
||||
if f == "none" {
|
||||
exp = Experiments{}
|
||||
continue
|
||||
}
|
||||
val := true
|
||||
set, ok := names[f]
|
||||
if !ok {
|
||||
return Experiments{}, fmt.Errorf("unknown AUTOMEMLIMIT_EXPERIMENT %s", f)
|
||||
}
|
||||
set(val)
|
||||
}
|
||||
|
||||
return exp, nil
|
||||
}
|
|
@ -22,16 +22,11 @@ const (
|
|||
var (
|
||||
// ErrNoLimit is returned when the memory limit is not set.
|
||||
ErrNoLimit = errors.New("memory is not limited")
|
||||
// ErrNoCgroup is returned when the process is not in cgroup.
|
||||
ErrNoCgroup = errors.New("process is not in cgroup")
|
||||
// ErrCgroupsNotSupported is returned when the system does not support cgroups.
|
||||
ErrCgroupsNotSupported = errors.New("cgroups is not supported on this system")
|
||||
)
|
||||
|
||||
type config struct {
|
||||
logger *log.Logger
|
||||
ratio float64
|
||||
env bool
|
||||
provider Provider
|
||||
}
|
||||
|
||||
|
@ -50,10 +45,10 @@ func WithRatio(ratio float64) Option {
|
|||
// WithEnv configures whether to use environment variables.
|
||||
//
|
||||
// Default: false
|
||||
//
|
||||
// Deprecated: currently this does nothing.
|
||||
func WithEnv() Option {
|
||||
return func(cfg *config) {
|
||||
cfg.env = true
|
||||
}
|
||||
return func(cfg *config) {}
|
||||
}
|
||||
|
||||
// WithProvider configures the provider.
|
||||
|
@ -65,17 +60,24 @@ func WithProvider(provider Provider) Option {
|
|||
}
|
||||
}
|
||||
|
||||
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options.
|
||||
// SetGoMemLimitWithOpts sets GOMEMLIMIT with options and environment variables.
|
||||
//
|
||||
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
|
||||
// through AUTOMEMLIMIT envrironment variable in the half-open range (0.0,1.0].
|
||||
//
|
||||
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
|
||||
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
|
||||
//
|
||||
// If AUTOMEMLIMIT_EXPERIMENT is set, it enables experimental features.
|
||||
// Please see the documentation of Experiments for more details.
|
||||
//
|
||||
// Options:
|
||||
// - WithRatio
|
||||
// - WithEnv (see more SetGoMemLimitWithEnv)
|
||||
// - WithProvider
|
||||
func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
||||
cfg := &config{
|
||||
logger: log.New(io.Discard, "", log.LstdFlags),
|
||||
ratio: defaultAUTOMEMLIMIT,
|
||||
env: false,
|
||||
provider: FromCgroup,
|
||||
}
|
||||
if os.Getenv(envAUTOMEMLIMIT_DEBUG) == "true" {
|
||||
|
@ -90,6 +92,15 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
|||
}
|
||||
}()
|
||||
|
||||
exps, err := parseExperiments()
|
||||
if err != nil {
|
||||
return 0, fmt.Errorf("failed to parse experiments: %w", err)
|
||||
}
|
||||
if exps.System {
|
||||
cfg.logger.Println("system experiment is enabled: using system memory limit as a fallback")
|
||||
cfg.provider = ApplyFallback(cfg.provider, FromSystem)
|
||||
}
|
||||
|
||||
snapshot := debug.SetMemoryLimit(-1)
|
||||
defer func() {
|
||||
err := recover()
|
||||
|
@ -122,6 +133,10 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
|||
|
||||
limit, err := setGoMemLimit(ApplyRatio(cfg.provider, ratio))
|
||||
if err != nil {
|
||||
if errors.Is(err, ErrNoLimit) {
|
||||
cfg.logger.Printf("memory is not limited, skipping: %v\n", err)
|
||||
return 0, nil
|
||||
}
|
||||
return 0, fmt.Errorf("failed to set GOMEMLIMIT: %w", err)
|
||||
}
|
||||
|
||||
|
@ -130,14 +145,8 @@ func SetGoMemLimitWithOpts(opts ...Option) (_ int64, _err error) {
|
|||
return limit, nil
|
||||
}
|
||||
|
||||
// SetGoMemLimitWithEnv sets GOMEMLIMIT with the value from the environment variable.
|
||||
// You can configure how much memory of the cgroup's memory limit to set as GOMEMLIMIT
|
||||
// through AUTOMEMLIMIT in the half-open range (0.0,1.0].
|
||||
//
|
||||
// If AUTOMEMLIMIT is not set, it defaults to 0.9. (10% is the headroom for memory sources the Go runtime is unaware of.)
|
||||
// If GOMEMLIMIT is already set or AUTOMEMLIMIT=off, this function does nothing.
|
||||
func SetGoMemLimitWithEnv() {
|
||||
_, _ = SetGoMemLimitWithOpts(WithEnv())
|
||||
_, _ = SetGoMemLimitWithOpts()
|
||||
}
|
||||
|
||||
// SetGoMemLimit sets GOMEMLIMIT with the value from the cgroup's memory limit and given ratio.
|
||||
|
|
|
@ -0,0 +1,29 @@
|
|||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2017, Jeremy Jay
|
||||
All rights reserved.
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
* Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
* Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
* Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
|
@ -0,0 +1,41 @@
|
|||
# memory
|
||||
|
||||
Package `memory` provides two methods reporting total physical system memory
|
||||
accessible to the kernel, and free memory available to the running application.
|
||||
|
||||
This package has no external dependency besides the standard library and default operating system tools.
|
||||
|
||||
Documentation:
|
||||
[![GoDoc](https://godoc.org/github.com/pbnjay/memory?status.svg)](https://godoc.org/github.com/pbnjay/memory)
|
||||
|
||||
This is useful for dynamic code to minimize thrashing and other contention, similar to the stdlib `runtime.NumCPU`
|
||||
See some history of the proposal at https://github.com/golang/go/issues/21816
|
||||
|
||||
|
||||
## Example
|
||||
|
||||
```go
|
||||
fmt.Printf("Total system memory: %d\n", memory.TotalMemory())
|
||||
fmt.Printf("Free memory: %d\n", memory.FreeMemory())
|
||||
```
|
||||
|
||||
|
||||
## Testing
|
||||
|
||||
Tested/working on:
|
||||
- macOS 10.12.6 (16G29), 10.15.7 (19H2)
|
||||
- Windows 10 1511 (10586.1045)
|
||||
- Linux RHEL (3.10.0-327.3.1.el7.x86_64)
|
||||
- Raspberry Pi 3 (ARMv8) on Raspbian, ODROID-C1+ (ARMv7) on Ubuntu, C.H.I.P
|
||||
(ARMv7).
|
||||
- Amazon Linux 2 aarch64 (m6a.large, 4.14.203-156.332.amzn2.aarch64)
|
||||
|
||||
Tested on virtual machines:
|
||||
- Windows 7 SP1 386
|
||||
- Debian stretch 386
|
||||
- NetBSD 7.1 amd64 + 386
|
||||
- OpenBSD 6.1 amd64 + 386
|
||||
- FreeBSD 11.1 amd64 + 386
|
||||
- DragonFly BSD 4.8.1 amd64
|
||||
|
||||
If you have access to untested systems please test and report success/bugs.
|
|
@ -0,0 +1,24 @@
|
|||
// Package memory provides a single method reporting total system memory
|
||||
// accessible to the kernel.
|
||||
package memory
|
||||
|
||||
// TotalMemory returns the total accessible system memory in bytes.
|
||||
//
|
||||
// The total accessible memory is installed physical memory size minus reserved
|
||||
// areas for the kernel and hardware, if such reservations are reported by
|
||||
// the operating system.
|
||||
//
|
||||
// If accessible memory size could not be determined, then 0 is returned.
|
||||
func TotalMemory() uint64 {
|
||||
return sysTotalMemory()
|
||||
}
|
||||
|
||||
// FreeMemory returns the total free system memory in bytes.
|
||||
//
|
||||
// The total free memory is installed physical memory size minus reserved
|
||||
// areas for other applications running on the same system.
|
||||
//
|
||||
// If free memory size could not be determined, then 0 is returned.
|
||||
func FreeMemory() uint64 {
|
||||
return sysFreeMemory()
|
||||
}
|
|
@ -0,0 +1,19 @@
|
|||
// +build freebsd openbsd dragonfly netbsd
|
||||
|
||||
package memory
|
||||
|
||||
func sysTotalMemory() uint64 {
|
||||
s, err := sysctlUint64("hw.physmem")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func sysFreeMemory() uint64 {
|
||||
s, err := sysctlUint64("hw.usermem")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return s
|
||||
}
|
|
@ -0,0 +1,49 @@
|
|||
// +build darwin
|
||||
|
||||
package memory
|
||||
|
||||
import (
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
func sysTotalMemory() uint64 {
|
||||
s, err := sysctlUint64("hw.memsize")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
return s
|
||||
}
|
||||
|
||||
func sysFreeMemory() uint64 {
|
||||
cmd := exec.Command("vm_stat")
|
||||
outBytes, err := cmd.Output()
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
|
||||
rePageSize := regexp.MustCompile("page size of ([0-9]*) bytes")
|
||||
reFreePages := regexp.MustCompile("Pages free: *([0-9]*)\\.")
|
||||
|
||||
// default: page size of 4096 bytes
|
||||
matches := rePageSize.FindSubmatchIndex(outBytes)
|
||||
pageSize := uint64(4096)
|
||||
if len(matches) == 4 {
|
||||
pageSize, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
|
||||
// ex: Pages free: 1126961.
|
||||
matches = reFreePages.FindSubmatchIndex(outBytes)
|
||||
freePages := uint64(0)
|
||||
if len(matches) == 4 {
|
||||
freePages, err = strconv.ParseUint(string(outBytes[matches[2]:matches[3]]), 10, 64)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
}
|
||||
return freePages * pageSize
|
||||
}
|
|
@ -0,0 +1,29 @@
|
|||
// +build linux
|
||||
|
||||
package memory
|
||||
|
||||
import "syscall"
|
||||
|
||||
func sysTotalMemory() uint64 {
|
||||
in := &syscall.Sysinfo_t{}
|
||||
err := syscall.Sysinfo(in)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// If this is a 32-bit system, then these fields are
|
||||
// uint32 instead of uint64.
|
||||
// So we always convert to uint64 to match signature.
|
||||
return uint64(in.Totalram) * uint64(in.Unit)
|
||||
}
|
||||
|
||||
func sysFreeMemory() uint64 {
|
||||
in := &syscall.Sysinfo_t{}
|
||||
err := syscall.Sysinfo(in)
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// If this is a 32-bit system, then these fields are
|
||||
// uint32 instead of uint64.
|
||||
// So we always convert to uint64 to match signature.
|
||||
return uint64(in.Freeram) * uint64(in.Unit)
|
||||
}
|
|
@ -0,0 +1,60 @@
|
|||
// +build windows
|
||||
|
||||
package memory
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
// omitting a few fields for brevity...
|
||||
// https://msdn.microsoft.com/en-us/library/windows/desktop/aa366589(v=vs.85).aspx
|
||||
type memStatusEx struct {
|
||||
dwLength uint32
|
||||
dwMemoryLoad uint32
|
||||
ullTotalPhys uint64
|
||||
ullAvailPhys uint64
|
||||
unused [5]uint64
|
||||
}
|
||||
|
||||
func sysTotalMemory() uint64 {
|
||||
kernel32, err := syscall.LoadDLL("kernel32.dll")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// GetPhysicallyInstalledSystemMemory is simpler, but broken on
|
||||
// older versions of windows (and uses this under the hood anyway).
|
||||
globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
msx := &memStatusEx{
|
||||
dwLength: 64,
|
||||
}
|
||||
r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
|
||||
if r == 0 {
|
||||
return 0
|
||||
}
|
||||
return msx.ullTotalPhys
|
||||
}
|
||||
|
||||
func sysFreeMemory() uint64 {
|
||||
kernel32, err := syscall.LoadDLL("kernel32.dll")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
// GetPhysicallyInstalledSystemMemory is simpler, but broken on
|
||||
// older versions of windows (and uses this under the hood anyway).
|
||||
globalMemoryStatusEx, err := kernel32.FindProc("GlobalMemoryStatusEx")
|
||||
if err != nil {
|
||||
return 0
|
||||
}
|
||||
msx := &memStatusEx{
|
||||
dwLength: 64,
|
||||
}
|
||||
r, _, _ := globalMemoryStatusEx.Call(uintptr(unsafe.Pointer(msx)))
|
||||
if r == 0 {
|
||||
return 0
|
||||
}
|
||||
return msx.ullAvailPhys
|
||||
}
|
|
@ -0,0 +1,21 @@
|
|||
// +build darwin freebsd openbsd dragonfly netbsd
|
||||
|
||||
package memory
|
||||
|
||||
import (
|
||||
"syscall"
|
||||
"unsafe"
|
||||
)
|
||||
|
||||
func sysctlUint64(name string) (uint64, error) {
|
||||
s, err := syscall.Sysctl(name)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
// hack because the string conversion above drops a \0
|
||||
b := []byte(s)
|
||||
if len(b) < 8 {
|
||||
b = append(b, 0)
|
||||
}
|
||||
return *(*uint64)(unsafe.Pointer(&b[0])), nil
|
||||
}
|
|
@ -0,0 +1,10 @@
|
|||
// +build !linux,!darwin,!windows,!freebsd,!dragonfly,!netbsd,!openbsd
|
||||
|
||||
package memory
|
||||
|
||||
func sysTotalMemory() uint64 {
|
||||
return 0
|
||||
}
|
||||
func sysFreeMemory() uint64 {
|
||||
return 0
|
||||
}
|
|
@ -68,7 +68,7 @@ codeberg.org/superseriousbusiness/exif-terminator
|
|||
# github.com/DmitriyVTitov/size v1.5.0
|
||||
## explicit; go 1.14
|
||||
github.com/DmitriyVTitov/size
|
||||
# github.com/KimMachineGun/automemlimit v0.4.0
|
||||
# github.com/KimMachineGun/automemlimit v0.5.0
|
||||
## explicit; go 1.19
|
||||
github.com/KimMachineGun/automemlimit
|
||||
github.com/KimMachineGun/automemlimit/memlimit
|
||||
|
@ -418,6 +418,9 @@ github.com/oklog/ulid
|
|||
# github.com/opencontainers/runtime-spec v1.0.2
|
||||
## explicit
|
||||
github.com/opencontainers/runtime-spec/specs-go
|
||||
# github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58
|
||||
## explicit; go 1.16
|
||||
github.com/pbnjay/memory
|
||||
# github.com/pelletier/go-toml/v2 v2.1.1
|
||||
## explicit; go 1.16
|
||||
github.com/pelletier/go-toml/v2
|
||||
|
|
Loading…
Reference in New Issue