mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

bump go-structr to v0.8.5 to improve memory usage (#2955)

This commit is contained in:
kim 2024-06-03 11:03:35 +00:00 committed by GitHub
parent addaba05c0
commit 6ed6824d5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 45 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -21,7 +21,7 @@ require (
codeberg.org/gruf/go-runners v1.6.2 codeberg.org/gruf/go-runners v1.6.2
codeberg.org/gruf/go-sched v1.2.3 codeberg.org/gruf/go-sched v1.2.3
codeberg.org/gruf/go-storage v0.1.1 codeberg.org/gruf/go-storage v0.1.1
codeberg.org/gruf/go-structr v0.8.4 codeberg.org/gruf/go-structr v0.8.5
codeberg.org/superseriousbusiness/exif-terminator v0.7.0 codeberg.org/superseriousbusiness/exif-terminator v0.7.0
github.com/DmitriyVTitov/size v1.5.0 github.com/DmitriyVTitov/size v1.5.0
github.com/KimMachineGun/automemlimit v0.6.1 github.com/KimMachineGun/automemlimit v0.6.1

4
go.sum
View File

@ -76,8 +76,8 @@ codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk
codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A= codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
codeberg.org/gruf/go-storage v0.1.1 h1:CSX1PMMg/7vqqK8aCFtq94xCrOB3xhj7eWIvzILdLpY= codeberg.org/gruf/go-storage v0.1.1 h1:CSX1PMMg/7vqqK8aCFtq94xCrOB3xhj7eWIvzILdLpY=
codeberg.org/gruf/go-storage v0.1.1/go.mod h1:145IWMUOc6YpIiZIiCIEwkkNZZPiSbwMnZxRjSc5q6c= codeberg.org/gruf/go-storage v0.1.1/go.mod h1:145IWMUOc6YpIiZIiCIEwkkNZZPiSbwMnZxRjSc5q6c=
codeberg.org/gruf/go-structr v0.8.4 h1:2eT1VOTWG6T9gIGZwF/1Jop6k6plvfdUY5yBcvbizVg= codeberg.org/gruf/go-structr v0.8.5 h1:WQuvLSQFyFwMjdU7dCWvgcjuhk07oWdSl9guShekzGQ=
codeberg.org/gruf/go-structr v0.8.4/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA= codeberg.org/gruf/go-structr v0.8.5/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0 h1:Y6VApSXhKqExG0H2hZ2JelRK4xmWdjDQjn13CpEfzko= codeberg.org/superseriousbusiness/exif-terminator v0.7.0 h1:Y6VApSXhKqExG0H2hZ2JelRK4xmWdjDQjn13CpEfzko=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE= codeberg.org/superseriousbusiness/exif-terminator v0.7.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

14
vendor/codeberg.org/gruf/go-structr/cache.go generated vendored
View File

@ -159,7 +159,7 @@ func (c *Cache[T]) Get(index *Index, keys ...Key) []T {
for i := range keys { for i := range keys {
// Concatenate all *values* from cached items. // Concatenate all *values* from cached items.
index.get(keys[i], func(item *indexed_item) { index.get(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok { if value, ok := item.data.(T); ok {
// Append value COPY. // Append value COPY.
value = c.copy(value) value = c.copy(value)
@ -344,7 +344,7 @@ func (c *Cache[T]) Load(index *Index, keys []Key, load func([]Key) ([]T, error))
before := len(values) before := len(values)
// Concatenate all *values* from cached items. // Concatenate all *values* from cached items.
index.get(keys[i], func(item *indexed_item) { index.get(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok { if value, ok := item.data.(T); ok {
// Append value COPY. // Append value COPY.
value = c.copy(value) value = c.copy(value)
@ -446,10 +446,10 @@ func (c *Cache[T]) Invalidate(index *Index, keys ...Key) {
// Preallocate expected ret slice. // Preallocate expected ret slice.
values := make([]T, 0, len(keys)) values := make([]T, 0, len(keys))
for _, key := range keys { for i := range keys {
// Delete all items under key from index, collecting // Delete all items under key from index, collecting
// value items and dropping them from all their indices. // value items and dropping them from all their indices.
index.delete(key, func(item *indexed_item) { index.delete(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok { if value, ok := item.data.(T); ok {
// No need to copy, as item // No need to copy, as item
@ -569,7 +569,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) {
if index != nil { if index != nil {
// Append item to index. // Append item to index.
index.append(key, item) index.append(key.key, item)
} }
// Get ptr to value data. // Get ptr to value data.
@ -596,7 +596,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) {
// Calculate index key. // Calculate index key.
key := idx.key(buf, parts) key := idx.key(buf, parts)
if key.Zero() { if key == "" {
continue continue
} }
@ -639,7 +639,7 @@ func (c *Cache[T]) store_error(index *Index, key Key, err error) {
item.data = err item.data = err
// Append item to index. // Append item to index.
index.append(key, item) index.append(key.key, item)
// Add item to main lru list. // Add item to main lru list.
c.lru.push_front(&item.elem) c.lru.push_front(&item.elem)

74
vendor/codeberg.org/gruf/go-structr/index.go generated vendored
View File

@ -96,7 +96,10 @@ func (i *Index) Key(parts ...any) Key {
buf := new_buffer() buf := new_buffer()
key := i.key(buf, parts) key := i.key(buf, parts)
free_buffer(buf) free_buffer(buf)
return key return Key{
raw: parts,
key: key,
}
} }
// Keys generates []Key{} from given (multiple) parts // Keys generates []Key{} from given (multiple) parts
@ -107,10 +110,13 @@ func (i *Index) Keys(parts ...[]any) []Key {
buf := new_buffer() buf := new_buffer()
for _, parts := range parts { for _, parts := range parts {
key := i.key(buf, parts) key := i.key(buf, parts)
if key.Zero() { if key == "" {
continue continue
} }
keys = append(keys, key) keys = append(keys, Key{
raw: parts,
key: key,
})
} }
free_buffer(buf) free_buffer(buf)
return keys return keys
@ -169,34 +175,21 @@ func (i *Index) get_one(key Key) *indexed_item {
// Extract entry from first list elem. // Extract entry from first list elem.
entry := (*index_entry)(l.head.data) entry := (*index_entry)(l.head.data)
// Check contains expected key.
if !entry.key.Equal(key) {
return nil
}
return entry.item return entry.item
} }
// get will fetch all indexed items under key, passing each to hook. // get will fetch all indexed items under key, passing each to hook.
func (i *Index) get(key Key, hook func(*indexed_item)) { func (i *Index) get(key string, hook func(*indexed_item)) {
if hook == nil { if hook == nil {
panic("nil hook") panic("nil hook")
} }
// Get list at hash. // Get list at hash.
l, _ := i.data.Get(key.key) l, _ := i.data.Get(key)
if l == nil { if l == nil {
return return
} }
// Extract entry from first list elem.
entry := (*index_entry)(l.head.data)
// Check contains expected key.
if !entry.key.Equal(key) {
return
}
// Iterate all entries in list. // Iterate all entries in list.
l.rangefn(func(elem *list_elem) { l.rangefn(func(elem *list_elem) {
@ -210,7 +203,7 @@ func (i *Index) get(key Key, hook func(*indexed_item)) {
} }
// key uses hasher to generate Key{} from given raw parts. // key uses hasher to generate Key{} from given raw parts.
func (i *Index) key(buf *byteutil.Buffer, parts []any) Key { func (i *Index) key(buf *byteutil.Buffer, parts []any) string {
if len(parts) != len(i.fields) { if len(parts) != len(i.fields) {
panicf("incorrect number key parts: want=%d received=%d", panicf("incorrect number key parts: want=%d received=%d",
len(i.fields), len(i.fields),
@ -223,7 +216,7 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key {
before := len(buf.B) before := len(buf.B)
buf.B = field.mangle(buf.B, parts[x]) buf.B = field.mangle(buf.B, parts[x])
if string(buf.B[before:]) == field.zerostr { if string(buf.B[before:]) == field.zerostr {
return Key{} return ""
} }
buf.B = append(buf.B, '.') buf.B = append(buf.B, '.')
} }
@ -233,24 +226,21 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key {
buf.B = append(buf.B, '.') buf.B = append(buf.B, '.')
} }
} }
return Key{ return string(buf.B)
raw: parts,
key: string(buf.B),
}
} }
// append will append the given index entry to appropriate // append will append the given index entry to appropriate
// doubly-linked-list in index hashmap. this handles case // doubly-linked-list in index hashmap. this handles case
// of key collisions and overwriting 'unique' entries. // of key collisions and overwriting 'unique' entries.
func (i *Index) append(key Key, item *indexed_item) { func (i *Index) append(key string, item *indexed_item) {
// Look for existing. // Look for existing.
l, _ := i.data.Get(key.key) l, _ := i.data.Get(key)
if l == nil { if l == nil {
// Allocate new. // Allocate new.
l = new_list() l = new_list()
i.data.Put(key.key, l) i.data.Put(key, l)
} else if is_unique(i.flags) { } else if is_unique(i.flags) {
@ -280,27 +270,19 @@ func (i *Index) append(key Key, item *indexed_item) {
} }
// delete will remove all indexed items under key, passing each to hook. // delete will remove all indexed items under key, passing each to hook.
func (i *Index) delete(key Key, hook func(*indexed_item)) { func (i *Index) delete(key string, hook func(*indexed_item)) {
if hook == nil { if hook == nil {
panic("nil hook") panic("nil hook")
} }
// Get list at hash. // Get list at hash.
l, _ := i.data.Get(key.key) l, _ := i.data.Get(key)
if l == nil { if l == nil {
return return
} }
// Extract entry from first list elem. // Delete at hash.
entry := (*index_entry)(l.head.data) i.data.Delete(key)
// Check contains expected key.
if !entry.key.Equal(key) {
return
}
// Delete data at hash.
i.data.Delete(key.key)
// Iterate entries in list. // Iterate entries in list.
for x := 0; x < l.len; x++ { for x := 0; x < l.len; x++ {
@ -330,7 +312,7 @@ func (i *Index) delete(key Key, hook func(*indexed_item)) {
// delete_entry deletes the given index entry. // delete_entry deletes the given index entry.
func (i *Index) delete_entry(entry *index_entry) { func (i *Index) delete_entry(entry *index_entry) {
// Get list at hash sum. // Get list at hash sum.
l, _ := i.data.Get(entry.key.key) l, _ := i.data.Get(entry.key)
if l == nil { if l == nil {
return return
} }
@ -339,8 +321,8 @@ func (i *Index) delete_entry(entry *index_entry) {
l.remove(&entry.elem) l.remove(&entry.elem)
if l.len == 0 { if l.len == 0 {
// Remove entry list from map. // Remove entry from map.
i.data.Delete(entry.key.key) i.data.Delete(entry.key)
// Release list. // Release list.
free_list(l) free_list(l)
@ -387,9 +369,9 @@ type index_entry struct {
// elem.data is ptr to index_entry. // elem.data is ptr to index_entry.
elem list_elem elem list_elem
// hash checksum // raw cache key
// + raw key data // for this entry.
key Key key string
// index this is stored in. // index this is stored in.
index *Index index *Index
@ -415,7 +397,7 @@ func new_index_entry() *index_entry {
// free_index_entry releases the index_entry. // free_index_entry releases the index_entry.
func free_index_entry(entry *index_entry) { func free_index_entry(entry *index_entry) {
entry.elem.data = nil entry.elem.data = nil
entry.key = Key{} entry.key = ""
entry.index = nil entry.index = nil
entry.item = nil entry.item = nil
index_entry_pool.Put(entry) index_entry_pool.Put(entry)

4
vendor/codeberg.org/gruf/go-structr/key.go generated vendored
View File

@ -22,7 +22,7 @@ func (k Key) Key() string {
// Equal returns whether keys are equal. // Equal returns whether keys are equal.
func (k Key) Equal(o Key) bool { func (k Key) Equal(o Key) bool {
return k.key == o.key return (k.key == o.key)
} }
// Value returns the raw slice of // Value returns the raw slice of
@ -33,7 +33,7 @@ func (k Key) Values() []any {
// Zero indicates a zero value key. // Zero indicates a zero value key.
func (k Key) Zero() bool { func (k Key) Zero() bool {
return k.raw == nil return (k.raw == nil)
} }
var buf_pool sync.Pool var buf_pool sync.Pool

8
vendor/codeberg.org/gruf/go-structr/queue.go generated vendored
View File

@ -127,7 +127,7 @@ func (q *Queue[T]) Pop(index *Index, keys ...Key) []T {
for i := range keys { for i := range keys {
// Delete all items under key from index, collecting // Delete all items under key from index, collecting
// value items and dropping them from all their indices. // value items and dropping them from all their indices.
index.delete(keys[i], func(item *indexed_item) { index.delete(keys[i].key, func(item *indexed_item) {
// Append deleted to values. // Append deleted to values.
value := item.data.(T) value := item.data.(T)
@ -179,7 +179,7 @@ func (q *Queue[T]) PushBack(values ...T) {
func (q *Queue[T]) MoveFront(index *Index, keys ...Key) { func (q *Queue[T]) MoveFront(index *Index, keys ...Key) {
q.mutex.Lock() q.mutex.Lock()
for i := range keys { for i := range keys {
index.get(keys[i], func(item *indexed_item) { index.get(keys[i].key, func(item *indexed_item) {
q.queue.move_front(&item.elem) q.queue.move_front(&item.elem)
}) })
} }
@ -190,7 +190,7 @@ func (q *Queue[T]) MoveFront(index *Index, keys ...Key) {
func (q *Queue[T]) MoveBack(index *Index, keys ...Key) { func (q *Queue[T]) MoveBack(index *Index, keys ...Key) {
q.mutex.Lock() q.mutex.Lock()
for i := range keys { for i := range keys {
index.get(keys[i], func(item *indexed_item) { index.get(keys[i].key, func(item *indexed_item) {
q.queue.move_back(&item.elem) q.queue.move_back(&item.elem)
}) })
} }
@ -305,7 +305,7 @@ func (q *Queue[T]) index(value T) *indexed_item {
// Calculate index key. // Calculate index key.
key := idx.key(buf, parts) key := idx.key(buf, parts)
if key.Zero() { if key == "" {
continue continue
} }

2
vendor/modules.txt vendored
View File

@ -68,7 +68,7 @@ codeberg.org/gruf/go-storage/disk
codeberg.org/gruf/go-storage/internal codeberg.org/gruf/go-storage/internal
codeberg.org/gruf/go-storage/memory codeberg.org/gruf/go-storage/memory
codeberg.org/gruf/go-storage/s3 codeberg.org/gruf/go-storage/s3
# codeberg.org/gruf/go-structr v0.8.4 # codeberg.org/gruf/go-structr v0.8.5
## explicit; go 1.21 ## explicit; go 1.21
codeberg.org/gruf/go-structr codeberg.org/gruf/go-structr
# codeberg.org/superseriousbusiness/exif-terminator v0.7.0 # codeberg.org/superseriousbusiness/exif-terminator v0.7.0