mirrors
/
gotosocial
mirror of
1
Fork 0
Code Issues Releases Activity

bump go-structr to v0.8.5 to improve memory usage (#2955)

This commit is contained in:
kim 2024-06-03 11:03:35 +00:00 committed by GitHub
parent addaba05c0
commit 6ed6824d5d
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
7 changed files with 45 additions and 63 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -21,7 +21,7 @@ require (
codeberg.org/gruf/go-runners v1.6.2
codeberg.org/gruf/go-sched v1.2.3
codeberg.org/gruf/go-storage v0.1.1
codeberg.org/gruf/go-structr v0.8.4
codeberg.org/gruf/go-structr v0.8.5
codeberg.org/superseriousbusiness/exif-terminator v0.7.0
github.com/DmitriyVTitov/size v1.5.0
github.com/KimMachineGun/automemlimit v0.6.1

4
go.sum
View File

@ -76,8 +76,8 @@ codeberg.org/gruf/go-sched v1.2.3 h1:H5ViDxxzOBR3uIyGBCf0eH8b1L8wMybOXcdtUUTXZHk
codeberg.org/gruf/go-sched v1.2.3/go.mod h1:vT9uB6KWFIIwnG9vcPY2a0alYNoqdL1mSzRM8I+PK7A=
codeberg.org/gruf/go-storage v0.1.1 h1:CSX1PMMg/7vqqK8aCFtq94xCrOB3xhj7eWIvzILdLpY=
codeberg.org/gruf/go-storage v0.1.1/go.mod h1:145IWMUOc6YpIiZIiCIEwkkNZZPiSbwMnZxRjSc5q6c=
codeberg.org/gruf/go-structr v0.8.4 h1:2eT1VOTWG6T9gIGZwF/1Jop6k6plvfdUY5yBcvbizVg=
codeberg.org/gruf/go-structr v0.8.4/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA=
codeberg.org/gruf/go-structr v0.8.5 h1:WQuvLSQFyFwMjdU7dCWvgcjuhk07oWdSl9guShekzGQ=
codeberg.org/gruf/go-structr v0.8.5/go.mod h1:c5UvVDSA3lZ1kv05V+7pXkO8u8Jea+VRWFDRFBCOxSA=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0 h1:Y6VApSXhKqExG0H2hZ2JelRK4xmWdjDQjn13CpEfzko=
codeberg.org/superseriousbusiness/exif-terminator v0.7.0/go.mod h1:gCWKduudUWFzsnixoMzu0FYVdxHWG+AbXnZ50DqxsUE=
dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=

14
vendor/codeberg.org/gruf/go-structr/cache.go generated vendored
View File

@ -159,7 +159,7 @@ func (c *Cache[T]) Get(index *Index, keys ...Key) []T {
for i := range keys {
// Concatenate all *values* from cached items.
index.get(keys[i], func(item *indexed_item) {
index.get(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok {
// Append value COPY.
value = c.copy(value)
@ -344,7 +344,7 @@ func (c *Cache[T]) Load(index *Index, keys []Key, load func([]Key) ([]T, error))
before := len(values)
// Concatenate all *values* from cached items.
index.get(keys[i], func(item *indexed_item) {
index.get(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok {
// Append value COPY.
value = c.copy(value)
@ -446,10 +446,10 @@ func (c *Cache[T]) Invalidate(index *Index, keys ...Key) {
// Preallocate expected ret slice.
values := make([]T, 0, len(keys))
for _, key := range keys {
for i := range keys {
// Delete all items under key from index, collecting
// value items and dropping them from all their indices.
index.delete(key, func(item *indexed_item) {
index.delete(keys[i].key, func(item *indexed_item) {
if value, ok := item.data.(T); ok {
// No need to copy, as item
@ -569,7 +569,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) {
if index != nil {
// Append item to index.
index.append(key, item)
index.append(key.key, item)
}
// Get ptr to value data.
@ -596,7 +596,7 @@ func (c *Cache[T]) store_value(index *Index, key Key, value T) {
// Calculate index key.
key := idx.key(buf, parts)
if key.Zero() {
if key == "" {
continue
}
@ -639,7 +639,7 @@ func (c *Cache[T]) store_error(index *Index, key Key, err error) {
item.data = err
// Append item to index.
index.append(key, item)
index.append(key.key, item)
// Add item to main lru list.
c.lru.push_front(&item.elem)

74
vendor/codeberg.org/gruf/go-structr/index.go generated vendored
View File

@ -96,7 +96,10 @@ func (i *Index) Key(parts ...any) Key {
buf := new_buffer()
key := i.key(buf, parts)
free_buffer(buf)
return key
return Key{
raw: parts,
key: key,
}
}
// Keys generates []Key{} from given (multiple) parts
@ -107,10 +110,13 @@ func (i *Index) Keys(parts ...[]any) []Key {
buf := new_buffer()
for _, parts := range parts {
key := i.key(buf, parts)
if key.Zero() {
if key == "" {
continue
}
keys = append(keys, key)
keys = append(keys, Key{
raw: parts,
key: key,
})
}
free_buffer(buf)
return keys
@ -169,34 +175,21 @@ func (i *Index) get_one(key Key) *indexed_item {
// Extract entry from first list elem.
entry := (*index_entry)(l.head.data)
// Check contains expected key.
if !entry.key.Equal(key) {
return nil
}
return entry.item
}
// get will fetch all indexed items under key, passing each to hook.
func (i *Index) get(key Key, hook func(*indexed_item)) {
func (i *Index) get(key string, hook func(*indexed_item)) {
if hook == nil {
panic("nil hook")
}
// Get list at hash.
l, _ := i.data.Get(key.key)
l, _ := i.data.Get(key)
if l == nil {
return
}
// Extract entry from first list elem.
entry := (*index_entry)(l.head.data)
// Check contains expected key.
if !entry.key.Equal(key) {
return
}
// Iterate all entries in list.
l.rangefn(func(elem *list_elem) {
@ -210,7 +203,7 @@ func (i *Index) get(key Key, hook func(*indexed_item)) {
}
// key uses hasher to generate Key{} from given raw parts.
func (i *Index) key(buf *byteutil.Buffer, parts []any) Key {
func (i *Index) key(buf *byteutil.Buffer, parts []any) string {
if len(parts) != len(i.fields) {
panicf("incorrect number key parts: want=%d received=%d",
len(i.fields),
@ -223,7 +216,7 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key {
before := len(buf.B)
buf.B = field.mangle(buf.B, parts[x])
if string(buf.B[before:]) == field.zerostr {
return Key{}
return ""
}
buf.B = append(buf.B, '.')
}
@ -233,24 +226,21 @@ func (i *Index) key(buf *byteutil.Buffer, parts []any) Key {
buf.B = append(buf.B, '.')
}
}
return Key{
raw: parts,
key: string(buf.B),
}
return string(buf.B)
}
// append will append the given index entry to appropriate
// doubly-linked-list in index hashmap. this handles case
// of key collisions and overwriting 'unique' entries.
func (i *Index) append(key Key, item *indexed_item) {
func (i *Index) append(key string, item *indexed_item) {
// Look for existing.
l, _ := i.data.Get(key.key)
l, _ := i.data.Get(key)
if l == nil {
// Allocate new.
l = new_list()
i.data.Put(key.key, l)
i.data.Put(key, l)
} else if is_unique(i.flags) {
@ -280,27 +270,19 @@ func (i *Index) append(key Key, item *indexed_item) {
}
// delete will remove all indexed items under key, passing each to hook.
func (i *Index) delete(key Key, hook func(*indexed_item)) {
func (i *Index) delete(key string, hook func(*indexed_item)) {
if hook == nil {
panic("nil hook")
}
// Get list at hash.
l, _ := i.data.Get(key.key)
l, _ := i.data.Get(key)
if l == nil {
return
}
// Extract entry from first list elem.
entry := (*index_entry)(l.head.data)
// Check contains expected key.
if !entry.key.Equal(key) {
return
}
// Delete data at hash.
i.data.Delete(key.key)
// Delete at hash.
i.data.Delete(key)
// Iterate entries in list.
for x := 0; x < l.len; x++ {
@ -330,7 +312,7 @@ func (i *Index) delete(key Key, hook func(*indexed_item)) {
// delete_entry deletes the given index entry.
func (i *Index) delete_entry(entry *index_entry) {
// Get list at hash sum.
l, _ := i.data.Get(entry.key.key)
l, _ := i.data.Get(entry.key)
if l == nil {
return
}
@ -339,8 +321,8 @@ func (i *Index) delete_entry(entry *index_entry) {
l.remove(&entry.elem)
if l.len == 0 {
// Remove entry list from map.
i.data.Delete(entry.key.key)
// Remove entry from map.
i.data.Delete(entry.key)
// Release list.
free_list(l)
@ -387,9 +369,9 @@ type index_entry struct {
// elem.data is ptr to index_entry.
elem list_elem
// hash checksum
// + raw key data
key Key
// raw cache key
// for this entry.
key string
// index this is stored in.
index *Index
@ -415,7 +397,7 @@ func new_index_entry() *index_entry {
// free_index_entry releases the index_entry.
func free_index_entry(entry *index_entry) {
entry.elem.data = nil
entry.key = Key{}
entry.key = ""
entry.index = nil
entry.item = nil
index_entry_pool.Put(entry)

4
vendor/codeberg.org/gruf/go-structr/key.go generated vendored
View File

@ -22,7 +22,7 @@ func (k Key) Key() string {
// Equal returns whether keys are equal.
func (k Key) Equal(o Key) bool {
return k.key == o.key
return (k.key == o.key)
}
// Value returns the raw slice of
@ -33,7 +33,7 @@ func (k Key) Values() []any {
// Zero indicates a zero value key.
func (k Key) Zero() bool {
return k.raw == nil
return (k.raw == nil)
}
var buf_pool sync.Pool

8
vendor/codeberg.org/gruf/go-structr/queue.go generated vendored
View File

@ -127,7 +127,7 @@ func (q *Queue[T]) Pop(index *Index, keys ...Key) []T {
for i := range keys {
// Delete all items under key from index, collecting
// value items and dropping them from all their indices.
index.delete(keys[i], func(item *indexed_item) {
index.delete(keys[i].key, func(item *indexed_item) {
// Append deleted to values.
value := item.data.(T)
@ -179,7 +179,7 @@ func (q *Queue[T]) PushBack(values ...T) {
func (q *Queue[T]) MoveFront(index *Index, keys ...Key) {
q.mutex.Lock()
for i := range keys {
index.get(keys[i], func(item *indexed_item) {
index.get(keys[i].key, func(item *indexed_item) {
q.queue.move_front(&item.elem)
})
}
@ -190,7 +190,7 @@ func (q *Queue[T]) MoveFront(index *Index, keys ...Key) {
func (q *Queue[T]) MoveBack(index *Index, keys ...Key) {
q.mutex.Lock()
for i := range keys {
index.get(keys[i], func(item *indexed_item) {
index.get(keys[i].key, func(item *indexed_item) {
q.queue.move_back(&item.elem)
})
}
@ -305,7 +305,7 @@ func (q *Queue[T]) index(value T) *indexed_item {
// Calculate index key.
key := idx.key(buf, parts)
if key.Zero() {
if key == "" {
continue
}

2
vendor/modules.txt vendored
View File

@ -68,7 +68,7 @@ codeberg.org/gruf/go-storage/disk
codeberg.org/gruf/go-storage/internal
codeberg.org/gruf/go-storage/memory
codeberg.org/gruf/go-storage/s3
# codeberg.org/gruf/go-structr v0.8.4
# codeberg.org/gruf/go-structr v0.8.5
## explicit; go 1.21
codeberg.org/gruf/go-structr
# codeberg.org/superseriousbusiness/exif-terminator v0.7.0