// Package sqlmem returns a SQLite connection's memory to the system
// when it has grown.
//
// A connection of the ncruces driver owns a region of memory that grows
// to the high-water mark of the heaviest query it has run and never
// shrinks: the region is unmapped only when the connection closes.
// database/sql hands a returned connection straight back out, so one
// heavy query (a big sort, a wide scan) leaves its pool holding that
// much for as long as the connection lives. Across a forge's stores
// that was two to three gigabytes at rest, most of it in a few dozen
// connections, and it rose for hours as more of them met a heavy query.
//
// Lean wraps a connection so the pool retires it, when it is returned,
// once its region has passed a limit. The next caller gets a new, small
// connection; the grown one is closed and its memory unmapped at once.
package sqlmem
import (
"context"
"database/sql/driver"
"reflect"
"sync/atomic"
"github.com/ncruces/go-sqlite3"
)
// conn is every interface the driver's connection implements and
// database/sql looks for. A wrapper that dropped one would silently
// change how statements run, so they are all named and forwarded.
type conn interface {
driver.Conn
driver.ConnBeginTx
driver.ConnPrepareContext
driver.ExecerContext
driver.NamedValueChecker
Raw() *sqlite3.Conn
}
// lean is a connection the pool drops once it has grown past limit.
type lean struct {
conn
limit int64
retired *atomic.Int64
}
// IsValid is asked by database/sql when a connection is returned to its
// pool and again before it is reused. False closes the connection.
func (l lean) IsValid() bool {
n, ok := Committed(l.Raw())
if !ok || n <= l.limit {
return true
}
if l.retired != nil {
l.retired.Add(1)
}
retiredN.Add(1)
retiredBytes.Add(n)
return false
}
var retiredN, retiredBytes atomic.Int64
// Retired is how many connections this process has retired for their
// size, and how many bytes they held between them: what the pools would
// otherwise have kept until those connections aged out.
func Retired() (n, bytes int64) { return retiredN.Load(), retiredBytes.Load() }
// Lean wraps c so its pool retires it once its memory passes limit
// bytes. retired, when not nil, counts the connections retired. A
// connection that is not the driver's own, or a limit of zero or less,
// is returned as it came.
func Lean(c driver.Conn, limit int64, retired *atomic.Int64) driver.Conn {
full, ok := c.(conn)
if !ok || limit <= 0 {
return c
}
return lean{conn: full, limit: limit, retired: retired}
}
// Connector opens connections through inner and wraps each with Lean.
// limit is read at each open, so a test can change it.
type Connector struct {
Inner driver.Connector
Limit *int64
Retired *atomic.Int64
}
func (c Connector) Connect(ctx context.Context) (driver.Conn, error) {
dc, err := c.Inner.Connect(ctx)
if err != nil {
return nil, err
}
return Lean(dc, *c.Limit, c.Retired), nil
}
func (c Connector) Driver() driver.Driver { return c.Inner.Driver() }
// Committed is how many bytes of memory c's region holds: what closing
// the connection would give back. The driver keeps that in a field it
// does not export, so it is read by reflection, and ok is false when
// the driver's layout is not the one this was written against. A caller
// then keeps the connection, which is the behaviour before this
// package; TestCommittedReadsTheDriver fails when that happens, so a
// driver upgrade that moves the field is noticed.
func Committed(c *sqlite3.Conn) (n int64, ok bool) {
defer func() {
if recover() != nil {
n, ok = 0, false
}
}()
if c == nil {
return 0, false
}
w := reflect.ValueOf(c).Elem().FieldByName("wrp")
if !w.IsValid() || w.Kind() != reflect.Pointer || w.IsNil() {
return 0, false
}
m := w.Elem().FieldByName("Memory")
if !m.IsValid() || m.Kind() != reflect.Pointer || m.IsNil() {
return 0, false
}
// On unix the region is committed in steps ahead of what is in use;
// committed is what the process holds. Elsewhere Buf's length is it.
if f := m.Elem().FieldByName("committed"); f.IsValid() && f.CanInt() {
return f.Int(), true
}
if f := m.Elem().FieldByName("Buf"); f.IsValid() && f.Kind() == reflect.Slice {
return int64(f.Len()), true
}
return 0, false
}