Skip to content
quarry Artifact [e6a231d70d]
Login

Artifact [e6a231d70d]

Artifact e6a231d70d771c4802e2556e6ed23e5eb4e545d5ea2ba01e5f8491ae5e13d4e5:


// Package sqlmem returns a SQLite connection's memory to the system
// when it has grown.
//
// A connection of the ncruces driver owns a region of memory that grows
// to the high-water mark of the heaviest query it has run and never
// shrinks: the region is unmapped only when the connection closes.
// database/sql hands a returned connection straight back out, so one
// heavy query (a big sort, a wide scan) leaves its pool holding that
// much for as long as the connection lives. Across a forge's stores
// that was two to three gigabytes at rest, most of it in a few dozen
// connections, and it rose for hours as more of them met a heavy query.
//
// Lean wraps a connection so the pool retires it, when it is returned,
// once its region has passed a limit. The next caller gets a new, small
// connection; the grown one is closed and its memory unmapped at once.
package sqlmem

import (
	"context"
	"database/sql/driver"
	"reflect"
	"sync/atomic"

	"github.com/ncruces/go-sqlite3"
)

// conn is every interface the driver's connection implements and
// database/sql looks for. A wrapper that dropped one would silently
// change how statements run, so they are all named and forwarded.
type conn interface {
	driver.Conn
	driver.ConnBeginTx
	driver.ConnPrepareContext
	driver.ExecerContext
	driver.NamedValueChecker
	Raw() *sqlite3.Conn
}

// lean is a connection the pool drops once it has grown past limit, or
// once the driver has panicked inside it (panic.go).
type lean struct {
	conn
	limit   int64
	retired *atomic.Int64
	dead    atomic.Pointer[PanicError]
	inTx    atomic.Bool
}

// IsValid is asked by database/sql when a connection is returned to its
// pool and again before it is reused. False closes the connection.
func (l *lean) IsValid() bool {
	if l.dead.Load() != nil {
		return false
	}
	n, ok := Committed(l.Raw())
	if !ok || l.limit <= 0 || n <= l.limit {
		return true
	}
	if l.retired != nil {
		l.retired.Add(1)
	}
	retiredN.Add(1)
	retiredBytes.Add(n)
	return false
}

var retiredN, retiredBytes atomic.Int64

// Retired is how many connections this process has retired for their
// size, and how many bytes they held between them: what the pools would
// otherwise have kept until those connections aged out.
func Retired() (n, bytes int64) { return retiredN.Load(), retiredBytes.Load() }

// Lean wraps c so its pool retires it once its memory passes limit
// bytes (a limit of zero or less never retires for size), and so a panic
// from the driver inside it is an error and the connection is dropped
// (panic.go). retired, when not nil, counts the connections retired for
// size. A connection that is not the driver's own is returned as it came.
func Lean(c driver.Conn, limit int64, retired *atomic.Int64) driver.Conn {
	full, ok := c.(conn)
	if !ok {
		return c
	}
	return &lean{conn: full, limit: limit, retired: retired}
}

// Connector opens connections through inner and wraps each with Lean.
// limit is read at each open, so a test can change it.
type Connector struct {
	Inner   driver.Connector
	Limit   *int64
	Retired *atomic.Int64
}

func (c Connector) Connect(ctx context.Context) (driver.Conn, error) {
	dc, err := c.Inner.Connect(ctx)
	if err != nil {
		return nil, err
	}
	return Lean(dc, *c.Limit, c.Retired), nil
}

func (c Connector) Driver() driver.Driver { return c.Inner.Driver() }

// Committed is how many bytes of memory c's region holds: what closing
// the connection would give back. The driver keeps that in a field it
// does not export, so it is read by reflection, and ok is false when
// the driver's layout is not the one this was written against. A caller
// then keeps the connection, which is the behaviour before this
// package; TestCommittedReadsTheDriver fails when that happens, so a
// driver upgrade that moves the field is noticed.
func Committed(c *sqlite3.Conn) (n int64, ok bool) {
	defer func() {
		if recover() != nil {
			n, ok = 0, false
		}
	}()
	if c == nil {
		return 0, false
	}
	w := reflect.ValueOf(c).Elem().FieldByName("wrp")
	if !w.IsValid() || w.Kind() != reflect.Pointer || w.IsNil() {
		return 0, false
	}
	m := w.Elem().FieldByName("Memory")
	if !m.IsValid() || m.Kind() != reflect.Pointer || m.IsNil() {
		return 0, false
	}
	// On unix the region is committed in steps ahead of what is in use;
	// committed is what the process holds. Elsewhere Buf's length is it.
	if f := m.Elem().FieldByName("committed"); f.IsValid() && f.CanInt() {
		return f.Int(), true
	}
	if f := m.Elem().FieldByName("Buf"); f.IsValid() && f.Kind() == reflect.Slice {
		return int64(f.Len()), true
	}
	return 0, false
}