package index
import (
"context"
"database/sql"
sqldriver "database/sql/driver"
"time"
"github.com/ncruces/go-sqlite3"
"github.com/ncruces/go-sqlite3/driver"
"github.com/ncruces/go-sqlite3/ext/fts5"
)
// writerMaxMemory is how much memory one writer connection may use. The
// driver's own ceiling is 256MB per connection (go-sqlite3 wrap.go),
// and a store of millions of artifacts went past it: netbsd-src (2.5M
// artifacts) ran SQLite out of memory folding and opening its index
// (#357). Memory grows only as a connection uses it, so this is a
// ceiling, not a reservation; wasm32 cannot address more than 4GB.
var writerMaxMemory int64 = 1 << 30
// Reader pool settings. Vars so a test can read them back and shrink
// them before Open.
//
// Measured with scripts/mem-bench.sh (28 stores, 30k-check-in largest)
// and a one-pool probe running eight heavy sorts and scans:
//
// cache_size 16 MiB -> 4 MiB took the probe's resident growth from
// 38 MB to 18 MB at the same speed (2.5s v 2.7s); the
// bench's route p95s stayed inside run-to-run noise.
// Kept. The writer gets 8 MiB: bulk folds reuse pages.
// temp_store FILE cut the probe's growth to 13 MB but took it from
// 2.6s to 11.5s. Left MEMORY.
// max conns 4 changed neither memory nor latency. Left unbounded (0):
// an out-of-memory panic leaves its connection checked out,
// and a bounded pool would lose a slot to each one.
// ceiling 256 MiB changed neither; a query past a ceiling panics
// "sqlite3: out of memory", which the web handler logs with
// its stack as a 500, so a ceiling with no evidence of a
// safe value would trade memory for failed pages. Left 1 GiB.
var (
readerMaxMemory int64 = 1 << 30
readerCacheKiB = 4000
readerTempStore = "MEMORY"
readerMaxConns = 0 // 0 = database/sql's default: unbounded
writerCacheKiB = 8000
)
// A connection's memory is mapped on demand, grows to the high-water
// mark of the heaviest query it ever ran, and is returned only when the
// connection is closed (go-sqlite3 Wrapper.Close). database/sql never
// closes an idle connection by default, so a forge holding ~58 stores
// (a writer and a reader pool each) ratcheted resident memory about
// 7MB a minute, 2.5GB at start to 10.9GB at 16 hours, and the kernel
// killed it. These three bound how long a connection can keep what it
// grew: idle ones go after connMaxIdleTime, busy ones are replaced
// after connMaxLifetime (database/sql closes them when returned to the
// pool, never inside a transaction), and one idle connection is all a
// pool keeps. Vars so a test can shrink them before Open.
var (
connMaxIdleTime = 2 * time.Minute
connMaxLifetime = 15 * time.Minute
connMaxIdle = 1
)
// openDB opens an index handle whose every connection carries
// connMaxMemory and the fts5 module, and whose pool recycles its
// connections. Every pool in this package comes through here, so none
// can be opened without the recycling policy; and because a recycled
// connection is a new one, everything a connection needs is set up in
// the DSN or in Connect below, never once on the *sql.DB. The limit travels on the context
// the connection is opened with, which database/sql supplies, so the
// driver's connector is wrapped rather than configured.
func openDB(dsn string, maxMem *int64) (*sql.DB, error) {
c, err := (&driver.SQLite{}).OpenConnector(dsn)
if err != nil {
return nil, err
}
db := sql.OpenDB(memConnector{c, maxMem})
db.SetConnMaxIdleTime(connMaxIdleTime)
db.SetConnMaxLifetime(connMaxLifetime)
db.SetMaxIdleConns(connMaxIdle)
return db, nil
}
type memConnector struct {
inner sqldriver.Connector
maxMem *int64
}
func (m memConnector) Connect(ctx context.Context) (sqldriver.Conn, error) {
c, err := m.inner.Connect(sqlite3.WithMaxMemory(ctx, *m.maxMem))
if err != nil {
return nil, err
}
if err := fts5.Register(c.(interface{ Raw() *sqlite3.Conn }).Raw()); err != nil {
c.Close()
return nil, err
}
return c, nil
}
func (m memConnector) Driver() sqldriver.Driver { return m.inner.Driver() }