APAW/.kilo/skills/sqlite-patterns/SKILL.md

SQLite Patterns Skill

Comprehensive guide to SQLite database patterns and best practices.

Overview

SQLite is a self-contained, serverless, zero-configuration, transactional SQL database engine. This skill covers schema design, querying, performance optimization, and integration patterns for Go applications.

Connection Management

Basic Connection (using modernc.org/sqlite driver)

import (
    "context"
    "database/sql"
    "time"
    
    "github.com/mattn/go-sqlite3" // or modernc.org/sqlite for pure Go
)

// ✅ Good: Connection with proper configuration
func NewSQLiteDB(ds string) (*sql.DB, error) {
    db, err := sql.Open("sqlite3", ds)
    if err != nil {
        return nil, fmt.Errorf("open sqlite: %w", err)
    }
    
    // Important pragmas for performance and safety
    _, err = db.Exec("PRAGMA journal_mode = WAL") // Write-Ahead Logging for better concurrency
    if err != nil {
        return nil, fmt.Errorf("set journal_mode: %w", err)
    }
    
    _, err = db.Exec("PRAGMA synchronous = NORMAL") // Balance between safety and speed
    if err != nil {
        return nil, fmt.Errorf("set synchronous: %w", err)
    }
    
    _, err = db.Exec("PRAGMA foreign_keys = ON") // Enforce foreign key constraints
    if err != nil {
        return nil, fmt.Errorf("set foreign_keys: %w", err)
    }
    
    _, err = db.Exec("PRAGMA busy_timeout = 5000") // Wait 5 seconds for lock
    if err != nil {
        return nil, fmt.Errorf("set busy_timeout: %w", err)
    }
    
    // Verify connection
    if err := db.Ping(); err != nil {
        return nil, fmt.Errorf("ping sqlite: %w", err)
    }
    
    return db, nil
}

// For high concurrency apps, consider connection pooling (though SQLite has limitations)
// SQLite works best with a single connection or limited concurrent writers

Schema Design

Data Types (SQLite uses dynamic typing but affinities matter)

-- ✅ Good: Table with appropriate column affinities
CREATE TABLE users (
    id INTEGER PRIMARY KEY AUTOINCREMENT, -- 64-bit signed integer
    email TEXT NOT NULL UNIQUE,           -- Text affinity
    password_hash TEXT NOT NULL,          -- Text affinity
    first_name TEXT,                      -- Text affinity
    last_name TEXT,                       -- Text affinity
    age INTEGER CHECK (age >= 0),         -- Integer affinity
    created_at DATETIME DEFAULT CURRENT_TIMESTAMP, -- Timestamp
    updated_at DATETIME DEFAULT CURRENT_TIMESTAMP,
    is_active BOOLEAN DEFAULT 1,          -- Stored as INTEGER (0/1)
    metadata TEXT                         -- JSON stored as TEXT
);

-- Indexes for performance
CREATE INDEX idx_users_email ON users(email);
CREATE INDEX idx_users_created_at ON users(created_at);

-- Trigger for automatic updated_at
CREATE TRIGGER update_users_updated_at
AFTER UPDATE ON users
FOR EACH ROW
BEGIN
    UPDATE users SET updated_at = CURRENT_TIMESTAMP WHERE id = NEW.id;
END;

Common Table Patterns

-- ✅ Good: Many-to-many relationship with junction table
CREATE TABLE posts (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    user_id INTEGER NOT NULL REFERENCES users(id) ON DELETE CASCADE,
    title TEXT NOT NULL,
    content TEXT,
    created_at DATETIME DEFAULT CURRENT_TIMESTAMP
);

CREATE TABLE tags (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    name TEXT NOT NULL UNIQUE
);

CREATE TABLE post_tags (
    post_id INTEGER NOT NULL REFERENCES posts(id) ON DELETE CASCADE,
    tag_id INTEGER NOT NULL REFERENCES tags(id) ON DELETE CASCADE,
    PRIMARY KEY (post_id, tag_id)
);

Query Patterns

Basic CRUD

// ✅ Good: Using sql with context and proper error handling
func GetUserByID(ctx context.Context, db *sql.DB, id int64) (*User, error) {
    var user User
    err := db.QueryRowContext(ctx, `
        SELECT id, email, first_name, last_name, age, created_at, updated_at, is_active
        FROM users
        WHERE id = ?
    `, id).Scan(
        &user.ID, &user.Email, &user.FirstName, &user.LastName,
        &user.Age, &user.CreatedAt, &user.UpdatedAt, &user.IsActive,
    )
    if err != nil {
        if errors.Is(err, sql.ErrNoRows) {
            return nil, ErrNotFound
        }
        return nil, fmt.Errorf("get user: %w", err)
    }
    return &user, nil
}

func CreateUser(ctx context.Context, db *sql.DB, user *User) (*User, error) {
    result, err := db.ExecContext(ctx, `
        INSERT INTO users (email, password_hash, first_name, last_name, age)
        VALUES (?, ?, ?, ?, ?)
    `, user.Email, user.PasswordHash, user.FirstName, user.LastName, user.Age)
    if err != nil {
        return nil, fmt.Errorf("create user: %w", err)
    }
    
    id, err := result.LastInsertId()
    if err != nil {
        return nil, fmt.Errorf("get last insert id: %w", err)
    }
    user.ID = id
    
    // Get the created timestamp
    err = db.QueryRowContext(ctx, `
        SELECT created_at, updated_at FROM users WHERE id = ?
    `, id).Scan(&user.CreatedAt, &user.UpdatedAt)
    if err != nil {
        return nil, fmt.Errorf("get timestamps: %w", err)
    }
    
    return user, nil
}

Transactions

// ✅ Good: Using tx with context and proper rollback
func TransferFunds(ctx context.Context, db *sql.DB, fromID, toID int64, amount float64) error {
    tx, err := db.BeginTx(ctx, nil)
    if err != nil {
        return fmt.Errorf("begin transaction: %w", err)
    }
    // Ensure rollback on error
    defer func() {
        if err != nil {
            tx.Rollback()
        }
    }()
    
    // Check sender balance
    var fromBalance float64
    err := tx.QueryRowContext(ctx, `
        SELECT balance FROM accounts WHERE user_id = ? FOR UPDATE
    `, fromID).Scan(&fromBalance)
    if err != nil {
        return fmt.Errorf("get sender balance: %w", err)
    }
    if fromBalance < amount {
        return ErrInsufficientFunds
    }
    
    // Update sender
    _, err = tx.ExecContext(ctx, `
        UPDATE accounts SET balance = balance - ? WHERE user_id = ?
    `, amount, fromID)
    if err != nil {
        return fmt.Errorf("update sender: %w", err)
    }
    
    // Update receiver
    _, err = tx.ExecContext(ctx, `
        UPDATE accounts SET balance = balance + ? WHERE user_id = ?
    `, amount, toID)
    if err != nil {
        return fmt.Errorf("update receiver: %w", err)
    }
    
    // Commit transaction
    if err := tx.Commit(); err != nil {
        return fmt.Errorf("commit transaction: %w", err)
    }
    
    return nil
}

Performance Optimization

Indexing Strategies

-- ✅ Good: Index for WHERE clauses
CREATE INDEX idx_orders_user_id ON orders(user_id);

-- ✅ Good: Composite index for filtering and sorting
CREATE INDEX idx_orders_date_status ON orders(created_at, status);

-- ✅ Good: Covering index (include all needed columns in index)
CREATE INDEX idx_orders_covering ON orders(status, created_at) 
INCLUDE (user_id, total_amount);

-- ✅ Good: Partial index for infrequent values
CREATE INDEX idx_orders_pending ON orders(id) 
WHERE status = 'pending';

Query Optimization

-- ✅ Good: Using EXPLAIN QUERY PLAN
EXPLAIN QUERY PLAN
SELECT o.id, o.total_amount, u.email
FROM orders o
JOIN users u ON o.user_id = u.id
WHERE o.order_date >= date('now', '-30 days')
  AND o.status = 'shipped'
ORDER BY o.order_date DESC
LIMIT 100;

-- ✅ Good: Avoid SELECT *
SELECT id, order_number, total_amount, order_date
FROM orders
WHERE order_date > date('now', '-7 days')

PRAGMA Settings for Performance

// ✅ Good: Applying performance pragmas
func ApplyPerformancePragmas(db *sql.DB) error {
    pragmas := []string{
        "PRAGMA journal_mode = WAL",           // Better concurrency
        "PRAGMA synchronous = NORMAL",         // Faster writes
        "PRAGMA cache_size = 10000",           // Increase cache (pages)
        "PRAGMA temp_store = MEMORY",          // Store temp tables in memory
        "PRAGMA mmap_size = 268435456",        // 256MB memory map
        "PRAGMA page_size = 4096",             // Optimal page size
        "PRAGMA locking_mode = EXCLUSIVE",     // For single-writer scenarios
    }
    
    for _, pragma := range pragmas {
        if _, err := db.Exec(pragma); err != nil {
            return fmt.Errorf("exec %s: %w", pragma, err)
        }
    }
    return nil
}

Testing

Test Database Isolation

// ✅ Good: Using in-memory database for unit tests
func NewTestDB(t *testing.T) *sql.DB {
    t.Helper()
    
    db, err := sql.Open("sqlite3", ":memory:")
    if err != nil {
        t.Fatalf("open sqlite: %v", err)
    }
    
    // Apply pragmas for test performance
    _, err = db.Exec("PRAGMA journal_mode = WAL")
    if err != nil {
        t.Fatalf("set journal_mode: %v", err)
    }
    
    // Apply schema
    if err := applySchema(db); err != nil {
        t.Fatalf("apply schema: %v", err)
    }
    
    return db
}

// ✅ Good: Using file-based database for integration tests
func NewIntegrationTestDB(t *testing.T) (*sql.DB, func()) {
    t.Helper()
    
    tmpDir := t.TempDir()
    ds := filepath.Join(tmpDir, "test.db")
    
    db, err := sql.Open("sqlite3", ds)
    if err != nil {
        t.Fatalf("open sqlite: %v", err)
    }
    
    // Apply pragmas
    _, err = db.Exec("PRAGMA journal_mode = WAL")
    if err != nil {
        t.Fatalf("set journal_mode: %v", err)
    }
    
    // Apply schema
    if err := applySchema(db); err != nil {
        t.Fatalf("apply schema: %v", err)
    }
    
    cleanup := func() {
        db.Close()
        // Files automatically cleaned up by t.TempDir()
    }
    
    return db, cleanup
}

Best Practices

❌ Bad Patterns

-- ❌ Bad: Using PRAGMA synchronous = OFF (can cause corruption)
PRAGMA synchronous = OFF;

-- ❌ Bad: No foreign key enforcement (defaults to off in SQLite)
-- Remember to always set: PRAGMA foreign_keys = ON;

-- ❌ Bad: Using TEXT PRIMARY KEY without UNIQUE (still works but inefficient)
CREATE TABLE bad (id TEXT PRIMARY KEY, data TEXT); -- Better: INTEGER PK

-- ❌ Bad: Long transactions blocking writers
// Keep transactions short in SQLite due to database-level locking

✅ Good Patterns

-- ✅ Good: Always enforce foreign keys
PRAGMA foreign_keys = ON;

-- ✅ Good: Use WAL mode for better concurrency
PRAGMA journal_mode = WAL;

-- ✅ Good: Keep transactions short
BEGIN;
-- Quick operations here
COMMIT;

-- ✅ Good: Use INTEGER PRIMARY KEY for auto-increment
CREATE TABLE good (
    id INTEGER PRIMARY KEY AUTOINCREMENT,
    data TEXT
);

-- ✅ Good: Index foreign key columns
CREATE INDEX idx_orders_user_id ON orders(user_id);

Common Operations

Backup and Restore

# ✅ Good: Backup using .dump command
sqlite3 production.db ".dump" > backup.sql

# Restore
sqlite3 development.db < backup.sql

# ✅ Good: Online backup using VACUUM INTO (SQLite 3.27+)
sqlite3 production.db "VACUUM INTO 'backup.db';"

# ✅ Good: Copy file while locked (use .backup command)
sqlite3 production.db ".backup backup.db"

Monitoring

-- ✅ Good: Checking database size
SELECT 
    page_count * page_size as size,
    page_count * page_size / 1024.0 as size_kb,
    page_count * page_size / 1048576.0 as size_mb
FROM pragma_page_count(), pragma_page_size();

-- ✅ Good: Checking schema integrity
PRAGMA integrity_check;

-- ✅ Good: Listing tables and indexes
SELECT name, type FROM sqlite_master WHERE type IN ('table', 'index') ORDER BY type, name;

-- ✅ Good: Table info
PRAGMA table_info(users);

-- ✅ Good: Index info
PRAGMA index_info(idx_users_email);

Go-Specific Patterns

Using sqlx with SQLite

import (
    "github.com/jmoiron/sqlx"
)

// ✅ Good: sqlx with SQLite
func NewSQLiteDBx(ds string) (*sqlx.DB, error) {
    db, err := sql.Open("sqlite3", ds)
    if err != nil {
        return nil, fmt.Errorf("open sqlite: %w", err)
    }
    
    // Apply pragmas
    if _, err := db.Exec("PRAGMA journal_mode = WAL"); err != nil {
        return nil, fmt.Errorf("set journal_mode: %w", err)
    }
    if _, err := db.Exec("PRAGMA foreign_keys = ON"); err != nil {
        return nil, fmt.Errorf("set foreign_keys: %w", err)
    }
    
    return sqlx.NewDb(db, "sqlite3"), nil
}

// ✅ Good: Named queries with sqlx
func GetUserByEmail(db *sqlx.DB, email string) (*User, error) {
    var user User
    err := db.Get(&user, `
        SELECT id, email, first_name, last_name, age, created_at, updated_at, is_active
        FROM users
        WHERE email = :email
    `, map[string]interface{}{"email": email})
    if err != nil {
        if errors.Is(err, sql.ErrNoRows) {
            return nil, ErrNotFound
        }
        return nil, fmt.Errorf("get user by email: %w", err)
    }
    return &user, nil
}

Using ORM (GORM) with SQLite

import (
    "gorm.io/driver/sqlite"
    "gorm.io/gorm"
)

// ✅ Good: GORM with SQLite
func NewGormDB(ds string) (*gorm.DB, error) {
    db, err := gorm.Open(sqlite.Open(ds), &gorm.Config{})
    if err != nil {
        return nil, fmt.Errorf("open gorm: %w", err)
    }
    
    // Configure connection pool (though SQLite limitations apply)
    sqlDB, err := db.DB()
    if err != nil {
        return nil, fmt.Errorf("get db: %w", err)
    }
    
    // SetMaxIdleConns and SetMaxOpenConns still apply to underlying connection
    sqlDB.SetMaxIdleConns(10)
    sqlDB.SetMaxOpenConns(100)
    sqlDB.SetConnMaxLifetime(time.Hour)
    
    // Apply pragmas
    if err := sqlDB.ExecContext(context.Background(), "PRAGMA journal_mode = WAL").Error; err != nil {
        return nil, fmt.Errorf("set journal_mode: %w", err)
    }
    if err := sqlDB.ExecContext(context.Background(), "PRAGMA foreign_keys = ON").Error; err != nil {
        return nil, fmt.Errorf("set foreign_keys: %w", err)
    }
    
    return db, nil
}

Limitations and Workarounds

Concurrency Limitations

SQLite has database-level locking for writes. For high-concurrency write scenarios:

1. Use WAL mode - Allows multiple readers while one writer is active
2. Queue writes - Use a worker pool to serialize writes
3. Consider client-side caching - Read-heavy workloads scale well
4. Sharding - Split data across multiple SQLite files by tenant/date

// ✅ Good: Write queue for high concurrency
type WriteQueue struct {
    db      *sql.DB
    ch      chan func() error
    wg      sync.WaitGroup
}

func NewWriteQueue(db *sql.DB, workerCount int) *WriteQueue {
    wq := &WriteQueue{
        db: db,
        ch: make(chan func() error, 100),
    }
    
    wq.wg.Add(workerCount)
    for i := 0; i < workerCount; i++ {
        go func() {
            defer wq.wg.Done()
            for fn := range wq.ch {
                fn() // Execute the write function
            }
        }()
    }
    return wq
}

func (wq *WriteQueue) Enqueue(fn func() error) {
    wq.ch <- fn
}

func (wq *WriteQueue) Close() {
    close(wq.ch)
    wq.wg.Wait()
}

Conclusion

SQLite is excellent for:

• Desktop and mobile applications
• Small to medium web applications (low to medium write concurrency)
• Development and testing
• Embedded devices
• Applications requiring zero-administration

Follow these patterns to get the best performance and reliability from SQLite in your Go applications.