updated layout for webmail and added http dns letsencrypt

internal/db/count_messages_by_folder_test.go

@@ -0,0 +1,44 @@
+package db
+
+import (
+	"testing"
+	"time"
+)
+
+func TestCountMessagesByFolder(t *testing.T) {
+	d := openTestDB(t)
+	const mailboxID = int64(1)
+
+	insert := func(folder, flags string) {
+		t.Helper()
+		if _, err := d.InsertMessage(mailboxID, folder, "", flags, time.Now(), 10, "/dev/null", []byte("nonce"), "a@example.com", "b@example.com", "subj", ""); err != nil {
+			t.Fatal(err)
+		}
+	}
+	insert("INBOX", "")
+	insert("INBOX", "")
+	insert("INBOX", `\Seen`)
+	insert("Sent", `\Seen`)
+
+	totals, err := d.CountMessagesByFolder(mailboxID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if totals["INBOX"] != 3 {
+		t.Errorf("INBOX total = %d, want 3", totals["INBOX"])
+	}
+	if totals["Sent"] != 1 {
+		t.Errorf("Sent total = %d, want 1", totals["Sent"])
+	}
+
+	unread, err := d.CountUnreadByFolder(mailboxID)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if unread["INBOX"] != 2 {
+		t.Errorf("INBOX unread = %d, want 2", unread["INBOX"])
+	}
+	if _, ok := unread["Sent"]; ok {
+		t.Errorf("expected Sent to have no unread entry, got %d", unread["Sent"])
+	}
+}

internal/db/crud_mailbox_messages.go

@@ -10,23 +10,23 @@ import (
 // InsertMessage records a stored message's index row (the ciphertext itself already
 // lives at storagePath — see internal/mailstore). Returns the new row's id, which
 // doubles as the IMAP UID in later milestones.
-func (d *DB) InsertMessage(mailboxID int64, folder, messageIDHeader, flags string, internalDate time.Time, sizeBytes int64, storagePath string, nonce []byte, cachedFrom, cachedTo, cachedSubject string) (int64, error) {
+func (d *DB) InsertMessage(mailboxID int64, folder, messageIDHeader, flags string, internalDate time.Time, sizeBytes int64, storagePath string, nonce []byte, cachedFrom, cachedTo, cachedSubject, cachedPreview string) (int64, error) {
 	res, err := d.Exec(`INSERT INTO esrv_mailbox_messages
-		(mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, storage_path, nonce, cached_from, cached_to, cached_subject)
-		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
-		mailboxID, folder, messageIDHeader, flags, internalDate, sizeBytes, storagePath, nonce, cachedFrom, cachedTo, cachedSubject)
+		(mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, storage_path, nonce, cached_from, cached_to, cached_subject, cached_preview)
+		VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`,
+		mailboxID, folder, messageIDHeader, flags, internalDate, sizeBytes, storagePath, nonce, cachedFrom, cachedTo, cachedSubject, cachedPreview)
 	if err != nil {
 		return 0, err
 	}
 	return res.LastInsertId()
 }
 
-const mailboxMessageColumns = `id, mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, cached_from, cached_to, cached_subject, storage_path, nonce, created_at`
+const mailboxMessageColumns = `id, mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, cached_from, cached_to, cached_subject, cached_preview, storage_path, nonce, created_at`
 
 func scanMailboxMessage(scan func(dest ...any) error) (MailboxMessage, error) {
 	var m MailboxMessage
 	var internalDate, createdAt string
-	err := scan(&m.ID, &m.MailboxID, &m.Folder, &m.MessageIDHeader, &m.Flags, &internalDate, &m.SizeBytes, &m.CachedFrom, &m.CachedTo, &m.CachedSubject, &m.StoragePath, &m.Nonce, &createdAt)
+	err := scan(&m.ID, &m.MailboxID, &m.Folder, &m.MessageIDHeader, &m.Flags, &internalDate, &m.SizeBytes, &m.CachedFrom, &m.CachedTo, &m.CachedSubject, &m.CachedPreview, &m.StoragePath, &m.Nonce, &createdAt)
 	if err != nil {
 		return m, err
 	}
@@ -103,6 +103,17 @@ func (d *DB) ListMessagesForMailbox(mailboxID int64) ([]MailboxMessage, error) {
 	return scanMailboxMessages(rows)
 }
 
+// UpdateMessageCachedFields overwrites a message's cached_from/cached_to/
+// cached_subject/cached_preview — the display-only fields derived from the message's
+// own content at store time. Used by mailstore.RebuildMessageCache to re-derive them
+// for messages stored before a caching fix/addition landed (those fields are
+// otherwise only ever computed once, at delivery time, never retroactively).
+func (d *DB) UpdateMessageCachedFields(id int64, cachedFrom, cachedTo, cachedSubject, cachedPreview string) error {
+	_, err := d.Exec(`UPDATE esrv_mailbox_messages SET cached_from = ?, cached_to = ?, cached_subject = ?, cached_preview = ? WHERE id = ?`,
+		cachedFrom, cachedTo, cachedSubject, cachedPreview, id)
+	return err
+}
+
 // SetMessageFlags overwrites a message's stored IMAP flags (space-separated), scoped
 // to mailboxID so a session can't touch another mailbox's message by guessing a UID.
 func (d *DB) SetMessageFlags(mailboxID, uid int64, flags string) error {
@@ -124,9 +135,35 @@ func (d *DB) ListMessagesInFolder(mailboxID int64, folder string) ([]MailboxMess
 // ListMessagesInFolderPage is ListMessagesInFolder with newest-first pagination, for
 // the webmail client's folder view — a mailbox can accumulate far more mail than is
 // reasonable to render in one page.
-func (d *DB) ListMessagesInFolderPage(mailboxID int64, folder string, offset, limit int) ([]MailboxMessage, error) {
-	rows, err := d.Query(`SELECT `+mailboxMessageColumns+` FROM esrv_mailbox_messages
-		WHERE mailbox_id = ? AND folder = ? ORDER BY id DESC LIMIT ? OFFSET ?`, mailboxID, folder, limit, offset)
+// sortColumnAndDir maps the folder view's ?sort=/&dir= query params to a safe,
+// hardcoded SQL ORDER BY fragment — never interpolates the raw query values
+// themselves, only picks between two known-safe literals, so this stays injection-safe
+// however sort/dir arrive from the URL. "date" (the default) sorts by id, which tracks
+// insertion/received order — the same ordering ListMessagesInFolderPage always used,
+// just now also selectable ascending.
+func sortColumnAndDir(sortBy, sortDir string) string {
+	col := "id"
+	if sortBy == "from" {
+		col = "cached_from"
+	}
+	dir := "DESC"
+	if sortDir == "asc" {
+		dir = "ASC"
+	}
+	// Tie-break on id in the same direction so same-sender/same-instant rows still
+	// have a stable, deterministic order across pages.
+	return col + " " + dir + ", id " + dir
+}
+
+func (d *DB) ListMessagesInFolderPage(mailboxID int64, folder string, unreadOnly bool, sortBy, sortDir string, offset, limit int) ([]MailboxMessage, error) {
+	query := `SELECT ` + mailboxMessageColumns + ` FROM esrv_mailbox_messages WHERE mailbox_id = ? AND folder = ?`
+	args := []any{mailboxID, folder}
+	if unreadOnly {
+		query += ` AND flags NOT LIKE '%\Seen%' ESCAPE '\'`
+	}
+	query += ` ORDER BY ` + sortColumnAndDir(sortBy, sortDir) + ` LIMIT ? OFFSET ?`
+	args = append(args, limit, offset)
+	rows, err := d.Query(query, args...)
 	if err != nil {
 		return nil, err
 	}
@@ -134,9 +171,14 @@ func (d *DB) ListMessagesInFolderPage(mailboxID int64, folder string, offset, li
 }
 
 // CountMessagesInFolder backs ListMessagesInFolderPage's pagination controls.
-func (d *DB) CountMessagesInFolder(mailboxID int64, folder string) (int, error) {
+func (d *DB) CountMessagesInFolder(mailboxID int64, folder string, unreadOnly bool) (int, error) {
+	query := `SELECT COUNT(*) FROM esrv_mailbox_messages WHERE mailbox_id = ? AND folder = ?`
+	args := []any{mailboxID, folder}
+	if unreadOnly {
+		query += ` AND flags NOT LIKE '%\Seen%' ESCAPE '\'`
+	}
 	var n int
-	err := d.QueryRow(`SELECT COUNT(*) FROM esrv_mailbox_messages WHERE mailbox_id = ? AND folder = ?`, mailboxID, folder).Scan(&n)
+	err := d.QueryRow(query, args...).Scan(&n)
 	return n, err
 }
 
@@ -210,6 +252,28 @@ func (d *DB) CountUnreadByFolder(mailboxID int64) (map[string]int, error) {
 	return out, rows.Err()
 }
 
+// CountMessagesByFolder returns every folder's total message count in one query — the
+// total half of the sidebar's "total / unread" display, mirroring CountUnreadByFolder's
+// shape exactly (a folder with zero messages simply has no entry in the returned map).
+func (d *DB) CountMessagesByFolder(mailboxID int64) (map[string]int, error) {
+	rows, err := d.Query(`SELECT folder, COUNT(*) FROM esrv_mailbox_messages
+		WHERE mailbox_id = ? GROUP BY folder`, mailboxID)
+	if err != nil {
+		return nil, err
+	}
+	defer rows.Close()
+	out := map[string]int{}
+	for rows.Next() {
+		var folder string
+		var n int
+		if err := rows.Scan(&folder, &n); err != nil {
+			return nil, err
+		}
+		out[folder] = n
+	}
+	return out, rows.Err()
+}
+
 // SuggestRecipients returns up to 10 distinct addresses (as originally cached — a
 // display name like "Name <addr@example.com>" is kept as-is, not parsed apart, since
 // that's exactly what a To/Cc/Bcc field already accepts) this mailbox has previously

internal/db/crud_mailboxes.go

@@ -5,13 +5,13 @@ import (
 	"errors"
 )
 
-const mailboxColumns = `id, email, domain_id, password_hash, is_active, quota_bytes, used_bytes, dek_wrapped, dek_nonce, created_at, created_by, totp_secret, totp_enabled, mfa_exempt`
+const mailboxColumns = `id, email, domain_id, password_hash, is_active, quota_bytes, used_bytes, dek_wrapped, dek_nonce, created_at, created_by, totp_secret, totp_enabled, mfa_exempt, group_messages`
 
 func scanMailbox(row *sql.Row) (*Mailbox, error) {
 	var m Mailbox
 	var createdAt string
 	var createdBy sql.NullInt64
-	if err := row.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt); err != nil {
+	if err := row.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt, &m.GroupMessages); err != nil {
 		if errors.Is(err, sql.ErrNoRows) {
 			return nil, nil
 		}
@@ -31,7 +31,7 @@ type MailboxWithDomain struct {
 }
 
 func (d *DB) ListMailboxes() ([]MailboxWithDomain, error) {
-	rows, err := d.Query(`SELECT m.id, m.email, m.domain_id, m.password_hash, m.is_active, m.quota_bytes, m.used_bytes, m.dek_wrapped, m.dek_nonce, m.created_at, m.created_by, m.totp_secret, m.totp_enabled, m.mfa_exempt, dm.domain_name
+	rows, err := d.Query(`SELECT m.id, m.email, m.domain_id, m.password_hash, m.is_active, m.quota_bytes, m.used_bytes, m.dek_wrapped, m.dek_nonce, m.created_at, m.created_by, m.totp_secret, m.totp_enabled, m.mfa_exempt, m.group_messages, dm.domain_name
 		FROM esrv_mailboxes m JOIN esrv_domains dm ON dm.id = m.domain_id ORDER BY m.email`)
 	if err != nil {
 		return nil, err
@@ -42,7 +42,7 @@ func (d *DB) ListMailboxes() ([]MailboxWithDomain, error) {
 		var m MailboxWithDomain
 		var createdAt string
 		var createdBy sql.NullInt64
-		if err := rows.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt, &m.DomainName); err != nil {
+		if err := rows.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt, &m.GroupMessages, &m.DomainName); err != nil {
 			return nil, err
 		}
 		m.CreatedAt, _ = parseTime(createdAt)
@@ -65,7 +65,7 @@ func (d *DB) ListMailboxesForDomain(domainID int64) ([]Mailbox, error) {
 		var m Mailbox
 		var createdAt string
 		var createdBy sql.NullInt64
-		if err := rows.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt); err != nil {
+		if err := rows.Scan(&m.ID, &m.Email, &m.DomainID, &m.PasswordHash, &m.IsActive, &m.QuotaBytes, &m.UsedBytes, &m.DEKWrapped, &m.DEKNonce, &createdAt, &createdBy, &m.TOTPSecret, &m.TOTPEnabled, &m.MFAExempt, &m.GroupMessages); err != nil {
 			return nil, err
 		}
 		m.CreatedAt, _ = parseTime(createdAt)
@@ -120,6 +120,11 @@ func (d *DB) SetMailboxMFAExempt(id int64, exempt bool) error {
 	return err
 }
 
+func (d *DB) SetMailboxGroupMessages(id int64, group bool) error {
+	_, err := d.Exec(`UPDATE esrv_mailboxes SET group_messages = ? WHERE id = ?`, group, id)
+	return err
+}
+
 func (d *DB) SetMailboxQuota(id int64, quotaBytes int64) error {
 	_, err := d.Exec(`UPDATE esrv_mailboxes SET quota_bytes = ? WHERE id = ?`, quotaBytes, id)
 	return err

internal/db/mailbox_models.go

@@ -25,6 +25,9 @@ type Mailbox struct {
 	// MFAExempt overrides [Auth] enforce_mailbox_mfa off for this mailbox specifically,
 	// even if its domain isn't exempt.
 	MFAExempt bool
+	// GroupMessages collapses a run of same-subject messages in a folder view into one
+	// expandable row when true. Off by default — a display preference, not a policy.
+	GroupMessages bool
 }
 
 // MailboxSession is a self-service webmail portal login — a parallel schema to
@@ -143,6 +146,7 @@ type MailboxMessage struct {
 	CachedFrom      string
 	CachedTo        string
 	CachedSubject   string
+	CachedPreview   string
 	StoragePath     string
 	Nonce           []byte
 	CreatedAt       time.Time

internal/db/schema.go

@@ -208,7 +208,10 @@ CREATE TABLE IF NOT EXISTS esrv_mailboxes (
 	created_by INTEGER REFERENCES esrv_admin_users(id),
 	totp_secret TEXT NOT NULL DEFAULT '',
 	totp_enabled INTEGER NOT NULL DEFAULT 0,
-	mfa_exempt INTEGER NOT NULL DEFAULT 0
+	mfa_exempt INTEGER NOT NULL DEFAULT 0,
+	-- Off by default: collapse a run of same-subject messages in a folder view into one
+	-- expandable row. Per-mailbox, not global, since this is purely a display preference.
+	group_messages INTEGER NOT NULL DEFAULT 0
 );
 
 -- Self-service webmail portal sessions — deliberately a parallel schema to
@@ -309,6 +312,10 @@ CREATE TABLE IF NOT EXISTS esrv_mailbox_messages (
 	cached_from TEXT NOT NULL DEFAULT '',
 	cached_to TEXT NOT NULL DEFAULT '',
 	cached_subject TEXT NOT NULL DEFAULT '',
+	-- First ~150 characters of the plain-text body, cached in plain text (like the
+	-- other cached_* columns) so the folder list can show a preview snippet without
+	-- decrypting the full message just to render the list.
+	cached_preview TEXT NOT NULL DEFAULT '',
 	storage_path TEXT NOT NULL,
 	nonce BLOB NOT NULL,
 	created_at DATETIME DEFAULT CURRENT_TIMESTAMP
@@ -432,6 +439,16 @@ func migrateAddedColumns(db *sql.DB) {
 		// unrecoverable-without-code-that-no-longer-exists) keys, same "not migrated"
 		// treatment as the singular-table identities before them.
 		`ALTER TABLE esrv_mailbox_smime_identities ADD COLUMN key_pem TEXT NOT NULL DEFAULT ''`,
+		`ALTER TABLE esrv_mailboxes ADD COLUMN group_messages INTEGER NOT NULL DEFAULT 0`,
+		`ALTER TABLE esrv_mailbox_messages ADD COLUMN cached_preview TEXT NOT NULL DEFAULT ''`,
+	}
+	// The three old columns above were NOT NULL with no default, so simply adding
+	// key_pem left them behind still blocking every new insert (which only ever sets
+	// key_pem, never these) on any DB created before this migration — confirmed live:
+	// "NOT NULL constraint failed: esrv_mailbox_smime_identities.key_ciphertext". Needs
+	// SQLite 3.35+ for DROP COLUMN; modernc.org/sqlite is well past that.
+	for _, col := range []string{"key_ciphertext", "key_nonce", "key_salt"} {
+		db.Exec(`ALTER TABLE esrv_mailbox_smime_identities DROP COLUMN ` + col)
 	}
 	for _, stmt := range stmts {
 		db.Exec(stmt)

internal/db/smime_legacy_migration_test.go

@@ -0,0 +1,50 @@
+package db
+
+import (
+	"database/sql"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+// TestSMIMEIdentityInsertWorksAfterLegacyColumnMigration reproduces a live bug: a DB
+// created before the S/MIME redesign (passphrase-wrapped key_ciphertext/key_nonce/
+// key_salt, all NOT NULL) only ever got key_pem ADDed by migrateAddedColumns, never had
+// the old NOT-NULL columns removed — so CreateSMIMEIdentity (which only sets key_pem)
+// failed with "NOT NULL constraint failed: esrv_mailbox_smime_identities.key_ciphertext"
+// on any pre-existing installation, confirmed against a real user's database.
+func TestSMIMEIdentityInsertWorksAfterLegacyColumnMigration(t *testing.T) {
+	path := filepath.Join(t.TempDir(), "test.db")
+
+	raw, err := sql.Open("sqlite", path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if _, err := raw.Exec(`
+		CREATE TABLE esrv_mailbox_smime_identities (
+			id INTEGER PRIMARY KEY AUTOINCREMENT,
+			mailbox_id INTEGER NOT NULL,
+			cert_pem TEXT NOT NULL,
+			key_ciphertext BLOB NOT NULL,
+			key_nonce BLOB NOT NULL,
+			key_salt BLOB NOT NULL,
+			not_after DATETIME NOT NULL,
+			created_at DATETIME DEFAULT CURRENT_TIMESTAMP
+		)
+	`); err != nil {
+		t.Fatal(err)
+	}
+	if err := raw.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	database, err := Open(path)
+	if err != nil {
+		t.Fatal(err)
+	}
+	t.Cleanup(func() { database.Close() })
+
+	if _, err := database.CreateSMIMEIdentity(1, "cert-pem", "key-pem", time.Now().Add(365*24*time.Hour)); err != nil {
+		t.Fatalf("CreateSMIMEIdentity after migrating a legacy DB: %v", err)
+	}
+}