diff --git a/internal/abuseguard/abuseguard.go b/internal/abuseguard/abuseguard.go index d7702cb..8a85fdd 100644 --- a/internal/abuseguard/abuseguard.go +++ b/internal/abuseguard/abuseguard.go @@ -64,21 +64,28 @@ type guardedListener struct { database *db.DB logger *toolbox.Logger maxPerIP int // <=0 means unlimited + maxTotal int // <=0 means unlimited mu sync.Mutex counts map[string]int + total int } // GuardListener wraps inner so every accepted connection is checked against the IP -// blacklist (skipping the check entirely for abuse-whitelisted IPs) and the per-IP -// concurrent-connection cap ([Security] max_connections_per_ip, default 20) before the -// caller ever sees it. +// blacklist (skipping the check entirely for abuse-whitelisted IPs), the per-IP +// concurrent-connection cap ([Security] max_connections_per_ip, default 20), and the +// total concurrent-connection cap across every source IP combined on this listener +// ([Security] max_total_connections, default 1000) before the caller ever sees it. Like +// max_connections_per_ip, this is enforced per listener instance (SMTP plain, SMTP TLS, +// IMAP plain, and IMAP TLS each get their own GuardListener/counter in main.go), not as +// one counter shared across all four. func GuardListener(inner net.Listener, database *db.DB, cfg *ini.File, logger *toolbox.Logger) net.Listener { - maxPerIP := 20 + maxPerIP, maxTotal := 20, 1000 if cfg != nil { maxPerIP = cfg.Section("Security").Key("max_connections_per_ip").MustInt(20) + maxTotal = cfg.Section("Security").Key("max_total_connections").MustInt(1000) } - return &guardedListener{Listener: inner, database: database, logger: logger, maxPerIP: maxPerIP, counts: make(map[string]int)} + return &guardedListener{Listener: inner, database: database, logger: logger, maxPerIP: maxPerIP, maxTotal: maxTotal, counts: make(map[string]int)} } func (g *guardedListener) Accept() (net.Conn, error) { @@ -91,26 +98,13 @@ func (g *guardedListener) Accept() (net.Conn, error) { if splitErr != nil { host = conn.RemoteAddr().String() } - abuseWhitelisted := false - if whitelisted, wErr := g.database.IsIPAbuseWhitelisted(host); wErr == nil && whitelisted { - abuseWhitelisted = true - } - if !abuseWhitelisted { - blocked, bErr := g.database.IsIPBlacklisted(host) - if bErr == nil && blocked { - if g.logger != nil { - g.logger.Warning("abuseguard: rejected connection from blacklisted IP %s", host) - } - conn.Close() - continue - } - } - // The concurrent-connection cap applies even to an abuse-whitelisted IP — + // Both concurrent-connection caps apply even to an abuse-whitelisted IP — // whitelisting exempts an IP from being auto-blacklisted over failed auth, not - // from basic resource-exhaustion protection, a different concern. - if g.maxPerIP > 0 { + // from basic resource-exhaustion protection, a different concern. Stays + // synchronous here: it's an in-memory counter, not a DB call. + if g.maxPerIP > 0 || g.maxTotal > 0 { g.mu.Lock() - if g.counts[host] >= g.maxPerIP { + if g.maxPerIP > 0 && g.counts[host] >= g.maxPerIP { g.mu.Unlock() if g.logger != nil { g.logger.Warning("abuseguard: rejected connection from %s: at the concurrent-connection limit (%d)", host, g.maxPerIP) @@ -118,14 +112,75 @@ func (g *guardedListener) Accept() (net.Conn, error) { conn.Close() continue } + if g.maxTotal > 0 && g.total >= g.maxTotal { + g.mu.Unlock() + if g.logger != nil { + g.logger.Warning("abuseguard: rejected connection from %s: at the total concurrent-connection limit (%d)", host, g.maxTotal) + } + conn.Close() + continue + } g.counts[host]++ + g.total++ g.mu.Unlock() conn = &countedConn{Conn: conn, g: g, host: host} } - return conn, nil + // The abuse-whitelist/blacklist DB lookups are deferred to first Read/Write + // (see checkedConn) rather than done here: this Accept loop is shared across + // every inbound connection on the listener, so a synchronous DB call here would + // serialize *every new connection* behind however long that query takes — + // worse under this app's deliberate SetMaxOpenConns(1), which serializes all DB + // access. Deferring lets each connection's own per-connection goroutine (spawned + // by the SMTP/IMAP server's Serve loop right after Accept returns) run its own + // check in parallel with every other connection's, while still guaranteeing the + // check completes strictly before any protocol byte — including the greeting + // banner — reaches the wire. + return &checkedConn{Conn: conn, g: g, host: host}, nil } } +// checkedConn defers the abuse-whitelist/blacklist DB check to the first Read or Write +// (see Accept's comment for why) and blocks that call — not the shared Accept loop — +// until the check resolves. +type checkedConn struct { + net.Conn + g *guardedListener + host string + once sync.Once + blocked bool +} + +func (c *checkedConn) ensureChecked() { + c.once.Do(func() { + if whitelisted, err := c.g.database.IsIPAbuseWhitelisted(c.host); err == nil && whitelisted { + return + } + if blocked, err := c.g.database.IsIPBlacklisted(c.host); err == nil && blocked { + if c.g.logger != nil { + c.g.logger.Warning("abuseguard: rejected connection from blacklisted IP %s", c.host) + } + c.blocked = true + c.Conn.Close() + } + }) +} + +func (c *checkedConn) Read(b []byte) (int, error) { + c.ensureChecked() + if c.blocked { + return 0, net.ErrClosed + } + return c.Conn.Read(b) +} + +func (c *checkedConn) Write(b []byte) (int, error) { + c.ensureChecked() + if c.blocked { + return 0, net.ErrClosed + } + return c.Conn.Write(b) +} + // countedConn decrements guardedListener's per-IP counter exactly once, however Close // ends up getting called (explicitly, via a defer, or both). type countedConn struct { @@ -142,6 +197,7 @@ func (c *countedConn) Close() error { if c.g.counts[c.host] <= 0 { delete(c.g.counts, c.host) } + c.g.total-- c.g.mu.Unlock() }) return c.Conn.Close() diff --git a/internal/abuseguard/abuseguard_test.go b/internal/abuseguard/abuseguard_test.go index 1b35457..0197c88 100644 --- a/internal/abuseguard/abuseguard_test.go +++ b/internal/abuseguard/abuseguard_test.go @@ -121,14 +121,22 @@ func TestGuardListenerRejectsBlacklistedIP(t *testing.T) { inner := newFakeListener(&addrOverrideConn{Conn: blockedConn, remote: hostPortAddr(blockedIP)}) guarded := GuardListener(inner, database, nil, nil) - go func() { - guarded.Accept() - inner.Close() - }() + conn, err := guarded.Accept() + if err != nil { + t.Fatal(err) + } + + // The blacklist check is deferred to first Read/Write (not done inside Accept + // itself — see checkedConn), so it only fires once something tries to use the + // connection, the way a real per-connection SMTP/IMAP session handler would. + buf := make([]byte, 1) + if _, err := conn.Read(buf); err == nil { + t.Fatal("expected Read on a blacklisted connection to fail") + } // The blocked connection's peer end should observe the connection close rather - // than any protocol banner, since GuardListener closes it before returning it. - buf := make([]byte, 1) + // than any protocol banner, since checkedConn closes it before any bytes reach + // the wire. if _, err := blockedPeer.Read(buf); err == nil { t.Fatal("expected blocked connection to be closed by GuardListener, got readable data instead") } @@ -209,3 +217,56 @@ func TestGuardListenerCapsConcurrentConnectionsPerIP(t *testing.T) { } _ = got2 } + +// TestGuardListenerCapsTotalConnections confirms max_total_connections rejects a +// connection once the cap is hit even across different source IPs (unlike +// max_connections_per_ip, which only tracks one IP at a time). +func TestGuardListenerCapsTotalConnections(t *testing.T) { + database := openTestDB(t) + cfg := ini.Empty() + sec, _ := cfg.NewSection("Security") + sec.NewKey("max_total_connections", "2") + + newConnFromIP := func(ip string) (*addrOverrideConn, net.Conn) { + local, peer := net.Pipe() + return &addrOverrideConn{Conn: local, remote: hostPortAddr(ip)}, peer + } + + c1, peer1 := newConnFromIP("198.51.100.10") + c2, peer2 := newConnFromIP("198.51.100.11") + c3, peer3 := newConnFromIP("198.51.100.12") + defer peer1.Close() + defer peer2.Close() + defer peer3.Close() + + inner := newFakeListener(c1, c2, c3) + guarded := GuardListener(inner, database, cfg, nil) + + if _, err := guarded.Accept(); err != nil { + t.Fatal(err) + } + if _, err := guarded.Accept(); err != nil { + t.Fatal(err) + } + + accepted := make(chan net.Conn, 1) + go func() { + c, err := guarded.Accept() + if err == nil { + accepted <- c + } + }() + + // The 3rd connection, from a distinct IP not previously seen, is still over the + // total cap of 2 — its peer should observe a close, proving this is a combined + // total, not per-IP. + buf := make([]byte, 1) + if _, err := peer3.Read(buf); err == nil { + t.Fatal("expected the 3rd connection (over the total cap) to be closed rather than accepted") + } + select { + case <-accepted: + t.Fatal("expected no connection to be accepted once the total cap is hit") + case <-time.After(200 * time.Millisecond): + } +} diff --git a/internal/config/config.go b/internal/config/config.go index ddf1cdd..b2a90b1 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -64,6 +64,10 @@ var defaults = []struct { {"LOG_LEVEL", "INFO", ""}, {"", "", "Hide verbose aiosmtpd-equivalent INFO messages when LOG_LEVEL = INFO"}, {"hide_info_aiosmtpd", "true", ""}, + {"", "", "Log line format: text (default, matches the original Python server's"}, + {"", "", "\"timestamp - name - level - message\" format) or json (one JSON object per"}, + {"", "", "line, for shipping to a log aggregator)"}, + {"format", "text", ""}, }}, {"Relay", []defaultKV{ {"", "", "Timeout in seconds for external SMTP connections"}, @@ -146,6 +150,10 @@ var defaults = []struct { {"", "", "source IP — a resource-exhaustion guard, separate from the failed-auth blacklist"}, {"", "", "above (a connection flood doesn't need to fail auth to hurt)"}, {"max_connections_per_ip", "20", ""}, + {"", "", "Reject a new SMTP/IMAP connection once this many are open in total, across every"}, + {"", "", "source IP combined — bounds worst-case resource use even from many distinct IPs."}, + {"", "", "0 = unlimited (previous behavior)."}, + {"max_total_connections", "1000", ""}, }}, {"IMAP", []defaultKV{ {"", "", "IMAP server configuration for mailbox retrieval (Thunderbird, etc.)"}, @@ -183,6 +191,9 @@ var defaults = []struct { {"virus_scan_enabled", "false", ""}, {"", "", "Address of the clamd instance to scan through, if enabled above"}, {"clamd_address", "127.0.0.1:3310", ""}, + {"", "", "Delete send/receive log history (and any stored message attachments) older than"}, + {"", "", "this many days. 0 (default) = keep forever — history only grows if left at 0."}, + {"email_log_retention_days", "0", ""}, }}, {"Rspamd", []defaultKV{ {"", "", "Optional rspamd integration for spam scoring (off by default; the built-in"}, diff --git a/internal/db/crud_monitoring.go b/internal/db/crud_monitoring.go index 4e58982..2fd7bb8 100644 --- a/internal/db/crud_monitoring.go +++ b/internal/db/crud_monitoring.go @@ -134,8 +134,12 @@ func (d *DB) SendCountsByDomain(hours int) ([]DomainSendCount, error) { // in this count either way. func (d *DB) CountRecentSendsForDomain(domain string, since time.Time) (int, error) { var n int + // mail_from_domain is computed once at insert time (see InsertEmailLog) and + // indexed, rather than recomputing substr/instr per row here on every relay send — + // SQLite can't use an index for a computed-expression WHERE clause, so the old form + // of this query was a full table scan against ever-growing history. err := d.QueryRow(`SELECT COUNT(*) FROM esrv_email_logs - WHERE substr(mail_from, instr(mail_from, '@') + 1) = ? AND timestamp >= ?`, + WHERE mail_from_domain = ? AND timestamp >= ?`, domain, since.UTC()).Scan(&n) return n, err } diff --git a/internal/db/crud_relay_queue.go b/internal/db/crud_relay_queue.go index d0411e0..340cc3d 100644 --- a/internal/db/crud_relay_queue.go +++ b/internal/db/crud_relay_queue.go @@ -115,3 +115,12 @@ func (d *DB) CountPendingRelayQueueItemsForEmailLog(emailLogID int64) (int, erro err := d.QueryRow(`SELECT COUNT(*) FROM esrv_relay_queue WHERE email_log_id = ?`, emailLogID).Scan(&n) return n, err } + +// CountPendingRelayQueueItems reports the total outbound relay backlog across every +// message, for the /metrics gauge (internal/webui/metrics.go) — a sustained rise here +// means deliveries aren't keeping up with intake. +func (d *DB) CountPendingRelayQueueItems() (int, error) { + var n int + err := d.QueryRow(`SELECT COUNT(*) FROM esrv_relay_queue`).Scan(&n) + return n, err +} diff --git a/internal/db/logs.go b/internal/db/logs.go index 8db35ab..43e8f37 100644 --- a/internal/db/logs.go +++ b/internal/db/logs.go @@ -6,9 +6,9 @@ import "time" // new row's id (needed before recipient/attachment child rows can be inserted). func (d *DB) InsertEmailLog(l EmailLog) (int64, error) { res, err := d.Exec(`INSERT INTO esrv_email_logs - (message_id, timestamp, peer_ip, mail_from, to_address, cc_addresses, bcc_addresses, subject, email_headers, message_body, status, dkim_signed, username) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - l.MessageID, l.Timestamp, l.PeerIP, l.MailFrom, l.ToAddress, l.CcAddresses, l.BccAddresses, l.Subject, l.EmailHeaders, l.MessageBody, l.Status, l.DKIMSigned, l.Username) + (message_id, timestamp, peer_ip, mail_from, mail_from_domain, to_address, cc_addresses, bcc_addresses, subject, email_headers, message_body, status, dkim_signed, username) + VALUES (?, ?, ?, ?, substr(?, instr(?, '@') + 1), ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + l.MessageID, l.Timestamp, l.PeerIP, l.MailFrom, l.MailFrom, l.MailFrom, l.ToAddress, l.CcAddresses, l.BccAddresses, l.Subject, l.EmailHeaders, l.MessageBody, l.Status, l.DKIMSigned, l.Username) if err != nil { return 0, err } @@ -47,6 +47,49 @@ func (d *DB) UpdateEmailLogStatus(id int64, status string) error { return err } +// PruneEmailLogsOlderThan deletes esrv_email_logs rows older than cutoff along with +// their esrv_email_recipient_logs and esrv_email_attachments rows (no FK cascade in +// this DB — see the schema comment on PRAGMA foreign_keys). A log still "queued" is +// never pruned regardless of age: that means the relay queue worker hasn't finished +// with it yet, and esrv_relay_queue's own row still references it. Returns the deleted +// attachments' file paths for the caller to remove from disk — this package does no +// file I/O — and the number of email logs deleted. +func (d *DB) PruneEmailLogsOlderThan(cutoff time.Time) (attachmentPaths []string, deleted int64, err error) { + rows, err := d.Query(`SELECT file_path FROM esrv_email_attachments + WHERE email_log_id IN (SELECT id FROM esrv_email_logs WHERE timestamp < ? AND status != 'queued')`, cutoff.UTC()) + if err != nil { + return nil, 0, err + } + for rows.Next() { + var p string + if err := rows.Scan(&p); err != nil { + rows.Close() + return nil, 0, err + } + attachmentPaths = append(attachmentPaths, p) + } + if err := rows.Err(); err != nil { + rows.Close() + return nil, 0, err + } + rows.Close() + + if _, err := d.Exec(`DELETE FROM esrv_email_attachments + WHERE email_log_id IN (SELECT id FROM esrv_email_logs WHERE timestamp < ? AND status != 'queued')`, cutoff.UTC()); err != nil { + return nil, 0, err + } + if _, err := d.Exec(`DELETE FROM esrv_email_recipient_logs + WHERE email_log_id IN (SELECT id FROM esrv_email_logs WHERE timestamp < ? AND status != 'queued')`, cutoff.UTC()); err != nil { + return nil, 0, err + } + res, err := d.Exec(`DELETE FROM esrv_email_logs WHERE timestamp < ? AND status != 'queued'`, cutoff.UTC()) + if err != nil { + return nil, 0, err + } + deleted, err = res.RowsAffected() + return attachmentPaths, deleted, err +} + // InsertEmailAttachment mirrors one EmailAttachment row creation. func (d *DB) InsertEmailAttachment(a EmailAttachment) error { _, err := d.Exec(`INSERT INTO esrv_email_attachments diff --git a/internal/db/logs_test.go b/internal/db/logs_test.go new file mode 100644 index 0000000..0018034 --- /dev/null +++ b/internal/db/logs_test.go @@ -0,0 +1,77 @@ +package db + +import ( + "os" + "path/filepath" + "testing" + "time" +) + +func TestPruneEmailLogsOlderThanDeletesOldRowsAndKeepsRecentAndQueued(t *testing.T) { + database := openMonitoringTestDB(t) + now := time.Now().UTC() + + oldID, err := database.InsertEmailLog(EmailLog{MessageID: "old@x", Timestamp: now.Add(-48 * time.Hour), MailFrom: "a@sender.example", EmailHeaders: "h", Status: "relayed"}) + if err != nil { + t.Fatal(err) + } + if err := database.InsertEmailRecipientLog(EmailRecipientLog{EmailLogID: oldID, Recipient: "r@x", RecipientType: "to", Status: "success"}); err != nil { + t.Fatal(err) + } + attachPath := filepath.Join(t.TempDir(), "attach.bin") + if err := os.WriteFile(attachPath, []byte("data"), 0o644); err != nil { + t.Fatal(err) + } + if err := database.InsertEmailAttachment(EmailAttachment{EmailLogID: oldID, Filename: "attach.bin", FilePath: attachPath, Size: 4}); err != nil { + t.Fatal(err) + } + + recentID, err := database.InsertEmailLog(EmailLog{MessageID: "recent@x", Timestamp: now, MailFrom: "a@sender.example", EmailHeaders: "h", Status: "relayed"}) + if err != nil { + t.Fatal(err) + } + + queuedOldID, err := database.InsertEmailLog(EmailLog{MessageID: "queued@x", Timestamp: now.Add(-48 * time.Hour), MailFrom: "a@sender.example", EmailHeaders: "h", Status: "queued"}) + if err != nil { + t.Fatal(err) + } + + paths, deleted, err := database.PruneEmailLogsOlderThan(now.Add(-24 * time.Hour)) + if err != nil { + t.Fatal(err) + } + if deleted != 1 { + t.Fatalf("expected exactly 1 email log deleted, got %d", deleted) + } + if len(paths) != 1 || paths[0] != attachPath { + t.Fatalf("expected the old attachment's path returned, got %v", paths) + } + + var count int + if err := database.QueryRow(`SELECT COUNT(*) FROM esrv_email_logs WHERE id = ?`, oldID).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Error("expected the old email log row to be deleted") + } + if err := database.QueryRow(`SELECT COUNT(*) FROM esrv_email_recipient_logs WHERE email_log_id = ?`, oldID).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 0 { + t.Error("expected the old email log's recipient log row to be deleted") + } + + if err := database.QueryRow(`SELECT COUNT(*) FROM esrv_email_logs WHERE id = ?`, recentID).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Error("expected the recent email log to survive pruning") + } + + if err := database.QueryRow(`SELECT COUNT(*) FROM esrv_email_logs WHERE id = ?`, queuedOldID).Scan(&count); err != nil { + t.Fatal(err) + } + if count != 1 { + t.Error("expected an old but still-'queued' email log to survive pruning (the relay worker isn't done with it)") + } +} diff --git a/internal/db/schema.go b/internal/db/schema.go index 68c8aa6..cd4ebda 100644 --- a/internal/db/schema.go +++ b/internal/db/schema.go @@ -104,6 +104,10 @@ CREATE TABLE IF NOT EXISTS esrv_email_logs ( timestamp DATETIME NOT NULL, peer_ip TEXT NOT NULL, mail_from TEXT NOT NULL, + -- Derived from mail_from at insert time (see InsertEmailLog) so domainSendRateLimited + -- can filter with a plain indexed equality instead of a per-row substr()/instr() + -- expression, which SQLite can't use an index for. + mail_from_domain TEXT NOT NULL DEFAULT '', to_address TEXT NOT NULL DEFAULT '', cc_addresses TEXT DEFAULT '', bcc_addresses TEXT DEFAULT '', @@ -115,6 +119,9 @@ CREATE TABLE IF NOT EXISTS esrv_email_logs ( username TEXT, created_at DATETIME DEFAULT CURRENT_TIMESTAMP ); +-- Not created here: on a pre-existing DB, mail_from_domain doesn't exist yet at this +-- point in Open() (migrateAddedColumns below adds it) — CREATE INDEX would fail with +-- "no such column". See migrateAddedColumns for this index. CREATE TABLE IF NOT EXISTS esrv_email_recipient_logs ( id INTEGER PRIMARY KEY AUTOINCREMENT, @@ -126,6 +133,10 @@ CREATE TABLE IF NOT EXISTS esrv_email_recipient_logs ( error_message TEXT, server_response TEXT ); +-- Resolved by UpdateEmailRecipientLogStatus (email_log_id + recipient, guarded on +-- status='queued') on every relay-queue worker tick — without this index that query +-- was a full table scan against a table that only grows. +CREATE INDEX IF NOT EXISTS idx_email_recipient_logs_email_log_id ON esrv_email_recipient_logs(email_log_id); -- In-flight outbound relay work — one row per recipient-domain-group (matching how -- RelayEmailAsync/EnqueueForDelivery already batch same-domain recipients into a @@ -770,6 +781,7 @@ func migrateAddedColumns(db *sql.DB) { `ALTER TABLE esrv_mailboxes ADD COLUMN carddav_enabled INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE esrv_domains ADD COLUMN dkim_dns_automation TEXT NOT NULL DEFAULT 'manual'`, `ALTER TABLE esrv_domains ADD COLUMN use_global_dkim INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE esrv_email_logs ADD COLUMN mail_from_domain TEXT NOT NULL DEFAULT ''`, } // The three old columns above were NOT NULL with no default, so simply adding // key_pem left them behind still blocking every new insert (which only ever sets @@ -787,6 +799,12 @@ func migrateAddedColumns(db *sql.DB) { // defaults to 0 for every pre-existing row above, which would otherwise let that // account skip its username change entirely once it re-hits /first-login next. db.Exec(`UPDATE esrv_admin_users SET must_change_username = 1 WHERE username = ? AND must_change_password = 1`, DefaultAdminUsername) + // Backfill mail_from_domain for rows written before this column existed, then index + // it — both deferred to here (after the ALTER TABLE above) since neither the column + // nor an index on it can exist yet on a pre-existing DB when the main `schema` + // constant runs. + db.Exec(`UPDATE esrv_email_logs SET mail_from_domain = substr(mail_from, instr(mail_from, '@') + 1) WHERE mail_from_domain = ''`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_email_logs_mail_from_domain_timestamp ON esrv_email_logs(mail_from_domain, timestamp)`) migrateSpamRenamedToJunk(db) migrateFilterRulesMarkAsSpamCheck(db) migrateFilterRulesAdvancedCheck(db) @@ -959,6 +977,21 @@ func Open(path string) (*DB, error) { // serialized through a single physical connection, so no connection can ever // collide with another's in-progress write. sqlDB.SetMaxOpenConns(1) + // WAL trades the default rollback journal's per-transaction create/fsync/delete + // cycle for periodic checkpointing — a large fsync-count reduction under the write + // volume this MTA generates (one email_log + N recipient_log rows per message). + // synchronous=NORMAL is the documented safe pairing with WAL (still fsyncs at each + // checkpoint; only risks losing the last few not-yet-checkpointed commits on an OS + // crash/power loss, never DB corruption). SetMaxOpenConns(1) above means WAL's + // concurrent-reader benefit doesn't apply here — this is purely about fsync count. + if _, err := sqlDB.Exec(`PRAGMA journal_mode = WAL`); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("set journal_mode: %w", err) + } + if _, err := sqlDB.Exec(`PRAGMA synchronous = NORMAL`); err != nil { + sqlDB.Close() + return nil, fmt.Errorf("set synchronous: %w", err) + } if _, err := sqlDB.Exec(`PRAGMA busy_timeout = 5000`); err != nil { sqlDB.Close() return nil, fmt.Errorf("set busy_timeout: %w", err) diff --git a/internal/imapserver/idletimeout.go b/internal/imapserver/idletimeout.go new file mode 100644 index 0000000..1b8faaf --- /dev/null +++ b/internal/imapserver/idletimeout.go @@ -0,0 +1,48 @@ +package imapserver + +import ( + "net" + "time" +) + +// IdleTimeoutListener wraps a listener so every accepted connection gets a rolling +// deadline extended by d on each read/write. Unlike SMTP (go-smtp's Server.ReadTimeout/ +// WriteTimeout), go-imap/v2's imapserver.Options has no timeout knob at all, so an idle +// or slow-drip connection can otherwise hold a goroutine (and a file descriptor) open +// indefinitely. 30 minutes (the caller's chosen d) matches RFC 2177's guidance that an +// IDLE-capable client re-issue IDLE at least that often, so a real IDLE session renews +// its own deadline in time and is never cut off by this. +func IdleTimeoutListener(inner net.Listener, d time.Duration) net.Listener { + return &idleTimeoutListener{Listener: inner, d: d} +} + +type idleTimeoutListener struct { + net.Listener + d time.Duration +} + +func (l *idleTimeoutListener) Accept() (net.Conn, error) { + conn, err := l.Listener.Accept() + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(l.d)) + return &idleTimeoutConn{Conn: conn, d: l.d}, nil +} + +type idleTimeoutConn struct { + net.Conn + d time.Duration +} + +func (c *idleTimeoutConn) Read(b []byte) (int, error) { + n, err := c.Conn.Read(b) + c.Conn.SetDeadline(time.Now().Add(c.d)) + return n, err +} + +func (c *idleTimeoutConn) Write(b []byte) (int, error) { + n, err := c.Conn.Write(b) + c.Conn.SetDeadline(time.Now().Add(c.d)) + return n, err +} diff --git a/internal/mailstore/store.go b/internal/mailstore/store.go index a8af482..3aa75a4 100644 --- a/internal/mailstore/store.go +++ b/internal/mailstore/store.go @@ -38,7 +38,7 @@ const previewSnippetLen = 150 // an accepted scope limit, not a bug: most real mail includes a text/plain // alternative regardless of whether the sender expects it to be shown. func previewSnippet(raw []byte) string { - parsed, err := mailview.Parse(raw) + parsed, err := mailview.Parse(bytes.NewReader(raw)) if err != nil { return "" } diff --git a/internal/mailview/mailview.go b/internal/mailview/mailview.go index c5c4a81..4115d8d 100644 --- a/internal/mailview/mailview.go +++ b/internal/mailview/mailview.go @@ -47,11 +47,15 @@ type Message struct { Attachments []Attachment } -// Parse walks raw's MIME structure (recursing into nested multiparts, e.g. a +// Parse walks r's MIME structure (recursing into nested multiparts, e.g. a // multipart/alternative inside a multipart/mixed) and classifies every leaf part as -// the text body, the HTML body, or an attachment. -func Parse(raw []byte) (*Message, error) { - msg, err := mail.ReadMessage(bytes.NewReader(raw)) +// the text body, the HTML body, or an attachment. Takes an io.Reader rather than +// []byte so a caller already holding a string (e.g. a message body pulled straight out +// of the DB) can pass strings.NewReader(s) directly instead of copying via []byte(s) +// first — callers already holding []byte pass bytes.NewReader(b), same as before, no +// copy either way. +func Parse(r io.Reader) (*Message, error) { + msg, err := mail.ReadMessage(r) if err != nil { return nil, err } diff --git a/internal/mailview/mailview_test.go b/internal/mailview/mailview_test.go index e47c2f0..f92e3eb 100644 --- a/internal/mailview/mailview_test.go +++ b/internal/mailview/mailview_test.go @@ -7,7 +7,7 @@ import ( func TestParseSimpleTextMessage(t *testing.T) { raw := "From: a@example.com\r\nTo: b@example.com\r\nSubject: hi\r\n\r\nhello there" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } @@ -29,7 +29,7 @@ func TestParseMultipartAlternativeKeepsBothBodies(t *testing.T) { "--B\r\nContent-Type: text/plain\r\n\r\nplain version\r\n" + "--B\r\nContent-Type: text/html\r\n\r\n

html version

\r\n" + "--B--\r\n" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } @@ -49,7 +49,7 @@ func TestParseAttachmentDecodesBase64(t *testing.T) { "--B\r\nContent-Type: application/octet-stream\r\nContent-Disposition: attachment; filename=\"a.txt\"\r\n" + "Content-Transfer-Encoding: BASE64\r\n\r\nSGVsbG8sIHdvcmxkIQ==\r\n" + "--B--\r\n" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } @@ -75,7 +75,7 @@ func TestParseNestedMultipartMixedWithAlternativeBody(t *testing.T) { "--OUTER\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=\"notes.txt\"\r\n\r\n" + "attached notes\r\n" + "--OUTER--\r\n" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } @@ -99,7 +99,7 @@ func TestParseInlineImageCapturesContentID(t *testing.T) { "Content-Id: \r\n" + "Content-Transfer-Encoding: BASE64\r\n\r\nSGVsbG8sIHdvcmxkIQ==\r\n" + "--B--\r\n" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } @@ -125,7 +125,7 @@ func TestParseInlineImageWithNoFilenameStillCaptured(t *testing.T) { "--B\r\nContent-Type: image/png\r\n" + "Content-Id: \r\n\r\nrawbytes" + "\r\n--B--\r\n" - m, err := Parse([]byte(raw)) + m, err := Parse(strings.NewReader(raw)) if err != nil { t.Fatal(err) } diff --git a/internal/relay/relay.go b/internal/relay/relay.go index fa4adba..dcc18a5 100644 --- a/internal/relay/relay.go +++ b/internal/relay/relay.go @@ -72,8 +72,18 @@ type Relay struct { // at a local stand-in MTA (combined with port above) without a real DNS MX record. // nil (the normal, non-test case) means "use net.LookupMX". mxLookup func(domain string) ([]*net.MX, error) + + // sideEffectSem bounds RelayEmailAsyncBounded's concurrency — see that method. + sideEffectSem chan struct{} } +// sideEffectConcurrency caps how many RelayEmailAsyncBounded sends (mailbox +// forwarding, forward-rule actions, auto-reply — see session.go) run at once, +// matching the main relay queue worker's own maxConcurrent (main.go's +// runRelayQueueWorker) so a burst of incoming messages that each trigger one of these +// can't spawn unbounded concurrent outbound SMTP connections. +const sideEffectConcurrency = 10 + func (r *Relay) targetPort() int { if r.port != 0 { return r.port @@ -91,7 +101,24 @@ func New(database *db.DB, cfg *ini.File, logger *toolbox.Logger) *Relay { if hostname == "" { hostname = cfg.Section("Server").Key("HOSTNAME").MustString("localhost") } - return &Relay{DB: database, Timeout: time.Duration(timeoutSecs) * time.Second, Hostname: hostname, Logger: logger} + return &Relay{DB: database, Timeout: time.Duration(timeoutSecs) * time.Second, Hostname: hostname, Logger: logger, sideEffectSem: make(chan struct{}, sideEffectConcurrency)} +} + +// RelayEmailAsyncBounded runs RelayEmailAsync in a new goroutine, gated by a shared +// semaphore (see sideEffectConcurrency) instead of a bare `go func(){ RelayEmailAsync +// (...) }()` — used for low-volume fire-and-forget side effects (mailbox forwarding, +// forward-rule actions, auto-reply) that each relay to a single recipient and have no +// esrv_email_logs row of their own to route through the main EnqueueForDelivery queue. +// Returns immediately either way; onDone (if non-nil) runs once delivery finishes. +func (r *Relay) RelayEmailAsyncBounded(mailFrom string, rcptTos []string, content string, recipientTypes []string, onDone func([]Result)) { + go func() { + r.sideEffectSem <- struct{}{} + defer func() { <-r.sideEffectSem }() + res := r.RelayEmailAsync(mailFrom, rcptTos, content, recipientTypes) + if onDone != nil { + onDone(res) + } + }() } // prepareEmailForRecipient mirrors email_relay._prepare_email_for_recipient: strips any diff --git a/internal/smtpserver/attachments.go b/internal/smtpserver/attachments.go index 74198c8..2b525df 100644 --- a/internal/smtpserver/attachments.go +++ b/internal/smtpserver/attachments.go @@ -32,6 +32,18 @@ func sanitizePathSegment(s string, chars string) string { return s } +// sanitizeAttachmentFilename confines an attacker-controlled MIME filename to a bare +// file name — filepath.Base strips any directory components (including "../" +// traversal), and the ".."/"." edge cases it can still return are replaced outright, +// so the caller's filepath.Join can never escape the intended storage directory. +func sanitizeAttachmentFilename(name string) string { + name = filepath.Base(name) + if name == "." || name == ".." || name == string(filepath.Separator) { + return "attachment" + } + return name +} + // cleanMessageIDPrefix strips everything from "@" onward, mirroring the // clean_message_id computation used to build attachment filenames. func cleanMessageIDPrefix(messageID string) string { @@ -49,14 +61,18 @@ type attachmentPart struct { type parsedMessage struct { HeaderLines []string // "Name: value" per header, in order - BodyText string // concatenated text/* parts Attachments []attachmentPart } // parseMessage mirrors the repeated BytesParser(policy=policy.default) passes in -// handle_DATA: it extracts header lines for logging, concatenated text body, and any -// attachment parts (Content-Disposition: attachment with a filename). -func parseMessage(raw []byte) (*parsedMessage, error) { +// handle_DATA: it extracts header lines for logging and, when wantAttachments is set, +// decodes every attachment part (Content-Disposition: attachment with a filename) to +// its real bytes. wantAttachments should be false whenever the caller doesn't actually +// need Attachments (Session.Data only does when the sender has attachment storage +// enabled) — decoding every attachment fully into memory just to discard it wastes a +// real allocation up to the size of the message's attachments, on every message, +// which matters on the small-RAM hosts this server targets. +func parseMessage(raw []byte, wantAttachments bool) (*parsedMessage, error) { msg, err := mail.ReadMessage(bytes.NewReader(raw)) if err != nil { return nil, err @@ -67,50 +83,45 @@ func parseMessage(raw []byte) (*parsedMessage, error) { out.HeaderLines = append(out.HeaderLines, k+": "+v) } } + if !wantAttachments { + return out, nil + } contentType := msg.Header.Get("Content-Type") mediaType, params, err := mime.ParseMediaType(contentType) if err != nil { mediaType = "text/plain" } + if !strings.HasPrefix(mediaType, "multipart/") { + return out, nil + } - if strings.HasPrefix(mediaType, "multipart/") { - mr := multipart.NewReader(msg.Body, params["boundary"]) - for { - part, err := mr.NextPart() - if err == io.EOF { - break - } - if err != nil { - break - } - data, _ := io.ReadAll(part) - // multipart.Reader auto-decodes quoted-printable transparently during Read, - // but not base64 (see the mime/multipart docs) — without this, a base64 - // attachment/body part is stored/relayed-for-display as raw base64 text - // instead of its actual decoded bytes. - data = decodeContentTransferEncoding(part.Header.Get("Content-Transfer-Encoding"), data) - disp, dispParams, _ := mime.ParseMediaType(part.Header.Get("Content-Disposition")) - partCT := part.Header.Get("Content-Type") - partMediaType, _, _ := mime.ParseMediaType(partCT) - - if disp == "attachment" && dispParams["filename"] != "" { - out.Attachments = append(out.Attachments, attachmentPart{ - Filename: dispParams["filename"], - ContentType: getContentType(partMediaType, dispParams["filename"]), - Data: data, - }) - continue - } - if strings.HasPrefix(partMediaType, "text/") && disp != "attachment" { - out.BodyText += string(data) + "\n" - } + mr := multipart.NewReader(msg.Body, params["boundary"]) + for { + part, err := mr.NextPart() + if err == io.EOF { + break + } + if err != nil { + break + } + disp, dispParams, _ := mime.ParseMediaType(part.Header.Get("Content-Disposition")) + if disp != "attachment" || dispParams["filename"] == "" { + continue } - } else if strings.HasPrefix(mediaType, "text/") { - data, _ := io.ReadAll(msg.Body) - out.BodyText = string(data) + data, _ := io.ReadAll(part) + // multipart.Reader auto-decodes quoted-printable transparently during Read, + // but not base64 (see the mime/multipart docs) — without this, a base64 + // attachment part is stored as raw base64 text instead of its actual decoded + // bytes. + data = decodeContentTransferEncoding(part.Header.Get("Content-Transfer-Encoding"), data) + partMediaType, _, _ := mime.ParseMediaType(part.Header.Get("Content-Type")) + out.Attachments = append(out.Attachments, attachmentPart{ + Filename: dispParams["filename"], + ContentType: getContentType(partMediaType, dispParams["filename"]), + Data: data, + }) } - out.BodyText = strings.TrimSpace(out.BodyText) return out, nil } diff --git a/internal/smtpserver/attachments_test.go b/internal/smtpserver/attachments_test.go index 18a372b..5c2ac89 100644 --- a/internal/smtpserver/attachments_test.go +++ b/internal/smtpserver/attachments_test.go @@ -29,7 +29,7 @@ func TestParseMessageDecodesBase64Attachment(t *testing.T) { "SGVsbG8sIHdvcmxkIQ==\r\n" + "--BOUND--\r\n" - parsed, err := parseMessage([]byte(raw)) + parsed, err := parseMessage([]byte(raw), true) if err != nil { t.Fatalf("parseMessage: %v", err) } @@ -65,7 +65,7 @@ func TestParseMessageLeavesNonBase64EncodingsAlone(t *testing.T) { "plain text content\r\n" + "--BOUND--\r\n" - parsed, err := parseMessage([]byte(raw)) + parsed, err := parseMessage([]byte(raw), true) if err != nil { t.Fatalf("parseMessage: %v", err) } diff --git a/internal/smtpserver/mailbox_spam_test.go b/internal/smtpserver/mailbox_spam_test.go index 3dc9c07..8ef1535 100644 --- a/internal/smtpserver/mailbox_spam_test.go +++ b/internal/smtpserver/mailbox_spam_test.go @@ -6,9 +6,11 @@ import ( "net/http/httptest" "net/smtp" "strings" + "sync/atomic" "testing" "mailgoserver/internal/db" + "mailgoserver/internal/mailstore" ) // fakeRspamd stands in for a real rspamd instance, always returning the fixed @@ -87,6 +89,75 @@ func TestRspamdScoreThresholdQuarantinesInsteadOfRejecting(t *testing.T) { } } +// TestRspamdCheckedOnceForMultipleLocalRecipients confirms one message to several +// local recipients hits rspamd exactly once, not once per recipient (see +// checkRspamdOnce in deliverLocally) — both recipients still land in Junk from that +// single check. +func TestRspamdCheckedOnceForMultipleLocalRecipients(t *testing.T) { + backend, mailboxID1 := newTestBackendWithMailbox(t) + + dek2 := mailstore.GenerateDEK() + wrapped2, nonce2, err := backend.Mailstore.WrapDEK(dek2) + if err != nil { + t.Fatal(err) + } + hash, err := db.HashPassword("portal-password-unused") + if err != nil { + t.Fatal(err) + } + mailboxID2, err := backend.DB.CreateMailbox("second@example.com", hash, 1, 5*1024*1024*1024, wrapped2, nonce2) + if err != nil { + t.Fatal(err) + } + + var calls int32 + rspamd := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + atomic.AddInt32(&calls, 1) + json.NewEncoder(w).Encode(map[string]any{"score": 20, "action": "add header"}) + })) + t.Cleanup(rspamd.Close) + backend.Cfg.Section("Rspamd").Key("enabled").SetValue("true") + backend.Cfg.Section("Rspamd").Key("url").SetValue(rspamd.URL) + backend.Cfg.Section("Rspamd").Key("reject_score").SetValue("15") + addr := startTestServer(t, backend) + + c, err := smtp.Dial(addr) + if err != nil { + t.Fatal(err) + } + defer c.Close() + if err := c.Auth(smtp.PlainAuth("", "test@example.com", "testpass123", "127.0.0.1")); err != nil { + t.Fatalf("auth: %v", err) + } + if err := c.Mail("test@example.com"); err != nil { + t.Fatalf("MAIL FROM: %v", err) + } + if err := c.Rcpt("inbox@example.com"); err != nil { + t.Fatalf("RCPT 1: %v", err) + } + if err := c.Rcpt("second@example.com"); err != nil { + t.Fatalf("RCPT 2: %v", err) + } + w, err := c.Data() + if err != nil { + t.Fatal(err) + } + w.Write([]byte("Subject: hi\r\n\r\nhi")) + if err := w.Close(); err != nil { + t.Fatalf("expected delivery accepted (quarantined), got: %v", err) + } + + if got := atomic.LoadInt32(&calls); got != 1 { + t.Fatalf("expected rspamd to be checked exactly once for 2 local recipients, got %d calls", got) + } + for _, mb := range []int64{mailboxID1, mailboxID2} { + spamMsgs, err := backend.DB.ListMessagesInFolder(mb, "Junk") + if err != nil || len(spamMsgs) != 1 { + t.Fatalf("expected 1 quarantined message in Spam for mailbox %d, got %d (err=%v)", mb, len(spamMsgs), err) + } + } +} + // firstMessage fetches the single message expected in folder, failing the test if // there isn't exactly one — a small shared helper for the tagging tests below, which // all need to inspect both the cached subject and the raw stored content. diff --git a/internal/smtpserver/server.go b/internal/smtpserver/server.go index 7fb05ca..60ea130 100644 --- a/internal/smtpserver/server.go +++ b/internal/smtpserver/server.go @@ -37,6 +37,10 @@ func applyLimits(s *smtp.Server, cfg *ini.File) { sec := cfg.Section("Mailstore") s.MaxMessageBytes = sec.Key("max_message_bytes").MustInt64(25 * 1024 * 1024) s.MaxRecipients = sec.Key("max_recipients").MustInt(100) + // Go strings are UTF-8 natively, so advertising SMTPUTF8 (RFC 6531) needs no + // backend change — unlike BINARYMIME/CHUNKING, which would require real BDAT + // support this backend doesn't implement, so those stay off. + s.EnableSMTPUTF8 = true } // NewPlainServer mirrors server_runner.py's PlainController: no TLS context at all, so diff --git a/internal/smtpserver/session.go b/internal/smtpserver/session.go index 88ff282..6bb666f 100644 --- a/internal/smtpserver/session.go +++ b/internal/smtpserver/session.go @@ -8,6 +8,7 @@ import ( "path/filepath" "strings" "time" + "unsafe" "github.com/emersion/go-smtp" "github.com/microcosm-cc/bluemonday" @@ -243,7 +244,13 @@ func (s *Session) Data(r io.Reader) error { if err != nil { return internalError("Internal server error") } - content := string(raw) + // unsafe.String views content directly over raw's own backing array instead of + // string(raw)'s real copy — on a message near [Mailstore] max_message_bytes (25MB + // default) that's a second full-message-sized allocation for no benefit, which + // matters on the small-RAM hosts this server targets. Safe only because raw is + // never mutated again below (only read, by parseMessage's own bytes.NewReader) — + // if that ever changes, this must go back to a real copy. + content := unsafe.String(unsafe.SliceData(raw), len(raw)) messageID := extractMessageID(content, s.backend.HeloHostname) senderDomain := domainOfAddr(s.mailFrom) @@ -305,7 +312,10 @@ func (s *Session) Data(r io.Reader) error { storeMessage = true } - parsed, parseErr := parseMessage(raw) + // wantAttachments=storeMessage: decoding every attachment fully into memory is + // only useful when they're about to be written to disk below — see parseMessage's + // own comment. + parsed, parseErr := parseMessage(raw, storeMessage) type savedAttachment struct { Filename, ContentType, FilePath string @@ -323,7 +333,7 @@ func (s *Session) Data(r io.Reader) error { if err := os.MkdirAll(storagePath, 0o755); err == nil { prefix := cleanMessageIDPrefix(messageID) for _, a := range parsed.Attachments { - filename := prefix + "_" + a.Filename + filename := prefix + "_" + sanitizeAttachmentFilename(a.Filename) fullPath := filepath.Join(storagePath, filename) if err := os.WriteFile(fullPath, a.Data, 0o644); err == nil { toSave = append(toSave, savedAttachment{Filename: a.Filename, ContentType: a.ContentType, FilePath: fullPath, Size: int64(len(a.Data))}) @@ -501,7 +511,7 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID // without every mailbox needing its own parse pass. Best-effort: a message this // package's own parser can't handle just never matches those two condition types. bodyText, hasAttachment := "", "no" - if parsedForRules, err := mailview.Parse([]byte(signedContent)); err == nil { + if parsedForRules, err := mailview.Parse(strings.NewReader(signedContent)); err == nil { bodyText = parsedForRules.TextBody if bodyText == "" && parsedForRules.HTMLBody != "" { bodyText = bluemonday.StrictPolicy().Sanitize(parsedForRules.HTMLBody) @@ -544,6 +554,26 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID } } + // rspamd is checked at most once per message and reused for every local recipient + // below, rather than once per recipient — confirmed safe for this deployment (no + // reliance on rspamd's per-recipient personalization, e.g. per-user Bayes/ + // whitelists); content and mail_from are identical for every recipient regardless, + // so the score/action rspamd would return doesn't actually vary by recipient here. + // Cuts what was N rspamd HTTP round-trips down to 1 for a large local fan-out. + rspamdChecked := false + var rspamdScore float64 + var rspamdAction string + var rspamdOK bool + checkRspamdOnce := func() (float64, string, bool) { + if !rspamdChecked { + if score, action, err := mailstore.CheckRspamd(rspamdURL, []byte(signedContent), s.mailFrom, rcpts[0]); err == nil { + rspamdScore, rspamdAction, rspamdOK = score, action, true + } + rspamdChecked = true + } + return rspamdScore, rspamdAction, rspamdOK + } + results := make([]relay.Result, 0, len(rcpts)) for i, rcpt := range rcpts { mbox := s.localMailboxes[strings.ToLower(rcpt)] @@ -596,7 +626,7 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID if !suppressSpam { quarantine := heuristicScore >= rejectScore if rspamdEnabled { - if score, rAction, err := mailstore.CheckRspamd(rspamdURL, []byte(signedContent), s.mailFrom, rcpt); err == nil { + if score, rAction, ok := checkRspamdOnce(); ok { // rspamd's own "reject" action is a considered policy decision // (DNSBL hit, greylisting, etc.) worth still hard-rejecting at // SMTP time to avoid backscatter; a bare score threshold hit @@ -656,12 +686,11 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID // accepted edge case, not engineered around). if mbox.ForwardTo != nil && *mbox.ForwardTo != "" { forwardTo, mailboxEmail, keepCopy := *mbox.ForwardTo, mbox.Email, mbox.ForwardKeepCopy - go func() { - res := s.backend.Relay.RelayEmailAsync(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}) + s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { if len(res) > 0 && res[0].Status != "success" { s.backend.Logger.Error("mailbox forwarding: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) } - }() + }) if !keepCopy { results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) continue @@ -685,12 +714,11 @@ func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID // mail; no SRS rewriting or Resent-* headers, matching every other // send path in this codebase. forwardTo, mailboxEmail := action.ForwardTo, mbox.Email - go func() { - res := s.backend.Relay.RelayEmailAsync(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}) + s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { if len(res) > 0 && res[0].Status != "success" { s.backend.Logger.Error("forward rule: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) } - }() + }) if !action.KeepCopy { results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) continue @@ -752,12 +780,11 @@ func (s *Session) sendAutoReply(mbox *db.Mailbox, subject, body, inReplyTo strin } mailboxEmail, replyTo := mbox.Email, s.mailFrom raw := buildAutoReplyMessage(s.backend.HeloHostname, mailboxEmail, replyTo, subject, body, inReplyTo) - go func() { - res := s.backend.Relay.RelayEmailAsync(mailboxEmail, []string{replyTo}, raw, []string{"to"}) + s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{replyTo}, raw, []string{"to"}, func(res []relay.Result) { if len(res) > 0 && res[0].Status != "success" { s.backend.Logger.Error("auto-reply to %s failed: %s", replyTo, res[0].ErrorMessage) } - }() + }) if err := s.backend.DB.RecordAutoReply(mbox.ID, replyTo); err != nil { s.backend.Logger.Error("record auto-reply to %s: %v", replyTo, err) } diff --git a/internal/toolbox/toolbox.go b/internal/toolbox/toolbox.go index 16a088e..c8c7a93 100644 --- a/internal/toolbox/toolbox.go +++ b/internal/toolbox/toolbox.go @@ -4,6 +4,7 @@ package toolbox import ( "crypto/rand" + "encoding/json" "fmt" "log" "math/big" @@ -62,13 +63,18 @@ func (l Level) String() string { } } -var globalLevel = LevelInfo +var ( + globalLevel = LevelInfo + globalJSONFormat = false +) // Configure sets the process-wide log level from settings.ini's [Logging] section, -// mirroring tool_box.setup_logging. +// mirroring tool_box.setup_logging. Also reads the optional format=json opt-in (see +// Logger.log) — default stays the original Python-parity text format. func Configure(cfg *ini.File) { section := cfg.Section("Logging") globalLevel = parseLevel(section.Key("LOG_LEVEL").MustString("INFO")) + globalJSONFormat = strings.EqualFold(section.Key("format").MustString("text"), "json") } // GetLogger returns a Logger for the given component name, mirroring tool_box.get_logger. @@ -83,6 +89,21 @@ func (l *Logger) log(level Level, format string, args ...any) { return } msg := fmt.Sprintf(format, args...) + if globalJSONFormat { + line, err := json.Marshal(struct { + Time string `json:"time"` + Logger string `json:"logger"` + Level string `json:"level"` + Message string `json:"message"` + }{time.Now().UTC().Format(time.RFC3339Nano), l.name, level.String(), msg}) + if err == nil { + l.out.Println(string(line)) + return + } + // json.Marshal only fails here on invalid UTF-8 in msg (e.g. from binary data + // interpolated into a log line) — fall through to the text format rather than + // silently dropping the line. + } ts := time.Now().Format("2006-01-02 15:04:05,000") l.out.Printf("%s - %s - %s - %s", ts, l.name, level, msg) } diff --git a/internal/webui/view_message.go b/internal/webui/view_message.go index 28a6ab8..835069a 100644 --- a/internal/webui/view_message.go +++ b/internal/webui/view_message.go @@ -62,7 +62,7 @@ func (a *App) viewMessageContent(w http.ResponseWriter, r *http.Request) { var plainBody string var attachments []viewedAttachment if log.MessageBody != "" { - if parsed, err := mailview.Parse([]byte(log.MessageBody)); err == nil { + if parsed, err := mailview.Parse(strings.NewReader(log.MessageBody)); err == nil { if parsed.HTMLBody != "" { htmlBody = template.HTML(htmlBodyPolicy.Sanitize(parsed.HTMLBody)) } diff --git a/internal/webui/webmail_client_test.go b/internal/webui/webmail_client_test.go index ac020f5..0172dec 100644 --- a/internal/webui/webmail_client_test.go +++ b/internal/webui/webmail_client_test.go @@ -337,7 +337,7 @@ func TestWebmailComposeHTMLBodyRoundTrip(t *testing.T) { if err != nil { t.Fatal(err) } - parsed, err := mailview.Parse(raw) + parsed, err := mailview.Parse(bytes.NewReader(raw)) if err != nil { t.Fatal(err) } diff --git a/internal/webui/webmail_compose.go b/internal/webui/webmail_compose.go index ef546e3..d58fb09 100644 --- a/internal/webui/webmail_compose.go +++ b/internal/webui/webmail_compose.go @@ -232,7 +232,7 @@ func (a *App) webmailLoadForPrefill(mailboxID int64, folder string, uid int64) * if err != nil { return nil } - parsed, err := mailview.Parse(raw) + parsed, err := mailview.Parse(bytes.NewReader(raw)) if err != nil { return nil } diff --git a/internal/webui/webmail_download_forward_test.go b/internal/webui/webmail_download_forward_test.go index ba28460..b16e5a8 100644 --- a/internal/webui/webmail_download_forward_test.go +++ b/internal/webui/webmail_download_forward_test.go @@ -1,6 +1,7 @@ package webui import ( + "bytes" "net/http" "net/http/httptest" "net/url" @@ -168,7 +169,7 @@ func TestWebmailComposeSendForwardAttach(t *testing.T) { if err != nil { t.Fatal(err) } - parsed, err := mailview.Parse(rawReceived) + parsed, err := mailview.Parse(bytes.NewReader(rawReceived)) if err != nil { t.Fatal(err) } @@ -225,7 +226,7 @@ func TestWebmailComposeSendMultipleForwardAttach(t *testing.T) { if err != nil { t.Fatal(err) } - parsed, err := mailview.Parse(rawReceived) + parsed, err := mailview.Parse(bytes.NewReader(rawReceived)) if err != nil { t.Fatal(err) } diff --git a/internal/webui/webmail_mail.go b/internal/webui/webmail_mail.go index 6f8c94d..bf1d774 100644 --- a/internal/webui/webmail_mail.go +++ b/internal/webui/webmail_mail.go @@ -2,6 +2,7 @@ package webui import ( "archive/zip" + "bytes" "encoding/base64" "fmt" "html/template" @@ -365,13 +366,13 @@ func (a *App) renderFolderOrSearch(w http.ResponseWriter, r *http.Request, folde "has_next": offset+len(rows) < total, "has_prev": page > 1, "search_query": query, "unread_counts": unreadCounts, "folder_counts": folderCounts, "unread_only": unreadOnly, "starred_only": starredOnly, "sort_by": sortBy, "sort_dir": sortDir, - "sort_from_href": sortLink("from", unreadOnly, starredOnly, sortBy, sortDir), - "sort_date_href": sortLink("", unreadOnly, starredOnly, sortBy, sortDir), - "unread_only_href": unreadOnlyHref, + "sort_from_href": sortLink("from", unreadOnly, starredOnly, sortBy, sortDir), + "sort_date_href": sortLink("", unreadOnly, starredOnly, sortBy, sortDir), + "unread_only_href": unreadOnlyHref, "starred_only_href": starredOnlyHref, - "prev_href": pageHref(page - 1), - "next_href": pageHref(page + 1), - "flashes": popFlashes(w, r), + "prev_href": pageHref(page - 1), + "next_href": pageHref(page + 1), + "flashes": popFlashes(w, r), }) } @@ -478,7 +479,7 @@ func (a *App) loadMessageForView(w http.ResponseWriter, r *http.Request, mbox *d return nil, false } unwrapped, smimeStatus, pgpStatus := a.unwrapCrypto(r, mbox.ID, raw) - parsed, err := mailview.Parse(unwrapped) + parsed, err := mailview.Parse(bytes.NewReader(unwrapped)) if err != nil { a.Logger.Error("parse message %d for mailbox %d: %v", uid, mbox.ID, err) setFlash(w, "error", "Error reading message") @@ -632,7 +633,7 @@ func (a *App) webmailAlwaysAllowImages(w http.ResponseWriter, r *http.Request) { return } unwrapped, _, _ := a.unwrapCrypto(r, mbox.ID, raw) - parsed, err := mailview.Parse(unwrapped) + parsed, err := mailview.Parse(bytes.NewReader(unwrapped)) senderEmail := "" if err == nil { senderEmail = extractAddress(parsed.Header.From) @@ -762,7 +763,7 @@ func (a *App) webmailMarkAsJunk(w http.ResponseWriter, r *http.Request) { return } unwrapped, _, _ := a.unwrapCrypto(r, mbox.ID, raw) - parsed, parseErr := mailview.Parse(unwrapped) + parsed, parseErr := mailview.Parse(bytes.NewReader(unwrapped)) if err := a.DB.MoveMessage(mbox.ID, uid, "Junk"); err != nil { setFlash(w, "error", "Error marking as junk") @@ -905,7 +906,7 @@ func (a *App) webmailAttachmentDownload(w http.ResponseWriter, r *http.Request) return } unwrapped, _, _ := a.unwrapCrypto(r, mbox.ID, raw) - parsed, err := mailview.Parse(unwrapped) + parsed, err := mailview.Parse(bytes.NewReader(unwrapped)) if err != nil || idx < 0 || idx >= len(parsed.Attachments) { http.NotFound(w, r) return @@ -933,7 +934,7 @@ func (a *App) webmailDownloadAllAttachments(w http.ResponseWriter, r *http.Reque return } unwrapped, _, _ := a.unwrapCrypto(r, mbox.ID, raw) - parsed, err := mailview.Parse(unwrapped) + parsed, err := mailview.Parse(bytes.NewReader(unwrapped)) if err != nil || len(parsed.Attachments) == 0 { http.NotFound(w, r) return diff --git a/main.go b/main.go index 8bc4561..71c612b 100644 --- a/main.go +++ b/main.go @@ -14,6 +14,7 @@ import ( "os" "os/signal" "path/filepath" + "sync" "sync/atomic" "syscall" "time" @@ -320,7 +321,7 @@ func main() { logger.Error("plain IMAP listen: %v", err) return } - if err := plainServer.Serve(abuseguard.GuardListener(l, database, cfg, logger)); err != nil && !errors.Is(err, net.ErrClosed) { + if err := plainServer.Serve(imapserver.IdleTimeoutListener(abuseguard.GuardListener(l, database, cfg, logger), 30*time.Minute)); err != nil && !errors.Is(err, net.ErrClosed) { logger.Error("plain IMAP server: %v", err) } }() @@ -330,7 +331,7 @@ func main() { logger.Error("TLS IMAP listen: %v", err) return } - if err := tlsServer.Serve(abuseguard.GuardListener(imapTLSListener, database, cfg, logger)); err != nil && !errors.Is(err, net.ErrClosed) { + if err := tlsServer.Serve(imapserver.IdleTimeoutListener(abuseguard.GuardListener(imapTLSListener, database, cfg, logger), 30*time.Minute)); err != nil && !errors.Is(err, net.ErrClosed) { logger.Error("TLS IMAP server: %v", err) } }() @@ -399,6 +400,45 @@ func main() { } go runScheduledBackups() + // Prunes send/receive log history (and any attachments stored alongside it) older + // than [Mailstore] email_log_retention_days — off by default (0), matching the + // "blank/0 means manual/off" convention used by the backup schedule and DKIM + // rotation above, since esrv_email_logs otherwise grows forever with no cleanup at + // all. Checked hourly, same cadence as the backup job, for the same reason (cheap, + // and a live Settings change takes effect without a restart). + runLogRetention := func() { + check := func() { + days := cfg.Section("Mailstore").Key("email_log_retention_days").MustInt(0) + if days <= 0 { + return + } + cutoff := time.Now().Add(-time.Duration(days) * 24 * time.Hour) + paths, deleted, err := database.PruneEmailLogsOlderThan(cutoff) + if err != nil { + logger.Error("log retention: %v", err) + return + } + for _, p := range paths { + if p == "" { + continue + } + if err := os.Remove(p); err != nil && !os.IsNotExist(err) { + logger.Error("log retention: remove attachment %s: %v", p, err) + } + } + if deleted > 0 { + logger.Info("log retention: pruned %d email log(s) older than %d days", deleted, days) + } + } + check() + ticker := time.NewTicker(1 * time.Hour) + defer ticker.Stop() + for range ticker.C { + check() + } + } + go runLogRetention() + // Optional auto-rotation of the shared/global DKIM key (CNAME delegation) — // manual-only (no ticker work beyond the no-op check) when [DKIM] // global_dkim_rotation_days is blank/0, same "blank means manual" convention as @@ -451,15 +491,32 @@ func main() { } go runGlobalDKIMRotation() + // Created once, up front, so runRelayQueueWorker (below) and waitForShutdown share + // the same signal — otherwise the worker would have no way to know shutdown had + // started and could keep kicking off fresh batches until the process is killed + // mid-delivery. + sigCtx, stopSignal := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer stopSignal() + // Delivers messages accepted onto esrv_relay_queue by Session.Data (see // internal/relay/queue.go) — a 5s tick keeps delivery prompt without polling too // aggressively; 10 concurrent deliveries / 50 per tick are fixed, not config, same - // "keeps this simple" precedent as retrySchedule. + // "keeps this simple" precedent as retrySchedule. Stops starting new batches once + // sigCtx fires; relayWorkerWG lets waitForShutdown wait for an already-in-progress + // batch to actually finish instead of killing it mid-delivery. + var relayWorkerWG sync.WaitGroup runRelayQueueWorker := func() { ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() - for range ticker.C { - relayer.ProcessQueueOnce(10, 50) + for { + select { + case <-sigCtx.Done(): + return + case <-ticker.C: + relayWorkerWG.Add(1) + relayer.ProcessQueueOnce(10, 50) + relayWorkerWG.Done() + } } } go runRelayQueueWorker() @@ -472,9 +529,7 @@ func main() { // go-imap/v2's Server only has Close() (force-close, confirmed no graceful variant // exists in that library) — still better than no shutdown handling at all. waitForShutdown := func(smtpPlain, smtpTLS *smtp.Server, imapPlain, imapTLS *goimapserver.Server, httpSrv, httpsSrv *http.Server) { - ctx, stop := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) - defer stop() - <-ctx.Done() + <-sigCtx.Done() logger.Info("shutdown signal received, draining connections...") shutdownCtx, cancel := context.WithTimeout(context.Background(), 25*time.Second) @@ -497,6 +552,17 @@ func main() { if httpsSrv != nil { httpsSrv.Shutdown(shutdownCtx) } + + relayDone := make(chan struct{}) + go func() { + relayWorkerWG.Wait() + close(relayDone) + }() + select { + case <-relayDone: + case <-shutdownCtx.Done(): + logger.Warning("shutdown: relay queue worker still delivering after the drain timeout, exiting anyway") + } logger.Info("shutdown complete") } @@ -526,6 +592,9 @@ func main() { mux.HandleFunc("GET /health", func(w http.ResponseWriter, r *http.Request) { writeHealthJSON(w, database, smtpRunning.Load()) }) + mux.HandleFunc("GET /metrics", func(w http.ResponseWriter, r *http.Request) { + writeMetricsText(w, database) + }) // Most visitors are mailbox owners, not admins — default the bare root to the // self-service webmail login, with a "Login as Admin" button there for the admin // dashboard's login instead of requiring the admin URL to be typed by hand. @@ -550,7 +619,11 @@ func main() { httpsPort := cfg.Section("Server").Key("WEB_HTTPS_PORT").MustInt(5001) httpsAddr := fmt.Sprintf("%s:%d", *host, httpsPort) - httpsServer := &http.Server{Addr: httpsAddr, Handler: handler, TLSConfig: webHTTPSConfig} + // ReadHeaderTimeout is the slowloris fix (bounds only the time to receive headers); + // ReadTimeout/WriteTimeout are deliberately left unset — webmailMux serves an + // SSE stream (webmail_mail.go's event-stream handler) and attachment downloads, + // both legitimately long-lived, and a global deadline would cut them off. + httpsServer := &http.Server{Addr: httpsAddr, Handler: handler, TLSConfig: webHTTPSConfig, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 120 * time.Second} go func() { logger.Info("Web interface (HTTPS) starting at https://%s", httpsAddr) // Empty cert/key paths: TLSConfig.GetCertificate (backed by certReloader) supplies @@ -567,7 +640,7 @@ func main() { httpPort = cfg.Section("Server").Key("WEB_HTTP_PORT").MustInt(5000) } addr := fmt.Sprintf("%s:%d", *host, httpPort) - httpServer := &http.Server{Addr: addr, Handler: handler} + httpServer := &http.Server{Addr: addr, Handler: handler, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 120 * time.Second} go func() { logger.Info("Web interface starting at http://%s (debug=%v)", addr, *debug) if err := httpServer.ListenAndServe(); err != nil && !errors.Is(err, http.ErrServerClosed) { @@ -602,3 +675,34 @@ func writeHealthJSON(w http.ResponseWriter, database *db.DB, smtpUp bool) { fmt.Fprintf(w, `{"status":%q,"timestamp":%q,"services":{"smtp_server":%q,"web_frontend":"running","database":%q},"version":"1.0.0"}`, overall, time.Now().Format(time.RFC3339), smtpStatus, dbStatus) } + +// writeMetricsText serves a hand-written Prometheus text-exposition response (no +// client_golang dependency needed for this small, fixed set of gauges) reusing the +// same DB queries the admin dashboard already runs, so this doesn't add new query +// load beyond what /health and the dashboard already cause. +func writeMetricsText(w http.ResponseWriter, database *db.DB) { + w.Header().Set("Content-Type", "text/plain; version=0.0.4") + + dbUp := 1 + if err := database.Ping(); err != nil { + dbUp = 0 + } + fmt.Fprintf(w, "# HELP mailgoserver_db_up Whether the database connection is healthy (1) or not (0).\n") + fmt.Fprintf(w, "# TYPE mailgoserver_db_up gauge\n") + fmt.Fprintf(w, "mailgoserver_db_up %d\n", dbUp) + + if pending, err := database.CountPendingRelayQueueItems(); err == nil { + fmt.Fprintf(w, "# HELP mailgoserver_relay_queue_pending Outbound relay domain-groups still awaiting delivery.\n") + fmt.Fprintf(w, "# TYPE mailgoserver_relay_queue_pending gauge\n") + fmt.Fprintf(w, "mailgoserver_relay_queue_pending %d\n", pending) + } + + if success, failed, err := database.DeliveryStats(24); err == nil { + fmt.Fprintf(w, "# HELP mailgoserver_delivery_success_24h Successful recipient deliveries in the last 24h.\n") + fmt.Fprintf(w, "# TYPE mailgoserver_delivery_success_24h gauge\n") + fmt.Fprintf(w, "mailgoserver_delivery_success_24h %d\n", success) + fmt.Fprintf(w, "# HELP mailgoserver_delivery_failed_24h Failed recipient deliveries in the last 24h.\n") + fmt.Fprintf(w, "# TYPE mailgoserver_delivery_failed_24h gauge\n") + fmt.Fprintf(w, "mailgoserver_delivery_failed_24h %d\n", failed) + } +}