diff --git a/.env.example b/.env.example index d5d5ee0..fcf9e01 100644 --- a/.env.example +++ b/.env.example @@ -9,3 +9,4 @@ IMAP_TLS_PORT=993 ACME_HTTP_PORT=80 WEB_HTTP_PORT=5000 WEB_HTTPS_PORT=5001 +JMAP_PORT=8443 diff --git a/Dockerfile.aio b/Dockerfile.aio index b2000d9..831c55d 100644 --- a/Dockerfile.aio +++ b/Dockerfile.aio @@ -71,7 +71,7 @@ VOLUME ["/app/server_data"] # anything mailgoserver itself uses. Add a `password` to rspamd's controller worker # config and publish it yourself if you want it. Port 6379 (redis) is never exposed at # all — loopback-only, see entrypoint-aio.sh. -EXPOSE 25 465 143 993 80 5000 5001 +EXPOSE 25 465 143 993 80 5000 5001 8443 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ CMD curl -fs http://127.0.0.1:5000/health || exit 1 diff --git a/README.md b/README.md index 0c1253f..5c6dc3a 100644 --- a/README.md +++ b/README.md @@ -15,6 +15,12 @@ a Python venv + separate services. client) with AES-encrypted-at-rest message storage, per-mailbox quotas, send-as aliases, and filter rules (move to folder, forward, discard, based on from/subject/ body conditions with AND/OR logic). +- **JMAP** (RFC 8620/8621) — a modern HTTP+JSON alternative to IMAP for mail clients that + support it, on its own dedicated port. Reads and writes the exact same encrypted + mailboxes IMAP uses (not a separate mail store): browse/search/flag/move/delete mail, + real thread grouping, send-as identities with signatures, sending mail, attachment + upload/download, and live push for new-mail notifications. On by default + (`JMAP_ENABLE`). - **Spam filtering** — a built-in heuristic score always runs; optionally point it at an [rspamd](https://rspamd.com) instance for a lot more signal (see `docker-deploy/` for a container that bundles rspamd for you). @@ -91,7 +97,9 @@ handles for you. Admin/webmail HTTP `5000` / HTTPS `5001` stay deliberately non-privileged; put a reverse proxy or your own `80`/`443` mapping in front of those if you want the dashboard on standard web ports too. Port `80` is also used, but only transiently, if you enable Let's Encrypt's HTTP-01 challenge (see below) — the same -setcap/root/Docker rule applies to it as to the mail ports. +setcap/root/Docker rule applies to it as to the mail ports. JMAP listens on its own +dedicated HTTPS port, `8443` by default (`[Server] JMAP_PORT`) — always TLS (JMAP mandates +it), non-privileged like the admin/webmail ports, no setcap needed. ## Build @@ -175,7 +183,8 @@ Compose profiles. None of the three run as root. Three independent listeners need a TLS certificate: SMTP direct-TLS (465), IMAP direct-TLS (993), and the admin/webmail HTTPS UI (5001). Each can be assigned a different one, from the admin dashboard's **Settings** page (TLS/SSL Configuration -card): +card). JMAP (8443) always reuses the admin/webmail UI's certificate — it's not a +fourth independently-assignable slot. - **Custom** — self-signed by default (generated on first run), or your own uploaded cert/key. diff --git a/docker-compose.yml b/docker-compose.yml index 2b92b2c..a85f49c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -16,6 +16,7 @@ services: - "${ACME_HTTP_PORT:-80}:80" - "${WEB_HTTP_PORT:-5000}:5000" - "${WEB_HTTPS_PORT:-5001}:5001" + - "${JMAP_PORT:-8443}:8443" volumes: - mailserver-aio-data:/app/server_data diff --git a/docker-deploy/Dockerfile b/docker-deploy/Dockerfile index 56fa361..5412166 100644 --- a/docker-deploy/Dockerfile +++ b/docker-deploy/Dockerfile @@ -55,7 +55,7 @@ USER mailgoserver # those too if you want them there). Port 80 is only actually bound while # [LetsEncrypt] challenge_type=http-01 is enabled and an obtain/renew is in flight — # harmless to expose even when unused. -EXPOSE 25 465 143 993 80 5000 5001 +EXPOSE 25 465 143 993 80 5000 5001 8443 # --host 0.0.0.0 is required: the binary's own default is 127.0.0.1, which would only # be reachable from inside this container, never through a published port. diff --git a/docker-deploy/Dockerfile.rspamd b/docker-deploy/Dockerfile.rspamd index 3257db2..2d26017 100644 --- a/docker-deploy/Dockerfile.rspamd +++ b/docker-deploy/Dockerfile.rspamd @@ -58,7 +58,7 @@ VOLUME ["/app/server_data"] # Port 80 is only actually bound while [LetsEncrypt] challenge_type=http-01 is enabled # and an obtain/renew is in flight — harmless to expose even when unused. -EXPOSE 25 465 143 993 80 5000 5001 +EXPOSE 25 465 143 993 80 5000 5001 8443 HEALTHCHECK --interval=30s --timeout=5s --start-period=15s --retries=3 \ CMD curl -fs http://127.0.0.1:5000/health || exit 1 diff --git a/docker-deploy/README.md b/docker-deploy/README.md index 2fe91cb..1040e06 100644 --- a/docker-deploy/README.md +++ b/docker-deploy/README.md @@ -25,13 +25,14 @@ docker compose --profile all-in-one up -d --build # mailserver + rspamd + r ``` Either way, the app itself binds the real standard mail ports by default — 25 -(SMTP), 465 (direct-TLS SMTP), 143 (IMAP), 993 (direct-TLS IMAP) — and the admin/webmail -UI on its usual non-privileged 5000/5001 (HTTP/HTTPS); put your own reverse proxy or a -`80:5000`/`443:5001` port mapping in front if you want those on 80/443 too (note port 80 -is already published here for Let's Encrypt HTTP-01, see below — pick a different host -port for the web UI's 80 mapping if you use both). Copy `.env.example` to `.env` in this -folder to change any host-side port — useful if something else on the host already owns -25/143/etc., or if you want to run more than one profile side by side. +(SMTP), 465 (direct-TLS SMTP), 143 (IMAP), 993 (direct-TLS IMAP) — the admin/webmail +UI on its usual non-privileged 5000/5001 (HTTP/HTTPS), and JMAP on 8443 (also +non-privileged); put your own reverse proxy or a `80:5000`/`443:5001` port mapping in +front if you want those on 80/443 too (note port 80 is already published here for +Let's Encrypt HTTP-01, see below — pick a different host port for the web UI's 80 +mapping if you use both). Copy `.env.example` to `.env` in this folder to change any +host-side port — useful if something else on the host already owns 25/143/etc., or if +you want to run more than one profile side by side. ## Security: nothing here runs as root diff --git a/docker-deploy/docker-compose.yml b/docker-deploy/docker-compose.yml index 11faa8c..a4ebcda 100644 --- a/docker-deploy/docker-compose.yml +++ b/docker-deploy/docker-compose.yml @@ -33,6 +33,7 @@ services: - "${ACME_HTTP_PORT:-80}:80" - "${WEB_HTTP_PORT:-5000}:5000" - "${WEB_HTTPS_PORT:-5001}:5001" + - "${JMAP_PORT:-8443}:8443" volumes: - mailserver-data:/app/server_data @@ -51,6 +52,7 @@ services: - "${ACME_HTTP_PORT:-80}:80" - "${WEB_HTTP_PORT:-5000}:5000" - "${WEB_HTTPS_PORT:-5001}:5001" + - "${JMAP_PORT:-8443}:8443" volumes: - mailserver-rspamd-data:/app/server_data @@ -69,6 +71,7 @@ services: - "${ACME_HTTP_PORT:-80}:80" - "${WEB_HTTP_PORT:-5000}:5000" - "${WEB_HTTPS_PORT:-5001}:5001" + - "${JMAP_PORT:-8443}:8443" volumes: - mailserver-aio-data:/app/server_data diff --git a/internal/config/config.go b/internal/config/config.go index b2a90b1..9c70e6b 100644 --- a/internal/config/config.go +++ b/internal/config/config.go @@ -23,6 +23,18 @@ var defaults = []struct { Keys []defaultKV }{ {"Server", []defaultKV{ + {"", "", "IMAP server configuration for mailbox retrieval (Thunderbird, etc.)"}, + {"", "", "Plain IMAP port (STARTTLS not offered, matching the SMTP plain-port design)"}, + {"IMAP_PORT", "143", ""}, + {"", "", "Implicit-TLS IMAP port (IMAPS)"}, + {"IMAP_TLS_PORT", "993", ""}, + {"", "", "JMAP (RFC 8620/8621) mail access, an HTTP+JSON alternative to IMAP — reads/"}, + {"", "", "writes the exact same mailboxes IMAP and webmail already use, not a separate"}, + {"", "", "mail store. Runs on its own dedicated HTTPS port (always TLS, the same"}, + {"", "", "certificate the web interface uses), not the web interface's own port."}, + {"JMAP_ENABLE", "true", ""}, + {"", "", "Dedicated JMAP port"}, + {"JMAP_PORT", "8443", ""}, {"", "", "Server configuration for SMTP ports and hostname"}, {"", "", "Plain SMTP port for internal/whitelisted IPs"}, {"", "", "(standard port 25 — the process needs CAP_NET_BIND_SERVICE or root to bind"}, @@ -155,13 +167,6 @@ var defaults = []struct { {"", "", "0 = unlimited (previous behavior)."}, {"max_total_connections", "1000", ""}, }}, - {"IMAP", []defaultKV{ - {"", "", "IMAP server configuration for mailbox retrieval (Thunderbird, etc.)"}, - {"", "", "Plain IMAP port (STARTTLS not offered, matching the SMTP plain-port design)"}, - {"IMAP_PORT", "143", ""}, - {"", "", "Implicit-TLS IMAP port (IMAPS)"}, - {"IMAP_TLS_PORT", "993", ""}, - }}, {"Mailstore", []defaultKV{ {"", "", "Local mailbox storage configuration"}, {"", "", "Directory where encrypted message blobs are written"}, @@ -327,12 +332,47 @@ func Load(path string) (*ini.File, error) { if err != nil { return nil, err } + migrated, err := migrateLegacySections(cfg) + if err != nil { + return nil, err + } + if migrated { + if err := cfg.SaveTo(path); err != nil { + return nil, err + } + } if err := backfillMissingDefaults(cfg, path); err != nil { return nil, err } return cfg, nil } +// migrateLegacySections moves IMAP_PORT/IMAP_TLS_PORT/JMAP_ENABLE/JMAP_PORT out of the +// standalone [IMAP]/[JMAP] sections an older version of this program wrote them into, and +// into [Server] where the defaults table now places them - preserving whatever value the +// admin already configured (not resetting it to default) - then deletes the now-empty +// legacy sections. A no-op after the first run, since those sections won't exist anymore. +func migrateLegacySections(cfg *ini.File) (bool, error) { + changed := false + server := cfg.Section("Server") + for _, secName := range []string{"IMAP", "JMAP"} { + if !cfg.HasSection(secName) { + continue + } + old := cfg.Section(secName) + for _, k := range old.Keys() { + if !server.HasKey(k.Name()) { + if _, err := server.NewKey(k.Name(), k.Value()); err != nil { + return false, err + } + } + } + cfg.DeleteSection(secName) + changed = true + } + return changed, nil +} + // backfillMissingDefaults adds any defaults-table key not already present in cfg, // leaving every existing key's value untouched, and saves to path only if it actually // added something. diff --git a/internal/config/config_test.go b/internal/config/config_test.go index bc05eaf..eac16ed 100644 --- a/internal/config/config_test.go +++ b/internal/config/config_test.go @@ -74,6 +74,60 @@ func TestLoadBackfillsMissingKeysWithoutTouchingExistingValues(t *testing.T) { } } +func TestLoadMigratesLegacyIMAPAndJMAPSectionsIntoServer(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, "settings.ini") + + // Simulate a settings.ini from before IMAP_PORT/JMAP_* moved into [Server], with a + // non-default port the admin actually configured - Load must preserve that value, + // not silently reset it to default just because it's now read from a new section. + old := "[Server]\nHOSTNAME = old.example.com\n\n[IMAP]\nIMAP_PORT = 1143\nIMAP_TLS_PORT = 1993\n\n[JMAP]\nJMAP_ENABLE = false\nJMAP_PORT = 9443\n" + if err := os.WriteFile(path, []byte(old), 0o644); err != nil { + t.Fatal(err) + } + + cfg, err := Load(path) + if err != nil { + t.Fatalf("Load: %v", err) + } + if got := cfg.Section("Server").Key("IMAP_PORT").String(); got != "1143" { + t.Errorf("IMAP_PORT not migrated with its configured value: got %q, want 1143", got) + } + if got := cfg.Section("Server").Key("IMAP_TLS_PORT").String(); got != "1993" { + t.Errorf("IMAP_TLS_PORT not migrated with its configured value: got %q, want 1993", got) + } + if got := cfg.Section("Server").Key("JMAP_ENABLE").String(); got != "false" { + t.Errorf("JMAP_ENABLE not migrated with its configured value: got %q, want false", got) + } + if got := cfg.Section("Server").Key("JMAP_PORT").String(); got != "9443" { + t.Errorf("JMAP_PORT not migrated with its configured value: got %q, want 9443", got) + } + if cfg.HasSection("IMAP") || cfg.HasSection("JMAP") { + t.Error("legacy [IMAP]/[JMAP] sections were not removed") + } + + // Migration must be persisted to disk, not just held in memory. + raw, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if strings.Contains(string(raw), "[IMAP]") || strings.Contains(string(raw), "[JMAP]") { + t.Error("legacy sections still present in saved settings.ini") + } + if !strings.Contains(string(raw), "IMAP_PORT") { + t.Error("migrated key not saved back to settings.ini") + } + + // Loading a second time (now already migrated) must be a stable no-op. + cfg2, err := Load(path) + if err != nil { + t.Fatalf("second Load: %v", err) + } + if got := cfg2.Section("Server").Key("IMAP_PORT").String(); got != "1143" { + t.Errorf("second Load: IMAP_PORT = %q, want 1143", got) + } +} + func TestAbsoluteSQLitePath(t *testing.T) { cases := []struct{ url, root, want string }{ {"sqlite:///server_data/db.sqlite", "/app", "/app/server_data/db.sqlite"}, diff --git a/internal/db/crud_jmap_state.go b/internal/db/crud_jmap_state.go new file mode 100644 index 0000000..cee3aaf --- /dev/null +++ b/internal/db/crud_jmap_state.go @@ -0,0 +1,125 @@ +package db + +import ( + "database/sql" + "strconv" +) + +// MessagesState returns the current JMAP (internal/jmap) Email state string for +// mailboxID — the highest modseq across both esrv_mailbox_messages and its tombstones +// table, since a destroy must advance the reported state even though the message row +// itself is gone by then. +func (d *DB) MessagesState(mailboxID int64) (string, error) { + var msgMax, tombMax int64 + if err := d.QueryRow(`SELECT COALESCE(MAX(modseq), 0) FROM esrv_mailbox_messages WHERE mailbox_id = ?`, mailboxID).Scan(&msgMax); err != nil { + return "", err + } + if err := d.QueryRow(`SELECT COALESCE(MAX(modseq), 0) FROM esrv_mailbox_message_tombstones WHERE mailbox_id = ?`, mailboxID).Scan(&tombMax); err != nil { + return "", err + } + max := msgMax + if tombMax > max { + max = tombMax + } + return strconv.FormatInt(max, 10), nil +} + +// FoldersState returns the current JMAP (internal/jmap) Mailbox state string for +// mailboxID. +func (d *DB) FoldersState(mailboxID int64) (string, error) { + var max int64 + err := d.QueryRow(`SELECT COALESCE(MAX(modseq), 0) FROM esrv_mailbox_folders WHERE mailbox_id = ?`, mailboxID).Scan(&max) + return strconv.FormatInt(max, 10), err +} + +// MessageChanges is one Email/changes result (internal/jmap): what changed for +// mailboxID since sinceState. No maxChanges paging in v1 (see the JMAP plan's +// disclosed scope cuts) — HasMore is always false. +type MessageChanges struct { + Created []int64 + Updated []int64 + Destroyed []int64 + NewState string + HasMore bool +} + +func scanInt64Rows(rows *sql.Rows) ([]int64, error) { + defer rows.Close() + var out []int64 + for rows.Next() { + var id int64 + if err := rows.Scan(&id); err != nil { + return nil, err + } + out = append(out, id) + } + return out, rows.Err() +} + +// MessageChangesSince computes MessageChanges for mailboxID against sinceState (a +// modseq value, as returned by a prior MessagesState/MessageChangesSince call) — see +// schema.go's comment on esrv_mailbox_messages.modseq/created_modseq for the created/ +// updated/destroyed derivation. +func (d *DB) MessageChangesSince(mailboxID, sinceState int64) (MessageChanges, error) { + var out MessageChanges + + createdRows, err := d.Query(`SELECT id FROM esrv_mailbox_messages WHERE mailbox_id = ? AND created_modseq > ?`, mailboxID, sinceState) + if err != nil { + return out, err + } + if out.Created, err = scanInt64Rows(createdRows); err != nil { + return out, err + } + + updatedRows, err := d.Query(`SELECT id FROM esrv_mailbox_messages WHERE mailbox_id = ? AND modseq > ? AND created_modseq <= ?`, mailboxID, sinceState, sinceState) + if err != nil { + return out, err + } + if out.Updated, err = scanInt64Rows(updatedRows); err != nil { + return out, err + } + + destroyedRows, err := d.Query(`SELECT message_id FROM esrv_mailbox_message_tombstones WHERE mailbox_id = ? AND modseq > ?`, mailboxID, sinceState) + if err != nil { + return out, err + } + if out.Destroyed, err = scanInt64Rows(destroyedRows); err != nil { + return out, err + } + + out.NewState, err = d.MessagesState(mailboxID) + return out, err +} + +// FolderChanges is one Mailbox/changes result (internal/jmap). No destroyed list: a +// JMAP Mailbox/set destroy reparents under Trash (db.MoveFolderToTrash), the row never +// disappears — see schema.go's comment on esrv_mailbox_folders.modseq. +type FolderChanges struct { + Created []int64 + Updated []int64 + NewState string + HasMore bool +} + +func (d *DB) FolderChangesSince(mailboxID, sinceState int64) (FolderChanges, error) { + var out FolderChanges + + createdRows, err := d.Query(`SELECT id FROM esrv_mailbox_folders WHERE mailbox_id = ? AND created_modseq > ?`, mailboxID, sinceState) + if err != nil { + return out, err + } + if out.Created, err = scanInt64Rows(createdRows); err != nil { + return out, err + } + + updatedRows, err := d.Query(`SELECT id FROM esrv_mailbox_folders WHERE mailbox_id = ? AND modseq > ? AND created_modseq <= ?`, mailboxID, sinceState, sinceState) + if err != nil { + return out, err + } + if out.Updated, err = scanInt64Rows(updatedRows); err != nil { + return out, err + } + + out.NewState, err = d.FoldersState(mailboxID) + return out, err +} diff --git a/internal/db/crud_mailbox_folders.go b/internal/db/crud_mailbox_folders.go index e41ce23..a1ac0a4 100644 --- a/internal/db/crud_mailbox_folders.go +++ b/internal/db/crud_mailbox_folders.go @@ -23,6 +23,36 @@ import ( // "Spam" folder before this rename. var StandardMailboxFolders = []string{"INBOX", "Junk", "Sent", "Drafts", "Trash"} +// nextFolderModseq returns the next per-mailbox modseq value for esrv_mailbox_folders, +// same reasoning as crud_mailbox_messages.go's nextMessageModseq (safe without its own +// transaction thanks to sqlDB.SetMaxOpenConns(1)). Backs JMAP's (internal/jmap) +// Mailbox/changes incremental sync. +func (d *DB) nextFolderModseq(mailboxID int64) (int64, error) { + var n int64 + err := d.QueryRow(`SELECT COALESCE(MAX(modseq), 0) + 1 FROM esrv_mailbox_folders WHERE mailbox_id = ?`, mailboxID).Scan(&n) + return n, err +} + +// EnsureFolderRow materializes a real esrv_mailbox_folders row for name if one doesn't +// already exist — needed because the 5 standard folders otherwise only get a row the +// first time they're renamed/reparented/reordered (see the schema comment above). +// JMAP's (internal/jmap) Mailbox.id must be stable and immutable, so every folder a +// JMAP client can see needs a real row by the time it's first exposed, standard or +// not. Idempotent via the same ON CONFLICT idiom SetFolderOrder already uses. +func (d *DB) EnsureFolderRow(mailboxID int64, name string) (int64, error) { + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return 0, err + } + if _, err := d.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, position, modseq, created_modseq) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(mailbox_id, name) DO NOTHING`, mailboxID, name, unpositionedFolder, modseq, modseq); err != nil { + return 0, err + } + var id int64 + err = d.QueryRow(`SELECT id FROM esrv_mailbox_folders WHERE mailbox_id = ? AND name = ?`, mailboxID, name).Scan(&id) + return id, err +} + func isStandardFolderName(name string) bool { for _, f := range StandardMailboxFolders { if f == name { @@ -101,14 +131,24 @@ func (d *DB) FolderPositions(mailboxID int64) (map[string]int, error) { // — order is that parent's full children list top to bottom (never the whole tree; // positions only mean something relative to siblings, see FolderPositions). func (d *DB) SetFolderOrder(mailboxID int64, order []string) error { + // Computed before Begin() — same self-deadlock reasoning as MoveFolderToTrash's. + // One shared modseq for the whole reorder — every listed sibling changed together + // as one logical event, same batch-mutation reasoning as MarkAllReadInFolder. + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } tx, err := d.Begin() if err != nil { return err } defer tx.Rollback() for i, name := range order { - if _, err := tx.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, position) VALUES (?, ?, ?) - ON CONFLICT(mailbox_id, name) DO UPDATE SET position = excluded.position`, mailboxID, name, i); err != nil { + // created_modseq is only set for a genuinely new row (the INSERT path) — the + // ON CONFLICT branch deliberately doesn't touch it, so an already-existing + // row's creation event stays put. + if _, err := tx.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, position, modseq, created_modseq) VALUES (?, ?, ?, ?, ?) + ON CONFLICT(mailbox_id, name) DO UPDATE SET position = excluded.position, modseq = excluded.modseq`, mailboxID, name, i, modseq, modseq); err != nil { return err } } @@ -345,7 +385,11 @@ func (d *DB) FolderTree(mailboxID int64) ([]*FolderNode, error) { // when the caller doesn't care about (or already knows) the parent — see // CreateMailboxFolderUnder for actually creating a new folder under a chosen parent. func (d *DB) CreateMailboxFolder(mailboxID int64, name string) error { - _, err := d.Exec(`INSERT OR IGNORE INTO esrv_mailbox_folders (mailbox_id, name, position) VALUES (?, ?, ?)`, mailboxID, name, unpositionedFolder) + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`INSERT OR IGNORE INTO esrv_mailbox_folders (mailbox_id, name, position, modseq, created_modseq) VALUES (?, ?, ?, ?, ?)`, mailboxID, name, unpositionedFolder, modseq, modseq) return err } @@ -354,17 +398,21 @@ func (d *DB) CreateMailboxFolder(mailboxID int64, name string) error { // its own row) by the time this is called; see webmailAddFolder for the validation // this relies on. func (d *DB) CreateMailboxFolderUnder(mailboxID int64, name, parent string) error { + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } if isStandardFolderName(parent) { - _, err := d.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, position) VALUES (?, ?, NULL, ?, ?)`, - mailboxID, name, parent, unpositionedFolder) + _, err := d.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, position, modseq, created_modseq) VALUES (?, ?, NULL, ?, ?, ?, ?)`, + mailboxID, name, parent, unpositionedFolder, modseq, modseq) return err } var parentID int64 if err := d.QueryRow(`SELECT id FROM esrv_mailbox_folders WHERE mailbox_id = ? AND name = ?`, mailboxID, parent).Scan(&parentID); err != nil { return err } - _, err := d.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, position) VALUES (?, ?, ?, '', ?)`, - mailboxID, name, parentID, unpositionedFolder) + _, err = d.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, position, modseq, created_modseq) VALUES (?, ?, ?, '', ?, ?, ?)`, + mailboxID, name, parentID, unpositionedFolder, modseq, modseq) return err } @@ -375,6 +423,15 @@ func (d *DB) CreateMailboxFolderUnder(mailboxID int64, name, parent string) erro // restore_parent_id/restore_parent_root first, so RestoreFolder can put it back where // it came from later. func (d *DB) MoveFolderToTrash(mailboxID int64, name string) error { + // Computed before Begin(), not inside the transaction below: sqlDB.SetMaxOpenConns(1) + // means a second query against d (the pool) while tx already holds the one + // connection would block forever waiting for a connection tx itself is holding — + // a self-deadlock. Same reasoning as every other nextFolderModseq call site. + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } + tx, err := d.Begin() if err != nil { return err @@ -391,8 +448,8 @@ func (d *DB) MoveFolderToTrash(mailboxID int64, name string) error { // No row yet (a message-derived-only folder, never explicitly parented) — its // restore target defaults to INBOX, the same default FolderParentMap already // uses for a row-less folder. - if _, err := tx.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, restore_parent_id, restore_parent_root, position) VALUES (?, ?, NULL, 'Trash', NULL, 'INBOX', ?)`, - mailboxID, name, unpositionedFolder); err != nil { + if _, err := tx.Exec(`INSERT INTO esrv_mailbox_folders (mailbox_id, name, parent_id, parent_root, restore_parent_id, restore_parent_root, position, modseq, created_modseq) VALUES (?, ?, NULL, 'Trash', NULL, 'INBOX', ?, ?, ?)`, + mailboxID, name, unpositionedFolder, modseq, modseq); err != nil { return err } case err != nil: @@ -402,8 +459,8 @@ func (d *DB) MoveFolderToTrash(mailboxID int64, name string) error { if !parentID.Valid && parentRoot == "" { restoreRoot = "INBOX" } - if _, err := tx.Exec(`UPDATE esrv_mailbox_folders SET restore_parent_id = ?, restore_parent_root = ?, parent_id = NULL, parent_root = 'Trash' WHERE id = ?`, - parentID, restoreRoot, id); err != nil { + if _, err := tx.Exec(`UPDATE esrv_mailbox_folders SET restore_parent_id = ?, restore_parent_root = ?, parent_id = NULL, parent_root = 'Trash', modseq = ? WHERE id = ?`, + parentID, restoreRoot, modseq, id); err != nil { return err } } @@ -443,8 +500,12 @@ func (d *DB) RestoreFolder(mailboxID int64, name string) error { parentRoot = "INBOX" } - _, err := d.Exec(`UPDATE esrv_mailbox_folders SET parent_id = ?, parent_root = ?, restore_parent_id = NULL, restore_parent_root = '' WHERE mailbox_id = ? AND name = ?`, - parentID, parentRoot, mailboxID, name) + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_folders SET parent_id = ?, parent_root = ?, restore_parent_id = NULL, restore_parent_root = '', modseq = ? WHERE mailbox_id = ? AND name = ?`, + parentID, parentRoot, modseq, mailboxID, name) return err } @@ -472,12 +533,17 @@ func (d *DB) ListMailboxFolders(mailboxID int64) ([]string, error) { // enough (no cascade needed): children reference their parent by row id, not by name, // so they stay correctly nested with zero further writes. func (d *DB) RenameMailboxFolder(mailboxID int64, oldName, newName string) error { + // Computed before Begin() — same self-deadlock reasoning as MoveFolderToTrash's. + modseq, err := d.nextFolderModseq(mailboxID) + if err != nil { + return err + } tx, err := d.Begin() if err != nil { return err } defer tx.Rollback() - if _, err := tx.Exec(`UPDATE esrv_mailbox_folders SET name = ? WHERE mailbox_id = ? AND name = ?`, newName, mailboxID, oldName); err != nil { + if _, err := tx.Exec(`UPDATE esrv_mailbox_folders SET name = ?, modseq = ? WHERE mailbox_id = ? AND name = ?`, newName, modseq, mailboxID, oldName); err != nil { return err } if _, err := tx.Exec(`UPDATE esrv_mailbox_messages SET folder = ? WHERE mailbox_id = ? AND folder = ?`, newName, mailboxID, oldName); err != nil { diff --git a/internal/db/crud_mailbox_messages.go b/internal/db/crud_mailbox_messages.go index b88ba1e..b30ebc8 100644 --- a/internal/db/crud_mailbox_messages.go +++ b/internal/db/crud_mailbox_messages.go @@ -11,22 +11,85 @@ import ( // lives at storagePath — see internal/mailstore). Returns the new row's id, which // doubles as the IMAP UID in later milestones. func (d *DB) InsertMessage(mailboxID int64, folder, messageIDHeader, flags string, internalDate time.Time, sizeBytes int64, storagePath string, nonce []byte, cachedFrom, cachedTo, cachedSubject, cachedPreview string) (int64, error) { + // modseq/created_modseq share one freshly computed value — this row is both + // "created" and "at its current state" as of the same event. thread_id is left at + // its column default (0) here: it depends on the message's own id, which doesn't + // exist until after this INSERT returns — see db.ThreadForMessage/ + // SetMessageThreadID, called separately by mailstore.storeMessage right after. + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return 0, err + } res, err := d.Exec(`INSERT INTO esrv_mailbox_messages - (mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, storage_path, nonce, cached_from, cached_to, cached_subject, cached_preview) - VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, - mailboxID, folder, messageIDHeader, flags, internalDate, sizeBytes, storagePath, nonce, cachedFrom, cachedTo, cachedSubject, cachedPreview) + (mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, storage_path, nonce, cached_from, cached_to, cached_subject, cached_preview, modseq, created_modseq) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)`, + mailboxID, folder, messageIDHeader, flags, internalDate, sizeBytes, storagePath, nonce, cachedFrom, cachedTo, cachedSubject, cachedPreview, modseq, modseq) if err != nil { return 0, err } return res.LastInsertId() } -const mailboxMessageColumns = `id, mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, cached_from, cached_to, cached_subject, cached_preview, storage_path, nonce, created_at` +// SetMessageThreadID sets a message's thread_id after insert (see InsertMessage's +// comment on why this can't happen at insert time) — called by mailstore.storeMessage +// right after InsertMessage returns the new uid. Does not bump modseq: threading is +// resolved once, immediately after creation, before any JMAP client could have +// observed the row without it — not a later, independently-visible mutation. +func (d *DB) SetMessageThreadID(mailboxID, uid, threadID int64) error { + _, err := d.Exec(`UPDATE esrv_mailbox_messages SET thread_id = ? WHERE id = ? AND mailbox_id = ?`, threadID, uid, mailboxID) + return err +} + +// ThreadForMessage resolves the JMAP (internal/jmap) thread_id a new message with the +// given raw References/In-Reply-To header values should join, within one mailbox. +// Checked most-recent-first (In-Reply-To, then References read newest-to-oldest per +// RFC 5322 §3.6.4's ordering) against every existing message's message_id_header; +// returns 0 if nothing matches — the caller (mailstore.storeMessage) then uses the new +// message's own id as its thread_id, the root of a new thread. Deliberately +// simplified: a message referencing two independently-threaded messages joins +// whichever is matched first, not a full union-find merge of both threads — not the +// full RFC 5256 THREAD REFERENCES algorithm, an accepted, documented scope cut. +func (d *DB) ThreadForMessage(mailboxID int64, references, inReplyTo string) (int64, error) { + ids := extractMessageIDTokens(inReplyTo) + ids = append(ids, extractMessageIDTokens(references)...) + for _, id := range ids { + var threadID int64 + err := d.QueryRow(`SELECT thread_id FROM esrv_mailbox_messages WHERE mailbox_id = ? AND message_id_header = ? LIMIT 1`, mailboxID, id).Scan(&threadID) + if err == nil { + return threadID, nil + } + if !errors.Is(err, sql.ErrNoRows) { + return 0, err + } + } + return 0, nil +} + +// extractMessageIDTokens splits a References/In-Reply-To header value into individual +// Message-ID tokens, angle brackets stripped (matching the bare-form storage +// convention smtpserver.extractMessageID already uses for message_id_header), most- +// recently-referenced first (the header itself lists oldest-first per RFC 5322 +// §3.6.4, reversed here). +func extractMessageIDTokens(header string) []string { + var out []string + for _, tok := range strings.Fields(header) { + tok = strings.TrimPrefix(strings.TrimSuffix(tok, ">"), "<") + if tok != "" { + out = append(out, tok) + } + } + for i, j := 0, len(out)-1; i < j; i, j = i+1, j-1 { + out[i], out[j] = out[j], out[i] + } + return out +} + +const mailboxMessageColumns = `id, mailbox_id, folder, message_id_header, flags, internal_date, size_bytes, cached_from, cached_to, cached_subject, cached_preview, storage_path, nonce, created_at, thread_id, modseq, created_modseq` func scanMailboxMessage(scan func(dest ...any) error) (MailboxMessage, error) { var m MailboxMessage var internalDate, createdAt string - err := scan(&m.ID, &m.MailboxID, &m.Folder, &m.MessageIDHeader, &m.Flags, &internalDate, &m.SizeBytes, &m.CachedFrom, &m.CachedTo, &m.CachedSubject, &m.CachedPreview, &m.StoragePath, &m.Nonce, &createdAt) + err := scan(&m.ID, &m.MailboxID, &m.Folder, &m.MessageIDHeader, &m.Flags, &internalDate, &m.SizeBytes, &m.CachedFrom, &m.CachedTo, &m.CachedSubject, &m.CachedPreview, &m.StoragePath, &m.Nonce, &createdAt, &m.ThreadID, &m.Modseq, &m.CreatedModseq) if err != nil { return m, err } @@ -35,6 +98,17 @@ func scanMailboxMessage(scan func(dest ...any) error) (MailboxMessage, error) { return m, nil } +// nextMessageModseq returns the next per-mailbox modseq value for esrv_mailbox_messages +// — a plain SELECT MAX+1, safe without a transaction because sqlDB.SetMaxOpenConns(1) +// already serializes every DB access through one connection app-wide (same reasoning +// as crud_relay_queue.go's ClaimDueRelayQueueItems select-then-update). Backs JMAP's +// (internal/jmap) Email/changes incremental sync. +func (d *DB) nextMessageModseq(mailboxID int64) (int64, error) { + var n int64 + err := d.QueryRow(`SELECT COALESCE(MAX(modseq), 0) + 1 FROM esrv_mailbox_messages WHERE mailbox_id = ?`, mailboxID).Scan(&n) + return n, err +} + func (d *DB) GetMessageByUID(mailboxID, uid int64) (*MailboxMessage, error) { row := d.QueryRow(`SELECT `+mailboxMessageColumns+` FROM esrv_mailbox_messages WHERE id = ? AND mailbox_id = ?`, uid, mailboxID) m, err := scanMailboxMessage(row.Scan) @@ -47,15 +121,29 @@ func (d *DB) GetMessageByUID(mailboxID, uid int64) (*MailboxMessage, error) { return &m, nil } +// DeleteMessage hard-deletes a message row, first leaving a tombstone behind so JMAP's +// (internal/jmap) Email/changes can still report it destroyed — a deleted row has no +// id of its own left to derive one from otherwise. func (d *DB) DeleteMessage(mailboxID, uid int64) error { - _, err := d.Exec(`DELETE FROM esrv_mailbox_messages WHERE id = ? AND mailbox_id = ?`, uid, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + if _, err := d.Exec(`INSERT INTO esrv_mailbox_message_tombstones (mailbox_id, message_id, modseq) VALUES (?, ?, ?)`, mailboxID, uid, modseq); err != nil { + return err + } + _, err = d.Exec(`DELETE FROM esrv_mailbox_messages WHERE id = ? AND mailbox_id = ?`, uid, mailboxID) return err } // MoveMessage reassigns a message to a different folder — pure metadata change, the // on-disk ciphertext at storage_path never moves. func (d *DB) MoveMessage(mailboxID, uid int64, newFolder string) error { - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET folder = ? WHERE id = ? AND mailbox_id = ?`, newFolder, uid, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET folder = ?, modseq = ? WHERE id = ? AND mailbox_id = ?`, newFolder, modseq, uid, mailboxID) return err } @@ -67,7 +155,11 @@ func (d *DB) MoveMessageToTrash(mailboxID, uid int64) error { if err := d.QueryRow(`SELECT folder FROM esrv_mailbox_messages WHERE id = ? AND mailbox_id = ?`, uid, mailboxID).Scan(&folder); err != nil { return err } - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET restore_folder = ?, folder = 'Trash' WHERE id = ? AND mailbox_id = ?`, folder, uid, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET restore_folder = ?, folder = 'Trash', modseq = ? WHERE id = ? AND mailbox_id = ?`, folder, modseq, uid, mailboxID) return err } @@ -82,7 +174,11 @@ func (d *DB) RestoreMessage(mailboxID, uid int64) error { if restoreFolder == "" { restoreFolder = "INBOX" } - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET folder = ?, restore_folder = '' WHERE id = ? AND mailbox_id = ?`, restoreFolder, uid, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET folder = ?, restore_folder = '', modseq = ? WHERE id = ? AND mailbox_id = ?`, restoreFolder, modseq, uid, mailboxID) return err } @@ -136,8 +232,16 @@ func (d *DB) ListMessagesForMailbox(mailboxID int64) ([]MailboxMessage, error) { // for messages stored before a caching fix/addition landed (those fields are // otherwise only ever computed once, at delivery time, never retroactively). func (d *DB) UpdateMessageCachedFields(id int64, cachedFrom, cachedTo, cachedSubject, cachedPreview string) error { - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET cached_from = ?, cached_to = ?, cached_subject = ?, cached_preview = ? WHERE id = ?`, - cachedFrom, cachedTo, cachedSubject, cachedPreview, id) + var mailboxID int64 + if err := d.QueryRow(`SELECT mailbox_id FROM esrv_mailbox_messages WHERE id = ?`, id).Scan(&mailboxID); err != nil { + return err + } + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET cached_from = ?, cached_to = ?, cached_subject = ?, cached_preview = ?, modseq = ? WHERE id = ?`, + cachedFrom, cachedTo, cachedSubject, cachedPreview, modseq, id) return err } @@ -147,15 +251,23 @@ func (d *DB) UpdateMessageCachedFields(id int64, cachedFrom, cachedTo, cachedSub // and storage_path, which don't change for an in-place content replacement. Scoped to // mailboxID so a caller can't touch another mailbox's message by guessing a uid. func (d *DB) UpdateMessageContent(mailboxID, id int64, sizeBytes int64, nonce []byte, messageIDHeader, cachedFrom, cachedTo, cachedSubject, cachedPreview string) error { - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET size_bytes = ?, nonce = ?, message_id_header = ?, cached_from = ?, cached_to = ?, cached_subject = ?, cached_preview = ? WHERE id = ? AND mailbox_id = ?`, - sizeBytes, nonce, messageIDHeader, cachedFrom, cachedTo, cachedSubject, cachedPreview, id, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET size_bytes = ?, nonce = ?, message_id_header = ?, cached_from = ?, cached_to = ?, cached_subject = ?, cached_preview = ?, modseq = ? WHERE id = ? AND mailbox_id = ?`, + sizeBytes, nonce, messageIDHeader, cachedFrom, cachedTo, cachedSubject, cachedPreview, modseq, id, mailboxID) return err } // SetMessageFlags overwrites a message's stored IMAP flags (space-separated), scoped // to mailboxID so a session can't touch another mailbox's message by guessing a UID. func (d *DB) SetMessageFlags(mailboxID, uid int64, flags string) error { - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET flags = ? WHERE id = ? AND mailbox_id = ?`, flags, uid, mailboxID) + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET flags = ?, modseq = ? WHERE id = ? AND mailbox_id = ?`, flags, modseq, uid, mailboxID) return err } @@ -254,8 +366,16 @@ func (d *DB) CountMessagesInFolder(mailboxID int64, folder string, unreadOnly, s // message individually. Messages that already have other flags (e.g. \Flagged) keep // them; only \Seen is added. func (d *DB) MarkAllReadInFolder(mailboxID int64, folder string) error { - _, err := d.Exec(`UPDATE esrv_mailbox_messages SET flags = TRIM(flags || ' \Seen') - WHERE mailbox_id = ? AND folder = ? AND flags NOT LIKE '%\Seen%' ESCAPE '\'`, mailboxID, folder) + // Every row this bulk update touches shares one modseq value — they all changed + // together as one logical event, same as any other batch mutation JMAP's + // Email/changes needs to report (it only needs modseq > sinceState per row, not a + // unique value per row). + modseq, err := d.nextMessageModseq(mailboxID) + if err != nil { + return err + } + _, err = d.Exec(`UPDATE esrv_mailbox_messages SET flags = TRIM(flags || ' \Seen'), modseq = ? + WHERE mailbox_id = ? AND folder = ? AND flags NOT LIKE '%\Seen%' ESCAPE '\'`, modseq, mailboxID, folder) return err } diff --git a/internal/db/mailbox_models.go b/internal/db/mailbox_models.go index ee92e66..ebecf04 100644 --- a/internal/db/mailbox_models.go +++ b/internal/db/mailbox_models.go @@ -239,6 +239,11 @@ type MailboxMessage struct { StoragePath string Nonce []byte CreatedAt time.Time + // ThreadID/Modseq/CreatedModseq back JMAP (internal/jmap) — see schema.go's + // comment on esrv_mailbox_messages for what each means. Unused by IMAP/webmail. + ThreadID int64 + Modseq int64 + CreatedModseq int64 } // MailboxSignature is one of a mailbox's saved email signatures — HTML content, shown diff --git a/internal/db/schema.go b/internal/db/schema.go index cd4ebda..d9fa9b2 100644 --- a/internal/db/schema.go +++ b/internal/db/schema.go @@ -555,13 +555,41 @@ CREATE TABLE IF NOT EXISTS esrv_mailbox_messages ( restore_folder TEXT NOT NULL DEFAULT '', storage_path TEXT NOT NULL, nonce BLOB NOT NULL, - created_at DATETIME DEFAULT CURRENT_TIMESTAMP + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + -- JMAP support (internal/jmap): thread_id groups messages by References/ + -- In-Reply-To (see db.ThreadForMessage), defaulting to the message's own id (a + -- singleton thread) until proven otherwise. modseq/created_modseq back JMAP's + -- Email/changes incremental sync — bumped on every mutating write to this row (see + -- the per-mailbox counter design in db.nextMessageModseq); created_modseq is set + -- once at insert and never changes, modseq is bumped again on every later update. + thread_id INTEGER NOT NULL DEFAULT 0, + modseq INTEGER NOT NULL DEFAULT 0, + created_modseq INTEGER NOT NULL DEFAULT 0 ); -- Matches the folder view's exact WHERE mailbox_id = ? AND folder = ? ORDER BY -- internal_date pattern — the single hottest query in the whole webmail client, and -- previously unindexed (this schema had no indexes at all before this one). CREATE INDEX IF NOT EXISTS idx_mailbox_messages_folder ON esrv_mailbox_messages(mailbox_id, folder, internal_date); +-- Not created here: thread_id/modseq don't exist yet at this point in Open() on a +-- pre-existing DB (migrateAddedColumns below adds them) — CREATE INDEX would fail +-- with "no such column". See migrateAddedColumns for these three indexes. + +-- Tombstones for JMAP's Email/changes "destroyed" list (internal/jmap) — a +-- hard-deleted esrv_mailbox_messages row leaves nothing behind to derive a JMAP id +-- from, so db.DeleteMessage inserts one of these in the same transaction as the +-- delete. Purely additive log, never pruned by this feature (a JMAP client's own +-- sinceState anchors how far back it needs to look) — if unbounded growth ever +-- matters, the log retention sweep (main.go's runLogRetention) is the natural place +-- to extend, not something this table manages itself. +CREATE TABLE IF NOT EXISTS esrv_mailbox_message_tombstones ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + mailbox_id INTEGER NOT NULL REFERENCES esrv_mailboxes(id), + message_id INTEGER NOT NULL, + modseq INTEGER NOT NULL, + deleted_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +CREATE INDEX IF NOT EXISTS idx_message_tombstones_mailbox_modseq ON esrv_mailbox_message_tombstones(mailbox_id, modseq); -- Explicit record of a mailbox's custom folders, so a freshly created (still empty) -- one shows up in the folder list — esrv_mailbox_messages.folder alone can only prove @@ -607,8 +635,18 @@ CREATE TABLE IF NOT EXISTS esrv_mailbox_folders ( restore_parent_id INTEGER REFERENCES esrv_mailbox_folders(id), restore_parent_root TEXT NOT NULL DEFAULT '', created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + -- JMAP support (internal/jmap): backs Mailbox/changes the same way + -- esrv_mailbox_messages.modseq/created_modseq back Email/changes — modseq bumped + -- on every create/rename/reparent/reorder, created_modseq set once at row creation + -- and never touched again. No destroyed-tombstone equivalent needed here: a JMAP + -- Mailbox/set destroy reparents under Trash (db.MoveFolderToTrash), it never + -- actually removes the row. + modseq INTEGER NOT NULL DEFAULT 0, + created_modseq INTEGER NOT NULL DEFAULT 0, UNIQUE(mailbox_id, name) ); +-- Not created here, same reason as idx_mailbox_messages_thread above — see +-- migrateAddedColumns. -- A mailbox's own S/MIME identities — a mailbox may hold several at once (e.g. one -- per external party it corresponds with, or after rotating an expiring one while @@ -782,6 +820,11 @@ func migrateAddedColumns(db *sql.DB) { `ALTER TABLE esrv_domains ADD COLUMN dkim_dns_automation TEXT NOT NULL DEFAULT 'manual'`, `ALTER TABLE esrv_domains ADD COLUMN use_global_dkim INTEGER NOT NULL DEFAULT 0`, `ALTER TABLE esrv_email_logs ADD COLUMN mail_from_domain TEXT NOT NULL DEFAULT ''`, + `ALTER TABLE esrv_mailbox_messages ADD COLUMN thread_id INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE esrv_mailbox_messages ADD COLUMN modseq INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE esrv_mailbox_messages ADD COLUMN created_modseq INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE esrv_mailbox_folders ADD COLUMN modseq INTEGER NOT NULL DEFAULT 0`, + `ALTER TABLE esrv_mailbox_folders ADD COLUMN created_modseq INTEGER NOT NULL DEFAULT 0`, } // The three old columns above were NOT NULL with no default, so simply adding // key_pem left them behind still blocking every new insert (which only ever sets @@ -805,6 +848,18 @@ func migrateAddedColumns(db *sql.DB) { // constant runs. db.Exec(`UPDATE esrv_email_logs SET mail_from_domain = substr(mail_from, instr(mail_from, '@') + 1) WHERE mail_from_domain = ''`) db.Exec(`CREATE INDEX IF NOT EXISTS idx_email_logs_mail_from_domain_timestamp ON esrv_email_logs(mail_from_domain, timestamp)`) + // JMAP support (internal/jmap): thread_id/modseq/created_modseq/tombstones — same + // deferred-to-here reasoning as mail_from_domain above. Every pre-existing message + // becomes its own singleton thread (no retroactive References/In-Reply-To grouping + // for historical mail — an accepted scope limit, matching every other backfill + // here). modseq/created_modseq need no backfill computation: 0 for every + // pre-existing row is already correct (a JMAP client's first sync simply sees + // everything as "changed since state 0"). + db.Exec(`UPDATE esrv_mailbox_messages SET thread_id = id WHERE thread_id = 0`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_mailbox_messages_thread ON esrv_mailbox_messages(mailbox_id, thread_id)`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_mailbox_messages_modseq ON esrv_mailbox_messages(mailbox_id, modseq)`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_mailbox_messages_msgid ON esrv_mailbox_messages(mailbox_id, message_id_header)`) + db.Exec(`CREATE INDEX IF NOT EXISTS idx_mailbox_folders_modseq ON esrv_mailbox_folders(mailbox_id, modseq)`) migrateSpamRenamedToJunk(db) migrateFilterRulesMarkAsSpamCheck(db) migrateFilterRulesAdvancedCheck(db) diff --git a/internal/imapserver/session.go b/internal/imapserver/session.go index 07dc4b0..97b7a03 100644 --- a/internal/imapserver/session.go +++ b/internal/imapserver/session.go @@ -329,6 +329,7 @@ func (s *Session) Append(mailbox string, r imap.LiteralReader, options *imap.App } } s.backend.Notify.Publish(s.mailbox.ID, folder) + s.backend.Notify.PublishAccountWide(s.mailbox.ID) return &imap.AppendData{UID: imap.UID(uid), UIDValidity: uint32(s.mailbox.ID)}, nil } diff --git a/internal/jmap/api.go b/internal/jmap/api.go new file mode 100644 index 0000000..2e27099 --- /dev/null +++ b/internal/jmap/api.go @@ -0,0 +1,48 @@ +package jmap + +import ( + "encoding/json" + "net/http" +) + +// handleAPI is POST /jmap/api (RFC 8620 §3.3) — the single endpoint every JMAP method +// call goes through, batched. +func (b *Backend) handleAPI(w http.ResponseWriter, r *http.Request) { + mbox := mailboxFromContext(r) + + var req Request + if err := json.NewDecoder(r.Body).Decode(&req); err != nil { + http.Error(w, "invalid request body", http.StatusBadRequest) + return + } + + cs := newCallState() + resp := Response{MethodResponses: make([]Invocation, 0, len(req.MethodCalls))} + for _, call := range req.MethodCalls { + fn, ok := methods[call.Name] + if !ok { + resp.MethodResponses = append(resp.MethodResponses, errorResult(call.ID, "unknownMethod", "")) + continue + } + args, err := cs.resolveBackReferences(call.Args) + if err != nil { + resp.MethodResponses = append(resp.MethodResponses, errorResult(call.ID, "invalidResultReference", err.Error())) + continue + } + result, methodErr := fn(b, mbox, args) + if methodErr != nil { + resp.MethodResponses = append(resp.MethodResponses, errorResult(call.ID, methodErr.Type, methodErr.Description)) + continue + } + body, err := json.Marshal(result) + if err != nil { + resp.MethodResponses = append(resp.MethodResponses, errorResult(call.ID, "serverFail", err.Error())) + continue + } + cs.record(call.ID, body) + resp.MethodResponses = append(resp.MethodResponses, Invocation{Name: call.Name, Args: body, ID: call.ID}) + } + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} diff --git a/internal/jmap/auth.go b/internal/jmap/auth.go new file mode 100644 index 0000000..7c62757 --- /dev/null +++ b/internal/jmap/auth.go @@ -0,0 +1,61 @@ +package jmap + +import ( + "context" + "net" + "net/http" + + "mailgoserver/internal/abuseguard" + "mailgoserver/internal/db" +) + +type contextKey int + +const mailboxContextKey contextKey = 0 + +// mailboxFromContext returns the *db.Mailbox requireAuth already verified for this +// request — safe to call unconditionally in any handler requireAuth wraps. +func mailboxFromContext(r *http.Request) *db.Mailbox { + mbox, _ := r.Context().Value(mailboxContextKey).(*db.Mailbox) + return mbox +} + +// requireAuth wraps next with HTTP Basic Auth against esrv_mailbox_app_passwords — +// the same credential check imapserver.Session.Login already uses (RFC 8620 leaves +// the auth scheme implementation-defined; Basic against app passwords matches this +// codebase's existing "no interactive MFA for a non-webui protocol" precedent for +// IMAP/SMTP). No session cookie, no CSRF — every request authenticates on its own. +func (b *Backend) requireAuth(next http.HandlerFunc) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + username, password, ok := r.BasicAuth() + if !ok { + b.unauthorized(w) + return + } + peerIP, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + peerIP = r.RemoteAddr + } + mbox, err := b.DB.VerifyMailboxAppPassword(username, password) + if err != nil { + b.DB.LogAuthAttempt("jmap_login", username, peerIP, false, "Authentication error: "+err.Error()) + abuseguard.RecordFailureAndMaybeBlacklist(b.DB, b.Cfg, b.Logger, peerIP) + b.unauthorized(w) + return + } + if mbox == nil { + b.DB.LogAuthAttempt("jmap_login", username, peerIP, false, "Invalid credentials") + abuseguard.RecordFailureAndMaybeBlacklist(b.DB, b.Cfg, b.Logger, peerIP) + b.unauthorized(w) + return + } + b.DB.LogAuthAttempt("jmap_login", username, peerIP, true, "Successful JMAP app-password authentication") + ctx := context.WithValue(r.Context(), mailboxContextKey, mbox) + next(w, r.WithContext(ctx)) + } +} + +func (b *Backend) unauthorized(w http.ResponseWriter) { + w.Header().Set("WWW-Authenticate", `Basic realm="jmap"`) + http.Error(w, "Unauthorized", http.StatusUnauthorized) +} diff --git a/internal/jmap/blob.go b/internal/jmap/blob.go new file mode 100644 index 0000000..58c373c --- /dev/null +++ b/internal/jmap/blob.go @@ -0,0 +1,140 @@ +package jmap + +import ( + "bytes" + "crypto/rand" + "encoding/hex" + "encoding/json" + "io" + "net/http" + "os" + "path/filepath" + "strconv" + "strings" + + "mailgoserver/internal/db" + "mailgoserver/internal/mailstore" + "mailgoserver/internal/mailview" +) + +// stagingDir is where an uploaded-but-not-yet-imported blob's raw bytes live — +// mailbox-scoped, alongside (not inside) that mailbox's real message storage +// directories, so it's never mistaken for a real stored message. A blob here is +// ephemeral: consumed once by Email/import (blobBytes just reads it — repeat imports +// of the same upload are harmless, nothing deletes it) or left to expire; cleanup is +// left as a best-effort TTL sweep to add later, not tracked in the DB — matching this +// codebase's "no migration framework, no fuss" posture for genuinely transient state +// (the same reasoning esrv_relay_queue's rows use). +func stagingDir(b *Backend, mbox *db.Mailbox) string { + return filepath.Join(b.Mailstore.BasePath, mailstore.SanitizePathSegment(mbox.Email), ".jmap-uploads") +} + +func writeJSON(w http.ResponseWriter, v any) { + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(v) +} + +func handleUpload(b *Backend, w http.ResponseWriter, r *http.Request) { + mbox := mailboxFromContext(r) + dir := stagingDir(b, mbox) + if err := os.MkdirAll(dir, 0o755); err != nil { + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + maxSize := b.Cfg.Section("Mailstore").Key("max_message_bytes").MustInt64(25 * 1024 * 1024) + data, err := io.ReadAll(io.LimitReader(r.Body, maxSize+1)) + if err != nil { + http.Error(w, "read error", http.StatusInternalServerError) + return + } + if int64(len(data)) > maxSize { + http.Error(w, "payload too large", http.StatusRequestEntityTooLarge) + return + } + + name := make([]byte, 16) + rand.Read(name) + blobID := hex.EncodeToString(name) + if err := os.WriteFile(filepath.Join(dir, blobID), data, 0o600); err != nil { + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + + contentType := r.Header.Get("Content-Type") + if contentType == "" { + contentType = "application/octet-stream" + } + writeJSON(w, map[string]any{ + "accountId": strconv.FormatInt(mbox.ID, 10), + "blobId": blobID, + "type": contentType, + "size": len(data), + }) +} + +// blobBytes resolves blobID to raw bytes, per Email/import's need: either a +// still-staged upload (see handleUpload) or — not applicable here, see +// handleDownload's attachment-extraction path for that case — nothing else. Returns +// (nil, false) if blobID isn't a staged upload for this mailbox. +func blobBytes(b *Backend, mbox *db.Mailbox, blobID string) ([]byte, bool) { + if strings.ContainsAny(blobID, "/\\.") { + return nil, false + } + data, err := os.ReadFile(filepath.Join(stagingDir(b, mbox), blobID)) + if err != nil { + return nil, false + } + return data, true +} + +// handleDownload serves GET /jmap/download/{accountId}/{blobId}/{name} — either a +// still-staged upload (served as-is) or an attachment inside an already-stored +// message, blobId shaped "-" (parsed and ownership-checked against +// the authenticated mailbox here, never trusting the path alone). +func handleDownload(b *Backend, w http.ResponseWriter, r *http.Request) { + mbox := mailboxFromContext(r) + blobID := r.PathValue("blobId") + + if data, ok := blobBytes(b, mbox, blobID); ok { + w.Header().Set("Content-Type", "application/octet-stream") + w.Write(data) + return + } + + uidStr, idxStr, found := strings.Cut(blobID, "-") + if !found { + http.NotFound(w, r) + return + } + uid, err := strconv.ParseInt(uidStr, 10, 64) + if err != nil { + http.NotFound(w, r) + return + } + idx, err := strconv.Atoi(idxStr) + if err != nil || idx < 0 { + http.NotFound(w, r) + return + } + msg, err := b.DB.GetMessageByUID(mbox.ID, uid) + if err != nil || msg == nil { + http.NotFound(w, r) + return + } + raw, err := b.Mailstore.FetchMessage(mbox.ID, uid) + if err != nil { + http.NotFound(w, r) + return + } + parsed, err := mailview.Parse(bytes.NewReader(raw)) + if err != nil || idx >= len(parsed.Attachments) { + http.NotFound(w, r) + return + } + att := parsed.Attachments[idx] + if att.ContentType != "" { + w.Header().Set("Content-Type", att.ContentType) + } + w.Write(att.Data) +} diff --git a/internal/jmap/blob_test.go b/internal/jmap/blob_test.go new file mode 100644 index 0000000..b50ff7f --- /dev/null +++ b/internal/jmap/blob_test.go @@ -0,0 +1,98 @@ +package jmap_test + +import ( + "bytes" + "encoding/json" + "io" + "net/http" + "strconv" + "testing" +) + +func TestJMAPBlobUploadThenDownload(t *testing.T) { + srv, _, _, email, password, mailboxID := newTestJMAPServer(t) + + body := []byte("hello, this is an uploaded blob") + uploadReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/upload/"+strconv.FormatInt(mailboxID, 10), bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + uploadReq.SetBasicAuth(email, password) + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatal(err) + } + defer uploadResp.Body.Close() + if uploadResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", uploadResp.StatusCode) + } + var uploaded struct { + BlobID string `json:"blobId"` + Size int `json:"size"` + } + if err := json.NewDecoder(uploadResp.Body).Decode(&uploaded); err != nil { + t.Fatal(err) + } + if uploaded.Size != len(body) { + t.Errorf("expected size %d, got %d", len(body), uploaded.Size) + } + + downloadReq, err := http.NewRequest(http.MethodGet, srv.URL+"/jmap/download/"+strconv.FormatInt(mailboxID, 10)+"/"+uploaded.BlobID+"/blob.txt", nil) + if err != nil { + t.Fatal(err) + } + downloadReq.SetBasicAuth(email, password) + downloadResp, err := http.DefaultClient.Do(downloadReq) + if err != nil { + t.Fatal(err) + } + defer downloadResp.Body.Close() + if downloadResp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", downloadResp.StatusCode) + } + got, err := io.ReadAll(downloadResp.Body) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(got, body) { + t.Errorf("expected byte-for-byte round trip, got %q want %q", got, body) + } +} + +func TestJMAPBlobDownloadAttachment(t *testing.T) { + srv, _, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\n" + + "Subject: With attachment\r\n" + + "Content-Type: multipart/mixed; boundary=\"B\"\r\n\r\n" + + "--B\r\nContent-Type: text/plain\r\n\r\nsee attached\r\n" + + "--B\r\nContent-Type: text/plain\r\nContent-Disposition: attachment; filename=\"notes.txt\"\r\n\r\n" + + "attachment contents\r\n" + + "--B--\r\n") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "With attachment") + if err != nil { + t.Fatal(err) + } + + blobID := strconv.FormatInt(uid, 10) + "-0" + req, err := http.NewRequest(http.MethodGet, srv.URL+"/jmap/download/"+strconv.FormatInt(mailboxID, 10)+"/"+blobID+"/notes.txt", nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth(email, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + got, err := io.ReadAll(resp.Body) + if err != nil { + t.Fatal(err) + } + if string(got) != "attachment contents" { + t.Errorf("expected the attachment's real bytes, got %q", got) + } +} diff --git a/internal/jmap/email.go b/internal/jmap/email.go new file mode 100644 index 0000000..db3c540 --- /dev/null +++ b/internal/jmap/email.go @@ -0,0 +1,784 @@ +package jmap + +import ( + "bytes" + "encoding/json" + "fmt" + "net/mail" + "sort" + "strconv" + "strings" + "time" + + "mailgoserver/internal/db" + "mailgoserver/internal/mailview" +) + +type jmapEmailAddress struct { + Name *string `json:"name"` + Email string `json:"email"` +} + +// parseAddressList turns a raw "Name , c@d" header value into JMAP +// EmailAddress[] — best-effort: an unparseable value degrades to a single entry with +// the raw text as the address rather than dropping it, since cached_from/cached_to +// are display-only fields anyway (never used for delivery decisions). +func parseAddressList(raw string) []jmapEmailAddress { + if strings.TrimSpace(raw) == "" { + return nil + } + addrs, err := mail.ParseAddressList(raw) + if err != nil { + return []jmapEmailAddress{{Email: raw}} + } + out := make([]jmapEmailAddress, len(addrs)) + for i, a := range addrs { + e := jmapEmailAddress{Email: a.Address} + if a.Name != "" { + name := a.Name + e.Name = &name + } + out[i] = e + } + return out +} + +type jmapBodyPart struct { + PartID string `json:"partId,omitempty"` + BlobID string `json:"blobId,omitempty"` + Size int `json:"size"` + Type string `json:"type"` + Name string `json:"name,omitempty"` + CID string `json:"cid,omitempty"` +} + +type jmapBodyValue struct { + Value string `json:"value"` + IsEncodingProblem bool `json:"isEncodingProblem"` + IsTruncated bool `json:"isTruncated"` +} + +type jmapEmail struct { + ID string `json:"id"` + ThreadID string `json:"threadId"` + MailboxIDs map[string]bool `json:"mailboxIds"` + Keywords map[string]bool `json:"keywords"` + Size int64 `json:"size"` + ReceivedAt string `json:"receivedAt"` + Subject *string `json:"subject"` + From []jmapEmailAddress `json:"from,omitempty"` + To []jmapEmailAddress `json:"to,omitempty"` + Preview string `json:"preview"` + HasAttachment bool `json:"hasAttachment"` + TextBody []jmapBodyPart `json:"textBody,omitempty"` + HTMLBody []jmapBodyPart `json:"htmlBody,omitempty"` + Attachments []jmapBodyPart `json:"attachments,omitempty"` + BodyValues map[string]jmapBodyValue `json:"bodyValues,omitempty"` +} + +// flagKeywords maps this app's stored IMAP flags to JMAP's "$"-prefixed keyword +// vocabulary (RFC 8621 §4.1.1) — the two systems share the same underlying concept +// (a message's own status markers), just different string spellings. +var flagToKeyword = map[string]string{ + `\Seen`: "$seen", + `\Flagged`: "$flagged", + `\Answered`: "$answered", + `\Draft`: "$draft", +} +var keywordToFlag = map[string]string{ + "$seen": `\Seen`, "$flagged": `\Flagged`, "$answered": `\Answered`, "$draft": `\Draft`, +} + +func keywordsFromFlags(flags string) map[string]bool { + out := map[string]bool{} + for _, f := range strings.Fields(flags) { + if kw, ok := flagToKeyword[f]; ok { + out[kw] = true + } + } + return out +} + +// buildEmailObject converts one stored message into a JMAP Email object. mailboxIDs +// maps this message's folder name to its JMAP mailboxId (see resolveMailboxes). +// wantBody triggers a full decrypt+MIME parse (mailstore.FetchMessage + +// mailview.Parse) for textBody/htmlBody/bodyValues/attachments/hasAttachment — the +// cheap path (list/query views) skips this entirely, same "cached columns avoid +// decryption" design IMAP/webmail already use. +func buildEmailObject(b *Backend, mbox *db.Mailbox, m db.MailboxMessage, mailboxIDByFolder map[string]string, wantBody bool) jmapEmail { + e := jmapEmail{ + ID: strconv.FormatInt(m.ID, 10), + ThreadID: strconv.FormatInt(m.ThreadID, 10), + MailboxIDs: map[string]bool{}, + Keywords: keywordsFromFlags(m.Flags), + Size: m.SizeBytes, + ReceivedAt: m.InternalDate.UTC().Format(time.RFC3339), + Preview: m.CachedPreview, + } + if mid, ok := mailboxIDByFolder[m.Folder]; ok { + e.MailboxIDs[mid] = true + } + if m.CachedSubject != "" { + s := m.CachedSubject + e.Subject = &s + } + e.From = parseAddressList(m.CachedFrom) + e.To = parseAddressList(m.CachedTo) + + if !wantBody { + return e + } + raw, err := b.Mailstore.FetchMessage(mbox.ID, m.ID) + if err != nil { + return e + } + parsed, err := mailview.Parse(strings.NewReader(string(raw))) + if err != nil { + return e + } + if parsed.TextBody != "" { + partID := strconv.FormatInt(m.ID, 10) + "-text" + e.TextBody = []jmapBodyPart{{PartID: partID, Type: "text/plain", Size: len(parsed.TextBody)}} + if e.BodyValues == nil { + e.BodyValues = map[string]jmapBodyValue{} + } + e.BodyValues[partID] = jmapBodyValue{Value: parsed.TextBody} + } + if parsed.HTMLBody != "" { + partID := strconv.FormatInt(m.ID, 10) + "-html" + e.HTMLBody = []jmapBodyPart{{PartID: partID, Type: "text/html", Size: len(parsed.HTMLBody)}} + if e.BodyValues == nil { + e.BodyValues = map[string]jmapBodyValue{} + } + e.BodyValues[partID] = jmapBodyValue{Value: parsed.HTMLBody} + } + for i, a := range parsed.Attachments { + e.Attachments = append(e.Attachments, jmapBodyPart{ + BlobID: strconv.FormatInt(m.ID, 10) + "-" + strconv.Itoa(i), + Size: len(a.Data), + Type: a.ContentType, + Name: a.Filename, + CID: a.ContentID, + }) + } + e.HasAttachment = len(e.Attachments) > 0 + return e +} + +// wantsBody reports whether the requested property set needs a full decrypt+parse — +// either explicitly via the fetch* flags, or implicitly because properties names one +// of the body-derived fields. +func wantsBody(args emailGetArgs) bool { + if args.FetchTextBodyValues || args.FetchHTMLBodyValues || args.FetchAllBodyValues { + return true + } + if args.Properties == nil { + return false + } + for _, p := range *args.Properties { + switch p { + case "textBody", "htmlBody", "bodyValues", "attachments", "hasAttachment": + return true + } + } + return false +} + +type emailGetArgs struct { + IDs *[]string `json:"ids"` + Properties *[]string `json:"properties"` + FetchTextBodyValues bool `json:"fetchTextBodyValues"` + FetchHTMLBodyValues bool `json:"fetchHTMLBodyValues"` + FetchAllBodyValues bool `json:"fetchAllBodyValues"` +} + +type emailGetResult struct { + AccountID string `json:"accountId"` + State string `json:"state"` + List []jmapEmail `json:"list"` + NotFound []string `json:"notFound"` +} + +func emailGet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailGetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + _, _, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + mailboxIDByFolder := make(map[string]string, len(idByName)) + for name, id := range idByName { + mailboxIDByFolder[name] = strconv.FormatInt(id, 10) + } + state, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + wantBody := wantsBody(args) + + result := emailGetResult{AccountID: strconv.FormatInt(mbox.ID, 10), State: state, List: []jmapEmail{}, NotFound: []string{}} + if args.IDs == nil { + return nil, &methodError{Type: "requestTooLarge", Description: "ids is required (fetching every Email in an account is not supported)"} + } + for _, idStr := range *args.IDs { + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + result.NotFound = append(result.NotFound, idStr) + continue + } + m, err := b.DB.GetMessageByUID(mbox.ID, id) + if err != nil || m == nil { + result.NotFound = append(result.NotFound, idStr) + continue + } + result.List = append(result.List, buildEmailObject(b, mbox, *m, mailboxIDByFolder, wantBody)) + } + return result, nil +} + +// emailQueryFilter is the bounded, flat FilterCondition subset this server supports +// in v1 — see the JMAP plan's disclosed scope cut on nested FilterOperator (AND/OR/ +// NOT) support. +type emailQueryFilter struct { + InMailbox *string `json:"inMailbox"` + Text *string `json:"text"` + Subject *string `json:"subject"` + From *string `json:"from"` + To *string `json:"to"` + HasKeyword *string `json:"hasKeyword"` + // Presence of "operator" means the client sent a nested FilterOperator — rejected + // below with unsupportedFilter rather than silently mis-evaluated. + Operator *string `json:"operator"` +} + +type emailQueryArgs struct { + Filter *emailQueryFilter `json:"filter"` + Limit *int `json:"limit"` + Position int `json:"position"` +} + +type emailQueryResult struct { + AccountID string `json:"accountId"` + QueryState string `json:"queryState"` + CanCalculateChanges bool `json:"canCalculateChanges"` + Position int `json:"position"` + IDs []string `json:"ids"` + Total int `json:"total"` +} + +const defaultQueryLimit = 50 + +func emailQuery(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailQueryArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + if args.Filter != nil && args.Filter.Operator != nil { + return nil, &methodError{Type: "unsupportedFilter", Description: "nested FilterOperator (AND/OR/NOT) is not supported — use a single flat FilterCondition"} + } + + limit := defaultQueryLimit + if args.Limit != nil && *args.Limit > 0 { + limit = *args.Limit + } + folder := "" + if args.Filter != nil && args.Filter.InMailbox != nil { + _, nameByID, _, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + name, ok := nameByID[*args.Filter.InMailbox] + if !ok { + return nil, &methodError{Type: "invalidArguments", Description: "unknown inMailbox id"} + } + folder = name + } + + var ( + messages []db.MailboxMessage + total int + err error + ) + searchText := "" + if args.Filter != nil { + switch { + case args.Filter.Text != nil: + searchText = *args.Filter.Text + case args.Filter.Subject != nil: + searchText = *args.Filter.Subject + case args.Filter.From != nil: + searchText = *args.Filter.From + case args.Filter.To != nil: + searchText = *args.Filter.To + } + } + if searchText != "" { + messages, err = b.DB.SearchMessagesInFolder(mbox.ID, folder, searchText, args.Position, limit) + if err == nil { + total, err = b.DB.CountSearchMessagesInFolder(mbox.ID, folder, searchText) + } + } else if folder != "" { + // Only hasKeyword=="$flagged" maps cleanly onto an existing WHERE clause + // (ListMessagesInFolderPage's starredOnly) — anything else (an unsupported + // keyword, or notKeyword, not modeled in emailQueryFilter at all) is rejected + // rather than silently ignored, same "never silently wrong" posture as the + // AND/OR/NOT rejection above. + starredOnly := false + if args.Filter != nil && args.Filter.HasKeyword != nil { + if *args.Filter.HasKeyword != "$flagged" { + return nil, &methodError{Type: "unsupportedFilter", Description: "hasKeyword is only supported for \"$flagged\" in v1"} + } + starredOnly = true + } + messages, err = b.DB.ListMessagesInFolderPage(mbox.ID, folder, false, starredOnly, "date", "desc", args.Position, limit) + if err == nil { + total, err = b.DB.CountMessagesInFolder(mbox.ID, folder, false, starredOnly) + } + } else { + return nil, &methodError{Type: "invalidArguments", Description: "filter.inMailbox or filter.text/subject/from/to is required"} + } + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + + state, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + ids := make([]string, len(messages)) + for i, m := range messages { + ids[i] = strconv.FormatInt(m.ID, 10) + } + return emailQueryResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), + QueryState: state, + CanCalculateChanges: false, + Position: args.Position, + IDs: ids, + Total: total, + }, nil +} + +type emailChangesArgs struct { + SinceState string `json:"sinceState"` +} + +type emailChangesResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + HasMoreChanges bool `json:"hasMoreChanges"` + Created []string `json:"created"` + Updated []string `json:"updated"` + Destroyed []string `json:"destroyed"` +} + +func emailChanges(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailChangesArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + since, err := strconv.ParseInt(args.SinceState, 10, 64) + if err != nil { + return nil, &methodError{Type: "invalidArguments", Description: "sinceState must be a modseq integer string"} + } + changes, err := b.DB.MessageChangesSince(mbox.ID, since) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + return emailChangesResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), + OldState: args.SinceState, + NewState: changes.NewState, + HasMoreChanges: changes.HasMore, + Created: int64sToStrings(changes.Created), + Updated: int64sToStrings(changes.Updated), + Destroyed: int64sToStrings(changes.Destroyed), + }, nil +} + +// singleMailboxFolder resolves a JMAP mailboxIds set (RFC 8621 §4.1.1: {mailboxId: +// true, ...}) to the one folder name it represents — this server stores each message +// in exactly one folder, so mailboxIds must name exactly one mailbox with value true; +// anything else (zero, or more than one) is rejected rather than silently picking one. +func singleMailboxFolder(ids map[string]bool, nameByID map[string]string) (string, error) { + var chosen string + n := 0 + for id, v := range ids { + if !v { + continue + } + name, ok := nameByID[id] + if !ok { + return "", fmt.Errorf("unknown mailbox id %q", id) + } + chosen, n = name, n+1 + } + if n != 1 { + return "", fmt.Errorf("mailboxIds must name exactly one mailbox (multiple mailbox membership per message isn't supported)") + } + return chosen, nil +} + +// flagsFromKeywords renders a JMAP keywords set back to this app's space-separated +// IMAP flags string — the inverse of keywordsFromFlags. Sorted only for a +// deterministic/diffable stored value, not load-bearing. +func flagsFromKeywords(kw map[string]bool) string { + var parts []string + for k, v := range kw { + if !v { + continue + } + if flag, ok := keywordToFlag[k]; ok { + parts = append(parts, flag) + } + } + sort.Strings(parts) + return strings.Join(parts, " ") +} + +// applyEmailUpdate handles one Email/set update patch (RFC 8620 §5.3): either a whole- +// property replacement ("keywords"/"mailboxIds") or the individual-property patch +// shorthand ("keywords/$seen", "mailboxIds/") real JMAP clients commonly send for +// a single flag toggle or move instead of resending the whole set. A "mailboxIds/" +// patch only acts on a "true" value — a bare removal (null/false) with no +// accompanying "true" elsewhere is only meaningful in a multi-mailbox model this +// server doesn't support, so it's a silent no-op rather than an error (the same +// message simply keeps its current single folder). +func applyEmailUpdate(b *Backend, mbox *db.Mailbox, m db.MailboxMessage, patch map[string]json.RawMessage, nameByID map[string]string) error { + keywords := keywordsFromFlags(m.Flags) + keywordsChanged := false + newFolder := m.Folder + + for key, raw := range patch { + switch { + case key == "keywords": + var kw map[string]bool + if err := json.Unmarshal(raw, &kw); err != nil { + return fmt.Errorf("keywords: %w", err) + } + keywords = kw + keywordsChanged = true + case key == "mailboxIds": + var ids map[string]bool + if err := json.Unmarshal(raw, &ids); err != nil { + return fmt.Errorf("mailboxIds: %w", err) + } + folder, err := singleMailboxFolder(ids, nameByID) + if err != nil { + return err + } + newFolder = folder + case strings.HasPrefix(key, "keywords/"): + kw := strings.TrimPrefix(key, "keywords/") + var val *bool + if err := json.Unmarshal(raw, &val); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + if val == nil || !*val { + delete(keywords, kw) + } else { + keywords[kw] = true + } + keywordsChanged = true + case strings.HasPrefix(key, "mailboxIds/"): + id := strings.TrimPrefix(key, "mailboxIds/") + var val *bool + if err := json.Unmarshal(raw, &val); err != nil { + return fmt.Errorf("%s: %w", key, err) + } + if val != nil && *val { + name, ok := nameByID[id] + if !ok { + return fmt.Errorf("mailboxIds/%s: unknown mailbox", id) + } + newFolder = name + } + default: + return fmt.Errorf("unsupported property %q", key) + } + } + + if keywordsChanged { + if err := b.DB.SetMessageFlags(mbox.ID, m.ID, flagsFromKeywords(keywords)); err != nil { + return err + } + } + if newFolder != m.Folder { + if err := b.DB.MoveMessage(mbox.ID, m.ID, newFolder); err != nil { + return err + } + } + return nil +} + +type emailSetArgs struct { + IfInState *string `json:"ifInState"` + Create map[string]json.RawMessage `json:"create"` + Update map[string]json.RawMessage `json:"update"` + Destroy []string `json:"destroy"` +} + +type emailSetResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + Created map[string]jmapEmail `json:"created"` + Updated map[string]any `json:"updated"` + Destroyed []string `json:"destroyed"` + NotCreated map[string]*methodError `json:"notCreated"` + NotUpdated map[string]*methodError `json:"notUpdated"` + NotDestroyed map[string]*methodError `json:"notDestroyed"` +} + +// emailSet is Email/set (RFC 8621 §4.6). "create" isn't supported — building a +// full MIME message from JMAP properties is a materially different job from mutating +// one; Email/import (raw RFC822 via a previously uploaded blob) is this server's +// supported creation path, and every "create" entry is reported notCreated pointing +// there rather than silently ignored. +func emailSet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailSetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + oldState, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + if args.IfInState != nil && *args.IfInState != oldState { + return nil, &methodError{Type: "stateMismatch", Description: "ifInState does not match current state"} + } + + result := emailSetResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), OldState: oldState, + Created: map[string]jmapEmail{}, Updated: map[string]any{}, Destroyed: []string{}, + NotCreated: map[string]*methodError{}, NotUpdated: map[string]*methodError{}, NotDestroyed: map[string]*methodError{}, + } + for clientID := range args.Create { + result.NotCreated[clientID] = &methodError{Type: "invalidArguments", Description: "Email/set create is not supported — use Email/import for raw RFC822 content"} + } + + _, nameByID, _, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + + for idStr, rawPatch := range args.Update { + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + result.NotUpdated[idStr] = &methodError{Type: "notFound"} + continue + } + m, err := b.DB.GetMessageByUID(mbox.ID, id) + if err != nil || m == nil { + result.NotUpdated[idStr] = &methodError{Type: "notFound"} + continue + } + var patch map[string]json.RawMessage + if err := json.Unmarshal(rawPatch, &patch); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidPatch", Description: err.Error()} + continue + } + if err := applyEmailUpdate(b, mbox, *m, patch, nameByID); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidPatch", Description: err.Error()} + continue + } + result.Updated[idStr] = nil + } + + for _, idStr := range args.Destroy { + id, err := strconv.ParseInt(idStr, 10, 64) + if err != nil { + result.NotDestroyed[idStr] = &methodError{Type: "notFound"} + continue + } + m, err := b.DB.GetMessageByUID(mbox.ID, id) + if err != nil || m == nil { + result.NotDestroyed[idStr] = &methodError{Type: "notFound"} + continue + } + if err := b.Mailstore.DeleteMessage(mbox.ID, id); err != nil { + result.NotDestroyed[idStr] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + result.Destroyed = append(result.Destroyed, idStr) + } + + if len(result.Updated) > 0 || len(result.Destroyed) > 0 { + // Wakes any connected EventSource (internal/jmap/eventsource.go) or IMAP IDLE + // session on this mailbox — same signal IMAP APPEND/SMTP delivery already + // publish, just triggered by a JMAP mutation instead. + b.Notify.PublishAccountWide(mbox.ID) + } + + newState, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + result.NewState = newState + return result, nil +} + +type emailCopyCreate struct { + ID string `json:"id"` + MailboxIDs map[string]bool `json:"mailboxIds"` +} + +type emailCopyArgs struct { + FromAccountID string `json:"fromAccountId"` + Create map[string]emailCopyCreate `json:"create"` + OnSuccessDestroyOriginal bool `json:"onSuccessDestroyOriginal"` +} + +type emailCopyResult struct { + FromAccountID string `json:"fromAccountId"` + AccountID string `json:"accountId"` + Created map[string]jmapEmail `json:"created"` + NotCreated map[string]*methodError `json:"notCreated"` +} + +// emailCopy is Email/copy (RFC 8621 §4.8) — always same-account here (this server has +// one JMAP account per mailbox, so cross-account copy has no meaning). +func emailCopy(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailCopyArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + accountID := strconv.FormatInt(mbox.ID, 10) + if args.FromAccountID != "" && args.FromAccountID != accountID { + return nil, &methodError{Type: "invalidArguments", Description: "cross-account copy is not supported — this server has one account per mailbox"} + } + + _, nameByID, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + mailboxIDByFolder := make(map[string]string, len(idByName)) + for name, id := range idByName { + mailboxIDByFolder[name] = strconv.FormatInt(id, 10) + } + + result := emailCopyResult{FromAccountID: accountID, AccountID: accountID, Created: map[string]jmapEmail{}, NotCreated: map[string]*methodError{}} + for clientID, c := range args.Create { + srcID, err := strconv.ParseInt(c.ID, 10, 64) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "notFound"} + continue + } + destFolder, err := singleMailboxFolder(c.MailboxIDs, nameByID) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "invalidArguments", Description: err.Error()} + continue + } + newUID, err := b.Mailstore.CopyMessage(mbox.ID, srcID, destFolder) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + if args.OnSuccessDestroyOriginal { + b.Mailstore.DeleteMessage(mbox.ID, srcID) + } + m, err := b.DB.GetMessageByUID(mbox.ID, newUID) + if err != nil || m == nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail"} + continue + } + result.Created[clientID] = buildEmailObject(b, mbox, *m, mailboxIDByFolder, false) + } + if len(result.Created) > 0 { + b.Notify.PublishAccountWide(mbox.ID) + } + return result, nil +} + +type emailImportEntry struct { + BlobID string `json:"blobId"` + MailboxIDs map[string]bool `json:"mailboxIds"` + Keywords map[string]bool `json:"keywords"` +} + +type emailImportArgs struct { + Emails map[string]emailImportEntry `json:"emails"` +} + +type emailImportResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + Created map[string]jmapEmail `json:"created"` + NotCreated map[string]*methodError `json:"notCreated"` +} + +// extractRawHeader reads a single header out of raw without a full MIME walk — enough +// for Email/import to seed message_id_header/cached_from/cached_subject the same way +// SMTP delivery and IMAP APPEND already do for a freshly arriving message. +func extractRawHeader(raw []byte, name string) string { + msg, err := mail.ReadMessage(bytes.NewReader(raw)) + if err != nil { + return "" + } + return msg.Header.Get(name) +} + +// emailImport is Email/import (RFC 8621 §4.7) — the supported way to create a new +// Email in this server (see emailSet's doc comment on why Email/set itself doesn't). +// blobId must reference a blob already uploaded via POST /jmap/upload (see blob.go). +func emailImport(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailImportArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + oldState, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + _, nameByID, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + mailboxIDByFolder := make(map[string]string, len(idByName)) + for name, id := range idByName { + mailboxIDByFolder[name] = strconv.FormatInt(id, 10) + } + + result := emailImportResult{AccountID: strconv.FormatInt(mbox.ID, 10), OldState: oldState, Created: map[string]jmapEmail{}, NotCreated: map[string]*methodError{}} + for clientID, entry := range args.Emails { + raw, ok := blobBytes(b, mbox, entry.BlobID) + if !ok { + result.NotCreated[clientID] = &methodError{Type: "notFound", Description: "unknown blobId"} + continue + } + folder, err := singleMailboxFolder(entry.MailboxIDs, nameByID) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "invalidArguments", Description: err.Error()} + continue + } + messageID := extractRawHeader(raw, "Message-Id") + fromHeader := extractRawHeader(raw, "From") + subject := extractRawHeader(raw, "Subject") + uid, err := b.Mailstore.StoreMessage(mbox.ID, folder, raw, messageID, fromHeader, subject) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + if len(entry.Keywords) > 0 { + b.DB.SetMessageFlags(mbox.ID, uid, flagsFromKeywords(entry.Keywords)) + } + m, err := b.DB.GetMessageByUID(mbox.ID, uid) + if err != nil || m == nil { + continue + } + result.Created[clientID] = buildEmailObject(b, mbox, *m, mailboxIDByFolder, false) + } + if len(result.Created) > 0 { + b.Notify.PublishAccountWide(mbox.ID) + } + newState, err := b.DB.MessagesState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + result.NewState = newState + return result, nil +} diff --git a/internal/jmap/email_test.go b/internal/jmap/email_test.go new file mode 100644 index 0000000..7838901 --- /dev/null +++ b/internal/jmap/email_test.go @@ -0,0 +1,400 @@ +package jmap_test + +import ( + "encoding/json" + "net/http" + "strconv" + "strings" + "testing" + + "mailgoserver/internal/jmap" +) + +func TestJMAPEmailGetQueryChanges(t *testing.T) { + srv, _, store, email, password, mailboxID := newTestJMAPServer(t) + + raw1 := []byte("From: Alice \r\nSubject: First\r\n\r\nBody one.") + if _, err := store.StoreMessage(mailboxID, "INBOX", raw1, "", "Alice ", "First"); err != nil { + t.Fatal(err) + } + + stateResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Email/query", Args: json.RawMessage(`{"filter":{"inMailbox":"1"}}`), ID: "c1"}}, + }) + var q1 struct { + QueryState string `json:"queryState"` + IDs []string `json:"ids"` + Total int `json:"total"` + } + if err := json.Unmarshal(stateResp.MethodResponses[0].Args, &q1); err != nil { + t.Fatal(err) + } + if q1.Total != 1 || len(q1.IDs) != 1 { + t.Fatalf("expected 1 message from Email/query, got %+v", q1) + } + + raw2 := []byte("From: Bob \r\nSubject: Second\r\n\r\nBody two.") + if _, err := store.StoreMessage(mailboxID, "INBOX", raw2, "", "Bob ", "Second"); err != nil { + t.Fatal(err) + } + + getResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Email/get", Args: json.RawMessage(`{"ids":["` + q1.IDs[0] + `"]}`), ID: "c2"}}, + }) + var get struct { + List []struct { + ID string `json:"id"` + Subject string `json:"subject"` + From []struct { + Email string `json:"email"` + } `json:"from"` + } `json:"list"` + } + if err := json.Unmarshal(getResp.MethodResponses[0].Args, &get); err != nil { + t.Fatal(err) + } + if len(get.List) != 1 || get.List[0].Subject != "First" { + t.Fatalf("expected Email/get to return the first message, got %+v", get.List) + } + if len(get.List[0].From) != 1 || get.List[0].From[0].Email != "alice@example.com" { + t.Fatalf("expected From address alice@example.com, got %+v", get.List[0].From) + } + + changesResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Email/changes", Args: json.RawMessage(`{"sinceState":"` + q1.QueryState + `"}`), ID: "c3"}}, + }) + var changes struct { + Created []string `json:"created"` + } + if err := json.Unmarshal(changesResp.MethodResponses[0].Args, &changes); err != nil { + t.Fatal(err) + } + if len(changes.Created) != 1 { + t.Fatalf("expected Email/changes to report exactly 1 new message since the query state, got %+v", changes.Created) + } +} + +func TestJMAPEmailGetFullBodyOnRequest(t *testing.T) { + srv, _, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\nSubject: Hi\r\n\r\nPlain body here.") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hi") + if err != nil { + t.Fatal(err) + } + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/get", + Args: json.RawMessage(`{"ids":["` + strconv.FormatInt(uid, 10) + `"],"fetchTextBodyValues":true}`), + ID: "c1", + }}, + }) + var get struct { + List []struct { + TextBody []struct { + PartID string `json:"partId"` + } `json:"textBody"` + BodyValues map[string]struct { + Value string `json:"value"` + } `json:"bodyValues"` + } `json:"list"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &get); err != nil { + t.Fatal(err) + } + if len(get.List) != 1 || len(get.List[0].TextBody) != 1 { + t.Fatalf("expected fetchTextBodyValues to populate textBody, got %+v", get.List) + } + partID := get.List[0].TextBody[0].PartID + if got := get.List[0].BodyValues[partID].Value; got != "Plain body here." { + t.Errorf("expected the plain body text, got %q", got) + } +} + +func TestJMAPThreadingGroupsRepliesByReferences(t *testing.T) { + _, database, store, _, _, mailboxID := newTestJMAPServer(t) + + root := []byte("From: Alice \r\nSubject: Thread root\r\n\r\nHi.") + rootUID, err := store.StoreMessage(mailboxID, "INBOX", root, "root@example.com", "Alice ", "Thread root") + if err != nil { + t.Fatal(err) + } + + reply := []byte("From: Bob \r\nSubject: Re: Thread root\r\nIn-Reply-To: \r\nReferences: \r\n\r\nReplying.") + replyUID, err := store.StoreMessage(mailboxID, "INBOX", reply, "reply@example.com", "Bob ", "Re: Thread root") + if err != nil { + t.Fatal(err) + } + + rootMsg, err := database.GetMessageByUID(mailboxID, rootUID) + if err != nil || rootMsg == nil { + t.Fatal(err) + } + replyMsg, err := database.GetMessageByUID(mailboxID, replyUID) + if err != nil || replyMsg == nil { + t.Fatal(err) + } + if rootMsg.ThreadID == 0 { + t.Fatal("expected the root message to have a non-zero thread_id") + } + if replyMsg.ThreadID != rootMsg.ThreadID { + t.Errorf("expected the reply to join the root's thread (%d), got %d", rootMsg.ThreadID, replyMsg.ThreadID) + } + + unrelated := []byte("From: Carol \r\nSubject: Unrelated\r\n\r\nSomething else entirely.") + unrelatedUID, err := store.StoreMessage(mailboxID, "INBOX", unrelated, "unrelated@example.com", "Carol ", "Unrelated") + if err != nil { + t.Fatal(err) + } + unrelatedMsg, err := database.GetMessageByUID(mailboxID, unrelatedUID) + if err != nil || unrelatedMsg == nil { + t.Fatal(err) + } + if unrelatedMsg.ThreadID != unrelatedUID { + t.Errorf("expected an unrelated message to start its own singleton thread (== its own id %d), got %d", unrelatedUID, unrelatedMsg.ThreadID) + } +} + +func TestJMAPEmailSetMoveAndFlags(t *testing.T) { + srv, database, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\nSubject: Hi\r\n\r\nBody.") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hi") + if err != nil { + t.Fatal(err) + } + + trashID, err := database.EnsureFolderRow(mailboxID, "Trash") + if err != nil { + t.Fatal(err) + } + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/set", + Args: json.RawMessage(`{"update":{"` + strconv.FormatInt(uid, 10) + `":{"keywords":{"$seen":true,"$flagged":true},"mailboxIds":{"` + strconv.FormatInt(trashID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + var result struct { + Updated map[string]any `json:"updated"` + NotUpdated map[string]any `json:"notUpdated"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + if len(result.NotUpdated) != 0 { + t.Fatalf("expected the update to succeed, got notUpdated %+v", result.NotUpdated) + } + + m, err := database.GetMessageByUID(mailboxID, uid) + if err != nil || m == nil { + t.Fatal(err) + } + if m.Folder != "Trash" { + t.Errorf("expected the message moved to Trash, got folder %q", m.Folder) + } + if !strings.Contains(m.Flags, `\Seen`) || !strings.Contains(m.Flags, `\Flagged`) { + t.Errorf("expected both $seen and $flagged applied, got flags %q", m.Flags) + } +} + +func TestJMAPEmailSetKeywordPatch(t *testing.T) { + srv, database, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\nSubject: Hi\r\n\r\nBody.") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hi") + if err != nil { + t.Fatal(err) + } + + doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/set", + Args: json.RawMessage(`{"update":{"` + strconv.FormatInt(uid, 10) + `":{"keywords/$seen":true}}}`), + ID: "c1", + }}, + }) + m, err := database.GetMessageByUID(mailboxID, uid) + if err != nil || m == nil { + t.Fatal(err) + } + if !strings.Contains(m.Flags, `\Seen`) { + t.Fatalf("expected keywords/$seen patch to add \\Seen, got flags %q", m.Flags) + } +} + +func TestJMAPEmailSetDestroyCreatesTombstone(t *testing.T) { + srv, database, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\nSubject: Hi\r\n\r\nBody.") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hi") + if err != nil { + t.Fatal(err) + } + state, err := database.MessagesState(mailboxID) + if err != nil { + t.Fatal(err) + } + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/set", + Args: json.RawMessage(`{"destroy":["` + strconv.FormatInt(uid, 10) + `"]}`), + ID: "c1", + }}, + }) + var result struct { + Destroyed []string `json:"destroyed"` + NotDestroyed map[string]any `json:"notDestroyed"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + if len(result.NotDestroyed) != 0 || len(result.Destroyed) != 1 { + t.Fatalf("expected the destroy to succeed, got %+v / notDestroyed %+v", result.Destroyed, result.NotDestroyed) + } + + m, err := database.GetMessageByUID(mailboxID, uid) + if err != nil { + t.Fatal(err) + } + if m != nil { + t.Fatal("expected the message row to be gone after destroy") + } + + changesResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Email/changes", Args: json.RawMessage(`{"sinceState":"` + state + `"}`), ID: "c2"}}, + }) + var changes struct { + Destroyed []string `json:"destroyed"` + } + if err := json.Unmarshal(changesResp.MethodResponses[0].Args, &changes); err != nil { + t.Fatal(err) + } + found := false + for _, id := range changes.Destroyed { + if id == strconv.FormatInt(uid, 10) { + found = true + } + } + if !found { + t.Fatalf("expected Email/changes to report %d as destroyed, got %+v", uid, changes.Destroyed) + } +} + +func TestJMAPEmailCopy(t *testing.T) { + srv, database, store, email, password, mailboxID := newTestJMAPServer(t) + + raw := []byte("From: Alice \r\nSubject: Hi\r\n\r\nBody.") + uid, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hi") + if err != nil { + t.Fatal(err) + } + trashID, err := database.EnsureFolderRow(mailboxID, "Trash") + if err != nil { + t.Fatal(err) + } + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/copy", + Args: json.RawMessage(`{"fromAccountId":"` + strconv.FormatInt(mailboxID, 10) + `","create":{"c1":{"id":"` + strconv.FormatInt(uid, 10) + `","mailboxIds":{"` + strconv.FormatInt(trashID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + var result struct { + Created map[string]struct{ ID string } `json:"created"` + NotCreated map[string]any `json:"notCreated"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + if len(result.NotCreated) != 0 { + t.Fatalf("expected the copy to succeed, got notCreated %+v", result.NotCreated) + } + newID, ok := result.Created["c1"] + if !ok { + t.Fatal("expected a created entry for c1") + } + newUID, err := strconv.ParseInt(newID.ID, 10, 64) + if err != nil { + t.Fatal(err) + } + if newUID == uid { + t.Fatal("expected the copy to have a distinct uid from the original") + } + + orig, err := database.GetMessageByUID(mailboxID, uid) + if err != nil || orig == nil { + t.Fatal("expected the original to still exist after copy") + } + if orig.Folder != "INBOX" { + t.Errorf("expected the original to remain in INBOX, got %q", orig.Folder) + } + copyMsg, err := database.GetMessageByUID(mailboxID, newUID) + if err != nil || copyMsg == nil { + t.Fatal("expected the copy to exist") + } + if copyMsg.Folder != "Trash" { + t.Errorf("expected the copy in Trash, got %q", copyMsg.Folder) + } +} + +func TestJMAPEmailImport(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServer(t) + + raw := "From: Alice \r\nSubject: Imported\r\n\r\nImported body." + uploadReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/upload/"+strconv.FormatInt(mailboxID, 10), strings.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + uploadReq.SetBasicAuth(email, password) + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatal(err) + } + defer uploadResp.Body.Close() + if uploadResp.StatusCode != http.StatusOK { + t.Fatalf("expected upload to succeed, got %d", uploadResp.StatusCode) + } + var uploaded struct { + BlobID string `json:"blobId"` + } + if err := json.NewDecoder(uploadResp.Body).Decode(&uploaded); err != nil { + t.Fatal(err) + } + if uploaded.BlobID == "" { + t.Fatal("expected a non-empty blobId from upload") + } + + inboxID, err := database.EnsureFolderRow(mailboxID, "INBOX") + if err != nil { + t.Fatal(err) + } + importResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/import", + Args: json.RawMessage(`{"emails":{"c1":{"blobId":"` + uploaded.BlobID + `","mailboxIds":{"` + strconv.FormatInt(inboxID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + var result struct { + Created map[string]struct { + ID string `json:"id"` + Subject string `json:"subject"` + } `json:"created"` + NotCreated map[string]any `json:"notCreated"` + } + if err := json.Unmarshal(importResp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + if len(result.NotCreated) != 0 { + t.Fatalf("expected the import to succeed, got notCreated %+v", result.NotCreated) + } + created, ok := result.Created["c1"] + if !ok || created.Subject != "Imported" { + t.Fatalf("expected the imported message with subject 'Imported', got %+v", result.Created) + } +} diff --git a/internal/jmap/eventsource.go b/internal/jmap/eventsource.go new file mode 100644 index 0000000..3515bf2 --- /dev/null +++ b/internal/jmap/eventsource.go @@ -0,0 +1,119 @@ +package jmap + +import ( + "encoding/json" + "fmt" + "net/http" + "strconv" + "strings" + "time" +) + +// stateChange is RFC 8620 §7.2's StateChange object, sent as one SSE "data:" line per +// push event. +type stateChange struct { + Type string `json:"@type"` + Changed map[string]map[string]any `json:"changed"` +} + +// currentStates returns the requested types' current state strings for mbox — only +// Email and Mailbox are tracked by this server (see state.go); an unrecognized type +// name in the request is silently skipped rather than erroring, matching JMAP's own +// "ignore unknown capabilities" posture elsewhere. +func currentStates(b *Backend, mboxID int64, types []string) (map[string]any, error) { + out := map[string]any{} + for _, t := range types { + switch t { + case "Email": + state, err := b.DB.MessagesState(mboxID) + if err != nil { + return nil, err + } + out["Email"] = state + case "Mailbox": + state, err := b.DB.FoldersState(mboxID) + if err != nil { + return nil, err + } + out["Mailbox"] = state + } + } + return out, nil +} + +// handleEventSource is GET /jmap/eventsource (RFC 8620 §7.3) — this server's push +// transport. Pushes account-wide, not per-folder like IMAP IDLE/webmail's own SSE +// (internal/webui's mailStream): subscribes to notify.Bus's account-wide sentinel key +// (see notify.Bus.PublishAccountWide) instead of fanning out across every folder, +// since a JMAP client's changed-state query is itself account-wide, not per-folder. +func (b *Backend) handleEventSource(w http.ResponseWriter, r *http.Request) { + mbox := mailboxFromContext(r) + + types := []string{"Email", "Mailbox"} + if raw := r.URL.Query().Get("types"); raw != "" && raw != "*" { + types = strings.Split(raw, ",") + } + closeAfterState := r.URL.Query().Get("closeafter") == "state" + ping := 30 * time.Second + if raw := r.URL.Query().Get("ping"); raw != "" { + if secs, err := strconv.Atoi(raw); err == nil && secs > 0 { + ping = time.Duration(secs) * time.Second + } + } + + flusher, ok := w.(http.Flusher) + if !ok { + http.Error(w, "streaming unsupported", http.StatusInternalServerError) + return + } + w.Header().Set("Content-Type", "text/event-stream") + w.Header().Set("Cache-Control", "no-cache") + w.Header().Set("Connection", "keep-alive") + w.WriteHeader(http.StatusOK) + flusher.Flush() + + sendState := func() bool { + changed, err := currentStates(b, mbox.ID, types) + if err != nil { + return true // keep the connection open — a transient DB error shouldn't kill it + } + body, err := json.Marshal(stateChange{ + Type: "StateChange", + Changed: map[string]map[string]any{strconv.FormatInt(mbox.ID, 10): changed}, + }) + if err != nil { + return true + } + if _, err := fmt.Fprintf(w, "event: state\ndata: %s\n\n", body); err != nil { + return false + } + flusher.Flush() + return true + } + + ch, unsubscribe := b.Notify.Subscribe(mbox.ID, "") + defer unsubscribe() + + keepalive := time.NewTicker(ping) + defer keepalive.Stop() + + ctx := r.Context() + for { + select { + case <-ctx.Done(): + return + case <-ch: + if !sendState() { + return + } + if closeAfterState { + return + } + case <-keepalive.C: + if _, err := fmt.Fprint(w, ": keepalive\n\n"); err != nil { + return + } + flusher.Flush() + } + } +} diff --git a/internal/jmap/eventsource_test.go b/internal/jmap/eventsource_test.go new file mode 100644 index 0000000..9e2dfed --- /dev/null +++ b/internal/jmap/eventsource_test.go @@ -0,0 +1,99 @@ +package jmap_test + +import ( + "bufio" + "context" + "encoding/json" + "net/http" + "strconv" + "strings" + "testing" + "time" + + "mailgoserver/internal/jmap" +) + +// TestJMAPEventSourcePushesOnNewMessage mirrors internal/imapserver's own +// TestIMAPIdlePushesOnNewMessage: open a push connection, create a new message via a +// real Email/import call from a second goroutine, assert a StateChange event arrives +// within a bounded timeout — exercising the actual PublishAccountWide wiring +// email.go's emailImport calls on success, not a manually-triggered test-only signal. +func TestJMAPEventSourcePushesOnNewMessage(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServer(t) + + inboxID, err := database.EnsureFolderRow(mailboxID, "INBOX") + if err != nil { + t.Fatal(err) + } + + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + req, err := http.NewRequestWithContext(ctx, http.MethodGet, srv.URL+"/jmap/eventsource?types=Email&ping=60", nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth(email, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + + dataCh := make(chan string, 4) + go func() { + scanner := bufio.NewScanner(resp.Body) + for scanner.Scan() { + line := scanner.Text() + if strings.HasPrefix(line, "data: ") { + dataCh <- strings.TrimPrefix(line, "data: ") + } + } + }() + + // A short wait for the server's handler to actually reach its Subscribe call + // before publishing — otherwise this notification could fire before anyone is + // listening and be missed (Publish is fire-and-forget, same reasoning + // TestIMAPIdlePushesOnNewMessage documents for the identical race). + time.Sleep(100 * time.Millisecond) + + uploadReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/upload/"+strconv.FormatInt(mailboxID, 10), strings.NewReader("Subject: new\r\n\r\nbody")) + if err != nil { + t.Fatal(err) + } + uploadReq.SetBasicAuth(email, password) + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatal(err) + } + defer uploadResp.Body.Close() + var uploaded struct{ BlobID string } + if err := json.NewDecoder(uploadResp.Body).Decode(&uploaded); err != nil { + t.Fatal(err) + } + doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/import", + Args: json.RawMessage(`{"emails":{"c1":{"blobId":"` + uploaded.BlobID + `","mailboxIds":{"` + strconv.FormatInt(inboxID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + + select { + case data := <-dataCh: + var change struct { + Type string `json:"@type"` + Changed map[string]map[string]any `json:"changed"` + } + if err := json.Unmarshal([]byte(data), &change); err != nil { + t.Fatalf("expected valid JSON StateChange, got %q: %v", data, err) + } + if change.Type != "StateChange" { + t.Errorf("expected @type StateChange, got %q", change.Type) + } + case <-time.After(3 * time.Second): + t.Fatal("timed out waiting for the EventSource push") + } +} diff --git a/internal/jmap/identity.go b/internal/jmap/identity.go new file mode 100644 index 0000000..8a0790b --- /dev/null +++ b/internal/jmap/identity.go @@ -0,0 +1,190 @@ +package jmap + +import ( + "encoding/json" + "strconv" + + "mailgoserver/internal/db" +) + +// jmapIdentity is RFC 8621 §6.1's Identity object. +type jmapIdentity struct { + ID string `json:"id"` + Name string `json:"name"` + Email string `json:"email"` + TextSignature string `json:"textSignature"` + HTMLSignature string `json:"htmlSignature"` + MayDelete bool `json:"mayDelete"` +} + +const primaryIdentityID = "primary" + +// identityRow pairs an Identity id with the address it represents — "primary" for the +// mailbox's own address, "alias-" for each active send-as alias +// (esrv_mailbox_aliases.can_send_as) — see identityGet's doc comment for why +// aliases (not S/MIME/PGP identities) are the source here. +type identityRow struct { + id, email string +} + +func identityRows(b *Backend, mbox *db.Mailbox) ([]identityRow, error) { + out := []identityRow{{id: primaryIdentityID, email: mbox.Email}} + aliases, err := b.DB.ListAliasesForMailbox(mbox.ID) + if err != nil { + return nil, err + } + for _, a := range aliases { + if a.IsActive && a.CanSendAs { + out = append(out, identityRow{id: "alias-" + strconv.FormatInt(a.ID, 10), email: a.Email}) + } + } + return out, nil +} + +func buildIdentity(b *Backend, mbox *db.Mailbox, row identityRow) jmapIdentity { + id := jmapIdentity{ID: row.id, Name: mbox.Email, Email: row.email, MayDelete: false} + sig, err := b.DB.GetDefaultSignature(mbox.ID, false, row.email) + if err == nil && sig != nil { + id.HTMLSignature = sig.ContentHTML + } + return id +} + +type identityGetArgs struct { + IDs *[]string `json:"ids"` +} + +type identityGetResult struct { + AccountID string `json:"accountId"` + State string `json:"state"` + List []jmapIdentity `json:"list"` + NotFound []string `json:"notFound"` +} + +// identityGet is Identity/get (RFC 8621 §6.2). Identities are synthesized from this +// mailbox's own address plus its active send-as aliases (esrv_mailbox_aliases) — not +// from the S/MIME/PGP identity tables, which are signing/encryption key material, a +// different concept from JMAP's "an address + display name + signature you can send +// from" Identity object. State is just the mailbox's own aliases/signatures being +// static within a session in practice; a fixed "1" is sufficient since neither +// changes via any method this server implements. +func identityGet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args identityGetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + rows, err := identityRows(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + result := identityGetResult{AccountID: strconv.FormatInt(mbox.ID, 10), State: "1", List: []jmapIdentity{}, NotFound: []string{}} + if args.IDs == nil { + for _, row := range rows { + result.List = append(result.List, buildIdentity(b, mbox, row)) + } + return result, nil + } + byID := make(map[string]identityRow, len(rows)) + for _, row := range rows { + byID[row.id] = row + } + for _, id := range *args.IDs { + row, ok := byID[id] + if !ok { + result.NotFound = append(result.NotFound, id) + continue + } + result.List = append(result.List, buildIdentity(b, mbox, row)) + } + return result, nil +} + +type identitySetArgs struct { + Update map[string]json.RawMessage `json:"update"` +} + +type identitySetResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + Updated map[string]any `json:"updated"` + NotCreated map[string]*methodError `json:"notCreated"` + NotUpdated map[string]*methodError `json:"notUpdated"` +} + +// identitySet is Identity/set (RFC 8621 §6.3) — limited to updating htmlSignature on +// an existing identity (routed to db.SetDefaultSignature/db.SetSignatureAliasDefault, +// creating a signature row via db.CreateSignature if none exists yet). No create/ +// destroy: the underlying send-as grant stays admin-controlled via +// esrv_mailbox_aliases.can_send_as, unchanged by this method — an Identity here isn't +// an independent object, it's a view onto that grant plus a signature. +func identitySet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args identitySetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + rows, err := identityRows(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + byID := make(map[string]identityRow, len(rows)) + for _, row := range rows { + byID[row.id] = row + } + + result := identitySetResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), OldState: "1", NewState: "1", + Updated: map[string]any{}, NotCreated: map[string]*methodError{}, NotUpdated: map[string]*methodError{}, + } + for idStr, rawPatch := range args.Update { + row, ok := byID[idStr] + if !ok { + result.NotUpdated[idStr] = &methodError{Type: "notFound"} + continue + } + var patch map[string]json.RawMessage + if err := json.Unmarshal(rawPatch, &patch); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidPatch", Description: err.Error()} + continue + } + htmlRaw, ok := patch["htmlSignature"] + if !ok { + if len(patch) == 0 { + result.Updated[idStr] = nil + continue + } + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: "only htmlSignature updates are supported"} + continue + } + var html string + if err := json.Unmarshal(htmlRaw, &html); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidPatch", Description: err.Error()} + continue + } + if err := setIdentitySignature(b, mbox, row, html); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + result.Updated[idStr] = nil + } + return result, nil +} + +func setIdentitySignature(b *Backend, mbox *db.Mailbox, row identityRow, html string) error { + sig, err := b.DB.GetDefaultSignature(mbox.ID, false, row.email) + if err != nil { + return err + } + if sig != nil { + return b.DB.UpdateSignature(mbox.ID, sig.ID, sig.Name, html) + } + name := "JMAP signature (" + row.email + ")" + newID, err := b.DB.CreateSignature(mbox.ID, name, html) + if err != nil { + return err + } + if row.id == primaryIdentityID { + return b.DB.SetDefaultSignature(mbox.ID, newID, false) + } + return b.DB.SetSignatureAliasDefault(mbox.ID, newID, row.email, false) +} diff --git a/internal/jmap/identity_test.go b/internal/jmap/identity_test.go new file mode 100644 index 0000000..183d265 --- /dev/null +++ b/internal/jmap/identity_test.go @@ -0,0 +1,94 @@ +package jmap_test + +import ( + "encoding/json" + "testing" + + "mailgoserver/internal/jmap" +) + +func TestJMAPIdentityGetReflectsSendAsAliases(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServer(t) + + mbox, err := database.GetMailboxByID(mailboxID) + if err != nil || mbox == nil { + t.Fatal(err) + } + if _, err := database.CreateAlias(mailboxID, "sales@example.com", mbox.DomainID, true); err != nil { + t.Fatal(err) + } + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Identity/get", Args: json.RawMessage(`{}`), ID: "c1"}}, + }) + var result struct { + List []struct { + ID string `json:"id"` + Email string `json:"email"` + } `json:"list"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + emails := map[string]bool{} + for _, id := range result.List { + emails[id.Email] = true + } + if !emails[email] { + t.Errorf("expected the mailbox's own address %q among identities, got %+v", email, result.List) + } + if !emails["sales@example.com"] { + t.Errorf("expected the send-as alias among identities, got %+v", result.List) + } +} + +func TestJMAPIdentitySetUpdatesSignature(t *testing.T) { + srv, _, _, email, password, _ := newTestJMAPServer(t) + + getResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Identity/get", Args: json.RawMessage(`{}`), ID: "c1"}}, + }) + var getResult struct { + List []struct{ ID string } `json:"list"` + } + if err := json.Unmarshal(getResp.MethodResponses[0].Args, &getResult); err != nil { + t.Fatal(err) + } + if len(getResult.List) == 0 { + t.Fatal("expected at least the primary identity") + } + primaryID := getResult.List[0].ID + + setResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Identity/set", + Args: json.RawMessage(`{"update":{"` + primaryID + `":{"htmlSignature":"

Best,
Alice

"}}}`), + ID: "c1", + }}, + }) + var setResult struct { + Updated map[string]any `json:"updated"` + NotUpdated map[string]any `json:"notUpdated"` + } + if err := json.Unmarshal(setResp.MethodResponses[0].Args, &setResult); err != nil { + t.Fatal(err) + } + if len(setResult.NotUpdated) != 0 { + t.Fatalf("expected the signature update to succeed, got notUpdated %+v", setResult.NotUpdated) + } + + getResp2 := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Identity/get", Args: json.RawMessage(`{"ids":["` + primaryID + `"]}`), ID: "c2"}}, + }) + var getResult2 struct { + List []struct { + HTMLSignature string `json:"htmlSignature"` + } `json:"list"` + } + if err := json.Unmarshal(getResp2.MethodResponses[0].Args, &getResult2); err != nil { + t.Fatal(err) + } + if len(getResult2.List) != 1 || getResult2.List[0].HTMLSignature != "

Best,
Alice

" { + t.Fatalf("expected the updated signature to be reflected, got %+v", getResult2.List) + } +} diff --git a/internal/jmap/jmap.go b/internal/jmap/jmap.go new file mode 100644 index 0000000..63b7644 --- /dev/null +++ b/internal/jmap/jmap.go @@ -0,0 +1,120 @@ +// Package jmap implements JMAP (RFC 8620 Core + RFC 8621 Mail) as an additional mail +// access protocol alongside this server's existing SMTP/IMAP/webmail stack. It is a +// new protocol *surface* only — every method here reads/writes the exact same +// esrv_mailboxes/esrv_mailbox_messages/esrv_mailbox_folders tables IMAP and webmail +// already use, via internal/mailstore and internal/db, the same relationship webmail +// already has to IMAP today. Served on its own dedicated port/http.Server (see +// main.go), never reachable through internal/webui's mux. +package jmap + +import ( + "encoding/json" + "net/http" + + "gopkg.in/ini.v1" + + "mailgoserver/internal/db" + "mailgoserver/internal/mailstore" + "mailgoserver/internal/notify" + "mailgoserver/internal/relay" + "mailgoserver/internal/smtpserver" + "mailgoserver/internal/toolbox" +) + +// Backend holds this protocol's shared dependencies, mirroring imapserver.Backend's +// shape (DB, Mailstore, Logger, Cfg, Notify). +type Backend struct { + DB *db.DB + Mailstore *mailstore.Store + Notify *notify.Bus + Cfg *ini.File + Logger *toolbox.Logger + Relay *relay.Relay + // SMTP backs EmailSubmission/set — RouteAndDeliver is the shared tail of SMTP's + // own Session.Data, reused rather than duplicated (see route.go's doc comment). + SMTP *smtpserver.Backend +} + +// Invocation is one JMAP method call/response — a 3-element JSON array +// [name, arguments, id] (RFC 8620 §3.2), not a JSON object, hence the custom +// (Un)MarshalJSON below. +type Invocation struct { + Name string + Args json.RawMessage + ID string +} + +func (i *Invocation) UnmarshalJSON(data []byte) error { + var raw [3]json.RawMessage + if err := json.Unmarshal(data, &raw); err != nil { + return err + } + if err := json.Unmarshal(raw[0], &i.Name); err != nil { + return err + } + i.Args = raw[1] + return json.Unmarshal(raw[2], &i.ID) +} + +func (i Invocation) MarshalJSON() ([]byte, error) { + args := i.Args + if args == nil { + args = json.RawMessage("{}") + } + return json.Marshal([3]any{i.Name, args, i.ID}) +} + +// Request is one POST /jmap/api body (RFC 8620 §3.3). +type Request struct { + Using []string `json:"using"` + MethodCalls []Invocation `json:"methodCalls"` +} + +// Response is one POST /jmap/api reply (RFC 8620 §3.4). +type Response struct { + MethodResponses []Invocation `json:"methodResponses"` +} + +// methodError is a JMAP method-level error result (RFC 8620 §3.5.2) — returned as a +// method response named "error" in place of the method's normal result name. +type methodError struct { + Type string `json:"type"` + Description string `json:"description,omitempty"` +} + +func errorResult(id, errType, description string) Invocation { + body, _ := json.Marshal(methodError{Type: errType, Description: description}) + return Invocation{Name: "error", Args: body, ID: id} +} + +// methodFunc is one dispatchable JMAP method (e.g. "Mailbox/get"). args has already +// had back-references resolved (see refs.go) by the time a methodFunc sees them. +type methodFunc func(b *Backend, mbox *db.Mailbox, args json.RawMessage) (result any, err *methodError) + +var methods = map[string]methodFunc{ + "Mailbox/get": mailboxGet, + "Mailbox/query": mailboxQuery, + "Mailbox/changes": mailboxChanges, + "Mailbox/set": mailboxSet, + "Email/get": emailGet, + "Email/query": emailQuery, + "Email/changes": emailChanges, + "Email/set": emailSet, + "Email/copy": emailCopy, + "Email/import": emailImport, + "Identity/get": identityGet, + "Identity/set": identitySet, + "EmailSubmission/set": emailSubmissionSet, +} + +// Mux builds this protocol's routes on their own *http.ServeMux — see the package doc +// comment on why this is never merged into internal/webui's mux. +func (b *Backend) Mux() *http.ServeMux { + mux := http.NewServeMux() + mux.HandleFunc("GET /jmap/session", b.requireAuth(b.handleSession)) + mux.HandleFunc("POST /jmap/api", b.requireAuth(b.handleAPI)) + mux.HandleFunc("POST /jmap/upload/{accountId}", b.requireAuth(func(w http.ResponseWriter, r *http.Request) { handleUpload(b, w, r) })) + mux.HandleFunc("GET /jmap/download/{accountId}/{blobId}/{name}", b.requireAuth(func(w http.ResponseWriter, r *http.Request) { handleDownload(b, w, r) })) + mux.HandleFunc("GET /jmap/eventsource", b.requireAuth(b.handleEventSource)) + return mux +} diff --git a/internal/jmap/jmap_test.go b/internal/jmap/jmap_test.go new file mode 100644 index 0000000..d1a0e58 --- /dev/null +++ b/internal/jmap/jmap_test.go @@ -0,0 +1,180 @@ +package jmap_test + +import ( + "bytes" + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "testing" + + "gopkg.in/ini.v1" + + "mailgoserver/internal/db" + "mailgoserver/internal/jmap" + "mailgoserver/internal/mailstore" + "mailgoserver/internal/notify" + "mailgoserver/internal/toolbox" +) + +// newTestJMAPServer seeds a domain + mailbox + app password (mirroring +// internal/imapserver/imapserver_test.go's newTestMailboxWithAppPassword) and starts +// a real in-process JMAP server over plain HTTP (httptest.Server — TLS is a +// deployment concern of main.go's real listener, not something these protocol-level +// tests need to exercise). +func newTestJMAPServer(t *testing.T) (srv *httptest.Server, database *db.DB, store *mailstore.Store, email, password string, mailboxID int64) { + t.Helper() + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + + domainID, err := database.CreateDomain("example.com") + if err != nil { + t.Fatal(err) + } + + store = mailstore.New(database, mailstore.GenerateDEK(), t.TempDir()) + dek := mailstore.GenerateDEK() + wrapped, nonce, err := store.WrapDEK(dek) + if err != nil { + t.Fatal(err) + } + portalHash, err := db.HashPassword("portal-password-unused") + if err != nil { + t.Fatal(err) + } + email = "inbox@example.com" + mailboxID, err = database.CreateMailbox(email, portalHash, domainID, 5*1024*1024*1024, wrapped, nonce) + if err != nil { + t.Fatal(err) + } + + password = db.GenerateAppPassword(25) + appHash, err := db.HashPassword(password) + if err != nil { + t.Fatal(err) + } + if _, err := database.CreateAppPassword(mailboxID, "test client", appHash, nil); err != nil { + t.Fatal(err) + } + + backend := &jmap.Backend{DB: database, Mailstore: store, Notify: notify.NewBus(), Cfg: ini.Empty(), Logger: toolbox.GetLogger("test")} + srv = httptest.NewServer(backend.Mux()) + t.Cleanup(srv.Close) + return srv, database, store, email, password, mailboxID +} + +// doJMAP posts req to srv's /jmap/api with Basic Auth and decodes the response. +func doJMAP(t *testing.T, srv *httptest.Server, email, password string, req jmap.Request) jmap.Response { + t.Helper() + body, err := json.Marshal(req) + if err != nil { + t.Fatal(err) + } + httpReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/api", bytes.NewReader(body)) + if err != nil { + t.Fatal(err) + } + httpReq.SetBasicAuth(email, password) + resp, err := http.DefaultClient.Do(httpReq) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + var out jmap.Response + if err := json.NewDecoder(resp.Body).Decode(&out); err != nil { + t.Fatal(err) + } + return out +} + +func TestJMAPSessionResource(t *testing.T) { + srv, _, _, email, password, mailboxID := newTestJMAPServer(t) + + req, err := http.NewRequest(http.MethodGet, srv.URL+"/jmap/session", nil) + if err != nil { + t.Fatal(err) + } + req.SetBasicAuth(email, password) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + t.Fatalf("expected 200, got %d", resp.StatusCode) + } + var session struct { + Accounts map[string]any `json:"accounts"` + APIURL string `json:"apiUrl"` + } + if err := json.NewDecoder(resp.Body).Decode(&session); err != nil { + t.Fatal(err) + } + accountID := strconv.FormatInt(mailboxID, 10) + if _, ok := session.Accounts[accountID]; !ok { + t.Fatalf("expected account %q in session accounts, got %+v", accountID, session.Accounts) + } + if session.APIURL == "" { + t.Error("expected a non-empty apiUrl") + } +} + +func TestJMAPSessionResourceRequiresAuth(t *testing.T) { + srv, _, _, _, _, _ := newTestJMAPServer(t) + resp, err := http.Get(srv.URL + "/jmap/session") + if err != nil { + t.Fatal(err) + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusUnauthorized { + t.Fatalf("expected 401 without credentials, got %d", resp.StatusCode) + } + if resp.Header.Get("WWW-Authenticate") == "" { + t.Error("expected a WWW-Authenticate challenge header") + } +} + +func TestJMAPBackReferenceResolution(t *testing.T) { + srv, _, store, email, password, mailboxID := newTestJMAPServer(t) + raw := []byte("From: Alice \r\nSubject: Hello there\r\n\r\nBody text.") + if _, err := store.StoreMessage(mailboxID, "INBOX", raw, "", "Alice ", "Hello there"); err != nil { + t.Fatal(err) + } + + // inMailbox:"1" relies on INBOX materializing as folder id 1 — safe here since + // resolveMailboxes processes StandardMailboxFolders (INBOX first, a fixed-order + // slice) before any other folder, and this test seeds no other folders first. + req := jmap.Request{ + Using: []string{"urn:ietf:params:jmap:core", "urn:ietf:params:jmap:mail"}, + MethodCalls: []jmap.Invocation{ + {Name: "Email/query", Args: json.RawMessage(`{"filter":{"inMailbox":"1"}}`), ID: "c1"}, + {Name: "Email/get", Args: json.RawMessage(`{"#ids":{"resultOf":"c1","name":"Email/query","path":"/ids"}}`), ID: "c2"}, + }, + } + resp := doJMAP(t, srv, email, password, req) + if len(resp.MethodResponses) != 2 { + t.Fatalf("expected 2 method responses, got %d", len(resp.MethodResponses)) + } + if resp.MethodResponses[1].Name == "error" { + t.Fatalf("expected Email/get to succeed via back-reference, got error: %s", resp.MethodResponses[1].Args) + } + var got struct { + List []struct { + ID string `json:"id"` + } `json:"list"` + } + if err := json.Unmarshal(resp.MethodResponses[1].Args, &got); err != nil { + t.Fatal(err) + } + if len(got.List) != 1 { + t.Fatalf("expected the back-referenced Email/get to resolve to 1 message, got %d", len(got.List)) + } +} diff --git a/internal/jmap/mailbox.go b/internal/jmap/mailbox.go new file mode 100644 index 0000000..688a079 --- /dev/null +++ b/internal/jmap/mailbox.go @@ -0,0 +1,526 @@ +package jmap + +import ( + "encoding/json" + "fmt" + "sort" + "strconv" + "strings" + + "mailgoserver/internal/db" +) + +// jmapMailbox is RFC 8621 §2's Mailbox object. +type jmapMailbox struct { + ID string `json:"id"` + Name string `json:"name"` + ParentID *string `json:"parentId"` + Role *string `json:"role"` + SortOrder int `json:"sortOrder"` + TotalEmails int `json:"totalEmails"` + UnreadEmails int `json:"unreadEmails"` + TotalThreads int `json:"totalThreads"` + UnreadThreads int `json:"unreadThreads"` + IsSubscribed bool `json:"isSubscribed"` + MyRights struct { + MayReadItems bool `json:"mayReadItems"` + MayAddItems bool `json:"mayAddItems"` + MayRemoveItems bool `json:"mayRemoveItems"` + MaySetSeen bool `json:"maySetSeen"` + MaySetKeywords bool `json:"maySetKeywords"` + MayCreateChild bool `json:"mayCreateChild"` + MayRename bool `json:"mayRename"` + MayDelete bool `json:"mayDelete"` + MaySubmit bool `json:"maySubmit"` + } `json:"myRights"` +} + +// standardRoles maps this app's fixed standard-folder names to JMAP's role vocabulary +// (RFC 8621 §2, the subset that applies here). +var standardRoles = map[string]string{ + "INBOX": "inbox", + "Sent": "sent", + "Drafts": "drafts", + "Trash": "trash", + "Junk": "junk", +} + +func strPtr(s string) *string { return &s } + +// resolvedMailbox pairs a folder name with everything needed to build a jmapMailbox +// or resolve a mailboxId back to a folder name. +type resolvedMailbox struct { + id, name, parentName string +} + +// resolveMailboxes materializes a stable esrv_mailbox_folders row for every folder +// mbox has (standard or custom — see db.EnsureFolderRow's doc comment on why the 5 +// standard folders need this) and returns them alongside id<->name lookup maps. +func resolveMailboxes(b *Backend, mbox *db.Mailbox) ([]resolvedMailbox, map[string]string, map[string]int64, error) { + names, err := b.DB.AllFoldersForMailbox(mbox.ID) + if err != nil { + return nil, nil, nil, err + } + parents, err := b.DB.FolderParentMap(mbox.ID) + if err != nil { + return nil, nil, nil, err + } + idByName := make(map[string]int64, len(names)) + for _, name := range names { + id, err := b.DB.EnsureFolderRow(mbox.ID, name) + if err != nil { + return nil, nil, nil, err + } + idByName[name] = id + } + idStrByName := make(map[string]string, len(idByName)) + nameByIDStr := make(map[string]string, len(idByName)) + for name, id := range idByName { + s := strconv.FormatInt(id, 10) + idStrByName[name] = s + nameByIDStr[s] = name + } + out := make([]resolvedMailbox, 0, len(names)) + for _, name := range names { + parentName := "" + if !isStandardFolder(name) { + parentName = parents[name] + } + out = append(out, resolvedMailbox{id: idStrByName[name], name: name, parentName: parentName}) + } + return out, nameByIDStr, idByName, nil +} + +func isStandardFolder(name string) bool { + _, ok := standardRoles[name] + return ok +} + +func buildMailboxObject(b *Backend, mbox *db.Mailbox, rm resolvedMailbox, idByName map[string]int64, positions map[string]int, totals, unread map[string]int) jmapMailbox { + m := jmapMailbox{ + ID: rm.id, + Name: rm.name, + SortOrder: unpositionedSortOrder, + TotalEmails: totals[rm.name], + UnreadEmails: unread[rm.name], + IsSubscribed: true, + } + if pos, ok := positions[rm.name]; ok { + m.SortOrder = pos + } + if role, ok := standardRoles[rm.name]; ok { + m.Role = strPtr(role) + } else if rm.parentName != "" { + if pid, ok := idByName[rm.parentName]; ok { + m.ParentID = strPtr(strconv.FormatInt(pid, 10)) + } + } + m.MyRights.MayReadItems = true + m.MyRights.MayAddItems = true + m.MyRights.MayRemoveItems = true + m.MyRights.MaySetSeen = true + m.MyRights.MaySetKeywords = true + m.MyRights.MayCreateChild = true + m.MyRights.MayRename = !isStandardFolder(rm.name) + m.MyRights.MayDelete = !isStandardFolder(rm.name) + m.MyRights.MaySubmit = rm.name == "Sent" + return m +} + +const unpositionedSortOrder = 0 + +type mailboxGetArgs struct { + IDs *[]string `json:"ids"` + Properties *[]string `json:"properties"` +} + +type mailboxGetResult struct { + AccountID string `json:"accountId"` + State string `json:"state"` + List []jmapMailbox `json:"list"` + NotFound []string `json:"notFound"` +} + +func mailboxGet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args mailboxGetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + resolved, _, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + positions, err := b.DB.FolderPositions(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + totals, err := b.DB.CountMessagesByFolder(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + unread, err := b.DB.CountUnreadByFolder(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + state, err := b.DB.FoldersState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + + result := mailboxGetResult{AccountID: strconv.FormatInt(mbox.ID, 10), State: state, List: []jmapMailbox{}, NotFound: []string{}} + if args.IDs == nil { + for _, rm := range resolved { + result.List = append(result.List, buildMailboxObject(b, mbox, rm, idByName, positions, totals, unread)) + } + return result, nil + } + byID := make(map[string]resolvedMailbox, len(resolved)) + for _, rm := range resolved { + byID[rm.id] = rm + } + for _, id := range *args.IDs { + rm, ok := byID[id] + if !ok { + result.NotFound = append(result.NotFound, id) + continue + } + result.List = append(result.List, buildMailboxObject(b, mbox, rm, idByName, positions, totals, unread)) + } + return result, nil +} + +type mailboxQueryFilter struct { + ParentID *string `json:"parentId"` + Name string `json:"name"` +} + +type mailboxQueryArgs struct { + Filter *mailboxQueryFilter `json:"filter"` +} + +type mailboxQueryResult struct { + AccountID string `json:"accountId"` + QueryState string `json:"queryState"` + CanCalculateChanges bool `json:"canCalculateChanges"` + Position int `json:"position"` + IDs []string `json:"ids"` + Total int `json:"total"` +} + +func mailboxQuery(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args mailboxQueryArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + resolved, _, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + positions, err := b.DB.FolderPositions(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + state, err := b.DB.FoldersState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + + matches := make([]resolvedMailbox, 0, len(resolved)) + for _, rm := range resolved { + if args.Filter != nil { + if args.Filter.Name != "" && rm.name != args.Filter.Name { + continue + } + if args.Filter.ParentID != nil { + var parentIDStr string + if pid, ok := idByName[rm.parentName]; ok { + parentIDStr = strconv.FormatInt(pid, 10) + } + if parentIDStr != *args.Filter.ParentID { + continue + } + } + } + matches = append(matches, rm) + } + sort.SliceStable(matches, func(i, j int) bool { + pi, pj := positions[matches[i].name], positions[matches[j].name] + return pi < pj + }) + + ids := make([]string, len(matches)) + for i, rm := range matches { + ids[i] = rm.id + } + return mailboxQueryResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), + QueryState: state, + CanCalculateChanges: false, + Position: 0, + IDs: ids, + Total: len(ids), + }, nil +} + +type mailboxChangesArgs struct { + SinceState string `json:"sinceState"` +} + +type mailboxChangesResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + HasMoreChanges bool `json:"hasMoreChanges"` + Created []string `json:"created"` + Updated []string `json:"updated"` + Destroyed []string `json:"destroyed"` +} + +func mailboxChanges(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args mailboxChangesArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + since, err := strconv.ParseInt(args.SinceState, 10, 64) + if err != nil { + return nil, &methodError{Type: "invalidArguments", Description: "sinceState must be a modseq integer string"} + } + changes, err := b.DB.FolderChangesSince(mbox.ID, since) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + return mailboxChangesResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), + OldState: args.SinceState, + NewState: changes.NewState, + HasMoreChanges: changes.HasMore, + Created: int64sToStrings(changes.Created), + Updated: int64sToStrings(changes.Updated), + Destroyed: []string{}, + }, nil +} + +// maxJMAPFolderNameLen mirrors internal/webui's own maxFolderNameLen — kept as a +// separate constant rather than importing internal/webui (jmap must not depend on the +// webui package) since it's a trivial, stable value both surfaces validate against. +const maxJMAPFolderNameLen = 60 + +func validFolderName(name string) error { + switch { + case name == "": + return fmt.Errorf("name is required") + case len(name) > maxJMAPFolderNameLen: + return fmt.Errorf("name is too long") + case strings.Contains(name, "/"): + return fmt.Errorf(`name can't contain "/"`) + case isStandardFolder(name): + return fmt.Errorf("%s already exists", name) + } + return nil +} + +type mailboxCreateRequest struct { + Name string `json:"name"` + ParentID *string `json:"parentId"` +} + +type mailboxSetArgs struct { + IfInState *string `json:"ifInState"` + Create map[string]mailboxCreateRequest `json:"create"` + Update map[string]json.RawMessage `json:"update"` + Destroy []string `json:"destroy"` +} + +type mailboxSetResult struct { + AccountID string `json:"accountId"` + OldState string `json:"oldState"` + NewState string `json:"newState"` + Created map[string]jmapMailbox `json:"created"` + Updated map[string]any `json:"updated"` + Destroyed []string `json:"destroyed"` + NotCreated map[string]*methodError `json:"notCreated"` + NotUpdated map[string]*methodError `json:"notUpdated"` + NotDestroyed map[string]*methodError `json:"notDestroyed"` +} + +// mailboxRenameFromPatch applies an Mailbox/set update patch, returning the resulting +// name. Only "name" is a supported patch key — this app has no existing primitive to +// re-parent an already-created custom folder to an arbitrary new parent (only +// CreateMailboxFolderUnder's one-time initial placement and MoveFolderToTrash's +// specific re-parent-to-Trash), and no per-folder subscription/sortOrder concept +// beyond the sidebar drag position SetFolderOrder already covers elsewhere — adding +// those is a real feature, out of scope here, so a patch touching them is rejected +// rather than silently ignored. +func mailboxRenameFromPatch(oldName string, patch map[string]json.RawMessage) (string, error) { + newName := oldName + for key, raw := range patch { + switch key { + case "name": + var n string + if err := json.Unmarshal(raw, &n); err != nil { + return "", fmt.Errorf("name: %w", err) + } + newName = strings.TrimSpace(n) + case "parentId", "sortOrder", "isSubscribed", "role": + return "", fmt.Errorf("%s updates are not supported", key) + default: + return "", fmt.Errorf("unsupported property %q", key) + } + } + return newName, nil +} + +// mailboxSet is Mailbox/set (RFC 8621 §2.5). destroy reparents under Trash +// (db.MoveFolderToTrash) — the same soft-delete this app's webmail already uses for +// folder deletion, a deliberate choice over a strict spec-literal hard delete so one +// consistent delete model covers webmail/IMAP/JMAP for the same underlying data. +func mailboxSet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args mailboxSetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + oldState, err := b.DB.FoldersState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + if args.IfInState != nil && *args.IfInState != oldState { + return nil, &methodError{Type: "stateMismatch", Description: "ifInState does not match current state"} + } + resolved, nameByID, idByName, err := resolveMailboxes(b, mbox) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + nameExists := func(name string) bool { + for _, rm := range resolved { + if strings.EqualFold(rm.name, name) { + return true + } + } + return false + } + + result := mailboxSetResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), OldState: oldState, + Created: map[string]jmapMailbox{}, Updated: map[string]any{}, Destroyed: []string{}, + NotCreated: map[string]*methodError{}, NotUpdated: map[string]*methodError{}, NotDestroyed: map[string]*methodError{}, + } + + for clientID, req := range args.Create { + name := strings.TrimSpace(req.Name) + if err := validFolderName(name); err != nil { + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: err.Error()} + continue + } + if nameExists(name) { + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: "a folder named " + name + " already exists"} + continue + } + parentName := "INBOX" + if req.ParentID != nil { + pn, ok := nameByID[*req.ParentID] + if !ok { + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: "unknown parentId"} + continue + } + parentName = pn + } + if root, err := b.DB.FolderRoot(mbox.ID, parentName); err != nil || root != "INBOX" { + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: "new folders can only go under Inbox"} + continue + } + if err := b.DB.CreateMailboxFolderUnder(mbox.ID, name, parentName); err != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + newID, err := b.DB.EnsureFolderRow(mbox.ID, name) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + m := jmapMailbox{ID: strconv.FormatInt(newID, 10), Name: name, SortOrder: unpositionedSortOrder, IsSubscribed: true} + if pid, ok := idByName[parentName]; ok { + m.ParentID = strPtr(strconv.FormatInt(pid, 10)) + } + m.MyRights.MayReadItems, m.MyRights.MayAddItems, m.MyRights.MayRemoveItems = true, true, true + m.MyRights.MaySetSeen, m.MyRights.MaySetKeywords, m.MyRights.MayCreateChild = true, true, true + m.MyRights.MayRename, m.MyRights.MayDelete = true, true + result.Created[clientID] = m + } + + for idStr, rawPatch := range args.Update { + oldName, ok := nameByID[idStr] + if !ok { + result.NotUpdated[idStr] = &methodError{Type: "notFound"} + continue + } + if isStandardFolder(oldName) { + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: "standard folders can't be renamed"} + continue + } + if root, err := b.DB.FolderRoot(mbox.ID, oldName); err != nil || root != "INBOX" { + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: "only a folder still under Inbox can be renamed"} + continue + } + var patch map[string]json.RawMessage + if err := json.Unmarshal(rawPatch, &patch); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidPatch", Description: err.Error()} + continue + } + newName, err := mailboxRenameFromPatch(oldName, patch) + if err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: err.Error()} + continue + } + if newName != oldName { + if err := validFolderName(newName); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: err.Error()} + continue + } + if nameExists(newName) { + result.NotUpdated[idStr] = &methodError{Type: "invalidProperties", Description: "a folder named " + newName + " already exists"} + continue + } + if err := b.DB.RenameMailboxFolder(mbox.ID, oldName, newName); err != nil { + result.NotUpdated[idStr] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + } + result.Updated[idStr] = nil + } + + for _, idStr := range args.Destroy { + name, ok := nameByID[idStr] + if !ok { + result.NotDestroyed[idStr] = &methodError{Type: "notFound"} + continue + } + if isStandardFolder(name) { + result.NotDestroyed[idStr] = &methodError{Type: "invalidProperties", Description: name + " is a standard folder and can't be removed"} + continue + } + if err := b.DB.MoveFolderToTrash(mbox.ID, name); err != nil { + result.NotDestroyed[idStr] = &methodError{Type: "serverFail", Description: err.Error()} + continue + } + result.Destroyed = append(result.Destroyed, idStr) + } + + if len(result.Created) > 0 || len(result.Updated) > 0 || len(result.Destroyed) > 0 { + b.Notify.PublishAccountWide(mbox.ID) + } + + newState, err := b.DB.FoldersState(mbox.ID) + if err != nil { + return nil, &methodError{Type: "serverFail", Description: err.Error()} + } + result.NewState = newState + return result, nil +} + +func int64sToStrings(ids []int64) []string { + out := make([]string, len(ids)) + for i, id := range ids { + out[i] = strconv.FormatInt(id, 10) + } + return out +} diff --git a/internal/jmap/mailbox_test.go b/internal/jmap/mailbox_test.go new file mode 100644 index 0000000..1007934 --- /dev/null +++ b/internal/jmap/mailbox_test.go @@ -0,0 +1,189 @@ +package jmap_test + +import ( + "encoding/json" + "testing" + + "mailgoserver/internal/jmap" +) + +func TestJMAPMailboxGetSynthesizesStandardFolders(t *testing.T) { + srv, _, _, email, password, _ := newTestJMAPServer(t) + + resp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{ + {Name: "Mailbox/get", Args: json.RawMessage(`{}`), ID: "c1"}, + }, + }) + var result struct { + List []struct { + ID string `json:"id"` + Name string `json:"name"` + Role *string `json:"role"` + } `json:"list"` + } + if err := json.Unmarshal(resp.MethodResponses[0].Args, &result); err != nil { + t.Fatal(err) + } + wantNames := map[string]bool{"INBOX": true, "Junk": true, "Sent": true, "Drafts": true, "Trash": true} + seen := map[string]bool{} + for _, m := range result.List { + if m.ID == "" { + t.Errorf("folder %q got no stable id", m.Name) + } + seen[m.Name] = true + } + for name := range wantNames { + if !seen[name] { + t.Errorf("expected standard folder %q in a fresh mailbox's Mailbox/get, got %+v", name, result.List) + } + } +} + +func TestJMAPMailboxQueryAndChanges(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServer(t) + + state1 := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Mailbox/get", Args: json.RawMessage(`{"ids":[]}`), ID: "c1"}}, + }) + var getResult struct { + State string `json:"state"` + } + if err := json.Unmarshal(state1.MethodResponses[0].Args, &getResult); err != nil { + t.Fatal(err) + } + + if err := database.CreateMailboxFolderUnder(mailboxID, "Projects", "INBOX"); err != nil { + t.Fatal(err) + } + + queryResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Mailbox/query", Args: json.RawMessage(`{"filter":{"name":"Projects"}}`), ID: "c2"}}, + }) + var queryResult struct { + IDs []string `json:"ids"` + } + if err := json.Unmarshal(queryResp.MethodResponses[0].Args, &queryResult); err != nil { + t.Fatal(err) + } + if len(queryResult.IDs) != 1 { + t.Fatalf("expected Mailbox/query to find the new 'Projects' folder, got %+v", queryResult.IDs) + } + + changesResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{Name: "Mailbox/changes", Args: json.RawMessage(`{"sinceState":"` + getResult.State + `"}`), ID: "c3"}}, + }) + var changesResult struct { + Created []string `json:"created"` + } + if err := json.Unmarshal(changesResp.MethodResponses[0].Args, &changesResult); err != nil { + t.Fatal(err) + } + found := false + for _, id := range changesResult.Created { + if id == queryResult.IDs[0] { + found = true + } + } + if !found { + t.Fatalf("expected Mailbox/changes to report the new folder %q as created, got %+v", queryResult.IDs[0], changesResult.Created) + } +} + +func TestJMAPMailboxSetCreateRenameDestroy(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServer(t) + + createResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Mailbox/set", + Args: json.RawMessage(`{"create":{"c1":{"name":"Projects"}}}`), + ID: "c1", + }}, + }) + var createResult struct { + Created map[string]struct{ ID string } `json:"created"` + NotCreated map[string]any `json:"notCreated"` + } + if err := json.Unmarshal(createResp.MethodResponses[0].Args, &createResult); err != nil { + t.Fatal(err) + } + if len(createResult.NotCreated) != 0 { + t.Fatalf("expected no notCreated entries, got %+v", createResult.NotCreated) + } + folderID, ok := createResult.Created["c1"] + if !ok || folderID.ID == "" { + t.Fatalf("expected Mailbox/set to create 'Projects', got %+v", createResult.Created) + } + folders, err := database.AllFoldersForMailbox(mailboxID) + if err != nil { + t.Fatal(err) + } + found := false + for _, f := range folders { + if f == "Projects" { + found = true + } + } + if !found { + t.Fatalf("expected 'Projects' to exist after Mailbox/set create, got %+v", folders) + } + + renameResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Mailbox/set", + Args: json.RawMessage(`{"update":{"` + folderID.ID + `":{"name":"Work"}}}`), + ID: "c1", + }}, + }) + var renameResult struct { + Updated map[string]any `json:"updated"` + NotUpdated map[string]any `json:"notUpdated"` + } + if err := json.Unmarshal(renameResp.MethodResponses[0].Args, &renameResult); err != nil { + t.Fatal(err) + } + if len(renameResult.NotUpdated) != 0 { + t.Fatalf("expected the rename to succeed, got notUpdated %+v", renameResult.NotUpdated) + } + folders, err = database.AllFoldersForMailbox(mailboxID) + if err != nil { + t.Fatal(err) + } + if !contains(folders, "Work") || contains(folders, "Projects") { + t.Fatalf("expected 'Projects' renamed to 'Work', got %+v", folders) + } + + destroyResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Mailbox/set", + Args: json.RawMessage(`{"destroy":["` + folderID.ID + `"]}`), + ID: "c1", + }}, + }) + var destroyResult struct { + Destroyed []string `json:"destroyed"` + NotDestroyed map[string]any `json:"notDestroyed"` + } + if err := json.Unmarshal(destroyResp.MethodResponses[0].Args, &destroyResult); err != nil { + t.Fatal(err) + } + if len(destroyResult.NotDestroyed) != 0 || len(destroyResult.Destroyed) != 1 { + t.Fatalf("expected the destroy to succeed, got %+v / notDestroyed %+v", destroyResult.Destroyed, destroyResult.NotDestroyed) + } + root, err := database.FolderRoot(mailboxID, "Work") + if err != nil { + t.Fatal(err) + } + if root != "Trash" { + t.Fatalf("expected 'Work' to have been soft-deleted under Trash, got root %q", root) + } +} + +func contains(list []string, s string) bool { + for _, v := range list { + if v == s { + return true + } + } + return false +} diff --git a/internal/jmap/refs.go b/internal/jmap/refs.go new file mode 100644 index 0000000..f88795a --- /dev/null +++ b/internal/jmap/refs.go @@ -0,0 +1,111 @@ +package jmap + +import ( + "encoding/json" + "fmt" + "strings" +) + +// resultRef is RFC 8620 §3.7's ResultReference shape: {"resultOf": callId, "name": +// methodName, "path": }, substituted in place of a "#property" key in a later +// call's arguments. +type resultRef struct { + ResultOf string `json:"resultOf"` + Name string `json:"name"` + Path string `json:"path"` +} + +// callState tracks each call's raw result within one API request, keyed by callId, so +// a later call in the same batch can reference an earlier one's result via a +// "#property" argument key (see resolveBackReferences). Deliberately does NOT +// implement general JSON Pointer/back-reference resolution: real JMAP Mail clients +// only ever chain a narrow set of path shapes (Email/query's "/ids" into a +// following Email/get or Email/set being the dominant real-world case), so this +// supports exactly: a plain field-path through object keys, and a single trailing +// "/*/" segment projecting one field out of an array of objects. Anything else +// is left unresolved and surfaces as whatever error the target method's own argument +// validation produces for an unrecognized "#foo" key — not silently misinterpreted. +type callState struct { + results map[string]json.RawMessage // callId -> raw method result object +} + +func newCallState() *callState { + return &callState{results: map[string]json.RawMessage{}} +} + +func (cs *callState) record(callID string, result json.RawMessage) { + cs.results[callID] = result +} + +// resolveBackReferences returns args with every "#foo" key replaced by a plain "foo" +// key holding the referenced value, per resultRef. A key that isn't a back-reference +// passes through unchanged. If args isn't a JSON object, it's returned unchanged — +// nothing to resolve, and the target method's own unmarshal will produce a clearer +// error than this layer could. +func (cs *callState) resolveBackReferences(args json.RawMessage) (json.RawMessage, error) { + var raw map[string]json.RawMessage + if err := json.Unmarshal(args, &raw); err != nil { + return args, nil + } + changed := false + out := make(map[string]json.RawMessage, len(raw)) + for k, v := range raw { + if !strings.HasPrefix(k, "#") { + out[k] = v + continue + } + var ref resultRef + if err := json.Unmarshal(v, &ref); err != nil { + return nil, fmt.Errorf("invalid result reference for %q: %w", k, err) + } + result, ok := cs.results[ref.ResultOf] + if !ok { + return nil, fmt.Errorf("result reference %q: no prior call %q in this request", k, ref.ResultOf) + } + resolved, err := jsonPointerLookup(result, ref.Path) + if err != nil { + return nil, fmt.Errorf("result reference %q: %w", k, err) + } + out[strings.TrimPrefix(k, "#")] = resolved + changed = true + } + if !changed { + return args, nil + } + return json.Marshal(out) +} + +// jsonPointerLookup applies the narrow path subset documented on callState to result. +func jsonPointerLookup(result json.RawMessage, path string) (json.RawMessage, error) { + segs := strings.Split(strings.TrimPrefix(path, "/"), "/") + cur := result + for i := 0; i < len(segs); i++ { + if segs[i] == "*" { + if i != len(segs)-2 { + return nil, fmt.Errorf("unsupported path %q: '*' must be the second-to-last segment", path) + } + field := segs[i+1] + var list []map[string]json.RawMessage + if err := json.Unmarshal(cur, &list); err != nil { + return nil, fmt.Errorf("path %q: expected an array of objects: %w", path, err) + } + out := make([]json.RawMessage, 0, len(list)) + for _, obj := range list { + if v, ok := obj[field]; ok { + out = append(out, v) + } + } + return json.Marshal(out) + } + var obj map[string]json.RawMessage + if err := json.Unmarshal(cur, &obj); err != nil { + return nil, fmt.Errorf("path %q: expected an object at segment %q: %w", path, segs[i], err) + } + v, ok := obj[segs[i]] + if !ok { + return nil, fmt.Errorf("path %q: no such field %q", path, segs[i]) + } + cur = v + } + return cur, nil +} diff --git a/internal/jmap/session.go b/internal/jmap/session.go new file mode 100644 index 0000000..8870619 --- /dev/null +++ b/internal/jmap/session.go @@ -0,0 +1,94 @@ +package jmap + +import ( + "encoding/json" + "net/http" + "strconv" +) + +// sessionCapability is one entry under the Session object's "capabilities" (account- +// independent) or an account's own "accountCapabilities" — both are just +// urn -> settings-object maps, an empty object being valid when a capability has no +// extra settings to advertise (RFC 8620 §2). +type sessionCapability = map[string]any + +type sessionAccount struct { + Name string `json:"name"` + IsPersonal bool `json:"isPersonal"` + IsReadOnly bool `json:"isReadOnly"` + AccountCapabilities map[string]sessionCapability `json:"accountCapabilities"` +} + +type sessionResponse struct { + Capabilities map[string]sessionCapability `json:"capabilities"` + Accounts map[string]sessionAccount `json:"accounts"` + PrimaryAccounts map[string]string `json:"primaryAccounts"` + Username string `json:"username"` + APIURL string `json:"apiUrl"` + DownloadURL string `json:"downloadUrl"` + UploadURL string `json:"uploadUrl"` + EventSourceURL string `json:"eventSourceUrl"` + State string `json:"state"` +} + +const ( + capCore = "urn:ietf:params:jmap:core" + capMail = "urn:ietf:params:jmap:mail" + capSubmission = "urn:ietf:params:jmap:submission" +) + +// handleSession serves the RFC 8620 §2 Session resource — the entry point a JMAP +// client fetches once (and re-fetches on the Basic realm challenge/whenever the +// server's "state" changes) to discover every other endpoint URL and this account's +// capabilities. +func (b *Backend) handleSession(w http.ResponseWriter, r *http.Request) { + mbox := mailboxFromContext(r) + accountID := strconv.FormatInt(mbox.ID, 10) + + maxSize := b.Cfg.Section("Mailstore").Key("max_message_bytes").MustInt64(25 * 1024 * 1024) + + resp := sessionResponse{ + Capabilities: map[string]sessionCapability{ + capCore: { + "maxSizeUpload": maxSize, + "maxConcurrentUpload": 4, + "maxSizeRequest": maxSize, + "maxConcurrentRequests": 4, + "maxCallsInRequest": 32, + "maxObjectsInGet": 500, + "maxObjectsInSet": 500, + "collationAlgorithms": []string{}, + }, + capMail: {}, + capSubmission: {}, + }, + Accounts: map[string]sessionAccount{ + accountID: { + Name: mbox.Email, + IsPersonal: true, + IsReadOnly: false, + AccountCapabilities: map[string]sessionCapability{ + capCore: {}, capMail: {}, capSubmission: {}, + }, + }, + }, + PrimaryAccounts: map[string]string{ + capMail: accountID, + capSubmission: accountID, + }, + Username: mbox.Email, + APIURL: "/jmap/api", + DownloadURL: "/jmap/download/{accountId}/{blobId}/{name}?type={type}", + UploadURL: "/jmap/upload/{accountId}", + EventSourceURL: "/jmap/eventsource?types={types}&closeafter={closeafter}&ping={ping}", + } + state, err := b.DB.MessagesState(mbox.ID) + if err != nil { + http.Error(w, "internal error", http.StatusInternalServerError) + return + } + resp.State = state + + w.Header().Set("Content-Type", "application/json") + json.NewEncoder(w).Encode(resp) +} diff --git a/internal/jmap/submission.go b/internal/jmap/submission.go new file mode 100644 index 0000000..f4ef0aa --- /dev/null +++ b/internal/jmap/submission.go @@ -0,0 +1,181 @@ +package jmap + +import ( + "encoding/json" + "net/mail" + "strconv" + "strings" + + "mailgoserver/internal/db" +) + +// jmapAddress is RFC 8621 §7.1's Envelope address shape ({"email": "..."}). +type jmapAddress struct { + Email string `json:"email"` +} + +type jmapEnvelope struct { + MailFrom jmapAddress `json:"mailFrom"` + RcptTo []jmapAddress `json:"rcptTo"` +} + +type emailSubmissionCreate struct { + EmailID string `json:"emailId"` + IdentityID string `json:"identityId"` + Envelope *jmapEnvelope `json:"envelope"` +} + +type emailSubmissionResult struct { + ID string `json:"id"` + EmailID string `json:"emailId"` + IdentityID string `json:"identityId"` + Envelope jmapEnvelope `json:"envelope"` + UndoStatus string `json:"undoStatus"` +} + +type emailSubmissionSetArgs struct { + Create map[string]emailSubmissionCreate `json:"create"` +} + +type emailSubmissionSetResult struct { + AccountID string `json:"accountId"` + Created map[string]emailSubmissionResult `json:"created"` + NotCreated map[string]*methodError `json:"notCreated"` +} + +// senderIdentity resolves identityID to the address it may send as, checking it +// against this mailbox's own address and its active send-as aliases — the same +// authorization esrv_mailbox_aliases.can_send_as already grants elsewhere (webmail +// compose, SMTP AUTH's own sender-spoofing check). An empty identityID defaults to +// the mailbox's own primary address. +func senderIdentity(b *Backend, mbox *db.Mailbox, identityID string) (email string, err *methodError) { + if identityID == "" || identityID == primaryIdentityID { + return mbox.Email, nil + } + rows, e := identityRows(b, mbox) + if e != nil { + return "", &methodError{Type: "serverFail", Description: e.Error()} + } + for _, row := range rows { + if row.id == identityID { + return row.email, nil + } + } + return "", &methodError{Type: "notFound", Description: "unknown identityId"} +} + +// defaultEnvelope derives an Envelope from raw's own To/Cc headers when the client +// didn't supply one (RFC 8621 §7.4: "If not supplied... derived from the Email +// object"). mailFrom is always the resolved sending identity's address, never +// client-controlled independently of identityId. +func defaultEnvelope(raw []byte, fromAddr string) jmapEnvelope { + env := jmapEnvelope{MailFrom: jmapAddress{Email: fromAddr}} + msg, err := mail.ReadMessage(strings.NewReader(string(raw))) + if err != nil { + return env + } + seen := map[string]bool{} + for _, header := range []string{"To", "Cc"} { + addrs, err := msg.Header.AddressList(header) + if err != nil { + continue + } + for _, a := range addrs { + lower := strings.ToLower(a.Address) + if !seen[lower] { + seen[lower] = true + env.RcptTo = append(env.RcptTo, jmapAddress{Email: a.Address}) + } + } + } + return env +} + +// emailSubmissionSet is EmailSubmission/set (RFC 8621 §7.4) — the JMAP send path, +// routing through the exact same internal/smtpserver.Backend.RouteAndDeliver SMTP's +// own DATA command uses, rather than a parallel send implementation (see route.go's +// doc comment). No EmailSubmission/get/persisted history in v1 — the result object +// is synthesized per-response, not independently queryable across sessions later; add +// a table if a client actually depends on that. +func emailSubmissionSet(b *Backend, mbox *db.Mailbox, rawArgs json.RawMessage) (any, *methodError) { + var args emailSubmissionSetArgs + if err := json.Unmarshal(rawArgs, &args); err != nil { + return nil, &methodError{Type: "invalidArguments", Description: err.Error()} + } + + result := emailSubmissionSetResult{ + AccountID: strconv.FormatInt(mbox.ID, 10), + Created: map[string]emailSubmissionResult{}, NotCreated: map[string]*methodError{}, + } + + for clientID, c := range args.Create { + emailID, err := strconv.ParseInt(c.EmailID, 10, 64) + if err != nil { + result.NotCreated[clientID] = &methodError{Type: "notFound", Description: "unknown emailId"} + continue + } + msg, err2 := b.DB.GetMessageByUID(mbox.ID, emailID) + if err2 != nil || msg == nil { + result.NotCreated[clientID] = &methodError{Type: "notFound", Description: "unknown emailId"} + continue + } + fromAddr, mErr := senderIdentity(b, mbox, c.IdentityID) + if mErr != nil { + result.NotCreated[clientID] = mErr + continue + } + + raw, fetchErr := b.Mailstore.FetchMessage(mbox.ID, emailID) + if fetchErr != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: fetchErr.Error()} + continue + } + + env := defaultEnvelope(raw, fromAddr) + if c.Envelope != nil { + env = *c.Envelope + if env.MailFrom.Email == "" { + env.MailFrom.Email = fromAddr + } + } + if len(env.RcptTo) == 0 { + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: "envelope.rcptTo is empty and could not be derived from the message"} + continue + } + + var localRcpts, relayRcpts []string + for _, a := range env.RcptTo { + if lm, lErr := b.Mailstore.ResolveRecipient(a.Email); lErr == nil && lm != nil { + localRcpts = append(localRcpts, a.Email) + } else { + relayRcpts = append(relayRcpts, a.Email) + } + } + + // peerIP "" — no real TCP peer for a JMAP-triggered send; username is this + // mailbox's own address, for the same log attribution SMTP AUTH's username + // already provides. + _, results, allSucceeded, anySucceeded, rdErr := b.SMTP.RouteAndDeliver(env.MailFrom.Email, localRcpts, relayRcpts, raw, "", mbox.Email) + if rdErr != nil { + result.NotCreated[clientID] = &methodError{Type: "serverFail", Description: rdErr.Error()} + continue + } + if !allSucceeded && !anySucceeded { + desc := "delivery failed for every recipient" + if len(results) > 0 && results[0].ErrorMessage != "" { + desc = results[0].ErrorMessage + } + result.NotCreated[clientID] = &methodError{Type: "invalidProperties", Description: desc} + continue + } + + identityID := c.IdentityID + if identityID == "" { + identityID = primaryIdentityID + } + result.Created[clientID] = emailSubmissionResult{ + ID: "sub-" + c.EmailID, EmailID: c.EmailID, IdentityID: identityID, Envelope: env, UndoStatus: "final", + } + } + return result, nil +} diff --git a/internal/jmap/submission_test.go b/internal/jmap/submission_test.go new file mode 100644 index 0000000..1b95a56 --- /dev/null +++ b/internal/jmap/submission_test.go @@ -0,0 +1,238 @@ +package jmap_test + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "path/filepath" + "strconv" + "strings" + "testing" + + "mailgoserver/internal/db" + "mailgoserver/internal/dkim" + "mailgoserver/internal/jmap" + "mailgoserver/internal/mailstore" + "mailgoserver/internal/notify" + "mailgoserver/internal/relay" + "mailgoserver/internal/smtpserver" + "mailgoserver/internal/toolbox" + + "gopkg.in/ini.v1" +) + +// newTestJMAPServerWithSMTP is newTestJMAPServer plus a wired smtpserver.Backend +// (jmap.Backend.SMTP), for EmailSubmission/set tests — mirrors how main.go wires the +// same *smtpserver.Backend into both the SMTP listener and jmap.Backend. +func newTestJMAPServerWithSMTP(t *testing.T) (srv *httptest.Server, database *db.DB, store *mailstore.Store, email, password string, mailboxID int64) { + t.Helper() + dir := t.TempDir() + database, err := db.Open(filepath.Join(dir, "test.db")) + if err != nil { + t.Fatal(err) + } + t.Cleanup(func() { database.Close() }) + + domainID, err := database.CreateDomain("example.com") + if err != nil { + t.Fatal(err) + } + store = mailstore.New(database, mailstore.GenerateDEK(), t.TempDir()) + dek := mailstore.GenerateDEK() + wrapped, nonce, err := store.WrapDEK(dek) + if err != nil { + t.Fatal(err) + } + portalHash, err := db.HashPassword("portal-password-unused") + if err != nil { + t.Fatal(err) + } + email = "inbox@example.com" + mailboxID, err = database.CreateMailbox(email, portalHash, domainID, 5*1024*1024*1024, wrapped, nonce) + if err != nil { + t.Fatal(err) + } + password = db.GenerateAppPassword(25) + appHash, err := db.HashPassword(password) + if err != nil { + t.Fatal(err) + } + if _, err := database.CreateAppPassword(mailboxID, "test client", appHash, nil); err != nil { + t.Fatal(err) + } + + cfg := ini.Empty() + // Mirrors internal/smtpserver's own newTestBackendWithMailbox: these tests + // exercise submission/delivery wiring, not spam/DKIM/SPF/DMARC scoring accuracy + // (a fresh test domain has no real DKIM key, published SPF, or DMARC record), so + // disable enforcement/scoring entirely rather than have an unrelated quarantine + // send the message to Junk instead of INBOX. + cfg.Section("Mailstore").Key("enforce_dkim").SetValue("false") + cfg.Section("Mailstore").Key("enforce_spf").SetValue("false") + cfg.Section("Mailstore").Key("enforce_dmarc").SetValue("false") + cfg.Section("Mailstore").Key("spam_reject_score").SetValue("1000000") + logger := toolbox.GetLogger("test") + smtpBackend := &smtpserver.Backend{ + DB: database, DKIM: dkim.New(database, 1024), Mailstore: store, Relay: relay.New(database, cfg, logger), + Cfg: cfg, HeloHostname: "mail.example.com", AttachmentsBasePath: t.TempDir(), + Notify: notify.NewBus(), Logger: logger, + } + backend := &jmap.Backend{DB: database, Mailstore: store, Notify: notify.NewBus(), Cfg: cfg, SMTP: smtpBackend, Logger: logger} + srv = httptest.NewServer(backend.Mux()) + t.Cleanup(srv.Close) + return srv, database, store, email, password, mailboxID +} + +func TestJMAPSubmissionToLocalMailbox(t *testing.T) { + srv, database, store, email, password, mailboxID := newTestJMAPServerWithSMTP(t) + + // Second local mailbox on the same domain, to submit to. + domains, err := database.ListDomains() + if err != nil || len(domains) == 0 { + t.Fatal(err) + } + dek2 := mailstore.GenerateDEK() + wrapped2, nonce2, err := store.WrapDEK(dek2) + if err != nil { + t.Fatal(err) + } + hash, err := db.HashPassword("unused") + if err != nil { + t.Fatal(err) + } + recipientMailboxID, err := database.CreateMailbox("recipient@example.com", hash, domains[0].ID, 5*1024*1024*1024, wrapped2, nonce2) + if err != nil { + t.Fatal(err) + } + + raw := "From: " + email + "\r\nTo: recipient@example.com\r\nSubject: Hi there\r\n\r\nBody." + uploadReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/upload/"+strconv.FormatInt(mailboxID, 10), strings.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + uploadReq.SetBasicAuth(email, password) + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatal(err) + } + defer uploadResp.Body.Close() + var uploaded struct{ BlobID string } + if err := json.NewDecoder(uploadResp.Body).Decode(&uploaded); err != nil { + t.Fatal(err) + } + + draftsID, err := database.EnsureFolderRow(mailboxID, "Drafts") + if err != nil { + t.Fatal(err) + } + importResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/import", + Args: json.RawMessage(`{"emails":{"c1":{"blobId":"` + uploaded.BlobID + `","mailboxIds":{"` + strconv.FormatInt(draftsID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + var importResult struct { + Created map[string]struct{ ID string } `json:"created"` + } + if err := json.Unmarshal(importResp.MethodResponses[0].Args, &importResult); err != nil { + t.Fatal(err) + } + emailID, ok := importResult.Created["c1"] + if !ok { + t.Fatalf("expected the import to succeed, got %s", importResp.MethodResponses[0].Args) + } + + submitResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "EmailSubmission/set", + Args: json.RawMessage(`{"create":{"s1":{"emailId":"` + emailID.ID + `"}}}`), + ID: "c2", + }}, + }) + var submitResult struct { + Created map[string]struct{ ID string } `json:"created"` + NotCreated map[string]any `json:"notCreated"` + } + if err := json.Unmarshal(submitResp.MethodResponses[0].Args, &submitResult); err != nil { + t.Fatal(err) + } + if len(submitResult.NotCreated) != 0 { + t.Fatalf("expected the submission to succeed, got notCreated %+v", submitResult.NotCreated) + } + if _, ok := submitResult.Created["s1"]; !ok { + t.Fatalf("expected a created entry for s1, got %+v", submitResult.Created) + } + + msgs, err := database.ListMessagesInFolder(recipientMailboxID, "INBOX") + if err != nil { + t.Fatal(err) + } + if len(msgs) != 1 { + t.Fatalf("expected 1 message delivered to the recipient's INBOX, got %d", len(msgs)) + } +} + +func TestJMAPSubmissionToRelayEnqueuesOnQueue(t *testing.T) { + srv, database, _, email, password, mailboxID := newTestJMAPServerWithSMTP(t) + + raw := "From: " + email + "\r\nTo: someone@elsewhere.example\r\nSubject: External\r\n\r\nBody." + uploadReq, err := http.NewRequest(http.MethodPost, srv.URL+"/jmap/upload/"+strconv.FormatInt(mailboxID, 10), strings.NewReader(raw)) + if err != nil { + t.Fatal(err) + } + uploadReq.SetBasicAuth(email, password) + uploadResp, err := http.DefaultClient.Do(uploadReq) + if err != nil { + t.Fatal(err) + } + defer uploadResp.Body.Close() + var uploaded struct{ BlobID string } + if err := json.NewDecoder(uploadResp.Body).Decode(&uploaded); err != nil { + t.Fatal(err) + } + + draftsID, err := database.EnsureFolderRow(mailboxID, "Drafts") + if err != nil { + t.Fatal(err) + } + importResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "Email/import", + Args: json.RawMessage(`{"emails":{"c1":{"blobId":"` + uploaded.BlobID + `","mailboxIds":{"` + strconv.FormatInt(draftsID, 10) + `":true}}}}`), + ID: "c1", + }}, + }) + var importResult struct { + Created map[string]struct{ ID string } `json:"created"` + } + if err := json.Unmarshal(importResp.MethodResponses[0].Args, &importResult); err != nil { + t.Fatal(err) + } + emailID := importResult.Created["c1"].ID + + submitResp := doJMAP(t, srv, email, password, jmap.Request{ + MethodCalls: []jmap.Invocation{{ + Name: "EmailSubmission/set", + Args: json.RawMessage(`{"create":{"s1":{"emailId":"` + emailID + `"}}}`), + ID: "c2", + }}, + }) + var submitResult struct { + NotCreated map[string]any `json:"notCreated"` + } + if err := json.Unmarshal(submitResp.MethodResponses[0].Args, &submitResult); err != nil { + t.Fatal(err) + } + if len(submitResult.NotCreated) != 0 { + t.Fatalf("expected the submission to succeed (queued), got notCreated %+v", submitResult.NotCreated) + } + + var pending int + if err := database.QueryRow(`SELECT COUNT(*) FROM esrv_relay_queue`).Scan(&pending); err != nil { + t.Fatal(err) + } + if pending != 1 { + t.Fatalf("expected exactly 1 esrv_relay_queue row for the relay recipient, got %d", pending) + } +} diff --git a/internal/mailstore/mailstore.go b/internal/mailstore/mailstore.go index 9b2ca9b..1efc57e 100644 --- a/internal/mailstore/mailstore.go +++ b/internal/mailstore/mailstore.go @@ -114,7 +114,10 @@ func openAESGCM(key, ciphertext, nonce []byte) ([]byte, error) { // so it can be used directly as a directory name, mirroring // smtpserver.sanitizePathSegment's spirit (that one only handles domains; this one // also strips "@" and ":" since a full address is used here, not just a domain). -func sanitizePathSegment(s string) string { +// SanitizePathSegment replaces path-unsafe characters in s (a mailbox email address, +// used as a directory name under BasePath) — exported so internal/jmap's blob staging +// area can build the same directory path without duplicating this logic. +func SanitizePathSegment(s string) string { for _, c := range []string{"/", "\\", ":", "@"} { s = strings.ReplaceAll(s, c, "_") } diff --git a/internal/mailstore/store.go b/internal/mailstore/store.go index 3aa75a4..72712fe 100644 --- a/internal/mailstore/store.go +++ b/internal/mailstore/store.go @@ -99,7 +99,7 @@ func (s *Store) storeMessage(mailboxID int64, folder string, raw []byte, message // the file is actually being written), not internalDate — a bulk import/APPEND of // old mail shouldn't retroactively create (or collide into) old dated directories; // INTERNALDATE is purely a DB column, unrelated to where the ciphertext blob lives. - dir := filepath.Join(s.BasePath, sanitizePathSegment(mbox.Email), folder, time.Now().Format("2006-02-Jan")) + dir := filepath.Join(s.BasePath, SanitizePathSegment(mbox.Email), folder, time.Now().Format("2006-02-Jan")) if err := os.MkdirAll(dir, 0o755); err != nil { return 0, err } @@ -118,6 +118,15 @@ func (s *Store) storeMessage(mailboxID int64, folder string, raw []byte, message if err := s.DB.AddMailboxUsedBytes(mailboxID, int64(len(raw))); err != nil { return 0, err } + // JMAP threading (internal/jmap): resolve after insert, since a message's own id + // (its default thread_id, if nothing else matches) doesn't exist until now — see + // db.ThreadForMessage. Best-effort: a lookup error here shouldn't fail the whole + // delivery, the message just starts as its own singleton thread instead. + threadID, err := s.DB.ThreadForMessage(mailboxID, extractHeaderValue(raw, "References"), extractHeaderValue(raw, "In-Reply-To")) + if err != nil || threadID == 0 { + threadID = uid + } + s.DB.SetMessageThreadID(mailboxID, uid, threadID) return uid, nil } @@ -245,6 +254,29 @@ func (s *Store) FetchMessage(mailboxID, uid int64) ([]byte, error) { return openAESGCM(dek, ciphertext, msg.Nonce) } +// CopyMessage duplicates uid into destFolder as a brand-new message — its own +// ciphertext file, its own row, its own uid/modseq/created_modseq (see storeMessage) — +// rather than sharing a ciphertext file across two rows with independent lifecycles +// (a delete of one must never affect the other). Backs JMAP's (internal/jmap) +// Email/copy. Reuses storeMessage directly: same quota check, same encryption, same +// InsertMessage call, and — since it re-parses the identical raw content — the same +// thread_id decision ThreadForMessage would make for the original, so a copy lands in +// the same thread with no special-casing needed here. +func (s *Store) CopyMessage(mailboxID, uid int64, destFolder string) (int64, error) { + m, err := s.DB.GetMessageByUID(mailboxID, uid) + if err != nil { + return 0, err + } + if m == nil { + return 0, fmt.Errorf("mailstore: message %d not found in mailbox %d", uid, mailboxID) + } + raw, err := s.FetchMessage(mailboxID, uid) + if err != nil { + return 0, err + } + return s.storeMessage(mailboxID, destFolder, raw, m.MessageIDHeader, m.CachedFrom, m.CachedSubject, m.InternalDate) +} + // DeleteMessage removes the on-disk ciphertext, the index row, and frees the quota. func (s *Store) DeleteMessage(mailboxID, uid int64) error { msg, err := s.DB.GetMessageByUID(mailboxID, uid) diff --git a/internal/notify/notify.go b/internal/notify/notify.go index a628acb..5debb58 100644 --- a/internal/notify/notify.go +++ b/internal/notify/notify.go @@ -75,3 +75,13 @@ func (b *Bus) Publish(mailboxID int64, folder string) { } } } + +// PublishAccountWide wakes every subscriber of mailboxID's account-wide sentinel key +// (folder ""), which never collides with a real folder name. Used by JMAP's +// EventSource push (internal/jmap), which pushes per-account rather than per-folder +// like IMAP IDLE/webmail's SSE — call this alongside the normal per-folder Publish at +// any call site that creates/mutates/deletes a message or folder, so a JMAP client's +// EventSource connection sees it too. +func (b *Bus) PublishAccountWide(mailboxID int64) { + b.Publish(mailboxID, "") +} diff --git a/internal/smtpserver/domain_ratelimit_test.go b/internal/smtpserver/domain_ratelimit_test.go index a644068..5288c5f 100644 --- a/internal/smtpserver/domain_ratelimit_test.go +++ b/internal/smtpserver/domain_ratelimit_test.go @@ -22,9 +22,8 @@ func TestDomainSendRateLimited(t *testing.T) { if err := backend.DB.SetDomainSendRateLimit(domainID, &limit); err != nil { t.Fatal(err) } - s := &Session{backend: backend} - if limited, err := s.domainSendRateLimited("ratelimited.example"); err != nil || limited { + if limited, err := backend.domainSendRateLimited("ratelimited.example"); err != nil || limited { t.Fatalf("expected not limited with zero sends so far, limited=%v err=%v", limited, err) } @@ -36,18 +35,18 @@ func TestDomainSendRateLimited(t *testing.T) { t.Fatal(err) } } - if limited, err := s.domainSendRateLimited("ratelimited.example"); err != nil || !limited { + if limited, err := backend.domainSendRateLimited("ratelimited.example"); err != nil || !limited { t.Fatalf("expected limited after hitting the cap of 2, limited=%v err=%v", limited, err) } // An unconfigured domain (no limit set) must never limit, regardless of volume. - if limited, err := s.domainSendRateLimited("example.com"); err != nil || limited { + if limited, err := backend.domainSendRateLimited("example.com"); err != nil || limited { t.Fatalf("expected no limit for a domain with send_rate_limit_per_hour unset, limited=%v err=%v", limited, err) } // An unrecognized domain must never limit either (not this server's problem to // cap, and GetDomainByName returning nil must fail open, not error). - if limited, err := s.domainSendRateLimited("nowhere.invalid"); err != nil || limited { + if limited, err := backend.domainSendRateLimited("nowhere.invalid"); err != nil || limited { t.Fatalf("expected no limit for an unrecognized domain, limited=%v err=%v", limited, err) } } diff --git a/internal/smtpserver/mailbox_autoreply_test.go b/internal/smtpserver/mailbox_autoreply_test.go index 488c4fb..86c99a2 100644 --- a/internal/smtpserver/mailbox_autoreply_test.go +++ b/internal/smtpserver/mailbox_autoreply_test.go @@ -88,8 +88,7 @@ func TestAutoReplyNeverFiresForNullSender(t *testing.T) { if err != nil || mbox == nil { t.Fatal(err) } - s := &Session{backend: backend, mailFrom: ""} - s.sendAutoReply(mbox, "Out of office", "away", "") + backend.sendAutoReply("", mbox, "Out of office", "away", "") var count int if err := backend.DB.QueryRow(`SELECT COUNT(*) FROM esrv_mailbox_autoreply_log WHERE mailbox_id = ?`, mailboxID).Scan(&count); err != nil { diff --git a/internal/smtpserver/route.go b/internal/smtpserver/route.go new file mode 100644 index 0000000..5e57df4 --- /dev/null +++ b/internal/smtpserver/route.go @@ -0,0 +1,565 @@ +package smtpserver + +import ( + "errors" + "fmt" + "os" + "path/filepath" + "strings" + "time" + "unsafe" + + "github.com/microcosm-cc/bluemonday" + + "mailgoserver/internal/db" + "mailgoserver/internal/dkim" + "mailgoserver/internal/mailstore" + "mailgoserver/internal/mailview" + "mailgoserver/internal/relay" +) + +// ErrVirusDetected marks RouteAndDeliver's virus-scan rejection — the message was +// never logged or delivered to anyone (distinguished from a per-recipient delivery +// failure, which is still "accepted" and reported via the returned results instead). +// Wrapped with the scanner's detected signature; a caller formats its own +// protocol-specific rejection message from err.Error() (e.g. Session.Data prefixes +// "Message rejected: "). +var ErrVirusDetected = errors.New("virus detected") + +// RouteAndDeliver signs, classifies every recipient (across both localRcpts and +// relayRcpts) as to/cc/bcc from the message's own headers, delivers to localRcpts +// (spam/DKIM/DMARC checks + filter rules + attachment storage opt-in), enqueues +// relayRcpts onto the outbound relay queue, sends a bounce for any partial failure, +// and writes the email log row — the shared tail of Session.Data (SMTP DATA) and +// JMAP EmailSubmission/set (internal/jmap), so both entry points route through +// exactly one implementation of "accept a fully-formed message and get it where it's +// going," rather than duplicating DKIM/DMARC/rspamd/filter-rule logic that must stay +// in sync. +// +// The caller has already decided which envelope recipients are local vs relay (SMTP +// already knows this from Rcpt() time; JMAP submission resolves it itself via +// Mailstore.ResolveRecipient) — RouteAndDeliver only needs the split, not how it was +// derived. peerIP/username are used only for logging/attribution and the +// attachment-storage opt-in check; a caller with no real TCP peer (JMAP submission) +// passes peerIP "". +// +// err is non-nil only for a whole-message rejection (currently just +// ErrVirusDetected) — a per-recipient delivery failure is never an error return, it's +// reported via results/allSucceeded/anySucceeded instead, matching how a real MTA +// splits a multi-recipient transaction's outcome. +func (b *Backend) RouteAndDeliver(mailFrom string, localRcpts, relayRcpts []string, raw []byte, peerIP, username string) (logID int64, results []relay.Result, allSucceeded, anySucceeded bool, err error) { + // unsafe.String views content directly over raw's own backing array instead of a + // real copy — see the original comment on this in Session.Data's history: matters + // on the small-RAM hosts this server targets, and raw is never mutated below, only + // read (via parseMessage's own bytes.NewReader). + content := unsafe.String(unsafe.SliceData(raw), len(raw)) + + messageID := extractMessageID(content, b.HeloHostname) + senderDomain := domainOfAddr(mailFrom) + + var customHeaders [][2]string + if senderDomain != "" { + customHeaders, _ = b.DKIM.GetActiveCustomHeaders(senderDomain) + } + customHeaders = append(customHeaders, + [2]string{"X-Originating-IP", "[" + peerIP + "]"}, + [2]string{"X-Mailer", "NetBro Mail Server 1.0"}, + [2]string{"X-Priority", "3"}, + ) + + allRcpts := make([]string, 0, len(localRcpts)+len(relayRcpts)) + allRcpts = append(allRcpts, localRcpts...) + allRcpts = append(allRcpts, relayRcpts...) + rebuilt := ensureRequiredHeaders(content, messageID, allRcpts, mailFrom, customHeaders) + + signedContent := rebuilt + dkimSigned := false + if senderDomain != "" { + signedContent = b.DKIM.Sign(rebuilt, senderDomain) + dkimSigned = signedContent != rebuilt + } + + // Virus scanning runs once per message (unlike rspamd's per-recipient-domain + // concept below in deliverLocally) — a virus is present or not regardless of who + // it's addressed to, so one scan covers both the relay and local-delivery paths + // that split further down. Hard-rejects the whole transaction on a positive + // match; fails OPEN on a scanner error/unreachable clamd (never blocks mail on a + // scanner outage) and is off entirely unless explicitly enabled. + if b.Cfg.Section("Mailstore").Key("virus_scan_enabled").MustBool(false) { + addr := b.Cfg.Section("Mailstore").Key("clamd_address").MustString("127.0.0.1:3310") + if infected, signature, scanErr := mailstore.ScanVirus(addr, []byte(signedContent)); scanErr != nil { + b.Logger.Error("virus scan unreachable/errored, delivering normally: %v", scanErr) + } else if infected { + b.Logger.Warning("rejected infected message from %s (%s)", mailFrom, signature) + return 0, nil, false, false, fmt.Errorf("%w (%s)", ErrVirusDetected, signature) + } + } + + rebuiltHeaders := existingHeaders(rebuilt) + toHeader := rebuiltHeaders["to"] + ccHeader := rebuiltHeaders["cc"] + subject := rebuiltHeaders["subject"] + // The message's own From: header, not the bare envelope address — used only for + // what's cached/displayed, never for delivery/auth decisions, which stay on + // mailFrom throughout. Falls back to the envelope address if missing/empty. + fromHeader := rebuiltHeaders["from"] + if fromHeader == "" { + fromHeader = mailFrom + } + + // Attachment storage: only if the authenticated sender or whitelisted IP opted in. + storeMessage := false + if sender, _ := b.DB.GetSenderByEmail(mailFrom); sender != nil && sender.StoreMessageContent { + storeMessage = true + } else if wl, _ := b.DB.GetWhitelistedIP(peerIP, senderDomain); wl != nil && wl.StoreMessageContent { + storeMessage = true + } + + // wantAttachments=storeMessage: decoding every attachment fully into memory is + // only useful when they're about to be written to disk below. + parsed, parseErr := parseMessage(raw, storeMessage) + + type savedAttachment struct { + Filename, ContentType, FilePath string + Size int64 + } + var toSave []savedAttachment + if storeMessage && parseErr == nil && len(parsed.Attachments) > 0 { + usernameOrIP := username + if usernameOrIP == "" && peerIP != "" { + usernameOrIP = sanitizePathSegment(peerIP, ":") + } else { + usernameOrIP = sanitizePathSegment(usernameOrIP, "/\\") + } + storagePath := attachmentStoragePath(b.AttachmentsBasePath, senderDomain, usernameOrIP, time.Now()) + if err := os.MkdirAll(storagePath, 0o755); err == nil { + prefix := cleanMessageIDPrefix(messageID) + for _, a := range parsed.Attachments { + filename := prefix + "_" + sanitizeAttachmentFilename(a.Filename) + fullPath := filepath.Join(storagePath, filename) + if err := os.WriteFile(fullPath, a.Data, 0o644); err == nil { + toSave = append(toSave, savedAttachment{Filename: a.Filename, ContentType: a.ContentType, FilePath: fullPath, Size: int64(len(a.Data))}) + } else { + b.Logger.Error("Failed to write attachment %s: %v", filename, err) + } + } + } + } + + // Classify each recipient as to/cc/bcc by presence in the To/Cc headers — + // anything not literally present in either is inferred BCC. Local and relay + // recipients are classified separately (both against the same headers) since the + // caller already split them; the classification itself doesn't care which group + // a recipient is in. + toList := parseAddressList(toHeader) + ccList := parseAddressList(ccHeader) + classify := func(rcpts []string) []string { + types := make([]string, len(rcpts)) + for i, rcpt := range rcpts { + lower := strings.ToLower(rcpt) + switch { + case containsStr(toList, lower): + types[i] = "to" + case containsStr(ccList, lower): + types[i] = "cc" + default: + types[i] = "bcc" + } + } + return types + } + localTypes := classify(localRcpts) + relayTypes := classify(relayRcpts) + + // Relay recipients are never delivered inline here — that would block the + // caller's response on however long the recipient domain's MX takes to answer. + // Instead a "queued" placeholder Result is recorded now and the real attempt + // happens later via EnqueueForDelivery (below, once logID exists) + the + // background worker in internal/relay/queue.go. + if len(relayRcpts) > 0 { + if limited, rlErr := b.domainSendRateLimited(senderDomain); rlErr != nil { + b.Logger.Error("send-rate-limit check for domain %s: %v", senderDomain, rlErr) + for i, rcpt := range relayRcpts { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "queued"}) + } + } else if limited { + for i, rcpt := range relayRcpts { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "failed", ErrorCode: "450", ErrorMessage: "Sending rate limit exceeded for domain " + senderDomain + ", try again later"}) + } + relayRcpts, relayTypes = nil, nil + } else { + for i, rcpt := range relayRcpts { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "queued"}) + } + } + } + if len(localRcpts) > 0 { + results = append(results, b.deliverLocally(mailFrom, peerIP, localRcpts, localTypes, signedContent, messageID, subject, fromHeader)...) + } + + // "queued" is neither a known success nor a known failure yet — the worker + // resolves it later (and bounces then, on genuine final failure). Only count + // actual failures here so the immediate bounce-on-partial-failure block below + // doesn't fire for a message that's simply still in flight. + var failed []relay.Result + for _, res := range results { + if res.Status != "success" && res.Status != "queued" { + failed = append(failed, res) + } + } + allSucceeded = len(results) > 0 && len(failed) == 0 + anySucceeded = len(results) > len(failed) + + // A single response can't express "delivered to some recipients, not others" — + // rejecting the whole transaction here would make a connecting SMTP server's own + // retry logic re-deliver to the recipients that already succeeded. So: accept + // (the caller's job) and bounce the failed subset back to the sender instead, + // exactly like a real MTA splitting a multi-recipient transaction's outcome. + // Never sent for a *total* failure (the caller rejects outright instead) or for a + // null-sender message or a currently-blacklisted peer, so a delivery failure + // never becomes a free "yes, that mailbox doesn't exist" oracle for abuse. + if len(failed) > 0 && anySucceeded && mailFrom != "" { + if blacklisted, _ := b.DB.IsIPBlacklisted(peerIP); !blacklisted { + if sbErr := b.Relay.SendBounce(mailFrom, subject, messageID, failed); sbErr != nil { + b.Logger.Error("send bounce to %s: %v", mailFrom, sbErr) + } + } + } + + var emailHeaders string + if parseErr == nil { + emailHeaders = strings.Join(parsed.HeaderLines, "\n") + } + + // Privacy default: only headers (and Subject) go into the admin-visible log, + // never the message itself — unless this sender/IP explicitly opted in via + // "Store Full Message Content" (storeMessage above), or the message was + // quarantined to Junk for at least one recipient, in which case an admin + // genuinely needs to see it to judge a spam/abuse report. + storeContent := storeMessage + if !storeContent { + for _, res := range results { + if res.Quarantined { + storeContent = true + break + } + } + } + loggedBody := "" + if storeContent { + loggedBody = signedContent + } + + logID, logErr := b.Relay.LogEmail(b.Cfg, peerIP, mailFrom, toHeader, ccHeader, "", subject, emailHeaders, loggedBody, messageID, username, dkimSigned, results) + if logErr != nil { + b.Logger.Error("Failed to log email: %v", logErr) + } else { + for _, a := range toSave { + if aErr := b.DB.InsertEmailAttachment(db.EmailAttachment{ + EmailLogID: logID, Filename: a.Filename, ContentType: a.ContentType, FilePath: a.FilePath, Size: a.Size, + }); aErr != nil { + b.Logger.Error("Failed to record attachment %s: %v", a.Filename, aErr) + } + } + if len(relayRcpts) > 0 { + if eErr := b.Relay.EnqueueForDelivery(logID, mailFrom, relayRcpts, relayTypes, signedContent); eErr != nil { + b.Logger.Error("Failed to enqueue relay delivery: %v", eErr) + } + } + } + + return logID, results, allSucceeded, anySucceeded, nil +} + +// deliverLocally runs the inbound DKIM/SPF/DMARC/spam checks once for the message +// (they don't vary per recipient) and stores it into each resolved local mailbox, +// producing one relay.Result per recipient so it can be merged into the same +// LogEmail/allSucceeded logic as relay results. Each recipient is resolved fresh via +// Mailstore.ResolveRecipient — the same resolution Session.Rcpt() itself already runs +// at RCPT time — rather than a pre-built cache, since RouteAndDeliver's callers may +// not have one (a JMAP submission has no RCPT phase at all); not a hot path that +// needs the micro-optimization of avoiding one extra DB query per recipient. +func (b *Backend) deliverLocally(mailFrom, peerIP string, rcpts, types []string, signedContent, messageID, subject, fromDisplay string) []relay.Result { + senderDomain := domainOfAddr(mailFrom) + dkimPass := senderDomain != "" && dkim.VerifyInbound(signedContent, senderDomain) + spfPass := mailstore.CheckSPF(mailFrom, peerIP) + heuristicScore := mailstore.SpamScore(peerIP, map[string]string{"subject": subject}, dkimPass, spfPass) + rejectScore := b.Cfg.Section("Mailstore").Key("spam_reject_score").MustInt(5) + rspamdEnabled := b.Cfg.Section("Rspamd").Key("enabled").MustBool(false) + rspamdURL := b.Cfg.Section("Rspamd").Key("url").MustString("http://127.0.0.1:11333") + rspamdRejectScore := b.Cfg.Section("Rspamd").Key("reject_score").MustInt(15) + + // Parsed once for every local recipient (not per-recipient — same message body + // for all of them) so filter rules can match on body text / attachment presence + // without every mailbox needing its own parse pass. + bodyText, hasAttachment := "", "no" + if parsedForRules, err := mailview.Parse(strings.NewReader(signedContent)); err == nil { + bodyText = parsedForRules.TextBody + if bodyText == "" && parsedForRules.HTMLBody != "" { + bodyText = bluemonday.StrictPolicy().Sanitize(parsedForRules.HTMLBody) + } + if len(parsedForRules.Attachments) > 0 { + hasAttachment = "yes" + } + } + + enforceDKIM := b.Cfg.Section("Mailstore").Key("enforce_dkim").MustBool(true) + enforceSPF := b.Cfg.Section("Mailstore").Key("enforce_spf").MustBool(true) + enforceDMARC := b.Cfg.Section("Mailstore").Key("enforce_dmarc").MustBool(true) + + // DMARC ties DKIM/SPF together via alignment to the visible From: header's + // domain — a materially different check from dkimPass/spfPass above (which align + // to the envelope's mailFrom domain). + fromHeaderAddrs := parseAddressList(fromDisplay) + fromHeaderDomain := "" + if len(fromHeaderAddrs) > 0 { + fromHeaderDomain = domainOfAddr(fromHeaderAddrs[0]) + } + dmarcFailPolicy := "" // "", "quarantine", or "reject" + if enforceDMARC && fromHeaderDomain != "" { + if pol := mailstore.LookupDMARCPolicy(fromHeaderDomain); pol != nil { + orgDomain := mailstore.OrganizationalDomain(fromHeaderDomain) + dkimAligned := dkim.VerifyInbound(signedContent, fromHeaderDomain) + spfAligned := spfPass && mailstore.OrganizationalDomain(senderDomain) == orgDomain + if !dkimAligned && !spfAligned { + dmarcFailPolicy = pol.EffectivePolicy(fromHeaderDomain, orgDomain) + } + } + } + + // rspamd is checked at most once per message and reused for every local + // recipient — safe for this deployment (no reliance on rspamd's per-recipient + // personalization); content and mailFrom are identical for every recipient + // regardless. + rspamdChecked := false + var rspamdScore float64 + var rspamdAction string + var rspamdOK bool + checkRspamdOnce := func() (float64, string, bool) { + if !rspamdChecked { + if score, action, err := mailstore.CheckRspamd(rspamdURL, []byte(signedContent), mailFrom, rcpts[0]); err == nil { + rspamdScore, rspamdAction, rspamdOK = score, action, true + } + rspamdChecked = true + } + return rspamdScore, rspamdAction, rspamdOK + } + + results := make([]relay.Result, 0, len(rcpts)) + for i, rcpt := range rcpts { + mbox, resolveErr := b.Mailstore.ResolveRecipient(rcpt) + if resolveErr != nil || mbox == nil { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: "550", ErrorMessage: "No such mailbox"}) + continue + } + folder := "INBOX" + markRead := false + spamGated := false + var tags []string + + // An explicit per-mailbox "allow" entry scoped to "all" bypasses every check + // below entirely. A narrower scope (spf/dkim/spam) only suppresses that one + // check. + scope, hasAllow, _ := b.DB.AllowScope(mbox.ID, mailFrom) + if hasAllow && scope == "all" { + // unchanged existing behavior: full bypass, no scoring, no tagging + } else if junked, _ := b.DB.IsJunked(mbox.ID, mailFrom); junked { + // The mailbox owner's own Blocklist also bypasses scoring entirely, + // straight to Junk — a *soft* quarantine (still delivered, just hidden), + // unlike the admin's separate hard-reject block list (checked at RCPT + // time, a different tool for a different job). + folder = "Junk" + spamGated = true + } else { + suppressDKIM := hasAllow && scope == "dkim" + suppressSPF := hasAllow && scope == "spf" + suppressSpam := hasAllow && scope == "spam" + suppressDMARC := hasAllow && scope == "dmarc" + + if enforceDKIM && !dkimPass && !suppressDKIM { + tags = append(tags, "Failed DKIM") + } + if enforceSPF && !spfPass && !suppressSPF { + tags = append(tags, "Failed SPF") + } + + hardReject := false + if !suppressDMARC { + switch dmarcFailPolicy { + case "reject": + tags = append(tags, "Failed DMARC") + hardReject = true + case "quarantine": + tags = append(tags, "Failed DMARC") + } + } + if !suppressSpam { + quarantine := heuristicScore >= rejectScore + if rspamdEnabled { + if score, rAction, ok := checkRspamdOnce(); ok { + // rspamd's own "reject" action is a considered policy + // decision worth still hard-rejecting to avoid backscatter; a + // bare score threshold hit is quarantined instead, so a false + // positive is recoverable from Junk rather than silently + // bounced with no trace. + if rAction == "reject" { + hardReject = true + } else if score >= float64(rspamdRejectScore) { + quarantine = true + } + } + } + if quarantine { + tags = append(tags, "SPAM") + } + } + if hardReject { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: "550", ErrorMessage: "Message rejected as spam"}) + continue + } + if len(tags) > 0 { + folder = "Junk" + spamGated = true + } + } + + // A tagged message's per-recipient copy gets its Subject prepended with why + // it was flagged — both in the stored raw content and the cached subject. + // This does invalidate that copy's own DKIM signature, harmless since + // spamGated is always true whenever tags are non-empty, and a tagged copy is + // never relayed/forwarded, only locally stored/read. + recipientContent := signedContent + recipientSubject := subject + if len(tags) > 0 { + tag := strings.Join(tags, ", ") + recipientContent = prependSubjectTag(signedContent, tag) + recipientSubject = "***" + tag + "***" + if subject != "" { + recipientSubject += " " + subject + } + } + + // Filter rules organize legitimate mail the recipient already trusts arriving + // in their INBOX — a quarantined message skips them entirely. + if !spamGated { + // Persistent mailbox-level forwarding, distinct from and independent of a + // filter rule's own "forward" action below. + if mbox.ForwardTo != nil && *mbox.ForwardTo != "" { + forwardTo, mailboxEmail, keepCopy := *mbox.ForwardTo, mbox.Email, mbox.ForwardKeepCopy + b.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { + if len(res) > 0 && res[0].Status != "success" { + b.Logger.Error("mailbox forwarding: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) + } + }) + if !keepCopy { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) + continue + } + } + + action, err := b.Mailstore.ApplyRules(mbox.ID, map[string]string{ + "from": mailFrom, "to": rcpt, "subject": subject, + "body": bodyText, "has_attachment": hasAttachment, "recipient_type": types[i], + }) + if err != nil { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: "450", ErrorMessage: err.Error()}) + continue + } + if action.ForwardTo != "" { + // Fire-and-forget: forwarding is a side effect layered on top of this + // recipient's own local delivery, not a substitute for it. + forwardTo, mailboxEmail := action.ForwardTo, mbox.Email + b.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { + if len(res) > 0 && res[0].Status != "success" { + b.Logger.Error("forward rule: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) + } + }) + if !action.KeepCopy { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) + continue + } + } + if action.Drop { + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Discarded by filter rule"}) + continue + } + if action.AutoReply { + b.sendAutoReply(mailFrom, mbox, action.AutoReplySubject, action.AutoReplyBody, messageID) + } + if action.Folder != "" { + folder = action.Folder + } + markRead = action.MarkRead + } + + uid, err := b.Mailstore.StoreMessage(mbox.ID, folder, []byte(recipientContent), messageID, fromDisplay, recipientSubject) + if err != nil { + errCode, errMsg := "450", err.Error() + if err == mailstore.ErrQuotaExceeded { + errCode, errMsg = "552", "Mailbox quota exceeded" + } + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: errCode, ErrorMessage: errMsg}) + continue + } + if markRead { + if err := b.DB.SetMessageFlags(mbox.ID, uid, `\Seen`); err != nil { + b.Logger.Error("mark_read rule failed to set flag for message %d: %v", uid, err) + } + } + b.Notify.Publish(mbox.ID, folder) + b.Notify.PublishAccountWide(mbox.ID) + serverResponse := "Delivered to local mailbox" + if spamGated { + serverResponse = "Quarantined to Junk folder" + } + results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: serverResponse, Quarantined: spamGated}) + } + return results +} + +// sendAutoReply fires a vacation-responder reply to mailFrom, fire-and-forget (a slow/ +// unreachable target must never delay the caller), after two loop/storm-prevention +// checks: never reply to a null-sender message (a bounce/DSN), and never reply to the +// same sender more than once per rolling 24h (esrv_mailbox_autoreply_log). +func (b *Backend) sendAutoReply(mailFrom string, mbox *db.Mailbox, subject, body, inReplyTo string) { + if mailFrom == "" { + return + } + if recent, err := b.DB.HasRecentAutoReply(mbox.ID, mailFrom); err != nil || recent { + return + } + if subject == "" { + subject = "Automatic reply" + } + mailboxEmail, replyTo := mbox.Email, mailFrom + raw := buildAutoReplyMessage(b.HeloHostname, mailboxEmail, replyTo, subject, body, inReplyTo) + b.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{replyTo}, raw, []string{"to"}, func(res []relay.Result) { + if len(res) > 0 && res[0].Status != "success" { + b.Logger.Error("auto-reply to %s failed: %s", replyTo, res[0].ErrorMessage) + } + }) + if err := b.DB.RecordAutoReply(mbox.ID, replyTo); err != nil { + b.Logger.Error("record auto-reply to %s: %v", replyTo, err) + } +} + +// domainSendRateLimited reports whether domain has hit its own admin-configured +// outbound send-rate cap (esrv_domains.send_rate_limit_per_hour) within the last +// rolling hour. Unconfigured (nil limit) or an unrecognized/empty domain never limits. +func (b *Backend) domainSendRateLimited(domain string) (bool, error) { + if domain == "" { + return false, nil + } + dom, err := b.DB.GetDomainByName(domain) + if err != nil { + return false, err + } + if dom == nil || dom.SendRateLimitPerHour == nil { + return false, nil + } + count, err := b.DB.CountRecentSendsForDomain(domain, time.Now().Add(-time.Hour)) + if err != nil { + return false, err + } + return count >= *dom.SendRateLimitPerHour, nil +} diff --git a/internal/smtpserver/session.go b/internal/smtpserver/session.go index 6bb666f..50eafe6 100644 --- a/internal/smtpserver/session.go +++ b/internal/smtpserver/session.go @@ -1,23 +1,19 @@ package smtpserver import ( + "errors" "fmt" "io" "net" - "os" - "path/filepath" "strings" "time" - "unsafe" "github.com/emersion/go-smtp" - "github.com/microcosm-cc/bluemonday" "gopkg.in/ini.v1" "mailgoserver/internal/abuseguard" "mailgoserver/internal/db" "mailgoserver/internal/dkim" "mailgoserver/internal/mailstore" - "mailgoserver/internal/mailview" "mailgoserver/internal/notify" "mailgoserver/internal/relay" "mailgoserver/internal/toolbox" @@ -244,552 +240,40 @@ func (s *Session) Data(r io.Reader) error { if err != nil { return internalError("Internal server error") } - // unsafe.String views content directly over raw's own backing array instead of - // string(raw)'s real copy — on a message near [Mailstore] max_message_bytes (25MB - // default) that's a second full-message-sized allocation for no benefit, which - // matters on the small-RAM hosts this server targets. Safe only because raw is - // never mutated again below (only read, by parseMessage's own bytes.NewReader) — - // if that ever changes, this must go back to a real copy. - content := unsafe.String(unsafe.SliceData(raw), len(raw)) - messageID := extractMessageID(content, s.backend.HeloHostname) - senderDomain := domainOfAddr(s.mailFrom) - - var customHeaders [][2]string - if senderDomain != "" { - customHeaders, _ = s.backend.DKIM.GetActiveCustomHeaders(senderDomain) - } - customHeaders = append(customHeaders, - [2]string{"X-Originating-IP", "[" + s.peerIP + "]"}, - [2]string{"X-Mailer", "NetBro Mail Server 1.0"}, - [2]string{"X-Priority", "3"}, - ) - - rebuilt := ensureRequiredHeaders(content, messageID, s.rcptTos, s.mailFrom, customHeaders) - - signedContent := rebuilt - dkimSigned := false - if senderDomain != "" { - signedContent = s.backend.DKIM.Sign(rebuilt, senderDomain) - dkimSigned = signedContent != rebuilt - } - - // Virus scanning runs once per message (unlike rspamd's per-recipient check - // below in deliverLocally) — a virus is present or not regardless of who it's - // addressed to, so one scan covers both the relay and local-delivery paths that - // split further down. Hard-rejects the whole transaction on a positive match, - // mirroring rspamd's own "reject" action precedent; fails OPEN on a scanner - // error/unreachable clamd (never blocks mail on a scanner outage, same posture - // CheckRspamd already has) and is off entirely unless explicitly enabled. - if s.backend.Cfg.Section("Mailstore").Key("virus_scan_enabled").MustBool(false) { - addr := s.backend.Cfg.Section("Mailstore").Key("clamd_address").MustString("127.0.0.1:3310") - if infected, signature, err := mailstore.ScanVirus(addr, []byte(signedContent)); err != nil { - s.backend.Logger.Error("virus scan unreachable/errored, delivering normally: %v", err) - } else if infected { - s.backend.Logger.Warning("rejected infected message from %s (%s)", s.mailFrom, signature) - return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.NoEnhancedCode, Message: "Message rejected: virus detected (" + signature + ")"} - } - } - - rebuiltHeaders := existingHeaders(rebuilt) - toHeader := rebuiltHeaders["to"] - ccHeader := rebuiltHeaders["cc"] - subject := rebuiltHeaders["subject"] - // The message's own From: header (e.g. "Bob Marley "), not the - // bare SMTP envelope address — used only for what's cached/displayed (webmail's - // folder list), never for delivery/auth decisions, which stay on s.mailFrom - // throughout. Falls back to the envelope address if the header's missing/empty. - fromHeader := rebuiltHeaders["from"] - if fromHeader == "" { - fromHeader = s.mailFrom - } - - // Attachment storage: only if the authenticated sender or whitelisted IP opted in. - storeMessage := false - if sender, _ := s.backend.DB.GetSenderByEmail(s.mailFrom); sender != nil && sender.StoreMessageContent { - storeMessage = true - } else if wl, _ := s.backend.DB.GetWhitelistedIP(s.peerIP, senderDomain); wl != nil && wl.StoreMessageContent { - storeMessage = true - } - - // wantAttachments=storeMessage: decoding every attachment fully into memory is - // only useful when they're about to be written to disk below — see parseMessage's - // own comment. - parsed, parseErr := parseMessage(raw, storeMessage) - - type savedAttachment struct { - Filename, ContentType, FilePath string - Size int64 - } - var toSave []savedAttachment - if storeMessage && parseErr == nil && len(parsed.Attachments) > 0 { - usernameOrIP := s.username - if usernameOrIP == "" && s.peerIP != "" { - usernameOrIP = sanitizePathSegment(s.peerIP, ":") - } else { - usernameOrIP = sanitizePathSegment(usernameOrIP, "/\\") - } - storagePath := attachmentStoragePath(s.backend.AttachmentsBasePath, senderDomain, usernameOrIP, time.Now()) - if err := os.MkdirAll(storagePath, 0o755); err == nil { - prefix := cleanMessageIDPrefix(messageID) - for _, a := range parsed.Attachments { - filename := prefix + "_" + sanitizeAttachmentFilename(a.Filename) - fullPath := filepath.Join(storagePath, filename) - if err := os.WriteFile(fullPath, a.Data, 0o644); err == nil { - toSave = append(toSave, savedAttachment{Filename: a.Filename, ContentType: a.ContentType, FilePath: fullPath, Size: int64(len(a.Data))}) - } else { - s.backend.Logger.Error("Failed to write attachment %s: %v", filename, err) - } - } - } - } - - // Classify each envelope recipient as to/cc/bcc by presence in the To/Cc headers — - // anything not literally present in either is inferred BCC. - toList := parseAddressList(toHeader) - ccList := parseAddressList(ccHeader) - recipientTypes := make([]string, len(s.rcptTos)) - for i, rcpt := range s.rcptTos { - lower := strings.ToLower(rcpt) - switch { - case containsStr(toList, lower): - recipientTypes[i] = "to" - case containsStr(ccList, lower): - recipientTypes[i] = "cc" - default: - recipientTypes[i] = "bcc" - } - } - - // Split recipients resolved to a local mailbox in Rcpt from everything else - // (still relayed exactly as before — unchanged for every non-local recipient). - var localRcpts, localTypes, relayRcpts, relayTypes []string - for i, rcpt := range s.rcptTos { + // Split recipients resolved to a local mailbox in Rcpt from everything else — + // RouteAndDeliver only needs the split, not how it was derived (SMTP already + // knows this from Rcpt() time; JMAP submission, internal/jmap, resolves it + // itself since it has no RCPT phase at all). + var localRcpts, relayRcpts []string + for _, rcpt := range s.rcptTos { if _, ok := s.localMailboxes[strings.ToLower(rcpt)]; ok { localRcpts = append(localRcpts, rcpt) - localTypes = append(localTypes, recipientTypes[i]) } else { relayRcpts = append(relayRcpts, rcpt) - relayTypes = append(relayTypes, recipientTypes[i]) } } - // Relay recipients are never delivered inline here — that would block this client's - // DATA response on however long the recipient domain's MX takes to answer (the - // concurrency/load issue this queue exists to fix). Instead a "queued" placeholder - // Result is recorded now and the real attempt happens later via EnqueueForDelivery - // (below, once logID exists) + the background worker in internal/relay/queue.go. - var results []relay.Result - if len(relayRcpts) > 0 { - if limited, err := s.domainSendRateLimited(senderDomain); err != nil { - s.backend.Logger.Error("send-rate-limit check for domain %s: %v", senderDomain, err) - for i, rcpt := range relayRcpts { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "queued"}) - } - } else if limited { - for i, rcpt := range relayRcpts { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "failed", ErrorCode: "450", ErrorMessage: "Sending rate limit exceeded for domain " + senderDomain + ", try again later"}) - } - relayRcpts, relayTypes = nil, nil - } else { - for i, rcpt := range relayRcpts { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: relayTypes[i], Status: "queued"}) - } - } - } - if len(localRcpts) > 0 { - results = append(results, s.deliverLocally(localRcpts, localTypes, signedContent, messageID, subject, fromHeader)...) - } - - // "queued" is neither a known success nor a known failure yet — the worker resolves - // it later (and bounces then, on genuine final failure). Only count actual failures - // here so the immediate bounce-on-partial-failure block below doesn't fire for a - // message that's simply still in flight. - var failed []relay.Result - for _, res := range results { - if res.Status != "success" && res.Status != "queued" { - failed = append(failed, res) - } - } - allSucceeded := len(results) > 0 && len(failed) == 0 - anySucceeded := len(results) > len(failed) - - // A single SMTP response to DATA can't express "delivered to some recipients, not - // others" — rejecting the whole transaction here would make the connecting - // server's own retry logic re-deliver to the recipients that already succeeded. - // So: accept (below) and bounce the failed subset back to our own sender instead, - // exactly like a real MTA splitting a multi-recipient transaction's outcome. A - // bounce is never sent for a *total* failure — that gets rejected outright (550) - // below instead, letting the connecting server's own MTA generate the bounce to - // its user, avoiding a double notification. Skipped entirely for a null-sender - // message (s.mailFrom == "", already itself a bounce/DSN — replying to one is the - // classic bounce-loop bug) and for a currently-blacklisted peer, so a delivery - // failure never becomes a free "yes, that mailbox doesn't exist" oracle for abuse. - if len(failed) > 0 && anySucceeded && s.mailFrom != "" { - if blacklisted, _ := s.backend.DB.IsIPBlacklisted(s.peerIP); !blacklisted { - if err := s.backend.Relay.SendBounce(s.mailFrom, subject, messageID, failed); err != nil { - s.backend.Logger.Error("send bounce to %s: %v", s.mailFrom, err) - } - } - } - - var emailHeaders string - if parseErr == nil { - emailHeaders = strings.Join(parsed.HeaderLines, "\n") - } - - // Privacy default: only headers (and the Subject field, logged separately below - // regardless) go into the admin-visible log, never the message itself — unless this - // sender/IP explicitly opted in via "Store Full Message Content" (storeMessage - // above), or the message was quarantined to Junk for at least one recipient, in - // which case an admin genuinely needs to see it to judge a spam/abuse report. When - // stored, it's the *entire* raw message (not a plain-text extraction) so the log - // viewer can render the real HTML body, inline images, and attachments — re-parsed - // on demand via internal/mailview, the same parser webmail's own message view uses - // — rather than a degraded text-only approximation. - storeContent := storeMessage - if !storeContent { - for _, res := range results { - if res.Quarantined { - storeContent = true - break - } - } - } - loggedBody := "" - if storeContent { - loggedBody = signedContent - } - - logID, logErr := s.backend.Relay.LogEmail(s.backend.Cfg, s.peerIP, s.mailFrom, toHeader, ccHeader, "", subject, emailHeaders, loggedBody, messageID, s.username, dkimSigned, results) - if logErr != nil { - s.backend.Logger.Error("Failed to log email: %v", logErr) - } else { - for _, a := range toSave { - if err := s.backend.DB.InsertEmailAttachment(db.EmailAttachment{ - EmailLogID: logID, Filename: a.Filename, ContentType: a.ContentType, FilePath: a.FilePath, Size: a.Size, - }); err != nil { - s.backend.Logger.Error("Failed to record attachment %s: %v", a.Filename, err) - } - } - if len(relayRcpts) > 0 { - if err := s.backend.Relay.EnqueueForDelivery(logID, s.mailFrom, relayRcpts, relayTypes, signedContent); err != nil { - s.backend.Logger.Error("Failed to enqueue relay delivery: %v", err) - } + _, _, allSucceeded, anySucceeded, err := s.backend.RouteAndDeliver(s.mailFrom, localRcpts, relayRcpts, raw, s.peerIP, s.username) + if err != nil { + if errors.Is(err, ErrVirusDetected) { + return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.NoEnhancedCode, Message: "Message rejected: " + err.Error()} } + return internalError("Internal server error") } if allSucceeded { return &smtp.SMTPError{Code: 250, EnhancedCode: smtp.NoEnhancedCode, Message: "Message accepted for delivery"} } if anySucceeded { - // Some recipients already have the message — 250 it (see the bounce comment - // above for why), not 550, which would tell the connecting server to retry - // the whole thing and re-deliver to those recipients a second time. + // Some recipients already have the message — 250 it (see RouteAndDeliver's + // bounce comment for why), not 550, which would tell the connecting server to + // retry the whole thing and re-deliver to those recipients a second time. return &smtp.SMTPError{Code: 250, EnhancedCode: smtp.NoEnhancedCode, Message: "Message accepted for delivery to some recipients"} } return &smtp.SMTPError{Code: 550, EnhancedCode: smtp.NoEnhancedCode, Message: "Message relay failed"} } -// deliverLocally runs the inbound DKIM/SPF/spam checks once for the message (they -// don't vary per recipient at this milestone — no per-mailbox allow/block-list yet) -// and stores it into each resolved local mailbox, producing one relay.Result per -// recipient so it can be merged into the same LogEmail/allSucceeded logic as relay -// results. -func (s *Session) deliverLocally(rcpts, types []string, signedContent, messageID, subject, fromDisplay string) []relay.Result { - senderDomain := domainOfAddr(s.mailFrom) - dkimPass := senderDomain != "" && dkim.VerifyInbound(signedContent, senderDomain) - spfPass := mailstore.CheckSPF(s.mailFrom, s.peerIP) - heuristicScore := mailstore.SpamScore(s.peerIP, map[string]string{"subject": subject}, dkimPass, spfPass) - rejectScore := s.backend.Cfg.Section("Mailstore").Key("spam_reject_score").MustInt(5) - rspamdEnabled := s.backend.Cfg.Section("Rspamd").Key("enabled").MustBool(false) - rspamdURL := s.backend.Cfg.Section("Rspamd").Key("url").MustString("http://127.0.0.1:11333") - rspamdRejectScore := s.backend.Cfg.Section("Rspamd").Key("reject_score").MustInt(15) - - // Parsed once for every local recipient (not per-recipient — same message body for - // all of them) so filter rules can match on body text / attachment presence - // without every mailbox needing its own parse pass. Best-effort: a message this - // package's own parser can't handle just never matches those two condition types. - bodyText, hasAttachment := "", "no" - if parsedForRules, err := mailview.Parse(strings.NewReader(signedContent)); err == nil { - bodyText = parsedForRules.TextBody - if bodyText == "" && parsedForRules.HTMLBody != "" { - bodyText = bluemonday.StrictPolicy().Sanitize(parsedForRules.HTMLBody) - } - if len(parsedForRules.Attachments) > 0 { - hasAttachment = "yes" - } - } - - enforceDKIM := s.backend.Cfg.Section("Mailstore").Key("enforce_dkim").MustBool(true) - enforceSPF := s.backend.Cfg.Section("Mailstore").Key("enforce_spf").MustBool(true) - enforceDMARC := s.backend.Cfg.Section("Mailstore").Key("enforce_dmarc").MustBool(true) - - // DMARC ties DKIM/SPF together via alignment to the visible From: header's domain - // — a materially different check from dkimPass/spfPass above (which align to the - // SMTP envelope's MAIL FROM domain, senderDomain — the two commonly match but - // DMARC specifically cares about the header, since that's what the recipient - // actually sees and what phishing spoofs). Computed once here, applied per - // recipient below (same shape as dkimPass/spfPass/heuristicScore) so the - // per-mailbox whitelist scope can still suppress it independently. - fromHeaderAddrs := parseAddressList(fromDisplay) - fromHeaderDomain := "" - if len(fromHeaderAddrs) > 0 { - fromHeaderDomain = domainOfAddr(fromHeaderAddrs[0]) - } - dmarcFailPolicy := "" // "", "quarantine", or "reject" — "" means DMARC didn't fail (or wasn't evaluated) - if enforceDMARC && fromHeaderDomain != "" { - if pol := mailstore.LookupDMARCPolicy(fromHeaderDomain); pol != nil { - orgDomain := mailstore.OrganizationalDomain(fromHeaderDomain) - dkimAligned := dkim.VerifyInbound(signedContent, fromHeaderDomain) - spfAligned := spfPass && mailstore.OrganizationalDomain(senderDomain) == orgDomain - if !dkimAligned && !spfAligned { - // ponytail: pct= sampling (gradual DMARC rollout) isn't applied — the - // full effective policy always enforces regardless of pct, which is - // strictly more cautious than what a pct<100 domain owner asked for, - // never less. Add real sampling if a pct<100 domain's mail needs to - // land in INBOX during a deliberate rollout. - dmarcFailPolicy = pol.EffectivePolicy(fromHeaderDomain, orgDomain) - } - } - } - - // rspamd is checked at most once per message and reused for every local recipient - // below, rather than once per recipient — confirmed safe for this deployment (no - // reliance on rspamd's per-recipient personalization, e.g. per-user Bayes/ - // whitelists); content and mail_from are identical for every recipient regardless, - // so the score/action rspamd would return doesn't actually vary by recipient here. - // Cuts what was N rspamd HTTP round-trips down to 1 for a large local fan-out. - rspamdChecked := false - var rspamdScore float64 - var rspamdAction string - var rspamdOK bool - checkRspamdOnce := func() (float64, string, bool) { - if !rspamdChecked { - if score, action, err := mailstore.CheckRspamd(rspamdURL, []byte(signedContent), s.mailFrom, rcpts[0]); err == nil { - rspamdScore, rspamdAction, rspamdOK = score, action, true - } - rspamdChecked = true - } - return rspamdScore, rspamdAction, rspamdOK - } - - results := make([]relay.Result, 0, len(rcpts)) - for i, rcpt := range rcpts { - mbox := s.localMailboxes[strings.ToLower(rcpt)] - folder := "INBOX" - markRead := false - spamGated := false - var tags []string - - // An explicit per-mailbox "allow" entry scoped to "all" bypasses every check - // below entirely — the pre-existing full-bypass behavior, still available via - // the scope picker (webmail_blocklist.html). A narrower scope (spf/dkim/spam) - // only suppresses that one check; the others below still apply independently. - scope, hasAllow, _ := s.backend.DB.AllowScope(mbox.ID, s.mailFrom) - if hasAllow && scope == "all" { - // unchanged existing behavior: full bypass, no scoring, no tagging - } else if junked, _ := s.backend.DB.IsJunked(mbox.ID, s.mailFrom); junked { - // The mailbox owner's own Blocklist (webmail Settings, or the message-view - // "Mark as Junk" action — internal/webui's webmailMarkAsJunk) also bypasses - // scoring entirely, straight to Junk: the user already told us how to - // treat this sender, so there's nothing left to compute (and no rspamd - // round-trip to make). Deliberately a *soft* quarantine (still delivered, - // just hidden), unlike admin's separate hard-reject block list - // (IsBlocked, checked at RCPT time — see Rcpt()) — those are different - // tools for different jobs, not two ways to do the same thing. - folder = "Junk" - spamGated = true - } else { - suppressDKIM := hasAllow && scope == "dkim" - suppressSPF := hasAllow && scope == "spf" - suppressSpam := hasAllow && scope == "spam" - suppressDMARC := hasAllow && scope == "dmarc" - - if enforceDKIM && !dkimPass && !suppressDKIM { - tags = append(tags, "Failed DKIM") - } - if enforceSPF && !spfPass && !suppressSPF { - tags = append(tags, "Failed SPF") - } - - hardReject := false - if !suppressDMARC { - switch dmarcFailPolicy { - case "reject": - tags = append(tags, "Failed DMARC") - hardReject = true - case "quarantine": - tags = append(tags, "Failed DMARC") - } - } - if !suppressSpam { - quarantine := heuristicScore >= rejectScore - if rspamdEnabled { - if score, rAction, ok := checkRspamdOnce(); ok { - // rspamd's own "reject" action is a considered policy decision - // (DNSBL hit, greylisting, etc.) worth still hard-rejecting at - // SMTP time to avoid backscatter; a bare score threshold hit - // (from either scorer) is quarantined instead of rejected, so a - // false positive is recoverable from the Junk folder rather than - // silently bounced with no trace. - if rAction == "reject" { - hardReject = true - } else if score >= float64(rspamdRejectScore) { - quarantine = true - } - } - // rspamd unreachable/erroring must not block mail — errors are swallowed, - // the built-in heuristic above is still the baseline gate either way. - } - if quarantine { - tags = append(tags, "SPAM") - } - } - if hardReject { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: "550", ErrorMessage: "Message rejected as spam"}) - continue - } - if len(tags) > 0 { - folder = "Junk" - spamGated = true - } - } - - // A tagged message's per-recipient copy gets its Subject prepended with why it - // was flagged (e.g. "***Failed SPF, Failed DKIM***") — both in the stored raw - // content (so any IMAP client sees it too, not just this webmail UI) and in the - // cached subject used for list views. This does invalidate that copy's own DKIM - // signature (rewritten after signing) — harmless here since spamGated is always - // true whenever tags are non-empty, and that already skips ApplyRules below - // (including its ForwardTo action), so a tagged copy is never relayed/forwarded - // anywhere; it's only ever locally stored and read via IMAP/webmail, neither of - // which re-verifies DKIM on read. - recipientContent := signedContent - recipientSubject := subject - if len(tags) > 0 { - tag := strings.Join(tags, ", ") - recipientContent = prependSubjectTag(signedContent, tag) - recipientSubject = "***" + tag + "***" - if subject != "" { - recipientSubject += " " + subject - } - } - - // Filter rules organize legitimate mail the recipient already trusts arriving - // in their INBOX — a quarantined message skips them entirely and always lands - // in Junk, rather than a rule accidentally routing spam back into view. - if !spamGated { - // Persistent mailbox-level forwarding (webmail account settings) — distinct - // from and independent of a filter rule's own "forward" action below; both - // can fire on the same message if a mailbox has both configured (a real but - // accepted edge case, not engineered around). - if mbox.ForwardTo != nil && *mbox.ForwardTo != "" { - forwardTo, mailboxEmail, keepCopy := *mbox.ForwardTo, mbox.Email, mbox.ForwardKeepCopy - s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { - if len(res) > 0 && res[0].Status != "success" { - s.backend.Logger.Error("mailbox forwarding: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) - } - }) - if !keepCopy { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) - continue - } - } - - action, err := s.backend.Mailstore.ApplyRules(mbox.ID, map[string]string{ - "from": s.mailFrom, "to": rcpt, "subject": subject, - "body": bodyText, "has_attachment": hasAttachment, "recipient_type": types[i], - }) - if err != nil { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: "450", ErrorMessage: err.Error()}) - continue - } - if action.ForwardTo != "" { - // Fire-and-forget: forwarding is a side effect layered on top of this - // recipient's own local delivery, not a substitute for it — a slow or - // unreachable forward target must never delay the SMTP response. - // Envelope-from is the mailbox's own address (not the original - // sender's) so this doesn't masquerade as a relay of someone else's - // mail; no SRS rewriting or Resent-* headers, matching every other - // send path in this codebase. - forwardTo, mailboxEmail := action.ForwardTo, mbox.Email - s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{forwardTo}, signedContent, []string{"to"}, func(res []relay.Result) { - if len(res) > 0 && res[0].Status != "success" { - s.backend.Logger.Error("forward rule: delivery to %s failed: %s", forwardTo, res[0].ErrorMessage) - } - }) - if !action.KeepCopy { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Forwarded to " + forwardTo + ", not kept locally"}) - continue - } - } - if action.Drop { - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: "Discarded by filter rule"}) - continue - } - if action.AutoReply { - s.sendAutoReply(mbox, action.AutoReplySubject, action.AutoReplyBody, messageID) - } - if action.Folder != "" { - folder = action.Folder - } - markRead = action.MarkRead - } - - uid, err := s.backend.Mailstore.StoreMessage(mbox.ID, folder, []byte(recipientContent), messageID, fromDisplay, recipientSubject) - if err != nil { - errCode, errMsg := "450", err.Error() - if err == mailstore.ErrQuotaExceeded { - errCode, errMsg = "552", "Mailbox quota exceeded" - } - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "failed", ErrorCode: errCode, ErrorMessage: errMsg}) - continue - } - if markRead { - if err := s.backend.DB.SetMessageFlags(mbox.ID, uid, `\Seen`); err != nil { - s.backend.Logger.Error("mark_read rule failed to set flag for message %d: %v", uid, err) - } - } - s.backend.Notify.Publish(mbox.ID, folder) - serverResponse := "Delivered to local mailbox" - if spamGated { - serverResponse = "Quarantined to Junk folder" - } - results = append(results, relay.Result{Recipient: rcpt, RecipientType: types[i], Status: "success", ServerResponse: serverResponse, Quarantined: spamGated}) - } - return results -} - -// sendAutoReply fires a vacation-responder reply to the current message's sender, -// fire-and-forget (same reasoning as the forward action: a slow/unreachable target -// must never delay the SMTP response), after two loop/storm-prevention checks: never -// reply to a null-sender message (a bounce/DSN — replying to one is the classic -// bounce-loop bug, same rule SendBounce itself already follows), and never reply to -// the same sender more than once per rolling 24h (esrv_mailbox_autoreply_log) — two -// auto-responders emailing each other would otherwise loop forever. -func (s *Session) sendAutoReply(mbox *db.Mailbox, subject, body, inReplyTo string) { - if s.mailFrom == "" { - return - } - if recent, err := s.backend.DB.HasRecentAutoReply(mbox.ID, s.mailFrom); err != nil || recent { - return - } - if subject == "" { - subject = "Automatic reply" - } - mailboxEmail, replyTo := mbox.Email, s.mailFrom - raw := buildAutoReplyMessage(s.backend.HeloHostname, mailboxEmail, replyTo, subject, body, inReplyTo) - s.backend.Relay.RelayEmailAsyncBounded(mailboxEmail, []string{replyTo}, raw, []string{"to"}, func(res []relay.Result) { - if len(res) > 0 && res[0].Status != "success" { - s.backend.Logger.Error("auto-reply to %s failed: %s", replyTo, res[0].ErrorMessage) - } - }) - if err := s.backend.DB.RecordAutoReply(mbox.ID, replyTo); err != nil { - s.backend.Logger.Error("record auto-reply to %s: %v", replyTo, err) - } -} - // buildAutoReplyMessage renders a simple vacation-responder reply — plain text/plain, // marked Auto-Submitted (RFC 3834) so it isn't itself replied to by another // auto-responder on the receiving end, mirroring relay.buildBounceMessage's shape. @@ -811,27 +295,6 @@ func buildAutoReplyMessage(hostname, from, to, subject, body, inReplyTo string) return strings.Join(headers, "\r\n") + "\r\n\r\n" + body + "\r\n" } -// domainSendRateLimited reports whether domain has hit its own admin-configured -// outbound send-rate cap (esrv_domains.send_rate_limit_per_hour) within the last -// rolling hour. Unconfigured (nil limit) or an unrecognized/empty domain never limits. -func (s *Session) domainSendRateLimited(domain string) (bool, error) { - if domain == "" { - return false, nil - } - dom, err := s.backend.DB.GetDomainByName(domain) - if err != nil { - return false, err - } - if dom == nil || dom.SendRateLimitPerHour == nil { - return false, nil - } - count, err := s.backend.DB.CountRecentSendsForDomain(domain, time.Now().Add(-time.Hour)) - if err != nil { - return false, err - } - return count >= *dom.SendRateLimitPerHour, nil -} - func containsStr(list []string, s string) bool { for _, v := range list { if v == s { diff --git a/internal/webui/settings_mfa_test.go b/internal/webui/settings_mfa_test.go index 4423d6e..9aeb780 100644 --- a/internal/webui/settings_mfa_test.go +++ b/internal/webui/settings_mfa_test.go @@ -86,13 +86,14 @@ func baseSettingsForm() url.Values { "Server.bind_ip": {"0.0.0.0"}, "Server.time_zone": {"UTC"}, "Server.hostname": {"mail.example.com"}, "Server.helo_hostname": {"mail.example.com"}, "Server.server_banner": {""}, + "Server.imap_port": {"1143"}, "Server.imap_tls_port": {"1993"}, + "Server.jmap_enable": {"true"}, "Server.jmap_port": {"8443"}, "Database.database_url": {"sqlite:///server_data/smtp_server.db"}, "Logging.log_level": {"INFO"}, "Logging.hide_info_aiosmtpd": {"true"}, "Relay.relay_timeout": {"30"}, "TLS.tls_cert_file": {"ssl_certs/server.crt"}, "TLS.tls_key_file": {"ssl_certs/server.key"}, "DKIM.dkim_key_size": {"2048"}, "DKIM.spf_server_ip": {"192.168.1.1"}, "Attachments.attachments_path": {"server_data/attachments"}, - "IMAP.imap_port": {"1143"}, "IMAP.imap_tls_port": {"1993"}, "Auth.enforce_admin_mfa": {"false"}, "Auth.enforce_mailbox_mfa": {"false"}, } } diff --git a/internal/webui/templates/settings.html b/internal/webui/templates/settings.html index 2704b1d..208c588 100644 --- a/internal/webui/templates/settings.html +++ b/internal/webui/templates/settings.html @@ -21,6 +21,29 @@
Server Configuration
+
+
+
Plain IMAP port (no STARTTLS offered)
+ +
+
+
Implicit-TLS IMAP port (IMAPS)
+ +
+
+
+
+
RFC 8620/8621 HTTP+JSON mail access — reads/writes the same mailboxes IMAP uses, not a separate store
+ +
+
+
Dedicated HTTPS port (always TLS, shares the admin/webmail UI's certificate)
+ +
+
Port for plain/IP-whitelisted SMTP connections
@@ -118,24 +141,6 @@
-
-
IMAP Configuration
-
-
-
-
-
Plain IMAP port (no STARTTLS offered)
- -
-
-
Implicit-TLS IMAP port (IMAPS)
- -
-
-
-
-
-
Rspamd (Spam Scoring)
@@ -394,7 +399,7 @@ } document.querySelector('form').addEventListener('submit', function(e) { - const ports = ['Server.smtp_port', 'Server.smtp_tls_port', 'Server.web_http_port', 'Server.web_https_port', 'IMAP.imap_port', 'IMAP.imap_tls_port']; + const ports = ['Server.smtp_port', 'Server.smtp_tls_port', 'Server.web_http_port', 'Server.web_https_port', 'Server.imap_port', 'Server.imap_tls_port', 'Server.jmap_port']; const seen = {}; for (const portField of ports) { const input = document.querySelector(`[name="${portField}"]`); diff --git a/internal/webui/webui_test.go b/internal/webui/webui_test.go index dc9b367..934c4dc 100644 --- a/internal/webui/webui_test.go +++ b/internal/webui/webui_test.go @@ -97,9 +97,10 @@ func newTestApp(t *testing.T) *App { serverSec.NewKey("hostname", "mail.example.com") serverSec.NewKey("helo_hostname", "mail.example.com") serverSec.NewKey("server_banner", "") - imapSec, _ := cfg.NewSection("IMAP") - imapSec.NewKey("imap_port", "1143") - imapSec.NewKey("imap_tls_port", "1993") + serverSec.NewKey("imap_port", "1143") + serverSec.NewKey("imap_tls_port", "1993") + serverSec.NewKey("jmap_enable", "true") + serverSec.NewKey("jmap_port", "8443") authSec, _ := cfg.NewSection("Auth") authSec.NewKey("enforce_admin_mfa", "false") authSec.NewKey("enforce_mailbox_mfa", "false") diff --git a/main.go b/main.go index 71c612b..bc9f1e3 100644 --- a/main.go +++ b/main.go @@ -30,6 +30,7 @@ import ( "mailgoserver/internal/dkim" "mailgoserver/internal/dnspublish" "mailgoserver/internal/imapserver" + "mailgoserver/internal/jmap" "mailgoserver/internal/mailstore" "mailgoserver/internal/notify" "mailgoserver/internal/relay" @@ -256,6 +257,7 @@ func main() { Notify: notifyBus, } imapBackend := &imapserver.Backend{DB: database, Mailstore: mstore, Logger: toolbox.GetLogger("imap"), Cfg: cfg, Notify: notifyBus} + jmapBackend := &jmap.Backend{DB: database, Mailstore: mstore, Notify: notifyBus, Cfg: cfg, Logger: toolbox.GetLogger("jmap"), Relay: relayer, SMTP: backend} var smtpRunning atomic.Bool @@ -305,8 +307,8 @@ func main() { } runIMAP := func() (plain, tlsSrv *goimapserver.Server) { - imapPort := cfg.Section("IMAP").Key("IMAP_PORT").MustInt(143) - imapTLSPort := cfg.Section("IMAP").Key("IMAP_TLS_PORT").MustInt(993) + imapPort := cfg.Section("Server").Key("IMAP_PORT").MustInt(143) + imapTLSPort := cfg.Section("Server").Key("IMAP_TLS_PORT").MustInt(993) plainServer := imapserver.NewPlainServer(imapBackend) tlsServer := imapserver.NewTLSServer(imapBackend, imapTLSConfig) @@ -338,6 +340,31 @@ func main() { return plainServer, tlsServer } + // runJMAP starts the JMAP (RFC 8620/8621) listener on its own dedicated port — + // deliberately not sharing the webui's HTTPS port (an explicit choice, not a + // technical requirement: webHTTPSConfig below is reused as-is for the cert, same + // hot-reloading Let's Encrypt/self-signed cert every other TLS listener already + // gets). JMAP mandates TLS (RFC 8620 §1.7), so there's no plaintext variant here, + // unlike SMTP/IMAP. Starts independent of -web-only/-smtp-only — it only depends + // on the DB/mailstore, never on SMTP or IMAP being up. + runJMAP := func() *http.Server { + jmapPort := cfg.Section("Server").Key("JMAP_PORT").MustInt(8443) + addr := fmt.Sprintf("0.0.0.0:%d", jmapPort) + srv := &http.Server{Addr: addr, Handler: jmapBackend.Mux(), TLSConfig: webHTTPSConfig, ReadHeaderTimeout: 10 * time.Second, IdleTimeout: 120 * time.Second} + go func() { + logger.Info("JMAP listening on :%d", jmapPort) + l, err := tls.Listen("tcp", addr, webHTTPSConfig) + if err != nil { + logger.Error("JMAP listen: %v", err) + return + } + if err := srv.Serve(abuseguard.GuardListener(l, database, cfg, logger)); err != nil && !errors.Is(err, http.ErrServerClosed) { + logger.Error("JMAP server: %v", err) + } + }() + return srv + } + // runCertRenewal is the first periodic/background job in this codebase — everything // else is purely request-driven. Checks soon after boot (so enabling Let's Encrypt // and restarting converges quickly) and every 12h thereafter. NeedsRenewal is a pure @@ -528,7 +555,7 @@ func main() { // restart/redeploy forever. go-smtp's Server has a real graceful Shutdown(ctx); // go-imap/v2's Server only has Close() (force-close, confirmed no graceful variant // exists in that library) — still better than no shutdown handling at all. - waitForShutdown := func(smtpPlain, smtpTLS *smtp.Server, imapPlain, imapTLS *goimapserver.Server, httpSrv, httpsSrv *http.Server) { + waitForShutdown := func(smtpPlain, smtpTLS *smtp.Server, imapPlain, imapTLS *goimapserver.Server, httpSrv, httpsSrv, jmapSrv *http.Server) { <-sigCtx.Done() logger.Info("shutdown signal received, draining connections...") @@ -552,6 +579,9 @@ func main() { if httpsSrv != nil { httpsSrv.Shutdown(shutdownCtx) } + if jmapSrv != nil { + jmapSrv.Shutdown(shutdownCtx) + } relayDone := make(chan struct{}) go func() { @@ -566,11 +596,19 @@ func main() { logger.Info("shutdown complete") } + // JMAP_ENABLE defaults true (unlike this codebase's usual new-feature-off + // convention) — meant to be on out of the box. Started here, before either branch + // below, since JMAP has no SMTP/IMAP/web-server dependency at the protocol level. + var jmapServer *http.Server + if cfg.Section("Server").Key("JMAP_ENABLE").MustBool(true) { + jmapServer = runJMAP() + } + if *smtpOnly { smtpPlain, smtpTLS := runSMTP() imapPlain, imapTLS := runIMAP() go runCertRenewal() - waitForShutdown(smtpPlain, smtpTLS, imapPlain, imapTLS, nil, nil) + waitForShutdown(smtpPlain, smtpTLS, imapPlain, imapTLS, nil, nil, jmapServer) return } @@ -648,7 +686,7 @@ func main() { } }() - waitForShutdown(smtpPlain, smtpTLS, imapPlain, imapTLS, httpServer, httpsServer) + waitForShutdown(smtpPlain, smtpTLS, imapPlain, imapTLS, httpServer, httpsServer, jmapServer) } func absPath(root, p string) string {